From: Davis Goodin Date: Tue, 12 Mar 2019 21:24:49 +0000 (-0500) Subject: Add pkgproj-based DEB/RPM build for targeting packs (dotnet/core-setup#5369) X-Git-Tag: submit/tizen/20210909.063632~11032^2~343 X-Git-Url: http://review.tizen.org/git/?a=commitdiff_plain;h=c5fd050642c9adf7e1b291cca798b3bc1459d318;p=platform%2Fupstream%2Fdotnet%2Fruntime.git Add pkgproj-based DEB/RPM build for targeting packs (dotnet/core-setup#5369) * Pkgproj-based DEB/RPM build for targeting packs * Code review fixes Make clean displayName clearer. Add '-a' to 'docker system prune'. Fix comments for fpm and debuild tool checks. * Fix bad indent Commit migrated from https://github.com/dotnet/core-setup/commit/f6441d4af6030bd40eb0440038bbe4d522d266cf --- diff --git a/eng/jobs/bash-build.yml b/eng/jobs/bash-build.yml index 3a4c6c9..9fff60b 100644 --- a/eng/jobs/bash-build.yml +++ b/eng/jobs/bash-build.yml @@ -6,8 +6,7 @@ parameters: displayName: null dockerImage: null osGroup: Linux - packageDistroListDeb: null - packageDistroListRpm: null + packageDistroList: null skipTests: $(SkipTests) strategy: matrix: @@ -52,6 +51,7 @@ jobs: -TargetArchitecture=${{ parameters.targetArchitecture }} -- /p:StabilizePackageVersion=$(IsStable) /nr:false + /bl:msbuild.binlog ${{ parameters.additionalMSBuildArgs }} PublishArguments: /p:PublishType=$(_PublishType) @@ -61,10 +61,10 @@ jobs: /p:StabilizePackageVersion=$(IsStable) /p:TargetArchitecture=${{ parameters.targetArchitecture }} /nr:false + /bl:msbuild.publish.binlog ${{ parameters.additionalMSBuildArgs }} - CommonMSBuildArgs: /bl - /p:ConfigurationGroup=$(_BuildConfig) + CommonMSBuildArgs: /p:ConfigurationGroup=$(_BuildConfig) /p:OfficialBuildId=$(OfficialBuildId) /p:OSGroup=${{ parameters.osGroup }} /p:PortableBuild=false @@ -72,32 +72,17 @@ jobs: /p:TargetArchitecture=${{ parameters.targetArchitecture }} /nr:false - # Packaging related variables - # Debian - CommonDebianRunCommand: run - -v $(Build.SourcesDirectory):/root/coresetup - -v $(Build.StagingDirectory)/sharedFrameworkPublish/:/root/sharedFrameworkPublish/ - -w=/root/coresetup - mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-14.04-debpkg-e5cf912-20175003025046 - /root/coresetup/Tools/msbuild.sh - DebianPackagingCommand: $(CommonDebianRunCommand) - /root/coresetup/src/pkg/packaging/dir.proj - /p:UsePrebuiltPortableBinariesForInstallers=true + # Tell the build to package up the bits from the portable build. + PackagePortableBitsArgs: >- /p:SharedFrameworkPublishDir=/root/sharedFrameworkPublish/ - $(CommonMSBuildArgs) + /p:InstallerSourceOSPlatformConfig=linux-x64.$(_BuildConfig) - # RPM - CommonRpmRunCommand: run + MSBuildScript: /root/coresetup/Tools/msbuild.sh + DockerRunMSBuild: >- + docker run -v $(Build.SourcesDirectory):/root/coresetup -v $(Build.StagingDirectory)/sharedFrameworkPublish/:/root/sharedFrameworkPublish/ -w=/root/coresetup - mcr.microsoft.com/dotnet-buildtools/prereqs:rhel-7-rpmpkg-c982313-20174116044113 - /root/coresetup/Tools/msbuild.sh - RpmPackagingCommand: $(CommonRpmRunCommand) - /root/coresetup/src/pkg/packaging/dir.proj - /p:UsePrebuiltPortableBinariesForInstallers=true - /p:SharedFrameworkPublishDir=/root/sharedFrameworkPublish/ - $(CommonMSBuildArgs) # Disable MSBuild node reuse in case this build is running on a persistent agent. # Use environment variable rather than /nr:false to make sure internal Execs that run MSBuild @@ -105,30 +90,39 @@ jobs: MSBUILDDISABLENODEREUSE: 1 steps: - - script: df -h - displayName: Check space (df -h) - continueOnError: true + # Builds don't set user ID, so files might be owned by root and unable to be cleaned up by AzDO. + # Clean up the build dirs ourselves in another Docker container to avoid failures. + # Using hosted agents is tracked by https://github.com/dotnet/core-setup/issues/4997 + - script: | + set -x + docker run --rm \ + -v "$(Agent.BuildDirectory):/root/build" \ + -w /root/build \ + ${{ parameters.dockerImage }} \ + bash -c ' + rm -v -rf a b + cd s + git clean -xdf' + displayName: Clean up old artifacts owned by root # Build binary and nuget packages - - script: $(RunArguments) - ./build.sh - $(BuildArguments) + - script: | + set -x + df -h + $(RunArguments) ./build.sh $(BuildArguments) displayName: Build - workingDirectory: '$(Build.SourcesDirectory)' - - - script: df -h - displayName: Check space (df -h) - continueOnError: true # Publish only if internal and not PR - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - - script: $(RunArguments) - ./Tools/msbuild.sh - ./publish/publish.proj - $(PublishArguments) - $(_CommonPublishArgs) + - script: | + set -x + df -h + $(RunArguments) \ + ./Tools/msbuild.sh \ + ./publish/publish.proj \ + $(PublishArguments) \ + $(_CommonPublishArgs) displayName: Publish - workingDirectory: '$(Build.SourcesDirectory)' condition: and(succeeded(), eq(variables._BuildConfig, 'Release')) # Only for glibc leg, here we produce RPMs and Debs @@ -138,121 +132,30 @@ jobs: inputs: SourceFolder: '$(Build.SourcesDirectory)/bin/obj/linux-x64.$(_BuildConfig)/sharedFrameworkPublish' TargetFolder: '$(Build.StagingDirectory)/sharedFrameworkPublish' - - script: df -h - displayName: Check space (df -h) - continueOnError: true - - script: docker - $(CommonDebianRunCommand) - /root/coresetup/build.proj - /t:BuildTraversalBuildDependencies - $(CommonMSBuildArgs) - displayName: Build traversal build dependencies - Ubuntu 14.04 - workingDirectory: '$(Build.SourcesDirectory)' - - script: df -h - displayName: Check space (df -h) - continueOnError: true - - # Debian packaging - - script: docker - $(DebianPackagingCommand) - displayName: 'Package Runtime packages and Runtime Dep - Ubuntu 14.04' - workingDirectory: '$(Build.SourcesDirectory)' - - script: df -h - displayName: Check space (df -h) - continueOnError: true - - - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - - script: docker - $(CommonDebianRunCommand) - /root/coresetup/publish/publish.proj - /p:PublishType=$(_PublishType) - /p:SharedFrameworkPublishDir=/root/sharedFrameworkPublish/ - $(CommonMSBuildArgs) - $(_CommonPublishArgs) - displayName: 'Publish Runtime and Runtime Dep - Ubuntu 14.04' - workingDirectory: '$(Build.SourcesDirectory)' - condition: and(succeeded(), eq(variables._BuildConfig, 'Release')) - - script: df -h - displayName: Check space (df -h) - continueOnError: true - - # Package and publish for each debian distros - - ${{ each debdistrorid in parameters.packageDistroListDeb }}: - - script: docker - $(DebianPackagingCommand) /p:BuildRuntimeDebs=false - /p:OutputRid=${{ debdistrorid }}-${{ parameters.targetArchitecture }} - displayName: 'Package Runtime Dep - ${{ debdistrorid }}' - - script: df -h - displayName: Check space (df -h) - continueOnError: true - - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - - script: docker - $(CommonDebianRunCommand) - /root/coresetup/publish/publish.proj - /p:PublishType=$(_PublishType) - /p:SharedFrameworkPublishDir=/root/sharedFrameworkPublish/ - $(CommonMSBuildArgs) - $(_CommonPublishArgs) - /p:OutputRid=${{ debdistrorid }}-${{ parameters.targetArchitecture }} - displayName: 'Publish Runtime Dep - ${{ debdistrorid }}' - workingDirectory: '$(Build.SourcesDirectory)' - condition: and(succeeded(), eq(variables._BuildConfig, 'Release')) - - script: df -h - displayName: Check space (df -h) - continueOnError: true - # RPM Packaging - - script: docker - $(CommonRpmRunCommand) - /root/coresetup/build.proj - /t:BuildTraversalBuildDependencies - $(CommonMSBuildArgs) - displayName: Build traversal build dependencies - Rhel7 - workingDirectory: '$(Build.SourcesDirectory)' - - script: docker - $(RpmPackagingCommand) - displayName: Package Runtime Dep - Rhel7 - - script: df -h - displayName: Check space (df -h) - continueOnError: true - - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - - script: docker - $(CommonRpmRunCommand) - /root/coresetup/publish/publish.proj - /p:PublishType=$(_PublishType) - /p:SharedFrameworkPublishDir=/root/sharedFrameworkPublish/ - $(CommonMSBuildArgs) - $(_CommonPublishArgs) - displayName: Publish Runtime Dep - Rhel7 - workingDirectory: '$(Build.SourcesDirectory)' - condition: and(succeeded(), eq(variables._BuildConfig, 'Release')) + - ${{ each packageBuild in parameters.packageDistroList }}: + # This leg's RID matches the build image. Build its distro-dependent packages, as well as + # the distro-independent installers. (There's no particular reason to build the distro- + # independent installers on this leg, but we need to do it somewhere.) + - template: steps/build-linux-package.yml + parameters: + buildTraversalBuildDependencies: true + distroRid: ${{ packageBuild.imageRid }} + image: ${{ packageBuild.image }} + packageStepDescription: Runtime Deps, Runtime, Framework Packs installers + + - ${{ each rid in packageBuild.rids }}: + # Build distro-dependent packages. + - template: steps/build-linux-package.yml + parameters: + distroRid: ${{ rid }} + image: ${{ packageBuild.image }} + outputRidArg: /p:OutputRid=${{ rid }}-${{ parameters.targetArchitecture }} + packageStepDescription: Runtime Deps installers + packagingArgs: /p:BuildDistroIndependentInstallers=false - # Package and publish for each rpm distros - - ${{ each rpmdistrorid in parameters.packageDistroListRpm }}: - - script: docker - $(RpmPackagingCommand) /p:BuildRuntimeRpms=false - /p:OutputRid=${{ rpmdistrorid }}-${{ parameters.targetArchitecture }} - displayName: 'Package Runtime Dep - ${{ rpmdistrorid }}' - - script: df -h - displayName: Check space (df -h) - continueOnError: true - - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - - script: docker - $(CommonRpmRunCommand) - /root/coresetup/publish/publish.proj - /p:PublishType=$(_PublishType) - /p:SharedFrameworkPublishDir=/root/sharedFrameworkPublish/ - $(CommonMSBuildArgs) - $(_CommonPublishArgs) - /p:OutputRid=${{ rpmdistrorid }}-${{ parameters.targetArchitecture }} - displayName: 'Publish Runtime Dep -${{ rpmdistrorid }}' - workingDirectory: '$(Build.SourcesDirectory)' - condition: and(succeeded(), eq(variables._BuildConfig, 'Release')) - - script: df -h - displayName: Check space (df -h) - continueOnError: true - task: CopyFiles@2 - displayName: Copy Files to $(Build.StagingDirectory)\BuildLogs + displayName: Copy logs to stage inputs: SourceFolder: '$(Build.SourcesDirectory)' Contents: | @@ -261,6 +164,7 @@ jobs: TargetFolder: '$(Build.StagingDirectory)\BuildLogs' continueOnError: true condition: succeededOrFailed() + - task: PublishBuildArtifacts@1 displayName: Publish Artifact BuildLogs inputs: @@ -269,6 +173,24 @@ jobs: continueOnError: true condition: succeededOrFailed() + - task: CopyFiles@2 + displayName: Copy packages to stage + inputs: + SourceFolder: '$(Build.SourcesDirectory)/bin/' + Contents: | + */packages/* + TargetFolder: '$(Build.StagingDirectory)\Packages' + continueOnError: true + condition: succeededOrFailed() + + - task: PublishBuildArtifacts@1 + displayName: Publish Artifact Packages + inputs: + PathtoPublish: '$(Build.StagingDirectory)\Packages' + ArtifactName: ${{ parameters.displayName }}-$(_BuildConfig)-Packages + continueOnError: true + condition: succeededOrFailed() + - script: df -h displayName: Check space (df -h) condition: always() @@ -276,10 +198,6 @@ jobs: # Force clean up machine in case any docker images are left behind - ${{ if ne(parameters.displayName, 'Build_FreeBSD_x64')}}: - - script: docker system prune -f + - script: docker system prune -af && df -h displayName: Run Docker clean up condition: succeededOrFailed() - - script: df -h - displayName: Check space (df -h) - condition: always() - continueOnError: true diff --git a/eng/jobs/finalize-publish.yml b/eng/jobs/finalize-publish.yml index ecd72bf..a956b04 100644 --- a/eng/jobs/finalize-publish.yml +++ b/eng/jobs/finalize-publish.yml @@ -15,6 +15,8 @@ jobs: name: dotnet-internal-temp # Double the default timeout. Publishing is subject to huge delays due to contention on the dotnet-core blob feed timeoutInMinutes: 120 + workspace: + clean: all variables: _PublishType: ${{ parameters._PublishType}} diff --git a/eng/jobs/osx-build.yml b/eng/jobs/osx-build.yml index 8c1e506..5d46490 100644 --- a/eng/jobs/osx-build.yml +++ b/eng/jobs/osx-build.yml @@ -13,6 +13,8 @@ jobs: release: _BuildConfig: Release _PublishType: blob + workspace: + clean: all steps: - script: $(Build.SourcesDirectory)/build.sh -OfficialBuildId=$(OfficialBuildId) diff --git a/eng/jobs/steps/build-linux-package.yml b/eng/jobs/steps/build-linux-package.yml new file mode 100644 index 0000000..ac81413 --- /dev/null +++ b/eng/jobs/steps/build-linux-package.yml @@ -0,0 +1,48 @@ +parameters: + buildTraversalBuildDependencies: false + distroRid: null + image: null + outputRidArg: '' + packageStepDescription: null + packagingArgs: '' + +steps: +- ${{ if eq(parameters.buildTraversalBuildDependencies, true) }}: + - script: | + set -x + df -h + $(DockerRunMSBuild) ${{ parameters.image }} $(MSBuildScript) \ + /root/coresetup/build.proj \ + /t:BuildTraversalBuildDependencies \ + $(CommonMSBuildArgs) \ + /bl:msbuild.${{ parameters.distroRid }}.traversaldependencies.binlog + displayName: ====== Build traversal build dependencies - ${{ parameters.distroRid }} + +- script: | + set -x + df -h + $(DockerRunMSBuild) ${{ parameters.image }} $(MSBuildScript) \ + /root/coresetup/src/pkg/packaging/dir.proj \ + /p:UsePrebuiltPortableBinariesForInstallers=true \ + $(PackagePortableBitsArgs) \ + /p:GenerateProjectInstallers=true \ + ${{ parameters.packagingArgs }} \ + $(CommonMSBuildArgs) \ + ${{ parameters.outputRidArg }} \ + /bl:msbuild.${{ parameters.distroRid }}.installers.binlog + displayName: Package ${{ parameters.packageStepDescription }} - ${{ parameters.distroRid }} + +- ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + - script: | + set -x + df -h + $(DockerRunMSBuild) ${{ parameters.image }} $(MSBuildScript) \ + /root/coresetup/publish/publish.proj \ + /p:PublishType=$(_PublishType) \ + $(PackagePortableBitsArgs) \ + $(CommonMSBuildArgs) \ + $(_CommonPublishArgs) \ + ${{ parameters.outputRidArg }} \ + /bl:msbuild.${{ parameters.distroRid }}.publish.binlog + displayName: -> Publish ${{ parameters.packageStepDescription }} - ${{ parameters.distroRid }} + condition: and(succeeded(), eq(variables._BuildConfig, 'Release')) diff --git a/eng/jobs/windows-build.yml b/eng/jobs/windows-build.yml index 92a425a..332ebb5d 100644 --- a/eng/jobs/windows-build.yml +++ b/eng/jobs/windows-build.yml @@ -27,6 +27,8 @@ jobs: release: _BuildConfig: Release _PublishType: blob + workspace: + clean: all variables: CommonMSBuildArgs: "/p:ConfigurationGroup=$(_BuildConfig) /p:TargetArchitecture=${{ parameters.targetArchitecture }} diff --git a/eng/pipelines/installer/azure-pipelines.yml b/eng/pipelines/installer/azure-pipelines.yml index 3808d70..906640a 100644 --- a/eng/pipelines/installer/azure-pipelines.yml +++ b/eng/pipelines/installer/azure-pipelines.yml @@ -144,8 +144,22 @@ jobs: parameters: displayName: Build_Linux_x64_glibc dockerImage: mcr.microsoft.com/dotnet-buildtools/prereqs:centos-7-d485f41-20173404063424 - packageDistroListDeb: [debian.8,debian.9,ubuntu.16.04,ubuntu.18.04] - packageDistroListRpm: [centos.7,fedora.27,opensuse.42,oraclelinux.7,sles.12] + packageDistroList: + - image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-14.04-debpkg-e5cf912-20175003025046 + imageRid: ubuntu.14.04 + rids: + - debian.8 + - debian.9 + - ubuntu.16.04 + - ubuntu.18.04 + - image: mcr.microsoft.com/dotnet-buildtools/prereqs:rhel-7-rpmpkg-c982313-20174116044113 + imageRid: rhel.7 + rids: + - centos.7 + - fedora.27 + - opensuse.42 + - oraclelinux.7 + - sles.12 portableBuild: true targetArchitecture: x64 diff --git a/src/installer/pkg/packaging/deb/dotnet-deb-tool-consumer.csproj b/src/installer/pkg/deps/dotnet-deb-tool-consumer.csproj similarity index 100% rename from src/installer/pkg/packaging/deb/dotnet-deb-tool-consumer.csproj rename to src/installer/pkg/deps/dotnet-deb-tool-consumer.csproj diff --git a/src/installer/pkg/deps/init-deb-tool-consumer.proj b/src/installer/pkg/deps/init-deb-tool-consumer.proj new file mode 100644 index 0000000..e80d214 --- /dev/null +++ b/src/installer/pkg/deps/init-deb-tool-consumer.proj @@ -0,0 +1,41 @@ + + + + + + $(ProjectDir)tools-local/setuptools/dotnet-deb-tool/ + $(DebtoolConsumerDeployDir)copied.sem + + + + + + + + + + --version-suffix $(VersionSuffix) + + + + + + + + + + + + + + + + diff --git a/src/installer/pkg/dir.props b/src/installer/pkg/dir.props index b83e911..ce85d6c 100644 --- a/src/installer/pkg/dir.props +++ b/src/installer/pkg/dir.props @@ -28,6 +28,37 @@ true + + $(IntermediateOutputRootPath)packages/ + + $(MSBuildThisFileDirectory)packaging/rpm/templates/ + + dotnet-deb-tool-consumer.csproj + $(MSBuildThisFileDirectory)deps/$(DebtoolConsumerProjectName) + + $(IntermediateOutputRootPath)$(DebtoolConsumerProjectName)/ + + true + true + + true + true + + + true + + diff --git a/src/installer/pkg/dir.targets b/src/installer/pkg/dir.targets index 4411425..9df96a0 100644 --- a/src/installer/pkg/dir.targets +++ b/src/installer/pkg/dir.targets @@ -1,7 +1,58 @@ - + + + $(PackagesDir)$(PlatformPackageId.ToLowerInvariant())\$(MicrosoftNETCorePlatformsPackageVersion)\runtime.json + + + + + + + + + + + + + + + + - $(PackagesDir)$(PlatformPackageId.ToLowerInvariant())\$(MicrosoftNETCorePlatformsPackageVersion)\runtime.json + false + true + + + + + + + + + + + + + + false + true + + + + diff --git a/src/installer/pkg/dir.traversal.targets b/src/installer/pkg/dir.traversal.targets index b68d12b..fa702d0 100644 --- a/src/installer/pkg/dir.traversal.targets +++ b/src/installer/pkg/dir.traversal.targets @@ -8,6 +8,11 @@ + + + + + diff --git a/src/installer/pkg/packaging/deb/package.props b/src/installer/pkg/packaging/deb/package.props index 64a80e3..0c57419 100644 --- a/src/installer/pkg/packaging/deb/package.props +++ b/src/installer/pkg/packaging/deb/package.props @@ -1,10 +1,6 @@ - $(ProjectDir)tools-local/setuptools/dotnet-deb-tool/ - $(PackageOutputPath) - dotnet-deb-tool-consumer.csproj - $(IntermediateOutputRootPath)$(toolConsumerProjectName) $(PackagingRoot)deb/ dotnet-host $(SharedHostDebPkgName.ToLower()) diff --git a/src/installer/pkg/packaging/deb/package.targets b/src/installer/pkg/packaging/deb/package.targets index 5aaa091..aeb5559 100644 --- a/src/installer/pkg/packaging/deb/package.targets +++ b/src/installer/pkg/packaging/deb/package.targets @@ -5,39 +5,9 @@ - true - true debian_config.json - true - - - --version-suffix $(VersionSuffix) - - - - - - - - - - - - - - - - - - + WorkingDirectory="$(DebtoolConsumerDeployDir)" /> @@ -185,7 +155,7 @@ ReplacementItems="@(HostFxrTokenValue)" /> + WorkingDirectory="$(DebtoolConsumerDeployDir)" /> @@ -278,7 +248,7 @@ ReplacementItems="@(SharedFrameworkTokenValue)" /> + WorkingDirectory="$(DebtoolConsumerDeployDir)" /> @@ -346,7 +316,7 @@ ReplacementItems="@(SharedFrameworkTokenValue)" /> + WorkingDirectory="$(DebtoolConsumerDeployDir)" /> @@ -364,23 +334,4 @@ - - - - - - - - - - false - true - - - - - - \ No newline at end of file diff --git a/src/installer/pkg/packaging/dir.proj b/src/installer/pkg/packaging/dir.proj index 6fe7bed..6141233 100644 --- a/src/installer/pkg/packaging/dir.proj +++ b/src/installer/pkg/packaging/dir.proj @@ -5,10 +5,11 @@ - GenerateVersionBadge; - GenerateCompressedFiles; - GenerateNugetPackages; - GenerateInstallers; + GenerateVersionBadge; + GenerateCompressedFiles; + GenerateNugetPackages; + GenerateInstallers; + GenerateProjectInstallers; @@ -164,6 +165,11 @@ + + + + + diff --git a/src/installer/pkg/packaging/dir.props b/src/installer/pkg/packaging/dir.props index 63f8f6f..b69f196 100644 --- a/src/installer/pkg/packaging/dir.props +++ b/src/installer/pkg/packaging/dir.props @@ -7,9 +7,12 @@ false + + + $(BuildDistroIndependentInstallers) + $(BuildDistroIndependentInstallers) - $(IntermediateOutputRootPath)packages/ $(IntermediateOutputRootPath)sharedHost/ $(IntermediateOutputRootPath)hostFxr/ $(IntermediateOutputRootPath)sharedFx/ diff --git a/src/installer/pkg/packaging/rpm/package.props b/src/installer/pkg/packaging/rpm/package.props index 2efbd93..50e7f01 100644 --- a/src/installer/pkg/packaging/rpm/package.props +++ b/src/installer/pkg/packaging/rpm/package.props @@ -12,7 +12,7 @@ $(DotnetRuntimeDependenciesPackageString)$(PackageNameSuffix) $(RuntimeDependenciesRpmPkgName.ToLower()) - $(MSBuildThisFileDirectory)/templates + $(RpmTemplatesDir) $(MSBuildThisFileDirectory)/scripts diff --git a/src/installer/pkg/packaging/rpm/package.targets b/src/installer/pkg/packaging/rpm/package.targets index c4f19e4..e590351 100644 --- a/src/installer/pkg/packaging/rpm/package.targets +++ b/src/installer/pkg/packaging/rpm/package.targets @@ -5,12 +5,6 @@ - - true - true - true - - @@ -387,22 +381,4 @@ - - - - - - - - - - false - true - - - - - diff --git a/src/installer/pkg/projects/dir.props b/src/installer/pkg/projects/dir.props index caad2a1..375b5c6 100644 --- a/src/installer/pkg/projects/dir.props +++ b/src/installer/pkg/projects/dir.props @@ -5,6 +5,13 @@ AnyCPU obj/$(Configuration)/ $(IntermediateOutputPath)$(NuGetRuntimeIdentifier)/ + + + $(OSPlatformConfig) + $(BaseIntermediateOutputPath)$(InstallerSourceOSPlatformConfig)\$(MSBuildProjectName)\ true @@ -19,6 +26,11 @@ true + + + $(MSBuildProjectName) + + @@ -65,7 +77,7 @@ - true + targeting - $(IntermediateOutputPath)layout/ + + $(InstallerSourceIntermediateOutputDir)layout/$(FrameworkPackType)/ + + $(ShortFrameworkName)-targeting-pack + + + + $(InstallerName)-$(PackageNameSuffix) + $(VersionedInstallerName.ToLowerInvariant()) + 1 + + $(ProductionVersion) + + + + $(ProductionVersion)~$(VersionSuffix) + + + + 0.1.$(VersionSuffix) + $([System.String]::Copy('$(InstallerPackageRelease)').Replace('-', '_')) + + + + $(PackagesIntermediateDir)$(InstallerName)/$(InstallerPackageVersion)/ + + $(ProductMoniker) + $(SharedFrameworkNugetVersion)-$(TargetArchitecture) + + + $(AssetOutputPath)$(InstallerName)-$(InstallerBuildPart)$(InstallerExtension) @@ -168,5 +210,17 @@ + + + + + \ No newline at end of file diff --git a/src/installer/pkg/projects/dir.targets b/src/installer/pkg/projects/dir.targets index 552318f..86ddb62 100644 --- a/src/installer/pkg/projects/dir.targets +++ b/src/installer/pkg/projects/dir.targets @@ -29,7 +29,10 @@ --> - + + + + @@ -209,48 +212,18 @@ - - - $(PackageOutputPath)$(Id).$(PackageVersion).nupkg - - - - - - - - - - - - - - - - + + + + + + + diff --git a/src/installer/pkg/projects/dir.traversal.targets b/src/installer/pkg/projects/dir.traversal.targets index c9c88c7..3597ba5 100644 --- a/src/installer/pkg/projects/dir.traversal.targets +++ b/src/installer/pkg/projects/dir.traversal.targets @@ -9,10 +9,17 @@ use this to build runtime packages, for example. --> + BeforeTargets="Build;GenerateInstallers"> - + + + + + + + + + + + + + + + + + + $(LayoutDirectory)debian_config.json + $(InstallerIntermediatesDir)out-deb + + -i $(LayoutDirectory) + $(DebToolArgs) -o $(DebIntermediatesDir) + $(DebToolArgs) -n $(VersionedInstallerName) + $(DebToolArgs) -v $(InstallerPackageVersion) + + + + + + + + + + + + + + + + + + + + + + + + + + + + $(LayoutDirectory)rpm_config.json + $(InstallerIntermediatesDir)out-rpm + + + $(LayoutDirectory)templates/ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + $(InstallerIntermediatesDir)/debianLayoutDirectory/ + $(LayoutDirectory)$ + $(LayoutDirectory)package_root + $(LayoutDirectory)samples + $(LayoutDirectory)docs + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + $([System.IO.File]::ReadAllText('$(ProjectDir)LICENSE.TXT').Replace('%0A', '\n').Replace('"', '\"')) + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + $(PackageOutputPath)$(Id).$(PackageVersion).nupkg + + + + + + + + + + + + + diff --git a/src/installer/pkg/projects/installer.builds b/src/installer/pkg/projects/installer.builds new file mode 100644 index 0000000..6f29147 --- /dev/null +++ b/src/installer/pkg/projects/installer.builds @@ -0,0 +1,10 @@ + + + + + + + + + + diff --git a/src/installer/pkg/projects/netcoreapp/pkg/dir.props b/src/installer/pkg/projects/netcoreapp/pkg/dir.props index 185c56b..83db445 100644 --- a/src/installer/pkg/projects/netcoreapp/pkg/dir.props +++ b/src/installer/pkg/projects/netcoreapp/pkg/dir.props @@ -2,6 +2,7 @@ true + dotnet diff --git a/src/installer/pkg/projects/windowsdesktop/pkg/dir.props b/src/installer/pkg/projects/windowsdesktop/pkg/dir.props index 4157e3c..2d57366 100644 --- a/src/installer/pkg/projects/windowsdesktop/pkg/dir.props +++ b/src/installer/pkg/projects/windowsdesktop/pkg/dir.props @@ -2,10 +2,16 @@ true + windowsdesktop + + false + false + + diff --git a/tools-local/scripts/dev-build-deb-docker.sh b/tools-local/scripts/dev-build-deb-docker.sh new file mode 100755 index 0000000..4d88fc0 --- /dev/null +++ b/tools-local/scripts/dev-build-deb-docker.sh @@ -0,0 +1,101 @@ +#!/usr/bin/env bash +# +# Copyright (c) .NET Foundation and contributors. All rights reserved. +# Licensed under the MIT license. See LICENSE file in the project root for full license information. +# + +# WARNING: This utility is not used by infra and very likely to be out of date. + +# This is a simple dev utility to easily perform clean builds for Debian and RPM +# package development. It emulates the official build, first producing a +# portable build using some other image, then using Deb/RPM build images to +# package up the bits. +# +# Run this script from the root of the repository. + +set -uex + +skipPortable= + +while [[ $# > 0 ]]; do + opt="$(echo "$1" | awk '{print tolower($0)}')" + case "$opt" in + --skip-portable) + skipPortable=true + ;; + *) + echo "Invalid argument: $1" + exit 1 + ;; + esac + shift +done + +containerized() { + image=$1 + shift + docker run -it --rm \ + -u="$(id -u):$(id -g)" \ + -e HOME=/work/.container-home \ + -v "$(pwd):/work:z" \ + -w "/work" \ + "$image" \ + "$@" +} + +package() { + name=$1 + shift + image=$1 + shift + type=$1 + shift + queryCommand=$1 + shift + + containerized "$image" \ + Tools/msbuild.sh \ + build.proj \ + /t:BuildTraversalBuildDependencies \ + /p:ConfigurationGroup=Release \ + /p:OSGroup=Linux \ + /p:PortableBuild=false \ + /p:TargetArchitecture=x64 \ + "/bl:bin/msbuild.$name.traversaldependencies.binlog" + + containerized "$image" \ + Tools/msbuild.sh \ + src/pkg/packaging/dir.proj \ + /p:UsePrebuiltPortableBinariesForInstallers=true \ + /p:SharedFrameworkPublishDir=/work/bin/obj/linux-x64.Release/sharedFrameworkPublish/ \ + /p:InstallerSourceOSPlatformConfig=linux-x64.Release \ + /p:GenerateProjectInstallers=true \ + /p:ConfigurationGroup=Release \ + /p:OSGroup=Linux \ + /p:PortableBuild=false \ + /p:TargetArchitecture=x64 \ + "/bl:bin/msbuild.$name.installers.binlog" + + containerized "$image" \ + find bin/*Release/ \ + -iname "*.$type" \ + -exec printf "\n{}\n========\n" \; \ + -exec $queryCommand '{}' \; \ + > "info-$type.txt" +} + +[ "$skipPortable" ] || containerized microsoft/dotnet-buildtools-prereqs:centos-7-b46d863-20180719033416 \ + ./build.sh \ + -skiptests=true \ + -ConfigurationGroup=Release \ + -PortableBuild=true \ + -strip-symbols \ + -TargetArchitecture=x64 \ + -- \ + /bl:bin/msbuild.portable.binlog + +ubuntu=microsoft/dotnet-buildtools-prereqs:ubuntu-14.04-debpkg-e5cf912-20175003025046 +rhel=microsoft/dotnet-buildtools-prereqs:rhel-7-rpmpkg-c982313-20174116044113 + +package ubuntu $ubuntu deb "dpkg-deb -I" +package rhel $rhel rpm "rpm -qpiR" diff --git a/tools-local/tasks/BuildFPMToolPreReqs.cs b/tools-local/tasks/BuildFPMToolPreReqs.cs index d04cb8c..f0c9fe8 100644 --- a/tools-local/tasks/BuildFPMToolPreReqs.cs +++ b/tools-local/tasks/BuildFPMToolPreReqs.cs @@ -1,14 +1,14 @@ // Copyright (c) .NET Foundation and contributors. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. +using Microsoft.Build.Framework; +using Newtonsoft.Json; +using Newtonsoft.Json.Linq; using System; using System.Collections.Generic; -using Microsoft.Build.Utilities; -using Microsoft.Build.Framework; using System.IO; using System.Linq; using System.Text; -using Newtonsoft.Json; namespace Microsoft.DotNet.Build.Tasks { @@ -61,7 +61,7 @@ namespace Microsoft.DotNet.Build.Tasks } catch (Exception e) { - Log.LogError("Exception while processing RPM paramters: " + e.Message); + Log.LogErrorFromException(e, true); } return !Log.HasLoggedErrors; @@ -95,7 +95,7 @@ namespace Microsoft.DotNet.Build.Tasks } } - public void UpdateCopyRight(ConfigJson configJson) + private void UpdateCopyRight(ConfigJson configJson) { try { @@ -152,7 +152,31 @@ namespace Microsoft.DotNet.Build.Tasks // Build the list of dependencies as -d -d if (configJson.Rpm_Dependencies != null) { - foreach (RpmDependency rpmdep in configJson.Rpm_Dependencies) + IEnumerable dependencies; + + switch (configJson.Rpm_Dependencies) + { + case JArray dependencyArray: + dependencies = dependencyArray.ToObject(); + break; + + case JObject dependencyDictionary: + dependencies = dependencyDictionary + .ToObject>() + .Select(pair => new RpmDependency + { + Package_Name = pair.Key, + Package_Version = pair.Value + }); + break; + + default: + throw new ArgumentException( + "Expected 'rpm_dependencies' to be JArray or JObject, but found " + + configJson.Rpm_Dependencies.Type); + } + + foreach (RpmDependency rpmdep in dependencies) { string dependency = ""; if (rpmdep.Package_Name != "") @@ -167,7 +191,10 @@ namespace Microsoft.DotNet.Build.Tasks dependency = string.Concat(rpmdep.Package_Name, " >= ", rpmdep.Package_Version); } } - if (dependency != "") parameters.Add(string.Concat("-d ", EscapeArg(dependency))); + if (dependency != "") + { + parameters.Add(string.Concat("-d ", EscapeArg(dependency))); + } } } @@ -267,54 +294,58 @@ namespace Microsoft.DotNet.Build.Tasks } return false; } - } - /// - /// Model classes for reading and storing the JSON. - /// - public class ConfigJson - { - public string Maintainer_Name { get; set; } - public string Maintainer_Email { get; set; } - public string Vendor { get; set; } - public string Package_Name { get; set; } - public string Install_Root { get; set; } - public string Install_Doc { get; set; } - public string Install_Man { get; set; } - public string Short_Description { get; set; } - public string Long_Description { get; set; } - public string Homepage { get; set; } - public string CopyRight { get; set; } - public Release Release { get; set; } - public Control Control { get; set; } - public License License { get; set; } - public List Rpm_Dependencies { get; set; } - public List Package_Conflicts { get; set; } - public List Directories { get; set; } - public string After_Install_Source { get; set; } - public string After_Remove_Source { get; set; } - } - public class Release - { - public string Package_Version { get; set; } - public string Package_Revision { get; set; } - public string Urgency { get; set; } - public string Changelog_Message { get; set; } - } - public class Control - { - public string Priority { get; set; } - public string Section { get; set; } - public string Architecture { get; set; } - } - public class License - { - public string Type { get; set; } - public string Full_Text { get; set; } - } - public class RpmDependency - { - public string Package_Name { get; set; } - public string Package_Version { get; set; } + /// + /// Model classes for reading and storing the JSON. + /// + private class ConfigJson + { + public string Maintainer_Name { get; set; } + public string Maintainer_Email { get; set; } + public string Vendor { get; set; } + public string Package_Name { get; set; } + public string Install_Root { get; set; } + public string Install_Doc { get; set; } + public string Install_Man { get; set; } + public string Short_Description { get; set; } + public string Long_Description { get; set; } + public string Homepage { get; set; } + public string CopyRight { get; set; } + public Release Release { get; set; } + public Control Control { get; set; } + public License License { get; set; } + public JContainer Rpm_Dependencies { get; set; } + public List Package_Conflicts { get; set; } + public List Directories { get; set; } + public string After_Install_Source { get; set; } + public string After_Remove_Source { get; set; } + } + + private class Release + { + public string Package_Version { get; set; } + public string Package_Revision { get; set; } + public string Urgency { get; set; } + public string Changelog_Message { get; set; } + } + + private class Control + { + public string Priority { get; set; } + public string Section { get; set; } + public string Architecture { get; set; } + } + + private class License + { + public string Type { get; set; } + public string Full_Text { get; set; } + } + + private class RpmDependency + { + public string Package_Name { get; set; } + public string Package_Version { get; set; } + } } } diff --git a/tools-local/tasks/GenerateJsonObjectString.cs b/tools-local/tasks/GenerateJsonObjectString.cs new file mode 100644 index 0000000..558c4b1 --- /dev/null +++ b/tools-local/tasks/GenerateJsonObjectString.cs @@ -0,0 +1,145 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using Microsoft.Build.Framework; +using System; +using System.IO; +using System.Linq; +using System.Text; + +namespace Microsoft.DotNet.Build.Tasks +{ + public class GenerateJsonObjectString : BuildTask + { + private static readonly string __indent1 = new string(' ', 4); + private static readonly string __indent2 = new string(' ', 8); + + /// + /// Properties to include. If multiple properties have the same name, each property value is + /// included in an array. Only specify one value metadata: the first value is used. + /// + /// %(Identity): Name of the property. + /// %(String): String value of the property. This task adds quotes around it in the JSON. + /// %(Object): Object value of the property. Create this with a nested call to this task. + /// + [Required] + public ITaskItem[] Properties { get; set; } + + /// + /// If set, also write the output JSON string to this file. + /// + public string TargetFile { get; set; } + + [Output] + public string Json { get; set; } + + public override bool Execute() + { + var result = new StringBuilder(); + result.AppendLine("{"); + + bool firstProperty = true; + + foreach (var group in Properties.GroupBy(item => item.ItemSpec)) + { + if (firstProperty) + { + firstProperty = false; + } + else + { + result.AppendLine(","); + } + + result.Append(__indent1); + result.Append("\""); + result.Append(group.Key); + result.Append("\": "); + + if (group.Count() == 1) + { + ITaskItem item = group.First(); + WriteProperty(result, item, __indent1); + } + else + { + result.AppendLine("["); + + bool firstArrayLine = true; + + foreach (ITaskItem item in group) + { + if (firstArrayLine) + { + firstArrayLine = false; + } + else + { + result.AppendLine(","); + } + + result.Append(__indent2); + WriteProperty(result, item, __indent2); + } + + result.AppendLine(); + result.Append(__indent1); + result.Append("]"); + } + } + + result.AppendLine(); + result.AppendLine("}"); + + Json = result.ToString(); + + if (!string.IsNullOrEmpty(TargetFile)) + { + Directory.CreateDirectory(Path.GetDirectoryName(TargetFile)); + File.WriteAllText(TargetFile, Json); + } + + return !Log.HasLoggedErrors; + } + + private void WriteProperty(StringBuilder result, ITaskItem item, string indent) + { + string stringValue = item.GetMetadata("String"); + string objectValue = item.GetMetadata("Object"); + + if (!string.IsNullOrEmpty(stringValue)) + { + result.Append("\""); + result.Append(stringValue); + result.Append("\""); + } + else if (!string.IsNullOrEmpty(objectValue)) + { + bool firstObjectLine = true; + + foreach (var line in objectValue.Split( + new[] {Environment.NewLine}, + StringSplitOptions.RemoveEmptyEntries)) + { + if (firstObjectLine) + { + firstObjectLine = false; + } + else + { + result.AppendLine(); + result.Append(indent); + } + + result.Append(line); + } + } + else + { + Log.LogError($"Item '{item.ItemSpec}' has no String or Object value."); + result.Append("null"); + } + } + } +}