From: David Wrighton Date: Fri, 14 May 2021 00:39:04 +0000 (-0700) Subject: Move copy of Crossgen2 tasks/targets to runtime repo and build composite image for... X-Git-Tag: submit/tizen/20210909.063632~1392 X-Git-Url: http://review.tizen.org/git/?a=commitdiff_plain;h=c538a9b39c5171e9f4ff877daf565a7061abe54a;p=platform%2Fupstream%2Fdotnet%2Fruntime.git Move copy of Crossgen2 tasks/targets to runtime repo and build composite image for runtime repo in parallel path (#52579) * Move all crossgen2 tasks/targets used in Runtime repo to be local to the repo * Move ResolveReadyToRunCompilers task into the Crossgen2Tasks dll * Create composite bundle and package - Note that it doesn't build properly, but hopefully that's a crossgen2 task bug * Composite image production * Refactor how composite image creation works - Move it to allow mixed composite and non-composite r2r in the same build, controlled by the PublishReadyToRunCompositeExclusions list - Enhance the accuracy of how msbuild tracks which files are input vs reference for composite images - Implement the R2R exclusion list for composite images - Remove the always on usage of --inputbubble when compiling a composite image * - Add a scheme to perform public signing with a PublicKeyToken to a composite of the composite image if required - Required adding a new parameter to crossgen2 --- diff --git a/Directory.Build.props b/Directory.Build.props index 295fb52..77b581d 100644 --- a/Directory.Build.props +++ b/Directory.Build.props @@ -75,6 +75,8 @@ $([MSBuild]::NormalizePath('$(ArtifactsBinDir)', 'installer.tasks', 'Debug', '$(NetCoreAppToolCurrent)', 'installer.tasks.dll')) $([MSBuild]::NormalizePath('$(ArtifactsBinDir)', 'installer.tasks', 'Debug', 'net461', 'installer.tasks.dll')) + $([MSBuild]::NormalizePath('$(ArtifactsBinDir)', 'Crossgen2Tasks', 'Debug', '$(NetCoreAppToolCurrent)', 'Microsoft.NET.CrossGen.props')) + $([MSBuild]::NormalizePath('$(ArtifactsBinDir)', 'Crossgen2Tasks', 'Debug', '$(NetCoreAppToolCurrent)', 'Microsoft.NET.CrossGen.targets')) $([MSBuild]::NormalizePath('$(AppleAppBuilderDir)', 'AppleAppBuilder.dll')) $([MSBuild]::NormalizePath('$(AndroidAppBuilderDir)', 'AndroidAppBuilder.dll')) $([MSBuild]::NormalizePath('$(WasmAppBuilderDir)', 'WasmAppBuilder.dll')) diff --git a/eng/Subsets.props b/eng/Subsets.props index 24b4bd2..f53c5f4 100644 --- a/eng/Subsets.props +++ b/eng/Subsets.props @@ -300,6 +300,10 @@ + + + + diff --git a/src/coreclr/tools/aot/ILCompiler.ReadyToRun/Compiler/CompositeImageSettings.cs b/src/coreclr/tools/aot/ILCompiler.ReadyToRun/Compiler/CompositeImageSettings.cs new file mode 100644 index 0000000..b9e720c --- /dev/null +++ b/src/coreclr/tools/aot/ILCompiler.ReadyToRun/Compiler/CompositeImageSettings.cs @@ -0,0 +1,14 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections.Immutable; + +namespace ILCompiler +{ + public class CompositeImageSettings + { + public ImmutableArray PublicKey; + public Version AssemblyVersion; + } +} \ No newline at end of file diff --git a/src/coreclr/tools/aot/ILCompiler.ReadyToRun/Compiler/DependencyAnalysis/ReadyToRun/CopiedCorHeaderNode.cs b/src/coreclr/tools/aot/ILCompiler.ReadyToRun/Compiler/DependencyAnalysis/ReadyToRun/CopiedCorHeaderNode.cs index 50a44f4..dc69bfd 100644 --- a/src/coreclr/tools/aot/ILCompiler.ReadyToRun/Compiler/DependencyAnalysis/ReadyToRun/CopiedCorHeaderNode.cs +++ b/src/coreclr/tools/aot/ILCompiler.ReadyToRun/Compiler/DependencyAnalysis/ReadyToRun/CopiedCorHeaderNode.cs @@ -150,7 +150,15 @@ namespace ILCompiler.DependencyAnalysis.ReadyToRun builder.EmitReloc(factory.ManifestMetadataTable, RelocType.IMAGE_REL_SYMBOL_SIZE); // Flags - builder.EmitUInt(0); + if (factory.CompositeImageSettings.PublicKey != null) + { + const uint COMIMAGE_FLAGS_STRONGNAMESIGNED = 8; + builder.EmitUInt(COMIMAGE_FLAGS_STRONGNAMESIGNED); + } + else + { + builder.EmitUInt(0); + } // Entrypoint builder.EmitInt(0); diff --git a/src/coreclr/tools/aot/ILCompiler.ReadyToRun/Compiler/DependencyAnalysis/ReadyToRun/ManifestMetadataTableNode.cs b/src/coreclr/tools/aot/ILCompiler.ReadyToRun/Compiler/DependencyAnalysis/ReadyToRun/ManifestMetadataTableNode.cs index ff85543..80f80b2 100644 --- a/src/coreclr/tools/aot/ILCompiler.ReadyToRun/Compiler/DependencyAnalysis/ReadyToRun/ManifestMetadataTableNode.cs +++ b/src/coreclr/tools/aot/ILCompiler.ReadyToRun/Compiler/DependencyAnalysis/ReadyToRun/ManifestMetadataTableNode.cs @@ -189,14 +189,34 @@ namespace ILCompiler.DependencyAnalysis.ReadyToRun MetadataBuilder metadataBuilder = new MetadataBuilder(); + AssemblyHashAlgorithm hashAlgorithm = AssemblyHashAlgorithm.None; + BlobHandle publicKeyBlob = default(BlobHandle); + AssemblyFlags manifestAssemblyFlags = default(AssemblyFlags); + Version manifestAssemblyVersion = new Version(0, 0, 0, 0); + + if ((factory.CompositeImageSettings != null) && factory.CompilationModuleGroup.IsCompositeBuildMode) + { + if (factory.CompositeImageSettings.PublicKey != null) + { + hashAlgorithm = AssemblyHashAlgorithm.Sha1; + publicKeyBlob = metadataBuilder.GetOrAddBlob(factory.CompositeImageSettings.PublicKey); + manifestAssemblyFlags |= AssemblyFlags.PublicKey; + } + + if (factory.CompositeImageSettings.AssemblyVersion != null) + { + manifestAssemblyVersion = factory.CompositeImageSettings.AssemblyVersion; + } + } + string manifestMetadataAssemblyName = "ManifestMetadata"; metadataBuilder.AddAssembly( metadataBuilder.GetOrAddString(manifestMetadataAssemblyName), - new Version(0, 0, 0, 0), + manifestAssemblyVersion, culture: default(StringHandle), - publicKey: default(BlobHandle), - flags: default(AssemblyFlags), - hashAlgorithm: AssemblyHashAlgorithm.None); + publicKey: publicKeyBlob, + flags: manifestAssemblyFlags, + hashAlgorithm: hashAlgorithm); metadataBuilder.AddModule( 0, diff --git a/src/coreclr/tools/aot/ILCompiler.ReadyToRun/Compiler/DependencyAnalysis/ReadyToRunCodegenNodeFactory.cs b/src/coreclr/tools/aot/ILCompiler.ReadyToRun/Compiler/DependencyAnalysis/ReadyToRunCodegenNodeFactory.cs index 4ffb3b7..1a577b3 100644 --- a/src/coreclr/tools/aot/ILCompiler.ReadyToRun/Compiler/DependencyAnalysis/ReadyToRunCodegenNodeFactory.cs +++ b/src/coreclr/tools/aot/ILCompiler.ReadyToRun/Compiler/DependencyAnalysis/ReadyToRunCodegenNodeFactory.cs @@ -61,6 +61,8 @@ namespace ILCompiler.DependencyAnalysis public MetadataManager MetadataManager { get; } + public CompositeImageSettings CompositeImageSettings { get; set; } + public bool MarkingComplete => _markingComplete; public void SetMarkingComplete() diff --git a/src/coreclr/tools/aot/ILCompiler.ReadyToRun/Compiler/ReadyToRunCodegenCompilationBuilder.cs b/src/coreclr/tools/aot/ILCompiler.ReadyToRun/Compiler/ReadyToRunCodegenCompilationBuilder.cs index cd77cbe..de5ca06 100644 --- a/src/coreclr/tools/aot/ILCompiler.ReadyToRun/Compiler/ReadyToRunCodegenCompilationBuilder.cs +++ b/src/coreclr/tools/aot/ILCompiler.ReadyToRun/Compiler/ReadyToRunCodegenCompilationBuilder.cs @@ -41,6 +41,7 @@ namespace ILCompiler private ReadyToRunFileLayoutAlgorithm _r2rFileLayoutAlgorithm; private int _customPESectionAlignment; private bool _verifyTypeAndFieldLayout; + private CompositeImageSettings _compositeImageSettings; private string _jitPath; private string _outputFile; @@ -205,6 +206,12 @@ namespace ILCompiler return this; } + public ReadyToRunCodegenCompilationBuilder UseCompositeImageSettings(CompositeImageSettings compositeImageSettings) + { + _compositeImageSettings = compositeImageSettings; + return this; + } + public override ICompilation ToCompilation() { // TODO: only copy COR headers for single-assembly build and for composite build with embedded MSIL @@ -249,6 +256,8 @@ namespace ILCompiler win32Resources, flags); + factory.CompositeImageSettings = _compositeImageSettings; + IComparer> comparer = new SortableDependencyNode.ObjectNodeComparer(new CompilerComparer()); DependencyAnalyzerBase graph = CreateDependencyGraph(factory, comparer); diff --git a/src/coreclr/tools/aot/ILCompiler.ReadyToRun/ILCompiler.ReadyToRun.csproj b/src/coreclr/tools/aot/ILCompiler.ReadyToRun/ILCompiler.ReadyToRun.csproj index 0175a47..8c387fa 100644 --- a/src/coreclr/tools/aot/ILCompiler.ReadyToRun/ILCompiler.ReadyToRun.csproj +++ b/src/coreclr/tools/aot/ILCompiler.ReadyToRun/ILCompiler.ReadyToRun.csproj @@ -109,6 +109,7 @@ + diff --git a/src/coreclr/tools/aot/crossgen2/CommandLineOptions.cs b/src/coreclr/tools/aot/crossgen2/CommandLineOptions.cs index f5a6846..8933085 100644 --- a/src/coreclr/tools/aot/crossgen2/CommandLineOptions.cs +++ b/src/coreclr/tools/aot/crossgen2/CommandLineOptions.cs @@ -31,6 +31,7 @@ namespace ILCompiler public bool CompileBubbleGenerics; public bool Verbose; public bool Composite; + public string CompositeKeyFile; public bool CompileNoMethods; public bool EmbedPgoData; public bool OutNearInput; @@ -101,6 +102,7 @@ namespace ILCompiler syntax.DefineOption("Ot|optimize-time", ref OptimizeTime, SR.OptimizeSpeedOption); syntax.DefineOption("inputbubble", ref InputBubble, SR.InputBubbleOption); syntax.DefineOption("composite", ref Composite, SR.CompositeBuildMode); + syntax.DefineOption("compositekeyfile", ref CompositeKeyFile, SR.CompositeKeyFile); syntax.DefineOption("compile-no-methods", ref CompileNoMethods, SR.CompileNoMethodsOption); syntax.DefineOption("out-near-input", ref OutNearInput, SR.OutNearInputOption); syntax.DefineOption("single-file-compilation", ref SingleFileCompilation, SR.SingleFileCompilationOption); diff --git a/src/coreclr/tools/aot/crossgen2/Program.cs b/src/coreclr/tools/aot/crossgen2/Program.cs index 836c08c..b5d93d1 100644 --- a/src/coreclr/tools/aot/crossgen2/Program.cs +++ b/src/coreclr/tools/aot/crossgen2/Program.cs @@ -3,6 +3,7 @@ using System; using System.Collections.Generic; +using System.Collections.Immutable; using System.IO; using System.Reflection.Metadata; using System.Reflection.PortableExecutable; @@ -634,6 +635,19 @@ namespace ILCompiler optimizationMode = ((EcmaAssembly)inputModules[0].Assembly).HasOptimizationsDisabled() ? OptimizationMode.None : OptimizationMode.Blended; } + CompositeImageSettings compositeImageSettings = new CompositeImageSettings(); + + if (_commandLineOptions.CompositeKeyFile != null) + { + ImmutableArray compositeStrongNameKey = File.ReadAllBytes(_commandLineOptions.CompositeKeyFile).ToImmutableArray(); + if (!IsValidPublicKey(compositeStrongNameKey)) + { + throw new Exception(string.Format(SR.ErrorCompositeKeyFileNotPublicKey)); + } + + compositeImageSettings.PublicKey = compositeStrongNameKey; + } + // // Compile // @@ -659,6 +673,7 @@ namespace ILCompiler .UseParallelism(_commandLineOptions.Parallelism) .UseProfileData(profileDataManager) .FileLayoutAlgorithms(_methodLayout, _fileLayout) + .UseCompositeImageSettings(compositeImageSettings) .UseJitPath(_commandLineOptions.JitPath) .UseInstructionSetSupport(instructionSetSupport) .UseCustomPESectionAlignment(_commandLineOptions.CustomPESectionAlignment) @@ -827,6 +842,128 @@ namespace ILCompiler return false; } + private enum AlgorithmClass + { + Signature = 1, + Hash = 4, + } + + private enum AlgorithmSubId + { + Sha1Hash = 4, + MacHash = 5, + RipeMdHash = 6, + RipeMd160Hash = 7, + Ssl3ShaMD5Hash = 8, + HmacHash = 9, + Tls1PrfHash = 10, + HashReplacOwfHash = 11, + Sha256Hash = 12, + Sha384Hash = 13, + Sha512Hash = 14, + } + + private struct AlgorithmId + { + // From wincrypt.h + private const int AlgorithmClassOffset = 13; + private const int AlgorithmClassMask = 0x7; + private const int AlgorithmSubIdOffset = 0; + private const int AlgorithmSubIdMask = 0x1ff; + + private readonly uint _flags; + + public const int RsaSign = 0x00002400; + public const int Sha = 0x00008004; + + public bool IsSet + { + get { return _flags != 0; } + } + + public AlgorithmClass Class + { + get { return (AlgorithmClass)((_flags >> AlgorithmClassOffset) & AlgorithmClassMask); } + } + + public AlgorithmSubId SubId + { + get { return (AlgorithmSubId)((_flags >> AlgorithmSubIdOffset) & AlgorithmSubIdMask); } + } + + public AlgorithmId(uint flags) + { + _flags = flags; + } + } + + private static readonly ImmutableArray s_ecmaKey = ImmutableArray.Create(new byte[] { 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0 }); + + private const int SnPublicKeyBlobSize = 13; + + // From wincrypt.h + private const byte PublicKeyBlobId = 0x06; + private const byte PrivateKeyBlobId = 0x07; + + // internal for testing + internal const int s_publicKeyHeaderSize = SnPublicKeyBlobSize - 1; + + // From StrongNameInternal.cpp + // Checks to see if a public key is a valid instance of a PublicKeyBlob as + // defined in StongName.h + internal static bool IsValidPublicKey(ImmutableArray blob) + { + // The number of public key bytes must be at least large enough for the header and one byte of data. + if (blob.IsDefault || blob.Length < s_publicKeyHeaderSize + 1) + { + return false; + } + + var blobReader = new BinaryReader(new MemoryStream(blob.ToArray())); + + // Signature algorithm ID + var sigAlgId = blobReader.ReadUInt32(); + // Hash algorithm ID + var hashAlgId = blobReader.ReadUInt32(); + // Size of public key data in bytes, not including the header + var publicKeySize = blobReader.ReadUInt32(); + // publicKeySize bytes of public key data + var publicKey = blobReader.ReadByte(); + + // The number of public key bytes must be the same as the size of the header plus the size of the public key data. + if (blob.Length != s_publicKeyHeaderSize + publicKeySize) + { + return false; + } + + // Check for the ECMA key, which does not obey the invariants checked below. + if (System.Linq.Enumerable.SequenceEqual(blob, s_ecmaKey)) + { + return true; + } + + // The public key must be in the wincrypto PUBLICKEYBLOB format + if (publicKey != PublicKeyBlobId) + { + return false; + } + + var signatureAlgorithmId = new AlgorithmId(sigAlgId); + if (signatureAlgorithmId.IsSet && signatureAlgorithmId.Class != AlgorithmClass.Signature) + { + return false; + } + + var hashAlgorithmId = new AlgorithmId(hashAlgId); + if (hashAlgorithmId.IsSet && (hashAlgorithmId.Class != AlgorithmClass.Hash || hashAlgorithmId.SubId < AlgorithmSubId.Sha1Hash)) + { + return false; + } + + return true; + } + + private static int Main(string[] args) { #if DEBUG diff --git a/src/coreclr/tools/aot/crossgen2/Properties/Resources.resx b/src/coreclr/tools/aot/crossgen2/Properties/Resources.resx index d94483a..bb6bcec 100644 --- a/src/coreclr/tools/aot/crossgen2/Properties/Resources.resx +++ b/src/coreclr/tools/aot/crossgen2/Properties/Resources.resx @@ -156,6 +156,12 @@ Emit a composite R2R image comprising a number of input assemblies + + KeyFile(.snk) for specifiying a Public Key for the composite image created + + + --CompositeKeyFile does not specify a valid public key + Input file(s) to compile diff --git a/src/installer/pkg/sfx/Microsoft.NETCore.App/Directory.Build.targets b/src/installer/pkg/sfx/Microsoft.NETCore.App/Directory.Build.targets index 9859603..671d6f0 100644 --- a/src/installer/pkg/sfx/Microsoft.NETCore.App/Directory.Build.targets +++ b/src/installer/pkg/sfx/Microsoft.NETCore.App/Directory.Build.targets @@ -9,7 +9,7 @@ - + diff --git a/src/installer/pkg/sfx/Microsoft.NETCore.App/Microsoft.NETCore.App.Crossgen2.sfxproj b/src/installer/pkg/sfx/Microsoft.NETCore.App/Microsoft.NETCore.App.Crossgen2.sfxproj index 6573cc7..e9cd505 100644 --- a/src/installer/pkg/sfx/Microsoft.NETCore.App/Microsoft.NETCore.App.Crossgen2.sfxproj +++ b/src/installer/pkg/sfx/Microsoft.NETCore.App/Microsoft.NETCore.App.Crossgen2.sfxproj @@ -74,8 +74,10 @@ + + diff --git a/src/installer/pkg/sfx/Microsoft.NETCore.App/Microsoft.NETCore.App.Runtime.Composite.sfxproj b/src/installer/pkg/sfx/Microsoft.NETCore.App/Microsoft.NETCore.App.Runtime.Composite.sfxproj new file mode 100644 index 0000000..950d06b --- /dev/null +++ b/src/installer/pkg/sfx/Microsoft.NETCore.App/Microsoft.NETCore.App.Runtime.Composite.sfxproj @@ -0,0 +1,37 @@ + + + + + + RuntimePack + dotnet-runtime-internal-composite + $(SharedFrameworkName).Composite + $(SharedFrameworkName).Composite.PGO + true + dotnet-runtime-composite-symbols + true + true + true + true + true + + + + + + + + + + + + + + + + + + + + + diff --git a/src/installer/pkg/sfx/Microsoft.NETCore.App/Microsoft.NETCore.App.Runtime.props b/src/installer/pkg/sfx/Microsoft.NETCore.App/Microsoft.NETCore.App.Runtime.props new file mode 100644 index 0000000..19b7ac9 --- /dev/null +++ b/src/installer/pkg/sfx/Microsoft.NETCore.App/Microsoft.NETCore.App.Runtime.props @@ -0,0 +1,147 @@ + + + true + + AddRuntimeFilesToPackage; + AddFrameworkFilesToPackage + + true + false + + false + + true + true + The .NET Shared Framework + + + + Mono + + + Mono.LLVM + + + Mono.LLVM.AOT + + + $(SharedFrameworkName).Runtime.$(RuntimeSpecificFrameworkSuffix).$(RuntimeIdentifier) + + + + + + + + + + + + + + + + + + + + + + + + + + + + runtimes/$(RuntimeIdentifier)/native + + + + tools + + + + runtimes/$(RuntimeIdentifier)/native/include/%(RecursiveDir) + + + + + runtimes/$(CoreCLRCrossTargetComponentDirName)_$(TargetArchitecture)/native + + + tools/$(CoreCLRCrossTargetComponentDirName)_$(TargetArchitecture) + + + tools/$(CoreCLRCrossTargetComponentDirName)_$(TargetArchitecture) + + + tools/$(CoreCLRCrossTargetComponentDirName)_$(TargetArchitecture) + + + tools + + + + + + + + + runtimes/$(RuntimeIdentifier)/native/%(LibrariesRuntimeFiles.NativeSubDirectory)%(RecursiveDir) + + + + + + + <_diaSymArch>$(_hostArch) + <_diaSymReaderPath>$(PkgMicrosoft_DiaSymReader_Native)/runtimes/win/native/Microsoft.DiaSymReader.Native.$(_diaSymArch).dll + + + <_diaSymTargetArch>$(TargetArchitecture) + <_diaSymTargetArch Condition="'$(TargetArchitecture)' == 'x64'">amd64 + <_diaSymReaderTargetArchPath>$(PkgMicrosoft_DiaSymReader_Native)/runtimes/win/native/Microsoft.DiaSymReader.Native.$(_diaSymTargetArch).dll + + + + + + runtimes/$(CoreCLRCrossTargetComponentDirName)_$(TargetArchitecture)/native + + + + + + + <_VersionFile Include="$(IntermediateOutputPath).version" TargetPath="shared/$(SharedFrameworkName)/$(Version)/" /> + + + + + + + + + + + + + + true + + diff --git a/src/installer/pkg/sfx/Microsoft.NETCore.App/Microsoft.NETCore.App.Runtime.sfxproj b/src/installer/pkg/sfx/Microsoft.NETCore.App/Microsoft.NETCore.App.Runtime.sfxproj index e6485dd..4fa48f4 100644 --- a/src/installer/pkg/sfx/Microsoft.NETCore.App/Microsoft.NETCore.App.Runtime.sfxproj +++ b/src/installer/pkg/sfx/Microsoft.NETCore.App/Microsoft.NETCore.App.Runtime.sfxproj @@ -12,144 +12,8 @@ dotnet-runtime-symbols NetCore.SharedFramework true - true - - AddRuntimeFilesToPackage; - AddFrameworkFilesToPackage - - true - false - - false - - true - true - The .NET Shared Framework - - Mono - - - Mono.LLVM - - - Mono.LLVM.AOT - - - $(SharedFrameworkName).Runtime.$(RuntimeSpecificFrameworkSuffix).$(RuntimeIdentifier) - - - - - - - - - - - - - - - - - - - - - - - - - - - - runtimes/$(RuntimeIdentifier)/native - - - - tools - - - - runtimes/$(RuntimeIdentifier)/native/include/%(RecursiveDir) - - - - - runtimes/$(CoreCLRCrossTargetComponentDirName)_$(TargetArchitecture)/native - - - tools/$(CoreCLRCrossTargetComponentDirName)_$(TargetArchitecture) - - - tools/$(CoreCLRCrossTargetComponentDirName)_$(TargetArchitecture) - - - tools/$(CoreCLRCrossTargetComponentDirName)_$(TargetArchitecture) - - - tools - - - - - - - - - runtimes/$(RuntimeIdentifier)/native/%(LibrariesRuntimeFiles.NativeSubDirectory)%(RecursiveDir) - - - - - - - <_diaSymArch>$(_hostArch) - <_diaSymReaderPath>$(PkgMicrosoft_DiaSymReader_Native)/runtimes/win/native/Microsoft.DiaSymReader.Native.$(_diaSymArch).dll - - - <_diaSymTargetArch>$(TargetArchitecture) - <_diaSymTargetArch Condition="'$(TargetArchitecture)' == 'x64'">amd64 - <_diaSymReaderTargetArchPath>$(PkgMicrosoft_DiaSymReader_Native)/runtimes/win/native/Microsoft.DiaSymReader.Native.$(_diaSymTargetArch).dll - - - - - - runtimes/$(CoreCLRCrossTargetComponentDirName)_$(TargetArchitecture)/native - - - - - - - <_VersionFile Include="$(IntermediateOutputPath).version" TargetPath="shared/$(SharedFrameworkName)/$(Version)/" /> - - - - - - - - - - + diff --git a/src/installer/pkg/sfx/bundle/Microsoft.NETCore.App.Composite.Bundle.bundleproj b/src/installer/pkg/sfx/bundle/Microsoft.NETCore.App.Composite.Bundle.bundleproj new file mode 100644 index 0000000..55046be --- /dev/null +++ b/src/installer/pkg/sfx/bundle/Microsoft.NETCore.App.Composite.Bundle.bundleproj @@ -0,0 +1,35 @@ + + + + true + false + dotnet-runtime-composite + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/src/tasks/Crossgen2Tasks/CommonFilePulledFromSdkRepo/BuildErrorException.cs b/src/tasks/Crossgen2Tasks/CommonFilePulledFromSdkRepo/BuildErrorException.cs new file mode 100644 index 0000000..82a886b --- /dev/null +++ b/src/tasks/Crossgen2Tasks/CommonFilePulledFromSdkRepo/BuildErrorException.cs @@ -0,0 +1,32 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Globalization; + +namespace Microsoft.NET.Build.Tasks +{ + /// + /// Represents an error that is neither avoidable in all cases nor indicative of a bug in this library. + /// It will be logged as a plain build error without the exception type or stack. + /// + internal class BuildErrorException : Exception + { + public BuildErrorException() + { + } + + public BuildErrorException(string message) : base(message) + { + } + + public BuildErrorException(string message, Exception innerException) : base(message, innerException) + { + } + + public BuildErrorException(string format, params string[] args) + : this(string.Format(CultureInfo.CurrentCulture, format, args)) + { + } + } +} diff --git a/src/tasks/Crossgen2Tasks/CommonFilePulledFromSdkRepo/LogAdapter.cs b/src/tasks/Crossgen2Tasks/CommonFilePulledFromSdkRepo/LogAdapter.cs new file mode 100644 index 0000000..de6c6b8 --- /dev/null +++ b/src/tasks/Crossgen2Tasks/CommonFilePulledFromSdkRepo/LogAdapter.cs @@ -0,0 +1,82 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using Microsoft.Build.Utilities; + +namespace Microsoft.NET.Build.Tasks +{ + internal sealed class LogAdapter : Logger + { + private TaskLoggingHelper _taskLogger; + + public LogAdapter(TaskLoggingHelper taskLogger) + { + _taskLogger = taskLogger; + } + + protected override void LogCore(in Message message) + { + switch (message.Level) + { + case MessageLevel.Error: + _taskLogger.LogError( + subcategory: default, + errorCode: message.Code, + helpKeyword: default, + file: message.File, + lineNumber: default, + columnNumber: default, + endLineNumber: default, + endColumnNumber: default, + message: message.Text); + break; + + case MessageLevel.Warning: + _taskLogger.LogWarning( + subcategory: default, + warningCode: message.Code, + helpKeyword: default, + file: message.File, + lineNumber: default, + columnNumber: default, + endLineNumber: default, + endColumnNumber: default, + message: message.Text); + break; + + case MessageLevel.HighImportance: + case MessageLevel.NormalImportance: + case MessageLevel.LowImportance: + if (message.Code == null && message.File == null) + { + // use shorter overload when there is no code and no file. Otherwise, msbuild + // will display: + // + // (,): message : + _taskLogger.LogMessage(message.Level.ToImportance(), message.Text); + } + else + { + _taskLogger.LogMessage( + subcategory: default, + code: message.Code, + helpKeyword: default, + file: message.File, + lineNumber: default, + columnNumber: default, + endLineNumber: default, + endColumnNumber: default, + importance: message.Level.ToImportance(), + message: message.Text); + } + break; + + default: + throw new ArgumentException( + $"Message \"{message.Code}: {message.Text}\" logged with invalid Level=${message.Level}", + paramName: nameof(message)); + } + } + } +} diff --git a/src/tasks/Crossgen2Tasks/CommonFilePulledFromSdkRepo/Logger.cs b/src/tasks/Crossgen2Tasks/CommonFilePulledFromSdkRepo/Logger.cs new file mode 100644 index 0000000..d77fed2 --- /dev/null +++ b/src/tasks/Crossgen2Tasks/CommonFilePulledFromSdkRepo/Logger.cs @@ -0,0 +1,143 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Diagnostics; +using Microsoft.Build.Framework; +using Microsoft.Build.Utilities; + +namespace Microsoft.NET.Build.Tasks +{ + /// + /// Replacement and abstraction for in our + /// build tasks. + /// + /// + /// Source compatible with usual Log.LogXxx MSBuild task code. (Subset of + /// API chosen based on actual usage in SDK, and with a deliberate goal of + /// eliminating some of the excessive overloading in TaskLoggingHelper. + /// + /// replaces the need for overloads taking over 10 + /// arguments. + /// + /// Also, string[] is used instead of object[] to avoid issues like passing + /// the importance out of order as a format argument. + /// + /// allows choosing Error/Warning/Message dynamically at a + /// single call site. + /// + /// Extracts error codes from the message prefix, and enforces that all of + /// our messages have a NETSDK code. + /// + /// Example: + /// C# + /// Log.LogError(Strings.SomethingIsWrong); + /// + /// Strings.resx: + /// Resource name: SomethingIsWrong + /// Resource value: NETSDK1234: Something is wrong. + /// + /// Results in LogCore getting a Message instance with Code="NETSDK1234" + /// and Text="Something is wrong." + /// + /// Pattern inspired by , + /// but retains completion via generated instead of + /// passing resource keys by name. + /// + /// All actual logging is deferred to subclass in , + /// which allows unit tests to verify task logging while mocking a single + /// method. adapts that to . + /// + internal abstract class Logger + { + public bool HasLoggedErrors { get; private set; } + + public void LogMessage(string format, params string[] args) + => Log(CreateMessage(MessageLevel.NormalImportance, format, args)); + + public void LogMessage(MessageImportance importance, string format, params string[] args) + => Log(CreateMessage(importance.ToLevel(), format, args)); + + public void LogWarning(string format, params string[] args) + => Log(CreateMessage(MessageLevel.Warning, format, args)); + + public void LogError(string format, params string[] args) + => Log(CreateMessage(MessageLevel.Error, format, args)); + + public void Log(in Message message) + { + HasLoggedErrors |= message.Level == MessageLevel.Error; + LogCore(message); + } + + protected abstract void LogCore(in Message message); + + private static Message CreateMessage(MessageLevel level, string format, string[] args) + { + string code; + + if (format.Length >= 12 + && format[0] == 'N' + && format[1] == 'E' + && format[2] == 'T' + && format[3] == 'S' + && format[4] == 'D' + && format[5] == 'K' + && IsAsciiDigit(format[6]) + && IsAsciiDigit(format[7]) + && IsAsciiDigit(format[8]) + && IsAsciiDigit(format[9]) + && format[10] == ':' + && format[11] == ' ') + { + code = format.Substring(0, 10); + format = format.Substring(12); + } + else + { + code = null; + } + + DebugThrowMissingOrIncorrectCode(code, format, level); + + return new Message( + level, + text: string.Format(format, args), + code: code); + } + + [Conditional("DEBUG")] + private static void DebugThrowMissingOrIncorrectCode(string code, string message, MessageLevel level) + { + // NB: This is not localized because it represents a bug in our code base, not a user error. + // To log message with external codes, use Log.Log(in Message, string[]) directly. + // It is not a Debug.Assert because it doesn't render well in unit tests. + + switch (level) + { + case MessageLevel.Error: + case MessageLevel.Warning: + if (code == null) + { + throw new ArgumentException( + "Message is not prefixed with NETSDK error code or error code is formatted incorrectly: " + + message); + } + break; + + default: + if (code != null) + { + throw new ArgumentException( + "Message is prefixed with NETSDK error, but error codes should not be used for informational messages: " + + $"{code}:{message}"); + } + break; + } + } + + private static bool IsAsciiDigit(char c) + => c >= '0' && c <= '9'; + } +} diff --git a/src/tasks/Crossgen2Tasks/CommonFilePulledFromSdkRepo/Message.cs b/src/tasks/Crossgen2Tasks/CommonFilePulledFromSdkRepo/Message.cs new file mode 100644 index 0000000..e5af900 --- /dev/null +++ b/src/tasks/Crossgen2Tasks/CommonFilePulledFromSdkRepo/Message.cs @@ -0,0 +1,25 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +namespace Microsoft.NET.Build.Tasks +{ + internal readonly struct Message + { + public readonly MessageLevel Level; + public readonly string Code; + public readonly string Text; + public readonly string File; + + public Message( + MessageLevel level, + string text, + string code = default, + string file = default) + { + Level = level; + Code = code; + Text = text; + File = file; + } + } +} diff --git a/src/tasks/Crossgen2Tasks/CommonFilePulledFromSdkRepo/MessageLevel.cs b/src/tasks/Crossgen2Tasks/CommonFilePulledFromSdkRepo/MessageLevel.cs new file mode 100644 index 0000000..ec95961 --- /dev/null +++ b/src/tasks/Crossgen2Tasks/CommonFilePulledFromSdkRepo/MessageLevel.cs @@ -0,0 +1,29 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using Microsoft.Build.Framework; + +namespace Microsoft.NET.Build.Tasks +{ + internal enum MessageLevel + { + // For efficient conversion, positive values map directly to MessageImportance: + LowImportance = MessageImportance.Low, + NormalImportance = MessageImportance.Normal, + HighImportance = MessageImportance.High, + + // And negative values are for levels that are not informational (warning/error): + Warning = -1, + Error = -2, + } + + internal static class MessageLevelExtensions + { + public static MessageLevel ToLevel(this MessageImportance importance) + => (MessageLevel)(importance); + + public static MessageImportance ToImportance(this MessageLevel level) + => level >= 0 ? (MessageImportance)level : throw new InvalidCastException(); + } +} diff --git a/src/tasks/Crossgen2Tasks/CommonFilePulledFromSdkRepo/MetadataKeys.cs b/src/tasks/Crossgen2Tasks/CommonFilePulledFromSdkRepo/MetadataKeys.cs new file mode 100644 index 0000000..e3df2f4 --- /dev/null +++ b/src/tasks/Crossgen2Tasks/CommonFilePulledFromSdkRepo/MetadataKeys.cs @@ -0,0 +1,127 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +namespace Microsoft.NET.Build.Tasks +{ + internal static class MetadataKeys + { + // General Metadata + public const string Name = "Name"; + public const string Type = "Type"; + public const string Version = "Version"; + public const string FileGroup = "FileGroup"; + public const string Path = "Path"; + public const string ResolvedPath = "ResolvedPath"; + public const string IsImplicitlyDefined = "IsImplicitlyDefined"; + public const string IsTopLevelDependency = "IsTopLevelDependency"; + public const string AllowExplicitVersion = "AllowExplicitVersion"; + public const string RelativePath = "RelativePath"; + public const string DiagnosticLevel = "DiagnosticLevel"; + + // Target Metadata + public const string RuntimeIdentifier = "RuntimeIdentifier"; + public const string TargetFrameworkMoniker = "TargetFrameworkMoniker"; + public const string TargetFramework = "TargetFramework"; + public const string FrameworkName = "FrameworkName"; + public const string FrameworkVersion = "FrameworkVersion"; + public const string IsTrimmable = "IsTrimmable"; + public const string RuntimeFrameworkName = "RuntimeFrameworkName"; + public const string RuntimePackRuntimeIdentifiers = "RuntimePackRuntimeIdentifiers"; + + // SDK Metadata + public const string SDKPackageItemSpec = "SDKPackageItemSpec"; + public const string OriginalItemSpec = "OriginalItemSpec"; + public const string SDKRootFolder = "SDKRootFolder"; + public const string ShimRuntimeIdentifier = "ShimRuntimeIdentifier"; + public const string RuntimePackAlwaysCopyLocal = "RuntimePackAlwaysCopyLocal"; + + // Foreign Keys + public const string ParentTarget = "ParentTarget"; + public const string ParentTargetLibrary = "ParentTargetLibrary"; + public const string ParentPackage = "ParentPackage"; + + // Tags + public const string Analyzer = "Analyzer"; + public const string AnalyzerLanguage = "AnalyzerLanguage"; + public const string TransitiveProjectReference = "TransitiveProjectReference"; + + // Diagnostics + public const string DiagnosticCode = "DiagnosticCode"; + public const string Message = "Message"; + public const string FilePath = "FilePath"; + public const string Severity = "Severity"; + public const string StartLine = "StartLine"; + public const string StartColumn = "StartColumn"; + public const string EndLine = "EndLine"; + public const string EndColumn = "EndColumn"; + + // Publish Target Manifest + public const string RuntimeStoreManifestNames = "RuntimeStoreManifestNames"; + + // Conflict Resolution + public const string OverriddenPackages = "OverriddenPackages"; + + // Package assets + public const string NuGetIsFrameworkReference = "NuGetIsFrameworkReference"; + public const string NuGetPackageId = "NuGetPackageId"; + public const string NuGetPackageVersion = "NuGetPackageVersion"; + public const string NuGetSourceType = "NuGetSourceType"; + public const string PathInPackage = "PathInPackage"; + public const string PackageDirectory = "PackageDirectory"; + public const string Publish = "Publish"; + + // References + public const string ExternallyResolved = "ExternallyResolved"; + public const string HintPath = "HintPath"; + public const string MSBuildSourceProjectFile = "MSBuildSourceProjectFile"; + public const string Private = "Private"; + public const string Pack = "Pack"; + public const string ReferenceSourceTarget = "ReferenceSourceTarget"; + public const string TargetPath = "TargetPath"; + public const string CopyLocal = "CopyLocal"; + + // Targeting packs + public const string PackageConflictPreferredPackages = "PackageConflictPreferredPackages"; + + // Runtime packs + public const string DropFromSingleFile = "DropFromSingleFile"; + public const string RuntimePackLabels = "RuntimePackLabels"; + public const string AdditionalFrameworkReferences = "AdditionalFrameworkReferences"; + + // Content files + public const string PPOutputPath = "PPOutputPath"; + public const string CodeLanguage = "CodeLanguage"; + public const string CopyToOutput = "CopyToOutput"; + public const string BuildAction = "BuildAction"; + public const string OutputPath = "OutputPath"; + public const string CopyToPublishDirectory = "CopyToPublishDirectory"; + public const string ExcludeFromSingleFile = "ExcludeFromSingleFile"; + + // Resource assemblies + public const string Culture = "Culture"; + // The DestinationSubDirectory is the directory containing the asset, relative to the destination folder. + public const string DestinationSubDirectory = "DestinationSubDirectory"; + + // Copy local assets + // The DestinationSubPath is the path to the asset, relative to the destination folder. + public const string DestinationSubPath = "DestinationSubPath"; + public const string AssetType = "AssetType"; + + public const string ReferenceOnly = "ReferenceOnly"; + + public const string Aliases = "Aliases"; + + // ReadyToRun + public const string DotNetHostPath = "DotNetHostPath"; + public const string JitPath = "JitPath"; + public const string TargetOS = "TargetOS"; + public const string TargetArch = "TargetArch"; + public const string DiaSymReader = "DiaSymReader"; + public const string CreatePDBCommand = "CreatePDBCommand"; + public const string OutputR2RImage = "OutputR2RImage"; + public const string OutputPDBImage = "OutputPDBImage"; + public const string EmitSymbols = "EmitSymbols"; + public const string IsVersion5 = "IsVersion5"; + public const string CreateCompositeImage = "CreateCompositeImage"; + } +} diff --git a/src/tasks/Crossgen2Tasks/CommonFilePulledFromSdkRepo/NuGetUtils.NuGet.cs b/src/tasks/Crossgen2Tasks/CommonFilePulledFromSdkRepo/NuGetUtils.NuGet.cs new file mode 100644 index 0000000..c104b28 --- /dev/null +++ b/src/tasks/Crossgen2Tasks/CommonFilePulledFromSdkRepo/NuGetUtils.NuGet.cs @@ -0,0 +1,101 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.IO; +using System.Linq; +using NuGet.Frameworks; +using NuGet.Packaging.Core; +using NuGet.ProjectModel; +using NuGet.RuntimeModel; + +namespace Microsoft.NET.Build.Tasks +{ + internal static partial class NuGetUtils + { + public static bool IsPlaceholderFile(string path) + { + // PERF: avoid allocations here as we check this for every file in project.assets.json + if (!path.EndsWith("_._", StringComparison.Ordinal)) + { + return false; + } + + if (path.Length == 3) + { + return true; + } + + char separator = path[path.Length - 4]; + return separator == '\\' || separator == '/'; + } + + public static string GetLockFileLanguageName(string projectLanguage) + { + switch (projectLanguage) + { + case "C#": return "cs"; + case "F#": return "fs"; + default: return projectLanguage?.ToLowerInvariant(); + } + } + + public static NuGetFramework ParseFrameworkName(string frameworkName) + { + return frameworkName == null ? null : NuGetFramework.Parse(frameworkName); + } + + public static bool IsApplicableAnalyzer(string file, string projectLanguage) + { + // This logic is preserved from previous implementations. + // See https://github.com/NuGet/Home/issues/6279#issuecomment-353696160 for possible issues with it. + + bool IsAnalyzer() + { + return file.StartsWith("analyzers", StringComparison.Ordinal) + && file.EndsWith(".dll", StringComparison.OrdinalIgnoreCase) + && !file.EndsWith(".resources.dll", StringComparison.OrdinalIgnoreCase); + } + + bool CS() => file.Contains("/cs/", StringComparison.OrdinalIgnoreCase); + bool VB() => file.Contains("/vb/", StringComparison.OrdinalIgnoreCase); + + bool FileMatchesProjectLanguage() + { + switch (projectLanguage) + { + case "C#": + return CS() || !VB(); + + case "VB": + return VB() || !CS(); + + default: + return false; + } + } + + return IsAnalyzer() && FileMatchesProjectLanguage(); + } + + public static string GetBestMatchingRid(RuntimeGraph runtimeGraph, string runtimeIdentifier, + IEnumerable availableRuntimeIdentifiers, out bool wasInGraph) + { + wasInGraph = runtimeGraph.Runtimes.ContainsKey(runtimeIdentifier); + + HashSet availableRids = new HashSet(availableRuntimeIdentifiers); + foreach (var candidateRuntimeIdentifier in runtimeGraph.ExpandRuntime(runtimeIdentifier)) + { + if (availableRids.Contains(candidateRuntimeIdentifier)) + { + return candidateRuntimeIdentifier; + } + } + + // No compatible RID found in availableRuntimeIdentifiers + return null; + } + } +} diff --git a/src/tasks/Crossgen2Tasks/CommonFilePulledFromSdkRepo/RuntimeGraphCache.cs b/src/tasks/Crossgen2Tasks/CommonFilePulledFromSdkRepo/RuntimeGraphCache.cs new file mode 100644 index 0000000..e74b12c --- /dev/null +++ b/src/tasks/Crossgen2Tasks/CommonFilePulledFromSdkRepo/RuntimeGraphCache.cs @@ -0,0 +1,58 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections.Generic; +using System.IO; +using System.Text; +using Microsoft.Build.Framework; +using NuGet.RuntimeModel; + +namespace Microsoft.NET.Build.Tasks +{ + internal class RuntimeGraphCache + { + private IBuildEngine4 _buildEngine; + private Logger _log; + + public RuntimeGraphCache(TaskBase task) + { + _buildEngine = task.BuildEngine4; + _log = task.Log; + } + + public RuntimeGraph GetRuntimeGraph(string runtimeJsonPath) + { + if (string.IsNullOrEmpty(runtimeJsonPath)) + { + throw new ArgumentNullException(nameof(runtimeJsonPath)); + } + if (!Path.IsPathRooted(runtimeJsonPath)) + { + throw new BuildErrorException("Path not rooted: {0}", runtimeJsonPath); + } + + string key = GetTaskObjectKey(runtimeJsonPath); + + RuntimeGraph result; + object existingRuntimeGraphTaskObject = _buildEngine.GetRegisteredTaskObject(key, RegisteredTaskObjectLifetime.AppDomain); + if (existingRuntimeGraphTaskObject == null) + { + result = JsonRuntimeFormat.ReadRuntimeGraph(runtimeJsonPath); + + _buildEngine.RegisterTaskObject(key, result, RegisteredTaskObjectLifetime.AppDomain, true); + } + else + { + result = (RuntimeGraph)existingRuntimeGraphTaskObject; + } + + return result; + } + + private static string GetTaskObjectKey(string runtimeJsonPath) + { + return $"{nameof(RuntimeGraphCache)}:{runtimeJsonPath}"; + } + } +} diff --git a/src/tasks/Crossgen2Tasks/CommonFilePulledFromSdkRepo/TaskBase.cs b/src/tasks/Crossgen2Tasks/CommonFilePulledFromSdkRepo/TaskBase.cs new file mode 100644 index 0000000..637ef41 --- /dev/null +++ b/src/tasks/Crossgen2Tasks/CommonFilePulledFromSdkRepo/TaskBase.cs @@ -0,0 +1,114 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using Microsoft.Build.Utilities; +using Microsoft.Build.Framework; +using System.Collections.Generic; +using System.Globalization; + +namespace Microsoft.NET.Build.Tasks +{ + public abstract class TaskBase : Task + { + private Logger _logger; + + internal TaskBase(Logger logger = null) + { + _logger = logger; + } + + internal new Logger Log + { + get + { + if (_logger == null) + { + _logger = new LogAdapter(base.Log); + } + + return _logger; + } + } + + public override bool Execute() + { + try + { + ExecuteCore(); + } + catch (BuildErrorException e) + { + Log.LogError(e.Message); + } + catch (Exception e) + { + LogErrorTelemetry("taskBaseCatchException", e); + throw; + } + + return !Log.HasLoggedErrors; + } + + private void LogErrorTelemetry(string eventName, Exception e) + { + (BuildEngine as IBuildEngine5)?.LogTelemetry(eventName, new Dictionary { + {"exceptionType", e.GetType().ToString() }, + {"detail", ExceptionToStringWithoutMessage(e) }}); + } + + private static string ExceptionToStringWithoutMessage(Exception e) + { + const string AggregateException_ToString = "{0}{1}---> (Inner Exception #{2}) {3}{4}{5}"; + if (e is AggregateException aggregate) + { + string text = NonAggregateExceptionToStringWithoutMessage(aggregate); + + for (int i = 0; i < aggregate.InnerExceptions.Count; i++) + { + text = string.Format(CultureInfo.InvariantCulture, + AggregateException_ToString, + text, + Environment.NewLine, + i, + ExceptionToStringWithoutMessage(aggregate.InnerExceptions[i]), + "<---", + Environment.NewLine); + } + + return text; + } + else + { + return NonAggregateExceptionToStringWithoutMessage(e); + } + } + + private static string NonAggregateExceptionToStringWithoutMessage(Exception e) + { + string s; + const string Exception_EndOfInnerExceptionStack = "--- End of inner exception stack trace ---"; + + + s = e.GetType().ToString(); + + if (e.InnerException != null) + { + s = s + " ---> " + ExceptionToStringWithoutMessage(e.InnerException) + Environment.NewLine + + " " + Exception_EndOfInnerExceptionStack; + + } + + var stackTrace = e.StackTrace; + + if (stackTrace != null) + { + s += Environment.NewLine + stackTrace; + } + + return s; + } + + protected abstract void ExecuteCore(); + } +} diff --git a/src/tasks/Crossgen2Tasks/Crossgen2Tasks.csproj b/src/tasks/Crossgen2Tasks/Crossgen2Tasks.csproj new file mode 100644 index 0000000..63d7016 --- /dev/null +++ b/src/tasks/Crossgen2Tasks/Crossgen2Tasks.csproj @@ -0,0 +1,27 @@ + + + $(NetCoreAppToolCurrent) + Library + true + $(NoWarn),CA1050 + + + $(NoWarn),CS8604,CS8602 + + + + + + + + + + + PreserveNewest + + + PreserveNewest + Microsoft.NET.CrossGen.props + + + diff --git a/src/tasks/Crossgen2Tasks/Microsoft.NET.CrossGen.targets b/src/tasks/Crossgen2Tasks/Microsoft.NET.CrossGen.targets new file mode 100644 index 0000000..1a133ed --- /dev/null +++ b/src/tasks/Crossgen2Tasks/Microsoft.NET.CrossGen.targets @@ -0,0 +1,508 @@ + + + + + + true + false + true + false + + + + + + + + <_CoreclrResolvedPath Include="@(CrossgenResolvedAssembliesToPublish)" + Condition="'%(CrossgenResolvedAssembliesToPublish.Filename)'=='coreclr'" /> + <_CoreclrResolvedPath Include="@(CrossgenResolvedAssembliesToPublish)" + Condition="'%(CrossgenResolvedAssembliesToPublish.Filename)'=='libcoreclr'" /> + <_JitResolvedPath Include="@(CrossgenResolvedAssembliesToPublish)" + Condition="'%(CrossgenResolvedAssembliesToPublish.Filename)'=='clrjit'" /> + <_JitResolvedPath Include="@(CrossgenResolvedAssembliesToPublish)" + Condition="'%(CrossgenResolvedAssembliesToPublish.Filename)'=='libclrjit'" /> + + + + + + + + + + + + + <_CoreclrPath>@(_CoreclrResolvedPath) + @(_JitResolvedPath) + <_CoreclrDir>$([System.IO.Path]::GetDirectoryName($(_CoreclrPath))) + <_CoreclrPkgDir>$([System.IO.Path]::Combine($(_CoreclrDir),"..\..\..\")) + $([System.IO.Path]::Combine($(_CoreclrPkgDir),"tools")) + + $([System.IO.Path]::Combine($(CrossgenDir),"crossgen")) + $([System.IO.Path]::Combine($(CrossgenDir),"crossgen.exe")) + + + + + + + + + + + $([System.IO.Path]::GetFullPath($([System.IO.Path]::Combine($(_NetCoreRefDir), $([System.IO.Path]::GetFileName($(Crossgen))))))) + + + + + + + + + + CrossgenExe=$(Crossgen); + CrossgenJit=$(JitPath); + CrossgenInputAssembly=%(_ManagedResolvedFilesToOptimize.Fullpath); + CrossgenOutputAssembly=$(_RuntimeOptimizedDir)$(DirectorySeparatorChar)%(_ManagedResolvedFilesToOptimize.FileName)%(_ManagedResolvedFilesToOptimize.Extension); + CrossgenSubOutputPath=%(_ManagedResolvedFilesToOptimize.DestinationSubPath); + _RuntimeOptimizedDir=$(_RuntimeOptimizedDir); + PublishDir=$(StoreStagingDir); + CrossgenPlatformAssembliesPath=$(_RuntimeRefDir)$(PathSeparator)$(_NetCoreRefDir); + CreateProfilingSymbols=$(CreateProfilingSymbols); + StoreSymbolsStagingDir=$(StoreSymbolsStagingDir); + _RuntimeSymbolsDir=$(_RuntimeSymbolsDir) + + + + + + + + + + + + + + + $([System.IO.Path]::GetDirectoryName($(_RuntimeSymbolsDir)\$(CrossgenSubOutputPath))) + $([System.IO.Path]::GetDirectoryName($(StoreSymbolsStagingDir)\$(CrossgenSubOutputPath))) + $(CrossgenExe) -nologo -readytorun -in "$(CrossgenInputAssembly)" -out "$(CrossgenOutputAssembly)" -jitpath "$(CrossgenJit)" -platform_assemblies_paths "$(CrossgenPlatformAssembliesPath)" + CreatePDB + CreatePerfMap + + + + + + + + + + + + + + + + + + <_ProfilingSymbols Include="$(CrossgenProfilingSymbolsOutputDirectory)\*" + Condition="'$(CreateProfilingSymbols)' == 'true'" /> + + + + + + + + + + $([System.IO.Path]::PathSeparator) + $([System.IO.Path]::DirectorySeparatorChar) + + + + + + + + + <_CrossProjFileDir>$([System.IO.Path]::Combine($(ComposeWorkingDir),"Optimize")) + <_NetCoreRefDir>$([System.IO.Path]::Combine($(_CrossProjFileDir), "netcoreapp")) + + + + <_CrossProjAssetsFile>$([System.IO.Path]::Combine($(_CrossProjFileDir), project.assets.json)) + + + + + + + + + + <_RuntimeRefDir>$([System.IO.Path]::Combine($(StoreWorkerWorkingDir), "runtimeref")) + <_RuntimeOptimizedDir>$([System.IO.Path]::Combine($(StoreWorkerWorkingDir), "runtimopt")) + <_RuntimeSymbolsDir>$([System.IO.Path]::Combine($(StoreWorkerWorkingDir), "runtimesymbols")) + + + + <_ManagedResolvedFilesToOptimize Include="@(_ManagedResolvedFileToPublishCandidates)" /> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + $(MSBuildThisFileDirectory)..\tasks\net6.0\Microsoft.NET.Sdk.Crossgen.dll + $(MSBuildThisFileDirectory)..\tasks\net472\Microsoft.NET.Sdk.Crossgen.dll + + + + + + + + + + + + + + + + + + + + + + + + + + <_ReadyToRunOutputPath>$(IntermediateOutputPath)R2R + + + + + + <_ReadyToRunImplementationAssemblies Include="@(ResolvedFileToPublish->WithMetadataValue('PostprocessAssembly', 'true'))" /> + + + + + <_ReadyToRunImplementationAssemblies Include="@(_ManagedRuntimePackAssembly)" ReferenceOnly="true" /> + + + + + + + + <_ReadyToRunImplementationAssemblies Remove="@(_ReadyToRunImplementationAssemblies)" /> + <_ReadyToRunImplementationAssemblies Include="@(_ReadyToRunImplementationAssembliesWithoutConflicts)" /> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + <_ReadyToRunCompilerHasWarnings Condition="'$(_ReadyToRunWarningsDetected)' == 'true'">true + + + + <_ReadyToRunCompilationFailures Condition="'$(_ReadyToRunCompilerExitCode)' != '' And $(_ReadyToRunCompilerExitCode) != 0" + Include="@(_ReadyToRunCompileList)" /> + + + + + + + + + + + + + <_ReadyToRunCompilerHasWarnings Condition="'$(_ReadyToRunWarningsDetected)' == 'true'">true + + + + <_ReadyToRunCompilationFailures Condition="'$(_ReadyToRunCompilerExitCode)' != '' And $(_ReadyToRunCompilerExitCode) != 0" + Include="@(_ReadyToRunSymbolsCompileList)" /> + + + diff --git a/src/tasks/Crossgen2Tasks/PrepareForReadyToRunCompilation.cs b/src/tasks/Crossgen2Tasks/PrepareForReadyToRunCompilation.cs new file mode 100644 index 0000000..7004f1d --- /dev/null +++ b/src/tasks/Crossgen2Tasks/PrepareForReadyToRunCompilation.cs @@ -0,0 +1,485 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Linq; +using System.IO; +using System.Runtime.InteropServices; +using System.Reflection.PortableExecutable; +using Microsoft.Build.Framework; +using Microsoft.Build.Utilities; +using System.Reflection.Metadata; +using System.Reflection; + +namespace Microsoft.NET.Build.Tasks +{ + public class PrepareForReadyToRunCompilation : TaskBase + { + [Required] + public ITaskItem MainAssembly { get; set; } + public ITaskItem[] Assemblies { get; set; } + public string[] ExcludeList { get; set; } + public bool EmitSymbols { get; set; } + public bool ReadyToRunUseCrossgen2 { get; set; } + public bool Crossgen2Composite { get; set; } + + [Required] + public string OutputPath { get; set; } + [Required] + public bool IncludeSymbolsInSingleFile { get; set; } + + public string[] PublishReadyToRunCompositeExclusions { get; set; } + + public ITaskItem CrossgenTool { get; set; } + public ITaskItem Crossgen2Tool { get; set; } + + // Output lists of files to compile. Currently crossgen has to run in two steps, the first to generate the R2R image + // and the second to create native PDBs for the compiled images (the output of the first step is an input to the second step) + [Output] + public ITaskItem[] ReadyToRunCompileList => _compileList.ToArray(); + [Output] + public ITaskItem[] ReadyToRunSymbolsCompileList => _symbolsCompileList.ToArray(); + + // Output files to publish after compilation. These lists are equivalent to the input list, but contain the new + // paths to the compiled R2R images and native PDBs. + [Output] + public ITaskItem[] ReadyToRunFilesToPublish => _r2rFiles.ToArray(); + + [Output] + public ITaskItem[] ReadyToRunAssembliesToReference => _r2rReferences.ToArray(); + + [Output] + public ITaskItem[] ReadyToRunCompositeBuildReferences => _r2rCompositeReferences.ToArray(); + + [Output] + public ITaskItem[] ReadyToRunCompositeBuildInput => _r2rCompositeInput.ToArray(); + + private bool _crossgen2IsVersion5; + + private List _compileList = new List(); + private List _symbolsCompileList = new List(); + private List _r2rFiles = new List(); + private List _r2rReferences = new List(); + private List _r2rCompositeReferences = new List(); + private List _r2rCompositeInput = new List(); + + protected override void ExecuteCore() + { + if (ReadyToRunUseCrossgen2) + { + string isVersion5 = Crossgen2Tool.GetMetadata(MetadataKeys.IsVersion5); + _crossgen2IsVersion5 = !string.IsNullOrEmpty(isVersion5) && bool.Parse(isVersion5); + + if (Crossgen2Composite && EmitSymbols && _crossgen2IsVersion5) + { + Log.LogError(Strings.Crossgen5CannotEmitSymbolsInCompositeMode); + return; + } + } + + string diaSymReaderPath = CrossgenTool?.GetMetadata(MetadataKeys.DiaSymReader); + + bool hasValidDiaSymReaderLib = + ReadyToRunUseCrossgen2 && !_crossgen2IsVersion5 || + !string.IsNullOrEmpty(diaSymReaderPath) && File.Exists(diaSymReaderPath); + + // Process input lists of files + ProcessInputFileList(Assemblies, _compileList, _symbolsCompileList, _r2rFiles, _r2rReferences, _r2rCompositeReferences, _r2rCompositeInput, hasValidDiaSymReaderLib); + } + + private void ProcessInputFileList( + ITaskItem[] inputFiles, + List imageCompilationList, + List symbolsCompilationList, + List r2rFilesPublishList, + List r2rReferenceList, + List r2rCompositeReferenceList, + List r2rCompositeInputList, + bool hasValidDiaSymReaderLib) + { + if (inputFiles == null) + { + return; + } + + var exclusionSet = ExcludeList == null || Crossgen2Composite ? null : new HashSet(ExcludeList, StringComparer.OrdinalIgnoreCase); + var compositeExclusionSet = PublishReadyToRunCompositeExclusions == null || !Crossgen2Composite ? null : new HashSet(PublishReadyToRunCompositeExclusions, StringComparer.OrdinalIgnoreCase); + + foreach (var file in inputFiles) + { + var eligibility = GetInputFileEligibility(file, Crossgen2Composite, exclusionSet, compositeExclusionSet); + + if (eligibility.NoEligibility) + { + continue; + } + + if (eligibility.IsReference) + r2rReferenceList.Add(file); + + if (eligibility.IsReference && !eligibility.ReferenceHiddenFromCompositeBuild && !eligibility.Compile) + r2rCompositeReferenceList.Add(file); + + if (!eligibility.Compile) + { + continue; + } + + var outputR2RImageRelativePath = file.GetMetadata(MetadataKeys.RelativePath); + var outputR2RImage = Path.Combine(OutputPath, outputR2RImageRelativePath); + + string outputPDBImage = null; + string outputPDBImageRelativePath = null; + string crossgen1CreatePDBCommand = null; + + if (EmitSymbols) + { + if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows) && hasValidDiaSymReaderLib) + { + outputPDBImage = Path.ChangeExtension(outputR2RImage, "ni.pdb"); + outputPDBImageRelativePath = Path.ChangeExtension(outputR2RImageRelativePath, "ni.pdb"); + crossgen1CreatePDBCommand = $"/CreatePDB \"{Path.GetDirectoryName(outputPDBImage)}\""; + } + else if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux)) + { + using (FileStream fs = new FileStream(file.ItemSpec, FileMode.Open, FileAccess.Read)) + { + PEReader pereader = new PEReader(fs); + MetadataReader mdReader = pereader.GetMetadataReader(); + Guid mvid = mdReader.GetGuid(mdReader.GetModuleDefinition().Mvid); + + outputPDBImage = Path.ChangeExtension(outputR2RImage, "ni.{" + mvid + "}.map"); + outputPDBImageRelativePath = Path.ChangeExtension(outputR2RImageRelativePath, "ni.{" + mvid + "}.map"); + crossgen1CreatePDBCommand = $"/CreatePerfMap \"{Path.GetDirectoryName(outputPDBImage)}\""; + } + } + } + + if (eligibility.CompileSeparately) + { + // This TaskItem is the IL->R2R entry, for an input assembly that needs to be compiled into a R2R image. This will be used as + // an input to the ReadyToRunCompiler task + TaskItem r2rCompilationEntry = new TaskItem(file); + r2rCompilationEntry.SetMetadata(MetadataKeys.OutputR2RImage, outputR2RImage); + if (outputPDBImage != null && ReadyToRunUseCrossgen2 && !_crossgen2IsVersion5) + { + r2rCompilationEntry.SetMetadata(MetadataKeys.EmitSymbols, "true"); + r2rCompilationEntry.SetMetadata(MetadataKeys.OutputPDBImage, outputPDBImage); + } + r2rCompilationEntry.RemoveMetadata(MetadataKeys.OriginalItemSpec); + imageCompilationList.Add(r2rCompilationEntry); + } + else if (eligibility.CompileIntoCompositeImage) + { + r2rCompositeInputList.Add(file); + } + + // This TaskItem corresponds to the output R2R image. It is equivalent to the input TaskItem, only the ItemSpec for it points to the new path + // for the newly created R2R image + TaskItem r2rFileToPublish = new TaskItem(file); + r2rFileToPublish.ItemSpec = outputR2RImage; + r2rFileToPublish.RemoveMetadata(MetadataKeys.OriginalItemSpec); + r2rFilesPublishList.Add(r2rFileToPublish); + + // Note: ReadyToRun PDB/Map files are not needed for debugging. They are only used for profiling, therefore the default behavior is to not generate them + // unless an explicit PublishReadyToRunEmitSymbols flag is enabled by the app developer. There is also another way to profile that the runtime supports, which does + // not rely on the native PDBs/Map files, so creating them is really an opt-in option, typically used by advanced users. + // For debugging, only the IL PDBs are required. + if (eligibility.CompileSeparately && outputPDBImage != null) + { + if (!ReadyToRunUseCrossgen2 || _crossgen2IsVersion5) + { + // This TaskItem is the R2R->R2RPDB entry, for a R2R image that was just created, and for which we need to create native PDBs. This will be used as + // an input to the ReadyToRunCompiler task + TaskItem pdbCompilationEntry = new TaskItem(file); + pdbCompilationEntry.ItemSpec = outputR2RImage; + pdbCompilationEntry.SetMetadata(MetadataKeys.OutputPDBImage, outputPDBImage); + pdbCompilationEntry.SetMetadata(MetadataKeys.CreatePDBCommand, crossgen1CreatePDBCommand); + symbolsCompilationList.Add(pdbCompilationEntry); + } + + // This TaskItem corresponds to the output PDB image. It is equivalent to the input TaskItem, only the ItemSpec for it points to the new path + // for the newly created PDB image. + TaskItem r2rSymbolsFileToPublish = new TaskItem(file); + r2rSymbolsFileToPublish.ItemSpec = outputPDBImage; + r2rSymbolsFileToPublish.SetMetadata(MetadataKeys.RelativePath, outputPDBImageRelativePath); + r2rSymbolsFileToPublish.RemoveMetadata(MetadataKeys.OriginalItemSpec); + if (!IncludeSymbolsInSingleFile) + { + r2rSymbolsFileToPublish.SetMetadata(MetadataKeys.ExcludeFromSingleFile, "true"); + } + + r2rFilesPublishList.Add(r2rSymbolsFileToPublish); + } + } + + if (Crossgen2Composite) + { + MainAssembly.SetMetadata(MetadataKeys.RelativePath, Path.GetFileName(MainAssembly.ItemSpec)); + + var compositeR2RImageRelativePath = MainAssembly.GetMetadata(MetadataKeys.RelativePath); + compositeR2RImageRelativePath = Path.ChangeExtension(compositeR2RImageRelativePath, "r2r" + Path.GetExtension(compositeR2RImageRelativePath)); + var compositeR2RImage = Path.Combine(OutputPath, compositeR2RImageRelativePath); + + TaskItem r2rCompilationEntry = new TaskItem(MainAssembly); + r2rCompilationEntry.ItemSpec = r2rCompositeInputList[0].ItemSpec; + r2rCompilationEntry.SetMetadata(MetadataKeys.OutputR2RImage, compositeR2RImage); + r2rCompilationEntry.SetMetadata(MetadataKeys.CreateCompositeImage, "true"); + r2rCompilationEntry.RemoveMetadata(MetadataKeys.OriginalItemSpec); + + if (EmitSymbols) + { + string compositePDBImage = null; + string compositePDBRelativePath = null; + if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows) && hasValidDiaSymReaderLib) + { + compositePDBImage = Path.ChangeExtension(compositeR2RImage, ".ni.pdb"); + compositePDBRelativePath = Path.ChangeExtension(compositeR2RImageRelativePath, ".ni.pdb"); + } + else if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux)) + { + compositePDBImage = Path.ChangeExtension(compositeR2RImage, ".ni.{composite}.map"); + compositePDBRelativePath = Path.ChangeExtension(compositeR2RImageRelativePath, ".ni.{composite}.map"); + } + + if (compositePDBImage != null && ReadyToRunUseCrossgen2 && !_crossgen2IsVersion5) + { + r2rCompilationEntry.SetMetadata(MetadataKeys.EmitSymbols, "true"); + r2rCompilationEntry.SetMetadata(MetadataKeys.OutputPDBImage, compositePDBImage); + + // Publish composite PDB file + TaskItem r2rSymbolsFileToPublish = new TaskItem(MainAssembly); + r2rSymbolsFileToPublish.ItemSpec = compositePDBImage; + r2rSymbolsFileToPublish.SetMetadata(MetadataKeys.RelativePath, compositePDBRelativePath); + r2rSymbolsFileToPublish.RemoveMetadata(MetadataKeys.OriginalItemSpec); + if (!IncludeSymbolsInSingleFile) + { + r2rSymbolsFileToPublish.SetMetadata(MetadataKeys.ExcludeFromSingleFile, "true"); + } + + r2rFilesPublishList.Add(r2rSymbolsFileToPublish); + } + } + + imageCompilationList.Add(r2rCompilationEntry); + + // Publish it + TaskItem compositeR2RFileToPublish = new TaskItem(MainAssembly); + compositeR2RFileToPublish.ItemSpec = compositeR2RImage; + compositeR2RFileToPublish.RemoveMetadata(MetadataKeys.OriginalItemSpec); + compositeR2RFileToPublish.SetMetadata(MetadataKeys.RelativePath, compositeR2RImageRelativePath); + r2rFilesPublishList.Add(compositeR2RFileToPublish); + } + } + + private struct Eligibility + { + [Flags] + private enum EligibilityEnum + { + None = 0, + Reference = 1, + HideReferenceFromComposite = 2, + CompileSeparately = 4, + CompileIntoCompositeImage = 8, + } + + private readonly EligibilityEnum _flags; + + public static Eligibility None => new Eligibility(EligibilityEnum.None); + + public bool NoEligibility => _flags == EligibilityEnum.None; + public bool IsReference => (_flags & EligibilityEnum.Reference) == EligibilityEnum.Reference; + public bool ReferenceHiddenFromCompositeBuild => (_flags & EligibilityEnum.HideReferenceFromComposite) == EligibilityEnum.HideReferenceFromComposite; + public bool CompileIntoCompositeImage => (_flags & EligibilityEnum.CompileIntoCompositeImage) == EligibilityEnum.CompileIntoCompositeImage; + public bool CompileSeparately => (_flags & EligibilityEnum.CompileSeparately) == EligibilityEnum.CompileSeparately; + public bool Compile => CompileIntoCompositeImage || CompileSeparately; + + private Eligibility(EligibilityEnum flags) + { + _flags = flags; + } + + public static Eligibility CreateReferenceEligibility(bool hideFromCompositeBuilds) + { + if (hideFromCompositeBuilds) + return new Eligibility(EligibilityEnum.Reference | EligibilityEnum.HideReferenceFromComposite); + else + return new Eligibility(EligibilityEnum.Reference); + } + + public static Eligibility CreateCompileEligibility(bool doNotBuildIntoComposite) + { + if (doNotBuildIntoComposite) + return new Eligibility(EligibilityEnum.Reference | EligibilityEnum.HideReferenceFromComposite | EligibilityEnum.CompileSeparately); + else + return new Eligibility(EligibilityEnum.Reference | EligibilityEnum.CompileIntoCompositeImage); + } + }; + + private static bool IsNonCompositeReadyToRunImage(PEReader peReader) + { + if (peReader.PEHeaders == null) + return false; + + if (peReader.PEHeaders.CorHeader == null) + return false; + + if ((peReader.PEHeaders.CorHeader.Flags & CorFlags.ILLibrary) == 0) + { + // This is likely a composite image, but those can't be re-r2r'd + return false; + } + else + { + return peReader.PEHeaders.CorHeader.ManagedNativeHeaderDirectory.Size != 0; + } + } + + private static Eligibility GetInputFileEligibility(ITaskItem file, bool compositeCompile, HashSet exclusionSet, HashSet r2rCompositeExclusionSet) + { + // Check to see if this is a valid ILOnly image that we can compile + using (FileStream fs = new FileStream(file.ItemSpec, FileMode.Open, FileAccess.Read)) + { + try + { + using (var pereader = new PEReader(fs)) + { + if (!pereader.HasMetadata) + { + return Eligibility.None; + } + + MetadataReader mdReader = pereader.GetMetadataReader(); + if (!mdReader.IsAssembly) + { + return Eligibility.None; + } + + if (IsReferenceAssembly(mdReader)) + { + // crossgen can only take implementation assemblies, even as references + return Eligibility.None; + } + + bool excludeFromR2R = (exclusionSet != null && exclusionSet.Contains(Path.GetFileName(file.ItemSpec))); + bool excludeFromComposite = (r2rCompositeExclusionSet != null && r2rCompositeExclusionSet.Contains(Path.GetFileName(file.ItemSpec))) || excludeFromR2R; + + if ((pereader.PEHeaders.CorHeader.Flags & CorFlags.ILOnly) != CorFlags.ILOnly) + { + // This can happen due to C++/CLI binaries or due to previously R2R compiled binaries. + + if (!IsNonCompositeReadyToRunImage(pereader)) + { + // For C++/CLI always treat as only a reference + return Eligibility.CreateReferenceEligibility(excludeFromComposite); + } + else + { + // If previously compiled as R2R, treat as reference if this would be compiled seperately + if (!compositeCompile || excludeFromComposite) + { + return Eligibility.CreateReferenceEligibility(excludeFromComposite); + } + } + } + + if (file.HasMetadataValue(MetadataKeys.ReferenceOnly, "true")) + { + return Eligibility.CreateReferenceEligibility(excludeFromComposite); + } + + if (excludeFromR2R) + { + return Eligibility.CreateReferenceEligibility(excludeFromComposite); + } + + // save these most expensive checks for last. We don't want to scan all references for IL code + if (ReferencesWinMD(mdReader) || !HasILCode(pereader, mdReader)) + { + // Forwarder assemblies are not separately compiled via R2R, but when performing composite compilation, they are included in the bundle + if (excludeFromComposite || !compositeCompile) + return Eligibility.CreateReferenceEligibility(excludeFromComposite); + } + + return Eligibility.CreateCompileEligibility(!compositeCompile || excludeFromComposite); + } + } + catch (BadImageFormatException) + { + // Not a valid assembly file + return Eligibility.None; + } + } + } + + private static bool IsReferenceAssembly(MetadataReader mdReader) + { + foreach (var attributeHandle in mdReader.GetAssemblyDefinition().GetCustomAttributes()) + { + EntityHandle attributeCtor = mdReader.GetCustomAttribute(attributeHandle).Constructor; + + StringHandle attributeTypeName = default; + StringHandle attributeTypeNamespace = default; + + if (attributeCtor.Kind == HandleKind.MemberReference) + { + EntityHandle attributeMemberParent = mdReader.GetMemberReference((MemberReferenceHandle)attributeCtor).Parent; + if (attributeMemberParent.Kind == HandleKind.TypeReference) + { + TypeReference attributeTypeRef = mdReader.GetTypeReference((TypeReferenceHandle)attributeMemberParent); + attributeTypeName = attributeTypeRef.Name; + attributeTypeNamespace = attributeTypeRef.Namespace; + } + } + else if (attributeCtor.Kind == HandleKind.MethodDefinition) + { + TypeDefinitionHandle attributeTypeDefHandle = mdReader.GetMethodDefinition((MethodDefinitionHandle)attributeCtor).GetDeclaringType(); + TypeDefinition attributeTypeDef = mdReader.GetTypeDefinition(attributeTypeDefHandle); + attributeTypeName = attributeTypeDef.Name; + attributeTypeNamespace = attributeTypeDef.Namespace; + } + + if (!attributeTypeName.IsNil && + !attributeTypeNamespace.IsNil && + mdReader.StringComparer.Equals(attributeTypeName, "ReferenceAssemblyAttribute") && + mdReader.StringComparer.Equals(attributeTypeNamespace, "System.Runtime.CompilerServices")) + { + return true; + } + } + + return false; + } + + private static bool ReferencesWinMD(MetadataReader mdReader) + { + foreach (var assemblyRefHandle in mdReader.AssemblyReferences) + { + AssemblyReference assemblyRef = mdReader.GetAssemblyReference(assemblyRefHandle); + if ((assemblyRef.Flags & AssemblyFlags.WindowsRuntime) == AssemblyFlags.WindowsRuntime) + { + return true; + } + } + + return false; + } + + private static bool HasILCode(PEReader peReader, MetadataReader mdReader) + { + foreach (var methoddefHandle in mdReader.MethodDefinitions) + { + MethodDefinition methodDef = mdReader.GetMethodDefinition(methoddefHandle); + if (methodDef.RelativeVirtualAddress > 0) + { + return true; + } + } + + return false; + } + } +} diff --git a/src/tasks/Crossgen2Tasks/README.md b/src/tasks/Crossgen2Tasks/README.md new file mode 100644 index 0000000..4bd2584 --- /dev/null +++ b/src/tasks/Crossgen2Tasks/README.md @@ -0,0 +1,4 @@ +The files in this directory are equivalent/identical to corresponding files in the sdk repo. They should be kept equivalent. + +The files in the subdirectory CommoonFilesPulledFromSdkRepo do not need to be updated aggressively as they change. +The files in the subdirectory ShimFilesSimulatingLogicInSdkRepo are simply in place to make the functionality be close enough so that these tasks can easily be compiled. \ No newline at end of file diff --git a/src/tasks/Crossgen2Tasks/ResolveReadyToRunCompilers.cs b/src/tasks/Crossgen2Tasks/ResolveReadyToRunCompilers.cs new file mode 100644 index 0000000..d44b985 --- /dev/null +++ b/src/tasks/Crossgen2Tasks/ResolveReadyToRunCompilers.cs @@ -0,0 +1,400 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.ComponentModel; +using System.IO; +using System.Linq; +using System.Runtime.InteropServices; + +using Microsoft.Build.Framework; +using Microsoft.Build.Utilities; +using NuGet.Versioning; + +namespace Microsoft.NET.Build.Tasks +{ + public class ResolveReadyToRunCompilers : TaskBase + { + public bool EmitSymbols { get; set; } + public bool ReadyToRunUseCrossgen2 { get; set; } + + [Required] + public ITaskItem[] RuntimePacks { get; set; } + public ITaskItem[] Crossgen2Packs { get; set; } + [Required] + public ITaskItem[] TargetingPacks { get; set; } + [Required] + public string RuntimeGraphPath { get; set; } + [Required] + public string NETCoreSdkRuntimeIdentifier { get; set; } + + [Output] + public ITaskItem CrossgenTool { get; set; } + [Output] + public ITaskItem Crossgen2Tool { get; set; } + + internal struct CrossgenToolInfo + { + public string ToolPath; + public string PackagePath; + public string ClrJitPath; + public string DiaSymReaderPath; + } + + private ITaskItem _runtimePack; + private ITaskItem _crossgen2Pack; + private string _targetRuntimeIdentifier; + private string _targetPlatform; + private string _hostRuntimeIdentifier; + + private CrossgenToolInfo _crossgenTool; + private CrossgenToolInfo _crossgen2Tool; + + private Architecture _targetArchitecture; + private bool _crossgen2IsVersion5; + + protected override void ExecuteCore() + { + _runtimePack = GetNETCoreAppRuntimePack(); + _crossgen2Pack = Crossgen2Packs?.FirstOrDefault(); + _targetRuntimeIdentifier = _runtimePack?.GetMetadata(MetadataKeys.RuntimeIdentifier); + + // Get the list of runtime identifiers that we support and can target + ITaskItem targetingPack = GetNETCoreAppTargetingPack(); + string supportedRuntimeIdentifiers = targetingPack?.GetMetadata(MetadataKeys.RuntimePackRuntimeIdentifiers); + + var runtimeGraph = new RuntimeGraphCache(this).GetRuntimeGraph(RuntimeGraphPath); + var supportedRIDsList = supportedRuntimeIdentifiers == null ? Array.Empty() : supportedRuntimeIdentifiers.Split(';'); + + // Get the best RID for the host machine, which will be used to validate that we can run crossgen for the target platform and architecture + _hostRuntimeIdentifier = NuGetUtils.GetBestMatchingRid( + runtimeGraph, + NETCoreSdkRuntimeIdentifier, + supportedRIDsList, + out _); + + if (_hostRuntimeIdentifier == null || _targetRuntimeIdentifier == null) + { + Log.LogError(Strings.ReadyToRunNoValidRuntimePackageError); + return; + } + + if (ReadyToRunUseCrossgen2) + { + if (!ValidateCrossgen2Support()) + { + return; + } + + // In .NET 5 Crossgen2 did not support emitting native symbols, so we use Crossgen to emit them + if (_crossgen2IsVersion5 && EmitSymbols && !ValidateCrossgenSupport()) + { + return; + } + } + else + { + if (!ValidateCrossgenSupport()) + { + return; + } + } + } + + private bool ValidateCrossgenSupport() + { + _crossgenTool.PackagePath = _runtimePack?.GetMetadata(MetadataKeys.PackageDirectory); + if (_crossgenTool.PackagePath == null) + { + Log.LogError(Strings.ReadyToRunNoValidRuntimePackageError); + return false; + } + + if (!ExtractTargetPlatformAndArchitecture(_targetRuntimeIdentifier, out _targetPlatform, out _targetArchitecture) || + !ExtractTargetPlatformAndArchitecture(_hostRuntimeIdentifier, out string hostPlatform, out Architecture hostArchitecture) || + _targetPlatform != hostPlatform || + !GetCrossgenComponentsPaths()) + { + Log.LogError(Strings.ReadyToRunTargetNotSupportedError); + return false; + } + + // Create tool task item + CrossgenTool = new TaskItem(_crossgenTool.ToolPath); + CrossgenTool.SetMetadata(MetadataKeys.JitPath, _crossgenTool.ClrJitPath); + if (!string.IsNullOrEmpty(_crossgenTool.DiaSymReaderPath)) + { + CrossgenTool.SetMetadata(MetadataKeys.DiaSymReader, _crossgenTool.DiaSymReaderPath); + } + + return true; + } + + private bool ValidateCrossgen2Support() + { + _crossgen2Tool.PackagePath = _crossgen2Pack?.GetMetadata(MetadataKeys.PackageDirectory); + if (_crossgen2Tool.PackagePath == null || + !NuGetVersion.TryParse(_crossgen2Pack.GetMetadata(MetadataKeys.NuGetPackageVersion), out NuGetVersion crossgen2PackVersion)) + { + Log.LogError(Strings.ReadyToRunNoValidRuntimePackageError); + return false; + } + + bool version5 = crossgen2PackVersion.Major < 6; + bool isSupportedTarget = ExtractTargetPlatformAndArchitecture(_targetRuntimeIdentifier, out _targetPlatform, out _targetArchitecture); + string targetOS = _targetPlatform switch + { + "linux" => "linux", + "linux-musl" => "linux", + "osx" => "osx", + "win" => "windows", + _ => null + }; + + // In .NET 5 Crossgen2 supported only the following host->target compilation scenarios: + // win-x64 -> win-x64 + // linux-x64 -> linux-x64 + // linux-musl-x64 -> linux-musl-x64 + isSupportedTarget = isSupportedTarget && + targetOS != null && + (!version5 || _targetRuntimeIdentifier == _hostRuntimeIdentifier) && + GetCrossgen2ComponentsPaths(version5); + + if (!isSupportedTarget) + { + Log.LogError(Strings.ReadyToRunTargetNotSupportedError); + return false; + } + + // Create tool task item + Crossgen2Tool = new TaskItem(_crossgen2Tool.ToolPath); + Crossgen2Tool.SetMetadata(MetadataKeys.IsVersion5, version5.ToString()); + if (version5) + { + Crossgen2Tool.SetMetadata(MetadataKeys.JitPath, _crossgen2Tool.ClrJitPath); + } + else + { + Crossgen2Tool.SetMetadata(MetadataKeys.TargetOS, targetOS); + Crossgen2Tool.SetMetadata(MetadataKeys.TargetArch, ArchitectureToString(_targetArchitecture)); + } + + _crossgen2IsVersion5 = version5; + return true; + } + + private ITaskItem GetNETCoreAppRuntimePack() + { + return GetNETCoreAppPack(RuntimePacks, MetadataKeys.FrameworkName); + } + + private ITaskItem GetNETCoreAppTargetingPack() + { + return GetNETCoreAppPack(TargetingPacks, MetadataKeys.RuntimeFrameworkName); + } + + private static ITaskItem GetNETCoreAppPack(ITaskItem[] packs, string metadataKey) + { + return packs.SingleOrDefault( + pack => pack.GetMetadata(metadataKey) + .Equals("Microsoft.NETCore.App", StringComparison.OrdinalIgnoreCase)); + } + + private static bool ExtractTargetPlatformAndArchitecture(string runtimeIdentifier, out string platform, out Architecture architecture) + { + platform = null; + architecture = default; + + // This will split RID like "linux-musl-arm64" into "linux-musl" and "arm64" components + int separator = runtimeIdentifier.LastIndexOf('-'); + if (separator < 0) + { + return false; + } + + string architectureStr = runtimeIdentifier.Substring(separator + 1).ToLowerInvariant(); + + switch (architectureStr) + { + case "arm": + architecture = Architecture.Arm; + break; + case "arm64": + architecture = Architecture.Arm64; + break; + case "x64": + architecture = Architecture.X64; + break; + case "x86": + architecture = Architecture.X86; + break; + default: + return false; + } + + platform = runtimeIdentifier.Substring(0, separator).ToLowerInvariant(); + return true; + } + + private bool GetCrossgenComponentsPaths() + { + if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) + { + if (_targetArchitecture == Architecture.Arm) + { + if (RuntimeInformation.OSArchitecture == _targetArchitecture) + { + // We can run native arm32 bits on an arm64 host in WOW mode + _crossgenTool.ToolPath = Path.Combine(_crossgenTool.PackagePath, "tools", "crossgen.exe"); + _crossgenTool.ClrJitPath = Path.Combine(_crossgenTool.PackagePath, "runtimes", _targetRuntimeIdentifier, "native", "clrjit.dll"); + _crossgenTool.DiaSymReaderPath = Path.Combine(_crossgenTool.PackagePath, "runtimes", _targetRuntimeIdentifier, "native", "Microsoft.DiaSymReader.Native.arm.dll"); + } + else + { + // We can use the x86-hosted crossgen compiler to target ARM + _crossgenTool.ToolPath = Path.Combine(_crossgenTool.PackagePath, "tools", "x86_arm", "crossgen.exe"); + _crossgenTool.ClrJitPath = Path.Combine(_crossgenTool.PackagePath, "runtimes", "x86_arm", "native", "clrjit.dll"); + _crossgenTool.DiaSymReaderPath = Path.Combine(_crossgenTool.PackagePath, "runtimes", "x86_arm", "native", "Microsoft.DiaSymReader.Native.x86.dll"); + } + } + else if (_targetArchitecture == Architecture.Arm64) + { + if (RuntimeInformation.OSArchitecture == _targetArchitecture) + { + _crossgenTool.ToolPath = Path.Combine(_crossgenTool.PackagePath, "tools", "crossgen.exe"); + _crossgenTool.ClrJitPath = Path.Combine(_crossgenTool.PackagePath, "runtimes", _targetRuntimeIdentifier, "native", "clrjit.dll"); + _crossgenTool.DiaSymReaderPath = Path.Combine(_crossgenTool.PackagePath, "runtimes", _targetRuntimeIdentifier, "native", "Microsoft.DiaSymReader.Native.arm64.dll"); + } + else + { + // We only have 64-bit hosted compilers for ARM64. + if (RuntimeInformation.OSArchitecture != Architecture.X64) + { + return false; + } + + _crossgenTool.ToolPath = Path.Combine(_crossgenTool.PackagePath, "tools", "x64_arm64", "crossgen.exe"); + _crossgenTool.ClrJitPath = Path.Combine(_crossgenTool.PackagePath, "runtimes", "x64_arm64", "native", "clrjit.dll"); + _crossgenTool.DiaSymReaderPath = Path.Combine(_crossgenTool.PackagePath, "runtimes", "x64_arm64", "native", "Microsoft.DiaSymReader.Native.amd64.dll"); + } + } + else + { + _crossgenTool.ToolPath = Path.Combine(_crossgenTool.PackagePath, "tools", "crossgen.exe"); + _crossgenTool.ClrJitPath = Path.Combine(_crossgenTool.PackagePath, "runtimes", _targetRuntimeIdentifier, "native", "clrjit.dll"); + if (_targetArchitecture == Architecture.X64) + { + _crossgenTool.DiaSymReaderPath = Path.Combine(_crossgenTool.PackagePath, "runtimes", _targetRuntimeIdentifier, "native", "Microsoft.DiaSymReader.Native.amd64.dll"); + } + else + { + _crossgenTool.DiaSymReaderPath = Path.Combine(_crossgenTool.PackagePath, "runtimes", _targetRuntimeIdentifier, "native", "Microsoft.DiaSymReader.Native.x86.dll"); + } + } + } + else if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux)) + { + if (_targetArchitecture == Architecture.Arm || _targetArchitecture == Architecture.Arm64) + { + if (RuntimeInformation.OSArchitecture == _targetArchitecture) + { + _crossgenTool.ToolPath = Path.Combine(_crossgenTool.PackagePath, "tools", "crossgen"); + _crossgenTool.ClrJitPath = Path.Combine(_crossgenTool.PackagePath, "runtimes", _targetRuntimeIdentifier, "native", "libclrjit.so"); + } + else if (RuntimeInformation.OSArchitecture == Architecture.X64) + { + string xarchPath = (_targetArchitecture == Architecture.Arm ? "x64_arm" : "x64_arm64"); + _crossgenTool.ToolPath = Path.Combine(_crossgenTool.PackagePath, "tools", xarchPath, "crossgen"); + _crossgenTool.ClrJitPath = Path.Combine(_crossgenTool.PackagePath, "runtimes", xarchPath, "native", "libclrjit.so"); + } + else + { + return false; + } + } + else + { + _crossgenTool.ToolPath = Path.Combine(_crossgenTool.PackagePath, "tools", "crossgen"); + _crossgenTool.ClrJitPath = Path.Combine(_crossgenTool.PackagePath, "runtimes", _targetRuntimeIdentifier, "native", "libclrjit.so"); + } + } + else if (RuntimeInformation.IsOSPlatform(OSPlatform.OSX)) + { + // Only x64 supported for OSX + if (_targetArchitecture != Architecture.X64 || RuntimeInformation.OSArchitecture != Architecture.X64) + { + return false; + } + + _crossgenTool.ToolPath = Path.Combine(_crossgenTool.PackagePath, "tools", "crossgen"); + _crossgenTool.ClrJitPath = Path.Combine(_crossgenTool.PackagePath, "runtimes", _targetRuntimeIdentifier, "native", "libclrjit.dylib"); + } + else + { + // Unknown platform + return false; + } + + return File.Exists(_crossgenTool.ToolPath) && File.Exists(_crossgenTool.ClrJitPath); + } + + private bool GetCrossgen2ComponentsPaths(bool version5) + { + string toolFileName, v5_clrJitFileNamePattern; + + if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) + { + toolFileName = "crossgen2.exe"; + v5_clrJitFileNamePattern = "clrjit-{0}.dll"; + } + else if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux)) + { + toolFileName = "crossgen2"; + v5_clrJitFileNamePattern = "libclrjit-{0}.so"; + } + else if (RuntimeInformation.IsOSPlatform(OSPlatform.OSX)) + { + toolFileName = "crossgen2"; + v5_clrJitFileNamePattern = "libclrjit-{0}.dylib"; + } + else + { + // Unknown platform + return false; + } + + if (version5) + { + string clrJitFileName = string.Format(v5_clrJitFileNamePattern, GetTargetSpecForVersion5()); + _crossgen2Tool.ClrJitPath = Path.Combine(_crossgen2Tool.PackagePath, "tools", clrJitFileName); + if (!File.Exists(_crossgen2Tool.ClrJitPath)) + { + return false; + } + } + + _crossgen2Tool.ToolPath = Path.Combine(_crossgen2Tool.PackagePath, "tools", toolFileName); + return File.Exists(_crossgen2Tool.ToolPath); + } + + // Keep in sync with JitConfigProvider.GetTargetSpec in .NET 5 + private string GetTargetSpecForVersion5() + { + string targetOSComponent = (_targetPlatform == "win" ? "win" : "unix"); + string targetArchComponent = ArchitectureToString(_targetArchitecture); + return targetOSComponent + '-' + targetArchComponent; + } + + private static string ArchitectureToString(Architecture architecture) + { + return architecture switch + { + Architecture.X86 => "x86", + Architecture.X64 => "x64", + Architecture.Arm => "arm", + Architecture.Arm64 => "arm64", + _ => null + }; + } + } +} diff --git a/src/tasks/Crossgen2Tasks/RunReadyToRunCompiler.cs b/src/tasks/Crossgen2Tasks/RunReadyToRunCompiler.cs new file mode 100644 index 0000000..2f6ed26 --- /dev/null +++ b/src/tasks/Crossgen2Tasks/RunReadyToRunCompiler.cs @@ -0,0 +1,396 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.IO; +using System.Runtime.InteropServices; +using System.Text; + +using Microsoft.Build.Framework; +using Microsoft.Build.Utilities; + +namespace Microsoft.NET.Build.Tasks +{ + public class RunReadyToRunCompiler : ToolTask + { + public ITaskItem CrossgenTool { get; set; } + public ITaskItem Crossgen2Tool { get; set; } + + [Required] + public ITaskItem CompilationEntry { get; set; } + [Required] + public ITaskItem[] ImplementationAssemblyReferences { get; set; } + public ITaskItem[] ReadyToRunCompositeBuildReferences { get; set; } + public ITaskItem[] ReadyToRunCompositeBuildInput { get; set; } + public bool ShowCompilerWarnings { get; set; } + public bool UseCrossgen2 { get; set; } + public string Crossgen2ExtraCommandLineArgs { get; set; } + + [Output] + public bool WarningsDetected { get; set; } + + private bool _emitSymbols; + private string _inputAssembly; + private string _outputR2RImage; + private string _outputPDBImage; + private string _createPDBCommand; + private bool _createCompositeImage; + + private bool IsPdbCompilation => !string.IsNullOrEmpty(_createPDBCommand); + private bool ActuallyUseCrossgen2 => UseCrossgen2 && !IsPdbCompilation; + + private string DotNetHostPath => Crossgen2Tool?.GetMetadata(MetadataKeys.DotNetHostPath); + + private bool Crossgen2IsVersion5 + { + get + { + string version5 = Crossgen2Tool?.GetMetadata(MetadataKeys.IsVersion5); + return !string.IsNullOrEmpty(version5) && bool.Parse(version5); + } + } + + protected override string ToolName + { + get + { + if (ActuallyUseCrossgen2) + { + string hostPath = DotNetHostPath; + if (!string.IsNullOrEmpty(hostPath)) + { + return hostPath; + } + return Crossgen2Tool.ItemSpec; + } + return CrossgenTool.ItemSpec; + } + } + + protected override string GenerateFullPathToTool() => ToolName; + + private string DiaSymReader => CrossgenTool.GetMetadata(MetadataKeys.DiaSymReader); + + public RunReadyToRunCompiler() + { + LogStandardErrorAsError = true; + } + + protected override bool ValidateParameters() + { + string emitSymbolsMetadata = CompilationEntry.GetMetadata(MetadataKeys.EmitSymbols); + _emitSymbols = !string.IsNullOrEmpty(emitSymbolsMetadata) && bool.Parse(emitSymbolsMetadata); + _createPDBCommand = CompilationEntry.GetMetadata(MetadataKeys.CreatePDBCommand); + string createCompositeImageMetadata = CompilationEntry.GetMetadata(MetadataKeys.CreateCompositeImage); + _createCompositeImage = !string.IsNullOrEmpty(createCompositeImageMetadata) && bool.Parse(createCompositeImageMetadata); + + if (IsPdbCompilation && CrossgenTool == null) + { + // PDB compilation is a step specific to Crossgen1 and 5.0 Crossgen2 + // which didn't support PDB generation. 6.0 Crossgen2 produces symbols + // directly during native compilation. + Log.LogError(Strings.CrossgenToolMissingInPDBCompilationMode); + return false; + } + + if (ActuallyUseCrossgen2) + { + if (Crossgen2Tool == null) + { + Log.LogError(Strings.Crossgen2ToolMissingWhenUseCrossgen2IsSet); + return false; + } + if (!File.Exists(Crossgen2Tool.ItemSpec)) + { + Log.LogError(Strings.Crossgen2ToolExecutableNotFound, Crossgen2Tool.ItemSpec); + return false; + } + string hostPath = DotNetHostPath; + if (!string.IsNullOrEmpty(hostPath) && !File.Exists(hostPath)) + { + Log.LogError(Strings.DotNetHostExecutableNotFound, hostPath); + return false; + } + string jitPath = Crossgen2Tool.GetMetadata(MetadataKeys.JitPath); + if (!string.IsNullOrEmpty(jitPath)) + { + if (!File.Exists(jitPath)) + { + Log.LogError(Strings.JitLibraryNotFound, jitPath); + return false; + } + } + else if (Crossgen2IsVersion5) + { + // We expect JitPath to be set for .NET 5 and {TargetOS, TargetArch} to be set for .NET 6 and later + Log.LogError(Strings.Crossgen2MissingRequiredMetadata, MetadataKeys.JitPath); + return false; + } + else + { + // For smooth switchover we accept both JitPath and TargetOS / TargetArch in .NET 6 Crossgen2 + if (string.IsNullOrEmpty(Crossgen2Tool.GetMetadata(MetadataKeys.TargetOS))) + { + Log.LogError(Strings.Crossgen2MissingRequiredMetadata, MetadataKeys.TargetOS); + return false; + } + if (string.IsNullOrEmpty(Crossgen2Tool.GetMetadata(MetadataKeys.TargetArch))) + { + Log.LogError(Strings.Crossgen2MissingRequiredMetadata, MetadataKeys.TargetArch); + return false; + } + } + } + else + { + if (CrossgenTool == null) + { + Log.LogError(Strings.CrossgenToolMissingWhenUseCrossgen2IsNotSet); + return false; + } + if (!File.Exists(CrossgenTool.ItemSpec)) + { + Log.LogError(Strings.CrossgenToolExecutableNotFound, CrossgenTool.ItemSpec); + return false; + } + if (!File.Exists(CrossgenTool.GetMetadata(MetadataKeys.JitPath))) + { + Log.LogError(Strings.JitLibraryNotFound, MetadataKeys.JitPath); + return false; + } + } + + _outputPDBImage = CompilationEntry.GetMetadata(MetadataKeys.OutputPDBImage); + + if (IsPdbCompilation) + { + _outputR2RImage = CompilationEntry.ItemSpec; + + if (!string.IsNullOrEmpty(DiaSymReader) && !File.Exists(DiaSymReader)) + { + Log.LogError(Strings.DiaSymReaderLibraryNotFound, DiaSymReader); + return false; + } + + // R2R image has to be created before emitting native symbols (crossgen needs this as an input argument) + if (string.IsNullOrEmpty(_outputPDBImage)) + { + Log.LogError(Strings.MissingOutputPDBImagePath); + } + + if (!File.Exists(_outputR2RImage)) + { + Log.LogError(Strings.PDBGeneratorInputExecutableNotFound, _outputR2RImage); + return false; + } + } + else + { + _outputR2RImage = CompilationEntry.GetMetadata(MetadataKeys.OutputR2RImage); + + if (!_createCompositeImage) + { + _inputAssembly = CompilationEntry.ItemSpec; + if (!File.Exists(_inputAssembly)) + { + Log.LogError(Strings.InputAssemblyNotFound, _inputAssembly); + return false; + } + } + else + { + _inputAssembly = "CompositeImage"; + } + + if (string.IsNullOrEmpty(_outputR2RImage)) + { + Log.LogError(Strings.MissingOutputR2RImageFileName); + return false; + } + + if (_emitSymbols && string.IsNullOrEmpty(_outputPDBImage)) + { + Log.LogError(Strings.MissingOutputPDBImagePath); + } + } + + return true; + } + + private string GetAssemblyReferencesCommands() + { + StringBuilder result = new StringBuilder(); + + var references = _createCompositeImage ? ReadyToRunCompositeBuildReferences : ImplementationAssemblyReferences; + + if (references != null) + { + foreach (var reference in (_createCompositeImage ? ReadyToRunCompositeBuildReferences : ImplementationAssemblyReferences)) + { + // When generating PDBs, we must not add a reference to the IL version of the R2R image for which we're trying to generate a PDB + if (IsPdbCompilation && string.Equals(Path.GetFileName(reference.ItemSpec), Path.GetFileName(_outputR2RImage), StringComparison.OrdinalIgnoreCase)) + continue; + + if (UseCrossgen2 && !IsPdbCompilation) + { + result.AppendLine($"-r:\"{reference}\""); + } + else + { + result.AppendLine($"-r \"{reference}\""); + } + } + } + + return result.ToString(); + } + + protected override string GenerateCommandLineCommands() + { + if (ActuallyUseCrossgen2 && !string.IsNullOrEmpty(DotNetHostPath)) + { + return $"\"{Crossgen2Tool.ItemSpec}\""; + } + return null; + } + + protected override string GenerateResponseFileCommands() + { + // Crossgen2 5.0 doesn't support PDB generation so Crossgen1 is used for that purpose. + if (ActuallyUseCrossgen2) + { + return GenerateCrossgen2ResponseFile(); + } + else + { + return GenerateCrossgenResponseFile(); + } + } + + private string GenerateCrossgenResponseFile() + { + StringBuilder result = new StringBuilder(); + + result.AppendLine("/nologo"); + + if (IsPdbCompilation) + { + result.Append(GetAssemblyReferencesCommands()); + + if (!string.IsNullOrEmpty(DiaSymReader)) + { + result.AppendLine($"/DiasymreaderPath \"{DiaSymReader}\""); + } + + result.AppendLine(_createPDBCommand); + result.AppendLine($"\"{_outputR2RImage}\""); + } + else + { + result.AppendLine("/MissingDependenciesOK"); + result.AppendLine($"/JITPath \"{CrossgenTool.GetMetadata(MetadataKeys.JitPath)}\""); + result.Append(GetAssemblyReferencesCommands()); + result.AppendLine($"/out \"{_outputR2RImage}\""); + result.AppendLine($"\"{_inputAssembly}\""); + } + + return result.ToString(); + } + + private string GenerateCrossgen2ResponseFile() + { + StringBuilder result = new StringBuilder(); + + string jitPath = Crossgen2Tool.GetMetadata(MetadataKeys.JitPath); + if (!string.IsNullOrEmpty(jitPath)) + { + result.AppendLine($"--jitpath:\"{jitPath}\""); + } + else + { + result.AppendLine($"--targetos:{Crossgen2Tool.GetMetadata(MetadataKeys.TargetOS)}"); + result.AppendLine($"--targetarch:{Crossgen2Tool.GetMetadata(MetadataKeys.TargetArch)}"); + } + + result.AppendLine("-O"); + + // 5.0 Crossgen2 doesn't support PDB generation. + if (!Crossgen2IsVersion5 && _emitSymbols) + { + if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) + { + result.AppendLine("--pdb"); + result.AppendLine($"--pdb-path:{Path.GetDirectoryName(_outputPDBImage)}"); + } + else + { + result.AppendLine("--perfmap"); + result.AppendLine($"--perfmap-path:{Path.GetDirectoryName(_outputPDBImage)}"); + } + } + + if (!string.IsNullOrEmpty(Crossgen2ExtraCommandLineArgs)) + { + foreach (string extraArg in Crossgen2ExtraCommandLineArgs.Split(new char[]{';'}, StringSplitOptions.RemoveEmptyEntries)) + { + result.AppendLine(extraArg); + } + } + + if (_createCompositeImage) + { + result.AppendLine("--composite"); + + // Crossgen2 v5 only supported compilation with --inputbubble specified + if (Crossgen2IsVersion5) + result.AppendLine("--inputbubble"); + + result.AppendLine($"--out:\"{_outputR2RImage}\""); + + result.Append(GetAssemblyReferencesCommands()); + + // Note: do not add double quotes around the input assembly, even if the file path contains spaces. The command line + // parsing logic will append this string to the working directory if it's a relative path, so any double quotes will result in errors. + foreach (var reference in ReadyToRunCompositeBuildInput) + { + result.AppendLine(reference.ItemSpec); + } + } + else + { + result.Append(GetAssemblyReferencesCommands()); + result.AppendLine($"--out:\"{_outputR2RImage}\""); + + // Note: do not add double quotes around the input assembly, even if the file path contains spaces. The command line + // parsing logic will append this string to the working directory if it's a relative path, so any double quotes will result in errors. + result.AppendLine($"{_inputAssembly}"); + } + + return result.ToString(); + } + + protected override int ExecuteTool(string pathToTool, string responseFileCommands, string commandLineCommands) + { + // Ensure output sub-directories exists - Crossgen does not create directories for output files. Any relative path used with the + // '/out' parameter has to have an existing directory. + Directory.CreateDirectory(Path.GetDirectoryName(_outputR2RImage)); + + WarningsDetected = false; + + return base.ExecuteTool(pathToTool, responseFileCommands, commandLineCommands); + } + + protected override void LogEventsFromTextOutput(string singleLine, MessageImportance messageImportance) + { + if (!ShowCompilerWarnings && singleLine.IndexOf("warning:", StringComparison.OrdinalIgnoreCase) != -1) + { + Log.LogMessage(MessageImportance.Normal, singleLine); + WarningsDetected = true; + } + else + { + base.LogEventsFromTextOutput(singleLine, messageImportance); + } + } + } +} diff --git a/src/tasks/Crossgen2Tasks/ShimFilesSimulatingLogicInSdkRepo/Microsoft.NET.CrossGen.props b/src/tasks/Crossgen2Tasks/ShimFilesSimulatingLogicInSdkRepo/Microsoft.NET.CrossGen.props new file mode 100644 index 0000000..99dab14 --- /dev/null +++ b/src/tasks/Crossgen2Tasks/ShimFilesSimulatingLogicInSdkRepo/Microsoft.NET.CrossGen.props @@ -0,0 +1,20 @@ + + + + true + + + + + + \ No newline at end of file diff --git a/src/tasks/Crossgen2Tasks/ShimFilesSimulatingLogicInSdkRepo/SimpleItemUtilities.cs b/src/tasks/Crossgen2Tasks/ShimFilesSimulatingLogicInSdkRepo/SimpleItemUtilities.cs new file mode 100644 index 0000000..1f975bc --- /dev/null +++ b/src/tasks/Crossgen2Tasks/ShimFilesSimulatingLogicInSdkRepo/SimpleItemUtilities.cs @@ -0,0 +1,40 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Microsoft.Build.Framework; +using System; +using System.IO; + +namespace Microsoft.NET.Build.Tasks +{ + internal static partial class ItemUtilities + { + public static bool? GetBooleanMetadata(this ITaskItem item, string metadataName) + { + bool? result = null; + + string value = item.GetMetadata(metadataName); + bool parsedResult; + if (bool.TryParse(value, out parsedResult)) + { + result = parsedResult; + } + + return result; + } + + public static bool HasMetadataValue(this ITaskItem item, string name) + { + string value = item.GetMetadata(name); + + return !string.IsNullOrEmpty(value); + } + + public static bool HasMetadataValue(this ITaskItem item, string name, string expectedValue) + { + string value = item.GetMetadata(name); + + return string.Equals(value, expectedValue, StringComparison.OrdinalIgnoreCase); + } + } +} \ No newline at end of file diff --git a/src/tasks/Crossgen2Tasks/ShimFilesSimulatingLogicInSdkRepo/Strings.cs b/src/tasks/Crossgen2Tasks/ShimFilesSimulatingLogicInSdkRepo/Strings.cs new file mode 100644 index 0000000..2321712 --- /dev/null +++ b/src/tasks/Crossgen2Tasks/ShimFilesSimulatingLogicInSdkRepo/Strings.cs @@ -0,0 +1,25 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +namespace Microsoft.NET.Build.Tasks +{ + public static class Strings + { + public static string Crossgen5CannotEmitSymbolsInCompositeMode = "Crossgen5CannotEmitSymbolsInCompositeMode"; + public static string Crossgen2ToolMissingWhenUseCrossgen2IsSet = "Crossgen2ToolMissingWhenUseCrossgen2IsSet"; + public static string Crossgen2ToolExecutableNotFound = "Crossgen2ToolExecutableNotFound"; + public static string DotNetHostExecutableNotFound = "DotNetHostExecutableNotFound"; + public static string JitLibraryNotFound = "JitLibraryNotFound"; + public static string Crossgen2MissingRequiredMetadata = "Crossgen2MissingRequiredMetadata"; + public static string CrossgenToolMissingWhenUseCrossgen2IsNotSet = "CrossgenToolMissingWhenUseCrossgen2IsNotSet"; + public static string DiaSymReaderLibraryNotFound = "DiaSymReaderLibraryNotFound"; + public static string MissingOutputPDBImagePath = "MissingOutputPDBImagePath"; + public static string PDBGeneratorInputExecutableNotFound = "PDBGeneratorInputExecutableNotFound"; + public static string InputAssemblyNotFound = "InputAssemblyNotFound"; + public static string MissingOutputR2RImageFileName = "MissingOutputR2RImageFileName"; + public static string CrossgenToolMissingInPDBCompilationMode = "CrossgenToolMissingInPDBCompilationMode"; + public static string CrossgenToolExecutableNotFound = "CrossgenToolExecutableNotFound"; + public static string ReadyToRunNoValidRuntimePackageError = "ReadyToRunNoValidRuntimePackageError"; + public static string ReadyToRunTargetNotSupportedError = "ReadyToRunTargetNotSupportedError"; + } +} \ No newline at end of file