Generate deterministic timestamp (#40122)
authorDavid Wrighton <davidwr@microsoft.com>
Thu, 30 Jul 2020 19:31:32 +0000 (12:31 -0700)
committerGitHub <noreply@github.com>
Thu, 30 Jul 2020 19:31:32 +0000 (12:31 -0700)
- Port the logic from the Roslyn compiler that handles deterministic timestamp generation
- Enable for composite images

This allows matching up generated pdb files with the composite image to work correctly in all known cases.

src/coreclr/src/tools/aot/ILCompiler.ReadyToRun/CodeGen/ReadyToRunObjectWriter.cs
src/coreclr/src/tools/aot/ILCompiler.ReadyToRun/Compiler/CryptographicHashProvider.cs [new file with mode: 0644]
src/coreclr/src/tools/aot/ILCompiler.ReadyToRun/Compiler/DependencyAnalysis/ReadyToRun/DebugDirectoryNode.cs
src/coreclr/src/tools/aot/ILCompiler.ReadyToRun/IBC/IBCProfileParser.cs
src/coreclr/src/tools/aot/ILCompiler.ReadyToRun/ILCompiler.ReadyToRun.csproj
src/coreclr/src/tools/aot/ILCompiler.ReadyToRun/ObjectWriter/R2RPEBuilder.cs

index 20d6dac..68239af 100644 (file)
@@ -6,6 +6,7 @@ using System.Collections.Generic;
 using System.Diagnostics;
 using System.IO;
 using System.Linq;
+using System.Reflection.Metadata;
 using System.Reflection.PortableExecutable;
 
 using ILCompiler.DependencyAnalysis.ReadyToRun;
@@ -102,8 +103,9 @@ namespace ILCompiler.DependencyAnalysis
                 stopwatch.Start();
 
                 PEHeaderBuilder headerBuilder;
-                int timeDateStamp;
+                int? timeDateStamp;
                 ISymbolNode r2rHeaderExportSymbol;
+                Func<IEnumerable<Blob>, BlobContentId> peIdProvider = null;
 
                 if (_nodeFactory.CompilationModuleGroup.IsCompositeBuildMode && _componentModule == null)
                 {
@@ -112,8 +114,8 @@ namespace ILCompiler.DependencyAnalysis
                         dllCharacteristics: default(DllCharacteristics),
                         Subsystem.Unknown,
                         _nodeFactory.Target);
-                    // TODO: generate a non-zero timestamp: https://github.com/dotnet/runtime/issues/32507
-                    timeDateStamp = 0;
+                    peIdProvider = new Func<IEnumerable<Blob>, BlobContentId>(content => BlobContentId.FromHash(CryptographicHashProvider.ComputeSourceHash(content)));
+                    timeDateStamp = null;
                     r2rHeaderExportSymbol = _nodeFactory.Header;
                 }
                 else
@@ -135,7 +137,8 @@ namespace ILCompiler.DependencyAnalysis
                     r2rHeaderExportSymbol,
                     Path.GetFileName(_objectFilePath),
                     getRuntimeFunctionsTable,
-                    _customPESectionAlignment);
+                    _customPESectionAlignment,
+                    peIdProvider);
 
                 NativeDebugDirectoryEntryNode nativeDebugDirectoryEntryNode = null;
                 ISymbolDefinitionNode firstImportThunk = null;
diff --git a/src/coreclr/src/tools/aot/ILCompiler.ReadyToRun/Compiler/CryptographicHashProvider.cs b/src/coreclr/src/tools/aot/ILCompiler.ReadyToRun/Compiler/CryptographicHashProvider.cs
new file mode 100644 (file)
index 0000000..dafd0c1
--- /dev/null
@@ -0,0 +1,252 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+#nullable enable
+
+using System;
+using System.Collections.Generic;
+using System.Collections.Immutable;
+using System.IO;
+using System.Linq;
+using System.Reflection;
+using System.Reflection.Metadata;
+using System.Security.Cryptography;
+
+namespace ILCompiler
+{
+    /// <summary>
+    /// Specifies a hash algorithms used for hashing source files.
+    /// </summary>
+    public enum SourceHashAlgorithm
+    {
+        /// <summary>
+        /// No algorithm specified.
+        /// </summary>
+        None = 0,
+
+        /// <summary>
+        /// Secure Hash Algorithm 1.
+        /// </summary>
+        Sha1 = 1,
+
+        /// <summary>
+        /// Secure Hash Algorithm 2 with a hash size of 256 bits.
+        /// </summary>
+        Sha256 = 2,
+    }
+
+    internal static class SourceHashAlgorithmUtils
+    {
+        public const SourceHashAlgorithm DefaultContentHashAlgorithm = SourceHashAlgorithm.Sha256;
+    }
+
+    internal abstract class CryptographicHashProvider
+    {
+        private ImmutableArray<byte> _lazySHA1Hash;
+        private ImmutableArray<byte> _lazySHA256Hash;
+        private ImmutableArray<byte> _lazySHA384Hash;
+        private ImmutableArray<byte> _lazySHA512Hash;
+        private ImmutableArray<byte> _lazyMD5Hash;
+
+        internal abstract ImmutableArray<byte> ComputeHash(HashAlgorithm algorithm);
+
+        internal ImmutableArray<byte> GetHash(AssemblyHashAlgorithm algorithmId)
+        {
+            using (HashAlgorithm? algorithm = TryGetAlgorithm(algorithmId))
+            {
+                // ERR_CryptoHashFailed has already been reported:
+                if (algorithm == null)
+                {
+                    return ImmutableArray.Create<byte>();
+                }
+
+                switch (algorithmId)
+                {
+                    case AssemblyHashAlgorithm.None:
+                    case AssemblyHashAlgorithm.Sha1:
+                        return GetHash(ref _lazySHA1Hash, algorithm);
+
+                    case AssemblyHashAlgorithm.Sha256:
+                        return GetHash(ref _lazySHA256Hash, algorithm);
+
+                    case AssemblyHashAlgorithm.Sha384:
+                        return GetHash(ref _lazySHA384Hash, algorithm);
+
+                    case AssemblyHashAlgorithm.Sha512:
+                        return GetHash(ref _lazySHA512Hash, algorithm);
+
+                    case AssemblyHashAlgorithm.MD5:
+                        return GetHash(ref _lazyMD5Hash, algorithm);
+
+                    default:
+                        throw new ArgumentException("algorithmId");
+                }
+            }
+        }
+
+        internal static int GetHashSize(SourceHashAlgorithm algorithmId)
+        {
+            switch (algorithmId)
+            {
+                case SourceHashAlgorithm.Sha1:
+                    return 160 / 8;
+
+                case SourceHashAlgorithm.Sha256:
+                    return 256 / 8;
+
+                default:
+                    throw new ArgumentException("algorithmId");
+            }
+        }
+
+        internal static HashAlgorithm? TryGetAlgorithm(SourceHashAlgorithm algorithmId)
+        {
+            switch (algorithmId)
+            {
+                case SourceHashAlgorithm.Sha1:
+                    return SHA1.Create();
+
+                case SourceHashAlgorithm.Sha256:
+                    return SHA256.Create();
+
+                default:
+                    return null;
+            }
+        }
+
+        internal static HashAlgorithmName GetAlgorithmName(SourceHashAlgorithm algorithmId)
+        {
+            switch (algorithmId)
+            {
+                case SourceHashAlgorithm.Sha1:
+                    return HashAlgorithmName.SHA1;
+
+                case SourceHashAlgorithm.Sha256:
+                    return HashAlgorithmName.SHA256;
+
+                default:
+                    throw new ArgumentException("algorithmId");
+            }
+        }
+
+        internal static HashAlgorithm? TryGetAlgorithm(AssemblyHashAlgorithm algorithmId)
+        {
+            switch (algorithmId)
+            {
+                case AssemblyHashAlgorithm.None:
+                case AssemblyHashAlgorithm.Sha1:
+                    return SHA1.Create();
+
+                case AssemblyHashAlgorithm.Sha256:
+                    return SHA256.Create();
+
+                case AssemblyHashAlgorithm.Sha384:
+                    return SHA384.Create();
+
+                case AssemblyHashAlgorithm.Sha512:
+                    return SHA512.Create();
+
+                case AssemblyHashAlgorithm.MD5:
+                    return MD5.Create();
+
+                default:
+                    return null;
+            }
+        }
+
+        internal static bool IsSupportedAlgorithm(AssemblyHashAlgorithm algorithmId)
+        {
+            switch (algorithmId)
+            {
+                case AssemblyHashAlgorithm.None:
+                case AssemblyHashAlgorithm.Sha1:
+                case AssemblyHashAlgorithm.Sha256:
+                case AssemblyHashAlgorithm.Sha384:
+                case AssemblyHashAlgorithm.Sha512:
+                case AssemblyHashAlgorithm.MD5:
+                    return true;
+
+                default:
+                    return false;
+            }
+        }
+
+        private ImmutableArray<byte> GetHash(ref ImmutableArray<byte> lazyHash, HashAlgorithm algorithm)
+        {
+            if (lazyHash.IsDefault)
+            {
+                ImmutableInterlocked.InterlockedCompareExchange(ref lazyHash, ComputeHash(algorithm), default(ImmutableArray<byte>));
+            }
+
+            return lazyHash;
+        }
+
+        internal const int Sha1HashSize = 20;
+
+        internal static ImmutableArray<byte> ComputeSha1(Stream stream)
+        {
+            if (stream != null)
+            {
+                stream.Seek(0, SeekOrigin.Begin);
+                using (var hashProvider = SHA1.Create())
+                {
+                    return ImmutableArray.Create(hashProvider.ComputeHash(stream));
+                }
+            }
+
+            return ImmutableArray<byte>.Empty;
+        }
+
+        internal static ImmutableArray<byte> ComputeSha1(ImmutableArray<byte> bytes)
+        {
+            return ComputeSha1(bytes.ToArray());
+        }
+
+        internal static ImmutableArray<byte> ComputeSha1(byte[] bytes)
+        {
+            using (var hashProvider = SHA1.Create())
+            {
+                return ImmutableArray.Create(hashProvider.ComputeHash(bytes));
+            }
+        }
+
+        internal static ImmutableArray<byte> ComputeHash(HashAlgorithmName algorithmName, IEnumerable<Blob> bytes)
+        {
+            using (var incrementalHash = IncrementalHash.CreateHash(algorithmName))
+            {
+                foreach (var blob in bytes)
+                {
+                    incrementalHash.AppendData(blob.GetBytes());
+                }
+                return ImmutableArray.Create(incrementalHash.GetHashAndReset());
+            }
+        }
+
+        internal static ImmutableArray<byte> ComputeHash(HashAlgorithmName algorithmName, IEnumerable<ArraySegment<byte>> bytes)
+        {
+            using (var incrementalHash = IncrementalHash.CreateHash(algorithmName))
+            {
+                foreach (var segment in bytes)
+                {
+                    incrementalHash.AppendData(segment);
+                }
+                return ImmutableArray.Create(incrementalHash.GetHashAndReset());
+            }
+        }
+
+        internal static ImmutableArray<byte> ComputeSourceHash(ImmutableArray<byte> bytes, SourceHashAlgorithm hashAlgorithm = SourceHashAlgorithmUtils.DefaultContentHashAlgorithm)
+        {
+            var algorithmName = GetAlgorithmName(hashAlgorithm);
+            using (var incrementalHash = IncrementalHash.CreateHash(algorithmName))
+            {
+                incrementalHash.AppendData(bytes.ToArray());
+                return ImmutableArray.Create(incrementalHash.GetHashAndReset());
+            }
+        }
+
+        internal static ImmutableArray<byte> ComputeSourceHash(IEnumerable<Blob> bytes, SourceHashAlgorithm hashAlgorithm = SourceHashAlgorithmUtils.DefaultContentHashAlgorithm)
+        {
+            return ComputeHash(GetAlgorithmName(hashAlgorithm), bytes);
+        }
+    }
+}
index 08093ec..1750254 100644 (file)
@@ -26,10 +26,12 @@ namespace ILCompiler.DependencyAnalysis.ReadyToRun
 
         private EcmaModule _module;
         private NativeDebugDirectoryEntryNode _nativeEntry;
+        private bool _insertDeterministicEntry;
 
         public DebugDirectoryNode(EcmaModule sourceModule, string outputFileName)
         {
             _module = sourceModule;
+            _insertDeterministicEntry = sourceModule == null; // Mark module as deterministic if generating composite image
             string pdbNameRoot = Path.GetFileNameWithoutExtension(outputFileName);
             if (sourceModule != null)
             {
@@ -50,7 +52,7 @@ namespace ILCompiler.DependencyAnalysis.ReadyToRun
 
         public int Offset => 0;
 
-        public int Size => (GetNumDebugDirectoryEntriesInModule() + 1) * ImageDebugDirectorySize;
+        public int Size => (GetNumDebugDirectoryEntriesInModule() + 1 + (_insertDeterministicEntry ? 1 : 0)) * ImageDebugDirectorySize;
 
         public void AppendMangledName(NameMangler nameMangler, Utf8StringBuilder sb)
         {
@@ -112,8 +114,21 @@ namespace ILCompiler.DependencyAnalysis.ReadyToRun
                 builder.EmitReloc(entry, RelocType.IMAGE_REL_FILE_ABSOLUTE);
             }
 
+            // If generating a composite image, emit the deterministic marker
+            if (_insertDeterministicEntry)
+            {
+                builder.EmitUInt(0 /* Characteristics */);
+                builder.EmitUInt(0);
+                builder.EmitUShort(0);
+                builder.EmitUShort(0);
+                builder.EmitInt((int)DebugDirectoryEntryType.Reproducible);
+                builder.EmitInt(0);
+                builder.EmitUInt(0);
+                builder.EmitUInt(0);
+            }
+
             // Second, copy existing entries from input module
-            for(int i = 0; i < numEntries; i++)
+            for (int i = 0; i < numEntries; i++)
             {
                 builder.EmitUInt(0 /* Characteristics */);
                 builder.EmitUInt(entries[i].Stamp);
index c42ecaa..718d4e6 100644 (file)
@@ -113,7 +113,8 @@ namespace ILCompiler.IBC
                             }
                             else
                             {
-                                _logger.Writer.WriteLine($"Token {0:x} does not refer to a method");
+                                if (_logger.IsVerbose)
+                                    _logger.Writer.WriteLine($"Token {(int)entry.Token:x} does not refer to a method");
                             }
                             break;
 
index 4839738..b8d0118 100644 (file)
@@ -98,6 +98,7 @@
     <Compile Include="ObjectWriter\MapFileBuilder.cs" />
     <Compile Include="CodeGen\ReadyToRunObjectWriter.cs" />
     <Compile Include="Compiler\CompilationModuleGroup.ReadyToRun.cs" />
+    <Compile Include="Compiler\CryptographicHashProvider.cs" />
     <Compile Include="Compiler\DependencyAnalysis\AllMethodsOnTypeNode.cs" />
     <Compile Include="Compiler\DependencyAnalysis\ArrayOfEmbeddedDataNode.cs" />
     <Compile Include="Compiler\DependencyAnalysis\ArrayOfEmbeddedPointersNode.cs" />
index 839b1f2..e1731e4 100644 (file)
@@ -173,8 +173,9 @@ namespace ILCompiler.PEWriter
             ISymbolNode r2rHeaderExportSymbol,
             string outputFileSimpleName,
             Func<RuntimeFunctionsTableNode> getRuntimeFunctionsTable,
-            int? customPESectionAlignment)
-            : base(peHeaderBuilder, deterministicIdProvider: null)
+            int? customPESectionAlignment,
+            Func<IEnumerable<Blob>, BlobContentId> deterministicIdProvider)
+            : base(peHeaderBuilder, deterministicIdProvider: deterministicIdProvider)
         {
             _target = target;
             _getRuntimeFunctionsTable = getRuntimeFunctionsTable;
@@ -288,7 +289,7 @@ namespace ILCompiler.PEWriter
         /// </summary>
         /// <param name="outputStream">Output stream for the final R2R PE file</param>
         /// <param name="timeDateStamp">Timestamp to set in the PE header of the output R2R executable</param>
-        public void Write(Stream outputStream, int timeDateStamp)
+        public void Write(Stream outputStream, int? timeDateStamp)
         {
             BlobBuilder outputPeFile = new BlobBuilder();
             Serialize(outputPeFile);
@@ -302,7 +303,8 @@ namespace ILCompiler.PEWriter
 
             ApplyMachineOSOverride(outputStream);
 
-            SetPEHeaderTimeStamp(outputStream, timeDateStamp);
+            if (timeDateStamp.HasValue)
+                SetPEHeaderTimeStamp(outputStream, timeDateStamp.Value);
 
             _written = true;
         }