R2RDump improvements in composite format support (#32499)
authorTomáš Rylek <trylek@microsoft.com>
Tue, 25 Feb 2020 08:55:31 +0000 (09:55 +0100)
committerGitHub <noreply@github.com>
Tue, 25 Feb 2020 08:55:31 +0000 (09:55 +0100)
1) Added back-translation table from component assembly names
to their indices;

2) Added clean split of available types and method entrypoints per
component module and modified the various dumps as appropriate;

3) Added dump of component header sections for composite files.

Thanks

Tomas

src/coreclr/src/tools/crossgen2/ILCompiler.Reflection.ReadyToRun/ReadyToRunReader.cs
src/coreclr/src/tools/r2rdump/R2RDiff.cs
src/coreclr/src/tools/r2rdump/R2RDump.cs
src/coreclr/src/tools/r2rdump/TextDumper.cs

index 01132e3..983a624 100644 (file)
@@ -18,6 +18,7 @@ using Internal.Runtime;
 using Internal.ReadyToRunConstants;
 
 using Debug = System.Diagnostics.Debug;
+using System.Linq;
 
 namespace ILCompiler.Reflection.ReadyToRun
 {
@@ -83,12 +84,13 @@ namespace ILCompiler.Reflection.ReadyToRun
         // ManifestReferences
         private MetadataReader _manifestReader;
         private List<AssemblyReferenceHandle> _manifestReferences;
+        private Dictionary<string, int> _manifestReferenceAssemblies;
 
         // ExceptionInfo
         private Dictionary<int, EHInfo> _runtimeFunctionToEHInfo;
 
         // Methods
-        private List<ReadyToRunMethod> _methods;
+        private Dictionary<ReadyToRunSection, List<ReadyToRunMethod>> _methods;
         private List<InstanceMethod> _instanceMethods;
 
         // ImportSections
@@ -96,7 +98,7 @@ namespace ILCompiler.Reflection.ReadyToRun
         private Dictionary<int, string> _importCellNames;
 
         // AvailableType
-        private List<string> _availableTypes;
+        private Dictionary<ReadyToRunSection, List<string>> _availableTypes;
 
         // CompilerIdentifier
         private string _compilerIdentifier;
@@ -127,15 +129,12 @@ namespace ILCompiler.Reflection.ReadyToRun
         /// The list originates in the top-level R2R image and is copied
         /// to all reference assemblies for the sake of simplicity.
         /// </summary>
-        public IEnumerable<string> ManifestReferenceAssemblies
+        public Dictionary<string, int> ManifestReferenceAssemblies
         {
             get
             {
-                // TODO (refactoring) make this a IReadOnlyList<string> to be consistent with the rest of the interface
-                foreach (AssemblyReferenceHandle manifestReference in ManifestReferences)
-                {
-                    yield return ManifestReader.GetString(ManifestReader.GetAssemblyReference(manifestReference).Name);
-                }
+                EnsureManifestReferenceAssemblies();
+                return _manifestReferenceAssemblies;
             }
         }
 
@@ -224,7 +223,7 @@ namespace ILCompiler.Reflection.ReadyToRun
         /// <summary>
         /// The runtime functions and method signatures of each method
         /// </summary>
-        public IReadOnlyList<ReadyToRunMethod> Methods
+        public Dictionary<ReadyToRunSection, List<ReadyToRunMethod>> Methods
         {
             get
             {
@@ -248,7 +247,7 @@ namespace ILCompiler.Reflection.ReadyToRun
         /// <summary>
         /// The available types from READYTORUN_SECTION_AVAILABLE_TYPES
         /// </summary>
-        public IReadOnlyList<string> AvailableTypes
+        public Dictionary<ReadyToRunSection, List<string>> AvailableTypes
         {
 
             get
@@ -402,7 +401,8 @@ namespace ILCompiler.Reflection.ReadyToRun
             {
                 return;
             }
-            _methods = new List<ReadyToRunMethod>();
+
+            _methods = new Dictionary<ReadyToRunSection, List<ReadyToRunMethod>>();
             _instanceMethods = new List<InstanceMethod>();
 
             if (ReadyToRunHeader.Sections.TryGetValue(ReadyToRunSectionType.RuntimeFunctions, out ReadyToRunSection runtimeFunctionSection))
@@ -557,6 +557,21 @@ namespace ILCompiler.Reflection.ReadyToRun
             }
         }
 
+        private void EnsureManifestReferenceAssemblies()
+        {
+            if (_manifestReferenceAssemblies != null)
+            {
+                return;
+            }
+            EnsureManifestReferences();
+            _manifestReferenceAssemblies = new Dictionary<string, int>(_manifestReferences.Count);
+            for (int assemblyIndex = 0; assemblyIndex < _manifestReferences.Count; assemblyIndex++)
+            {
+                string assemblyName = ManifestReader.GetString(ManifestReader.GetAssemblyReference(_manifestReferences[assemblyIndex]).Name);
+                _manifestReferenceAssemblies.Add(assemblyName, assemblyIndex);
+            }
+        }
+
         private unsafe void EnsureExceptionInfo()
         {
             if (_runtimeFunctionToEHInfo != null)
@@ -659,7 +674,12 @@ namespace ILCompiler.Reflection.ReadyToRun
                         throw new BadImageFormatException("EntryPointRuntimeFunctionId out of bounds");
                     }
                     isEntryPoint[method.EntryPointRuntimeFunctionId] = true;
-                    _methods.Add(method);
+                    if (!_methods.TryGetValue(section, out List<ReadyToRunMethod> sectionMethods))
+                    {
+                        sectionMethods = new List<ReadyToRunMethod>();
+                        _methods.Add(section, sectionMethods);
+                    }
+                    sectionMethods.Add(method);
                 }
             }
         }
@@ -744,7 +764,12 @@ namespace ILCompiler.Reflection.ReadyToRun
                 {
                     isEntryPoint[method.EntryPointRuntimeFunctionId] = true;
                 }
-                _methods.Add(method);
+                if (!Methods.TryGetValue(instMethodEntryPointSection, out List<ReadyToRunMethod> sectionMethods))
+                {
+                    sectionMethods = new List<ReadyToRunMethod>();
+                    Methods.Add(instMethodEntryPointSection, sectionMethods);
+                }
+                sectionMethods.Add(method);
                 _instanceMethods.Add(new InstanceMethod(curParser.LowHashcode, method));
                 curParser = allEntriesEnum.GetNext();
             }
@@ -752,7 +777,7 @@ namespace ILCompiler.Reflection.ReadyToRun
 
         private void CountRuntimeFunctions(bool[] isEntryPoint)
         {
-            foreach (ReadyToRunMethod method in _methods)
+            foreach (ReadyToRunMethod method in Methods.Values.SelectMany(sectionMethods => sectionMethods))
             {
                 int runtimeFunctionId = method.EntryPointRuntimeFunctionId;
                 if (runtimeFunctionId == -1)
@@ -778,7 +803,7 @@ namespace ILCompiler.Reflection.ReadyToRun
             {
                 return;
             }
-            _availableTypes = new List<string>();
+            _availableTypes = new Dictionary<ReadyToRunSection, List<string>>();
             ReadyToRunSection availableTypesSection;
             if (ReadyToRunHeader.Sections.TryGetValue(ReadyToRunSectionType.AvailableTypes, out availableTypesSection))
             {
@@ -820,13 +845,23 @@ namespace ILCompiler.Reflection.ReadyToRun
                 {
                     ExportedTypeHandle exportedTypeHandle = MetadataTokens.ExportedTypeHandle((int)rid);
                     string exportedTypeName = GetExportedTypeFullName(metadataReader, exportedTypeHandle);
-                    _availableTypes.Add("exported " + exportedTypeName);
+                    if (!AvailableTypes.TryGetValue(availableTypesSection, out List<string> sectionTypes))
+                    {
+                        sectionTypes = new List<string>();
+                        AvailableTypes.Add(availableTypesSection, sectionTypes);
+                    }
+                    sectionTypes.Add("exported " + exportedTypeName);
                 }
                 else
                 {
                     TypeDefinitionHandle typeDefHandle = MetadataTokens.TypeDefinitionHandle((int)rid);
                     string typeDefName = MetadataNameFormatter.FormatHandle(metadataReader, typeDefHandle);
-                    _availableTypes.Add(typeDefName);
+                    if (!AvailableTypes.TryGetValue(availableTypesSection, out List<string> sectionTypes))
+                    {
+                        sectionTypes = new List<string>();
+                        AvailableTypes.Add(availableTypesSection, sectionTypes);
+                    }
+                    sectionTypes.Add(typeDefName);
                 }
 
                 curParser = allEntriesEnum.GetNext();
index 8a7021b..e1134bb 100644 (file)
@@ -57,9 +57,54 @@ namespace R2RDump
             DiffR2RSections();
             DiffR2RMethods();
 
-            Dictionary<string, ReadyToRunMethod> leftMethods = new Dictionary<string, ReadyToRunMethod>(_leftDumper.Reader.Methods
+            if (!_leftDumper.Reader.ReadyToRunHeader.Sections.TryGetValue(ReadyToRunSectionType.MethodDefEntryPoints, out ReadyToRunSection leftSection))
+            {
+                leftSection = new ReadyToRunSection();
+            }
+            if (!_rightDumper.Reader.ReadyToRunHeader.Sections.TryGetValue(ReadyToRunSectionType.MethodDefEntryPoints, out ReadyToRunSection rightSection))
+            {
+                rightSection = new ReadyToRunSection();
+            }
+
+            DiffMethodsForModule(leftSection, rightSection);
+
+            if (_leftDumper.Reader.Composite && _rightDumper.Reader.Composite)
+            {
+                HashSet<string> allComponentAssemblies = new HashSet<string>(_leftDumper.Reader.ManifestReferenceAssemblies.Keys);
+                allComponentAssemblies.UnionWith(_rightDumper.Reader.ManifestReferenceAssemblies.Keys);
+                foreach (string assemblyName in allComponentAssemblies.OrderBy(name => name))
+                {
+                    int leftIndex = _leftDumper.Reader.ManifestReferenceAssemblies[assemblyName];
+                    int rightIndex = _rightDumper.Reader.ManifestReferenceAssemblies[assemblyName];
+
+                    if (leftIndex < 0 ||  !_leftDumper.Reader.ReadyToRunAssemblyHeaders[leftIndex].Sections.TryGetValue(ReadyToRunSectionType.MethodDefEntryPoints, out leftSection))
+                    {
+                        leftSection = new ReadyToRunSection();
+                    }
+                    if (rightIndex < 0 || !_rightDumper.Reader.ReadyToRunAssemblyHeaders[rightIndex].Sections.TryGetValue(ReadyToRunSectionType.MethodDefEntryPoints, out rightSection))
+                    {
+                        rightSection = new ReadyToRunSection();
+                    }
+                    _writer.WriteLine($@"{assemblyName}: component method diff");
+                    DiffMethodsForModule(leftSection, rightSection);
+                }
+            }
+        }
+
+        private void DiffMethodsForModule(ReadyToRunSection leftSection, ReadyToRunSection rightSection)
+        {
+            if (!_leftDumper.Reader.Methods.TryGetValue(leftSection, out List<ReadyToRunMethod> leftSectionMethods))
+            {
+                leftSectionMethods = new List<ReadyToRunMethod>();
+            }
+            if (!_rightDumper.Reader.Methods.TryGetValue(leftSection, out List<ReadyToRunMethod> rightSectionMethods))
+            {
+                rightSectionMethods = new List<ReadyToRunMethod>();
+            }
+
+            Dictionary<string, ReadyToRunMethod> leftMethods = new Dictionary<string, ReadyToRunMethod>(leftSectionMethods
                 .Select(method => new KeyValuePair<string, ReadyToRunMethod>(method.SignatureString, method)));
-            Dictionary<string, ReadyToRunMethod> rightMethods = new Dictionary<string, ReadyToRunMethod>(_rightDumper.Reader.Methods
+            Dictionary<string, ReadyToRunMethod> rightMethods = new Dictionary<string, ReadyToRunMethod>(rightSectionMethods
                 .Select(method => new KeyValuePair<string, ReadyToRunMethod>(method.SignatureString, method)));
             Dictionary<string, MethodPair> commonMethods = new Dictionary<string, MethodPair>(leftMethods
                 .Select(kvp => new KeyValuePair<string, MethodPair>(kvp.Key,
@@ -69,8 +114,8 @@ namespace R2RDump
             {
                 commonMethods = new Dictionary<string, MethodPair>(HideMethodsWithSameDisassembly(commonMethods));
             }
-            DumpCommonMethods(_leftDumper, commonMethods);
-            DumpCommonMethods(_rightDumper, commonMethods);
+            DumpCommonMethods(_leftDumper, leftSection, commonMethods);
+            DumpCommonMethods(_rightDumper, rightSection, commonMethods);
         }
 
         /// <summary>
@@ -104,7 +149,40 @@ namespace R2RDump
         /// </summary>
         private void DiffR2RMethods()
         {
-            ShowDiff(GetR2RMethodMap(_leftDumper.Reader), GetR2RMethodMap(_rightDumper.Reader), "R2R methods");
+            DiffR2RMethodsForHeader(_leftDumper.Reader.ReadyToRunHeader, _rightDumper.Reader.ReadyToRunHeader);
+        }
+
+        private void DiffR2RMethodsForHeader(ReadyToRunCoreHeader leftHeader, ReadyToRunCoreHeader rightHeader)
+        {
+            if (!leftHeader.Sections.TryGetValue(ReadyToRunSectionType.MethodDefEntryPoints, out ReadyToRunSection leftSection))
+            {
+                leftSection = new ReadyToRunSection();
+            }
+            if (!rightHeader.Sections.TryGetValue(ReadyToRunSectionType.MethodDefEntryPoints, out ReadyToRunSection rightSection))
+            {
+                rightSection = new ReadyToRunSection();
+            }
+            ShowDiff(GetR2RMethodMap(_leftDumper.Reader, leftSection), GetR2RMethodMap(_rightDumper.Reader, rightSection), "R2R methods");
+
+            if (_leftDumper.Reader.Composite && _rightDumper.Reader.Composite)
+            {
+                HashSet<string> allComponentAssemblies = new HashSet<string>(_leftDumper.Reader.ManifestReferenceAssemblies.Keys);
+                allComponentAssemblies.UnionWith(_rightDumper.Reader.ManifestReferenceAssemblies.Keys);
+                foreach (string assemblyName in allComponentAssemblies.OrderBy(name => name))
+                {
+                    int leftIndex = _leftDumper.Reader.ManifestReferenceAssemblies[assemblyName];
+                    int rightIndex = _rightDumper.Reader.ManifestReferenceAssemblies[assemblyName];
+                    if (leftIndex < 0 || !_leftDumper.Reader.ReadyToRunAssemblyHeaders[leftIndex].Sections.TryGetValue(ReadyToRunSectionType.MethodDefEntryPoints, out leftSection))
+                    {
+                        leftSection = new ReadyToRunSection();
+                    }
+                    if (rightIndex < 0 || !_rightDumper.Reader.ReadyToRunAssemblyHeaders[rightIndex].Sections.TryGetValue(ReadyToRunSectionType.MethodDefEntryPoints, out rightSection))
+                    {
+                        rightSection = new ReadyToRunSection();
+                    }
+                    ShowDiff(GetR2RMethodMap(_leftDumper.Reader, leftSection), GetR2RMethodMap(_rightDumper.Reader, rightSection), $"{assemblyName}: component R2R methods");
+                }
+            }
         }
 
         /// <summary>
@@ -209,14 +287,17 @@ namespace R2RDump
         /// </summary>
         /// <param name="reader">R2R image to scan</param>
         /// <returns></returns>
-        private Dictionary<string, int> GetR2RMethodMap(ReadyToRunReader reader)
+        private Dictionary<string, int> GetR2RMethodMap(ReadyToRunReader reader, ReadyToRunSection section)
         {
             Dictionary<string, int> methodMap = new Dictionary<string, int>();
 
-            foreach (ReadyToRunMethod method in reader.Methods)
+            if (reader.Methods.TryGetValue(section, out List<ReadyToRunMethod> sectionMethods))
             {
-                int size = method.RuntimeFunctions.Sum(rf => rf.Size);
-                methodMap.Add(method.SignatureString, size);
+                foreach (ReadyToRunMethod method in sectionMethods)
+                {
+                    int size = method.RuntimeFunctions.Sum(rf => rf.Size);
+                    methodMap.Add(method.SignatureString, size);
+                }
             }
 
             return methodMap;
@@ -227,17 +308,18 @@ namespace R2RDump
         /// </summary>
         /// <param name="dumper">Output dumper to use</param>
         /// <param name="signatureFilter">Set of common signatures of methods to dump</param>
-        private void DumpCommonMethods(Dumper dumper, Dictionary<string, MethodPair> signatureFilter)
+        private void DumpCommonMethods(Dumper dumper, ReadyToRunSection section, Dictionary<string, MethodPair> signatureFilter)
         {
-            IEnumerable<ReadyToRunMethod> filteredMethods = dumper
-                .Reader
-                .Methods
-                .Where(method => signatureFilter.ContainsKey(method.SignatureString))
-                .OrderBy(method => method.SignatureString);
-
-            foreach (ReadyToRunMethod method in filteredMethods)
+            if (dumper.Reader.Methods.TryGetValue(section, out List<ReadyToRunMethod> sectionMethods))
             {
-                dumper.DumpMethod(method);
+                IEnumerable<ReadyToRunMethod> filteredMethods = sectionMethods
+                    .Where(method => signatureFilter.ContainsKey(method.SignatureString))
+                    .OrderBy(method => method.SignatureString);
+
+                foreach (ReadyToRunMethod method in filteredMethods)
+                {
+                    dumper.DumpMethod(method);
+                }
             }
         }
 
index ce459a5..b874fa6 100644 (file)
@@ -142,9 +142,9 @@ namespace R2RDump
             _options = options;
         }
 
-        public IEnumerable<ReadyToRunSection> NormalizedSections()
+        public IEnumerable<ReadyToRunSection> NormalizedSections(ReadyToRunCoreHeader header)
         {
-            IEnumerable<ReadyToRunSection> sections = _r2r.ReadyToRunHeader.Sections.Values;
+            IEnumerable<ReadyToRunSection> sections = header.Sections.Values;
             if (_options.Normalize)
             {
                 sections = sections.OrderBy((s) => s.Type);
@@ -154,7 +154,7 @@ namespace R2RDump
 
         public IEnumerable<ReadyToRunMethod> NormalizedMethods()
         {
-            IEnumerable<ReadyToRunMethod> methods = _r2r.Methods;
+            IEnumerable<ReadyToRunMethod> methods = _r2r.Methods.Values.SelectMany(sectionMethods => sectionMethods);
             if (_options.Normalize)
             {
                 methods = methods.OrderBy((m) => m.SignatureString);
@@ -441,7 +441,7 @@ namespace R2RDump
         public IList<ReadyToRunMethod> FindMethod(ReadyToRunReader r2r, string query, bool exact)
         {
             List<ReadyToRunMethod> res = new List<ReadyToRunMethod>();
-            foreach (ReadyToRunMethod method in r2r.Methods)
+            foreach (ReadyToRunMethod method in r2r.Methods.Values.SelectMany(sectionMethods => sectionMethods))
             {
                 if (Match(method, query, exact))
                 {
@@ -478,7 +478,7 @@ namespace R2RDump
         /// <param name="rtfQuery">The name or value to search for</param>
         public RuntimeFunction FindRuntimeFunction(ReadyToRunReader r2r, int rtfQuery)
         {
-            foreach (ReadyToRunMethod m in r2r.Methods)
+            foreach (ReadyToRunMethod m in r2r.Methods.Values.SelectMany(sectionMethods => sectionMethods))
             {
                 foreach (RuntimeFunction rtf in m.RuntimeFunctions)
                 {
index b5610d7..1b3cf27 100644 (file)
@@ -43,8 +43,8 @@ namespace R2RDump
 
         internal override void WriteDivider(string title)
         {
-            int len = 61 - title.Length - 2;
-            _writer.WriteLine(new String('=', len / 2) + " " + title + " " + new String('=', (int)Math.Ceiling(len / 2.0)));
+            int len = Math.Max(61 - title.Length - 2, 2);
+            _writer.WriteLine(new String('=', len / 2) + " " + title + " " + new String('=', (len + 1) / 2));
             SkipLine();
         }
 
@@ -77,10 +77,27 @@ namespace R2RDump
                 _writer.WriteLine($"{_r2r.ReadyToRunHeader.Sections.Count} sections");
                 SkipLine();
 
-                foreach (ReadyToRunSection section in NormalizedSections())
+                foreach (ReadyToRunSection section in NormalizedSections(_r2r.ReadyToRunHeader))
                 {
                     DumpSection(section);
                 }
+
+                if (_r2r.Composite)
+                {
+                    WriteDivider("Component Assembly Sections");
+                    int assemblyIndex = 0;
+                    foreach (string assemblyName in _r2r.ManifestReferenceAssemblies.OrderBy(kvp => kvp.Value).Select(kvp => kvp.Key))
+                    {
+                        WriteDivider($@"Component Assembly [{assemblyIndex}]: {assemblyName}");
+                        ReadyToRunCoreHeader assemblyHeader = _r2r.ReadyToRunAssemblyHeaders[assemblyIndex];
+                        foreach (ReadyToRunSection section in NormalizedSections(assemblyHeader))
+                        {
+                            DumpSection(section);
+                        }
+                        assemblyIndex++;
+                    }
+
+                }
             }
             SkipLine();
         }
@@ -289,9 +306,13 @@ namespace R2RDump
                         _writer.WriteLine(availableTypes.ToString());
                     }
 
-                    foreach (string name in _r2r.AvailableTypes)
+                    if (_r2r.AvailableTypes.TryGetValue(section, out List<string> sectionTypes))
                     {
-                        _writer.WriteLine(name);
+                        _writer.WriteLine();
+                        foreach (string name in sectionTypes)
+                        {
+                            _writer.WriteLine(name);
+                        }
                     }
                     break;
                 case ReadyToRunSectionType.MethodDefEntryPoints:
@@ -300,6 +321,15 @@ namespace R2RDump
                         NativeArray methodEntryPoints = new NativeArray(_r2r.Image, (uint)_r2r.GetOffset(section.RelativeVirtualAddress));
                         _writer.Write(methodEntryPoints.ToString());
                     }
+
+                    if (_r2r.Methods.TryGetValue(section, out List<ReadyToRunMethod> sectionMethods))
+                    {
+                        _writer.WriteLine();
+                        foreach (ReadyToRunMethod method in sectionMethods)
+                        {
+                            _writer.WriteLine($@"{MetadataTokens.GetToken(method.MethodHandle):X8}: {method.SignatureString}");
+                        }
+                    }
                     break;
                 case ReadyToRunSectionType.InstanceMethodEntryPoints:
                     if (!_options.Naked)
@@ -391,9 +421,9 @@ namespace R2RDump
                         }
                     }
 
-                    _writer.WriteLine($"Manifest metadata AssemblyRef's ({_r2r.ManifestReferenceAssemblies.Count()} entries):");
+                    _writer.WriteLine($"Manifest metadata AssemblyRef's ({_r2r.ManifestReferenceAssemblies.Count} entries):");
                     int manifestAsmIndex = 0;
-                    foreach (string manifestReferenceAssembly in _r2r.ManifestReferenceAssemblies)
+                    foreach (string manifestReferenceAssembly in _r2r.ManifestReferenceAssemblies.OrderBy(kvp => kvp.Value).Select(kvp => kvp.Key))
                     {
                         _writer.WriteLine($"[ID 0x{manifestAsmIndex + assemblyRefCount + 2:X2}]: {manifestReferenceAssembly}");
                         manifestAsmIndex++;