tests/scripts/crossgen_comparison.py

   1 #!/usr/bin/env python
   2 #
   3 # Licensed to the .NET Foundation under one or more agreements.
   4 # The .NET Foundation licenses this file to you under the MIT license.
   5 # See the LICENSE file in the project root for more information.
   6 #
   7 ################################################################################
   8 #
   9 # Module: crossgen_comparison.py
  10 #
  11 # Notes:
  12 #
  13 # Script that
  14 #   1) runs crossgen on System.Private.CoreLib.dll and CoreFX assemblies and
  15 #   collects information about the crossgen behaviour (such as the return code,
  16 #   stdout/stderr streams, SHA256 hash sum of the resulting file).
  17 #   2) compares the collected information from two crossgen scenarios (e.g.
  18 #   x86_arm vs. arm_arm) and report all the differences in their behaviour
  19 #   (such as mismatches in the resulting files; hash sums, or missing files).
  20 #
  21 # Example:
  22 #
  23 # The following command
  24 #
  25 #  ~/git/coreclr$ python tests/scripts/crossgen_comparison.py crossgen_corelib
  26 #  --crossgen bin/Product/Linux.arm.Checked/crossgen
  27 #  --il_corelib bin/Product/Linux.arm.Checked/IL/System.Private.CoreLib.dll
  28 #  --result_dir Linux.arm_arm.Checked
  29 #
  30 # runs arm_arm crossgen on System.Private.CoreLib.dll and puts all the
  31 # information in file Linux.arm_arm.Checked/System.Private.CoreLib.dll.json
  32 #
  33 #  ~/git/coreclr$ cat Linux.arm_arm.Checked/System.Private.CoreLib.dll.json
  34 #   {
  35 #     "AssemblyName": "System.Private.CoreLib.dll",
  36 #     "ReturnCode": 0,
  37 #     "OutputFileHash": "4d27c7f694c20974945e4f7cb43263286a18c56f4d00aac09f6124caa372ba0a",
  38 #     "StdErr": [],
  39 #     "StdOut": [
  40 #       "Native image /tmp/System.Private.CoreLib.dll generated successfully."
  41 #     ]
  42 #   }
  43 ################################################################################
  44 ################################################################################
  45
  46 import argparse
  47 import glob
  48 import json
  49 import hashlib
  50 import os
  51 import tempfile
  52 import sys
  53 import subprocess
  54
  55 ################################################################################
  56 # Argument Parser
  57 ################################################################################
  58
  59 def build_argument_parser():
  60     description = """Script that runs crossgen on different assemblies and
  61         collects/compares information about the crossgen behaviour."""
  62
  63     parser = argparse.ArgumentParser(description=description)
  64
  65     subparsers = parser.add_subparsers()
  66
  67     crossgen_corelib_description = """Runs crossgen on IL System.Private.CoreLib.dll and
  68         collects information about the run."""
  69
  70     corelib_parser = subparsers.add_parser('crossgen_corelib', description=crossgen_corelib_description)
  71     corelib_parser.add_argument('--crossgen', dest='crossgen_executable_filename', required=True)
  72     corelib_parser.add_argument('--il_corelib', dest='il_corelib_filename', required=True)
  73     corelib_parser.add_argument('--result_dir', dest='result_dirname', required=True)
  74     corelib_parser.set_defaults(func=crossgen_corelib)
  75
  76     frameworks_parser_description = """Runs crossgen on each assembly in Core_Root and
  77         collects information about all the runs."""
  78
  79     frameworks_parser = subparsers.add_parser('crossgen_framework', description=frameworks_parser_description)
  80     frameworks_parser.add_argument('--crossgen', dest='crossgen_executable_filename', required=True)
  81     frameworks_parser.add_argument('--core_root', dest='core_root', required=True)
  82     frameworks_parser.add_argument('--result_dir', dest='result_dirname', required=True)
  83     frameworks_parser.set_defaults(func=crossgen_framework)
  84
  85     compare_parser_description = """Compares collected information from two crossgen scenarios - base vs diff"""
  86
  87     compare_parser = subparsers.add_parser('compare', description=compare_parser_description)
  88     compare_parser.add_argument('--base_dir', dest='base_dirname', required=True)
  89     compare_parser.add_argument('--diff_dir', dest='diff_dirname', required=True)
  90     compare_parser.set_defaults(func=compare)
  91
  92     return parser
  93
  94 ################################################################################
  95 # Globals
  96 ################################################################################
  97
  98 # List of framework assemblies used for crossgen_framework command
  99 g_Framework_Assemblies = [
 100     'CommandLine.dll',
 101     'Microsoft.CodeAnalysis.CSharp.dll',
 102     'Microsoft.CodeAnalysis.dll',
 103     'Microsoft.CodeAnalysis.VisualBasic.dll',
 104     'Microsoft.CSharp.dll',
 105     'Microsoft.Diagnostics.FastSerialization.dll',
 106     'Microsoft.Diagnostics.Tracing.TraceEvent.dll',
 107     'Microsoft.DotNet.Cli.Utils.dll',
 108     'Microsoft.DotNet.InternalAbstractions.dll',
 109     'Microsoft.DotNet.ProjectModel.dll',
 110     'Microsoft.Extensions.DependencyModel.dll',
 111 #   'Microsoft.VisualBasic.dll', see https://github.com/dotnet/coreclr/issues/19370
 112     'Microsoft.Win32.Primitives.dll',
 113     'Microsoft.Win32.Registry.dll',
 114     'netstandard.dll',
 115     'Newtonsoft.Json.dll',
 116     'NuGet.Common.dll',
 117     'NuGet.Configuration.dll',
 118     'NuGet.DependencyResolver.Core.dll',
 119     'NuGet.Frameworks.dll',
 120     'NuGet.LibraryModel.dll',
 121     'NuGet.Packaging.Core.dll',
 122     'NuGet.Packaging.Core.Types.dll',
 123     'NuGet.Packaging.dll',
 124     'NuGet.ProjectModel.dll',
 125     'NuGet.Protocol.Core.Types.dll',
 126     'NuGet.Protocol.Core.v3.dll',
 127     'NuGet.Repositories.dll',
 128     'NuGet.RuntimeModel.dll',
 129     'NuGet.Versioning.dll',
 130     'System.AppContext.dll',
 131     'System.Buffers.dll',
 132     'System.Collections.Concurrent.dll',
 133     'System.Collections.dll',
 134     'System.Collections.Immutable.dll',
 135     'System.Collections.NonGeneric.dll',
 136     'System.Collections.Specialized.dll',
 137     'System.CommandLine.dll',
 138     'System.ComponentModel.Annotations.dll',
 139     'System.ComponentModel.DataAnnotations.dll',
 140     'System.ComponentModel.dll',
 141     'System.ComponentModel.EventBasedAsync.dll',
 142     'System.ComponentModel.Primitives.dll',
 143     'System.ComponentModel.TypeConverter.dll',
 144     'System.Configuration.dll',
 145     'System.Console.dll',
 146     'System.Core.dll',
 147     'System.Data.Common.dll',
 148     'System.Data.dll',
 149     'System.Diagnostics.Contracts.dll',
 150     'System.Diagnostics.Debug.dll',
 151     'System.Diagnostics.DiagnosticSource.dll',
 152     'System.Diagnostics.EventLog.dll',
 153     'System.Diagnostics.FileVersionInfo.dll',
 154     'System.Diagnostics.Process.dll',
 155     'System.Diagnostics.StackTrace.dll',
 156     'System.Diagnostics.TextWriterTraceListener.dll',
 157     'System.Diagnostics.Tools.dll',
 158     'System.Diagnostics.TraceSource.dll',
 159     'System.Diagnostics.Tracing.dll',
 160     'System.dll',
 161     'System.Drawing.dll',
 162     'System.Drawing.Primitives.dll',
 163     'System.Dynamic.Runtime.dll',
 164     'System.Globalization.Calendars.dll',
 165     'System.Globalization.dll',
 166     'System.Globalization.Extensions.dll',
 167     'System.IO.Compression.Brotli.dll',
 168     'System.IO.Compression.dll',
 169     'System.IO.Compression.FileSystem.dll',
 170     'System.IO.Compression.ZipFile.dll',
 171     'System.IO.dll',
 172     'System.IO.FileSystem.AccessControl.dll',
 173     'System.IO.FileSystem.dll',
 174     'System.IO.FileSystem.DriveInfo.dll',
 175     'System.IO.FileSystem.Primitives.dll',
 176     'System.IO.FileSystem.Watcher.dll',
 177     'System.IO.IsolatedStorage.dll',
 178     'System.IO.MemoryMappedFiles.dll',
 179     'System.IO.Pipes.AccessControl.dll',
 180     'System.IO.Pipes.dll',
 181     'System.IO.UnmanagedMemoryStream.dll',
 182     'System.Linq.dll',
 183     'System.Linq.Expressions.dll',
 184     'System.Linq.Parallel.dll',
 185     'System.Linq.Queryable.dll',
 186     'System.Memory.dll',
 187     'System.Net.dll',
 188     'System.Net.Http.dll',
 189     'System.Net.HttpListener.dll',
 190     'System.Net.Mail.dll',
 191     'System.Net.NameResolution.dll',
 192     'System.Net.NetworkInformation.dll',
 193     'System.Net.Ping.dll',
 194     'System.Net.Primitives.dll',
 195     'System.Net.Requests.dll',
 196     'System.Net.Security.dll',
 197     'System.Net.ServicePoint.dll',
 198     'System.Net.Sockets.dll',
 199     'System.Net.WebClient.dll',
 200     'System.Net.WebHeaderCollection.dll',
 201     'System.Net.WebProxy.dll',
 202     'System.Net.WebSockets.Client.dll',
 203     'System.Net.WebSockets.dll',
 204     'System.Numerics.dll',
 205     'System.Numerics.Vectors.dll',
 206     'System.ObjectModel.dll',
 207     'System.Private.DataContractSerialization.dll',
 208     'System.Private.Uri.dll',
 209     'System.Private.Xml.dll',
 210     'System.Private.Xml.Linq.dll',
 211     'System.Reflection.DispatchProxy.dll',
 212     'System.Reflection.dll',
 213     'System.Reflection.Emit.dll',
 214     'System.Reflection.Emit.ILGeneration.dll',
 215     'System.Reflection.Emit.Lightweight.dll',
 216     'System.Reflection.Extensions.dll',
 217     'System.Reflection.Metadata.dll',
 218     'System.Reflection.Primitives.dll',
 219     'System.Reflection.TypeExtensions.dll',
 220     'System.Resources.Reader.dll',
 221     'System.Resources.ResourceManager.dll',
 222     'System.Resources.Writer.dll',
 223     'System.Runtime.CompilerServices.Unsafe.dll',
 224     'System.Runtime.CompilerServices.VisualC.dll',
 225     'System.Runtime.dll',
 226     'System.Runtime.Extensions.dll',
 227     'System.Runtime.Handles.dll',
 228     'System.Runtime.InteropServices.dll',
 229     'System.Runtime.InteropServices.RuntimeInformation.dll',
 230     'System.Runtime.InteropServices.WindowsRuntime.dll',
 231     'System.Runtime.Loader.dll',
 232     'System.Runtime.Numerics.dll',
 233     'System.Runtime.Serialization.dll',
 234     'System.Runtime.Serialization.Formatters.dll',
 235     'System.Runtime.Serialization.Json.dll',
 236     'System.Runtime.Serialization.Primitives.dll',
 237     'System.Runtime.Serialization.Xml.dll',
 238     'System.Security.AccessControl.dll',
 239     'System.Security.Claims.dll',
 240     'System.Security.Cryptography.Algorithms.dll',
 241     'System.Security.Cryptography.Cng.dll',
 242     'System.Security.Cryptography.Csp.dll',
 243     'System.Security.Cryptography.Encoding.dll',
 244     'System.Security.Cryptography.OpenSsl.dll',
 245     'System.Security.Cryptography.Primitives.dll',
 246     'System.Security.Cryptography.X509Certificates.dll',
 247     'System.Security.dll',
 248     'System.Security.Permissions.dll',
 249     'System.Security.Principal.dll',
 250     'System.Security.Principal.Windows.dll',
 251     'System.Security.SecureString.dll',
 252     'System.ServiceModel.Web.dll',
 253     'System.ServiceProcess.dll',
 254     'System.Text.Encoding.CodePages.dll',
 255     'System.Text.Encoding.dll',
 256     'System.Text.Encoding.Extensions.dll',
 257     'System.Text.RegularExpressions.dll',
 258     'System.Threading.AccessControl.dll',
 259     'System.Threading.dll',
 260     'System.Threading.Overlapped.dll',
 261     'System.Threading.Tasks.Dataflow.dll',
 262     'System.Threading.Tasks.dll',
 263     'System.Threading.Tasks.Extensions.dll',
 264     'System.Threading.Tasks.Parallel.dll',
 265     'System.Threading.Thread.dll',
 266     'System.Threading.ThreadPool.dll',
 267     'System.Threading.Timer.dll',
 268     'System.Transactions.dll',
 269     'System.Transactions.Local.dll',
 270     'System.ValueTuple.dll',
 271     'System.Web.dll',
 272     'System.Web.HttpUtility.dll',
 273     'System.Windows.dll',
 274     'System.Xml.dll',
 275     'System.Xml.Linq.dll',
 276     'System.Xml.ReaderWriter.dll',
 277     'System.Xml.Serialization.dll',
 278     'System.Xml.XDocument.dll',
 279     'System.Xml.XmlDocument.dll',
 280     'System.Xml.XmlSerializer.dll',
 281     'System.Xml.XPath.dll',
 282     'System.Xml.XPath.XDocument.dll',
 283     'TraceReloggerLib.dll',
 284     'WindowsBase.dll']
 285
 286 class CrossGenRunner:
 287     def __init__(self, crossgen_executable_filename, no_logo=True):
 288         self.crossgen_executable_filename = crossgen_executable_filename
 289         self.no_logo = no_logo
 290         self.platform_assemblies_paths_sep = ";"
 291
 292     """
 293         Creates a subprocess running crossgen with specified set of arguments,
 294         communicates with the owner process - waits for its termination and pulls
 295         returncode, stdour, stderr.
 296     """
 297     def run(self, in_filename, out_filename, platform_assemblies_paths):
 298         p = subprocess.Popen(self._build_args(in_filename, out_filename, platform_assemblies_paths), stdout=subprocess.PIPE, stderr=subprocess.PIPE)
 299         stdout, stderr = p.communicate()
 300         return (p.returncode, stdout.decode(), stderr.decode())
 301
 302     def _build_args(self, in_filename, out_filename, platform_assemblies_paths):
 303         args = []
 304         args.append(self.crossgen_executable_filename)
 305         if self.no_logo:
 306            args.append('/nologo')
 307         args.append('/Platform_Assemblies_Paths')
 308         args.append(self.platform_assemblies_paths_sep.join(platform_assemblies_paths))
 309         args.append('/out')
 310         args.append(out_filename)
 311         args.append(in_filename)
 312         return args
 313
 314 def compute_file_hashsum(filename):
 315     """
 316         Compute SHA256 file hashsum for {filename}.
 317     """
 318     algo=hashlib.sha256()
 319     maximum_block_size_in_bytes = 65536
 320     with open(filename, 'rb') as file:
 321         while True:
 322             block = file.read(maximum_block_size_in_bytes)
 323             if block:
 324                 algo.update(block)
 325             else:
 326                 break
 327     return algo.hexdigest()
 328
 329
 330 ################################################################################
 331 # This describes collected during crossgen information.
 332 ################################################################################
 333 class CrossGenResult:
 334     def __init__(self, assembly_name, returncode, stdout, stderr, out_file_hashsum):
 335         self.assembly_name = assembly_name
 336         self.returncode = returncode
 337         self.stdout = stdout
 338         self.stderr = stderr
 339         self.out_file_hashsum = out_file_hashsum
 340
 341 ################################################################################
 342 # JSON Encoder for CrossGenResult objects.
 343 ################################################################################
 344 class CrossGenResultEncoder(json.JSONEncoder):
 345     def default(self, obj):
 346         if isinstance(obj, CrossGenResult):
 347             return {
 348                 'AssemblyName': obj.assembly_name,
 349                 'ReturnCode': obj.returncode,
 350                 'StdOut': obj.stdout.splitlines(),
 351                 'StdErr': obj.stderr.splitlines(),
 352                 'OutputFileHash': obj.out_file_hashsum }
 353         # Let the base class default method raise the TypeError
 354         return json.JSONEncoder.default(self, obj)
 355
 356 ################################################################################
 357 # JSON Decoder for CrossGenResult objects.
 358 ################################################################################
 359 class CrossGenResultDecoder(json.JSONDecoder):
 360     def __init__(self, *args, **kwargs):
 361         json.JSONDecoder.__init__(self, object_hook=self._decode_object, *args, **kwargs)
 362     def _decode_object(self, dict):
 363         try:
 364             assembly_name = dict['AssemblyName']
 365             returncode = dict['ReturnCode']
 366             stdout = dict['StdOut']
 367             stderr = dict['StdErr']
 368             out_file_hashsum = dict['OutputFileHash']
 369             return CrossGenResult(assembly_name, returncode, stdout, stderr, out_file_hashsum)
 370         except KeyError:
 371             return dict
 372
 373 ################################################################################
 374 # Helper Functions
 375 ################################################################################
 376
 377 def crossgen_assembly(crossgen_executable_filename, in_filename, out_filename, platform_assemblies_paths):
 378     runner = CrossGenRunner(crossgen_executable_filename)
 379     returncode, stdout, stderr = runner.run(in_filename, out_filename, platform_assemblies_paths)
 380     assembly_name = os.path.basename(in_filename)
 381     out_file_hashsum = compute_file_hashsum(out_filename) if returncode == 0 else None
 382     return CrossGenResult(assembly_name, returncode, stdout, stderr, out_file_hashsum)
 383
 384 def save_crossgen_result_to_json_file(crossgen_result, json_filename):
 385     with open(json_filename, 'wt') as json_file:
 386         json.dump(crossgen_result, json_file, cls=CrossGenResultEncoder, indent=2)
 387
 388 def crossgen_corelib(args):
 389     il_corelib_filename = args.il_corelib_filename
 390     ni_corelib_filename = os.path.join(tempfile.gettempdir(), os.path.basename(il_corelib_filename))
 391     crossgen_result = crossgen_assembly(args.crossgen_executable_filename, il_corelib_filename, ni_corelib_filename, [os.path.dirname(il_corelib_filename)])
 392     result_filename = os.path.join(args.result_dirname, crossgen_result.assembly_name + '.json')
 393     save_crossgen_result_to_json_file(crossgen_result, result_filename)
 394
 395 def add_ni_extension(filename):
 396     filename,ext = os.path.splitext(filename)
 397     return filename + '.ni' + ext
 398
 399 def crossgen_framework(args):
 400     global g_Framework_Assemblies
 401     platform_assemblies_paths = [args.core_root]
 402     for assembly_name in g_Framework_Assemblies:
 403         il_filename = os.path.join(args.core_root, assembly_name)
 404         ni_filename = os.path.join(tempfile.gettempdir(), add_ni_extension(assembly_name))
 405         crossgen_result = crossgen_assembly(args.crossgen_executable_filename, il_filename, ni_filename, platform_assemblies_paths)
 406         result_filename = os.path.join(args.result_dirname, crossgen_result.assembly_name + '.json')
 407         save_crossgen_result_to_json_file(crossgen_result, result_filename)
 408
 409 def load_crossgen_result_from_json_file(json_filename):
 410     with open(json_filename, 'rt') as json_file:
 411         return json.load(json_file, cls=CrossGenResultDecoder)
 412
 413 def load_crossgen_results_from_dir(dirname):
 414     results_by_assembly_name = dict()
 415     for filename in glob.glob(os.path.join(dirname, '*.json')):
 416         result = load_crossgen_result_from_json_file(filename)
 417         results_by_assembly_name[result.assembly_name] = result
 418     return results_by_assembly_name
 419
 420 def compare(args):
 421     base_results = load_crossgen_results_from_dir(args.base_dirname)
 422     diff_results = load_crossgen_results_from_dir(args.diff_dirname)
 423
 424     base_assemblies = set(base_results.keys())
 425     diff_assemblies = set(diff_results.keys())
 426
 427     column_width = max(len(assembly_name) for assembly_name in base_assemblies | diff_assemblies)
 428     has_mismatch_error = False
 429
 430     for assembly_name in sorted(base_assemblies & diff_assemblies):
 431         base_result = base_results[assembly_name]
 432         diff_result = diff_results[assembly_name]
 433
 434         if base_result.out_file_hashsum == diff_result.out_file_hashsum:
 435             print('{0}  [OK]'.format(assembly_name.ljust(column_width)))
 436         else:
 437             print('{0}  [MISMATCH]'.format(assembly_name.ljust(column_width)))
 438             has_mismatch_error = True
 439
 440     for assembly_name in sorted(base_assemblies - diff_assemblies):
 441         print('{0}  [BASE ONLY]'.format(assembly_name.ljust(column_width)))
 442         has_mismatch_error = True
 443
 444     for assembly_name in sorted(diff_assemblies - base_assemblies):
 445         print('{0}  [DIFF ONLY]'.format(assembly_name.ljust(column_width)))
 446         has_mismatch_error = True
 447
 448     sys.exit(1 if has_mismatch_error else 0)
 449
 450 ################################################################################
 451 # __main__
 452 ################################################################################
 453
 454 if __name__ == '__main__':
 455     parser = build_argument_parser()
 456     args = parser.parse_args()
 457     func = args.func(args)