From 5ff4fdd0250e73b34cecb3c994746bcb9d7325cb Mon Sep 17 00:00:00 2001 From: David Wrighton Date: Thu, 22 Jun 2023 16:35:25 -0700 Subject: [PATCH] Adjust determinism tests to store away the 2 different binaries that are generated (#87852) - This should allow comparison without needing full rebuilds - as well as comparison in cases where locally running the tooling doesn't result in equivalent results - In addition, generate a map file to better understand what portion of the two files does not match Finally, fix comparison failure identified by using the above tech: - Fix culture sensive compare in instruction set handling - This should be culture insensitive, as otherwise different machines can reliably generate different output due to different culture settings --- eng/pipelines/coreclr/templates/crossgen2-comparison-job.yml | 8 ++++---- .../ReadyToRun/ReadyToRunInstructionSetSupportSignature.cs | 2 +- src/tests/Common/scripts/crossgen2_comparison.py | 7 ++++--- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/eng/pipelines/coreclr/templates/crossgen2-comparison-job.yml b/eng/pipelines/coreclr/templates/crossgen2-comparison-job.yml index 0e2cb6c..4faa6c5 100644 --- a/eng/pipelines/coreclr/templates/crossgen2-comparison-job.yml +++ b/eng/pipelines/coreclr/templates/crossgen2-comparison-job.yml @@ -149,13 +149,13 @@ jobs: --crossgen $HELIX_WORKITEM_PAYLOAD/crossgen2/crossgen2.dll --dotnet $HELIX_WORKITEM_PAYLOAD/corerun --core_root $HELIX_WORKITEM_PAYLOAD/prebuiltWork/dlls - --result_dir $HELIX_WORKITEM_PAYLOAD/log + --result_dir $HELIX_WORKITEM_UPLOAD_ROOT --target_os $(target_crossgen2_os) --target_arch $(targetarch) --compiler_arch_os $(compiler_arch_os); python3 -u $HELIX_CORRELATION_PAYLOAD/crossgen2_comparison.py compare --base_dir $HELIX_WORKITEM_PAYLOAD/prebuiltWork/log - --diff_dir $HELIX_WORKITEM_PAYLOAD/log + --diff_dir $HELIX_WORKITEM_UPLOAD_ROOT --testresults $HELIX_WORKITEM_ROOT/testResults.xml --target_arch_os $(target_crossgen2_os)_$(targetarch) ${{ if eq(parameters.osGroup, 'windows') }}: @@ -168,13 +168,13 @@ jobs: --crossgen %HELIX_WORKITEM_PAYLOAD%\crossgen2\crossgen2.dll --dotnet %HELIX_WORKITEM_PAYLOAD%\corerun.exe --core_root %HELIX_WORKITEM_PAYLOAD%\prebuiltWork\dlls - --result_dir %HELIX_WORKITEM_PAYLOAD%\log + --result_dir %HELIX_WORKITEM_UPLOAD_ROOT% --target_os $(target_crossgen2_os) --target_arch $(targetarch) --compiler_arch_os $(compiler_arch_os) & python -u %HELIX_CORRELATION_PAYLOAD%\crossgen2_comparison.py compare --base_dir %HELIX_WORKITEM_PAYLOAD%\prebuiltWork\log - --diff_dir %HELIX_WORKITEM_PAYLOAD%\log + --diff_dir %HELIX_WORKITEM_UPLOAD_ROOT% --testresults %HELIX_WORKITEM_ROOT%\testResults.xml --target_arch_os $(target_crossgen2_os)_$(targetarch) # Publish log diff --git a/src/coreclr/tools/aot/ILCompiler.ReadyToRun/Compiler/DependencyAnalysis/ReadyToRun/ReadyToRunInstructionSetSupportSignature.cs b/src/coreclr/tools/aot/ILCompiler.ReadyToRun/Compiler/DependencyAnalysis/ReadyToRun/ReadyToRunInstructionSetSupportSignature.cs index c889c0b..bcb2949 100644 --- a/src/coreclr/tools/aot/ILCompiler.ReadyToRun/Compiler/DependencyAnalysis/ReadyToRun/ReadyToRunInstructionSetSupportSignature.cs +++ b/src/coreclr/tools/aot/ILCompiler.ReadyToRun/Compiler/DependencyAnalysis/ReadyToRun/ReadyToRunInstructionSetSupportSignature.cs @@ -113,7 +113,7 @@ namespace ILCompiler.DependencyAnalysis.ReadyToRun public override int CompareToImpl(ISortableNode other, CompilerComparer comparer) { - return _instructionSetsSupport.CompareTo(((ReadyToRunInstructionSetSupportSignature)other)._instructionSetsSupport); + return string.CompareOrdinal(_instructionSetsSupport, ((ReadyToRunInstructionSetSupportSignature)other)._instructionSetsSupport); } } } diff --git a/src/tests/Common/scripts/crossgen2_comparison.py b/src/tests/Common/scripts/crossgen2_comparison.py index 198b80a..65b0e06 100644 --- a/src/tests/Common/scripts/crossgen2_comparison.py +++ b/src/tests/Common/scripts/crossgen2_comparison.py @@ -529,7 +529,8 @@ class CrossGenRunner: args.append('"' + platform_assemblies_paths + self.platform_directory_sep + '*.dll"' ) args.append('-O') args.append('--determinism-stress') - args.append('3') + args.append('6') + args.append('--map') args.append('--out') args.append(ni_filename) args.append('--targetos ') @@ -720,6 +721,7 @@ def add_ni_extension(filename): def crossgen_framework(args): ni_files_dirname, debugging_files_dirname = create_output_folders() + ni_files_dirname = args.result_dirname async def run_crossgen_helper(print_prefix, assembly_name): global g_frameworkcompile_failed @@ -737,7 +739,6 @@ def crossgen_framework(args): helper = AsyncSubprocessHelper(g_Framework_Assemblies, verbose=True) helper.run_to_completion(run_crossgen_helper) - shutil.rmtree(ni_files_dirname, ignore_errors=True) if g_frameworkcompile_failed: sys.exit(1) @@ -955,7 +956,7 @@ def compare_results(args): base_result_string = json.dumps(base_result, cls=CrossGenResultEncoder, indent=2) diff_result_string = json.dumps(diff_result, cls=CrossGenResultEncoder, indent=2) - message = 'Expected {0} got {1}'.format(base_result_string, diff_result_string) + message = 'Expected {0} got {1} Attached to this helix job will be the binary produced during the helix run and you can find the binary to compare to in the artifacts for the pipeline.'.format(base_result_string, diff_result_string) testresult = root.createElement('test') testresult.setAttribute('name', 'CrossgenCompile_{3}_Target_{0}_{1}_vs_{2}'.format(args.target_arch_os, base_result.compiler_arch_os, diff_result.compiler_arch_os, assembly_name)) testresult.setAttribute('type', 'Target_{0}'.format(args.target_arch_os)) -- 2.7.4