From: Simon Nattress Date: Wed, 24 Feb 2021 18:11:30 +0000 (-0800) Subject: R2RTest cleanup for Serp compile (#48528) X-Git-Tag: submit/tizen/20210909.063632~3050 X-Git-Url: http://review.tizen.org/git/?a=commitdiff_plain;h=75065d1d521bb57381a20e6898e58df7f48ccc57;p=platform%2Fupstream%2Fdotnet%2Fruntime.git R2RTest cleanup for Serp compile (#48528) * Simplify compilation logic now a single mega composite image is functional * Remove composite scenario switch and support just composite or individual assemblies * Use the same list of allowed Dlls in the serp/bin directory that Serp deployment uses --- diff --git a/src/coreclr/tools/r2rtest/BuildOptions.cs b/src/coreclr/tools/r2rtest/BuildOptions.cs index e2e57ca..0a2a8fe 100644 --- a/src/coreclr/tools/r2rtest/BuildOptions.cs +++ b/src/coreclr/tools/r2rtest/BuildOptions.cs @@ -45,7 +45,6 @@ namespace R2RTest public DirectoryInfo[] RewriteOldPath { get; set; } public DirectoryInfo[] RewriteNewPath { get; set; } public DirectoryInfo AspNetPath { get; set; } - public SerpCompositeScenario CompositeScenario { get; set; } public bool MeasurePerf { get; set; } public string InputFileSearchString { get; set; } public string ConfigurationSuffix => (Release ? "-ret.out" : "-chk.out"); diff --git a/src/coreclr/tools/r2rtest/CommandLineOptions.cs b/src/coreclr/tools/r2rtest/CommandLineOptions.cs index 86da62a..adae37e 100644 --- a/src/coreclr/tools/r2rtest/CommandLineOptions.cs +++ b/src/coreclr/tools/r2rtest/CommandLineOptions.cs @@ -165,9 +165,11 @@ namespace R2RTest InputDirectory(), DegreeOfParallelism(), AspNetPath(), - CompositeScenario(), + Composite(), Map(), Pdb(), + CompilationTimeoutMinutes(), + Crossgen2Path(), }, options => { @@ -295,9 +297,6 @@ namespace R2RTest // Option AspNetPath() => new Option(new[] { "--asp-net-path", "-asp" }, "Path to SERP's ASP.NET Core folder").ExistingOnly(); - - Option CompositeScenario() => - new Option(new [] { "--composite-scenario", "-cs" }, "Specifies which layers of a shared framework application are compiled as composite" ); } } } diff --git a/src/coreclr/tools/r2rtest/Commands/CompileSerpCommand.cs b/src/coreclr/tools/r2rtest/Commands/CompileSerpCommand.cs index dbe99e7..702359b 100644 --- a/src/coreclr/tools/r2rtest/Commands/CompileSerpCommand.cs +++ b/src/coreclr/tools/r2rtest/Commands/CompileSerpCommand.cs @@ -11,28 +11,12 @@ using System.Text; namespace R2RTest { - public enum SerpCompositeScenario - { - /// - /// Compiles all Serp, Asp.Net, and Framework assemblies in their own version bubble - /// - NoComposite, - /// - /// Compiles Serp Core, Asp.Net, and Framework into their own composite images. Compiles Serp packages assemblies individually. - /// - SerpAspNetSharedFramework, - /// - /// Composite image containing Serp Core, Asp.Net, Shared Framework, plus composite image containing package assemblies - /// - SerpAspNetSharedFrameworkCompositeAndPackagesComposite, - } - class CompileSerpCommand { private static readonly string BackupFolder = "backup"; private static readonly string CompileFolder = "compile"; - private static readonly string FrameworkCompositeFilename = "framework-r2r.dll"; - private static readonly string PackagesCompositeFilename = "packages-r2r.dll"; + private static readonly string SerpCompositeFilename = "Microsoft.Search.Frontend.CoreUX.R2R.dll"; + private static readonly string AnswersGlueAssemblyFilename = "Microsoft.Search.Frontend.AnswersGlue.dll"; private List _packageCompileAssemblies; private List _packageReferenceAssemblies; @@ -52,7 +36,6 @@ namespace R2RTest options.NoJit = true; options.NoEtw = true; options.Release = true; - options.LargeBubble = true; _options = options; @@ -79,10 +62,11 @@ namespace R2RTest throw new ArgumentException($"Error: InputDirectory must point at a SERP build. Could not find {Path.Combine(SerpDir, "runserp.cmd")}"); } - string allowListFilePath = Path.Combine(SerpDir, "App_Data", "AllowedDllList.txt"); - if (!File.Exists(allowListFilePath)) + // Check that the "Microsoft.Search.Frontend.AnswersGlue.dll" assembly was generated in this drop + string answersGlueAssembly = Path.Combine(BinDir, AnswersGlueAssemblyFilename); + if (!File.Exists(answersGlueAssembly)) { - throw new ArgumentException($"File {allowListFilePath} was not found"); + throw new FileNotFoundException($"Could not find {answersGlueAssembly}. Run {SerpDir}\\GenAnswersEntryPoints.bat first!"); } // Add all assemblies from the various SERP packages (filtered by ShouldInclude) @@ -109,7 +93,7 @@ namespace R2RTest _coreReferenceAssemblies = new List(); { // Add an allow-list of assemblies from bin. This unified list includes binaries from /bin and /App_data so filter just the /bin assemblies - foreach (string item in new HashSet(File.ReadAllLines(allowListFilePath))) + foreach (string item in GetCrossgenAllowedBinDlls()) { string binAssembly = Path.Combine(BinDir, item); if (File.Exists(binAssembly) && @@ -178,130 +162,75 @@ namespace R2RTest _aspReferenceAssemblies = new List(_aspCompileAssemblies); } - } + public int CompileSerpAssemblies() { - Console.WriteLine($"Compiling serp in {_options.CompositeScenario} scenario"); + Console.WriteLine("Compiling serp in " + (_options.Composite ? "composite" : "single assembly") + " mode"); string serpRoot = Directory.GetParent(SerpDir).Parent.Parent.Parent.FullName; string compileOutRoot = Path.Combine(serpRoot, CompileFolder); if (Directory.Exists(compileOutRoot)) Directory.Delete(compileOutRoot, true); - // Composite FX, Composite ASP.NET, Composite Serp core, Individual package assemblies List fileCompilations = new List(); - bool combinedComposite = false; - bool compositeFramework = false; - bool compositeAspNet = false; - bool compositeSerpCore = false; - if (_options.CompositeScenario == SerpCompositeScenario.SerpAspNetSharedFramework) - { - compositeFramework = true; - compositeAspNet = true; - compositeSerpCore = true; - } - if (_options.CompositeScenario == SerpCompositeScenario.SerpAspNetSharedFrameworkCompositeAndPackagesComposite) - { - combinedComposite = true; - } - - // Single composite image for Serp, Asp, Fx + // Single composite image for all of Serp + if (_options.Composite) { - if (combinedComposite) - { - List combinedCompileAssemblies = new List(); - HashSet simpleNameList = new HashSet(); - combinedCompileAssemblies.AddRange(FilterAssembliesNoSimpleNameDuplicates(simpleNameList, _coreCompileAssemblies)); - combinedCompileAssemblies.AddRange(FilterAssembliesNoSimpleNameDuplicates(simpleNameList, _aspCompileAssemblies)); - combinedCompileAssemblies.AddRange(FilterAssembliesNoSimpleNameDuplicates(simpleNameList, _frameworkCompileAssemblies)); - - List combinedCompileAssembliesBackup = BackupAndUseOriginalAssemblies(serpRoot, combinedCompileAssemblies); - string frameworkCompositeDll = Path.Combine(_options.CoreRootDirectory.FullName, FrameworkCompositeFilename); - if (File.Exists(frameworkCompositeDll)) - File.Delete(frameworkCompositeDll); - - string frameworkCompositeDllCompile = GetCompileFile(serpRoot, frameworkCompositeDll); - Crossgen2RunnerOptions crossgen2Options = new Crossgen2RunnerOptions() { Composite = true, CompositeRoot = GetBackupFile(serpRoot, SerpDir) }; - var runner = new Crossgen2Runner(_options, crossgen2Options, combinedCompileAssembliesBackup); - var compilationProcess = new ProcessInfo(new CompilationProcessConstructor(runner, frameworkCompositeDllCompile, combinedCompileAssembliesBackup)); - fileCompilations.Add(compilationProcess); - } + List combinedCompileAssemblies = new List(); + HashSet simpleNameList = new HashSet(); + combinedCompileAssemblies.AddRange(FilterAssembliesNoSimpleNameDuplicates(simpleNameList, _coreCompileAssemblies)); + combinedCompileAssemblies.AddRange(FilterAssembliesNoSimpleNameDuplicates(simpleNameList, _aspCompileAssemblies)); + combinedCompileAssemblies.AddRange(FilterAssembliesNoSimpleNameDuplicates(simpleNameList, _frameworkCompileAssemblies)); + combinedCompileAssemblies.AddRange(FilterAssembliesNoSimpleNameDuplicates(simpleNameList, _packageCompileAssemblies)); + + List combinedCompileAssembliesBackup = BackupAndUseOriginalAssemblies(serpRoot, combinedCompileAssemblies); + string compositeDllPath = Path.Combine(BinDir, SerpCompositeFilename); + if (File.Exists(compositeDllPath)) + File.Delete(compositeDllPath); + + string compositeDllCompile = GetCompileFile(serpRoot, compositeDllPath); + Crossgen2RunnerOptions crossgen2Options = new Crossgen2RunnerOptions() { Composite = true, CompositeRoot = GetBackupFile(serpRoot, SerpDir), PartialComposite = true }; + var runner = new Crossgen2Runner(_options, crossgen2Options, combinedCompileAssembliesBackup); + var compilationProcess = new ProcessInfo(new CompilationProcessConstructor(runner, compositeDllCompile, combinedCompileAssembliesBackup)); + fileCompilations.Add(compilationProcess); } - - if (!combinedComposite) + else { - // Composite FX + // Framework assemblies { List frameworkCompileAssembliesBackup = BackupAndUseOriginalAssemblies(serpRoot, _frameworkCompileAssemblies); - string frameworkCompositeDll = Path.Combine(_options.CoreRootDirectory.FullName, FrameworkCompositeFilename); - if (File.Exists(frameworkCompositeDll)) - File.Delete(frameworkCompositeDll); - - // Always restore the framework from the backup if present first since we run CG2 on it - var backupFrameworkDir = Path.GetDirectoryName(GetBackupFile(serpRoot, frameworkCompositeDll)); - var backedUpFiles = Directory.GetFiles(backupFrameworkDir, "*.dll", SearchOption.AllDirectories); - foreach (var file in backedUpFiles) - { - string destinationFile = GetOriginalFile(serpRoot, file); - File.Copy(file, destinationFile, true); - } - - if (compositeFramework) + + Crossgen2RunnerOptions crossgen2Options = new Crossgen2RunnerOptions() { Composite = false }; + var runner = new Crossgen2Runner(_options, crossgen2Options, new List()); + foreach (string assembly in frameworkCompileAssembliesBackup) { - string frameworkCompositeDllCompile = GetCompileFile(serpRoot, frameworkCompositeDll); - Crossgen2RunnerOptions crossgen2Options = new Crossgen2RunnerOptions() { Composite = true }; - var runner = new Crossgen2Runner(_options, crossgen2Options, new List()); - var compilationProcess = new ProcessInfo(new CompilationProcessConstructor(runner, frameworkCompositeDllCompile, frameworkCompileAssembliesBackup)); + string dllCompile = GetCompileFile(serpRoot, assembly); + var compilationProcess = new ProcessInfo(new CompilationProcessConstructor(runner, dllCompile, new string[] { assembly })); fileCompilations.Add(compilationProcess); } - else - { - Crossgen2RunnerOptions crossgen2Options = new Crossgen2RunnerOptions() { Composite = false }; - var runner = new Crossgen2Runner(_options, crossgen2Options, new List()); - foreach (string assembly in frameworkCompileAssembliesBackup) - { - string dllCompile = GetCompileFile(serpRoot, assembly); - var compilationProcess = new ProcessInfo(new CompilationProcessConstructor(runner, dllCompile, new string[] { assembly })); - fileCompilations.Add(compilationProcess); - } - } } - // Composite Asp.Net + // Asp.Net { List aspCombinedReferences = new List(); aspCombinedReferences.AddRange(_aspReferenceAssemblies); aspCombinedReferences.AddRange(_frameworkCompileAssemblies); List aspCombinedReferencesBackup = BackupAndUseOriginalAssemblies(serpRoot, aspCombinedReferences); List aspCompileAssembliesBackup = BackupAndUseOriginalAssemblies(serpRoot, _aspCompileAssemblies); - string aspCompositeDll = Path.Combine(_options.AspNetPath.FullName, "asp-r2r.dll"); - if (File.Exists(aspCompositeDll)) - File.Delete(aspCompositeDll); - - if (compositeAspNet) + + Crossgen2RunnerOptions crossgen2Options = new Crossgen2RunnerOptions() { Composite = false }; + var runner = new Crossgen2Runner(_options, crossgen2Options, aspCombinedReferencesBackup); + foreach (string assembly in aspCompileAssembliesBackup) { - string aspCompositeDllCompile = GetCompileFile(serpRoot, aspCompositeDll); - Crossgen2RunnerOptions crossgen2Options = new Crossgen2RunnerOptions() { Composite = true, PartialComposite = true }; - var runner = new Crossgen2Runner(_options, crossgen2Options, aspCombinedReferencesBackup); - var compilationProcess = new ProcessInfo(new CompilationProcessConstructor(runner, aspCompositeDllCompile, aspCompileAssembliesBackup)); + string dllCompile = GetCompileFile(serpRoot, assembly); + var compilationProcess = new ProcessInfo(new CompilationProcessConstructor(runner, dllCompile, new string[] { assembly })); fileCompilations.Add(compilationProcess); } - else - { - Crossgen2RunnerOptions crossgen2Options = new Crossgen2RunnerOptions() { Composite = false }; - var runner = new Crossgen2Runner(_options, crossgen2Options, aspCombinedReferencesBackup); - foreach (string assembly in aspCompileAssembliesBackup) - { - string dllCompile = GetCompileFile(serpRoot, assembly); - var compilationProcess = new ProcessInfo(new CompilationProcessConstructor(runner, dllCompile, new string[] { assembly })); - fileCompilations.Add(compilationProcess); - } - } } - // Composite Serp core + // Serp core { List coreCombinedReferences = new List(); coreCombinedReferences.AddRange(_coreReferenceAssemblies); @@ -309,59 +238,28 @@ namespace R2RTest coreCombinedReferences.AddRange(_frameworkReferenceAssemblies); List coreCombinedReferencesBackup = BackupAndUseOriginalAssemblies(serpRoot, coreCombinedReferences); List coreCompileAssembliesBackup = BackupAndUseOriginalAssemblies(serpRoot, _coreCompileAssemblies); - string serpCompositeDll = Path.Combine(BinDir, "serp-r2r.dll"); - if (File.Exists(serpCompositeDll)) - File.Delete(serpCompositeDll); - if (compositeSerpCore) + Crossgen2RunnerOptions crossgen2Options = new Crossgen2RunnerOptions() { Composite = false }; + var runner = new Crossgen2Runner(_options, crossgen2Options, coreCombinedReferencesBackup); + foreach (string assembly in coreCompileAssembliesBackup) { - string coreCompositeDllCompile = GetCompileFile(serpRoot, serpCompositeDll); - Crossgen2RunnerOptions crossgen2Options = new Crossgen2RunnerOptions() { Composite = true, PartialComposite = true }; - var runner = new Crossgen2Runner(_options, crossgen2Options, coreCombinedReferencesBackup); - var compilationProcess = new ProcessInfo(new CompilationProcessConstructor(runner, coreCompositeDllCompile, coreCompileAssembliesBackup)); + string dllCompile = GetCompileFile(serpRoot, assembly); + var compilationProcess = new ProcessInfo(new CompilationProcessConstructor(runner, dllCompile, new string[] { assembly })); fileCompilations.Add(compilationProcess); } - else - { - Crossgen2RunnerOptions crossgen2Options = new Crossgen2RunnerOptions() { Composite = false }; - var runner = new Crossgen2Runner(_options, crossgen2Options, coreCombinedReferencesBackup); - foreach (string assembly in coreCompileAssembliesBackup) - { - string dllCompile = GetCompileFile(serpRoot, assembly); - var compilationProcess = new ProcessInfo(new CompilationProcessConstructor(runner, dllCompile, new string[] { assembly })); - fileCompilations.Add(compilationProcess); - } - } } - } - string packagesCompositeDirectory = null; - // Individual Serp package assemblies - { - List packageCombinedReferences = new List(); - HashSet simpleNameList = new HashSet(); - packageCombinedReferences.AddRange(FilterAssembliesNoSimpleNameDuplicates(simpleNameList, _packageReferenceAssemblies)); - packageCombinedReferences.AddRange(FilterAssembliesNoSimpleNameDuplicates(simpleNameList, _coreReferenceAssemblies)); - packageCombinedReferences.AddRange(FilterAssembliesNoSimpleNameDuplicates(simpleNameList, _aspReferenceAssemblies)); - packageCombinedReferences.AddRange(FilterAssembliesNoSimpleNameDuplicates(simpleNameList, _frameworkReferenceAssemblies)); - List packageCombinedReferencesBackup = BackupAndUseOriginalAssemblies(serpRoot, packageCombinedReferences); - List packageCompileAssembliesBackup = BackupAndUseOriginalAssemblies(serpRoot, _packageCompileAssemblies); - - string packagesCompositeDll = Path.Combine(_options.CoreRootDirectory.FullName, PackagesCompositeFilename); - packagesCompositeDirectory = SerpDir; - if (File.Exists(packagesCompositeDll)) - File.Delete(packagesCompositeDll); - - if (combinedComposite) - { - string packagesCompositeDllCompile = GetCompileFile(serpRoot, packagesCompositeDll); - Crossgen2RunnerOptions crossgen2Options = new Crossgen2RunnerOptions() { Composite = true, CompositeRoot = GetBackupFile(serpRoot, SerpDir), PartialComposite = true }; - var runner = new Crossgen2Runner(_options, crossgen2Options, packageCombinedReferencesBackup); - var compilationProcess = new ProcessInfo(new CompilationProcessConstructor(runner, packagesCompositeDllCompile, packageCompileAssembliesBackup)); - fileCompilations.Add(compilationProcess); - } - else + // Individual Serp package assemblies { + List packageCombinedReferences = new List(); + HashSet simpleNameList = new HashSet(); + packageCombinedReferences.AddRange(FilterAssembliesNoSimpleNameDuplicates(simpleNameList, _packageReferenceAssemblies)); + packageCombinedReferences.AddRange(FilterAssembliesNoSimpleNameDuplicates(simpleNameList, _coreReferenceAssemblies)); + packageCombinedReferences.AddRange(FilterAssembliesNoSimpleNameDuplicates(simpleNameList, _aspReferenceAssemblies)); + packageCombinedReferences.AddRange(FilterAssembliesNoSimpleNameDuplicates(simpleNameList, _frameworkReferenceAssemblies)); + List packageCombinedReferencesBackup = BackupAndUseOriginalAssemblies(serpRoot, packageCombinedReferences); + List packageCompileAssembliesBackup = BackupAndUseOriginalAssemblies(serpRoot, _packageCompileAssemblies); + Crossgen2RunnerOptions crossgen2Options = new Crossgen2RunnerOptions() { Composite = false }; var runner = new Crossgen2Runner(_options, crossgen2Options, packageCombinedReferencesBackup); foreach (string assembly in packageCompileAssembliesBackup) @@ -388,21 +286,17 @@ namespace R2RTest if (!success) return 1; - - if (combinedComposite) + if (_options.Composite) { // For combined composite, move the component assemblies we added the R2R header from the composite out folder // to the correct compile tree destination folder so they get copied into the right place - string compositeOutputRootDir = GetCompileFile(serpRoot, _options.CoreRootDirectory.FullName); - string frameworkCompositeDll = Path.Combine(compositeOutputRootDir, FrameworkCompositeFilename); + string compositeOutputRootDir = GetCompileFile(serpRoot, BinDir); + string frameworkCompositeDll = Path.Combine(compositeOutputRootDir, SerpCompositeFilename); Debug.Assert(File.Exists(frameworkCompositeDll)); var compiledCompositeFiles = Directory.GetFiles(compositeOutputRootDir, "*.dll", SearchOption.AllDirectories); foreach (var componentAssembly in compiledCompositeFiles) { - if (Path.GetFileName(componentAssembly).Equals(FrameworkCompositeFilename)) - continue; - - if (Path.GetFileName(componentAssembly).Equals(PackagesCompositeFilename)) + if (Path.GetFileName(componentAssembly).Equals(SerpCompositeFilename)) continue; string assemblyRelativePath = Path.GetRelativePath(compositeOutputRootDir, componentAssembly); @@ -443,6 +337,40 @@ namespace R2RTest return false; } + private IEnumerable GetCrossgenAllowedBinDlls() + { + // This list is hard-coded somewhere in Serp's deployment scripting. + return new string[] + { + "AjaxMin.dll", + "AntiXssLibrary.dll", + "DomainNameProcessorLibrary.dll", + "dotless.Core.dll", + "Frontend.PixelBasedTrimmer.dll", + "Microsoft.Search.Frontend.AnswersFramework.ServiceInterfaces.dll", + "Microsoft.Search.Frontend.AnswersGlue.dll", + "Microsoft.Search.Frontend.ApiSchemas.Current.dll", + "Microsoft.Search.Frontend.ApiSchemas.Interfaces.dll", + "Microsoft.Search.Frontend.CoreUX.dll", + "Microsoft.Search.Frontend.CoreUX.Data.dll", + "Microsoft.Search.Frontend.CoreUX.Instrumentation.dll", + "Microsoft.Search.Frontend.DataSchemas.dll", + "Microsoft.Search.Frontend.Data.Bond.dll", + "Microsoft.Search.Frontend.Data.BondRepository.dll", + "Microsoft.Search.Frontend.Data.LinksModels.dll", + "Microsoft.Search.Frontend.Data.Kif.dll", + "Microsoft.Search.Frontend.Data.PropertyPath.dll", + "Microsoft.Search.Frontend.DeviceCapabilities.dll", + "Microsoft.Search.Frontend.dll", + "Microsoft.Search.Frontend.Federation.Federators.dll", + "Microsoft.Search.Frontend.Hosting.Serp.Core.dll", + "Microsoft.Search.Frontend.Hosting.Serp.Runtime.dll", + "Microsoft.Search.Frontend.SchemaInterfaces.dll", + "Microsoft.Search.Frontend.Serialization.Models.dll", + "Microsoft.Search.Frontend.SharedComponents.dll", + "SparkleAssembly.dll" + }; + } private static IEnumerable ResolveReferences(IEnumerable folders) { foreach (string referenceFolder in folders)