New SuperIlc command "compile-framework" and some fixes (dotnet/coreclr#26804)
authorTomáš Rylek <trylek@microsoft.com>
Tue, 24 Sep 2019 06:30:51 +0000 (23:30 -0700)
committerGitHub <noreply@github.com>
Tue, 24 Sep 2019 06:30:51 +0000 (23:30 -0700)
1) New command "compile-framework" does just what it says on the tin.
My thinking is that I'll use this option to build the CPAOT
framework during product build in the envisioned pipeline. I have
added hard-coded exclusions for assemblies that fail to build right
now - about half of them aren't interesting as framework assemblies
(e.g. R2RDump.dll), some other fail with IBC bugs and one other bug
I'm about to investigate.

2) Based on JanV's suggestion I have added a somewhat hacky logic
to exclude the "testhost" folder when building the entire Pridotnet/coreclr#1
test tree. I'm open to suggestions how to tackle this in a cleaner
manner, I have shared my thoughts on the subject in code comments
next to the problematic spots.

Thanks

Tomas

Commit migrated from https://github.com/dotnet/coreclr/commit/ddbdf5739bdfbcc94b245090806371afb3dbbef5

src/coreclr/src/tools/ReadyToRun.SuperIlc/BuildFolderSet.cs
src/coreclr/src/tools/ReadyToRun.SuperIlc/CommandLineOptions.cs
src/coreclr/src/tools/ReadyToRun.SuperIlc/Commands/CompileDirectoryCommand.cs
src/coreclr/src/tools/ReadyToRun.SuperIlc/Commands/CompileFrameworkCommand.cs [new file with mode: 0644]
src/coreclr/src/tools/ReadyToRun.SuperIlc/Commands/CompileNugetCommand.cs
src/coreclr/src/tools/ReadyToRun.SuperIlc/Commands/CompileSubtreeCommand.cs
src/coreclr/src/tools/ReadyToRun.SuperIlc/ParallelRunner.cs
src/coreclr/src/tools/ReadyToRun.SuperIlc/ProcessRunner.cs

index 36193e44c815f4407da7dd4fcba7a4e37282e85f..a582dc858874148aefe7659816c2cffb825421a3 100644 (file)
@@ -8,39 +8,71 @@ using System.Collections.Generic;
 using System.Diagnostics;
 using System.IO;
 using System.Linq;
+using System.Text;
+using Microsoft.Diagnostics.Tracing.Parsers;
 using Microsoft.Diagnostics.Tracing.Parsers.Clr;
 
 namespace ReadyToRun.SuperIlc
 {
     public class BuildFolderSet
     {
-        private IEnumerable<BuildFolder> _buildFolders;
+        class FrameworkExclusion
+        {
+            public readonly string SimpleName;
+            public readonly string Reason;
 
-        private IEnumerable<CompilerRunner> _compilerRunners;
+            public FrameworkExclusion(string simpleName, string reason)
+            {
+                SimpleName = simpleName;
+                Reason = reason;
+            }
+        }
 
-        private BuildOptions _options;
+        private static readonly FrameworkExclusion[] s_frameworkExclusions =
+        {
+            new FrameworkExclusion("CommandLine", "Not a framework assembly"),
+            new FrameworkExclusion("R2RDump", "Not a framework assembly"),
+            new FrameworkExclusion("System.Runtime.WindowsRuntime", "WinRT is currently not supported"),
+            new FrameworkExclusion("xunit.performance.api", "Not a framework assembly"),
 
-        private Buckets _frameworkCompilationFailureBuckets;
+            // TODO (DavidWr): IBC-related failures
+            new FrameworkExclusion("Microsoft.CodeAnalysis.CSharp", "Ibc TypeToken 6200019a has type token which resolves to a nil token"),
+            new FrameworkExclusion("Microsoft.CodeAnalysis", "Ibc TypeToken 620001af unable to find external typedef"),
+            new FrameworkExclusion("Microsoft.CodeAnalysis.VisualBasic", "Ibc TypeToken 620002ce unable to find external typedef"),
 
-        private Buckets _compilationFailureBuckets;
+            // TODO (TRylek): problem related to devirtualization of method without IL - System.Enum.Equals(object)
+            new FrameworkExclusion("System.ComponentModel.TypeConverter", "TODO trylek - devirtualization of method without IL"),
+        };
 
-        private Buckets _executionFailureBuckets;
+        private readonly IEnumerable<BuildFolder> _buildFolders;
 
-        private long _frameworkCompilationMilliseconds;
+        private readonly IEnumerable<CompilerRunner> _compilerRunners;
 
-        private long _compilationMilliseconds;
+        private readonly BuildOptions _options;
 
-        private long _executionMilliseconds;
+        private readonly Buckets _frameworkCompilationFailureBuckets;
 
-        private long _buildMilliseconds;
+        private readonly Buckets _compilationFailureBuckets;
+
+        private readonly Buckets _executionFailureBuckets;
+
+        private readonly Dictionary<string, byte> _cpaotManagedSequentialResults;
+
+        private readonly Dictionary<string, byte> _crossgenManagedSequentialResults;
 
-        private Dictionary<string, byte> _cpaotManagedSequentialResults;
+        private readonly Dictionary<string, byte> _cpaotRequiresMarshalingResults;
 
-        private Dictionary<string, byte> _crossgenManagedSequentialResults;
+        private readonly Dictionary<string, byte> _crossgenRequiresMarshalingResults;
 
-        private Dictionary<string, byte> _cpaotRequiresMarshalingResults;
+        private readonly Dictionary<string, string> _frameworkExclusions;
 
-        private Dictionary<string, byte> _crossgenRequiresMarshalingResults;
+        private long _frameworkCompilationMilliseconds;
+
+        private long _compilationMilliseconds;
+
+        private long _executionMilliseconds;
+
+        private long _buildMilliseconds;
 
         public BuildFolderSet(
             IEnumerable<BuildFolder> buildFolders,
@@ -61,6 +93,7 @@ namespace ReadyToRun.SuperIlc
             _cpaotRequiresMarshalingResults = new Dictionary<string, byte>();
             _crossgenRequiresMarshalingResults = new Dictionary<string, byte>();
 
+            _frameworkExclusions = new Dictionary<string, string>();
         }
 
         private void WriteJittedMethodSummary(StreamWriter logWriter)
@@ -92,7 +125,10 @@ namespace ReadyToRun.SuperIlc
 
         public bool Compile()
         {
-            CompileFramework();
+            if (!CompileFramework())
+            {
+                return false;
+            }
 
             Stopwatch stopwatch = new Stopwatch();
             stopwatch.Start();
@@ -133,7 +169,7 @@ namespace ReadyToRun.SuperIlc
                     foreach (CompilerRunner runner in _compilerRunners)
                     {
                         ProcessInfo runnerProcess = compilation[(int)runner.Index];
-                        if (runnerProcess == null)
+                        if (runnerProcess == null || runnerProcess.IsEmpty)
                         {
                             // No runner process
                         }
@@ -235,8 +271,17 @@ namespace ReadyToRun.SuperIlc
 
             List<ProcessInfo> compilationsToRun = new List<ProcessInfo>();
             List<KeyValuePair<string, ProcessInfo[]>> compilationsPerRunner = new List<KeyValuePair<string, ProcessInfo[]>>();
+            List<string> excludedAssemblies = new List<string>();
             foreach (string frameworkDll in ComputeManagedAssemblies.GetManagedAssembliesInFolder(_options.CoreRootDirectory.FullName))
             {
+                string simpleName = Path.GetFileNameWithoutExtension(frameworkDll);
+                FrameworkExclusion exclusion = s_frameworkExclusions.FirstOrDefault(asm => asm.SimpleName.Equals(simpleName, StringComparison.OrdinalIgnoreCase));
+                if (exclusion != null)
+                {
+                    _frameworkExclusions.Add(exclusion.SimpleName, exclusion.Reason);
+                    continue;
+                }
+
                 ProcessInfo[] processes = new ProcessInfo[(int)CompilerIndex.Count];
                 compilationsPerRunner.Add(new KeyValuePair<string, ProcessInfo[]>(frameworkDll, processes));
                 foreach (CompilerRunner runner in frameworkRunners)
@@ -434,7 +479,7 @@ namespace ReadyToRun.SuperIlc
             return success;
         }
 
-        public bool Build(IEnumerable<CompilerRunner> runners)
+        public bool Build()
         {
             Stopwatch stopwatch = new Stopwatch();
             stopwatch.Start();
@@ -538,6 +583,7 @@ namespace ReadyToRun.SuperIlc
             EXIT_CODE = 1,
             CRASHED = 2,
             TIMED_OUT = 3,
+            BUILD_FAILED = 4,
 
             Count
         }
@@ -620,24 +666,35 @@ namespace ReadyToRun.SuperIlc
                                 executionOutcomes[(int)outcome, (int)runner.Index]++;
                                 executionFailureOutcomeMask |= 1 << (int)outcome;
                             }
-                            if (!compilationFailed && !executionFailed)
+                            else if (compilationFailed)
+                            {
+                                executionOutcomes[(int)ExecutionOutcome.BUILD_FAILED, (int)runner.Index]++;
+                            }
+                            else
                             {
                                 executionOutcomes[(int)ExecutionOutcome.PASS, (int)runner.Index]++;
                             }
                         }
-                        if (executionFailureOutcomeMask != 0)
+                        if (!anyCompilationFailed)
                         {
-                            for (int outcomeIndex = 0; outcomeIndex < (int)ExecutionOutcome.Count; outcomeIndex++)
+                            if (executionFailureOutcomeMask != 0)
                             {
-                                if ((executionFailureOutcomeMask & (1 << outcomeIndex)) != 0)
+                                for (int outcomeIndex = 0; outcomeIndex < (int)ExecutionOutcome.Count; outcomeIndex++)
                                 {
-                                    executionOutcomes[outcomeIndex, (int)CompilerIndex.Count]++;
+                                    if ((executionFailureOutcomeMask & (1 << outcomeIndex)) != 0)
+                                    {
+                                        executionOutcomes[outcomeIndex, (int)CompilerIndex.Count]++;
+                                    }
                                 }
                             }
+                            else
+                            {
+                                executionOutcomes[(int)ExecutionOutcome.PASS, (int)CompilerIndex.Count]++;
+                            }
                         }
                         else
                         {
-                            executionOutcomes[(int)ExecutionOutcome.PASS, (int)CompilerIndex.Count]++;
+                            executionOutcomes[(int)ExecutionOutcome.BUILD_FAILED, (int)CompilerIndex.Count]++;
                         }
                     }
                 }
@@ -674,7 +731,7 @@ namespace ReadyToRun.SuperIlc
                 if (!_options.Exe)
                 {
                     logWriter.WriteLine();
-                    logWriter.Write($"{totalCompilations,7} ILC |");
+                    logWriter.Write($"{totalCompilations,8} ILC |");
                     foreach (CompilerRunner runner in _compilerRunners)
                     {
                         logWriter.Write($"{runner.CompilerName,8} |");
@@ -683,7 +740,7 @@ namespace ReadyToRun.SuperIlc
                     logWriter.WriteLine(separator);
                     for (int outcomeIndex = 0; outcomeIndex < (int)CompilationOutcome.Count; outcomeIndex++)
                     {
-                        logWriter.Write($"{((CompilationOutcome)outcomeIndex).ToString(),11} |");
+                        logWriter.Write($"{((CompilationOutcome)outcomeIndex).ToString(),12} |");
                         foreach (CompilerRunner runner in _compilerRunners)
                         {
                             logWriter.Write($"{compilationOutcomes[outcomeIndex, (int)runner.Index],8} |");
@@ -695,7 +752,7 @@ namespace ReadyToRun.SuperIlc
                 if (!_options.NoExe)
                 {
                     logWriter.WriteLine();
-                    logWriter.Write($"{totalExecutions,7} EXE |");
+                    logWriter.Write($"{totalExecutions,8} EXE |");
                     foreach (CompilerRunner runner in _compilerRunners)
                     {
                         logWriter.Write($"{runner.CompilerName,8} |");
@@ -704,7 +761,7 @@ namespace ReadyToRun.SuperIlc
                     logWriter.WriteLine(separator);
                     for (int outcomeIndex = 0; outcomeIndex < (int)ExecutionOutcome.Count; outcomeIndex++)
                     {
-                        logWriter.Write($"{((ExecutionOutcome)outcomeIndex).ToString(),11} |");
+                        logWriter.Write($"{((ExecutionOutcome)outcomeIndex).ToString(),12} |");
                         foreach (CompilerRunner runner in _compilerRunners)
                         {
                             logWriter.Write($"{executionOutcomes[outcomeIndex, (int)runner.Index],8} |");
@@ -734,6 +791,10 @@ namespace ReadyToRun.SuperIlc
                 logWriter.WriteLine();
                 logWriter.WriteLine("Framework compilation failures:");
                 FrameworkCompilationFailureBuckets.WriteToStream(logWriter, detailed: false);
+
+                logWriter.WriteLine();
+                logWriter.WriteLine("Framework exclusions:");
+                WriteFrameworkExclusions(logWriter);
             }
 
             if (foldersToBuild != 0)
@@ -753,7 +814,32 @@ namespace ReadyToRun.SuperIlc
                 }
             }
 
-            WriteFoldersBlockedWithIssues(logWriter);
+            if (_buildFolders.Count() != 0)
+            {
+                WriteFoldersBlockedWithIssues(logWriter);
+            }
+        }
+
+        private void WriteFrameworkExclusions(StreamWriter logWriter)
+        {
+            int keyLength = _frameworkExclusions.Keys.Max(key => key.Length);
+            const string SimpleNameTitle = "SIMPLE_NAME";
+            keyLength = Math.Max(keyLength, SimpleNameTitle.Length);
+            StringBuilder title = new StringBuilder();
+            title.Append(SimpleNameTitle);
+            title.Append(' ', keyLength - SimpleNameTitle.Length);
+            title.Append(" | REASON");
+            logWriter.WriteLine(title.ToString());
+            logWriter.WriteLine(new string('-', title.Length));
+            foreach (KeyValuePair<string, string> exclusion in _frameworkExclusions.OrderBy(kvp => kvp.Key, StringComparer.OrdinalIgnoreCase))
+            {
+                StringBuilder line = new StringBuilder();
+                line.Append(exclusion.Key);
+                line.Append(' ', keyLength - exclusion.Key.Length);
+                line.Append(" | ");
+                line.Append(exclusion.Value);
+                logWriter.WriteLine(line.ToString());
+            }
         }
 
         private void WritePerFolderStatistics(StreamWriter logWriter)
@@ -1109,7 +1195,7 @@ namespace ReadyToRun.SuperIlc
                             foreach (CompilerRunner runner in _compilerRunners)
                             {
                                 ProcessInfo compilationProcess = compilation[(int)runner.Index];
-                                if (compilationProcess != null)
+                                if (compilationProcess != null && !compilationProcess.IsEmpty)
                                 {
                                     string log = $"\nCOMPILE {runner.CompilerName}:{compilationProcess.Parameters.InputFileName}";
                                     StreamWriter runnerLog = perRunnerLog[(int)runner.Index];
@@ -1206,6 +1292,15 @@ namespace ReadyToRun.SuperIlc
             string combinedSetLogPath = Path.Combine(_options.OutputDirectory.FullName, "combined-" + suffix);
             WriteCombinedLog(combinedSetLogPath);
 
+            if (_options.Framework)
+            {
+                string frameworkExclusionsFile = Path.Combine(_options.OutputDirectory.FullName, "framework-exclusions-" + suffix);
+                using (StreamWriter writer = new StreamWriter(frameworkExclusionsFile))
+                {
+                    WriteFrameworkExclusions(writer);
+                }
+            }
+
             if (!_options.Exe && _options.Framework)
             {
                 string frameworkBucketsFile = Path.Combine(_options.OutputDirectory.FullName, "framework-buckets-" + suffix);
index 3a283dade85aa41f86e835a281199e7b1a84aeea..6c2342fca2855d1aa968a04e6081065977179e00 100644 (file)
@@ -17,6 +17,7 @@ namespace ReadyToRun.SuperIlc
             var parser = new CommandLineBuilder()
                 .AddCommand(CompileFolder())
                 .AddCommand(CompileSubtree())
+                .AddCommand(CompileFramework())
                 .AddCommand(CompileNugetPackages())
                 .AddCommand(CompileCrossgenRsp());
 
@@ -80,6 +81,26 @@ namespace ReadyToRun.SuperIlc
                     },
                     handler: CommandHandler.Create<BuildOptions>(CompileSubtreeCommand.CompileSubtree));
 
+            Command CompileFramework() =>
+                new Command("compile-framework", "Compile managed framework assemblies in Core_Root",
+                    new Option[]
+                    {
+                            CoreRootDirectory(),
+                            Crossgen(),
+                            NoCleanup(),
+                            DegreeOfParallelism(),
+                            Sequential(),
+                            Release(),
+                            LargeBubble(),
+                            ReferencePath(),
+                            IssuesPath(),
+                            CompilationTimeoutMinutes(),
+                            R2RDumpPath(),
+                            MeasurePerf(),
+                            InputFileSearchString(),
+                    },
+                    handler: CommandHandler.Create<BuildOptions>(CompileFrameworkCommand.CompileFramework));
+
             Command CompileNugetPackages() =>
                 new Command("compile-nuget", "Restore a list of Nuget packages into an empty console app, publish, and optimize with Crossgen / CPAOT",
                     new Option[]
index 4ea858a79780625b4600ce398c17a4d7e227fd02..03ae21aa3af7d5d04f248e23f2cb9a6c46278559 100644 (file)
@@ -51,7 +51,7 @@ namespace ReadyToRun.SuperIlc
             }
 
             BuildFolderSet folderSet = new BuildFolderSet(new BuildFolder[] { folder }, runners, options);
-            bool success = folderSet.Build(runners);
+            bool success = folderSet.Build();
             folderSet.WriteLogs();
 
             if (!options.NoCleanup && !options.Exe)
diff --git a/src/coreclr/src/tools/ReadyToRun.SuperIlc/Commands/CompileFrameworkCommand.cs b/src/coreclr/src/tools/ReadyToRun.SuperIlc/Commands/CompileFrameworkCommand.cs
new file mode 100644 (file)
index 0000000..96a6baa
--- /dev/null
@@ -0,0 +1,40 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using System;
+using System.Collections.Generic;
+using System.Diagnostics;
+using System.IO;
+using System.Linq;
+
+namespace ReadyToRun.SuperIlc
+{
+    class CompileFrameworkCommand
+    {
+        public static int CompileFramework(BuildOptions options)
+        {
+            if (options.CoreRootDirectory == null)
+            {
+                Console.Error.WriteLine("--core-root-directory (--cr) is a required argument.");
+                return 1;
+            }
+
+            string logsFolder = Path.Combine(options.CoreRootDirectory.FullName, "logs");
+            Directory.CreateDirectory(logsFolder);
+            options.OutputDirectory = new DirectoryInfo(logsFolder);
+            options.Framework = true;
+            options.NoJit = true;
+            options.NoEtw = true;
+            options.NoExe = true;
+
+            IEnumerable<CompilerRunner> runners = options.CompilerRunners(isFramework: false);
+
+            BuildFolderSet folderSet = new BuildFolderSet(Array.Empty<BuildFolder>(), runners, options);
+            bool success = folderSet.Build();
+            folderSet.WriteLogs();
+
+            return success ? 0 : 1;
+        }
+    }
+}
index 34acaae5936ebc619155bdb57581b7e1e1ff13f7..4ac34c5fd264d2cbb3233cd00e62cb8c19d3a8ed 100644 (file)
@@ -92,7 +92,7 @@ namespace ReadyToRun.SuperIlc
                 }
 
                 BuildFolderSet folderSet = new BuildFolderSet(publishedAppFoldersToCompile, runners, options);
-                bool success = folderSet.Build(runners);
+                bool success = folderSet.Build();
                 folderSet.WriteLogs();
 
                 if (!options.NoCleanup && !options.Exe)
index 08ebccc7f1a1ed073e34558d499dd246ecb604a5..52d5d411c9b385005977d7f417c6456a521cd90c 100644 (file)
@@ -102,7 +102,7 @@ namespace ReadyToRun.SuperIlc
             Console.WriteLine($@"{directories.Length} folders scanned)");
 
             BuildFolderSet folderSet = new BuildFolderSet(folders, runners, options);
-            bool success = folderSet.Build(runners);
+            bool success = folderSet.Build();
             folderSet.WriteLogs();
 
             if (!options.NoCleanup)
@@ -116,20 +116,24 @@ namespace ReadyToRun.SuperIlc
         private static string[] LocateSubtree(string folder, string coreRootFolder)
         {
             ConcurrentBag<string> directories = new ConcurrentBag<string>();
-            LocateSubtreeAsync(folder, coreRootFolder, directories).Wait();
+            // TODO: this is somewhat hacky - should we introduce a new option -bt (bin/tests/OS.arch.config) we'd use
+            // to derive the location of Core_Root and testhost?
+            string testHostFolder = Path.Combine(Path.GetDirectoryName(Path.GetDirectoryName(coreRootFolder)), "testhost");
+            LocateSubtreeAsync(folder, coreRootFolder, testHostFolder, directories).Wait();
             return directories.ToArray();
         }
 
-        private static async Task LocateSubtreeAsync(string folder, string coreRootFolder, ConcurrentBag<string> directories)
+        private static async Task LocateSubtreeAsync(string folder, string coreRootFolder, string testHostFolder, ConcurrentBag<string> directories)
         {
             if (!Path.GetExtension(folder).Equals(".out", StringComparison.OrdinalIgnoreCase) &&
-                !folder.Equals(coreRootFolder, StringComparison.OrdinalIgnoreCase))
+                !folder.Equals(coreRootFolder, StringComparison.OrdinalIgnoreCase) &&
+                !folder.Equals(testHostFolder, StringComparison.OrdinalIgnoreCase))
             {
                 directories.Add(folder);
                 List<Task> subfolderTasks = new List<Task>();
                 foreach (string subdir in Directory.EnumerateDirectories(folder))
                 {
-                    subfolderTasks.Add(Task.Run(() => LocateSubtreeAsync(subdir, coreRootFolder, directories)));
+                    subfolderTasks.Add(Task.Run(() => LocateSubtreeAsync(subdir, coreRootFolder, testHostFolder, directories)));
                 }
                 await Task.WhenAll(subfolderTasks);
             }
index 5d46c76dd981c7b45589b1937c05d52983b88733..3a9eed3958435a090a3cf51c7f46292c8842cb21 100644 (file)
@@ -108,9 +108,11 @@ public sealed class ParallelRunner
         collectEtwTraces |= measurePerf;
         foreach (ProcessInfo process in processesToRun)
         {
-            process.Construct();
-            processList.Add(process);
-            collectEtwTraces |= process.Parameters.CollectJittedMethods;
+            if (process.Construct())
+            {
+                processList.Add(process);
+                collectEtwTraces |= process.Parameters.CollectJittedMethods;
+            }
         }
 
         processList.Sort((a, b) => b.Parameters.CompilationCostHeuristic.CompareTo(a.Parameters.CompilationCostHeuristic));
index 9209724cf6acf6fb4502340ab95d2873d91d4849..846b48357a7396e6db3c6b9824e985ef07af2dac 100644 (file)
@@ -57,6 +57,8 @@ public class ProcessInfo
     public int ExitCode;
     public Dictionary<string, HashSet<string>> JittedMethods;
 
+    public bool IsEmpty => Parameters == null;
+
     public bool Crashed => ExitCode < -1000 * 1000;
 
     public ProcessInfo(ProcessConstructor constructor)
@@ -64,10 +66,11 @@ public class ProcessInfo
         Constructor = constructor;
     }
 
-    public void Construct()
+    public bool Construct()
     {
         Parameters = Constructor.Construct();
         Constructor = null;
+        return Parameters != null;
     }
 }