From: Johan Lorensson Date: Thu, 10 Aug 2023 11:06:11 +0000 (+0200) Subject: Add additional filter capabilities to dotnet-pgo tool. (#89853) X-Git-Tag: accepted/tizen/unified/riscv/20231226.055536~389 X-Git-Url: http://review.tizen.org/git/?a=commitdiff_plain;h=6d9baea7c228dfbb11833ae08137bba6a0d262ac;p=platform%2Fupstream%2Fdotnet%2Fruntime.git Add additional filter capabilities to dotnet-pgo tool. (#89853) Mono adopted PGO in .net7 as a replacement for a solution called profiled AOT in mono/mono used by Android SDK. In the replaced solution there was a concept of a stop trigger, meaning that the user could collect methods up to a stop trigger (a method name) meaning that the profiled AOT image would only include methods up to that point. When using EventPipe and nettrace there is limited ability to get the same fine grained control over what methods that ends up in the nettrace file. dotnet-monitor includes ways to stop tracing if it hits for example a specific method, but due to the nature of EventPipe, there could still be additional methods added to the trace when closing session. dotnet-trace currently don't offer any ability to do something similar, but if implemented, it would probably come with the same limitations as dotnet-monitor. Adding better filter capabilities to dotnet-pgo would add additional capabilities, giving users more control on what methods that gets included into the generated mibc file. That would give Mono's profiled AOT better control to include methods up to a stop trigger. dotnet-pgo alread had capabilities to include events based on timestamp interval. This commit extends that to select the lower/upper timestamp based on a regular expression matching methods. This commit also adds capabilities to add a method include/exclude filters using regular expression, giving users fine grained control on what methods to include/exclude in the generated mibc file. The additional filter capabilities could be used by Android SDK to for example create a mibc file including all methods up to Main, replacing the stop trigger features used in old profiled AOT solution. --------- Co-authored-by: mdh1418 --- diff --git a/src/coreclr/tools/dotnet-pgo/PgoRootCommand.cs b/src/coreclr/tools/dotnet-pgo/PgoRootCommand.cs index 06f63db..b9f3fab5 100644 --- a/src/coreclr/tools/dotnet-pgo/PgoRootCommand.cs +++ b/src/coreclr/tools/dotnet-pgo/PgoRootCommand.cs @@ -45,6 +45,14 @@ namespace Microsoft.Diagnostics.Tools.Pgo new("--exclude-events-before") { DefaultValueFactory = _ => Double.MinValue, Description = "Exclude data from events before specified time. Time is specified as milliseconds from the start of the trace" }; public CliOption ExcludeEventsAfter { get; } = new("--exclude-events-after") { DefaultValueFactory = _ => Double.MaxValue, Description = "Exclude data from events after specified time. Time is specified as milliseconds from the start of the trace" }; + public CliOption ExcludeEventsBeforeJittingMethod { get; } = + new("--exclude-events-before-jitting-method") { DefaultValueFactory = _ => string.Empty, Description = "Exclude data from events before observing a specific method getting jitted. Method is matched using a regular expression against the method name. Note that the method name is formatted the same as in PerfView which includes typed parameters." }; + public CliOption ExcludeEventsAfterJittingMethod { get; } = + new("--exclude-events-after-jitting-method") { DefaultValueFactory = _ => string.Empty, Description = "Exclude data from events after observing a specific method getting jitted. Method is matched using a regular expression against the method name. Note that the method name is formatted the same as in PerfView which includes typed parameters." }; + public CliOption IncludeMethods { get; } = + new("--include-methods") { DefaultValueFactory = _ => string.Empty, Description = "Include methods with names matching regular expression. Note that the method names are formatted the same as in PerfView which includes typed parameters." }; + public CliOption ExcludeMethods { get; } = + new("--exclude-methods") { DefaultValueFactory = _ => string.Empty, Description = "Exclude methods with names matching regular expression. Note that the method names are formatted the same as in PerfView which includes typed parameters." }; public CliOption Compressed { get; } = new("--compressed") { DefaultValueFactory = _ => true, Description = "Generate compressed mibc" }; public CliOption DumpWorstOverlapGraphs { get; } = @@ -99,6 +107,10 @@ namespace Microsoft.Diagnostics.Tools.Pgo ClrInstanceId, ExcludeEventsBefore, ExcludeEventsAfter, + ExcludeEventsBeforeJittingMethod, + ExcludeEventsAfterJittingMethod, + IncludeMethods, + ExcludeMethods, AutomaticReferences, _verbosity, Compressed, diff --git a/src/coreclr/tools/dotnet-pgo/Program.cs b/src/coreclr/tools/dotnet-pgo/Program.cs index c65c1d2..400d03c 100644 --- a/src/coreclr/tools/dotnet-pgo/Program.cs +++ b/src/coreclr/tools/dotnet-pgo/Program.cs @@ -21,6 +21,7 @@ using System.Runtime.Serialization.Json; using System.Text; using System.Text.Json; using System.Text.Encodings.Web; +using System.Text.RegularExpressions; using System.Threading.Tasks; using Microsoft.Diagnostics.Tools.Pgo; @@ -1343,6 +1344,12 @@ namespace Microsoft.Diagnostics.Tools.Pgo double excludeEventsBefore = Get(_command.ExcludeEventsBefore); double excludeEventsAfter = Get(_command.ExcludeEventsAfter); + Regex excludeEventsBeforeJittingMethod = !string.IsNullOrEmpty(Get(_command.ExcludeEventsBeforeJittingMethod)) ? new Regex(Get(_command.ExcludeEventsBeforeJittingMethod)) : null; + Regex excludeEventsAfterJittingMethod = !string.IsNullOrEmpty(Get(_command.ExcludeEventsAfterJittingMethod)) ? new Regex(Get(_command.ExcludeEventsAfterJittingMethod)) : null; + Regex includeMethods = !string.IsNullOrEmpty(Get(_command.IncludeMethods)) ? new Regex(Get(_command.IncludeMethods)) : null; + Regex excludeMethods = !string.IsNullOrEmpty(Get(_command.ExcludeMethods)) ? new Regex(Get(_command.ExcludeMethods)) : null; + + // Find all the R2RLoad events. if (_command.ProcessR2REvents) { foreach (var e in p.EventsInProcess.ByEventType()) @@ -1351,6 +1358,7 @@ namespace Microsoft.Diagnostics.Tools.Pgo string retArg = e.MethodSignature.Substring(0, parenIndex); string paramsArgs = e.MethodSignature.Substring(parenIndex); string methodNameFromEventDirectly = retArg + e.MethodNamespace + "." + e.MethodName + paramsArgs; + if (e.ClrInstanceID != clrInstanceId) { if (!_command.Warnings) @@ -1359,6 +1367,7 @@ namespace Microsoft.Diagnostics.Tools.Pgo PrintWarning($"Skipped R2REntryPoint {methodNameFromEventDirectly} due to ClrInstanceID of {e.ClrInstanceID}"); continue; } + MethodDesc method = null; string extraWarningText = null; bool failedDueToNonloadableModule = false; @@ -1382,8 +1391,80 @@ namespace Microsoft.Diagnostics.Tools.Pgo continue; } - if ((e.TimeStampRelativeMSec >= excludeEventsBefore) && (e.TimeStampRelativeMSec <= excludeEventsAfter)) + if (e.TimeStampRelativeMSec < excludeEventsBefore) + { + continue; + } + + if (e.TimeStampRelativeMSec > excludeEventsAfter) + { + break; + } + + string perfviewMethodName = e.MethodNamespace + "." + e.MethodName + paramsArgs; + if (PassesMethodFilter(includeMethods, excludeMethods, perfviewMethodName)) + { methodsToAttemptToPrepare.Add((int)e.EventIndex, new ProcessedMethodData(e.TimeStampRelativeMSec, method, "R2RLoad")); + } + } + } + + // In case requesting events before/after jitting a method, discover the + // corresponding excludeEventsBefore/excludeEventsAfter in event stream based + // on filter criterias. + if (_command.ProcessJitEvents && (excludeEventsBeforeJittingMethod != null || excludeEventsAfterJittingMethod != null)) + { + double firstMatchEventsBeforeJittingMethod = double.PositiveInfinity; + double lastMatchEventsAfterJittingMethod = double.NegativeInfinity; + foreach (var e in p.EventsInProcess.ByEventType()) + { + if (e.ClrInstanceID != clrInstanceId) + { + continue; + } + + MethodDesc method = null; + bool failedDueToNonloadableModule = false; + try + { + method = idParser.ResolveMethodID(e.MethodID, out failedDueToNonloadableModule, false); + } + catch { } + + if (method == null) + { + continue; + } + + int parenIndex = e.MethodSignature.IndexOf('('); + string paramsArgs = e.MethodSignature.Substring(parenIndex); + string perfviewMethodName = e.MethodNamespace + "." + e.MethodName + paramsArgs; + + if (e.TimeStampRelativeMSec > excludeEventsBefore && e.TimeStampRelativeMSec < firstMatchEventsBeforeJittingMethod && excludeEventsBeforeJittingMethod != null && excludeEventsBeforeJittingMethod.IsMatch(perfviewMethodName)) + { + firstMatchEventsBeforeJittingMethod = e.TimeStampRelativeMSec; + } + + if (e.TimeStampRelativeMSec < excludeEventsAfter && e.TimeStampRelativeMSec > lastMatchEventsAfterJittingMethod && excludeEventsAfterJittingMethod != null && excludeEventsAfterJittingMethod.IsMatch(perfviewMethodName)) + { + lastMatchEventsAfterJittingMethod = e.TimeStampRelativeMSec; + } + } + + if (firstMatchEventsBeforeJittingMethod < double.PositiveInfinity) + { + excludeEventsBefore = firstMatchEventsBeforeJittingMethod; + } + + if (lastMatchEventsAfterJittingMethod > double.NegativeInfinity) + { + excludeEventsAfter = lastMatchEventsAfterJittingMethod; + } + + if (excludeEventsBefore > excludeEventsAfter) + { + PrintError($"Exclude events before timestamp: \"{excludeEventsBefore}\" can't be later than exclude events after timestamp: \"{excludeEventsAfter}\""); + return -1; } } @@ -1396,6 +1477,7 @@ namespace Microsoft.Diagnostics.Tools.Pgo string retArg = e.MethodSignature.Substring(0, parenIndex); string paramsArgs = e.MethodSignature.Substring(parenIndex); string methodNameFromEventDirectly = retArg + e.MethodNamespace + "." + e.MethodName + paramsArgs; + if (e.ClrInstanceID != clrInstanceId) { if (!_command.Warnings) @@ -1428,8 +1510,21 @@ namespace Microsoft.Diagnostics.Tools.Pgo continue; } - if ((e.TimeStampRelativeMSec >= excludeEventsBefore) && (e.TimeStampRelativeMSec <= excludeEventsAfter)) + if (e.TimeStampRelativeMSec < excludeEventsBefore) + { + continue; + } + + if (e.TimeStampRelativeMSec > excludeEventsAfter) + { + break; + } + + string perfviewMethodName = e.MethodNamespace + "." + e.MethodName + paramsArgs; + if (PassesMethodFilter(includeMethods, excludeMethods, perfviewMethodName)) + { methodsToAttemptToPrepare.Add((int)e.EventIndex, new ProcessedMethodData(e.TimeStampRelativeMSec, method, "JitStart")); + } } } @@ -1783,6 +1878,24 @@ namespace Microsoft.Diagnostics.Tools.Pgo return 0; } + private static bool PassesMethodFilter(Regex includeMethods, Regex excludeMethods, string methodName) + { + if (includeMethods != null || excludeMethods != null) + { + if (includeMethods != null && !includeMethods.IsMatch(methodName)) + { + return false; + } + + if (excludeMethods != null && excludeMethods.IsMatch(methodName)) + { + return false; + } + } + + return true; + } + private static void GenerateJittraceFile(FileInfo outputFileName, IEnumerable methodsToAttemptToPrepare, JitTraceOptions jittraceOptions) { PrintMessage($"JitTrace options {jittraceOptions}");