Add additional filter capabilities to dotnet-pgo tool. (#89853)

Mono adopted PGO in .net7 as a replacement for a
solution called profiled AOT in mono/mono used by Android
SDK. In the replaced solution there was a concept of
a stop trigger, meaning that the user could collect methods
up to a stop trigger (a method name) meaning that the
profiled AOT image would only include methods up to that point.

When using EventPipe and nettrace there is limited ability to
get the same fine grained control over what methods that ends
up in the nettrace file. dotnet-monitor includes ways to stop
tracing if it hits for example a specific method, but due to the
nature of EventPipe, there could still be additional methods added
to the trace when closing session. dotnet-trace currently don't offer
any ability to do something similar, but if implemented, it
would probably come with the same limitations as dotnet-monitor.

Adding better filter capabilities to dotnet-pgo would add additional
capabilities, giving users more control on what methods that gets
included into the generated mibc file. That would give Mono's
profiled AOT better control to include methods up to a stop trigger.

dotnet-pgo alread had capabilities to include events based on timestamp
interval. This commit extends that to select the lower/upper timestamp
based on a regular expression matching methods. This commit also
adds capabilities to add a method include/exclude filters using regular
expression, giving users fine grained control on what methods to
include/exclude in the generated mibc file.

The additional filter capabilities could be used by Android SDK to
for example create a mibc file including all methods up to Main,
replacing the stop trigger features used in old profiled AOT solution.

---------

Co-authored-by: mdh1418 <mitchhwang1418@gmail.com>

diff --git a/src/coreclr/tools/dotnet-pgo/PgoRootCommand.cs b/src/coreclr/tools/dotnet-pgo/PgoRootCommand.cs
index 06f63db..b9f3fab 100644 (file)
--- a/src/coreclr/tools/dotnet-pgo/PgoRootCommand.cs
+++ b/src/coreclr/tools/dotnet-pgo/PgoRootCommand.cs
@@ -45,6 +45,14 @@ namespace Microsoft.Diagnostics.Tools.Pgo
             new("--exclude-events-before") { DefaultValueFactory = _ => Double.MinValue, Description = "Exclude data from events before specified time. Time is specified as milliseconds from the start of the trace" };
         public CliOption<double> ExcludeEventsAfter { get; } =
             new("--exclude-events-after") { DefaultValueFactory = _ => Double.MaxValue, Description = "Exclude data from events after specified time. Time is specified as milliseconds from the start of the trace" };
+        public CliOption<string> ExcludeEventsBeforeJittingMethod { get; } =
+            new("--exclude-events-before-jitting-method") { DefaultValueFactory = _ => string.Empty, Description = "Exclude data from events before observing a specific method getting jitted. Method is matched using a regular expression against the method name. Note that the method name is formatted the same as in PerfView which includes typed parameters." };
+        public CliOption<string> ExcludeEventsAfterJittingMethod { get; } =
+            new("--exclude-events-after-jitting-method") { DefaultValueFactory = _ => string.Empty, Description = "Exclude data from events after observing a specific method getting jitted. Method is matched using a regular expression against the method name. Note that the method name is formatted the same as in PerfView which includes typed parameters." };
+        public CliOption<string> IncludeMethods { get; } =
+            new("--include-methods") { DefaultValueFactory = _ => string.Empty, Description = "Include methods with names matching regular expression. Note that the method names are formatted the same as in PerfView which includes typed parameters." };
+        public CliOption<string> ExcludeMethods { get; } =
+            new("--exclude-methods") { DefaultValueFactory = _ => string.Empty, Description = "Exclude methods with names matching regular expression. Note that the method names are formatted the same as in PerfView which includes typed parameters." };
         public CliOption<bool> Compressed { get; } =
             new("--compressed") { DefaultValueFactory = _ => true, Description = "Generate compressed mibc" };
         public CliOption<int> DumpWorstOverlapGraphs { get; } =
@@ -99,6 +107,10 @@ namespace Microsoft.Diagnostics.Tools.Pgo
                 ClrInstanceId,
                 ExcludeEventsBefore,
                 ExcludeEventsAfter,
+                ExcludeEventsBeforeJittingMethod,
+                ExcludeEventsAfterJittingMethod,
+                IncludeMethods,
+                ExcludeMethods,
                 AutomaticReferences,
                 _verbosity,
                 Compressed,

diff --git a/src/coreclr/tools/dotnet-pgo/Program.cs b/src/coreclr/tools/dotnet-pgo/Program.cs
index c65c1d2..400d03c 100644 (file)
--- a/src/coreclr/tools/dotnet-pgo/Program.cs
+++ b/src/coreclr/tools/dotnet-pgo/Program.cs
@@ -21,6 +21,7 @@ using System.Runtime.Serialization.Json;
 using System.Text;
 using System.Text.Json;
 using System.Text.Encodings.Web;
+using System.Text.RegularExpressions;
 using System.Threading.Tasks;
 
 using Microsoft.Diagnostics.Tools.Pgo;
@@ -1343,6 +1344,12 @@ namespace Microsoft.Diagnostics.Tools.Pgo
 
                 double excludeEventsBefore = Get(_command.ExcludeEventsBefore);
                 double excludeEventsAfter = Get(_command.ExcludeEventsAfter);
+                Regex excludeEventsBeforeJittingMethod = !string.IsNullOrEmpty(Get(_command.ExcludeEventsBeforeJittingMethod)) ? new Regex(Get(_command.ExcludeEventsBeforeJittingMethod)) : null;
+                Regex excludeEventsAfterJittingMethod = !string.IsNullOrEmpty(Get(_command.ExcludeEventsAfterJittingMethod)) ? new Regex(Get(_command.ExcludeEventsAfterJittingMethod)) : null;
+                Regex includeMethods = !string.IsNullOrEmpty(Get(_command.IncludeMethods)) ? new Regex(Get(_command.IncludeMethods)) : null;
+                Regex excludeMethods = !string.IsNullOrEmpty(Get(_command.ExcludeMethods)) ? new Regex(Get(_command.ExcludeMethods)) : null;
+
+                // Find all the R2RLoad events.
                 if (_command.ProcessR2REvents)
                 {
                     foreach (var e in p.EventsInProcess.ByEventType<R2RGetEntryPointTraceData>())
@@ -1351,6 +1358,7 @@ namespace Microsoft.Diagnostics.Tools.Pgo
                         string retArg = e.MethodSignature.Substring(0, parenIndex);
                         string paramsArgs = e.MethodSignature.Substring(parenIndex);
                         string methodNameFromEventDirectly = retArg + e.MethodNamespace + "." + e.MethodName + paramsArgs;
+
                         if (e.ClrInstanceID != clrInstanceId)
                         {
                             if (!_command.Warnings)
@@ -1359,6 +1367,7 @@ namespace Microsoft.Diagnostics.Tools.Pgo
                             PrintWarning($"Skipped R2REntryPoint {methodNameFromEventDirectly} due to ClrInstanceID of {e.ClrInstanceID}");
                             continue;
                         }
+
                         MethodDesc method = null;
                         string extraWarningText = null;
                         bool failedDueToNonloadableModule = false;
@@ -1382,8 +1391,80 @@ namespace Microsoft.Diagnostics.Tools.Pgo
                             continue;
                         }
 
-                        if ((e.TimeStampRelativeMSec >= excludeEventsBefore) && (e.TimeStampRelativeMSec <= excludeEventsAfter))
+                        if (e.TimeStampRelativeMSec < excludeEventsBefore)
+                        {
+                            continue;
+                        }
+
+                        if (e.TimeStampRelativeMSec > excludeEventsAfter)
+                        {
+                            break;
+                        }
+
+                        string perfviewMethodName = e.MethodNamespace + "." + e.MethodName + paramsArgs;
+                        if (PassesMethodFilter(includeMethods, excludeMethods, perfviewMethodName))
+                        {
                             methodsToAttemptToPrepare.Add((int)e.EventIndex, new ProcessedMethodData(e.TimeStampRelativeMSec, method, "R2RLoad"));
+                        }
+                    }
+                }
+
+                // In case requesting events before/after jitting a method, discover the
+                // corresponding excludeEventsBefore/excludeEventsAfter in event stream based
+                // on filter criterias.
+                if (_command.ProcessJitEvents && (excludeEventsBeforeJittingMethod != null || excludeEventsAfterJittingMethod != null))
+                {
+                    double firstMatchEventsBeforeJittingMethod = double.PositiveInfinity;
+                    double lastMatchEventsAfterJittingMethod = double.NegativeInfinity;
+                    foreach (var e in p.EventsInProcess.ByEventType<MethodJittingStartedTraceData>())
+                    {
+                        if (e.ClrInstanceID != clrInstanceId)
+                        {
+                            continue;
+                        }
+
+                        MethodDesc method = null;
+                        bool failedDueToNonloadableModule = false;
+                        try
+                        {
+                            method = idParser.ResolveMethodID(e.MethodID, out failedDueToNonloadableModule, false);
+                        }
+                        catch { }
+
+                        if (method == null)
+                        {
+                            continue;
+                        }
+
+                        int parenIndex = e.MethodSignature.IndexOf('(');
+                        string paramsArgs = e.MethodSignature.Substring(parenIndex);
+                        string perfviewMethodName = e.MethodNamespace + "." + e.MethodName + paramsArgs;
+
+                        if (e.TimeStampRelativeMSec > excludeEventsBefore && e.TimeStampRelativeMSec < firstMatchEventsBeforeJittingMethod && excludeEventsBeforeJittingMethod != null && excludeEventsBeforeJittingMethod.IsMatch(perfviewMethodName))
+                        {
+                            firstMatchEventsBeforeJittingMethod = e.TimeStampRelativeMSec;
+                        }
+
+                        if (e.TimeStampRelativeMSec < excludeEventsAfter && e.TimeStampRelativeMSec > lastMatchEventsAfterJittingMethod && excludeEventsAfterJittingMethod != null && excludeEventsAfterJittingMethod.IsMatch(perfviewMethodName))
+                        {
+                            lastMatchEventsAfterJittingMethod = e.TimeStampRelativeMSec;
+                        }
+                    }
+
+                    if (firstMatchEventsBeforeJittingMethod < double.PositiveInfinity)
+                    {
+                        excludeEventsBefore = firstMatchEventsBeforeJittingMethod;
+                    }
+
+                    if (lastMatchEventsAfterJittingMethod > double.NegativeInfinity)
+                    {
+                        excludeEventsAfter = lastMatchEventsAfterJittingMethod;
+                    }
+
+                    if (excludeEventsBefore > excludeEventsAfter)
+                    {
+                        PrintError($"Exclude events before timestamp: \"{excludeEventsBefore}\" can't be later than exclude events after timestamp: \"{excludeEventsAfter}\"");
+                        return -1;
                     }
                 }
 
@@ -1396,6 +1477,7 @@ namespace Microsoft.Diagnostics.Tools.Pgo
                         string retArg = e.MethodSignature.Substring(0, parenIndex);
                         string paramsArgs = e.MethodSignature.Substring(parenIndex);
                         string methodNameFromEventDirectly = retArg + e.MethodNamespace + "." + e.MethodName + paramsArgs;
+
                         if (e.ClrInstanceID != clrInstanceId)
                         {
                             if (!_command.Warnings)
@@ -1428,8 +1510,21 @@ namespace Microsoft.Diagnostics.Tools.Pgo
                             continue;
                         }
 
-                        if ((e.TimeStampRelativeMSec >= excludeEventsBefore) && (e.TimeStampRelativeMSec <= excludeEventsAfter))
+                        if (e.TimeStampRelativeMSec < excludeEventsBefore)
+                        {
+                            continue;
+                        }
+
+                        if (e.TimeStampRelativeMSec > excludeEventsAfter)
+                        {
+                            break;
+                        }
+
+                        string perfviewMethodName = e.MethodNamespace + "." + e.MethodName + paramsArgs;
+                        if (PassesMethodFilter(includeMethods, excludeMethods, perfviewMethodName))
+                        {
                             methodsToAttemptToPrepare.Add((int)e.EventIndex, new ProcessedMethodData(e.TimeStampRelativeMSec, method, "JitStart"));
+                        }
                     }
                 }
 
@@ -1783,6 +1878,24 @@ namespace Microsoft.Diagnostics.Tools.Pgo
             return 0;
         }
 
+        private static bool PassesMethodFilter(Regex includeMethods, Regex excludeMethods, string methodName)
+        {
+            if (includeMethods != null || excludeMethods != null)
+            {
+                if (includeMethods != null && !includeMethods.IsMatch(methodName))
+                {
+                    return false;
+                }
+
+                if (excludeMethods != null && excludeMethods.IsMatch(methodName))
+                {
+                    return false;
+                }
+            }
+
+            return true;
+        }
+
         private static void GenerateJittraceFile(FileInfo outputFileName, IEnumerable<ProcessedMethodData> methodsToAttemptToPrepare, JitTraceOptions jittraceOptions)
         {
             PrintMessage($"JitTrace options {jittraceOptions}");