Add support for configuration of custom metrics (#1713)

* Add support for configuration of custom metrics

* CR Feedback

diff --git a/src/Microsoft.Diagnostics.Monitoring.RestServer/MetricsOptions.cs b/src/Microsoft.Diagnostics.Monitoring.RestServer/MetricsOptions.cs
index 3126be4663df70c478c10d453684a2e51215f9f0..5c363a18417c009cb68b6fd7e1b4f5b73252eec0 100644 (file)
--- a/src/Microsoft.Diagnostics.Monitoring.RestServer/MetricsOptions.cs
+++ b/src/Microsoft.Diagnostics.Monitoring.RestServer/MetricsOptions.cs
@@ -5,6 +5,7 @@
 using Microsoft.AspNetCore.Http;
 using Microsoft.AspNetCore.Http.Extensions;
 using System;
+using System.Collections.Generic;
 
 namespace Microsoft.Diagnostics.Monitoring.RestServer
 {
@@ -51,5 +52,15 @@ namespace Microsoft.Diagnostics.Monitoring.RestServer
         public int UpdateIntervalSeconds { get; set; }
 
         public int MetricCount { get; set; }
+
+        public bool IncludeDefaultProviders { get; set; } = true;
+
+        public List<MetricProvider> Providers { get; set; } = new List<MetricProvider>(0);
+    }
+
+    public class MetricProvider
+    {
+        public string ProviderName { get; set; }
+        public List<string> CounterNames { get; set; } = new List<string>(0);
     }
 }

diff --git a/src/Microsoft.Diagnostics.Monitoring.RestServer/MetricsService.cs b/src/Microsoft.Diagnostics.Monitoring.RestServer/MetricsService.cs
index 173bcaab6592320d4015830f968d03c60b65dfe7..dce4672cca260628489fd0ffb2d9399922faa41d 100644 (file)
--- a/src/Microsoft.Diagnostics.Monitoring.RestServer/MetricsService.cs
+++ b/src/Microsoft.Diagnostics.Monitoring.RestServer/MetricsService.cs
@@ -6,7 +6,9 @@ using Microsoft.Diagnostics.Monitoring.EventPipe;
 using Microsoft.Diagnostics.NETCore.Client;
 using Microsoft.Extensions.Hosting;
 using Microsoft.Extensions.Options;
+using Microsoft.Extensions.Primitives;
 using System;
+using System.Collections.Generic;
 using System.Threading;
 using System.Threading.Tasks;
 
@@ -20,53 +22,81 @@ namespace Microsoft.Diagnostics.Monitoring.RestServer
         private EventCounterPipeline _counterPipeline;
         private readonly IDiagnosticServices _services;
         private readonly MetricsStoreService _store;
-        private readonly MetricsOptions _options;
+        private IOptionsMonitor<MetricsOptions> _optionsMonitor;
 
         public MetricsService(IDiagnosticServices services,
-            IOptions<MetricsOptions> metricsOptions,
+            IOptionsMonitor<MetricsOptions> optionsMonitor,
             MetricsStoreService metricsStore)
         {
             _store = metricsStore;
             _services = services;
-            _options = metricsOptions.Value;
+            _optionsMonitor = optionsMonitor;
         }
         
-        protected override Task ExecuteAsync(CancellationToken stoppingToken)
+        protected override async Task ExecuteAsync(CancellationToken stoppingToken)
         {
-            return Task.Run( async () =>
+            while (!stoppingToken.IsCancellationRequested)
             {
-                while (!stoppingToken.IsCancellationRequested)
+                stoppingToken.ThrowIfCancellationRequested();
+
+                try
                 {
-                    stoppingToken.ThrowIfCancellationRequested();
+                    //TODO In multi-process scenarios, how do we decide which process to choose?
+                    //One possibility is to enable metrics after a request to begin polling for metrics
+                    IProcessInfo pi = await _services.GetProcessAsync(filter: null, stoppingToken);
 
-                    try
-                    {
-                        //TODO In multi-process scenarios, how do we decide which process to choose?
-                        //One possibility is to enable metrics after a request to begin polling for metrics
-                        IProcessInfo pi = await _services.GetProcessAsync(filter: null, stoppingToken);
+                    var client = new DiagnosticsClient(pi.EndpointInfo.Endpoint);
 
-                        var client = new DiagnosticsClient(pi.EndpointInfo.Endpoint);
+                    MetricsOptions options = _optionsMonitor.CurrentValue;
+                    using var optionsTokenSource = new CancellationTokenSource();
 
-                        _counterPipeline = new EventCounterPipeline(client, new EventPipeCounterPipelineSettings
-                        {
-                            CounterGroups = Array.Empty<EventPipeCounterGroup>(),
-                            Duration = Timeout.InfiniteTimeSpan,
-                            RefreshInterval = TimeSpan.FromSeconds(_options.UpdateIntervalSeconds)
-                        }, metricsLogger: new[] { new MetricsLogger(_store.MetricsStore) });
+                    //If metric options change, we need to cancel the existing metrics pipeline and restart with the new settings.
+                    using IDisposable monitorListener = _optionsMonitor.OnChange((_, _) => optionsTokenSource.Cancel());
 
-                        await _counterPipeline.RunAsync(stoppingToken);
-                    }
-                    catch (Exception e) when (!(e is OperationCanceledException))
+                    EventPipeCounterGroup[] counterGroups = Array.Empty<EventPipeCounterGroup>();
+                    if (options.Providers.Count > 0)
                     {
-                        //Most likely we failed to resolve the pid. Attempt to do this again.
-                        if (_counterPipeline != null)
+                        //In the dotnet-monitor case, custom metrics are additive to the default counters.
+                        var eventPipeCounterGroups = new List<EventPipeCounterGroup>();
+                        if (options.IncludeDefaultProviders)
                         {
-                            await _counterPipeline.DisposeAsync();
+                            eventPipeCounterGroups.Add(new EventPipeCounterGroup { ProviderName = MonitoringSourceConfiguration.SystemRuntimeEventSourceName });
+                            eventPipeCounterGroups.Add(new EventPipeCounterGroup { ProviderName = MonitoringSourceConfiguration.MicrosoftAspNetCoreHostingEventSourceName });
+                            eventPipeCounterGroups.Add(new EventPipeCounterGroup { ProviderName = MonitoringSourceConfiguration.GrpcAspNetCoreServer });
                         }
-                        await Task.Delay(5000);
+
+                        foreach (MetricProvider customProvider in options.Providers)
+                        {
+                            var customCounterGroup = new EventPipeCounterGroup { ProviderName = customProvider.ProviderName };
+                            if (customProvider.CounterNames.Count > 0)
+                            {
+                                customCounterGroup.CounterNames = customProvider.CounterNames.ToArray();
+                            }
+                            eventPipeCounterGroups.Add(customCounterGroup);
+                        }
+                        counterGroups = eventPipeCounterGroups.ToArray();
                     }
+
+                    _counterPipeline = new EventCounterPipeline(client, new EventPipeCounterPipelineSettings
+                    {
+                        CounterGroups = counterGroups,
+                        Duration = Timeout.InfiniteTimeSpan,
+                        RefreshInterval = TimeSpan.FromSeconds(options.UpdateIntervalSeconds)
+                    }, metricsLogger: new[] { new MetricsLogger(_store.MetricsStore) });
+
+                    using var linkedTokenSource = CancellationTokenSource.CreateLinkedTokenSource(stoppingToken, optionsTokenSource.Token);
+                    await _counterPipeline.RunAsync(linkedTokenSource.Token);
                 }
-            }, stoppingToken);
+                catch (Exception e) when (e is not OperationCanceledException || !stoppingToken.IsCancellationRequested)
+                {
+                    //Most likely we failed to resolve the pid or metric configuration change. Attempt to do this again.
+                    if (_counterPipeline != null)
+                    {
+                        await _counterPipeline.DisposeAsync();
+                    }
+                    await Task.Delay(5000);
+                }
+            }
         }
 
         public override async void Dispose()

diff --git a/src/Tools/dotnet-monitor/DiagnosticsMonitorCommandHandler.cs b/src/Tools/dotnet-monitor/DiagnosticsMonitorCommandHandler.cs
index ddfd017cc0d8163f83a476694d76b5ce4309099f..aa2902de6c1075ed9b1442b3e819ed359ba32ed9 100644 (file)
--- a/src/Tools/dotnet-monitor/DiagnosticsMonitorCommandHandler.cs
+++ b/src/Tools/dotnet-monitor/DiagnosticsMonitorCommandHandler.cs
@@ -84,7 +84,7 @@ namespace Microsoft.Diagnostics.Tools.Monitor
                     services.ConfigureEgress(context.Configuration);
                     if (metrics)
                     {
-                        services.Configure<MetricsOptions>(context.Configuration.GetSection(MetricsOptions.ConfigurationKey));
+                        services.ConfigureMetrics(context.Configuration);
                     }
                 })
                 .UseUrls(urls)

diff --git a/src/Tools/dotnet-monitor/ServiceCollectionExtensions.cs b/src/Tools/dotnet-monitor/ServiceCollectionExtensions.cs
index 029eaebcd6351df98ee14603f98d3f921b8085d6..1c5be2f3f26d41ed6a33c5df9d938db71909224a 100644 (file)
--- a/src/Tools/dotnet-monitor/ServiceCollectionExtensions.cs
+++ b/src/Tools/dotnet-monitor/ServiceCollectionExtensions.cs
@@ -13,6 +13,11 @@ namespace Microsoft.Diagnostics.Tools.Monitor
 {
     internal static class ServiceCollectionExtensions
     {
+        public static IServiceCollection ConfigureMetrics(this IServiceCollection services, IConfiguration configuration)
+        {
+            return services.Configure<MetricsOptions>(configuration.GetSection(MetricsOptions.ConfigurationKey));
+        }
+
         public static IServiceCollection ConfigureEgress(this IServiceCollection services, IConfiguration configuration)
         {
             // Register change token for EgressOptions binding