From: Sung Yoon Whang Date: Fri, 25 Oct 2019 20:03:40 +0000 (-0700) Subject: Export feature for dotnet-counters (#493) X-Git-Tag: submit/tizen/20200402.013218~14^2^2~27 X-Git-Url: http://review.tizen.org/git/?a=commitdiff_plain;h=c7a9317aa20b7bc7ae5ae0e8ec3a983c41273ce8;p=platform%2Fcore%2Fdotnet%2Fdiagnostics.git Export feature for dotnet-counters (#493) * Command line stuff * Implement CSV/JSON Exporters * some cleanup * Overwrite the output if needed * remove newline * build error * Refactor to remove duplicacy between ConsoleWriter and Exporters * Remove ICounterExporter * Add license headers * Docs change * Remove unused using * Add file extension to output path only when necessary, remove some remaining comments and useless code * Add unit tests for dotnet-counters * remove useless binaries from the commit * change comment * tabs -> spaces * Remove unused tests * Address PR feedback on tests * More PR feedback * Fix PR comments --- diff --git a/.gitignore b/.gitignore index 83a2d8a5e..32d9b12f2 100644 --- a/.gitignore +++ b/.gitignore @@ -120,3 +120,4 @@ StressLog.txt *.netperf *.nettrace *.speedscope.json +*.csv diff --git a/documentation/design-docs/dotnet-tools.md b/documentation/design-docs/dotnet-tools.md index 5d100ae7b..98d482d90 100644 --- a/documentation/design-docs/dotnet-tools.md +++ b/documentation/design-docs/dotnet-tools.md @@ -150,6 +150,7 @@ COMMANDS list Display a list of counter names and descriptions ps Display a list of dotnet processes that can be monitored monitor Display periodically refreshing values of selected counters + collect Periodically collect selected counter values and export them into a specified file format for post-processing. LIST @@ -229,6 +230,55 @@ MONITOR provider and counter names, use the list command. +COLLECT + + + Examples: + + 1. Collect the runtime performance counters at a refresh interval of 10 seconds and export it as a JSON file named "test.json". +``` + dotnet run collect --process-id 863148 --refresh-interval 10 --output test --format json +``` + + 2. Collect the runtime performance counters as well as the ASP.NET hosting performance counters at the default refresh interval (1 second) and export it as a CSV file named "mycounter.csv". +``` + dotnet run collect --process-id 863148 --output mycounter --format csv System.Runtime Microsoft.AspNetCore.Hosting +``` + + + Syntax: + +``` + dotnet-counters collect [-h||--help] + [-p|--process-id ] + [-o|--output ] + [--format ] + [--refreshInterval ] + counter_list + + Periodically collect selected counter values and export them into a specified file format for post-processing. + + -h, --help + Show command line help + + -p,--process-id + The ID of the process that will be monitored + + -o, --output + The name of the output file + + --format + The format to be exported. Currently available: csv, json + + --refresh-interval + The number of seconds to delay between updating the displayed counters + + counter_list + A space separated list of counters. Counters can be specified provider_name[:counter_name]. If the + provider_name is used without a qualifying counter_name then all counters will be shown. To discover + provider and counter names, use the list command. + +``` ### dotnet-trace diff --git a/documentation/dotnet-counters-instructions.md b/documentation/dotnet-counters-instructions.md index 3c673799d..66f174384 100644 --- a/documentation/dotnet-counters-instructions.md +++ b/documentation/dotnet-counters-instructions.md @@ -34,6 +34,7 @@ dotnet tool install --global dotnet-counters list Display a list of counter names and descriptions ps Display a list of dotnet processes that can be monitored monitor Display periodically refreshing values of selected counters + collect Periodically collect selected counter values and export them into a specified file format for post-processing. *PS* dotnet-counters ps @@ -127,3 +128,50 @@ dotnet tool install --global dotnet-counters provider_name is used without a qualifying counter_name then all counters will be shown. To discover provider and counter names, use the list command. +*COLLECT* + +### Examples: + +1. Collect the runtime performance counters at a refresh interval of 10 seconds and export it as a JSON file named "test.json". + +``` + dotnet run collect --process-id 863148 --refresh-interval 10 --output test --format json +``` + +2. Collect the runtime performance counters as well as the ASP.NET hosting performance counters at the default refresh interval (1 second) and export it as a CSV file named "mycounter.csv". + +``` + dotnet run collect --process-id 863148 --output mycounter --format csv System.Runtime Microsoft.AspNetCore.Hosting +``` + + + ### Syntax: + + dotnet-counters collect [-h||--help] + [-p|--process-id ] + [-o|--output ] + [--format ] + [--refreshInterval ] + counter_list + + Periodically collect selected counter values and export them into a specified file format for post-processing. + + -h, --help + Show command line help + + -p,--process-id + The ID of the process that will be monitored + + -o, --output + The name of the output file + + --format + The format to be exported. Currently available: csv, json + + --refresh-interval + The number of seconds to delay between updating the displayed counters + + counter_list + A space separated list of counters. Counters can be specified provider_name[:counter_name]. If the + provider_name is used without a qualifying counter_name then all counters will be shown. To discover + provider and counter names, use the list command. diff --git a/src/Tools/dotnet-counters/ConsoleWriter.cs b/src/Tools/dotnet-counters/ConsoleWriter.cs deleted file mode 100644 index 5778480b2..000000000 --- a/src/Tools/dotnet-counters/ConsoleWriter.cs +++ /dev/null @@ -1,138 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. -// See the LICENSE file in the project root for more information. - -using System; -using System.Collections.Generic; -using System.Linq; - -namespace Microsoft.Diagnostics.Tools.Counters -{ - public class ConsoleWriter - { - /// Information about an observed provider. - private class ObservedProvider - { - public ObservedProvider(string name) - { - Name = name; - KnownData.TryGetProvider(name, out KnownProvider); - } - - public string Name { get; } // Name of the category. - public Dictionary Counters { get; } = new Dictionary(); // Counters in this category. - public readonly CounterProvider KnownProvider; - } - - /// Information about an observed counter. - private class ObservedCounter - { - public ObservedCounter(string displayName) => DisplayName = displayName; - public string DisplayName { get; } // Display name for this counter. - public int Row { get; set; } // Assigned row for this counter. May change during operation. - } - - private readonly Dictionary providers = new Dictionary(); // Tracks observed providers and counters. - private const int Indent = 4; // Counter name indent size. - private int maxNameLength = 40; // Allow room for 40 character counter names by default. - private int maxPreDecimalDigits = 11; // Allow room for values up to 999 million by default. - - private int STATUS_ROW; // Row # of where we print the status of dotnet-counters - private bool paused = false; - private bool initialized = false; - - private void UpdateStatus() - { - Console.SetCursorPosition(0, STATUS_ROW); - Console.Write($" Status: {GetStatus()}{new string(' ', 40)}"); // Write enough blanks to clear previous status. - } - - private string GetStatus() => !initialized ? "Waiting for initial payload..." : (paused ? "Paused" : "Running"); - - /// Clears display and writes out category and counter name layout. - public void AssignRowsAndInitializeDisplay() - { - Console.Clear(); - int row = Console.CursorTop; - Console.WriteLine("Press p to pause, r to resume, q to quit."); row++; - Console.WriteLine($" Status: {GetStatus()}"); STATUS_ROW = row++; - Console.WriteLine(); row++; // Blank line. - - foreach (ObservedProvider provider in providers.Values.OrderBy(p => p.KnownProvider == null).ThenBy(p => p.Name)) // Known providers first. - { - Console.WriteLine($"[{provider.Name}]"); row++; - foreach (ObservedCounter counter in provider.Counters.Values.OrderBy(c => c.DisplayName)) - { - Console.WriteLine($"{new string(' ', Indent)}{counter.DisplayName}"); - counter.Row = row++; - } - } - } - - public void ToggleStatus(bool pauseCmdSet) - { - if (paused == pauseCmdSet) - { - return; - } - - paused = pauseCmdSet; - UpdateStatus(); - } - - public void Update(string providerName, ICounterPayload payload, bool pauseCmdSet) - { - if (!initialized) - { - initialized = true; - AssignRowsAndInitializeDisplay(); - } - - if (pauseCmdSet) - { - return; - } - - string name = payload.GetName(); - - bool redraw = false; - if (!providers.TryGetValue(providerName, out ObservedProvider provider)) - { - providers[providerName] = provider = new ObservedProvider(providerName); - redraw = true; - } - - if (!provider.Counters.TryGetValue(name, out ObservedCounter counter)) - { - string displayName = provider.KnownProvider?.TryGetDisplayName(name) ?? (string.IsNullOrWhiteSpace(payload.GetDisplay()) ? name : payload.GetDisplay()); - provider.Counters[name] = counter = new ObservedCounter(displayName); - maxNameLength = Math.Max(maxNameLength, displayName.Length); - redraw = true; - } - - const string DecimalPlaces = "###"; - string payloadVal = payload.GetValue().ToString("#,0." + DecimalPlaces); - int decimalIndex = payloadVal.IndexOf('.'); - if (decimalIndex == -1) - { - decimalIndex = payloadVal.Length; - } - - if (decimalIndex > maxPreDecimalDigits) - { - maxPreDecimalDigits = decimalIndex; - redraw = true; - } - - if (redraw) - { - AssignRowsAndInitializeDisplay(); - } - - Console.SetCursorPosition(Indent + maxNameLength + 1, counter.Row); - int prefixSpaces = maxPreDecimalDigits - decimalIndex; - int postfixSpaces = DecimalPlaces.Length - (payloadVal.Length - decimalIndex - 1); - Console.Write($"{new string(' ', prefixSpaces)}{payloadVal}{new string(' ', postfixSpaces)}"); - } - } -} diff --git a/src/Tools/dotnet-counters/CounterMonitor.cs b/src/Tools/dotnet-counters/CounterMonitor.cs index 05b27465a..d4097ecb2 100644 --- a/src/Tools/dotnet-counters/CounterMonitor.cs +++ b/src/Tools/dotnet-counters/CounterMonitor.cs @@ -13,6 +13,7 @@ using System.Threading; using System.Threading.Tasks; using Microsoft.Diagnostics.Tracing; +using Microsoft.Diagnostics.Tools.Counters.Exporters; namespace Microsoft.Diagnostics.Tools.Counters { @@ -23,23 +24,23 @@ namespace Microsoft.Diagnostics.Tools.Counters private List _counterList; private CancellationToken _ct; private IConsole _console; - private ConsoleWriter writer; + private ICounterRenderer _renderer; private CounterFilter filter; private ulong _sessionId; + private string _output; private bool pauseCmdSet; public CounterMonitor() { - writer = new ConsoleWriter(); filter = new CounterFilter(); pauseCmdSet = false; } - private void Dynamic_All(TraceEvent obj) + private void DynamicAllMonitor(TraceEvent obj) { // If we are paused, ignore the event. // There's a potential race here between the two tasks but not a huge deal if we miss by one event. - writer.ToggleStatus(pauseCmdSet); + _renderer.ToggleStatus(pauseCmdSet); if (obj.EventName.Equals("EventCounters")) { @@ -49,19 +50,8 @@ namespace Microsoft.Diagnostics.Tools.Counters // If it's not a counter we asked for, ignore it. if (!filter.Filter(obj.ProviderName, payloadFields["Name"].ToString())) return; - // There really isn't a great way to tell whether an EventCounter payload is an instance of - // IncrementingCounterPayload or CounterPayload, so here we check the number of fields - // to distinguish the two. - ICounterPayload payload; - if (payloadFields.ContainsKey("CounterType")) - { - payload = payloadFields["CounterType"].Equals("Sum") ? (ICounterPayload)new IncrementingCounterPayload(payloadFields, _interval) : (ICounterPayload)new CounterPayload(payloadFields); - } - else - { - payload = payloadFields.Count == 6 ? (ICounterPayload)new IncrementingCounterPayload(payloadFields, _interval) : (ICounterPayload)new CounterPayload(payloadFields); - } - writer.Update(obj.ProviderName, payload, pauseCmdSet); + ICounterPayload payload = payloadFields["CounterType"].Equals("Sum") ? (ICounterPayload)new IncrementingCounterPayload(payloadFields, _interval) : (ICounterPayload)new CounterPayload(payloadFields); + _renderer.CounterPayloadReceived(obj.ProviderName, payload, pauseCmdSet); } } @@ -88,6 +78,7 @@ namespace Microsoft.Diagnostics.Tools.Counters catch (PlatformNotSupportedException) { } + _renderer.Stop(); } public async Task Monitor(CancellationToken ct, List counter_list, IConsole console, int processId, int refreshInterval) @@ -99,8 +90,9 @@ namespace Microsoft.Diagnostics.Tools.Counters _console = console; _processId = processId; _interval = refreshInterval; + _renderer = new ConsoleWriter(); - return await StartMonitor(); + return await Start(); } catch (OperationCanceledException) @@ -116,8 +108,55 @@ namespace Microsoft.Diagnostics.Tools.Counters } } + public async Task Collect(CancellationToken ct, List counter_list, IConsole console, int processId, int refreshInterval, CountersExportFormat format, string output) + { + try + { + _ct = ct; + _counterList = counter_list; // NOTE: This variable name has an underscore because that's the "name" that the CLI displays. System.CommandLine doesn't like it if we change the variable to camelcase. + _console = console; + _processId = processId; + _interval = refreshInterval; + _output = output; + + if (_output.Length == 0) + { + _console.Error.WriteLine("Output cannot be an empty string"); + return 0; + } + + if (format == CountersExportFormat.csv) + { + _renderer = new CSVExporter(output); + } + else if (format == CountersExportFormat.json) + { + // Try getting the process name. + string processName = ""; + try + { + processName = Process.GetProcessById(_processId).ProcessName; + } + catch (Exception) { } + _renderer = new JSONExporter(output, processName); ; + } + else + { + _console.Error.WriteLine($"The output format {format} is not a valid output format."); + return 0; + } + return await Start(); + } + catch (OperationCanceledException) + { + } + + return 1; + } + + // Use EventPipe CollectTracing2 command to start monitoring. This may throw. - private void RequestTracingV2(string providerString) + private EventPipeEventSource RequestTracingV2(string providerString) { var configuration = new SessionConfigurationV2( circularBufferSizeMB: 1000, @@ -125,32 +164,23 @@ namespace Microsoft.Diagnostics.Tools.Counters requestRundown: false, providers: Trace.Extensions.ToProviders(providerString)); var binaryReader = EventPipeClient.CollectTracing2(_processId, configuration, out _sessionId); - EventPipeEventSource source = new EventPipeEventSource(binaryReader); - source.Dynamic.All += Dynamic_All; - source.Process(); + return new EventPipeEventSource(binaryReader); } + // Use EventPipe CollectTracing command to start monitoring. This may throw. - private void RequestTracingV1(string providerString) + private EventPipeEventSource RequestTracingV1(string providerString) { var configuration = new SessionConfiguration( circularBufferSizeMB: 1000, format: EventPipeSerializationFormat.NetTrace, providers: Trace.Extensions.ToProviders(providerString)); var binaryReader = EventPipeClient.CollectTracing(_processId, configuration, out _sessionId); - EventPipeEventSource source = new EventPipeEventSource(binaryReader); - source.Dynamic.All += Dynamic_All; - source.Process(); + return new EventPipeEventSource(binaryReader); } - private async Task StartMonitor() + private string BuildProviderString() { - if (_processId == 0) { - _console.Error.WriteLine("--process-id is required."); - return 1; - } - - String providerString; - + string providerString; if (_counterList.Count == 0) { CounterProvider defaultProvider = null; @@ -160,7 +190,7 @@ namespace Microsoft.Diagnostics.Tools.Counters if (!KnownData.TryGetProvider("System.Runtime", out defaultProvider)) { _console.Error.WriteLine("No providers or profiles were specified and there is no default profile available."); - return 1; + return ""; } providerString = defaultProvider.ToProviderString(_interval); filter.AddFilter("System.Runtime"); @@ -202,22 +232,47 @@ namespace Microsoft.Diagnostics.Tools.Counters } providerString = sb.ToString(); } + return providerString; + } + + + private async Task Start() + { + if (_processId == 0) + { + _console.Error.WriteLine("--process-id is required."); + return 1; + } - ManualResetEvent shouldExit = new ManualResetEvent(false); - _ct.Register(() => shouldExit.Set()); + string providerString = BuildProviderString(); + if (providerString.Length == 0) + { + return 1; + } - var terminated = false; - writer.AssignRowsAndInitializeDisplay(); + _renderer.Initialize(); + ManualResetEvent shouldExit = new ManualResetEvent(false); + _ct.Register(() => shouldExit.Set()); Task monitorTask = new Task(() => { try { - RequestTracingV2(providerString); - } - catch (EventPipeUnknownCommandException) - { - // If unknown command exception is thrown, it's likely the app being monitored is running an older version of runtime that doesn't support CollectTracingV2. Try again with V1. - RequestTracingV1(providerString); + EventPipeEventSource source = null; + + try + { + source = RequestTracingV2(providerString); + } + catch (EventPipeUnknownCommandException) + { + // If unknown command exception is thrown, it's likely the app being monitored is + // running an older version of runtime that doesn't support CollectTracingV2. Try again with V1. + source = RequestTracingV1(providerString); + } + + source.Dynamic.All += DynamicAllMonitor; + _renderer.EventPipeSourceConnected(); + source.Process(); } catch (Exception ex) { @@ -225,7 +280,6 @@ namespace Microsoft.Diagnostics.Tools.Counters } finally { - terminated = true; // This indicates that the runtime is done. We shouldn't try to talk to it anymore. shouldExit.Set(); } }); @@ -249,6 +303,7 @@ namespace Microsoft.Diagnostics.Tools.Counters ConsoleKey cmd = Console.ReadKey(true).Key; if (cmd == ConsoleKey.Q) { + StopMonitor(); break; } else if (cmd == ConsoleKey.P) @@ -260,11 +315,7 @@ namespace Microsoft.Diagnostics.Tools.Counters pauseCmdSet = false; } } - if (!terminated) - { - StopMonitor(); - } - + return await Task.FromResult(0); } } diff --git a/src/Tools/dotnet-counters/CounterPayload.cs b/src/Tools/dotnet-counters/CounterPayload.cs index 2b1e4ddee..f0f9d26d5 100644 --- a/src/Tools/dotnet-counters/CounterPayload.cs +++ b/src/Tools/dotnet-counters/CounterPayload.cs @@ -13,6 +13,7 @@ namespace Microsoft.Diagnostics.Tools.Counters string GetName(); double GetValue(); string GetDisplay(); + string GetCounterType(); } @@ -42,6 +43,11 @@ namespace Microsoft.Diagnostics.Tools.Counters { return m_DisplayName; } + + public string GetCounterType() + { + return "Metric"; + } } class IncrementingCounterPayload : ICounterPayload @@ -76,5 +82,10 @@ namespace Microsoft.Diagnostics.Tools.Counters { return $"{m_DisplayName} / {m_DisplayRateTimeScale}"; } + + public string GetCounterType() + { + return "Rate"; + } } } diff --git a/src/Tools/dotnet-counters/Exporters/CSVExporter.cs b/src/Tools/dotnet-counters/Exporters/CSVExporter.cs new file mode 100644 index 000000000..9cccdd323 --- /dev/null +++ b/src/Tools/dotnet-counters/Exporters/CSVExporter.cs @@ -0,0 +1,73 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.IO; +using System.Text; + +namespace Microsoft.Diagnostics.Tools.Counters.Exporters +{ + class CSVExporter : ICounterRenderer + { + private string _output; + private StringBuilder builder; + private int flushLength = 10_000; // Arbitrary length to flush + + public string Output { get; set; } + + public CSVExporter(string output) + { + if (output.EndsWith(".csv")) + { + _output = output; + } + else + { + _output = output + ".csv"; + } + } + + public void Initialize() + { + if (File.Exists(_output)) + { + Console.WriteLine($"[Warning] {_output} already exists. This file will be overwritten."); + File.Delete(_output); + } + builder = new StringBuilder(); + builder.AppendLine("Timestamp,Provider,Counter Name,Counter Type,Mean/Increment"); + } + + public void EventPipeSourceConnected() + { + Console.WriteLine("Starting a counter session. Press Q to quit."); + } + + public void ToggleStatus(bool paused) + { + // Do nothing + } + + public void CounterPayloadReceived(string providerName, ICounterPayload payload, bool _) + { + if (builder.Length > flushLength) + { + File.AppendAllText(_output, builder.ToString()); + builder.Clear(); + } + builder.Append(DateTime.UtcNow.ToString() + ","); + builder.Append(providerName + ","); + builder.Append(payload.GetDisplay() + ","); + builder.Append(payload.GetCounterType() + ","); + builder.Append(payload.GetValue() + "\n"); + } + + public void Stop() + { + // Append all the remaining text to the file. + File.AppendAllText(_output, builder.ToString()); + Console.WriteLine("File saved to " + _output); + } + } +} diff --git a/src/Tools/dotnet-counters/Exporters/ConsoleWriter.cs b/src/Tools/dotnet-counters/Exporters/ConsoleWriter.cs new file mode 100644 index 000000000..29720b64e --- /dev/null +++ b/src/Tools/dotnet-counters/Exporters/ConsoleWriter.cs @@ -0,0 +1,157 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Collections.Generic; +using System.Linq; + +namespace Microsoft.Diagnostics.Tools.Counters.Exporters +{ + /// + /// ConsoleWriter is an implementation of ICounterRenderer for rendering the counter values in real-time + /// to the console. This is the renderer for the `dotnet-counters monitor` command. + /// + public class ConsoleWriter : ICounterRenderer + { + /// Information about an observed provider. + private class ObservedProvider + { + public ObservedProvider(string name) + { + Name = name; + KnownData.TryGetProvider(name, out KnownProvider); + } + + public string Name { get; } // Name of the category. + public Dictionary Counters { get; } = new Dictionary(); // Counters in this category. + public readonly CounterProvider KnownProvider; + } + + /// Information about an observed counter. + private class ObservedCounter + { + public ObservedCounter(string displayName) => DisplayName = displayName; + public string DisplayName { get; } // Display name for this counter. + public int Row { get; set; } // Assigned row for this counter. May change during operation. + } + + private readonly Dictionary providers = new Dictionary(); // Tracks observed providers and counters. + private const int Indent = 4; // Counter name indent size. + private int maxNameLength = 40; // Allow room for 40 character counter names by default. + private int maxPreDecimalDigits = 11; // Allow room for values up to 999 million by default. + + private int STATUS_ROW; // Row # of where we print the status of dotnet-counters + private bool paused = false; + private bool initialized = false; + + public void Initialize() + { + AssignRowsAndInitializeDisplay(); + } + + public void EventPipeSourceConnected() + { + // Do nothing + } + + private void UpdateStatus() + { + Console.SetCursorPosition(0, STATUS_ROW); + Console.Write($" Status: {GetStatus()}{new string(' ', 40)}"); // Write enough blanks to clear previous status. + } + + private string GetStatus() => !initialized ? "Waiting for initial payload..." : (paused ? "Paused" : "Running"); + + /// Clears display and writes out category and counter name layout. + public void AssignRowsAndInitializeDisplay() + { + Console.Clear(); + int row = Console.CursorTop; + Console.WriteLine("Press p to pause, r to resume, q to quit."); row++; + Console.WriteLine($" Status: {GetStatus()}"); STATUS_ROW = row++; + Console.WriteLine(); row++; // Blank line. + + foreach (ObservedProvider provider in providers.Values.OrderBy(p => p.KnownProvider == null).ThenBy(p => p.Name)) // Known providers first. + { + Console.WriteLine($"[{provider.Name}]"); row++; + foreach (ObservedCounter counter in provider.Counters.Values.OrderBy(c => c.DisplayName)) + { + Console.WriteLine($"{new string(' ', Indent)}{counter.DisplayName}"); + counter.Row = row++; + } + } + } + + public void ToggleStatus(bool pauseCmdSet) + { + if (paused == pauseCmdSet) + { + return; + } + + paused = pauseCmdSet; + UpdateStatus(); + } + + public void CounterPayloadReceived(string providerName, ICounterPayload payload, bool pauseCmdSet) + { + if (!initialized) + { + initialized = true; + AssignRowsAndInitializeDisplay(); + } + + if (pauseCmdSet) + { + return; + } + + string name = payload.GetName(); + + bool redraw = false; + if (!providers.TryGetValue(providerName, out ObservedProvider provider)) + { + providers[providerName] = provider = new ObservedProvider(providerName); + redraw = true; + } + + if (!provider.Counters.TryGetValue(name, out ObservedCounter counter)) + { + string displayName = provider.KnownProvider?.TryGetDisplayName(name) ?? (string.IsNullOrWhiteSpace(payload.GetDisplay()) ? name : payload.GetDisplay()); + provider.Counters[name] = counter = new ObservedCounter(displayName); + maxNameLength = Math.Max(maxNameLength, displayName.Length); + redraw = true; + } + + const string DecimalPlaces = "###"; + string payloadVal = payload.GetValue().ToString("#,0." + DecimalPlaces); + int decimalIndex = payloadVal.IndexOf('.'); + if (decimalIndex == -1) + { + decimalIndex = payloadVal.Length; + } + + if (decimalIndex > maxPreDecimalDigits) + { + maxPreDecimalDigits = decimalIndex; + redraw = true; + } + + if (redraw) + { + AssignRowsAndInitializeDisplay(); + } + + Console.SetCursorPosition(Indent + maxNameLength + 1, counter.Row); + int prefixSpaces = maxPreDecimalDigits - decimalIndex; + int postfixSpaces = DecimalPlaces.Length - (payloadVal.Length - decimalIndex - 1); + Console.Write($"{new string(' ', prefixSpaces)}{payloadVal}{new string(' ', postfixSpaces)}"); + } + + public void Stop() + { + // Nothing to do here. + } + } +} diff --git a/src/Tools/dotnet-counters/Exporters/ICounterRenderer.cs b/src/Tools/dotnet-counters/Exporters/ICounterRenderer.cs new file mode 100644 index 000000000..9d75d55f1 --- /dev/null +++ b/src/Tools/dotnet-counters/Exporters/ICounterRenderer.cs @@ -0,0 +1,15 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +namespace Microsoft.Diagnostics.Tools.Counters.Exporters +{ + public interface ICounterRenderer + { + void Initialize(); + void EventPipeSourceConnected(); + void ToggleStatus(bool paused); + void CounterPayloadReceived(string providerName, ICounterPayload payload, bool paused); + void Stop(); + } +} diff --git a/src/Tools/dotnet-counters/Exporters/JSONExporter.cs b/src/Tools/dotnet-counters/Exporters/JSONExporter.cs new file mode 100644 index 000000000..ee18c0ab8 --- /dev/null +++ b/src/Tools/dotnet-counters/Exporters/JSONExporter.cs @@ -0,0 +1,75 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.IO; +using System.Text; + +namespace Microsoft.Diagnostics.Tools.Counters.Exporters +{ + class JSONExporter : ICounterRenderer + { + private string _output; + private string _processName; + private StringBuilder builder; + private int flushLength = 10_000; // Arbitrary length to flush + + public JSONExporter(string output, string processName) + { + if (output.EndsWith(".json")) + { + _output = output; + } + else + { + _output = output + ".json"; + } + _processName = processName; + } + public void Initialize() + { + if (File.Exists(_output)) + { + Console.WriteLine($"[Warning] {_output} already exists. This file will be overwritten."); + File.Delete(_output); + } + + builder = new StringBuilder(); + builder.Append($"{{ \"Target Process\": \"{_processName}\", "); + builder.Append($"\"Start Time\": \"{DateTime.Now.ToString()}\", "); + builder.Append($"\"Events\": ["); + } + + public void EventPipeSourceConnected() + { + Console.WriteLine("Starting a counter session. Press Q to quit."); + } + public void ToggleStatus(bool paused) + { + // Do nothing + } + + public void CounterPayloadReceived(string providerName, ICounterPayload payload, bool _) + { + if (builder.Length > flushLength) + { + File.AppendAllText(_output, builder.ToString()); + builder.Clear(); + } + builder.Append($"{{ \"timestamp\": \"{DateTime.Now.ToString()}\", "); + builder.Append($" \"provider\": \"{providerName}\", "); + builder.Append($" \"name\": \"{payload.GetDisplay()}\", "); + builder.Append($" \"counter type\": \"{payload.GetCounterType()}\", "); + builder.Append($" \"value\": {payload.GetValue()} }},"); + } + + public void Stop() + { + builder.Append($"] }}"); + // Append all the remaining text to the file. + File.AppendAllText(_output, builder.ToString()); + Console.WriteLine("File saved to " + _output); + } + } +} diff --git a/src/Tools/dotnet-counters/Program.cs b/src/Tools/dotnet-counters/Program.cs index c9a36fe53..318d40fba 100644 --- a/src/Tools/dotnet-counters/Program.cs +++ b/src/Tools/dotnet-counters/Program.cs @@ -4,9 +4,9 @@ using System; using System.CommandLine; +using System.CommandLine.Binding; using System.CommandLine.Builder; using System.CommandLine.Invocation; -using System.Diagnostics; using System.Linq; using System.Threading; using System.Threading.Tasks; @@ -16,8 +16,12 @@ using Microsoft.Internal.Common.Commands; namespace Microsoft.Diagnostics.Tools.Counters { + public enum CountersExportFormat { csv, json }; + internal class Program { + delegate Task ExportDelegate(CancellationToken ct, List counter_list, IConsole console, int processId, int refreshInterval, CountersExportFormat format, string output); + private static Command MonitorCommand() => new Command( "monitor", @@ -26,6 +30,14 @@ namespace Microsoft.Diagnostics.Tools.Counters argument: CounterList(), handler: CommandHandler.Create, IConsole, int, int>(new CounterMonitor().Monitor)); + private static Command CollectCommand() => + new Command( + "collect", + "Monitor counters in a .NET application and export the result into a file", + new Option[] { ProcessIdOption(), RefreshIntervalOption(), ExportFormatOption(), ExportFileNameOption() }, + argument: CounterList(), + handler: HandlerDescriptor.FromDelegate((ExportDelegate)new CounterMonitor().Collect).GetCommandHandler()); + private static Option ProcessIdOption() => new Option( new[] { "-p", "--process-id" }, @@ -38,6 +50,18 @@ namespace Microsoft.Diagnostics.Tools.Counters "The number of seconds to delay between updating the displayed counters.", new Argument(defaultValue: 1) { Name = "refresh-interval" }); + private static Option ExportFormatOption() => + new Option( + new[] { "--format" }, + "The format of exported counter data.", + new Argument(defaultValue: CountersExportFormat.csv) { Name = "format" }); + + private static Option ExportFileNameOption() => + new Option( + new[] { "-o", "--output" }, + "The output file name.", + new Argument(defaultValue: "counter") { Name = "output" }); + private static Argument CounterList() => new Argument> { Name = "counter_list", @@ -85,6 +109,7 @@ namespace Microsoft.Diagnostics.Tools.Counters { var parser = new CommandLineBuilder() .AddCommand(MonitorCommand()) + .AddCommand(CollectCommand()) .AddCommand(ListCommand()) .AddCommand(ProcessStatusCommand()) .UseDefaults() diff --git a/src/Tools/dotnet-counters/dotnet-counters.csproj b/src/Tools/dotnet-counters/dotnet-counters.csproj index adcd8a4fa..b8418707e 100644 --- a/src/Tools/dotnet-counters/dotnet-counters.csproj +++ b/src/Tools/dotnet-counters/dotnet-counters.csproj @@ -24,4 +24,8 @@ + + + + \ No newline at end of file diff --git a/src/tests/dotnet-counters/CSVExporterTests.cs b/src/tests/dotnet-counters/CSVExporterTests.cs new file mode 100644 index 000000000..907383c8b --- /dev/null +++ b/src/tests/dotnet-counters/CSVExporterTests.cs @@ -0,0 +1,105 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; + +using Xunit; +using Microsoft.Diagnostics.Tools.Counters; +using Microsoft.Diagnostics.Tools.Counters.Exporters; + +namespace DotnetCounters.UnitTests +{ + /// + /// These test the some of the known providers that we provide as a default configuration for customers to use. + /// + public class CSVExporterTests + { + [Fact] + public void IncrementingCounterTest() + { + string fileName = "IncrementingCounterTest.csv"; + CSVExporter exporter = new CSVExporter(fileName); + exporter.Initialize(); + for (int i = 0; i < 100; i++) + { + exporter.CounterPayloadReceived("myProvider", TestHelpers.GenerateCounterPayload(true, "incrementingCounterOne", i, 1, "Incrementing Counter One: " + i.ToString()), false); + } + exporter.Stop(); + + Assert.True(File.Exists(fileName)); + + try + { + List lines = File.ReadLines(fileName).ToList(); + Assert.Equal(101, lines.Count); // should be 101 including the headers + + string[] headerTokens = lines[0].Split(','); + Assert.Equal("Provider", headerTokens[1]); + Assert.Equal("Counter Name", headerTokens[2]); + Assert.Equal("Counter Type", headerTokens[3]); + Assert.Equal("Mean/Increment", headerTokens[4]); + + for (int i = 1; i < lines.Count; i++) + { + string[] tokens = lines[i].Split(','); + + Assert.Equal("myProvider", tokens[1]); + Assert.Equal($"Incrementing Counter One: {i-1} / 1 sec", tokens[2]); + Assert.Equal("Rate", tokens[3]); + Assert.Equal((i - 1).ToString(), tokens[4]); + } + } + finally + { + File.Delete(fileName); + } + } + + [Fact] + public void CounterTest() + { + string fileName = "CounterTest.csv"; + CSVExporter exporter = new CSVExporter(fileName); + exporter.Initialize(); + for (int i = 0; i < 10; i++) + { + exporter.CounterPayloadReceived("myProvider", TestHelpers.GenerateCounterPayload(false, "counterOne", i, 1, "Counter One: " + i.ToString()), false); + } + exporter.Stop(); + + Assert.True(File.Exists(fileName)); + + try + { + List lines = File.ReadLines(fileName).ToList(); + Assert.Equal(11, lines.Count); // should be 11 including the headers + + string[] headerTokens = lines[0].Split(','); + Assert.Equal("Provider", headerTokens[1]); + Assert.Equal("Counter Name", headerTokens[2]); + Assert.Equal("Counter Type", headerTokens[3]); + Assert.Equal("Mean/Increment", headerTokens[4]); + + + for (int i = 1; i < lines.Count; i++) + { + string[] tokens = lines[i].Split(','); + + Assert.Equal("myProvider", tokens[1]); + Assert.Equal("Counter One: " + (i - 1).ToString(), tokens[2]); + Assert.Equal("Metric", tokens[3]); + Assert.Equal((i - 1).ToString(), tokens[4]); + } + } + finally + { + File.Delete(fileName); + } + + } + } +} diff --git a/src/tests/dotnet-counters/DotnetCounters.UnitTests.csproj b/src/tests/dotnet-counters/DotnetCounters.UnitTests.csproj new file mode 100644 index 000000000..1d1fa5489 --- /dev/null +++ b/src/tests/dotnet-counters/DotnetCounters.UnitTests.csproj @@ -0,0 +1,15 @@ + + + + netcoreapp3.0 + + + + + + + + + + + diff --git a/src/tests/dotnet-counters/JSONExporterTests.cs b/src/tests/dotnet-counters/JSONExporterTests.cs new file mode 100644 index 000000000..12411fe02 --- /dev/null +++ b/src/tests/dotnet-counters/JSONExporterTests.cs @@ -0,0 +1,109 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.IO; +using System.Collections.Generic; +using Xunit; +using Microsoft.Diagnostics.Tools.Counters.Exporters; +using Newtonsoft.Json; + +namespace DotnetCounters.UnitTests +{ + /// + /// These test the some of the known providers that we provide as a default configuration for customers to use. + /// + public class JSONExporterTests + { + [Fact] + public void IncrementingCounterTest() + { + string fileName = "IncrementingCounterTest.json"; + JSONExporter exporter = new JSONExporter(fileName, "myProcess.exe"); + exporter.Initialize(); + for (int i = 0; i < 10; i++) + { + exporter.CounterPayloadReceived("myProvider", TestHelpers.GenerateCounterPayload(true, "incrementingCounterOne", 1.0, 1, "Incrementing Counter One"), false); + } + exporter.Stop(); + + Assert.True(File.Exists(fileName)); + using (StreamReader r = new StreamReader(fileName)) + { + string json = r.ReadToEnd(); + JSONCounterTrace counterTrace = JsonConvert.DeserializeObject(json); + + Assert.Equal("myProcess.exe", counterTrace.targetProcess); + Assert.Equal(10, counterTrace.events.Length); + foreach (JSONCounterPayload payload in counterTrace.events) + { + Assert.Equal("myProvider", payload.provider); + Assert.Equal("Incrementing Counter One / 1 sec", payload.name); + Assert.Equal("Rate", payload.counterType); + Assert.Equal(1.0, payload.value); + } + } + } + + [Fact] + public void CounterTest() + { + string fileName = "CounterTest.json"; + JSONExporter exporter = new JSONExporter(fileName, "myProcess.exe"); + exporter.Initialize(); + for (int i = 0; i < 10; i++) + { + exporter.CounterPayloadReceived("myProvider", TestHelpers.GenerateCounterPayload(false, "counterOne", 1.0, 1, "Counter One"), false); + } + exporter.Stop(); + + Assert.True(File.Exists(fileName)); + using (StreamReader r = new StreamReader(fileName)) + { + string json = r.ReadToEnd(); + JSONCounterTrace counterTrace = JsonConvert.DeserializeObject(json); + + Assert.Equal("myProcess.exe", counterTrace.targetProcess); + Assert.Equal(10, counterTrace.events.Length); + foreach (JSONCounterPayload payload in counterTrace.events) + { + Assert.Equal("myProvider", payload.provider); + Assert.Equal("Counter One", payload.name); + Assert.Equal("Metric", payload.counterType); + Assert.Equal(1.0, payload.value); + } + } + } + } + + class JSONCounterPayload + { + [JsonProperty("timestamp")] + public string timestamp { get; set; } + + [JsonProperty("provider")] + public string provider { get; set; } + + [JsonProperty("name")] + public string name { get; set; } + + [JsonProperty("counter type")] + public string counterType { get; set; } + + [JsonProperty("value")] + public double value { get; set; } + } + + class JSONCounterTrace + { + [JsonProperty("Target Process")] + public string targetProcess { get; set; } + + [JsonProperty("Start Time")] + public string startTime { get; set; } + + [JsonProperty("Events")] + public JSONCounterPayload[] events { get; set; } + } +} diff --git a/src/tests/dotnet-counters/KnownProviderTests.cs b/src/tests/dotnet-counters/KnownProviderTests.cs new file mode 100644 index 000000000..eda0a143a --- /dev/null +++ b/src/tests/dotnet-counters/KnownProviderTests.cs @@ -0,0 +1,48 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using Xunit; +using Microsoft.Diagnostics.Tools.Counters; + +namespace Microsoft.Diagnostics.Tools.Counters +{ + /// + /// These test the some of the known providers that we provide as a default configuration for customers to use. + /// + public class KnownProviderTests + { + [Fact] + public void TestRuntimeProvider() + { + KnownData.TryGetProvider("System.Runtime", out CounterProvider runtimeProvider); + + Assert.Equal("System.Runtime", runtimeProvider.Name); + Assert.Equal("0xffffffff", runtimeProvider.Keywords); + Assert.Equal("5", runtimeProvider.Level); + Assert.Equal("System.Runtime:0xffffffff:5:EventCounterIntervalSec=1", runtimeProvider.ToProviderString(1)); + } + + [Fact] + public void TestASPNETProvider() + { + KnownData.TryGetProvider("Microsoft.AspNetCore.Hosting", out CounterProvider aspnetProvider); + + Assert.Equal("Microsoft.AspNetCore.Hosting", aspnetProvider.Name); + Assert.Equal("0x0", aspnetProvider.Keywords); + Assert.Equal("4", aspnetProvider.Level); + Assert.Equal("Microsoft.AspNetCore.Hosting:0x0:4:EventCounterIntervalSec=5", aspnetProvider.ToProviderString(5)); + } + + [Fact] + public void UnknownProvider() + { + KnownData.TryGetProvider("SomeRandomProvider", out CounterProvider randomProvider); + + Assert.Null(randomProvider); + } + + // TODO: Add more as we add more providers as known providers to the tool... + } +} diff --git a/src/tests/dotnet-counters/TestHelpers.cs b/src/tests/dotnet-counters/TestHelpers.cs new file mode 100644 index 000000000..c1d1a7cc9 --- /dev/null +++ b/src/tests/dotnet-counters/TestHelpers.cs @@ -0,0 +1,47 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Collections.Generic; +using System.Text; + +using Microsoft.Diagnostics.Tools.Counters; + +namespace DotnetCounters.UnitTests +{ + class TestHelpers + { + public static ICounterPayload GenerateCounterPayload( + bool isIncrementingCounter, + string counterName, + double counterValue, + int displayRateTimeScaleSeconds = 0, + string displayName = "") + { + if (isIncrementingCounter) + { + Dictionary payloadFields = new Dictionary() + { + { "Name", counterName }, + { "Increment", counterValue }, + { "DisplayName", displayName }, + { "DisplayRateTimeScale", displayRateTimeScaleSeconds == 0 ? "" : TimeSpan.FromSeconds(displayRateTimeScaleSeconds).ToString() }, + }; + ICounterPayload payload = new IncrementingCounterPayload(payloadFields, 1); + return payload; + } + else + { + Dictionary payloadFields = new Dictionary() + { + { "Name", counterName }, + { "Mean", counterValue }, + { "DisplayName", displayName }, + }; + ICounterPayload payload = new CounterPayload(payloadFields); + return payload; + } + } + } +}