add eventpipe test harness (#618)
authorJohn Salem <josalem@microsoft.com>
Tue, 19 Nov 2019 23:10:29 +0000 (15:10 -0800)
committerGitHub <noreply@github.com>
Tue, 19 Nov 2019 23:10:29 +0000 (15:10 -0800)
* Add eventpipe test harness to diagnostics repo

.gitignore
diagnostics.sln
eng/Version.Details.xml
eng/Versions.props
eng/build.yml
src/tests/eventpipe/EventPipe.UnitTests.csproj [new file with mode: 0644]
src/tests/eventpipe/README.md [new file with mode: 0644]
src/tests/eventpipe/common/IpcTraceTest.cs [new file with mode: 0644]
src/tests/eventpipe/common/RemoteTestExecutorHelper.cs [new file with mode: 0644]
src/tests/eventpipe/common/StreamProxy.cs [new file with mode: 0644]
src/tests/eventpipe/providers.cs [new file with mode: 0644]

index 32d9b12f2bcd89f1966a7ca49691fd40ed59f693..29f04c6a159bf1bad716c5637c5c879db7209b7b 100644 (file)
@@ -121,3 +121,5 @@ StressLog.txt
 *.nettrace
 *.speedscope.json
 *.csv
+*.gcdump
+*.binlog
index 31d100231969b01fb8a43e005e8e7f6003271ca4..6796cda20753ff4f6e916edac704d7ecf3a5d9f5 100644 (file)
@@ -55,6 +55,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "dotnet-gcdump", "src\Tools\
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "DotnetCounters.UnitTests", "src\tests\dotnet-counters\DotnetCounters.UnitTests.csproj", "{E5A7DC6C-BF8D-418A-BCBD-094EB748FA82}"
 EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "EventPipe.UnitTests", "src\tests\eventpipe\EventPipe.UnitTests.csproj", "{CED9ABBA-861E-4C0A-9359-22351208EF27}"
+EndProject
 Global
        GlobalSection(SolutionConfigurationPlatforms) = preSolution
                Checked|Any CPU = Checked|Any CPU
@@ -888,6 +890,46 @@ Global
                {E5A7DC6C-BF8D-418A-BCBD-094EB748FA82}.RelWithDebInfo|x64.Build.0 = Release|Any CPU
                {E5A7DC6C-BF8D-418A-BCBD-094EB748FA82}.RelWithDebInfo|x86.ActiveCfg = Release|Any CPU
                {E5A7DC6C-BF8D-418A-BCBD-094EB748FA82}.RelWithDebInfo|x86.Build.0 = Release|Any CPU
+               {CED9ABBA-861E-4C0A-9359-22351208EF27}.Checked|Any CPU.ActiveCfg = Debug|Any CPU
+               {CED9ABBA-861E-4C0A-9359-22351208EF27}.Checked|Any CPU.Build.0 = Debug|Any CPU
+               {CED9ABBA-861E-4C0A-9359-22351208EF27}.Checked|ARM.ActiveCfg = Debug|Any CPU
+               {CED9ABBA-861E-4C0A-9359-22351208EF27}.Checked|ARM.Build.0 = Debug|Any CPU
+               {CED9ABBA-861E-4C0A-9359-22351208EF27}.Checked|ARM64.ActiveCfg = Debug|Any CPU
+               {CED9ABBA-861E-4C0A-9359-22351208EF27}.Checked|ARM64.Build.0 = Debug|Any CPU
+               {CED9ABBA-861E-4C0A-9359-22351208EF27}.Checked|x64.ActiveCfg = Debug|Any CPU
+               {CED9ABBA-861E-4C0A-9359-22351208EF27}.Checked|x64.Build.0 = Debug|Any CPU
+               {CED9ABBA-861E-4C0A-9359-22351208EF27}.Checked|x86.ActiveCfg = Debug|Any CPU
+               {CED9ABBA-861E-4C0A-9359-22351208EF27}.Checked|x86.Build.0 = Debug|Any CPU
+               {CED9ABBA-861E-4C0A-9359-22351208EF27}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+               {CED9ABBA-861E-4C0A-9359-22351208EF27}.Debug|Any CPU.Build.0 = Debug|Any CPU
+               {CED9ABBA-861E-4C0A-9359-22351208EF27}.Debug|ARM.ActiveCfg = Debug|Any CPU
+               {CED9ABBA-861E-4C0A-9359-22351208EF27}.Debug|ARM.Build.0 = Debug|Any CPU
+               {CED9ABBA-861E-4C0A-9359-22351208EF27}.Debug|ARM64.ActiveCfg = Debug|Any CPU
+               {CED9ABBA-861E-4C0A-9359-22351208EF27}.Debug|ARM64.Build.0 = Debug|Any CPU
+               {CED9ABBA-861E-4C0A-9359-22351208EF27}.Debug|x64.ActiveCfg = Debug|Any CPU
+               {CED9ABBA-861E-4C0A-9359-22351208EF27}.Debug|x64.Build.0 = Debug|Any CPU
+               {CED9ABBA-861E-4C0A-9359-22351208EF27}.Debug|x86.ActiveCfg = Debug|Any CPU
+               {CED9ABBA-861E-4C0A-9359-22351208EF27}.Debug|x86.Build.0 = Debug|Any CPU
+               {CED9ABBA-861E-4C0A-9359-22351208EF27}.Release|Any CPU.ActiveCfg = Release|Any CPU
+               {CED9ABBA-861E-4C0A-9359-22351208EF27}.Release|Any CPU.Build.0 = Release|Any CPU
+               {CED9ABBA-861E-4C0A-9359-22351208EF27}.Release|ARM.ActiveCfg = Release|Any CPU
+               {CED9ABBA-861E-4C0A-9359-22351208EF27}.Release|ARM.Build.0 = Release|Any CPU
+               {CED9ABBA-861E-4C0A-9359-22351208EF27}.Release|ARM64.ActiveCfg = Release|Any CPU
+               {CED9ABBA-861E-4C0A-9359-22351208EF27}.Release|ARM64.Build.0 = Release|Any CPU
+               {CED9ABBA-861E-4C0A-9359-22351208EF27}.Release|x64.ActiveCfg = Release|Any CPU
+               {CED9ABBA-861E-4C0A-9359-22351208EF27}.Release|x64.Build.0 = Release|Any CPU
+               {CED9ABBA-861E-4C0A-9359-22351208EF27}.Release|x86.ActiveCfg = Release|Any CPU
+               {CED9ABBA-861E-4C0A-9359-22351208EF27}.Release|x86.Build.0 = Release|Any CPU
+               {CED9ABBA-861E-4C0A-9359-22351208EF27}.RelWithDebInfo|Any CPU.ActiveCfg = Release|Any CPU
+               {CED9ABBA-861E-4C0A-9359-22351208EF27}.RelWithDebInfo|Any CPU.Build.0 = Release|Any CPU
+               {CED9ABBA-861E-4C0A-9359-22351208EF27}.RelWithDebInfo|ARM.ActiveCfg = Release|Any CPU
+               {CED9ABBA-861E-4C0A-9359-22351208EF27}.RelWithDebInfo|ARM.Build.0 = Release|Any CPU
+               {CED9ABBA-861E-4C0A-9359-22351208EF27}.RelWithDebInfo|ARM64.ActiveCfg = Release|Any CPU
+               {CED9ABBA-861E-4C0A-9359-22351208EF27}.RelWithDebInfo|ARM64.Build.0 = Release|Any CPU
+               {CED9ABBA-861E-4C0A-9359-22351208EF27}.RelWithDebInfo|x64.ActiveCfg = Release|Any CPU
+               {CED9ABBA-861E-4C0A-9359-22351208EF27}.RelWithDebInfo|x64.Build.0 = Release|Any CPU
+               {CED9ABBA-861E-4C0A-9359-22351208EF27}.RelWithDebInfo|x86.ActiveCfg = Release|Any CPU
+               {CED9ABBA-861E-4C0A-9359-22351208EF27}.RelWithDebInfo|x86.Build.0 = Release|Any CPU
        EndGlobalSection
        GlobalSection(SolutionProperties) = preSolution
                HideSolutionNode = FALSE
@@ -918,6 +960,7 @@ Global
                {AEDCCF5B-5AD0-4D64-BF73-5CF468E07D22} = {03479E19-3F18-49A6-910A-F5041E27E7C0}
                {936678B3-3392-4F4F-943C-B6A4BFCBAADC} = {B62728C8-1267-4043-B46F-5537BBAEC692}
                {E5A7DC6C-BF8D-418A-BCBD-094EB748FA82} = {03479E19-3F18-49A6-910A-F5041E27E7C0}
+               {CED9ABBA-861E-4C0A-9359-22351208EF27} = {03479E19-3F18-49A6-910A-F5041E27E7C0}
        EndGlobalSection
        GlobalSection(ExtensibilityGlobals) = postSolution
                SolutionGuid = {46465737-C938-44FC-BE1A-4CE139EBB5E0}
index fafc1cfdaa889f03482952ba703f36276b1c7dad..5e52a57a9a1944d14419de6f84b5626250ed1152 100644 (file)
@@ -4,6 +4,10 @@
         <Uri>https://github.com/dotnet/command-line-api</Uri>
         <Sha>166610c56ff732093f0145a2911d4f6c40b786da</Sha>
     </Dependency>
+    <Dependency Name="Microsoft.DotNet.RemoteExecutor" Version=">5.0.0-beta.19562.5">
+        <Uri>https://github.com/dotnet/arcade</Uri>
+        <Sha>993af9410c505680b9a260f3bfd79515c936de12</Sha>
+    </Dependency>
   </ProductDependencies>
   <ToolsetDependencies>
     <Dependency Name="Microsoft.DotNet.Arcade.Sdk" Version="1.0.0-beta.19358.1">
index a6a55b8b8dce2a8037cd894763cc22addc6c3fd4..fb7c062b45f2e8b2cdc286f8e7d3bf1bed8b1e9c 100644 (file)
@@ -34,6 +34,8 @@
     <XUnitVersion>2.4.1</XUnitVersion>
     <XUnitAbstractionsVersion>2.0.3</XUnitAbstractionsVersion>
 
+    <MicrosoftDotNetRemoteExecutorVersion>5.0.0-beta.19562.5</MicrosoftDotNetRemoteExecutorVersion>
+
     <cdbsosversion>10.0.18362</cdbsosversion>
 
   </PropertyGroup>
index 4b0ccb13f60ec7e62a353aea28bcfd0a38ae1e92..f7924466b72a66003435520dbdf384206e3e7958 100644 (file)
@@ -84,6 +84,7 @@ jobs:
     - _PhaseName : ${{ parameters.name }}
     - _HelixType: build/product
     - _HelixBuildConfig: $(_BuildConfig)
+    - _Pipeline_StreamDumpDir: $(Build.SourcesDirectory)/artifacts/tmp/$(_BuildConfig)/streams
 
     # Only enable publishing in non-public, non PR scenarios.
     - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
@@ -176,6 +177,15 @@ jobs:
       continueOnError: true
       condition: failed()
 
+    - task: PublishBuildArtifacts@1
+      displayName: Publish Stream Artifacts on failure
+      inputs:
+        PathtoPublish: $(_Pipeline_StreamDumpDir)
+        PublishLocation: Container
+        ArtifactName: Streams_$(_PhaseName)_$(_BuildArch)_$(_BuildConfig)
+      continueOnError: true
+      condition: failed()
+
     - task: CopyFiles@2
       displayName: Gather Logs
       inputs:
diff --git a/src/tests/eventpipe/EventPipe.UnitTests.csproj b/src/tests/eventpipe/EventPipe.UnitTests.csproj
new file mode 100644 (file)
index 0000000..1f98507
--- /dev/null
@@ -0,0 +1,14 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <TargetFramework>netcoreapp3.0</TargetFramework>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <PackageReference Include="Microsoft.DotNet.RemoteExecutor" Version="$(MicrosoftDotNetRemoteExecutorVersion)" />
+    <PackageReference Include="Microsoft.Diagnostics.Tracing.TraceEvent" Version="$(MicrosoftDiagnosticsTracingTraceEventVersion)" />
+    <PackageReference Include="xunit.abstractions" Version="$(XUnitAbstractionsVersion)" />
+    <ProjectReference Include="$(MSBuildThisFileDirectory)..\..\Microsoft.Diagnostics.Tools.RuntimeClient\Microsoft.Diagnostics.Tools.RuntimeClient.csproj" />
+  </ItemGroup>
+
+</Project>
diff --git a/src/tests/eventpipe/README.md b/src/tests/eventpipe/README.md
new file mode 100644 (file)
index 0000000..335d4da
--- /dev/null
@@ -0,0 +1,16 @@
+# EventPipe testing in dotnet/diagnostics
+
+This directory contains the dotnet/diagnostics end of testing for EventPipe and related infrastructure. Not all aspects of EventPipe are tested here. The table below indicates where specific parts of the feature are being tested, including the tests in this directory.
+
+| completed | functionality    | location |
+| --------- | ---------------- | -------- |
+| ✅        | IPC protocol     | dotnet/coreclr |
+| ✅        | EventPipe Provider Enable/Disable | dotnet/coreclr |
+| ✅        | EventPipe Event-Provider coherence | dotnet/coreclr |
+| ✅        | `dotnet trace` provider parsing | dotnet/diagnostics |
+| ✅        | `dotnet trace` provider-profile merging | dotnet/diagnostics |
+
+The tests here are meant to cover the diagnostic scenarios rather than correctness of the feature. They will transitively test that, but the main focus is whether typical scenarios work _using_ the technology.  In short, these test should answer the following questions:
+* Does EventPipeEventSource + IPC Protocol + EventPipe collect the events we expect to see?
+* Do typical diagnostic pairings of Issue + "Event to diagnose the issue" work via EventPipe?
+  * e.g., if my app is starving for threads, can I turn on Thread events and successfully collect them.
\ No newline at end of file
diff --git a/src/tests/eventpipe/common/IpcTraceTest.cs b/src/tests/eventpipe/common/IpcTraceTest.cs
new file mode 100644 (file)
index 0000000..6d5b908
--- /dev/null
@@ -0,0 +1,424 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using System;
+using System.Diagnostics;
+using System.Diagnostics.Tracing;
+using System.IO;
+using System.Threading;
+using System.Threading.Tasks;
+using System.Collections.Generic;
+using Microsoft.Diagnostics.Tracing;
+using Microsoft.Diagnostics.Tools.RuntimeClient;
+using System.Runtime.InteropServices;
+using System.Linq;
+using System.Text.RegularExpressions;
+
+namespace EventPipe.UnitTests.Common
+{
+    public class Logger
+    {
+        public static Logger logger = new Logger();
+        private TextWriter _log;
+        private Stopwatch _sw;
+        public Logger(TextWriter log = null)
+        {
+            _log = log ?? Console.Out;
+            _sw = new Stopwatch();
+        }
+
+        public void Log(string message)
+        {
+            if (!_sw.IsRunning)
+                _sw.Start();
+            _log.WriteLine($"{_sw.Elapsed.TotalSeconds,5:f1}s: {message}");
+        }
+    }
+
+    public class ExpectedEventCount
+    {
+        // The acceptable percent error on the expected value
+        // represented as a floating point value in [0,1].
+        public float Error { get; private set; }
+
+        // The expected count of events. A value of -1 indicates
+        // that count does not matter, and we are simply testing
+        // that the provider exists in the trace.
+        public int Count { get; private set; }
+
+        public ExpectedEventCount(int count, float error = 0.0f)
+        {
+            Count = count;
+            Error = error;
+        }
+
+        public bool Validate(int actualValue)
+        {
+            return Count == -1 || CheckErrorBounds(actualValue);
+        }
+
+        public bool CheckErrorBounds(int actualValue)
+        {
+            return Math.Abs(actualValue - Count) <= (Count * Error);
+        }
+
+        public static implicit operator ExpectedEventCount(int i)
+        {
+            return new ExpectedEventCount(i);
+        }
+
+        public override string ToString()
+        {
+            return $"{Count} +- {Count * Error}";
+        }
+    }
+
+    // This event source is used by the test infra to
+    // to insure that providers have finished being enabled
+    // for the session being observed. Since the client API
+    // returns the pipe for reading _before_ it finishes
+    // enabling the providers to write to that session,
+    // we need to guarantee that our providers are on before
+    // sending events. This is a _unique_ problem I imagine
+    // should _only_ affect scenarios like these tests
+    // where the reading and sending of events are required
+    // to synchronize.
+    public sealed class SentinelEventSource : EventSource
+    {
+        private SentinelEventSource() {}
+        public static SentinelEventSource Log = new SentinelEventSource();
+        public void SentinelEvent() { WriteEvent(1, "SentinelEvent"); }
+    }
+
+    public static class SessionConfigurationExtensions
+    {
+        public static SessionConfiguration InjectSentinel(this SessionConfiguration sessionConfiguration)
+        {
+            var newProviderList = new List<Provider>(sessionConfiguration.Providers);
+            newProviderList.Add(new Provider("SentinelEventSource"));
+            return new SessionConfiguration(sessionConfiguration.CircularBufferSizeInMB, sessionConfiguration.Format, newProviderList.AsReadOnly());
+        }
+    }
+
+    public class IpcTraceTest
+    {
+        // This Action is executed while the trace is being collected.
+        private Action _eventGeneratingAction;
+
+        // A dictionary of event providers to number of events.
+        // A count of -1 indicates that you are only testing for the presence of the provider
+        // and don't care about the number of events sent
+        private Dictionary<string, ExpectedEventCount> _expectedEventCounts;
+        private Dictionary<string, int> _actualEventCounts = new Dictionary<string, int>();
+        private int _droppedEvents = 0;
+        private SessionConfiguration _sessionConfiguration;
+
+        // A function to be called with the EventPipeEventSource _before_
+        // the call to `source.Process()`.  The function should return another
+        // function that will be called to check whether the optional test was validated.
+        // Example in situ: providervalidation.cs
+        private Func<EventPipeEventSource, Func<int>> _optionalTraceValidator;
+
+        IpcTraceTest(
+            Dictionary<string, ExpectedEventCount> expectedEventCounts,
+            Action eventGeneratingAction,
+            SessionConfiguration sessionConfiguration = null,
+            Func<EventPipeEventSource, Func<int>> optionalTraceValidator = null)
+        {
+            _eventGeneratingAction = eventGeneratingAction;
+            _expectedEventCounts = expectedEventCounts;
+            _sessionConfiguration = sessionConfiguration?.InjectSentinel() ?? new SessionConfiguration(
+                circularBufferSizeMB: 1000,
+                format: EventPipeSerializationFormat.NetTrace,
+                providers: new List<Provider> { 
+                    new Provider("Microsoft-Windows-DotNETRuntime"),
+                    new Provider("SentinelEventSource")
+                });
+            _optionalTraceValidator = optionalTraceValidator;
+        }
+
+        private int Fail(string message = "")
+        {
+            Logger.logger.Log("Test FAILED!");
+            Logger.logger.Log(message);
+            Logger.logger.Log("Configuration:");
+            Logger.logger.Log("{");
+            Logger.logger.Log($"\tbufferSize: {_sessionConfiguration.CircularBufferSizeInMB},");
+            Logger.logger.Log("\tproviders: [");
+            foreach (var provider in _sessionConfiguration.Providers)
+            {
+                Logger.logger.Log($"\t\t{provider.ToString()},");
+            }
+            Logger.logger.Log("\t]");
+            Logger.logger.Log("}\n");
+            Logger.logger.Log("Expected:");
+            Logger.logger.Log("{");
+            foreach (var (k, v) in _expectedEventCounts)
+            {
+                Logger.logger.Log($"\t\"{k}\" = {v}");
+            }
+            Logger.logger.Log("}\n");
+
+            Logger.logger.Log("Actual:");
+            Logger.logger.Log("{");
+            foreach (var (k, v) in _actualEventCounts)
+            {
+                Logger.logger.Log($"\t\"{k}\" = {v}");
+            }
+            Logger.logger.Log("}");
+
+            return -1;
+        }
+
+        private int Validate()
+        {
+            var isClean = EnsureCleanEnvironment();
+            if (!isClean)
+                return -1;
+            // CollectTracing returns before EventPipe::Enable has returned, so the
+            // the sources we want to listen for may not have been enabled yet.
+            // We'll use this sentinel EventSource to check if Enable has finished
+            ManualResetEvent sentinelEventReceived = new ManualResetEvent(false);
+            var sentinelTask = new Task(() =>
+            {
+                Logger.logger.Log("Started sending sentinel events...");
+                while (!sentinelEventReceived.WaitOne(50))
+                {
+                    SentinelEventSource.Log.SentinelEvent();
+                }
+                Logger.logger.Log("Stopped sending sentinel events");
+            });
+            sentinelTask.Start();
+
+            int processId = Process.GetCurrentProcess().Id;
+            object threadSync = new object(); // for locking eventpipeSessionId access
+            ulong eventpipeSessionId = 0;
+            Func<int> optionalTraceValidationCallback = null;
+            var readerTask = new Task(() =>
+            {
+                Logger.logger.Log("Connecting to EventPipe...");
+                using var eventPipeStream = new StreamProxy(EventPipeClient.CollectTracing(processId, _sessionConfiguration, out var sessionId));
+                if (sessionId == 0)
+                {
+                    Logger.logger.Log("Failed to connect to EventPipe!");
+                    throw new ApplicationException("Failed to connect to EventPipe");
+                }
+                Logger.logger.Log($"Connected to EventPipe with sessionID '0x{sessionId:x}'");
+
+                lock (threadSync)
+                {
+                    eventpipeSessionId = sessionId;
+                }
+
+                Logger.logger.Log("Creating EventPipeEventSource...");
+                using EventPipeEventSource source = new EventPipeEventSource(eventPipeStream);
+                Logger.logger.Log("EventPipeEventSource created");
+
+                source.Dynamic.All += (eventData) =>
+                {
+                    try
+                    {
+                        if (eventData.ProviderName == "SentinelEventSource")
+                        {
+                            if (!sentinelEventReceived.WaitOne(0))
+                                Logger.logger.Log("Saw sentinel event");
+                            sentinelEventReceived.Set();
+                        }
+
+                        else if (_actualEventCounts.TryGetValue(eventData.ProviderName, out _))
+                        {
+                            _actualEventCounts[eventData.ProviderName]++;
+                        }
+                        else
+                        {
+                            Logger.logger.Log($"Saw new provider '{eventData.ProviderName}'");
+                            _actualEventCounts[eventData.ProviderName] = 1;
+                        }
+                    }
+                    catch (Exception e)
+                    {
+                        Logger.logger.Log("Exception in Dynamic.All callback " + e.ToString());
+                    }
+                };
+                Logger.logger.Log("Dynamic.All callback registered");
+
+                if (_optionalTraceValidator != null)
+                {
+                    Logger.logger.Log("Running optional trace validator");
+                    optionalTraceValidationCallback = _optionalTraceValidator(source);
+                    Logger.logger.Log("Finished running optional trace validator");
+                }
+
+                Logger.logger.Log("Starting stream processing...");
+                try
+                {
+                    source.Process();
+                    _droppedEvents = source.EventsLost;
+                }
+                catch (Exception e)
+                {
+                    Logger.logger.Log($"Exception thrown while reading; dumping culprit stream to disk...");
+                    eventPipeStream.DumpStreamToDisk();
+                    // rethrow it to fail the test
+                    throw e;
+                }
+                Logger.logger.Log("Stopping stream processing");
+                Logger.logger.Log($"Dropped {source.EventsLost} events");
+            });
+
+            readerTask.Start();
+            sentinelEventReceived.WaitOne();
+
+            Logger.logger.Log("Starting event generating action...");
+            _eventGeneratingAction();
+            Logger.logger.Log("Stopping event generating action");
+
+            // Should throw if the reader task throws any exceptions
+            var tokenSource = new CancellationTokenSource();
+            CancellationToken ct = tokenSource.Token;
+            readerTask.ContinueWith((task) =>
+            {
+                // if our reader task died earlier, we need to break the infinite wait below.
+                // We'll allow the AggregateException to be thrown and fail the test though.
+                Logger.logger.Log($"Task stats: isFaulted: {task.IsFaulted}, Exception == null: {task.Exception == null}");
+                if (task.IsFaulted || task.Exception != null)
+                {
+                    tokenSource.Cancel();
+                }
+
+                return task;
+            });
+
+            var stopTask = Task.Run(() => 
+            {
+                Logger.logger.Log("Sending StopTracing command...");
+                lock (threadSync) // eventpipeSessionId
+                {
+                    EventPipeClient.StopTracing(processId, eventpipeSessionId);
+                }
+                Logger.logger.Log("Finished StopTracing command");
+            }, ct);
+
+            try
+            {
+                Task.WaitAll(new Task[] { readerTask, stopTask }, ct);
+            }
+            catch (OperationCanceledException)
+            {
+                Logger.logger.Log($"A task faulted");
+                Logger.logger.Log($"\treaderTask.IsFaulted = {readerTask.IsFaulted}");
+                if (readerTask.Exception != null)
+                {
+                    throw readerTask.Exception;
+                }
+                return -1;
+            }
+
+            Logger.logger.Log("Reader task finished");
+            Logger.logger.Log($"Dropped {_droppedEvents} events");
+
+            foreach (var (provider, expectedCount) in _expectedEventCounts)
+            {
+                if (_actualEventCounts.TryGetValue(provider, out var actualCount))
+                {
+                    if (!expectedCount.Validate(actualCount))
+                    {
+                        return Fail($"Event count mismatch for provider \"{provider}\": expected {expectedCount}, but saw {actualCount}");
+                    }
+                }
+                else
+                {
+                    return Fail($"No events for provider \"{provider}\"");
+                }
+            }
+
+            if (optionalTraceValidationCallback != null)
+            {
+                Logger.logger.Log("Validating optional callback...");
+                // reader thread should be dead now, no need to lock
+                return optionalTraceValidationCallback();
+            }
+            else
+            {
+                return 100;
+            }
+        }
+
+        // Ensure that we have a clean environment for running the test.
+        // Specifically check that we don't have more than one match for 
+        // Diagnostic IPC sockets in the TempPath.  These can be left behind
+        // by bugs, catastrophic test failures, etc. from previous testing.
+        // The tmp directory is only cleared on reboot, so it is possible to
+        // run into these zombie pipes if there are failures over time.
+        // Note: Windows has some guarantees about named pipes not living longer
+        // the process that created them, so we don't need to check on that platform.
+        private bool EnsureCleanEnvironment()
+        {
+            if (!RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
+            {
+                Logger.logger.Log("Validating clean environment...");
+                // mimic the RuntimeClient's code for finding OS Transports
+                IEnumerable<FileInfo> ipcPorts = Directory.GetFiles(Path.GetTempPath())
+                    .Select(namedPipe => new FileInfo(namedPipe))
+                    .Where(input => Regex.IsMatch(input.Name, $"^dotnet-diagnostic-{System.Diagnostics.Process.GetCurrentProcess().Id}-(\\d+)-socket$"));
+                
+                if (ipcPorts.Count() > 1)
+                {
+                    Logger.logger.Log($"Found {ipcPorts.Count()} OS transports for pid {System.Diagnostics.Process.GetCurrentProcess().Id}:");
+                    foreach (var match in ipcPorts)
+                    {
+                        Logger.logger.Log($"\t{match.Name}");
+                    }
+
+                    // Get everything _except_ the newest pipe
+                    var duplicates = ipcPorts.OrderBy(fileInfo => fileInfo.CreationTime.Ticks).SkipLast(1);
+                    foreach (var duplicate in duplicates)
+                    {
+                        Logger.logger.Log($"Attempting to delete the oldest pipe: {duplicate.FullName}");
+                        duplicate.Delete(); // should throw if we can't delete and be caught in Validate
+                        Logger.logger.Log($"Deleted");
+                    }
+
+                    var afterIpcPorts = Directory.GetFiles(Path.GetTempPath())
+                        .Select(namedPipe => new FileInfo(namedPipe))
+                        .Where(input => Regex.IsMatch(input.Name, $"^dotnet-diagnostic-{System.Diagnostics.Process.GetCurrentProcess().Id}-(\\d+)-socket$"));
+
+                    if (afterIpcPorts.Count() == 1)
+                    {
+                        return true;
+                    }
+                    else
+                    {
+                        Logger.logger.Log($"Unable to clean the environment.  The following transports are on the system:");
+                        foreach(var transport in afterIpcPorts)
+                        {
+                            Logger.logger.Log($"\t{transport.FullName}");
+                        }
+                        return false;
+                    }
+                }
+                Logger.logger.Log("Environment was clean.");
+                return true;
+            }
+
+            return true;
+        }
+
+        public static int RunAndValidateEventCounts(
+            Dictionary<string, ExpectedEventCount> expectedEventCounts,
+            Action eventGeneratingAction,
+            SessionConfiguration sessionConfiguration = null,
+            Func<EventPipeEventSource, Func<int>> optionalTraceValidator = null)
+        {
+            Logger.logger.Log("==TEST STARTING==");
+            var test = new IpcTraceTest(expectedEventCounts, eventGeneratingAction, sessionConfiguration, optionalTraceValidator);
+            var ret = test.Validate();
+            if (ret == 100)
+                Logger.logger.Log("==TEST FINISHED: PASSED!==");
+            else
+                Logger.logger.Log("==TEST FINISHED: FAILED!==");
+            return ret;
+        }
+    }
+}
\ No newline at end of file
diff --git a/src/tests/eventpipe/common/RemoteTestExecutorHelper.cs b/src/tests/eventpipe/common/RemoteTestExecutorHelper.cs
new file mode 100644 (file)
index 0000000..2558978
--- /dev/null
@@ -0,0 +1,38 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using Microsoft.DotNet.RemoteExecutor;
+using System;
+using System.Diagnostics;
+using System.Threading.Tasks;
+using Xunit.Abstractions;
+
+namespace EventPipe.UnitTests.Common
+{
+    public static class RemoteTestExecutorHelper
+    {
+        public static async Task RunTestCaseAsync(Action testCase, ITestOutputHelper output)
+        {
+            var options = new RemoteInvokeOptions()
+            {
+                StartInfo = new ProcessStartInfo() { RedirectStandardOutput = true, RedirectStandardError = true }
+            };
+
+            using RemoteInvokeHandle remoteInvokeHandle = RemoteExecutor.Invoke(testCase, options);
+
+            try
+            {
+                Task<string> stdOutputTask = remoteInvokeHandle.Process.StandardOutput.ReadToEndAsync();
+                Task<string> stdErrorTask = remoteInvokeHandle.Process.StandardError.ReadToEndAsync();
+                await Task.WhenAll(stdErrorTask, stdOutputTask);
+                output.WriteLine(stdOutputTask.Result);
+                Console.Error.Write(stdErrorTask.Result);
+            }
+            catch (ObjectDisposedException)
+            {
+                Console.Error.WriteLine("Failed to collect remote process's output");
+            }
+        }
+    }
+}
diff --git a/src/tests/eventpipe/common/StreamProxy.cs b/src/tests/eventpipe/common/StreamProxy.cs
new file mode 100644 (file)
index 0000000..f0f4677
--- /dev/null
@@ -0,0 +1,111 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using System;
+using System.Buffers;
+using System.IO;
+
+namespace EventPipe.UnitTests.Common
+{
+    // This Stream implementation takes one stream
+    // and proxies the Stream API to it while
+    // saving any read bytes to an internal stream.
+    // Should an error occur, the internal stream
+    // is dumped to disk for reproducing the error.
+    public class StreamProxy : Stream
+    {
+        private Stream ProxiedStream { get; }
+        private MemoryStream InternalStream => new MemoryStream();
+        public override bool CanRead => ProxiedStream.CanRead;
+
+        public override bool CanSeek => ProxiedStream.CanSeek;
+
+        public override bool CanWrite => ProxiedStream.CanWrite;
+
+        public override long Length => ProxiedStream.Length;
+
+        public override long Position { get => ProxiedStream.Position; set => ProxiedStream.Position = value; }
+
+        public StreamProxy(Stream streamToProxy)
+        {
+            ProxiedStream = streamToProxy;
+        }
+
+        public override void Flush() => ProxiedStream.Flush();
+
+        // Read the actual desired amount of bytes into an empty buffer
+        // copy those bytes into an internal MemoryStream THEN
+        // forward those bytes to the caller. If the caller
+        // would have thrown an exception with its Read, copy
+        // the bytes to the internal MemoryStream and then throw
+        public override int Read(byte[] buffer, int offset, int count)
+        {
+            if (buffer == null || offset < 0 || count < 0)
+                throw new ArgumentException("Invalid input into Read");
+
+            byte[] localBuffer = ArrayPool<byte>.Shared.Rent(count);
+            var readCount = ProxiedStream.Read(localBuffer, 0, count);
+            if (readCount == 0)
+                return readCount;
+
+            InternalStream.Write(localBuffer, 0, readCount);
+
+            if (buffer.Length - offset < count)
+            {
+                // This is the error that EventPipeEventSource is causing,
+                // so this is when we should throw an exception, just like
+                // System.IO.PipeStream. This will result in the dispose method
+                // being called and the culprit stream data being dumped to disk
+                Logger.logger.Log($"[Error] Attempted to read {count} bytes into a buffer of length {buffer.Length} at offset {offset}");
+
+                // Throw the exception like what would have happened in System.IO.PipeStream
+                throw new ArgumentException($"Attempted to read {count} bytes into a buffer of length {buffer.Length} at offset {offset}");
+            }
+
+            // copy the data into the caller's buffer
+            Array.Copy(localBuffer, 0, buffer, offset, readCount);
+
+            ArrayPool<byte>.Shared.Return(localBuffer, true);
+            return readCount;
+        }
+
+        public override long Seek(long offset, SeekOrigin origin) => ProxiedStream.Seek(offset, origin);
+
+        public override void SetLength(long value) => ProxiedStream.SetLength(value);
+
+        public override void Write(byte[] buffer, int offset, int count)
+        {
+            // This stream is only for "reading" from. No need for this method.
+            throw new System.NotImplementedException();
+        }
+
+        protected bool disposed = false;
+        protected override void Dispose(bool disposing)
+        {
+            if (!disposed)
+            {
+                if (disposing)
+                {
+                    ProxiedStream.Dispose();
+                    InternalStream.Dispose();
+                }
+
+                disposed = true;
+            }
+        }
+
+        public void DumpStreamToDisk()
+        {
+            var streamDumpDir = System.Environment.GetEnvironmentVariable("_PIPELINE_STREAMDUMPDIR") ?? Path.GetTempPath();
+            Logger.logger.Log($"\t streamDumpDir = {streamDumpDir}");
+            var filePath = Path.Combine(streamDumpDir, Path.GetRandomFileName() + ".nettrace");
+            using (var streamDumpFile = File.Create(filePath))
+            {
+                Logger.logger.Log($"\t Writing stream for PID {System.Diagnostics.Process.GetCurrentProcess().Id} to {filePath}");
+                InternalStream.Seek(0, SeekOrigin.Begin);
+                InternalStream.CopyTo(streamDumpFile);
+            }
+        }
+    }
+}
\ No newline at end of file
diff --git a/src/tests/eventpipe/providers.cs b/src/tests/eventpipe/providers.cs
new file mode 100644 (file)
index 0000000..75a96ca
--- /dev/null
@@ -0,0 +1,69 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using System;
+using Xunit;
+using Xunit.Abstractions;
+using Microsoft.Diagnostics.Tools.RuntimeClient;
+using System.Collections.Generic;
+using System.Diagnostics.Tracing;
+using EventPipe.UnitTests.Common;
+
+// Use this test as an example of how to write tests for EventPipe in
+// the dotnet/diagnostics repo
+
+namespace EventPipe.UnitTests.ProviderValidation
+{
+    public sealed class MyEventSource : EventSource
+    {
+        private MyEventSource() {}
+        public static MyEventSource Log = new MyEventSource();
+        public void MyEvent() { WriteEvent(1, "MyEvent"); }
+    }
+
+    public class ProviderTests
+    {
+        private readonly ITestOutputHelper output;
+
+        public ProviderTests(ITestOutputHelper outputHelper)
+        {
+            output = outputHelper;
+        }
+
+        [Fact]
+        public async void UserDefinedEventSource_ProducesEvents()
+        {
+            await RemoteTestExecutorHelper.RunTestCaseAsync(() => 
+            {
+                Dictionary<string, ExpectedEventCount> expectedEventCounts = new Dictionary<string, ExpectedEventCount>()
+                {
+                    { "MyEventSource", new ExpectedEventCount(100_000, 0.30f) },
+                    { "Microsoft-Windows-DotNETRuntimeRundown", -1 },
+                    { "Microsoft-DotNETCore-SampleProfiler", -1 }
+                };
+
+                var providers = new List<Provider>()
+                {
+                    new Provider("MyEventSource"),
+                    new Provider("Microsoft-DotNETCore-SampleProfiler")
+                };
+
+                Action eventGeneratingAction = () => 
+                {
+                    for (int i = 0; i < 100_000; i++)
+                    {
+                        if (i % 10_000 == 0)
+                            Logger.logger.Log($"Fired MyEvent {i:N0}/100,000 times...");
+                        MyEventSource.Log.MyEvent();
+                    }
+                };
+
+                var config = new SessionConfiguration(circularBufferSizeMB: (uint)Math.Pow(2, 10), format: EventPipeSerializationFormat.NetTrace,  providers: providers);
+
+                var ret = IpcTraceTest.RunAndValidateEventCounts(expectedEventCounts, eventGeneratingAction, config);
+                Assert.Equal(100, ret);
+            }, output);
+        }
+    }
+}
\ No newline at end of file