1 // ***********************************************************************
2 // Copyright (c) 2015 Charlie Poole
4 // Permission is hereby granted, free of charge, to any person obtaining
5 // a copy of this software and associated documentation files (the
6 // "Software"), to deal in the Software without restriction, including
7 // without limitation the rights to use, copy, modify, merge, publish,
8 // distribute, sublicense, and/or sell copies of the Software, and to
9 // permit persons to whom the Software is furnished to do so, subject to
10 // the following conditions:
12 // The above copyright notice and this permission notice shall be
13 // included in all copies or substantial portions of the Software.
15 // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
16 // EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
17 // MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
18 // NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
19 // LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
20 // OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
21 // WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
22 // ***********************************************************************
25 #define NUNIT_FRAMEWORK
31 using System.Collections.Generic;
32 using System.Diagnostics;
34 using System.Reflection;
36 using NUnit.Framework.Api;
37 using NUnit.Framework.Interfaces;
38 using NUnit.Framework.Internal;
39 using NUnit.Framework.Internal.Filters;
41 using NUnitLite.TUnit;
42 using NUnit.Framework.TUnit;
48 /// TextRunner is a general purpose class that runs tests and
49 /// outputs to a text-based user interface (TextUI).
51 /// Call it from your Main like this:
52 /// new TextRunner(textWriter).Execute(args);
54 /// new TextUI().Execute(args);
55 /// The provided TextWriter is used by default, unless the
56 /// arguments to Execute override it using -out. The second
57 /// form uses the Console, provided it exists on the platform.
59 /// NOTE: When running on a platform without a Console, such
60 /// as Windows Phone, the results will simply not appear if
61 /// you fail to specify a file in the call itself or as an option.
63 public class TextRunner : ITestListener
65 #region Runner Return Codes
67 /// <summary>OK</summary>
68 public const int OK = 0;
69 /// <summary>Invalid Arguments</summary>
70 public const int INVALID_ARG = -1;
71 /// <summary>File not found</summary>
72 public const int FILE_NOT_FOUND = -2;
73 /// <summary>Test fixture not found - No longer in use</summary>
74 //public const int FIXTURE_NOT_FOUND = -3;
75 /// <summary>Invalid test suite</summary>
76 public const int INVALID_TEST_FIXTURE = -4;
77 /// <summary>Unexpected error occurred</summary>
78 public const int UNEXPECTED_ERROR = -100;
82 private Assembly _testAssembly;
83 private readonly List<Assembly> _assemblies = new List<Assembly>();
84 private ITestAssemblyRunner _runner;
86 private NUnitLiteOptions _options;
87 private ITestListener _teamCity = null;
89 private TextUI _textUI;
95 //// Initializes a new instance of the <see cref="TextRunner"/> class.
97 public TextRunner() { }
100 /// Initializes a new instance of the <see cref="TextRunner"/> class
101 /// specifying a test assembly whose tests are to be run.
103 /// <param name="testAssembly"></param>
105 public TextRunner(Assembly testAssembly)
107 _testAssembly = testAssembly;
114 public ResultSummary Summary { get; private set; }
118 #region Public Methods
120 public Dictionary<string, ITest> TestcaseList
122 get { return _runner.GetTestcaseIDList(); }
125 public void LoadTest(string[] args)
127 //TLogger.Write("LoadTest ..................");
128 _options = new NUnitLiteOptions(args);
129 ExtendedTextWriter outWriter = null;
130 outWriter = new ColorConsoleWriter();
131 _textUI = new TextUI(outWriter, Console.In, _options);
132 _runner = new NUnitTestAssemblyRunner(new DefaultTestAssemblyBuilder());
135 #if !SILVERLIGHT && !PORTABLE
136 if (!Directory.Exists(_options.WorkDirectory))
137 Directory.CreateDirectory(_options.WorkDirectory);
140 if (_options.TeamCity)
141 _teamCity = new TeamCityEventListener();
145 if (_options.ShowVersion || !_options.NoHeader)
146 _textUI.DisplayHeader();
148 if (_options.ShowHelp)
150 _textUI.DisplayHelp();
154 // We already showed version as a part of the header
155 if (_options.ShowVersion)
158 if (_options.ErrorMessages.Count > 0)
160 _textUI.DisplayErrors(_options.ErrorMessages);
161 _textUI.DisplayHelp();
166 _textUI.DisplayRuntimeEnvironment();
168 var testFile = _testAssembly != null
169 ? AssemblyHelper.GetAssemblyPath(_testAssembly)
170 : _options.InputFiles.Count > 0
171 ? _options.InputFiles[0]
174 //TLogger.Write("Input File [0]:" + _options.InputFiles[0]);
175 //TLogger.Write("Input File Format Size :"+ _options.InputFiles.Count);
177 if (testFile != null)
179 _textUI.DisplayTestFiles(new string[] { testFile });
180 //TLogger.Write("after DisplayTestFiles");
181 if (_testAssembly == null)
182 _testAssembly = AssemblyHelper.Load(testFile);
186 if (_options.WaitBeforeExit && _options.OutFile != null)
187 _textUI.DisplayWarning("Ignoring /wait option - only valid for Console");
189 foreach (string nameOrPath in _options.InputFiles) {
190 //TLogger.Write("In foreach" + nameOrPath);
191 _assemblies.Add(AssemblyHelper.Load(nameOrPath));
194 // We display the filters at this point so that any exception message
195 // thrown by CreateTestFilter will be understandable.
196 _textUI.DisplayTestFilters();
197 var runSettings = MakeRunSettings(_options);
199 TestFilter filter = CreateTestFilter(_options);
201 _runner.Load(_testAssembly, runSettings);
203 catch (FileNotFoundException ex)
205 _textUI.DisplayError(ex.Message);
210 _textUI.DisplayError(ex.ToString());
216 if (_options.WaitBeforeExit)
217 _textUI.WaitForUser("Press Enter key to continue . . .");
218 if (_options.OutFile != null && outWriter != null)
225 #if !SILVERLIGHT && !PORTABLE
226 public int Execute(string[] args)
228 var options = new NUnitLiteOptions(args);
230 InitializeInternalTrace(options);
232 ExtendedTextWriter outWriter = null;
233 if (options.OutFile != null)
235 outWriter = new ExtendedTextWrapper(TextWriter.Synchronized(new StreamWriter(Path.Combine(options.WorkDirectory, options.OutFile))));
236 Console.SetOut(outWriter);
240 outWriter = new ColorConsoleWriter();
243 TextWriter errWriter = null;
244 if (options.ErrFile != null)
246 errWriter = TextWriter.Synchronized(new StreamWriter(Path.Combine(options.WorkDirectory, options.ErrFile)));
247 Console.SetError(errWriter);
252 return Execute(outWriter, Console.In, options);
256 if (options.OutFile != null && outWriter != null)
259 if (options.ErrFile != null && errWriter != null)
264 public int Execute(string[] args)
266 var options = new NUnitLiteOptions(args);
267 ExtendedTextWriter outWriter = null;
268 outWriter = new ColorConsoleWriter();
271 return Execute(outWriter, Console.In, options);
275 if (options.OutFile != null && outWriter != null)
281 public int Execute(ExtendedTextWriter writer, TextReader reader, NUnitLiteOptions options)
283 var textUI = new TextUI(writer, reader, options);
284 return Execute(textUI, options);
288 /// Execute a test run
290 /// <param name="callingAssembly">The assembly from which tests are loaded</param>
291 public int Execute(TextUI textUI, NUnitLiteOptions options)
296 _runner = new NUnitTestAssemblyRunner(new DefaultTestAssemblyBuilder());
301 #if !SILVERLIGHT && !PORTABLE
302 if (!Directory.Exists(_options.WorkDirectory))
303 Directory.CreateDirectory(_options.WorkDirectory);
306 if (_options.TeamCity)
307 _teamCity = new TeamCityEventListener();
311 if (_options.ShowVersion || !_options.NoHeader)
312 _textUI.DisplayHeader();
314 if (_options.ShowHelp)
316 _textUI.DisplayHelp();
317 return TextRunner.OK;
320 // We already showed version as a part of the header
321 if (_options.ShowVersion)
322 return TextRunner.OK;
324 if (_options.ErrorMessages.Count > 0)
326 _textUI.DisplayErrors(_options.ErrorMessages);
327 _textUI.DisplayHelp();
329 return TextRunner.INVALID_ARG;
332 _textUI.DisplayRuntimeEnvironment();
334 var testFile = _testAssembly != null
335 ? AssemblyHelper.GetAssemblyPath(_testAssembly)
336 : _options.InputFiles.Count > 0
337 ? _options.InputFiles[0]
340 if (testFile != null)
342 _textUI.DisplayTestFiles(new string[] { testFile });
343 if (_testAssembly == null)
344 _testAssembly = AssemblyHelper.Load(testFile);
348 if (_options.WaitBeforeExit && _options.OutFile != null)
349 _textUI.DisplayWarning("Ignoring /wait option - only valid for Console");
351 foreach (string nameOrPath in _options.InputFiles){
352 _assemblies.Add(AssemblyHelper.Load(nameOrPath));
355 // We display the filters at this point so that any exception message
356 // thrown by CreateTestFilter will be understandable.
357 _textUI.DisplayTestFilters();
359 var runSettings = MakeRunSettings(_options);
361 TestFilter filter = CreateTestFilter(_options);
363 //_runner.Load(_testAssembly, runSettings);
365 return _options.Explore ? ExploreTests() : RunTests(filter, runSettings);
367 catch (FileNotFoundException ex)
369 _textUI.DisplayError(ex.Message);
370 return FILE_NOT_FOUND;
374 _textUI.DisplayError(ex.ToString());
375 return UNEXPECTED_ERROR;
380 if (_options.WaitBeforeExit)
381 _textUI.WaitForUser("Press Enter key to continue . . .");
388 #region Helper Methods
390 public int RunTests(TestFilter filter, IDictionary<string, object> runSettings)
392 var startTime = DateTime.UtcNow;
394 ITestResult result = _runner.Run(this, filter);
397 // Silverlight can't display results while the test is running
398 // so we do it afterwards.
399 foreach(ITestResult testResult in _results)
400 _textUI.TestFinished(testResult);
402 ReportResults(result);
404 #if !SILVERLIGHT && !PORTABLE
405 if (_options.ResultOutputSpecifications.Count > 0)
408 // [DuongNT]: Create Tizen OutputManager
409 var outputManager = new TOutputManager(_options.WorkDirectory);
411 var outputManager = new OutputManager(_options.WorkDirectory);
413 foreach (var spec in _options.ResultOutputSpecifications)
414 outputManager.WriteResultFile(result, spec, runSettings, filter);
417 if (Summary.InvalidTestFixtures > 0)
418 return INVALID_TEST_FIXTURE;
420 return Summary.FailureCount + Summary.ErrorCount + Summary.InvalidCount;
423 public void ReportResults(ITestResult result)
425 Summary = new ResultSummary(result);
427 if (Summary.ExplicitCount + Summary.SkipCount + Summary.IgnoreCount > 0)
428 _textUI.DisplayNotRunReport(result);
430 if (result.ResultState.Status == TestStatus.Failed)
431 _textUI.DisplayErrorsAndFailuresReport(result);
435 _textUI.PrintFullReport(_result);
437 if (TSettings.GetInstance().IsManual == false)
439 _textUI.DisplayRunSettings();
441 _textUI.DisplaySummaryReport(Summary);
445 private int ExploreTests()
447 #if !PORTABLE && !SILVERLIGHT
448 ITest testNode = _runner.LoadedTest;
450 var specs = _options.ExploreOutputSpecifications;
452 if (specs.Count == 0)
453 new TestCaseOutputWriter().WriteTestFile(testNode, Console.Out);
456 var outputManager = new OutputManager(_options.WorkDirectory);
458 foreach (var spec in _options.ExploreOutputSpecifications)
459 outputManager.WriteTestFile(testNode, spec);
467 /// Make the settings for this run - this is public for testing
469 public static Dictionary<string, object> MakeRunSettings(NUnitLiteOptions options)
471 // Transfer command line options to run settings
472 var runSettings = new Dictionary<string, object>();
474 if (options.RandomSeed >= 0)
475 runSettings[PackageSettings.RandomSeed] = options.RandomSeed;
478 if (options.WorkDirectory != null)
479 runSettings[PackageSettings.WorkDirectory] = Path.GetFullPath(options.WorkDirectory);
481 if (options.DefaultTimeout >= 0)
482 runSettings[PackageSettings.DefaultTimeout] = options.DefaultTimeout;
484 if (options.StopOnError)
485 runSettings[PackageSettings.StopOnError] = true;
487 if (options.DefaultTestNamePattern != null)
488 runSettings[PackageSettings.DefaultTestNamePattern] = options.DefaultTestNamePattern;
494 /// Create the test filter for this run - public for testing
496 /// <param name="options"></param>
497 /// <returns></returns>
498 public static TestFilter CreateTestFilter(NUnitLiteOptions options)
500 var filter = TestFilter.Empty;
502 if (options.TestList.Count > 0)
504 var testFilters = new List<TestFilter>();
505 foreach (var test in options.TestList)
506 testFilters.Add(new FullNameFilter(test));
508 filter = testFilters.Count > 1
509 ? new OrFilter(testFilters.ToArray())
514 if (options.WhereClauseSpecified)
516 string xmlText = new TestSelectionParser().Parse(options.WhereClause);
517 var whereFilter = TestFilter.FromXml(TNode.FromXml(xmlText));
518 filter = filter.IsEmpty
520 : new AndFilter(filter, whereFilter);
526 #if !PORTABLE && !SILVERLIGHT
527 private void InitializeInternalTrace(NUnitLiteOptions _options)
529 var traceLevel = (InternalTraceLevel)Enum.Parse(typeof(InternalTraceLevel), _options.InternalTraceLevel ?? "Off", true);
531 if (traceLevel != InternalTraceLevel.Off)
533 var logName = GetLogFileName();
535 #if NETCF // NETCF: Try to encapsulate this
536 InternalTrace.Initialize(Path.Combine(NUnit.Env.DocumentFolder, logName), traceLevel);
538 StreamWriter streamWriter = null;
539 if (traceLevel > InternalTraceLevel.Off)
541 string logPath = Path.Combine(Environment.CurrentDirectory, logName);
542 streamWriter = new StreamWriter(new FileStream(logPath, FileMode.Append, FileAccess.Write, FileShare.Write));
543 streamWriter.AutoFlush = true;
545 InternalTrace.Initialize(streamWriter, traceLevel);
550 private string GetLogFileName()
552 const string LOG_FILE_FORMAT = "InternalTrace.{0}.{1}.{2}";
554 // Some mobiles don't have an Open With menu item,
555 // so we use .txt, which is opened easily.
557 const string ext = "txt";
559 const string ext = "log";
561 var baseName = _testAssembly != null
562 ? _testAssembly.GetName().Name
563 : _options.InputFiles.Count > 0
564 ? Path.GetFileNameWithoutExtension(_options.InputFiles[0])
567 return string.Format(LOG_FILE_FORMAT, Process.GetCurrentProcess().Id, baseName, ext);
573 #region ITestListener Members
576 /// Called when a test or suite has just started
578 /// <param name="test">The test that is starting</param>
579 public void TestStarted(ITest test)
581 if (_teamCity != null)
582 _teamCity.TestStarted(test);
586 /// Called when a test has finished
588 /// <param name="result">The result of the test</param>
589 public void TestFinished(ITestResult result)
591 if (_teamCity != null)
592 _teamCity.TestFinished(result);
595 _textUI.TestFinished(result);
597 // For Silverlight, we can't display the results
598 // until the run is completed. We don't save anything
599 // unless there is associated output, since that's
600 // the only time we display anything in Silverlight.
601 if (result.Output.Length > 0)
602 _results.Add(result);
607 /// Called when a test produces output for immediate display
609 /// <param name="output">A TestOutput object containing the text to display</param>
610 public void TestOutput(TestOutput output)
612 _textUI.TestOutput(output);