test/Tizen.NUI.Devel.Tests.Ubuntu/nunitlite/TextRunner.cs

   1 // ***********************************************************************
   2 // Copyright (c) 2015 Charlie Poole
   3 //
   4 // Permission is hereby granted, free of charge, to any person obtaining
   5 // a copy of this software and associated documentation files (the
   6 // "Software"), to deal in the Software without restriction, including
   7 // without limitation the rights to use, copy, modify, merge, publish,
   8 // distribute, sublicense, and/or sell copies of the Software, and to
   9 // permit persons to whom the Software is furnished to do so, subject to
  10 // the following conditions:
  11 //
  12 // The above copyright notice and this permission notice shall be
  13 // included in all copies or substantial portions of the Software.
  14 //
  15 // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  16 // EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  17 // MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  18 // NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
  19 // LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
  20 // OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
  21 // WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  22 // ***********************************************************************
  23 #define PORTABLE
  24 #define TIZEN
  25 #define NUNIT_FRAMEWORK
  26 #define NUNITLITE
  27 #define NET_4_5
  28 #define PARALLEL
  29 #define TIZEN
  30 using System;
  31 using System.Collections.Generic;
  32 using System.Diagnostics;
  33 using System.IO;
  34 using System.Reflection;
  35 using NUnit.Common;
  36 using NUnit.Framework.Api;
  37 using NUnit.Framework.Interfaces;
  38 using NUnit.Framework.Internal;
  39 using NUnit.Framework.Internal.Filters;
  40 #if TIZEN
  41 using NUnitLite.TUnit;
  42 using NUnit.Framework.TUnit;
  43 #endif
  44
  45 namespace NUnitLite
  46 {
  47         /// <summary>
  48         /// TextRunner is a general purpose class that runs tests and
  49         /// outputs to a text-based user interface (TextUI).
  50         ///
  51         /// Call it from your Main like this:
  52         ///   new TextRunner(textWriter).Execute(args);
  53         ///     OR
  54         ///   new TextUI().Execute(args);
  55         /// The provided TextWriter is used by default, unless the
  56         /// arguments to Execute override it using -out. The second
  57         /// form uses the Console, provided it exists on the platform.
  58         ///
  59         /// NOTE: When running on a platform without a Console, such
  60         /// as Windows Phone, the results will simply not appear if
  61         /// you fail to specify a file in the call itself or as an option.
  62         /// </summary>
  63         public class TextRunner : ITestListener
  64         {
  65                 #region Runner Return Codes
  66
  67                 /// <summary>OK</summary>
  68                 public const int OK = 0;
  69                 /// <summary>Invalid Arguments</summary>
  70                 public const int INVALID_ARG = -1;
  71                 /// <summary>File not found</summary>
  72                 public const int FILE_NOT_FOUND = -2;
  73                 /// <summary>Test fixture not found - No longer in use</summary>
  74                 //public const int FIXTURE_NOT_FOUND = -3;
  75                 /// <summary>Invalid test suite</summary>
  76                 public const int INVALID_TEST_FIXTURE = -4;
  77                 /// <summary>Unexpected error occurred</summary>
  78                 public const int UNEXPECTED_ERROR = -100;
  79
  80                 #endregion
  81
  82                 private Assembly _testAssembly;
  83                 private readonly List<Assembly> _assemblies = new List<Assembly>();
  84                 private ITestAssemblyRunner _runner;
  85
  86                 private NUnitLiteOptions _options;
  87                 private ITestListener _teamCity = null;
  88
  89                 private TextUI _textUI;
  90
  91
  92                 #region Constructors
  93
  94                 //// <summary>
  95                 //// Initializes a new instance of the <see cref="TextRunner"/> class.
  96                 //// </summary>
  97                 public TextRunner() { }
  98
  99                 /// <summary>
 100                 /// Initializes a new instance of the <see cref="TextRunner"/> class
 101                 /// specifying a test assembly whose tests are to be run.
 102                 /// </summary>
 103                 /// <param name="testAssembly"></param>
 104                 /// </summary>
 105                 public TextRunner(Assembly testAssembly)
 106                 {
 107                         _testAssembly = testAssembly;
 108                 }
 109
 110                 #endregion
 111
 112                 #region Properties
 113
 114                 public ResultSummary Summary { get; private set; }
 115
 116                 #endregion
 117
 118                 #region Public Methods
 119                 #if TIZEN
 120                 public Dictionary<string, ITest> TestcaseList
 121                 {
 122                         get { return _runner.GetTestcaseIDList(); }
 123                 }
 124
 125                 public void LoadTest(string[] args)
 126                 {
 127             //TLogger.Write("LoadTest ..................");
 128             _options = new NUnitLiteOptions(args);
 129                         ExtendedTextWriter outWriter = null;
 130                         outWriter = new ColorConsoleWriter();
 131                         _textUI = new TextUI(outWriter, Console.In, _options);
 132                         _runner = new NUnitTestAssemblyRunner(new DefaultTestAssemblyBuilder());
 133                         try
 134                         {
 135                 #if !SILVERLIGHT && !PORTABLE
 136                 if (!Directory.Exists(_options.WorkDirectory))
 137                 Directory.CreateDirectory(_options.WorkDirectory);
 138
 139                 #if !NETCF
 140                                 if (_options.TeamCity)
 141                                         _teamCity = new TeamCityEventListener();
 142                 #endif
 143                 #endif
 144
 145                                 if (_options.ShowVersion || !_options.NoHeader)
 146                                         _textUI.DisplayHeader();
 147
 148                                 if (_options.ShowHelp)
 149                                 {
 150                                         _textUI.DisplayHelp();
 151                                         return;
 152                                 }
 153
 154                                 // We already showed version as a part of the header
 155                                 if (_options.ShowVersion)
 156                                         return;
 157
 158                                 if (_options.ErrorMessages.Count > 0)
 159                                 {
 160                                         _textUI.DisplayErrors(_options.ErrorMessages);
 161                                         _textUI.DisplayHelp();
 162
 163                                         return;
 164                                 }
 165
 166                                 _textUI.DisplayRuntimeEnvironment();
 167
 168                                 var testFile = _testAssembly != null
 169                                         ? AssemblyHelper.GetAssemblyPath(_testAssembly)
 170                                         : _options.InputFiles.Count > 0
 171                                         ? _options.InputFiles[0]
 172                                         : null;
 173
 174                 //TLogger.Write("Input File [0]:" + _options.InputFiles[0]);
 175                 //TLogger.Write("Input File Format Size :"+ _options.InputFiles.Count);
 176
 177                 if (testFile != null)
 178                                 {
 179                                         _textUI.DisplayTestFiles(new string[] { testFile });
 180                     //TLogger.Write("after DisplayTestFiles");
 181                                         if (_testAssembly == null)
 182                                                 _testAssembly = AssemblyHelper.Load(testFile);
 183                                 }
 184
 185
 186                                 if (_options.WaitBeforeExit && _options.OutFile != null)
 187                                         _textUI.DisplayWarning("Ignoring /wait option - only valid for Console");
 188
 189                                 foreach (string nameOrPath in _options.InputFiles) {
 190                     //TLogger.Write("In foreach" + nameOrPath);
 191                     _assemblies.Add(AssemblyHelper.Load(nameOrPath));
 192                 }
 193
 194                 // We display the filters at this point so  that any exception message
 195                 // thrown by CreateTestFilter will be understandable.
 196                 _textUI.DisplayTestFilters();
 197                                 var runSettings = MakeRunSettings(_options);
 198
 199                                 TestFilter filter = CreateTestFilter(_options);
 200
 201                                 _runner.Load(_testAssembly, runSettings);
 202                         }
 203                         catch (FileNotFoundException ex)
 204                         {
 205                                 _textUI.DisplayError(ex.Message);
 206                                 return;
 207                         }
 208                         catch (Exception ex)
 209                         {
 210                                 _textUI.DisplayError(ex.ToString());
 211                                 return;
 212                         }
 213                 #if !SILVERLIGHT
 214                         finally
 215                         {
 216                                 if (_options.WaitBeforeExit)
 217                                         _textUI.WaitForUser("Press Enter key to continue . . .");
 218                                 if (_options.OutFile != null && outWriter != null)
 219                                         outWriter.Flush();
 220                         }
 221                 #endif
 222                 }
 223                 #endif
 224
 225                 #if !SILVERLIGHT && !PORTABLE
 226                 public int Execute(string[] args)
 227                 {
 228                 var options = new NUnitLiteOptions(args);
 229
 230                 InitializeInternalTrace(options);
 231
 232                 ExtendedTextWriter outWriter = null;
 233                 if (options.OutFile != null)
 234                 {
 235                 outWriter = new ExtendedTextWrapper(TextWriter.Synchronized(new StreamWriter(Path.Combine(options.WorkDirectory, options.OutFile))));
 236                 Console.SetOut(outWriter);
 237                 }
 238                 else
 239                 {
 240                 outWriter = new ColorConsoleWriter();
 241                 }
 242
 243                 TextWriter errWriter = null;
 244                 if (options.ErrFile != null)
 245                 {
 246                 errWriter = TextWriter.Synchronized(new StreamWriter(Path.Combine(options.WorkDirectory, options.ErrFile)));
 247                 Console.SetError(errWriter);
 248                 }
 249
 250                 try
 251                 {
 252                 return Execute(outWriter, Console.In, options);
 253                 }
 254                 finally
 255                 {
 256                 if (options.OutFile != null && outWriter != null)
 257                 outWriter.Close();
 258
 259                 if (options.ErrFile != null && errWriter != null)
 260                 errWriter.Close();
 261                 }
 262                 }
 263                 #else
 264                 public int Execute(string[] args)
 265                 {
 266                         var options = new NUnitLiteOptions(args);
 267                         ExtendedTextWriter outWriter = null;
 268                         outWriter = new ColorConsoleWriter();
 269                         try
 270                         {
 271                                 return Execute(outWriter, Console.In, options);
 272                         }
 273                         finally
 274                         {
 275                                 if (options.OutFile != null && outWriter != null)
 276                                         outWriter.Flush();
 277                         }
 278                 }
 279                 #endif
 280
 281                 public int Execute(ExtendedTextWriter writer, TextReader reader, NUnitLiteOptions options)
 282                 {
 283                         var textUI = new TextUI(writer, reader, options);
 284                         return Execute(textUI, options);
 285                 }
 286
 287                 /// <summary>
 288                 /// Execute a test run
 289                 /// </summary>
 290                 /// <param name="callingAssembly">The assembly from which tests are loaded</param>
 291                 public int Execute(TextUI textUI, NUnitLiteOptions options)
 292                 {
 293                         _textUI = textUI;
 294                         _options = options;
 295                         if (_runner == null)
 296                                 _runner = new NUnitTestAssemblyRunner(new DefaultTestAssemblyBuilder());
 297
 298                         try
 299                         {
 300 #if !TIZEN
 301 #if !SILVERLIGHT && !PORTABLE
 302                                 if (!Directory.Exists(_options.WorkDirectory))
 303                                 Directory.CreateDirectory(_options.WorkDirectory);
 304
 305 #if !NETCF
 306                                 if (_options.TeamCity)
 307                                         _teamCity = new TeamCityEventListener();
 308 #endif
 309 #endif
 310
 311                                 if (_options.ShowVersion || !_options.NoHeader)
 312                                         _textUI.DisplayHeader();
 313
 314                                 if (_options.ShowHelp)
 315                                 {
 316                                         _textUI.DisplayHelp();
 317                                         return TextRunner.OK;
 318                                 }
 319
 320                                 // We already showed version as a part of the header
 321                                 if (_options.ShowVersion)
 322                                         return TextRunner.OK;
 323
 324                                 if (_options.ErrorMessages.Count > 0)
 325                                 {
 326                                         _textUI.DisplayErrors(_options.ErrorMessages);
 327                                         _textUI.DisplayHelp();
 328
 329                                         return TextRunner.INVALID_ARG;
 330                                 }
 331
 332                                 _textUI.DisplayRuntimeEnvironment();
 333
 334                                 var testFile = _testAssembly != null
 335                                         ? AssemblyHelper.GetAssemblyPath(_testAssembly)
 336                                         : _options.InputFiles.Count > 0
 337                                         ? _options.InputFiles[0]
 338                                         : null;
 339
 340                                 if (testFile != null)
 341                                 {
 342                                         _textUI.DisplayTestFiles(new string[] { testFile });
 343                                         if (_testAssembly == null)
 344                                                 _testAssembly = AssemblyHelper.Load(testFile);
 345                                 }
 346
 347
 348                                 if (_options.WaitBeforeExit && _options.OutFile != null)
 349                                         _textUI.DisplayWarning("Ignoring /wait option - only valid for Console");
 350
 351                                 foreach (string nameOrPath in _options.InputFiles){
 352                         _assemblies.Add(AssemblyHelper.Load(nameOrPath));
 353                 }
 354
 355                                 // We display the filters at this point so  that any exception message
 356                                 // thrown by CreateTestFilter will be understandable.
 357                                 _textUI.DisplayTestFilters();
 358 #endif
 359                 var runSettings = MakeRunSettings(_options);
 360
 361                                 TestFilter filter = CreateTestFilter(_options);
 362
 363                                 //_runner.Load(_testAssembly, runSettings);
 364
 365                                 return _options.Explore ? ExploreTests() : RunTests(filter, runSettings);
 366                         }
 367                         catch (FileNotFoundException ex)
 368                         {
 369                                 _textUI.DisplayError(ex.Message);
 370                                 return FILE_NOT_FOUND;
 371                         }
 372                         catch (Exception ex)
 373                         {
 374                                 _textUI.DisplayError(ex.ToString());
 375                                 return UNEXPECTED_ERROR;
 376                         }
 377                         #if !SILVERLIGHT
 378                         finally
 379                         {
 380                                 if (_options.WaitBeforeExit)
 381                                         _textUI.WaitForUser("Press Enter key to continue . . .");
 382                         }
 383                         #endif
 384                 }
 385
 386                 #endregion
 387
 388                 #region Helper Methods
 389
 390                 public int RunTests(TestFilter filter, IDictionary<string, object> runSettings)
 391                 {
 392                         var startTime = DateTime.UtcNow;
 393
 394                         ITestResult result = _runner.Run(this, filter);
 395
 396                         #if SILVERLIGHT
 397                         // Silverlight can't display results while the test is running
 398                         // so we do it afterwards.
 399                         foreach(ITestResult testResult in _results)
 400                         _textUI.TestFinished(testResult);
 401                         #endif
 402                         ReportResults(result);
 403
 404                         #if !SILVERLIGHT && !PORTABLE
 405                         if (_options.ResultOutputSpecifications.Count > 0)
 406                         {
 407                         #if TIZEN
 408                         // [DuongNT]: Create Tizen OutputManager
 409                         var outputManager = new TOutputManager(_options.WorkDirectory);
 410                         #else
 411                         var outputManager = new OutputManager(_options.WorkDirectory);
 412                         #endif
 413                         foreach (var spec in _options.ResultOutputSpecifications)
 414                                 outputManager.WriteResultFile(result, spec, runSettings, filter);
 415                 }
 416                         #endif
 417                 if (Summary.InvalidTestFixtures > 0)
 418                         return INVALID_TEST_FIXTURE;
 419
 420                 return Summary.FailureCount + Summary.ErrorCount + Summary.InvalidCount;
 421         }
 422
 423         public void ReportResults(ITestResult result)
 424         {
 425                 Summary = new ResultSummary(result);
 426
 427                 if (Summary.ExplicitCount + Summary.SkipCount + Summary.IgnoreCount > 0)
 428                         _textUI.DisplayNotRunReport(result);
 429
 430                 if (result.ResultState.Status == TestStatus.Failed)
 431                         _textUI.DisplayErrorsAndFailuresReport(result);
 432
 433                 #if FULL
 434                 if (_options.Full)
 435                 _textUI.PrintFullReport(_result);
 436                 #endif
 437                 if (TSettings.GetInstance().IsManual == false)
 438                 {
 439                         _textUI.DisplayRunSettings();
 440
 441                         _textUI.DisplaySummaryReport(Summary);
 442                 }
 443         }
 444
 445         private int ExploreTests()
 446         {
 447                 #if !PORTABLE && !SILVERLIGHT
 448                 ITest testNode = _runner.LoadedTest;
 449
 450                 var specs = _options.ExploreOutputSpecifications;
 451
 452                 if (specs.Count == 0)
 453                 new TestCaseOutputWriter().WriteTestFile(testNode, Console.Out);
 454                 else
 455                 {
 456                 var outputManager = new OutputManager(_options.WorkDirectory);
 457
 458                 foreach (var spec in _options.ExploreOutputSpecifications)
 459                 outputManager.WriteTestFile(testNode, spec);
 460                 }
 461                 #endif
 462
 463                 return OK;
 464         }
 465
 466         /// <summary>
 467         /// Make the settings for this run - this is public for testing
 468         /// </summary>
 469         public static Dictionary<string, object> MakeRunSettings(NUnitLiteOptions options)
 470         {
 471                 // Transfer command line options to run settings
 472                 var runSettings = new Dictionary<string, object>();
 473
 474                 if (options.RandomSeed >= 0)
 475                         runSettings[PackageSettings.RandomSeed] = options.RandomSeed;
 476
 477                 #if !PORTABLE
 478                 if (options.WorkDirectory != null)
 479                 runSettings[PackageSettings.WorkDirectory] = Path.GetFullPath(options.WorkDirectory);
 480                 #endif
 481                 if (options.DefaultTimeout >= 0)
 482                         runSettings[PackageSettings.DefaultTimeout] = options.DefaultTimeout;
 483
 484                 if (options.StopOnError)
 485                         runSettings[PackageSettings.StopOnError] = true;
 486
 487                 if (options.DefaultTestNamePattern != null)
 488                         runSettings[PackageSettings.DefaultTestNamePattern] = options.DefaultTestNamePattern;
 489
 490                 return runSettings;
 491         }
 492
 493         /// <summary>
 494         /// Create the test filter for this run - public for testing
 495         /// </summary>
 496         /// <param name="options"></param>
 497         /// <returns></returns>
 498         public static TestFilter CreateTestFilter(NUnitLiteOptions options)
 499         {
 500                 var filter = TestFilter.Empty;
 501
 502                 if (options.TestList.Count > 0)
 503                 {
 504                         var testFilters = new List<TestFilter>();
 505                         foreach (var test in options.TestList)
 506                                 testFilters.Add(new FullNameFilter(test));
 507
 508                         filter = testFilters.Count > 1
 509                                 ? new OrFilter(testFilters.ToArray())
 510                                 : testFilters[0];
 511                 }
 512
 513
 514                 if (options.WhereClauseSpecified)
 515                 {
 516                         string xmlText = new TestSelectionParser().Parse(options.WhereClause);
 517                         var whereFilter = TestFilter.FromXml(TNode.FromXml(xmlText));
 518                         filter = filter.IsEmpty
 519                                 ? whereFilter
 520                                 : new AndFilter(filter, whereFilter);
 521                 }
 522
 523                 return filter;
 524         }
 525
 526         #if !PORTABLE && !SILVERLIGHT
 527         private void InitializeInternalTrace(NUnitLiteOptions _options)
 528         {
 529         var traceLevel = (InternalTraceLevel)Enum.Parse(typeof(InternalTraceLevel), _options.InternalTraceLevel ?? "Off", true);
 530
 531         if (traceLevel != InternalTraceLevel.Off)
 532         {
 533         var logName = GetLogFileName();
 534
 535         #if NETCF // NETCF: Try to encapsulate this
 536         InternalTrace.Initialize(Path.Combine(NUnit.Env.DocumentFolder, logName), traceLevel);
 537         #else
 538         StreamWriter streamWriter = null;
 539         if (traceLevel > InternalTraceLevel.Off)
 540         {
 541                 string logPath = Path.Combine(Environment.CurrentDirectory, logName);
 542                 streamWriter = new StreamWriter(new FileStream(logPath, FileMode.Append, FileAccess.Write, FileShare.Write));
 543                 streamWriter.AutoFlush = true;
 544         }
 545         InternalTrace.Initialize(streamWriter, traceLevel);
 546         #endif
 547 }
 548 }
 549
 550 private string GetLogFileName()
 551 {
 552         const string LOG_FILE_FORMAT = "InternalTrace.{0}.{1}.{2}";
 553
 554         // Some mobiles don't have an Open With menu item,
 555         // so we use .txt, which is opened easily.
 556         #if NETCF
 557         const string ext = "txt";
 558         #else
 559         const string ext = "log";
 560         #endif
 561         var baseName = _testAssembly != null
 562                 ? _testAssembly.GetName().Name
 563                 : _options.InputFiles.Count > 0
 564                 ? Path.GetFileNameWithoutExtension(_options.InputFiles[0])
 565                 : "NUnitLite";
 566
 567         return string.Format(LOG_FILE_FORMAT, Process.GetCurrentProcess().Id, baseName, ext);
 568 }
 569         #endif
 570
 571 #endregion
 572
 573 #region ITestListener Members
 574
 575 /// <summary>
 576 /// Called when a test or suite has just started
 577 /// </summary>
 578 /// <param name="test">The test that is starting</param>
 579 public void TestStarted(ITest test)
 580 {
 581         if (_teamCity != null)
 582                 _teamCity.TestStarted(test);
 583 }
 584
 585 /// <summary>
 586 /// Called when a test has finished
 587 /// </summary>
 588 /// <param name="result">The result of the test</param>
 589 public void TestFinished(ITestResult result)
 590 {
 591         if (_teamCity != null)
 592                 _teamCity.TestFinished(result);
 593
 594         #if !SILVERLIGHT
 595         _textUI.TestFinished(result);
 596         #else
 597         // For Silverlight, we can't display the results
 598         // until the run is completed. We don't save anything
 599         // unless there is associated output, since that's
 600         // the only time we display anything in Silverlight.
 601         if (result.Output.Length > 0)
 602         _results.Add(result);
 603         #endif
 604 }
 605
 606 /// <summary>
 607 /// Called when a test produces output for immediate display
 608 /// </summary>
 609 /// <param name="output">A TestOutput object containing the text to display</param>
 610 public void TestOutput(TestOutput output)
 611 {
 612         _textUI.TestOutput(output);
 613 }
 614
 615 #endregion
 616 }
 617 }