executor/tools/xeTestLogCompare.cpp

   1 /*-------------------------------------------------------------------------
   2  * drawElements Quality Program Test Executor
   3  * ------------------------------------------
   4  *
   5  * Copyright 2014 The Android Open Source Project
   6  *
   7  * Licensed under the Apache License, Version 2.0 (the "License");
   8  * you may not use this file except in compliance with the License.
   9  * You may obtain a copy of the License at
  10  *
  11  *      http://www.apache.org/licenses/LICENSE-2.0
  12  *
  13  * Unless required by applicable law or agreed to in writing, software
  14  * distributed under the License is distributed on an "AS IS" BASIS,
  15  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  16  * See the License for the specific language governing permissions and
  17  * limitations under the License.
  18  *
  19  *//*!
  20  * \file
  21  * \brief Test log compare utility.
  22  *//*--------------------------------------------------------------------*/
  23
  24 #include "xeTestLogParser.hpp"
  25 #include "xeTestResultParser.hpp"
  26 #include "deFilePath.hpp"
  27 #include "deString.h"
  28 #include "deThread.hpp"
  29 #include "deCommandLine.hpp"
  30
  31 #include <vector>
  32 #include <string>
  33 #include <cstdio>
  34 #include <cstdlib>
  35 #include <fstream>
  36 #include <iostream>
  37 #include <set>
  38 #include <map>
  39
  40 using std::vector;
  41 using std::string;
  42 using std::set;
  43 using std::map;
  44
  45 enum OutputMode
  46 {
  47         OUTPUTMODE_ALL = 0,
  48         OUTPUTMODE_DIFF,
  49
  50         OUTPUTMODE_LAST
  51 };
  52
  53 enum OutputFormat
  54 {
  55         OUTPUTFORMAT_TEXT = 0,
  56         OUTPUTFORMAT_CSV,
  57
  58         OUTPUTFORMAT_LAST
  59 };
  60
  61 enum OutputValue
  62 {
  63         OUTPUTVALUE_STATUS_CODE = 0,
  64         OUTPUTVALUE_STATUS_DETAILS,
  65
  66         OUTPUTVALUE_LAST
  67 };
  68
  69 namespace opt
  70 {
  71
  72 DE_DECLARE_COMMAND_LINE_OPT(OutMode,    OutputMode);
  73 DE_DECLARE_COMMAND_LINE_OPT(OutFormat,  OutputFormat);
  74 DE_DECLARE_COMMAND_LINE_OPT(OutValue,   OutputValue);
  75
  76 static void registerOptions (de::cmdline::Parser& parser)
  77 {
  78         using de::cmdline::Option;
  79         using de::cmdline::NamedValue;
  80
  81         static const NamedValue<OutputMode> s_outputModes[] =
  82         {
  83                 { "all",        OUTPUTMODE_ALL  },
  84                 { "diff",       OUTPUTMODE_DIFF }
  85         };
  86         static const NamedValue<OutputFormat> s_outputFormats[] =
  87         {
  88                 { "text",       OUTPUTFORMAT_TEXT       },
  89                 { "csv",        OUTPUTFORMAT_CSV        }
  90         };
  91         static const NamedValue<OutputValue> s_outputValues[] =
  92         {
  93                 { "code",               OUTPUTVALUE_STATUS_CODE         },
  94                 { "details",    OUTPUTVALUE_STATUS_DETAILS      }
  95         };
  96
  97         parser << Option<OutFormat>             ("f",   "format",               "Output format",        s_outputFormats,        "csv")
  98                    << Option<OutMode>           ("m",   "mode",                 "Output mode",          s_outputModes,          "all")
  99                    << Option<OutValue>          ("v",   "value",                "Value to extract",     s_outputValues,         "code");
 100 }
 101
 102 } // opt
 103
 104 struct CommandLine
 105 {
 106         CommandLine (void)
 107                 : outMode       (OUTPUTMODE_ALL)
 108                 , outFormat     (OUTPUTFORMAT_CSV)
 109                 , outValue      (OUTPUTVALUE_STATUS_CODE)
 110         {
 111         }
 112
 113         OutputMode                      outMode;
 114         OutputFormat            outFormat;
 115         OutputValue                     outValue;
 116         vector<string>          filenames;
 117 };
 118
 119 struct ShortBatchResult
 120 {
 121         vector<xe::TestCaseResultHeader>        resultHeaders;
 122         map<string, int>                                        resultMap;
 123 };
 124
 125 class ShortResultHandler : public xe::TestLogHandler
 126 {
 127 public:
 128         ShortResultHandler (ShortBatchResult& result)
 129                 : m_result(result)
 130         {
 131         }
 132
 133         void setSessionInfo (const xe::SessionInfo&)
 134         {
 135                 // Ignored.
 136         }
 137
 138         xe::TestCaseResultPtr startTestCaseResult (const char* casePath)
 139         {
 140                 return xe::TestCaseResultPtr(new xe::TestCaseResultData(casePath));
 141         }
 142
 143         void testCaseResultUpdated (const xe::TestCaseResultPtr&)
 144         {
 145                 // Ignored.
 146         }
 147
 148         void testCaseResultComplete (const xe::TestCaseResultPtr& caseData)
 149         {
 150                 xe::TestCaseResultHeader        header;
 151                 int                                                     caseNdx = (int)m_result.resultHeaders.size();
 152
 153                 header.casePath                 = caseData->getTestCasePath();
 154                 header.caseType                 = xe::TESTCASETYPE_SELF_VALIDATE;
 155                 header.statusCode               = caseData->getStatusCode();
 156                 header.statusDetails    = caseData->getStatusDetails();
 157
 158                 if (header.statusCode == xe::TESTSTATUSCODE_LAST)
 159                 {
 160                         xe::TestCaseResult fullResult;
 161
 162                         xe::parseTestCaseResultFromData(&m_testResultParser, &fullResult, *caseData.get());
 163
 164                         header = xe::TestCaseResultHeader(fullResult);
 165                 }
 166
 167                 // Insert into result list & map.
 168                 m_result.resultHeaders.push_back(header);
 169                 m_result.resultMap[header.casePath] = caseNdx;
 170         }
 171
 172 private:
 173         ShortBatchResult&               m_result;
 174         xe::TestResultParser    m_testResultParser;
 175 };
 176
 177 static void readLogFile (ShortBatchResult& batchResult, const char* filename)
 178 {
 179         std::ifstream           in                              (filename, std::ifstream::binary|std::ifstream::in);
 180         ShortResultHandler      resultHandler   (batchResult);
 181         xe::TestLogParser       parser                  (&resultHandler);
 182         deUint8                         buf                             [1024];
 183         int                                     numRead                 = 0;
 184
 185         for (;;)
 186         {
 187                 in.read((char*)&buf[0], DE_LENGTH_OF_ARRAY(buf));
 188                 numRead = (int)in.gcount();
 189
 190                 if (numRead <= 0)
 191                         break;
 192
 193                 parser.parse(&buf[0], numRead);
 194         }
 195
 196         in.close();
 197 }
 198
 199 class LogFileReader : public de::Thread
 200 {
 201 public:
 202         LogFileReader (ShortBatchResult& batchResult, const char* filename)
 203                 : m_batchResult (batchResult)
 204                 , m_filename    (filename)
 205         {
 206         }
 207
 208         void run (void)
 209         {
 210                 readLogFile(m_batchResult, m_filename.c_str());
 211         }
 212
 213 private:
 214         ShortBatchResult&       m_batchResult;
 215         std::string                     m_filename;
 216 };
 217
 218 static void computeCaseList (vector<string>& cases, const vector<ShortBatchResult>& batchResults)
 219 {
 220         // \todo [2012-07-10 pyry] Do proper case ordering (eg. handle missing cases nicely).
 221         set<string> addedCases;
 222
 223         for (vector<ShortBatchResult>::const_iterator batchIter = batchResults.begin(); batchIter != batchResults.end(); batchIter++)
 224         {
 225                 for (vector<xe::TestCaseResultHeader>::const_iterator caseIter = batchIter->resultHeaders.begin(); caseIter != batchIter->resultHeaders.end(); caseIter++)
 226                 {
 227                         if (addedCases.find(caseIter->casePath) == addedCases.end())
 228                         {
 229                                 cases.push_back(caseIter->casePath);
 230                                 addedCases.insert(caseIter->casePath);
 231                         }
 232                 }
 233         }
 234 }
 235
 236 static void getTestResultHeaders (vector<xe::TestCaseResultHeader>& headers, const vector<ShortBatchResult>& batchResults, const char* casePath)
 237 {
 238         headers.resize(batchResults.size());
 239
 240         for (int ndx = 0; ndx < (int)batchResults.size(); ndx++)
 241         {
 242                 const ShortBatchResult&                         batchResult     = batchResults[ndx];
 243                 map<string, int>::const_iterator        resultPos       = batchResult.resultMap.find(casePath);
 244
 245                 if (resultPos != batchResult.resultMap.end())
 246                         headers[ndx] = batchResult.resultHeaders[resultPos->second];
 247                 else
 248                 {
 249                         headers[ndx].casePath   = casePath;
 250                         headers[ndx].caseType   = xe::TESTCASETYPE_SELF_VALIDATE;
 251                         headers[ndx].statusCode = xe::TESTSTATUSCODE_LAST;
 252                 }
 253         }
 254 }
 255
 256 static const char* getStatusCodeName (xe::TestStatusCode code)
 257 {
 258         if (code == xe::TESTSTATUSCODE_LAST)
 259                 return "Missing";
 260         else
 261                 return xe::getTestStatusCodeName(code);
 262 }
 263
 264 static bool runCompare (const CommandLine& cmdLine, std::ostream& dst)
 265 {
 266         vector<ShortBatchResult>        results;
 267         vector<string>                          batchNames;
 268         bool                                            compareOk       = true;
 269
 270         XE_CHECK(!cmdLine.filenames.empty());
 271
 272         try
 273         {
 274                 // Read in batch results
 275                 results.resize(cmdLine.filenames.size());
 276                 {
 277                         std::vector<de::SharedPtr<LogFileReader> > readers;
 278
 279                         for (int ndx = 0; ndx < (int)cmdLine.filenames.size(); ndx++)
 280                         {
 281                                 readers.push_back(de::SharedPtr<LogFileReader>(new LogFileReader(results[ndx], cmdLine.filenames[ndx].c_str())));
 282                                 readers.back()->start();
 283                         }
 284
 285                         for (int ndx = 0; ndx < (int)cmdLine.filenames.size(); ndx++)
 286                         {
 287                                 readers[ndx]->join();
 288
 289                                 // Use file name as batch name.
 290                                 batchNames.push_back(de::FilePath(cmdLine.filenames[ndx].c_str()).getBaseName());
 291                         }
 292                 }
 293
 294                 // Compute unified case list.
 295                 vector<string> caseList;
 296                 computeCaseList(caseList, results);
 297
 298                 // Stats.
 299                 int             numCases                = (int)caseList.size();
 300                 int             numEqual                = 0;
 301
 302                 if (cmdLine.outFormat == OUTPUTFORMAT_CSV)
 303                 {
 304                         dst << "TestCasePath";
 305                         for (vector<string>::const_iterator nameIter = batchNames.begin(); nameIter != batchNames.end(); nameIter++)
 306                                 dst << "," << *nameIter;
 307                         dst << "\n";
 308                 }
 309
 310                 // Compare cases.
 311                 for (vector<string>::const_iterator caseIter = caseList.begin(); caseIter != caseList.end(); caseIter++)
 312                 {
 313                         const string&                                           caseName        = *caseIter;
 314                         vector<xe::TestCaseResultHeader>        headers;
 315                         bool                                                            allEqual        = true;
 316
 317                         getTestResultHeaders(headers, results, caseName.c_str());
 318
 319                         for (vector<xe::TestCaseResultHeader>::const_iterator iter = headers.begin()+1; iter != headers.end(); iter++)
 320                         {
 321                                 if (iter->statusCode != headers[0].statusCode)
 322                                 {
 323                                         allEqual = false;
 324                                         break;
 325                                 }
 326                         }
 327
 328                         if (allEqual)
 329                                 numEqual += 1;
 330
 331                         if (cmdLine.outMode == OUTPUTMODE_ALL || !allEqual)
 332                         {
 333                                 if (cmdLine.outFormat == OUTPUTFORMAT_TEXT)
 334                                 {
 335                                         dst << caseName << "\n";
 336                                         for (int ndx = 0; ndx < (int)headers.size(); ndx++)
 337                                                 dst << "  " << batchNames[ndx] << ": " << getStatusCodeName(headers[ndx].statusCode) << " (" << headers[ndx].statusDetails << ")\n";
 338                                         dst << "\n";
 339                                 }
 340                                 else if (cmdLine.outFormat == OUTPUTFORMAT_CSV)
 341                                 {
 342                                         dst << caseName;
 343                                         for (vector<xe::TestCaseResultHeader>::const_iterator iter = headers.begin(); iter != headers.end(); iter++)
 344                                                 dst << "," << (cmdLine.outValue == OUTPUTVALUE_STATUS_CODE ? getStatusCodeName(iter->statusCode) : iter->statusDetails.c_str());
 345                                         dst << "\n";
 346                                 }
 347                         }
 348                 }
 349
 350                 compareOk = numEqual == numCases;
 351
 352                 if (cmdLine.outFormat == OUTPUTFORMAT_TEXT)
 353                 {
 354                         dst << "  " << numEqual << " / " << numCases << " test case results match.\n";
 355                         dst << "  Comparison " << (compareOk ? "passed" : "FAILED") << "!\n";
 356                 }
 357         }
 358         catch (const std::exception& e)
 359         {
 360                 printf("%s\n", e.what());
 361                 compareOk = false;
 362         }
 363
 364         return compareOk;
 365 }
 366
 367 static bool parseCommandLine (CommandLine& cmdLine, int argc, const char* const* argv)
 368 {
 369         de::cmdline::Parser                     parser;
 370         de::cmdline::CommandLine        opts;
 371
 372         XE_CHECK(argc >= 1);
 373
 374         opt::registerOptions(parser);
 375
 376         if (!parser.parse(argc-1, &argv[1], &opts, std::cerr)   ||
 377                 opts.getArgs().empty())
 378         {
 379                 std::cout << argv[0] << ": [options] [filenames]\n";
 380                 parser.help(std::cout);
 381                 return false;
 382         }
 383
 384         cmdLine.outFormat       = opts.getOption<opt::OutFormat>();
 385         cmdLine.outMode         = opts.getOption<opt::OutMode>();
 386         cmdLine.outValue        = opts.getOption<opt::OutValue>();
 387         cmdLine.filenames       = opts.getArgs();
 388
 389         return true;
 390 }
 391
 392 int main (int argc, const char* const* argv)
 393 {
 394         CommandLine cmdLine;
 395
 396         if (!parseCommandLine(cmdLine, argc, argv))
 397                 return -1;
 398
 399         try
 400         {
 401                 bool compareOk = runCompare(cmdLine, std::cout);
 402                 return compareOk ? 0 : -1;
 403         }
 404         catch (const std::exception& e)
 405         {
 406                 printf("FATAL ERROR: %s\n", e.what());
 407                 return -1;
 408         }
 409 }