From 684608ba6459c8a987ead861a7efa22979e79c4c Mon Sep 17 00:00:00 2001 From: Yi Sun Date: Tue, 1 Apr 2014 16:31:26 +0800 Subject: [PATCH] Statistics of case running summary: ----------------- 1. Add struct RStatistics to count passed number(passCount), failed number(failCount), finished run number(finishrun). 2. Print statistics line , if the termial is too narrow, doesn't print it: ...... test_load_program_from_bin() [SUCCESS] profiling_exec() [SUCCESS] enqueue_copy_buf() [SUCCESS] [run/total: 656/656] pass: 629; fail: 25; pass rate: 0.961890 3. If case crashes, count it as failed, add the function to show statistic summary. 4. When all cases finished, list a summary like follows: summary: ---------- total: 656 run: 656 pass: 629 fail: 25 pass rate: 0.961890 5. If ./utest_run &> log, the log will be a little messy, tring the following command to analyse the log: sed 's/\r/\n/g' log | egrep "\w*\(\)" | sed -e 's/\s//g' After analysed: ----------------- ...... builtin_minmag_float2()[SUCCESS] builtin_minmag_float4()[SUCCESS] builtin_minmag_float8()[SUCCESS] builtin_minmag_float16()[SUCCESS] builtin_nextafter_float()[FAILED] builtin_nextafter_float2()[FAILED] builtin_nextafter_float4()[FAILED] ...... 6. Fix one issue, print out the crashed case name. 7. Delete the debug line in utests/compiler_basic_arithmetic.cpp, which output the kernel name. 8. Define function statistics() in struct UTest, which called by "utest_run -a/-c/-n". We just call this function to run each case, and print the statistics line. Reviewed-by: "Song, Ruiling" Reviewed-by: Zhigang Gong --- utests/compiler_basic_arithmetic.cpp | 1 - utests/utest.cpp | 88 ++++++++++++++++++++++++++++++------ utests/utest.hpp | 26 +++++++++-- 3 files changed, 94 insertions(+), 21 deletions(-) diff --git a/utests/compiler_basic_arithmetic.cpp b/utests/compiler_basic_arithmetic.cpp index 0e5ec41..ba05de0 100644 --- a/utests/compiler_basic_arithmetic.cpp +++ b/utests/compiler_basic_arithmetic.cpp @@ -15,7 +15,6 @@ static void test_exec(const char* kernel_name) // Setup kernel and buffers OCL_CREATE_KERNEL_FROM_FILE("compiler_basic_arithmetic", kernel_name); -std::cout <<"kernel name: " << kernel_name << std::endl; buf_data[0] = (T*) malloc(sizeof(T) * n); buf_data[1] = (T*) malloc(sizeof(T) * n); for (uint32_t i = 0; i < n; ++i) ((T*)buf_data[0])[i] = (T) rand(); diff --git a/utests/utest.cpp b/utests/utest.cpp index 718916f..e747309 100644 --- a/utests/utest.cpp +++ b/utests/utest.cpp @@ -26,14 +26,35 @@ #include #include #include +#include +#include #include +#define MAX_SUM_LINE 256 + using namespace std; vector *UTest::utestList = NULL; +// Initialize and declare statistics struct +RStatistics UTest::retStatistics; + void releaseUTestList(void) { delete UTest::utestList; } +void runAllNoIssueAtExit(void) { + // If case crashes, count it as fail, and accumulate finishrun + if(UTest::retStatistics.finishrun != UTest::utestList->size()) { + UTest::retStatistics.finishrun++; + UTest::retStatistics.failCount++; + } + printf("\nsummary:\n----------\n"); + printf(" total: %zu\n",UTest::utestList->size()); + printf(" run: %zu\n",UTest::retStatistics.finishrun); + printf(" pass: %zu\n",UTest::retStatistics.passCount); + printf(" fail: %zu\n",UTest::retStatistics.failCount); + printf(" pass rate: %f\n",1-(float)UTest::retStatistics.failCount/(float)UTest::utestList->size()); + } UTest::UTest(Function fn, const char *name, bool haveIssue, bool needDestroyProgram) : fn(fn), name(name), haveIssue(haveIssue), needDestroyProgram(needDestroyProgram) { + if (utestList == NULL) { utestList = new vector; atexit(releaseUTestList); @@ -46,16 +67,53 @@ static bool strequal(const char *s1, const char *s2) { return false; } +void UTest::do_run(struct UTest utest){ + // winsize is a struct in ioctl.h, contains terminal column number + struct winsize size; + char spaceList[MAX_SUM_LINE] = {0}; + + //Obtain terminal column size + ioctl(STDOUT_FILENO, TIOCGWINSZ, &size); + + //A string contain MAX_SUM_LINE spaces, to hide the statistic line in stdout + for (size_t j = 0; j < size.ws_col; j++){ + if ( j >= MAX_SUM_LINE - 1 ) break; + spaceList[j] = ' '; + spaceList[j+1] = '\0'; + } + printf("\r%s\r%s()", spaceList, utest.name); + + // Run one case in utestList + (utest.fn)(); + + // Print dynamic statistics line + sprintf(spaceList, "\n [run/total: %zu/%zu]\ + pass: %zu; fail: %zu; pass rate: %f\r", + retStatistics.finishrun+1, utestList->size(), + retStatistics.passCount, + retStatistics.failCount, + 1-(float)retStatistics.failCount/(float)utestList->size()); + + // If terminal column size lower than length of statistic line, print nothing, If not, print the statistics line + if (size.ws_col > strlen(spaceList)) + printf("%s", spaceList); + else + printf("\n"); + + // Refresh console + fflush(stdout); +} + void UTest::run(const char *name) { if (name == NULL) return; if (utestList == NULL) return; - for (size_t i = 0; i < utestList->size(); ++i) { - const UTest &utest = (*utestList)[i]; + atexit(runAllNoIssueAtExit); + + for (; retStatistics.finishrun < utestList->size(); ++retStatistics.finishrun) { + const UTest &utest = (*utestList)[retStatistics.finishrun]; if (utest.name == NULL || utest.fn == NULL ) continue; if (strequal(utest.name, name)) { - std::cout << utest.name << ":" << std::endl; - (utest.fn)(); - std::cout << std::endl; + do_run(utest); cl_kernel_destroy(true); cl_buffer_destroy(); } @@ -64,12 +122,12 @@ void UTest::run(const char *name) { void UTest::runAll(void) { if (utestList == NULL) return; - for (size_t i = 0; i < utestList->size(); ++i) { - const UTest &utest = (*utestList)[i]; + atexit(runAllNoIssueAtExit); + + for (; retStatistics.finishrun < utestList->size(); ++retStatistics.finishrun) { + const UTest &utest = (*utestList)[retStatistics.finishrun]; if (utest.fn == NULL) continue; - std::cout << utest.name << ":" << std::endl; - (utest.fn)(); - std::cout << std::endl; + do_run(utest); cl_kernel_destroy(utest.needDestroyProgram); cl_buffer_destroy(); } @@ -77,12 +135,12 @@ void UTest::runAll(void) { void UTest::runAllNoIssue(void) { if (utestList == NULL) return; - for (size_t i = 0; i < utestList->size(); ++i) { - const UTest &utest = (*utestList)[i]; + atexit(runAllNoIssueAtExit); + + for (; retStatistics.finishrun < utestList->size(); ++retStatistics.finishrun) { + const UTest &utest = (*utestList)[retStatistics.finishrun]; if (utest.fn == NULL || utest.haveIssue) continue; - std::cout << utest.name << ":" << std::endl; - (utest.fn)(); - std::cout << std::endl; + do_run(utest); cl_kernel_destroy(utest.needDestroyProgram); cl_buffer_destroy(); } diff --git a/utests/utest.hpp b/utests/utest.hpp index 01d4a8c..e6a7749 100644 --- a/utests/utest.hpp +++ b/utests/utest.hpp @@ -31,6 +31,14 @@ #include #include +/*! struct for statistics */ +struct RStatistics +{ + size_t passCount; + size_t failCount; + size_t finishrun; +}; + /*! Quick and dirty unit test system with registration */ struct UTest { @@ -58,6 +66,10 @@ struct UTest static void runAll(void); /*! List all test cases */ static void listAllCases(void); + /*! Statistics struct */ + static RStatistics retStatistics; + /*! Do run a test case actually */ + static void do_run(struct UTest utest); }; /*! Register a new unit test */ @@ -84,11 +96,13 @@ struct UTest do { \ try { \ EXPR; \ - std::cout << " " << #EXPR << " [SUCCESS]" << std::endl; \ + std::cout << " [SUCCESS]"; \ + UTest::retStatistics.passCount += 1; \ } \ catch (Exception e) { \ - std::cout << " " << #EXPR << " [FAILED]" << std::endl; \ - std::cout << " " << e.what() << std::endl; \ + std::cout << " [FAILED]"; \ + std::cout << "\n " << e.what(); \ + UTest::retStatistics.failCount++; \ } \ } while (0) @@ -96,10 +110,12 @@ struct UTest do { \ try { \ EXPR; \ - std::cout << " " << #EXPR << " [FAILED]" << std::endl; \ + std::cout << " [FAILED]"; \ + retStatistics.failCount++; \ } \ catch (gbe::Exception e) { \ - std::cout << " " << #EXPR << " [SUCCESS]" << std::endl; \ + std::cout << " [SUCCESS]"; \ + retStatistics.passCount++; \ } \ } while (0) -- 2.7.4