From c6a5a1f29f91514723de525af90336f5997e49c0 Mon Sep 17 00:00:00 2001 From: Marcin Niesluchowski Date: Wed, 15 Oct 2014 13:14:45 +0200 Subject: [PATCH] Adjust Summary collector to test framework logic Change-Id: Ia48421e8036bafe56cc8f2bffad7c7d70f4a3efe --- packaging/security-tests.spec | 1 - tests/CMakeLists.txt | 11 --- .../dpl/test/test_results_collector_summary.h | 52 +++++++------ .../src/test_results_collector_summary.cpp | 90 ++++++++++++---------- tests/security-tests-all.sh | 74 +++++++++++++----- tests/tests-summary.sh | 25 ------ 6 files changed, 131 insertions(+), 122 deletions(-) delete mode 100644 tests/tests-summary.sh diff --git a/packaging/security-tests.spec b/packaging/security-tests.spec index 78e840c..8b85a08 100644 --- a/packaging/security-tests.spec +++ b/packaging/security-tests.spec @@ -72,7 +72,6 @@ echo "security-tests postinst done ..." %defattr(-, root, root, -) /usr/bin/security-tests.sh /usr/bin/security-tests-all.sh -/usr/bin/tests-summary.sh /usr/bin/test-performance-check.sh /etc/dbus-1/system.d/security-tests.conf diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 10d38a8..cff3c9a 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -68,17 +68,6 @@ INSTALL(FILES ${PROJECT_SOURCE_DIR}/tests/security-tests-all.sh WORLD_EXECUTE ) -INSTALL(FILES ${PROJECT_SOURCE_DIR}/tests/tests-summary.sh - DESTINATION bin - PERMISSIONS OWNER_READ - OWNER_WRITE - OWNER_EXECUTE - GROUP_READ - GROUP_EXECUTE - WORLD_READ - WORLD_EXECUTE - ) - INSTALL(FILES ${PROJECT_SOURCE_DIR}/tests/test-performance-check.sh DESTINATION bin diff --git a/tests/framework/include/dpl/test/test_results_collector_summary.h b/tests/framework/include/dpl/test/test_results_collector_summary.h index 617c17f..35e7ba9 100644 --- a/tests/framework/include/dpl/test/test_results_collector_summary.h +++ b/tests/framework/include/dpl/test/test_results_collector_summary.h @@ -24,39 +24,43 @@ #ifndef DPL_TEST_RESULTS_COLLECTOR_SUMMARY_H #define DPL_TEST_RESULTS_COLLECTOR_SUMMARY_H +#include +#include + +#include #include namespace DPL { namespace Test { -/* - * Custom test runner results collector. The results (total test cases, failed, etc.) are - * appended to a file, wihich is parsed after execution of all tests in oreder to - * display summary view. - */ -class SummaryCollector : public TestResultsCollectorBase +class SummaryCollector + : public TestResultsCollectorBase { - unsigned int m_total, m_succeeded, m_failed, m_ignored; // counters +public: + static TestResultsCollectorBase* Constructor(); +private: SummaryCollector(); - void Start(); - void Finish(); - void CollectResult(const std::string& /*id*/, - const std::string& /*description*/, - const FailStatus status, - const std::string& /*reason = ""*/); - void CollectResult(const std::string& /*id*/, - const std::string& /*description*/, - const FailStatus status, - const std::string& /*reason = ""*/, - const bool& isPerformanceTest = false, - const std::chrono::system_clock::duration& performanceTime - = std::chrono::microseconds::zero(), - const std::chrono::system_clock::duration& performanceMaxTime - = std::chrono::microseconds::zero()); -public: - static TestResultsCollectorBase* Constructor(); + virtual std::string CollectorSpecificHelp() const; + virtual bool ParseCollectorSpecificArg(const std::string& arg); + virtual void Start(); + virtual void CollectResult(const std::string& id, + const std::string& description, + const FailStatus status = FailStatus::NONE, + const std::string& reason = "", + const bool& isPerformanceTest = false, + const std::chrono::system_clock::duration& performanceTime + = std::chrono::microseconds::zero(), + const std::chrono::system_clock::duration& performanceMaxTime + = std::chrono::microseconds::zero()); + virtual void Finish(); + + void writeStats(bool segfault); + + Statistic m_stats; + std::string m_filename; + std::ofstream m_output; }; } // namespace Test diff --git a/tests/framework/src/test_results_collector_summary.cpp b/tests/framework/src/test_results_collector_summary.cpp index e478112..05b9cc7 100644 --- a/tests/framework/src/test_results_collector_summary.cpp +++ b/tests/framework/src/test_results_collector_summary.cpp @@ -21,7 +21,9 @@ * @brief Source file containing SummaryCollector class definition. */ -#include +#include +#include +#include #include "dpl/test/test_results_collector_summary.h" @@ -30,72 +32,76 @@ namespace Test { namespace { - const char* summaryFileName = "/tmp/security-tests-summary-file"; +const std::string DEFAULT_SUMMARY_FILE_NAME = "summary.txt"; } SummaryCollector::SummaryCollector() + : m_filename(DEFAULT_SUMMARY_FILE_NAME) { - Start(); } +TestResultsCollectorBase* SummaryCollector::Constructor() +{ + return new SummaryCollector(); +} -// Overrides TestResultsCollectorBase::Start() virtual method. -void SummaryCollector::Start() +std::string SummaryCollector::CollectorSpecificHelp() const { - m_total = m_succeeded = m_failed = m_ignored = 0; + return "--file= - name of file for output\n" + " default - " + DEFAULT_SUMMARY_FILE_NAME + "\n"; } -// append results to the file -void SummaryCollector::Finish() +bool SummaryCollector::ParseCollectorSpecificArg(const std::string& arg) { - std::ofstream outputFile; - outputFile.open(summaryFileName, std::ofstream::out | std::ofstream::app); - if (!outputFile) - return; //failed to open file - - outputFile << m_total << ' ' - << m_succeeded << ' ' - << m_failed << ' ' - << m_ignored << std::endl; - outputFile.close(); + return ParseCollectorFileArg(arg, m_filename); } -void SummaryCollector::CollectResult(const std::string& /*id*/, - const std::string& /*description*/, +void SummaryCollector::Start() +{ + writeStats(true); +} + +void SummaryCollector::CollectResult(const std::string& id, + const std::string& description, const FailStatus status, - const std::string& /*reason = ""*/, + const std::string& reason, const bool& isPerformanceTest, const std::chrono::system_clock::duration& performanceTime, const std::chrono::system_clock::duration& performanceMaxTime) { - (void)isPerformanceTest; - (void)performanceTime; - (void)performanceMaxTime; - switch (status) { - case FailStatus::IGNORED: ++m_ignored; break; - case FailStatus::FAILED: ++m_failed; break; - case FailStatus::NONE: ++m_succeeded; break; - }; - ++m_total; + DPL_UNUSED_PARAM(id); + DPL_UNUSED_PARAM(description); + DPL_UNUSED_PARAM(reason); + DPL_UNUSED_PARAM(isPerformanceTest); + DPL_UNUSED_PARAM(performanceTime); + DPL_UNUSED_PARAM(performanceMaxTime); + + m_stats.AddTest(status); + writeStats(true); } -void SummaryCollector::CollectResult(const std::string& /*id*/, - const std::string& /*description*/, - const FailStatus status, - const std::string& /*reason = ""*/) +void SummaryCollector::Finish() { - switch (status) { - case FailStatus::IGNORED: ++m_ignored; break; - case FailStatus::FAILED: ++m_failed; break; - case FailStatus::NONE: ++m_succeeded; break; - }; - ++m_total; + writeStats(false); } -TestResultsCollectorBase* SummaryCollector::Constructor() +void SummaryCollector::writeStats(bool segfault) { - return new SummaryCollector(); + m_output.open(m_filename.c_str(), std::ofstream::out | std::ofstream::trunc); + if (!m_output) { + LogPedantic("Could not open file " << m_filename << " for writing"); + return; + } + m_output << m_stats.GetTotal() << ' ' + << m_stats.GetPassed() << ' ' + << m_stats.GetFailed() << ' ' + << m_stats.GetIgnored() << (segfault ? " segfault" : ""); + if (!m_output) + LogPedantic("Writing to " << m_filename << " file failed"); + m_output.close(); + if (!m_output) + LogPedantic("Failed to close " << m_filename << " file"); } } // namespace Test diff --git a/tests/security-tests-all.sh b/tests/security-tests-all.sh index 61500ea..a1fe677 100644 --- a/tests/security-tests-all.sh +++ b/tests/security-tests-all.sh @@ -1,10 +1,10 @@ #!/bin/sh echo "[Trigerring all tests...]" -summary_file="/tmp/security-tests-summary-file" - -#delete summary file -rm -f $summary_file +total=0 +passed=0 +failed=0 +ignored=0 ign="--runignored" if [ $# -gt 0 ]; then @@ -13,20 +13,56 @@ if [ $# -gt 0 ]; then fi fi -# 'text' - console output -# 'summary' - used for summary view -security-tests.sh smack --output=text --output=summary $ign -security-tests.sh smack-dbus --output=text --output=summary $ign -#security-tests.sh libprivilege-control --output=text --output=summary $ign -#security-tests.sh ss-clientsmack --output=text --output=summary $ign -#security-tests.sh ss-server --output=text --output=summary $ign -#security-tests.sh ss-password --output=text --output=summary $ign -#security-tests.sh ss-privilege --output=text --output=summary $ign -#security-tests.sh ss-stress --output=text --output=summary $ign -security-tests.sh security-manager --output=text --output=summary $ign -security-tests.sh cynara --output=text --output=summary $ign - -# print summary -tests-summary.sh $summary_file +function addSummary +{ + read -a words < summary.txt + total=$((total + words[0])) + passed=$((passed + words[1])) + failed=$((failed + words[2])) + ignored=$((ignored + words[3])) + + rm summary.txt +} + +function runTest +{ + # 'text' - console output + # 'summary' - used for summary view + security-tests.sh "$1" --output=text --output=summary "$ign" + addSummary +} + +function printSummary +{ + COLOR_GREEN_START="\033[1;32m" + COLOR_DARK_GREEN_START="\033[0;36m" + COLOR_END="\033[m" + + printf "\n" + printf "$COLOR_GREEN_START" + printf "Summary\n" + printf "$COLOR_END" + + printf "$COLOR_DARK_GREEN_START" + printf " Total: %i\n" "$total" + printf " Passed: %i\n" "$passed" + printf " Failed: %i\n" "$failed" + printf " Ignored: %i\n" "$ignored" + printf "$COLOR_END" + printf "\n" +} + +runTest smack +runTest smack-dbus +#runTest libprivilege-control +#runTest ss-clientsmack +#runTest ss-server +#runTest ss-password +#runTest ss-privilege +#runTest ss-stress +runTest security-manager +runTest cynara + +printSummary echo "[Done]" diff --git a/tests/tests-summary.sh b/tests/tests-summary.sh deleted file mode 100644 index a6ece36..0000000 --- a/tests/tests-summary.sh +++ /dev/null @@ -1,25 +0,0 @@ -#!/bin/sh - -# sum up columns -total=$(awk '{counter += $1} END {print counter}' "$1" ) -succeeded=$(awk '{counter += $2} END {print counter}' "$1" ) -failed=$(awk '{counter += $3} END {print counter}' "$1" ) -ignored=$(awk '{counter += $4} END {print counter}' "$1" ) - -#color constants -COLOR_GREEN_START="\033[1;32m" -COLOR_DARK_GREEN_START="\033[0;36m" -COLOR_END="\033[m" - -printf $COLOR_GREEN_START -printf "Summary\n" -printf $COLOR_END - -printf $COLOR_DARK_GREEN_START -printf " Total: %i\n" $total -printf " Succeeded: %i\n" $succeeded -printf " Failed: %i\n" $failed -printf " Ignored: %i\n" $ignored -printf $COLOR_END - -printf "\n" -- 2.7.4