From 1f83ced0374bff81725dd25e2fe68a406d567716 Mon Sep 17 00:00:00 2001 From: Michal Witanowski Date: Wed, 20 Nov 2013 11:52:55 +0100 Subject: [PATCH] Add summary view to all security tests. [Issue#] SSDWSSP-522 [Bug/Feature] Running security tests via security-tests-all.sh prints summary view (all tests results summed together). [Cause] N/A [Solution] A new results collector for WRT tests runner was created (SummaryCollector in summary_collector.h). The collector counts all the test cases results and appends a line to a temporary file containing: number of all test cases, succeeded, failed and ignored. Then, after the tests are executed, tests-summary.sh sums the columns and prints summary in the console. [Verification] Build, install, run all security tests via security-tests-all.sh and wait for summary at the very end of script output. Change-Id: Id44cfb90acbc426e17e1971807e6eef0496c7805 --- packaging/security-tests.spec | 1 + tests/CMakeLists.txt | 11 +++ tests/common/CMakeLists.txt | 1 + tests/common/summary_collector.cpp | 83 ++++++++++++++++++++++ tests/common/summary_collector.h | 51 +++++++++++++ .../libprivilege-control-test.cpp | 3 + tests/libsmack-tests/libsmack-test.cpp | 2 + .../security_server_measurer_API_speed.cpp | 2 + .../security_server_tests_client_smack.cpp | 8 +-- .../security_server_tests_dbus.cpp | 6 +- .../security_server_tests_password.cpp | 6 +- .../security_server_tests_stress.cpp | 6 +- tests/security-server-tests/server.cpp | 2 + tests/security-tests-all.sh | 25 +++++-- tests/security-tests.sh | 16 ++--- tests/tests-summary.sh | 25 +++++++ 16 files changed, 220 insertions(+), 28 deletions(-) create mode 100644 tests/common/summary_collector.cpp create mode 100644 tests/common/summary_collector.h create mode 100644 tests/tests-summary.sh diff --git a/packaging/security-tests.spec b/packaging/security-tests.spec index e86f39c..a21c76e 100644 --- a/packaging/security-tests.spec +++ b/packaging/security-tests.spec @@ -75,6 +75,7 @@ osp-installer -u V5LKqDFBXm %defattr(-, root, root, -) /usr/bin/security-tests.sh /usr/bin/security-tests-all.sh +/usr/bin/tests-summary.sh /usr/bin/test-performance-check.sh /usr/bin/perf diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 251ebed..27b1f8b 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -35,6 +35,17 @@ INSTALL(FILES ${PROJECT_SOURCE_DIR}/tests/security-tests-all.sh WORLD_EXECUTE ) +INSTALL(FILES ${PROJECT_SOURCE_DIR}/tests/tests-summary.sh + DESTINATION bin + PERMISSIONS OWNER_READ + OWNER_WRITE + OWNER_EXECUTE + GROUP_READ + GROUP_EXECUTE + WORLD_READ + WORLD_EXECUTE + ) + INSTALL(FILES ${PROJECT_SOURCE_DIR}/tests/test-performance-check.sh ${PROJECT_SOURCE_DIR}/tests/perf diff --git a/tests/common/CMakeLists.txt b/tests/common/CMakeLists.txt index 0727e76..7f63585 100644 --- a/tests/common/CMakeLists.txt +++ b/tests/common/CMakeLists.txt @@ -13,6 +13,7 @@ SET(COMMON_TARGET_TEST_SOURCES ${PROJECT_SOURCE_DIR}/tests/common/tests_common.cpp ${PROJECT_SOURCE_DIR}/tests/common/access_provider.cpp ${PROJECT_SOURCE_DIR}/tests/common/smack_access.cpp + ${PROJECT_SOURCE_DIR}/tests/common/summary_collector.cpp ) #header directories diff --git a/tests/common/summary_collector.cpp b/tests/common/summary_collector.cpp new file mode 100644 index 0000000..b4d69ac --- /dev/null +++ b/tests/common/summary_collector.cpp @@ -0,0 +1,83 @@ +/* + * Copyright (c) 2013 Samsung Electronics Co., Ltd All Rights Reserved + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * @file summary_collector.cpp + * @author Michal Witanowski (m.witanowski@samsung.com) + * @version 1.0 + * @brief Implementation of custom test results collector needed by summary view. + */ + +#include +#include + +namespace { + const char* summaryFileName = "/tmp/security-tests-summary-file"; + const char* summaryCollectorName = "summary"; +}; + +SummaryCollector::SummaryCollector() +{ + Start(); +} + + +// Overrides DPL::Test::TestResultsCollectorBase::Start() virtual method. +void SummaryCollector::Start() +{ + m_total = m_succeeded = m_failed = m_ignored = 0; +} + +// append results to the file +void SummaryCollector::Finish() +{ + std::ofstream outputFile; + outputFile.open(summaryFileName, std::ofstream::out | std::ofstream::app); + if (!outputFile) + return; //failed to open file + + outputFile << m_total << ' ' + << m_succeeded << ' ' + << m_failed << ' ' + << m_ignored << std::endl; + outputFile.close(); +} + +void SummaryCollector::CollectResult(const std::string& /*id*/, + const std::string& /*description*/, + const FailStatus::Type status, + const std::string& /*reason = ""*/) +{ + switch (status) { + case FailStatus::IGNORED: ++m_ignored; break; + case FailStatus::INTERNAL: // internal error count as fail + case FailStatus::FAILED: ++m_failed; break; + case FailStatus::NONE: ++m_succeeded; break; + }; + ++m_total; +} + +DPL::Test::TestResultsCollectorBase* SummaryCollector::Constructor() +{ + return new SummaryCollector(); +} + +void SummaryCollector::Register() +{ + //register custom results collector + DPL::Test::TestResultsCollectorBase::RegisterCollectorConstructor(summaryCollectorName, + &SummaryCollector::Constructor); +} diff --git a/tests/common/summary_collector.h b/tests/common/summary_collector.h new file mode 100644 index 0000000..3a1cdf9 --- /dev/null +++ b/tests/common/summary_collector.h @@ -0,0 +1,51 @@ +/* + * Copyright (c) 2013 Samsung Electronics Co., Ltd All Rights Reserved + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * @file summary_collector.h + * @author Michal Witanowski (m.witanowski@samsung.com) + * @version 1.0 + * @brief Implementation of custom test results collector needed by summary view. + */ + +#ifndef _RAW_RESULTS_COLLECTOR_H_ +#define _RAW_RESULTS_COLLECTOR_H_ + +#include + +/* + * Custom test runner results collector. The results (total test cases, failed, etc.) are + * appended to a file, wihich is parsed after execution of all tests in oreder to + * display summary view. + */ +class SummaryCollector : public DPL::Test::TestResultsCollectorBase +{ + unsigned int m_total, m_succeeded, m_failed, m_ignored; // counters + + SummaryCollector(); + void Start(); + void Finish(); + void CollectResult(const std::string& /*id*/, + const std::string& /*description*/, + const FailStatus::Type status, + const std::string& /*reason = ""*/); + +public: + static TestResultsCollectorBase* Constructor(); + static void Register(); +}; + +#endif diff --git a/tests/libprivilege-control-tests/libprivilege-control-test.cpp b/tests/libprivilege-control-tests/libprivilege-control-test.cpp index 1f1f18f..57e9780 100644 --- a/tests/libprivilege-control-tests/libprivilege-control-test.cpp +++ b/tests/libprivilege-control-tests/libprivilege-control-test.cpp @@ -22,10 +22,13 @@ #include #include +#include int main (int argc, char *argv[]) { LogInfo("Starting libprivilege-control tests"); + + SummaryCollector::Register(); int status = DPL::Test::TestRunnerSingleton::Instance().ExecTestRunner(argc, argv); return status; } diff --git a/tests/libsmack-tests/libsmack-test.cpp b/tests/libsmack-tests/libsmack-test.cpp index 93c853d..556e2a1 100644 --- a/tests/libsmack-tests/libsmack-test.cpp +++ b/tests/libsmack-tests/libsmack-test.cpp @@ -20,9 +20,11 @@ * @brief libsmack test runer */ #include +#include int main (int argc, char *argv[]) { + SummaryCollector::Register(); int status = DPL::Test::TestRunnerSingleton::Instance().ExecTestRunner(argc, argv); return status; } diff --git a/tests/security-server-tests/security_server_measurer_API_speed.cpp b/tests/security-server-tests/security_server_measurer_API_speed.cpp index 3ea2130..27d0aa3 100644 --- a/tests/security-server-tests/security_server_measurer_API_speed.cpp +++ b/tests/security-server-tests/security_server_measurer_API_speed.cpp @@ -50,6 +50,7 @@ #include #include #include "security_server_mockup.h" +#include /*Number of calls in a single test*/ #define NUMBER_OF_CALLS (5) @@ -718,6 +719,7 @@ RUNNER_TEST(m170_security_server_check_privilege_by_pid) { int main(int argc, char *argv[]) { + SummaryCollector::Register(); securityClientEnableLogSystem(); DPL::Test::TestRunnerSingleton::Instance().ExecTestRunner(argc, argv); return 0; diff --git a/tests/security-server-tests/security_server_tests_client_smack.cpp b/tests/security-server-tests/security_server_tests_client_smack.cpp index 5d35cd7..2f977f2 100644 --- a/tests/security-server-tests/security_server_tests_client_smack.cpp +++ b/tests/security-server-tests/security_server_tests_client_smack.cpp @@ -29,11 +29,11 @@ #include "security_server_mockup.h" #include - #include #include - #include "tests_common.h" +#include + #define PROPER_COOKIE_SIZE 20 @@ -952,6 +952,6 @@ RUNNER_TEST_NOSMACK(tc18_security_server_get_smacklabel_cookie_nosmack) { int main(int argc, char *argv[]) { - return - DPL::Test::TestRunnerSingleton::Instance().ExecTestRunner(argc, argv); + SummaryCollector::Register(); + return DPL::Test::TestRunnerSingleton::Instance().ExecTestRunner(argc, argv); } diff --git a/tests/security-server-tests/security_server_tests_dbus.cpp b/tests/security-server-tests/security_server_tests_dbus.cpp index be179db..cc8f329 100644 --- a/tests/security-server-tests/security_server_tests_dbus.cpp +++ b/tests/security-server-tests/security_server_tests_dbus.cpp @@ -7,6 +7,7 @@ #include #include #include +#include #include "tests_common.h" #include "security-server.h" #include "privilege-control.h" @@ -348,7 +349,6 @@ RUNNER_MULTIPROCESS_TEST_NOSMACK(tc01_smack_context_from_DBus_nosmack) int main(int argc, char *argv[]) { - int status = DPL::Test::TestRunnerSingleton::Instance().ExecTestRunner(argc, argv); - - return status; + SummaryCollector::Register(); + return DPL::Test::TestRunnerSingleton::Instance().ExecTestRunner(argc, argv); } diff --git a/tests/security-server-tests/security_server_tests_password.cpp b/tests/security-server-tests/security_server_tests_password.cpp index 4074e1e..860f8dd 100644 --- a/tests/security-server-tests/security_server_tests_password.cpp +++ b/tests/security-server-tests/security_server_tests_password.cpp @@ -32,6 +32,7 @@ #include #include "security_server_clean_env.h" #include +#include // security server retry timeout in microseconds @@ -1493,7 +1494,6 @@ RUNNER_TEST(tc49_security_server_no_retry_timeout_set_pwd_max_challenge) int main(int argc, char *argv[]) { - int status = DPL::Test::TestRunnerSingleton::Instance().ExecTestRunner(argc, argv); - - return status; + SummaryCollector::Register(); + return DPL::Test::TestRunnerSingleton::Instance().ExecTestRunner(argc, argv); } diff --git a/tests/security-server-tests/security_server_tests_stress.cpp b/tests/security-server-tests/security_server_tests_stress.cpp index da3959b..dfa6996 100644 --- a/tests/security-server-tests/security_server_tests_stress.cpp +++ b/tests/security-server-tests/security_server_tests_stress.cpp @@ -12,6 +12,7 @@ #include #include #include +#include #include #include #include @@ -21,7 +22,6 @@ #include #include "security-server.h" - std::mutex g_mutex; std::mutex g_msgMutex; size_t g_successes = 0; @@ -213,7 +213,7 @@ RUNNER_CHILD_TEST_NOSMACK(tc_stress_cookie_api_no_smack) int main (int argc, char *argv[]) { - int status = DPL::Test::TestRunnerSingleton::Instance().ExecTestRunner(argc, argv); - return status; + SummaryCollector::Register(); + return DPL::Test::TestRunnerSingleton::Instance().ExecTestRunner(argc, argv); } diff --git a/tests/security-server-tests/server.cpp b/tests/security-server-tests/server.cpp index f86cb10..76f733b 100644 --- a/tests/security-server-tests/server.cpp +++ b/tests/security-server-tests/server.cpp @@ -34,6 +34,7 @@ #include "tests_common.h" #include #include +#include const char *TEST03_SUBJECT = "subject_0f09f7cc"; const char *TEST04_SUBJECT = "subject_57dfbfc5"; @@ -850,5 +851,6 @@ int main(int argc, char *argv[]) { printf("Error: %s must be executed by root\n", argv[0]); exit(1); } + SummaryCollector::Register(); return DPL::Test::TestRunnerSingleton::Instance().ExecTestRunner(argc, argv); } diff --git a/tests/security-tests-all.sh b/tests/security-tests-all.sh index d1c4bab..32063c0 100644 --- a/tests/security-tests-all.sh +++ b/tests/security-tests-all.sh @@ -1,6 +1,11 @@ #!/bin/sh echo "[Trigerring all tests...]" +summary_file="/tmp/security-tests-summary-file" + +#delete summary file +rm -f $summary_file + ign="--runignored" if [ $# -gt 0 ]; then if [ "$1" = "--noignored" ]; then @@ -8,11 +13,17 @@ if [ $# -gt 0 ]; then fi fi -security-tests.sh smack --output=text $ign -security-tests.sh libprivilege-control --output=text $ign -security-tests.sh ss-clientsmack --output=text $ign -security-tests.sh ss-server --output=text $ign -security-tests.sh ss-password --output=text $ign -security-tests.sh ss-dbus --output=text $ign -security-tests.sh ss-stress --output=text $ign +# 'text' - console output +# 'summary' - used for summary view +security-tests.sh smack --output=text --output=summary $ign +security-tests.sh libprivilege-control --output=text --output=summary $ign +security-tests.sh ss-clientsmack --output=text --output=summary $ign +security-tests.sh ss-server --output=text --output=summary $ign +security-tests.sh ss-password --output=text --output=summary $ign +security-tests.sh ss-dbus --output=text --output=summary $ign +security-tests.sh ss-stress --output=text --output=summary $ign + +# print summary +tests-summary.sh $summary_file + echo "[Done]" diff --git a/tests/security-tests.sh b/tests/security-tests.sh index 8d876e9..2f87cbc 100644 --- a/tests/security-tests.sh +++ b/tests/security-tests.sh @@ -26,49 +26,49 @@ case $1 in echo "=========================================================================" echo $1 echo - libsmack-test $2 $3 + libsmack-test "${@:2}" # propagate all remaining arguments (except first) ;; "libprivilege-control") echo "=========================================================================" echo $1 echo - libprivilege-control-test $2 $3 + libprivilege-control-test "${@:2}" ;; "ss-clientsmack") echo "=========================================================================" echo "SECURITY SERVER TEST CLIENT SMACK" echo - security-server-tests-client-smack $2 $3 + security-server-tests-client-smack "${@:2}" ;; "ss-stress") echo "=========================================================================" echo "SECURITY SERVER TEST STRESS" echo - security-server-tests-stress $2 $3 + security-server-tests-stress "${@:2}" ;; "ss-server") echo "=========================================================================" echo "SECURITY SERVER TEST SERVER" echo - security-server-tests-server $2 $3 + security-server-tests-server "${@:2}" ;; "ss-api-speed") echo "=========================================================================" echo "SECURITY SERVER MEASURER SERVER" echo - security-server-tests-api-speed $2 $3 + security-server-tests-api-speed "${@:2}" ;; "ss-password") echo "=========================================================================" echo "SECURITY SERVER TEST PASSWORD" echo - security-server-tests-password $2 $3 + security-server-tests-password "${@:2}" ;; "ss-dbus") echo "=========================================================================" echo "SECURITY SERVER TEST DBUS" echo - security-server-tests-dbus $2 $3 + security-server-tests-dbus "${@:2}" ;; *) echo "Correct using:" diff --git a/tests/tests-summary.sh b/tests/tests-summary.sh new file mode 100644 index 0000000..a6ece36 --- /dev/null +++ b/tests/tests-summary.sh @@ -0,0 +1,25 @@ +#!/bin/sh + +# sum up columns +total=$(awk '{counter += $1} END {print counter}' "$1" ) +succeeded=$(awk '{counter += $2} END {print counter}' "$1" ) +failed=$(awk '{counter += $3} END {print counter}' "$1" ) +ignored=$(awk '{counter += $4} END {print counter}' "$1" ) + +#color constants +COLOR_GREEN_START="\033[1;32m" +COLOR_DARK_GREEN_START="\033[0;36m" +COLOR_END="\033[m" + +printf $COLOR_GREEN_START +printf "Summary\n" +printf $COLOR_END + +printf $COLOR_DARK_GREEN_START +printf " Total: %i\n" $total +printf " Succeeded: %i\n" $succeeded +printf " Failed: %i\n" $failed +printf " Ignored: %i\n" $ignored +printf $COLOR_END + +printf "\n" -- 2.7.4