Database Performance Test 43/143843/27
authorErnest Borowski <e.borowski@partner.samsung.com>
Fri, 11 Aug 2017 11:10:24 +0000 (13:10 +0200)
committerZofia Grzelewska <z.abramowska@samsung.com>
Mon, 19 Feb 2018 19:06:32 +0000 (20:06 +0100)
Tests are measuring performance loss when Apps count increase
Tests are measuring: Adding app, Removing app, Adding Privileges for app

Change-Id: Ia091c67a9e36f499ada7194d6d751ffe511a981c
Signed-off-by: Ernest Borowski <e.borowski@partner.samsung.com>
packaging/security-manager.spec
test/CMakeLists.txt
test/privilege_db_fixture.cpp
test/privilege_db_fixture.h
test/test_performance_db.cpp [new file with mode: 0644]

index 7df3049..a597130 100644 (file)
@@ -302,6 +302,7 @@ chsmack -a System %{db_test_dir}/.security-manager-test.db-journal
 %files -n security-manager-tests
 %manifest %{name}.manifest
 %attr(755,root,root) %{_bindir}/security-manager-unit-tests
+%attr(755,root,root) %{_bindir}/security-manager-performance-tests
 %attr(0600,root,root) %{db_test_dir}/.security-manager-test.db
 %attr(0600,root,root) %{db_test_dir}/.security-manager-test.db-journal
 
index 8edb441..49e0c34 100644 (file)
@@ -38,6 +38,7 @@ SET(SM_TEST_SRC  ${PROJECT_SOURCE_DIR}/test)
 ################################################################################
 
 SET(TARGET_SM_TESTS "security-manager-unit-tests")
+SET(TARGET_SM_PERFORMANCE_TESTS "security-manager-performance-tests")
 
 SET(SM_TESTS_SOURCES
     ${SM_TEST_SRC}/colour_log_formatter.cpp
@@ -73,16 +74,47 @@ SET(SM_TESTS_SOURCES
     ${PROJECT_SOURCE_DIR}/src/common/tzplatform-config.cpp
 )
 
+SET(SM_PERFORMANCE_TESTS_SOURCES
+    ${SM_TEST_SRC}/test_performance_db.cpp
+    ${SM_TEST_SRC}/colour_log_formatter.cpp
+    ${SM_TEST_SRC}/security-manager-tests.cpp
+    ${SM_TEST_SRC}/privilege_db_fixture.cpp
+    ${DPL_PATH}/core/src/assert.cpp
+    ${DPL_PATH}/core/src/colors.cpp
+    ${DPL_PATH}/core/src/errno_string.cpp
+    ${DPL_PATH}/core/src/exception.cpp
+    ${DPL_PATH}/core/src/noncopyable.cpp
+    ${DPL_PATH}/db/src/sql_connection.cpp
+    ${DPL_PATH}/db/src/naive_synchronization_object.cpp
+    ${DPL_PATH}/log/src/abstract_log_provider.cpp
+    ${DPL_PATH}/log/src/log.cpp
+    ${DPL_PATH}/log/src/old_style_log_provider.cpp
+    ${PROJECT_SOURCE_DIR}/src/common/config-file.cpp
+    #${PROJECT_SOURCE_DIR}/src/common/file-lock.cpp
+    ${PROJECT_SOURCE_DIR}/src/common/privilege_db.cpp
+    #${PROJECT_SOURCE_DIR}/src/common/smack-check.cpp
+    #${PROJECT_SOURCE_DIR}/src/common/smack-labels.cpp
+    #${PROJECT_SOURCE_DIR}/src/common/smack-rules.cpp
+    ${PROJECT_SOURCE_DIR}/src/common/filesystem.cpp
+    ${PROJECT_SOURCE_DIR}/src/common/tzplatform-config.cpp
+)
+
 IF(DPL_WITH_DLOG)
     SET(SM_TESTS_SOURCES
         ${SM_TESTS_SOURCES}
         ${DPL_PATH}/log/src/dlog_log_provider.cpp)
+    SET(SM_PERFORMANCE_TESTS_SOURCES
+        ${SM_PERFORMANCE_TESTS_SOURCES}
+        ${DPL_PATH}/log/src/dlog_log_provider.cpp)
 ENDIF(DPL_WITH_DLOG)
 
 IF(DPL_WITH_SYSTEMD_JOURNAL)
     SET(SM_TESTS_SOURCES
         ${SM_TESTS_SOURCES}
         ${DPL_PATH}/log/src/sd_journal_provider.cpp)
+    SET(SM_PERFORMANCE_TESTS_SOURCES
+        ${SM_PERFORMANCE_TESTS_SOURCES}
+        ${DPL_PATH}/log/src/sd_journal_provider.cpp)
 ENDIF(DPL_WITH_SYSTEMD_JOURNAL)
 
 INCLUDE_DIRECTORIES(
@@ -101,6 +133,7 @@ INCLUDE_DIRECTORIES(
 )
 
 ADD_EXECUTABLE(${TARGET_SM_TESTS} ${SM_TESTS_SOURCES})
+ADD_EXECUTABLE(${TARGET_SM_PERFORMANCE_TESTS} ${SM_PERFORMANCE_TESTS_SOURCES})
 
 TARGET_LINK_LIBRARIES(${TARGET_SM_TESTS}
     ${COMMON_DEP_LIBRARIES}
@@ -110,4 +143,12 @@ TARGET_LINK_LIBRARIES(${TARGET_SM_TESTS}
     -lcrypt
 )
 
-INSTALL(TARGETS ${TARGET_SM_TESTS} DESTINATION ${BIN_INSTALL_DIR})
+TARGET_LINK_LIBRARIES(${TARGET_SM_PERFORMANCE_TESTS}
+    ${COMMON_DEP_LIBRARIES}
+    ${DLOG_DEP_LIBRARIES}
+    boost_unit_test_framework
+    -ldl
+    -lcrypt
+)
+
+INSTALL(TARGETS ${TARGET_SM_TESTS} ${TARGET_SM_PERFORMANCE_TESTS} DESTINATION ${BIN_INSTALL_DIR})
index 732ac81..4cb0206 100644 (file)
@@ -79,6 +79,14 @@ PrivilegeDb* PrivilegeDBFixture::getPrivDb() {
     return testPrivDb;
 }
 
+void PrivilegeDBFixture::addAppNoCheck(const std::string &appName,
+    const std::string &pkgName, uid_t uid, const std::string &tizenVer,
+    const std::string &authorName, bool isHybrid)
+{
+    BOOST_REQUIRE_NO_THROW(testPrivDb->AddApplication(appName, pkgName, uid, tizenVer,
+        authorName, isHybrid));
+}
+
 void PrivilegeDBFixture::addAppSuccess(const std::string &appName,
     const std::string &pkgName, const uid_t uid, const std::string &tizenVer,
     const std::string &authorName, bool isHybrid)
@@ -132,6 +140,12 @@ void PrivilegeDBFixture::addAppFail(const std::string &appName,
     }
 }
 
+void PrivilegeDBFixture::addAppDefinedPrivileges(const std::string &appName, uid_t uid,
+    const AppDefinedPrivilegesVector &privileges)
+{
+    BOOST_REQUIRE_NO_THROW(testPrivDb->AddAppDefinedPrivileges(appName, uid, privileges));
+}
+
 void PrivilegeDBFixture::removeApp(const std::string &appName,
     const uid_t uid, bool expAppNameIsNoMore, bool expPkgNameIsNoMore, bool expAuthorNameIsNoMore)
 {
@@ -153,6 +167,13 @@ void PrivilegeDBFixture::removeApp(const std::string &appName,
          << expAuthorNameIsNoMore);
 }
 
+void PrivilegeDBFixture::removeAppNoCheck(const std::string &appName, uid_t uid)
+{
+    bool temp = false;
+    BOOST_REQUIRE_NO_THROW(testPrivDb->RemoveApplication(appName, uid, temp, temp, temp));
+}
+
+
 void PrivilegeDBFixture::removeAppSuccess(const std::string &appName,
     const uid_t uid)
 {
index 1064d08..9b56ed3 100644 (file)
@@ -38,12 +38,17 @@ public:
 
     PrivilegeDb* getPrivDb();
 
+    void addAppDefinedPrivileges(const std::string &appName, uid_t uid,
+        const AppDefinedPrivilegesVector &privileges);
+    void addAppNoCheck(const std::string &appName, const std::string &pkgName,
+        uid_t uid, const std::string &tizenVer, const std::string &authorName, bool isHybrid);
     void addAppSuccess(const std::string &appName, const std::string &pkgName,
         const uid_t uid, const std::string &tizenVer, const std::string &authorName, bool isHybrid);
     void addAppFail(const std::string &appName, const std::string &pkgName,
         const uid_t uid, const std::string &tizenVer, const std::string &authorName, bool isHybrid);
     void removeApp(const std::string &appName, const uid_t uid, bool expAppNameIsNoMore,
         bool expPkgNameIsNoMore, bool expAuthorNameIsNoMore);
+    void removeAppNoCheck(const std::string &appName, uid_t uid);
     void removeAppSuccess(const std::string &appName, const uid_t uid);
     void checkPrivateSharing(const std::string &ownerAppName,
         const std::string &targetAppName, const std::string &path, int expectedPathCount,
diff --git a/test/test_performance_db.cpp b/test/test_performance_db.cpp
new file mode 100644 (file)
index 0000000..385adc4
--- /dev/null
@@ -0,0 +1,363 @@
+/*
+ *  Copyright (c) 2017 Samsung Electronics Co., Ltd All Rights Reserved
+ *
+ *  Contact: Rafal Krypa <r.krypa@samsung.com>
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License
+ *
+ *
+ * @file        test_performance_db.cpp
+ * @author      Ernest Borowski (e.borowski@partner.samsung.com)
+ * @version     1.0
+ * @brief       This file is implementation of test_performance_db functions.
+ */
+
+#define BOOST_TEST_MODULE SecurityManagerTest
+
+#include <boost/test/results_reporter.hpp>
+#include <boost/test/unit_test.hpp>
+#include <chrono>
+#include <cmath>
+#include <ctime>
+#include <functional>
+#include <iomanip>
+#include <iostream>
+#include <sstream>
+#include <string>
+#include <tuple>
+#include <vector>
+
+#include "privilege_db_fixture.h"
+
+const int AVG_LOOPS = 300;
+const int MANY_LOOPS = 3000;
+const int FEW_LOOPS  = 10;
+using namespace std::chrono;
+using IncrementAppAll = std::function <void (int&, int&, int&, int&)>;
+using IncrementAppPart = std::function <void (int&, int&)>;
+
+std::vector<AppDefinedPrivilege> getPrivileges()
+{
+    static int uniqueNumber = 0;
+
+    uniqueNumber++;
+    std::string privilegeArray[] = {"org.tizen.my_app.gps", "org.tizen.my_app.maps",
+                                    "org.tizen.my_app.browser",
+                                    "org.tizen.my_app.nfc", "org.tizen.my_app.message",
+                                    "org.tizen.my_app.audio"};
+    AppDefinedPrivilegesVector privileges;
+
+    for (const std::string& privilege : privilegeArray) {
+        privileges.push_back(std::make_tuple(privilege + std::to_string(uniqueNumber),
+        SM_APP_DEFINED_PRIVILEGE_TYPE_UNTRUSTED, ""));
+    }
+
+    return(privileges);
+}
+
+void incrementAppFull(int &appId, int &pkgId, int &uid, int &authorId)
+{
+    ++appId;
+    ++pkgId;
+    ++uid;
+    ++authorId;
+}
+void incrementAppPart(int &appId, int &uid)
+{
+    ++appId;
+    ++uid;
+}
+
+duration <double> measureAddAppTime(int appCount, PrivilegeDBFixture &pdf, int incStartVal = 0,
+     IncrementAppAll inc = incrementAppFull)
+{
+    int iApp = incStartVal, iPkg = incStartVal;
+    int iAuthor = incStartVal, iUid = incStartVal;
+
+    auto start = system_clock::now();
+
+    for (int j = 0; j < appCount; j++) {
+        inc(iApp, iPkg, iAuthor, iUid);
+        BOOST_REQUIRE_NO_THROW(pdf.addAppNoCheck(pdf.app(iApp), pdf.pkg(iPkg), pdf.uid(iUid),
+            pdf.tizenVer(1), pdf.author(iAuthor), pdf.Hybrid));
+    }
+    auto end = system_clock::now();
+    return (end - start);
+}
+
+duration <double> measureRemoveTime(int appCount, PrivilegeDBFixture &pdf, int incStartVal = 0,
+     IncrementAppPart inc = incrementAppPart)
+{
+    int iApp = incStartVal, iUid = incStartVal;
+    auto start = system_clock::now();
+
+    for (int j = 0; j < appCount; j++) {
+        inc(iApp, iUid);
+        BOOST_REQUIRE_NO_THROW(pdf.removeAppNoCheck(pdf.app(iApp), pdf.uid(iUid)));
+    }
+    auto end = system_clock::now();
+    return (end - start);
+}
+
+duration <double> measureAddPrivilegeTime(int appCount, PrivilegeDBFixture &pdf,
+    int incStartValue = 0, IncrementAppPart inc = incrementAppPart)
+{
+    int iApp = incStartValue, iUid = incStartValue;
+    auto start = system_clock::now();
+
+    for (int j = 0; j < appCount; j++) {
+        inc(iApp, iUid);
+        BOOST_REQUIRE_NO_THROW(pdf.addAppDefinedPrivileges(pdf.app(iApp), pdf.uid(iUid),
+                                                           getPrivileges()));
+    }
+    auto end = system_clock::now();
+    return (end - start);
+}
+
+std::string convertDurationToString(const duration <double> &time)
+{
+    double t = double(duration_cast<microseconds>(time).count());
+    std::stringstream stream;
+
+    stream << std::setprecision(3);
+    if (t < 1e3) {
+        stream << t << "us";
+    } else if (t < 1e6) {
+        t /= 1e3;
+        stream << t << "ms";
+    } else {
+        t /= 1e6;
+        stream << t << "s";
+    }
+    return stream.str();
+}
+
+double getRatio(const duration<double> &largeDbAvgTime, const duration<double> &smallDbAvgTime)
+{
+    return double(duration_cast<microseconds>(largeDbAvgTime).count()) /
+                (duration_cast<microseconds>(smallDbAvgTime).count());
+}
+
+void displayResult(const std::string &msg, const duration<double> &largeDbAvgTime,
+                   const duration<double> &smallDbAvgTime,
+                   int fullCount = AVG_LOOPS)
+{
+    double ratio = getRatio(largeDbAvgTime, smallDbAvgTime);
+    BOOST_TEST_MESSAGE("\tAverage time for " << msg << convertDurationToString(smallDbAvgTime)
+                       << "\n\tAverage time for " << fullCount
+                       << " apps: " << convertDurationToString(largeDbAvgTime)
+                       << "\n\tRatio: " << ratio);
+}
+
+std::string durationToMicroseconds(const duration <double> &time)
+{
+    return std::to_string(double(duration_cast<microseconds>(time).count()));
+}
+
+BOOST_AUTO_TEST_SUITE(PERFORMANCE_DB_TEST)
+
+BOOST_AUTO_TEST_CASE(TXX_db_perf_add)
+{
+    duration <double> addAppTimeSmallDb, addAppTimeLargeDb;
+    {
+        PrivilegeDBFixture pdf;
+        addAppTimeSmallDb = measureAddAppTime(FEW_LOOPS, pdf) / FEW_LOOPS;
+    }
+    {
+        PrivilegeDBFixture pdf;
+        addAppTimeLargeDb = measureAddAppTime(AVG_LOOPS, pdf);
+        displayResult("adding " + std::to_string(FEW_LOOPS) + " apps: ", addAppTimeLargeDb / AVG_LOOPS,
+                      addAppTimeSmallDb);
+    }
+}
+
+BOOST_AUTO_TEST_CASE(TXX_db_perf_priv)
+{
+    duration <double> addPrivTimeSmallDb, addPrivTimeLargeDb;
+    {
+        PrivilegeDBFixture pdf;
+        measureAddAppTime(FEW_LOOPS, pdf);
+        addPrivTimeSmallDb = measureAddPrivilegeTime(FEW_LOOPS, pdf) / FEW_LOOPS;
+    }
+    {
+        PrivilegeDBFixture pdf;
+        measureAddAppTime(AVG_LOOPS, pdf);
+        addPrivTimeLargeDb = measureAddPrivilegeTime(AVG_LOOPS, pdf);
+
+        displayResult("adding privileges to " + std::to_string(FEW_LOOPS) + " apps: ",
+                      addPrivTimeLargeDb / AVG_LOOPS, addPrivTimeSmallDb);
+    }
+}
+
+BOOST_AUTO_TEST_CASE(TXX_db_perf_remove)
+{
+    duration <double> rmAppTimeSmallDb, rmAppTimeLargeDb;
+    {
+        PrivilegeDBFixture pdf;
+        measureAddAppTime(FEW_LOOPS, pdf);
+        measureAddPrivilegeTime(FEW_LOOPS, pdf);
+        rmAppTimeSmallDb = measureRemoveTime(FEW_LOOPS, pdf) / FEW_LOOPS;
+    }
+    {
+        PrivilegeDBFixture pdf;
+        measureAddAppTime(AVG_LOOPS, pdf);
+        measureAddPrivilegeTime(AVG_LOOPS, pdf);
+        rmAppTimeLargeDb = measureRemoveTime(AVG_LOOPS, pdf);
+
+        displayResult("removing " + std::to_string(FEW_LOOPS) + " apps: ",
+                      rmAppTimeLargeDb / AVG_LOOPS, rmAppTimeSmallDb);
+    }
+}
+
+BOOST_AUTO_TEST_CASE(TXX_db_perf_add_custom_author)
+{
+    duration <double> addAppTimeSmallDb, addAppTimeLargeDb;
+
+    auto incrementSetAuthor = [](int &appId, int &pkgId, int &authorId, int &uid) {
+        ++appId;
+        ++pkgId;
+        ++uid;
+        authorId = AVG_LOOPS + FEW_LOOPS;
+    };
+
+    {
+        PrivilegeDBFixture pdf;
+        addAppTimeSmallDb = measureAddAppTime(FEW_LOOPS, pdf, 0, incrementSetAuthor) / FEW_LOOPS;
+    }
+    {
+        PrivilegeDBFixture pdf;
+        addAppTimeLargeDb = measureAddAppTime(AVG_LOOPS, pdf, 0, incrementSetAuthor);
+        displayResult("adding " + std::to_string(FEW_LOOPS) + " apps: ",
+                      addAppTimeLargeDb / AVG_LOOPS, addAppTimeSmallDb);
+    }
+}
+
+BOOST_AUTO_TEST_CASE(TXX_db_perf_priv_custom_author)
+{
+    duration <double> addPrivTimeSmallDb, addPrivTimeLargeDb;
+
+    auto incrementSetAuthor = [](int& appId, int& pkgId, int& authorId, int& uid) {
+        ++appId;
+        ++pkgId;
+        ++uid;
+        authorId = AVG_LOOPS + FEW_LOOPS;
+    };
+
+    {
+        PrivilegeDBFixture pdf;
+        measureAddAppTime(FEW_LOOPS, pdf, 0, incrementSetAuthor);
+        addPrivTimeSmallDb = measureAddPrivilegeTime(FEW_LOOPS, pdf) / FEW_LOOPS;
+    }
+    {
+        PrivilegeDBFixture pdf;
+        measureAddAppTime(AVG_LOOPS, pdf, 0, incrementSetAuthor);
+        addPrivTimeLargeDb = measureAddPrivilegeTime(AVG_LOOPS, pdf);
+
+        displayResult("adding privileges to " + std::to_string(FEW_LOOPS) + " apps: ",
+                      addPrivTimeLargeDb / AVG_LOOPS, addPrivTimeSmallDb);
+    }
+}
+
+BOOST_AUTO_TEST_CASE(TXX_db_perf_remove_custom_author)
+{
+    duration <double> rmAppTimeSmallDb, rmAppTimeLargeDb;
+
+    auto incrementSetAuthor = [](int& appId, int& pkgId, int& authorId, int& uid) {
+        ++appId;
+        ++pkgId;
+        ++uid;
+        authorId = AVG_LOOPS + FEW_LOOPS;
+    };
+
+    {
+        PrivilegeDBFixture pdf;
+        measureAddAppTime(FEW_LOOPS, pdf, 0, incrementSetAuthor);
+        measureAddPrivilegeTime(FEW_LOOPS, pdf);
+        rmAppTimeSmallDb = measureRemoveTime(FEW_LOOPS, pdf) / FEW_LOOPS;
+    }
+    {
+        PrivilegeDBFixture pdf;
+        measureAddAppTime(AVG_LOOPS, pdf, 0, incrementSetAuthor);
+        measureAddPrivilegeTime(AVG_LOOPS, pdf);
+        rmAppTimeLargeDb = measureRemoveTime(AVG_LOOPS, pdf);
+
+        displayResult("removing " + std::to_string(FEW_LOOPS) + " apps: ",
+                      rmAppTimeLargeDb / AVG_LOOPS, rmAppTimeSmallDb);
+    }
+}
+
+BOOST_AUTO_TEST_CASE(TXX_db_perf_add_large)
+{
+    //measures average add time of adding 300 apps when there is 2700 apps installed.
+    duration <double> addAppTimeSmallDb, addAppTimeLargeDb;
+    const int ADD_APP_COUNT = MANY_LOOPS * 0.1;
+    const int EXISTING_APP_COUNT = MANY_LOOPS * 0.9;
+    {
+        PrivilegeDBFixture pdf;
+        addAppTimeSmallDb = measureAddAppTime(FEW_LOOPS, pdf) / FEW_LOOPS;
+    }
+    {
+        PrivilegeDBFixture pdf;
+        measureAddAppTime(EXISTING_APP_COUNT, pdf);
+        addAppTimeLargeDb = measureAddAppTime(ADD_APP_COUNT, pdf);
+
+        displayResult("adding " + std::to_string(FEW_LOOPS) + " apps: ",
+                      addAppTimeLargeDb / ADD_APP_COUNT, addAppTimeSmallDb, ADD_APP_COUNT);
+    }
+}
+
+BOOST_AUTO_TEST_CASE(TXX_db_perf_priv_large)
+{
+    //measures average adding privileges time of adding 300 apps privileges when there is 2700 apps with privileges installed.
+    duration <double> addPrivTimeSmallDb, addPrivTimeLargeDb;
+    const int ADD_PRIV_COUNT = MANY_LOOPS * 0.1;
+    const int EXISTING_PRIV_COUNT = MANY_LOOPS * 0.9;
+    {
+        PrivilegeDBFixture pdf;
+        measureAddAppTime(FEW_LOOPS, pdf);
+        addPrivTimeSmallDb = measureAddPrivilegeTime(FEW_LOOPS, pdf) / FEW_LOOPS;
+    }
+    {
+        PrivilegeDBFixture pdf;
+        measureAddAppTime(MANY_LOOPS, pdf);
+        measureAddPrivilegeTime(EXISTING_PRIV_COUNT, pdf);
+        addPrivTimeLargeDb = measureAddPrivilegeTime(ADD_PRIV_COUNT, pdf);
+
+        displayResult("adding privileges to " + std::to_string(FEW_LOOPS) + " apps: ",
+                      addPrivTimeLargeDb / ADD_PRIV_COUNT, addPrivTimeSmallDb, ADD_PRIV_COUNT);
+    }
+}
+
+BOOST_AUTO_TEST_CASE(TXX_db_perf_remove_large)
+{
+    //measures average remove time of 10% apps when there is 3000 apps installed.
+    duration <double> rmAppTimeSmallDb, rmAppTimeLargeDb;
+    const int REMOVE_APP_COUNT = MANY_LOOPS * 0.1;
+    {
+        PrivilegeDBFixture pdf;
+        measureAddAppTime(FEW_LOOPS, pdf);
+        measureAddPrivilegeTime(FEW_LOOPS, pdf);
+        rmAppTimeSmallDb = measureRemoveTime(FEW_LOOPS, pdf) / FEW_LOOPS;
+    }
+    {
+        PrivilegeDBFixture pdf;
+        measureAddAppTime(MANY_LOOPS, pdf);
+        measureAddPrivilegeTime(MANY_LOOPS, pdf);
+        rmAppTimeLargeDb = measureRemoveTime(REMOVE_APP_COUNT, pdf);
+
+        displayResult("removing " + std::to_string(FEW_LOOPS) + " apps: ",
+                      rmAppTimeLargeDb / REMOVE_APP_COUNT, rmAppTimeSmallDb, REMOVE_APP_COUNT);
+    }
+}
+
+BOOST_AUTO_TEST_SUITE_END()