From 4b58867e7d32255e04eeb63d7772f51a8221384f Mon Sep 17 00:00:00 2001 From: Piotr Bartosiewicz Date: Tue, 26 Aug 2014 12:18:30 +0200 Subject: [PATCH 01/16] Fix clang compilation problem [Bug/Feature] Does not compile with clang [Cause] N/A [Solution] N/A [Verification] Compile using gcc and clang Change-Id: I2c065d87b66473c1a80b402fb925ee62180c15df --- src/config/from-json-visitor.hpp | 6 +++--- src/config/from-kvstore-visitor.hpp | 6 +++--- src/config/to-json-visitor.hpp | 6 +++--- src/config/to-kvstore-visitor.hpp | 6 +++--- 4 files changed, 12 insertions(+), 12 deletions(-) diff --git a/src/config/from-json-visitor.hpp b/src/config/from-json-visitor.hpp index 9ebe1fc..41b09cd 100644 --- a/src/config/from-json-visitor.hpp +++ b/src/config/from-json-visitor.hpp @@ -57,7 +57,7 @@ public: FromJsonVisitor& operator=(const FromJsonVisitor&) = delete; - template + template void visit(const std::string& name, T& value) { json_object* object = nullptr; @@ -117,7 +117,7 @@ private: value = json_object_get_string(object); } - template + template static void fromJsonObject(json_object* object, std::vector& value) { checkType(object, json_type_array); @@ -128,7 +128,7 @@ private: } } - template::value>::type> + template::value>::type> static void fromJsonObject(json_object* object, T& value) { checkType(object, json_type_object); diff --git a/src/config/from-kvstore-visitor.hpp b/src/config/from-kvstore-visitor.hpp index 5b63410..25c4182 100644 --- a/src/config/from-kvstore-visitor.hpp +++ b/src/config/from-kvstore-visitor.hpp @@ -54,7 +54,7 @@ public: FromKVStoreVisitor& operator=(const FromKVStoreVisitor&) = delete; - template + template void visit(const std::string& name, T& value) { getInternal(key(mKeyPrefix, name), value); @@ -64,13 +64,13 @@ private: std::shared_ptr mStorePtr; std::string mKeyPrefix; - template::value, int>::type = 0> + template::value, int>::type = 0> void getInternal(const std::string& name, T& value) { value = mStorePtr->get(name); } - template::value, int>::type = 0> + template::value, int>::type = 0> void getInternal(const std::string& name, T& value) { FromKVStoreVisitor visitor(*this, name); diff --git a/src/config/to-json-visitor.hpp b/src/config/to-json-visitor.hpp index ade4ef3..85f8a16 100644 --- a/src/config/to-json-visitor.hpp +++ b/src/config/to-json-visitor.hpp @@ -58,7 +58,7 @@ public: return json_object_to_json_string(mObject); } - template + template void visit(const std::string& name, const T& value) { json_object_object_add(mObject, name.c_str(), toJsonObject(value)); @@ -99,7 +99,7 @@ private: return json_object_new_string(value.c_str()); } - template + template static json_object* toJsonObject(const std::vector& value) { json_object* array = json_object_new_array(); @@ -109,7 +109,7 @@ private: return array; } - template::value>::type> + template::value>::type> static json_object* toJsonObject(const T& value) { ToJsonVisitor visitor; diff --git a/src/config/to-kvstore-visitor.hpp b/src/config/to-kvstore-visitor.hpp index 8e6f577..dcb7d8c 100644 --- a/src/config/to-kvstore-visitor.hpp +++ b/src/config/to-kvstore-visitor.hpp @@ -54,7 +54,7 @@ public: ToKVStoreVisitor& operator=(const ToKVStoreVisitor&) = delete; - template + template void visit(const std::string& name, const T& value) { setInternal(key(mKeyPrefix, name), value); @@ -65,13 +65,13 @@ private: std::string mKeyPrefix; - template::value, int>::type = 0> + template::value, int>::type = 0> void setInternal(const std::string& name, const T& value) { mStorePtr->set(name, value); } - template::value, int>::type = 0> + template::value, int>::type = 0> void setInternal(const std::string& name, const T& value) { ToKVStoreVisitor visitor(*this, name); -- 2.7.4 From 99363ac913cfd91a2946b2ab51995854f7c475cb Mon Sep 17 00:00:00 2001 From: Piotr Bartosiewicz Date: Wed, 3 Sep 2014 14:14:05 +0200 Subject: [PATCH 02/16] Structs serialization to KVStore finished [Bug/Feature] Missing serialization/deserialization of vector of structs. [Cause] N/A [Solution] N/A [Verification] Build, install, run tests Change-Id: I562bb89550b4af5ce86c3f818738cb878830f828 --- src/config/from-kvstore-visitor.hpp | 15 +++++++++++++++ src/config/to-kvstore-visitor.hpp | 9 +++++++++ 2 files changed, 24 insertions(+) diff --git a/src/config/from-kvstore-visitor.hpp b/src/config/from-kvstore-visitor.hpp index 25c4182..40ecaaa 100644 --- a/src/config/from-kvstore-visitor.hpp +++ b/src/config/from-kvstore-visitor.hpp @@ -76,6 +76,21 @@ private: FromKVStoreVisitor visitor(*this, name); value.accept(visitor); } + + template + void getInternal(const std::string& name, std::vector& values) + { + values.clear(); + for (unsigned int i = 0;; ++i) { + const std::string k = key(name, std::to_string(i)); + if (mStorePtr->count(k) == 0) { + return; + } + T value; + getInternal(k, value); + values.push_back(value); + } + } }; } // namespace config diff --git a/src/config/to-kvstore-visitor.hpp b/src/config/to-kvstore-visitor.hpp index dcb7d8c..8da093b 100644 --- a/src/config/to-kvstore-visitor.hpp +++ b/src/config/to-kvstore-visitor.hpp @@ -77,6 +77,15 @@ private: ToKVStoreVisitor visitor(*this, name); value.accept(visitor); } + + template + void setInternal(const std::string& name, const std::vector& values) + { + mStorePtr->remove(name); + for (size_t i = 0; i < values.size(); ++i) { + setInternal(key(name, std::to_string(i)), values[i]); + } + } }; } // namespace config -- 2.7.4 From 656141e7ebb3642ec6f01e02e77dc2cc202f8ad2 Mon Sep 17 00:00:00 2001 From: Jan Olszak Date: Thu, 18 Sep 2014 10:46:53 +0200 Subject: [PATCH 03/16] Writing the size of a vector in the KVStore [Bug/Feature] No way to write an empty vector in the KVStore [Cause] N/A [Solution] N/A [Verification] Build, install, run tests Change-Id: Ia4d67e33e9320c27cf6d86e809bf5c54edd940cc --- src/config/from-kvstore-visitor.hpp | 17 ++-- src/config/kvstore.cpp | 176 ++++++++++++++++++++---------------- src/config/kvstore.hpp | 20 ++-- src/config/to-kvstore-visitor.hpp | 3 +- 4 files changed, 120 insertions(+), 96 deletions(-) diff --git a/src/config/from-kvstore-visitor.hpp b/src/config/from-kvstore-visitor.hpp index 40ecaaa..0c91eb5 100644 --- a/src/config/from-kvstore-visitor.hpp +++ b/src/config/from-kvstore-visitor.hpp @@ -77,18 +77,19 @@ private: value.accept(visitor); } - template + template::value, int>::type = 0> void getInternal(const std::string& name, std::vector& values) { values.clear(); - for (unsigned int i = 0;; ++i) { + size_t vectorSize = mStorePtr->get(name); + if (vectorSize == 0) { + return; + } + + values.resize(vectorSize); + for (size_t i = 0; i < vectorSize; ++i) { const std::string k = key(name, std::to_string(i)); - if (mStorePtr->count(k) == 0) { - return; - } - T value; - getInternal(k, value); - values.push_back(value); + getInternal(k, values[i]); } } }; diff --git a/src/config/kvstore.cpp b/src/config/kvstore.cpp index 03231be..6c26e64 100644 --- a/src/config/kvstore.cpp +++ b/src/config/kvstore.cpp @@ -39,24 +39,6 @@ namespace { const int AUTO_DETERM_SIZE = -1; const int FIRST_COLUMN = 0; -struct Transaction { - Transaction(sqlite3::Connection& connRef) - : mConnRef(connRef) - { - mConnRef.exec("BEGIN EXCLUSIVE TRANSACTION"); - } - ~Transaction() - { - if (std::uncaught_exception()) { - mConnRef.exec("ROLLBACK TRANSACTION"); - } else { - mConnRef.exec("COMMIT TRANSACTION"); - } - } -private: - sqlite3::Connection& mConnRef; -}; - struct ScopedReset { ScopedReset(std::unique_ptr& stmtPtr) : mStmtPtr(stmtPtr) {} @@ -68,9 +50,19 @@ private: std::unique_ptr& mStmtPtr; }; -std::string escape(const std::string& in) +/** + * Escape characters used by the GLOB function. + */ +void sqliteEscapeStr(::sqlite3_context* context, int argc, ::sqlite3_value** values) { - const std::set toEscape({'?', '*', '[', ']'}); + char* inBuff = (char*)sqlite3_value_text(values[0]); + if (argc != 1 || inBuff == NULL) { + sqlite3_result_error(context, "SQL function escapeSequence() called with invalid arguments.\n", -1); + return; + } + + std::string in(inBuff); + static const std::set toEscape({'?', '*', '[', ']'}); // Compute the out size auto isEscapeChar = [&](char c) { @@ -80,7 +72,7 @@ std::string escape(const std::string& in) in.end(), isEscapeChar); if (numEscape == 0) { - return in; + sqlite3_result_text(context, in.c_str(), AUTO_DETERM_SIZE, SQLITE_TRANSIENT); } // Escape characters @@ -98,16 +90,46 @@ std::string escape(const std::string& in) out[j] = in[i]; } } - return out; + sqlite3_result_text(context, out.c_str(), AUTO_DETERM_SIZE, SQLITE_TRANSIENT); } + } // namespace + +struct KVStore::Transaction { + Transaction(KVStore* kvstorePtr) + : mKVStorePtr(kvstorePtr) + { + if (mKVStorePtr->mTransactionCounter == 0) { + mKVStorePtr->mConn.exec("BEGIN EXCLUSIVE TRANSACTION"); + } + mKVStorePtr->mTransactionCounter += 1; + } + + ~Transaction() + { + mKVStorePtr->mTransactionCounter -= 1; + if (mKVStorePtr->mTransactionCounter == 0) { + if (std::uncaught_exception()) { + mKVStorePtr->mConn.exec("ROLLBACK TRANSACTION"); + } else { + mKVStorePtr->mConn.exec("COMMIT TRANSACTION"); + } + } + } + +private: + config::KVStore* mKVStorePtr; +}; + + KVStore::KVStore(const std::string& path) : mPath(path), mConn(path) { setupDb(); + createFunctions(); prepareStatements(); } @@ -115,6 +137,7 @@ KVStore::KVStore(const KVStore& store) : mPath(store.mPath), mConn(mPath) { + createFunctions(); prepareStatements(); } @@ -125,69 +148,75 @@ KVStore::~KVStore() void KVStore::setupDb() { Lock lock(mConnMtx); - Transaction transaction(mConn); + Transaction transaction(this); mConn.exec("CREATE TABLE IF NOT EXISTS data (key TEXT PRIMARY KEY, value TEXT NOT NULL)"); } void KVStore::prepareStatements() { mGetValueStmt.reset( - new sqlite3::Statement(mConn, "SELECT value FROM data WHERE key GLOB ? LIMIT 1")); - mGetValueListStmt.reset( - new sqlite3::Statement(mConn, "SELECT value FROM data WHERE key GLOB ? ||'*' ORDER BY key")); - mGetValueCountStmt.reset( - new sqlite3::Statement(mConn, "SELECT count(key) FROM data WHERE key GLOB ? ||'*' ")); - mGetSizeStmt.reset( - new sqlite3::Statement(mConn, "SELECT count(key) FROM data")); + new sqlite3::Statement(mConn, "SELECT value FROM data WHERE key = ? LIMIT 1")); + mGetKeyExistsStmt.reset( + new sqlite3::Statement(mConn, "SELECT 1 FROM data WHERE key = ?1 OR key GLOB escapeStr(?1) || '.*' LIMIT 1")); + mGetIsEmptyStmt.reset( + new sqlite3::Statement(mConn, "SELECT 1 FROM data LIMIT 1")); mSetValueStmt.reset( new sqlite3::Statement(mConn, "INSERT OR REPLACE INTO data (key, value) VALUES (?,?)")); mRemoveValuesStmt.reset( - new sqlite3::Statement(mConn, "DELETE FROM data WHERE key GLOB ? ||'*' ")); + new sqlite3::Statement(mConn, "DELETE FROM data WHERE key = ?1 OR key GLOB escapeStr(?1) ||'.*' ")); +} + +void KVStore::createFunctions() +{ + int ret = sqlite3_create_function(mConn.get(), "escapeStr", 1, SQLITE_ANY, 0, &sqliteEscapeStr, 0, 0); + if (ret != SQLITE_OK) { + throw ConfigException("Error during creating functions: " + mConn.getErrorMessage()); + } } void KVStore::clear() { Lock lock(mConnMtx); - Transaction transaction(mConn); + Transaction transaction(this); mConn.exec("DELETE FROM data"); } -unsigned int KVStore::size() +bool KVStore::isEmpty() { Lock lock(mConnMtx); - ScopedReset scopedReset(mGetSizeStmt); + ScopedReset scopedReset(mGetIsEmptyStmt); - if (::sqlite3_step(mGetSizeStmt->get()) != SQLITE_ROW) { + int ret = ::sqlite3_step(mGetIsEmptyStmt->get()); + if (ret == SQLITE_DONE) { + return true; + } else if (ret == SQLITE_ROW) { + return false; + } else { throw ConfigException("Error during stepping: " + mConn.getErrorMessage()); } - - return static_cast(::sqlite3_column_int(mGetSizeStmt->get(), FIRST_COLUMN)); } -unsigned int KVStore::count(const std::string& key) +bool KVStore::exists(const std::string& key) { Lock lock(mConnMtx); - Transaction transaction(mConn); - return countInternal(key); -} + ScopedReset scopedReset(mGetKeyExistsStmt); -unsigned int KVStore::countInternal(const std::string& key) -{ - ScopedReset scopedReset(mGetValueCountStmt); - - ::sqlite3_bind_text(mGetValueCountStmt->get(), 1, escape(key).c_str(), AUTO_DETERM_SIZE, SQLITE_TRANSIENT); + ::sqlite3_bind_text(mGetKeyExistsStmt->get(), 1, key.c_str(), AUTO_DETERM_SIZE, SQLITE_TRANSIENT); - if (::sqlite3_step(mGetValueCountStmt->get()) != SQLITE_ROW) { + int ret = ::sqlite3_step(mGetKeyExistsStmt->get()); + if (ret == SQLITE_DONE) { + return false; + } else if (ret == SQLITE_ROW) { + return true; + } else { throw ConfigException("Error during stepping: " + mConn.getErrorMessage()); } - - return static_cast(::sqlite3_column_int(mGetValueCountStmt->get(), FIRST_COLUMN)); } void KVStore::remove(const std::string& key) { Lock lock(mConnMtx); - Transaction transaction(mConn); + Transaction transaction(this); removeInternal(key); } @@ -209,7 +238,7 @@ void KVStore::setInternal(const std::string& key, const std::string& value) ::sqlite3_bind_text(mSetValueStmt->get(), 1, key.c_str(), AUTO_DETERM_SIZE, SQLITE_STATIC); ::sqlite3_bind_text(mSetValueStmt->get(), 2, value.c_str(), AUTO_DETERM_SIZE, SQLITE_STATIC); - Transaction transaction(mConn); + Transaction transaction(this); ScopedReset scopedReset(mSetValueStmt); if (::sqlite3_step(mSetValueStmt->get()) != SQLITE_DONE) { @@ -229,20 +258,17 @@ void KVStore::setInternal(const std::string& key, const std::vector } Lock lock(mConnMtx); - Transaction transaction(mConn); + Transaction transaction(this); removeInternal(key); - for (unsigned int i = 0; i < values.size(); ++i) { - ScopedReset scopedReset(mSetValueStmt); - const std::string modifiedKey = key + "." + std::to_string(i);; - - ::sqlite3_bind_text(mSetValueStmt->get(), 1, modifiedKey.c_str(), AUTO_DETERM_SIZE, SQLITE_STATIC); - ::sqlite3_bind_text(mSetValueStmt->get(), 2, values[i].c_str(), AUTO_DETERM_SIZE, SQLITE_STATIC); + // Save vector's capacity + setInternal(key, values.size()); - if (::sqlite3_step(mSetValueStmt->get()) != SQLITE_DONE) { - throw ConfigException("Error during stepping: " + mConn.getErrorMessage()); - } + // Save vector's elements + for (unsigned int i = 0; i < values.size(); ++i) { + setInternal(config::key(key, std::to_string(i)), + values[i]); } } @@ -250,14 +276,14 @@ std::string KVStore::getInternal(const std::string& key, std::string*) { Lock lock(mConnMtx); - ::sqlite3_bind_text(mGetValueStmt->get(), 1, escape(key).c_str(), AUTO_DETERM_SIZE, SQLITE_TRANSIENT); + ::sqlite3_bind_text(mGetValueStmt->get(), 1, key.c_str(), AUTO_DETERM_SIZE, SQLITE_TRANSIENT); - Transaction transaction(mConn); + Transaction transaction(this); ScopedReset scopedReset(mGetValueStmt); int ret = ::sqlite3_step(mGetValueStmt->get()); if (ret == SQLITE_DONE) { - throw ConfigException("No value corresponding to the key"); + throw ConfigException("No value corresponding to the key: " + key); } if (ret != SQLITE_ROW) { throw ConfigException("Error during stepping: " + mConn.getErrorMessage()); @@ -269,24 +295,18 @@ std::string KVStore::getInternal(const std::string& key, std::string*) std::vector KVStore::getInternal(const std::string& key, std::vector*) { Lock lock(mConnMtx); + Transaction transaction(this); - ::sqlite3_bind_text(mGetValueListStmt->get(), 1, escape(key).c_str(), AUTO_DETERM_SIZE, SQLITE_TRANSIENT); - - Transaction transaction(mConn); - ScopedReset scopedReset(mGetValueListStmt); - - unsigned int valuesSize = countInternal(key); + unsigned int valuesSize = get(key); + std::vector values(valuesSize); if (valuesSize == 0) { - throw ConfigException("No value corresponding to the key"); + return values; } - std::vector values(valuesSize); - for (std::string& value : values) { - if (::sqlite3_step(mGetValueListStmt->get()) != SQLITE_ROW) { - throw ConfigException("Error during stepping: " + mConn.getErrorMessage()); - } - value = reinterpret_cast( - sqlite3_column_text(mGetValueListStmt->get(), FIRST_COLUMN)); + for (unsigned int i = 0; i < values.size(); ++i) { + values[i] = getInternal(config::key(key, std::to_string(i)), + static_cast(nullptr)); + } return values; diff --git a/src/config/kvstore.hpp b/src/config/kvstore.hpp index 6e30d99..3a2455b 100644 --- a/src/config/kvstore.hpp +++ b/src/config/kvstore.hpp @@ -55,16 +55,16 @@ public: void clear(); /** - * @return Number of all stored values + * @return Is there any data stored */ - unsigned int size(); + bool isEmpty(); /** * @param key string regexp of the stored values * - * @return Number of values corresponding to the passed key + * @return Does this key exist in the database */ - unsigned int count(const std::string& key); + bool exists(const std::string& key); /** * Removes values corresponding to the passed key. @@ -103,7 +103,9 @@ public: private: - typedef std::lock_guard Lock; + struct Transaction; + typedef std::lock_guard Lock; + unsigned int mTransactionCounter; void setInternal(const std::string& key, const std::string& value); void setInternal(const std::string& key, const std::initializer_list& values); @@ -120,22 +122,22 @@ private: template std::vector getInternal(const std::string& key, std::vector*); - std::mutex mConnMtx; + std::recursive_mutex mConnMtx; std::string mPath; sqlite3::Connection mConn; std::unique_ptr mGetValueStmt; - std::unique_ptr mGetValueCountStmt; - std::unique_ptr mGetSizeStmt; + std::unique_ptr mGetKeyExistsStmt; + std::unique_ptr mGetIsEmptyStmt; std::unique_ptr mGetValueListStmt; std::unique_ptr mSetValueStmt; std::unique_ptr mRemoveValuesStmt; void setupDb(); void prepareStatements(); + void createFunctions(); void removeInternal(const std::string& key); - unsigned int countInternal(const std::string& key); }; diff --git a/src/config/to-kvstore-visitor.hpp b/src/config/to-kvstore-visitor.hpp index 8da093b..477ff83 100644 --- a/src/config/to-kvstore-visitor.hpp +++ b/src/config/to-kvstore-visitor.hpp @@ -78,10 +78,11 @@ private: value.accept(visitor); } - template + template::value, int>::type = 0> void setInternal(const std::string& name, const std::vector& values) { mStorePtr->remove(name); + mStorePtr->set(name, values.size()); for (size_t i = 0; i < values.size(); ++i) { setInternal(key(name, std::to_string(i)), values[i]); } -- 2.7.4 From abe56504c381aaf48d459a92b83f59ebe3a19af6 Mon Sep 17 00:00:00 2001 From: Jan Olszak Date: Thu, 2 Oct 2014 16:05:45 +0200 Subject: [PATCH 04/16] Transaction and synchronization guard [Bug/Feature] KVStore returns a transaction guard [Cause] N/A [Solution] N/A [Verification] Build, install, run tests Change-Id: I44d421e5817ca359bf28f70b9182c9e2e0618137 --- src/config/from-kvstore-visitor.hpp | 8 +++- src/config/kvstore.cpp | 74 ++++++++++++++++++------------------- src/config/kvstore.hpp | 20 ++++++---- src/config/to-kvstore-visitor.hpp | 8 ++-- 4 files changed, 58 insertions(+), 52 deletions(-) diff --git a/src/config/from-kvstore-visitor.hpp b/src/config/from-kvstore-visitor.hpp index 0c91eb5..c7e3540 100644 --- a/src/config/from-kvstore-visitor.hpp +++ b/src/config/from-kvstore-visitor.hpp @@ -38,13 +38,15 @@ class FromKVStoreVisitor { public: explicit FromKVStoreVisitor(const std::string& filePath, const std::string& prefix) : mStorePtr(new KVStore(filePath)), - mKeyPrefix(prefix) + mKeyPrefix(prefix), + mTransaction(mStorePtr->getTransaction()) { } FromKVStoreVisitor(const FromKVStoreVisitor& visitor, const std::string& prefix) : mStorePtr(visitor.mStorePtr), - mKeyPrefix(prefix) + mKeyPrefix(prefix), + mTransaction(visitor.mTransaction) { } @@ -63,6 +65,7 @@ public: private: std::shared_ptr mStorePtr; std::string mKeyPrefix; + KVStore::Transaction mTransaction; template::value, int>::type = 0> void getInternal(const std::string& name, T& value) @@ -81,6 +84,7 @@ private: void getInternal(const std::string& name, std::vector& values) { values.clear(); + size_t vectorSize = mStorePtr->get(name); if (vectorSize == 0) { return; diff --git a/src/config/kvstore.cpp b/src/config/kvstore.cpp index 6c26e64..621f7f6 100644 --- a/src/config/kvstore.cpp +++ b/src/config/kvstore.cpp @@ -95,34 +95,43 @@ void sqliteEscapeStr(::sqlite3_context* context, int argc, ::sqlite3_value** val } // namespace - - -struct KVStore::Transaction { - Transaction(KVStore* kvstorePtr) +struct KVStore::TransactionImpl { + TransactionImpl(KVStore* kvstorePtr) : mKVStorePtr(kvstorePtr) { - if (mKVStorePtr->mTransactionCounter == 0) { - mKVStorePtr->mConn.exec("BEGIN EXCLUSIVE TRANSACTION"); - } - mKVStorePtr->mTransactionCounter += 1; + mKVStorePtr->mConnMtx.lock(); + mKVStorePtr->mConn.exec("BEGIN EXCLUSIVE TRANSACTION"); } - ~Transaction() + ~TransactionImpl() { - mKVStorePtr->mTransactionCounter -= 1; - if (mKVStorePtr->mTransactionCounter == 0) { - if (std::uncaught_exception()) { + if (std::uncaught_exception()) { + try { mKVStorePtr->mConn.exec("ROLLBACK TRANSACTION"); - } else { + } catch (ConfigException&) {} + } else { + try { mKVStorePtr->mConn.exec("COMMIT TRANSACTION"); - } + } catch (ConfigException&) {} } + mKVStorePtr->mConnMtx.unlock(); } private: config::KVStore* mKVStorePtr; }; +KVStore::Transaction KVStore::getTransaction() +{ + Lock lock(getTransactionMutex); + + auto t = mTransactionImplPtr.lock(); + if (!t) { + t = std::make_shared(this); + mTransactionImplPtr = t; + } + return t; +} KVStore::KVStore(const std::string& path) : mPath(path), @@ -147,8 +156,7 @@ KVStore::~KVStore() void KVStore::setupDb() { - Lock lock(mConnMtx); - Transaction transaction(this); + Transaction transaction = getTransaction(); mConn.exec("CREATE TABLE IF NOT EXISTS data (key TEXT PRIMARY KEY, value TEXT NOT NULL)"); } @@ -176,14 +184,13 @@ void KVStore::createFunctions() void KVStore::clear() { - Lock lock(mConnMtx); - Transaction transaction(this); + Transaction transaction = getTransaction(); mConn.exec("DELETE FROM data"); } bool KVStore::isEmpty() { - Lock lock(mConnMtx); + Transaction transaction = getTransaction(); ScopedReset scopedReset(mGetIsEmptyStmt); int ret = ::sqlite3_step(mGetIsEmptyStmt->get()); @@ -198,7 +205,7 @@ bool KVStore::isEmpty() bool KVStore::exists(const std::string& key) { - Lock lock(mConnMtx); + Transaction transaction = getTransaction(); ScopedReset scopedReset(mGetKeyExistsStmt); ::sqlite3_bind_text(mGetKeyExistsStmt->get(), 1, key.c_str(), AUTO_DETERM_SIZE, SQLITE_TRANSIENT); @@ -215,13 +222,7 @@ bool KVStore::exists(const std::string& key) void KVStore::remove(const std::string& key) { - Lock lock(mConnMtx); - Transaction transaction(this); - removeInternal(key); -} - -void KVStore::removeInternal(const std::string& key) -{ + Transaction transaction = getTransaction(); ScopedReset scopedReset(mRemoveValuesStmt); ::sqlite3_bind_text(mRemoveValuesStmt->get(), 1, key.c_str(), AUTO_DETERM_SIZE, SQLITE_STATIC); @@ -233,13 +234,12 @@ void KVStore::removeInternal(const std::string& key) void KVStore::setInternal(const std::string& key, const std::string& value) { - Lock lock(mConnMtx); + Transaction transaction = getTransaction(); + ScopedReset scopedReset(mSetValueStmt); ::sqlite3_bind_text(mSetValueStmt->get(), 1, key.c_str(), AUTO_DETERM_SIZE, SQLITE_STATIC); ::sqlite3_bind_text(mSetValueStmt->get(), 2, value.c_str(), AUTO_DETERM_SIZE, SQLITE_STATIC); - Transaction transaction(this); - ScopedReset scopedReset(mSetValueStmt); if (::sqlite3_step(mSetValueStmt->get()) != SQLITE_DONE) { throw ConfigException("Error during stepping: " + mConn.getErrorMessage()); @@ -257,10 +257,9 @@ void KVStore::setInternal(const std::string& key, const std::vector throw ConfigException("Too many values to insert"); } - Lock lock(mConnMtx); - Transaction transaction(this); + Transaction transaction = getTransaction(); - removeInternal(key); + remove(key); // Save vector's capacity setInternal(key, values.size()); @@ -274,13 +273,11 @@ void KVStore::setInternal(const std::string& key, const std::vector std::string KVStore::getInternal(const std::string& key, std::string*) { - Lock lock(mConnMtx); + Transaction transaction = getTransaction(); + ScopedReset scopedReset(mGetValueStmt); ::sqlite3_bind_text(mGetValueStmt->get(), 1, key.c_str(), AUTO_DETERM_SIZE, SQLITE_TRANSIENT); - Transaction transaction(this); - ScopedReset scopedReset(mGetValueStmt); - int ret = ::sqlite3_step(mGetValueStmt->get()); if (ret == SQLITE_DONE) { throw ConfigException("No value corresponding to the key: " + key); @@ -294,8 +291,7 @@ std::string KVStore::getInternal(const std::string& key, std::string*) std::vector KVStore::getInternal(const std::string& key, std::vector*) { - Lock lock(mConnMtx); - Transaction transaction(this); + Transaction transaction = getTransaction(); unsigned int valuesSize = get(key); std::vector values(valuesSize); diff --git a/src/config/kvstore.hpp b/src/config/kvstore.hpp index 3a2455b..44c0aa1 100644 --- a/src/config/kvstore.hpp +++ b/src/config/kvstore.hpp @@ -35,12 +35,17 @@ #include #include #include +#include namespace config { class KVStore { public: + /** + * A guard struct for thread synchronization and transaction management. + */ + typedef std::shared_ptr Transaction; /** * @param path configuration database file path @@ -101,11 +106,15 @@ public: return getInternal(key, static_cast(nullptr)); } + KVStore::Transaction getTransaction(); private: - struct Transaction; - typedef std::lock_guard Lock; - unsigned int mTransactionCounter; + typedef std::lock_guard Lock; + + struct TransactionImpl; + std::weak_ptr mTransactionImplPtr; + std::mutex getTransactionMutex; + std::mutex mConnMtx; void setInternal(const std::string& key, const std::string& value); void setInternal(const std::string& key, const std::initializer_list& values); @@ -122,8 +131,6 @@ private: template std::vector getInternal(const std::string& key, std::vector*); - std::recursive_mutex mConnMtx; - std::string mPath; sqlite3::Connection mConn; std::unique_ptr mGetValueStmt; @@ -136,9 +143,6 @@ private: void setupDb(); void prepareStatements(); void createFunctions(); - - void removeInternal(const std::string& key); - }; namespace { diff --git a/src/config/to-kvstore-visitor.hpp b/src/config/to-kvstore-visitor.hpp index 477ff83..37b70a9 100644 --- a/src/config/to-kvstore-visitor.hpp +++ b/src/config/to-kvstore-visitor.hpp @@ -38,13 +38,15 @@ class ToKVStoreVisitor { public: ToKVStoreVisitor(const std::string& filePath, const std::string& prefix) : mStorePtr(new KVStore(filePath)), - mKeyPrefix(prefix) + mKeyPrefix(prefix), + mTransaction(mStorePtr->getTransaction()) { } ToKVStoreVisitor(const ToKVStoreVisitor& visitor, const std::string& prefix) : mStorePtr(visitor.mStorePtr), - mKeyPrefix(prefix) + mKeyPrefix(prefix), + mTransaction(visitor.mTransaction) { } @@ -63,7 +65,7 @@ public: private: std::shared_ptr mStorePtr; std::string mKeyPrefix; - + KVStore::Transaction mTransaction; template::value, int>::type = 0> void setInternal(const std::string& name, const T& value) -- 2.7.4 From 37f5674162881418068e85000da5b1aa689eb336 Mon Sep 17 00:00:00 2001 From: Jan Olszak Date: Fri, 3 Oct 2014 18:16:59 +0200 Subject: [PATCH 05/16] FDStore class for binary serialization [Bug/Feature] Class for writing and reading binary data to a file descriptor [Cause] N/A [Solution] N/A [Verification] Build, install, run tests Change-Id: I18713366449c77a17970eeb7f0f6b3ae1feac656 --- src/config/fdstore.cpp | 89 ++++++++++++++++++++++++++++++++++++ src/config/fdstore.hpp | 70 ++++++++++++++++++++++++++++ src/config/from-fdstore-visitor.hpp | 91 +++++++++++++++++++++++++++++++++++++ src/config/manager.hpp | 34 +++++++++++++- src/config/to-fdstore-visitor.hpp | 88 +++++++++++++++++++++++++++++++++++ 5 files changed, 371 insertions(+), 1 deletion(-) create mode 100644 src/config/fdstore.cpp create mode 100644 src/config/fdstore.hpp create mode 100644 src/config/from-fdstore-visitor.hpp create mode 100644 src/config/to-fdstore-visitor.hpp diff --git a/src/config/fdstore.cpp b/src/config/fdstore.cpp new file mode 100644 index 0000000..050e99f --- /dev/null +++ b/src/config/fdstore.cpp @@ -0,0 +1,89 @@ +/* + * Copyright (c) 2014 Samsung Electronics Co., Ltd All Rights Reserved + * + * Contact: Jan Olszak + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License + */ + +/** + * @file + * @author Jan Olszak (j.olszak@samsung.com) + * @brief Definition of a class for writing and reading data from a file descriptor + */ + +#include "config.hpp" + +#include "config/fdstore.hpp" +#include "config/exception.hpp" + +#include +#include +#include + +namespace config { + + +FDStore::FDStore(int fd) + : mFD(fd) +{ +} + +FDStore::FDStore(const FDStore& store) + : mFD(store.mFD) +{ +} + +FDStore::~FDStore() +{ +} + +void FDStore::write(const void* bufferPtr, const size_t size) +{ + size_t nTotal = 0; + int n; + + do { + n = ::write(mFD, + reinterpret_cast(bufferPtr) + nTotal, + size - nTotal); + if (n < 0) { + if (errno == EINTR) { + continue; + } + throw ConfigException("Error during witting: " + std::string(strerror(errno))); + } + nTotal += n; + } while (nTotal < size); +} + +void FDStore::read(void* bufferPtr, const size_t size) +{ + size_t nTotal = 0; + int n; + + do { + n = ::read(mFD, + reinterpret_cast(bufferPtr) + nTotal, + size - nTotal); + if (n < 0) { + if (errno == EINTR) { + continue; + } + throw ConfigException("Error during reading: " + std::string(strerror(errno))); + } + nTotal += n; + } while (nTotal < size); +} + +} // namespace config diff --git a/src/config/fdstore.hpp b/src/config/fdstore.hpp new file mode 100644 index 0000000..d99971d --- /dev/null +++ b/src/config/fdstore.hpp @@ -0,0 +1,70 @@ +/* + * Copyright (c) 2014 Samsung Electronics Co., Ltd All Rights Reserved + * + * Contact: Jan Olszak + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License + */ + +/** + * @file + * @author Jan Olszak (j.olszak@samsung.com) + * @brief Declaration of a class for writing and reading data from a file descriptor + */ + +#ifndef COMMON_CONFIG_FDSTORE_HPP +#define COMMON_CONFIG_FDSTORE_HPP + +#include + +namespace config { + +class FDStore { + +public: + /** + * Constructor. One can pass any kind of file descriptor. + * Serialization is NOT written for network purposes, + * rather local communication. + * + * @param fd file descriptor + */ + FDStore(int fd = -1); + FDStore(const FDStore& store); + ~FDStore(); + + /** + * Write data using the file descriptor + * + * @param bufferPtr buffer with the data + * @param size size of the buffer + */ + void write(const void* bufferPtr, const size_t size); + + /** + * Reads a value of the given type. + * + * @param bufferPtr buffer with the data + * @param size size of the buffer + */ + void read(void* bufferPtr, const size_t size); + +private: + int mFD; +}; + +} // namespace config + +#endif // COMMON_CONFIG_FDSTORE_HPP + + diff --git a/src/config/from-fdstore-visitor.hpp b/src/config/from-fdstore-visitor.hpp new file mode 100644 index 0000000..4266255 --- /dev/null +++ b/src/config/from-fdstore-visitor.hpp @@ -0,0 +1,91 @@ +/* + * Copyright (c) 2014 Samsung Electronics Co., Ltd All Rights Reserved + * + * Contact: Jan Olszak (j.olszak@samsung.com) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License + */ + +/** + * @file + * @author Jan Olszak (j.olszak@samsung.com) + * @brief Visitor for loading from FDStore + */ + +#ifndef CONFIG_FROM_FDSTORE_VISITOR_HPP +#define CONFIG_FROM_FDSTORE_VISITOR_HPP + +#include "config/is-visitable.hpp" +#include "config/fdstore.hpp" + +#include + +namespace config { + +class FromFDStoreVisitor { +public: + explicit FromFDStoreVisitor(int fd) + : mStore(fd) + { + } + + FromFDStoreVisitor(const FromFDStoreVisitor& visitor) = default; + + FromFDStoreVisitor& operator=(const FromFDStoreVisitor&) = delete; + + template + void visit(const std::string& name, T& value) + { + readInternal(value); + } + +private: + FDStore mStore; + + void readInternal(std::string& value) + { + size_t size; + readInternal(size); + value.resize(size); + mStore.read(&value.front(), size); + } + + template::value, int>::type = 0> + void readInternal(T& value) + { + mStore.read(&value, sizeof(T)); + } + + template::value, int>::type = 0> + void readInternal(T& value) + { + FromFDStoreVisitor visitor(*this); + value.accept(visitor); + } + + template + void readInternal(std::vector& values) + { + size_t vectorSize; + readInternal(vectorSize); + values.resize(vectorSize); + + for (T& value : values) { + readInternal(value); + } + } +}; + +} // namespace config + +#endif // CONFIG_FROM_FDSTORE_VISITOR_HPP diff --git a/src/config/manager.hpp b/src/config/manager.hpp index 688bc54..a2623ea 100644 --- a/src/config/manager.hpp +++ b/src/config/manager.hpp @@ -27,8 +27,10 @@ #include "config/to-json-visitor.hpp" #include "config/to-kvstore-visitor.hpp" +#include "config/to-fdstore-visitor.hpp" #include "config/from-json-visitor.hpp" #include "config/from-kvstore-visitor.hpp" +#include "config/from-fdstore-visitor.hpp" #include "config/is-visitable.hpp" #include "config/fs-utils.hpp" @@ -121,7 +123,7 @@ void loadFromKVStore(const std::string& filename, Config& config, const std::str * Saves the config to a KVStore. * * @param filename path to the KVStore db - * @param config visitable structure to load + * @param config visitable structure to save * @param configName name of the config inside the KVStore db */ template @@ -133,6 +135,36 @@ void saveToKVStore(const std::string& filename, const Config& config, const std: config.accept(visitor); } +/** + * Load binary data from a file/socket/pipe represented by the fd + * + * @param fd file descriptor + * @param config visitable structure to load + */ +template +void loadFromFD(const int fd, Config& config) +{ + static_assert(isVisitable::value, "Use CONFIG_REGISTER macro"); + + FromFDStoreVisitor visitor(fd); + config.accept(visitor); +} + +/** + * Save binary data to a file/socket/pipe represented by the fd + * + * @param fd file descriptor + * @param config visitable structure to save + */ +template +void saveToFD(const int fd, const Config& config) +{ + static_assert(isVisitable::value, "Use CONFIG_REGISTER macro"); + + ToFDStoreVisitor visitor(fd); + config.accept(visitor); +} + } // namespace config #endif // CONFIG_MANAGER_HPP diff --git a/src/config/to-fdstore-visitor.hpp b/src/config/to-fdstore-visitor.hpp new file mode 100644 index 0000000..7a2f358 --- /dev/null +++ b/src/config/to-fdstore-visitor.hpp @@ -0,0 +1,88 @@ +/* + * Copyright (c) 2014 Samsung Electronics Co., Ltd All Rights Reserved + * + * Contact: Jan Olszak (j.olszak@samsung.com) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License + */ + +/** + * @file + * @author Jan Olszak (j.olszak@samsung.com) + * @brief Visitor for saving to FDStore + */ + +#ifndef CONFIG_TO_FDSTORE_VISITOR_HPP +#define CONFIG_TO_FDSTORE_VISITOR_HPP + +#include "config/is-visitable.hpp" +#include "config/fdstore.hpp" + +#include + +namespace config { + +class ToFDStoreVisitor { + +public: + ToFDStoreVisitor(int fd) + : mStore(fd) + { + } + + ToFDStoreVisitor(const ToFDStoreVisitor& visitor) = default; + + ToFDStoreVisitor& operator=(const ToFDStoreVisitor&) = delete; + + template + void visit(const std::string& name, const T& value) + { + writeInternal(value); + } + +private: + FDStore mStore; + + void writeInternal(const std::string& value) + { + writeInternal(value.size()); + mStore.write(value.c_str(), value.size()); + } + + template::value, int>::type = 0> + void writeInternal(const T& value) + { + mStore.write(&value, sizeof(T)); + } + + template::value, int>::type = 0> + void writeInternal(const T& value) + { + ToFDStoreVisitor visitor(*this); + value.accept(visitor); + } + + template + void writeInternal(const std::vector& values) + { + writeInternal(values.size()); + for (const T& value : values) { + writeInternal(value); + } + } + +}; + +} // namespace config + +#endif // CONFIG_TO_FDSTORE_VISITOR_HPP -- 2.7.4 From 4b7005976ac880e883d18a394846f53034a5b976 Mon Sep 17 00:00:00 2001 From: Jan Olszak Date: Fri, 31 Oct 2014 16:54:25 +0100 Subject: [PATCH 06/16] Macro for creating empty Config structures [Bug/Feature] N/A [Cause] N/A [Solution] N/A [Verification] Build, install, run tests Change-Id: I1e2c468f2d05855b71938b20e7b63bb5aa112d54 --- src/config/fields.hpp | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/config/fields.hpp b/src/config/fields.hpp index 3269856..d5443b7 100644 --- a/src/config/fields.hpp +++ b/src/config/fields.hpp @@ -55,6 +55,15 @@ * ) * }; */ + +#define CONFIG_REGISTER_EMPTY \ + template \ + void accept(Visitor ) { \ + } \ + template \ + void accept(Visitor ) const { \ + } \ + #define CONFIG_REGISTER(...) \ template \ void accept(Visitor v) { \ -- 2.7.4 From d555a457fcf245abf8972e5f9ab3317e811ff58f Mon Sep 17 00:00:00 2001 From: Jan Olszak Date: Wed, 19 Nov 2014 10:28:41 +0100 Subject: [PATCH 07/16] Timeout in read and write to a FDStore [Bug/Feature] N/A [Cause] N/A [Solution] N/A [Verification] Build, install, run tests Change-Id: I42a358ce87d1dd6e02030a0e0e2431838ddae29c --- src/config/fdstore.cpp | 112 +++++++++++++++++++++++++++++++++++++------------ src/config/fdstore.hpp | 6 ++- 2 files changed, 89 insertions(+), 29 deletions(-) diff --git a/src/config/fdstore.cpp b/src/config/fdstore.cpp index 050e99f..7934893 100644 --- a/src/config/fdstore.cpp +++ b/src/config/fdstore.cpp @@ -30,9 +30,52 @@ #include #include #include +#include +#include namespace config { +namespace { + +void waitForEvent(int fd, + short event, + const std::chrono::high_resolution_clock::time_point deadline) +{ + // Wait for the rest of the data + struct pollfd fds[1]; + fds[0].fd = fd; + fds[0].events = event | POLLHUP; + + for (;;) { + std::chrono::milliseconds timeoutMS = + std::chrono::duration_cast(deadline - std::chrono::high_resolution_clock::now()); + if (timeoutMS.count() < 0) { + throw ConfigException("Timeout"); + } + + int ret = ::poll(fds, 1 /*fds size*/, timeoutMS.count()); + + if (ret == -1) { + if (errno == EINTR) { + continue; + } + throw ConfigException("Error in poll: " + std::string(strerror(errno))); + } + + if (ret == 0) { + throw ConfigException("Timeout"); + } + + if (fds[0].revents & POLLHUP) { + throw ConfigException("Peer disconnected"); + } + + // Here Comes the Sun + break; + } +} + +} // namespace FDStore::FDStore(int fd) : mFD(fd) @@ -48,42 +91,57 @@ FDStore::~FDStore() { } -void FDStore::write(const void* bufferPtr, const size_t size) +void FDStore::write(const void* bufferPtr, const size_t size, const unsigned int timeoutMS) { + std::chrono::high_resolution_clock::time_point deadline = std::chrono::high_resolution_clock::now() + + std::chrono::milliseconds(timeoutMS); + size_t nTotal = 0; - int n; + for (;;) { + int n = ::write(mFD, + reinterpret_cast(bufferPtr) + nTotal, + size - nTotal); + if (n > 0) { + nTotal += n; + } else if (errno == EAGAIN || errno == EWOULDBLOCK || errno == EINTR) { + // Neglected errors + } else { + throw ConfigException("Error during reading: " + std::string(strerror(errno))); + } - do { - n = ::write(mFD, - reinterpret_cast(bufferPtr) + nTotal, - size - nTotal); - if (n < 0) { - if (errno == EINTR) { - continue; - } - throw ConfigException("Error during witting: " + std::string(strerror(errno))); + if (nTotal >= size) { + // All data is written, break loop + break; + } else { + waitForEvent(mFD, POLLOUT, deadline); } - nTotal += n; - } while (nTotal < size); + } } -void FDStore::read(void* bufferPtr, const size_t size) +void FDStore::read(void* bufferPtr, const size_t size, const unsigned int timeoutMS) { - size_t nTotal = 0; - int n; + std::chrono::high_resolution_clock::time_point deadline = std::chrono::high_resolution_clock::now() + + std::chrono::milliseconds(timeoutMS); - do { - n = ::read(mFD, - reinterpret_cast(bufferPtr) + nTotal, - size - nTotal); - if (n < 0) { - if (errno == EINTR) { - continue; - } + size_t nTotal = 0; + for (;;) { + int n = ::read(mFD, + reinterpret_cast(bufferPtr) + nTotal, + size - nTotal); + if (n > 0) { + nTotal += n; + } else if (errno == EAGAIN || errno == EWOULDBLOCK || errno == EINTR) { + // Neglected errors + } else { throw ConfigException("Error during reading: " + std::string(strerror(errno))); } - nTotal += n; - } while (nTotal < size); -} + if (nTotal >= size) { + // All data is read, break loop + break; + } else { + waitForEvent(mFD, POLLIN, deadline); + } + } +} } // namespace config diff --git a/src/config/fdstore.hpp b/src/config/fdstore.hpp index d99971d..d34ea14 100644 --- a/src/config/fdstore.hpp +++ b/src/config/fdstore.hpp @@ -48,16 +48,18 @@ public: * * @param bufferPtr buffer with the data * @param size size of the buffer + * @param timeoutMS timeout in milliseconds */ - void write(const void* bufferPtr, const size_t size); + void write(const void* bufferPtr, const size_t size, const unsigned int timeoutMS = 500); /** * Reads a value of the given type. * * @param bufferPtr buffer with the data * @param size size of the buffer + * @param timeoutMS timeout in milliseconds */ - void read(void* bufferPtr, const size_t size); + void read(void* bufferPtr, const size_t size, const unsigned int timeoutMS = 500); private: int mFD; -- 2.7.4 From bb674f609e25e5f7a216f9c30676c4381c2f99c9 Mon Sep 17 00:00:00 2001 From: Mateusz Malicki Date: Wed, 26 Nov 2014 15:11:51 +0100 Subject: [PATCH 08/16] Added possibility to declare union in config file [Bug/Feature] Possibility to declare something like union in config file [Cause] Necessity to create vector with element of more than one type [Solution] Added macro CONFIG_DECLARE_OPTIONS [Verification] Build, install, run scs tests Change-Id: I094eb63f5cda6435836b3d76674d3cad5f89fe14 --- src/config/fields-union.hpp | 158 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 158 insertions(+) create mode 100644 src/config/fields-union.hpp diff --git a/src/config/fields-union.hpp b/src/config/fields-union.hpp new file mode 100644 index 0000000..8f4d997 --- /dev/null +++ b/src/config/fields-union.hpp @@ -0,0 +1,158 @@ +/* + * Copyright (c) 2014 Samsung Electronics Co., Ltd All Rights Reserved + * + * Contact: Mateusz Malicki (m.malicki2@samsung.com) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License + */ + +/** + * @file + * @author Mateusz Malicki (m.malicki2@samsung.com) + * @brief Macros for declare configuration fields + */ + +#ifndef CONFIG_FIELDS_UNION_HPP +#define CONFIG_FIELDS_UNION_HPP + +#include "config/fields.hpp" +#include +#include + +/** + * Use this macro to declare and register config fields + * + * Example: + * struct Foo { + * std::string bar; + * + * CONFIG_REGISTER + * ( + * bar, + * ) + * }; + * + * struct Config + * { + * CONFIG_DECLARE_UNION + * ( + * Foo, + * int + * ) + * }; + * + * Example of valid configuration: + * 1. { + * "type": "Foo", + * "value": { "bar": "some string" } + * } + * 2. { + * "type": "int", + * "value": 1 + * } + * + * + * Usage: + * Config config; + * if (config.is()) { + * Foo& foo = config.as(); + * // ... + * } + * if (config.is())) { + * int field = config.as(); + * // ... + * } + */ + + +#define CONFIG_DECLARE_UNION(...) \ + struct TypeWrapperBase \ + { \ + virtual ~TypeWrapperBase() {} \ + }; \ + \ + template \ + struct TypeWrapper : TypeWrapperBase \ + { \ + Class value; \ + ~TypeWrapper() {} \ + }; \ + \ + std::unique_ptr mConfigDeclareField__; \ + \ + template \ + void accept(Visitor v) { \ + std::string name; \ + v.visit("type", name); \ + visitOption(v, name); \ + } \ + \ + template \ + void accept(Visitor v) const { \ + const std::string name = getOptionName(); \ + if (!name.empty()) { \ + v.visit("type", name); \ + visitOption(v, name); \ + } else { \ + /* Unsupported type in config file */ \ + } \ + } \ + \ + template \ + void visitOption(Visitor& v, const std::string& name) { \ + GENERATE_CODE(GENERATE_UNION_VISIT__, __VA_ARGS__) \ + } \ + template \ + void visitOption(Visitor& v, const std::string& name) const { \ + GENERATE_CODE(GENERATE_UNION_VISIT_CONST__, __VA_ARGS__) \ + } \ + std::string getOptionName() const { \ + GENERATE_CODE(GENERATE_UNION_NAME__, __VA_ARGS__) \ + return std::string(); \ + } \ + \ + template \ + bool is() const { \ + return dynamic_cast*>(mConfigDeclareField__.get()) != NULL; \ + } \ + template \ + Type& as() { \ + assert(mConfigDeclareField__.get()); \ + return dynamic_cast&>(*mConfigDeclareField__.get()).value; \ + } \ + template \ + const Type& as() const { \ + assert(mConfigDeclareField__.get()); \ + return dynamic_cast&>(*mConfigDeclareField__.get()).value; \ + } + +#define GENERATE_CODE(MACRO, ...) \ + BOOST_PP_LIST_FOR_EACH(MACRO, _, BOOST_PP_VARIADIC_TO_LIST(__VA_ARGS__)) + +#define GENERATE_UNION_VISIT__(r, _, TYPE_) \ + if (#TYPE_ == name) { \ + mConfigDeclareField__.reset(new TypeWrapper()); \ + v.visit("value", as()); \ + } + +#define GENERATE_UNION_VISIT_CONST__(r, _, TYPE_) \ + if (#TYPE_ == name) { \ + v.visit("value", as()); \ + } + +#define GENERATE_UNION_NAME__(r, _, TYPE_) \ + if (is()) { \ + return #TYPE_; \ + } + +#endif /* CONFIG_FIELDS_UNION_HPP */ -- 2.7.4 From 24abd8462353da9c7f0865fdfc268713c7402fd0 Mon Sep 17 00:00:00 2001 From: Mateusz Malicki Date: Tue, 2 Dec 2014 15:01:32 +0100 Subject: [PATCH 09/16] Ability to copy union and to set union type [Feature] Ability to copy union and to set union type [Cause] Need to copy and add new union elements [Solution] Use of boost::any class; add isSet and set methods [Verification] Build, install, run scs tests Change-Id: I61e3211c27e62af93b36a5309e50ebaf90ab70ad --- src/config/fields-union.hpp | 91 ++++++++++++++++++++++++--------------------- 1 file changed, 49 insertions(+), 42 deletions(-) diff --git a/src/config/fields-union.hpp b/src/config/fields-union.hpp index 8f4d997..57e54a0 100644 --- a/src/config/fields-union.hpp +++ b/src/config/fields-union.hpp @@ -26,7 +26,8 @@ #define CONFIG_FIELDS_UNION_HPP #include "config/fields.hpp" -#include +#include +#include #include /** @@ -64,31 +65,43 @@ * * Usage: * Config config; - * if (config.is()) { - * Foo& foo = config.as(); - * // ... - * } - * if (config.is())) { - * int field = config.as(); - * // ... + * // ... + * if (config.isSet()) { + * if (config.is()) { + * Foo& foo = config.as(); + * // ... + * } + * if (config.is())) { + * int field = config.as(); + * // ... + * } + * } else { + * // Config is not set + * Foo foo({"some string"}); + * config.set(foo); + * config.set(std::move(foo)); + * config.set(Foo({"some string"})); * } */ #define CONFIG_DECLARE_UNION(...) \ - struct TypeWrapperBase \ - { \ - virtual ~TypeWrapperBase() {} \ - }; \ +private: \ + boost::any mConfigDeclareField__; \ \ - template \ - struct TypeWrapper : TypeWrapperBase \ - { \ - Class value; \ - ~TypeWrapper() {} \ - }; \ - \ - std::unique_ptr mConfigDeclareField__; \ + template \ + void visitOption(Visitor& v, const std::string& name) { \ + GENERATE_CODE(GENERATE_UNION_VISIT__, __VA_ARGS__) \ + } \ + template \ + void visitOption(Visitor& v, const std::string& name) const { \ + GENERATE_CODE(GENERATE_UNION_VISIT_CONST__, __VA_ARGS__) \ + } \ + std::string getOptionName() const { \ + GENERATE_CODE(GENERATE_UNION_NAME__, __VA_ARGS__) \ + return std::string(); \ + } \ +public: \ \ template \ void accept(Visitor v) { \ @@ -104,45 +117,39 @@ v.visit("type", name); \ visitOption(v, name); \ } else { \ - /* Unsupported type in config file */ \ + /* Unsupported type */ \ } \ } \ \ - template \ - void visitOption(Visitor& v, const std::string& name) { \ - GENERATE_CODE(GENERATE_UNION_VISIT__, __VA_ARGS__) \ - } \ - template \ - void visitOption(Visitor& v, const std::string& name) const { \ - GENERATE_CODE(GENERATE_UNION_VISIT_CONST__, __VA_ARGS__) \ - } \ - std::string getOptionName() const { \ - GENERATE_CODE(GENERATE_UNION_NAME__, __VA_ARGS__) \ - return std::string(); \ - } \ - \ template \ bool is() const { \ - return dynamic_cast*>(mConfigDeclareField__.get()) != NULL; \ + return boost::any_cast(&mConfigDeclareField__) != NULL; \ } \ template \ Type& as() { \ - assert(mConfigDeclareField__.get()); \ - return dynamic_cast&>(*mConfigDeclareField__.get()).value; \ + assert(!mConfigDeclareField__.empty()); \ + return boost::any_cast(mConfigDeclareField__); \ } \ template \ const Type& as() const { \ - assert(mConfigDeclareField__.get()); \ - return dynamic_cast&>(*mConfigDeclareField__.get()).value; \ - } + assert(!mConfigDeclareField__.empty()); \ + return boost::any_cast(mConfigDeclareField__); \ + } \ + bool isSet() { \ + return !getOptionName().empty(); \ + } \ + template \ + Type& set(Type&& src) { \ + mConfigDeclareField__ = std::forward(src); \ + return as(); \ + } \ #define GENERATE_CODE(MACRO, ...) \ BOOST_PP_LIST_FOR_EACH(MACRO, _, BOOST_PP_VARIADIC_TO_LIST(__VA_ARGS__)) #define GENERATE_UNION_VISIT__(r, _, TYPE_) \ if (#TYPE_ == name) { \ - mConfigDeclareField__.reset(new TypeWrapper()); \ - v.visit("value", as()); \ + v.visit("value", set(std::move(TYPE_()))); \ } #define GENERATE_UNION_VISIT_CONST__(r, _, TYPE_) \ -- 2.7.4 From 715efc8dc52494fd06faca5dcddb43817c08a4d2 Mon Sep 17 00:00:00 2001 From: Mateusz Malicki Date: Wed, 10 Dec 2014 18:07:09 +0100 Subject: [PATCH 10/16] Disable rvalue references [Bug] Boost any rvalue support is optional [Cause] rvalues is provided in 1.54 and can be disabled [Solution] Disable rvalue references [Verification] N/A Change-Id: I395adec94bbd07e33aa8bf703c0732ffa3b69ea6 --- src/config/fields-union.hpp | 40 +++++++++++++++++++++++++++++----------- 1 file changed, 29 insertions(+), 11 deletions(-) diff --git a/src/config/fields-union.hpp b/src/config/fields-union.hpp index 57e54a0..53620ca 100644 --- a/src/config/fields-union.hpp +++ b/src/config/fields-union.hpp @@ -79,15 +79,27 @@ * // Config is not set * Foo foo({"some string"}); * config.set(foo); - * config.set(std::move(foo)); + * config.set(std::move(foo)); //< copy sic! * config.set(Foo({"some string"})); * } */ +class DisbaleMoveAnyWrapper : public boost::any +{ + public: + DisbaleMoveAnyWrapper() {} + DisbaleMoveAnyWrapper(const DisbaleMoveAnyWrapper& any) + : boost::any(static_cast(any)) {}; + DisbaleMoveAnyWrapper& operator=(DisbaleMoveAnyWrapper&& any) = delete; + DisbaleMoveAnyWrapper& operator=(const DisbaleMoveAnyWrapper& any) { + static_cast(*this) = static_cast(any); + return *this; + } +}; #define CONFIG_DECLARE_UNION(...) \ private: \ - boost::any mConfigDeclareField__; \ + DisbaleMoveAnyWrapper mConfigDeclareField; \ \ template \ void visitOption(Visitor& v, const std::string& name) { \ @@ -101,6 +113,12 @@ private: GENERATE_CODE(GENERATE_UNION_NAME__, __VA_ARGS__) \ return std::string(); \ } \ + boost::any& getHolder() { \ + return static_cast(mConfigDeclareField); \ + } \ + const boost::any& getHolder() const { \ + return static_cast(mConfigDeclareField); \ + } \ public: \ \ template \ @@ -123,24 +141,24 @@ public: \ template \ bool is() const { \ - return boost::any_cast(&mConfigDeclareField__) != NULL; \ + return boost::any_cast(&getHolder()) != NULL; \ } \ template \ - Type& as() { \ - assert(!mConfigDeclareField__.empty()); \ - return boost::any_cast(mConfigDeclareField__); \ + typename std::enable_if::value, Type>::type& as() { \ + assert(!getHolder().empty()); \ + return boost::any_cast(getHolder()); \ } \ template \ const Type& as() const { \ - assert(!mConfigDeclareField__.empty()); \ - return boost::any_cast(mConfigDeclareField__); \ + assert(!getHolder().empty()); \ + return boost::any_cast(getHolder()); \ } \ bool isSet() { \ return !getOptionName().empty(); \ } \ template \ - Type& set(Type&& src) { \ - mConfigDeclareField__ = std::forward(src); \ + Type& set(const Type& src) { \ + getHolder() = std::forward(src); \ return as(); \ } \ @@ -154,7 +172,7 @@ public: #define GENERATE_UNION_VISIT_CONST__(r, _, TYPE_) \ if (#TYPE_ == name) { \ - v.visit("value", as()); \ + v.visit("value", as()); \ } #define GENERATE_UNION_NAME__(r, _, TYPE_) \ -- 2.7.4 From 8aa153f871f2f7ba005458dde3104abc191570ef Mon Sep 17 00:00:00 2001 From: Krzysztof Dynowski Date: Sat, 6 Dec 2014 12:53:07 +0100 Subject: [PATCH 11/16] config (dynamic): implemented kvjson visitor and loading methods [Bug/Feature] load config from kvstore with defaults from json [Cause] N/A [Solution] N/A [Verification] Build, install Change-Id: I58c70c0089102b8c4ec1c659937e43d698cd6197 --- src/config/from-kvjson-visitor.hpp | 215 +++++++++++++++++++++++++++++++++++++ src/config/fs-utils.hpp | 4 + src/config/kvstore.cpp | 6 +- src/config/manager.hpp | 39 +++++++ 4 files changed, 262 insertions(+), 2 deletions(-) create mode 100644 src/config/from-kvjson-visitor.hpp diff --git a/src/config/from-kvjson-visitor.hpp b/src/config/from-kvjson-visitor.hpp new file mode 100644 index 0000000..174bbad --- /dev/null +++ b/src/config/from-kvjson-visitor.hpp @@ -0,0 +1,215 @@ +#ifndef CONFIG_FROM_KVJSON_VISITOR_HPP +#define CONFIG_FROM_KVJSON_VISITOR_HPP + +/* + * Copyright (c) 2014 Samsung Electronics Co., Ltd All Rights Reserved + * + * Contact: Krzysztof Dynowski (k.dynowski@samsumg.com) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License + */ + + +/** + * @file + * @author Krzysztof Dynowski (k.dynowski@samsumg.com) + * @brief KVStore visitor with defaults values from json + */ + +#include "config/manager.hpp" + + +namespace config { + +class FromKVJsonVisitor { +public: + FromKVJsonVisitor(const std::string& db, const std::string& json) : + mStorePtr(new KVStore(db)) + { + mObject = json_tokener_parse(json.c_str()); + if (mObject == nullptr) { + throw ConfigException("Json parsing error"); + } + } + + ~FromKVJsonVisitor() { + json_object_put(mObject); + } + + FromKVJsonVisitor(const FromKVJsonVisitor& v) : + mObject(json_object_get(v.mObject)), + mStorePtr(v.mStorePtr), + mKeyPrefix(v.mKeyPrefix) + { + } + FromKVJsonVisitor& operator=(const FromKVJsonVisitor&) = delete; + + template + void visit(const std::string& name, T& value) { + getValue(name, value); + } + +private: + std::shared_ptr mStorePtr; + std::string mKeyPrefix; + json_object* mObject; + + // create visitor for field object (visitable object) + FromKVJsonVisitor(const FromKVJsonVisitor& v, const std::string& name) : + mStorePtr(v.mStorePtr) + { + json_object* object; + if (!json_object_object_get_ex(v.mObject, name.c_str(), &object)) { + throw ConfigException("Missing json field " + key(mKeyPrefix, name)); + } + mObject = json_object_get(object); + mKeyPrefix = key(v.mKeyPrefix, name); + } + + // create visitor for vector i-th element (visitable object) + FromKVJsonVisitor(const FromKVJsonVisitor& v, int i) : + mStorePtr(v.mStorePtr) + { + json_object* object = json_object_array_get_idx(v.mObject, i); + checkType(object, json_type_object); + mObject = json_object_get(object); + mKeyPrefix = key(v.mKeyPrefix, std::to_string(i)); + } + + template::value, int>::type = 0> + void getValue(const std::string& name, T& t) + { + json_object* object; + if (!json_object_object_get_ex(mObject, name.c_str(), &object)) { + throw ConfigException("Missing json field " + key(mKeyPrefix, name)); + } + std::string k = key(mKeyPrefix, name); + if (mStorePtr->exists(k)) { + t = mStorePtr->get(k); + } + else { + fromJsonObject(object, t); + } + } + + template::value, int>::type = 0> + void getValue(const std::string& name, T& t) + { + FromKVJsonVisitor visitor(*this, name); + t.accept(visitor); + } + + int getArraySize(std::string& name, json_object* object) + { + int length = -1, jlength = json_object_array_length(object); + if (mStorePtr->exists(name)) { + length = mStorePtr->get(name); + } + return length != jlength ? jlength : length; + } + + template + void getValue(const std::string& name, std::vector& value) + { + json_object* object; + if (!json_object_object_get_ex(mObject, name.c_str(), &object)) { + throw ConfigException("Missing json field " + key(mKeyPrefix, name)); + } + checkType(object, json_type_array); + + std::string k = key(mKeyPrefix, name); + int length = getArraySize(k, object); + value.resize(static_cast(length)); + FromKVJsonVisitor visitor(*this, name); + for (int i = 0; i < length; ++i) { + visitor.getValue(i, value[i]); + } + } + + template::value, int>::type = 0> + void getValue(int i, T& t) + { + json_object* object = json_object_array_get_idx(mObject, i); + std::string k = key(mKeyPrefix, std::to_string(i)); + if (mStorePtr->exists(k)) { + t = mStorePtr->get(k); + } + else { + fromJsonObject(object, t); + } + } + + template::value, int>::type = 0> + void getValue(int i, T& t) + { + FromKVJsonVisitor visitor(*this, i); + t.accept(visitor); + } + + template + void getValue(int i, std::vector& value) + { + std::string k = key(mKeyPrefix, std::to_string(i)); + int length = getArraySize(k, mObject); + value.resize(static_cast(length)); + FromKVJsonVisitor visitor(*this, i); + for (int i = 0; i < length; ++i) { + visitor.getValue(i, value[i]); + } + } + + static void checkType(json_object* object, json_type type) + { + if (type != json_object_get_type(object)) { + throw ConfigException("Invalid field type " + std::to_string(type)); + } + } + + static void fromJsonObject(json_object* object, int& value) + { + checkType(object, json_type_int); + std::int64_t value64 = json_object_get_int64(object); + if (value64 > INT32_MAX || value64 < INT32_MIN) { + throw ConfigException("Value out of range"); + } + value = static_cast(value64); + } + + static void fromJsonObject(json_object* object, std::int64_t& value) + { + checkType(object, json_type_int); + value = json_object_get_int64(object); + } + + static void fromJsonObject(json_object* object, bool& value) + { + checkType(object, json_type_boolean); + value = json_object_get_boolean(object); + } + + static void fromJsonObject(json_object* object, double& value) + { + checkType(object, json_type_double); + value = json_object_get_double(object); + } + + static void fromJsonObject(json_object* object, std::string& value) + { + checkType(object, json_type_string); + value = json_object_get_string(object); + } +}; + +} // namespace config + +#endif // CONFIG_FROM_KVJSON_VISITOR_HPP diff --git a/src/config/fs-utils.hpp b/src/config/fs-utils.hpp index 924a642..82b85f8 100644 --- a/src/config/fs-utils.hpp +++ b/src/config/fs-utils.hpp @@ -33,6 +33,10 @@ namespace fsutils { bool readFileContent(const std::string& path, std::string& result); bool saveFileContent(const std::string& path, const std::string& content); +inline std::string readFileContent(const std::string& path) { + std::string content; + return readFileContent(path, content) ? content : std::string(); +} } // namespace fsutils } // namespace config diff --git a/src/config/kvstore.cpp b/src/config/kvstore.cpp index 621f7f6..d6223fc 100644 --- a/src/config/kvstore.cpp +++ b/src/config/kvstore.cpp @@ -165,7 +165,9 @@ void KVStore::prepareStatements() mGetValueStmt.reset( new sqlite3::Statement(mConn, "SELECT value FROM data WHERE key = ? LIMIT 1")); mGetKeyExistsStmt.reset( - new sqlite3::Statement(mConn, "SELECT 1 FROM data WHERE key = ?1 OR key GLOB escapeStr(?1) || '.*' LIMIT 1")); + // following line left in comment to have example of any subkey matching + //new sqlite3::Statement(mConn, "SELECT 1 FROM data WHERE key = ?1 OR key GLOB escapeStr(?1) || '.*' LIMIT 1")); + new sqlite3::Statement(mConn, "SELECT 1 FROM data WHERE key = ?1 LIMIT 1")); mGetIsEmptyStmt.reset( new sqlite3::Statement(mConn, "SELECT 1 FROM data LIMIT 1")); mSetValueStmt.reset( @@ -280,7 +282,7 @@ std::string KVStore::getInternal(const std::string& key, std::string*) int ret = ::sqlite3_step(mGetValueStmt->get()); if (ret == SQLITE_DONE) { - throw ConfigException("No value corresponding to the key: " + key); + throw ConfigException("No value corresponding to the key: " + key + "@" + mPath); } if (ret != SQLITE_ROW) { throw ConfigException("Error during stepping: " + mConn.getErrorMessage()); diff --git a/src/config/manager.hpp b/src/config/manager.hpp index a2623ea..eb7c740 100644 --- a/src/config/manager.hpp +++ b/src/config/manager.hpp @@ -31,6 +31,7 @@ #include "config/from-json-visitor.hpp" #include "config/from-kvstore-visitor.hpp" #include "config/from-fdstore-visitor.hpp" +#include "config/from-kvjson-visitor.hpp" #include "config/is-visitable.hpp" #include "config/fs-utils.hpp" @@ -136,6 +137,44 @@ void saveToKVStore(const std::string& filename, const Config& config, const std: } /** + * Load the config from KVStore with defaults given in json + * + * @param kvfile path to the KVStore db + * @param jsonfile path to json file with defaults + * @param config visitable structure to save + */ +template +void loadFromKVStoreWithJson(const std::string& kvfile, const std::string& json, Config& config) +{ + static_assert(isVisitable::value, "Use CONFIG_REGISTER macro"); + + FromKVJsonVisitor visitor(kvfile, json); + config.accept(visitor); +} +/** + * Load the config from KVStore with defaults given in json file + * + * @param kvfile path to the KVStore db + * @param jsonfile path to json file with defaults + * @param config visitable structure to save + */ +template +void loadFromKVStoreWithJsonFile(const std::string& kvfile, const std::string& jsonfile, Config& config) +{ + static_assert(isVisitable::value, "Use CONFIG_REGISTER macro"); + + std::string content; + if (!fsutils::readFileContent(jsonfile, content)) { + throw ConfigException("Could not load " + jsonfile); + } + try { + loadFromKVStoreWithJson(kvfile, content, config); + } catch (ConfigException& e) { + throw ConfigException("Error in " + jsonfile + ": " + e.what()); + } +} + +/** * Load binary data from a file/socket/pipe represented by the fd * * @param fd file descriptor -- 2.7.4 From a16f0812ed167378199adeaa5c30f938d49df0bc Mon Sep 17 00:00:00 2001 From: Mateusz Malicki Date: Fri, 9 Jan 2015 16:23:19 +0100 Subject: [PATCH 12/16] Prevent for writing unset union [Bug] Unset can be written to file (can't be read) [Cause] N/A [Solution] Assert in set and accept function, throw config::ConfigException when unsupported type is read or writen [Verification] N/A Change-Id: I5e30d53fe64a9fb132178c1f191af5a5910336d4 --- src/config/fields-union.hpp | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/src/config/fields-union.hpp b/src/config/fields-union.hpp index 53620ca..ec4ea3c 100644 --- a/src/config/fields-union.hpp +++ b/src/config/fields-union.hpp @@ -26,6 +26,8 @@ #define CONFIG_FIELDS_UNION_HPP #include "config/fields.hpp" +#include "config/exception.hpp" + #include #include #include @@ -104,10 +106,12 @@ private: template \ void visitOption(Visitor& v, const std::string& name) { \ GENERATE_CODE(GENERATE_UNION_VISIT__, __VA_ARGS__) \ + throw config::ConfigException("Union type error. Unsupported type"); \ } \ template \ void visitOption(Visitor& v, const std::string& name) const { \ GENERATE_CODE(GENERATE_UNION_VISIT_CONST__, __VA_ARGS__) \ + throw config::ConfigException("Union type error. Unsupported type"); \ } \ std::string getOptionName() const { \ GENERATE_CODE(GENERATE_UNION_NAME__, __VA_ARGS__) \ @@ -131,12 +135,9 @@ public: template \ void accept(Visitor v) const { \ const std::string name = getOptionName(); \ - if (!name.empty()) { \ - v.visit("type", name); \ - visitOption(v, name); \ - } else { \ - /* Unsupported type */ \ - } \ + assert(!name.empty() && "Type is not set"); \ + v.visit("type", name); \ + visitOption(v, name); \ } \ \ template \ @@ -159,6 +160,7 @@ public: template \ Type& set(const Type& src) { \ getHolder() = std::forward(src); \ + assert(!getOptionName().empty() && "Unsupported type"); \ return as(); \ } \ @@ -168,11 +170,13 @@ public: #define GENERATE_UNION_VISIT__(r, _, TYPE_) \ if (#TYPE_ == name) { \ v.visit("value", set(std::move(TYPE_()))); \ + return; \ } #define GENERATE_UNION_VISIT_CONST__(r, _, TYPE_) \ if (#TYPE_ == name) { \ v.visit("value", as()); \ + return; \ } #define GENERATE_UNION_NAME__(r, _, TYPE_) \ -- 2.7.4 From e28d57b64695d9f9a4a5939a9e8b71459d1fb9ae Mon Sep 17 00:00:00 2001 From: Piotr Bartosiewicz Date: Mon, 12 Jan 2015 16:34:00 +0100 Subject: [PATCH 13/16] Ability to list all keys from kv store [Bug/Feature] Add listKeys method. Mainly for debug purposes. [Cause] N/A [Solution] N/A [Verification] Build Change-Id: I1d760e047e0393e8ecdb22c242f3ef52e7fdf9d5 --- src/config/kvstore.cpp | 23 +++++++++++++++++++++++ src/config/kvstore.hpp | 6 ++++++ 2 files changed, 29 insertions(+) diff --git a/src/config/kvstore.cpp b/src/config/kvstore.cpp index d6223fc..a9d3fb6 100644 --- a/src/config/kvstore.cpp +++ b/src/config/kvstore.cpp @@ -174,6 +174,8 @@ void KVStore::prepareStatements() new sqlite3::Statement(mConn, "INSERT OR REPLACE INTO data (key, value) VALUES (?,?)")); mRemoveValuesStmt.reset( new sqlite3::Statement(mConn, "DELETE FROM data WHERE key = ?1 OR key GLOB escapeStr(?1) ||'.*' ")); + mGetKeysStmt.reset( + new sqlite3::Statement(mConn, "SELECT key FROM data")); } void KVStore::createFunctions() @@ -310,4 +312,25 @@ std::vector KVStore::getInternal(const std::string& key, std::vecto return values; } +std::vector KVStore::getKeys() +{ + Transaction transaction = getTransaction(); + ScopedReset scopedReset(mGetKeysStmt); + + std::vector result; + + for (;;) { + int ret = ::sqlite3_step(mGetKeysStmt->get()); + if (ret == SQLITE_DONE) { + return result; + } + if (ret != SQLITE_ROW) { + throw ConfigException("Error during stepping: " + mConn.getErrorMessage()); + } + const char* key = reinterpret_cast(sqlite3_column_text(mGetKeysStmt->get(), + FIRST_COLUMN)); + result.push_back(key); + } +} + } // namespace config diff --git a/src/config/kvstore.hpp b/src/config/kvstore.hpp index 44c0aa1..4dc978a 100644 --- a/src/config/kvstore.hpp +++ b/src/config/kvstore.hpp @@ -106,6 +106,11 @@ public: return getInternal(key, static_cast(nullptr)); } + /** + * Returns all stored keys. + */ + std::vector getKeys(); + KVStore::Transaction getTransaction(); private: @@ -139,6 +144,7 @@ private: std::unique_ptr mGetValueListStmt; std::unique_ptr mSetValueStmt; std::unique_ptr mRemoveValuesStmt; + std::unique_ptr mGetKeysStmt; void setupDb(); void prepareStatements(); -- 2.7.4 From c9099ead16ca369eec52a5d1fbfdca5df6019ed4 Mon Sep 17 00:00:00 2001 From: Mateusz Malicki Date: Tue, 13 Jan 2015 15:03:39 +0100 Subject: [PATCH 14/16] Fix: Crashing when writing unset union type [Bug] Assert instead exception [Cause] Assert [Solution] Assert was removed [Verification] Build in debug mode, run test Change-Id: I93e9a481d008ea50882b7886f0a63c9f6136bbd0 --- src/config/fields-union.hpp | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/src/config/fields-union.hpp b/src/config/fields-union.hpp index ec4ea3c..d37791d 100644 --- a/src/config/fields-union.hpp +++ b/src/config/fields-union.hpp @@ -30,7 +30,6 @@ #include #include -#include /** * Use this macro to declare and register config fields @@ -135,7 +134,9 @@ public: template \ void accept(Visitor v) const { \ const std::string name = getOptionName(); \ - assert(!name.empty() && "Type is not set"); \ + if (name.empty()) { \ + throw config::ConfigException("Type is not set"); \ + } \ v.visit("type", name); \ visitOption(v, name); \ } \ @@ -146,12 +147,16 @@ public: } \ template \ typename std::enable_if::value, Type>::type& as() { \ - assert(!getHolder().empty()); \ + if (getHolder().empty()) { \ + throw config::ConfigException("Type is not set"); \ + } \ return boost::any_cast(getHolder()); \ } \ template \ const Type& as() const { \ - assert(!getHolder().empty()); \ + if (getHolder().empty()) { \ + throw config::ConfigException("Type is not set"); \ + } \ return boost::any_cast(getHolder()); \ } \ bool isSet() { \ @@ -160,7 +165,9 @@ public: template \ Type& set(const Type& src) { \ getHolder() = std::forward(src); \ - assert(!getOptionName().empty() && "Unsupported type"); \ + if (getOptionName().empty()) { \ + throw config::ConfigException("Unsupported type"); \ + } \ return as(); \ } \ -- 2.7.4 From fbce303855c4dc63be84ee80dbce7d561bdd9f39 Mon Sep 17 00:00:00 2001 From: Piotr Bartosiewicz Date: Thu, 15 Jan 2015 10:13:33 +0100 Subject: [PATCH 15/16] Fix arrays in kvjson visitor, prohibit empty db name [Bug/Feature] Arrays were not correctly handled. [Cause] N/A [Solution] N/A [Verification] Build, run tests Change-Id: Idd5c2c35baf602e09961ee8af80857dda313a5da --- src/config/from-kvjson-visitor.hpp | 5 ++--- src/config/sqlite3/connection.cpp | 5 +++++ 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/src/config/from-kvjson-visitor.hpp b/src/config/from-kvjson-visitor.hpp index 174bbad..bbae07e 100644 --- a/src/config/from-kvjson-visitor.hpp +++ b/src/config/from-kvjson-visitor.hpp @@ -111,11 +111,10 @@ private: int getArraySize(std::string& name, json_object* object) { - int length = -1, jlength = json_object_array_length(object); if (mStorePtr->exists(name)) { - length = mStorePtr->get(name); + return mStorePtr->get(name); } - return length != jlength ? jlength : length; + return json_object_array_length(object); } template diff --git a/src/config/sqlite3/connection.cpp b/src/config/sqlite3/connection.cpp index e5d0552..cb25694 100644 --- a/src/config/sqlite3/connection.cpp +++ b/src/config/sqlite3/connection.cpp @@ -31,6 +31,11 @@ namespace sqlite3 { Connection::Connection(const std::string& path) { + if (path.empty()) { + // Sqlite creates temporary database in case of empty path + // but we want to forbid this. + throw ConfigException("Error opening the database: empty path"); + } if (::sqlite3_open_v2(path.c_str(), &mDbPtr, SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE, -- 2.7.4 From f181b5f2d77b6981c0c05f88af63f1bf99535749 Mon Sep 17 00:00:00 2001 From: Piotr Bartosiewicz Date: Fri, 16 Jan 2015 14:50:09 +0100 Subject: [PATCH 16/16] Config manager api refinements [Bug/Feature] Always use prefix in db. Function renames. [Cause] N/A [Solution] N/A [Verification] Build, run tests Change-Id: Ia32747c15c904b3c67b00aeb7a6bf051c9591de9 --- src/config/from-kvjson-visitor.hpp | 5 +++-- src/config/from-kvstore-visitor.hpp | 18 +++++++----------- src/config/manager.hpp | 36 +++++++++++++++++++++--------------- src/config/to-kvstore-visitor.hpp | 18 +++++++----------- 4 files changed, 38 insertions(+), 39 deletions(-) diff --git a/src/config/from-kvjson-visitor.hpp b/src/config/from-kvjson-visitor.hpp index bbae07e..d2c1060 100644 --- a/src/config/from-kvjson-visitor.hpp +++ b/src/config/from-kvjson-visitor.hpp @@ -33,8 +33,9 @@ namespace config { class FromKVJsonVisitor { public: - FromKVJsonVisitor(const std::string& db, const std::string& json) : - mStorePtr(new KVStore(db)) + FromKVJsonVisitor(const std::string& db, const std::string& json, const std::string& prefix) + : mStorePtr(new KVStore(db)) + , mKeyPrefix(prefix) { mObject = json_tokener_parse(json.c_str()); if (mObject == nullptr) { diff --git a/src/config/from-kvstore-visitor.hpp b/src/config/from-kvstore-visitor.hpp index c7e3540..3981bfd 100644 --- a/src/config/from-kvstore-visitor.hpp +++ b/src/config/from-kvstore-visitor.hpp @@ -43,17 +43,6 @@ public: { } - FromKVStoreVisitor(const FromKVStoreVisitor& visitor, const std::string& prefix) - : mStorePtr(visitor.mStorePtr), - mKeyPrefix(prefix), - mTransaction(visitor.mTransaction) - { - } - - ~FromKVStoreVisitor() - { - } - FromKVStoreVisitor& operator=(const FromKVStoreVisitor&) = delete; template @@ -67,6 +56,13 @@ private: std::string mKeyPrefix; KVStore::Transaction mTransaction; + FromKVStoreVisitor(const FromKVStoreVisitor& visitor, const std::string& prefix) + : mStorePtr(visitor.mStorePtr), + mKeyPrefix(prefix), + mTransaction(visitor.mTransaction) + { + } + template::value, int>::type = 0> void getInternal(const std::string& name, T& value) { diff --git a/src/config/manager.hpp b/src/config/manager.hpp index eb7c740..0785094 100644 --- a/src/config/manager.hpp +++ b/src/config/manager.hpp @@ -45,7 +45,7 @@ namespace config { * @param config visitable structure to fill */ template -void loadFromString(const std::string& jsonString, Config& config) +void loadFromJsonString(const std::string& jsonString, Config& config) { static_assert(isVisitable::value, "Use CONFIG_REGISTER macro"); @@ -53,14 +53,13 @@ void loadFromString(const std::string& jsonString, Config& config) config.accept(visitor); } - /** * Creates a string representation of the configuration in json format * * @param config visitable structure to convert */ template -std::string saveToString(const Config& config) +std::string saveToJsonString(const Config& config) { static_assert(isVisitable::value, "Use CONFIG_REGISTER macro"); @@ -76,14 +75,14 @@ std::string saveToString(const Config& config) * @param config visitable structure to load */ template -void loadFromFile(const std::string& filename, Config& config) +void loadFromJsonFile(const std::string& filename, Config& config) { std::string content; if (!fsutils::readFileContent(filename, content)) { throw ConfigException("Could not load " + filename); } try { - loadFromString(content, config); + loadFromJsonString(content, config); } catch (ConfigException& e) { throw ConfigException("Error in " + filename + ": " + e.what()); } @@ -96,9 +95,9 @@ void loadFromFile(const std::string& filename, Config& config) * @param config visitable structure to save */ template -void saveToFile(const std::string& filename, const Config& config) +void saveToJsonFile(const std::string& filename, const Config& config) { - const std::string content = saveToString(config); + const std::string content = saveToJsonString(config); if (!fsutils::saveFileContent(filename, content)) { throw ConfigException("Could not save " + filename); } @@ -112,7 +111,7 @@ void saveToFile(const std::string& filename, const Config& config) * @param configName name of the configuration inside the KVStore db */ template -void loadFromKVStore(const std::string& filename, Config& config, const std::string& configName = "") +void loadFromKVStore(const std::string& filename, Config& config, const std::string& configName) { static_assert(isVisitable::value, "Use CONFIG_REGISTER macro"); @@ -128,7 +127,7 @@ void loadFromKVStore(const std::string& filename, Config& config, const std::str * @param configName name of the config inside the KVStore db */ template -void saveToKVStore(const std::string& filename, const Config& config, const std::string& configName = "") +void saveToKVStore(const std::string& filename, const Config& config, const std::string& configName) { static_assert(isVisitable::value, "Use CONFIG_REGISTER macro"); @@ -142,33 +141,40 @@ void saveToKVStore(const std::string& filename, const Config& config, const std: * @param kvfile path to the KVStore db * @param jsonfile path to json file with defaults * @param config visitable structure to save + * @param kvConfigName name of the config inside the KVStore db */ template -void loadFromKVStoreWithJson(const std::string& kvfile, const std::string& json, Config& config) +void loadFromKVStoreWithJson(const std::string& kvfile, + const std::string& json, + Config& config, + const std::string& kvConfigName) { static_assert(isVisitable::value, "Use CONFIG_REGISTER macro"); - FromKVJsonVisitor visitor(kvfile, json); + FromKVJsonVisitor visitor(kvfile, json, kvConfigName); config.accept(visitor); } + /** * Load the config from KVStore with defaults given in json file * * @param kvfile path to the KVStore db * @param jsonfile path to json file with defaults * @param config visitable structure to save + * @param kvConfigName name of the config inside the KVStore db */ template -void loadFromKVStoreWithJsonFile(const std::string& kvfile, const std::string& jsonfile, Config& config) +void loadFromKVStoreWithJsonFile(const std::string& kvfile, + const std::string& jsonfile, + Config& config, + const std::string& kvConfigName) { - static_assert(isVisitable::value, "Use CONFIG_REGISTER macro"); - std::string content; if (!fsutils::readFileContent(jsonfile, content)) { throw ConfigException("Could not load " + jsonfile); } try { - loadFromKVStoreWithJson(kvfile, content, config); + loadFromKVStoreWithJson(kvfile, content, config, kvConfigName); } catch (ConfigException& e) { throw ConfigException("Error in " + jsonfile + ": " + e.what()); } diff --git a/src/config/to-kvstore-visitor.hpp b/src/config/to-kvstore-visitor.hpp index 37b70a9..071dece 100644 --- a/src/config/to-kvstore-visitor.hpp +++ b/src/config/to-kvstore-visitor.hpp @@ -43,17 +43,6 @@ public: { } - ToKVStoreVisitor(const ToKVStoreVisitor& visitor, const std::string& prefix) - : mStorePtr(visitor.mStorePtr), - mKeyPrefix(prefix), - mTransaction(visitor.mTransaction) - { - } - - ~ToKVStoreVisitor() - { - } - ToKVStoreVisitor& operator=(const ToKVStoreVisitor&) = delete; template @@ -67,6 +56,13 @@ private: std::string mKeyPrefix; KVStore::Transaction mTransaction; + ToKVStoreVisitor(const ToKVStoreVisitor& visitor, const std::string& prefix) + : mStorePtr(visitor.mStorePtr), + mKeyPrefix(prefix), + mTransaction(visitor.mTransaction) + { + } + template::value, int>::type = 0> void setInternal(const std::string& name, const T& value) { -- 2.7.4