From: Jaeyun Jung Date: Tue, 7 Feb 2023 08:45:34 +0000 (+0900) Subject: [Service] replace ml-service DB X-Git-Tag: accepted/tizen/unified/20230223.162150~12 X-Git-Url: http://review.tizen.org/git/?a=commitdiff_plain;h=c31450ba119a6d4435a4fc017a26e186e5d5be34;p=platform%2Fcore%2Fapi%2Fmachine-learning.git [Service] replace ml-service DB Replace ml-service database to sqlite3. TODO: - handle database version. - implement methods for model info. Signed-off-by: Jaeyun Jung --- diff --git a/daemon/includes/service-db.hh b/daemon/includes/service-db.hh index 0331d5a..c84b480 100644 --- a/daemon/includes/service-db.hh +++ b/daemon/includes/service-db.hh @@ -13,7 +13,7 @@ #ifndef __SERVICE_DB_HH__ #define __SERVICE_DB_HH__ -#include +#include #include /** @@ -29,9 +29,12 @@ public: virtual void connectDB (); virtual void disconnectDB (); - virtual void put (const std::string name, const std::string value); - virtual void get (std::string name, std::string &out_value); - virtual void del (std::string name); + virtual void set_pipeline (const std::string name, const std::string description); + virtual void get_pipeline (const std::string name, std::string &description); + virtual void delete_pipeline (const std::string name); + virtual void set_model (const std::string name, const std::string model); + virtual void get_model (const std::string name, std::string &model); + virtual void delete_model (const std::string name); static MLServiceDB & getInstance (void); @@ -39,10 +42,9 @@ private: MLServiceDB (std::string path); virtual ~MLServiceDB (); - std::string path; - leveldb_t *db_obj; - leveldb_readoptions_t *db_roptions; - leveldb_writeoptions_t *db_woptions; + std::string _path; + bool _initialized; + sqlite3 *_db; }; #endif /* __SERVICE_DB_HH__ */ diff --git a/daemon/meson.build b/daemon/meson.build index f89ae1b..48b60e8 100644 --- a/daemon/meson.build +++ b/daemon/meson.build @@ -43,6 +43,7 @@ if get_option('enable-ml-service') gio_dep, gst_dep, leveldb_dep, + sqlite_dep, libsystemd_dep ] diff --git a/daemon/model-dbus-impl.cc b/daemon/model-dbus-impl.cc index 68af61a..50373eb 100644 --- a/daemon/model-dbus-impl.cc +++ b/daemon/model-dbus-impl.cc @@ -60,7 +60,7 @@ dbus_cb_model_set_path (MachinelearningServiceModel *obj, try { db.connectDB(); - db.put (name, std::string (path)); + db.set_model (name, std::string (path)); } catch (const std::runtime_error & e) { @@ -100,7 +100,7 @@ dbus_cb_model_get_path (MachinelearningServiceModel *obj, try { db.connectDB (); - db.get (name, ret_path); + db.get_model (name, ret_path); } catch (const std::invalid_argument & e) { @@ -139,7 +139,7 @@ gdbus_cb_model_delete (MachinelearningServiceModel *obj, try { db.connectDB (); - db.del (name); + db.delete_model (name); } catch (const std::invalid_argument & e) { diff --git a/daemon/pipeline-module.cc b/daemon/pipeline-module.cc index 272416c..cd93a88 100644 --- a/daemon/pipeline-module.cc +++ b/daemon/pipeline-module.cc @@ -79,7 +79,7 @@ static gboolean dbus_cb_core_set_pipeline (MachinelearningServicePipeline *obj, try { db.connectDB (); - db.put (service_name, pipeline_desc); + db.set_pipeline (service_name, pipeline_desc); } catch (const std::invalid_argument &e) { _E ("An exception occurred during write to the DB. Error message: %s", e.what ()); result = -EINVAL; @@ -116,7 +116,7 @@ static gboolean dbus_cb_core_get_pipeline (MachinelearningServicePipeline *obj, try { db.connectDB (); - db.get (service_name, stored_pipeline_description); + db.get_pipeline (service_name, stored_pipeline_description); } catch (const std::invalid_argument &e) { _E ("An exception occurred during read the DB. Error message: %s", e.what ()); result = -EINVAL; @@ -152,7 +152,7 @@ static gboolean dbus_cb_core_delete_pipeline (MachinelearningServicePipeline *ob try { db.connectDB (); - db.del (service_name); + db.delete_pipeline (service_name); } catch (const std::invalid_argument &e) { _E ("An exception occurred during delete an item in the DB. Error message: %s", e.what ()); result = -EINVAL; @@ -195,7 +195,7 @@ static gboolean dbus_cb_core_launch_pipeline (MachinelearningServicePipeline *ob /** get pipeline description from the DB */ try { db.connectDB (); - db.get (service_name, stored_pipeline_description); + db.get_pipeline (service_name, stored_pipeline_description); } catch (const std::invalid_argument &e) { _E ("An exception occurred during read the DB. Error message: %s", e.what ()); result = -EINVAL; diff --git a/daemon/service-db.cc b/daemon/service-db.cc index 63db2d7..9f69b9a 100644 --- a/daemon/service-db.cc +++ b/daemon/service-db.cc @@ -14,9 +14,31 @@ #include "service-db.hh" -#define ML_DATABASE_PATH DB_PATH"/.ml-service-leveldb" +#define sqlite3_clear_errmsg(m) do { if (m) { sqlite3_free (m); (m) = nullptr; } } while (0) + +#define ML_DATABASE_PATH DB_PATH"/.ml-service.db" #define DB_KEY_PREFIX MESON_KEY_PREFIX +typedef enum +{ + TBL_DB_INFO = 0, + TBL_PIPELINE_DESCRIPTION = 1, + TBL_MODEL_INFO = 2, + + TBL_MAX +} mlsvc_table_e; + +const char *g_mlsvc_table_schema_v1[] = +{ + [TBL_DB_INFO] = "tblMLDBInfo (name TEXT PRIMARY KEY NOT NULL, version INTEGER DEFAULT 1)", + [TBL_PIPELINE_DESCRIPTION] = "tblPipeline (key TEXT PRIMARY KEY NOT NULL, description TEXT, CHECK (length(description) > 0))", + [TBL_MODEL_INFO] = "tblModel (key TEXT NOT NULL, version INTEGER DEFAULT 1, stable TEXT DEFAULT 'N', valid TEXT DEFAULT 'N', " + "path TEXT, PRIMARY KEY (key, version), CHECK (length(path) > 0), CHECK (stable IN ('T', 'F')), CHECK (valid IN ('T', 'F')))", + NULL +}; + +const char **g_mlsvc_table_schema = g_mlsvc_table_schema_v1; + /** * @brief Get an instance of MLServiceDB, which is created only once at runtime. * @return MLServiceDB& MLServiceDB instance @@ -33,11 +55,8 @@ MLServiceDB & MLServiceDB::getInstance (void) * @param path database path */ MLServiceDB::MLServiceDB (std::string path) -: path (path), db_obj (nullptr), db_roptions (nullptr), db_woptions (nullptr) +: _path (path), _initialized (false), _db (nullptr) { - db_roptions = leveldb_readoptions_create (); - db_woptions = leveldb_writeoptions_create (); - leveldb_writeoptions_set_sync (db_woptions, 1); } /** @@ -46,8 +65,7 @@ MLServiceDB::MLServiceDB (std::string path) MLServiceDB::~MLServiceDB () { disconnectDB (); - leveldb_readoptions_destroy (db_roptions); - leveldb_writeoptions_destroy (db_woptions); + _initialized = false; } /** @@ -56,149 +74,291 @@ MLServiceDB::~MLServiceDB () void MLServiceDB::connectDB () { - char *err = nullptr; - leveldb_options_t *db_options; + int rc; + char *sql; + char *errmsg = nullptr; - if (db_obj) + if (_db != nullptr) return; - db_options = leveldb_options_create (); - leveldb_options_set_create_if_missing (db_options, 1); + rc = sqlite3_open (_path.c_str (), &_db); + if (rc != SQLITE_OK) { + g_warning ("Failed to open database: %s (%d)", sqlite3_errmsg (_db), rc); + goto error; + } + + /* Create table and handle database version. */ + if (!_initialized) { + /** + * @todo handle database version + * - check old level db and insert pipeline description into sqlite. + * - check database info (TBL_DB_INFO), fetch data and update table schema when version is updated, ... + * - need transaction + */ - db_obj = leveldb_open (db_options, path.c_str (), &err); - leveldb_options_destroy (db_options); - if (err != nullptr) { - g_warning ("Error! Failed to open database located at '%s': leveldb_open() has returned an error: %s", - path.c_str (), err); - leveldb_free (err); + rc = sqlite3_exec (_db, "BEGIN TRANSACTION;", nullptr, nullptr, &errmsg); + if (rc != SQLITE_OK) { + g_warning ("Failed to begin transaction: %s (%d)", errmsg, rc); + sqlite3_clear_errmsg (errmsg); + goto error; + } + + /* Create tables. */ + sql = g_strdup_printf ("CREATE TABLE IF NOT EXISTS %s;", g_mlsvc_table_schema[TBL_DB_INFO]); + rc = sqlite3_exec (_db, sql, nullptr, nullptr, &errmsg); + g_free (sql); + if (rc != SQLITE_OK) { + g_warning ("Failed to create table for database info: %s (%d)", errmsg, rc); + sqlite3_clear_errmsg (errmsg); + goto error; + } + + sql = g_strdup_printf ("CREATE TABLE IF NOT EXISTS %s;", g_mlsvc_table_schema[TBL_PIPELINE_DESCRIPTION]); + rc = sqlite3_exec (_db, sql, nullptr, nullptr, &errmsg); + g_free (sql); + if (rc != SQLITE_OK) { + g_warning ("Failed to create table for pipeline description: %s (%d)", errmsg, rc); + sqlite3_clear_errmsg (errmsg); + goto error; + } + + sql = g_strdup_printf ("CREATE TABLE IF NOT EXISTS %s;", g_mlsvc_table_schema[TBL_MODEL_INFO]); + rc = sqlite3_exec (_db, sql, nullptr, nullptr, &errmsg); + g_free (sql); + if (rc != SQLITE_OK) { + g_warning ("Failed to create table for model info: %s (%d)", errmsg, rc); + sqlite3_clear_errmsg (errmsg); + goto error; + } + + rc = sqlite3_exec (_db, "END TRANSACTION;", nullptr, nullptr, &errmsg); + if (rc != SQLITE_OK) { + g_warning ("Failed to end transaction: %s (%d)", errmsg, rc); + sqlite3_clear_errmsg (errmsg); + goto error; + } + + _initialized = true; + } + +error: + if (!_initialized) { disconnectDB (); - throw std::runtime_error ("Failed to connectDB()!"); + throw std::runtime_error ("Failed to connect DB."); } } /** * @brief Disconnect the DB. - * @note LevelDB does not support multi-process and it might cause - * the IO exception when multiple clients write the key simultaneously. */ void MLServiceDB::disconnectDB () { - if (db_obj) { - leveldb_close (db_obj); - db_obj = nullptr; + if (_db) { + sqlite3_close (_db); + _db = nullptr; } } /** - * @brief Set the value with the given name. + * @brief Set the pipeline description with the given name. * @note If the name already exists, the pipeline description is overwritten. * @param[in] name Unique name to set the associated pipeline description. - * @param[in] value The pipeline description to be stored. + * @param[in] description The pipeline description to be stored. */ void -MLServiceDB::put (const std::string name, const std::string value) +MLServiceDB::set_pipeline (const std::string name, const std::string description) { - char *err = nullptr; + int rc; + char *sql; + char *errmsg = nullptr; - if (name.empty() || value.empty()) + if (name.empty () || description.empty ()) throw std::invalid_argument ("Invalid name or value parameters!"); std::string key_with_prefix = DB_KEY_PREFIX; key_with_prefix += name; - std::size_t hash[2] = { std::hash{}(key_with_prefix), 0U }; - - leveldb_put (db_obj, db_woptions, (char *) hash, sizeof (std::size_t) * 2, - value.c_str (), value.size (), &err); - if (err != nullptr) { - g_warning - ("Failed to call leveldb_put () for the name, '%s' of the pipeline description (size: %zu bytes / description: '%.40s')", - name.c_str (), value.size (), value.c_str ()); - g_warning ("leveldb_put () has returned an error: %s", err); - leveldb_free (err); - throw std::runtime_error ("Failed to put()!"); + + /* (key, description) */ + sql = g_strdup_printf ("INSERT OR REPLACE INTO tblPipeline VALUES ('%s', '%s');", + key_with_prefix.c_str (), description.c_str ()); + rc = sqlite3_exec (_db, sql, nullptr, nullptr, &errmsg); + g_free (sql); + if (rc != SQLITE_OK) { + g_warning ("Failed to insert pipeline description with name %s: %s (%d)", name.c_str (), errmsg, rc); + sqlite3_clear_errmsg (errmsg); + throw std::runtime_error ("Failed to insert pipeline description."); } } /** - * @brief Get the value with the given name. + * @brief Get the pipeline description with the given name. * @param[in] name The unique name to retrieve. - * @param[out] value The pipeline corresponding with the given name. + * @param[out] description The pipeline corresponding with the given name. */ void -MLServiceDB::get (const std::string name, std::string & out_value) +MLServiceDB::get_pipeline (const std::string name, std::string &description) { - char *err = nullptr; + int rc; + char *sql; char *value = nullptr; - gsize read_len; + sqlite3_stmt *res; - if (name.empty()) + if (name.empty ()) throw std::invalid_argument ("Invalid name parameters!"); std::string key_with_prefix = DB_KEY_PREFIX; key_with_prefix += name; - std::size_t hash[2] = { std::hash{}(key_with_prefix), 0U }; - - value = leveldb_get (db_obj, db_roptions, (char *) hash, sizeof (std::size_t) * 2, - &read_len, &err); - if (err != nullptr) { - g_warning - ("Failed to call leveldb_get() for the name %s. Error message is %s.", - name.c_str (), err); - leveldb_free (err); - if (value) - leveldb_free (value); - throw std::runtime_error ("Failed to get()!"); - } - if (!value) { - g_warning - ("Failed to find the key %s. The key should be set before reading it.", - name.c_str ()); - throw std::invalid_argument ("Failed to find the key."); + sql = g_strdup_printf ("SELECT description FROM tblPipeline WHERE key = '%s';", + key_with_prefix.c_str ()); + rc = sqlite3_prepare_v2 (_db, sql, -1, &res, nullptr); + g_free (sql); + if (rc != SQLITE_OK) { + g_warning ("Failed to get pipeline description with name %s: %s (%d)", name.c_str (), sqlite3_errmsg (_db), rc); + goto error; } - out_value = std::string (value, read_len); - leveldb_free (value); - return; + rc = sqlite3_step (res); + if (rc == SQLITE_ROW) + value = g_strdup_printf ("%s", sqlite3_column_text (res, 0)); + + sqlite3_finalize (res); + +error: + if (value) { + description = std::string (value); + g_free (value); + } else { + throw std::invalid_argument ("Failed to get pipeline description."); + } } /** - * @brief Delete the value with a given name. + * @brief Delete the pipeline description with a given name. * @param[in] name The unique name to delete */ void -MLServiceDB::del (const std::string name) +MLServiceDB::delete_pipeline (const std::string name) +{ + int rc; + char *sql; + char *errmsg = nullptr; + + if (name.empty ()) + throw std::invalid_argument ("Invalid name parameters!"); + + std::string key_with_prefix = DB_KEY_PREFIX; + key_with_prefix += name; + + sql = g_strdup_printf ("DELETE FROM tblPipeline WHERE key = '%s';", + key_with_prefix.c_str ()); + rc = sqlite3_exec (_db, sql, nullptr, nullptr, &errmsg); + g_free (sql); + if (rc != SQLITE_OK) { + g_warning ("Failed to delete pipeline description with name %s: %s (%d)", name.c_str (), errmsg, rc); + sqlite3_clear_errmsg (errmsg); + throw std::invalid_argument ("Failed to delete pipeline description."); + } +} + +/** + * @brief Set the model with the given name. + * @param[in] name Unique name for model. + * @param[in] model The model to be stored. + */ +void +MLServiceDB::set_model (const std::string name, const std::string model) { - char *err = nullptr; + int rc; + char *sql; + char *errmsg = nullptr; + + if (name.empty () || model.empty ()) + throw std::invalid_argument ("Invalid name or value parameters!"); + + std::string key_with_prefix = DB_KEY_PREFIX; + key_with_prefix += name; + + /* (key, version, stable, valid, path) */ + sql = g_strdup_printf ("INSERT OR REPLACE INTO tblModel VALUES ('%s', IFNULL ((SELECT version from tblModel WHERE key = '%s' ORDER BY version DESC LIMIT 1) + 1, 1), 'T', 'T', '%s');", + key_with_prefix.c_str (), key_with_prefix.c_str (), model.c_str ()); + rc = sqlite3_exec (_db, sql, nullptr, nullptr, &errmsg); + g_free (sql); + if (rc != SQLITE_OK) { + g_warning ("Failed to insert pipeline description with name %s: %s (%d)", name.c_str (), errmsg, rc); + sqlite3_clear_errmsg (errmsg); + throw std::runtime_error ("Failed to insert model."); + } +} + +/** + * @brief Get the model with the given name. + * @param[in] name The unique name to retrieve. + * @param[out] model The model corresponding with the given name. + */ +void +MLServiceDB::get_model (const std::string name, std::string &model) +{ + int rc; + char *sql; char *value = nullptr; - gsize read_len; + sqlite3_stmt *res; - if (name.empty()) + if (name.empty ()) throw std::invalid_argument ("Invalid name parameters!"); std::string key_with_prefix = DB_KEY_PREFIX; key_with_prefix += name; - std::size_t hash[2] = { std::hash{}(key_with_prefix), 0U }; - - /* Check whether the key exists or not. */ - value = leveldb_get (db_obj, db_roptions, (char *) hash, sizeof (std::size_t) * 2, - &read_len, &err); - if (!value || err) { - g_warning - ("Failed to find the key %s. The key should be set before reading it.", - name.c_str ()); - if (err) - leveldb_free (err); - throw std::invalid_argument ("Failed to find the key."); + + sql = g_strdup_printf ("SELECT path FROM tblModel WHERE key = '%s' ORDER BY version DESC LIMIT 1;", + key_with_prefix.c_str ()); + rc = sqlite3_prepare_v2 (_db, sql, -1, &res, nullptr); + g_free (sql); + if (rc != SQLITE_OK) { + g_warning ("Failed to get pipeline description with name %s: %s (%d)", name.c_str (), sqlite3_errmsg (_db), rc); + goto error; } - leveldb_free (value); - - leveldb_delete (db_obj, db_woptions, (char *) hash, sizeof (std::size_t) * 2, &err); - if (err != nullptr) { - g_warning ("Failed to delete the key %s. Error message is %s.", - name.c_str (), err); - leveldb_free (err); - throw std::runtime_error ("Failed to del()!"); + + rc = sqlite3_step (res); + if (rc == SQLITE_ROW) + value = g_strdup_printf ("%s", sqlite3_column_text (res, 0)); + + sqlite3_finalize (res); + +error: + if (value) { + model = std::string (value); + g_free (value); + } else { + throw std::invalid_argument ("Failed to get model."); + } +} + +/** + * @brief Delete the model. + * @param[in] name The unique name to delete + */ +void +MLServiceDB::delete_model (const std::string name) +{ + int rc; + char *sql; + char *errmsg = nullptr; + + if (name.empty ()) + throw std::invalid_argument ("Invalid name parameters!"); + + std::string key_with_prefix = DB_KEY_PREFIX; + key_with_prefix += name; + + sql = g_strdup_printf ("DELETE FROM tblModel WHERE key = '%s';", + key_with_prefix.c_str ()); + rc = sqlite3_exec (_db, sql, nullptr, nullptr, &errmsg); + g_free (sql); + if (rc != SQLITE_OK) { + g_warning ("Failed to delete model with name %s: %s (%d)", name.c_str (), errmsg, rc); + sqlite3_clear_errmsg (errmsg); + throw std::invalid_argument ("Failed to delete model."); } }