From eb45b4d93a76503e3195a36ee64005a14db5e228 Mon Sep 17 00:00:00 2001 From: Sangwan Kwon Date: Tue, 14 Jan 2020 14:29:47 +0900 Subject: [PATCH] osquery: Remove database Signed-off-by: Sangwan Kwon --- src/osquery/CMakeLists.txt | 1 - src/osquery/core/database/in_memory_database.cpp | 4 - src/osquery/core/init.cpp | 90 ---- src/osquery/core/query.cpp | 63 +-- src/osquery/core/system.cpp | 21 +- src/osquery/core/tables.cpp | 18 +- src/osquery/database/CMakeLists.txt | 18 - src/osquery/database/database.cpp | 572 --------------------- src/osquery/database/tests/database.cpp | 302 ----------- src/osquery/database/tests/results.cpp | 154 ------ src/osquery/devtools/shell.cpp | 10 - src/osquery/include/osquery/database.h | 295 ----------- .../include/osquery/database/plugins/ephemeral.h | 1 - .../include/osquery/database/plugins/rocksdb.h | 1 - .../include/osquery/database/plugins/sqlite.h | 1 - src/osquery/logger/logger.cpp | 3 +- src/osquery/logger/tests/logger.cpp | 5 - src/osquery/main/main.cpp | 5 - src/osquery/plugins/CMakeLists.txt | 3 - src/osquery/plugins/database/ephemeral.cpp | 143 ------ src/osquery/plugins/database/ephemeral.h | 85 --- src/osquery/plugins/database/sqlite.cpp | 314 ----------- src/osquery/plugins/database/sqlite.h | 87 ---- src/osquery/plugins/database/tests/sqlite.cpp | 22 - src/osquery/plugins/database/tests/utils.cpp | 268 ---------- src/osquery/plugins/database/tests/utils.h | 95 ---- src/osquery/sql/tests/virtual_table.cpp | 6 - src/osquery/tests/test_util.cpp | 11 - src/osquery/tests/test_util.h | 1 - 29 files changed, 8 insertions(+), 2591 deletions(-) delete mode 100644 src/osquery/database/CMakeLists.txt delete mode 100644 src/osquery/database/database.cpp delete mode 100644 src/osquery/database/tests/database.cpp delete mode 100644 src/osquery/database/tests/results.cpp delete mode 100644 src/osquery/include/osquery/database.h delete mode 120000 src/osquery/include/osquery/database/plugins/ephemeral.h delete mode 120000 src/osquery/include/osquery/database/plugins/rocksdb.h delete mode 120000 src/osquery/include/osquery/database/plugins/sqlite.h delete mode 100644 src/osquery/plugins/database/ephemeral.cpp delete mode 100644 src/osquery/plugins/database/ephemeral.h delete mode 100644 src/osquery/plugins/database/sqlite.cpp delete mode 100644 src/osquery/plugins/database/sqlite.h delete mode 100644 src/osquery/plugins/database/tests/sqlite.cpp delete mode 100644 src/osquery/plugins/database/tests/utils.cpp delete mode 100644 src/osquery/plugins/database/tests/utils.h diff --git a/src/osquery/CMakeLists.txt b/src/osquery/CMakeLists.txt index cb80e4e..b177d88 100644 --- a/src/osquery/CMakeLists.txt +++ b/src/osquery/CMakeLists.txt @@ -44,7 +44,6 @@ ENDIF(DEFINED GBS_BUILD) ## osquery v4.0.0 ADD_SUBDIRECTORY(core) -ADD_SUBDIRECTORY(database) ADD_SUBDIRECTORY(filesystem) ADD_SUBDIRECTORY(logger) ADD_SUBDIRECTORY(plugins) diff --git a/src/osquery/core/database/in_memory_database.cpp b/src/osquery/core/database/in_memory_database.cpp index bee2b9f..63002da 100644 --- a/src/osquery/core/database/in_memory_database.cpp +++ b/src/osquery/core/database/in_memory_database.cpp @@ -7,7 +7,6 @@ */ #include -#include #include #include @@ -61,9 +60,6 @@ ExpectedSuccess InMemoryDatabase::destroyDB() { ExpectedSuccess InMemoryDatabase::open() { debug_only::verifyTrue(!is_open_, "database is already open"); - for (const auto& domain : kDomains) { - storage_[domain] = std::make_unique>(); - } is_open_ = true; return Success(); } diff --git a/src/osquery/core/init.cpp b/src/osquery/core/init.cpp index 340a547..8482428 100644 --- a/src/osquery/core/init.cpp +++ b/src/osquery/core/init.cpp @@ -42,7 +42,6 @@ #include #include #include -#include #ifdef __linux__ #include @@ -151,17 +150,12 @@ namespace osquery { DECLARE_string(logger_plugin); DECLARE_bool(config_check); DECLARE_bool(config_dump); -DECLARE_bool(database_dump); -DECLARE_string(database_path); -DECLARE_bool(disable_database); DECLARE_bool(disable_logging); CLI_FLAG(bool, S, false, "Run as a shell process"); CLI_FLAG(bool, D, false, "Run as a daemon process"); CLI_FLAG(bool, daemonize, false, "Attempt to daemonize (POSIX only)"); -FLAG(bool, ephemeral, false, "Skip pidfile and database state checks"); - ToolType kToolType{ToolType::UNKNOWN}; /// The saved exit code from a thread's request to stop the process. @@ -176,29 +170,9 @@ unsigned long kLegacyThreadId; /// When no flagfile is provided via CLI, attempt to read flag 'defaults'. const std::string kBackupDefaultFlagfile{OSQUERY_HOME "osquery.flags.default"}; -const size_t kDatabaseMaxRetryCount{25}; -const size_t kDatabaseRetryDelay{200}; std::function Initializer::shutdown_{nullptr}; RecursiveMutex Initializer::shutdown_mutex_; -namespace { - -void initWorkDirectories() { - if (!FLAGS_disable_database) { - auto const recursive = true; - auto const ignore_existence = true; - auto const status = createDirectory( - boost::filesystem::path(FLAGS_database_path).parent_path(), - recursive, - ignore_existence); - if (!status.ok()) { - LOG(ERROR) << "Could not initialize db directory: " << status.what(); - } - } -} - -} // namespace - static inline void printUsage(const std::string& binary, ToolType tool) { // Parse help options before gflags. Only display osquery-related options. fprintf(stdout, DESCRIPTION, kVersion.c_str()); @@ -315,22 +289,10 @@ Initializer::Initializer(int& argc, // Initialize registries and plugins registryAndPluginInit(); - if (isShell() || FLAGS_ephemeral) { - if (Flag::isDefault("database_path") && - Flag::isDefault("disable_database")) { - // The shell should not use a database by default, but should use the DB - // specified by database_path if it is set - FLAGS_disable_database = true; - } - } - if (isShell()) { // Initialize the shell after setting modified defaults and parsing flags. initShell(); } - if (isDaemon()) { - initWorkDirectories(); - } if (FLAGS_enable_signal_handler) { std::signal(SIGABRT, signalHandler); @@ -368,32 +330,11 @@ void Initializer::initDaemon() const { // Print the version to the OS system log. systemLog(binary_ + " started [version=" + kVersion + "]"); - - if (!FLAGS_ephemeral) { - // Create a process mutex around the daemon. - auto pid_status = createPidFile(); - if (!pid_status.ok()) { - LOG(ERROR) << binary_ << " initialize failed: " << pid_status.toString(); - shutdown(EXIT_FAILURE); - } - } } void Initializer::initShell() const { // Get the caller's home dir for temporary storage/state management. auto homedir = osqueryHomeDirectory(); - if (osquery::pathExists(homedir).ok()) { - // Only apply user/shell-specific paths if not overridden by CLI flag. - if (Flag::isDefault("database_path")) { - osquery::FLAGS_database_path = - (fs::path(homedir) / "shell.db").make_preferred().string(); - } - } else { - fprintf( - stderr, "Cannot access or create osquery home: %s", homedir.c_str()); - FLAGS_disable_database = true; - } - if (Flag::isDefault("hash_delay")) { // The hash_delay is designed for daemons only. Flag::updateValue("hash_delay", "0"); @@ -430,39 +371,9 @@ void Initializer::installShutdown(std::function& handler) { } void Initializer::start() const { - DatabasePlugin::setAllowOpen(true); - // A daemon must always have R/W access to the database. - DatabasePlugin::setRequireWrite(isDaemon()); - - for (size_t i = 1; i <= kDatabaseMaxRetryCount; i++) { - if (DatabasePlugin::initPlugin().ok()) { - break; - } - - if (i == kDatabaseMaxRetryCount) { - LOG(ERROR) << RLOG(1629) << binary_ - << " initialize failed: Could not initialize database"; - auto retcode = (isWorker()) ? EXIT_CATASTROPHIC : EXIT_FAILURE; - requestShutdown(retcode); - } - } - - // Ensure the database results version is up to date before proceeding - if (!upgradeDatabase()) { - LOG(ERROR) << "Failed to upgrade database"; - auto retcode = (isWorker()) ? EXIT_CATASTROPHIC : EXIT_FAILURE; - requestShutdown(retcode); - } - - // Run the setup for all lazy registries (tables, SQL). Registry::setUp(); - if (FLAGS_database_dump) { - dumpDatabase(); - requestShutdown(); - } - // Initialize the status and result plugin logger. if (!FLAGS_disable_logging) { initActivePlugin("logger", FLAGS_logger_plugin); @@ -485,7 +396,6 @@ void Initializer::waitForShutdown() { // Hopefully release memory used by global string constructors in gflags. GFLAGS_NAMESPACE::ShutDownCommandLineFlags(); - DatabasePlugin::shutdown(); auto excode = (kExitCode != 0) ? kExitCode : EXIT_SUCCESS; exit(excode); diff --git a/src/osquery/core/query.cpp b/src/osquery/core/query.cpp index 72a4431..855f91e 100644 --- a/src/osquery/core/query.cpp +++ b/src/osquery/core/query.cpp @@ -10,7 +10,6 @@ #include #include -#include #include #include #include @@ -28,46 +27,19 @@ FLAG(bool, "Use numeric JSON syntax for numeric values"); uint64_t Query::getPreviousEpoch() const { - uint64_t epoch = 0; - std::string raw; - auto status = getDatabaseValue(kQueries, name_ + "epoch", raw); - if (status.ok()) { - epoch = std::stoull(raw); - } - return epoch; + return 0; } uint64_t Query::getQueryCounter(bool new_query) const { - uint64_t counter = 0; - if (new_query) { - return counter; - } - - std::string raw; - auto status = getDatabaseValue(kQueries, name_ + "counter", raw); - if (status.ok()) { - counter = std::stoull(raw) + 1; - } - return counter; + return 0; } Status Query::getPreviousQueryResults(QueryDataSet& results) const { - std::string raw; - auto status = getDatabaseValue(kQueries, name_, raw); - if (!status.ok()) { - return status; - } - - status = deserializeQueryDataJSON(raw, results); - if (!status.ok()) { - return status; - } return Status::success(); } std::vector Query::getStoredQueryNames() { std::vector results; - scanDatabaseKeys(kQueries, results); return results; } @@ -78,13 +50,10 @@ bool Query::isQueryNameInDatabase() const { static inline void saveQuery(const std::string& name, const std::string& query) { - setDatabaseValue(kQueries, "query." + name, query); } bool Query::isNewQuery() const { - std::string query; - getDatabaseValue(kQueries, "query." + name_, query); - return (query != query_); + return true; } Status Query::addNewResults(QueryDataTyped qd, @@ -140,32 +109,6 @@ Status Query::addNewResults(QueryDataTyped current_qd, target_gd = &dr.added; } - counter = getQueryCounter(fresh_results || new_query); - auto status = - setDatabaseValue(kQueries, name_ + "counter", std::to_string(counter)); - if (!status.ok()) { - return status; - } - - if (update_db) { - // Replace the "previous" query data with the current. - std::string json; - status = serializeQueryDataJSON(*target_gd, json, true); - if (!status.ok()) { - return status; - } - - status = setDatabaseValue(kQueries, name_, json); - if (!status.ok()) { - return status; - } - - status = setDatabaseValue( - kQueries, name_ + "epoch", std::to_string(current_epoch)); - if (!status.ok()) { - return status; - } - } return Status::success(); } diff --git a/src/osquery/core/system.cpp b/src/osquery/core/system.cpp index 1ae042f..f83f764 100644 --- a/src/osquery/core/system.cpp +++ b/src/osquery/core/system.cpp @@ -46,7 +46,6 @@ #include #include -#include #include #include #include @@ -224,16 +223,9 @@ std::string generateHostUUID() { } Status getInstanceUUID(std::string& ident) { - // Lookup the instance identifier (UUID) previously generated and stored. - auto status = - getDatabaseValue(kPersistentSettings, "instance_uuid_v1", ident); - if (ident.size() == 0) { - // There was no UUID stored in the database, generate one and store it. - ident = osquery::generateNewUUID(); - return setDatabaseValue(kPersistentSettings, "instance_uuid_v1", ident); - } + ident = osquery::generateNewUUID(); - return status; + return Status(-1, "Not supported"); } Status getEphemeralUUID(std::string& ident) { @@ -244,14 +236,7 @@ Status getEphemeralUUID(std::string& ident) { } Status getHostUUID(std::string& ident) { - // Lookup the host identifier (UUID) previously generated and stored. - auto status = getDatabaseValue(kPersistentSettings, "host_uuid_v3", ident); - if (ident.size() == 0) { - // There was no UUID stored in the database, generate one and store it. - ident = osquery::generateHostUUID(); - return setDatabaseValue(kPersistentSettings, "host_uuid_v3", ident); - } - return status; + return Status(-1, "Not supported"); } Status getSpecifiedUUID(std::string& ident) { diff --git a/src/osquery/core/tables.cpp b/src/osquery/core/tables.cpp index 0d6bde2..bd0737e 100644 --- a/src/osquery/core/tables.cpp +++ b/src/osquery/core/tables.cpp @@ -8,7 +8,6 @@ #include -#include #include #include #include @@ -226,12 +225,7 @@ bool TablePlugin::isCached(size_t step, const QueryContext& ctx) const { } TableRows TablePlugin::getCache() const { - VLOG(1) << "Retrieving results from cache for table: " << getName(); - // Lookup results from database and deserialize. - std::string content; - getDatabaseValue(kQueries, "cache." + getName(), content); TableRows results; - deserializeTableRowsJSON(content, results); return results; } @@ -239,17 +233,7 @@ void TablePlugin::setCache(size_t step, size_t interval, const QueryContext& ctx, const TableRows& results) { - if (FLAGS_disable_caching || !cacheAllowed(columns(), ctx)) { - return; - } - - // Serialize QueryData and save to database. - std::string content; - if (serializeTableRowsJSON(results, content)) { - last_cached_ = step; - last_interval_ = interval; - setDatabaseValue(kQueries, "cache." + getName(), content); - } + return; } std::string columnDefinition(const TableColumns& columns, bool is_extension) { diff --git a/src/osquery/database/CMakeLists.txt b/src/osquery/database/CMakeLists.txt deleted file mode 100644 index b1daf1c..0000000 --- a/src/osquery/database/CMakeLists.txt +++ /dev/null @@ -1,18 +0,0 @@ -# Copyright (c) 2019 Samsung Electronics Co., Ltd All Rights Reserved -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License - -ADD_OSQUERY_LIBRARY(osquery_database database.cpp) - -FILE(GLOB OSQUERY_DATABASE_TESTS "tests/*.cpp") -ADD_OSQUERY_TEST(${OSQUERY_DATABASE_TESTS}) diff --git a/src/osquery/database/database.cpp b/src/osquery/database/database.cpp deleted file mode 100644 index 59e5646..0000000 --- a/src/osquery/database/database.cpp +++ /dev/null @@ -1,572 +0,0 @@ -/** - * Copyright (c) 2014-present, Facebook, Inc. - * All rights reserved. - * - * This source code is licensed in accordance with the terms specified in - * the LICENSE file found in the root directory of this source tree. - */ - -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include -#include - -namespace pt = boost::property_tree; -namespace rj = rapidjson; - -namespace osquery { - -/// Generate a specific-use registry for database access abstraction. -CREATE_REGISTRY(DatabasePlugin, "database"); - -CLI_FLAG(bool, database_dump, false, "Dump the contents of the backing store"); - -CLI_FLAG(string, - database_path, - OSQUERY_DB_HOME "osquery.db", - "If using a disk-based backing store, specify a path"); -FLAG_ALIAS(std::string, db_path, database_path); - -FLAG(bool, disable_database, false, "Disable the persistent RocksDB storage"); - -const std::string kInternalDatabase = "rocksdb"; -const std::string kPersistentSettings = "configurations"; -const std::string kQueries = "queries"; -const std::string kEvents = "events"; -const std::string kCarves = "carves"; -const std::string kLogs = "logs"; - -const std::string kDbEpochSuffix = "epoch"; -const std::string kDbCounterSuffix = "counter"; - -const std::string kDbVersionKey = "results_version"; - -const std::vector kDomains = { - kPersistentSettings, kQueries, kEvents, kLogs, kCarves}; - -std::atomic DatabasePlugin::kDBAllowOpen(false); -std::atomic DatabasePlugin::kDBRequireWrite(false); -std::atomic DatabasePlugin::kDBInitialized(false); -std::atomic DatabasePlugin::kDBChecking(false); - -/** - * @brief A reader/writer mutex protecting database resets. - * - * A write is locked while using reset flows. A read is locked when calling - * database plugin APIs. - */ -Mutex kDatabaseReset; - -Status DatabasePlugin::initPlugin() { - // Initialize the database plugin using the flag. - auto plugin = (FLAGS_disable_database) ? "ephemeral" : kInternalDatabase; - { - auto const status = RegistryFactory::get().setActive("database", plugin); - if (status.ok()) { - kDBInitialized = true; - return status; - } - LOG(WARNING) << "Failed to activate database plugin " << boost::io::quoted(plugin) << ": " << status.what(); - } - // If the database did not setUp override the active plugin. - auto const status = RegistryFactory::get().setActive("database", "ephemeral"); - if (!status.ok()) { - LOG(ERROR) << "Failed to activate database plugin \"ephemeral\": " << status.what(); - } - kDBInitialized = status.ok(); - return status; -} - -void DatabasePlugin::shutdown() { - auto database_registry = RegistryFactory::get().registry("database"); - for (auto& plugin : RegistryFactory::get().names("database")) { - database_registry->remove(plugin); - } -} - -Status DatabasePlugin::reset() { - // Keep this simple, scope the critical section to the broader methods. - tearDown(); - return setUp(); -} - -bool DatabasePlugin::checkDB() { - kDBChecking = true; - bool result = true; - try { - auto status = setUp(); - if (kDBRequireWrite && read_only_) { - result = false; - } - tearDown(); - result = status.ok(); - } catch (const std::exception& e) { - VLOG(1) << "Database plugin check failed: " << e.what(); - result = false; - } - kDBChecking = false; - return result; -} - -Status DatabasePlugin::scan(const std::string& domain, - std::vector& results, - const std::string& prefix, - size_t max) const { - return Status::success(); -} - -Status DatabasePlugin::call(const PluginRequest& request, - PluginResponse& response) { - if (request.count("action") == 0) { - return Status(1, "Database plugin must include a request action"); - } - - // Get a domain/key, which are used for most database plugin actions. - auto domain = (request.count("domain") > 0) ? request.at("domain") : ""; - auto key = (request.count("key") > 0) ? request.at("key") : ""; - - if (request.at("action") == "reset") { - WriteLock lock(kDatabaseReset); - DatabasePlugin::kDBInitialized = false; - // Prevent RocksDB reentrancy by logger plugins during plugin setup. - VLOG(1) << "Resetting the database plugin: " << getName(); - auto status = this->reset(); - if (!status.ok()) { - // The active database could not be reset, fallback to an ephemeral. - Registry::get().setActive("database", "ephemeral"); - LOG(WARNING) << "Unable to reset database plugin: " << getName(); - } - DatabasePlugin::kDBInitialized = true; - return status; - } - - // Switch over the possible database plugin actions. - ReadLock lock(kDatabaseReset); - if (request.at("action") == "get") { - std::string value; - auto status = this->get(domain, key, value); - response.push_back({{"v", value}}); - return status; - } else if (request.at("action") == "put") { - if (request.count("value") == 0) { - return Status(1, "Database plugin put action requires a value"); - } - return this->put(domain, key, request.at("value")); - } else if (request.at("action") == "putBatch") { - if (request.count("json") == 0) { - return Status( - 1, - "Database plugin putBatch action requires a json-encoded value list"); - } - - auto json_object = JSON::newObject(); - - auto status = json_object.fromString(request.at("json")); - if (!status.ok()) { - VLOG(1) << status.getMessage(); - return status; - } - - const auto& json_object_list = json_object.doc().GetObject(); - - DatabaseStringValueList data; - data.reserve(json_object_list.MemberCount()); - - for (auto& item : json_object_list) { - if (!item.value.IsString()) { - return Status(1, - "Database plugin putBatch action with an invalid json " - "received. Only string values are supported"); - } - - data.push_back( - std::make_pair(item.name.GetString(), item.value.GetString())); - } - - return this->putBatch(domain, data); - } else if (request.at("action") == "remove") { - return this->remove(domain, key); - } else if (request.at("action") == "remove_range") { - auto key_high = - (request.count("key_high") > 0) ? request.at("key_high") : ""; - if (!key_high.empty() && !key.empty()) { - return this->removeRange(domain, key, key_high); - } - return Status(1, "Missing range"); - } else if (request.at("action") == "scan") { - // Accumulate scanned keys into a vector. - std::vector keys; - // Optionally allow the caller to request a max number of keys. - size_t max = 0; - if (request.count("max") > 0) { - max = std::stoul(request.at("max")); - } - auto status = this->scan(domain, keys, request.at("prefix"), max); - for (const auto& k : keys) { - response.push_back({{"k", k}}); - } - return status; - } - - return Status(1, "Unknown database plugin action"); -} - -static inline std::shared_ptr getDatabasePlugin() { - auto& rf = RegistryFactory::get(); - if (!rf.exists("database", rf.getActive("database"), true)) { - return nullptr; - } - - auto plugin = rf.plugin("database", rf.getActive("database")); - return std::dynamic_pointer_cast(plugin); -} - -namespace { -Status sendPutBatchDatabaseRequest(const std::string& domain, - const DatabaseStringValueList& data) { - auto json_object = JSON::newObject(); - for (const auto& p : data) { - const auto& key = p.first; - const auto& value = p.second; - - json_object.addRef(key, value); - } - - std::string serialized_data; - auto status = json_object.toString(serialized_data); - if (!status.ok()) { - VLOG(1) << status.getMessage(); - return status; - } - - PluginRequest request = {{"action", "putBatch"}, - {"domain", domain}, - {"json", std::move(serialized_data)}}; - - status = Registry::call("database", request); - if (!status.ok()) { - VLOG(1) << status.getMessage(); - } - - return status; -} - -Status sendPutDatabaseRequest(const std::string& domain, - const DatabaseStringValueList& data) { - const auto& key = data[0].first; - const auto& value = data[0].second; - - PluginRequest request = { - {"action", "put"}, {"domain", domain}, {"key", key}, {"value", value}}; - - auto status = Registry::call("database", request); - if (!status.ok()) { - VLOG(1) << status.getMessage(); - } - - return status; -} -} // namespace - -Status getDatabaseValue(const std::string& domain, - const std::string& key, - std::string& value) { - if (domain.empty()) { - return Status(1, "Missing domain"); - } - - ReadLock lock(kDatabaseReset); - if (!DatabasePlugin::kDBInitialized) { - throw std::runtime_error("Cannot get database value: " + key); - } else { - auto plugin = getDatabasePlugin(); - return plugin->get(domain, key, value); - } -} - -Status getDatabaseValue(const std::string& domain, - const std::string& key, - int& value) { - std::string result; - auto s = getDatabaseValue(domain, key, result); - if (s.ok()) { - value = std::stoi(result); - } - return s; -} - -Status setDatabaseValue(const std::string& domain, - const std::string& key, - const std::string& value) { - return setDatabaseBatch(domain, {std::make_pair(key, value)}); -} - -Status setDatabaseBatch(const std::string& domain, - const DatabaseStringValueList& data) { - if (domain.empty()) { - return Status(1, "Missing domain"); - } - - ReadLock lock(kDatabaseReset); - if (!DatabasePlugin::kDBInitialized) { - throw std::runtime_error("Cannot set database values"); - } - - auto plugin = getDatabasePlugin(); - return plugin->putBatch(domain, data); -} - -Status setDatabaseValue(const std::string& domain, - const std::string& key, - int value) { - return setDatabaseBatch(domain, {std::make_pair(key, std::to_string(value))}); -} - -Status deleteDatabaseValue(const std::string& domain, const std::string& key) { - if (domain.empty()) { - return Status(1, "Missing domain"); - } - - ReadLock lock(kDatabaseReset); - if (!DatabasePlugin::kDBInitialized) { - throw std::runtime_error("Cannot delete database value: " + key); - } else { - auto plugin = getDatabasePlugin(); - return plugin->remove(domain, key); - } -} - -Status deleteDatabaseRange(const std::string& domain, - const std::string& low, - const std::string& high) { - if (domain.empty()) { - return Status(1, "Missing domain"); - } - - ReadLock lock(kDatabaseReset); - if (!DatabasePlugin::kDBInitialized) { - throw std::runtime_error("Cannot delete database values: " + low + " - " + - high); - } else { - auto plugin = getDatabasePlugin(); - return plugin->removeRange(domain, low, high); - } -} - -Status scanDatabaseKeys(const std::string& domain, - std::vector& keys, - size_t max) { - return scanDatabaseKeys(domain, keys, "", max); -} - -/// Get a list of keys for a given domain. -Status scanDatabaseKeys(const std::string& domain, - std::vector& keys, - const std::string& prefix, - size_t max) { - if (domain.empty()) { - return Status(1, "Missing domain"); - } - - ReadLock lock(kDatabaseReset); - if (!DatabasePlugin::kDBInitialized) { - throw std::runtime_error("Cannot scan database values: " + prefix); - } else { - auto plugin = getDatabasePlugin(); - return plugin->scan(domain, keys, prefix, max); - } -} - -void resetDatabase() { - PluginRequest request = {{"action", "reset"}}; - Registry::call("database", request); -} - -void dumpDatabase() { - auto plugin = getDatabasePlugin(); - plugin->dumpDatabase(); -} - -Status ptreeToRapidJSON(const std::string& in, std::string& out) { - pt::ptree tree; - try { - std::stringstream ss; - ss << in; - pt::read_json(ss, tree); - } catch (const pt::json_parser::json_parser_error& /* e */) { - return Status(1, "Failed to parse JSON"); - } - - auto json = JSON::newArray(); - for (const auto& t : tree) { - std::stringstream ss; - pt::write_json(ss, t.second); - - rj::Document row; - if (row.Parse(ss.str().c_str()).HasParseError()) { - return Status(1, "Failed to serialize JSON"); - } - json.push(row); - } - - json.toString(out); - - return Status::success(); -} - -static Status migrateV0V1(void) { - std::vector keys; - auto s = scanDatabaseKeys(kQueries, keys); - if (!s.ok()) { - return Status(1, "Failed to lookup legacy query data from database"); - } - - for (const auto& key : keys) { - // Skip over epoch and counter entries, as 0 is parsed by ptree - if (boost::algorithm::ends_with(key, kDbEpochSuffix) || - boost::algorithm::ends_with(key, kDbCounterSuffix) || - boost::algorithm::starts_with(key, "query.")) { - continue; - } - - std::string value{""}; - if (!getDatabaseValue(kQueries, key, value)) { - LOG(WARNING) << "Failed to get value from database " << key; - continue; - } - - std::string out; - s = ptreeToRapidJSON(value, out); - if (!s.ok()) { - LOG(WARNING) << "Conversion from ptree to RapidJSON failed for '" << key - << ": " << value << "': " << s.what() << ". Dropping key!"; - continue; - } - - if (!setDatabaseValue(kQueries, key, out)) { - LOG(WARNING) << "Failed to update value in database " << key << ": " - << value; - } - } - - return Status::success(); -} - -static Status migrateV1V2(void) { - std::vector keys; - const std::string audit_str(".audit."); - - Status s = scanDatabaseKeys(kEvents, keys); - if (!s.ok()) { - return Status::failure( - 1, "Failed to scan event keys from database: " + s.what()); - } - - for (const auto& key : keys) { - const auto pos = key.find(audit_str); - if (pos != std::string::npos) { - std::string value; - std::string new_key = key; - new_key.replace(pos, audit_str.length(), ".auditeventpublisher."); - - s = getDatabaseValue(kEvents, key, value); - if (!s.ok()) { - LOG(ERROR) << "Failed to read value for key '" << key - << "'. Key will be kept but won't be migrated!"; - continue; - } - - s = setDatabaseValue(kEvents, new_key, value); - if (!s.ok()) { - LOG(ERROR) << "Failed to set value for key '" << new_key - << "' migrated from '" << key - << "'. Original key will be kept but won't be migrated!"; - continue; - } - - s = deleteDatabaseValue(kEvents, key); - if (!s.ok()) { - LOG(WARNING) << "Failed to delete key '" << key - << "' after migration to new key '" << new_key - << "'. Original key will be kept but data was migrated!"; - } - } - } - - return Status::success(); -} - -Status upgradeDatabase(int to_version) { - LOG(INFO) << "Checking database version for migration"; - - std::string value; - Status st = getDatabaseValue(kPersistentSettings, kDbVersionKey, value); - - int db_version = 0; - /* Since there isn't a reliable way to determined what happen when the read - * fails we just assume the key doesn't exist which indicates database - * version 0. - */ - if (st.ok()) { - auto ret = tryTo(value); - if (ret.isError()) { - LOG(ERROR) << "Invalid value '" << value << "'for " << kDbVersionKey - << " key. Database is corrupted."; - return Status(1, "Invalid value for database version."); - } else { - db_version = ret.get(); - } - } - - while (db_version != to_version) { - Status migrate_status; - - LOG(INFO) << "Performing migration: " << db_version << " -> " - << (db_version + 1); - - switch (db_version) { - case 0: - migrate_status = migrateV0V1(); - break; - - case 1: - migrate_status = migrateV1V2(); - break; - - default: - LOG(ERROR) << "Logic error: the migration code is broken!"; - migrate_status = Status(1); - break; - } - - if (!migrate_status.ok()) { - return Status(1, "Database migration failed."); - } - - st = setDatabaseValue( - kPersistentSettings, kDbVersionKey, std::to_string(db_version + 1)); - if (!st.ok()) { - LOG(ERROR) << "Failed to set new database version after migration. " - << "The DB was correctly migrated from version " << db_version - << " to version " << (db_version + 1) - << " but persisting the new version failed."; - return Status(1, "Database migration failed."); - } - - LOG(INFO) << "Migration " << db_version << " -> " << (db_version + 1) - << " successfully completed!"; - - db_version++; - } - - return Status::success(); -} -} // namespace osquery diff --git a/src/osquery/database/tests/database.cpp b/src/osquery/database/tests/database.cpp deleted file mode 100644 index 3c3409e..0000000 --- a/src/osquery/database/tests/database.cpp +++ /dev/null @@ -1,302 +0,0 @@ -/** - * Copyright (c) 2014-present, Facebook, Inc. - * All rights reserved. - * - * This source code is licensed in accordance with the terms specified in - * the LICENSE file found in the root directory of this source tree. - */ - -#include -#include -#include -#include - -#include - -#include - -#include - -namespace rj = rapidjson; - -namespace osquery { - -DECLARE_bool(disable_database); - -class DatabaseTests : public testing::Test { - public: - void SetUp() override { - Initializer::platformSetup(); - registryAndPluginInit(); - - // Force registry to use ephemeral database plugin - FLAGS_disable_database = true; - DatabasePlugin::setAllowOpen(true); - DatabasePlugin::initPlugin(); - } -}; - -TEST_F(DatabaseTests, test_set_value_str) { - auto s = setDatabaseValue(kLogs, "str", "{}"); - EXPECT_TRUE(s.ok()); -} - -TEST_F(DatabaseTests, test_set_value_int) { - auto s = setDatabaseValue(kLogs, "int", -1); - EXPECT_TRUE(s.ok()); -} - -TEST_F(DatabaseTests, test_set_str_batch) { - DatabaseStringValueList batch = { - {"str1", "{a}"}, {"str2", "{b}"}, {"str3", "{c}"}}; - - auto s = setDatabaseBatch(kLogs, batch); - EXPECT_TRUE(s.ok()); -} - -TEST_F(DatabaseTests, test_set_value_mix1) { - auto s = setDatabaseValue(kLogs, "intstr", -1); - EXPECT_TRUE(s.ok()); - - s = setDatabaseValue(kLogs, "intstr", "{}"); - EXPECT_TRUE(s.ok()); -} - -TEST_F(DatabaseTests, test_set_value_mix2) { - auto s = setDatabaseValue(kLogs, "strint", "{}"); - EXPECT_TRUE(s.ok()); - - s = setDatabaseValue(kLogs, "strint", -1); - EXPECT_TRUE(s.ok()); -} - -TEST_F(DatabaseTests, test_get_value_does_not_exist) { - // Unknown keys return failed, but will return empty data. - std::string value; - auto s = getDatabaseValue(kLogs, "does_not_exist", value); - EXPECT_FALSE(s.ok()); - EXPECT_TRUE(value.empty()); -} - -TEST_F(DatabaseTests, test_get_value_str) { - std::string expected; - for (unsigned char i = std::numeric_limits::min(); - i < std::numeric_limits::max(); - i++) { - if (std::isprint(i)) { - expected += i; - } - } - - setDatabaseValue(kLogs, "str", expected); - - std::string value; - auto s = getDatabaseValue(kLogs, "str", value); - - EXPECT_TRUE(s.ok()); - EXPECT_EQ(value, expected); -} - -TEST_F(DatabaseTests, test_get_value_int) { - int expected = std::numeric_limits::min(); - setDatabaseValue(kLogs, "int", expected); - - int value = 0; - auto s = getDatabaseValue(kLogs, "int", value); - - EXPECT_TRUE(s.ok()); - EXPECT_EQ(value, expected); -} - -TEST_F(DatabaseTests, test_get_value_mix1) { - int expected = std::numeric_limits::max(); - setDatabaseValue(kLogs, "strint", "{}"); - setDatabaseValue(kLogs, "strint", expected); - - int value; - auto s = getDatabaseValue(kLogs, "strint", value); - - EXPECT_TRUE(s.ok()); - EXPECT_EQ(value, expected); -} - -TEST_F(DatabaseTests, test_get_str_batch) { - DatabaseStringValueList batch = { - {"str1", "{a}"}, {"str2", "{b}"}, {"str3", "{c}"}}; - auto s = setDatabaseBatch(kLogs, batch); - EXPECT_TRUE(s.ok()); - - for (const auto& p : batch) { - const auto& key = p.first; - const auto& expected_value = p.second; - - std::string value; - s = getDatabaseValue(kLogs, key, value); - EXPECT_TRUE(s.ok()); - - EXPECT_EQ(value, expected_value); - } -} - -TEST_F(DatabaseTests, test_get_value_mix2) { - std::string expected = "{}"; - setDatabaseValue(kLogs, "intstr", -1); - setDatabaseValue(kLogs, "intstr", expected); - - std::string value; - auto s = getDatabaseValue(kLogs, "intstr", value); - - EXPECT_TRUE(s.ok()); - EXPECT_EQ(value, expected); -} - -TEST_F(DatabaseTests, test_scan_values) { - setDatabaseValue(kLogs, "1", "0"); - setDatabaseValue(kLogs, "2", 0); - setDatabaseValue(kLogs, "3", "0"); - - std::vector keys; - auto s = scanDatabaseKeys(kLogs, keys); - EXPECT_TRUE(s.ok()); - EXPECT_GT(keys.size(), 2U); - - keys.clear(); - s = scanDatabaseKeys(kLogs, keys, 3); - EXPECT_TRUE(s.ok()); - EXPECT_EQ(keys.size(), 3U); -} - -TEST_F(DatabaseTests, test_delete_values_str) { - setDatabaseValue(kLogs, "k", "0"); - - std::string value; - getDatabaseValue(kLogs, "k", value); - EXPECT_FALSE(value.empty()); - - auto s = deleteDatabaseValue(kLogs, "k"); - EXPECT_TRUE(s.ok()); - - // Make sure the key has been deleted. - value.clear(); - s = getDatabaseValue(kLogs, "k", value); - EXPECT_FALSE(s.ok()); - EXPECT_TRUE(value.empty()); -} - -TEST_F(DatabaseTests, test_delete_values_int) { - int expected = 0; - setDatabaseValue(kLogs, "k", expected); - - int value; - getDatabaseValue(kLogs, "k", value); - EXPECT_EQ(value, expected); - - auto s = deleteDatabaseValue(kLogs, "k"); - EXPECT_TRUE(s.ok()); - - // Make sure the key has been deleted. - value = -5; - s = getDatabaseValue(kLogs, "k", value); - EXPECT_FALSE(s.ok()); - EXPECT_EQ(value, -5); -} - -TEST_F(DatabaseTests, test_ptree_upgrade_to_rj_empty_v0v1) { - auto empty_results{"{}"}; - auto status = setDatabaseValue(kQueries, "old_empty_results", empty_results); - EXPECT_TRUE(status.ok()); - - // Stage our database to be pre-upgrade to ensure the logic runs - status = setDatabaseValue(kPersistentSettings, kDbVersionKey, "0"); - EXPECT_TRUE(status.ok()); - - status = upgradeDatabase(1); - EXPECT_TRUE(status.ok()); - - std::string new_empty_list; - status = getDatabaseValue(kQueries, "old_empty_results", new_empty_list); - EXPECT_TRUE(status.ok()); - - rj::Document empty_list; - EXPECT_FALSE(empty_list.Parse(new_empty_list.c_str()).HasParseError()); - EXPECT_TRUE(empty_list.IsArray()); - - // Expect our DB upgrade logic to have been set - std::string db_results_version{""}; - getDatabaseValue(kPersistentSettings, kDbVersionKey, db_results_version); - EXPECT_EQ(db_results_version, "1"); -} - -TEST_F(DatabaseTests, test_ptree_upgrade_to_rj_results_v0v1) { - auto bad_json = - "{\"\":{\"disabled\":\"0\",\"network_name\":\"BTWifi-Starbucks\"},\"\":{" - "\"disabled\":\"0\",\"network_name\":\"Lobo-Guest\"},\"\":{\"disabled\":" - "\"0\",\"network_name\":\"GoogleGuest\"}}"; - auto status = setDatabaseValue(kQueries, "bad_wifi_json", bad_json); - EXPECT_TRUE(status.ok()); - - // Add an integer value to ensure we don't munge non-json objects - status = setDatabaseValue(kQueries, "bad_wifi_jsonepoch", "1521583712"); - EXPECT_TRUE(status.ok()); - - // Stage our database to be pre-upgrade to ensure the logic runs - status = setDatabaseValue(kPersistentSettings, kDbVersionKey, "0"); - EXPECT_TRUE(status.ok()); - - rj::Document bad_doc; - - // Potential bug with RJ, in that parsing should fail with empty keys - // EXPECT_TRUE(bad_doc.Parse(bad_json).HasParseError()); - EXPECT_FALSE(bad_doc.IsArray()); - - status = upgradeDatabase(1); - EXPECT_TRUE(status.ok()); - - std::string good_json; - status = getDatabaseValue(kQueries, "bad_wifi_json", good_json); - EXPECT_TRUE(status.ok()); - - rj::Document clean_doc; - EXPECT_FALSE(clean_doc.Parse(good_json.c_str()).HasParseError()); - EXPECT_TRUE(clean_doc.IsArray()); - EXPECT_EQ(clean_doc.Size(), 3U); - - // Ensure our non-json thing was not destroyed - std::string query_epoch{""}; - status = getDatabaseValue(kQueries, "bad_wifi_jsonepoch", query_epoch); - auto ulepoch = std::stoull(query_epoch); - EXPECT_EQ(ulepoch, 1521583712U); - - // Expect our DB upgrade logic to have been set - std::string db_results_version{""}; - getDatabaseValue(kPersistentSettings, "results_version", db_results_version); - EXPECT_EQ(db_results_version, "1"); -} - -TEST_F(DatabaseTests, test_migration_v1v2) { - /* Testing migration from 1 to 2 */ - Status status = setDatabaseValue(kPersistentSettings, kDbVersionKey, "1"); - ASSERT_TRUE(status.ok()); - - status = setDatabaseValue( - kEvents, "data.audit.process_events.0123456789", "event_data"); - ASSERT_TRUE(status.ok()); - - status = upgradeDatabase(2); - ASSERT_TRUE(status.ok()); - - std::string value; - status = getDatabaseValue(kPersistentSettings, kDbVersionKey, value); - EXPECT_EQ(value, "2"); - - status = - getDatabaseValue(kEvents, "data.audit.process_events.0123456789", value); - EXPECT_FALSE(status.ok()); - - status = getDatabaseValue( - kEvents, "data.auditeventpublisher.process_events.0123456789", value); - EXPECT_TRUE(status.ok()); - EXPECT_EQ(value, "event_data"); -} - -} // namespace osquery diff --git a/src/osquery/database/tests/results.cpp b/src/osquery/database/tests/results.cpp deleted file mode 100644 index 62d99f7..0000000 --- a/src/osquery/database/tests/results.cpp +++ /dev/null @@ -1,154 +0,0 @@ -/** - * Copyright (c) 2014-present, Facebook, Inc. - * All rights reserved. - * - * This source code is licensed in accordance with the terms specified in - * the LICENSE file found in the root directory of this source tree. - */ - -#include - -#include -#include -#include -#include - -#include - -#include - -namespace osquery { - -class ResultsTests : public testing::Test {}; - -TEST_F(ResultsTests, test_simple_diff) { - QueryDataSet os; - QueryDataTyped o; - QueryDataTyped n; - - RowTyped r1; - r1["foo"] = "bar"; - n.push_back(r1); - - auto results = diff(os, n); - EXPECT_EQ(results.added, n); - EXPECT_EQ(results.removed, o); -} - -TEST_F(ResultsTests, test_serialize_row) { - auto results = getSerializedRow(); - auto doc = JSON::newObject(); - auto s = serializeRow(results.second, doc, doc.doc(), true); - EXPECT_TRUE(s.ok()); - EXPECT_EQ(s.toString(), "OK"); - EXPECT_EQ(doc.doc()["meaning_of_life"], "meaning_of_life_value"); - EXPECT_EQ(doc.doc()["alphabetical"], "alphabetical_value"); -} - -TEST_F(ResultsTests, test_deserialize_row_json) { - auto results = getSerializedRow(); - std::string input; - serializeRowJSON(results.second, input, true); - - // Pull the serialized JSON back into a Row output container. - RowTyped output; - auto s = deserializeRowJSON(input, output); - EXPECT_TRUE(s.ok()); - // The output container should match the input row. - EXPECT_EQ(output, results.second); -} - -TEST_F(ResultsTests, test_serialize_query_data) { - auto results = getSerializedQueryData(); - auto doc = JSON::newArray(); - auto s = serializeQueryData(results.second, doc, doc.doc(), true); - EXPECT_TRUE(s.ok()); - EXPECT_EQ(s.toString(), "OK"); - EXPECT_EQ(results.first.doc(), doc.doc()); -} - -TEST_F(ResultsTests, test_serialize_query_data_json) { - auto results = getSerializedQueryDataJSON(); - std::string json; - auto s = serializeQueryDataJSON(results.second, json, true); - EXPECT_TRUE(s.ok()); - EXPECT_EQ(s.toString(), "OK"); - EXPECT_EQ(results.first, json); -} - -TEST_F(ResultsTests, test_deserialize_query_data_json) { - auto results = getSerializedQueryDataJSON(); - QueryDataSet resultSet = - QueryDataSet(results.second.begin(), results.second.end()); - - // Pull the serialized JSON back into a QueryData output container. - QueryDataSet output; - auto s = deserializeQueryDataJSON(results.first, output); - EXPECT_TRUE(s.ok()); - // The output container should match the input query data. - EXPECT_EQ(output, resultSet); -} - -TEST_F(ResultsTests, test_serialize_diff_results) { - auto results = getSerializedDiffResults(); - auto doc = JSON::newObject(); - auto s = serializeDiffResults(results.second, doc, doc.doc(), true); - EXPECT_TRUE(s.ok()); - EXPECT_EQ(s.toString(), "OK"); - EXPECT_EQ(results.first.doc(), doc.doc()); -} - -TEST_F(ResultsTests, test_serialize_diff_results_json) { - auto results = getSerializedDiffResultsJSON(); - std::string json; - auto s = serializeDiffResultsJSON(results.second, json, true); - EXPECT_TRUE(s.ok()); - EXPECT_EQ(s.toString(), "OK"); - EXPECT_EQ(results.first, json); -} - -TEST_F(ResultsTests, test_serialize_query_log_item) { - auto results = getSerializedQueryLogItem(); - auto doc = JSON::newObject(); - auto s = serializeQueryLogItem(results.second, doc); - EXPECT_TRUE(s.ok()); - EXPECT_EQ(s.toString(), "OK"); - EXPECT_EQ(results.first.doc(), doc.doc()); -} - -TEST_F(ResultsTests, test_serialize_query_log_item_json) { - auto results = getSerializedQueryLogItemJSON(); - std::string json; - auto s = serializeQueryLogItemJSON(results.second, json); - EXPECT_TRUE(s.ok()); - EXPECT_EQ(s.toString(), "OK"); - EXPECT_EQ(results.first, json); -} - -TEST_F(ResultsTests, test_adding_duplicate_rows_to_query_data) { - RowTyped r1, r2, r3; - r1["foo"] = "bar"; - r1["baz"] = "boo"; - - r2["foo"] = "baz"; - r2["baz"] = "bop"; - - r3["foo"] = "baz"; - r3["baz"] = "bop"; - - QueryDataTyped q; - bool s; - - s = addUniqueRowToQueryData(q, r1); - EXPECT_TRUE(s); - EXPECT_EQ(q.size(), 1U); - - s = addUniqueRowToQueryData(q, r2); - EXPECT_TRUE(s); - EXPECT_EQ(q.size(), 2U); - - s = addUniqueRowToQueryData(q, r3); - EXPECT_FALSE(s); - EXPECT_EQ(q.size(), 2U); -} -} diff --git a/src/osquery/devtools/shell.cpp b/src/osquery/devtools/shell.cpp index 0b56208..7013459 100644 --- a/src/osquery/devtools/shell.cpp +++ b/src/osquery/devtools/shell.cpp @@ -33,7 +33,6 @@ #include -#include #include #include #include @@ -71,7 +70,6 @@ SHELL_FLAG(string, A, "", "Select all from a table"); DECLARE_string(nullvalue); DECLARE_string(logger_plugin); DECLARE_string(logger_path); -DECLARE_string(database_path); } // namespace osquery static char zHelp[] = @@ -1152,14 +1150,6 @@ inline void meta_show(struct callback_data* p) { fprintf(p->out, "\n"); } - auto database = osquery::RegistryFactory::get().getActive("database"); - fprintf(p->out, "%13.13s: %s", "Database", database.c_str()); - if (database == "rocksdb") { - fprintf(p->out, " (%s)\n", osquery::FLAGS_database_path.c_str()); - } else { - fprintf(p->out, "\n"); - } - fprintf(p->out, "\nShell settings:\n"); fprintf(p->out, "%13.13s: %s\n", "echo", p->echoOn != 0 ? "on" : "off"); fprintf( diff --git a/src/osquery/include/osquery/database.h b/src/osquery/include/osquery/database.h deleted file mode 100644 index ebab6eb..0000000 --- a/src/osquery/include/osquery/database.h +++ /dev/null @@ -1,295 +0,0 @@ -/** - * Copyright (c) 2014-present, Facebook, Inc. - * All rights reserved. - * - * This source code is licensed in accordance with the terms specified in - * the LICENSE file found in the root directory of this source tree. - */ - -#pragma once - -#include -#include -#include - -#include - -namespace osquery { -// A list of key/str pairs; used for write batching with setDatabaseBatch -using DatabaseStringValueList = - std::vector>; - -class Status; -/** - * @brief A list of supported backing storage categories: called domains. - * - * RocksDB has a concept of "column families" which are kind of like tables - * in other databases. kDomains is populated with a list of all column - * families. If a string exists in kDomains, it's a column family in the - * database. - * - * For SQLite-backed storage these are tables using a keyed index. - */ -extern const std::vector kDomains; - -/** - * @brief A backing storage domain name, used for key/value based storage. - * - * There are certain "cached" variables such as a node-unique UUID or negotiated - * 'node_key' following enrollment. If a value or setting must persist between - * osqueryi or osqueryd runs it should be stored using the kPersistentSetting%s - * domain. - */ -extern const std::string kPersistentSettings; - -/// The "domain" where the results of scheduled queries are stored. -extern const std::string kQueries; - -/// The "domain" where event results are stored, queued for querytime retrieval. -extern const std::string kEvents; - -/// The "domain" where the results of carve queries are stored. -extern const std::string kCarves; - -/// The key for the DB version -extern const std::string kDbVersionKey; - -/// The running version of our database schema -const int kDbCurrentVersion = 2; - -/** - * @brief The "domain" where buffered log results are stored. - * - * Logger plugins may shuttle logs to a remote endpoint or API call - * asynchronously. The backing store can be used to buffer results and status - * logs until the logger plugin-specific thread decided to flush. - */ -extern const std::string kLogs; - -/** - * @brief An osquery backing storage (database) type that persists executions. - * - * The osquery tools need a high-performance storage and indexing mechanism for - * storing intermediate results from EventPublisher%s, persisting one-time - * generated values, and performing non-memory backed differentials. - * - * Practically, osquery is built around RocksDB's performance guarantees and - * all of the internal APIs expect RocksDB's indexing and read performance. - * However, access to this representation of a backing-store is still abstracted - * to removing RocksDB as a dependency for the osquery SDK. - */ -class DatabasePlugin : public Plugin { - public: - /** - * @brief Perform a domain and key lookup from the backing store. - * - * Database value access indexing is abstracted into domains and keys. - * Both are string values but exist separately for simple indexing without - * API-enforcing tokenization. In some cases we do add a component-specific - * tokeninzation to keys. - * - * @param domain A string value representing abstract storage indexing. - * @param key A string value representing the lookup/retrieval key. - * @param value The output parameter, left empty if the key does not exist. - * @return Failure if the data could not be accessed. It is up to the plugin - * to determine if a missing key means a non-success status. - */ - virtual Status get(const std::string& domain, - const std::string& key, - std::string& value) const = 0; - - virtual Status get(const std::string& domain, - const std::string& key, - int& value) const = 0; - - /** - * @brief Store a string-represented value using a domain and key index. - * - * See DatabasePlugin::get for discussion around domain and key use. - * - * @param domain A string value representing abstract storage indexing. - * @param key A string value representing the lookup/retrieval key. - * @param value A string value representing the data. - * @return Failure if the data could not be stored. It is up to the plugin - * to determine if a conflict/overwrite should return different status text. - */ - virtual Status put(const std::string& domain, - const std::string& key, - const std::string& value) = 0; - - virtual Status put(const std::string& domain, - const std::string& key, - int value) = 0; - - virtual Status putBatch(const std::string& domain, - const DatabaseStringValueList& data) = 0; - - virtual void dumpDatabase() const = 0; - - /// Data removal method. - virtual Status remove(const std::string& domain, const std::string& k) = 0; - - /// Data removal with range bounds. - virtual Status removeRange(const std::string& domain, - const std::string& low, - const std::string& high) = 0; - - virtual Status scan(const std::string& domain, - std::vector& results, - const std::string& prefix, - size_t max) const; - - /** - * @brief Shutdown the database and release initialization resources. - * - * Assume that a plugin may override #tearDown and choose to close resources - * when the registry is stopping. Most plugins will implement a mutex around - * initialization and destruction and assume #setUp and #tearDown will - * dictate the flow in most situations. - */ - ~DatabasePlugin() override = default; - - /** - * @brief Support the registry calling API for extensions. - * - * The database plugin "fast-calls" directly to local plugins. - * Extensions cannot use an extension-local backing store so their requests - * are routed like all other plugins. - */ - Status call(const PluginRequest& request, PluginResponse& response) override; - - public: - /// Database-specific workflow: reset the originally request instance. - Status reset(); - - /// Database-specific workflow: perform an initialize, then reset. - bool checkDB(); - - /// Require all DBHandle accesses to open a read and write handle. - static void setRequireWrite(bool rw) { - kDBRequireWrite = rw; - } - - /// Allow DBHandle creations. - static void setAllowOpen(bool ao) { - kDBAllowOpen = ao; - } - - public: - /// Control availability of the RocksDB handle (default false). - static std::atomic kDBAllowOpen; - - /// The database must be opened in a R/W mode (default false). - static std::atomic kDBRequireWrite; - - /// An internal mutex around database sanity checking. - static std::atomic kDBChecking; - - /// An internal status protecting database access. - static std::atomic kDBInitialized; - - public: - /** - * @brief Allow the initializer to check the active database plugin. - * - * Unlink the initializer's Initializer::initActivePlugin helper method, the - * database plugin should always be within the core. There is no need to - * discover the active plugin via the registry or extensions API. - * - * The database should setUp in preparation for accesses. - */ - static Status initPlugin(); - - /// Allow shutdown before exit. - static void shutdown(); - - protected: - /// The database was opened in a ReadOnly mode. - bool read_only_{false}; - - /// Original requested path on disk. - std::string path_; -}; - -/** - * @brief Lookup a value from the active osquery DatabasePlugin storage. - * - * See DatabasePlugin::get for discussion around domain and key use. - * Extensions, components, plugins, and core code should use getDatabaseValue - * as a wrapper around the current tool's choice of a backing storage plugin. - * - * @param domain A string value representing abstract storage indexing. - * @param key A string value representing the lookup/retrieval key. - * @param value The output parameter, left empty if the key does not exist. - * @return Storage operation status. - */ -Status getDatabaseValue(const std::string& domain, - const std::string& key, - std::string& value); - -Status getDatabaseValue(const std::string& domain, - const std::string& key, - int& value); - -/** - * @brief Set or put a value into the active osquery DatabasePlugin storage. - * - * See DatabasePlugin::get for discussion around domain and key use. - * Extensions, components, plugins, and core code should use setDatabaseValue - * as a wrapper around the current tool's choice of a backing storage plugin. - * - * @param domain A string value representing abstract storage indexing. - * @param key A string value representing the lookup/retrieval key. - * @param value A string value representing the data. - * @return Storage operation status. - */ -Status setDatabaseValue(const std::string& domain, - const std::string& key, - const std::string& value); - -Status setDatabaseValue(const std::string& domain, - const std::string& key, - int value); - -Status setDatabaseBatch(const std::string& domain, - const DatabaseStringValueList& data); - -/// Remove a domain/key identified value from backing-store. -Status deleteDatabaseValue(const std::string& domain, const std::string& key); - -/// Remove a range of keys in domain. -Status deleteDatabaseRange(const std::string& domain, - const std::string& low, - const std::string& high); - -/// Get a list of keys for a given domain. -Status scanDatabaseKeys(const std::string& domain, - std::vector& keys, - size_t max = 0); - -/// Get a list of keys for a given domain. -Status scanDatabaseKeys(const std::string& domain, - std::vector& keys, - const std::string& prefix, - size_t max = 0); - -/// Allow callers to reload or reset the database plugin. -void resetDatabase(); - -/// Allow callers to scan each column family and print each value. -void dumpDatabase(); - -Status ptreeToRapidJSON(const std::string& in, std::string& out); - -/** - * @brief Upgrades the legacy database json format from ptree to RapidJSON - * - * This helper function was required as Boost property trees contain json - * which leverages empty strings for keys in json arrays. This is incompatible - * with rapidjson, thus we require a converter function to upgrade any cached - * results in the database. - * - * @return Success status of upgrading the database - */ -Status upgradeDatabase(int to_version = kDbCurrentVersion); -} // namespace osquery diff --git a/src/osquery/include/osquery/database/plugins/ephemeral.h b/src/osquery/include/osquery/database/plugins/ephemeral.h deleted file mode 120000 index 6486abb..0000000 --- a/src/osquery/include/osquery/database/plugins/ephemeral.h +++ /dev/null @@ -1 +0,0 @@ -../../../../plugins/database/ephemeral.h \ No newline at end of file diff --git a/src/osquery/include/osquery/database/plugins/rocksdb.h b/src/osquery/include/osquery/database/plugins/rocksdb.h deleted file mode 120000 index f46583c..0000000 --- a/src/osquery/include/osquery/database/plugins/rocksdb.h +++ /dev/null @@ -1 +0,0 @@ -../../../../plugins/database/rocksdb.h \ No newline at end of file diff --git a/src/osquery/include/osquery/database/plugins/sqlite.h b/src/osquery/include/osquery/database/plugins/sqlite.h deleted file mode 120000 index fff0be6..0000000 --- a/src/osquery/include/osquery/database/plugins/sqlite.h +++ /dev/null @@ -1 +0,0 @@ -../../../../plugins/database/sqlite.h \ No newline at end of file diff --git a/src/osquery/logger/logger.cpp b/src/osquery/logger/logger.cpp index c41e3c9..0cb1c80 100644 --- a/src/osquery/logger/logger.cpp +++ b/src/osquery/logger/logger.cpp @@ -18,7 +18,6 @@ #include #include -#include #include #include #include @@ -450,7 +449,7 @@ size_t queuedSenders() { } void relayStatusLogs(bool async) { - if (FLAGS_disable_logging || !DatabasePlugin::kDBInitialized) { + if (FLAGS_disable_logging) { // The logger plugins may not be setUp if logging is disabled. // If the database is not setUp, or is in a reset, status logs continue // to buffer. diff --git a/src/osquery/logger/tests/logger.cpp b/src/osquery/logger/tests/logger.cpp index 83b2c54..9ff5a37 100644 --- a/src/osquery/logger/tests/logger.cpp +++ b/src/osquery/logger/tests/logger.cpp @@ -11,7 +11,6 @@ #include #include -#include #include #include #include @@ -21,7 +20,6 @@ namespace osquery { -DECLARE_bool(disable_database); DECLARE_int32(logger_min_status); DECLARE_int32(logger_min_stderr); DECLARE_bool(logger_secondary_status_only); @@ -34,9 +32,6 @@ class LoggerTests : public testing::Test { void SetUp() override { Initializer::platformSetup(); registryAndPluginInit(); - FLAGS_disable_database = true; - DatabasePlugin::setAllowOpen(true); - DatabasePlugin::initPlugin(); // Backup the logging status, then disable. FLAGS_disable_logging = false; diff --git a/src/osquery/main/main.cpp b/src/osquery/main/main.cpp index 1b954ba..4d891c6 100644 --- a/src/osquery/main/main.cpp +++ b/src/osquery/main/main.cpp @@ -15,7 +15,6 @@ #include #include -#include #include #include #include @@ -56,10 +55,6 @@ int profile(int argc, char* argv[]) { query = std::string(argv[1]); } - // Perform some duplication from Initializer with respect to database setup. - osquery::DatabasePlugin::setAllowOpen(true); - osquery::RegistryFactory::get().setActive("database", "ephemeral"); - auto dbc = osquery::SQLiteDBManager::get(); for (size_t i = 0; i < static_cast(osquery::FLAGS_profile); ++i) { osquery::QueryData results; diff --git a/src/osquery/plugins/CMakeLists.txt b/src/osquery/plugins/CMakeLists.txt index c6c418e..e81b6bf 100644 --- a/src/osquery/plugins/CMakeLists.txt +++ b/src/osquery/plugins/CMakeLists.txt @@ -5,6 +5,3 @@ # the LICENSE file found in the root directory of this source tree. ADD_OSQUERY_LIBRARY(osquery_pluing_logger logger/filesystem_logger.cpp) - -ADD_OSQUERY_LIBRARY(osquery_plugin_db database/ephemeral.cpp - database/sqlite.cpp) diff --git a/src/osquery/plugins/database/ephemeral.cpp b/src/osquery/plugins/database/ephemeral.cpp deleted file mode 100644 index 607f9ac..0000000 --- a/src/osquery/plugins/database/ephemeral.cpp +++ /dev/null @@ -1,143 +0,0 @@ -/** - * Copyright (c) 2014-present, Facebook, Inc. - * All rights reserved. - * - * This source code is licensed in accordance with the terms specified in - * the LICENSE file found in the root directory of this source tree. - */ - -#include - -#include "osquery/plugins/database/ephemeral.h" - -namespace osquery { - -template -Status EphemeralDatabasePlugin::getAny(const std::string& domain, - const std::string& key, - T& value) const { - auto domainIterator = db_.find(domain); - if (domainIterator == db_.end()) { - return Status(1, "Domain " + domain + " does not exist"); - } - - auto keyIterator = domainIterator->second.find(key); - if (keyIterator == domainIterator->second.end()) { - return Status(1, "Key " + key + " in domain " + domain + " does not exist"); - } - - try { - value = boost::get(keyIterator->second); - } catch (const boost::bad_get& e) { - LOG(WARNING) << "Type error getting string value for (domain,key) : (" - << key << "," << domain << ") " << e.what(); - return Status( - 1, "EphemeralDatabasePlugin::get was requested incorrect type(string)"); - } - return Status(0); -} - -Status EphemeralDatabasePlugin::get(const std::string& domain, - const std::string& key, - std::string& value) const { - return this->getAny(domain, key, value); -} -Status EphemeralDatabasePlugin::get(const std::string& domain, - const std::string& key, - int& value) const { - return this->getAny(domain, key, value); -} - -void EphemeralDatabasePlugin::setValue(const std::string& domain, - const std::string& key, - const std::string& value) { - db_[domain][key] = value; -} - -void EphemeralDatabasePlugin::setValue(const std::string& domain, - const std::string& key, - int value) { - db_[domain][key] = value; -} - -Status EphemeralDatabasePlugin::put(const std::string& domain, - const std::string& key, - const std::string& value) { - setValue(domain, key, value); - return Status(0); -} - -Status EphemeralDatabasePlugin::put(const std::string& domain, - const std::string& key, - int value) { - setValue(domain, key, value); - return Status(0); -} - -Status EphemeralDatabasePlugin::putBatch(const std::string& domain, - const DatabaseStringValueList& data) { - for (const auto& p : data) { - const auto& key = p.first; - const auto& value = p.second; - - setValue(domain, key, value); - } - - return Status::success(); -} - -void EphemeralDatabasePlugin::dumpDatabase() const { - for (const auto& domainValue : db_) { - const auto& domain = domainValue.first; - for (const auto& keyValue : domainValue.second) { - const auto& key = keyValue.first; - const auto& value = keyValue.second; - std::cout << domain << "[" << key << "]: " << value << std::endl; - } - } -} - -Status EphemeralDatabasePlugin::remove(const std::string& domain, - const std::string& k) { - db_[domain].erase(k); - return Status(0); -} - -Status EphemeralDatabasePlugin::removeRange(const std::string& domain, - const std::string& low, - const std::string& high) { - std::vector keys; - for (const auto& it : db_[domain]) { - if (it.first >= low && it.first <= high) { - keys.push_back(it.first); - } - } - - for (const auto& key : keys) { - db_[domain].erase(key); - } - return Status(0); -} - -Status EphemeralDatabasePlugin::scan(const std::string& domain, - std::vector& results, - const std::string& prefix, - size_t max) const { - if (db_.count(domain) == 0) { - return Status(0); - } - - for (const auto& key : db_.at(domain)) { - if (!prefix.empty() && - !(std::mismatch(prefix.begin(), prefix.end(), key.first.begin()) - .first == prefix.end())) { - continue; - } - results.push_back(key.first); - if (max > 0 && results.size() >= max) { - break; - } - } - return Status(0); -} -} // namespace osquery diff --git a/src/osquery/plugins/database/ephemeral.h b/src/osquery/plugins/database/ephemeral.h deleted file mode 100644 index 13c10fe..0000000 --- a/src/osquery/plugins/database/ephemeral.h +++ /dev/null @@ -1,85 +0,0 @@ -/** - * Copyright (c) 2014-present, Facebook, Inc. - * All rights reserved. - * - * This source code is licensed in accordance with the terms specified in - * the LICENSE file found in the root directory of this source tree. - */ - -#include -#include -#include -#include - -#include - -namespace osquery { - -DECLARE_string(database_path); - -class EphemeralDatabasePlugin : public DatabasePlugin { - using DBType = - std::map>>; - template - Status getAny(const std::string& domain, - const std::string& key, - T& value) const; - - private: - void setValue(const std::string& domain, - const std::string& key, - const std::string& value); - - void setValue(const std::string& domain, const std::string& key, int value); - - public: - /// Data retrieval method. - - Status get(const std::string& domain, - const std::string& key, - std::string& value) const override; - Status get(const std::string& domain, - const std::string& key, - int& value) const override; - /// Data storage method. - Status put(const std::string& domain, - const std::string& key, - const std::string& value) override; - Status put(const std::string& domain, - const std::string& key, - int value) override; - - Status putBatch(const std::string& domain, - const DatabaseStringValueList& data) override; - - void dumpDatabase() const override; - - /// Data removal method. - Status remove(const std::string& domain, const std::string& k) override; - - Status removeRange(const std::string& domain, - const std::string& low, - const std::string& high) override; - - /// Key/index lookup method. - Status scan(const std::string& domain, - std::vector& results, - const std::string& prefix, - size_t max) const override; - - public: - /// Database workflow: open and setup. - Status setUp() override { - DBType().swap(db_); - return Status(0); - } - - private: - DBType db_; -}; - -/// Backing-storage provider for osquery internal/core. -REGISTER_INTERNAL(EphemeralDatabasePlugin, "database", "ephemeral"); - -} // namespace osquery diff --git a/src/osquery/plugins/database/sqlite.cpp b/src/osquery/plugins/database/sqlite.cpp deleted file mode 100644 index 4e1f253..0000000 --- a/src/osquery/plugins/database/sqlite.cpp +++ /dev/null @@ -1,314 +0,0 @@ -/** - * Copyright (c) 2014-present, Facebook, Inc. - * All rights reserved. - * - * This source code is licensed in accordance with the terms specified in - * the LICENSE file found in the root directory of this source tree. - */ - -#include - -#include -#include - -#include -#include -#include -#include -#include -#include - -namespace osquery { - -const std::map kDBSettings = { - {"synchronous", "OFF"}, - {"count_changes", "OFF"}, - {"default_temp_store", "2"}, - {"auto_vacuum", "FULL"}, - {"journal_mode", "OFF"}, - {"cache_size", "1000"}, - {"page_count", "1000"}, -}; - -Status SQLiteDatabasePlugin::setUp() { - if (!DatabasePlugin::kDBAllowOpen) { - LOG(WARNING) << RLOG(1629) << "Not allowed to set up database plugin"; - } - - // Consume the current settings. - // A configuration update may change them, but that does not affect state. - path_ = FLAGS_database_path; - - if (pathExists(path_).ok() && !isReadable(path_).ok()) { - return Status(1, "Cannot read database path: " + path_); - } - - if (!DatabasePlugin::kDBChecking) { - VLOG(1) << "Opening database handle: " << path_; - } - - // Tests may trash calls to setUp, make sure subsequent calls do not leak. - close(); - - // Open the SQLite backing storage at path_ - auto result = sqlite3_open_v2( - path_.c_str(), - &db_, - (SQLITE_OPEN_FULLMUTEX | SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE), - nullptr); - - if (result != SQLITE_OK || db_ == nullptr) { - if (DatabasePlugin::kDBRequireWrite) { - close(); - // A failed open in R/W mode is a runtime error. - return Status(1, "Cannot open database: " + std::to_string(result)); - } - - if (!DatabasePlugin::kDBChecking) { - VLOG(1) << "Opening database failed: Continuing with read-only support"; - } - - read_only_ = true; - } - - if (!read_only_) { - for (const auto& domain : kDomains) { - std::string q = "create table if not exists " + domain + - " (key TEXT PRIMARY KEY, value TEXT);"; - result = sqlite3_exec(db_, q.c_str(), nullptr, nullptr, nullptr); - if (result != SQLITE_OK) { - close(); - return Status(1, "Cannot create domain: " + domain); - } - } - - std::string settings; - for (const auto& setting : kDBSettings) { - settings += "PRAGMA " + setting.first + "=" + setting.second + "; "; - } - sqlite3_exec(db_, settings.c_str(), nullptr, nullptr, nullptr); - } - - // RocksDB may not create/append a directory with acceptable permissions. - if (!read_only_ && platformSetSafeDbPerms(path_) == false) { - close(); - return Status(1, "Cannot set permissions on database path: " + path_); - } - return Status(0); -} - -void SQLiteDatabasePlugin::close() { - WriteLock lock(close_mutex_); - if (db_ != nullptr) { - sqlite3_close(db_); - db_ = nullptr; - } -} - -static int getData(void* argument, int argc, char* argv[], char* column[]) { - if (argument == nullptr) { - return SQLITE_MISUSE; - } - - QueryData* qData = (QueryData*)argument; - Row r; - for (int i = 0; i < argc; i++) { - if (column[i] != nullptr) { - r[column[i]] = (argv[i] != nullptr) ? argv[i] : ""; - } - } - (*qData).push_back(std::move(r)); - return 0; -} - -Status SQLiteDatabasePlugin::get(const std::string& domain, - const std::string& key, - std::string& value) const { - QueryData results; - char* err = nullptr; - std::string q = "select value from " + domain + " where key = '" + key + "';"; - sqlite3_exec(db_, q.c_str(), getData, &results, &err); - if (err != nullptr) { - sqlite3_free(err); - } - - // Only assign value if the query found a result. - if (results.size() > 0) { - value = std::move(results[0]["value"]); - return Status(0); - } - return Status(1); -} - -Status SQLiteDatabasePlugin::get(const std::string& domain, - const std::string& key, - int& value) const { - std::string result; - auto s = this->get(domain, key, result); - if (s.ok()) { - auto expectedValue = tryTo(result); - if (expectedValue.isError()) { - return Status::failure("Could not deserialize str to int"); - } else { - value = expectedValue.take(); - } - } - return s; -} - -static void tryVacuum(sqlite3* db) { - std::string q = - "SELECT (sum(s1.pageno + 1 == s2.pageno) * 1.0 / count(*)) < 0.01 as v " - " FROM " - "(SELECT pageno FROM dbstat ORDER BY path) AS s1," - "(SELECT pageno FROM dbstat ORDER BY path) AS s2 WHERE " - "s1.rowid + 1 = s2.rowid; "; - - QueryData results; - sqlite3_exec(db, q.c_str(), getData, &results, nullptr); - if (results.size() > 0 && results[0]["v"].back() == '1') { - sqlite3_exec(db, "vacuum;", nullptr, nullptr, nullptr); - } -} - -Status SQLiteDatabasePlugin::put(const std::string& domain, - const std::string& key, - const std::string& value) { - return putBatch(domain, {std::make_pair(key, value)}); -} - -Status SQLiteDatabasePlugin::put(const std::string& domain, - const std::string& key, - int value) { - return putBatch(domain, {std::make_pair(key, std::to_string(value))}); -} - -Status SQLiteDatabasePlugin::putBatch(const std::string& domain, - const DatabaseStringValueList& data) { - if (read_only_) { - return Status::success(); - } - - // Prepare the query, adding placeholders for all the rows we have in `data` - std::stringstream buffer; - buffer << "insert or replace into " + domain + " values "; - - for (auto i = 1U; i <= data.size(); i++) { - auto index = i * 2; - buffer << "(?" << index - 1 << ", ?" << index << ")"; - - if (i + 1 > data.size()) { - buffer << ";"; - } else { - buffer << ", "; - } - } - - const auto& q = buffer.str(); - - // Bind each value from the rows we got - sqlite3_stmt* stmt = nullptr; - sqlite3_prepare_v2(db_, q.c_str(), -1, &stmt, nullptr); - - { - int i = 1; - - for (const auto& p : data) { - const auto& key = p.first; - const auto& value = p.second; - - sqlite3_bind_text(stmt, i, key.c_str(), -1, SQLITE_STATIC); - sqlite3_bind_text(stmt, i + 1, value.c_str(), -1, SQLITE_STATIC); - - i += 2; - } - } - - auto rc = sqlite3_step(stmt); - if (rc != SQLITE_DONE) { - return Status(1); - } - - sqlite3_finalize(stmt); - if (rand() % 10 == 0) { - tryVacuum(db_); - } - - return Status::success(); -} - -Status SQLiteDatabasePlugin::remove(const std::string& domain, - const std::string& key) { - if (read_only_) { - return Status::success(); - } - - sqlite3_stmt* stmt = nullptr; - std::string q = "delete from " + domain + " where key IN (?1);"; - sqlite3_prepare_v2(db_, q.c_str(), -1, &stmt, nullptr); - - sqlite3_bind_text(stmt, 1, key.c_str(), -1, SQLITE_STATIC); - auto rc = sqlite3_step(stmt); - if (rc != SQLITE_DONE) { - return Status(1); - } - - sqlite3_finalize(stmt); - if (rand() % 10 == 0) { - tryVacuum(db_); - } - return Status(0); -} - -void SQLiteDatabasePlugin::dumpDatabase() const {} - -Status SQLiteDatabasePlugin::removeRange(const std::string& domain, - const std::string& low, - const std::string& high) { - if (read_only_) { - return Status::success(); - } - - sqlite3_stmt* stmt = nullptr; - std::string q = "delete from " + domain + " where key >= ?1 and key <= ?2;"; - sqlite3_prepare_v2(db_, q.c_str(), -1, &stmt, nullptr); - - sqlite3_bind_text(stmt, 1, low.c_str(), -1, SQLITE_STATIC); - sqlite3_bind_text(stmt, 2, high.c_str(), -1, SQLITE_STATIC); - auto rc = sqlite3_step(stmt); - if (rc != SQLITE_DONE) { - return Status(1); - } - - sqlite3_finalize(stmt); - if (rand() % 10 == 0) { - tryVacuum(db_); - } - return Status(0); -} - -Status SQLiteDatabasePlugin::scan(const std::string& domain, - std::vector& results, - const std::string& prefix, - size_t max) const { - QueryData _results; - char* err = nullptr; - - std::string q = - "select key from " + domain + " where key LIKE '" + prefix + "%'"; - if (max > 0) { - q += " limit " + std::to_string(max); - } - sqlite3_exec(db_, q.c_str(), getData, &_results, &err); - if (err != nullptr) { - sqlite3_free(err); - } - - // Only assign value if the query found a result. - for (auto& r : _results) { - results.push_back(std::move(r["key"])); - } - - return Status::success(); -} -} // namespace osquery diff --git a/src/osquery/plugins/database/sqlite.h b/src/osquery/plugins/database/sqlite.h deleted file mode 100644 index ef0ea1f..0000000 --- a/src/osquery/plugins/database/sqlite.h +++ /dev/null @@ -1,87 +0,0 @@ -/** - * Copyright (c) 2014-present, Facebook, Inc. - * All rights reserved. - * - * This source code is licensed in accordance with the terms specified in - * the LICENSE file found in the root directory of this source tree. - */ - -#include - -#include - -#include -#include -#include -#include - -namespace osquery { - -DECLARE_string(database_path); - -class SQLiteDatabasePlugin : public DatabasePlugin { - public: - /// Data retrieval method. - Status get(const std::string& domain, - const std::string& key, - std::string& value) const override; - Status get(const std::string& domain, - const std::string& key, - int& value) const override; - - /// Data storage method. - Status put(const std::string& domain, - const std::string& key, - const std::string& value) override; - Status put(const std::string& domain, - const std::string& key, - int value) override; - - Status putBatch(const std::string& domain, - const DatabaseStringValueList& data) override; - - void dumpDatabase() const override; - - /// Data removal method. - Status remove(const std::string& domain, const std::string& k) override; - - /// Data range removal method. - Status removeRange(const std::string& domain, - const std::string& low, - const std::string& high) override; - - /// Key/index lookup method. - Status scan(const std::string& domain, - std::vector& results, - const std::string& prefix, - size_t max) const override; - - public: - /// Database workflow: open and setup. - Status setUp() override; - - /// Database workflow: close and cleanup. - void tearDown() override { - close(); - } - - /// Need to tear down open resources, - virtual ~SQLiteDatabasePlugin() { - close(); - } - - private: - void close(); - - private: - /// The long-lived sqlite3 database. - sqlite3* db_{nullptr}; - - /// Deconstruction mutex. - Mutex close_mutex_; -}; - -/// Backing-storage provider for osquery internal/core. -REGISTER_INTERNAL(SQLiteDatabasePlugin, "database", "sqlite"); - -} // namespace osquery diff --git a/src/osquery/plugins/database/tests/sqlite.cpp b/src/osquery/plugins/database/tests/sqlite.cpp deleted file mode 100644 index 57c81b8..0000000 --- a/src/osquery/plugins/database/tests/sqlite.cpp +++ /dev/null @@ -1,22 +0,0 @@ -/** - * Copyright (c) 2014-present, Facebook, Inc. - * All rights reserved. - * - * This source code is licensed in accordance with the terms specified in - * the LICENSE file found in the root directory of this source tree. - */ - -#include - -namespace osquery { - -class SQLiteDatabasePluginTests : public DatabasePluginTests { - protected: - std::string name() override { - return "sqlite"; - } -}; - -// Define the default set of database plugin operation tests. -CREATE_DATABASE_TESTS(SQLiteDatabasePluginTests); -} diff --git a/src/osquery/plugins/database/tests/utils.cpp b/src/osquery/plugins/database/tests/utils.cpp deleted file mode 100644 index 35dd376..0000000 --- a/src/osquery/plugins/database/tests/utils.cpp +++ /dev/null @@ -1,268 +0,0 @@ -/** - * Copyright (c) 2014-present, Facebook, Inc. - * All rights reserved. - * - * This source code is licensed in accordance with the terms specified in - * the LICENSE file found in the root directory of this source tree. - */ - -#include - -#include -#include -#include -#include -#include -#include -#include - -#include - -namespace fs = boost::filesystem; - -namespace osquery { - -DECLARE_bool(disable_database); -DECLARE_string(database_path); - -class EphemeralDatabasePluginTests : public DatabasePluginTests { - protected: - std::string name() override { - return "ephemeral"; - } -}; - -// Define the default set of database plugin operation tests. -CREATE_DATABASE_TESTS(EphemeralDatabasePluginTests); - -void DatabasePluginTests::SetUp() { - Initializer::platformSetup(); - registryAndPluginInit(); - FLAGS_disable_database = true; - DatabasePlugin::setAllowOpen(true); - DatabasePlugin::initPlugin(); - - auto& rf = RegistryFactory::get(); - existing_plugin_ = rf.getActive("database"); - rf.plugin("database", existing_plugin_)->tearDown(); - - setName(name()); - path_ = FLAGS_database_path; - FLAGS_database_path = ( - fs::temp_directory_path() / - fs::unique_path("osquery.database_plugin_tests.%%%%.%%%%.%%%%.%%%%.db") - ).string(); - // removePath(path_); - - auto plugin = rf.plugin("database", getName()); - plugin_ = std::dynamic_pointer_cast(plugin); - plugin_->reset(); - - rf.setActive("database", getName()); -} - -void DatabasePluginTests::TearDown() { - auto& rf = RegistryFactory::get(); - rf.plugin("database", name_)->tearDown(); - rf.setActive("database", existing_plugin_); - fs::remove_all(fs::path(FLAGS_database_path)); - FLAGS_database_path = path_; -} - -void DatabasePluginTests::testPluginCheck() { - auto& rf = RegistryFactory::get(); - - // Do not worry about multiple set-active calls. - // For testing purposes they should be idempotent. - EXPECT_TRUE(rf.setActive("database", getName())); - - // Get an instance of the database plugin and call check. - auto plugin = rf.plugin("database", getName()); - auto db_plugin = std::dynamic_pointer_cast(plugin); - EXPECT_TRUE(db_plugin->checkDB()); - - // Testing relies on database resetting too. - EXPECT_TRUE(db_plugin->reset()); -} - -auto kTestReseter = ([]() { resetDatabase(); }); - -void DatabasePluginTests::testReset() { - RegistryFactory::get().setActive("database", getName()); - setDatabaseValue(kLogs, "reset", "1"); - resetDatabase(); - - if ("ephemeral" != getName()) { - // The ephemeral plugin is special and does not persist after reset. - std::string value; - EXPECT_TRUE(getDatabaseValue(kLogs, "reset", value)); - EXPECT_EQ(value, "1"); - } -} - -void DatabasePluginTests::testPut() { - auto s = getPlugin()->put(kQueries, "test_put", "bar"); - EXPECT_TRUE(s.ok()); - EXPECT_EQ(s.getMessage(), "OK"); - - s = setDatabaseValue(kQueries, "test_put", ""); - EXPECT_TRUE(s.ok()); - - PluginRequest req = {{"action", "put"}, - {"domain", kQueries}, - {"key", "test_put"}, - {"value", "bar"}}; - s = Registry::call("database", getName(), req); - EXPECT_TRUE(s.ok()); - - auto reset = std::async(std::launch::async, kTestReseter); - reset.get(); -} - -void DatabasePluginTests::testPutBatch() { - DatabaseStringValueList string_batch = { - {"test_put_str1", "test_put_str1_value"}, - {"test_put_str2", "test_put_str2_value"}}; - - auto s = getPlugin()->putBatch(kQueries, string_batch); - EXPECT_TRUE(s.ok()); - EXPECT_EQ(s.getMessage(), "OK"); - - for (const auto& p : string_batch) { - const auto& key = p.first; - const auto& expected_value = p.second; - - std::string value; - s = getDatabaseValue(kQueries, key, value); - EXPECT_EQ(s.getMessage(), "OK"); - - EXPECT_EQ(expected_value, value); - } - - DatabaseStringValueList str_batch2 = { - {"test_plugin_put_json_str1", "test_put_str1_value"}, - {"test_plugin_put_json_str2", "test_put_str2_value"}}; - - auto json_object = JSON::newObject(); - for (const auto& p : str_batch2) { - const auto& key = p.first; - const auto& value = p.second; - - json_object.addRef(key, value); - } - - std::string serialized_data; - s = json_object.toString(serialized_data); - EXPECT_TRUE(s.ok()); - - PluginRequest request = { - {"action", "putBatch"}, {"domain", kQueries}, {"json", serialized_data}}; - - s = Registry::call("database", getName(), request); - EXPECT_TRUE(s.ok()); - - for (const auto& p : str_batch2) { - const auto& key = p.first; - const auto& expected_value = p.second; - - std::string value; - s = getDatabaseValue(kQueries, key, value); - EXPECT_EQ(s.getMessage(), "OK"); - - EXPECT_EQ(expected_value, value); - } - - auto reset = std::async(std::launch::async, kTestReseter); - reset.get(); -} - -void DatabasePluginTests::testGet() { - getPlugin()->put(kQueries, "test_get", "bar"); - - std::string r; - auto s = getPlugin()->get(kQueries, "test_get", r); - EXPECT_TRUE(s.ok()); - EXPECT_EQ(s.getMessage(), "OK"); - EXPECT_EQ(r, "bar"); - - auto reset = std::async(std::launch::async, kTestReseter); - reset.get(); -} - -void DatabasePluginTests::testDelete() { - getPlugin()->put(kQueries, "test_delete", "baz"); - auto s = getPlugin()->remove(kQueries, "test_delete"); - EXPECT_TRUE(s.ok()); - EXPECT_EQ(s.getMessage(), "OK"); -} - -void DatabasePluginTests::testDeleteRange() { - getPlugin()->put(kQueries, "test_delete", "baz"); - getPlugin()->put(kQueries, "test1", "1"); - getPlugin()->put(kQueries, "test2", "2"); - getPlugin()->put(kQueries, "test3", "3"); - getPlugin()->put(kQueries, "test4", "4"); - auto s = getPlugin()->removeRange(kQueries, "test1", "test3"); - EXPECT_TRUE(s.ok()); - EXPECT_EQ(s.getMessage(), "OK"); - - std::string r; - getPlugin()->get(kQueries, "test4", r); - EXPECT_EQ(r, "4"); - getPlugin()->get(kQueries, "test_delete", r); - EXPECT_EQ(r, "baz"); - s = getPlugin()->get(kQueries, "test1", r); - EXPECT_FALSE(s.ok()); - s = getPlugin()->get(kQueries, "test2", r); - EXPECT_FALSE(s.ok()); - s = getPlugin()->get(kQueries, "test3", r); - EXPECT_FALSE(s.ok()); - - // Expect invalid logically ranges to have no effect. - getPlugin()->put(kQueries, "new_test1", "1"); - getPlugin()->put(kQueries, "new_test2", "2"); - getPlugin()->put(kQueries, "new_test3", "3"); - getPlugin()->put(kQueries, "new_test4", "4"); - s = getPlugin()->removeRange(kQueries, "new_test3", "new_test2"); - EXPECT_TRUE(s.ok()); - getPlugin()->get(kQueries, "new_test2", r); - EXPECT_EQ(r, "2"); - getPlugin()->get(kQueries, "new_test3", r); - EXPECT_EQ(r, "3"); - - // An equality range will not delete that single item. - s = getPlugin()->removeRange(kQueries, "new_test2", "new_test2"); - EXPECT_TRUE(s.ok()); - s = getPlugin()->get(kQueries, "new_test2", r); - EXPECT_FALSE(s.ok()); -} - -void DatabasePluginTests::testScan() { - getPlugin()->put(kQueries, "test_scan_foo1", "baz"); - getPlugin()->put(kQueries, "test_scan_foo2", "baz"); - getPlugin()->put(kQueries, "test_scan_foo3", "baz"); - - std::vector keys; - std::vector expected = { - "test_scan_foo1", "test_scan_foo2", "test_scan_foo3"}; - auto s = getPlugin()->scan(kQueries, keys, "", 0); - EXPECT_TRUE(s.ok()); - EXPECT_EQ(s.getMessage(), "OK"); - EXPECT_EQ(keys.size(), 3U); - for (const auto& i : expected) { - EXPECT_NE(std::find(keys.begin(), keys.end(), i), keys.end()); - } -} - -void DatabasePluginTests::testScanLimit() { - getPlugin()->put(kQueries, "test_scan_foo1", "baz"); - getPlugin()->put(kQueries, "test_scan_foo2", "baz"); - getPlugin()->put(kQueries, "test_scan_foo3", "baz"); - - std::vector keys; - auto s = getPlugin()->scan(kQueries, keys, "", 2); - EXPECT_TRUE(s.ok()); - EXPECT_EQ(s.getMessage(), "OK"); - EXPECT_EQ(keys.size(), 2U); -} -} // namespace osquery diff --git a/src/osquery/plugins/database/tests/utils.h b/src/osquery/plugins/database/tests/utils.h deleted file mode 100644 index ccc5cf5..0000000 --- a/src/osquery/plugins/database/tests/utils.h +++ /dev/null @@ -1,95 +0,0 @@ -/** - * Copyright (c) 2014-present, Facebook, Inc. - * All rights reserved. - * - * This source code is licensed in accordance with the terms specified in - * the LICENSE file found in the root directory of this source tree. - */ - -#pragma once - -#include - -#include - -/// The following test macros allow pretty test output. -#define CREATE_DATABASE_TESTS(n) \ - TEST_F(n, test_plugin_check) { \ - testPluginCheck(); \ - } \ - TEST_F(n, test_reset) { \ - testReset(); \ - } \ - TEST_F(n, test_put) { \ - testPut(); \ - } \ - TEST_F(n, test_putBatch) { \ - testPutBatch(); \ - } \ - TEST_F(n, test_get) { \ - testGet(); \ - } \ - TEST_F(n, test_delete) { \ - testDelete(); \ - } \ - TEST_F(n, test_delete_range) { \ - testDeleteRange(); \ - } \ - TEST_F(n, test_scan) { \ - testScan(); \ - } \ - TEST_F(n, test_scan_limit) { \ - testScanLimit(); \ - } - -namespace osquery { - -class DatabasePluginTests : public testing::Test { - public: - void SetUp() override; - - void TearDown() override; - - protected: - /// Path to testing database. - std::string path_; - - protected: - /// Require each plugin tester to implement a set name. - virtual std::string name() = 0; - - private: - void setName(const std::string& name) { - name_ = name; - } - - const std::string& getName() { - return name_; - } - - std::shared_ptr getPlugin() { - return plugin_; - } - - private: - /// Plugin name - std::string name_; - - /// Plugin casted from setUp, ready to run tests. - std::shared_ptr plugin_{nullptr}; - - /// Previous active database plugin. - std::string existing_plugin_; - - protected: - void testPluginCheck(); - void testReset(); - void testPut(); - void testPutBatch(); - void testGet(); - void testDelete(); - void testDeleteRange(); - void testScan(); - void testScanLimit(); -}; -} // namespace osquery diff --git a/src/osquery/sql/tests/virtual_table.cpp b/src/osquery/sql/tests/virtual_table.cpp index 24b4d8a..67aa30c 100644 --- a/src/osquery/sql/tests/virtual_table.cpp +++ b/src/osquery/sql/tests/virtual_table.cpp @@ -9,7 +9,6 @@ #include #include -#include #include #include #include @@ -20,16 +19,11 @@ namespace osquery { -DECLARE_bool(disable_database); - class VirtualTableTests : public testing::Test { public: void SetUp() override { Initializer::platformSetup(); registryAndPluginInit(); - FLAGS_disable_database = true; - DatabasePlugin::setAllowOpen(true); - DatabasePlugin::initPlugin(); } }; diff --git a/src/osquery/tests/test_util.cpp b/src/osquery/tests/test_util.cpp index 0078732..6ffaeec 100644 --- a/src/osquery/tests/test_util.cpp +++ b/src/osquery/tests/test_util.cpp @@ -42,10 +42,8 @@ std::string kTestWorkingDirectory; /// The relative path within the source repo to find test content. std::string kTestDataPath{"../../../tools/tests/"}; -DECLARE_string(database_path); DECLARE_string(enroll_tls_endpoint); DECLARE_bool(disable_logging); -DECLARE_bool(disable_database); using chrono_clock = std::chrono::high_resolution_clock; @@ -100,22 +98,13 @@ void initTesting() { fs::remove_all(kTestWorkingDirectory); fs::create_directories(kTestWorkingDirectory); - FLAGS_database_path = kTestWorkingDirectory + "unittests.db"; FLAGS_disable_logging = true; - FLAGS_disable_database = true; - - // Tests need a database plugin. - // Set up the database instance for the unittests. - DatabasePlugin::setAllowOpen(true); - DatabasePlugin::initPlugin(); Initializer::platformSetup(); } void shutdownTesting() { - DatabasePlugin::shutdown(); - Initializer::platformTeardown(); } diff --git a/src/osquery/tests/test_util.h b/src/osquery/tests/test_util.h index 8edb254..f02fc03 100644 --- a/src/osquery/tests/test_util.h +++ b/src/osquery/tests/test_util.h @@ -13,7 +13,6 @@ #include #include -#include #include namespace osquery { -- 2.7.4