## osquery v4.0.0
ADD_SUBDIRECTORY(core)
-ADD_SUBDIRECTORY(database)
ADD_SUBDIRECTORY(filesystem)
ADD_SUBDIRECTORY(logger)
ADD_SUBDIRECTORY(plugins)
*/
#include <osquery/core/database/in_memory_database.h>
-#include <osquery/database.h>
#include <osquery/logger.h>
#include <boost/algorithm/string.hpp>
ExpectedSuccess<DatabaseError> InMemoryDatabase::open() {
debug_only::verifyTrue(!is_open_, "database is already open");
- for (const auto& domain : kDomains) {
- storage_[domain] = std::make_unique<InMemoryStorage<DataType>>();
- }
is_open_ = true;
return Success();
}
#include <osquery/registry.h>
#include <osquery/utils/info/version.h>
#include <osquery/utils/system/time.h>
-#include <osquery/database.h>
#ifdef __linux__
#include <sys/syscall.h>
DECLARE_string(logger_plugin);
DECLARE_bool(config_check);
DECLARE_bool(config_dump);
-DECLARE_bool(database_dump);
-DECLARE_string(database_path);
-DECLARE_bool(disable_database);
DECLARE_bool(disable_logging);
CLI_FLAG(bool, S, false, "Run as a shell process");
CLI_FLAG(bool, D, false, "Run as a daemon process");
CLI_FLAG(bool, daemonize, false, "Attempt to daemonize (POSIX only)");
-FLAG(bool, ephemeral, false, "Skip pidfile and database state checks");
-
ToolType kToolType{ToolType::UNKNOWN};
/// The saved exit code from a thread's request to stop the process.
/// When no flagfile is provided via CLI, attempt to read flag 'defaults'.
const std::string kBackupDefaultFlagfile{OSQUERY_HOME "osquery.flags.default"};
-const size_t kDatabaseMaxRetryCount{25};
-const size_t kDatabaseRetryDelay{200};
std::function<void()> Initializer::shutdown_{nullptr};
RecursiveMutex Initializer::shutdown_mutex_;
-namespace {
-
-void initWorkDirectories() {
- if (!FLAGS_disable_database) {
- auto const recursive = true;
- auto const ignore_existence = true;
- auto const status = createDirectory(
- boost::filesystem::path(FLAGS_database_path).parent_path(),
- recursive,
- ignore_existence);
- if (!status.ok()) {
- LOG(ERROR) << "Could not initialize db directory: " << status.what();
- }
- }
-}
-
-} // namespace
-
static inline void printUsage(const std::string& binary, ToolType tool) {
// Parse help options before gflags. Only display osquery-related options.
fprintf(stdout, DESCRIPTION, kVersion.c_str());
// Initialize registries and plugins
registryAndPluginInit();
- if (isShell() || FLAGS_ephemeral) {
- if (Flag::isDefault("database_path") &&
- Flag::isDefault("disable_database")) {
- // The shell should not use a database by default, but should use the DB
- // specified by database_path if it is set
- FLAGS_disable_database = true;
- }
- }
-
if (isShell()) {
// Initialize the shell after setting modified defaults and parsing flags.
initShell();
}
- if (isDaemon()) {
- initWorkDirectories();
- }
if (FLAGS_enable_signal_handler) {
std::signal(SIGABRT, signalHandler);
// Print the version to the OS system log.
systemLog(binary_ + " started [version=" + kVersion + "]");
-
- if (!FLAGS_ephemeral) {
- // Create a process mutex around the daemon.
- auto pid_status = createPidFile();
- if (!pid_status.ok()) {
- LOG(ERROR) << binary_ << " initialize failed: " << pid_status.toString();
- shutdown(EXIT_FAILURE);
- }
- }
}
void Initializer::initShell() const {
// Get the caller's home dir for temporary storage/state management.
auto homedir = osqueryHomeDirectory();
- if (osquery::pathExists(homedir).ok()) {
- // Only apply user/shell-specific paths if not overridden by CLI flag.
- if (Flag::isDefault("database_path")) {
- osquery::FLAGS_database_path =
- (fs::path(homedir) / "shell.db").make_preferred().string();
- }
- } else {
- fprintf(
- stderr, "Cannot access or create osquery home: %s", homedir.c_str());
- FLAGS_disable_database = true;
- }
-
if (Flag::isDefault("hash_delay")) {
// The hash_delay is designed for daemons only.
Flag::updateValue("hash_delay", "0");
}
void Initializer::start() const {
- DatabasePlugin::setAllowOpen(true);
- // A daemon must always have R/W access to the database.
- DatabasePlugin::setRequireWrite(isDaemon());
-
- for (size_t i = 1; i <= kDatabaseMaxRetryCount; i++) {
- if (DatabasePlugin::initPlugin().ok()) {
- break;
- }
-
- if (i == kDatabaseMaxRetryCount) {
- LOG(ERROR) << RLOG(1629) << binary_
- << " initialize failed: Could not initialize database";
- auto retcode = (isWorker()) ? EXIT_CATASTROPHIC : EXIT_FAILURE;
- requestShutdown(retcode);
- }
- }
-
- // Ensure the database results version is up to date before proceeding
- if (!upgradeDatabase()) {
- LOG(ERROR) << "Failed to upgrade database";
- auto retcode = (isWorker()) ? EXIT_CATASTROPHIC : EXIT_FAILURE;
- requestShutdown(retcode);
- }
-
-
// Run the setup for all lazy registries (tables, SQL).
Registry::setUp();
- if (FLAGS_database_dump) {
- dumpDatabase();
- requestShutdown();
- }
-
// Initialize the status and result plugin logger.
if (!FLAGS_disable_logging) {
initActivePlugin("logger", FLAGS_logger_plugin);
// Hopefully release memory used by global string constructors in gflags.
GFLAGS_NAMESPACE::ShutDownCommandLineFlags();
- DatabasePlugin::shutdown();
auto excode = (kExitCode != 0) ? kExitCode : EXIT_SUCCESS;
exit(excode);
#include <string>
#include <vector>
-#include <osquery/database.h>
#include <osquery/flags.h>
#include <osquery/logger.h>
#include <osquery/query.h>
"Use numeric JSON syntax for numeric values");
uint64_t Query::getPreviousEpoch() const {
- uint64_t epoch = 0;
- std::string raw;
- auto status = getDatabaseValue(kQueries, name_ + "epoch", raw);
- if (status.ok()) {
- epoch = std::stoull(raw);
- }
- return epoch;
+ return 0;
}
uint64_t Query::getQueryCounter(bool new_query) const {
- uint64_t counter = 0;
- if (new_query) {
- return counter;
- }
-
- std::string raw;
- auto status = getDatabaseValue(kQueries, name_ + "counter", raw);
- if (status.ok()) {
- counter = std::stoull(raw) + 1;
- }
- return counter;
+ return 0;
}
Status Query::getPreviousQueryResults(QueryDataSet& results) const {
- std::string raw;
- auto status = getDatabaseValue(kQueries, name_, raw);
- if (!status.ok()) {
- return status;
- }
-
- status = deserializeQueryDataJSON(raw, results);
- if (!status.ok()) {
- return status;
- }
return Status::success();
}
std::vector<std::string> Query::getStoredQueryNames() {
std::vector<std::string> results;
- scanDatabaseKeys(kQueries, results);
return results;
}
static inline void saveQuery(const std::string& name,
const std::string& query) {
- setDatabaseValue(kQueries, "query." + name, query);
}
bool Query::isNewQuery() const {
- std::string query;
- getDatabaseValue(kQueries, "query." + name_, query);
- return (query != query_);
+ return true;
}
Status Query::addNewResults(QueryDataTyped qd,
target_gd = &dr.added;
}
- counter = getQueryCounter(fresh_results || new_query);
- auto status =
- setDatabaseValue(kQueries, name_ + "counter", std::to_string(counter));
- if (!status.ok()) {
- return status;
- }
-
- if (update_db) {
- // Replace the "previous" query data with the current.
- std::string json;
- status = serializeQueryDataJSON(*target_gd, json, true);
- if (!status.ok()) {
- return status;
- }
-
- status = setDatabaseValue(kQueries, name_, json);
- if (!status.ok()) {
- return status;
- }
-
- status = setDatabaseValue(
- kQueries, name_ + "epoch", std::to_string(current_epoch));
- if (!status.ok()) {
- return status;
- }
- }
return Status::success();
}
#include <boost/uuid/uuid_io.hpp>
#include <osquery/core.h>
-#include <osquery/database.h>
#include <osquery/filesystem/filesystem.h>
#include <osquery/flags.h>
#include <osquery/logger.h>
}
Status getInstanceUUID(std::string& ident) {
- // Lookup the instance identifier (UUID) previously generated and stored.
- auto status =
- getDatabaseValue(kPersistentSettings, "instance_uuid_v1", ident);
- if (ident.size() == 0) {
- // There was no UUID stored in the database, generate one and store it.
- ident = osquery::generateNewUUID();
- return setDatabaseValue(kPersistentSettings, "instance_uuid_v1", ident);
- }
+ ident = osquery::generateNewUUID();
- return status;
+ return Status(-1, "Not supported");
}
Status getEphemeralUUID(std::string& ident) {
}
Status getHostUUID(std::string& ident) {
- // Lookup the host identifier (UUID) previously generated and stored.
- auto status = getDatabaseValue(kPersistentSettings, "host_uuid_v3", ident);
- if (ident.size() == 0) {
- // There was no UUID stored in the database, generate one and store it.
- ident = osquery::generateHostUUID();
- return setDatabaseValue(kPersistentSettings, "host_uuid_v3", ident);
- }
- return status;
+ return Status(-1, "Not supported");
}
Status getSpecifiedUUID(std::string& ident) {
#include <osquery/utils/json/json.h>
-#include <osquery/database.h>
#include <osquery/flags.h>
#include <osquery/logger.h>
#include <osquery/registry_factory.h>
}
TableRows TablePlugin::getCache() const {
- VLOG(1) << "Retrieving results from cache for table: " << getName();
- // Lookup results from database and deserialize.
- std::string content;
- getDatabaseValue(kQueries, "cache." + getName(), content);
TableRows results;
- deserializeTableRowsJSON(content, results);
return results;
}
size_t interval,
const QueryContext& ctx,
const TableRows& results) {
- if (FLAGS_disable_caching || !cacheAllowed(columns(), ctx)) {
- return;
- }
-
- // Serialize QueryData and save to database.
- std::string content;
- if (serializeTableRowsJSON(results, content)) {
- last_cached_ = step;
- last_interval_ = interval;
- setDatabaseValue(kQueries, "cache." + getName(), content);
- }
+ return;
}
std::string columnDefinition(const TableColumns& columns, bool is_extension) {
+++ /dev/null
-# Copyright (c) 2019 Samsung Electronics Co., Ltd All Rights Reserved
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License
-
-ADD_OSQUERY_LIBRARY(osquery_database database.cpp)
-
-FILE(GLOB OSQUERY_DATABASE_TESTS "tests/*.cpp")
-ADD_OSQUERY_TEST(${OSQUERY_DATABASE_TESTS})
+++ /dev/null
-/**
- * Copyright (c) 2014-present, Facebook, Inc.
- * All rights reserved.
- *
- * This source code is licensed in accordance with the terms specified in
- * the LICENSE file found in the root directory of this source tree.
- */
-
-#include <boost/algorithm/string/predicate.hpp>
-#include <boost/io/detail/quoted_manip.hpp>
-#include <boost/property_tree/json_parser.hpp>
-
-#include <osquery/database.h>
-#include <osquery/flagalias.h>
-#include <osquery/flags.h>
-#include <osquery/logger.h>
-#include <osquery/registry.h>
-#include <osquery/utils/config/default_paths.h>
-#include <osquery/utils/conversions/tryto.h>
-#include <osquery/utils/json/json.h>
-
-namespace pt = boost::property_tree;
-namespace rj = rapidjson;
-
-namespace osquery {
-
-/// Generate a specific-use registry for database access abstraction.
-CREATE_REGISTRY(DatabasePlugin, "database");
-
-CLI_FLAG(bool, database_dump, false, "Dump the contents of the backing store");
-
-CLI_FLAG(string,
- database_path,
- OSQUERY_DB_HOME "osquery.db",
- "If using a disk-based backing store, specify a path");
-FLAG_ALIAS(std::string, db_path, database_path);
-
-FLAG(bool, disable_database, false, "Disable the persistent RocksDB storage");
-
-const std::string kInternalDatabase = "rocksdb";
-const std::string kPersistentSettings = "configurations";
-const std::string kQueries = "queries";
-const std::string kEvents = "events";
-const std::string kCarves = "carves";
-const std::string kLogs = "logs";
-
-const std::string kDbEpochSuffix = "epoch";
-const std::string kDbCounterSuffix = "counter";
-
-const std::string kDbVersionKey = "results_version";
-
-const std::vector<std::string> kDomains = {
- kPersistentSettings, kQueries, kEvents, kLogs, kCarves};
-
-std::atomic<bool> DatabasePlugin::kDBAllowOpen(false);
-std::atomic<bool> DatabasePlugin::kDBRequireWrite(false);
-std::atomic<bool> DatabasePlugin::kDBInitialized(false);
-std::atomic<bool> DatabasePlugin::kDBChecking(false);
-
-/**
- * @brief A reader/writer mutex protecting database resets.
- *
- * A write is locked while using reset flows. A read is locked when calling
- * database plugin APIs.
- */
-Mutex kDatabaseReset;
-
-Status DatabasePlugin::initPlugin() {
- // Initialize the database plugin using the flag.
- auto plugin = (FLAGS_disable_database) ? "ephemeral" : kInternalDatabase;
- {
- auto const status = RegistryFactory::get().setActive("database", plugin);
- if (status.ok()) {
- kDBInitialized = true;
- return status;
- }
- LOG(WARNING) << "Failed to activate database plugin " << boost::io::quoted(plugin) << ": " << status.what();
- }
- // If the database did not setUp override the active plugin.
- auto const status = RegistryFactory::get().setActive("database", "ephemeral");
- if (!status.ok()) {
- LOG(ERROR) << "Failed to activate database plugin \"ephemeral\": " << status.what();
- }
- kDBInitialized = status.ok();
- return status;
-}
-
-void DatabasePlugin::shutdown() {
- auto database_registry = RegistryFactory::get().registry("database");
- for (auto& plugin : RegistryFactory::get().names("database")) {
- database_registry->remove(plugin);
- }
-}
-
-Status DatabasePlugin::reset() {
- // Keep this simple, scope the critical section to the broader methods.
- tearDown();
- return setUp();
-}
-
-bool DatabasePlugin::checkDB() {
- kDBChecking = true;
- bool result = true;
- try {
- auto status = setUp();
- if (kDBRequireWrite && read_only_) {
- result = false;
- }
- tearDown();
- result = status.ok();
- } catch (const std::exception& e) {
- VLOG(1) << "Database plugin check failed: " << e.what();
- result = false;
- }
- kDBChecking = false;
- return result;
-}
-
-Status DatabasePlugin::scan(const std::string& domain,
- std::vector<std::string>& results,
- const std::string& prefix,
- size_t max) const {
- return Status::success();
-}
-
-Status DatabasePlugin::call(const PluginRequest& request,
- PluginResponse& response) {
- if (request.count("action") == 0) {
- return Status(1, "Database plugin must include a request action");
- }
-
- // Get a domain/key, which are used for most database plugin actions.
- auto domain = (request.count("domain") > 0) ? request.at("domain") : "";
- auto key = (request.count("key") > 0) ? request.at("key") : "";
-
- if (request.at("action") == "reset") {
- WriteLock lock(kDatabaseReset);
- DatabasePlugin::kDBInitialized = false;
- // Prevent RocksDB reentrancy by logger plugins during plugin setup.
- VLOG(1) << "Resetting the database plugin: " << getName();
- auto status = this->reset();
- if (!status.ok()) {
- // The active database could not be reset, fallback to an ephemeral.
- Registry::get().setActive("database", "ephemeral");
- LOG(WARNING) << "Unable to reset database plugin: " << getName();
- }
- DatabasePlugin::kDBInitialized = true;
- return status;
- }
-
- // Switch over the possible database plugin actions.
- ReadLock lock(kDatabaseReset);
- if (request.at("action") == "get") {
- std::string value;
- auto status = this->get(domain, key, value);
- response.push_back({{"v", value}});
- return status;
- } else if (request.at("action") == "put") {
- if (request.count("value") == 0) {
- return Status(1, "Database plugin put action requires a value");
- }
- return this->put(domain, key, request.at("value"));
- } else if (request.at("action") == "putBatch") {
- if (request.count("json") == 0) {
- return Status(
- 1,
- "Database plugin putBatch action requires a json-encoded value list");
- }
-
- auto json_object = JSON::newObject();
-
- auto status = json_object.fromString(request.at("json"));
- if (!status.ok()) {
- VLOG(1) << status.getMessage();
- return status;
- }
-
- const auto& json_object_list = json_object.doc().GetObject();
-
- DatabaseStringValueList data;
- data.reserve(json_object_list.MemberCount());
-
- for (auto& item : json_object_list) {
- if (!item.value.IsString()) {
- return Status(1,
- "Database plugin putBatch action with an invalid json "
- "received. Only string values are supported");
- }
-
- data.push_back(
- std::make_pair(item.name.GetString(), item.value.GetString()));
- }
-
- return this->putBatch(domain, data);
- } else if (request.at("action") == "remove") {
- return this->remove(domain, key);
- } else if (request.at("action") == "remove_range") {
- auto key_high =
- (request.count("key_high") > 0) ? request.at("key_high") : "";
- if (!key_high.empty() && !key.empty()) {
- return this->removeRange(domain, key, key_high);
- }
- return Status(1, "Missing range");
- } else if (request.at("action") == "scan") {
- // Accumulate scanned keys into a vector.
- std::vector<std::string> keys;
- // Optionally allow the caller to request a max number of keys.
- size_t max = 0;
- if (request.count("max") > 0) {
- max = std::stoul(request.at("max"));
- }
- auto status = this->scan(domain, keys, request.at("prefix"), max);
- for (const auto& k : keys) {
- response.push_back({{"k", k}});
- }
- return status;
- }
-
- return Status(1, "Unknown database plugin action");
-}
-
-static inline std::shared_ptr<DatabasePlugin> getDatabasePlugin() {
- auto& rf = RegistryFactory::get();
- if (!rf.exists("database", rf.getActive("database"), true)) {
- return nullptr;
- }
-
- auto plugin = rf.plugin("database", rf.getActive("database"));
- return std::dynamic_pointer_cast<DatabasePlugin>(plugin);
-}
-
-namespace {
-Status sendPutBatchDatabaseRequest(const std::string& domain,
- const DatabaseStringValueList& data) {
- auto json_object = JSON::newObject();
- for (const auto& p : data) {
- const auto& key = p.first;
- const auto& value = p.second;
-
- json_object.addRef(key, value);
- }
-
- std::string serialized_data;
- auto status = json_object.toString(serialized_data);
- if (!status.ok()) {
- VLOG(1) << status.getMessage();
- return status;
- }
-
- PluginRequest request = {{"action", "putBatch"},
- {"domain", domain},
- {"json", std::move(serialized_data)}};
-
- status = Registry::call("database", request);
- if (!status.ok()) {
- VLOG(1) << status.getMessage();
- }
-
- return status;
-}
-
-Status sendPutDatabaseRequest(const std::string& domain,
- const DatabaseStringValueList& data) {
- const auto& key = data[0].first;
- const auto& value = data[0].second;
-
- PluginRequest request = {
- {"action", "put"}, {"domain", domain}, {"key", key}, {"value", value}};
-
- auto status = Registry::call("database", request);
- if (!status.ok()) {
- VLOG(1) << status.getMessage();
- }
-
- return status;
-}
-} // namespace
-
-Status getDatabaseValue(const std::string& domain,
- const std::string& key,
- std::string& value) {
- if (domain.empty()) {
- return Status(1, "Missing domain");
- }
-
- ReadLock lock(kDatabaseReset);
- if (!DatabasePlugin::kDBInitialized) {
- throw std::runtime_error("Cannot get database value: " + key);
- } else {
- auto plugin = getDatabasePlugin();
- return plugin->get(domain, key, value);
- }
-}
-
-Status getDatabaseValue(const std::string& domain,
- const std::string& key,
- int& value) {
- std::string result;
- auto s = getDatabaseValue(domain, key, result);
- if (s.ok()) {
- value = std::stoi(result);
- }
- return s;
-}
-
-Status setDatabaseValue(const std::string& domain,
- const std::string& key,
- const std::string& value) {
- return setDatabaseBatch(domain, {std::make_pair(key, value)});
-}
-
-Status setDatabaseBatch(const std::string& domain,
- const DatabaseStringValueList& data) {
- if (domain.empty()) {
- return Status(1, "Missing domain");
- }
-
- ReadLock lock(kDatabaseReset);
- if (!DatabasePlugin::kDBInitialized) {
- throw std::runtime_error("Cannot set database values");
- }
-
- auto plugin = getDatabasePlugin();
- return plugin->putBatch(domain, data);
-}
-
-Status setDatabaseValue(const std::string& domain,
- const std::string& key,
- int value) {
- return setDatabaseBatch(domain, {std::make_pair(key, std::to_string(value))});
-}
-
-Status deleteDatabaseValue(const std::string& domain, const std::string& key) {
- if (domain.empty()) {
- return Status(1, "Missing domain");
- }
-
- ReadLock lock(kDatabaseReset);
- if (!DatabasePlugin::kDBInitialized) {
- throw std::runtime_error("Cannot delete database value: " + key);
- } else {
- auto plugin = getDatabasePlugin();
- return plugin->remove(domain, key);
- }
-}
-
-Status deleteDatabaseRange(const std::string& domain,
- const std::string& low,
- const std::string& high) {
- if (domain.empty()) {
- return Status(1, "Missing domain");
- }
-
- ReadLock lock(kDatabaseReset);
- if (!DatabasePlugin::kDBInitialized) {
- throw std::runtime_error("Cannot delete database values: " + low + " - " +
- high);
- } else {
- auto plugin = getDatabasePlugin();
- return plugin->removeRange(domain, low, high);
- }
-}
-
-Status scanDatabaseKeys(const std::string& domain,
- std::vector<std::string>& keys,
- size_t max) {
- return scanDatabaseKeys(domain, keys, "", max);
-}
-
-/// Get a list of keys for a given domain.
-Status scanDatabaseKeys(const std::string& domain,
- std::vector<std::string>& keys,
- const std::string& prefix,
- size_t max) {
- if (domain.empty()) {
- return Status(1, "Missing domain");
- }
-
- ReadLock lock(kDatabaseReset);
- if (!DatabasePlugin::kDBInitialized) {
- throw std::runtime_error("Cannot scan database values: " + prefix);
- } else {
- auto plugin = getDatabasePlugin();
- return plugin->scan(domain, keys, prefix, max);
- }
-}
-
-void resetDatabase() {
- PluginRequest request = {{"action", "reset"}};
- Registry::call("database", request);
-}
-
-void dumpDatabase() {
- auto plugin = getDatabasePlugin();
- plugin->dumpDatabase();
-}
-
-Status ptreeToRapidJSON(const std::string& in, std::string& out) {
- pt::ptree tree;
- try {
- std::stringstream ss;
- ss << in;
- pt::read_json(ss, tree);
- } catch (const pt::json_parser::json_parser_error& /* e */) {
- return Status(1, "Failed to parse JSON");
- }
-
- auto json = JSON::newArray();
- for (const auto& t : tree) {
- std::stringstream ss;
- pt::write_json(ss, t.second);
-
- rj::Document row;
- if (row.Parse(ss.str().c_str()).HasParseError()) {
- return Status(1, "Failed to serialize JSON");
- }
- json.push(row);
- }
-
- json.toString(out);
-
- return Status::success();
-}
-
-static Status migrateV0V1(void) {
- std::vector<std::string> keys;
- auto s = scanDatabaseKeys(kQueries, keys);
- if (!s.ok()) {
- return Status(1, "Failed to lookup legacy query data from database");
- }
-
- for (const auto& key : keys) {
- // Skip over epoch and counter entries, as 0 is parsed by ptree
- if (boost::algorithm::ends_with(key, kDbEpochSuffix) ||
- boost::algorithm::ends_with(key, kDbCounterSuffix) ||
- boost::algorithm::starts_with(key, "query.")) {
- continue;
- }
-
- std::string value{""};
- if (!getDatabaseValue(kQueries, key, value)) {
- LOG(WARNING) << "Failed to get value from database " << key;
- continue;
- }
-
- std::string out;
- s = ptreeToRapidJSON(value, out);
- if (!s.ok()) {
- LOG(WARNING) << "Conversion from ptree to RapidJSON failed for '" << key
- << ": " << value << "': " << s.what() << ". Dropping key!";
- continue;
- }
-
- if (!setDatabaseValue(kQueries, key, out)) {
- LOG(WARNING) << "Failed to update value in database " << key << ": "
- << value;
- }
- }
-
- return Status::success();
-}
-
-static Status migrateV1V2(void) {
- std::vector<std::string> keys;
- const std::string audit_str(".audit.");
-
- Status s = scanDatabaseKeys(kEvents, keys);
- if (!s.ok()) {
- return Status::failure(
- 1, "Failed to scan event keys from database: " + s.what());
- }
-
- for (const auto& key : keys) {
- const auto pos = key.find(audit_str);
- if (pos != std::string::npos) {
- std::string value;
- std::string new_key = key;
- new_key.replace(pos, audit_str.length(), ".auditeventpublisher.");
-
- s = getDatabaseValue(kEvents, key, value);
- if (!s.ok()) {
- LOG(ERROR) << "Failed to read value for key '" << key
- << "'. Key will be kept but won't be migrated!";
- continue;
- }
-
- s = setDatabaseValue(kEvents, new_key, value);
- if (!s.ok()) {
- LOG(ERROR) << "Failed to set value for key '" << new_key
- << "' migrated from '" << key
- << "'. Original key will be kept but won't be migrated!";
- continue;
- }
-
- s = deleteDatabaseValue(kEvents, key);
- if (!s.ok()) {
- LOG(WARNING) << "Failed to delete key '" << key
- << "' after migration to new key '" << new_key
- << "'. Original key will be kept but data was migrated!";
- }
- }
- }
-
- return Status::success();
-}
-
-Status upgradeDatabase(int to_version) {
- LOG(INFO) << "Checking database version for migration";
-
- std::string value;
- Status st = getDatabaseValue(kPersistentSettings, kDbVersionKey, value);
-
- int db_version = 0;
- /* Since there isn't a reliable way to determined what happen when the read
- * fails we just assume the key doesn't exist which indicates database
- * version 0.
- */
- if (st.ok()) {
- auto ret = tryTo<int>(value);
- if (ret.isError()) {
- LOG(ERROR) << "Invalid value '" << value << "'for " << kDbVersionKey
- << " key. Database is corrupted.";
- return Status(1, "Invalid value for database version.");
- } else {
- db_version = ret.get();
- }
- }
-
- while (db_version != to_version) {
- Status migrate_status;
-
- LOG(INFO) << "Performing migration: " << db_version << " -> "
- << (db_version + 1);
-
- switch (db_version) {
- case 0:
- migrate_status = migrateV0V1();
- break;
-
- case 1:
- migrate_status = migrateV1V2();
- break;
-
- default:
- LOG(ERROR) << "Logic error: the migration code is broken!";
- migrate_status = Status(1);
- break;
- }
-
- if (!migrate_status.ok()) {
- return Status(1, "Database migration failed.");
- }
-
- st = setDatabaseValue(
- kPersistentSettings, kDbVersionKey, std::to_string(db_version + 1));
- if (!st.ok()) {
- LOG(ERROR) << "Failed to set new database version after migration. "
- << "The DB was correctly migrated from version " << db_version
- << " to version " << (db_version + 1)
- << " but persisting the new version failed.";
- return Status(1, "Database migration failed.");
- }
-
- LOG(INFO) << "Migration " << db_version << " -> " << (db_version + 1)
- << " successfully completed!";
-
- db_version++;
- }
-
- return Status::success();
-}
-} // namespace osquery
+++ /dev/null
-/**
- * Copyright (c) 2014-present, Facebook, Inc.
- * All rights reserved.
- *
- * This source code is licensed in accordance with the terms specified in
- * the LICENSE file found in the root directory of this source tree.
- */
-
-#include <osquery/database.h>
-#include <osquery/flags.h>
-#include <osquery/registry.h>
-#include <osquery/system.h>
-
-#include <osquery/utils/json/json.h>
-
-#include <gtest/gtest.h>
-
-#include <limits>
-
-namespace rj = rapidjson;
-
-namespace osquery {
-
-DECLARE_bool(disable_database);
-
-class DatabaseTests : public testing::Test {
- public:
- void SetUp() override {
- Initializer::platformSetup();
- registryAndPluginInit();
-
- // Force registry to use ephemeral database plugin
- FLAGS_disable_database = true;
- DatabasePlugin::setAllowOpen(true);
- DatabasePlugin::initPlugin();
- }
-};
-
-TEST_F(DatabaseTests, test_set_value_str) {
- auto s = setDatabaseValue(kLogs, "str", "{}");
- EXPECT_TRUE(s.ok());
-}
-
-TEST_F(DatabaseTests, test_set_value_int) {
- auto s = setDatabaseValue(kLogs, "int", -1);
- EXPECT_TRUE(s.ok());
-}
-
-TEST_F(DatabaseTests, test_set_str_batch) {
- DatabaseStringValueList batch = {
- {"str1", "{a}"}, {"str2", "{b}"}, {"str3", "{c}"}};
-
- auto s = setDatabaseBatch(kLogs, batch);
- EXPECT_TRUE(s.ok());
-}
-
-TEST_F(DatabaseTests, test_set_value_mix1) {
- auto s = setDatabaseValue(kLogs, "intstr", -1);
- EXPECT_TRUE(s.ok());
-
- s = setDatabaseValue(kLogs, "intstr", "{}");
- EXPECT_TRUE(s.ok());
-}
-
-TEST_F(DatabaseTests, test_set_value_mix2) {
- auto s = setDatabaseValue(kLogs, "strint", "{}");
- EXPECT_TRUE(s.ok());
-
- s = setDatabaseValue(kLogs, "strint", -1);
- EXPECT_TRUE(s.ok());
-}
-
-TEST_F(DatabaseTests, test_get_value_does_not_exist) {
- // Unknown keys return failed, but will return empty data.
- std::string value;
- auto s = getDatabaseValue(kLogs, "does_not_exist", value);
- EXPECT_FALSE(s.ok());
- EXPECT_TRUE(value.empty());
-}
-
-TEST_F(DatabaseTests, test_get_value_str) {
- std::string expected;
- for (unsigned char i = std::numeric_limits<unsigned char>::min();
- i < std::numeric_limits<unsigned char>::max();
- i++) {
- if (std::isprint(i)) {
- expected += i;
- }
- }
-
- setDatabaseValue(kLogs, "str", expected);
-
- std::string value;
- auto s = getDatabaseValue(kLogs, "str", value);
-
- EXPECT_TRUE(s.ok());
- EXPECT_EQ(value, expected);
-}
-
-TEST_F(DatabaseTests, test_get_value_int) {
- int expected = std::numeric_limits<int>::min();
- setDatabaseValue(kLogs, "int", expected);
-
- int value = 0;
- auto s = getDatabaseValue(kLogs, "int", value);
-
- EXPECT_TRUE(s.ok());
- EXPECT_EQ(value, expected);
-}
-
-TEST_F(DatabaseTests, test_get_value_mix1) {
- int expected = std::numeric_limits<int>::max();
- setDatabaseValue(kLogs, "strint", "{}");
- setDatabaseValue(kLogs, "strint", expected);
-
- int value;
- auto s = getDatabaseValue(kLogs, "strint", value);
-
- EXPECT_TRUE(s.ok());
- EXPECT_EQ(value, expected);
-}
-
-TEST_F(DatabaseTests, test_get_str_batch) {
- DatabaseStringValueList batch = {
- {"str1", "{a}"}, {"str2", "{b}"}, {"str3", "{c}"}};
- auto s = setDatabaseBatch(kLogs, batch);
- EXPECT_TRUE(s.ok());
-
- for (const auto& p : batch) {
- const auto& key = p.first;
- const auto& expected_value = p.second;
-
- std::string value;
- s = getDatabaseValue(kLogs, key, value);
- EXPECT_TRUE(s.ok());
-
- EXPECT_EQ(value, expected_value);
- }
-}
-
-TEST_F(DatabaseTests, test_get_value_mix2) {
- std::string expected = "{}";
- setDatabaseValue(kLogs, "intstr", -1);
- setDatabaseValue(kLogs, "intstr", expected);
-
- std::string value;
- auto s = getDatabaseValue(kLogs, "intstr", value);
-
- EXPECT_TRUE(s.ok());
- EXPECT_EQ(value, expected);
-}
-
-TEST_F(DatabaseTests, test_scan_values) {
- setDatabaseValue(kLogs, "1", "0");
- setDatabaseValue(kLogs, "2", 0);
- setDatabaseValue(kLogs, "3", "0");
-
- std::vector<std::string> keys;
- auto s = scanDatabaseKeys(kLogs, keys);
- EXPECT_TRUE(s.ok());
- EXPECT_GT(keys.size(), 2U);
-
- keys.clear();
- s = scanDatabaseKeys(kLogs, keys, 3);
- EXPECT_TRUE(s.ok());
- EXPECT_EQ(keys.size(), 3U);
-}
-
-TEST_F(DatabaseTests, test_delete_values_str) {
- setDatabaseValue(kLogs, "k", "0");
-
- std::string value;
- getDatabaseValue(kLogs, "k", value);
- EXPECT_FALSE(value.empty());
-
- auto s = deleteDatabaseValue(kLogs, "k");
- EXPECT_TRUE(s.ok());
-
- // Make sure the key has been deleted.
- value.clear();
- s = getDatabaseValue(kLogs, "k", value);
- EXPECT_FALSE(s.ok());
- EXPECT_TRUE(value.empty());
-}
-
-TEST_F(DatabaseTests, test_delete_values_int) {
- int expected = 0;
- setDatabaseValue(kLogs, "k", expected);
-
- int value;
- getDatabaseValue(kLogs, "k", value);
- EXPECT_EQ(value, expected);
-
- auto s = deleteDatabaseValue(kLogs, "k");
- EXPECT_TRUE(s.ok());
-
- // Make sure the key has been deleted.
- value = -5;
- s = getDatabaseValue(kLogs, "k", value);
- EXPECT_FALSE(s.ok());
- EXPECT_EQ(value, -5);
-}
-
-TEST_F(DatabaseTests, test_ptree_upgrade_to_rj_empty_v0v1) {
- auto empty_results{"{}"};
- auto status = setDatabaseValue(kQueries, "old_empty_results", empty_results);
- EXPECT_TRUE(status.ok());
-
- // Stage our database to be pre-upgrade to ensure the logic runs
- status = setDatabaseValue(kPersistentSettings, kDbVersionKey, "0");
- EXPECT_TRUE(status.ok());
-
- status = upgradeDatabase(1);
- EXPECT_TRUE(status.ok());
-
- std::string new_empty_list;
- status = getDatabaseValue(kQueries, "old_empty_results", new_empty_list);
- EXPECT_TRUE(status.ok());
-
- rj::Document empty_list;
- EXPECT_FALSE(empty_list.Parse(new_empty_list.c_str()).HasParseError());
- EXPECT_TRUE(empty_list.IsArray());
-
- // Expect our DB upgrade logic to have been set
- std::string db_results_version{""};
- getDatabaseValue(kPersistentSettings, kDbVersionKey, db_results_version);
- EXPECT_EQ(db_results_version, "1");
-}
-
-TEST_F(DatabaseTests, test_ptree_upgrade_to_rj_results_v0v1) {
- auto bad_json =
- "{\"\":{\"disabled\":\"0\",\"network_name\":\"BTWifi-Starbucks\"},\"\":{"
- "\"disabled\":\"0\",\"network_name\":\"Lobo-Guest\"},\"\":{\"disabled\":"
- "\"0\",\"network_name\":\"GoogleGuest\"}}";
- auto status = setDatabaseValue(kQueries, "bad_wifi_json", bad_json);
- EXPECT_TRUE(status.ok());
-
- // Add an integer value to ensure we don't munge non-json objects
- status = setDatabaseValue(kQueries, "bad_wifi_jsonepoch", "1521583712");
- EXPECT_TRUE(status.ok());
-
- // Stage our database to be pre-upgrade to ensure the logic runs
- status = setDatabaseValue(kPersistentSettings, kDbVersionKey, "0");
- EXPECT_TRUE(status.ok());
-
- rj::Document bad_doc;
-
- // Potential bug with RJ, in that parsing should fail with empty keys
- // EXPECT_TRUE(bad_doc.Parse(bad_json).HasParseError());
- EXPECT_FALSE(bad_doc.IsArray());
-
- status = upgradeDatabase(1);
- EXPECT_TRUE(status.ok());
-
- std::string good_json;
- status = getDatabaseValue(kQueries, "bad_wifi_json", good_json);
- EXPECT_TRUE(status.ok());
-
- rj::Document clean_doc;
- EXPECT_FALSE(clean_doc.Parse(good_json.c_str()).HasParseError());
- EXPECT_TRUE(clean_doc.IsArray());
- EXPECT_EQ(clean_doc.Size(), 3U);
-
- // Ensure our non-json thing was not destroyed
- std::string query_epoch{""};
- status = getDatabaseValue(kQueries, "bad_wifi_jsonepoch", query_epoch);
- auto ulepoch = std::stoull(query_epoch);
- EXPECT_EQ(ulepoch, 1521583712U);
-
- // Expect our DB upgrade logic to have been set
- std::string db_results_version{""};
- getDatabaseValue(kPersistentSettings, "results_version", db_results_version);
- EXPECT_EQ(db_results_version, "1");
-}
-
-TEST_F(DatabaseTests, test_migration_v1v2) {
- /* Testing migration from 1 to 2 */
- Status status = setDatabaseValue(kPersistentSettings, kDbVersionKey, "1");
- ASSERT_TRUE(status.ok());
-
- status = setDatabaseValue(
- kEvents, "data.audit.process_events.0123456789", "event_data");
- ASSERT_TRUE(status.ok());
-
- status = upgradeDatabase(2);
- ASSERT_TRUE(status.ok());
-
- std::string value;
- status = getDatabaseValue(kPersistentSettings, kDbVersionKey, value);
- EXPECT_EQ(value, "2");
-
- status =
- getDatabaseValue(kEvents, "data.audit.process_events.0123456789", value);
- EXPECT_FALSE(status.ok());
-
- status = getDatabaseValue(
- kEvents, "data.auditeventpublisher.process_events.0123456789", value);
- EXPECT_TRUE(status.ok());
- EXPECT_EQ(value, "event_data");
-}
-
-} // namespace osquery
+++ /dev/null
-/**
- * Copyright (c) 2014-present, Facebook, Inc.
- * All rights reserved.
- *
- * This source code is licensed in accordance with the terms specified in
- * the LICENSE file found in the root directory of this source tree.
- */
-
-#include <osquery/database.h>
-
-#include <osquery/core/sql/diff_results.h>
-#include <osquery/core/sql/query_data.h>
-#include <osquery/query.h>
-#include <osquery/sql/tests/sql_test_utils.h>
-
-#include <gtest/gtest.h>
-
-#include <string>
-
-namespace osquery {
-
-class ResultsTests : public testing::Test {};
-
-TEST_F(ResultsTests, test_simple_diff) {
- QueryDataSet os;
- QueryDataTyped o;
- QueryDataTyped n;
-
- RowTyped r1;
- r1["foo"] = "bar";
- n.push_back(r1);
-
- auto results = diff(os, n);
- EXPECT_EQ(results.added, n);
- EXPECT_EQ(results.removed, o);
-}
-
-TEST_F(ResultsTests, test_serialize_row) {
- auto results = getSerializedRow();
- auto doc = JSON::newObject();
- auto s = serializeRow(results.second, doc, doc.doc(), true);
- EXPECT_TRUE(s.ok());
- EXPECT_EQ(s.toString(), "OK");
- EXPECT_EQ(doc.doc()["meaning_of_life"], "meaning_of_life_value");
- EXPECT_EQ(doc.doc()["alphabetical"], "alphabetical_value");
-}
-
-TEST_F(ResultsTests, test_deserialize_row_json) {
- auto results = getSerializedRow();
- std::string input;
- serializeRowJSON(results.second, input, true);
-
- // Pull the serialized JSON back into a Row output container.
- RowTyped output;
- auto s = deserializeRowJSON(input, output);
- EXPECT_TRUE(s.ok());
- // The output container should match the input row.
- EXPECT_EQ(output, results.second);
-}
-
-TEST_F(ResultsTests, test_serialize_query_data) {
- auto results = getSerializedQueryData();
- auto doc = JSON::newArray();
- auto s = serializeQueryData(results.second, doc, doc.doc(), true);
- EXPECT_TRUE(s.ok());
- EXPECT_EQ(s.toString(), "OK");
- EXPECT_EQ(results.first.doc(), doc.doc());
-}
-
-TEST_F(ResultsTests, test_serialize_query_data_json) {
- auto results = getSerializedQueryDataJSON();
- std::string json;
- auto s = serializeQueryDataJSON(results.second, json, true);
- EXPECT_TRUE(s.ok());
- EXPECT_EQ(s.toString(), "OK");
- EXPECT_EQ(results.first, json);
-}
-
-TEST_F(ResultsTests, test_deserialize_query_data_json) {
- auto results = getSerializedQueryDataJSON();
- QueryDataSet resultSet =
- QueryDataSet(results.second.begin(), results.second.end());
-
- // Pull the serialized JSON back into a QueryData output container.
- QueryDataSet output;
- auto s = deserializeQueryDataJSON(results.first, output);
- EXPECT_TRUE(s.ok());
- // The output container should match the input query data.
- EXPECT_EQ(output, resultSet);
-}
-
-TEST_F(ResultsTests, test_serialize_diff_results) {
- auto results = getSerializedDiffResults();
- auto doc = JSON::newObject();
- auto s = serializeDiffResults(results.second, doc, doc.doc(), true);
- EXPECT_TRUE(s.ok());
- EXPECT_EQ(s.toString(), "OK");
- EXPECT_EQ(results.first.doc(), doc.doc());
-}
-
-TEST_F(ResultsTests, test_serialize_diff_results_json) {
- auto results = getSerializedDiffResultsJSON();
- std::string json;
- auto s = serializeDiffResultsJSON(results.second, json, true);
- EXPECT_TRUE(s.ok());
- EXPECT_EQ(s.toString(), "OK");
- EXPECT_EQ(results.first, json);
-}
-
-TEST_F(ResultsTests, test_serialize_query_log_item) {
- auto results = getSerializedQueryLogItem();
- auto doc = JSON::newObject();
- auto s = serializeQueryLogItem(results.second, doc);
- EXPECT_TRUE(s.ok());
- EXPECT_EQ(s.toString(), "OK");
- EXPECT_EQ(results.first.doc(), doc.doc());
-}
-
-TEST_F(ResultsTests, test_serialize_query_log_item_json) {
- auto results = getSerializedQueryLogItemJSON();
- std::string json;
- auto s = serializeQueryLogItemJSON(results.second, json);
- EXPECT_TRUE(s.ok());
- EXPECT_EQ(s.toString(), "OK");
- EXPECT_EQ(results.first, json);
-}
-
-TEST_F(ResultsTests, test_adding_duplicate_rows_to_query_data) {
- RowTyped r1, r2, r3;
- r1["foo"] = "bar";
- r1["baz"] = "boo";
-
- r2["foo"] = "baz";
- r2["baz"] = "bop";
-
- r3["foo"] = "baz";
- r3["baz"] = "bop";
-
- QueryDataTyped q;
- bool s;
-
- s = addUniqueRowToQueryData(q, r1);
- EXPECT_TRUE(s);
- EXPECT_EQ(q.size(), 1U);
-
- s = addUniqueRowToQueryData(q, r2);
- EXPECT_TRUE(s);
- EXPECT_EQ(q.size(), 2U);
-
- s = addUniqueRowToQueryData(q, r3);
- EXPECT_FALSE(s);
- EXPECT_EQ(q.size(), 2U);
-}
-}
#include <boost/algorithm/string/predicate.hpp>
-#include <osquery/database.h>
#include <osquery/devtools/devtools.h>
#include <osquery/filesystem/filesystem.h>
#include <osquery/flags.h>
DECLARE_string(nullvalue);
DECLARE_string(logger_plugin);
DECLARE_string(logger_path);
-DECLARE_string(database_path);
} // namespace osquery
static char zHelp[] =
fprintf(p->out, "\n");
}
- auto database = osquery::RegistryFactory::get().getActive("database");
- fprintf(p->out, "%13.13s: %s", "Database", database.c_str());
- if (database == "rocksdb") {
- fprintf(p->out, " (%s)\n", osquery::FLAGS_database_path.c_str());
- } else {
- fprintf(p->out, "\n");
- }
-
fprintf(p->out, "\nShell settings:\n");
fprintf(p->out, "%13.13s: %s\n", "echo", p->echoOn != 0 ? "on" : "off");
fprintf(
+++ /dev/null
-/**
- * Copyright (c) 2014-present, Facebook, Inc.
- * All rights reserved.
- *
- * This source code is licensed in accordance with the terms specified in
- * the LICENSE file found in the root directory of this source tree.
- */
-
-#pragma once
-
-#include <atomic>
-#include <string>
-#include <vector>
-
-#include <osquery/plugins/plugin.h>
-
-namespace osquery {
-// A list of key/str pairs; used for write batching with setDatabaseBatch
-using DatabaseStringValueList =
- std::vector<std::pair<std::string, std::string>>;
-
-class Status;
-/**
- * @brief A list of supported backing storage categories: called domains.
- *
- * RocksDB has a concept of "column families" which are kind of like tables
- * in other databases. kDomains is populated with a list of all column
- * families. If a string exists in kDomains, it's a column family in the
- * database.
- *
- * For SQLite-backed storage these are tables using a keyed index.
- */
-extern const std::vector<std::string> kDomains;
-
-/**
- * @brief A backing storage domain name, used for key/value based storage.
- *
- * There are certain "cached" variables such as a node-unique UUID or negotiated
- * 'node_key' following enrollment. If a value or setting must persist between
- * osqueryi or osqueryd runs it should be stored using the kPersistentSetting%s
- * domain.
- */
-extern const std::string kPersistentSettings;
-
-/// The "domain" where the results of scheduled queries are stored.
-extern const std::string kQueries;
-
-/// The "domain" where event results are stored, queued for querytime retrieval.
-extern const std::string kEvents;
-
-/// The "domain" where the results of carve queries are stored.
-extern const std::string kCarves;
-
-/// The key for the DB version
-extern const std::string kDbVersionKey;
-
-/// The running version of our database schema
-const int kDbCurrentVersion = 2;
-
-/**
- * @brief The "domain" where buffered log results are stored.
- *
- * Logger plugins may shuttle logs to a remote endpoint or API call
- * asynchronously. The backing store can be used to buffer results and status
- * logs until the logger plugin-specific thread decided to flush.
- */
-extern const std::string kLogs;
-
-/**
- * @brief An osquery backing storage (database) type that persists executions.
- *
- * The osquery tools need a high-performance storage and indexing mechanism for
- * storing intermediate results from EventPublisher%s, persisting one-time
- * generated values, and performing non-memory backed differentials.
- *
- * Practically, osquery is built around RocksDB's performance guarantees and
- * all of the internal APIs expect RocksDB's indexing and read performance.
- * However, access to this representation of a backing-store is still abstracted
- * to removing RocksDB as a dependency for the osquery SDK.
- */
-class DatabasePlugin : public Plugin {
- public:
- /**
- * @brief Perform a domain and key lookup from the backing store.
- *
- * Database value access indexing is abstracted into domains and keys.
- * Both are string values but exist separately for simple indexing without
- * API-enforcing tokenization. In some cases we do add a component-specific
- * tokeninzation to keys.
- *
- * @param domain A string value representing abstract storage indexing.
- * @param key A string value representing the lookup/retrieval key.
- * @param value The output parameter, left empty if the key does not exist.
- * @return Failure if the data could not be accessed. It is up to the plugin
- * to determine if a missing key means a non-success status.
- */
- virtual Status get(const std::string& domain,
- const std::string& key,
- std::string& value) const = 0;
-
- virtual Status get(const std::string& domain,
- const std::string& key,
- int& value) const = 0;
-
- /**
- * @brief Store a string-represented value using a domain and key index.
- *
- * See DatabasePlugin::get for discussion around domain and key use.
- *
- * @param domain A string value representing abstract storage indexing.
- * @param key A string value representing the lookup/retrieval key.
- * @param value A string value representing the data.
- * @return Failure if the data could not be stored. It is up to the plugin
- * to determine if a conflict/overwrite should return different status text.
- */
- virtual Status put(const std::string& domain,
- const std::string& key,
- const std::string& value) = 0;
-
- virtual Status put(const std::string& domain,
- const std::string& key,
- int value) = 0;
-
- virtual Status putBatch(const std::string& domain,
- const DatabaseStringValueList& data) = 0;
-
- virtual void dumpDatabase() const = 0;
-
- /// Data removal method.
- virtual Status remove(const std::string& domain, const std::string& k) = 0;
-
- /// Data removal with range bounds.
- virtual Status removeRange(const std::string& domain,
- const std::string& low,
- const std::string& high) = 0;
-
- virtual Status scan(const std::string& domain,
- std::vector<std::string>& results,
- const std::string& prefix,
- size_t max) const;
-
- /**
- * @brief Shutdown the database and release initialization resources.
- *
- * Assume that a plugin may override #tearDown and choose to close resources
- * when the registry is stopping. Most plugins will implement a mutex around
- * initialization and destruction and assume #setUp and #tearDown will
- * dictate the flow in most situations.
- */
- ~DatabasePlugin() override = default;
-
- /**
- * @brief Support the registry calling API for extensions.
- *
- * The database plugin "fast-calls" directly to local plugins.
- * Extensions cannot use an extension-local backing store so their requests
- * are routed like all other plugins.
- */
- Status call(const PluginRequest& request, PluginResponse& response) override;
-
- public:
- /// Database-specific workflow: reset the originally request instance.
- Status reset();
-
- /// Database-specific workflow: perform an initialize, then reset.
- bool checkDB();
-
- /// Require all DBHandle accesses to open a read and write handle.
- static void setRequireWrite(bool rw) {
- kDBRequireWrite = rw;
- }
-
- /// Allow DBHandle creations.
- static void setAllowOpen(bool ao) {
- kDBAllowOpen = ao;
- }
-
- public:
- /// Control availability of the RocksDB handle (default false).
- static std::atomic<bool> kDBAllowOpen;
-
- /// The database must be opened in a R/W mode (default false).
- static std::atomic<bool> kDBRequireWrite;
-
- /// An internal mutex around database sanity checking.
- static std::atomic<bool> kDBChecking;
-
- /// An internal status protecting database access.
- static std::atomic<bool> kDBInitialized;
-
- public:
- /**
- * @brief Allow the initializer to check the active database plugin.
- *
- * Unlink the initializer's Initializer::initActivePlugin helper method, the
- * database plugin should always be within the core. There is no need to
- * discover the active plugin via the registry or extensions API.
- *
- * The database should setUp in preparation for accesses.
- */
- static Status initPlugin();
-
- /// Allow shutdown before exit.
- static void shutdown();
-
- protected:
- /// The database was opened in a ReadOnly mode.
- bool read_only_{false};
-
- /// Original requested path on disk.
- std::string path_;
-};
-
-/**
- * @brief Lookup a value from the active osquery DatabasePlugin storage.
- *
- * See DatabasePlugin::get for discussion around domain and key use.
- * Extensions, components, plugins, and core code should use getDatabaseValue
- * as a wrapper around the current tool's choice of a backing storage plugin.
- *
- * @param domain A string value representing abstract storage indexing.
- * @param key A string value representing the lookup/retrieval key.
- * @param value The output parameter, left empty if the key does not exist.
- * @return Storage operation status.
- */
-Status getDatabaseValue(const std::string& domain,
- const std::string& key,
- std::string& value);
-
-Status getDatabaseValue(const std::string& domain,
- const std::string& key,
- int& value);
-
-/**
- * @brief Set or put a value into the active osquery DatabasePlugin storage.
- *
- * See DatabasePlugin::get for discussion around domain and key use.
- * Extensions, components, plugins, and core code should use setDatabaseValue
- * as a wrapper around the current tool's choice of a backing storage plugin.
- *
- * @param domain A string value representing abstract storage indexing.
- * @param key A string value representing the lookup/retrieval key.
- * @param value A string value representing the data.
- * @return Storage operation status.
- */
-Status setDatabaseValue(const std::string& domain,
- const std::string& key,
- const std::string& value);
-
-Status setDatabaseValue(const std::string& domain,
- const std::string& key,
- int value);
-
-Status setDatabaseBatch(const std::string& domain,
- const DatabaseStringValueList& data);
-
-/// Remove a domain/key identified value from backing-store.
-Status deleteDatabaseValue(const std::string& domain, const std::string& key);
-
-/// Remove a range of keys in domain.
-Status deleteDatabaseRange(const std::string& domain,
- const std::string& low,
- const std::string& high);
-
-/// Get a list of keys for a given domain.
-Status scanDatabaseKeys(const std::string& domain,
- std::vector<std::string>& keys,
- size_t max = 0);
-
-/// Get a list of keys for a given domain.
-Status scanDatabaseKeys(const std::string& domain,
- std::vector<std::string>& keys,
- const std::string& prefix,
- size_t max = 0);
-
-/// Allow callers to reload or reset the database plugin.
-void resetDatabase();
-
-/// Allow callers to scan each column family and print each value.
-void dumpDatabase();
-
-Status ptreeToRapidJSON(const std::string& in, std::string& out);
-
-/**
- * @brief Upgrades the legacy database json format from ptree to RapidJSON
- *
- * This helper function was required as Boost property trees contain json
- * which leverages empty strings for keys in json arrays. This is incompatible
- * with rapidjson, thus we require a converter function to upgrade any cached
- * results in the database.
- *
- * @return Success status of upgrading the database
- */
-Status upgradeDatabase(int to_version = kDbCurrentVersion);
-} // namespace osquery
+++ /dev/null
-../../../../plugins/database/ephemeral.h
\ No newline at end of file
+++ /dev/null
-../../../../plugins/database/rocksdb.h
\ No newline at end of file
+++ /dev/null
-../../../../plugins/database/sqlite.h
\ No newline at end of file
#include <boost/noncopyable.hpp>
#include <osquery/data_logger.h>
-#include <osquery/database.h>
#include <osquery/filesystem/filesystem.h>
#include <osquery/flags.h>
#include <osquery/plugins/logger.h>
}
void relayStatusLogs(bool async) {
- if (FLAGS_disable_logging || !DatabasePlugin::kDBInitialized) {
+ if (FLAGS_disable_logging) {
// The logger plugins may not be setUp if logging is disabled.
// If the database is not setUp, or is in a reset, status logs continue
// to buffer.
#include <gtest/gtest.h>
#include <osquery/data_logger.h>
-#include <osquery/database.h>
#include <osquery/filesystem/filesystem.h>
#include <osquery/plugins/logger.h>
#include <osquery/registry_factory.h>
namespace osquery {
-DECLARE_bool(disable_database);
DECLARE_int32(logger_min_status);
DECLARE_int32(logger_min_stderr);
DECLARE_bool(logger_secondary_status_only);
void SetUp() override {
Initializer::platformSetup();
registryAndPluginInit();
- FLAGS_disable_database = true;
- DatabasePlugin::setAllowOpen(true);
- DatabasePlugin::initPlugin();
// Backup the logging status, then disable.
FLAGS_disable_logging = false;
#include <boost/algorithm/string/predicate.hpp>
#include <osquery/core.h>
-#include <osquery/database.h>
#include <osquery/devtools/devtools.h>
#include <osquery/filesystem/fileops.h>
#include <osquery/flags.h>
query = std::string(argv[1]);
}
- // Perform some duplication from Initializer with respect to database setup.
- osquery::DatabasePlugin::setAllowOpen(true);
- osquery::RegistryFactory::get().setActive("database", "ephemeral");
-
auto dbc = osquery::SQLiteDBManager::get();
for (size_t i = 0; i < static_cast<size_t>(osquery::FLAGS_profile); ++i) {
osquery::QueryData results;
# the LICENSE file found in the root directory of this source tree.
ADD_OSQUERY_LIBRARY(osquery_pluing_logger logger/filesystem_logger.cpp)
-
-ADD_OSQUERY_LIBRARY(osquery_plugin_db database/ephemeral.cpp
- database/sqlite.cpp)
+++ /dev/null
-/**
- * Copyright (c) 2014-present, Facebook, Inc.
- * All rights reserved.
- *
- * This source code is licensed in accordance with the terms specified in
- * the LICENSE file found in the root directory of this source tree.
- */
-
-#include <iostream>
-
-#include "osquery/plugins/database/ephemeral.h"
-
-namespace osquery {
-
-template <typename T>
-Status EphemeralDatabasePlugin::getAny(const std::string& domain,
- const std::string& key,
- T& value) const {
- auto domainIterator = db_.find(domain);
- if (domainIterator == db_.end()) {
- return Status(1, "Domain " + domain + " does not exist");
- }
-
- auto keyIterator = domainIterator->second.find(key);
- if (keyIterator == domainIterator->second.end()) {
- return Status(1, "Key " + key + " in domain " + domain + " does not exist");
- }
-
- try {
- value = boost::get<T>(keyIterator->second);
- } catch (const boost::bad_get& e) {
- LOG(WARNING) << "Type error getting string value for (domain,key) : ("
- << key << "," << domain << ") " << e.what();
- return Status(
- 1, "EphemeralDatabasePlugin::get was requested incorrect type(string)");
- }
- return Status(0);
-}
-
-Status EphemeralDatabasePlugin::get(const std::string& domain,
- const std::string& key,
- std::string& value) const {
- return this->getAny(domain, key, value);
-}
-Status EphemeralDatabasePlugin::get(const std::string& domain,
- const std::string& key,
- int& value) const {
- return this->getAny(domain, key, value);
-}
-
-void EphemeralDatabasePlugin::setValue(const std::string& domain,
- const std::string& key,
- const std::string& value) {
- db_[domain][key] = value;
-}
-
-void EphemeralDatabasePlugin::setValue(const std::string& domain,
- const std::string& key,
- int value) {
- db_[domain][key] = value;
-}
-
-Status EphemeralDatabasePlugin::put(const std::string& domain,
- const std::string& key,
- const std::string& value) {
- setValue(domain, key, value);
- return Status(0);
-}
-
-Status EphemeralDatabasePlugin::put(const std::string& domain,
- const std::string& key,
- int value) {
- setValue(domain, key, value);
- return Status(0);
-}
-
-Status EphemeralDatabasePlugin::putBatch(const std::string& domain,
- const DatabaseStringValueList& data) {
- for (const auto& p : data) {
- const auto& key = p.first;
- const auto& value = p.second;
-
- setValue(domain, key, value);
- }
-
- return Status::success();
-}
-
-void EphemeralDatabasePlugin::dumpDatabase() const {
- for (const auto& domainValue : db_) {
- const auto& domain = domainValue.first;
- for (const auto& keyValue : domainValue.second) {
- const auto& key = keyValue.first;
- const auto& value = keyValue.second;
- std::cout << domain << "[" << key << "]: " << value << std::endl;
- }
- }
-}
-
-Status EphemeralDatabasePlugin::remove(const std::string& domain,
- const std::string& k) {
- db_[domain].erase(k);
- return Status(0);
-}
-
-Status EphemeralDatabasePlugin::removeRange(const std::string& domain,
- const std::string& low,
- const std::string& high) {
- std::vector<std::string> keys;
- for (const auto& it : db_[domain]) {
- if (it.first >= low && it.first <= high) {
- keys.push_back(it.first);
- }
- }
-
- for (const auto& key : keys) {
- db_[domain].erase(key);
- }
- return Status(0);
-}
-
-Status EphemeralDatabasePlugin::scan(const std::string& domain,
- std::vector<std::string>& results,
- const std::string& prefix,
- size_t max) const {
- if (db_.count(domain) == 0) {
- return Status(0);
- }
-
- for (const auto& key : db_.at(domain)) {
- if (!prefix.empty() &&
- !(std::mismatch(prefix.begin(), prefix.end(), key.first.begin())
- .first == prefix.end())) {
- continue;
- }
- results.push_back(key.first);
- if (max > 0 && results.size() >= max) {
- break;
- }
- }
- return Status(0);
-}
-} // namespace osquery
+++ /dev/null
-/**
- * Copyright (c) 2014-present, Facebook, Inc.
- * All rights reserved.
- *
- * This source code is licensed in accordance with the terms specified in
- * the LICENSE file found in the root directory of this source tree.
- */
-
-#include <osquery/database.h>
-#include <osquery/flags.h>
-#include <osquery/logger.h>
-#include <osquery/registry_factory.h>
-
-#include <boost/variant.hpp>
-
-namespace osquery {
-
-DECLARE_string(database_path);
-
-class EphemeralDatabasePlugin : public DatabasePlugin {
- using DBType =
- std::map<std::string,
- std::map<std::string, boost::variant<int, std::string>>>;
- template <typename T>
- Status getAny(const std::string& domain,
- const std::string& key,
- T& value) const;
-
- private:
- void setValue(const std::string& domain,
- const std::string& key,
- const std::string& value);
-
- void setValue(const std::string& domain, const std::string& key, int value);
-
- public:
- /// Data retrieval method.
-
- Status get(const std::string& domain,
- const std::string& key,
- std::string& value) const override;
- Status get(const std::string& domain,
- const std::string& key,
- int& value) const override;
- /// Data storage method.
- Status put(const std::string& domain,
- const std::string& key,
- const std::string& value) override;
- Status put(const std::string& domain,
- const std::string& key,
- int value) override;
-
- Status putBatch(const std::string& domain,
- const DatabaseStringValueList& data) override;
-
- void dumpDatabase() const override;
-
- /// Data removal method.
- Status remove(const std::string& domain, const std::string& k) override;
-
- Status removeRange(const std::string& domain,
- const std::string& low,
- const std::string& high) override;
-
- /// Key/index lookup method.
- Status scan(const std::string& domain,
- std::vector<std::string>& results,
- const std::string& prefix,
- size_t max) const override;
-
- public:
- /// Database workflow: open and setup.
- Status setUp() override {
- DBType().swap(db_);
- return Status(0);
- }
-
- private:
- DBType db_;
-};
-
-/// Backing-storage provider for osquery internal/core.
-REGISTER_INTERNAL(EphemeralDatabasePlugin, "database", "ephemeral");
-
-} // namespace osquery
+++ /dev/null
-/**
- * Copyright (c) 2014-present, Facebook, Inc.
- * All rights reserved.
- *
- * This source code is licensed in accordance with the terms specified in
- * the LICENSE file found in the root directory of this source tree.
- */
-
-#include <sstream>
-
-#include <sqlite3.h>
-#include <sys/stat.h>
-
-#include <osquery/filesystem/fileops.h>
-#include <osquery/filesystem/filesystem.h>
-#include <osquery/logger.h>
-#include <osquery/query.h>
-#include <osquery/utils/conversions/tryto.h>
-#include <osquery/plugins/database/sqlite.h>
-
-namespace osquery {
-
-const std::map<std::string, std::string> kDBSettings = {
- {"synchronous", "OFF"},
- {"count_changes", "OFF"},
- {"default_temp_store", "2"},
- {"auto_vacuum", "FULL"},
- {"journal_mode", "OFF"},
- {"cache_size", "1000"},
- {"page_count", "1000"},
-};
-
-Status SQLiteDatabasePlugin::setUp() {
- if (!DatabasePlugin::kDBAllowOpen) {
- LOG(WARNING) << RLOG(1629) << "Not allowed to set up database plugin";
- }
-
- // Consume the current settings.
- // A configuration update may change them, but that does not affect state.
- path_ = FLAGS_database_path;
-
- if (pathExists(path_).ok() && !isReadable(path_).ok()) {
- return Status(1, "Cannot read database path: " + path_);
- }
-
- if (!DatabasePlugin::kDBChecking) {
- VLOG(1) << "Opening database handle: " << path_;
- }
-
- // Tests may trash calls to setUp, make sure subsequent calls do not leak.
- close();
-
- // Open the SQLite backing storage at path_
- auto result = sqlite3_open_v2(
- path_.c_str(),
- &db_,
- (SQLITE_OPEN_FULLMUTEX | SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE),
- nullptr);
-
- if (result != SQLITE_OK || db_ == nullptr) {
- if (DatabasePlugin::kDBRequireWrite) {
- close();
- // A failed open in R/W mode is a runtime error.
- return Status(1, "Cannot open database: " + std::to_string(result));
- }
-
- if (!DatabasePlugin::kDBChecking) {
- VLOG(1) << "Opening database failed: Continuing with read-only support";
- }
-
- read_only_ = true;
- }
-
- if (!read_only_) {
- for (const auto& domain : kDomains) {
- std::string q = "create table if not exists " + domain +
- " (key TEXT PRIMARY KEY, value TEXT);";
- result = sqlite3_exec(db_, q.c_str(), nullptr, nullptr, nullptr);
- if (result != SQLITE_OK) {
- close();
- return Status(1, "Cannot create domain: " + domain);
- }
- }
-
- std::string settings;
- for (const auto& setting : kDBSettings) {
- settings += "PRAGMA " + setting.first + "=" + setting.second + "; ";
- }
- sqlite3_exec(db_, settings.c_str(), nullptr, nullptr, nullptr);
- }
-
- // RocksDB may not create/append a directory with acceptable permissions.
- if (!read_only_ && platformSetSafeDbPerms(path_) == false) {
- close();
- return Status(1, "Cannot set permissions on database path: " + path_);
- }
- return Status(0);
-}
-
-void SQLiteDatabasePlugin::close() {
- WriteLock lock(close_mutex_);
- if (db_ != nullptr) {
- sqlite3_close(db_);
- db_ = nullptr;
- }
-}
-
-static int getData(void* argument, int argc, char* argv[], char* column[]) {
- if (argument == nullptr) {
- return SQLITE_MISUSE;
- }
-
- QueryData* qData = (QueryData*)argument;
- Row r;
- for (int i = 0; i < argc; i++) {
- if (column[i] != nullptr) {
- r[column[i]] = (argv[i] != nullptr) ? argv[i] : "";
- }
- }
- (*qData).push_back(std::move(r));
- return 0;
-}
-
-Status SQLiteDatabasePlugin::get(const std::string& domain,
- const std::string& key,
- std::string& value) const {
- QueryData results;
- char* err = nullptr;
- std::string q = "select value from " + domain + " where key = '" + key + "';";
- sqlite3_exec(db_, q.c_str(), getData, &results, &err);
- if (err != nullptr) {
- sqlite3_free(err);
- }
-
- // Only assign value if the query found a result.
- if (results.size() > 0) {
- value = std::move(results[0]["value"]);
- return Status(0);
- }
- return Status(1);
-}
-
-Status SQLiteDatabasePlugin::get(const std::string& domain,
- const std::string& key,
- int& value) const {
- std::string result;
- auto s = this->get(domain, key, result);
- if (s.ok()) {
- auto expectedValue = tryTo<int>(result);
- if (expectedValue.isError()) {
- return Status::failure("Could not deserialize str to int");
- } else {
- value = expectedValue.take();
- }
- }
- return s;
-}
-
-static void tryVacuum(sqlite3* db) {
- std::string q =
- "SELECT (sum(s1.pageno + 1 == s2.pageno) * 1.0 / count(*)) < 0.01 as v "
- " FROM "
- "(SELECT pageno FROM dbstat ORDER BY path) AS s1,"
- "(SELECT pageno FROM dbstat ORDER BY path) AS s2 WHERE "
- "s1.rowid + 1 = s2.rowid; ";
-
- QueryData results;
- sqlite3_exec(db, q.c_str(), getData, &results, nullptr);
- if (results.size() > 0 && results[0]["v"].back() == '1') {
- sqlite3_exec(db, "vacuum;", nullptr, nullptr, nullptr);
- }
-}
-
-Status SQLiteDatabasePlugin::put(const std::string& domain,
- const std::string& key,
- const std::string& value) {
- return putBatch(domain, {std::make_pair(key, value)});
-}
-
-Status SQLiteDatabasePlugin::put(const std::string& domain,
- const std::string& key,
- int value) {
- return putBatch(domain, {std::make_pair(key, std::to_string(value))});
-}
-
-Status SQLiteDatabasePlugin::putBatch(const std::string& domain,
- const DatabaseStringValueList& data) {
- if (read_only_) {
- return Status::success();
- }
-
- // Prepare the query, adding placeholders for all the rows we have in `data`
- std::stringstream buffer;
- buffer << "insert or replace into " + domain + " values ";
-
- for (auto i = 1U; i <= data.size(); i++) {
- auto index = i * 2;
- buffer << "(?" << index - 1 << ", ?" << index << ")";
-
- if (i + 1 > data.size()) {
- buffer << ";";
- } else {
- buffer << ", ";
- }
- }
-
- const auto& q = buffer.str();
-
- // Bind each value from the rows we got
- sqlite3_stmt* stmt = nullptr;
- sqlite3_prepare_v2(db_, q.c_str(), -1, &stmt, nullptr);
-
- {
- int i = 1;
-
- for (const auto& p : data) {
- const auto& key = p.first;
- const auto& value = p.second;
-
- sqlite3_bind_text(stmt, i, key.c_str(), -1, SQLITE_STATIC);
- sqlite3_bind_text(stmt, i + 1, value.c_str(), -1, SQLITE_STATIC);
-
- i += 2;
- }
- }
-
- auto rc = sqlite3_step(stmt);
- if (rc != SQLITE_DONE) {
- return Status(1);
- }
-
- sqlite3_finalize(stmt);
- if (rand() % 10 == 0) {
- tryVacuum(db_);
- }
-
- return Status::success();
-}
-
-Status SQLiteDatabasePlugin::remove(const std::string& domain,
- const std::string& key) {
- if (read_only_) {
- return Status::success();
- }
-
- sqlite3_stmt* stmt = nullptr;
- std::string q = "delete from " + domain + " where key IN (?1);";
- sqlite3_prepare_v2(db_, q.c_str(), -1, &stmt, nullptr);
-
- sqlite3_bind_text(stmt, 1, key.c_str(), -1, SQLITE_STATIC);
- auto rc = sqlite3_step(stmt);
- if (rc != SQLITE_DONE) {
- return Status(1);
- }
-
- sqlite3_finalize(stmt);
- if (rand() % 10 == 0) {
- tryVacuum(db_);
- }
- return Status(0);
-}
-
-void SQLiteDatabasePlugin::dumpDatabase() const {}
-
-Status SQLiteDatabasePlugin::removeRange(const std::string& domain,
- const std::string& low,
- const std::string& high) {
- if (read_only_) {
- return Status::success();
- }
-
- sqlite3_stmt* stmt = nullptr;
- std::string q = "delete from " + domain + " where key >= ?1 and key <= ?2;";
- sqlite3_prepare_v2(db_, q.c_str(), -1, &stmt, nullptr);
-
- sqlite3_bind_text(stmt, 1, low.c_str(), -1, SQLITE_STATIC);
- sqlite3_bind_text(stmt, 2, high.c_str(), -1, SQLITE_STATIC);
- auto rc = sqlite3_step(stmt);
- if (rc != SQLITE_DONE) {
- return Status(1);
- }
-
- sqlite3_finalize(stmt);
- if (rand() % 10 == 0) {
- tryVacuum(db_);
- }
- return Status(0);
-}
-
-Status SQLiteDatabasePlugin::scan(const std::string& domain,
- std::vector<std::string>& results,
- const std::string& prefix,
- size_t max) const {
- QueryData _results;
- char* err = nullptr;
-
- std::string q =
- "select key from " + domain + " where key LIKE '" + prefix + "%'";
- if (max > 0) {
- q += " limit " + std::to_string(max);
- }
- sqlite3_exec(db_, q.c_str(), getData, &_results, &err);
- if (err != nullptr) {
- sqlite3_free(err);
- }
-
- // Only assign value if the query found a result.
- for (auto& r : _results) {
- results.push_back(std::move(r["key"]));
- }
-
- return Status::success();
-}
-} // namespace osquery
+++ /dev/null
-/**
- * Copyright (c) 2014-present, Facebook, Inc.
- * All rights reserved.
- *
- * This source code is licensed in accordance with the terms specified in
- * the LICENSE file found in the root directory of this source tree.
- */
-
-#include <mutex>
-
-#include <sqlite3.h>
-
-#include <osquery/database.h>
-#include <osquery/filesystem/fileops.h>
-#include <osquery/flags.h>
-#include <osquery/registry_factory.h>
-
-namespace osquery {
-
-DECLARE_string(database_path);
-
-class SQLiteDatabasePlugin : public DatabasePlugin {
- public:
- /// Data retrieval method.
- Status get(const std::string& domain,
- const std::string& key,
- std::string& value) const override;
- Status get(const std::string& domain,
- const std::string& key,
- int& value) const override;
-
- /// Data storage method.
- Status put(const std::string& domain,
- const std::string& key,
- const std::string& value) override;
- Status put(const std::string& domain,
- const std::string& key,
- int value) override;
-
- Status putBatch(const std::string& domain,
- const DatabaseStringValueList& data) override;
-
- void dumpDatabase() const override;
-
- /// Data removal method.
- Status remove(const std::string& domain, const std::string& k) override;
-
- /// Data range removal method.
- Status removeRange(const std::string& domain,
- const std::string& low,
- const std::string& high) override;
-
- /// Key/index lookup method.
- Status scan(const std::string& domain,
- std::vector<std::string>& results,
- const std::string& prefix,
- size_t max) const override;
-
- public:
- /// Database workflow: open and setup.
- Status setUp() override;
-
- /// Database workflow: close and cleanup.
- void tearDown() override {
- close();
- }
-
- /// Need to tear down open resources,
- virtual ~SQLiteDatabasePlugin() {
- close();
- }
-
- private:
- void close();
-
- private:
- /// The long-lived sqlite3 database.
- sqlite3* db_{nullptr};
-
- /// Deconstruction mutex.
- Mutex close_mutex_;
-};
-
-/// Backing-storage provider for osquery internal/core.
-REGISTER_INTERNAL(SQLiteDatabasePlugin, "database", "sqlite");
-
-} // namespace osquery
+++ /dev/null
-/**
- * Copyright (c) 2014-present, Facebook, Inc.
- * All rights reserved.
- *
- * This source code is licensed in accordance with the terms specified in
- * the LICENSE file found in the root directory of this source tree.
- */
-
-#include <plugins/database/tests/utils.h>
-
-namespace osquery {
-
-class SQLiteDatabasePluginTests : public DatabasePluginTests {
- protected:
- std::string name() override {
- return "sqlite";
- }
-};
-
-// Define the default set of database plugin operation tests.
-CREATE_DATABASE_TESTS(SQLiteDatabasePluginTests);
-}
+++ /dev/null
-/**
- * Copyright (c) 2014-present, Facebook, Inc.
- * All rights reserved.
- *
- * This source code is licensed in accordance with the terms specified in
- * the LICENSE file found in the root directory of this source tree.
- */
-
-#include <future>
-
-#include <osquery/filesystem/filesystem.h>
-#include <osquery/flags.h>
-#include <osquery/registry.h>
-#include <osquery/registry_factory.h>
-#include <osquery/system.h>
-#include <osquery/utils/json/json.h>
-#include <plugins/database/tests/utils.h>
-
-#include <boost/filesystem.hpp>
-
-namespace fs = boost::filesystem;
-
-namespace osquery {
-
-DECLARE_bool(disable_database);
-DECLARE_string(database_path);
-
-class EphemeralDatabasePluginTests : public DatabasePluginTests {
- protected:
- std::string name() override {
- return "ephemeral";
- }
-};
-
-// Define the default set of database plugin operation tests.
-CREATE_DATABASE_TESTS(EphemeralDatabasePluginTests);
-
-void DatabasePluginTests::SetUp() {
- Initializer::platformSetup();
- registryAndPluginInit();
- FLAGS_disable_database = true;
- DatabasePlugin::setAllowOpen(true);
- DatabasePlugin::initPlugin();
-
- auto& rf = RegistryFactory::get();
- existing_plugin_ = rf.getActive("database");
- rf.plugin("database", existing_plugin_)->tearDown();
-
- setName(name());
- path_ = FLAGS_database_path;
- FLAGS_database_path = (
- fs::temp_directory_path() /
- fs::unique_path("osquery.database_plugin_tests.%%%%.%%%%.%%%%.%%%%.db")
- ).string();
- // removePath(path_);
-
- auto plugin = rf.plugin("database", getName());
- plugin_ = std::dynamic_pointer_cast<DatabasePlugin>(plugin);
- plugin_->reset();
-
- rf.setActive("database", getName());
-}
-
-void DatabasePluginTests::TearDown() {
- auto& rf = RegistryFactory::get();
- rf.plugin("database", name_)->tearDown();
- rf.setActive("database", existing_plugin_);
- fs::remove_all(fs::path(FLAGS_database_path));
- FLAGS_database_path = path_;
-}
-
-void DatabasePluginTests::testPluginCheck() {
- auto& rf = RegistryFactory::get();
-
- // Do not worry about multiple set-active calls.
- // For testing purposes they should be idempotent.
- EXPECT_TRUE(rf.setActive("database", getName()));
-
- // Get an instance of the database plugin and call check.
- auto plugin = rf.plugin("database", getName());
- auto db_plugin = std::dynamic_pointer_cast<DatabasePlugin>(plugin);
- EXPECT_TRUE(db_plugin->checkDB());
-
- // Testing relies on database resetting too.
- EXPECT_TRUE(db_plugin->reset());
-}
-
-auto kTestReseter = ([]() { resetDatabase(); });
-
-void DatabasePluginTests::testReset() {
- RegistryFactory::get().setActive("database", getName());
- setDatabaseValue(kLogs, "reset", "1");
- resetDatabase();
-
- if ("ephemeral" != getName()) {
- // The ephemeral plugin is special and does not persist after reset.
- std::string value;
- EXPECT_TRUE(getDatabaseValue(kLogs, "reset", value));
- EXPECT_EQ(value, "1");
- }
-}
-
-void DatabasePluginTests::testPut() {
- auto s = getPlugin()->put(kQueries, "test_put", "bar");
- EXPECT_TRUE(s.ok());
- EXPECT_EQ(s.getMessage(), "OK");
-
- s = setDatabaseValue(kQueries, "test_put", "");
- EXPECT_TRUE(s.ok());
-
- PluginRequest req = {{"action", "put"},
- {"domain", kQueries},
- {"key", "test_put"},
- {"value", "bar"}};
- s = Registry::call("database", getName(), req);
- EXPECT_TRUE(s.ok());
-
- auto reset = std::async(std::launch::async, kTestReseter);
- reset.get();
-}
-
-void DatabasePluginTests::testPutBatch() {
- DatabaseStringValueList string_batch = {
- {"test_put_str1", "test_put_str1_value"},
- {"test_put_str2", "test_put_str2_value"}};
-
- auto s = getPlugin()->putBatch(kQueries, string_batch);
- EXPECT_TRUE(s.ok());
- EXPECT_EQ(s.getMessage(), "OK");
-
- for (const auto& p : string_batch) {
- const auto& key = p.first;
- const auto& expected_value = p.second;
-
- std::string value;
- s = getDatabaseValue(kQueries, key, value);
- EXPECT_EQ(s.getMessage(), "OK");
-
- EXPECT_EQ(expected_value, value);
- }
-
- DatabaseStringValueList str_batch2 = {
- {"test_plugin_put_json_str1", "test_put_str1_value"},
- {"test_plugin_put_json_str2", "test_put_str2_value"}};
-
- auto json_object = JSON::newObject();
- for (const auto& p : str_batch2) {
- const auto& key = p.first;
- const auto& value = p.second;
-
- json_object.addRef(key, value);
- }
-
- std::string serialized_data;
- s = json_object.toString(serialized_data);
- EXPECT_TRUE(s.ok());
-
- PluginRequest request = {
- {"action", "putBatch"}, {"domain", kQueries}, {"json", serialized_data}};
-
- s = Registry::call("database", getName(), request);
- EXPECT_TRUE(s.ok());
-
- for (const auto& p : str_batch2) {
- const auto& key = p.first;
- const auto& expected_value = p.second;
-
- std::string value;
- s = getDatabaseValue(kQueries, key, value);
- EXPECT_EQ(s.getMessage(), "OK");
-
- EXPECT_EQ(expected_value, value);
- }
-
- auto reset = std::async(std::launch::async, kTestReseter);
- reset.get();
-}
-
-void DatabasePluginTests::testGet() {
- getPlugin()->put(kQueries, "test_get", "bar");
-
- std::string r;
- auto s = getPlugin()->get(kQueries, "test_get", r);
- EXPECT_TRUE(s.ok());
- EXPECT_EQ(s.getMessage(), "OK");
- EXPECT_EQ(r, "bar");
-
- auto reset = std::async(std::launch::async, kTestReseter);
- reset.get();
-}
-
-void DatabasePluginTests::testDelete() {
- getPlugin()->put(kQueries, "test_delete", "baz");
- auto s = getPlugin()->remove(kQueries, "test_delete");
- EXPECT_TRUE(s.ok());
- EXPECT_EQ(s.getMessage(), "OK");
-}
-
-void DatabasePluginTests::testDeleteRange() {
- getPlugin()->put(kQueries, "test_delete", "baz");
- getPlugin()->put(kQueries, "test1", "1");
- getPlugin()->put(kQueries, "test2", "2");
- getPlugin()->put(kQueries, "test3", "3");
- getPlugin()->put(kQueries, "test4", "4");
- auto s = getPlugin()->removeRange(kQueries, "test1", "test3");
- EXPECT_TRUE(s.ok());
- EXPECT_EQ(s.getMessage(), "OK");
-
- std::string r;
- getPlugin()->get(kQueries, "test4", r);
- EXPECT_EQ(r, "4");
- getPlugin()->get(kQueries, "test_delete", r);
- EXPECT_EQ(r, "baz");
- s = getPlugin()->get(kQueries, "test1", r);
- EXPECT_FALSE(s.ok());
- s = getPlugin()->get(kQueries, "test2", r);
- EXPECT_FALSE(s.ok());
- s = getPlugin()->get(kQueries, "test3", r);
- EXPECT_FALSE(s.ok());
-
- // Expect invalid logically ranges to have no effect.
- getPlugin()->put(kQueries, "new_test1", "1");
- getPlugin()->put(kQueries, "new_test2", "2");
- getPlugin()->put(kQueries, "new_test3", "3");
- getPlugin()->put(kQueries, "new_test4", "4");
- s = getPlugin()->removeRange(kQueries, "new_test3", "new_test2");
- EXPECT_TRUE(s.ok());
- getPlugin()->get(kQueries, "new_test2", r);
- EXPECT_EQ(r, "2");
- getPlugin()->get(kQueries, "new_test3", r);
- EXPECT_EQ(r, "3");
-
- // An equality range will not delete that single item.
- s = getPlugin()->removeRange(kQueries, "new_test2", "new_test2");
- EXPECT_TRUE(s.ok());
- s = getPlugin()->get(kQueries, "new_test2", r);
- EXPECT_FALSE(s.ok());
-}
-
-void DatabasePluginTests::testScan() {
- getPlugin()->put(kQueries, "test_scan_foo1", "baz");
- getPlugin()->put(kQueries, "test_scan_foo2", "baz");
- getPlugin()->put(kQueries, "test_scan_foo3", "baz");
-
- std::vector<std::string> keys;
- std::vector<std::string> expected = {
- "test_scan_foo1", "test_scan_foo2", "test_scan_foo3"};
- auto s = getPlugin()->scan(kQueries, keys, "", 0);
- EXPECT_TRUE(s.ok());
- EXPECT_EQ(s.getMessage(), "OK");
- EXPECT_EQ(keys.size(), 3U);
- for (const auto& i : expected) {
- EXPECT_NE(std::find(keys.begin(), keys.end(), i), keys.end());
- }
-}
-
-void DatabasePluginTests::testScanLimit() {
- getPlugin()->put(kQueries, "test_scan_foo1", "baz");
- getPlugin()->put(kQueries, "test_scan_foo2", "baz");
- getPlugin()->put(kQueries, "test_scan_foo3", "baz");
-
- std::vector<std::string> keys;
- auto s = getPlugin()->scan(kQueries, keys, "", 2);
- EXPECT_TRUE(s.ok());
- EXPECT_EQ(s.getMessage(), "OK");
- EXPECT_EQ(keys.size(), 2U);
-}
-} // namespace osquery
+++ /dev/null
-/**
- * Copyright (c) 2014-present, Facebook, Inc.
- * All rights reserved.
- *
- * This source code is licensed in accordance with the terms specified in
- * the LICENSE file found in the root directory of this source tree.
- */
-
-#pragma once
-
-#include <gtest/gtest.h>
-
-#include <osquery/database.h>
-
-/// The following test macros allow pretty test output.
-#define CREATE_DATABASE_TESTS(n) \
- TEST_F(n, test_plugin_check) { \
- testPluginCheck(); \
- } \
- TEST_F(n, test_reset) { \
- testReset(); \
- } \
- TEST_F(n, test_put) { \
- testPut(); \
- } \
- TEST_F(n, test_putBatch) { \
- testPutBatch(); \
- } \
- TEST_F(n, test_get) { \
- testGet(); \
- } \
- TEST_F(n, test_delete) { \
- testDelete(); \
- } \
- TEST_F(n, test_delete_range) { \
- testDeleteRange(); \
- } \
- TEST_F(n, test_scan) { \
- testScan(); \
- } \
- TEST_F(n, test_scan_limit) { \
- testScanLimit(); \
- }
-
-namespace osquery {
-
-class DatabasePluginTests : public testing::Test {
- public:
- void SetUp() override;
-
- void TearDown() override;
-
- protected:
- /// Path to testing database.
- std::string path_;
-
- protected:
- /// Require each plugin tester to implement a set name.
- virtual std::string name() = 0;
-
- private:
- void setName(const std::string& name) {
- name_ = name;
- }
-
- const std::string& getName() {
- return name_;
- }
-
- std::shared_ptr<DatabasePlugin> getPlugin() {
- return plugin_;
- }
-
- private:
- /// Plugin name
- std::string name_;
-
- /// Plugin casted from setUp, ready to run tests.
- std::shared_ptr<DatabasePlugin> plugin_{nullptr};
-
- /// Previous active database plugin.
- std::string existing_plugin_;
-
- protected:
- void testPluginCheck();
- void testReset();
- void testPut();
- void testPutBatch();
- void testGet();
- void testDelete();
- void testDeleteRange();
- void testScan();
- void testScanLimit();
-};
-} // namespace osquery
#include <gtest/gtest.h>
#include <osquery/core.h>
-#include <osquery/database.h>
#include <osquery/logger.h>
#include <osquery/registry.h>
#include <osquery/sql.h>
namespace osquery {
-DECLARE_bool(disable_database);
-
class VirtualTableTests : public testing::Test {
public:
void SetUp() override {
Initializer::platformSetup();
registryAndPluginInit();
- FLAGS_disable_database = true;
- DatabasePlugin::setAllowOpen(true);
- DatabasePlugin::initPlugin();
}
};
/// The relative path within the source repo to find test content.
std::string kTestDataPath{"../../../tools/tests/"};
-DECLARE_string(database_path);
DECLARE_string(enroll_tls_endpoint);
DECLARE_bool(disable_logging);
-DECLARE_bool(disable_database);
using chrono_clock = std::chrono::high_resolution_clock;
fs::remove_all(kTestWorkingDirectory);
fs::create_directories(kTestWorkingDirectory);
- FLAGS_database_path = kTestWorkingDirectory + "unittests.db";
FLAGS_disable_logging = true;
- FLAGS_disable_database = true;
-
- // Tests need a database plugin.
- // Set up the database instance for the unittests.
- DatabasePlugin::setAllowOpen(true);
- DatabasePlugin::initPlugin();
Initializer::platformSetup();
}
void shutdownTesting() {
- DatabasePlugin::shutdown();
-
Initializer::platformTeardown();
}
#include <vector>
#include <osquery/core.h>
-#include <osquery/database.h>
#include <osquery/filesystem/filesystem.h>
namespace osquery {
projects / platform / core / security / vist.git / commitdiff