From 9f2179c81cfe6779cc5979c0061fcfaf9a78cbae Mon Sep 17 00:00:00 2001 From: Sangwan Kwon Date: Tue, 14 Jan 2020 13:01:43 +0900 Subject: [PATCH] osquery: Remove config Signed-off-by: Sangwan Kwon --- src/osquery/CMakeLists.txt | 1 - src/osquery/config/CMakeLists.txt | 19 - src/osquery/config/config.cpp | 1029 -------------------- src/osquery/config/config.h | 565 ----------- src/osquery/config/packs.cpp | 298 ------ src/osquery/config/tests/config_tests.cpp | 636 ------------ src/osquery/config/tests/packs.cpp | 262 ----- src/osquery/config/tests/test_utils.cpp | 92 -- src/osquery/config/tests/test_utils.h | 32 - src/osquery/core/init.cpp | 34 - src/osquery/core/query.cpp | 10 +- src/osquery/devtools/shell.cpp | 29 - src/osquery/dispatcher/scheduler.cpp | 21 - src/osquery/dispatcher/tests/scheduler.cpp | 269 ----- src/osquery/events/events.cpp | 55 +- src/osquery/events/tests/events_database_tests.cpp | 7 +- src/osquery/events/tests/events_tests.cpp | 40 - src/osquery/include/osquery/config.h | 448 --------- src/osquery/plugins/CMakeLists.txt | 3 - src/osquery/plugins/config/filesystem_config.cpp | 118 --- src/osquery/plugins/config/parsers/decorators.cpp | 302 ------ src/osquery/plugins/config/parsers/decorators.h | 58 -- src/osquery/tests/test_util.cpp | 2 - src/osquery/tests/test_util.h | 1 - 24 files changed, 5 insertions(+), 4326 deletions(-) delete mode 100644 src/osquery/config/CMakeLists.txt delete mode 100644 src/osquery/config/config.cpp delete mode 100644 src/osquery/config/config.h delete mode 100644 src/osquery/config/packs.cpp delete mode 100644 src/osquery/config/tests/config_tests.cpp delete mode 100644 src/osquery/config/tests/packs.cpp delete mode 100644 src/osquery/config/tests/test_utils.cpp delete mode 100644 src/osquery/config/tests/test_utils.h delete mode 100644 src/osquery/dispatcher/tests/scheduler.cpp delete mode 100644 src/osquery/include/osquery/config.h delete mode 100644 src/osquery/plugins/config/filesystem_config.cpp delete mode 100644 src/osquery/plugins/config/parsers/decorators.cpp delete mode 100644 src/osquery/plugins/config/parsers/decorators.h diff --git a/src/osquery/CMakeLists.txt b/src/osquery/CMakeLists.txt index 0dbae5c..850c3cf 100644 --- a/src/osquery/CMakeLists.txt +++ b/src/osquery/CMakeLists.txt @@ -43,7 +43,6 @@ ELSE(DEFINED GBS_BUILD) ENDIF(DEFINED GBS_BUILD) ## osquery v4.0.0 -ADD_SUBDIRECTORY(config) ADD_SUBDIRECTORY(core) ADD_SUBDIRECTORY(database) ADD_SUBDIRECTORY(dispatcher) diff --git a/src/osquery/config/CMakeLists.txt b/src/osquery/config/CMakeLists.txt deleted file mode 100644 index 10016f1..0000000 --- a/src/osquery/config/CMakeLists.txt +++ /dev/null @@ -1,19 +0,0 @@ -# Copyright (c) 2019 Samsung Electronics Co., Ltd All Rights Reserved -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License - -ADD_OSQUERY_LIBRARY(osquery_config config.cpp - packs.cpp) - -FILE(GLOB OSQUERY_CONFIG_TESTS "tests/*.cpp") -ADD_OSQUERY_TEST(${OSQUERY_CONFIG_TESTS}) diff --git a/src/osquery/config/config.cpp b/src/osquery/config/config.cpp deleted file mode 100644 index 0f84963..0000000 --- a/src/osquery/config/config.cpp +++ /dev/null @@ -1,1029 +0,0 @@ -/** - * Copyright (c) 2014-present, Facebook, Inc. - * All rights reserved. - * - * This source code is licensed in accordance with the terms specified in - * the LICENSE file found in the root directory of this source tree. - */ - -#include -#include -#include -#include -#include -#include - -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace rj = rapidjson; - -namespace osquery { -namespace { -/// Prefix to persist config data -const std::string kConfigPersistencePrefix{"config_persistence."}; - -using ConfigMap = std::map; - -std::atomic is_first_time_refresh(true); -}; // namespace - -/** - * @brief Config plugin registry. - * - * This creates an osquery registry for "config" which may implement - * ConfigPlugin. A ConfigPlugin's call API should make use of a genConfig - * after reading JSON data in the plugin implementation. - */ -CREATE_REGISTRY(ConfigPlugin, "config"); - -/** - * @brief ConfigParser plugin registry. - * - * This creates an osquery registry for "config_parser" which may implement - * ConfigParserPlugin. A ConfigParserPlugin should not export any call actions - * but rather have a simple property tree-accessor API through Config. - */ -CREATE_LAZY_REGISTRY(ConfigParserPlugin, "config_parser"); - -/// The config plugin must be known before reading options. -CLI_FLAG(string, config_plugin, "filesystem", "Config plugin name"); - -CLI_FLAG(bool, - config_check, - false, - "Check the format of an osquery config and exit"); - -CLI_FLAG(bool, config_dump, false, "Dump the contents of the configuration"); - -CLI_FLAG(uint64, - config_refresh, - 0, - "Optional interval in seconds to re-read configuration"); -FLAG_ALIAS(google::uint64, config_tls_refresh, config_refresh); - -/// How long to wait when config update fails -CLI_FLAG(uint64, - config_accelerated_refresh, - 300, - "Interval to wait if reading a configuration fails"); - -CLI_FLAG(bool, - config_enable_backup, - false, - "Backup config and use it when refresh fails"); - -FLAG_ALIAS(google::uint64, - config_tls_accelerated_refresh, - config_accelerated_refresh); - -DECLARE_string(config_plugin); -DECLARE_string(pack_delimiter); - -/** - * @brief The backing store key name for the executing query. - * - * The config maintains schedule statistics and tracks failed executions. - * On process or worker resume an initializer or config may check if the - * resume was the result of a failure during an executing query. - */ -const std::string kExecutingQuery{"executing_query"}; -const std::string kFailedQueries{"failed_queries"}; - -/// The time osquery was started. -std::atomic kStartTime; - -// The config may be accessed and updated asynchronously; use mutexes. -Mutex config_hash_mutex_; -Mutex config_refresh_mutex_; -Mutex config_backup_mutex_; - -/// Several config methods require enumeration via predicate lambdas. -RecursiveMutex config_schedule_mutex_; -RecursiveMutex config_files_mutex_; -RecursiveMutex config_performance_mutex_; - -using PackRef = std::unique_ptr; - -/** - * The schedule is an iterable collection of Packs. When you iterate through - * a schedule, you only get the packs that should be running on the host that - * you're currently operating on. - */ -class Schedule : private boost::noncopyable { - public: - /// Under the hood, the schedule is just a list of the Pack objects - using container = std::vector; - - /** - * @brief Create a schedule maintained by the configuration. - * - * This will check for previously executing queries. If any query was - * executing it is considered in a 'dirty' state and should generate logs. - * The schedule may also choose to blacklist this query. - */ - Schedule(); - - /** - * @brief This class' iteration function - * - * Our step operation will be called on each element in packs_. It is - * responsible for determining if that element should be returned as the - * next iterator element or skipped. - */ - struct Step { - bool operator()(const PackRef& pack) const; - }; - - /// Add a pack to the schedule - void add(PackRef pack); - - /// Remove a pack, by name. - void remove(const std::string& pack); - - /// Remove a pack by name and source. - void remove(const std::string& pack, const std::string& source); - - /// Remove all packs by source. - void removeAll(const std::string& source); - - /// Boost gives us a nice template for maintaining the state of the iterator - using iterator = boost::filter_iterator; - - iterator begin(); - - iterator end(); - - PackRef& last(); - - private: - /// Underlying storage for the packs - container packs_; - - /** - * @brief The schedule will check and record previously executing queries. - * - * If a query is found on initialization, the name will be recorded, it is - * possible to skip previously failed queries. - */ - std::string failed_query_; - - /** - * @brief List of blacklisted queries. - * - * A list of queries that are blacklisted from executing due to prior - * failures. If a query caused a worker to fail it will be recorded during - * the next execution and saved to the blacklist. - */ - std::map blacklist_; - - private: - friend class Config; -}; - -bool Schedule::Step::operator()(const PackRef& pack) const { - return pack->shouldPackExecute(); -} - -void Schedule::add(PackRef pack) { - remove(pack->getName(), pack->getSource()); - packs_.push_back(std::move(pack)); -} - -void Schedule::remove(const std::string& pack) { - remove(pack, ""); -} - -void Schedule::remove(const std::string& pack, const std::string& source) { - auto new_end = std::remove_if( - packs_.begin(), packs_.end(), [pack, source](const PackRef& p) { - if (p->getName() == pack && - (p->getSource() == source || source == "")) { - Config::get().removeFiles(source + FLAGS_pack_delimiter + - p->getName()); - return true; - } - return false; - }); - packs_.erase(new_end, packs_.end()); -} - -void Schedule::removeAll(const std::string& source) { - auto new_end = - std::remove_if(packs_.begin(), packs_.end(), [source](const PackRef& p) { - if (p->getSource() == source) { - Config::get().removeFiles(source + FLAGS_pack_delimiter + - p->getName()); - return true; - } - return false; - }); - packs_.erase(new_end, packs_.end()); -} - -Schedule::iterator Schedule::begin() { - return Schedule::iterator(packs_.begin(), packs_.end()); -} - -Schedule::iterator Schedule::end() { - return Schedule::iterator(packs_.end(), packs_.end()); -} - -PackRef& Schedule::last() { - return packs_.back(); -} - -/** - * @brief A thread that periodically reloads configuration state. - * - * This refresh runner thread can refresh any configuration plugin. - * It may accelerate the time between checks if the configuration fails to load. - * For configurations pulled from the network this assures that configuration - * is fresh when re-attaching. - */ -class ConfigRefreshRunner : public InternalRunnable { - public: - ConfigRefreshRunner() : InternalRunnable("ConfigRefreshRunner") {} - - /// A simple wait/interruptible lock. - void start() override; - - private: - /// The current refresh rate in seconds. - std::atomic refresh_sec_{0}; - - private: - friend class Config; -}; - -void restoreScheduleBlacklist(std::map& blacklist) { - std::string content; - getDatabaseValue(kPersistentSettings, kFailedQueries, content); - auto blacklist_pairs = osquery::split(content, ":"); - if (blacklist_pairs.size() == 0 || blacklist_pairs.size() % 2 != 0) { - // Nothing in the blacklist, or malformed data. - return; - } - - size_t current_time = getUnixTime(); - for (size_t i = 0; i < blacklist_pairs.size() / 2; i++) { - // Fill in a mapping of query name to time the blacklist expires. - auto expire = - tryTo(blacklist_pairs[(i * 2) + 1], 10).takeOr(0ll); - if (expire > 0 && current_time < (size_t)expire) { - blacklist[blacklist_pairs[(i * 2)]] = (size_t)expire; - } - } -} - -void saveScheduleBlacklist(const std::map& blacklist) { - std::string content; - for (const auto& query : blacklist) { - if (!content.empty()) { - content += ":"; - } - content += query.first + ":" + std::to_string(query.second); - } - setDatabaseValue(kPersistentSettings, kFailedQueries, content); -} - -Schedule::Schedule() { - // Parse the schedule's query blacklist from backing storage. - restoreScheduleBlacklist(blacklist_); - - // Check if any queries were executing when the tool last stopped. - getDatabaseValue(kPersistentSettings, kExecutingQuery, failed_query_); - if (!failed_query_.empty()) { - LOG(WARNING) << "Scheduled query may have failed: " << failed_query_; - setDatabaseValue(kPersistentSettings, kExecutingQuery, ""); - // Add this query name to the blacklist and save the blacklist. - blacklist_[failed_query_] = getUnixTime() + 86400; - saveScheduleBlacklist(blacklist_); - } -} - -Config::Config() - : schedule_(std::make_unique()), - valid_(false), - refresh_runner_(std::make_shared()) {} - -Config& Config::get() { - static Config instance; - return instance; -} - -void Config::addPack(const std::string& name, - const std::string& source, - const rj::Value& obj) { - assert(obj.IsObject()); - - auto addSinglePack = ([this, &source](const std::string pack_name, - const rj::Value& pack_obj) { - RecursiveLock wlock(config_schedule_mutex_); - try { - schedule_->add(std::make_unique(pack_name, source, pack_obj)); - if (schedule_->last()->shouldPackExecute()) { - applyParsers(source + FLAGS_pack_delimiter + pack_name, pack_obj, true); - } - } catch (const std::exception& e) { - LOG(WARNING) << "Error adding pack: " << pack_name << ": " << e.what(); - } - }); - - if (name == "*") { - // This is a multi-pack, expect the config plugin to have generated a - // "name": {pack-content} response similar to embedded pack content - // within the configuration. - for (const auto& pack : obj.GetObject()) { - addSinglePack(pack.name.GetString(), pack.value); - } - } else { - addSinglePack(name, obj); - } -} - -size_t Config::getStartTime() { - return kStartTime; -} - -void Config::setStartTime(size_t st) { - kStartTime = st; -} - -void Config::removePack(const std::string& pack) { - RecursiveLock wlock(config_schedule_mutex_); - return schedule_->remove(pack); -} - -void Config::addFile(const std::string& source, - const std::string& category, - const std::string& path) { - RecursiveLock wlock(config_files_mutex_); - files_[source][category].push_back(path); -} - -void Config::removeFiles(const std::string& source) { - RecursiveLock wlock(config_files_mutex_); - if (files_.count(source)) { - FileCategories().swap(files_[source]); - } -} - -/** - * @brief Return true if the failed query is no longer blacklisted. - * - * There are two scenarios where a blacklisted query becomes 'unblacklisted'. - * The first is simple, the amount of time it was blacklisted for has expired. - * The second is more complex, the query failed but the schedule has requested - * that the query should not be blacklisted. - * - * @param blt The time the query was originally blacklisted. - * @param query The scheduled query and its options. - */ -static inline bool blacklistExpired(size_t blt, const ScheduledQuery& query) { - if (getUnixTime() > blt) { - return true; - } - - auto blo = query.options.find("blacklist"); - if (blo != query.options.end() && blo->second == false) { - // The schedule requested that we do not blacklist this query. - return true; - } - return false; -} - -void Config::scheduledQueries( - std::function - predicate, - bool blacklisted) const { - RecursiveLock lock(config_schedule_mutex_); - for (PackRef& pack : *schedule_) { - for (auto& it : pack->getSchedule()) { - std::string name = it.first; - // The query name may be synthetic. - if (pack->getName() != "main") { - name = "pack" + FLAGS_pack_delimiter + pack->getName() + - FLAGS_pack_delimiter + it.first; - } - - // They query may have failed and been added to the schedule's blacklist. - auto blacklisted_query = schedule_->blacklist_.find(name); - if (blacklisted_query != schedule_->blacklist_.end()) { - if (blacklistExpired(blacklisted_query->second, it.second)) { - // The blacklisted query passed the expiration time (remove). - schedule_->blacklist_.erase(blacklisted_query); - saveScheduleBlacklist(schedule_->blacklist_); - it.second.blacklisted = false; - } else { - // The query is still blacklisted. - it.second.blacklisted = true; - if (!blacklisted) { - // The caller does not want blacklisted queries. - continue; - } - } - } - - // Call the predicate. - predicate(std::move(name), it.second); - } - } -} - -void Config::packs(std::function predicate) const { - RecursiveLock lock(config_schedule_mutex_); - for (PackRef& pack : schedule_->packs_) { - predicate(std::cref(*pack.get())); - } -} - -Status Config::refresh() { - PluginResponse response; - auto status = Registry::call("config", {{"action", "genConfig"}}, response); - - WriteLock lock(config_refresh_mutex_); - if (!status.ok()) { - if (FLAGS_config_refresh > 0 && getRefresh() == FLAGS_config_refresh) { - VLOG(1) << "Using accelerated configuration delay"; - setRefresh(FLAGS_config_accelerated_refresh); - } - - loaded_ = true; - return status; - } else if (getRefresh() != FLAGS_config_refresh) { - VLOG(1) << "Normal configuration delay restored"; - setRefresh(FLAGS_config_refresh); - } - - // if there was a response, parse it and update internal state - valid_ = true; - if (response.size() > 0) { - if (FLAGS_config_dump) { - // If config checking is enabled, debug-write the raw config data. - for (const auto& content : response[0]) { - fprintf(stdout, - "{\"%s\": %s}\n", - content.first.c_str(), - content.second.c_str()); - } - // Don't force because the config plugin may have started services. - Initializer::requestShutdown(); - return Status::success(); - } - status = update(response[0]); - } - - is_first_time_refresh = false; - loaded_ = true; - return status; -} - -void Config::setRefresh(size_t refresh_sec) { - refresh_runner_->refresh_sec_ = refresh_sec; -} - -size_t Config::getRefresh() const { - return refresh_runner_->refresh_sec_; -} - -Status Config::load() { - valid_ = false; - auto config_plugin = RegistryFactory::get().getActive("config"); - if (!RegistryFactory::get().exists("config", config_plugin)) { - return Status(1, "Missing config plugin " + config_plugin); - } - - // Set the initial and optional refresh value. - setRefresh(FLAGS_config_refresh); - - /* - * If the initial configuration includes a non-0 refresh, start an - * additional service that sleeps and periodically regenerates the - * configuration. - */ - if (!FLAGS_config_check && !started_thread_ && getRefresh() > 0) { - Dispatcher::addService(refresh_runner_); - started_thread_ = true; - } - - return refresh(); -} - -void stripConfigComments(std::string& json) { - std::string sink; - - boost::replace_all(json, "\\\n", ""); - for (auto& line : osquery::split(json, "\n")) { - boost::trim(line); - if (line.size() > 0 && line[0] == '#') { - continue; - } - if (line.size() > 1 && line[0] == '/' && line[1] == '/') { - continue; - } - sink += line + '\n'; - } - json = sink; -} - -Expected Config::restoreConfigBackup() { - LOG(INFO) << "Restoring backed up config from the database"; - std::vector keys; - ConfigMap config; - - WriteLock lock(config_backup_mutex_); - scanDatabaseKeys(kPersistentSettings, keys, kConfigPersistencePrefix); - - for (const auto& key : keys) { - std::string value; - Status status = getDatabaseValue(kPersistentSettings, key, value); - if (!status.ok()) { - LOG(ERROR) - << "restoreConfigBackup database failed to retrieve config for key " - << key; - return createError(Config::RestoreConfigError::DatabaseError) - << "Could not retrieve value for the key: " << key; - } - config[key.substr(kConfigPersistencePrefix.length())] = std::move(value); - } - - return config; -} - -void Config::backupConfig(const ConfigMap& config) { - LOG(INFO) << "BackupConfig started"; - std::vector keys; - - WriteLock lock(config_backup_mutex_); - scanDatabaseKeys(kPersistentSettings, keys, kConfigPersistencePrefix); - for (const auto& key : keys) { - if (config.find(key.substr(kConfigPersistencePrefix.length())) == - config.end()) { - deleteDatabaseValue(kPersistentSettings, key); - } - } - - for (const auto& source : config) { - setDatabaseValue(kPersistentSettings, - kConfigPersistencePrefix + source.first, - source.second); - } -} - -Status Config::updateSource(const std::string& source, - const std::string& json) { - { - RecursiveLock lock(config_schedule_mutex_); - // Remove all packs from this source. - schedule_->removeAll(source); - // Remove all files from this source. - removeFiles(source); - } - - // load the config (source.second) into a JSON object. - auto doc = JSON::newObject(); - auto clone = json; - stripConfigComments(clone); - - if (!doc.fromString(clone) || !doc.doc().IsObject()) { - return Status(1, "Error parsing the config JSON"); - } - - // extract the "schedule" key and store it as the main pack - auto& rf = RegistryFactory::get(); - if (doc.doc().HasMember("schedule")) { - auto& schedule = doc.doc()["schedule"]; - if (schedule.IsObject()) { - auto main_doc = JSON::newObject(); - auto queries_obj = main_doc.getObject(); - main_doc.copyFrom(schedule, queries_obj); - main_doc.add("queries", queries_obj); - addPack("main", source, main_doc.doc()); - } - } - - // extract the "packs" key into additional pack objects - if (doc.doc().HasMember("packs")) { - auto& packs = doc.doc()["packs"]; - if (packs.IsObject()) { - for (const auto& pack : packs.GetObject()) { - std::string pack_name = pack.name.GetString(); - if (pack.value.IsObject()) { - // The pack is a JSON object, treat the content as pack data. - addPack(pack_name, source, pack.value); - } else if (pack.value.IsString()) { - genPack(pack_name, source, pack.value.GetString()); - } - } - } - } - - applyParsers(source, doc.doc(), false); - return Status::success(); -} - -Status Config::genPack(const std::string& name, - const std::string& source, - const std::string& target) { - // If the pack value is a string (and not a JSON object) then it is a - // resource to be handled by the config plugin. - PluginResponse response; - PluginRequest request = { - {"action", "genPack"}, {"name", name}, {"value", target}}; - Registry::call("config", request, response); - - if (response.size() == 0 || response[0].count(name) == 0) { - return Status(1, "Invalid plugin response"); - } - - auto clone = response[0][name]; - if (clone.empty()) { - LOG(WARNING) << "Error reading the query pack named: " << name; - return Status::success(); - } - - stripConfigComments(clone); - auto doc = JSON::newObject(); - if (!doc.fromString(clone) || !doc.doc().IsObject()) { - LOG(WARNING) << "Error parsing the \"" << name << "\" pack JSON"; - } else { - addPack(name, source, doc.doc()); - } - - return Status::success(); -} - -void Config::applyParsers(const std::string& source, - const rj::Value& obj, - bool pack) { - assert(obj.IsObject()); - - // Iterate each parser. - RecursiveLock lock(config_schedule_mutex_); - for (const auto& plugin : RegistryFactory::get().plugins("config_parser")) { - std::shared_ptr parser = nullptr; - try { - parser = std::dynamic_pointer_cast(plugin.second); - } catch (const std::bad_cast& /* e */) { - LOG(ERROR) << "Error casting config parser plugin: " << plugin.first; - } - if (parser == nullptr || parser.get() == nullptr) { - continue; - } - - // For each key requested by the parser, add a property tree reference. - std::map parser_config; - for (const auto& key : parser->keys()) { - if (obj.HasMember(key.c_str()) && !obj[key.c_str()].IsNull()) { - if (!obj[key.c_str()].IsArray() && !obj[key.c_str()].IsObject()) { - LOG(WARNING) << "Error config " << key - << " should be an array or object"; - continue; - } - - auto doc = JSON::newFromValue(obj[key.c_str()]); - parser_config.emplace(std::make_pair(key, std::move(doc))); - } - } - // The config parser plugin will receive a copy of each property tree for - // each top-level-config key. The parser may choose to update the config's - // internal state - parser->update(source, parser_config); - } -} - -Status Config::update(const ConfigMap& config) { - // Iterate though each source and overwrite config data. - // This will add/overwrite pack data, append to the schedule, change watched - // files, set options, etc. - // Before this occurs, take an opportunity to purge stale state. - purge(); - - bool needs_reconfigure = false; - for (const auto& source : config) { - auto status = updateSource(source.first, source.second); - if (status.getCode() == 2) { - // The source content did not change. - continue; - } - - if (!status.ok()) { - LOG(ERROR) << "updateSource failed to parse config, of source: " - << source.first << " and content: " << source.second; - return status; - } - // If a source was updated and the content has changed, then the registry - // should be reconfigured. File watches may have changed, etc. - needs_reconfigure = true; - } - - if (loaded_ && needs_reconfigure) { - // The config has since been loaded. - // This update call is most likely a response to an async update request - // from a config plugin. This request should request all plugins to update. - for (const auto& registry : RegistryFactory::get().all()) { - if (registry.first == "event_publisher" || - registry.first == "event_subscriber") { - continue; - } - registry.second->configure(); - } - - EventFactory::configUpdate(); - } - - // This cannot be under the previous if block because on extensions loaded_ - // allways false. - if (needs_reconfigure) { - std::string loggers = RegistryFactory::get().getActive("logger"); - for (const auto& logger : osquery::split(loggers, ",")) { - LOG(INFO) << "Calling configure for logger " << logger; - PluginRef plugin = Registry::get().plugin("logger", logger); - - if (plugin) { - plugin->configure(); - } - } - } - - if (FLAGS_config_enable_backup) { - backupConfig(config); - } - - return Status::success(); -} - -void Config::purge() { - // The first use of purge is removing expired query results. - std::vector saved_queries; - scanDatabaseKeys(kQueries, saved_queries); - - auto queryExists = [schedule = static_cast(schedule_.get())]( - const std::string& query_name) { - for (const auto& pack : schedule->packs_) { - const auto& pack_queries = pack->getSchedule(); - if (pack_queries.count(query_name)) { - return true; - } - } - return false; - }; - - RecursiveLock lock(config_schedule_mutex_); - // Iterate over each result set in the database. - for (const auto& saved_query : saved_queries) { - if (queryExists(saved_query)) { - continue; - } - - std::string content; - getDatabaseValue(kPersistentSettings, "timestamp." + saved_query, content); - if (content.empty()) { - // No timestamp is set for this query, perhaps this is the first time - // query results expiration is applied. - setDatabaseValue(kPersistentSettings, - "timestamp." + saved_query, - std::to_string(getUnixTime())); - continue; - } - - // Parse the timestamp and compare. - size_t last_executed = 0; - try { - last_executed = boost::lexical_cast(content); - } catch (const boost::bad_lexical_cast& /* e */) { - // Erase the timestamp as is it potentially corrupt. - deleteDatabaseValue(kPersistentSettings, "timestamp." + saved_query); - continue; - } - - if (last_executed < getUnixTime() - 592200) { - // Query has not run in the last week, expire results and interval. - deleteDatabaseValue(kQueries, saved_query); - deleteDatabaseValue(kQueries, saved_query + "epoch"); - deleteDatabaseValue(kPersistentSettings, "interval." + saved_query); - deleteDatabaseValue(kPersistentSettings, "timestamp." + saved_query); - VLOG(1) << "Expiring results for scheduled query: " << saved_query; - } - } -} - -void Config::reset() { - setStartTime(getUnixTime()); - - schedule_ = std::make_unique(); - std::map().swap(performance_); - std::map().swap(files_); - std::map().swap(hash_); - valid_ = false; - loaded_ = false; - is_first_time_refresh = true; - - refresh_runner_ = std::make_shared(); - started_thread_ = false; - - // Also request each parse to reset state. - for (const auto& plugin : RegistryFactory::get().plugins("config_parser")) { - std::shared_ptr parser = nullptr; - try { - parser = std::dynamic_pointer_cast(plugin.second); - } catch (const std::bad_cast& /* e */) { - continue; - } - if (parser == nullptr || parser.get() == nullptr) { - continue; - } - parser->reset(); - } -} - -void ConfigParserPlugin::reset() { - // Resets will clear all top-level keys from the parser's data store. - for (auto& category : data_.doc().GetObject()) { - auto obj = data_.getObject(); - data_.add(category.name.GetString(), obj, data_.doc()); - } -} - -void Config::recordQueryPerformance(const std::string& name, - size_t delay, - const Row& r0, - const Row& r1) { - RecursiveLock lock(config_performance_mutex_); - if (performance_.count(name) == 0) { - performance_[name] = QueryPerformance(); - } - - // Grab access to the non-const schedule item. - auto& query = performance_.at(name); - if (!r1.at("user_time").empty() && !r0.at("user_time").empty()) { - auto ut1 = tryTo(r1.at("user_time")); - auto ut0 = tryTo(r0.at("user_time")); - auto diff = (ut1 && ut0) ? ut1.take() - ut0.take() : 0; - if (diff > 0) { - query.user_time += diff; - } - } - - if (!r1.at("system_time").empty() && !r0.at("system_time").empty()) { - auto st1 = tryTo(r1.at("system_time")); - auto st0 = tryTo(r0.at("system_time")); - auto diff = (st1 && st0) ? st1.take() - st0.take() : 0; - if (diff > 0) { - query.system_time += diff; - } - } - - if (!r1.at("resident_size").empty() && !r0.at("resident_size").empty()) { - auto rs1 = tryTo(r1.at("resident_size")); - auto rs0 = tryTo(r0.at("resident_size")); - auto diff = (rs1 && rs0) ? rs1.take() - rs0.take() : 0; - if (diff > 0) { - // Memory is stored as an average of RSS changes between query executions. - query.average_memory = (query.average_memory * query.executions) + diff; - query.average_memory = (query.average_memory / (query.executions + 1)); - } - } - - query.wall_time += delay; - query.executions += 1; - query.last_executed = getUnixTime(); - - // Clear the executing query (remove the dirty bit). - setDatabaseValue(kPersistentSettings, kExecutingQuery, ""); -} - -void Config::recordQueryStart(const std::string& name) { - // There should only ever be a single executing query in the schedule. - setDatabaseValue(kPersistentSettings, kExecutingQuery, name); - // Store the time this query name last executed for later results eviction. - // When configuration updates occur the previous schedule is searched for - // 'stale' query names, aka those that have week-old or longer last execute - // timestamps. Offending queries have their database results purged. - setDatabaseValue( - kPersistentSettings, "timestamp." + name, std::to_string(getUnixTime())); -} - -void Config::getPerformanceStats( - const std::string& name, - std::function predicate) const { - if (performance_.count(name) > 0) { - RecursiveLock lock(config_performance_mutex_); - predicate(performance_.at(name)); - } -} - -const std::shared_ptr Config::getParser( - const std::string& parser) { - if (!RegistryFactory::get().exists("config_parser", parser, true)) { - return nullptr; - } - - auto plugin = RegistryFactory::get().plugin("config_parser", parser); - // This is an error, need to check for existence (and not nullptr). - return std::dynamic_pointer_cast(plugin); -} - -void Config::files(std::function& files)> - predicate) const { - RecursiveLock lock(config_files_mutex_); - for (const auto& it : files_) { - for (const auto& category : it.second) { - predicate(category.first, category.second); - } - } -} - -Config::~Config() = default; - -Status ConfigPlugin::genPack(const std::string& name, - const std::string& value, - std::string& pack) { - return Status(1, "Not implemented"); -} - -Status ConfigPlugin::call(const PluginRequest& request, - PluginResponse& response) { - auto action = request.find("action"); - if (action == request.end()) { - return Status::failure("Config plugins require an action"); - } - - if (action->second == "genConfig") { - std::map config; - auto stat = genConfig(config); - response.push_back(config); - return stat; - } else if (action->second == "genPack") { - auto name = request.find("name"); - auto value = request.find("value"); - if (name == request.end() || value == request.end()) { - return Status(1, "Missing name or value"); - } - - std::string pack; - auto stat = genPack(name->second, value->second, pack); - response.push_back({{name->second, pack}}); - return stat; - } else if (action->second == "update") { - auto source = request.find("source"); - auto data = request.find("data"); - if (source == request.end() || data == request.end()) { - return Status(1, "Missing source or data"); - } - - return Config::get().update({{source->second, data->second}}); - } else if (action->second == "option") { - auto name = request.find("name"); - if (name == request.end()) { - return Status(1, "Missing option name"); - } - - response.push_back( - {{"name", name->second}, {"value", Flag::getValue(name->second)}}); - return Status::success(); - } - return Status(1, "Config plugin action unknown: " + action->second); -} - -Status ConfigParserPlugin::setUp() { - for (const auto& key : keys()) { - auto obj = data_.getObject(); - data_.add(key, obj); - } - return Status::success(); -} - -void ConfigRefreshRunner::start() { - while (!interrupted()) { - // Cool off and time wait the configured period. - // Apply this interruption initially as at t=0 the config was read. - pause(std::chrono::seconds(refresh_sec_)); - // Since the pause occurs before the logic, we need to check for an - // interruption request. - if (interrupted()) { - return; - } - - VLOG(1) << "Refreshing configuration state"; - Config::get().refresh(); - } -} -} diff --git a/src/osquery/config/config.h b/src/osquery/config/config.h deleted file mode 100644 index 327bae7..0000000 --- a/src/osquery/config/config.h +++ /dev/null @@ -1,565 +0,0 @@ -/** - * Copyright (c) 2014-present, Facebook, Inc. - * All rights reserved. - * - * This source code is licensed in accordance with the terms specified in - * the LICENSE file found in the root directory of this source tree. - */ - -#pragma once - -#include -#include -#include -#include - -#include -#include -#include -#include -#include - -#include - -namespace osquery { - -class Status; -class Config; -class Pack; -class Schedule; -class ConfigParserPlugin; -class ConfigRefreshRunner; - -/// The name of the executing query within the single-threaded schedule. -extern const std::string kExecutingQuery; - -/** - * @brief The programmatic representation of osquery's configuration - * - * @code{.cpp} - * auto c = Config::get(); - * // use methods in osquery::Config on `c` - * @endcode - */ -class Config : private boost::noncopyable { - private: - Config(); - - void backupConfig(const std::map& config); - - public: - ~Config(); - - /// Singleton accessor. - static Config& get(); - - enum class RestoreConfigError { DatabaseError = 1 }; - /** - * @brief restoreConfigBackup retrieve backed up config - * @return config persisted int the database - */ - static Expected, - Config::RestoreConfigError> - restoreConfigBackup(); - - /** - * @brief Update the internal config data. - * - * @param config A map of domain or namespace to config data. - * @return If the config changes were applied. - */ - Status update(const std::map& config); - - /** - * @brief Record performance (monitoring) information about a scheduled query. - * - * The daemon and query scheduler will optionally record process metadata - * before and after executing each query. This can be compared and reported - * on an interval or within the osquery_schedule table. - * - * The config consumes and calculates the optional performance differentials. - * It would also be possible to store this in the RocksDB backing store or - * report directly to a LoggerPlugin sink. The Config is the most appropriate - * as the metrics are transient to the process running the schedule and apply - * to the updates/changes reflected in the schedule, from the config. - * - * @param name The unique name of the scheduled item - * @param delay Number of seconds (wall time) taken by the query - * @param size Number of characters generated by query - * @param r0 the process row before the query - * @param r1 the process row after the query - */ - void recordQueryPerformance(const std::string& name, - size_t delay, - const Row& r0, - const Row& r1); - - /** - * @brief Record a query 'initialization', meaning the query will run. - * - * Recording initializations if queries helps to identify when queries do not - * complete. The Config::recordQueryPerformance method will clear a dirty - * status set by this method. This status is saved in the backing database - * store. On process start, or worker state, if any dirty bit is set then - * it is assumed that the current start is a result of a previous abort. - * - * @param name THe unique name of the scheduled item - */ - void recordQueryStart(const std::string& name); - - /// Whether or not the last loaded config was valid. - bool isValid() const { - return valid_; - } - - /// Get start time of config. - static size_t getStartTime(); - - /// Set the start time if the config. - static void setStartTime(size_t st); - - /** - * @brief Add a pack to the osquery schedule - */ - void addPack(const std::string& name, - const std::string& source, - const rapidjson::Value& obj); - - /** - * @brief Remove a pack from the osquery schedule - */ - void removePack(const std::string& pack); - - /** - * @brief Iterate through all packs - */ - void packs(std::function predicate) const; - - /** - * @brief Add a file - * - * @param source source of config file - * @param category is the category which the file exists in - * @param path is the file path to add - */ - void addFile(const std::string& source, - const std::string& category, - const std::string& path); - - /// Remove files by source. - void removeFiles(const std::string& source); - - /** - * @brief Map a function across the set of scheduled queries - * - * @param predicate is a function which accepts two parameters, the name of - * the query and the ScheduledQuery struct of the queries data. predicate - * will be called on each currently scheduled query. - * - * @param blacklisted [optional] return blacklisted queries if true. - * - * @code{.cpp} - * std::map queries; - * Config::get().scheduledQueries( - * ([&queries](std::string name, const ScheduledQuery& query) { - * queries[name] = query; - * })); - * @endcode - */ - void scheduledQueries( - std::function - predicate, - bool blacklisted = false) const; - - /** - * @brief Map a function across the set of configured files - * - * @param predicate is a function which accepts two parameters, the name of - * the file category and a vector of files in that category. predicate will be - * called on each pair in files_ - * - * @code{.cpp} - * std::map> file_map; - * Config::get().files( - * ([&file_map](const std::string& category, - * const std::vector& files) { - * file_map[category] = files; - * })); - * @endcode - */ - void files(std::function& files)> - predicate) const; - - /** - * @brief Get the performance stats for a specific query, by name - * - * @param name is the name of the query which you'd like to retrieve - * @param predicate is a function which accepts a const reference to a - * QueryPerformance struct. predicate will be called on name's related - * QueryPerformance struct, if it exists. - * - * @code{.cpp} - * Config::get().getPerformanceStats( - * "my_awesome_query", - * [](const QueryPerformance& query) { - * // use "query" here - * }); - * @endcode - */ - void getPerformanceStats( - const std::string& name, - std::function predicate) const; - - /** - * @brief Helper to access config parsers via the registry - * - * This may return a nullptr if an exception is thrown attempting to retrieve - * the requested config parser. - * - * @param parser is the string name of the parser that you want - * - * @return a shared pointer to the config parser plugin - */ - static const std::shared_ptr getParser( - const std::string& parser); - - protected: - /** - * @brief Call the genConfig method of the config retriever plugin. - * - * This may perform a resource load such as TCP request or filesystem read. - * If a non-zero value is passed to --config_refresh, this starts a thread - * that periodically calls genConfig to reload config state - */ - Status refresh(); - - /// Update the refresh rate. - void setRefresh(size_t refresh_sec); - - /// Inspect the refresh rate. - size_t getRefresh() const; - - /** - * @brief Check if a config plugin is registered and load configs. - * - * Calls refresh after confirming a config plugin is registered - */ - Status load(); - - /// A step method for Config::update. - Status updateSource(const std::string& source, const std::string& json); - - /** - * @brief Generate pack content from a resource handled by the Plugin. - * - * Configuration content may set pack values to JSON strings instead of an - * embedded dictionary representing the pack content. When a string is - * encountered the config assumes this is a 'resource' handled by the Plugin. - * - * The value, or target, is sent to the ConfigPlugin via a registry request. - * The plugin response is assumed, and used, as the pack content. - * - * @param name A pack name provided and handled by the ConfigPlugin. - * @param source The config content source identifier. - * @param target A resource (path, URL, etc) handled by the ConfigPlugin. - * @return status On success the response will be JSON parsed. - */ - Status genPack(const std::string& name, - const std::string& source, - const std::string& target); - - /** - * @brief Apply each ConfigParser to an input JSON document. - * - * This iterates each discovered ConfigParser Plugin and the plugin's keys - * to match keys within the input JSON document. If a key matches then the - * associated value is passed to the parser. - * - * Use this utility method for both the top-level configuration JSON and - * the content of each configuration pack. There is an optional black list - * parameter to differentiate pack content. - * - * @param source The input configuration source name. - * @param obj The input configuration JSON. - * @param pack True if the JSON was built from pack data, otherwise false. - */ - void applyParsers(const std::string& source, - const rapidjson::Value& obj, - bool pack = false); - - /** - * @brief When config sources are updated the config will 'purge'. - * - * The general 'purge' action applies to searching for outdated query results, - * timestamps, saved intervals, etc. This event only occurs before a source - * is updated. Since updating the configuration may have expected side effects - * such as changing watched files or overwriting (modifying) pack content, - * this 'purge' action is assumed to be destructive and potentially expensive. - */ - void purge(); - - /** - * @brief Reset the configuration state, reserved for testing only. - */ - void reset(); - - private: - /// Schedule of packs and their queries. - std::unique_ptr schedule_; - - /// A set of performance stats for each query in the schedule. - std::map performance_; - - /// A set of named categories filled with filesystem globbing paths. - using FileCategories = std::map>; - std::map files_; - - /// A set of hashes for each source of the config. - std::map hash_; - - /// Check if the config received valid/parsable content from a config plugin. - bool valid_{false}; - - /** - * @brief Check if the configuration has attempted a load. - * - * Check if the config is updating sources in response to an async update - * or the initialization load step. - */ - bool loaded_{false}; - - /// Try if the configuration has started an auto-refresh thread. - bool started_thread_{false}; - - private: - /// Hold a reference to the refresh runner to update the acceleration. - std::shared_ptr refresh_runner_{nullptr}; - - private: - friend class Initializer; - - private: - friend class ConfigTests; - friend class ConfigRefreshRunner; - friend class FilePathsConfigParserPluginTests; - friend class FileEventsTableTests; - friend class DecoratorsConfigParserPluginTests; - friend class SchedulerTests; - friend class WatcherTests; - FRIEND_TEST(ConfigTests, test_config_backup); - FRIEND_TEST(ConfigTests, test_config_backup_integrate); - FRIEND_TEST(ConfigTests, test_config_refresh); - FRIEND_TEST(ConfigTests, test_get_scheduled_queries); - FRIEND_TEST(ConfigTests, test_nonblacklist_query); - FRIEND_TEST(OptionsConfigParserPluginTests, test_get_option); - FRIEND_TEST(ViewsConfigParserPluginTests, test_add_view); - FRIEND_TEST(ViewsConfigParserPluginTests, test_swap_view); - FRIEND_TEST(ViewsConfigParserPluginTests, test_update_view); - FRIEND_TEST(OptionsConfigParserPluginTests, test_unknown_option); - FRIEND_TEST(OptionsConfigParserPluginTests, test_json_option); - FRIEND_TEST(EventsConfigParserPluginTests, test_get_event); - FRIEND_TEST(PacksTests, test_discovery_cache); - FRIEND_TEST(PacksTests, test_multi_pack); - FRIEND_TEST(SchedulerTests, test_monitor); - FRIEND_TEST(SchedulerTests, test_config_results_purge); - FRIEND_TEST(EventsTests, test_event_subscriber_configure); - FRIEND_TEST(TLSConfigTests, test_retrieve_config); - FRIEND_TEST(TLSConfigTests, test_runner_and_scheduler); -}; - -/** - * @brief Superclass for the pluggable config component. - * - * In order to make the distribution of configurations to hosts running - * osquery, we take advantage of a plugin interface which allows you to - * integrate osquery with your internal configuration distribution mechanisms. - * You may use ZooKeeper, files on disk, a custom solution, etc. In order to - * use your specific configuration distribution system, one simply needs to - * create a custom subclass of ConfigPlugin. That subclass should implement - * the ConfigPlugin::genConfig method. - * - * Consider the following example: - * - * @code{.cpp} - * class TestConfigPlugin : public ConfigPlugin { - * public: - * virtual Status genConfig(std::map& config) { - * config["my_source"] = "{}"; - * return Status::success(); - * } - * }; - * - * REGISTER(TestConfigPlugin, "config", "test"); - * @endcode - */ -class ConfigPlugin : public Plugin { - public: - /** - * @brief Virtual method which should implemented custom config retrieval - * - * ConfigPlugin::genConfig should be implemented by a subclasses of - * ConfigPlugin which needs to retrieve config data in a custom way. - * - * @param config The output ConfigSourceMap, a map of JSON to source names. - * - * @return A failure status will prevent the source map from merging. - */ - virtual Status genConfig(std::map& config) = 0; - - /** - * @brief Virtual method which could implement custom query pack retrieval - * - * The default config syntax for query packs is like the following: - * - * @code - * { - * "packs": { - * "foo": { - * "version": "1.5.0", - * "platform:" "any", - * "queries": { - * // ... - * } - * } - * } - * } - * @endcode - * - * Alternatively, you can define packs like the following as well: - * - * @code - * { - * "packs": { - * "foo": "/var/osquery/packs/foo.json", - * "bar": "/var/osquery/packs/bar.json" - * } - * } - * @endcode - * - * If you defined the "value" of your pack as a string instead of an inline - * data structure, then osquery will pass the responsibility of retrieving - * the pack to the active config plugin. In the above example, it seems - * obvious that the value is a local file path. Alternatively, if the - * filesystem config plugin wasn't being used, the string could be a remote - * URL, etc. - * - * genPack is not a pure virtual, so you don't have to define it if you don't - * want to use the shortened query pack syntax. The default implementation - * returns a failed status. - * - * @param name is the name of the query pack - * @param value is the string based value that was provided with the pack - * @param pack should be populated with the string JSON pack content - * - * @return a Status instance indicating the success or failure of the call - */ - virtual Status genPack(const std::string& name, - const std::string& value, - std::string& pack); - - /// Main entrypoint for config plugin requests - Status call(const PluginRequest& request, PluginResponse& response) override; -}; - -/** - * @brief A pluggable configuration parser. - * - * An osquery config instance is populated from JSON using a ConfigPlugin. - * That plugin may update the config data asynchronously and read from - * several sources, as is the case with "filesystem" and reading multiple files. - * - * A ConfigParserPlugin will receive the merged configuration at osquery start - * and the updated (still merged) config if any ConfigPlugin updates the - * instance asynchronously. Each parser specifies a set of top-level JSON - * keys to receive. The config instance will auto-merge the key values - * from multiple sources. - * - * The keys must contain either dictionaries or lists. - * - * If a top-level key is a dictionary, each source with the top-level key - * will have its own dictionary keys merged and replaced based on the lexical - * order of sources. For the "filesystem" config plugin this is the lexical - * sorting of filenames. If the top-level key is a list, each source with the - * top-level key will have its contents appended. - * - * Each config parser plugin will live alongside the config instance for the - * life of the osquery process. The parser may perform actions at config load - * and config update "time" as well as keep its own data members and be - * accessible through the Config class API. - */ -class ConfigParserPlugin : public Plugin { - public: - using ParserConfig = std::map; - - public: - /** - * @brief Return a list of top-level config keys to receive in updates. - * - * The ConfigParserPlugin::update method will receive a map of these keys - * with a JSON-parsed document of configuration data. - * - * @return A list of string top-level JSON keys. - */ - virtual std::vector keys() const = 0; - - /** - * @brief Receive a merged JSON document for each top-level config key. - * - * Called when the Config instance is initially loaded with data from the - * active config plugin and when it is updated via an async ConfigPlugin - * update. Every config parser will receive a map of merged data for each key - * they requested in keys(). - * - * @param source source of the config data - * @param config A JSON-parsed document map. - * @return Failure if the parser should no longer receive updates. - */ - virtual Status update(const std::string& source, - const ParserConfig& config) = 0; - - /// Allow parsers to perform some setup before the configuration is loaded. - Status setUp() override; - - Status call(const PluginRequest& /*request*/, - PluginResponse& /*response*/) override { - return Status(0); - } - - /** - * @brief Accessor for parser-manipulated data. - * - * Parsers should be used generically, for places within the code base that - * request a parser (check for its existence), should only use this - * ConfigParserPlugin::getData accessor. - * - * More complex parsers that require dynamic casting are not recommended. - */ - const JSON& getData() const { - return data_; - } - - protected: - /// Allow the config to request parser state resets. - virtual void reset(); - - protected: - /// Allow the config parser to keep some global state. - JSON data_; - - private: - friend class Config; -}; - -/** - * @brief JSON parsers may accept comments. - * - * For semi-compatibility with existing configurations we will attempt to strip - * hash and C++ style comments. It is OK for the config update to be latent - * as it is a single event. But some configuration plugins may update running - * configurations. - * - * @param json A mutable input/output string that will contain stripped JSON. - */ -void stripConfigComments(std::string& json); -} // namespace osquery diff --git a/src/osquery/config/packs.cpp b/src/osquery/config/packs.cpp deleted file mode 100644 index b878ad6..0000000 --- a/src/osquery/config/packs.cpp +++ /dev/null @@ -1,298 +0,0 @@ -/** - * Copyright (c) 2014-present, Facebook, Inc. - * All rights reserved. - * - * This source code is licensed in accordance with the terms specified in - * the LICENSE file found in the root directory of this source tree. - */ - -#include -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace rj = rapidjson; - -namespace osquery { - -FLAG(uint64, - pack_refresh_interval, - 3600, - "Cache expiration for a packs discovery queries"); - -FLAG(string, pack_delimiter, "_", "Delimiter for pack and query names"); - -FLAG(uint64, schedule_splay_percent, 10, "Percent to splay config times"); - -FLAG(uint64, - schedule_default_interval, - 3600, - "Query interval to use if none is provided"); - -size_t kMaxQueryInterval = 604800; - -size_t splayValue(size_t original, size_t splayPercent) { - if (splayPercent == 0 || splayPercent > 100) { - return original; - } - - float percent_to_modify_by = (float)splayPercent / 100; - size_t possible_difference = - static_cast(original * percent_to_modify_by); - size_t max_value = original + possible_difference; - size_t min_value = std::max((size_t)1, original - possible_difference); - - if (max_value == min_value) { - return max_value; - } - - std::default_random_engine generator; - generator.seed(static_cast( - std::chrono::high_resolution_clock::now().time_since_epoch().count())); - std::uniform_int_distribution distribution(min_value, max_value); - return distribution(generator); -} - -size_t restoreSplayedValue(const std::string& name, size_t interval) { - // Attempt to restore a previously-calculated splay. - std::string content; - getDatabaseValue(kPersistentSettings, "interval." + name, content); - if (!content.empty()) { - // This query name existed before, check the last requested interval. - auto details = osquery::split(content, ":"); - if (details.size() == 2) { - auto const last_interval_exp = tryTo(details[0], 10); - auto const last_splay_exp = tryTo(details[1], 10); - if (last_interval_exp.isValue() && last_splay_exp.isValue()) { - if (last_interval_exp.get() == static_cast(interval) && - last_splay_exp.get() > 0) { - // This is a matching interval, use the previous splay. - return static_cast(last_splay_exp.get()); - } - } - } - } - - // If the splayed interval was not restored from the database. - auto splay = splayValue(interval, FLAGS_schedule_splay_percent); - content = std::to_string(interval) + ":" + std::to_string(splay); - setDatabaseValue(kPersistentSettings, "interval." + name, content); - return splay; -} - -void Pack::initialize(const std::string& name, - const std::string& source, - const rj::Value& obj) { - name_ = name; - source_ = source; - // Check the shard limitation, shards falling below this value are included. - if (obj.HasMember("shard")) { - shard_ = JSON::valueToSize(obj["shard"]); - } - - // Check for a platform restriction. - platform_.clear(); - if (obj.HasMember("platform") && obj["platform"].IsString()) { - platform_ = obj["platform"].GetString(); - } - - // Check for a version restriction. - version_.clear(); - if (obj.HasMember("version") && obj["version"].IsString()) { - version_ = obj["version"].GetString(); - } - - std::string oncall; - if (obj.HasMember("oncall") && obj["oncall"].IsString()) { - oncall = obj["oncall"].GetString(); - } else { - oncall = "unknown"; - } - - discovery_queries_.clear(); - if (obj.HasMember("discovery") && obj["discovery"].IsArray()) { - for (const auto& item : obj["discovery"].GetArray()) { - discovery_queries_.push_back(item.GetString()); - } - } - - // Initialize a discovery cache at time 0. - discovery_cache_ = std::make_pair(0, false); - valid_ = true; - - // If the splay percent is less than 1 reset to a sane estimate. - if (FLAGS_schedule_splay_percent <= 1) { - FLAGS_schedule_splay_percent = 10; - } - - schedule_.clear(); - if (!obj.HasMember("queries") || !obj["queries"].IsObject()) { - // This pack contained no queries. - VLOG(1) << "No queries defined for pack " << name; - return; - } - - // Iterate the queries (or schedule) and check platform/version/sanity. - for (const auto& q : obj["queries"].GetObject()) { - if (!q.value.IsObject()) { - VLOG(1) << "The pack " << name << " must contain a dictionary of queries"; - continue; - } - - if (q.value.HasMember("platform") && q.value["platform"].IsString()) { - if (!checkPlatform(q.value["platform"].GetString())) { - continue; - } - } - - if (q.value.HasMember("version") && q.value["version"].IsString()) { - if (!checkVersion(q.value["version"].GetString())) { - continue; - } - } - - if (!q.value.HasMember("query") || !q.value["query"].IsString()) { - VLOG(1) << "No query string defined for query " << q.name.GetString(); - continue; - } - - ScheduledQuery query( - name_, q.name.GetString(), q.value["query"].GetString()); - - query.oncall = oncall; - - if (!q.value.HasMember("interval")) { - query.interval = FLAGS_schedule_default_interval; - } else { - query.interval = JSON::valueToSize(q.value["interval"]); - } - - if (query.interval <= 0 || query.query.empty() || - query.interval > kMaxQueryInterval) { - // Invalid pack query. - LOG(WARNING) << "Query has invalid interval: " << q.name.GetString() - << ": " << query.interval; - continue; - } - - query.splayed_interval = - restoreSplayedValue(q.name.GetString(), query.interval); - - if (!q.value.HasMember("snapshot")) { - query.options["snapshot"] = false; - } else { - query.options["snapshot"] = JSON::valueToBool(q.value["snapshot"]); - } - - if (!q.value.HasMember("removed")) { - query.options["removed"] = true; - } else { - query.options["removed"] = JSON::valueToBool(q.value["removed"]); - } - - query.options["blacklist"] = true; - if (q.value.HasMember("blacklist")) { - query.options["blacklist"] = JSON::valueToBool(q.value["blacklist"]); - } - - schedule_.emplace(std::make_pair(q.name.GetString(), std::move(query))); - } -} - -const std::map& Pack::getSchedule() const { - return schedule_; -} - -std::map& Pack::getSchedule() { - return schedule_; -} - -const std::vector& Pack::getDiscoveryQueries() const { - return discovery_queries_; -} - -const PackStats& Pack::getStats() const { - return stats_; -} - -const std::string& Pack::getPlatform() const { - return platform_; -} - -const std::string& Pack::getVersion() const { - return version_; -} - -bool Pack::shouldPackExecute() { - active_ = (valid_ && checkDiscovery()); - return active_; -} - -const std::string& Pack::getName() const { - return name_; -} - -const std::string& Pack::getSource() const { - return source_; -} - -bool Pack::checkPlatform() const { - return checkPlatform(platform_); -} - -bool Pack::checkPlatform(const std::string& platform) const { - return ::osquery::checkPlatform(platform); -} - -bool Pack::checkVersion() const { - return checkVersion(version_); -} - -bool Pack::checkVersion(const std::string& version) const { - if (version.empty() || version == "null") { - return true; - } - - return versionAtLeast(version, kSDKVersion); -} - -bool Pack::checkDiscovery() { - stats_.total++; - size_t current = osquery::getUnixTime(); - if ((current - discovery_cache_.first) < FLAGS_pack_refresh_interval) { - stats_.hits++; - return discovery_cache_.second; - } - - stats_.misses++; - discovery_cache_.first = current; - discovery_cache_.second = true; - for (const auto& q : discovery_queries_) { - SQL results(q); - if (!results.ok()) { - LOG(WARNING) << "Discovery query failed (" << q - << "): " << results.getMessageString(); - discovery_cache_.second = false; - break; - } - if (results.rows().size() == 0) { - discovery_cache_.second = false; - break; - } - } - return discovery_cache_.second; -} - -bool Pack::isActive() const { - return active_; -} -} diff --git a/src/osquery/config/tests/config_tests.cpp b/src/osquery/config/tests/config_tests.cpp deleted file mode 100644 index e64e1ff..0000000 --- a/src/osquery/config/tests/config_tests.cpp +++ /dev/null @@ -1,636 +0,0 @@ -/** - * Copyright (c) 2014-present, Facebook, Inc. - * All rights reserved. - * - * This source code is licensed in accordance with the terms specified in - * the LICENSE file found in the root directory of this source tree. - */ - -#include - -#include - -#include -#include - -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include - -#include - -#include - -#include -#include -#include -#include -#include -#include -#include - - -namespace osquery { - -DECLARE_uint64(config_refresh); -DECLARE_uint64(config_accelerated_refresh); -DECLARE_bool(config_enable_backup); -DECLARE_bool(disable_database); - -namespace fs = boost::filesystem; - -// Blacklist testing methods, internal to config implementations. -extern void restoreScheduleBlacklist(std::map& blacklist); -extern void saveScheduleBlacklist( - const std::map& blacklist); - -class ConfigTests : public testing::Test { - public: - ConfigTests() { - Initializer::platformSetup(); - registryAndPluginInit(); - FLAGS_disable_database = true; - DatabasePlugin::setAllowOpen(true); - DatabasePlugin::initPlugin(); - - Config::get().reset(); - } - - protected: - fs::path fake_directory_; - - protected: - void SetUp() { - fake_directory_ = fs::canonical(createMockFileStructure()); - - refresh_ = FLAGS_config_refresh; - FLAGS_config_refresh = 0; - - createMockFileStructure(); - } - - void TearDown() { - fs::remove_all(fake_directory_); - FLAGS_config_refresh = refresh_; - } - - protected: - Status load() { - return Config::get().load(); - } - - void setLoaded() { - Config::get().loaded_ = true; - } - - Config& get() { - return Config::get(); - } - - private: - size_t refresh_{0}; -}; - -class TestConfigPlugin : public ConfigPlugin { - public: - TestConfigPlugin() { - gen_config_count_ = 0; - gen_pack_count_ = 0; - } - - Status genConfig(std::map& config) override { - gen_config_count_++; - if (fail_) { - return Status(1); - } - - std::string content; - auto s = readFile(getTestConfigDirectory() / "test_noninline_packs.conf", content); - config["data"] = content; - return s; - } - - Status genPack(const std::string& name, - const std::string& value, - std::string& pack) override { - gen_pack_count_++; - getUnrestrictedPack().toString(pack); - return Status::success(); - } - - public: - std::atomic gen_config_count_{0}; - std::atomic gen_pack_count_{0}; - std::atomic fail_{false}; -}; - -class TestDataConfigParserPlugin : public ConfigParserPlugin { - public: - std::vector keys() const override { - return {"data"}; - } - - Status setUp() override { - return Status::success(); - } - - Status update(const std::string& source, - const ParserConfig& config) override { - source_ = source; - config_.clear(); - for (const auto& entry : config) { - std::string content; - entry.second.toString(content); - config_[entry.first] = content; - } - return Status::success(); - } - std::string source_{""}; - std::map config_; -}; - -TEST_F(ConfigTests, test_plugin) { - auto& rf = RegistryFactory::get(); - auto plugin = std::make_shared(); - rf.registry("config")->add("test", plugin); - // Change the active config plugin. - EXPECT_TRUE(rf.setActive("config", "test").ok()); - - PluginResponse response; - auto status = Registry::call("config", {{"action", "genConfig"}}, response); - - EXPECT_EQ(status.ok(), true) << status.what(); - - Registry::call("config", {{"action", "genConfig"}}); - EXPECT_EQ(2U, plugin->gen_config_count_); - rf.registry("config")->remove("test"); -} - -TEST_F(ConfigTests, test_invalid_content) { - std::string bad_json = "{\"options\": {},}"; - ASSERT_NO_THROW(get().update({{"bad_source", bad_json}})); -} - -TEST_F(ConfigTests, test_strip_comments) { - std::string json_comments = - "// Comment\n // Comment //\n # Comment\n# Comment\n{\"options\":{}}"; - - // Test support for stripping C++ and hash style comments from config JSON. - auto actual = json_comments; - stripConfigComments(actual); - std::string expected = "{\"options\":{}}\n"; - EXPECT_EQ(actual, expected); - - // Make sure the config update source logic applies the stripping. - EXPECT_TRUE(get().update({{"data", json_comments}})); -} - -TEST_F(ConfigTests, test_schedule_blacklist) { - auto current_time = getUnixTime(); - std::map blacklist; - saveScheduleBlacklist(blacklist); - restoreScheduleBlacklist(blacklist); - EXPECT_EQ(blacklist.size(), 0U); - - // Create some entries. - blacklist["test_1"] = current_time * 2; - blacklist["test_2"] = current_time * 3; - saveScheduleBlacklist(blacklist); - blacklist.clear(); - restoreScheduleBlacklist(blacklist); - ASSERT_EQ(blacklist.count("test_1"), 1U); - ASSERT_EQ(blacklist.count("test_2"), 1U); - EXPECT_EQ(blacklist.at("test_1"), current_time * 2); - EXPECT_EQ(blacklist.at("test_2"), current_time * 3); - - // Now save an expired query. - blacklist["test_1"] = 1; - saveScheduleBlacklist(blacklist); - blacklist.clear(); - - // When restoring, the values below the current time will not be included. - restoreScheduleBlacklist(blacklist); - EXPECT_EQ(blacklist.size(), 1U); -} - -TEST_F(ConfigTests, test_pack_noninline) { - auto& rf = RegistryFactory::get(); - rf.registry("config")->add("test", std::make_shared()); - // Change the active config plugin. - EXPECT_TRUE(rf.setActive("config", "test").ok()); - - // Get a specialized config/test plugin. - const auto& plugin = - std::dynamic_pointer_cast(rf.plugin("config", "test")); - - this->load(); - // Expect the test plugin to have recorded 1 pack. - // This value is incremented when its genPack method is called. - EXPECT_EQ(plugin->gen_pack_count_, 1U); - - int total_packs = 0; - // Expect the config to have recorded a pack for the inline and non-inline. - get().packs([&total_packs](const Pack& pack) { total_packs++; }); - EXPECT_EQ(total_packs, 2); - rf.registry("config")->remove("test"); -} - -TEST_F(ConfigTests, test_pack_restrictions) { - auto doc = getExamplePacksConfig(); - auto& packs = doc.doc()["packs"]; - for (const auto& pack : packs.GetObject()) { - get().addPack(pack.name.GetString(), "", pack.value); - } - - std::map results = { - {"unrestricted_pack", true}, - {"discovery_pack", false}, - {"fake_version_pack", false}, - {"valid_discovery_pack", true}, - {"restricted_pack", false}, - }; - - get().packs(([&results](const Pack& pack) { - if (results[pack.getName()]) { - EXPECT_TRUE(const_cast(pack).shouldPackExecute()) - << "Pack " << pack.getName() << " should have executed"; - } else { - EXPECT_FALSE(const_cast(pack).shouldPackExecute()) - << "Pack " << pack.getName() << " should not have executed"; - } - })); -} - -TEST_F(ConfigTests, test_pack_removal) { - size_t pack_count = 0; - get().packs(([&pack_count](const Pack& pack) { pack_count++; })); - EXPECT_EQ(pack_count, 0U); - - pack_count = 0; - get().addPack("unrestricted_pack", "", getUnrestrictedPack().doc()); - get().packs(([&pack_count](const Pack& pack) { pack_count++; })); - EXPECT_EQ(pack_count, 1U); - - pack_count = 0; - get().removePack("unrestricted_pack"); - get().packs(([&pack_count](const Pack& pack) { pack_count++; })); - EXPECT_EQ(pack_count, 0U); -} - -TEST_F(ConfigTests, test_get_scheduled_queries) { - std::vector query_names; - get().addPack("unrestricted_pack", "", getUnrestrictedPack().doc()); - get().scheduledQueries( - ([&query_names](std::string name, const ScheduledQuery& query) { - query_names.push_back(std::move(name)); - })); - - auto expected_size = getUnrestrictedPack().doc()["queries"].MemberCount(); - EXPECT_EQ(query_names.size(), expected_size) - << "The number of queries in the schedule (" << query_names.size() - << ") should equal " << expected_size; - ASSERT_FALSE(query_names.empty()); - - // Construct a schedule blacklist and place the first scheduled query. - std::map blacklist; - std::string query_name = query_names[0]; - blacklist[query_name] = getUnixTime() * 2; - saveScheduleBlacklist(blacklist); - blacklist.clear(); - - // When the blacklist is edited externally, the config must re-read. - get().reset(); - get().addPack("unrestricted_pack", "", getUnrestrictedPack().doc()); - - // Clear the query names in the scheduled queries and request again. - query_names.clear(); - get().scheduledQueries( - ([&query_names](std::string name, const ScheduledQuery&) { - query_names.push_back(std::move(name)); - })); - // The query should not exist. - EXPECT_EQ(std::find(query_names.begin(), query_names.end(), query_name), - query_names.end()); - - // Try again, this time requesting scheduled queries. - query_names.clear(); - bool blacklisted = false; - get().scheduledQueries(([&blacklisted, &query_names, &query_name]( - std::string name, const ScheduledQuery& query) { - if (name == query_name) { - // Only populate the query we've blacklisted. - query_names.push_back(std::move(name)); - blacklisted = query.blacklisted; - } - }), - true); - ASSERT_EQ(query_names.size(), std::size_t{1}); - EXPECT_EQ(query_names[0], query_name); - EXPECT_TRUE(blacklisted); -} - -TEST_F(ConfigTests, test_nonblacklist_query) { - std::map blacklist; - - const std::string kConfigTestNonBlacklistQuery{"pack_unrestricted_pack_process_heartbeat"}; - - blacklist[kConfigTestNonBlacklistQuery] = getUnixTime() * 2; - saveScheduleBlacklist(blacklist); - - get().reset(); - get().addPack("unrestricted_pack", "", getUnrestrictedPack().doc()); - - std::map blacklisted; - get().scheduledQueries( - ([&blacklisted](std::string name, const ScheduledQuery& query) { - blacklisted[name] = query.blacklisted; - })); - - // This query cannot be blacklisted. - auto query = blacklisted.find(kConfigTestNonBlacklistQuery); - ASSERT_NE(query, blacklisted.end()); - EXPECT_FALSE(query->second); -} - -class TestConfigParserPlugin : public ConfigParserPlugin { - public: - std::vector keys() const override { - // This config parser requests the follow top-level-config keys. - return {"dictionary", "dictionary2", "list"}; - } - - Status update(const std::string& source, - const ParserConfig& config) override { - // Set a simple boolean indicating the update callin occurred. - update_called = true; - // Copy all expected keys into the parser's data. - for (const auto& key : config) { - auto obj = data_.getObject(); - data_.copyFrom(key.second.doc(), obj); - data_.add(key.first, obj, data_.doc()); - } - - // Set parser-rendered additional data. - auto obj2 = data_.getObject(); - data_.addRef("key2", "value2", obj2); - data_.add("dictionary3", obj2, data_.doc()); - return Status::success(); - } - - // Flag tracking that the update method was called. - static bool update_called; - - private: - FRIEND_TEST(ConfigTests, test_config_parser); -}; - -// An intermediate boolean to check parser updates. -bool TestConfigParserPlugin::update_called = false; - -TEST_F(ConfigTests, test_get_parser) { - auto& rf = RegistryFactory::get(); - rf.registry("config_parser") - ->add("test", std::make_shared()); - - auto s = get().update(getTestConfigMap("test_parse_items.conf")); - EXPECT_TRUE(s.ok()); - EXPECT_EQ(s.toString(), "OK"); - - auto plugin = get().getParser("test"); - EXPECT_TRUE(plugin != nullptr); - EXPECT_TRUE(plugin.get() != nullptr); - - const auto& parser = - std::dynamic_pointer_cast(plugin); - const auto& doc = parser->getData(); - - EXPECT_TRUE(doc.doc().HasMember("list")); - EXPECT_TRUE(doc.doc().HasMember("dictionary")); - rf.registry("config_parser")->remove("test"); -} - -class PlaceboConfigParserPlugin : public ConfigParserPlugin { - public: - std::vector keys() const override { - return {}; - } - Status update(const std::string&, const ParserConfig&) override { - return Status::success(); - } - - /// Make sure configure is called. - void configure() override { - configures++; - } - - size_t configures{0}; -}; - -TEST_F(ConfigTests, test_plugin_reconfigure) { - auto& rf = RegistryFactory::get(); - // Add a configuration plugin (could be any plugin) that will react to - // config updates. - rf.registry("config_parser") - ->add("placebo", std::make_shared()); - - // Create a config that has been loaded. - setLoaded(); - get().update({{"data", "{}"}}); - // Get the placebo. - auto placebo = std::static_pointer_cast( - rf.plugin("config_parser", "placebo")); - EXPECT_EQ(placebo->configures, 1U); - - // Updating with the same content does not reconfigure parsers. - get().update({{"data", "{}"}}); - EXPECT_EQ(placebo->configures, 1U); - - // Updating with different content will reconfigure. - get().update({{"data", "{\"options\":{}}"}}); - EXPECT_EQ(placebo->configures, 2U); - get().update({{"data", "{\"options\":{}}"}}); - EXPECT_EQ(placebo->configures, 2U); - - // Updating with a new source will reconfigure. - get().update({{"data", "{\"options\":{}}"}, {"data1", "{}"}}); - EXPECT_EQ(placebo->configures, 3U); - // Updating and not including a source is handled by the config plugin. - // The config will expect the other source to update asynchronously and does - // not consider the missing key as a delete request. - get().update({{"data", "{\"options\":{}}"}}); - EXPECT_EQ(placebo->configures, 3U); - - rf.registry("config_parser")->remove("placebo"); -} - -TEST_F(ConfigTests, test_pack_file_paths) { - size_t count = 0; - auto fileCounter = [&count](const std::string& c, - const std::vector& files) { - count += files.size(); - }; - - get().addPack("unrestricted_pack", "", getUnrestrictedPack().doc()); - get().files(fileCounter); - EXPECT_EQ(count, 2U); - - count = 0; - get().removePack("unrestricted_pack"); - get().files(fileCounter); - EXPECT_EQ(count, 0U); - - count = 0; - get().addPack("restricted_pack", "", getRestrictedPack().doc()); - get().files(fileCounter); - EXPECT_EQ(count, 0U); - - // Test a more-generic update. - count = 0; - get().update({{"data", "{\"file_paths\": {\"new\": [\"/new\"]}}"}}); - get().files(fileCounter); - EXPECT_EQ(count, 1U); - - count = 0; - get().update({{"data", "{}"}}); - get().files(fileCounter); - EXPECT_EQ(count, 0U); -} - -void waitForConfig(std::shared_ptr& plugin, size_t count) { - // Max wait of 3 seconds. - auto delay = std::chrono::milliseconds{3000}; - auto const step = std::chrono::milliseconds{20}; - while (delay.count() > 0) { - if (plugin->gen_config_count_ > count) { - break; - } - delay -= step; - std::this_thread::sleep_for(step); - } -} - -TEST_F(ConfigTests, test_config_refresh) { - auto& rf = RegistryFactory::get(); - auto refresh = FLAGS_config_refresh; - auto refresh_acceleratred = FLAGS_config_accelerated_refresh; - - // Create and add a test plugin. - auto plugin = std::make_shared(); - EXPECT_TRUE(rf.registry("config")->add("test", plugin)); - EXPECT_TRUE(rf.setActive("config", "test")); - - // Reset the configuration and stop the refresh thread. - get().reset(); - - // Stop the existing refresh runner thread. - Dispatcher::stopServices(); - Dispatcher::joinServices(); - - // Set a config_refresh value to convince the Config to start the thread. - FLAGS_config_refresh = 2; - FLAGS_config_accelerated_refresh = 1; - get().setRefresh(FLAGS_config_refresh); - - // Fail the first config load. - plugin->fail_ = true; - - // The runner will wait at least one refresh-delay. - auto count = static_cast(plugin->gen_config_count_); - - get().load(); - EXPECT_TRUE(get().started_thread_); - EXPECT_GT(plugin->gen_config_count_, count); - EXPECT_EQ(get().getRefresh(), FLAGS_config_accelerated_refresh); - - plugin->fail_ = false; - count = static_cast(plugin->gen_config_count_); - - waitForConfig(plugin, count + 1); - EXPECT_GT(plugin->gen_config_count_, count); - EXPECT_EQ(get().getRefresh(), FLAGS_config_refresh); - - // Now make the configuration break. - plugin->fail_ = true; - count = static_cast(plugin->gen_config_count_); - - waitForConfig(plugin, count + 1); - EXPECT_GT(plugin->gen_config_count_, count); - EXPECT_EQ(get().getRefresh(), FLAGS_config_accelerated_refresh); - - // Test that the normal acceleration is restored. - plugin->fail_ = false; - count = static_cast(plugin->gen_config_count_); - - waitForConfig(plugin, count + 1); - EXPECT_GT(plugin->gen_config_count_, count); - EXPECT_EQ(get().getRefresh(), FLAGS_config_refresh); - - // Stop the new refresh runner thread. - Dispatcher::stopServices(); - Dispatcher::joinServices(); - - FLAGS_config_refresh = refresh; - FLAGS_config_accelerated_refresh = refresh_acceleratred; - rf.registry("config")->remove("test"); -} - -TEST_F(ConfigTests, test_config_backup) { - get().reset(); - const std::map expected_config = {{"a", "b"}, - {"c", "d"}}; - get().backupConfig(expected_config); - const auto config = get().restoreConfigBackup(); - EXPECT_TRUE(config); - EXPECT_EQ(*config, expected_config); -} - -TEST_F(ConfigTests, test_config_backup_integrate) { - const auto config_enable_backup_saved = FLAGS_config_enable_backup; - FLAGS_config_enable_backup = true; - - get().reset(); - auto& rf = RegistryFactory::get(); - auto data_parser = std::make_shared(); - auto success_plugin = std::make_shared(); - auto fail_plugin = std::make_shared(); - fail_plugin->fail_ = true; - success_plugin->fail_ = false; - - rf.registry("config")->add("test_success", success_plugin); - rf.registry("config")->add("test_fail", fail_plugin); - rf.registry("config_parser")->add("test", data_parser); - // Change the active config plugin. - EXPECT_TRUE(rf.setActive("config", "test_success").ok()); - - auto status = get().refresh(); - EXPECT_TRUE(status.ok()); - EXPECT_EQ(success_plugin->gen_config_count_, 1); - - auto source_backup = data_parser->source_; - auto config_backup = data_parser->config_; - - EXPECT_TRUE(source_backup.length() > 0); - - get().reset(); - data_parser->source_.clear(); - data_parser->config_.clear(); - EXPECT_TRUE(rf.setActive("config", "test_fail").ok()); - - status = get().refresh(); - EXPECT_FALSE(status.ok()); - EXPECT_EQ(fail_plugin->gen_config_count_, 1); - - EXPECT_EQ(data_parser->source_, source_backup); - EXPECT_EQ(data_parser->config_, config_backup); - - FLAGS_config_enable_backup = config_enable_backup_saved; -} -} diff --git a/src/osquery/config/tests/packs.cpp b/src/osquery/config/tests/packs.cpp deleted file mode 100644 index b7deffc..0000000 --- a/src/osquery/config/tests/packs.cpp +++ /dev/null @@ -1,262 +0,0 @@ -/** - * Copyright (c) 2014-present, Facebook, Inc. - * All rights reserved. - * - * This source code is licensed in accordance with the terms specified in - * the LICENSE file found in the root directory of this source tree. - */ - -#include - -#include - -#include -#include -#include -#include -#include -#include - -#include - -#include - -#include - -#include -#include - - -namespace osquery { - -DECLARE_bool(disable_database); - -class PacksTests : public testing::Test { - public: - PacksTests() { - Initializer::platformSetup(); - registryAndPluginInit(); - FLAGS_disable_database = true; - DatabasePlugin::setAllowOpen(true); - DatabasePlugin::initPlugin(); - } -}; - -TEST_F(PacksTests, test_parse) { - auto doc = getExamplePacksConfig(); - EXPECT_TRUE(doc.doc().HasMember("packs")); -} - -TEST_F(PacksTests, test_should_pack_execute) { - Pack kpack("unrestricted_pack", getUnrestrictedPack().doc()); - EXPECT_TRUE(kpack.shouldPackExecute()); - - Pack fpack("discovery_pack", getPackWithDiscovery().doc()); - EXPECT_FALSE(fpack.shouldPackExecute()); -} - -TEST_F(PacksTests, test_get_discovery_queries) { - std::vector expected; - - Pack kpack("unrestricted_pack", getUnrestrictedPack().doc()); - EXPECT_EQ(kpack.getDiscoveryQueries(), expected); - - expected = {"select pid from processes where name = 'foobar';"}; - Pack fpack("discovery_pack", getPackWithDiscovery().doc()); - EXPECT_EQ(fpack.getDiscoveryQueries(), expected); -} - -TEST_F(PacksTests, test_platform) { - Pack fpack("discovery_pack", getPackWithDiscovery().doc()); - EXPECT_EQ(fpack.getPlatform(), "all"); -} - -TEST_F(PacksTests, test_version) { - Pack fpack("discovery_pack", getPackWithDiscovery().doc()); - EXPECT_EQ(fpack.getVersion(), "1.5.0"); -} - -TEST_F(PacksTests, test_check_platform) { - // First we exercise some basic functionality which should behave the same - // regardless of the current build platform. - Pack fpack("discovery_pack", getPackWithDiscovery().doc()); - EXPECT_TRUE(fpack.checkPlatform()); - - fpack.platform_ = "null"; - EXPECT_TRUE(fpack.checkPlatform()); - - fpack.platform_ = ""; - EXPECT_TRUE(fpack.checkPlatform()); - - fpack.platform_ = "bad_value"; - EXPECT_FALSE(fpack.checkPlatform()); - - // We should execute the query if the SDK platform is specifed. - fpack.platform_ = kSDKPlatform; - EXPECT_TRUE(fpack.checkPlatform()); - // But not if something other than the SDK platform is speciifed. - fpack.platform_ = (kSDKPlatform == "darwin") ? "linux" : "darwin"; - EXPECT_FALSE(fpack.checkPlatform()); - - // For the remaining tests, we exercise all of the valid platform values. - fpack.platform_ = "darwin"; - if (isPlatform(PlatformType::TYPE_OSX)) { - EXPECT_TRUE(fpack.checkPlatform()); - } else { - EXPECT_FALSE(fpack.checkPlatform()); - } - - fpack.platform_ = "freebsd"; - if (isPlatform(PlatformType::TYPE_FREEBSD)) { - EXPECT_TRUE(fpack.checkPlatform()); - } else { - EXPECT_FALSE(fpack.checkPlatform()); - } - - // Although officially no longer supported, we still treat the platform - // values of "centos" and "ubuntu" just like "linux". We execute any query - // with any of these platform values on any Linux system. For what it's - // worth, we never actually differentiated between Linux distributions. - for (auto p : std::set{"centos", "linux", "ubuntu"}) { - fpack.platform_ = p; - if (isPlatform(PlatformType::TYPE_LINUX)) { - EXPECT_TRUE(fpack.checkPlatform()); - } else { - EXPECT_FALSE(fpack.checkPlatform()); - } - } - - fpack.platform_ = "posix"; - if (isPlatform(PlatformType::TYPE_POSIX) || - isPlatform(PlatformType::TYPE_LINUX) || - isPlatform(PlatformType::TYPE_OSX) || - isPlatform(PlatformType::TYPE_FREEBSD)) { - EXPECT_TRUE(fpack.checkPlatform()); - } else { - EXPECT_FALSE(fpack.checkPlatform()); - } - - fpack.platform_ = "windows"; - if (isPlatform(PlatformType::TYPE_WINDOWS)) { - EXPECT_TRUE(fpack.checkPlatform()); - } else { - EXPECT_FALSE(fpack.checkPlatform()); - } -} - -TEST_F(PacksTests, test_check_version) { - Pack zpack("fake_version_pack", getPackWithFakeVersion().doc()); - EXPECT_FALSE(zpack.checkVersion()); - - Pack fpack("discovery_pack", getPackWithDiscovery().doc()); - EXPECT_TRUE(fpack.checkVersion()); -} - -TEST_F(PacksTests, test_restriction_population) { - // Require that all potential restrictions are populated before being checked. - auto doc = getExamplePacksConfig(); - const auto& packs = doc.doc()["packs"]; - Pack fpack("fake_pack", packs["restricted_pack"]); - - ASSERT_FALSE(fpack.getPlatform().empty()); - ASSERT_FALSE(fpack.getVersion().empty()); - ASSERT_EQ(fpack.getShard(), 1U); -} - -TEST_F(PacksTests, test_schedule) { - Pack fpack("discovery_pack", getPackWithDiscovery().doc()); - // Expect a single query in the schedule since one query has an explicit - // invalid/fake platform requirement. - EXPECT_EQ(fpack.getSchedule().size(), 1U); -} - -TEST_F(PacksTests, test_discovery_cache) { - Config c; - // This pack and discovery query are valid, expect the SQL to execute. - c.addPack("valid_discovery_pack", "", getPackWithValidDiscovery().doc()); - size_t query_count = 0U; - size_t query_attempts = 5U; - for (size_t i = 0; i < query_attempts; i++) { - c.scheduledQueries( - ([&query_count](std::string name, const ScheduledQuery& query) { - query_count++; - })); - } - EXPECT_EQ(query_count, query_attempts); - - size_t pack_count = 0U; - c.packs(([&pack_count, query_attempts](const Pack& p) { - pack_count++; - // There is one pack without a discovery query. - EXPECT_EQ(p.getStats().total, query_attempts + 1); - EXPECT_EQ(p.getStats().hits, query_attempts); - EXPECT_EQ(p.getStats().misses, 1U); - })); - - EXPECT_EQ(pack_count, 1U); - c.reset(); -} - -TEST_F(PacksTests, test_multi_pack) { - std::string multi_pack_content = "{\"first\": {}, \"second\": {}}"; - auto multi_pack = JSON::newObject(); - multi_pack.fromString(multi_pack_content); - - Config c; - c.addPack("*", "", multi_pack.doc()); - - std::vector pack_names; - c.packs( - ([&pack_names](const Pack& p) { pack_names.push_back(p.getName()); })); - - std::vector expected = {"first", "second"}; - ASSERT_EQ(expected.size(), pack_names.size()); - EXPECT_EQ(expected, pack_names); -} - -TEST_F(PacksTests, test_discovery_zero_state) { - Pack pack("discovery_pack", getPackWithDiscovery().doc()); - auto stats = pack.getStats(); - EXPECT_EQ(stats.total, 0U); - EXPECT_EQ(stats.hits, 0U); - EXPECT_EQ(stats.misses, 0U); -} - -TEST_F(PacksTests, test_splay) { - auto val1 = splayValue(100, 10); - EXPECT_GE(val1, 90U); - EXPECT_LE(val1, 110U); - - auto val2 = splayValue(100, 10); - EXPECT_GE(val2, 90U); - EXPECT_LE(val2, 110U); - - auto val3 = splayValue(10, 0); - EXPECT_EQ(val3, 10U); - - auto val4 = splayValue(100, 1); - EXPECT_GE(val4, 99U); - EXPECT_LE(val4, 101U); - - auto val5 = splayValue(1, 10); - EXPECT_EQ(val5, 1U); -} - -TEST_F(PacksTests, test_restore_splay) { - auto splay = restoreSplayedValue("pack_test_query_name", 3600); - EXPECT_GE(splay, 3600U - 360); - EXPECT_LE(splay, 3600U + 360); - - // If we restore, the splay should always be equal. - for (size_t i = 0; i < 10; i++) { - auto splay2 = restoreSplayedValue("pack_test_query_name", 3600); - EXPECT_EQ(splay, splay2); - } - - // If we modify the input interval the splay will change. - auto splay3 = restoreSplayedValue("pack_test_query_name", 3600 * 10); - EXPECT_GE(splay3, 3600U * 10 - (360 * 10)); - EXPECT_LE(splay3, 3600U * 10 + (360 * 10)); - EXPECT_NE(splay, splay3); -} -} diff --git a/src/osquery/config/tests/test_utils.cpp b/src/osquery/config/tests/test_utils.cpp deleted file mode 100644 index a4e6e9d..0000000 --- a/src/osquery/config/tests/test_utils.cpp +++ /dev/null @@ -1,92 +0,0 @@ -/** - * Copyright (c) 2014-present, Facebook, Inc. - * All rights reserved. - * - * This source code is licensed in accordance with the terms specified in - * the LICENSE file found in the root directory of this source tree. - */ - -#include - -#include - -#include - -#include - -#include - -#include - -namespace { - -namespace fs = boost::filesystem; - -fs::path getConfDirPathImpl() { - return fs::path("../../../tools/tests/"); -} - -} - -namespace osquery { - -fs::path const& getTestConfigDirectory() { - static auto const path = getConfDirPathImpl(); - return path; -} - -std::map getTestConfigMap(const std::string& file) { - std::string content; - auto const filepath = getTestConfigDirectory() / file; - auto status = readFile(filepath, content); - EXPECT_TRUE(status.ok()) - << "Could not read file: " << boost::io::quoted(filepath.string()) - << ", because: " << status.what(); - std::map config; - config["awesome"] = content; - return config; -} - -JSON getExamplePacksConfig() { - std::string content; - auto const filepath = getTestConfigDirectory() / "test_inline_pack.conf"; - auto status = readFile(filepath, content); - EXPECT_TRUE(status.ok()) - << "Could not read file: " << boost::io::quoted(filepath.string()) - << ", because: " << status.what(); - JSON doc = JSON::newObject(); - doc.fromString(content); - return doc; -} - -/// no discovery queries, no platform restriction -JSON getUnrestrictedPack() { - auto doc = getExamplePacksConfig(); - return JSON::newFromValue(doc.doc()["packs"]["unrestricted_pack"]); -} - -// several restrictions (version, platform, shard) -JSON getRestrictedPack() { - auto doc = getExamplePacksConfig(); - return JSON::newFromValue(doc.doc()["packs"]["restricted_pack"]); -} - -/// 1 discovery query, darwin platform restriction -JSON getPackWithDiscovery() { - auto doc = getExamplePacksConfig(); - return JSON::newFromValue(doc.doc()["packs"]["discovery_pack"]); -} - -/// 1 discovery query which will always pass -JSON getPackWithValidDiscovery() { - auto doc = getExamplePacksConfig(); - return JSON::newFromValue(doc.doc()["packs"]["valid_discovery_pack"]); -} - -/// no discovery queries, no platform restriction, fake version string -JSON getPackWithFakeVersion() { - auto doc = getExamplePacksConfig(); - return JSON::newFromValue(doc.doc()["packs"]["fake_version_pack"]); -} - -} // namespace osquery diff --git a/src/osquery/config/tests/test_utils.h b/src/osquery/config/tests/test_utils.h deleted file mode 100644 index ab8eb2c..0000000 --- a/src/osquery/config/tests/test_utils.h +++ /dev/null @@ -1,32 +0,0 @@ -/** - * Copyright (c) 2014-present, Facebook, Inc. - * All rights reserved. - * - * This source code is licensed in accordance with the terms specified in - * the LICENSE file found in the root directory of this source tree. - */ - -#pragma once - -#include - -#include - -#include -#include - -namespace osquery { - -boost::filesystem::path const& getTestConfigDirectory(); - -// Get an example generate config with one static source name to JSON content. -std::map getTestConfigMap(const std::string& file); - -JSON getExamplePacksConfig(); -JSON getUnrestrictedPack(); -JSON getRestrictedPack(); -JSON getPackWithDiscovery(); -JSON getPackWithValidDiscovery(); -JSON getPackWithFakeVersion(); - -} // namespace osquery diff --git a/src/osquery/core/init.cpp b/src/osquery/core/init.cpp index 5a91314..53bdf24 100644 --- a/src/osquery/core/init.cpp +++ b/src/osquery/core/init.cpp @@ -33,7 +33,6 @@ #include "osquery/utils/config/default_paths.h" #include "osquery/utils/info/platform_type.h" -#include #include #include #include @@ -152,7 +151,6 @@ DECLARE_string(flagfile); namespace osquery { -DECLARE_string(config_plugin); DECLARE_string(logger_plugin); DECLARE_bool(config_check); DECLARE_bool(config_dump); @@ -237,8 +235,6 @@ Initializer::Initializer(int& argc, // Initialize random number generated based on time. std::srand(static_cast( chrono_clock::now().time_since_epoch().count())); - // The config holds the initialization time for easy access. - Config::setStartTime(getUnixTime()); // osquery can function as the daemon or shell depending on argv[0]. if (tool == ToolType::SHELL_DAEMON) { @@ -382,11 +378,6 @@ void Initializer::initDaemon() const { return; } - if (FLAGS_config_check) { - // No need to daemonize, emit log lines, or create process mutexes. - return; - } - #if !defined(__APPLE__) && !defined(WIN32) // OS X uses launchd to daemonize. if (osquery::FLAGS_daemonize) { @@ -487,39 +478,14 @@ void Initializer::start() const { } - // Then set the config plugin, which uses a single/active plugin. - initActivePlugin("config", FLAGS_config_plugin); - // Run the setup for all lazy registries (tables, SQL). Registry::setUp(); - if (FLAGS_config_check) { - // The initiator requested an initialization and config check. - auto s = Config::get().load(); - if (!s.ok()) { - std::cerr << "Error reading config: " << s.toString() << "\n"; - } - // A configuration check exits the application. - // Make sure to request a shutdown as plugins may have created services. - requestShutdown(s.getCode()); - } - if (FLAGS_database_dump) { dumpDatabase(); requestShutdown(); } - // Load the osquery config using the default/active config plugin. - auto s = Config::get().load(); - if (!s.ok()) { - auto message = "Error reading config: " + s.toString(); - if (isDaemon()) { - LOG(WARNING) << message; - } else { - VLOG(1) << message; - } - } - // Initialize the status and result plugin logger. if (!FLAGS_disable_logging) { initActivePlugin("logger", FLAGS_logger_plugin); diff --git a/src/osquery/core/query.cpp b/src/osquery/core/query.cpp index aaea66d..72a4431 100644 --- a/src/osquery/core/query.cpp +++ b/src/osquery/core/query.cpp @@ -21,8 +21,6 @@ namespace rj = rapidjson; namespace osquery { -DECLARE_bool(decorations_top_level); - /// Log numeric values as numbers (in JSON syntax) FLAG(bool, log_numerics_as_numbers, @@ -210,15 +208,11 @@ inline void addLegacyFieldsAndDecorations(const QueryLogItem& item, if (!item.decorations.empty()) { auto dec_obj = doc.getObject(); auto target_obj = std::ref(dec_obj); - if (FLAGS_decorations_top_level) { - target_obj = std::ref(obj); - } + target_obj = std::ref(obj); + for (const auto& name : item.decorations) { doc.addRef(name.first, name.second, target_obj); } - if (!FLAGS_decorations_top_level) { - doc.add("decorations", dec_obj, obj); - } } } diff --git a/src/osquery/devtools/shell.cpp b/src/osquery/devtools/shell.cpp index 131710e..19eabdc 100644 --- a/src/osquery/devtools/shell.cpp +++ b/src/osquery/devtools/shell.cpp @@ -33,7 +33,6 @@ #include -#include #include #include #include @@ -73,8 +72,6 @@ SHELL_FLAG(string, A, "", "Select all from a table"); DECLARE_string(nullvalue); DECLARE_string(logger_plugin); DECLARE_string(logger_path); -DECLARE_string(config_plugin); -DECLARE_string(config_path); DECLARE_string(database_path); } // namespace osquery @@ -1146,14 +1143,6 @@ inline void meta_show(struct callback_data* p) { fprintf(p->out, "\nGeneral settings:\n"); fprintf(p->out, "%13.13s: %s\n", "Flagfile", FLAGS_flagfile.c_str()); - // Show helpful config-related settings. - fprintf( - p->out, "%13.13s: %s", "Config", osquery::FLAGS_config_plugin.c_str()); - if (std::strcmp(osquery::FLAGS_config_plugin.c_str(), "filesystem") == 0) { - fprintf(p->out, " (%s)\n", osquery::FLAGS_config_path.c_str()); - } else { - fprintf(p->out, "\n"); - } // Show helpful logger-related settings. fprintf( @@ -1644,24 +1633,6 @@ int runQuery(struct callback_data* data, const char* query) { int runPack(struct callback_data* data) { int rc = 0; - - // Check every pack for a name matching the requested --pack flag. - Config::get().packs([data, &rc](const Pack& pack) { - if (pack.getName() != FLAGS_pack) { - return; - } - - for (const auto& query : pack.getSchedule()) { - rc = runQuery(data, query.second.query.c_str()); - if (rc != 0) { - fprintf(stderr, - "Could not execute query %s: %s\n", - query.first.c_str(), - query.second.query.c_str()); - return; - } - } - }); return rc; } diff --git a/src/osquery/dispatcher/scheduler.cpp b/src/osquery/dispatcher/scheduler.cpp index f08e6d1..2902b75 100644 --- a/src/osquery/dispatcher/scheduler.cpp +++ b/src/osquery/dispatcher/scheduler.cpp @@ -12,7 +12,6 @@ #include #include -#include #include #include #include @@ -23,7 +22,6 @@ #include "osquery/dispatcher/scheduler.h" #include "osquery/sql/sqlite_util.h" -#include "osquery/plugins/config/parsers/decorators.h" namespace osquery { @@ -63,7 +61,6 @@ SQLInternal monitor(const std::string& name, const ScheduledQuery& query) { EQUALS, pid); auto t0 = getUnixTime(); - Config::get().recordQueryStart(name); SQLInternal sql(query.query, true); // Snapshot the performance after, and compare. auto t1 = getUnixTime(); @@ -72,17 +69,12 @@ SQLInternal monitor(const std::string& name, const ScheduledQuery& query) { "pid", EQUALS, pid); - if (r0.size() > 0 && r1.size() > 0) { - // Always called while processes table is working. - Config::get().recordQueryPerformance(name, t1 - t0, r0[0], r1[0]); - } return sql; } Status launchQuery(const std::string& name, const ScheduledQuery& query) { // Execute the scheduled query and create a named query object. LOG(INFO) << "Executing scheduled query " << name << ": " << query.query; - runDecorators(DECORATE_ALWAYS); auto sql = monitor(name, query); if (!sql.getStatus().ok()) { @@ -102,7 +94,6 @@ Status launchQuery(const std::string& name, const ScheduledQuery& query) { item.time = osquery::getUnixTime(); item.epoch = FLAGS_schedule_epoch; item.calendar_time = osquery::getAsciiTime(); - getDecorations(item.decorations); if (query.options.count("snapshot") && query.options.at("snapshot")) { // This is a snapshot query, emit results with a differential or state. @@ -162,18 +153,6 @@ void SchedulerRunner::start() { auto i = osquery::getUnixTime(); for (; (timeout_ == 0) || (i <= timeout_); ++i) { auto start_time_point = std::chrono::steady_clock::now(); - Config::get().scheduledQueries( - ([&i](const std::string& name, const ScheduledQuery& query) { - if (query.splayed_interval > 0 && i % query.splayed_interval == 0) { - TablePlugin::kCacheInterval = query.splayed_interval; - TablePlugin::kCacheStep = i; - const auto status = launchQuery(name, query); - } - })); - // Configuration decorators run on 60 second intervals only. - if ((i % 60) == 0) { - runDecorators(DECORATE_INTERVAL, i); - } if (FLAGS_schedule_reload > 0 && (i % FLAGS_schedule_reload) == 0) { if (FLAGS_schedule_reload_sql) { SQLiteDBManager::resetPrimary(); diff --git a/src/osquery/dispatcher/tests/scheduler.cpp b/src/osquery/dispatcher/tests/scheduler.cpp deleted file mode 100644 index b38228a..0000000 --- a/src/osquery/dispatcher/tests/scheduler.cpp +++ /dev/null @@ -1,269 +0,0 @@ -/** - * Copyright (c) 2014-present, Facebook, Inc. - * All rights reserved. - * - * This source code is licensed in accordance with the terms specified in - * the LICENSE file found in the root directory of this source tree. - */ - -#include - -#include -#include -#include -#include - -#include -#include -#include -#include - -namespace osquery { - -DECLARE_bool(disable_database); -DECLARE_bool(disable_logging); -DECLARE_uint64(schedule_reload); - -class SchedulerTests : public testing::Test { - void SetUp() override { - Initializer::platformSetup(); - registryAndPluginInit(); - FLAGS_disable_database = true; - DatabasePlugin::setAllowOpen(true); - DatabasePlugin::initPlugin(); - - logging_ = FLAGS_disable_logging; - FLAGS_disable_logging = true; - Config::get().reset(); - } - - void TearDown() override { - FLAGS_disable_logging = logging_; - Config::get().reset(); - } - - private: - bool logging_{false}; -}; - -TEST_F(SchedulerTests, test_monitor) { - std::string name = "pack_test_test_query"; - - // This query has never run so it will not have a timestamp. - std::string timestamp; - getDatabaseValue(kPersistentSettings, "timestamp." + name, timestamp); - ASSERT_TRUE(timestamp.empty()); - - // Fill in a scheduled query and execute it via the query monitor wrapper. - ScheduledQuery query("time_pack", "time", "select * from time"); - query.interval = 10; - query.splayed_interval = 11; - - auto results = monitor(name, query); - EXPECT_EQ(results.rowsTyped().size(), 1U); - - // Ask the config instance for the monitored performance. - QueryPerformance perf; - Config::get().getPerformanceStats( - name, ([&perf](const QueryPerformance& r) { perf = r; })); - // Make sure it was recorded query ran. - // There is no pack for this query within the config, that is fine as these - // performance stats are tracked independently. - EXPECT_EQ(perf.executions, 1U); - - // A bit more testing, potentially redundant, check the database results. - // Since we are only monitoring, no 'actual' results are stored. - std::string content; - getDatabaseValue(kQueries, name, content); - EXPECT_TRUE(content.empty()); - - // Finally, make sure there is a recorded timestamp for the execution. - // We are not concerned with the APPROX value, only that it was recorded. - getDatabaseValue(kPersistentSettings, "timestamp." + name, timestamp); - EXPECT_FALSE(timestamp.empty()); -} - -TEST_F(SchedulerTests, test_config_results_purge) { - // Set a query time for now (time is only important relative to a week ago). - auto query_time = osquery::getUnixTime(); - setDatabaseValue( - kPersistentSettings, "timestamp.test_query", std::to_string(query_time)); - // Store a meaningless saved query interval splay. - setDatabaseValue(kPersistentSettings, "interval.test_query", "11"); - // Store meaningless query differential results. - setDatabaseValue(kQueries, "test_query", "{}"); - - // We do not need "THE" config instance. - // We only need to trigger a 'purge' event, this occurs when configuration - // content is updated by a plugin or on load. - Config::get().purge(); - - // Nothing should have been purged. - { - std::string content; - getDatabaseValue(kPersistentSettings, "timestamp.test_query", content); - EXPECT_FALSE(content.empty()); - } - - { - std::string content; - getDatabaseValue(kPersistentSettings, "interval.test_query", content); - EXPECT_FALSE(content.empty()); - } - - { - std::string content; - getDatabaseValue(kQueries, "test_query", content); - EXPECT_FALSE(content.empty()); - } - - // Update the timestamp to have run a week and a day ago. - query_time -= (84600 * (7 + 1)); - setDatabaseValue( - kPersistentSettings, "timestamp.test_query", std::to_string(query_time)); - - // Trigger another purge. - Config::get().purge(); - // Now ALL 'test_query' related storage will have been purged. - { - std::string content; - getDatabaseValue(kPersistentSettings, "timestamp.test_query", content); - EXPECT_TRUE(content.empty()); - } - - { - std::string content; - getDatabaseValue(kPersistentSettings, "interval.test_query", content); - EXPECT_TRUE(content.empty()); - } - - { - std::string content; - getDatabaseValue(kQueries, "test_query", content); - EXPECT_TRUE(content.empty()); - } -} - -TEST_F(SchedulerTests, test_scheduler) { - auto backup_step = TablePlugin::kCacheStep; - auto backup_interval = TablePlugin::kCacheInterval; - - // Start the scheduler now. - auto now = osquery::getUnixTime(); - TablePlugin::kCacheStep = now; - - // Update the config with a pack/schedule that contains several queries. - std::string config = - "{" - "\"packs\": {" - "\"scheduler\": {" - "\"queries\": {" - "\"1\": {\"query\": \"select * from osquery_schedule\", \"interval\": 1}," - "\"2\": {\"query\": \"select * from osquery_info\", \"interval\": 1}," - "\"3\": {\"query\": \"select * from processes\", \"interval\": 1}," - "\"4\": {\"query\": \"select * from osquery_packs\", \"interval\": 1}" - "}" - "}" - "}" - "}"; - Config::get().update({{"data", config}}); - - // Run the scheduler for 1 second with a second interval. - SchedulerRunner runner(static_cast(now + 1), 1); - runner.start(); - - // If a query was executed the cache step will have been advanced. - EXPECT_GT(TablePlugin::kCacheStep, now); - - // Restore plugin settings. - TablePlugin::kCacheStep = backup_step; - TablePlugin::kCacheInterval = backup_interval; -} - -TEST_F(SchedulerTests, test_scheduler_zero_drift) { - const auto backup_step = TablePlugin::kCacheStep; - const auto backup_interval = TablePlugin::kCacheInterval; - - // Start the scheduler now. - const auto now = osquery::getUnixTime(); - TablePlugin::kCacheStep = now; - - // Update the config with a pack/schedule that contains several queries. - std::string config = R"config( - { - "packs": { - "scheduler": { - "queries": { - "1": {"query": "select 1 as number", "interval": 1}, - "2": {"query": "select 2 as number", "interval": 1} - } - } - } - })config"; - Config::get().update({{"data", config}}); - - // Run the scheduler for 1 second with a second interval. - SchedulerRunner runner( - static_cast(now), size_t{1}, std::chrono::seconds{10}); - runner.start(); - - EXPECT_EQ(runner.getCurrentTimeDrift(), std::chrono::milliseconds::zero()); - - // Restore plugin settings. - TablePlugin::kCacheStep = backup_step; - TablePlugin::kCacheInterval = backup_interval; -} - -TEST_F(SchedulerTests, test_scheduler_drift_accumulation) { - const auto backup_step = TablePlugin::kCacheStep; - const auto backup_interval = TablePlugin::kCacheInterval; - - // Start the scheduler now. - const auto now = osquery::getUnixTime(); - TablePlugin::kCacheStep = now; - - // Update the config with a pack/schedule that contains several queries. - std::string config = R"config( - { - "packs": { - "scheduler": { - "queries": { - "3": {"query": "select 3 as number", "interval": 1}, - "4": {"query": "select 4 as number", "interval": 1}, - "5": {"query": "select 5 as number", "interval": 1}, - "6": {"query": "select 6 as number", "interval": 1}, - "7": {"query": "select 7 as number", "interval": 1}, - "8": {"query": "select 1 as number", "interval": 1}, - "9": {"query": "select 2 as number", "interval": 1} - } - } - } - })config"; - Config::get().update({{"data", config}}); - - // Run the scheduler for 1 second with a second interval. - SchedulerRunner runner(static_cast(now + 3), - size_t{0}, - std::chrono::seconds{10}); - runner.start(); - - EXPECT_GE(runner.getCurrentTimeDrift(), std::chrono::milliseconds{1}); - - // Restore plugin settings. - TablePlugin::kCacheStep = backup_step; - TablePlugin::kCacheInterval = backup_interval; -} - -TEST_F(SchedulerTests, test_scheduler_reload) { - std::string config = - "{\"schedule\":{\"1\":{" - "\"query\":\"select * from processes\", \"interval\":1}}}"; - auto backup_reload = FLAGS_schedule_reload; - - // Start the scheduler; - auto expire = static_cast(getUnixTime() + 1); - FLAGS_schedule_reload = 1; - SchedulerRunner runner(expire, 1); - FLAGS_schedule_reload = backup_reload; -} -} diff --git a/src/osquery/events/events.cpp b/src/osquery/events/events.cpp index a3cf6f3..ace461e 100644 --- a/src/osquery/events/events.cpp +++ b/src/osquery/events/events.cpp @@ -14,7 +14,6 @@ #include #include -#include #include #include #include @@ -68,7 +67,7 @@ static inline void getOptimizeData(EventTime& o_time, std::string& query_name, const std::string& publisher) { // Read the optimization time for the current executing query. - getDatabaseValue(kPersistentSettings, kExecutingQuery, query_name); + getDatabaseValue(kPersistentSettings, "", query_name); if (query_name.empty()) { o_time = 0; o_eid = 0; @@ -93,7 +92,7 @@ static inline void setOptimizeData(EventTime time, const std::string& publisher) { // Store the optimization time and eid. std::string query_name; - getDatabaseValue(kPersistentSettings, kExecutingQuery, query_name); + getDatabaseValue(kPersistentSettings, "", query_name); if (query_name.empty()) { return; } @@ -607,32 +606,6 @@ void EventFactory::configUpdate() { // Scan the schedule for queries that touch "_events" tables. // We will count the queries std::map subscriber_details; - Config::get().scheduledQueries( - [&subscriber_details](std::string name, const ScheduledQuery& query) { - std::vector tables; - // Convert query string into a list of virtual tables effected. - if (!getQueryTables(query.query, tables)) { - VLOG(1) << "Cannot get tables from query: " << name; - return; - } - - // Remove duplicates and select only the subscriber tables. - std::set subscribers; - for (const auto& table : tables) { - if (Registry::get().exists("event_subscriber", table)) { - subscribers.insert(table); - } - } - - for (const auto& subscriber : subscribers) { - auto& details = subscriber_details[subscriber]; - details.max_interval = (query.interval > details.max_interval) - ? query.interval - : details.max_interval; - details.query_count++; - } - }); - auto& ef = EventFactory::getInstance(); for (const auto& details : subscriber_details) { if (!ef.exists(details.first)) { @@ -797,30 +770,6 @@ Status EventFactory::registerEventSubscriber(const PluginRef& sub) { return Status(1, "Subscribers must have set a name"); } - auto plugin = Config::get().getParser("events"); - if (plugin != nullptr && plugin.get() != nullptr) { - const auto& data = plugin->getData().doc(); - // First perform explicit enabling. - if (data["events"].HasMember("enable_subscribers")) { - for (const auto& item : data["events"]["enable_subscribers"].GetArray()) { - if (item.GetString() == name) { - VLOG(1) << "Enabling event subscriber: " << name; - specialized_sub->disabled = false; - } - } - } - // Then use explicit disabling as an ultimate override. - if (data["events"].HasMember("disable_subscribers")) { - for (const auto& item : - data["events"]["disable_subscribers"].GetArray()) { - if (item.GetString() == name) { - VLOG(1) << "Disabling event subscriber: " << name; - specialized_sub->disabled = true; - } - } - } - } - if (specialized_sub->state() != EventState::EVENT_NONE) { specialized_sub->tearDown(); } diff --git a/src/osquery/events/tests/events_database_tests.cpp b/src/osquery/events/tests/events_database_tests.cpp index 1896ee9..a66d67c 100644 --- a/src/osquery/events/tests/events_database_tests.cpp +++ b/src/osquery/events/tests/events_database_tests.cpp @@ -13,7 +13,6 @@ #include -#include #include #include #include @@ -47,10 +46,6 @@ class EventsDatabaseTests : public ::testing::Test { DatabasePlugin::setAllowOpen(true); DatabasePlugin::initPlugin(); - RegistryFactory::get().registry("config_parser")->setUp(); - optimize_ = FLAGS_events_optimize; - FLAGS_events_optimize = false; - std::vector event_keys; scanDatabaseKeys(kEvents, event_keys); for (const auto& key : event_keys) { @@ -355,7 +350,7 @@ TEST_F(EventsDatabaseTests, test_optimize) { FLAGS_events_optimize = true; // Must also define an executing query. - setDatabaseValue(kPersistentSettings, kExecutingQuery, "events_db_test"); + setDatabaseValue(kPersistentSettings, "", "events_db_test"); auto t = getUnixTime(); auto results = genRows(sub.get()); diff --git a/src/osquery/events/tests/events_tests.cpp b/src/osquery/events/tests/events_tests.cpp index 34553f4..5341a15 100644 --- a/src/osquery/events/tests/events_tests.cpp +++ b/src/osquery/events/tests/events_tests.cpp @@ -11,7 +11,6 @@ #include #include -#include #include #include #include @@ -448,45 +447,6 @@ TEST_F(EventsTests, test_event_subscriber_context) { EXPECT_TRUE(status.ok()); } -TEST_F(EventsTests, test_event_subscriber_configure) { - auto sub = std::make_shared(); - // Register this subscriber (within the RegistryFactory), so it receives - // configure/reconfigure events. - auto& rf = RegistryFactory::get(); - rf.registry("event_subscriber")->add("fake_events", sub); - - // Register it within the event factory too. - auto status = EventFactory::registerEventSubscriber(sub); - EXPECT_TRUE(status.ok()); - - // Assure we start from a base state. - EXPECT_EQ(sub->timesConfigured, 0U); - // Force the config into a loaded state. - Config::get().loaded_ = true; - Config::get().update({{"data", "{}"}}); - EXPECT_EQ(sub->timesConfigured, 1U); - - // Now update the config to contain sets of scheduled queries. - Config::get().update( - {{"data", - "{\"schedule\": {\"1\": {\"query\": \"select * from fake_events\", " - "\"interval\": 10}, \"2\":{\"query\": \"select * from time, " - "fake_events\", \"interval\": 19}, \"3\":{\"query\": \"select * " - "from fake_events, fake_events\", \"interval\": 5}}}"}}); - - // This will become 19 * 3, rounded up 60. - EXPECT_EQ(sub->min_expiration_, 60U); - EXPECT_EQ(sub->query_count_, 3U); - - // Register it within the event factory too. - EventFactory::deregisterEventSubscriber(sub->getName()); - rf.registry("event_subscriber")->remove(sub->getName()); - - // Final check to make sure updates are not effecting this subscriber. - Config::get().update({{"data", "{}"}}); - EXPECT_EQ(sub->timesConfigured, 2U); -} - TEST_F(EventsTests, test_fire_event) { auto pub = std::make_shared(); pub->setName("BasicPublisher"); diff --git a/src/osquery/include/osquery/config.h b/src/osquery/include/osquery/config.h deleted file mode 100644 index 90097f2..0000000 --- a/src/osquery/include/osquery/config.h +++ /dev/null @@ -1,448 +0,0 @@ -/* - * Copyright (c) 2014, Facebook, Inc. - * All rights reserved. - * - * This source code is licensed under the BSD-style license found in the - * LICENSE file in the root directory of this source tree. An additional grant - * of patent rights can be found in the PATENTS file in the same directory. - * - */ - -#pragma once - -#include -#include -#include - -#include -#include -#include -#include - -#include -#include -#include -#include - -namespace pt = boost::property_tree; - -namespace osquery { - -/// The builder or invoker may change the default config plugin. -DECLARE_string(config_plugin); - -/** - * @brief The osquery config is updated names sources containing JSON. - * - * A ConfigSourceMap is a named mapping from source (the key) to a JSON blob. - * This map is generated by a ConfigPlugin an provided to the Config via an - * update call. ConfigPlugin%s may update the Config asynchronously. - * - * The osquery Config instance will perform source merging by amalgamating - * the JSON literal types (lists and maps) for well known top-level keys. - * The merging will happen in lexicographical order based on source name. - */ -typedef std::map ConfigSourceMap; - -/** - * @brief A native representation of osquery configuration data. - * - * When you use osquery::Config::getInstance(), you are getting a singleton - * handle to interact with the data stored in an instance of this struct. - */ -struct ConfigData { - /// A vector of all of the queries that are scheduled to execute. - std::map schedule; - std::map options; - std::map > files; - /// All data catches optional/plugin-parsed configuration keys. - pt::ptree all_data; -}; - -class ConfigParserPlugin; -typedef std::shared_ptr ConfigPluginRef; - -/** - * @brief A singleton that exposes accessors to osquery's configuration data. - * - * osquery has two types on configurations. Things that don't change during - * the execution of the process should be configured as command-line - * arguments. Things that can change during the lifetime of program execution - * should be defined using the osquery::config::Config class and the pluggable - * plugin interface that is included with it. - */ -class Config : private boost::noncopyable { - public: - /** - * @brief The primary way to access the Config singleton. - * - * osquery::config::Config::getInstance() provides access to the Config - * singleton - * - * @code{.cpp} - * auto config = osquery::config::Config::getInstance(); - * @endcode - * - * @return a singleton instance of Config. - */ - static Config& getInstance() { - static Config cfg; - return cfg; - } - - /** - * @brief Call the genConfig method of the config retriever plugin. - * - * This may perform a resource load such as TCP request or filesystem read. - */ - static Status load(); - - /** - * @brief Update the internal config data. - * - * @param config A map of domain or namespace to config data. - * @return If the config changes were applied. - */ - static Status update(const ConfigSourceMap& config); - - /** - * @brief Calculate the has of the osquery config - * - * @return The MD5 of the osquery config - */ - static Status getMD5(std::string& hashString); - - /** - * @brief Adds a new query to the scheduled queries. - * - */ - static void addScheduledQuery(const std::string& name, - const std::string& query, - int interval); - - /** - * @brief Checks if a query exists in the query schedule. - * - */ - static bool checkScheduledQuery(const std::string& query); - - /** - * @brief Checks if the query name exists in the query schedule. - * - */ - static bool checkScheduledQueryName(const std::string& query_name); - - /** - * @brief Check to ensure that the config is accessible and properly - * formatted - * - * @return an instance of osquery::Status, indicating the success or failure - * of the operation. - */ - static Status checkConfig(); - - private: - /** - * @brief Default constructor. - * - * Since instances of Config should only be created via getInstance(), - * Config's constructor is private - */ - Config() : force_merge_success_(false) {} - ~Config(){} - Config(Config const&); - void operator=(Config const&); - - /** - * @brief Uses the specified config retriever to populate a string with the - * config JSON. - * - * Internally, genConfig checks to see if there was a config retriever - * specified on the command-line. If there was, it checks to see if that - * config retriever actually exists. If it does, it gets used to generate - * configuration data. If it does not, an error is logged. - * - * @return status indicating the success or failure of the operation. - */ - static Status genConfig(); - - /// Merge a retrieved config source JSON into a working ConfigData. - static Status mergeConfig(const std::string& source, ConfigData& conf); - - public: - /** - * @brief Record performance (monitoring) information about a scheduled query. - * - * The daemon and query scheduler will optionally record process metadata - * before and after executing each query. This can be compared and reported - * on an interval or within the osquery_schedule table. - * - * The config consumes and calculates the optional performance differentials. - * It would also be possible to store this in the RocksDB backing store or - * report directly to a LoggerPlugin sink. The Config is the most appropriate - * as the metrics are transient to the process running the schedule and apply - * to the updates/changes reflected in the schedule, from the config. - * - * @param name The unique name of the scheduled item - * @param delay Number of seconds (wall time) taken by the query - * @param size Number of characters generated by query - * @param t0 the process row before the query - * @param t1 the process row after the query - */ - static void recordQueryPerformance(const std::string& name, - size_t delay, - size_t size, - const Row& t0, - const Row& t1); - - private: - /// The raw osquery config data in a native format - ConfigData data_; - - /// The raw JSON source map from the config plugin. - std::map raw_; - - /// The reader/writer config data mutex. - boost::shared_mutex mutex_; - - /// Enforce merge success. - bool force_merge_success_; - - private: - /** - * @brief A ConfigDataInstance requests read-only access to ConfigParser data. - * - * A ConfigParser plugin will receive several top-level-config keys and - * optionally parse and store information. That information is a property tree - * called ConfigParser::data_. Use ConfigDataInstance::getParsedData to - * retrieve read-only access to this data. - * - * @param parser The name of the config parser. - */ - static const pt::ptree& getParsedData(const std::string& parser); - - /// See getParsedData but request access to the parser plugin. - static const ConfigPluginRef getParser(const std::string& parser); - - /// A default, empty property tree used when a missing parser is requested. - pt::ptree empty_data_; - - private: - /// Config accessors, `ConfigDataInstance`, are the forced use of the config - /// data. This forces the caller to use a shared read lock. - friend class ConfigDataInstance; - - private: - FRIEND_TEST(ConfigTests, test_locking); -}; - -/** - * @brief All accesses to the Config's data must request a ConfigDataInstance. - * - * This class will request a read-only lock of the config's changeable internal - * data structures such as query schedule, options, monitored files, etc. - * - * Since a variable config plugin may implement `update` calls, internal uses - * of config data needs simple read and write locking. - */ -class ConfigDataInstance { - public: - ConfigDataInstance() : lock_(Config::getInstance().mutex_) {} - ~ConfigDataInstance() { lock_.unlock(); } - - /// Helper accessor for Config::data_.schedule. - const std::map schedule() const { - return Config::getInstance().data_.schedule; - } - - /// Helper accessor for Config::data_.options. - const std::map& options() const { - return Config::getInstance().data_.options; - } - - /// Helper accessor for Config::data_.files. - const std::map >& files() const { - return Config::getInstance().data_.files; - } - - const pt::ptree& getParsedData(const std::string& parser) const { - return Config::getParsedData(parser); - } - - const ConfigPluginRef getParser(const std::string& parser) const { - return Config::getParser(parser); - } - - /// Helper accessor for Config::data_.all_data. - const pt::ptree& data() const { return Config::getInstance().data_.all_data; } - - private: - /** - * @brief ConfigParser plugin's may update the internal config representation. - * - * If the config parser reads and calculates new information it should store - * that derived data itself and rely on ConfigDataInstance::getParsedData. - * This means another plugin is aware of the ConfigParser and knowns to make - * getParsedData calls. If the parser is augmenting/changing internal state, - * such as modifying the osquery schedule or options, then it must write - * changed back into the default data. - * - * Note that this returns the ConfigData instance, not the raw property tree. - */ - ConfigData& mutableConfigData() { return Config::getInstance().data_; } - - private: - /// A read lock on the reader/writer config data accessor/update mutex. - boost::shared_lock lock_; - - private: - friend class ConfigParserPlugin; -}; - -/** - * @brief Superclass for the pluggable config component. - * - * In order to make the distribution of configurations to hosts running - * osquery, we take advantage of a plugin interface which allows you to - * integrate osquery with your internal configuration distribution mechanisms. - * You may use ZooKeeper, files on disk, a custom solution, etc. In order to - * use your specific configuration distribution system, one simply needs to - * create a custom subclass of ConfigPlugin. That subclass should implement - * the ConfigPlugin::genConfig method. - * - * Consider the following example: - * - * @code{.cpp} - * class TestConfigPlugin : public ConfigPlugin { - * public: - * virtual std::pair genConfig() { - * std::string config; - * auto status = getMyConfig(config); - * return std::make_pair(status, config); - * } - * }; - * - * REGISTER(TestConfigPlugin, "config", "test"); - * @endcode - */ -class ConfigPlugin : public Plugin { - public: - /** - * @brief Virtual method which should implemented custom config retrieval - * - * ConfigPlugin::genConfig should be implemented by a subclasses of - * ConfigPlugin which needs to retrieve config data in a custom way. - * - * @param config The output ConfigSourceMap, a map of JSON to source names. - * @return A failure status will prevent the source map from merging. - */ - virtual Status genConfig(ConfigSourceMap& config) = 0; - Status call(const PluginRequest& request, PluginResponse& response) override; -}; - -/// Helper merged and parsed property tree. -typedef pt::ptree ConfigTree; - -/// Helper for a map of requested keys to their merged and parsed property tree. -typedef std::map ConfigTreeMap; - -/** - * @brief A pluggable configuration parser. - * - * An osquery config instance is populated from JSON using a ConfigPlugin. - * That plugin may update the config data asynchronously and read from - * several sources, as is the case with "filesystem" and reading multiple files. - * - * A ConfigParserPlugin will receive the merged configuration at osquery start - * and the updated (still merged) config if any ConfigPlugin updates the - * instance asynchronously. Each parser specifies a set of top-level JSON - * keys to receive. The config instance will auto-merge the key values - * from multiple sources if they are dictionaries or lists. - * - * If a top-level key is a dictionary, each source with the top-level key - * will have its own dictionary keys merged and replaced based on the lexical - * order of sources. For the "filesystem" config plugin this is the lexical - * sorting of filenames. If the top-level key is a list, each source with the - * top-level key will have its contents appended. - * - * Each config parser plugin will live alongside the config instance for the - * life of the osquery process. The parser may perform actions at config load - * and config update "time" as well as keep its own data members and be - * accessible through the Config class API. - */ -class ConfigParserPlugin : public Plugin { - protected: - /** - * @brief Return a list of top-level config keys to receive in updates. - * - * The ::update method will receive a map of these keys with a JSON-parsed - * property tree of configuration data. - * - * @return A list of string top-level JSON keys. - */ - virtual std::vector keys() = 0; - - /** - * @brief Receive a merged property tree for each top-level config key. - * - * Called when the Config instance is initially loaded with data from the - * active config plugin and when it is updated via an async ConfigPlugin - * update. Every config parser will receive a map of merged data for each key - * they requested in keys(). - * - * @param config A JSON-parsed property tree map. - * @return Failure if the parser should no longer receive updates. - */ - virtual Status update(const ConfigTreeMap& config) = 0; - - protected: - /// Mutable config data accessor for ConfigParser%s. - ConfigData& mutableConfigData(ConfigDataInstance& cdi) { - return cdi.mutableConfigData(); - } - - protected: - /// Allow the config parser to keep some global state. - pt::ptree data_; - - private: - Status setUp(); - - private: - /// Config::update will call all appropriate parser updates. - friend class Config; - /// A config data instance implements a read/write lock around data_ access. - friend class ConfigDataInstance; -}; - -/** - * @brief Calculate a splayed integer based on a variable splay percentage - * - * The value of splayPercent must be between 1 and 100. If it's not, the - * value of original will be returned. - * - * @param original The original value to be modified - * @param splayPercent The percent in which to splay the original value by - * - * @return The modified version of original - */ -int splayValue(int original, int splayPercent); - -/** - * @brief Config plugin registry. - * - * This creates an osquery registry for "config" which may implement - * ConfigPlugin. A ConfigPlugin's call API should make use of a genConfig - * after reading JSON data in the plugin implementation. - */ -CREATE_REGISTRY(ConfigPlugin, "config"); - -/** - * @brief ConfigParser plugin registry. - * - * This creates an osquery registry for "config_parser" which may implement - * ConfigParserPlugin. A ConfigParserPlugin should not export any call actions - * but rather have a simple property tree-accessor API through Config. - */ -CREATE_LAZY_REGISTRY(ConfigParserPlugin, "config_parser"); -} diff --git a/src/osquery/plugins/CMakeLists.txt b/src/osquery/plugins/CMakeLists.txt index 5d5b333..c6c418e 100644 --- a/src/osquery/plugins/CMakeLists.txt +++ b/src/osquery/plugins/CMakeLists.txt @@ -4,9 +4,6 @@ # This source code is licensed in accordance with the terms specified in # the LICENSE file found in the root directory of this source tree. -ADD_OSQUERY_LIBRARY(osquery_pluing_config config/parsers/decorators.cpp - config/filesystem_config.cpp) - ADD_OSQUERY_LIBRARY(osquery_pluing_logger logger/filesystem_logger.cpp) ADD_OSQUERY_LIBRARY(osquery_plugin_db database/ephemeral.cpp diff --git a/src/osquery/plugins/config/filesystem_config.cpp b/src/osquery/plugins/config/filesystem_config.cpp deleted file mode 100644 index 3fe2450..0000000 --- a/src/osquery/plugins/config/filesystem_config.cpp +++ /dev/null @@ -1,118 +0,0 @@ -/** - * Copyright (c) 2014-present, Facebook, Inc. - * All rights reserved. - * - * This source code is licensed as defined on the LICENSE file found in the - * root directory of this source tree. - */ - -#include - -#include -#include -#include - -#include -#include -#include -#include -#include -#include - -#include - -namespace errc = boost::system::errc; -namespace fs = boost::filesystem; -namespace pt = boost::property_tree; - -namespace osquery { - -CLI_FLAG(string, - config_path, - (fs::path(OSQUERY_HOME) / "osquery.conf").make_preferred().string(), - "Path to JSON config file"); - -class FilesystemConfigPlugin : public ConfigPlugin { - public: - Status genConfig(std::map& config); - Status genPack(const std::string& name, - const std::string& value, - std::string& pack); -}; - -REGISTER(FilesystemConfigPlugin, "config", "filesystem"); - -Status FilesystemConfigPlugin::genConfig( - std::map& config) { - boost::system::error_code ec; - if (!fs::is_regular_file(FLAGS_config_path, ec) || - ec.value() != errc::success) { - return Status::failure("config file does not exist: " + FLAGS_config_path); - } - - std::vector conf_files; - resolveFilePattern(FLAGS_config_path + ".d/%.conf", conf_files); - std::sort(conf_files.begin(), conf_files.end()); - conf_files.push_back(FLAGS_config_path); - - for (const auto& path : conf_files) { - std::string content; - if (readFile(path, content).ok()) { - config[path] = content; - } - } - - return Status(0, "OK"); -} - -Status FilesystemConfigPlugin::genPack(const std::string& name, - const std::string& value, - std::string& pack) { - if (name == "*") { - // The config requested a multi-pack. - std::vector paths; - resolveFilePattern(value, paths); - - pt::ptree multi_pack; - for (const auto& path : paths) { - std::string content; - if (!readFile(path, content)) { - LOG(WARNING) << "Cannot read multi-pack file: " << path; - continue; - } - - // Assemble an intermediate property tree for simplified parsing. - pt::ptree single_pack; - stripConfigComments(content); - try { - std::stringstream json_stream; - json_stream << content; - pt::read_json(json_stream, single_pack); - } catch (const pt::json_parser::json_parser_error& /* e */) { - LOG(WARNING) << "Cannot read multi-pack JSON: " << path; - continue; - } - - multi_pack.put_child(fs::path(path).stem().string(), single_pack); - } - - // We should have a property tree of pack content mimicking embedded - // configuration packs, ready to parse as a string. - std::ostringstream output; - pt::write_json(output, multi_pack, false); - pack = output.str(); - if (pack.empty()) { - return Status(1, "Multi-pack content empty"); - } - - return Status(0); - } - - boost::system::error_code ec; - if (!fs::is_regular_file(value, ec) || ec.value() != errc::success) { - return Status(1, value + " is not a valid path"); - } - - return readFile(value, pack); -} -} // namespace osquery diff --git a/src/osquery/plugins/config/parsers/decorators.cpp b/src/osquery/plugins/config/parsers/decorators.cpp deleted file mode 100644 index 6b72171..0000000 --- a/src/osquery/plugins/config/parsers/decorators.cpp +++ /dev/null @@ -1,302 +0,0 @@ -/** - * Copyright (c) 2014-present, Facebook, Inc. - * All rights reserved. - * - * This source code is licensed in accordance with the terms specified in - * the LICENSE file found in the root directory of this source tree. - */ - -#include -#include -#include -#include -#include -#include -#include - -namespace osquery { - -FLAG(bool, disable_decorators, false, "Disable log result decoration"); - -FLAG(bool, - decorations_top_level, - false, - "Add decorators as top level JSON objects"); - -/// Statically define the parser name to avoid mistakes. -const std::string kDecorationsName{"decorators"}; - -const std::map kDecorationPointKeys = { - {DECORATE_LOAD, "load"}, - {DECORATE_ALWAYS, "always"}, - {DECORATE_INTERVAL, "interval"}, -}; - -using KeyValueMap = std::map; -using DecorationStore = std::map; - -namespace { - -/** - * @brief A simple ConfigParserPlugin for a "decorators" dictionary key. - * - * Decorators append data to results, snapshots, and status log lines. - * They can be used to add arbitrary additional datums within the 'decorators' - * subkey. - * - * Decorators come in three basic flavors, defined by when they are run: - * load: run these decorators when the config is loaded. - * always: run these decorators for every query immediate before - * interval: run these decorators on an interval. - * - * When 'interval' is used, the value is a dictionary of intervals, each of the - * subkeys are treated as the requested interval in sections. The internals - * are emulated by the query schedule. - * - * Decorators are sets of queries, and each selected column within the set is - * added to the 'decorators' dictionary. Including two queries with the same - * column name is undefined behavior and will most likely lead to either - * duplicate keys or overwriting. Issuing a query that emits more than one row - * will also lead to undefined behavior. The decorator executor will ignore any - * rows past the first. - */ -class DecoratorsConfigParserPlugin : public ConfigParserPlugin { - public: - std::vector keys() const override { - return {kDecorationsName}; - } - - Status setUp() override; - - Status update(const std::string& source, const ParserConfig& config) override; - - /// Update the set of decorators for a given source. - void updateDecorations(const std::string& source, const JSON& doc); - - /// Clear the decorations created from decorators for the given source. - void clearSources(const std::string& source); - - /// Clear all decorations. - virtual void reset() override; - - public: - /// Set of configuration sources to the set of decorator queries. - std::map> always_; - - /// Set of configuration sources to the set of on-load decorator queries. - std::map> load_; - - /// Set of configuration sources to valid intervals. - std::map>> intervals_; - - public: - /// The result set of decorations, column names and their values. - static DecorationStore kDecorations; - - /// Protect additions to the decorator set. - static Mutex kDecorationsMutex; - - /// Protect the configuration controlled content. - static Mutex kDecorationsConfigMutex; -}; -} - -DecorationStore DecoratorsConfigParserPlugin::kDecorations; -Mutex DecoratorsConfigParserPlugin::kDecorationsMutex; -Mutex DecoratorsConfigParserPlugin::kDecorationsConfigMutex; - -Status DecoratorsConfigParserPlugin::setUp() { - // Decorators are kept within customized data structures. - // No need to define a key for the ::getData API. - return Status(0, "OK"); -} - -Status DecoratorsConfigParserPlugin::update(const std::string& source, - const ParserConfig& config) { - clearSources(source); - clearDecorations(source); - auto decorations = config.find(kDecorationsName); - if (decorations != config.end()) { - // Each of these methods acquires the decorator lock separately. - // The run decorators method is designed to have call sites throughout - // the code base. - updateDecorations(source, decorations->second); - runDecorators(DECORATE_LOAD, 0, source); - } - - return Status(0, "OK"); -} - -void DecoratorsConfigParserPlugin::clearSources(const std::string& source) { - // Reset the internal data store. - WriteLock lock(DecoratorsConfigParserPlugin::kDecorationsConfigMutex); - if (intervals_.count(source) > 0) { - intervals_[source].clear(); - } - - if (always_.count(source) > 0) { - always_[source].clear(); - } - - if (load_.count(source) > 0) { - load_[source].clear(); - } -} - -void DecoratorsConfigParserPlugin::reset() { - // Reset the internal data store (for all sources). - for (const auto& source : DecoratorsConfigParserPlugin::kDecorations) { - clearSources(source.first); - clearDecorations(source.first); - } -} - -void DecoratorsConfigParserPlugin::updateDecorations(const std::string& source, - const JSON& doc) { - WriteLock lock(DecoratorsConfigParserPlugin::kDecorationsConfigMutex); - // Assign load decorators. - auto& load_key = kDecorationPointKeys.at(DECORATE_LOAD); - if (doc.doc().HasMember(load_key.c_str())) { - auto& load = doc.doc()[load_key.c_str()]; - if (load.IsArray()) { - for (const auto& item : load.GetArray()) { - if (item.IsString()) { - load_[source].push_back(item.GetString()); - } - } - } - } - - // Assign always decorators. - auto& always_key = kDecorationPointKeys.at(DECORATE_ALWAYS); - if (doc.doc().HasMember(always_key.c_str())) { - auto& always = doc.doc()[always_key.c_str()]; - if (always.IsArray()) { - for (const auto& item : always.GetArray()) { - if (item.IsString()) { - always_[source].push_back(item.GetString()); - } - } - } - } - - // Check if intervals are defined. - auto& interval_key = kDecorationPointKeys.at(DECORATE_INTERVAL); - if (doc.doc().HasMember(interval_key.c_str())) { - const auto& interval = doc.doc()[interval_key.c_str()]; - if (interval.IsObject()) { - for (const auto& item : interval.GetObject()) { - auto rate = doc.valueToSize(item.name); - // size_t rate = std::stoll(item.name.GetString()); - if (rate % 60 != 0) { - LOG(WARNING) << "Invalid decorator interval rate " << rate - << " in config source: " << source; - continue; - } - - // This is a valid interval, update the set of intervals to include - // this value. When intervals are checked this set is scanned, if a - // match is found, then the associated config data is executed. - if (item.value.IsArray()) { - for (const auto& interval_query : item.value.GetArray()) { - if (interval_query.IsString()) { - intervals_[source][rate].push_back(interval_query.GetString()); - } - } - } - } - } - } -} - -inline void addDecoration(const std::string& source, - const std::string& name, - const std::string& value) { - WriteLock lock(DecoratorsConfigParserPlugin::kDecorationsMutex); - DecoratorsConfigParserPlugin::kDecorations[source][name] = value; -} - -inline void runDecorators(const std::string& source, - const std::vector& queries) { - for (const auto& query : queries) { - SQL results(query); - if (results.rows().size() > 0) { - // Notice the warning above about undefined behavior when: - // 1: You include decorators that emit the same column name - // 2: You include a query that returns more than 1 row. - for (const auto& column : results.rows()[0]) { - addDecoration(source, column.first, column.second); - } - } - - if (results.rows().size() > 1) { - // Multiple rows exhibit undefined behavior. - LOG(WARNING) << "Multiple rows returned for decorator query: " << query; - } - } -} - -void clearDecorations(const std::string& source) { - WriteLock lock(DecoratorsConfigParserPlugin::kDecorationsMutex); - DecoratorsConfigParserPlugin::kDecorations[source].clear(); -} - -void runDecorators(DecorationPoint point, - size_t time, - const std::string& source) { - if (FLAGS_disable_decorators) { - return; - } - - auto parser = Config::getParser(kDecorationsName); - if (parser == nullptr) { - // The decorators parser does not exist. - return; - } - - // Abstract the use of the decorator parser API. - ReadLock lock(DecoratorsConfigParserPlugin::kDecorationsConfigMutex); - auto dp = std::dynamic_pointer_cast(parser); - if (point == DECORATE_LOAD) { - for (const auto& target_source : dp->load_) { - if (source.empty() || target_source.first == source) { - runDecorators(target_source.first, target_source.second); - } - } - } else if (point == DECORATE_ALWAYS) { - for (const auto& target_source : dp->always_) { - if (source.empty() || target_source.first == source) { - runDecorators(target_source.first, target_source.second); - } - } - } else if (point == DECORATE_INTERVAL) { - for (const auto& target_source : dp->intervals_) { - for (const auto& interval : target_source.second) { - if (time % interval.first == 0) { - if (source.empty() || target_source.first == source) { - runDecorators(target_source.first, interval.second); - } - } - } - } - } -} - -void getDecorations(std::map& results) { - if (FLAGS_disable_decorators) { - return; - } - - ReadLock lock(DecoratorsConfigParserPlugin::kDecorationsMutex); - // Copy the decorations into the log_item. - for (const auto& source : DecoratorsConfigParserPlugin::kDecorations) { - for (const auto& decoration : source.second) { - results[decoration.first] = decoration.second; - } - } -} - -REGISTER_INTERNAL(DecoratorsConfigParserPlugin, - "config_parser", - kDecorationsName.c_str()); -} diff --git a/src/osquery/plugins/config/parsers/decorators.h b/src/osquery/plugins/config/parsers/decorators.h deleted file mode 100644 index 9c7d367..0000000 --- a/src/osquery/plugins/config/parsers/decorators.h +++ /dev/null @@ -1,58 +0,0 @@ -/** - * Copyright (c) 2014-present, Facebook, Inc. - * All rights reserved. - * - * This source code is licensed in accordance with the terms specified in - * the LICENSE file found in the root directory of this source tree. - */ - -#pragma once - -#include -#include - -#include -#include - -namespace osquery { - -/// Enforce specific types of decoration. -enum DecorationPoint { - DECORATE_LOAD, - DECORATE_ALWAYS, - DECORATE_INTERVAL, -}; - -/// Define a map of decoration points to their expected configuration key. -extern const std::map kDecorationPointKeys; - -/** - * @brief Iterate the discovered decorators for a given point type. - * - * The configuration maintains various sources, each may contain a set of - * decorators. The source tracking is abstracted for the decorator iterator. - * - * @param point request execution of decorators for this given point. - * @param time an optional time for points using intervals. - * @param source restrict run to a specific config source. - */ -void runDecorators(DecorationPoint point, - size_t time = 0, - const std::string& source = ""); - -/** - * @brief Access the internal storage of the Decorator parser. - * - * The decoration set is a map of column name to value. It contains the opaque - * set of decoration point results. - * - * Decorations are applied to log items before they are sent to the downstream - * logging APIs: logString, logSnapshot, etc. - * - * @param results the output parameter to write decorations. - */ -void getDecorations(std::map& results); - -/// Clear decorations for a source when it updates. -void clearDecorations(const std::string& source); -} diff --git a/src/osquery/tests/test_util.cpp b/src/osquery/tests/test_util.cpp index 59b290f..83a9aa9 100644 --- a/src/osquery/tests/test_util.cpp +++ b/src/osquery/tests/test_util.cpp @@ -51,8 +51,6 @@ DECLARE_bool(disable_database); using chrono_clock = std::chrono::high_resolution_clock; void initTesting() { - Config::setStartTime(getUnixTime()); - kToolType = ToolType::TEST; if (osquery::isPlatform(PlatformType::TYPE_OSX)) { kTestWorkingDirectory = "/private/tmp/osquery-tests"; diff --git a/src/osquery/tests/test_util.h b/src/osquery/tests/test_util.h index fe2be45..4de9428 100644 --- a/src/osquery/tests/test_util.h +++ b/src/osquery/tests/test_util.h @@ -12,7 +12,6 @@ #include #include -#include #include #include #include -- 2.7.4