ENDIF(DEFINED GBS_BUILD)
## osquery v4.0.0
-ADD_SUBDIRECTORY(config)
ADD_SUBDIRECTORY(core)
ADD_SUBDIRECTORY(database)
ADD_SUBDIRECTORY(dispatcher)
+++ /dev/null
-# Copyright (c) 2019 Samsung Electronics Co., Ltd All Rights Reserved
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License
-
-ADD_OSQUERY_LIBRARY(osquery_config config.cpp
- packs.cpp)
-
-FILE(GLOB OSQUERY_CONFIG_TESTS "tests/*.cpp")
-ADD_OSQUERY_TEST(${OSQUERY_CONFIG_TESTS})
+++ /dev/null
-/**
- * Copyright (c) 2014-present, Facebook, Inc.
- * All rights reserved.
- *
- * This source code is licensed in accordance with the terms specified in
- * the LICENSE file found in the root directory of this source tree.
- */
-
-#include <algorithm>
-#include <chrono>
-#include <functional>
-#include <map>
-#include <string>
-#include <vector>
-
-#include <boost/algorithm/string/replace.hpp>
-#include <boost/algorithm/string/trim.hpp>
-#include <boost/iterator/filter_iterator.hpp>
-
-#include <osquery/config/config.h>
-#include <osquery/database.h>
-#include <osquery/events.h>
-#include <osquery/flagalias.h>
-#include <osquery/flags.h>
-#include <osquery/logger.h>
-#include <osquery/packs.h>
-#include <osquery/registry.h>
-#include <osquery/system.h>
-#include <osquery/tables.h>
-#include <osquery/utils/conversions/split.h>
-#include <osquery/utils/conversions/tryto.h>
-#include <osquery/utils/system/time.h>
-
-namespace rj = rapidjson;
-
-namespace osquery {
-namespace {
-/// Prefix to persist config data
-const std::string kConfigPersistencePrefix{"config_persistence."};
-
-using ConfigMap = std::map<std::string, std::string>;
-
-std::atomic<bool> is_first_time_refresh(true);
-}; // namespace
-
-/**
- * @brief Config plugin registry.
- *
- * This creates an osquery registry for "config" which may implement
- * ConfigPlugin. A ConfigPlugin's call API should make use of a genConfig
- * after reading JSON data in the plugin implementation.
- */
-CREATE_REGISTRY(ConfigPlugin, "config");
-
-/**
- * @brief ConfigParser plugin registry.
- *
- * This creates an osquery registry for "config_parser" which may implement
- * ConfigParserPlugin. A ConfigParserPlugin should not export any call actions
- * but rather have a simple property tree-accessor API through Config.
- */
-CREATE_LAZY_REGISTRY(ConfigParserPlugin, "config_parser");
-
-/// The config plugin must be known before reading options.
-CLI_FLAG(string, config_plugin, "filesystem", "Config plugin name");
-
-CLI_FLAG(bool,
- config_check,
- false,
- "Check the format of an osquery config and exit");
-
-CLI_FLAG(bool, config_dump, false, "Dump the contents of the configuration");
-
-CLI_FLAG(uint64,
- config_refresh,
- 0,
- "Optional interval in seconds to re-read configuration");
-FLAG_ALIAS(google::uint64, config_tls_refresh, config_refresh);
-
-/// How long to wait when config update fails
-CLI_FLAG(uint64,
- config_accelerated_refresh,
- 300,
- "Interval to wait if reading a configuration fails");
-
-CLI_FLAG(bool,
- config_enable_backup,
- false,
- "Backup config and use it when refresh fails");
-
-FLAG_ALIAS(google::uint64,
- config_tls_accelerated_refresh,
- config_accelerated_refresh);
-
-DECLARE_string(config_plugin);
-DECLARE_string(pack_delimiter);
-
-/**
- * @brief The backing store key name for the executing query.
- *
- * The config maintains schedule statistics and tracks failed executions.
- * On process or worker resume an initializer or config may check if the
- * resume was the result of a failure during an executing query.
- */
-const std::string kExecutingQuery{"executing_query"};
-const std::string kFailedQueries{"failed_queries"};
-
-/// The time osquery was started.
-std::atomic<size_t> kStartTime;
-
-// The config may be accessed and updated asynchronously; use mutexes.
-Mutex config_hash_mutex_;
-Mutex config_refresh_mutex_;
-Mutex config_backup_mutex_;
-
-/// Several config methods require enumeration via predicate lambdas.
-RecursiveMutex config_schedule_mutex_;
-RecursiveMutex config_files_mutex_;
-RecursiveMutex config_performance_mutex_;
-
-using PackRef = std::unique_ptr<Pack>;
-
-/**
- * The schedule is an iterable collection of Packs. When you iterate through
- * a schedule, you only get the packs that should be running on the host that
- * you're currently operating on.
- */
-class Schedule : private boost::noncopyable {
- public:
- /// Under the hood, the schedule is just a list of the Pack objects
- using container = std::vector<PackRef>;
-
- /**
- * @brief Create a schedule maintained by the configuration.
- *
- * This will check for previously executing queries. If any query was
- * executing it is considered in a 'dirty' state and should generate logs.
- * The schedule may also choose to blacklist this query.
- */
- Schedule();
-
- /**
- * @brief This class' iteration function
- *
- * Our step operation will be called on each element in packs_. It is
- * responsible for determining if that element should be returned as the
- * next iterator element or skipped.
- */
- struct Step {
- bool operator()(const PackRef& pack) const;
- };
-
- /// Add a pack to the schedule
- void add(PackRef pack);
-
- /// Remove a pack, by name.
- void remove(const std::string& pack);
-
- /// Remove a pack by name and source.
- void remove(const std::string& pack, const std::string& source);
-
- /// Remove all packs by source.
- void removeAll(const std::string& source);
-
- /// Boost gives us a nice template for maintaining the state of the iterator
- using iterator = boost::filter_iterator<Step, container::iterator>;
-
- iterator begin();
-
- iterator end();
-
- PackRef& last();
-
- private:
- /// Underlying storage for the packs
- container packs_;
-
- /**
- * @brief The schedule will check and record previously executing queries.
- *
- * If a query is found on initialization, the name will be recorded, it is
- * possible to skip previously failed queries.
- */
- std::string failed_query_;
-
- /**
- * @brief List of blacklisted queries.
- *
- * A list of queries that are blacklisted from executing due to prior
- * failures. If a query caused a worker to fail it will be recorded during
- * the next execution and saved to the blacklist.
- */
- std::map<std::string, size_t> blacklist_;
-
- private:
- friend class Config;
-};
-
-bool Schedule::Step::operator()(const PackRef& pack) const {
- return pack->shouldPackExecute();
-}
-
-void Schedule::add(PackRef pack) {
- remove(pack->getName(), pack->getSource());
- packs_.push_back(std::move(pack));
-}
-
-void Schedule::remove(const std::string& pack) {
- remove(pack, "");
-}
-
-void Schedule::remove(const std::string& pack, const std::string& source) {
- auto new_end = std::remove_if(
- packs_.begin(), packs_.end(), [pack, source](const PackRef& p) {
- if (p->getName() == pack &&
- (p->getSource() == source || source == "")) {
- Config::get().removeFiles(source + FLAGS_pack_delimiter +
- p->getName());
- return true;
- }
- return false;
- });
- packs_.erase(new_end, packs_.end());
-}
-
-void Schedule::removeAll(const std::string& source) {
- auto new_end =
- std::remove_if(packs_.begin(), packs_.end(), [source](const PackRef& p) {
- if (p->getSource() == source) {
- Config::get().removeFiles(source + FLAGS_pack_delimiter +
- p->getName());
- return true;
- }
- return false;
- });
- packs_.erase(new_end, packs_.end());
-}
-
-Schedule::iterator Schedule::begin() {
- return Schedule::iterator(packs_.begin(), packs_.end());
-}
-
-Schedule::iterator Schedule::end() {
- return Schedule::iterator(packs_.end(), packs_.end());
-}
-
-PackRef& Schedule::last() {
- return packs_.back();
-}
-
-/**
- * @brief A thread that periodically reloads configuration state.
- *
- * This refresh runner thread can refresh any configuration plugin.
- * It may accelerate the time between checks if the configuration fails to load.
- * For configurations pulled from the network this assures that configuration
- * is fresh when re-attaching.
- */
-class ConfigRefreshRunner : public InternalRunnable {
- public:
- ConfigRefreshRunner() : InternalRunnable("ConfigRefreshRunner") {}
-
- /// A simple wait/interruptible lock.
- void start() override;
-
- private:
- /// The current refresh rate in seconds.
- std::atomic<size_t> refresh_sec_{0};
-
- private:
- friend class Config;
-};
-
-void restoreScheduleBlacklist(std::map<std::string, size_t>& blacklist) {
- std::string content;
- getDatabaseValue(kPersistentSettings, kFailedQueries, content);
- auto blacklist_pairs = osquery::split(content, ":");
- if (blacklist_pairs.size() == 0 || blacklist_pairs.size() % 2 != 0) {
- // Nothing in the blacklist, or malformed data.
- return;
- }
-
- size_t current_time = getUnixTime();
- for (size_t i = 0; i < blacklist_pairs.size() / 2; i++) {
- // Fill in a mapping of query name to time the blacklist expires.
- auto expire =
- tryTo<long long>(blacklist_pairs[(i * 2) + 1], 10).takeOr(0ll);
- if (expire > 0 && current_time < (size_t)expire) {
- blacklist[blacklist_pairs[(i * 2)]] = (size_t)expire;
- }
- }
-}
-
-void saveScheduleBlacklist(const std::map<std::string, size_t>& blacklist) {
- std::string content;
- for (const auto& query : blacklist) {
- if (!content.empty()) {
- content += ":";
- }
- content += query.first + ":" + std::to_string(query.second);
- }
- setDatabaseValue(kPersistentSettings, kFailedQueries, content);
-}
-
-Schedule::Schedule() {
- // Parse the schedule's query blacklist from backing storage.
- restoreScheduleBlacklist(blacklist_);
-
- // Check if any queries were executing when the tool last stopped.
- getDatabaseValue(kPersistentSettings, kExecutingQuery, failed_query_);
- if (!failed_query_.empty()) {
- LOG(WARNING) << "Scheduled query may have failed: " << failed_query_;
- setDatabaseValue(kPersistentSettings, kExecutingQuery, "");
- // Add this query name to the blacklist and save the blacklist.
- blacklist_[failed_query_] = getUnixTime() + 86400;
- saveScheduleBlacklist(blacklist_);
- }
-}
-
-Config::Config()
- : schedule_(std::make_unique<Schedule>()),
- valid_(false),
- refresh_runner_(std::make_shared<ConfigRefreshRunner>()) {}
-
-Config& Config::get() {
- static Config instance;
- return instance;
-}
-
-void Config::addPack(const std::string& name,
- const std::string& source,
- const rj::Value& obj) {
- assert(obj.IsObject());
-
- auto addSinglePack = ([this, &source](const std::string pack_name,
- const rj::Value& pack_obj) {
- RecursiveLock wlock(config_schedule_mutex_);
- try {
- schedule_->add(std::make_unique<Pack>(pack_name, source, pack_obj));
- if (schedule_->last()->shouldPackExecute()) {
- applyParsers(source + FLAGS_pack_delimiter + pack_name, pack_obj, true);
- }
- } catch (const std::exception& e) {
- LOG(WARNING) << "Error adding pack: " << pack_name << ": " << e.what();
- }
- });
-
- if (name == "*") {
- // This is a multi-pack, expect the config plugin to have generated a
- // "name": {pack-content} response similar to embedded pack content
- // within the configuration.
- for (const auto& pack : obj.GetObject()) {
- addSinglePack(pack.name.GetString(), pack.value);
- }
- } else {
- addSinglePack(name, obj);
- }
-}
-
-size_t Config::getStartTime() {
- return kStartTime;
-}
-
-void Config::setStartTime(size_t st) {
- kStartTime = st;
-}
-
-void Config::removePack(const std::string& pack) {
- RecursiveLock wlock(config_schedule_mutex_);
- return schedule_->remove(pack);
-}
-
-void Config::addFile(const std::string& source,
- const std::string& category,
- const std::string& path) {
- RecursiveLock wlock(config_files_mutex_);
- files_[source][category].push_back(path);
-}
-
-void Config::removeFiles(const std::string& source) {
- RecursiveLock wlock(config_files_mutex_);
- if (files_.count(source)) {
- FileCategories().swap(files_[source]);
- }
-}
-
-/**
- * @brief Return true if the failed query is no longer blacklisted.
- *
- * There are two scenarios where a blacklisted query becomes 'unblacklisted'.
- * The first is simple, the amount of time it was blacklisted for has expired.
- * The second is more complex, the query failed but the schedule has requested
- * that the query should not be blacklisted.
- *
- * @param blt The time the query was originally blacklisted.
- * @param query The scheduled query and its options.
- */
-static inline bool blacklistExpired(size_t blt, const ScheduledQuery& query) {
- if (getUnixTime() > blt) {
- return true;
- }
-
- auto blo = query.options.find("blacklist");
- if (blo != query.options.end() && blo->second == false) {
- // The schedule requested that we do not blacklist this query.
- return true;
- }
- return false;
-}
-
-void Config::scheduledQueries(
- std::function<void(std::string name, const ScheduledQuery& query)>
- predicate,
- bool blacklisted) const {
- RecursiveLock lock(config_schedule_mutex_);
- for (PackRef& pack : *schedule_) {
- for (auto& it : pack->getSchedule()) {
- std::string name = it.first;
- // The query name may be synthetic.
- if (pack->getName() != "main") {
- name = "pack" + FLAGS_pack_delimiter + pack->getName() +
- FLAGS_pack_delimiter + it.first;
- }
-
- // They query may have failed and been added to the schedule's blacklist.
- auto blacklisted_query = schedule_->blacklist_.find(name);
- if (blacklisted_query != schedule_->blacklist_.end()) {
- if (blacklistExpired(blacklisted_query->second, it.second)) {
- // The blacklisted query passed the expiration time (remove).
- schedule_->blacklist_.erase(blacklisted_query);
- saveScheduleBlacklist(schedule_->blacklist_);
- it.second.blacklisted = false;
- } else {
- // The query is still blacklisted.
- it.second.blacklisted = true;
- if (!blacklisted) {
- // The caller does not want blacklisted queries.
- continue;
- }
- }
- }
-
- // Call the predicate.
- predicate(std::move(name), it.second);
- }
- }
-}
-
-void Config::packs(std::function<void(const Pack& pack)> predicate) const {
- RecursiveLock lock(config_schedule_mutex_);
- for (PackRef& pack : schedule_->packs_) {
- predicate(std::cref(*pack.get()));
- }
-}
-
-Status Config::refresh() {
- PluginResponse response;
- auto status = Registry::call("config", {{"action", "genConfig"}}, response);
-
- WriteLock lock(config_refresh_mutex_);
- if (!status.ok()) {
- if (FLAGS_config_refresh > 0 && getRefresh() == FLAGS_config_refresh) {
- VLOG(1) << "Using accelerated configuration delay";
- setRefresh(FLAGS_config_accelerated_refresh);
- }
-
- loaded_ = true;
- return status;
- } else if (getRefresh() != FLAGS_config_refresh) {
- VLOG(1) << "Normal configuration delay restored";
- setRefresh(FLAGS_config_refresh);
- }
-
- // if there was a response, parse it and update internal state
- valid_ = true;
- if (response.size() > 0) {
- if (FLAGS_config_dump) {
- // If config checking is enabled, debug-write the raw config data.
- for (const auto& content : response[0]) {
- fprintf(stdout,
- "{\"%s\": %s}\n",
- content.first.c_str(),
- content.second.c_str());
- }
- // Don't force because the config plugin may have started services.
- Initializer::requestShutdown();
- return Status::success();
- }
- status = update(response[0]);
- }
-
- is_first_time_refresh = false;
- loaded_ = true;
- return status;
-}
-
-void Config::setRefresh(size_t refresh_sec) {
- refresh_runner_->refresh_sec_ = refresh_sec;
-}
-
-size_t Config::getRefresh() const {
- return refresh_runner_->refresh_sec_;
-}
-
-Status Config::load() {
- valid_ = false;
- auto config_plugin = RegistryFactory::get().getActive("config");
- if (!RegistryFactory::get().exists("config", config_plugin)) {
- return Status(1, "Missing config plugin " + config_plugin);
- }
-
- // Set the initial and optional refresh value.
- setRefresh(FLAGS_config_refresh);
-
- /*
- * If the initial configuration includes a non-0 refresh, start an
- * additional service that sleeps and periodically regenerates the
- * configuration.
- */
- if (!FLAGS_config_check && !started_thread_ && getRefresh() > 0) {
- Dispatcher::addService(refresh_runner_);
- started_thread_ = true;
- }
-
- return refresh();
-}
-
-void stripConfigComments(std::string& json) {
- std::string sink;
-
- boost::replace_all(json, "\\\n", "");
- for (auto& line : osquery::split(json, "\n")) {
- boost::trim(line);
- if (line.size() > 0 && line[0] == '#') {
- continue;
- }
- if (line.size() > 1 && line[0] == '/' && line[1] == '/') {
- continue;
- }
- sink += line + '\n';
- }
- json = sink;
-}
-
-Expected<ConfigMap, Config::RestoreConfigError> Config::restoreConfigBackup() {
- LOG(INFO) << "Restoring backed up config from the database";
- std::vector<std::string> keys;
- ConfigMap config;
-
- WriteLock lock(config_backup_mutex_);
- scanDatabaseKeys(kPersistentSettings, keys, kConfigPersistencePrefix);
-
- for (const auto& key : keys) {
- std::string value;
- Status status = getDatabaseValue(kPersistentSettings, key, value);
- if (!status.ok()) {
- LOG(ERROR)
- << "restoreConfigBackup database failed to retrieve config for key "
- << key;
- return createError(Config::RestoreConfigError::DatabaseError)
- << "Could not retrieve value for the key: " << key;
- }
- config[key.substr(kConfigPersistencePrefix.length())] = std::move(value);
- }
-
- return config;
-}
-
-void Config::backupConfig(const ConfigMap& config) {
- LOG(INFO) << "BackupConfig started";
- std::vector<std::string> keys;
-
- WriteLock lock(config_backup_mutex_);
- scanDatabaseKeys(kPersistentSettings, keys, kConfigPersistencePrefix);
- for (const auto& key : keys) {
- if (config.find(key.substr(kConfigPersistencePrefix.length())) ==
- config.end()) {
- deleteDatabaseValue(kPersistentSettings, key);
- }
- }
-
- for (const auto& source : config) {
- setDatabaseValue(kPersistentSettings,
- kConfigPersistencePrefix + source.first,
- source.second);
- }
-}
-
-Status Config::updateSource(const std::string& source,
- const std::string& json) {
- {
- RecursiveLock lock(config_schedule_mutex_);
- // Remove all packs from this source.
- schedule_->removeAll(source);
- // Remove all files from this source.
- removeFiles(source);
- }
-
- // load the config (source.second) into a JSON object.
- auto doc = JSON::newObject();
- auto clone = json;
- stripConfigComments(clone);
-
- if (!doc.fromString(clone) || !doc.doc().IsObject()) {
- return Status(1, "Error parsing the config JSON");
- }
-
- // extract the "schedule" key and store it as the main pack
- auto& rf = RegistryFactory::get();
- if (doc.doc().HasMember("schedule")) {
- auto& schedule = doc.doc()["schedule"];
- if (schedule.IsObject()) {
- auto main_doc = JSON::newObject();
- auto queries_obj = main_doc.getObject();
- main_doc.copyFrom(schedule, queries_obj);
- main_doc.add("queries", queries_obj);
- addPack("main", source, main_doc.doc());
- }
- }
-
- // extract the "packs" key into additional pack objects
- if (doc.doc().HasMember("packs")) {
- auto& packs = doc.doc()["packs"];
- if (packs.IsObject()) {
- for (const auto& pack : packs.GetObject()) {
- std::string pack_name = pack.name.GetString();
- if (pack.value.IsObject()) {
- // The pack is a JSON object, treat the content as pack data.
- addPack(pack_name, source, pack.value);
- } else if (pack.value.IsString()) {
- genPack(pack_name, source, pack.value.GetString());
- }
- }
- }
- }
-
- applyParsers(source, doc.doc(), false);
- return Status::success();
-}
-
-Status Config::genPack(const std::string& name,
- const std::string& source,
- const std::string& target) {
- // If the pack value is a string (and not a JSON object) then it is a
- // resource to be handled by the config plugin.
- PluginResponse response;
- PluginRequest request = {
- {"action", "genPack"}, {"name", name}, {"value", target}};
- Registry::call("config", request, response);
-
- if (response.size() == 0 || response[0].count(name) == 0) {
- return Status(1, "Invalid plugin response");
- }
-
- auto clone = response[0][name];
- if (clone.empty()) {
- LOG(WARNING) << "Error reading the query pack named: " << name;
- return Status::success();
- }
-
- stripConfigComments(clone);
- auto doc = JSON::newObject();
- if (!doc.fromString(clone) || !doc.doc().IsObject()) {
- LOG(WARNING) << "Error parsing the \"" << name << "\" pack JSON";
- } else {
- addPack(name, source, doc.doc());
- }
-
- return Status::success();
-}
-
-void Config::applyParsers(const std::string& source,
- const rj::Value& obj,
- bool pack) {
- assert(obj.IsObject());
-
- // Iterate each parser.
- RecursiveLock lock(config_schedule_mutex_);
- for (const auto& plugin : RegistryFactory::get().plugins("config_parser")) {
- std::shared_ptr<ConfigParserPlugin> parser = nullptr;
- try {
- parser = std::dynamic_pointer_cast<ConfigParserPlugin>(plugin.second);
- } catch (const std::bad_cast& /* e */) {
- LOG(ERROR) << "Error casting config parser plugin: " << plugin.first;
- }
- if (parser == nullptr || parser.get() == nullptr) {
- continue;
- }
-
- // For each key requested by the parser, add a property tree reference.
- std::map<std::string, JSON> parser_config;
- for (const auto& key : parser->keys()) {
- if (obj.HasMember(key.c_str()) && !obj[key.c_str()].IsNull()) {
- if (!obj[key.c_str()].IsArray() && !obj[key.c_str()].IsObject()) {
- LOG(WARNING) << "Error config " << key
- << " should be an array or object";
- continue;
- }
-
- auto doc = JSON::newFromValue(obj[key.c_str()]);
- parser_config.emplace(std::make_pair(key, std::move(doc)));
- }
- }
- // The config parser plugin will receive a copy of each property tree for
- // each top-level-config key. The parser may choose to update the config's
- // internal state
- parser->update(source, parser_config);
- }
-}
-
-Status Config::update(const ConfigMap& config) {
- // Iterate though each source and overwrite config data.
- // This will add/overwrite pack data, append to the schedule, change watched
- // files, set options, etc.
- // Before this occurs, take an opportunity to purge stale state.
- purge();
-
- bool needs_reconfigure = false;
- for (const auto& source : config) {
- auto status = updateSource(source.first, source.second);
- if (status.getCode() == 2) {
- // The source content did not change.
- continue;
- }
-
- if (!status.ok()) {
- LOG(ERROR) << "updateSource failed to parse config, of source: "
- << source.first << " and content: " << source.second;
- return status;
- }
- // If a source was updated and the content has changed, then the registry
- // should be reconfigured. File watches may have changed, etc.
- needs_reconfigure = true;
- }
-
- if (loaded_ && needs_reconfigure) {
- // The config has since been loaded.
- // This update call is most likely a response to an async update request
- // from a config plugin. This request should request all plugins to update.
- for (const auto& registry : RegistryFactory::get().all()) {
- if (registry.first == "event_publisher" ||
- registry.first == "event_subscriber") {
- continue;
- }
- registry.second->configure();
- }
-
- EventFactory::configUpdate();
- }
-
- // This cannot be under the previous if block because on extensions loaded_
- // allways false.
- if (needs_reconfigure) {
- std::string loggers = RegistryFactory::get().getActive("logger");
- for (const auto& logger : osquery::split(loggers, ",")) {
- LOG(INFO) << "Calling configure for logger " << logger;
- PluginRef plugin = Registry::get().plugin("logger", logger);
-
- if (plugin) {
- plugin->configure();
- }
- }
- }
-
- if (FLAGS_config_enable_backup) {
- backupConfig(config);
- }
-
- return Status::success();
-}
-
-void Config::purge() {
- // The first use of purge is removing expired query results.
- std::vector<std::string> saved_queries;
- scanDatabaseKeys(kQueries, saved_queries);
-
- auto queryExists = [schedule = static_cast<const Schedule*>(schedule_.get())](
- const std::string& query_name) {
- for (const auto& pack : schedule->packs_) {
- const auto& pack_queries = pack->getSchedule();
- if (pack_queries.count(query_name)) {
- return true;
- }
- }
- return false;
- };
-
- RecursiveLock lock(config_schedule_mutex_);
- // Iterate over each result set in the database.
- for (const auto& saved_query : saved_queries) {
- if (queryExists(saved_query)) {
- continue;
- }
-
- std::string content;
- getDatabaseValue(kPersistentSettings, "timestamp." + saved_query, content);
- if (content.empty()) {
- // No timestamp is set for this query, perhaps this is the first time
- // query results expiration is applied.
- setDatabaseValue(kPersistentSettings,
- "timestamp." + saved_query,
- std::to_string(getUnixTime()));
- continue;
- }
-
- // Parse the timestamp and compare.
- size_t last_executed = 0;
- try {
- last_executed = boost::lexical_cast<size_t>(content);
- } catch (const boost::bad_lexical_cast& /* e */) {
- // Erase the timestamp as is it potentially corrupt.
- deleteDatabaseValue(kPersistentSettings, "timestamp." + saved_query);
- continue;
- }
-
- if (last_executed < getUnixTime() - 592200) {
- // Query has not run in the last week, expire results and interval.
- deleteDatabaseValue(kQueries, saved_query);
- deleteDatabaseValue(kQueries, saved_query + "epoch");
- deleteDatabaseValue(kPersistentSettings, "interval." + saved_query);
- deleteDatabaseValue(kPersistentSettings, "timestamp." + saved_query);
- VLOG(1) << "Expiring results for scheduled query: " << saved_query;
- }
- }
-}
-
-void Config::reset() {
- setStartTime(getUnixTime());
-
- schedule_ = std::make_unique<Schedule>();
- std::map<std::string, QueryPerformance>().swap(performance_);
- std::map<std::string, FileCategories>().swap(files_);
- std::map<std::string, std::string>().swap(hash_);
- valid_ = false;
- loaded_ = false;
- is_first_time_refresh = true;
-
- refresh_runner_ = std::make_shared<ConfigRefreshRunner>();
- started_thread_ = false;
-
- // Also request each parse to reset state.
- for (const auto& plugin : RegistryFactory::get().plugins("config_parser")) {
- std::shared_ptr<ConfigParserPlugin> parser = nullptr;
- try {
- parser = std::dynamic_pointer_cast<ConfigParserPlugin>(plugin.second);
- } catch (const std::bad_cast& /* e */) {
- continue;
- }
- if (parser == nullptr || parser.get() == nullptr) {
- continue;
- }
- parser->reset();
- }
-}
-
-void ConfigParserPlugin::reset() {
- // Resets will clear all top-level keys from the parser's data store.
- for (auto& category : data_.doc().GetObject()) {
- auto obj = data_.getObject();
- data_.add(category.name.GetString(), obj, data_.doc());
- }
-}
-
-void Config::recordQueryPerformance(const std::string& name,
- size_t delay,
- const Row& r0,
- const Row& r1) {
- RecursiveLock lock(config_performance_mutex_);
- if (performance_.count(name) == 0) {
- performance_[name] = QueryPerformance();
- }
-
- // Grab access to the non-const schedule item.
- auto& query = performance_.at(name);
- if (!r1.at("user_time").empty() && !r0.at("user_time").empty()) {
- auto ut1 = tryTo<long long>(r1.at("user_time"));
- auto ut0 = tryTo<long long>(r0.at("user_time"));
- auto diff = (ut1 && ut0) ? ut1.take() - ut0.take() : 0;
- if (diff > 0) {
- query.user_time += diff;
- }
- }
-
- if (!r1.at("system_time").empty() && !r0.at("system_time").empty()) {
- auto st1 = tryTo<long long>(r1.at("system_time"));
- auto st0 = tryTo<long long>(r0.at("system_time"));
- auto diff = (st1 && st0) ? st1.take() - st0.take() : 0;
- if (diff > 0) {
- query.system_time += diff;
- }
- }
-
- if (!r1.at("resident_size").empty() && !r0.at("resident_size").empty()) {
- auto rs1 = tryTo<long long>(r1.at("resident_size"));
- auto rs0 = tryTo<long long>(r0.at("resident_size"));
- auto diff = (rs1 && rs0) ? rs1.take() - rs0.take() : 0;
- if (diff > 0) {
- // Memory is stored as an average of RSS changes between query executions.
- query.average_memory = (query.average_memory * query.executions) + diff;
- query.average_memory = (query.average_memory / (query.executions + 1));
- }
- }
-
- query.wall_time += delay;
- query.executions += 1;
- query.last_executed = getUnixTime();
-
- // Clear the executing query (remove the dirty bit).
- setDatabaseValue(kPersistentSettings, kExecutingQuery, "");
-}
-
-void Config::recordQueryStart(const std::string& name) {
- // There should only ever be a single executing query in the schedule.
- setDatabaseValue(kPersistentSettings, kExecutingQuery, name);
- // Store the time this query name last executed for later results eviction.
- // When configuration updates occur the previous schedule is searched for
- // 'stale' query names, aka those that have week-old or longer last execute
- // timestamps. Offending queries have their database results purged.
- setDatabaseValue(
- kPersistentSettings, "timestamp." + name, std::to_string(getUnixTime()));
-}
-
-void Config::getPerformanceStats(
- const std::string& name,
- std::function<void(const QueryPerformance& query)> predicate) const {
- if (performance_.count(name) > 0) {
- RecursiveLock lock(config_performance_mutex_);
- predicate(performance_.at(name));
- }
-}
-
-const std::shared_ptr<ConfigParserPlugin> Config::getParser(
- const std::string& parser) {
- if (!RegistryFactory::get().exists("config_parser", parser, true)) {
- return nullptr;
- }
-
- auto plugin = RegistryFactory::get().plugin("config_parser", parser);
- // This is an error, need to check for existence (and not nullptr).
- return std::dynamic_pointer_cast<ConfigParserPlugin>(plugin);
-}
-
-void Config::files(std::function<void(const std::string& category,
- const std::vector<std::string>& files)>
- predicate) const {
- RecursiveLock lock(config_files_mutex_);
- for (const auto& it : files_) {
- for (const auto& category : it.second) {
- predicate(category.first, category.second);
- }
- }
-}
-
-Config::~Config() = default;
-
-Status ConfigPlugin::genPack(const std::string& name,
- const std::string& value,
- std::string& pack) {
- return Status(1, "Not implemented");
-}
-
-Status ConfigPlugin::call(const PluginRequest& request,
- PluginResponse& response) {
- auto action = request.find("action");
- if (action == request.end()) {
- return Status::failure("Config plugins require an action");
- }
-
- if (action->second == "genConfig") {
- std::map<std::string, std::string> config;
- auto stat = genConfig(config);
- response.push_back(config);
- return stat;
- } else if (action->second == "genPack") {
- auto name = request.find("name");
- auto value = request.find("value");
- if (name == request.end() || value == request.end()) {
- return Status(1, "Missing name or value");
- }
-
- std::string pack;
- auto stat = genPack(name->second, value->second, pack);
- response.push_back({{name->second, pack}});
- return stat;
- } else if (action->second == "update") {
- auto source = request.find("source");
- auto data = request.find("data");
- if (source == request.end() || data == request.end()) {
- return Status(1, "Missing source or data");
- }
-
- return Config::get().update({{source->second, data->second}});
- } else if (action->second == "option") {
- auto name = request.find("name");
- if (name == request.end()) {
- return Status(1, "Missing option name");
- }
-
- response.push_back(
- {{"name", name->second}, {"value", Flag::getValue(name->second)}});
- return Status::success();
- }
- return Status(1, "Config plugin action unknown: " + action->second);
-}
-
-Status ConfigParserPlugin::setUp() {
- for (const auto& key : keys()) {
- auto obj = data_.getObject();
- data_.add(key, obj);
- }
- return Status::success();
-}
-
-void ConfigRefreshRunner::start() {
- while (!interrupted()) {
- // Cool off and time wait the configured period.
- // Apply this interruption initially as at t=0 the config was read.
- pause(std::chrono::seconds(refresh_sec_));
- // Since the pause occurs before the logic, we need to check for an
- // interruption request.
- if (interrupted()) {
- return;
- }
-
- VLOG(1) << "Refreshing configuration state";
- Config::get().refresh();
- }
-}
-}
+++ /dev/null
-/**
- * Copyright (c) 2014-present, Facebook, Inc.
- * All rights reserved.
- *
- * This source code is licensed in accordance with the terms specified in
- * the LICENSE file found in the root directory of this source tree.
- */
-
-#pragma once
-
-#include <functional>
-#include <map>
-#include <memory>
-#include <vector>
-
-#include <osquery/core/sql/query_performance.h>
-#include <osquery/plugins/plugin.h>
-#include <osquery/query.h>
-#include <osquery/utils/expected/expected.h>
-#include <osquery/utils/json/json.h>
-
-#include <gtest/gtest_prod.h>
-
-namespace osquery {
-
-class Status;
-class Config;
-class Pack;
-class Schedule;
-class ConfigParserPlugin;
-class ConfigRefreshRunner;
-
-/// The name of the executing query within the single-threaded schedule.
-extern const std::string kExecutingQuery;
-
-/**
- * @brief The programmatic representation of osquery's configuration
- *
- * @code{.cpp}
- * auto c = Config::get();
- * // use methods in osquery::Config on `c`
- * @endcode
- */
-class Config : private boost::noncopyable {
- private:
- Config();
-
- void backupConfig(const std::map<std::string, std::string>& config);
-
- public:
- ~Config();
-
- /// Singleton accessor.
- static Config& get();
-
- enum class RestoreConfigError { DatabaseError = 1 };
- /**
- * @brief restoreConfigBackup retrieve backed up config
- * @return config persisted int the database
- */
- static Expected<std::map<std::string, std::string>,
- Config::RestoreConfigError>
- restoreConfigBackup();
-
- /**
- * @brief Update the internal config data.
- *
- * @param config A map of domain or namespace to config data.
- * @return If the config changes were applied.
- */
- Status update(const std::map<std::string, std::string>& config);
-
- /**
- * @brief Record performance (monitoring) information about a scheduled query.
- *
- * The daemon and query scheduler will optionally record process metadata
- * before and after executing each query. This can be compared and reported
- * on an interval or within the osquery_schedule table.
- *
- * The config consumes and calculates the optional performance differentials.
- * It would also be possible to store this in the RocksDB backing store or
- * report directly to a LoggerPlugin sink. The Config is the most appropriate
- * as the metrics are transient to the process running the schedule and apply
- * to the updates/changes reflected in the schedule, from the config.
- *
- * @param name The unique name of the scheduled item
- * @param delay Number of seconds (wall time) taken by the query
- * @param size Number of characters generated by query
- * @param r0 the process row before the query
- * @param r1 the process row after the query
- */
- void recordQueryPerformance(const std::string& name,
- size_t delay,
- const Row& r0,
- const Row& r1);
-
- /**
- * @brief Record a query 'initialization', meaning the query will run.
- *
- * Recording initializations if queries helps to identify when queries do not
- * complete. The Config::recordQueryPerformance method will clear a dirty
- * status set by this method. This status is saved in the backing database
- * store. On process start, or worker state, if any dirty bit is set then
- * it is assumed that the current start is a result of a previous abort.
- *
- * @param name THe unique name of the scheduled item
- */
- void recordQueryStart(const std::string& name);
-
- /// Whether or not the last loaded config was valid.
- bool isValid() const {
- return valid_;
- }
-
- /// Get start time of config.
- static size_t getStartTime();
-
- /// Set the start time if the config.
- static void setStartTime(size_t st);
-
- /**
- * @brief Add a pack to the osquery schedule
- */
- void addPack(const std::string& name,
- const std::string& source,
- const rapidjson::Value& obj);
-
- /**
- * @brief Remove a pack from the osquery schedule
- */
- void removePack(const std::string& pack);
-
- /**
- * @brief Iterate through all packs
- */
- void packs(std::function<void(const Pack& pack)> predicate) const;
-
- /**
- * @brief Add a file
- *
- * @param source source of config file
- * @param category is the category which the file exists in
- * @param path is the file path to add
- */
- void addFile(const std::string& source,
- const std::string& category,
- const std::string& path);
-
- /// Remove files by source.
- void removeFiles(const std::string& source);
-
- /**
- * @brief Map a function across the set of scheduled queries
- *
- * @param predicate is a function which accepts two parameters, the name of
- * the query and the ScheduledQuery struct of the queries data. predicate
- * will be called on each currently scheduled query.
- *
- * @param blacklisted [optional] return blacklisted queries if true.
- *
- * @code{.cpp}
- * std::map<std::string, ScheduledQuery> queries;
- * Config::get().scheduledQueries(
- * ([&queries](std::string name, const ScheduledQuery& query) {
- * queries[name] = query;
- * }));
- * @endcode
- */
- void scheduledQueries(
- std::function<void(std::string name, const ScheduledQuery& query)>
- predicate,
- bool blacklisted = false) const;
-
- /**
- * @brief Map a function across the set of configured files
- *
- * @param predicate is a function which accepts two parameters, the name of
- * the file category and a vector of files in that category. predicate will be
- * called on each pair in files_
- *
- * @code{.cpp}
- * std::map<std::string, std::vector<std::string>> file_map;
- * Config::get().files(
- * ([&file_map](const std::string& category,
- * const std::vector<std::string>& files) {
- * file_map[category] = files;
- * }));
- * @endcode
- */
- void files(std::function<void(const std::string& category,
- const std::vector<std::string>& files)>
- predicate) const;
-
- /**
- * @brief Get the performance stats for a specific query, by name
- *
- * @param name is the name of the query which you'd like to retrieve
- * @param predicate is a function which accepts a const reference to a
- * QueryPerformance struct. predicate will be called on name's related
- * QueryPerformance struct, if it exists.
- *
- * @code{.cpp}
- * Config::get().getPerformanceStats(
- * "my_awesome_query",
- * [](const QueryPerformance& query) {
- * // use "query" here
- * });
- * @endcode
- */
- void getPerformanceStats(
- const std::string& name,
- std::function<void(const QueryPerformance& query)> predicate) const;
-
- /**
- * @brief Helper to access config parsers via the registry
- *
- * This may return a nullptr if an exception is thrown attempting to retrieve
- * the requested config parser.
- *
- * @param parser is the string name of the parser that you want
- *
- * @return a shared pointer to the config parser plugin
- */
- static const std::shared_ptr<ConfigParserPlugin> getParser(
- const std::string& parser);
-
- protected:
- /**
- * @brief Call the genConfig method of the config retriever plugin.
- *
- * This may perform a resource load such as TCP request or filesystem read.
- * If a non-zero value is passed to --config_refresh, this starts a thread
- * that periodically calls genConfig to reload config state
- */
- Status refresh();
-
- /// Update the refresh rate.
- void setRefresh(size_t refresh_sec);
-
- /// Inspect the refresh rate.
- size_t getRefresh() const;
-
- /**
- * @brief Check if a config plugin is registered and load configs.
- *
- * Calls refresh after confirming a config plugin is registered
- */
- Status load();
-
- /// A step method for Config::update.
- Status updateSource(const std::string& source, const std::string& json);
-
- /**
- * @brief Generate pack content from a resource handled by the Plugin.
- *
- * Configuration content may set pack values to JSON strings instead of an
- * embedded dictionary representing the pack content. When a string is
- * encountered the config assumes this is a 'resource' handled by the Plugin.
- *
- * The value, or target, is sent to the ConfigPlugin via a registry request.
- * The plugin response is assumed, and used, as the pack content.
- *
- * @param name A pack name provided and handled by the ConfigPlugin.
- * @param source The config content source identifier.
- * @param target A resource (path, URL, etc) handled by the ConfigPlugin.
- * @return status On success the response will be JSON parsed.
- */
- Status genPack(const std::string& name,
- const std::string& source,
- const std::string& target);
-
- /**
- * @brief Apply each ConfigParser to an input JSON document.
- *
- * This iterates each discovered ConfigParser Plugin and the plugin's keys
- * to match keys within the input JSON document. If a key matches then the
- * associated value is passed to the parser.
- *
- * Use this utility method for both the top-level configuration JSON and
- * the content of each configuration pack. There is an optional black list
- * parameter to differentiate pack content.
- *
- * @param source The input configuration source name.
- * @param obj The input configuration JSON.
- * @param pack True if the JSON was built from pack data, otherwise false.
- */
- void applyParsers(const std::string& source,
- const rapidjson::Value& obj,
- bool pack = false);
-
- /**
- * @brief When config sources are updated the config will 'purge'.
- *
- * The general 'purge' action applies to searching for outdated query results,
- * timestamps, saved intervals, etc. This event only occurs before a source
- * is updated. Since updating the configuration may have expected side effects
- * such as changing watched files or overwriting (modifying) pack content,
- * this 'purge' action is assumed to be destructive and potentially expensive.
- */
- void purge();
-
- /**
- * @brief Reset the configuration state, reserved for testing only.
- */
- void reset();
-
- private:
- /// Schedule of packs and their queries.
- std::unique_ptr<Schedule> schedule_;
-
- /// A set of performance stats for each query in the schedule.
- std::map<std::string, QueryPerformance> performance_;
-
- /// A set of named categories filled with filesystem globbing paths.
- using FileCategories = std::map<std::string, std::vector<std::string>>;
- std::map<std::string, FileCategories> files_;
-
- /// A set of hashes for each source of the config.
- std::map<std::string, std::string> hash_;
-
- /// Check if the config received valid/parsable content from a config plugin.
- bool valid_{false};
-
- /**
- * @brief Check if the configuration has attempted a load.
- *
- * Check if the config is updating sources in response to an async update
- * or the initialization load step.
- */
- bool loaded_{false};
-
- /// Try if the configuration has started an auto-refresh thread.
- bool started_thread_{false};
-
- private:
- /// Hold a reference to the refresh runner to update the acceleration.
- std::shared_ptr<ConfigRefreshRunner> refresh_runner_{nullptr};
-
- private:
- friend class Initializer;
-
- private:
- friend class ConfigTests;
- friend class ConfigRefreshRunner;
- friend class FilePathsConfigParserPluginTests;
- friend class FileEventsTableTests;
- friend class DecoratorsConfigParserPluginTests;
- friend class SchedulerTests;
- friend class WatcherTests;
- FRIEND_TEST(ConfigTests, test_config_backup);
- FRIEND_TEST(ConfigTests, test_config_backup_integrate);
- FRIEND_TEST(ConfigTests, test_config_refresh);
- FRIEND_TEST(ConfigTests, test_get_scheduled_queries);
- FRIEND_TEST(ConfigTests, test_nonblacklist_query);
- FRIEND_TEST(OptionsConfigParserPluginTests, test_get_option);
- FRIEND_TEST(ViewsConfigParserPluginTests, test_add_view);
- FRIEND_TEST(ViewsConfigParserPluginTests, test_swap_view);
- FRIEND_TEST(ViewsConfigParserPluginTests, test_update_view);
- FRIEND_TEST(OptionsConfigParserPluginTests, test_unknown_option);
- FRIEND_TEST(OptionsConfigParserPluginTests, test_json_option);
- FRIEND_TEST(EventsConfigParserPluginTests, test_get_event);
- FRIEND_TEST(PacksTests, test_discovery_cache);
- FRIEND_TEST(PacksTests, test_multi_pack);
- FRIEND_TEST(SchedulerTests, test_monitor);
- FRIEND_TEST(SchedulerTests, test_config_results_purge);
- FRIEND_TEST(EventsTests, test_event_subscriber_configure);
- FRIEND_TEST(TLSConfigTests, test_retrieve_config);
- FRIEND_TEST(TLSConfigTests, test_runner_and_scheduler);
-};
-
-/**
- * @brief Superclass for the pluggable config component.
- *
- * In order to make the distribution of configurations to hosts running
- * osquery, we take advantage of a plugin interface which allows you to
- * integrate osquery with your internal configuration distribution mechanisms.
- * You may use ZooKeeper, files on disk, a custom solution, etc. In order to
- * use your specific configuration distribution system, one simply needs to
- * create a custom subclass of ConfigPlugin. That subclass should implement
- * the ConfigPlugin::genConfig method.
- *
- * Consider the following example:
- *
- * @code{.cpp}
- * class TestConfigPlugin : public ConfigPlugin {
- * public:
- * virtual Status genConfig(std::map<std::string, std::string>& config) {
- * config["my_source"] = "{}";
- * return Status::success();
- * }
- * };
- *
- * REGISTER(TestConfigPlugin, "config", "test");
- * @endcode
- */
-class ConfigPlugin : public Plugin {
- public:
- /**
- * @brief Virtual method which should implemented custom config retrieval
- *
- * ConfigPlugin::genConfig should be implemented by a subclasses of
- * ConfigPlugin which needs to retrieve config data in a custom way.
- *
- * @param config The output ConfigSourceMap, a map of JSON to source names.
- *
- * @return A failure status will prevent the source map from merging.
- */
- virtual Status genConfig(std::map<std::string, std::string>& config) = 0;
-
- /**
- * @brief Virtual method which could implement custom query pack retrieval
- *
- * The default config syntax for query packs is like the following:
- *
- * @code
- * {
- * "packs": {
- * "foo": {
- * "version": "1.5.0",
- * "platform:" "any",
- * "queries": {
- * // ...
- * }
- * }
- * }
- * }
- * @endcode
- *
- * Alternatively, you can define packs like the following as well:
- *
- * @code
- * {
- * "packs": {
- * "foo": "/var/osquery/packs/foo.json",
- * "bar": "/var/osquery/packs/bar.json"
- * }
- * }
- * @endcode
- *
- * If you defined the "value" of your pack as a string instead of an inline
- * data structure, then osquery will pass the responsibility of retrieving
- * the pack to the active config plugin. In the above example, it seems
- * obvious that the value is a local file path. Alternatively, if the
- * filesystem config plugin wasn't being used, the string could be a remote
- * URL, etc.
- *
- * genPack is not a pure virtual, so you don't have to define it if you don't
- * want to use the shortened query pack syntax. The default implementation
- * returns a failed status.
- *
- * @param name is the name of the query pack
- * @param value is the string based value that was provided with the pack
- * @param pack should be populated with the string JSON pack content
- *
- * @return a Status instance indicating the success or failure of the call
- */
- virtual Status genPack(const std::string& name,
- const std::string& value,
- std::string& pack);
-
- /// Main entrypoint for config plugin requests
- Status call(const PluginRequest& request, PluginResponse& response) override;
-};
-
-/**
- * @brief A pluggable configuration parser.
- *
- * An osquery config instance is populated from JSON using a ConfigPlugin.
- * That plugin may update the config data asynchronously and read from
- * several sources, as is the case with "filesystem" and reading multiple files.
- *
- * A ConfigParserPlugin will receive the merged configuration at osquery start
- * and the updated (still merged) config if any ConfigPlugin updates the
- * instance asynchronously. Each parser specifies a set of top-level JSON
- * keys to receive. The config instance will auto-merge the key values
- * from multiple sources.
- *
- * The keys must contain either dictionaries or lists.
- *
- * If a top-level key is a dictionary, each source with the top-level key
- * will have its own dictionary keys merged and replaced based on the lexical
- * order of sources. For the "filesystem" config plugin this is the lexical
- * sorting of filenames. If the top-level key is a list, each source with the
- * top-level key will have its contents appended.
- *
- * Each config parser plugin will live alongside the config instance for the
- * life of the osquery process. The parser may perform actions at config load
- * and config update "time" as well as keep its own data members and be
- * accessible through the Config class API.
- */
-class ConfigParserPlugin : public Plugin {
- public:
- using ParserConfig = std::map<std::string, JSON>;
-
- public:
- /**
- * @brief Return a list of top-level config keys to receive in updates.
- *
- * The ConfigParserPlugin::update method will receive a map of these keys
- * with a JSON-parsed document of configuration data.
- *
- * @return A list of string top-level JSON keys.
- */
- virtual std::vector<std::string> keys() const = 0;
-
- /**
- * @brief Receive a merged JSON document for each top-level config key.
- *
- * Called when the Config instance is initially loaded with data from the
- * active config plugin and when it is updated via an async ConfigPlugin
- * update. Every config parser will receive a map of merged data for each key
- * they requested in keys().
- *
- * @param source source of the config data
- * @param config A JSON-parsed document map.
- * @return Failure if the parser should no longer receive updates.
- */
- virtual Status update(const std::string& source,
- const ParserConfig& config) = 0;
-
- /// Allow parsers to perform some setup before the configuration is loaded.
- Status setUp() override;
-
- Status call(const PluginRequest& /*request*/,
- PluginResponse& /*response*/) override {
- return Status(0);
- }
-
- /**
- * @brief Accessor for parser-manipulated data.
- *
- * Parsers should be used generically, for places within the code base that
- * request a parser (check for its existence), should only use this
- * ConfigParserPlugin::getData accessor.
- *
- * More complex parsers that require dynamic casting are not recommended.
- */
- const JSON& getData() const {
- return data_;
- }
-
- protected:
- /// Allow the config to request parser state resets.
- virtual void reset();
-
- protected:
- /// Allow the config parser to keep some global state.
- JSON data_;
-
- private:
- friend class Config;
-};
-
-/**
- * @brief JSON parsers may accept comments.
- *
- * For semi-compatibility with existing configurations we will attempt to strip
- * hash and C++ style comments. It is OK for the config update to be latent
- * as it is a single event. But some configuration plugins may update running
- * configurations.
- *
- * @param json A mutable input/output string that will contain stripped JSON.
- */
-void stripConfigComments(std::string& json);
-} // namespace osquery
+++ /dev/null
-/**
- * Copyright (c) 2014-present, Facebook, Inc.
- * All rights reserved.
- *
- * This source code is licensed in accordance with the terms specified in
- * the LICENSE file found in the root directory of this source tree.
- */
-
-#include <algorithm>
-#include <random>
-
-#include <osquery/database.h>
-#include <osquery/logger.h>
-#include <osquery/packs.h>
-#include <osquery/sql.h>
-#include <osquery/system.h>
-#include <osquery/utils/conversions/split.h>
-#include <osquery/utils/conversions/tryto.h>
-#include <osquery/utils/info/version.h>
-#include <osquery/utils/json/json.h>
-#include <osquery/utils/system/time.h>
-
-namespace rj = rapidjson;
-
-namespace osquery {
-
-FLAG(uint64,
- pack_refresh_interval,
- 3600,
- "Cache expiration for a packs discovery queries");
-
-FLAG(string, pack_delimiter, "_", "Delimiter for pack and query names");
-
-FLAG(uint64, schedule_splay_percent, 10, "Percent to splay config times");
-
-FLAG(uint64,
- schedule_default_interval,
- 3600,
- "Query interval to use if none is provided");
-
-size_t kMaxQueryInterval = 604800;
-
-size_t splayValue(size_t original, size_t splayPercent) {
- if (splayPercent == 0 || splayPercent > 100) {
- return original;
- }
-
- float percent_to_modify_by = (float)splayPercent / 100;
- size_t possible_difference =
- static_cast<size_t>(original * percent_to_modify_by);
- size_t max_value = original + possible_difference;
- size_t min_value = std::max((size_t)1, original - possible_difference);
-
- if (max_value == min_value) {
- return max_value;
- }
-
- std::default_random_engine generator;
- generator.seed(static_cast<unsigned int>(
- std::chrono::high_resolution_clock::now().time_since_epoch().count()));
- std::uniform_int_distribution<size_t> distribution(min_value, max_value);
- return distribution(generator);
-}
-
-size_t restoreSplayedValue(const std::string& name, size_t interval) {
- // Attempt to restore a previously-calculated splay.
- std::string content;
- getDatabaseValue(kPersistentSettings, "interval." + name, content);
- if (!content.empty()) {
- // This query name existed before, check the last requested interval.
- auto details = osquery::split(content, ":");
- if (details.size() == 2) {
- auto const last_interval_exp = tryTo<long>(details[0], 10);
- auto const last_splay_exp = tryTo<long>(details[1], 10);
- if (last_interval_exp.isValue() && last_splay_exp.isValue()) {
- if (last_interval_exp.get() == static_cast<long>(interval) &&
- last_splay_exp.get() > 0) {
- // This is a matching interval, use the previous splay.
- return static_cast<size_t>(last_splay_exp.get());
- }
- }
- }
- }
-
- // If the splayed interval was not restored from the database.
- auto splay = splayValue(interval, FLAGS_schedule_splay_percent);
- content = std::to_string(interval) + ":" + std::to_string(splay);
- setDatabaseValue(kPersistentSettings, "interval." + name, content);
- return splay;
-}
-
-void Pack::initialize(const std::string& name,
- const std::string& source,
- const rj::Value& obj) {
- name_ = name;
- source_ = source;
- // Check the shard limitation, shards falling below this value are included.
- if (obj.HasMember("shard")) {
- shard_ = JSON::valueToSize(obj["shard"]);
- }
-
- // Check for a platform restriction.
- platform_.clear();
- if (obj.HasMember("platform") && obj["platform"].IsString()) {
- platform_ = obj["platform"].GetString();
- }
-
- // Check for a version restriction.
- version_.clear();
- if (obj.HasMember("version") && obj["version"].IsString()) {
- version_ = obj["version"].GetString();
- }
-
- std::string oncall;
- if (obj.HasMember("oncall") && obj["oncall"].IsString()) {
- oncall = obj["oncall"].GetString();
- } else {
- oncall = "unknown";
- }
-
- discovery_queries_.clear();
- if (obj.HasMember("discovery") && obj["discovery"].IsArray()) {
- for (const auto& item : obj["discovery"].GetArray()) {
- discovery_queries_.push_back(item.GetString());
- }
- }
-
- // Initialize a discovery cache at time 0.
- discovery_cache_ = std::make_pair<size_t, bool>(0, false);
- valid_ = true;
-
- // If the splay percent is less than 1 reset to a sane estimate.
- if (FLAGS_schedule_splay_percent <= 1) {
- FLAGS_schedule_splay_percent = 10;
- }
-
- schedule_.clear();
- if (!obj.HasMember("queries") || !obj["queries"].IsObject()) {
- // This pack contained no queries.
- VLOG(1) << "No queries defined for pack " << name;
- return;
- }
-
- // Iterate the queries (or schedule) and check platform/version/sanity.
- for (const auto& q : obj["queries"].GetObject()) {
- if (!q.value.IsObject()) {
- VLOG(1) << "The pack " << name << " must contain a dictionary of queries";
- continue;
- }
-
- if (q.value.HasMember("platform") && q.value["platform"].IsString()) {
- if (!checkPlatform(q.value["platform"].GetString())) {
- continue;
- }
- }
-
- if (q.value.HasMember("version") && q.value["version"].IsString()) {
- if (!checkVersion(q.value["version"].GetString())) {
- continue;
- }
- }
-
- if (!q.value.HasMember("query") || !q.value["query"].IsString()) {
- VLOG(1) << "No query string defined for query " << q.name.GetString();
- continue;
- }
-
- ScheduledQuery query(
- name_, q.name.GetString(), q.value["query"].GetString());
-
- query.oncall = oncall;
-
- if (!q.value.HasMember("interval")) {
- query.interval = FLAGS_schedule_default_interval;
- } else {
- query.interval = JSON::valueToSize(q.value["interval"]);
- }
-
- if (query.interval <= 0 || query.query.empty() ||
- query.interval > kMaxQueryInterval) {
- // Invalid pack query.
- LOG(WARNING) << "Query has invalid interval: " << q.name.GetString()
- << ": " << query.interval;
- continue;
- }
-
- query.splayed_interval =
- restoreSplayedValue(q.name.GetString(), query.interval);
-
- if (!q.value.HasMember("snapshot")) {
- query.options["snapshot"] = false;
- } else {
- query.options["snapshot"] = JSON::valueToBool(q.value["snapshot"]);
- }
-
- if (!q.value.HasMember("removed")) {
- query.options["removed"] = true;
- } else {
- query.options["removed"] = JSON::valueToBool(q.value["removed"]);
- }
-
- query.options["blacklist"] = true;
- if (q.value.HasMember("blacklist")) {
- query.options["blacklist"] = JSON::valueToBool(q.value["blacklist"]);
- }
-
- schedule_.emplace(std::make_pair(q.name.GetString(), std::move(query)));
- }
-}
-
-const std::map<std::string, ScheduledQuery>& Pack::getSchedule() const {
- return schedule_;
-}
-
-std::map<std::string, ScheduledQuery>& Pack::getSchedule() {
- return schedule_;
-}
-
-const std::vector<std::string>& Pack::getDiscoveryQueries() const {
- return discovery_queries_;
-}
-
-const PackStats& Pack::getStats() const {
- return stats_;
-}
-
-const std::string& Pack::getPlatform() const {
- return platform_;
-}
-
-const std::string& Pack::getVersion() const {
- return version_;
-}
-
-bool Pack::shouldPackExecute() {
- active_ = (valid_ && checkDiscovery());
- return active_;
-}
-
-const std::string& Pack::getName() const {
- return name_;
-}
-
-const std::string& Pack::getSource() const {
- return source_;
-}
-
-bool Pack::checkPlatform() const {
- return checkPlatform(platform_);
-}
-
-bool Pack::checkPlatform(const std::string& platform) const {
- return ::osquery::checkPlatform(platform);
-}
-
-bool Pack::checkVersion() const {
- return checkVersion(version_);
-}
-
-bool Pack::checkVersion(const std::string& version) const {
- if (version.empty() || version == "null") {
- return true;
- }
-
- return versionAtLeast(version, kSDKVersion);
-}
-
-bool Pack::checkDiscovery() {
- stats_.total++;
- size_t current = osquery::getUnixTime();
- if ((current - discovery_cache_.first) < FLAGS_pack_refresh_interval) {
- stats_.hits++;
- return discovery_cache_.second;
- }
-
- stats_.misses++;
- discovery_cache_.first = current;
- discovery_cache_.second = true;
- for (const auto& q : discovery_queries_) {
- SQL results(q);
- if (!results.ok()) {
- LOG(WARNING) << "Discovery query failed (" << q
- << "): " << results.getMessageString();
- discovery_cache_.second = false;
- break;
- }
- if (results.rows().size() == 0) {
- discovery_cache_.second = false;
- break;
- }
- }
- return discovery_cache_.second;
-}
-
-bool Pack::isActive() const {
- return active_;
-}
-}
+++ /dev/null
-/**
- * Copyright (c) 2014-present, Facebook, Inc.
- * All rights reserved.
- *
- * This source code is licensed in accordance with the terms specified in
- * the LICENSE file found in the root directory of this source tree.
- */
-
-#include <osquery/config/config.h>
-
-#include <osquery/config/tests/test_utils.h>
-
-#include <osquery/filesystem/filesystem.h>
-#include <osquery/filesystem/mock_file_structure.h>
-
-#include <osquery/core.h>
-#include <osquery/database.h>
-#include <osquery/dispatcher.h>
-#include <osquery/flags.h>
-#include <osquery/packs.h>
-#include <osquery/registry.h>
-#include <osquery/system.h>
-
-#include <osquery/utils/info/platform_type.h>
-#include <osquery/utils/json/json.h>
-#include <osquery/utils/system/time.h>
-
-#include <boost/filesystem/path.hpp>
-
-#include <gtest/gtest.h>
-
-#include <atomic>
-#include <chrono>
-#include <map>
-#include <memory>
-#include <string>
-#include <thread>
-#include <vector>
-
-
-namespace osquery {
-
-DECLARE_uint64(config_refresh);
-DECLARE_uint64(config_accelerated_refresh);
-DECLARE_bool(config_enable_backup);
-DECLARE_bool(disable_database);
-
-namespace fs = boost::filesystem;
-
-// Blacklist testing methods, internal to config implementations.
-extern void restoreScheduleBlacklist(std::map<std::string, size_t>& blacklist);
-extern void saveScheduleBlacklist(
- const std::map<std::string, size_t>& blacklist);
-
-class ConfigTests : public testing::Test {
- public:
- ConfigTests() {
- Initializer::platformSetup();
- registryAndPluginInit();
- FLAGS_disable_database = true;
- DatabasePlugin::setAllowOpen(true);
- DatabasePlugin::initPlugin();
-
- Config::get().reset();
- }
-
- protected:
- fs::path fake_directory_;
-
- protected:
- void SetUp() {
- fake_directory_ = fs::canonical(createMockFileStructure());
-
- refresh_ = FLAGS_config_refresh;
- FLAGS_config_refresh = 0;
-
- createMockFileStructure();
- }
-
- void TearDown() {
- fs::remove_all(fake_directory_);
- FLAGS_config_refresh = refresh_;
- }
-
- protected:
- Status load() {
- return Config::get().load();
- }
-
- void setLoaded() {
- Config::get().loaded_ = true;
- }
-
- Config& get() {
- return Config::get();
- }
-
- private:
- size_t refresh_{0};
-};
-
-class TestConfigPlugin : public ConfigPlugin {
- public:
- TestConfigPlugin() {
- gen_config_count_ = 0;
- gen_pack_count_ = 0;
- }
-
- Status genConfig(std::map<std::string, std::string>& config) override {
- gen_config_count_++;
- if (fail_) {
- return Status(1);
- }
-
- std::string content;
- auto s = readFile(getTestConfigDirectory() / "test_noninline_packs.conf", content);
- config["data"] = content;
- return s;
- }
-
- Status genPack(const std::string& name,
- const std::string& value,
- std::string& pack) override {
- gen_pack_count_++;
- getUnrestrictedPack().toString(pack);
- return Status::success();
- }
-
- public:
- std::atomic<size_t> gen_config_count_{0};
- std::atomic<size_t> gen_pack_count_{0};
- std::atomic<bool> fail_{false};
-};
-
-class TestDataConfigParserPlugin : public ConfigParserPlugin {
- public:
- std::vector<std::string> keys() const override {
- return {"data"};
- }
-
- Status setUp() override {
- return Status::success();
- }
-
- Status update(const std::string& source,
- const ParserConfig& config) override {
- source_ = source;
- config_.clear();
- for (const auto& entry : config) {
- std::string content;
- entry.second.toString(content);
- config_[entry.first] = content;
- }
- return Status::success();
- }
- std::string source_{""};
- std::map<std::string, std::string> config_;
-};
-
-TEST_F(ConfigTests, test_plugin) {
- auto& rf = RegistryFactory::get();
- auto plugin = std::make_shared<TestConfigPlugin>();
- rf.registry("config")->add("test", plugin);
- // Change the active config plugin.
- EXPECT_TRUE(rf.setActive("config", "test").ok());
-
- PluginResponse response;
- auto status = Registry::call("config", {{"action", "genConfig"}}, response);
-
- EXPECT_EQ(status.ok(), true) << status.what();
-
- Registry::call("config", {{"action", "genConfig"}});
- EXPECT_EQ(2U, plugin->gen_config_count_);
- rf.registry("config")->remove("test");
-}
-
-TEST_F(ConfigTests, test_invalid_content) {
- std::string bad_json = "{\"options\": {},}";
- ASSERT_NO_THROW(get().update({{"bad_source", bad_json}}));
-}
-
-TEST_F(ConfigTests, test_strip_comments) {
- std::string json_comments =
- "// Comment\n // Comment //\n # Comment\n# Comment\n{\"options\":{}}";
-
- // Test support for stripping C++ and hash style comments from config JSON.
- auto actual = json_comments;
- stripConfigComments(actual);
- std::string expected = "{\"options\":{}}\n";
- EXPECT_EQ(actual, expected);
-
- // Make sure the config update source logic applies the stripping.
- EXPECT_TRUE(get().update({{"data", json_comments}}));
-}
-
-TEST_F(ConfigTests, test_schedule_blacklist) {
- auto current_time = getUnixTime();
- std::map<std::string, size_t> blacklist;
- saveScheduleBlacklist(blacklist);
- restoreScheduleBlacklist(blacklist);
- EXPECT_EQ(blacklist.size(), 0U);
-
- // Create some entries.
- blacklist["test_1"] = current_time * 2;
- blacklist["test_2"] = current_time * 3;
- saveScheduleBlacklist(blacklist);
- blacklist.clear();
- restoreScheduleBlacklist(blacklist);
- ASSERT_EQ(blacklist.count("test_1"), 1U);
- ASSERT_EQ(blacklist.count("test_2"), 1U);
- EXPECT_EQ(blacklist.at("test_1"), current_time * 2);
- EXPECT_EQ(blacklist.at("test_2"), current_time * 3);
-
- // Now save an expired query.
- blacklist["test_1"] = 1;
- saveScheduleBlacklist(blacklist);
- blacklist.clear();
-
- // When restoring, the values below the current time will not be included.
- restoreScheduleBlacklist(blacklist);
- EXPECT_EQ(blacklist.size(), 1U);
-}
-
-TEST_F(ConfigTests, test_pack_noninline) {
- auto& rf = RegistryFactory::get();
- rf.registry("config")->add("test", std::make_shared<TestConfigPlugin>());
- // Change the active config plugin.
- EXPECT_TRUE(rf.setActive("config", "test").ok());
-
- // Get a specialized config/test plugin.
- const auto& plugin =
- std::dynamic_pointer_cast<TestConfigPlugin>(rf.plugin("config", "test"));
-
- this->load();
- // Expect the test plugin to have recorded 1 pack.
- // This value is incremented when its genPack method is called.
- EXPECT_EQ(plugin->gen_pack_count_, 1U);
-
- int total_packs = 0;
- // Expect the config to have recorded a pack for the inline and non-inline.
- get().packs([&total_packs](const Pack& pack) { total_packs++; });
- EXPECT_EQ(total_packs, 2);
- rf.registry("config")->remove("test");
-}
-
-TEST_F(ConfigTests, test_pack_restrictions) {
- auto doc = getExamplePacksConfig();
- auto& packs = doc.doc()["packs"];
- for (const auto& pack : packs.GetObject()) {
- get().addPack(pack.name.GetString(), "", pack.value);
- }
-
- std::map<std::string, bool> results = {
- {"unrestricted_pack", true},
- {"discovery_pack", false},
- {"fake_version_pack", false},
- {"valid_discovery_pack", true},
- {"restricted_pack", false},
- };
-
- get().packs(([&results](const Pack& pack) {
- if (results[pack.getName()]) {
- EXPECT_TRUE(const_cast<Pack&>(pack).shouldPackExecute())
- << "Pack " << pack.getName() << " should have executed";
- } else {
- EXPECT_FALSE(const_cast<Pack&>(pack).shouldPackExecute())
- << "Pack " << pack.getName() << " should not have executed";
- }
- }));
-}
-
-TEST_F(ConfigTests, test_pack_removal) {
- size_t pack_count = 0;
- get().packs(([&pack_count](const Pack& pack) { pack_count++; }));
- EXPECT_EQ(pack_count, 0U);
-
- pack_count = 0;
- get().addPack("unrestricted_pack", "", getUnrestrictedPack().doc());
- get().packs(([&pack_count](const Pack& pack) { pack_count++; }));
- EXPECT_EQ(pack_count, 1U);
-
- pack_count = 0;
- get().removePack("unrestricted_pack");
- get().packs(([&pack_count](const Pack& pack) { pack_count++; }));
- EXPECT_EQ(pack_count, 0U);
-}
-
-TEST_F(ConfigTests, test_get_scheduled_queries) {
- std::vector<std::string> query_names;
- get().addPack("unrestricted_pack", "", getUnrestrictedPack().doc());
- get().scheduledQueries(
- ([&query_names](std::string name, const ScheduledQuery& query) {
- query_names.push_back(std::move(name));
- }));
-
- auto expected_size = getUnrestrictedPack().doc()["queries"].MemberCount();
- EXPECT_EQ(query_names.size(), expected_size)
- << "The number of queries in the schedule (" << query_names.size()
- << ") should equal " << expected_size;
- ASSERT_FALSE(query_names.empty());
-
- // Construct a schedule blacklist and place the first scheduled query.
- std::map<std::string, size_t> blacklist;
- std::string query_name = query_names[0];
- blacklist[query_name] = getUnixTime() * 2;
- saveScheduleBlacklist(blacklist);
- blacklist.clear();
-
- // When the blacklist is edited externally, the config must re-read.
- get().reset();
- get().addPack("unrestricted_pack", "", getUnrestrictedPack().doc());
-
- // Clear the query names in the scheduled queries and request again.
- query_names.clear();
- get().scheduledQueries(
- ([&query_names](std::string name, const ScheduledQuery&) {
- query_names.push_back(std::move(name));
- }));
- // The query should not exist.
- EXPECT_EQ(std::find(query_names.begin(), query_names.end(), query_name),
- query_names.end());
-
- // Try again, this time requesting scheduled queries.
- query_names.clear();
- bool blacklisted = false;
- get().scheduledQueries(([&blacklisted, &query_names, &query_name](
- std::string name, const ScheduledQuery& query) {
- if (name == query_name) {
- // Only populate the query we've blacklisted.
- query_names.push_back(std::move(name));
- blacklisted = query.blacklisted;
- }
- }),
- true);
- ASSERT_EQ(query_names.size(), std::size_t{1});
- EXPECT_EQ(query_names[0], query_name);
- EXPECT_TRUE(blacklisted);
-}
-
-TEST_F(ConfigTests, test_nonblacklist_query) {
- std::map<std::string, size_t> blacklist;
-
- const std::string kConfigTestNonBlacklistQuery{"pack_unrestricted_pack_process_heartbeat"};
-
- blacklist[kConfigTestNonBlacklistQuery] = getUnixTime() * 2;
- saveScheduleBlacklist(blacklist);
-
- get().reset();
- get().addPack("unrestricted_pack", "", getUnrestrictedPack().doc());
-
- std::map<std::string, bool> blacklisted;
- get().scheduledQueries(
- ([&blacklisted](std::string name, const ScheduledQuery& query) {
- blacklisted[name] = query.blacklisted;
- }));
-
- // This query cannot be blacklisted.
- auto query = blacklisted.find(kConfigTestNonBlacklistQuery);
- ASSERT_NE(query, blacklisted.end());
- EXPECT_FALSE(query->second);
-}
-
-class TestConfigParserPlugin : public ConfigParserPlugin {
- public:
- std::vector<std::string> keys() const override {
- // This config parser requests the follow top-level-config keys.
- return {"dictionary", "dictionary2", "list"};
- }
-
- Status update(const std::string& source,
- const ParserConfig& config) override {
- // Set a simple boolean indicating the update callin occurred.
- update_called = true;
- // Copy all expected keys into the parser's data.
- for (const auto& key : config) {
- auto obj = data_.getObject();
- data_.copyFrom(key.second.doc(), obj);
- data_.add(key.first, obj, data_.doc());
- }
-
- // Set parser-rendered additional data.
- auto obj2 = data_.getObject();
- data_.addRef("key2", "value2", obj2);
- data_.add("dictionary3", obj2, data_.doc());
- return Status::success();
- }
-
- // Flag tracking that the update method was called.
- static bool update_called;
-
- private:
- FRIEND_TEST(ConfigTests, test_config_parser);
-};
-
-// An intermediate boolean to check parser updates.
-bool TestConfigParserPlugin::update_called = false;
-
-TEST_F(ConfigTests, test_get_parser) {
- auto& rf = RegistryFactory::get();
- rf.registry("config_parser")
- ->add("test", std::make_shared<TestConfigParserPlugin>());
-
- auto s = get().update(getTestConfigMap("test_parse_items.conf"));
- EXPECT_TRUE(s.ok());
- EXPECT_EQ(s.toString(), "OK");
-
- auto plugin = get().getParser("test");
- EXPECT_TRUE(plugin != nullptr);
- EXPECT_TRUE(plugin.get() != nullptr);
-
- const auto& parser =
- std::dynamic_pointer_cast<TestConfigParserPlugin>(plugin);
- const auto& doc = parser->getData();
-
- EXPECT_TRUE(doc.doc().HasMember("list"));
- EXPECT_TRUE(doc.doc().HasMember("dictionary"));
- rf.registry("config_parser")->remove("test");
-}
-
-class PlaceboConfigParserPlugin : public ConfigParserPlugin {
- public:
- std::vector<std::string> keys() const override {
- return {};
- }
- Status update(const std::string&, const ParserConfig&) override {
- return Status::success();
- }
-
- /// Make sure configure is called.
- void configure() override {
- configures++;
- }
-
- size_t configures{0};
-};
-
-TEST_F(ConfigTests, test_plugin_reconfigure) {
- auto& rf = RegistryFactory::get();
- // Add a configuration plugin (could be any plugin) that will react to
- // config updates.
- rf.registry("config_parser")
- ->add("placebo", std::make_shared<PlaceboConfigParserPlugin>());
-
- // Create a config that has been loaded.
- setLoaded();
- get().update({{"data", "{}"}});
- // Get the placebo.
- auto placebo = std::static_pointer_cast<PlaceboConfigParserPlugin>(
- rf.plugin("config_parser", "placebo"));
- EXPECT_EQ(placebo->configures, 1U);
-
- // Updating with the same content does not reconfigure parsers.
- get().update({{"data", "{}"}});
- EXPECT_EQ(placebo->configures, 1U);
-
- // Updating with different content will reconfigure.
- get().update({{"data", "{\"options\":{}}"}});
- EXPECT_EQ(placebo->configures, 2U);
- get().update({{"data", "{\"options\":{}}"}});
- EXPECT_EQ(placebo->configures, 2U);
-
- // Updating with a new source will reconfigure.
- get().update({{"data", "{\"options\":{}}"}, {"data1", "{}"}});
- EXPECT_EQ(placebo->configures, 3U);
- // Updating and not including a source is handled by the config plugin.
- // The config will expect the other source to update asynchronously and does
- // not consider the missing key as a delete request.
- get().update({{"data", "{\"options\":{}}"}});
- EXPECT_EQ(placebo->configures, 3U);
-
- rf.registry("config_parser")->remove("placebo");
-}
-
-TEST_F(ConfigTests, test_pack_file_paths) {
- size_t count = 0;
- auto fileCounter = [&count](const std::string& c,
- const std::vector<std::string>& files) {
- count += files.size();
- };
-
- get().addPack("unrestricted_pack", "", getUnrestrictedPack().doc());
- get().files(fileCounter);
- EXPECT_EQ(count, 2U);
-
- count = 0;
- get().removePack("unrestricted_pack");
- get().files(fileCounter);
- EXPECT_EQ(count, 0U);
-
- count = 0;
- get().addPack("restricted_pack", "", getRestrictedPack().doc());
- get().files(fileCounter);
- EXPECT_EQ(count, 0U);
-
- // Test a more-generic update.
- count = 0;
- get().update({{"data", "{\"file_paths\": {\"new\": [\"/new\"]}}"}});
- get().files(fileCounter);
- EXPECT_EQ(count, 1U);
-
- count = 0;
- get().update({{"data", "{}"}});
- get().files(fileCounter);
- EXPECT_EQ(count, 0U);
-}
-
-void waitForConfig(std::shared_ptr<TestConfigPlugin>& plugin, size_t count) {
- // Max wait of 3 seconds.
- auto delay = std::chrono::milliseconds{3000};
- auto const step = std::chrono::milliseconds{20};
- while (delay.count() > 0) {
- if (plugin->gen_config_count_ > count) {
- break;
- }
- delay -= step;
- std::this_thread::sleep_for(step);
- }
-}
-
-TEST_F(ConfigTests, test_config_refresh) {
- auto& rf = RegistryFactory::get();
- auto refresh = FLAGS_config_refresh;
- auto refresh_acceleratred = FLAGS_config_accelerated_refresh;
-
- // Create and add a test plugin.
- auto plugin = std::make_shared<TestConfigPlugin>();
- EXPECT_TRUE(rf.registry("config")->add("test", plugin));
- EXPECT_TRUE(rf.setActive("config", "test"));
-
- // Reset the configuration and stop the refresh thread.
- get().reset();
-
- // Stop the existing refresh runner thread.
- Dispatcher::stopServices();
- Dispatcher::joinServices();
-
- // Set a config_refresh value to convince the Config to start the thread.
- FLAGS_config_refresh = 2;
- FLAGS_config_accelerated_refresh = 1;
- get().setRefresh(FLAGS_config_refresh);
-
- // Fail the first config load.
- plugin->fail_ = true;
-
- // The runner will wait at least one refresh-delay.
- auto count = static_cast<size_t>(plugin->gen_config_count_);
-
- get().load();
- EXPECT_TRUE(get().started_thread_);
- EXPECT_GT(plugin->gen_config_count_, count);
- EXPECT_EQ(get().getRefresh(), FLAGS_config_accelerated_refresh);
-
- plugin->fail_ = false;
- count = static_cast<size_t>(plugin->gen_config_count_);
-
- waitForConfig(plugin, count + 1);
- EXPECT_GT(plugin->gen_config_count_, count);
- EXPECT_EQ(get().getRefresh(), FLAGS_config_refresh);
-
- // Now make the configuration break.
- plugin->fail_ = true;
- count = static_cast<size_t>(plugin->gen_config_count_);
-
- waitForConfig(plugin, count + 1);
- EXPECT_GT(plugin->gen_config_count_, count);
- EXPECT_EQ(get().getRefresh(), FLAGS_config_accelerated_refresh);
-
- // Test that the normal acceleration is restored.
- plugin->fail_ = false;
- count = static_cast<size_t>(plugin->gen_config_count_);
-
- waitForConfig(plugin, count + 1);
- EXPECT_GT(plugin->gen_config_count_, count);
- EXPECT_EQ(get().getRefresh(), FLAGS_config_refresh);
-
- // Stop the new refresh runner thread.
- Dispatcher::stopServices();
- Dispatcher::joinServices();
-
- FLAGS_config_refresh = refresh;
- FLAGS_config_accelerated_refresh = refresh_acceleratred;
- rf.registry("config")->remove("test");
-}
-
-TEST_F(ConfigTests, test_config_backup) {
- get().reset();
- const std::map<std::string, std::string> expected_config = {{"a", "b"},
- {"c", "d"}};
- get().backupConfig(expected_config);
- const auto config = get().restoreConfigBackup();
- EXPECT_TRUE(config);
- EXPECT_EQ(*config, expected_config);
-}
-
-TEST_F(ConfigTests, test_config_backup_integrate) {
- const auto config_enable_backup_saved = FLAGS_config_enable_backup;
- FLAGS_config_enable_backup = true;
-
- get().reset();
- auto& rf = RegistryFactory::get();
- auto data_parser = std::make_shared<TestDataConfigParserPlugin>();
- auto success_plugin = std::make_shared<TestConfigPlugin>();
- auto fail_plugin = std::make_shared<TestConfigPlugin>();
- fail_plugin->fail_ = true;
- success_plugin->fail_ = false;
-
- rf.registry("config")->add("test_success", success_plugin);
- rf.registry("config")->add("test_fail", fail_plugin);
- rf.registry("config_parser")->add("test", data_parser);
- // Change the active config plugin.
- EXPECT_TRUE(rf.setActive("config", "test_success").ok());
-
- auto status = get().refresh();
- EXPECT_TRUE(status.ok());
- EXPECT_EQ(success_plugin->gen_config_count_, 1);
-
- auto source_backup = data_parser->source_;
- auto config_backup = data_parser->config_;
-
- EXPECT_TRUE(source_backup.length() > 0);
-
- get().reset();
- data_parser->source_.clear();
- data_parser->config_.clear();
- EXPECT_TRUE(rf.setActive("config", "test_fail").ok());
-
- status = get().refresh();
- EXPECT_FALSE(status.ok());
- EXPECT_EQ(fail_plugin->gen_config_count_, 1);
-
- EXPECT_EQ(data_parser->source_, source_backup);
- EXPECT_EQ(data_parser->config_, config_backup);
-
- FLAGS_config_enable_backup = config_enable_backup_saved;
-}
-}
+++ /dev/null
-/**
- * Copyright (c) 2014-present, Facebook, Inc.
- * All rights reserved.
- *
- * This source code is licensed in accordance with the terms specified in
- * the LICENSE file found in the root directory of this source tree.
- */
-
-#include <osquery/config/config.h>
-
-#include <osquery/config/tests/test_utils.h>
-
-#include <osquery/core.h>
-#include <osquery/database.h>
-#include <osquery/flags.h>
-#include <osquery/packs.h>
-#include <osquery/registry.h>
-#include <osquery/system.h>
-
-#include <osquery/filesystem/filesystem.h>
-
-#include <osquery/utils/info/platform_type.h>
-
-#include <gtest/gtest.h>
-
-#include <string>
-#include <vector>
-
-
-namespace osquery {
-
-DECLARE_bool(disable_database);
-
-class PacksTests : public testing::Test {
- public:
- PacksTests() {
- Initializer::platformSetup();
- registryAndPluginInit();
- FLAGS_disable_database = true;
- DatabasePlugin::setAllowOpen(true);
- DatabasePlugin::initPlugin();
- }
-};
-
-TEST_F(PacksTests, test_parse) {
- auto doc = getExamplePacksConfig();
- EXPECT_TRUE(doc.doc().HasMember("packs"));
-}
-
-TEST_F(PacksTests, test_should_pack_execute) {
- Pack kpack("unrestricted_pack", getUnrestrictedPack().doc());
- EXPECT_TRUE(kpack.shouldPackExecute());
-
- Pack fpack("discovery_pack", getPackWithDiscovery().doc());
- EXPECT_FALSE(fpack.shouldPackExecute());
-}
-
-TEST_F(PacksTests, test_get_discovery_queries) {
- std::vector<std::string> expected;
-
- Pack kpack("unrestricted_pack", getUnrestrictedPack().doc());
- EXPECT_EQ(kpack.getDiscoveryQueries(), expected);
-
- expected = {"select pid from processes where name = 'foobar';"};
- Pack fpack("discovery_pack", getPackWithDiscovery().doc());
- EXPECT_EQ(fpack.getDiscoveryQueries(), expected);
-}
-
-TEST_F(PacksTests, test_platform) {
- Pack fpack("discovery_pack", getPackWithDiscovery().doc());
- EXPECT_EQ(fpack.getPlatform(), "all");
-}
-
-TEST_F(PacksTests, test_version) {
- Pack fpack("discovery_pack", getPackWithDiscovery().doc());
- EXPECT_EQ(fpack.getVersion(), "1.5.0");
-}
-
-TEST_F(PacksTests, test_check_platform) {
- // First we exercise some basic functionality which should behave the same
- // regardless of the current build platform.
- Pack fpack("discovery_pack", getPackWithDiscovery().doc());
- EXPECT_TRUE(fpack.checkPlatform());
-
- fpack.platform_ = "null";
- EXPECT_TRUE(fpack.checkPlatform());
-
- fpack.platform_ = "";
- EXPECT_TRUE(fpack.checkPlatform());
-
- fpack.platform_ = "bad_value";
- EXPECT_FALSE(fpack.checkPlatform());
-
- // We should execute the query if the SDK platform is specifed.
- fpack.platform_ = kSDKPlatform;
- EXPECT_TRUE(fpack.checkPlatform());
- // But not if something other than the SDK platform is speciifed.
- fpack.platform_ = (kSDKPlatform == "darwin") ? "linux" : "darwin";
- EXPECT_FALSE(fpack.checkPlatform());
-
- // For the remaining tests, we exercise all of the valid platform values.
- fpack.platform_ = "darwin";
- if (isPlatform(PlatformType::TYPE_OSX)) {
- EXPECT_TRUE(fpack.checkPlatform());
- } else {
- EXPECT_FALSE(fpack.checkPlatform());
- }
-
- fpack.platform_ = "freebsd";
- if (isPlatform(PlatformType::TYPE_FREEBSD)) {
- EXPECT_TRUE(fpack.checkPlatform());
- } else {
- EXPECT_FALSE(fpack.checkPlatform());
- }
-
- // Although officially no longer supported, we still treat the platform
- // values of "centos" and "ubuntu" just like "linux". We execute any query
- // with any of these platform values on any Linux system. For what it's
- // worth, we never actually differentiated between Linux distributions.
- for (auto p : std::set<std::string>{"centos", "linux", "ubuntu"}) {
- fpack.platform_ = p;
- if (isPlatform(PlatformType::TYPE_LINUX)) {
- EXPECT_TRUE(fpack.checkPlatform());
- } else {
- EXPECT_FALSE(fpack.checkPlatform());
- }
- }
-
- fpack.platform_ = "posix";
- if (isPlatform(PlatformType::TYPE_POSIX) ||
- isPlatform(PlatformType::TYPE_LINUX) ||
- isPlatform(PlatformType::TYPE_OSX) ||
- isPlatform(PlatformType::TYPE_FREEBSD)) {
- EXPECT_TRUE(fpack.checkPlatform());
- } else {
- EXPECT_FALSE(fpack.checkPlatform());
- }
-
- fpack.platform_ = "windows";
- if (isPlatform(PlatformType::TYPE_WINDOWS)) {
- EXPECT_TRUE(fpack.checkPlatform());
- } else {
- EXPECT_FALSE(fpack.checkPlatform());
- }
-}
-
-TEST_F(PacksTests, test_check_version) {
- Pack zpack("fake_version_pack", getPackWithFakeVersion().doc());
- EXPECT_FALSE(zpack.checkVersion());
-
- Pack fpack("discovery_pack", getPackWithDiscovery().doc());
- EXPECT_TRUE(fpack.checkVersion());
-}
-
-TEST_F(PacksTests, test_restriction_population) {
- // Require that all potential restrictions are populated before being checked.
- auto doc = getExamplePacksConfig();
- const auto& packs = doc.doc()["packs"];
- Pack fpack("fake_pack", packs["restricted_pack"]);
-
- ASSERT_FALSE(fpack.getPlatform().empty());
- ASSERT_FALSE(fpack.getVersion().empty());
- ASSERT_EQ(fpack.getShard(), 1U);
-}
-
-TEST_F(PacksTests, test_schedule) {
- Pack fpack("discovery_pack", getPackWithDiscovery().doc());
- // Expect a single query in the schedule since one query has an explicit
- // invalid/fake platform requirement.
- EXPECT_EQ(fpack.getSchedule().size(), 1U);
-}
-
-TEST_F(PacksTests, test_discovery_cache) {
- Config c;
- // This pack and discovery query are valid, expect the SQL to execute.
- c.addPack("valid_discovery_pack", "", getPackWithValidDiscovery().doc());
- size_t query_count = 0U;
- size_t query_attempts = 5U;
- for (size_t i = 0; i < query_attempts; i++) {
- c.scheduledQueries(
- ([&query_count](std::string name, const ScheduledQuery& query) {
- query_count++;
- }));
- }
- EXPECT_EQ(query_count, query_attempts);
-
- size_t pack_count = 0U;
- c.packs(([&pack_count, query_attempts](const Pack& p) {
- pack_count++;
- // There is one pack without a discovery query.
- EXPECT_EQ(p.getStats().total, query_attempts + 1);
- EXPECT_EQ(p.getStats().hits, query_attempts);
- EXPECT_EQ(p.getStats().misses, 1U);
- }));
-
- EXPECT_EQ(pack_count, 1U);
- c.reset();
-}
-
-TEST_F(PacksTests, test_multi_pack) {
- std::string multi_pack_content = "{\"first\": {}, \"second\": {}}";
- auto multi_pack = JSON::newObject();
- multi_pack.fromString(multi_pack_content);
-
- Config c;
- c.addPack("*", "", multi_pack.doc());
-
- std::vector<std::string> pack_names;
- c.packs(
- ([&pack_names](const Pack& p) { pack_names.push_back(p.getName()); }));
-
- std::vector<std::string> expected = {"first", "second"};
- ASSERT_EQ(expected.size(), pack_names.size());
- EXPECT_EQ(expected, pack_names);
-}
-
-TEST_F(PacksTests, test_discovery_zero_state) {
- Pack pack("discovery_pack", getPackWithDiscovery().doc());
- auto stats = pack.getStats();
- EXPECT_EQ(stats.total, 0U);
- EXPECT_EQ(stats.hits, 0U);
- EXPECT_EQ(stats.misses, 0U);
-}
-
-TEST_F(PacksTests, test_splay) {
- auto val1 = splayValue(100, 10);
- EXPECT_GE(val1, 90U);
- EXPECT_LE(val1, 110U);
-
- auto val2 = splayValue(100, 10);
- EXPECT_GE(val2, 90U);
- EXPECT_LE(val2, 110U);
-
- auto val3 = splayValue(10, 0);
- EXPECT_EQ(val3, 10U);
-
- auto val4 = splayValue(100, 1);
- EXPECT_GE(val4, 99U);
- EXPECT_LE(val4, 101U);
-
- auto val5 = splayValue(1, 10);
- EXPECT_EQ(val5, 1U);
-}
-
-TEST_F(PacksTests, test_restore_splay) {
- auto splay = restoreSplayedValue("pack_test_query_name", 3600);
- EXPECT_GE(splay, 3600U - 360);
- EXPECT_LE(splay, 3600U + 360);
-
- // If we restore, the splay should always be equal.
- for (size_t i = 0; i < 10; i++) {
- auto splay2 = restoreSplayedValue("pack_test_query_name", 3600);
- EXPECT_EQ(splay, splay2);
- }
-
- // If we modify the input interval the splay will change.
- auto splay3 = restoreSplayedValue("pack_test_query_name", 3600 * 10);
- EXPECT_GE(splay3, 3600U * 10 - (360 * 10));
- EXPECT_LE(splay3, 3600U * 10 + (360 * 10));
- EXPECT_NE(splay, splay3);
-}
-}
+++ /dev/null
-/**
- * Copyright (c) 2014-present, Facebook, Inc.
- * All rights reserved.
- *
- * This source code is licensed in accordance with the terms specified in
- * the LICENSE file found in the root directory of this source tree.
- */
-
-#include <osquery/config/tests/test_utils.h>
-
-#include <osquery/filesystem/filesystem.h>
-
-#include <osquery/utils/system/env.h>
-
-#include <gtest/gtest.h>
-
-#include <boost/io/detail/quoted_manip.hpp>
-
-#include <cstdlib>
-
-namespace {
-
-namespace fs = boost::filesystem;
-
-fs::path getConfDirPathImpl() {
- return fs::path("../../../tools/tests/");
-}
-
-}
-
-namespace osquery {
-
-fs::path const& getTestConfigDirectory() {
- static auto const path = getConfDirPathImpl();
- return path;
-}
-
-std::map<std::string, std::string> getTestConfigMap(const std::string& file) {
- std::string content;
- auto const filepath = getTestConfigDirectory() / file;
- auto status = readFile(filepath, content);
- EXPECT_TRUE(status.ok())
- << "Could not read file: " << boost::io::quoted(filepath.string())
- << ", because: " << status.what();
- std::map<std::string, std::string> config;
- config["awesome"] = content;
- return config;
-}
-
-JSON getExamplePacksConfig() {
- std::string content;
- auto const filepath = getTestConfigDirectory() / "test_inline_pack.conf";
- auto status = readFile(filepath, content);
- EXPECT_TRUE(status.ok())
- << "Could not read file: " << boost::io::quoted(filepath.string())
- << ", because: " << status.what();
- JSON doc = JSON::newObject();
- doc.fromString(content);
- return doc;
-}
-
-/// no discovery queries, no platform restriction
-JSON getUnrestrictedPack() {
- auto doc = getExamplePacksConfig();
- return JSON::newFromValue(doc.doc()["packs"]["unrestricted_pack"]);
-}
-
-// several restrictions (version, platform, shard)
-JSON getRestrictedPack() {
- auto doc = getExamplePacksConfig();
- return JSON::newFromValue(doc.doc()["packs"]["restricted_pack"]);
-}
-
-/// 1 discovery query, darwin platform restriction
-JSON getPackWithDiscovery() {
- auto doc = getExamplePacksConfig();
- return JSON::newFromValue(doc.doc()["packs"]["discovery_pack"]);
-}
-
-/// 1 discovery query which will always pass
-JSON getPackWithValidDiscovery() {
- auto doc = getExamplePacksConfig();
- return JSON::newFromValue(doc.doc()["packs"]["valid_discovery_pack"]);
-}
-
-/// no discovery queries, no platform restriction, fake version string
-JSON getPackWithFakeVersion() {
- auto doc = getExamplePacksConfig();
- return JSON::newFromValue(doc.doc()["packs"]["fake_version_pack"]);
-}
-
-} // namespace osquery
+++ /dev/null
-/**
- * Copyright (c) 2014-present, Facebook, Inc.
- * All rights reserved.
- *
- * This source code is licensed in accordance with the terms specified in
- * the LICENSE file found in the root directory of this source tree.
- */
-
-#pragma once
-
-#include <osquery/utils/json/json.h>
-
-#include <boost/filesystem/path.hpp>
-
-#include <map>
-#include <string>
-
-namespace osquery {
-
-boost::filesystem::path const& getTestConfigDirectory();
-
-// Get an example generate config with one static source name to JSON content.
-std::map<std::string, std::string> getTestConfigMap(const std::string& file);
-
-JSON getExamplePacksConfig();
-JSON getUnrestrictedPack();
-JSON getRestrictedPack();
-JSON getPackWithDiscovery();
-JSON getPackWithValidDiscovery();
-JSON getPackWithFakeVersion();
-
-} // namespace osquery
#include "osquery/utils/config/default_paths.h"
#include "osquery/utils/info/platform_type.h"
-#include <osquery/config/config.h>
#include <osquery/core.h>
#include <osquery/data_logger.h>
#include <osquery/dispatcher.h>
namespace osquery {
-DECLARE_string(config_plugin);
DECLARE_string(logger_plugin);
DECLARE_bool(config_check);
DECLARE_bool(config_dump);
// Initialize random number generated based on time.
std::srand(static_cast<unsigned int>(
chrono_clock::now().time_since_epoch().count()));
- // The config holds the initialization time for easy access.
- Config::setStartTime(getUnixTime());
// osquery can function as the daemon or shell depending on argv[0].
if (tool == ToolType::SHELL_DAEMON) {
return;
}
- if (FLAGS_config_check) {
- // No need to daemonize, emit log lines, or create process mutexes.
- return;
- }
-
#if !defined(__APPLE__) && !defined(WIN32)
// OS X uses launchd to daemonize.
if (osquery::FLAGS_daemonize) {
}
- // Then set the config plugin, which uses a single/active plugin.
- initActivePlugin("config", FLAGS_config_plugin);
-
// Run the setup for all lazy registries (tables, SQL).
Registry::setUp();
- if (FLAGS_config_check) {
- // The initiator requested an initialization and config check.
- auto s = Config::get().load();
- if (!s.ok()) {
- std::cerr << "Error reading config: " << s.toString() << "\n";
- }
- // A configuration check exits the application.
- // Make sure to request a shutdown as plugins may have created services.
- requestShutdown(s.getCode());
- }
-
if (FLAGS_database_dump) {
dumpDatabase();
requestShutdown();
}
- // Load the osquery config using the default/active config plugin.
- auto s = Config::get().load();
- if (!s.ok()) {
- auto message = "Error reading config: " + s.toString();
- if (isDaemon()) {
- LOG(WARNING) << message;
- } else {
- VLOG(1) << message;
- }
- }
-
// Initialize the status and result plugin logger.
if (!FLAGS_disable_logging) {
initActivePlugin("logger", FLAGS_logger_plugin);
namespace osquery {
-DECLARE_bool(decorations_top_level);
-
/// Log numeric values as numbers (in JSON syntax)
FLAG(bool,
log_numerics_as_numbers,
if (!item.decorations.empty()) {
auto dec_obj = doc.getObject();
auto target_obj = std::ref(dec_obj);
- if (FLAGS_decorations_top_level) {
- target_obj = std::ref(obj);
- }
+ target_obj = std::ref(obj);
+
for (const auto& name : item.decorations) {
doc.addRef(name.first, name.second, target_obj);
}
- if (!FLAGS_decorations_top_level) {
- doc.add("decorations", dec_obj, obj);
- }
}
}
#include <boost/algorithm/string/predicate.hpp>
-#include <osquery/config/config.h>
#include <osquery/database.h>
#include <osquery/devtools/devtools.h>
#include <osquery/filesystem/filesystem.h>
DECLARE_string(nullvalue);
DECLARE_string(logger_plugin);
DECLARE_string(logger_path);
-DECLARE_string(config_plugin);
-DECLARE_string(config_path);
DECLARE_string(database_path);
} // namespace osquery
fprintf(p->out, "\nGeneral settings:\n");
fprintf(p->out, "%13.13s: %s\n", "Flagfile", FLAGS_flagfile.c_str());
- // Show helpful config-related settings.
- fprintf(
- p->out, "%13.13s: %s", "Config", osquery::FLAGS_config_plugin.c_str());
- if (std::strcmp(osquery::FLAGS_config_plugin.c_str(), "filesystem") == 0) {
- fprintf(p->out, " (%s)\n", osquery::FLAGS_config_path.c_str());
- } else {
- fprintf(p->out, "\n");
- }
// Show helpful logger-related settings.
fprintf(
int runPack(struct callback_data* data) {
int rc = 0;
-
- // Check every pack for a name matching the requested --pack flag.
- Config::get().packs([data, &rc](const Pack& pack) {
- if (pack.getName() != FLAGS_pack) {
- return;
- }
-
- for (const auto& query : pack.getSchedule()) {
- rc = runQuery(data, query.second.query.c_str());
- if (rc != 0) {
- fprintf(stderr,
- "Could not execute query %s: %s\n",
- query.first.c_str(),
- query.second.query.c_str());
- return;
- }
- }
- });
return rc;
}
#include <boost/format.hpp>
#include <boost/io/detail/quoted_manip.hpp>
-#include <osquery/config/config.h>
#include <osquery/core.h>
#include <osquery/data_logger.h>
#include <osquery/database.h>
#include "osquery/dispatcher/scheduler.h"
#include "osquery/sql/sqlite_util.h"
-#include "osquery/plugins/config/parsers/decorators.h"
namespace osquery {
EQUALS,
pid);
auto t0 = getUnixTime();
- Config::get().recordQueryStart(name);
SQLInternal sql(query.query, true);
// Snapshot the performance after, and compare.
auto t1 = getUnixTime();
"pid",
EQUALS,
pid);
- if (r0.size() > 0 && r1.size() > 0) {
- // Always called while processes table is working.
- Config::get().recordQueryPerformance(name, t1 - t0, r0[0], r1[0]);
- }
return sql;
}
Status launchQuery(const std::string& name, const ScheduledQuery& query) {
// Execute the scheduled query and create a named query object.
LOG(INFO) << "Executing scheduled query " << name << ": " << query.query;
- runDecorators(DECORATE_ALWAYS);
auto sql = monitor(name, query);
if (!sql.getStatus().ok()) {
item.time = osquery::getUnixTime();
item.epoch = FLAGS_schedule_epoch;
item.calendar_time = osquery::getAsciiTime();
- getDecorations(item.decorations);
if (query.options.count("snapshot") && query.options.at("snapshot")) {
// This is a snapshot query, emit results with a differential or state.
auto i = osquery::getUnixTime();
for (; (timeout_ == 0) || (i <= timeout_); ++i) {
auto start_time_point = std::chrono::steady_clock::now();
- Config::get().scheduledQueries(
- ([&i](const std::string& name, const ScheduledQuery& query) {
- if (query.splayed_interval > 0 && i % query.splayed_interval == 0) {
- TablePlugin::kCacheInterval = query.splayed_interval;
- TablePlugin::kCacheStep = i;
- const auto status = launchQuery(name, query);
- }
- }));
- // Configuration decorators run on 60 second intervals only.
- if ((i % 60) == 0) {
- runDecorators(DECORATE_INTERVAL, i);
- }
if (FLAGS_schedule_reload > 0 && (i % FLAGS_schedule_reload) == 0) {
if (FLAGS_schedule_reload_sql) {
SQLiteDBManager::resetPrimary();
+++ /dev/null
-/**
- * Copyright (c) 2014-present, Facebook, Inc.
- * All rights reserved.
- *
- * This source code is licensed in accordance with the terms specified in
- * the LICENSE file found in the root directory of this source tree.
- */
-
-#include <gtest/gtest.h>
-
-#include <osquery/database.h>
-#include <osquery/logger.h>
-#include <osquery/registry.h>
-#include <osquery/system.h>
-
-#include <osquery/config/config.h>
-#include <osquery/dispatcher/scheduler.h>
-#include <osquery/sql/sqlite_util.h>
-#include <osquery/utils/system/time.h>
-
-namespace osquery {
-
-DECLARE_bool(disable_database);
-DECLARE_bool(disable_logging);
-DECLARE_uint64(schedule_reload);
-
-class SchedulerTests : public testing::Test {
- void SetUp() override {
- Initializer::platformSetup();
- registryAndPluginInit();
- FLAGS_disable_database = true;
- DatabasePlugin::setAllowOpen(true);
- DatabasePlugin::initPlugin();
-
- logging_ = FLAGS_disable_logging;
- FLAGS_disable_logging = true;
- Config::get().reset();
- }
-
- void TearDown() override {
- FLAGS_disable_logging = logging_;
- Config::get().reset();
- }
-
- private:
- bool logging_{false};
-};
-
-TEST_F(SchedulerTests, test_monitor) {
- std::string name = "pack_test_test_query";
-
- // This query has never run so it will not have a timestamp.
- std::string timestamp;
- getDatabaseValue(kPersistentSettings, "timestamp." + name, timestamp);
- ASSERT_TRUE(timestamp.empty());
-
- // Fill in a scheduled query and execute it via the query monitor wrapper.
- ScheduledQuery query("time_pack", "time", "select * from time");
- query.interval = 10;
- query.splayed_interval = 11;
-
- auto results = monitor(name, query);
- EXPECT_EQ(results.rowsTyped().size(), 1U);
-
- // Ask the config instance for the monitored performance.
- QueryPerformance perf;
- Config::get().getPerformanceStats(
- name, ([&perf](const QueryPerformance& r) { perf = r; }));
- // Make sure it was recorded query ran.
- // There is no pack for this query within the config, that is fine as these
- // performance stats are tracked independently.
- EXPECT_EQ(perf.executions, 1U);
-
- // A bit more testing, potentially redundant, check the database results.
- // Since we are only monitoring, no 'actual' results are stored.
- std::string content;
- getDatabaseValue(kQueries, name, content);
- EXPECT_TRUE(content.empty());
-
- // Finally, make sure there is a recorded timestamp for the execution.
- // We are not concerned with the APPROX value, only that it was recorded.
- getDatabaseValue(kPersistentSettings, "timestamp." + name, timestamp);
- EXPECT_FALSE(timestamp.empty());
-}
-
-TEST_F(SchedulerTests, test_config_results_purge) {
- // Set a query time for now (time is only important relative to a week ago).
- auto query_time = osquery::getUnixTime();
- setDatabaseValue(
- kPersistentSettings, "timestamp.test_query", std::to_string(query_time));
- // Store a meaningless saved query interval splay.
- setDatabaseValue(kPersistentSettings, "interval.test_query", "11");
- // Store meaningless query differential results.
- setDatabaseValue(kQueries, "test_query", "{}");
-
- // We do not need "THE" config instance.
- // We only need to trigger a 'purge' event, this occurs when configuration
- // content is updated by a plugin or on load.
- Config::get().purge();
-
- // Nothing should have been purged.
- {
- std::string content;
- getDatabaseValue(kPersistentSettings, "timestamp.test_query", content);
- EXPECT_FALSE(content.empty());
- }
-
- {
- std::string content;
- getDatabaseValue(kPersistentSettings, "interval.test_query", content);
- EXPECT_FALSE(content.empty());
- }
-
- {
- std::string content;
- getDatabaseValue(kQueries, "test_query", content);
- EXPECT_FALSE(content.empty());
- }
-
- // Update the timestamp to have run a week and a day ago.
- query_time -= (84600 * (7 + 1));
- setDatabaseValue(
- kPersistentSettings, "timestamp.test_query", std::to_string(query_time));
-
- // Trigger another purge.
- Config::get().purge();
- // Now ALL 'test_query' related storage will have been purged.
- {
- std::string content;
- getDatabaseValue(kPersistentSettings, "timestamp.test_query", content);
- EXPECT_TRUE(content.empty());
- }
-
- {
- std::string content;
- getDatabaseValue(kPersistentSettings, "interval.test_query", content);
- EXPECT_TRUE(content.empty());
- }
-
- {
- std::string content;
- getDatabaseValue(kQueries, "test_query", content);
- EXPECT_TRUE(content.empty());
- }
-}
-
-TEST_F(SchedulerTests, test_scheduler) {
- auto backup_step = TablePlugin::kCacheStep;
- auto backup_interval = TablePlugin::kCacheInterval;
-
- // Start the scheduler now.
- auto now = osquery::getUnixTime();
- TablePlugin::kCacheStep = now;
-
- // Update the config with a pack/schedule that contains several queries.
- std::string config =
- "{"
- "\"packs\": {"
- "\"scheduler\": {"
- "\"queries\": {"
- "\"1\": {\"query\": \"select * from osquery_schedule\", \"interval\": 1},"
- "\"2\": {\"query\": \"select * from osquery_info\", \"interval\": 1},"
- "\"3\": {\"query\": \"select * from processes\", \"interval\": 1},"
- "\"4\": {\"query\": \"select * from osquery_packs\", \"interval\": 1}"
- "}"
- "}"
- "}"
- "}";
- Config::get().update({{"data", config}});
-
- // Run the scheduler for 1 second with a second interval.
- SchedulerRunner runner(static_cast<unsigned long int>(now + 1), 1);
- runner.start();
-
- // If a query was executed the cache step will have been advanced.
- EXPECT_GT(TablePlugin::kCacheStep, now);
-
- // Restore plugin settings.
- TablePlugin::kCacheStep = backup_step;
- TablePlugin::kCacheInterval = backup_interval;
-}
-
-TEST_F(SchedulerTests, test_scheduler_zero_drift) {
- const auto backup_step = TablePlugin::kCacheStep;
- const auto backup_interval = TablePlugin::kCacheInterval;
-
- // Start the scheduler now.
- const auto now = osquery::getUnixTime();
- TablePlugin::kCacheStep = now;
-
- // Update the config with a pack/schedule that contains several queries.
- std::string config = R"config(
- {
- "packs": {
- "scheduler": {
- "queries": {
- "1": {"query": "select 1 as number", "interval": 1},
- "2": {"query": "select 2 as number", "interval": 1}
- }
- }
- }
- })config";
- Config::get().update({{"data", config}});
-
- // Run the scheduler for 1 second with a second interval.
- SchedulerRunner runner(
- static_cast<unsigned long int>(now), size_t{1}, std::chrono::seconds{10});
- runner.start();
-
- EXPECT_EQ(runner.getCurrentTimeDrift(), std::chrono::milliseconds::zero());
-
- // Restore plugin settings.
- TablePlugin::kCacheStep = backup_step;
- TablePlugin::kCacheInterval = backup_interval;
-}
-
-TEST_F(SchedulerTests, test_scheduler_drift_accumulation) {
- const auto backup_step = TablePlugin::kCacheStep;
- const auto backup_interval = TablePlugin::kCacheInterval;
-
- // Start the scheduler now.
- const auto now = osquery::getUnixTime();
- TablePlugin::kCacheStep = now;
-
- // Update the config with a pack/schedule that contains several queries.
- std::string config = R"config(
- {
- "packs": {
- "scheduler": {
- "queries": {
- "3": {"query": "select 3 as number", "interval": 1},
- "4": {"query": "select 4 as number", "interval": 1},
- "5": {"query": "select 5 as number", "interval": 1},
- "6": {"query": "select 6 as number", "interval": 1},
- "7": {"query": "select 7 as number", "interval": 1},
- "8": {"query": "select 1 as number", "interval": 1},
- "9": {"query": "select 2 as number", "interval": 1}
- }
- }
- }
- })config";
- Config::get().update({{"data", config}});
-
- // Run the scheduler for 1 second with a second interval.
- SchedulerRunner runner(static_cast<unsigned long int>(now + 3),
- size_t{0},
- std::chrono::seconds{10});
- runner.start();
-
- EXPECT_GE(runner.getCurrentTimeDrift(), std::chrono::milliseconds{1});
-
- // Restore plugin settings.
- TablePlugin::kCacheStep = backup_step;
- TablePlugin::kCacheInterval = backup_interval;
-}
-
-TEST_F(SchedulerTests, test_scheduler_reload) {
- std::string config =
- "{\"schedule\":{\"1\":{"
- "\"query\":\"select * from processes\", \"interval\":1}}}";
- auto backup_reload = FLAGS_schedule_reload;
-
- // Start the scheduler;
- auto expire = static_cast<unsigned long int>(getUnixTime() + 1);
- FLAGS_schedule_reload = 1;
- SchedulerRunner runner(expire, 1);
- FLAGS_schedule_reload = backup_reload;
-}
-}
#include <boost/algorithm/string/classification.hpp>
#include <boost/lexical_cast.hpp>
-#include <osquery/config/config.h>
#include <osquery/database.h>
#include <osquery/events.h>
#include <osquery/flags.h>
std::string& query_name,
const std::string& publisher) {
// Read the optimization time for the current executing query.
- getDatabaseValue(kPersistentSettings, kExecutingQuery, query_name);
+ getDatabaseValue(kPersistentSettings, "", query_name);
if (query_name.empty()) {
o_time = 0;
o_eid = 0;
const std::string& publisher) {
// Store the optimization time and eid.
std::string query_name;
- getDatabaseValue(kPersistentSettings, kExecutingQuery, query_name);
+ getDatabaseValue(kPersistentSettings, "", query_name);
if (query_name.empty()) {
return;
}
// Scan the schedule for queries that touch "_events" tables.
// We will count the queries
std::map<std::string, SubscriberExpirationDetails> subscriber_details;
- Config::get().scheduledQueries(
- [&subscriber_details](std::string name, const ScheduledQuery& query) {
- std::vector<std::string> tables;
- // Convert query string into a list of virtual tables effected.
- if (!getQueryTables(query.query, tables)) {
- VLOG(1) << "Cannot get tables from query: " << name;
- return;
- }
-
- // Remove duplicates and select only the subscriber tables.
- std::set<std::string> subscribers;
- for (const auto& table : tables) {
- if (Registry::get().exists("event_subscriber", table)) {
- subscribers.insert(table);
- }
- }
-
- for (const auto& subscriber : subscribers) {
- auto& details = subscriber_details[subscriber];
- details.max_interval = (query.interval > details.max_interval)
- ? query.interval
- : details.max_interval;
- details.query_count++;
- }
- });
-
auto& ef = EventFactory::getInstance();
for (const auto& details : subscriber_details) {
if (!ef.exists(details.first)) {
return Status(1, "Subscribers must have set a name");
}
- auto plugin = Config::get().getParser("events");
- if (plugin != nullptr && plugin.get() != nullptr) {
- const auto& data = plugin->getData().doc();
- // First perform explicit enabling.
- if (data["events"].HasMember("enable_subscribers")) {
- for (const auto& item : data["events"]["enable_subscribers"].GetArray()) {
- if (item.GetString() == name) {
- VLOG(1) << "Enabling event subscriber: " << name;
- specialized_sub->disabled = false;
- }
- }
- }
- // Then use explicit disabling as an ultimate override.
- if (data["events"].HasMember("disable_subscribers")) {
- for (const auto& item :
- data["events"]["disable_subscribers"].GetArray()) {
- if (item.GetString() == name) {
- VLOG(1) << "Disabling event subscriber: " << name;
- specialized_sub->disabled = true;
- }
- }
- }
- }
-
if (specialized_sub->state() != EventState::EVENT_NONE) {
specialized_sub->tearDown();
}
#include <gtest/gtest.h>
-#include <osquery/config/config.h>
#include <osquery/core.h>
#include <osquery/core/sql/row.h>
#include <osquery/database.h>
DatabasePlugin::setAllowOpen(true);
DatabasePlugin::initPlugin();
- RegistryFactory::get().registry("config_parser")->setUp();
- optimize_ = FLAGS_events_optimize;
- FLAGS_events_optimize = false;
-
std::vector<std::string> event_keys;
scanDatabaseKeys(kEvents, event_keys);
for (const auto& key : event_keys) {
FLAGS_events_optimize = true;
// Must also define an executing query.
- setDatabaseValue(kPersistentSettings, kExecutingQuery, "events_db_test");
+ setDatabaseValue(kPersistentSettings, "", "events_db_test");
auto t = getUnixTime();
auto results = genRows(sub.get());
#include <gflags/gflags.h>
#include <gtest/gtest.h>
-#include <osquery/config/config.h>
#include <osquery/database.h>
#include <osquery/events.h>
#include <osquery/registry_factory.h>
EXPECT_TRUE(status.ok());
}
-TEST_F(EventsTests, test_event_subscriber_configure) {
- auto sub = std::make_shared<FakeEventSubscriber>();
- // Register this subscriber (within the RegistryFactory), so it receives
- // configure/reconfigure events.
- auto& rf = RegistryFactory::get();
- rf.registry("event_subscriber")->add("fake_events", sub);
-
- // Register it within the event factory too.
- auto status = EventFactory::registerEventSubscriber(sub);
- EXPECT_TRUE(status.ok());
-
- // Assure we start from a base state.
- EXPECT_EQ(sub->timesConfigured, 0U);
- // Force the config into a loaded state.
- Config::get().loaded_ = true;
- Config::get().update({{"data", "{}"}});
- EXPECT_EQ(sub->timesConfigured, 1U);
-
- // Now update the config to contain sets of scheduled queries.
- Config::get().update(
- {{"data",
- "{\"schedule\": {\"1\": {\"query\": \"select * from fake_events\", "
- "\"interval\": 10}, \"2\":{\"query\": \"select * from time, "
- "fake_events\", \"interval\": 19}, \"3\":{\"query\": \"select * "
- "from fake_events, fake_events\", \"interval\": 5}}}"}});
-
- // This will become 19 * 3, rounded up 60.
- EXPECT_EQ(sub->min_expiration_, 60U);
- EXPECT_EQ(sub->query_count_, 3U);
-
- // Register it within the event factory too.
- EventFactory::deregisterEventSubscriber(sub->getName());
- rf.registry("event_subscriber")->remove(sub->getName());
-
- // Final check to make sure updates are not effecting this subscriber.
- Config::get().update({{"data", "{}"}});
- EXPECT_EQ(sub->timesConfigured, 2U);
-}
-
TEST_F(EventsTests, test_fire_event) {
auto pub = std::make_shared<BasicEventPublisher>();
pub->setName("BasicPublisher");
+++ /dev/null
-/*
- * Copyright (c) 2014, Facebook, Inc.
- * All rights reserved.
- *
- * This source code is licensed under the BSD-style license found in the
- * LICENSE file in the root directory of this source tree. An additional grant
- * of patent rights can be found in the PATENTS file in the same directory.
- *
- */
-
-#pragma once
-
-#include <map>
-#include <memory>
-#include <vector>
-
-#include <boost/noncopyable.hpp>
-#include <boost/property_tree/ptree.hpp>
-#include <boost/property_tree/json_parser.hpp>
-#include <boost/thread/shared_mutex.hpp>
-
-#include <osquery/database.h>
-#include <osquery/flags.h>
-#include <osquery/registry.h>
-#include <osquery/status.h>
-
-namespace pt = boost::property_tree;
-
-namespace osquery {
-
-/// The builder or invoker may change the default config plugin.
-DECLARE_string(config_plugin);
-
-/**
- * @brief The osquery config is updated names sources containing JSON.
- *
- * A ConfigSourceMap is a named mapping from source (the key) to a JSON blob.
- * This map is generated by a ConfigPlugin an provided to the Config via an
- * update call. ConfigPlugin%s may update the Config asynchronously.
- *
- * The osquery Config instance will perform source merging by amalgamating
- * the JSON literal types (lists and maps) for well known top-level keys.
- * The merging will happen in lexicographical order based on source name.
- */
-typedef std::map<std::string, std::string> ConfigSourceMap;
-
-/**
- * @brief A native representation of osquery configuration data.
- *
- * When you use osquery::Config::getInstance(), you are getting a singleton
- * handle to interact with the data stored in an instance of this struct.
- */
-struct ConfigData {
- /// A vector of all of the queries that are scheduled to execute.
- std::map<std::string, ScheduledQuery> schedule;
- std::map<std::string, std::string> options;
- std::map<std::string, std::vector<std::string> > files;
- /// All data catches optional/plugin-parsed configuration keys.
- pt::ptree all_data;
-};
-
-class ConfigParserPlugin;
-typedef std::shared_ptr<ConfigParserPlugin> ConfigPluginRef;
-
-/**
- * @brief A singleton that exposes accessors to osquery's configuration data.
- *
- * osquery has two types on configurations. Things that don't change during
- * the execution of the process should be configured as command-line
- * arguments. Things that can change during the lifetime of program execution
- * should be defined using the osquery::config::Config class and the pluggable
- * plugin interface that is included with it.
- */
-class Config : private boost::noncopyable {
- public:
- /**
- * @brief The primary way to access the Config singleton.
- *
- * osquery::config::Config::getInstance() provides access to the Config
- * singleton
- *
- * @code{.cpp}
- * auto config = osquery::config::Config::getInstance();
- * @endcode
- *
- * @return a singleton instance of Config.
- */
- static Config& getInstance() {
- static Config cfg;
- return cfg;
- }
-
- /**
- * @brief Call the genConfig method of the config retriever plugin.
- *
- * This may perform a resource load such as TCP request or filesystem read.
- */
- static Status load();
-
- /**
- * @brief Update the internal config data.
- *
- * @param config A map of domain or namespace to config data.
- * @return If the config changes were applied.
- */
- static Status update(const ConfigSourceMap& config);
-
- /**
- * @brief Calculate the has of the osquery config
- *
- * @return The MD5 of the osquery config
- */
- static Status getMD5(std::string& hashString);
-
- /**
- * @brief Adds a new query to the scheduled queries.
- *
- */
- static void addScheduledQuery(const std::string& name,
- const std::string& query,
- int interval);
-
- /**
- * @brief Checks if a query exists in the query schedule.
- *
- */
- static bool checkScheduledQuery(const std::string& query);
-
- /**
- * @brief Checks if the query name exists in the query schedule.
- *
- */
- static bool checkScheduledQueryName(const std::string& query_name);
-
- /**
- * @brief Check to ensure that the config is accessible and properly
- * formatted
- *
- * @return an instance of osquery::Status, indicating the success or failure
- * of the operation.
- */
- static Status checkConfig();
-
- private:
- /**
- * @brief Default constructor.
- *
- * Since instances of Config should only be created via getInstance(),
- * Config's constructor is private
- */
- Config() : force_merge_success_(false) {}
- ~Config(){}
- Config(Config const&);
- void operator=(Config const&);
-
- /**
- * @brief Uses the specified config retriever to populate a string with the
- * config JSON.
- *
- * Internally, genConfig checks to see if there was a config retriever
- * specified on the command-line. If there was, it checks to see if that
- * config retriever actually exists. If it does, it gets used to generate
- * configuration data. If it does not, an error is logged.
- *
- * @return status indicating the success or failure of the operation.
- */
- static Status genConfig();
-
- /// Merge a retrieved config source JSON into a working ConfigData.
- static Status mergeConfig(const std::string& source, ConfigData& conf);
-
- public:
- /**
- * @brief Record performance (monitoring) information about a scheduled query.
- *
- * The daemon and query scheduler will optionally record process metadata
- * before and after executing each query. This can be compared and reported
- * on an interval or within the osquery_schedule table.
- *
- * The config consumes and calculates the optional performance differentials.
- * It would also be possible to store this in the RocksDB backing store or
- * report directly to a LoggerPlugin sink. The Config is the most appropriate
- * as the metrics are transient to the process running the schedule and apply
- * to the updates/changes reflected in the schedule, from the config.
- *
- * @param name The unique name of the scheduled item
- * @param delay Number of seconds (wall time) taken by the query
- * @param size Number of characters generated by query
- * @param t0 the process row before the query
- * @param t1 the process row after the query
- */
- static void recordQueryPerformance(const std::string& name,
- size_t delay,
- size_t size,
- const Row& t0,
- const Row& t1);
-
- private:
- /// The raw osquery config data in a native format
- ConfigData data_;
-
- /// The raw JSON source map from the config plugin.
- std::map<std::string, std::string> raw_;
-
- /// The reader/writer config data mutex.
- boost::shared_mutex mutex_;
-
- /// Enforce merge success.
- bool force_merge_success_;
-
- private:
- /**
- * @brief A ConfigDataInstance requests read-only access to ConfigParser data.
- *
- * A ConfigParser plugin will receive several top-level-config keys and
- * optionally parse and store information. That information is a property tree
- * called ConfigParser::data_. Use ConfigDataInstance::getParsedData to
- * retrieve read-only access to this data.
- *
- * @param parser The name of the config parser.
- */
- static const pt::ptree& getParsedData(const std::string& parser);
-
- /// See getParsedData but request access to the parser plugin.
- static const ConfigPluginRef getParser(const std::string& parser);
-
- /// A default, empty property tree used when a missing parser is requested.
- pt::ptree empty_data_;
-
- private:
- /// Config accessors, `ConfigDataInstance`, are the forced use of the config
- /// data. This forces the caller to use a shared read lock.
- friend class ConfigDataInstance;
-
- private:
- FRIEND_TEST(ConfigTests, test_locking);
-};
-
-/**
- * @brief All accesses to the Config's data must request a ConfigDataInstance.
- *
- * This class will request a read-only lock of the config's changeable internal
- * data structures such as query schedule, options, monitored files, etc.
- *
- * Since a variable config plugin may implement `update` calls, internal uses
- * of config data needs simple read and write locking.
- */
-class ConfigDataInstance {
- public:
- ConfigDataInstance() : lock_(Config::getInstance().mutex_) {}
- ~ConfigDataInstance() { lock_.unlock(); }
-
- /// Helper accessor for Config::data_.schedule.
- const std::map<std::string, ScheduledQuery> schedule() const {
- return Config::getInstance().data_.schedule;
- }
-
- /// Helper accessor for Config::data_.options.
- const std::map<std::string, std::string>& options() const {
- return Config::getInstance().data_.options;
- }
-
- /// Helper accessor for Config::data_.files.
- const std::map<std::string, std::vector<std::string> >& files() const {
- return Config::getInstance().data_.files;
- }
-
- const pt::ptree& getParsedData(const std::string& parser) const {
- return Config::getParsedData(parser);
- }
-
- const ConfigPluginRef getParser(const std::string& parser) const {
- return Config::getParser(parser);
- }
-
- /// Helper accessor for Config::data_.all_data.
- const pt::ptree& data() const { return Config::getInstance().data_.all_data; }
-
- private:
- /**
- * @brief ConfigParser plugin's may update the internal config representation.
- *
- * If the config parser reads and calculates new information it should store
- * that derived data itself and rely on ConfigDataInstance::getParsedData.
- * This means another plugin is aware of the ConfigParser and knowns to make
- * getParsedData calls. If the parser is augmenting/changing internal state,
- * such as modifying the osquery schedule or options, then it must write
- * changed back into the default data.
- *
- * Note that this returns the ConfigData instance, not the raw property tree.
- */
- ConfigData& mutableConfigData() { return Config::getInstance().data_; }
-
- private:
- /// A read lock on the reader/writer config data accessor/update mutex.
- boost::shared_lock<boost::shared_mutex> lock_;
-
- private:
- friend class ConfigParserPlugin;
-};
-
-/**
- * @brief Superclass for the pluggable config component.
- *
- * In order to make the distribution of configurations to hosts running
- * osquery, we take advantage of a plugin interface which allows you to
- * integrate osquery with your internal configuration distribution mechanisms.
- * You may use ZooKeeper, files on disk, a custom solution, etc. In order to
- * use your specific configuration distribution system, one simply needs to
- * create a custom subclass of ConfigPlugin. That subclass should implement
- * the ConfigPlugin::genConfig method.
- *
- * Consider the following example:
- *
- * @code{.cpp}
- * class TestConfigPlugin : public ConfigPlugin {
- * public:
- * virtual std::pair<osquery::Status, std::string> genConfig() {
- * std::string config;
- * auto status = getMyConfig(config);
- * return std::make_pair(status, config);
- * }
- * };
- *
- * REGISTER(TestConfigPlugin, "config", "test");
- * @endcode
- */
-class ConfigPlugin : public Plugin {
- public:
- /**
- * @brief Virtual method which should implemented custom config retrieval
- *
- * ConfigPlugin::genConfig should be implemented by a subclasses of
- * ConfigPlugin which needs to retrieve config data in a custom way.
- *
- * @param config The output ConfigSourceMap, a map of JSON to source names.
- * @return A failure status will prevent the source map from merging.
- */
- virtual Status genConfig(ConfigSourceMap& config) = 0;
- Status call(const PluginRequest& request, PluginResponse& response) override;
-};
-
-/// Helper merged and parsed property tree.
-typedef pt::ptree ConfigTree;
-
-/// Helper for a map of requested keys to their merged and parsed property tree.
-typedef std::map<std::string, ConfigTree> ConfigTreeMap;
-
-/**
- * @brief A pluggable configuration parser.
- *
- * An osquery config instance is populated from JSON using a ConfigPlugin.
- * That plugin may update the config data asynchronously and read from
- * several sources, as is the case with "filesystem" and reading multiple files.
- *
- * A ConfigParserPlugin will receive the merged configuration at osquery start
- * and the updated (still merged) config if any ConfigPlugin updates the
- * instance asynchronously. Each parser specifies a set of top-level JSON
- * keys to receive. The config instance will auto-merge the key values
- * from multiple sources if they are dictionaries or lists.
- *
- * If a top-level key is a dictionary, each source with the top-level key
- * will have its own dictionary keys merged and replaced based on the lexical
- * order of sources. For the "filesystem" config plugin this is the lexical
- * sorting of filenames. If the top-level key is a list, each source with the
- * top-level key will have its contents appended.
- *
- * Each config parser plugin will live alongside the config instance for the
- * life of the osquery process. The parser may perform actions at config load
- * and config update "time" as well as keep its own data members and be
- * accessible through the Config class API.
- */
-class ConfigParserPlugin : public Plugin {
- protected:
- /**
- * @brief Return a list of top-level config keys to receive in updates.
- *
- * The ::update method will receive a map of these keys with a JSON-parsed
- * property tree of configuration data.
- *
- * @return A list of string top-level JSON keys.
- */
- virtual std::vector<std::string> keys() = 0;
-
- /**
- * @brief Receive a merged property tree for each top-level config key.
- *
- * Called when the Config instance is initially loaded with data from the
- * active config plugin and when it is updated via an async ConfigPlugin
- * update. Every config parser will receive a map of merged data for each key
- * they requested in keys().
- *
- * @param config A JSON-parsed property tree map.
- * @return Failure if the parser should no longer receive updates.
- */
- virtual Status update(const ConfigTreeMap& config) = 0;
-
- protected:
- /// Mutable config data accessor for ConfigParser%s.
- ConfigData& mutableConfigData(ConfigDataInstance& cdi) {
- return cdi.mutableConfigData();
- }
-
- protected:
- /// Allow the config parser to keep some global state.
- pt::ptree data_;
-
- private:
- Status setUp();
-
- private:
- /// Config::update will call all appropriate parser updates.
- friend class Config;
- /// A config data instance implements a read/write lock around data_ access.
- friend class ConfigDataInstance;
-};
-
-/**
- * @brief Calculate a splayed integer based on a variable splay percentage
- *
- * The value of splayPercent must be between 1 and 100. If it's not, the
- * value of original will be returned.
- *
- * @param original The original value to be modified
- * @param splayPercent The percent in which to splay the original value by
- *
- * @return The modified version of original
- */
-int splayValue(int original, int splayPercent);
-
-/**
- * @brief Config plugin registry.
- *
- * This creates an osquery registry for "config" which may implement
- * ConfigPlugin. A ConfigPlugin's call API should make use of a genConfig
- * after reading JSON data in the plugin implementation.
- */
-CREATE_REGISTRY(ConfigPlugin, "config");
-
-/**
- * @brief ConfigParser plugin registry.
- *
- * This creates an osquery registry for "config_parser" which may implement
- * ConfigParserPlugin. A ConfigParserPlugin should not export any call actions
- * but rather have a simple property tree-accessor API through Config.
- */
-CREATE_LAZY_REGISTRY(ConfigParserPlugin, "config_parser");
-}
# This source code is licensed in accordance with the terms specified in
# the LICENSE file found in the root directory of this source tree.
-ADD_OSQUERY_LIBRARY(osquery_pluing_config config/parsers/decorators.cpp
- config/filesystem_config.cpp)
-
ADD_OSQUERY_LIBRARY(osquery_pluing_logger logger/filesystem_logger.cpp)
ADD_OSQUERY_LIBRARY(osquery_plugin_db database/ephemeral.cpp
+++ /dev/null
-/**
- * Copyright (c) 2014-present, Facebook, Inc.
- * All rights reserved.
- *
- * This source code is licensed as defined on the LICENSE file found in the
- * root directory of this source tree.
- */
-
-#include <vector>
-
-#include <boost/filesystem/operations.hpp>
-#include <boost/property_tree/json_parser.hpp>
-#include <boost/property_tree/ptree.hpp>
-
-#include <osquery/config/config.h>
-#include <osquery/filesystem/filesystem.h>
-#include <osquery/flags.h>
-#include <osquery/logger.h>
-#include <osquery/registry_factory.h>
-#include <osquery/utils/config/default_paths.h>
-
-#include <osquery/utils/json/json.h>
-
-namespace errc = boost::system::errc;
-namespace fs = boost::filesystem;
-namespace pt = boost::property_tree;
-
-namespace osquery {
-
-CLI_FLAG(string,
- config_path,
- (fs::path(OSQUERY_HOME) / "osquery.conf").make_preferred().string(),
- "Path to JSON config file");
-
-class FilesystemConfigPlugin : public ConfigPlugin {
- public:
- Status genConfig(std::map<std::string, std::string>& config);
- Status genPack(const std::string& name,
- const std::string& value,
- std::string& pack);
-};
-
-REGISTER(FilesystemConfigPlugin, "config", "filesystem");
-
-Status FilesystemConfigPlugin::genConfig(
- std::map<std::string, std::string>& config) {
- boost::system::error_code ec;
- if (!fs::is_regular_file(FLAGS_config_path, ec) ||
- ec.value() != errc::success) {
- return Status::failure("config file does not exist: " + FLAGS_config_path);
- }
-
- std::vector<std::string> conf_files;
- resolveFilePattern(FLAGS_config_path + ".d/%.conf", conf_files);
- std::sort(conf_files.begin(), conf_files.end());
- conf_files.push_back(FLAGS_config_path);
-
- for (const auto& path : conf_files) {
- std::string content;
- if (readFile(path, content).ok()) {
- config[path] = content;
- }
- }
-
- return Status(0, "OK");
-}
-
-Status FilesystemConfigPlugin::genPack(const std::string& name,
- const std::string& value,
- std::string& pack) {
- if (name == "*") {
- // The config requested a multi-pack.
- std::vector<std::string> paths;
- resolveFilePattern(value, paths);
-
- pt::ptree multi_pack;
- for (const auto& path : paths) {
- std::string content;
- if (!readFile(path, content)) {
- LOG(WARNING) << "Cannot read multi-pack file: " << path;
- continue;
- }
-
- // Assemble an intermediate property tree for simplified parsing.
- pt::ptree single_pack;
- stripConfigComments(content);
- try {
- std::stringstream json_stream;
- json_stream << content;
- pt::read_json(json_stream, single_pack);
- } catch (const pt::json_parser::json_parser_error& /* e */) {
- LOG(WARNING) << "Cannot read multi-pack JSON: " << path;
- continue;
- }
-
- multi_pack.put_child(fs::path(path).stem().string(), single_pack);
- }
-
- // We should have a property tree of pack content mimicking embedded
- // configuration packs, ready to parse as a string.
- std::ostringstream output;
- pt::write_json(output, multi_pack, false);
- pack = output.str();
- if (pack.empty()) {
- return Status(1, "Multi-pack content empty");
- }
-
- return Status(0);
- }
-
- boost::system::error_code ec;
- if (!fs::is_regular_file(value, ec) || ec.value() != errc::success) {
- return Status(1, value + " is not a valid path");
- }
-
- return readFile(value, pack);
-}
-} // namespace osquery
+++ /dev/null
-/**
- * Copyright (c) 2014-present, Facebook, Inc.
- * All rights reserved.
- *
- * This source code is licensed in accordance with the terms specified in
- * the LICENSE file found in the root directory of this source tree.
- */
-
-#include <osquery/plugins/config/parsers/decorators.h>
-#include <osquery/config/config.h>
-#include <osquery/flags.h>
-#include <osquery/logger.h>
-#include <osquery/registry_factory.h>
-#include <osquery/sql.h>
-#include <osquery/utils/json/json.h>
-
-namespace osquery {
-
-FLAG(bool, disable_decorators, false, "Disable log result decoration");
-
-FLAG(bool,
- decorations_top_level,
- false,
- "Add decorators as top level JSON objects");
-
-/// Statically define the parser name to avoid mistakes.
-const std::string kDecorationsName{"decorators"};
-
-const std::map<DecorationPoint, std::string> kDecorationPointKeys = {
- {DECORATE_LOAD, "load"},
- {DECORATE_ALWAYS, "always"},
- {DECORATE_INTERVAL, "interval"},
-};
-
-using KeyValueMap = std::map<std::string, std::string>;
-using DecorationStore = std::map<std::string, KeyValueMap>;
-
-namespace {
-
-/**
- * @brief A simple ConfigParserPlugin for a "decorators" dictionary key.
- *
- * Decorators append data to results, snapshots, and status log lines.
- * They can be used to add arbitrary additional datums within the 'decorators'
- * subkey.
- *
- * Decorators come in three basic flavors, defined by when they are run:
- * load: run these decorators when the config is loaded.
- * always: run these decorators for every query immediate before
- * interval: run these decorators on an interval.
- *
- * When 'interval' is used, the value is a dictionary of intervals, each of the
- * subkeys are treated as the requested interval in sections. The internals
- * are emulated by the query schedule.
- *
- * Decorators are sets of queries, and each selected column within the set is
- * added to the 'decorators' dictionary. Including two queries with the same
- * column name is undefined behavior and will most likely lead to either
- * duplicate keys or overwriting. Issuing a query that emits more than one row
- * will also lead to undefined behavior. The decorator executor will ignore any
- * rows past the first.
- */
-class DecoratorsConfigParserPlugin : public ConfigParserPlugin {
- public:
- std::vector<std::string> keys() const override {
- return {kDecorationsName};
- }
-
- Status setUp() override;
-
- Status update(const std::string& source, const ParserConfig& config) override;
-
- /// Update the set of decorators for a given source.
- void updateDecorations(const std::string& source, const JSON& doc);
-
- /// Clear the decorations created from decorators for the given source.
- void clearSources(const std::string& source);
-
- /// Clear all decorations.
- virtual void reset() override;
-
- public:
- /// Set of configuration sources to the set of decorator queries.
- std::map<std::string, std::vector<std::string>> always_;
-
- /// Set of configuration sources to the set of on-load decorator queries.
- std::map<std::string, std::vector<std::string>> load_;
-
- /// Set of configuration sources to valid intervals.
- std::map<std::string, std::map<size_t, std::vector<std::string>>> intervals_;
-
- public:
- /// The result set of decorations, column names and their values.
- static DecorationStore kDecorations;
-
- /// Protect additions to the decorator set.
- static Mutex kDecorationsMutex;
-
- /// Protect the configuration controlled content.
- static Mutex kDecorationsConfigMutex;
-};
-}
-
-DecorationStore DecoratorsConfigParserPlugin::kDecorations;
-Mutex DecoratorsConfigParserPlugin::kDecorationsMutex;
-Mutex DecoratorsConfigParserPlugin::kDecorationsConfigMutex;
-
-Status DecoratorsConfigParserPlugin::setUp() {
- // Decorators are kept within customized data structures.
- // No need to define a key for the ::getData API.
- return Status(0, "OK");
-}
-
-Status DecoratorsConfigParserPlugin::update(const std::string& source,
- const ParserConfig& config) {
- clearSources(source);
- clearDecorations(source);
- auto decorations = config.find(kDecorationsName);
- if (decorations != config.end()) {
- // Each of these methods acquires the decorator lock separately.
- // The run decorators method is designed to have call sites throughout
- // the code base.
- updateDecorations(source, decorations->second);
- runDecorators(DECORATE_LOAD, 0, source);
- }
-
- return Status(0, "OK");
-}
-
-void DecoratorsConfigParserPlugin::clearSources(const std::string& source) {
- // Reset the internal data store.
- WriteLock lock(DecoratorsConfigParserPlugin::kDecorationsConfigMutex);
- if (intervals_.count(source) > 0) {
- intervals_[source].clear();
- }
-
- if (always_.count(source) > 0) {
- always_[source].clear();
- }
-
- if (load_.count(source) > 0) {
- load_[source].clear();
- }
-}
-
-void DecoratorsConfigParserPlugin::reset() {
- // Reset the internal data store (for all sources).
- for (const auto& source : DecoratorsConfigParserPlugin::kDecorations) {
- clearSources(source.first);
- clearDecorations(source.first);
- }
-}
-
-void DecoratorsConfigParserPlugin::updateDecorations(const std::string& source,
- const JSON& doc) {
- WriteLock lock(DecoratorsConfigParserPlugin::kDecorationsConfigMutex);
- // Assign load decorators.
- auto& load_key = kDecorationPointKeys.at(DECORATE_LOAD);
- if (doc.doc().HasMember(load_key.c_str())) {
- auto& load = doc.doc()[load_key.c_str()];
- if (load.IsArray()) {
- for (const auto& item : load.GetArray()) {
- if (item.IsString()) {
- load_[source].push_back(item.GetString());
- }
- }
- }
- }
-
- // Assign always decorators.
- auto& always_key = kDecorationPointKeys.at(DECORATE_ALWAYS);
- if (doc.doc().HasMember(always_key.c_str())) {
- auto& always = doc.doc()[always_key.c_str()];
- if (always.IsArray()) {
- for (const auto& item : always.GetArray()) {
- if (item.IsString()) {
- always_[source].push_back(item.GetString());
- }
- }
- }
- }
-
- // Check if intervals are defined.
- auto& interval_key = kDecorationPointKeys.at(DECORATE_INTERVAL);
- if (doc.doc().HasMember(interval_key.c_str())) {
- const auto& interval = doc.doc()[interval_key.c_str()];
- if (interval.IsObject()) {
- for (const auto& item : interval.GetObject()) {
- auto rate = doc.valueToSize(item.name);
- // size_t rate = std::stoll(item.name.GetString());
- if (rate % 60 != 0) {
- LOG(WARNING) << "Invalid decorator interval rate " << rate
- << " in config source: " << source;
- continue;
- }
-
- // This is a valid interval, update the set of intervals to include
- // this value. When intervals are checked this set is scanned, if a
- // match is found, then the associated config data is executed.
- if (item.value.IsArray()) {
- for (const auto& interval_query : item.value.GetArray()) {
- if (interval_query.IsString()) {
- intervals_[source][rate].push_back(interval_query.GetString());
- }
- }
- }
- }
- }
- }
-}
-
-inline void addDecoration(const std::string& source,
- const std::string& name,
- const std::string& value) {
- WriteLock lock(DecoratorsConfigParserPlugin::kDecorationsMutex);
- DecoratorsConfigParserPlugin::kDecorations[source][name] = value;
-}
-
-inline void runDecorators(const std::string& source,
- const std::vector<std::string>& queries) {
- for (const auto& query : queries) {
- SQL results(query);
- if (results.rows().size() > 0) {
- // Notice the warning above about undefined behavior when:
- // 1: You include decorators that emit the same column name
- // 2: You include a query that returns more than 1 row.
- for (const auto& column : results.rows()[0]) {
- addDecoration(source, column.first, column.second);
- }
- }
-
- if (results.rows().size() > 1) {
- // Multiple rows exhibit undefined behavior.
- LOG(WARNING) << "Multiple rows returned for decorator query: " << query;
- }
- }
-}
-
-void clearDecorations(const std::string& source) {
- WriteLock lock(DecoratorsConfigParserPlugin::kDecorationsMutex);
- DecoratorsConfigParserPlugin::kDecorations[source].clear();
-}
-
-void runDecorators(DecorationPoint point,
- size_t time,
- const std::string& source) {
- if (FLAGS_disable_decorators) {
- return;
- }
-
- auto parser = Config::getParser(kDecorationsName);
- if (parser == nullptr) {
- // The decorators parser does not exist.
- return;
- }
-
- // Abstract the use of the decorator parser API.
- ReadLock lock(DecoratorsConfigParserPlugin::kDecorationsConfigMutex);
- auto dp = std::dynamic_pointer_cast<DecoratorsConfigParserPlugin>(parser);
- if (point == DECORATE_LOAD) {
- for (const auto& target_source : dp->load_) {
- if (source.empty() || target_source.first == source) {
- runDecorators(target_source.first, target_source.second);
- }
- }
- } else if (point == DECORATE_ALWAYS) {
- for (const auto& target_source : dp->always_) {
- if (source.empty() || target_source.first == source) {
- runDecorators(target_source.first, target_source.second);
- }
- }
- } else if (point == DECORATE_INTERVAL) {
- for (const auto& target_source : dp->intervals_) {
- for (const auto& interval : target_source.second) {
- if (time % interval.first == 0) {
- if (source.empty() || target_source.first == source) {
- runDecorators(target_source.first, interval.second);
- }
- }
- }
- }
- }
-}
-
-void getDecorations(std::map<std::string, std::string>& results) {
- if (FLAGS_disable_decorators) {
- return;
- }
-
- ReadLock lock(DecoratorsConfigParserPlugin::kDecorationsMutex);
- // Copy the decorations into the log_item.
- for (const auto& source : DecoratorsConfigParserPlugin::kDecorations) {
- for (const auto& decoration : source.second) {
- results[decoration.first] = decoration.second;
- }
- }
-}
-
-REGISTER_INTERNAL(DecoratorsConfigParserPlugin,
- "config_parser",
- kDecorationsName.c_str());
-}
+++ /dev/null
-/**
- * Copyright (c) 2014-present, Facebook, Inc.
- * All rights reserved.
- *
- * This source code is licensed in accordance with the terms specified in
- * the LICENSE file found in the root directory of this source tree.
- */
-
-#pragma once
-
-#include <map>
-#include <functional>
-
-#include <osquery/config/config.h>
-#include <osquery/database.h>
-
-namespace osquery {
-
-/// Enforce specific types of decoration.
-enum DecorationPoint {
- DECORATE_LOAD,
- DECORATE_ALWAYS,
- DECORATE_INTERVAL,
-};
-
-/// Define a map of decoration points to their expected configuration key.
-extern const std::map<DecorationPoint, std::string> kDecorationPointKeys;
-
-/**
- * @brief Iterate the discovered decorators for a given point type.
- *
- * The configuration maintains various sources, each may contain a set of
- * decorators. The source tracking is abstracted for the decorator iterator.
- *
- * @param point request execution of decorators for this given point.
- * @param time an optional time for points using intervals.
- * @param source restrict run to a specific config source.
- */
-void runDecorators(DecorationPoint point,
- size_t time = 0,
- const std::string& source = "");
-
-/**
- * @brief Access the internal storage of the Decorator parser.
- *
- * The decoration set is a map of column name to value. It contains the opaque
- * set of decoration point results.
- *
- * Decorations are applied to log items before they are sent to the downstream
- * logging APIs: logString, logSnapshot, etc.
- *
- * @param results the output parameter to write decorations.
- */
-void getDecorations(std::map<std::string, std::string>& results);
-
-/// Clear decorations for a source when it updates.
-void clearDecorations(const std::string& source);
-}
using chrono_clock = std::chrono::high_resolution_clock;
void initTesting() {
- Config::setStartTime(getUnixTime());
-
kToolType = ToolType::TEST;
if (osquery::isPlatform(PlatformType::TYPE_OSX)) {
kTestWorkingDirectory = "/private/tmp/osquery-tests";
#include <utility>
#include <vector>
-#include <osquery/config/config.h>
#include <osquery/core.h>
#include <osquery/database.h>
#include <osquery/events.h>
projects / platform / core / security / vist.git / commitdiff