From 54377e3bb0c3a99fcadf794d4c0027d9dba7daf9 Mon Sep 17 00:00:00 2001 From: Sangwan Kwon Date: Tue, 24 Feb 2015 21:29:57 -0700 Subject: [PATCH] Bump version to upstream-1.4.3 [experimental] Known issues - extension tests failed - Tizen build failed Added: os_version Signed-off-by: Sangwan Kwon --- CMakeLists.txt | 2 +- include/osquery/config.h | 25 +- include/osquery/core.h | 99 ++++-- include/osquery/database/db_handle.h | 2 +- include/osquery/events.h | 5 +- include/osquery/extensions.h | 43 +++ include/osquery/filesystem.h | 57 +++- include/osquery/flags.h | 49 ++- include/osquery/logger.h | 23 ++ include/osquery/registry.h | 210 ++++++++++-- include/osquery/sdk.h | 89 ++++- include/osquery/sql.h | 2 +- include/osquery/tables.h | 2 +- osquery.thrift | 13 + osquery/CMakeLists.txt | 17 +- osquery/config/config.cpp | 153 +++++---- osquery/config/config_tests.cpp | 26 +- osquery/config/plugins/filesystem.cpp | 39 ++- osquery/core/CMakeLists.txt | 1 + osquery/core/flags.cpp | 47 ++- osquery/core/hash_tests.cpp | 1 - osquery/core/init.cpp | 272 +++++++++++---- osquery/core/system.cpp | 13 +- osquery/core/tables_tests.cpp | 1 - osquery/core/test_util.cpp | 2 +- osquery/core/watcher.cpp | 371 +++++++++++++++------ osquery/core/watcher.h | 223 ++++++++++--- osquery/database/db_handle.cpp | 13 +- osquery/devtools/CMakeLists.txt | 3 +- osquery/distributed/distributed_tests.cpp | 1 - osquery/events/events.cpp | 17 +- osquery/events/events_tests.cpp | 2 +- osquery/events/linux/inotify.cpp | 34 +- osquery/events/linux/inotify.h | 3 + osquery/examples/example_extension.cpp | 15 +- osquery/examples/example_module.cpp | 44 +++ osquery/extensions/extensions.cpp | 232 ++++++++++--- osquery/extensions/extensions_tests.cpp | 24 +- osquery/extensions/interface.cpp | 21 +- osquery/extensions/interface.h | 19 +- osquery/filesystem/filesystem.cpp | 252 ++++++++++---- osquery/filesystem/filesystem_tests.cpp | 112 ++++++- osquery/logger/logger.cpp | 34 +- osquery/logger/logger_tests.cpp | 2 +- osquery/main/daemon.cpp | 27 +- osquery/main/shell.cpp | 26 +- osquery/registry/registry.cpp | 179 +++++++++- osquery/registry/registry_tests.cpp | 43 ++- osquery/scheduler/scheduler.cpp | 4 +- osquery/sql/sql_tests.cpp | 3 +- osquery/sql/sqlite_util.cpp | 4 +- osquery/sql/virtual_table_tests.cpp | 1 - osquery/tables/CMakeLists.txt | 3 + osquery/tables/events/linux/file_changes.cpp | 12 +- .../tables/specs/linux/process_memory_map.table | 2 +- osquery/tables/specs/{x => utility}/file.table | 0 osquery/tables/specs/{x => utility}/hash.table | 0 .../tables/specs/utility/osquery_extensions.table | 12 + .../specs/{x => utility}/osquery_flags.table | 0 .../tables/specs/{x => utility}/osquery_info.table | 0 .../tables/specs/utility/osquery_registry.table | 11 + osquery/tables/specs/{x => utility}/time.table | 0 osquery/tables/specs/x/os_version.table | 8 + osquery/tables/specs/x/osquery_extensions.table | 11 - osquery/tables/specs/x/system_controls.table | 11 + osquery/tables/system/linux/os_version.cpp | 61 ++++ osquery/tables/system/linux/processes.cpp | 2 +- osquery/tables/system/linux/sysctl_utils.cpp | 116 +++++++ osquery/tables/system/sysctl_utils.h | 41 +++ osquery/tables/system/system_controls.cpp | 126 +++++++ osquery/tables/utility/file.cpp | 10 +- osquery/tables/utility/osquery.cpp | 60 +++- osquery/tables/utility/time.cpp | 6 +- packaging/osquery.spec | 2 +- tools/sync.sh | 3 +- tools/tests/test.config.d/extra_options.conf | 6 + tools/tests/test.config.d/osquery.conf | 16 + 77 files changed, 2739 insertions(+), 682 deletions(-) create mode 100644 osquery/examples/example_module.cpp rename osquery/tables/specs/{x => utility}/file.table (100%) rename osquery/tables/specs/{x => utility}/hash.table (100%) create mode 100644 osquery/tables/specs/utility/osquery_extensions.table rename osquery/tables/specs/{x => utility}/osquery_flags.table (100%) rename osquery/tables/specs/{x => utility}/osquery_info.table (100%) create mode 100644 osquery/tables/specs/utility/osquery_registry.table rename osquery/tables/specs/{x => utility}/time.table (100%) create mode 100644 osquery/tables/specs/x/os_version.table delete mode 100644 osquery/tables/specs/x/osquery_extensions.table create mode 100644 osquery/tables/specs/x/system_controls.table create mode 100644 osquery/tables/system/linux/os_version.cpp create mode 100644 osquery/tables/system/linux/sysctl_utils.cpp create mode 100644 osquery/tables/system/sysctl_utils.h create mode 100644 osquery/tables/system/system_controls.cpp create mode 100644 tools/tests/test.config.d/extra_options.conf create mode 100644 tools/tests/test.config.d/osquery.conf diff --git a/CMakeLists.txt b/CMakeLists.txt index a0c37b3..8ee3629 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -35,7 +35,7 @@ ADD_DEFINITIONS("-fPIC") #ADD_DEFINITIONS("-pedantic-errors") # TODO(sangwan.kwon): Get version from packing spec. -SET(OSQUERY_BUILD_VERSION "1.4.2") +SET(OSQUERY_BUILD_VERSION "1.4.3") # Set various platform/platform-version/build version/etc defines. ADD_DEFINITIONS(-DOSQUERY_BUILD_VERSION=${OSQUERY_BUILD_VERSION} diff --git a/include/osquery/config.h b/include/osquery/config.h index 407533b..5daed00 100644 --- a/include/osquery/config.h +++ b/include/osquery/config.h @@ -14,11 +14,16 @@ #include #include +#include +#include + #include #include #include #include +namespace pt = boost::property_tree; + namespace osquery { /// The builder or invoker may change the default config plugin. @@ -35,6 +40,7 @@ struct OsqueryConfig { std::vector scheduledQueries; std::map options; std::map > eventFiles; + pt::ptree all_data; }; /** @@ -80,7 +86,7 @@ class Config { * * This may perform a resource load such as TCP request or filesystem read. */ - Status load(); + static Status load(); /** * @brief Get a vector of all scheduled queries. @@ -105,14 +111,23 @@ class Config { * * @return A map all the files in the JSON blob organized by category */ - static std::map >& getWatchedFiles(); + static std::map > getWatchedFiles(); + + /** + * @brief Return the configuration ptree + * + * + * + * @return Returns the unparsed, ptree representation of the given config + */ + static pt::ptree getEntireConfiguration(); /** * @brief Calculate the has of the osquery config * * @return The MD5 of the osquery config */ - Status getMD5(std::string& hashString); + static Status getMD5(std::string& hashString); /** * @brief Check to ensure that the config is accessible and properly @@ -172,7 +187,7 @@ class Config { * @return an instance of osquery::Status, indicating the success or failure * of the operation. */ - static osquery::Status genConfig(std::string& conf); + static osquery::Status genConfig(std::vector& conf); /// Prevent ConfigPlugins from implementing setUp. osquery::Status setUp() { return Status(0, "Not used"); } @@ -224,7 +239,7 @@ class ConfigPlugin : public Plugin { * indicates that config retrieval was successful, then the config data * should be returned in pair.second. */ - virtual std::pair genConfig() = 0; + virtual Status genConfig(std::map& config) = 0; Status call(const PluginRequest& request, PluginResponse& response); }; diff --git a/include/osquery/core.h b/include/osquery/core.h index f13eb6a..f8a8698 100644 --- a/include/osquery/core.h +++ b/include/osquery/core.h @@ -29,6 +29,9 @@ #endif // clang-format on +/// A configuration error is catastrophic and should exit the watcher. +#define EXIT_CATASTROPHIC 78 + namespace osquery { /** @@ -44,35 +47,75 @@ extern const std::string kSDKVersion; /** * @brief A helpful tool type to report when logging, print help, or debugging. */ -enum osqueryTool { +enum ToolType { OSQUERY_TOOL_SHELL, OSQUERY_TOOL_DAEMON, OSQUERY_TOOL_TEST, OSQUERY_EXTENSION, }; -/** - * @brief Sets up various aspects of osquery execution state. - * - * osquery needs a few things to happen as soon as the executable begins - * executing. initOsquery takes care of setting up the relevant parameters. - * initOsquery should be called in an executable's `main()` function. - * - * @param argc the number of elements in argv - * @param argv the command-line arguments passed to `main()` - */ -void initOsquery(int argc, char* argv[], int tool = OSQUERY_TOOL_TEST); - -/** - * @brief Sets up a process as a osquery daemon. - */ -void initOsqueryDaemon(); - -/** - * @brief Turns of various aspects of osquery such as event loops. - * - */ -void shutdownOsquery(); +class Initializer { + public: + /** + * @brief Sets up various aspects of osquery execution state. + * + * osquery needs a few things to happen as soon as the process begins + * executing. Initializer takes care of setting up the relevant parameters. + * Initializer should be called in an executable's `main()` function. + * + * @param argc the number of elements in argv + * @param argv the command-line arguments passed to `main()` + * @param tool the type of osquery main (daemon, shell, test, extension). + */ + Initializer(int argc, char* argv[], ToolType tool = OSQUERY_TOOL_TEST); + + /** + * @brief Sets up the process as an osquery daemon. + * + * A daemon has additional constraints, it can use a process mutext, check + * for sane/non-default configurations, etc. + */ + void initDaemon(); + + /** + * @brief Daemon tools may want to continually spawn worker processes + * and monitor their utilization. + * + * A daemon may call initWorkerWatcher to begin watching child daemon + * processes until it-itself is unscheduled. The basic guarentee is that only + * workers will return from the function. + * + * The worker-watcher will implement performance bounds on CPU utilization + * and memory, as well as check for zombie/defunct workers and respawn them + * if appropriate. The appropriateness is determined from heuristics around + * how the worker exitted. Various exit states and velocities may cause the + * watcher to resign. + * + * @param name The name of the worker process. + */ + void initWorkerWatcher(const std::string& name); + + /// Assume initialization finished, start work. + void start(); + /// Turns off various aspects of osquery such as event loops. + void shutdown(); + /// Check if a process is an osquery worker. + bool isWorker(); + + private: + /// Initialize this process as an osquery daemon worker. + void initWorker(const std::string& name); + /// Initialize the osquery watcher, optionally spawn a worker. + void initWatcher(); + /// Set the config and logger plugins, optionally depend on an extension. + void initConfigLogger(); + + private: + int argc_; + char** argv_; + int tool_; + std::string binary_; +}; /** * @brief Split a given string based on an optional delimiter. @@ -121,14 +164,17 @@ int getUnixTime(); */ template inline size_t incUtf8StringIterator(_Iterator1& it, const _Iterator2& last) { - if (it == last) + if (it == last) { return 0; + } + unsigned char c; size_t res = 1; for (++it; last != it; ++it, ++res) { c = *it; - if (!(c & 0x80) || ((c & 0xC0) == 0xC0)) + if (!(c & 0x80) || ((c & 0xC0) == 0xC0)) { break; + } } return res; @@ -144,8 +190,9 @@ inline size_t incUtf8StringIterator(_Iterator1& it, const _Iterator2& last) { inline size_t utf8StringSize(const std::string& str) { size_t res = 0; std::string::const_iterator it = str.begin(); - for (; it != str.end(); incUtf8StringIterator(it, str.end())) + for (; it != str.end(); incUtf8StringIterator(it, str.end())) { res++; + } return res; } diff --git a/include/osquery/database/db_handle.h b/include/osquery/database/db_handle.h index 3d77118..a11009c 100644 --- a/include/osquery/database/db_handle.h +++ b/include/osquery/database/db_handle.h @@ -21,7 +21,7 @@ namespace osquery { -DECLARE_string(db_path); +DECLARE_string(database_path); ///////////////////////////////////////////////////////////////////////////// // Constants diff --git a/include/osquery/events.h b/include/osquery/events.h index 9c07e48..e61389a 100644 --- a/include/osquery/events.h +++ b/include/osquery/events.h @@ -520,6 +520,9 @@ class EventSubscriberPlugin : public Plugin { void operator=(EventSubscriberPlugin const&); private: + Status setUp() { return Status(0, "Setup never used"); } + + private: /// Do not respond to periodic/scheduled/triggered event expiration requests. bool expire_events_; @@ -770,6 +773,6 @@ void attachEvents(); /// Sleep in a boost::thread interruptable state. void interruptableSleep(size_t milli); -CREATE_LAZY_REGISTRY(EventPublisherPlugin, "event_publisher"); +CREATE_REGISTRY(EventPublisherPlugin, "event_publisher"); CREATE_REGISTRY(EventSubscriberPlugin, "event_subscriber"); } diff --git a/include/osquery/extensions.h b/include/osquery/extensions.h index 9e91b46..07ce6e6 100644 --- a/include/osquery/extensions.h +++ b/include/osquery/extensions.h @@ -18,6 +18,11 @@ namespace osquery { DECLARE_int32(worker_threads); DECLARE_string(extensions_socket); +DECLARE_string(extensions_autoload); +DECLARE_string(extensions_timeout); + +/// A millisecond internal applied to extension initialization. +extern const int kExtensionInitializeMLatency; /** * @brief Helper struct for managing extenion metadata. @@ -73,6 +78,44 @@ Status getExtensions(const std::string& manager_path, Status pingExtension(const std::string& path); /** + * @brief Request the extensions API to autoload any appropriate extensions. + * + * Extensions may be 'autoloaded' using the `extensions_autoload` command line + * argument. loadExtensions should be called before any plugin or registry item + * is used. This allows appropriate extensions to expose plugin requirements. + * + * An 'appropriate' extension is one within the `extensions_autoload` search + * path with file ownership equivilent or greater (root) than the osquery + * process requesting autoload. + */ +void loadExtensions(); + +/** + * @brief Load extensions from a delimited search path string. + * + * @param paths A colon-delimited path variable, e.g: '/path1:/path2'. + */ +Status loadExtensions(const std::string& loadfile); + +/** + * @brief Request the extensions API to autoload any appropriate modules. + * + * Extension modules are shared libraries that add Plugins to the osquery + * core's registry at runtime. + */ +void loadModules(); + +/** + * @brief Load extenion modules from a delimited search path string. + * + * @param paths A colon-delimited path variable, e.g: '/path1:/path2'. + */ +Status loadModules(const std::string& loadfile); + +/// Load all modules in a direcotry. +Status loadModuleFile(const std::string& path); + +/** * @brief Call a Plugin exposed by an Extension Registry route. * * This is mostly a Registry%-internal method used to call an ExtensionHandler diff --git a/include/osquery/filesystem.h b/include/osquery/filesystem.h index 5a3de6f..90d7ca8 100644 --- a/include/osquery/filesystem.h +++ b/include/osquery/filesystem.h @@ -11,6 +11,7 @@ #pragma once #include +#include #include #include @@ -26,6 +27,13 @@ namespace osquery { * this many wildcards. */ const unsigned int kMaxDirectoryTraversalDepth = 40; +typedef unsigned int ReturnSetting; +enum { + REC_LIST_FILES = 0x1, // Return only files + REC_LIST_FOLDERS = 0x2, // Return only folders + REC_EVENT_OPT = 0x4, // Enable optimizations for file event resolutions + REC_LIST_ALL = REC_LIST_FILES | REC_LIST_FOLDERS +}; const std::string kWildcardCharacter = "%"; const std::string kWildcardCharacterRecursive = @@ -87,7 +95,8 @@ Status pathExists(const boost::filesystem::path& path); * of the operation. */ Status listFilesInDirectory(const boost::filesystem::path& path, - std::vector& results); + std::vector& results, + bool ignore_error = 1); /** * @brief List all of the directories in a specific directory, non-recursively. @@ -101,7 +110,8 @@ Status listFilesInDirectory(const boost::filesystem::path& path, * of the operation. */ Status listDirectoriesInDirectory(const boost::filesystem::path& path, - std::vector& results); + std::vector& results, + bool ignore_error = 1); /** * @brief Given a wildcard filesystem patten, resolve all possible paths @@ -126,6 +136,30 @@ Status resolveFilePattern(const boost::filesystem::path& fs_path, std::vector& results); /** + * @brief Given a wildcard filesystem patten, resolve all possible paths + * + * @code{.cpp} + * std::vector results; + * auto s = resolveFilePattern("/Users/marpaia/Downloads/%", results); + * if (s.ok()) { + * for (const auto& result : results) { + * LOG(INFO) << result; + * } + * } + * @endcode + * + * @param fs_path The filesystem pattern + * @param results The vector in which all results will be returned + * @param setting Do you want files returned, folders or both? + * + * @return An instance of osquery::Status which indicates the success or + * failure of the operation + */ +Status resolveFilePattern(const boost::filesystem::path& fs_path, + std::vector& results, + ReturnSetting setting); + +/** * @brief Get directory portion of a path. * * @param path The input path, either a filename or directory. @@ -153,7 +187,24 @@ Status isDirectory(const boost::filesystem::path& path); * * @return a vector of strings representing the path of all home directories */ -std::vector getHomeDirectories(); +std::set getHomeDirectories(); + +/** + * @brief Check the permissions of a file and it's directory. + * + * 'Safe' implies the directory is not a /tmp-like directory in that users + * cannot control super-user-owner files. The file should be owned by the + * process's UID or the file should be owned by root. + * + * @param dir the directory to check /tmp mode + * @param path a path to a file to check + * @param executable the file must also be executable + * + * @return true if the file is 'safe' else false + */ +bool safePermissions(const std::string& dir, + const std::string& path, + bool executable = false); /// Return bit-mask-style permissions. std::string lsperms(int mode); diff --git a/include/osquery/flags.h b/include/osquery/flags.h index 0a94979..7233a49 100644 --- a/include/osquery/flags.h +++ b/include/osquery/flags.h @@ -37,6 +37,7 @@ struct FlagDetail { std::string description; bool shell; bool external; + bool cli; }; struct FlagInfo { @@ -66,7 +67,7 @@ class Flag { static int create(const std::string& name, const FlagDetail& flag); /// Create a Gflags alias to name, using the Flag::getValue accessor. - static int createAlias(const std::string& alias, const std::string& name); + static int createAlias(const std::string& alias, const FlagDetail& flag); static Flag& instance() { static Flag f; @@ -126,16 +127,25 @@ class Flag { static std::string getType(const std::string& name); /* + * @brief Get the description as a string of an osquery flag. + * + * @param name the flag name. + */ + static std::string getDescription(const std::string& name); + + /* * @brief Print help-style output to stdout for a given flag set. * * @param shell Only print shell flags. * @param external Only print external flags (from extensions). */ - static void printFlags(bool shell = false, bool external = false); + static void printFlags(bool shell = false, + bool external = false, + bool cli = false); private: std::map flags_; - std::map aliases_; + std::map aliases_; }; /** @@ -175,20 +185,25 @@ class FlagAlias { * @param value The default value, use a C++ literal. * @param desc A string literal used for help display. */ -#define OSQUERY_FLAG(t, n, v, d, s, e) \ - DEFINE_##t(n, v, d); \ - namespace flags { \ - const int flag_##n = Flag::create(#n, {d, s, e}); \ +#define OSQUERY_FLAG(t, n, v, d, s, e, c) \ + DEFINE_##t(n, v, d); \ + namespace flags { \ + const int flag_##n = Flag::create(#n, {d, s, e, c}); \ } -#define FLAG(t, n, v, d) OSQUERY_FLAG(t, n, v, d, false, false) -#define SHELL_FLAG(t, n, v, d) OSQUERY_FLAG(t, n, v, d, true, false) -#define EXTENSION_FLAG(t, n, v, d) OSQUERY_FLAG(t, n, v, d, false, true) - -#define FLAG_ALIAS(t, a, n) \ - FlagAlias FLAGS_##a(#a, #t, #n, &FLAGS_##n); \ - namespace flags { \ - static GFLAGS_NAMESPACE::FlagRegisterer oflag_##a( \ - #a, #t, #a, #a, &FLAGS_##n, &FLAGS_##n); \ - const int flag_alias_##a = Flag::createAlias(#a, #n); \ +#define FLAG(t, n, v, d) OSQUERY_FLAG(t, n, v, d, 0, 0, 0) +#define SHELL_FLAG(t, n, v, d) OSQUERY_FLAG(t, n, v, d, 1, 0, 0) +#define EXTENSION_FLAG(t, n, v, d) OSQUERY_FLAG(t, n, v, d, 0, 1, 0) +#define CLI_FLAG(t, n, v, d) OSQUERY_FLAG(t, n, v, d, 0, 0, 1) + +#define OSQUERY_FLAG_ALIAS(t, a, n, s, e) \ + FlagAlias FLAGS_##a(#a, #t, #n, &FLAGS_##n); \ + namespace flags { \ + static GFLAGS_NAMESPACE::FlagRegisterer oflag_##a( \ + #a, #t, #a, #a, &FLAGS_##n, &FLAGS_##n); \ + const int flag_alias_##a = Flag::createAlias(#a, {#n, s, e, 0}); \ } + +#define FLAG_ALIAS(t, a, n) OSQUERY_FLAG_ALIAS(t, a, n, 0, 0) +#define SHELL_FLAG_ALIAS(t, a, n) _OSQUERY_FLAG_ALIAS(t, a, n, 1, 0) +#define EXTENSION_FLAG_ALIAS(a, n) OSQUERY_FLAG_ALIAS(std::string, a, n, 0, 1) diff --git a/include/osquery/logger.h b/include/osquery/logger.h index a2154c6..6a6c202 100644 --- a/include/osquery/logger.h +++ b/include/osquery/logger.h @@ -15,12 +15,16 @@ #include +#include #include #include #include namespace osquery { +DECLARE_bool(disable_logging); +DECLARE_string(logger_plugin); + /** * @breif An internal severity set mapping to Glog's LogSeverity levels. */ @@ -45,6 +49,25 @@ struct StatusLogLine { }; /** + * @brief Helper logging macro for table-generated verbose log lines. + * + * Since logging in tables does not always mean a critical warning or error + * but more likely a parsing or expected edge-case, we provide a TLOG. + * + * The tool user can set within config or via the CLI what level of logging + * to tolerate. It's the table developer's job to assume consistency in logging. + */ +#define TLOG VLOG(1) + +/** + * @brief Prepend a reference number to the log line. + * + * A reference number is an external-search helper for somewhat confusing or + * seeminly-critical log lines. + */ +#define RLOG(n) "[Ref #" #n "] " + +/** * @brief Superclass for the pluggable logging facilities. * * In order to make the logging of osquery results and inline debug, warning, diff --git a/include/osquery/registry.h b/include/osquery/registry.h index 728dcfa..3802b4d 100644 --- a/include/osquery/registry.h +++ b/include/osquery/registry.h @@ -18,7 +18,7 @@ #include #include -#include +#include namespace osquery { @@ -41,22 +41,26 @@ namespace osquery { * @param type A typename that derives from Plugin. * @param name A string identifier for the registry. */ -#define CREATE_REGISTRY(type, name) \ - namespace registry { \ - const auto type##Registry = Registry::create(name); \ +#define CREATE_REGISTRY(type, name) \ + namespace registry { \ + __attribute__((constructor)) static void type##Registry() { \ + Registry::create(name); \ + } \ } /** * @brief A boilerplate code helper to create a registry given a name and - * plugin base class type. This 'lazy' registry does not automatically run - * Plugin::setUp on all items. + * plugin base class type. This 'lazy' registry does not run + * Plugin::setUp on its items, so the registry will do it. * * @param type A typename that derives from Plugin. * @param name A string identifier for the registry. */ -#define CREATE_LAZY_REGISTRY(type, name) \ - namespace registry { \ - const auto type##Registry = Registry::create(name, false); \ +#define CREATE_LAZY_REGISTRY(type, name) \ + namespace registry { \ + __attribute__((constructor)) static void type##Registry() { \ + Registry::create(name, true); \ + } \ } /** @@ -71,12 +75,16 @@ namespace osquery { * @param registry The string name for the registry. * @param name A string identifier for this registry item. */ -#define REGISTER(type, registry, name) \ - const auto type##RegistryItem = Registry::add(registry, name); +#define REGISTER(type, registry, name) \ + __attribute__((constructor)) static void type##RegistryItem() { \ + Registry::add(registry, name); \ + } /// The same as REGISTER but prevents the plugin item from being broadcasted. -#define REGISTER_INTERNAL(type, registry, name) \ - const auto type##RegistryItem = Registry::add(registry, name, true); +#define REGISTER_INTERNAL(type, registry, name) \ + __attribute__((constructor)) static void type##RegistryItem() { \ + Registry::add(registry, name, true); \ + } /** * @brief The request part of a plugin (registry item's) call. @@ -106,6 +114,18 @@ typedef std::function AddExternalCallback; typedef std::function RemoveExternalCallback; +/// When a module is being initialized its information is kept in a transient +/// RegistryFactory lookup location. +struct ModuleInfo { + std::string path; + std::string name; + std::string version; + std::string sdk_version; +}; + +/// The call-in prototype for Registry modules. +typedef void (*ModuleInitalizer)(void); + class Plugin { public: Plugin() { name_ = "unnamed"; } @@ -162,7 +182,7 @@ class Plugin { class RegistryHelperCore { public: - explicit RegistryHelperCore(bool auto_setup = true) + explicit RegistryHelperCore(bool auto_setup = false) : auto_setup_(auto_setup) {} virtual ~RegistryHelperCore() {} @@ -195,6 +215,8 @@ class RegistryHelperCore { const PluginRequest& request, PluginResponse& response); + Status add(const std::string& item_name, bool internal = false); + /** * @brief Allow a plugin to perform some setup functions when osquery starts. * @@ -226,6 +248,23 @@ class RegistryHelperCore { /// Allow the registry to introspect into the registered name (for logging). void setName(const std::string& name); + /// Allow others to introspect into the registered name (for reporting). + const std::string& getName() const { return name_; } + + /// Check if a given plugin name is considered internal. + bool isInternal(const std::string& item_name) const; + + /// Allow others to introspect into the routes from extensions. + const std::map& getExternal() const { + return external_; + } + + /// Set an 'active' plugin to receive registry calls when no item name given. + Status setActive(const std::string& item_name); + + /// Get the 'active' plugin, return success with the active plugin name. + const std::string& getActive() const; + protected: /// The identifier for this registry, used to register items. std::string name_; @@ -244,6 +283,11 @@ class RegistryHelperCore { std::map routes_; /// Keep a lookup of registry items that are blacklisted from broadcast. std::vector internal_; + /// Support an 'active' mode where calls without a specific item name will + /// be directed to the 'active' plugin. + std::string active_; + /// If a module was initialized/declared then store lookup information. + std::map modules_; }; /** @@ -259,7 +303,7 @@ class RegistryHelper : public RegistryHelperCore { typedef std::shared_ptr RegistryTypeRef; public: - explicit RegistryHelper(bool auto_setup = true) + explicit RegistryHelper(bool auto_setup = false) : RegistryHelperCore(auto_setup), add_(&RegistryType::addExternal), remove_(&RegistryType::removeExternal) {} @@ -331,13 +375,7 @@ class RegistryHelper : public RegistryHelperCore { std::shared_ptr item((RegistryType*)new Item()); item->setName(item_name); items_[item_name] = item; - - // The item can be listed as internal, meaning it does not broadcast. - if (internal) { - internal_.push_back(item_name); - } - - return Status(0, "OK"); + return RegistryHelperCore::add(item_name, internal); } /** @@ -369,10 +407,47 @@ class RegistryHelper : public RegistryHelperCore { RemoveExternalCallback remove_; }; +/// Helper defintion for a shared pointer to a Plugin. typedef std::shared_ptr PluginRef; +/// Helper definition for a basic-templated Registry type using a base Plugin. typedef RegistryHelper PluginRegistryHelper; +/// Helper definitions for a shared pointer to the basic Registry type. typedef std::shared_ptr PluginRegistryHelperRef; +/** + * @basic A workflow manager for opening a module path and appending to the + * core registry. + * + * osquery Registry modules are part of the extensions API, in that they use + * the osquery SDK to expose additional features to the osquery core. Modules + * do not require the Thrift interface and may be compiled as shared objects + * and loaded late at run time once the core and internal registry has been + * initialized and setUp. + * + * A ModuleLoader interprets search paths, dynamically loads the modules, + * maintains identification within the RegistryFactory and any registries + * the module adds items into. + */ +class RegistryModuleLoader : private boost::noncopyable { + public: + /// Unlock the registry, open, construct, and allow the module to declare. + RegistryModuleLoader(const std::string& path); + /// Keep the symbol resolution/calling out of construction. + void init(); + + /// Clear module information, 'lock' the registry. + ~RegistryModuleLoader(); + + private: + // Keep the handle for symbol resolution/calling. + void* handle_; + // Keep the path for debugging/logging. + std::string path_; + + private: + FRIEND_TEST(RegistryTests, test_registry_modules); +}; + class RegistryFactory : private boost::noncopyable { public: static RegistryFactory& instance() { @@ -394,12 +469,12 @@ class RegistryFactory : private boost::noncopyable { * @endcode * * @param registry_name The canonical name for this registry. - * @param auto_setup Optionally set false if the registry handles setup + * @param auto_setup Set true if the registry does not setup itself * @return A non-sense int that must be casted const. */ template - static int create(const std::string& registry_name, bool auto_setup = true) { - if (instance().registries_.count(registry_name) > 0) { + static int create(const std::string& registry_name, bool auto_setup = false) { + if (locked() || instance().registries_.count(registry_name) > 0) { return 0; } @@ -432,8 +507,11 @@ class RegistryFactory : private boost::noncopyable { static Status add(const std::string& registry_name, const std::string& item_name, bool internal = false) { - auto registry = instance().registry(registry_name); - return registry->template add(item_name, internal); + if (!locked()) { + auto registry = instance().registry(registry_name); + return registry->template add(item_name, internal); + } + return Status(0, "Registry locked"); } /// Direct access to all registries. @@ -497,6 +575,22 @@ class RegistryFactory : private boost::noncopyable { const std::string& item_name, const PluginRequest& request); + /// A helper call that uses the active plugin (if the registry has one). + static Status call(const std::string& registry_name, + const PluginRequest& request, + PluginResponse& response); + + /// A helper call that uses the active plugin (if the registry has one). + static Status call(const std::string& registry_name, + const PluginRequest& request); + + /// Set a registry's active plugin. + static Status setActive(const std::string& registry_name, + const std::string& item_name); + + /// Get a registry's active plugin. + static const std::string& getActive(const std::string& registry_nane); + /// Run `setUp` on every registry that is not marked 'lazy'. static void setUp(); @@ -505,6 +599,9 @@ class RegistryFactory : private boost::noncopyable { const std::string& item_name, bool local = false); + /// Get a list of the registry names. + static std::vector names(); + /// Get a list of the registry item names for a given registry. static std::vector names(const std::string& registry_name); @@ -525,16 +622,73 @@ class RegistryFactory : private boost::noncopyable { /// Check if duplicate registry items using registry aliasing are allowed. static bool allowDuplicates() { return instance().allow_duplicates_; } + /// Declare a module for initialization and subsequent registration attempts + static void declareModule(const std::string& name, + const std::string& version, + const std::string& min_sdk_version, + const std::string& sdk_version); + + /// Access module metadata. + static const std::map& getModules(); + + private: + /// Access the current initializing module UUID. + static RouteUUID getModule(); + + /// Check if the registry is allowing module registrations. + static bool usingModule(); + + /// Initialize a module for lookup, resolution, and its registrations. + static void initModule(const std::string& path); + + static void shutdownModule(); + + /// Check if the registries are locked. + static bool locked() { return instance().locked_; } + + /// Set the registry locked status. + static void locked(bool locked) { instance().locked_ = locked; } + protected: - RegistryFactory() : allow_duplicates_(false) {} + RegistryFactory() : allow_duplicates_(false), locked_(false) {} RegistryFactory(RegistryFactory const&); void operator=(RegistryFactory const&); virtual ~RegistryFactory() {} private: + /// Track duplicate registry item support, used for testing. bool allow_duplicates_; + /// Track registry "locking", while locked a registry cannot add/create. + bool locked_; + + /// The primary storage for constructed registries. std::map registries_; + /** + * @brief The registry tracks the set of active extension routes. + * + * If an extension dies (the process ends or does not respond to a ping), + * the registry will be notified via the extension watcher. + * When an operation requests to use that extension route the extension + * manager will lazily check the registry for changes. + */ std::set extensions_; + + /** + * @brief The registry tracks loaded extension module metadata/info. + * + * Each extension module is assigned a transient RouteUUID for identification + * those route IDs are passed to each registry to identify which plugin + * items belong to modules, similarly to extensions. + */ + std::map modules_; + + // During module initialization store the current-working module ID. + RouteUUID module_uuid_; + + private: + friend class RegistryHelperCore; + friend class RegistryModuleLoader; + FRIEND_TEST(RegistryTests, test_registry_modules); }; /** diff --git a/include/osquery/sdk.h b/include/osquery/sdk.h index b985e83..0868299 100644 --- a/include/osquery/sdk.h +++ b/include/osquery/sdk.h @@ -29,7 +29,92 @@ #include namespace osquery { -/// Anything built with only libosquery.a (SDK) will not include an SQL -/// provider (aka "sql" registry). +/** + * @brief Create the external SQLite implementation wrapper. + * + * Anything built with only libosquery and not the 'additional' library will + * not include a native SQL implementation. This applies to extensions and + * separate applications built with the osquery SDK. + * + * The ExternalSQLPlugin is a wrapper around the SQLite API, which forwards + * calls to an osquery extension manager (core). + */ REGISTER_INTERNAL(ExternalSQLPlugin, "sql", "sql"); + +/** + * @brief Mimic the REGISTER macro, extensions should use this helper. + * + * The SDK does not provide a REGISTER macro for modules or extensions. + * Tools built with the osquery SDK should use REGISTER_EXTERNAL to add to + * their own 'external' registry. This registry will broadcast to the osquery + * extension manager (core) in an extension. + * + * osquery 'modules' should not construct their plugin registrations in + * global scope (global construction time). Instead they should use the + * module call-in well defined symbol, declare their SDK constraints, then + * use the REGISTER_MODULE call within `initModule`. + */ +#define REGISTER_EXTERNAL(type, registry, name) \ + __attribute__((constructor)) static void type##ExtensionRegistryItem() { \ + Registry::add(registry, name); \ + } + +/// Helper macro to write the `initModule` symbol without rewrites. +#define DECLARE_MODULE(name) \ + extern "C" void initModule(void); \ + __attribute__((constructor)) static void declareModule() + +/** + * @brief Create an osquery extension 'module'. + * + * This helper macro creates a constructor to declare an osquery module is + * loading. The osquery registry is set up when modules (shared objects) are + * discovered via search paths and opened. At that phase the registry is locked + * meaning no additional plugins can be registered. To unlock the registry + * for modifications a module must call Registry::declareModule. This declares + * and any plugins added will use the metadata in the declare to determine: + * - The name of the module adding the plugin + * - The SDK version the module was built with, to determine compatibility + * - The minimum SDK the module requires from osquery core + * + * The registry is again locked when the module load is complete and a well + * known module-exported symbol is called. + */ +#define CREATE_MODULE(name, version, min_sdk_version) \ + DECLARE_MODULE(name) { \ + Registry::declareModule( \ + name, version, min_sdk_version, OSQUERY_SDK_VERSION); \ + } + +/** + * @brief Create an osquery extension 'module', if an expression is true. + * + * This is a helper testing wrapper around CREATE_MODULE and DECLARE_MODULE. + * It allows unit and integration tests to generate global construction code + * that depends on data/variables available during global construction. + * + * And example use includes checking if a process environment variable is + * defined. If defined the module is declared. + */ +#define CREATE_MODULE_IF(expr, name, version, min_sdk_version) \ + DECLARE_MODULE(name) { \ + if ((expr)) { \ + Registry::declareModule( \ + name, version, min_sdk_version, OSQUERY_SDK_VERSION); \ + } \ + } + +/// Helper replacement for REGISTER, used within extension modules. +#define REGISTER_MODULE(type, registry, name) \ + auto type##ModuleRegistryItem = Registry::add(registry, name) + +// Remove registry-helper macros from the SDK. +#undef REGISTER +#define REGISTER "Do not REGISTER in the osquery SDK" +#undef REGISTER_INTERNAL +#define REGISTER_INTERNAL "Do not REGISTER_INTERNAL in the osquery SDK" +#undef CREATE_REGISTRY +#define CREATE_REGISTRY "Do not CREATE_REGISTRY in the osquery SDK" +#undef CREATE_LAZY_REGISTRY +#define CREATE_LAZY_REGISTRY "Do not CREATE_LAZY_REGISTRY in the osquery SDK" } diff --git a/include/osquery/sql.h b/include/osquery/sql.h index 9cea7f9..9d927cb 100644 --- a/include/osquery/sql.h +++ b/include/osquery/sql.h @@ -226,5 +226,5 @@ class MockSQL : public SQL { } }; -CREATE_REGISTRY(SQLPlugin, "sql"); +CREATE_LAZY_REGISTRY(SQLPlugin, "sql"); } diff --git a/include/osquery/tables.h b/include/osquery/tables.h index dd43084..6b814e4 100644 --- a/include/osquery/tables.h +++ b/include/osquery/tables.h @@ -315,6 +315,6 @@ class TablePlugin : public Plugin { std::string columnDefinition(const TableColumns& columns); std::string columnDefinition(const PluginResponse& response); -CREATE_REGISTRY(TablePlugin, "table"); +CREATE_LAZY_REGISTRY(TablePlugin, "table"); } } diff --git a/osquery.thrift b/osquery.thrift index 2fdf3d6..59f9765 100644 --- a/osquery.thrift +++ b/osquery.thrift @@ -4,6 +4,17 @@ namespace cpp osquery.extensions typedef map ExtensionPluginRequest typedef list> ExtensionPluginResponse +/// Extensions should request osquery options to set active registries and +/// bootstrap any config/logger plugins. +struct InternalOptionInfo { + 1:string value, + 2:string default_value, + 3:string type, +} + +/// Each option (CLI flag) has a unique name. +typedef map InternalOptionList + /// When communicating extension metadata, use a thrift-internal structure. struct InternalExtensionInfo { 1:string name, @@ -63,6 +74,8 @@ service Extension { service ExtensionManager extends Extension { /// Return the list of active registered extensions. InternalExtensionList extensions(), + /// Return the list of bootstrap or configuration options. + InternalOptionList options(), /// The API endpoint used by an extension to register its plugins. ExtensionStatus registerExtension( 1:InternalExtensionInfo info, diff --git a/osquery/CMakeLists.txt b/osquery/CMakeLists.txt index 70dabd5..c0bc610 100644 --- a/osquery/CMakeLists.txt +++ b/osquery/CMakeLists.txt @@ -66,6 +66,7 @@ MACRO(ADD_OSQUERY_TEST IS_CORE TEST_NAME SOURCE) TARGET_LINK_LIBRARIES(${TEST_NAME} ${TARGET_OSQUERY_LIB}) TARGET_LINK_LIBRARIES(${TEST_NAME} gtest) +# SET_TARGET_PROPERTIES(${TARGET} PROPERTIES COMPILE_FLAGS "-DGTEST_HAS_TR1_TUPLE=0") ADD_TEST(${TEST_NAME} ${TEST_NAME}) INSTALL(TARGETS ${TEST_NAME} DESTINATION ${CMAKE_INSTALL_BINDIR} @@ -84,6 +85,14 @@ MACRO(TARGET_OSQUERY_LINK_WHOLE TARGET LIBRARY) TARGET_LINK_LIBRARIES(${TARGET} "-Wl,-no-whole-archive") ENDMACRO(TARGET_OSQUERY_LINK_WHOLE) +MACRO(ADD_OSQUERY_MODULE TARGET) + ADD_LIBRARY(${TARGET} SHARED ${ARGN}) + TARGET_LINK_LIBRARIES(${TARGET} dl) + ADD_DEPENDENCIES(${TARGET} ${TARGET_OSQUERY_LIB} glog) + SET_TARGET_PROPERTIES(${TARGET} PROPERTIES COMPILE_FLAGS "-fPIC") + SET_TARGET_PROPERTIES(${TARGET} PROPERTIES OUTPUT_NAME ${TARGET}) +ENDMACRO(ADD_OSQUERY_MODULE) + ADD_SUBDIRECTORY(core) ADD_SUBDIRECTORY(config) ADD_SUBDIRECTORY(dispatcher) @@ -162,7 +171,7 @@ TARGET_LINK_LIBRARIES(${TARGET_OSQUERY_LIB} ${${TARGET_OSQUERY_LIB}_DEP}) # DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}) ## osqueryi generation ########################################################## -ADD_EXECUTABLE(${TARGET_OSQUERY_SHELL} main/shell.cpp) +ADD_EXECUTABLE(${TARGET_OSQUERY_SHELL} devtools/shell.cpp main/shell.cpp) TARGET_LINK_LIBRARIES(${TARGET_OSQUERY_SHELL} ${TARGET_OSQUERY_LIB}) INSTALL(TARGETS ${TARGET_OSQUERY_SHELL} DESTINATION ${CMAKE_INSTALL_BINDIR} @@ -190,4 +199,8 @@ INSTALL(TARGETS ${TARGET_OSQUERY_DAEMON} ## example extension with the SDK ############################################## ADD_EXECUTABLE(example_extension examples/example_extension.cpp) TARGET_LINK_LIBRARIES(example_extension ${TARGET_OSQUERY_LIB}) -SET_TARGET_PROPERTIES(example_extension PROPERTIES OUTPUT_NAME example_extension) +SET_TARGET_PROPERTIES(example_extension PROPERTIES OUTPUT_NAME example_extension.ext) + +# Build the example extension module with the SDK +ADD_OSQUERY_MODULE(modexample examples/example_module.cpp) + diff --git a/osquery/config/config.cpp b/osquery/config/config.cpp index 5a2c3f8..eabc398 100644 --- a/osquery/config/config.cpp +++ b/osquery/config/config.cpp @@ -11,8 +11,6 @@ #include #include -#include -#include #include #include @@ -21,14 +19,14 @@ #include #include -#include "osquery/core/watcher.h" - namespace pt = boost::property_tree; + +typedef pt::ptree::value_type tree_node; typedef std::map > EventFileMap_t; namespace osquery { -FLAG(string, config_plugin, "filesystem", "Config type (plugin)"); +CLI_FLAG(string, config_plugin, "filesystem", "Config plugin name"); // This lock is used to protect the entirety of the OSqueryConfig struct // Is should be used when ever accessing the structs members, reading or @@ -36,11 +34,15 @@ FLAG(string, config_plugin, "filesystem", "Config type (plugin)"); static boost::shared_mutex rw_lock; Status Config::load() { + auto& config_plugin = Registry::getActive("config"); + if (!Registry::exists("config", config_plugin)) { + return Status(1, "Missing config plugin " + config_plugin); + } + boost::unique_lock lock(rw_lock); - OsqueryConfig conf; - auto s = Config::genConfig(conf); - if (!s.ok()) { + OsqueryConfig conf; + if (!genConfig(conf).ok()) { return Status(1, "Cannot generate config"); } @@ -49,79 +51,99 @@ Status Config::load() { if (Flag::isDefault(option.first)) { // Only override if option was NOT given as an argument. Flag::updateValue(option.first, option.second); - if (!osquery::isOsqueryWorker()) { - VLOG(1) << "Setting flag option: " << option.first << "=" - << option.second; - } + VLOG(1) << "Setting flag option: " << option.first << "=" + << option.second; } } - cfg_ = conf; + getInstance().cfg_ = conf; return Status(0, "OK"); } -Status Config::genConfig(std::string& conf) { - if (!Registry::exists("config", FLAGS_config_plugin)) { - LOG(ERROR) << "Config retriever " << FLAGS_config_plugin << " not found"; - return Status(1, "Config retriever not found"); +Status Config::genConfig(std::vector& conf) { + auto& config_plugin = Registry::getActive("config"); + if (!Registry::exists("config", config_plugin)) { + return Status(1, "Missing config plugin " + config_plugin); } PluginResponse response; - auto status = Registry::call( - "config", FLAGS_config_plugin, {{"action", "genConfig"}}, response); - + auto status = Registry::call("config", {{"action", "genConfig"}}, response); if (!status.ok()) { return status; } - conf = response[0].at("data"); + if (response.size() > 0) { + for (const auto& it : response[0]) { + conf.push_back(it.second); + } + } return Status(0, "OK"); } +inline void mergeOption(const tree_node& option, OsqueryConfig& conf) { + conf.options[option.first.data()] = option.second.data(); + conf.all_data.add_child("options." + option.first, option.second); +} + +inline void mergeAdditional(const tree_node& node, OsqueryConfig& conf) { + conf.all_data.add_child("additional_monitoring." + node.first, node.second); + + // Support special merging of file paths. + if (node.first != "file_paths") { + return; + } + + for (const auto& category : node.second) { + for (const auto& path : category.second) { + resolveFilePattern(path.second.data(), + conf.eventFiles[category.first], + REC_LIST_FOLDERS | REC_EVENT_OPT); + } + } +} + +inline void mergeScheduledQuery(const tree_node& node, OsqueryConfig& conf) { + // Read tree/JSON into a query structure. + OsqueryScheduledQuery query; + query.name = node.second.get("name", ""); + query.query = node.second.get("query", ""); + query.interval = node.second.get("interval", 0); + // Also store the raw node in the property tree list. + conf.scheduledQueries.push_back(query); + conf.all_data.add_child("scheduledQueries", node.second); +} + Status Config::genConfig(OsqueryConfig& conf) { - std::string config_string; - auto s = genConfig(config_string); + std::vector configs; + auto s = genConfig(configs); if (!s.ok()) { return s; } - std::stringstream json; - pt::ptree tree; - try { - json << config_string; - pt::read_json(json, tree); - // Parse each scheduled query from the config. - for (const pt::ptree::value_type& v : tree.get_child("scheduledQueries")) { - osquery::OsqueryScheduledQuery q; - q.name = (v.second).get("name"); - q.query = (v.second).get("query"); - q.interval = (v.second).get("interval"); - conf.scheduledQueries.push_back(q); - } - // Flags may be set as 'options' within the config. - if (tree.count("options") > 0) { - for (const pt::ptree::value_type& v : tree.get_child("options")) { - conf.options[v.first.data()] = v.second.data(); + for (const auto& config_data : configs) { + std::stringstream json_data; + json_data << config_data; + + pt::ptree tree; + pt::read_json(json_data, tree); + + if (tree.count("scheduledQueries") > 0) { + for (const auto& node : tree.get_child("scheduledQueries")) { + mergeScheduledQuery(node, conf); } } if (tree.count("additional_monitoring") > 0) { - for (const pt::ptree::value_type& v : - tree.get_child("additional_monitoring")) { - if (v.first == "file_paths") { - for (const pt::ptree::value_type& file_cat : v.second) { - for (const pt::ptree::value_type& file : file_cat.second) { - osquery::resolveFilePattern(file.second.get_value(), - conf.eventFiles[file_cat.first]); - } - } - } + for (const auto& node : tree.get_child("additional_monitoring")) { + mergeAdditional(node, conf); } } - } catch (const std::exception& e) { - LOG(ERROR) << "Error parsing config JSON: " << e.what(); - return Status(1, e.what()); - } + if (tree.count("options") > 0) { + for (const auto& option : tree.get_child("options")) { + mergeOption(option, conf); + } + } + } return Status(0, "OK"); } @@ -130,20 +152,22 @@ std::vector Config::getScheduledQueries() { return getInstance().cfg_.scheduledQueries; } -std::map >& Config::getWatchedFiles() { +std::map > Config::getWatchedFiles() { boost::shared_lock lock(rw_lock); return getInstance().cfg_.eventFiles; } +pt::ptree Config::getEntireConfiguration() { + boost::shared_lock lock(rw_lock); + return getInstance().cfg_.all_data; +} + Status Config::getMD5(std::string& hash_string) { - std::string config_string; - auto s = genConfig(config_string); - if (!s.ok()) { - return s; - } + std::stringstream out; + write_json(out, getEntireConfiguration()); hash_string = osquery::hashFromBuffer( - HASH_TYPE_MD5, (void*)config_string.c_str(), config_string.length()); + HASH_TYPE_MD5, (void*)out.str().c_str(), out.str().length()); return Status(0, "OK"); } @@ -160,9 +184,10 @@ Status ConfigPlugin::call(const PluginRequest& request, } if (request.at("action") == "genConfig") { - auto config_data = genConfig(); - response.push_back({{"data", config_data.second}}); - return config_data.first; + std::map config; + auto stat = genConfig(config); + response.push_back(config); + return stat; } return Status(1, "Config plugin action unknown: " + request.at("action")); } diff --git a/osquery/config/config_tests.cpp b/osquery/config/config_tests.cpp index 3b44538..b2110ab 100644 --- a/osquery/config/config_tests.cpp +++ b/osquery/config/config_tests.cpp @@ -7,6 +7,7 @@ * of patent rights can be found in the PATENTS file in the same directory. * */ +#include #include @@ -26,7 +27,7 @@ DECLARE_string(config_path); class ConfigTests : public testing::Test { public: ConfigTests() { - FLAGS_config_plugin = "filesystem"; + Registry::setActive("config", "filesystem"); FLAGS_config_path = kTestDataPath + "test.config"; } @@ -35,7 +36,7 @@ class ConfigTests : public testing::Test { void SetUp() { createMockFileStructure(); Registry::setUp(); - Config::getInstance().load(); + Config::load(); } void TearDown() { tearDownMockFileStructure(); } @@ -44,18 +45,21 @@ class ConfigTests : public testing::Test { class TestConfigPlugin : public ConfigPlugin { public: TestConfigPlugin() {} - - std::pair genConfig() { - return std::make_pair(Status(0, "OK"), "foobar"); + Status genConfig(std::map& config) { + config["data"] = "foobar"; + return Status(0, "OK"); + ; } }; TEST_F(ConfigTests, test_plugin) { Registry::add("config", "test"); + // Change the active config plugin. + EXPECT_TRUE(Registry::setActive("config", "test").ok()); + PluginResponse response; - auto status = - Registry::call("config", "test", {{"action", "genConfig"}}, response); + auto status = Registry::call("config", {{"action", "genConfig"}}, response); EXPECT_EQ(status.ok(), true); EXPECT_EQ(status.toString(), "OK"); @@ -64,15 +68,15 @@ TEST_F(ConfigTests, test_plugin) { TEST_F(ConfigTests, test_queries_execute) { auto queries = Config::getInstance().getScheduledQueries(); - EXPECT_EQ(queries.size(), 1); + EXPECT_EQ(queries.size(), 2); } TEST_F(ConfigTests, test_threatfiles_execute) { - auto files = Config::getInstance().getWatchedFiles(); + auto files = Config::getWatchedFiles(); EXPECT_EQ(files.size(), 2); - EXPECT_EQ(files["downloads"].size(), 9); - EXPECT_EQ(files["system_binaries"].size(), 5); + EXPECT_EQ(files["downloads"].size(), 1); + EXPECT_EQ(files["system_binaries"].size(), 2); } } diff --git a/osquery/config/plugins/filesystem.cpp b/osquery/config/plugins/filesystem.cpp index 8b31bec..78e0948 100644 --- a/osquery/config/plugins/filesystem.cpp +++ b/osquery/config/plugins/filesystem.cpp @@ -8,43 +8,52 @@ * */ -#include +#include #include #include #include #include +#include namespace fs = boost::filesystem; -using osquery::Status; namespace osquery { -FLAG(string, config_path, "/var/osquery/osquery.conf", "Path to config file"); +CLI_FLAG(string, + config_path, + "/var/osquery/osquery.conf", + "(filesystem) config plugin path to JSON config file"); class FilesystemConfigPlugin : public ConfigPlugin { public: - virtual std::pair genConfig(); + Status genConfig(std::map& config); }; REGISTER(FilesystemConfigPlugin, "config", "filesystem"); -std::pair FilesystemConfigPlugin::genConfig() { - std::string config; +Status FilesystemConfigPlugin::genConfig( + std::map& config) { if (!fs::exists(FLAGS_config_path)) { - return std::make_pair(Status(1, "config file does not exist"), config); + return Status(1, "config file does not exist"); } - VLOG(1) << "Filesystem ConfigPlugin reading: " << FLAGS_config_path; - std::ifstream config_stream(FLAGS_config_path); + std::vector conf_files; + resolveFilePattern(FLAGS_config_path + ".d/%.conf", conf_files); + if (conf_files.size() > 0) { + VLOG(1) << "Discovered (" << conf_files.size() << ") additional configs"; + } - config_stream.seekg(0, std::ios::end); - config.reserve(config_stream.tellg()); - config_stream.seekg(0, std::ios::beg); + std::sort(conf_files.begin(), conf_files.end()); + conf_files.push_back(FLAGS_config_path); - config.assign((std::istreambuf_iterator(config_stream)), - std::istreambuf_iterator()); - return std::make_pair(Status(0, "OK"), config); + for (const auto& path : conf_files) { + std::string content; + if (readFile(path, content).ok()) { + config[path] = content; + } + } + return Status(0, "OK"); } } diff --git a/osquery/core/CMakeLists.txt b/osquery/core/CMakeLists.txt index 40143a0..af314d3 100644 --- a/osquery/core/CMakeLists.txt +++ b/osquery/core/CMakeLists.txt @@ -7,6 +7,7 @@ ADD_OSQUERY_LIBRARY(TRUE osquery_core init.cpp hash.cpp watcher.cpp) +# TODO(Sangwan): Detach from core ADD_OSQUERY_LIBRARY(TRUE osquery_test_util test_util.cpp) ADD_OSQUERY_TEST(TRUE osquery_flags_tests flags_tests.cpp) diff --git a/osquery/core/flags.cpp b/osquery/core/flags.cpp index 5d2ae92..5e9338c 100644 --- a/osquery/core/flags.cpp +++ b/osquery/core/flags.cpp @@ -34,8 +34,8 @@ int Flag::create(const std::string& name, const FlagDetail& flag) { return 0; } -int Flag::createAlias(const std::string& alias, const std::string& name) { - instance().aliases_.insert(std::make_pair(alias, name)); +int Flag::createAlias(const std::string& alias, const FlagDetail& flag) { + instance().aliases_.insert(std::make_pair(alias, flag)); return 0; } @@ -72,6 +72,17 @@ std::string Flag::getType(const std::string& name) { return info.type; } +std::string Flag::getDescription(const std::string& name) { + if (instance().flags_.count(name)) { + return instance().flags_.at(name).description; + } + + if (instance().aliases_.count(name)) { + return getDescription(instance().aliases_.at(name).description); + } + return ""; +} + Status Flag::updateValue(const std::string& name, const std::string& value) { GFLAGS_NAMESPACE::SetCommandLineOption(name.c_str(), value.c_str()); return Status(0, "OK"); @@ -100,7 +111,7 @@ std::map Flag::flags() { return flags; } -void Flag::printFlags(bool shell, bool external) { +void Flag::printFlags(bool shell, bool external, bool cli) { std::vector info; GFLAGS_NAMESPACE::GetAllFlags(&info); @@ -110,19 +121,25 @@ void Flag::printFlags(bool shell, bool external) { } max += 7; + auto& aliases = instance().aliases_; auto& details = instance().flags_; for (const auto& flag : info) { - if (details.count(flag.name) == 0) { - // This flag was not defined within osquery code, skip. - continue; - } - - const auto& detail = details.at(flag.name); - if ((shell && !detail.shell) || (!shell && detail.shell)) { - continue; - } - - if ((external && !detail.external) || (!external && detail.external)) { + if (details.count(flag.name) > 0) { + const auto& detail = details.at(flag.name); + if ((shell && !detail.shell) || (!shell && detail.shell) || + (external && !detail.external) || (!external && detail.external) || + (cli && !detail.cli) || (!cli && detail.cli)) { + continue; + } + } else if (aliases.count(flag.name) > 0) { + const auto& alias = aliases.at(flag.name); + // Aliases are only printed if this is an external tool and the alias + // is external. + if (!alias.external || !external) { + continue; + } + } else { + // This flag was not defined as an osquery flag or flag alias. continue; } @@ -135,7 +152,7 @@ void Flag::printFlags(bool shell, bool external) { } fprintf(stdout, "%s", std::string(pad - flag.name.size(), ' ').c_str()); - fprintf(stdout, "%s\n", detail.description.c_str()); + fprintf(stdout, "%s\n", getDescription(flag.name).c_str()); } } } diff --git a/osquery/core/hash_tests.cpp b/osquery/core/hash_tests.cpp index 4c5757e..1c5f57c 100644 --- a/osquery/core/hash_tests.cpp +++ b/osquery/core/hash_tests.cpp @@ -50,6 +50,5 @@ TEST_F(HashTests, test_file_hashing) { int main(int argc, char* argv[]) { testing::InitGoogleTest(&argc, argv); - osquery::initOsquery(argc, argv); return RUN_ALL_TESTS(); } diff --git a/osquery/core/init.cpp b/osquery/core/init.cpp index 1700eb0..367a011 100644 --- a/osquery/core/init.cpp +++ b/osquery/core/init.cpp @@ -8,83 +8,97 @@ * */ +#include #include #include +#include +#include + #include #include #include +#include #include #include #include #include +#include "osquery/core/watcher.h" + namespace osquery { -const std::string kDescription = - "your operating system as a high-performance " - "relational database"; -const std::string kEpilog = "osquery project page ."; +#define DESCRIPTION \ + "osquery %s, your OS as a high-performance relational database\n" +#define EPILOG "\nosquery project page .\n" +#define OPTIONS \ + "\nosquery configuration options (set by config or CLI flags):\n\n" +#define OPTIONS_SHELL "\nosquery shell-only CLI flags:\n\n" +#define OPTIONS_CLI "osquery%s command line flags:\n\n" +#define USAGE "Usage: %s [OPTION]... %s\n\n" +#define CONFIG_ERROR \ + "You are using default configurations for osqueryd for one or more of the " \ + "following\n" \ + "flags: pidfile, db_path.\n\n" \ + "These options create files in /var/osquery but it looks like that path " \ + "has not\n" \ + "been created. Please consider explicitly defining those " \ + "options as a different \n" \ + "path. Additionally, review the \"using osqueryd\" wiki page:\n" \ + " - https://github.com/facebook/osquery/wiki/using-osqueryd\n\n"; -FLAG(bool, config_check, false, "Check the format of an osquery config"); +CLI_FLAG(bool, + config_check, + false, + "Check the format of an osquery config and exit"); #ifndef __APPLE__ -namespace osquery { -FLAG(bool, daemonize, false, "Run as daemon (osqueryd only)"); -} +CLI_FLAG(bool, daemonize, false, "Run as daemon (osqueryd only)"); #endif namespace fs = boost::filesystem; void printUsage(const std::string& binary, int tool) { // Parse help options before gflags. Only display osquery-related options. - fprintf(stdout, "osquery " OSQUERY_VERSION ", %s\n", kDescription.c_str()); + fprintf(stdout, DESCRIPTION, OSQUERY_VERSION); if (tool == OSQUERY_TOOL_SHELL) { // The shell allows a caller to run a single SQL statement and exit. - fprintf( - stdout, "Usage: %s [OPTION]... [SQL STATEMENT]\n\n", binary.c_str()); + fprintf(stdout, USAGE, binary.c_str(), "[SQL STATEMENT]"); } else { - fprintf(stdout, "Usage: %s [OPTION]...\n\n", binary.c_str()); + fprintf(stdout, USAGE, binary.c_str(), ""); } - fprintf(stdout, "The following options control osquery.\n\n"); - // Print only the core/internal flags. - Flag::printFlags(); + if (tool == OSQUERY_EXTENSION) { + fprintf(stdout, OPTIONS_CLI, " extension"); + Flag::printFlags(false, true); + } else { + fprintf(stdout, OPTIONS_CLI, ""); + Flag::printFlags(false, false, true); + fprintf(stdout, OPTIONS); + Flag::printFlags(); + } if (tool == OSQUERY_TOOL_SHELL) { // Print shell flags. - fprintf(stdout, "\nThe following control the osquery shell.\n\n"); + fprintf(stdout, OPTIONS_SHELL); Flag::printFlags(true); } - fprintf(stdout, "\n%s\n", kEpilog.c_str()); -} - -void printConfigWarning() { - std::cerr << "You are using default configurations for osqueryd for one or " - "more of the following\n" - << "flags: pidfile, db_path.\n\n" - << "These options create files in /var/osquery but it looks like " - "that path has not\n" - << "been created. Please consider explicitly defining those " - "options as a different \n" - << "path. Additionally, review the \"using osqueryd\" wiki page:\n" - << " - https://github.com/facebook/osquery/wiki/using-osqueryd\n\n"; + fprintf(stdout, EPILOG); } -void announce() { - syslog(LOG_NOTICE, "osqueryd started [version=" OSQUERY_VERSION "]"); -} - -void initOsquery(int argc, char* argv[], int tool) { +Initializer::Initializer(int argc, char* argv[], ToolType tool) + : argc_(argc), + argv_((char**)argv), + tool_(tool), + binary_(fs::path(std::string(argv[0])).filename().string()) { std::srand(time(nullptr)); - std::string binary(fs::path(std::string(argv[0])).filename().string()); - std::string first_arg = (argc > 1) ? std::string(argv[1]) : ""; // osquery implements a custom help/usage output. + std::string first_arg = (argc_ > 1) ? std::string(argv_[1]) : ""; if ((first_arg == "--help" || first_arg == "-h" || first_arg == "-help") && tool != OSQUERY_TOOL_TEST) { - printUsage(binary, tool); + printUsage(binary_, tool_); ::exit(0); } @@ -100,38 +114,50 @@ void initOsquery(int argc, char* argv[], int tool) { FLAGS_logger_plugin = STR(OSQUERY_DEFAULT_LOGGER_PLUGIN); #endif + if (tool == OSQUERY_TOOL_SHELL) { + // The shell is transient, rewrite config-loaded paths. + osquery::FLAGS_disable_logging = true; + + // Get the caller's home dir for temporary storage/state management. + auto user = getpwuid(getuid()); + std::string homedir; + if (getenv("HOME") != nullptr) { + homedir = std::string(getenv("HOME")) + "/.osquery"; + } else if (user != nullptr || user->pw_dir != nullptr) { + homedir = std::string(user->pw_dir) + "/.osquery"; + } else { + homedir = "/tmp/osquery"; + } + + if (osquery::pathExists(homedir).ok() || + boost::filesystem::create_directory(homedir)) { + osquery::FLAGS_database_path = homedir + "/shell.db"; + osquery::FLAGS_extensions_socket = homedir + "/shell.em"; + } + } + // Set version string from CMake build GFLAGS_NAMESPACE::SetVersionString(OSQUERY_VERSION); // Let gflags parse the non-help options/flags. - GFLAGS_NAMESPACE::ParseCommandLineFlags(&argc, &argv, false); + GFLAGS_NAMESPACE::ParseCommandLineFlags(&argc_, &argv_, false); - // Initialize the status and results logger. - initStatusLogger(binary); - VLOG(1) << "osquery initialized [version=" OSQUERY_VERSION "]"; + // If the caller is checking configuration, disable the watchdog/worker. + if (FLAGS_config_check) { + FLAGS_disable_watchdog = true; + } + // Initialize the status and results logger. + initStatusLogger(binary_); if (tool != OSQUERY_EXTENSION) { - // Load the osquery config using the default/active config plugin. - Config::getInstance().load(); - - if (FLAGS_config_check) { - // The initiator requested an initialization and config check. - auto s = Config::checkConfig(); - if (!s.ok()) { - std::cerr << "Error reading config: " << s.toString() << "\n"; - } - // A configuration check exits the application. - ::exit(s.getCode()); - } + VLOG(1) << "osquery initialized [version=" << OSQUERY_VERSION << "]"; + } else { + VLOG(1) << "osquery extension initialized [sdk=" << OSQUERY_SDK_VERSION + << "]"; } - - // Run the setup for all non-lazy registries. - Registry::setUp(); - // Initialize the status and result plugin logger. - initLogger(binary); } -void initOsqueryDaemon() { +void Initializer::initDaemon() { #ifndef __APPLE__ // OSX uses launchd to daemonize. if (osquery::FLAGS_daemonize) { @@ -142,29 +168,143 @@ void initOsqueryDaemon() { #endif // Print the version to SYSLOG. - announce(); + syslog( + LOG_NOTICE, "%s started [version=%s]", binary_.c_str(), OSQUERY_VERSION); // check if /var/osquery exists if ((Flag::isDefault("pidfile") || Flag::isDefault("db_path")) && !isDirectory("/var/osquery")) { - printConfigWarning(); + std::cerr << CONFIG_ERROR } // Create a process mutex around the daemon. auto pid_status = createPidFile(); if (!pid_status.ok()) { - LOG(ERROR) << "osqueryd initialize failed: " << pid_status.toString(); + LOG(ERROR) << binary_ << " initialize failed: " << pid_status.toString(); ::exit(EXIT_FAILURE); } +} - // Check the backing store by allocating and exitting on error. - if (!DBHandle::checkDB()) { - LOG(ERROR) << "osqueryd initialize failed: Could not create DB handle"; +void Initializer::initWatcher() { + // The watcher takes a list of paths to autoload extensions from. + loadExtensions(); + + // Add a watcher service thread to start/watch an optional worker and set + // of optional extensions in the autoload paths. + if (Watcher::hasManagedExtensions() || !FLAGS_disable_watchdog) { + Dispatcher::getInstance().addService( + std::make_shared(argc_, argv_, !FLAGS_disable_watchdog)); + } + + // If there are no autoloaded extensions, the watcher service will end, + // otherwise it will continue as a background thread and respawn them. + // If the watcher is also a worker watchdog it will do nothing but monitor + // the extensions and worker process. + if (!FLAGS_disable_watchdog) { + Dispatcher::joinServices(); + // Executation should never reach this point. ::exit(EXIT_FAILURE); } } -void shutdownOsquery() { +void Initializer::initWorker(const std::string& name) { + // Set the worker's process name. + size_t name_size = strlen(argv_[0]); + for (int i = 0; i < argc_; i++) { + if (argv_[i] != nullptr) { + memset(argv_[i], 0, strlen(argv_[i])); + } + } + strncpy(argv_[0], name.c_str(), name_size); + + // Start a watcher watcher thread to exit the process if the watcher exits. + Dispatcher::getInstance().addService( + std::make_shared(getppid())); +} + +void Initializer::initWorkerWatcher(const std::string& name) { + if (isWorker()) { + initWorker(name); + } else { + // The watcher will forever monitor and spawn additional workers. + initWatcher(); + } +} + +bool Initializer::isWorker() { return (getenv("OSQUERY_WORKER") != nullptr); } + +void Initializer::initConfigLogger() { + // Use a delay, meaning the amount of milliseconds waited for extensions. + size_t delay = 0; + // The timeout is the maximum time in seconds to wait for extensions. + size_t timeout = atoi(FLAGS_extensions_timeout.c_str()); + while (!Registry::setActive("config", FLAGS_config_plugin)) { + // If there is at least 1 autoloaded extension, it may broadcast a route + // to the active config plugin. + if (!Watcher::hasManagedExtensions() || delay > timeout * 1000) { + LOG(ERROR) << "Config plugin not found: " << FLAGS_config_plugin; + ::exit(EXIT_CATASTROPHIC); + } + ::usleep(kExtensionInitializeMLatency * 1000); + delay += kExtensionInitializeMLatency; + } + + // Try the same wait for a logger pluing too. + while (!Registry::setActive("logger", FLAGS_logger_plugin)) { + if (!Watcher::hasManagedExtensions() || delay > timeout * 1000) { + LOG(ERROR) << "Logger plugin not found: " << FLAGS_logger_plugin; + ::exit(EXIT_CATASTROPHIC); + } + ::usleep(kExtensionInitializeMLatency * 1000); + delay += kExtensionInitializeMLatency; + } +} + +void Initializer::start() { + // Load registry/extension modules before extensions. + osquery::loadModules(); + + // Bind to an extensions socket and wait for registry additions. + osquery::startExtensionManager(); + + // Then set the config/logger plugins, which use a single/active plugin. + initConfigLogger(); + + // Run the setup for all lazy registries (tables, SQL). + Registry::setUp(); + + if (FLAGS_config_check) { + // The initiator requested an initialization and config check. + auto s = Config::checkConfig(); + if (!s.ok()) { + std::cerr << "Error reading config: " << s.toString() << "\n"; + } + // A configuration check exits the application. + ::exit(s.getCode()); + } + + // Load the osquery config using the default/active config plugin. + Config::load(); + + // Check the backing store by allocating and exiting on error. + if (!DBHandle::checkDB()) { + LOG(ERROR) << binary_ << " initialize failed: Could not create DB handle"; + if (isWorker()) { + ::exit(EXIT_CATASTROPHIC); + } else { + ::exit(EXIT_FAILURE); + } + } + + // Initialize the status and result plugin logger. + initLogger(binary_); + + // Start event threads. + osquery::attachEvents(); + osquery::EventFactory::delay(); +} + +void Initializer::shutdown() { // End any event type run loops. EventFactory::end(); diff --git a/osquery/core/system.cpp b/osquery/core/system.cpp index 64cf913..8ee85e3 100644 --- a/osquery/core/system.cpp +++ b/osquery/core/system.cpp @@ -31,13 +31,16 @@ namespace fs = boost::filesystem; namespace osquery { /// The path to the pidfile for osqueryd -FLAG(string, - pidfile, - "/var/osquery/osqueryd.pidfile", - "The path to the pidfile for osqueryd"); +CLI_FLAG(string, + pidfile, + "/var/osquery/osqueryd.pidfile", + "Path to the daemon pidfile mutex"); /// Should the daemon force unload previously-running osqueryd daemons. -FLAG(bool, force, false, "Force osqueryd to kill previously-running daemons"); +CLI_FLAG(bool, + force, + false, + "Force osqueryd to kill previously-running daemons"); std::string getHostname() { char hostname[256]; // Linux max should be 64. diff --git a/osquery/core/tables_tests.cpp b/osquery/core/tables_tests.cpp index d3f5f06..7c4f558 100644 --- a/osquery/core/tables_tests.cpp +++ b/osquery/core/tables_tests.cpp @@ -115,6 +115,5 @@ TEST_F(TablesTests, test_constraint_map) { int main(int argc, char* argv[]) { testing::InitGoogleTest(&argc, argv); - osquery::initOsquery(argc, argv); return RUN_ALL_TESTS(); } diff --git a/osquery/core/test_util.cpp b/osquery/core/test_util.cpp index f4aa2b6..fd814a2 100644 --- a/osquery/core/test_util.cpp +++ b/osquery/core/test_util.cpp @@ -250,7 +250,7 @@ void createMockFileStructure() { "/deep11/deep2/deep3/"); boost::filesystem::create_directories(kFakeDirectory + "/deep1/deep2/"); writeTextFile(kFakeDirectory + "/root.txt", "root"); - writeTextFile(kFakeDirectory + "/toor.txt", "toor"); + writeTextFile(kFakeDirectory + "/door.txt", "toor"); writeTextFile(kFakeDirectory + "/roto.txt", "roto"); writeTextFile(kFakeDirectory + "/deep1/level1.txt", "l1"); writeTextFile(kFakeDirectory + "/deep11/not_bash", "l1"); diff --git a/osquery/core/watcher.cpp b/osquery/core/watcher.cpp index 618ff0a..eaf8ed8 100644 --- a/osquery/core/watcher.cpp +++ b/osquery/core/watcher.cpp @@ -9,14 +9,12 @@ */ #include -#include #include #include #include -#include #include #include #include @@ -32,92 +30,257 @@ namespace osquery { const std::map > kWatchdogLimits = { // Maximum MB worker can privately allocate. - {MEMORY_LIMIT, {50, 20, 10, 10}}, + {MEMORY_LIMIT, {50, 30, 10, 10}}, // Percent of user or system CPU worker can utilize for LATENCY_LIMIT // seconds. - {UTILIZATION_LIMIT, {90, 70, 60, 50}}, + {UTILIZATION_LIMIT, {90, 80, 60, 50}}, // Number of seconds the worker should run, else consider the exit fatal. {RESPAWN_LIMIT, {20, 20, 20, 5}}, // If the worker respawns too quickly, backoff on creating additional. {RESPAWN_DELAY, {5, 5, 5, 1}}, // Seconds of tolerable UTILIZATION_LIMIT sustained latency. - {LATENCY_LIMIT, {5, 5, 3, 1}}, + {LATENCY_LIMIT, {12, 6, 3, 1}}, // How often to poll for performance limit violations. {INTERVAL, {3, 3, 3, 1}}, }; -FLAG(int32, - watchdog_level, - 1, - "Performance limit level (0=loose, 1=normal, 2=restrictive, 3=debug)"); +const std::string kExtensionExtension = ".ext"; -FLAG(bool, disable_watchdog, false, "Disable userland watchdog process"); +CLI_FLAG(int32, + watchdog_level, + 1, + "Performance limit level (0=loose, 1=normal, 2=restrictive, 3=debug)"); -bool Watcher::ok() { - ::sleep(getWorkerLimit(INTERVAL)); - return (worker_ >= 0); +CLI_FLAG(bool, disable_watchdog, false, "Disable userland watchdog process"); + +/// If the worker exits the watcher will inspect the return code. +void childHandler(int signum) { + siginfo_t info; + // Make sure WNOWAIT is used to the wait information is not removed. + // Watcher::watch implements a thread to poll for this information. + waitid(P_ALL, 0, &info, WEXITED | WSTOPPED | WNOHANG | WNOWAIT); + if (info.si_code == CLD_EXITED && info.si_status == EXIT_CATASTROPHIC) { + // A child process had a catastrophic error, abort the watcher. + ::exit(EXIT_FAILURE); + } +} + +void Watcher::resetWorkerCounters(size_t respawn_time) { + // Reset the monitoring counters for the watcher. + auto& state = instance().state_; + state.sustained_latency = 0; + state.user_time = 0; + state.system_time = 0; + state.last_respawn_time = respawn_time; +} + +void Watcher::resetExtensionCounters(const std::string& extension, + size_t respawn_time) { + WatcherLocker locker; + auto& state = instance().extension_states_[extension]; + state.sustained_latency = 0; + state.user_time = 0; + state.system_time = 0; + state.last_respawn_time = respawn_time; +} + +std::string Watcher::getExtensionPath(pid_t child) { + for (const auto& extension : extensions()) { + if (extension.second == child) { + return extension.first; + } + } + return ""; +} + +void Watcher::removeExtensionPath(const std::string& extension) { + WatcherLocker locker; + instance().extensions_.erase(extension); + instance().extension_states_.erase(extension); +} + +PerformanceState& Watcher::getState(pid_t child) { + if (child == instance().worker_) { + return instance().state_; + } else { + return instance().extension_states_[getExtensionPath(child)]; + } +} + +PerformanceState& Watcher::getState(const std::string& extension) { + return instance().extension_states_[extension]; } -bool Watcher::watch() { +void Watcher::setExtension(const std::string& extension, pid_t child) { + WatcherLocker locker; + instance().extensions_[extension] = child; +} + +void Watcher::reset(pid_t child) { + if (child == instance().worker_) { + instance().worker_ = 0; + resetWorkerCounters(0); + return; + } + + // If it was not the worker pid then find the extension name to reset. + for (const auto& extension : extensions()) { + if (extension.second == child) { + setExtension(extension.first, 0); + resetExtensionCounters(extension.first, 0); + } + } +} + +void Watcher::addExtensionPath(const std::string& path) { + // Resolve acceptable extension binaries from autoload paths. + if (isDirectory(path).ok()) { + VLOG(1) << "Cannot autoload extension from directory: " << path; + return; + } + + // Only autoload extensions which were safe at the time of discovery. + // If the extension binary later becomes unsafe (permissions change) then + // it will fail to reload if a reload is ever needed. + fs::path extension(path); + if (safePermissions(extension.parent_path().string(), path, true)) { + if (extension.extension().string() == kExtensionExtension) { + setExtension(extension.string(), 0); + resetExtensionCounters(extension.string(), 0); + VLOG(1) << "Found autoloadable extension: " << extension.string(); + } + } +} + +bool Watcher::hasManagedExtensions() { + if (instance().extensions_.size() > 0) { + return true; + } + + // A watchdog process may hint to a worker the number of managed extensions. + // Setting this counter to 0 will prevent the worker from waiting for missing + // dependent config plugins. Otherwise, its existance, will cause a worker to + // wait for missing plugins to broadcast from managed extensions. + return (getenv("OSQUERY_EXTENSIONS") != nullptr); +} + +bool WatcherRunner::ok() { + interruptableSleep(getWorkerLimit(INTERVAL) * 1000); + // Watcher is OK to run if a worker or at least one extension exists. + return (Watcher::getWorker() >= 0 || Watcher::hasManagedExtensions()); +} + +void WatcherRunner::enter() { + // Set worker performance counters to an initial state. + Watcher::resetWorkerCounters(0); + signal(SIGCHLD, childHandler); + + // Enter the watch loop. + do { + if (use_worker_ && !watch(Watcher::getWorker())) { + // The watcher failed, create a worker. + createWorker(); + } + + // Loop over every managed extension and check sanity. + std::vector failing_extensions; + for (const auto& extension : Watcher::extensions()) { + if (!watch(extension.second)) { + if (!createExtension(extension.first)) { + failing_extensions.push_back(extension.first); + } + } + } + // If any extension creations failed, stop managing them. + for (const auto& failed_extension : failing_extensions) { + Watcher::removeExtensionPath(failed_extension); + } + } while (ok()); +} + +bool WatcherRunner::watch(pid_t child) { int status; - pid_t result = waitpid(worker_, &status, WNOHANG); - if (worker_ == 0 || result == worker_) { + pid_t result = waitpid(child, &status, WNOHANG); + if (child == 0 || result == child) { // Worker does not exist or never existed. return false; } else if (result == 0) { // If the inspect finds problems it will stop/restart the worker. - if (!isWorkerSane()) { - stopWorker(); + if (!isChildSane(child)) { + stopChild(child); return false; } } return true; } -void Watcher::stopWorker() { - kill(worker_, SIGKILL); - worker_ = 0; +void WatcherRunner::stopChild(pid_t child) { + kill(child, SIGKILL); + child = 0; + // Clean up the defunct (zombie) process. - waitpid(-1, 0, 0); + waitpid(-1, 0, WNOHANG); } -bool Watcher::isWorkerSane() { +bool WatcherRunner::isChildSane(pid_t child) { auto rows = - SQL::selectAllFrom("processes", "pid", tables::EQUALS, INTEGER(worker_)); + SQL::selectAllFrom("processes", "pid", tables::EQUALS, INTEGER(child)); if (rows.size() == 0) { // Could not find worker process? return false; } + // Get the performance state for the worker or extension. + size_t sustained_latency = 0; // Compare CPU utilization since last check. - BIGINT_LITERAL footprint, user_time, system_time; + BIGINT_LITERAL footprint, user_time, system_time, parent; // IV is the check interval in seconds, and utilization is set per-second. auto iv = getWorkerLimit(INTERVAL); - try { - user_time = AS_LITERAL(BIGINT_LITERAL, rows[0].at("user_time")) / iv; - system_time = AS_LITERAL(BIGINT_LITERAL, rows[0].at("system_time")) / iv; - footprint = AS_LITERAL(BIGINT_LITERAL, rows[0].at("phys_footprint")); - } catch (const std::exception& e) { - sustained_latency_ = 0; - } + { + WatcherLocker locker; + auto state = Watcher::getState(child); + try { + parent = AS_LITERAL(BIGINT_LITERAL, rows[0].at("parent")); + user_time = AS_LITERAL(BIGINT_LITERAL, rows[0].at("user_time")) / iv; + system_time = AS_LITERAL(BIGINT_LITERAL, rows[0].at("system_time")) / iv; + footprint = AS_LITERAL(BIGINT_LITERAL, rows[0].at("phys_footprint")); + } catch (const std::exception& e) { + state.sustained_latency = 0; + } - if (current_user_time_ + getWorkerLimit(UTILIZATION_LIMIT) < user_time || - current_system_time_ + getWorkerLimit(UTILIZATION_LIMIT) < system_time) { - sustained_latency_++; - } else { - sustained_latency_ = 0; + // Check the different of CPU time used since last check. + if (state.user_time + getWorkerLimit(UTILIZATION_LIMIT) < user_time || + state.system_time + getWorkerLimit(UTILIZATION_LIMIT) < system_time) { + state.sustained_latency++; + } else { + state.sustained_latency = 0; + } + // Update the current CPU time. + state.user_time = user_time; + state.system_time = system_time; + + // Check if the sustained difference exceeded the acceptable latency limit. + sustained_latency = state.sustained_latency; } - current_user_time_ = user_time; - current_system_time_ = system_time; + // Only make a decision about the child sanity if it is still the watcher's + // child. It's possible for the child to die, and its pid reused. + if (parent != getpid()) { + // The child's parent is not the watcher. + Watcher::reset(child); + // Do not stop or call the child insane, since it is not our child. + return true; + } - if (sustained_latency_ * iv >= getWorkerLimit(LATENCY_LIMIT)) { + if (sustained_latency > 0 && + sustained_latency * iv >= getWorkerLimit(LATENCY_LIMIT)) { LOG(WARNING) << "osqueryd worker system performance limits exceeded"; return false; } - if (footprint > getWorkerLimit(MEMORY_LIMIT) * 1024 * 1024) { - LOG(WARNING) << "osqueryd worker memory limits exceeded"; + // Check if the private memory exceeds a memory limit. + if (footprint > 0 && footprint > getWorkerLimit(MEMORY_LIMIT) * 1024 * 1024) { + LOG(WARNING) << "osqueryd worker memory limits exceeded: " << footprint; return false; } @@ -125,10 +288,14 @@ bool Watcher::isWorkerSane() { return true; } -void Watcher::createWorker() { - if (last_respawn_time_ > getUnixTime() - getWorkerLimit(RESPAWN_LIMIT)) { - LOG(WARNING) << "osqueryd worker respawning too quickly"; - ::sleep(getWorkerLimit(RESPAWN_DELAY)); +void WatcherRunner::createWorker() { + { + WatcherLocker locker; + if (Watcher::getState(Watcher::getWorker()).last_respawn_time > + getUnixTime() - getWorkerLimit(RESPAWN_LIMIT)) { + LOG(WARNING) << "osqueryd worker respawning too quickly"; + interruptableSleep(getWorkerLimit(RESPAWN_DELAY) * 1000); + } } // Get the path of the current process. @@ -139,50 +306,84 @@ void Watcher::createWorker() { ::exit(EXIT_FAILURE); } - worker_ = fork(); - if (worker_ < 0) { + // Set an environment signaling to potential plugin-dependent workers to wait + // for extensions to broadcast. + if (Watcher::hasManagedExtensions()) { + setenv("OSQUERY_EXTENSIONS", "true", 1); + } + + auto worker_pid = fork(); + if (worker_pid < 0) { // Unrecoverable error, cannot create a worker process. LOG(ERROR) << "osqueryd could not create a worker process"; ::exit(EXIT_FAILURE); - } else if (worker_ == 0) { + } else if (worker_pid == 0) { // This is the new worker process, no watching needed. - setenv("OSQUERYD_WORKER", std::to_string(getpid()).c_str(), 1); + setenv("OSQUERY_WORKER", std::to_string(getpid()).c_str(), 1); // Get the complete path of the osquery process binary. auto exec_path = fs::system_complete(fs::path(qd[0]["path"])); execve(exec_path.string().c_str(), argv_, environ); - // Code will never reach this point. - ::exit(EXIT_FAILURE); + // Code should never reach this point. + LOG(ERROR) << "osqueryd could not start worker process"; + ::exit(EXIT_CATASTROPHIC); } + Watcher::setWorker(worker_pid); + Watcher::resetWorkerCounters(getUnixTime()); VLOG(1) << "osqueryd watcher (" << getpid() << ") executing worker (" - << worker_ << ")"; + << worker_pid << ")"; } -void Watcher::resetCounters() { - // Reset the monitoring counters for the watcher. - sustained_latency_ = 0; - current_user_time_ = 0; - current_system_time_ = 0; - last_respawn_time_ = getUnixTime(); -} - -void Watcher::initWorker() { - // Set the worker's process name. - size_t name_size = strlen(argv_[0]); - for (int i = 0; i < argc_; i++) { - if (argv_[i] != nullptr) { - memset(argv_[i], 0, strlen(argv_[i])); +bool WatcherRunner::createExtension(const std::string& extension) { + { + WatcherLocker locker; + if (Watcher::getState(extension).last_respawn_time > + getUnixTime() - getWorkerLimit(RESPAWN_LIMIT)) { + LOG(WARNING) << "Extension respawning too quickly: " << extension; + // Unlike a worker, if an extension respawns to quickly we give up. + return false; } } - strncpy(argv_[0], name_.c_str(), name_size); - // Start a watcher watcher thread to exit the process if the watcher exits. - Dispatcher::getInstance().addService( - std::make_shared(getppid())); -} + // Check the path to the previously-discovered extension binary. + auto exec_path = fs::system_complete(fs::path(extension)); + if (!safePermissions( + exec_path.parent_path().string(), exec_path.string(), true)) { + // Extension binary has become unsafe. + LOG(WARNING) << "Extension binary has unsafe permissions: " << extension; + return false; + } + + auto ext_pid = fork(); + if (ext_pid < 0) { + // Unrecoverable error, cannot create an extension process. + LOG(ERROR) << "Cannot create extension process: " << extension; + ::exit(EXIT_FAILURE); + } else if (ext_pid == 0) { + // Pass the current extension socket and a set timeout to the extension. + setenv("OSQUERY_EXTENSION", std::to_string(getpid()).c_str(), 1); + // Execute extension with very specific arguments. + execle(exec_path.string().c_str(), + ("osquery extension: " + extension).c_str(), + "--socket", + Flag::getValue("extensions_socket").c_str(), + "--timeout", + Flag::getValue("extensions_timeout").c_str(), + "--interval", + Flag::getValue("extensions_interval").c_str(), + (Flag::getValue("verbose") == "true") ? "--verbose" : (char*)nullptr, + (char*)nullptr, + environ); + // Code should never reach this point. + VLOG(1) << "Could not start extension process: " << extension; + ::exit(EXIT_FAILURE); + } -bool isOsqueryWorker() { - return (getenv("OSQUERYD_WORKER") != nullptr); + Watcher::setExtension(extension, ext_pid); + Watcher::resetExtensionCounters(extension, getUnixTime()); + VLOG(1) << "Created and monitoring extension child (" << ext_pid << "): " + << extension; + return true; } void WatcherWatcherRunner::enter() { @@ -211,26 +412,4 @@ size_t getWorkerLimit(WatchdogLimitType name, int level) { } return kWatchdogLimits.at(name).at(level); } - -void initWorkerWatcher(const std::string& name, int argc, char* argv[]) { - // The watcher will forever monitor and spawn additional workers. - Watcher watcher(argc, argv); - watcher.setWorkerName(name); - - if (isOsqueryWorker()) { - // Do not start watching/spawning if this process is a worker. - watcher.initWorker(); - } else { - do { - if (!watcher.watch()) { - // The watcher failed, create a worker. - watcher.createWorker(); - watcher.resetCounters(); - } - } while (watcher.ok()); - - // Executation should never reach this point. - ::exit(EXIT_FAILURE); - } -} } diff --git a/osquery/core/watcher.h b/osquery/core/watcher.h index ffa7396..6756c36 100644 --- a/osquery/core/watcher.h +++ b/osquery/core/watcher.h @@ -14,6 +14,9 @@ #include +#include +#include + #include #include "osquery/dispatcher/dispatcher.h" @@ -22,6 +25,14 @@ namespace osquery { DECLARE_bool(disable_watchdog); +/** + * @brief Categories of process performance limitations. + * + * Performance limits are applied by a watcher thread on autoloaded extensions + * and optional a daemon worker process. The performance types are identified + * here, and organized into levels. Such that a caller may enforce rigor or + * relax the performance expectations of a osquery daemon. + */ enum WatchdogLimitType { MEMORY_LIMIT, UTILIZATION_LIMIT, @@ -31,47 +42,186 @@ enum WatchdogLimitType { INTERVAL, }; -class Watcher { +/** + * @brief A performance state structure for an autoloaded extension or worker. + * + * A watcher thread will continue to check the performance state, and keep a + * last-checked snapshot for each autoloaded extension and worker process. + */ +struct PerformanceState { + /// A counter of how many intervals the process exceeded performance limits. + size_t sustained_latency; + /// The last checked user CPU time. + size_t user_time; + /// The last checked system CPU time. + size_t system_time; + /// A timestamp when the process/worker was last created. + size_t last_respawn_time; + + PerformanceState() { + sustained_latency = 0; + user_time = 0; + system_time = 0; + last_respawn_time = 0; + } +}; + +/** + * @brief Thread-safe watched child process state manager. + * + * The Watcher instance is separated from the WatcherRunner thread to allow + * signals and osquery-introspection to monitor the autoloaded extensions + * and optional worker stats. A child-process change signal may indicate an + * autoloaded extension ended. Tables may also report on the historic worker + * or extension utilizations. + * + * Though not critical, it is perferred to remove the extension's broadcasted + * routes quickly. Locking access to the extensions list between signals and + * the WatcherRunner thread allows osquery to tearDown registry changes before + * attempting to respawn an extension process. + */ +class Watcher : private boost::noncopyable { public: - Watcher(int argc, char* argv[]) : worker_(0), argc_(argc), argv_(argv) { - resetCounters(); - last_respawn_time_ = 0; + /// Instance accessor + static Watcher& instance() { + static Watcher instance; + return instance; } - void setWorkerName(const std::string& name) { name_ = name; } - const std::string& getWorkerName() { return name_; } + /// Reset counters after a worker exits. + static void resetWorkerCounters(size_t respawn_time); - /// Boilerplate function to sleep for some configured latency - bool ok(); - /// Begin the worker-watcher process. - bool watch(); - /// Fork a worker process. - void createWorker(); - /// If the process is a worker, clean up identification. - void initWorker(); - void resetCounters(); + /// Reset counters for an extension path. + static void resetExtensionCounters(const std::string& extension, + size_t respawn_time); + + /// Lock access to extensions. + static void lock() { instance().lock_.lock(); } + + /// Unlock access to extensions. + static void unlock() { instance().lock_.unlock(); } + + /// Accessor for autoloadable extension paths. + static const std::map& extensions() { + return instance().extensions_; + } + + /// Lookup extension path from pid. + static std::string getExtensionPath(pid_t child); + + /// Remove an autoloadable extension path. + static void removeExtensionPath(const std::string& extension); + + /// Add extensions autoloadable paths. + static void addExtensionPath(const std::string& path); + + /// Get state information for a worker or extension child. + static PerformanceState& getState(pid_t child); + static PerformanceState& getState(const std::string& extension); + + /// Accessor for the worker process. + static pid_t getWorker() { return instance().worker_; } + + /// Setter for worker process. + static void setWorker(pid_t child) { instance().worker_ = child; } + + /// Setter for an extension process. + static void setExtension(const std::string& extension, pid_t child); + + /// Reset pid and performance counters for a worker or extension process. + static void reset(pid_t child); + + /// Return the number of autoloadable extensions. + static bool hasManagedExtensions(); private: - /// Inspect into the memory, CPU, and other worker process states. - bool isWorkerSane(); - /// If a worker as otherwise gone insane, stop it. - void stopWorker(); + /// Do not request the lock until extensions are used. + Watcher() : worker_(-1), lock_(mutex_, boost::defer_lock) {} + Watcher(Watcher const&); + void operator=(Watcher const&); + virtual ~Watcher() {} private: - size_t sustained_latency_; - size_t current_user_time_; - size_t current_system_time_; - size_t last_respawn_time_; + /// Performance state for the worker process. + PerformanceState state_; + /// Performance states for each autoloadable extension binary. + std::map extension_states_; private: /// Keep the single worker process/thread ID for inspection. pid_t worker_; + /// Keep a list of resolved extension paths and their managed pids. + std::map extensions_; + /// Path to autoload extensions from. + std::vector extensions_paths_; + + private: + /// Mutex and lock around extensions access. + boost::mutex mutex_; + /// Mutex and lock around extensions access. + boost::unique_lock lock_; +}; + +/** + * @brief A scoped locker for iterating over watcher extensions. + * + * A lock must be used if any part of osquery wants to enumerate the autoloaded + * extensions or autoloadable extension paths a Watcher may be monitoring. + * A signal or WatcherRunner thread may stop or start extensions. + */ +class WatcherLocker { + public: + /// Construct and gain watcher lock. + WatcherLocker() { Watcher::lock(); } + /// Destruct and release watcher lock. + ~WatcherLocker() { Watcher::unlock(); } +}; + +/** + * @brief The watchdog thread responsible for spawning/monitoring children. + * + * The WatcherRunner thread will spawn any autoloaded modules or optional + * osquery daemon worker processes. It will then poll for their performance + * state and kill/respawn osquery child processes. + */ +class WatcherRunner : public InternalRunnable { + public: + /** + * @brief Construct a watcher thread. + * + * @param argc The osquery process argc. + * @param argv The osquery process argv. + * @param use_worker True if the process should spawn and monitor a worker. + */ + explicit WatcherRunner(int argc, char** argv, bool use_worker) + : argc_(argc), argv_(argv), use_worker_(use_worker) { + (void)argc_; + } + + private: + void enter(); + /// Boilerplate function to sleep for some configured latency + bool ok(); + /// Begin the worker-watcher process. + bool watch(pid_t child); + /// Inspect into the memory, CPU, and other worker/extension process states. + bool isChildSane(pid_t child); + + private: + /// Fork a worker process. + void createWorker(); + /// Fork an extension process. + bool createExtension(const std::string& extension); + /// If a worker/extension has otherwise gone insane, stop it. + void stopChild(pid_t child); + + private: /// Keep the invocation daemon's argc to iterate through argv. int argc_; /// When a worker child is spawned the argv will be scrubed. char** argv_; - /// When a worker child is spawned the process name will be changed. - std::string name_; + /// Spawn/monitor a worker process. + bool use_worker_; }; /// The WatcherWatcher is spawned within the worker and watches the watcher. @@ -84,29 +234,6 @@ class WatcherWatcherRunner : public InternalRunnable { pid_t watcher_; }; -/// Check if the current process is already a worker. -bool isOsqueryWorker(); - /// Get a performance limit by name and optional level. size_t getWorkerLimit(WatchdogLimitType limit, int level = -1); - -/** - * @brief Daemon tools may want to continually spawn worker processes - * and monitor their utilization. - * - * A daemon may call initWorkerWatcher to begin watching child daemon - * processes until it-itself is unscheduled. The basic guarentee is that only - * workers will return from the function. - * - * The worker-watcher will implement performance bounds on CPU utilization - * and memory, as well as check for zombie/defunct workers and respawn them - * if appropriate. The appropriateness is determined from heuristics around - * how the worker exitted. Various exit states and velocities may cause the - * watcher to resign. - * - * @param name The name of the worker process. - * @param argc The daemon's argc. - * @param argv The daemon's volitle argv. - */ -void initWorkerWatcher(const std::string& name, int argc, char* argv[]); } diff --git a/osquery/database/db_handle.cpp b/osquery/database/db_handle.cpp index d393135..2e605d9 100644 --- a/osquery/database/db_handle.cpp +++ b/osquery/database/db_handle.cpp @@ -33,14 +33,13 @@ const std::string kEvents = "events"; const std::vector kDomains = {kConfigurations, kQueries, kEvents}; FLAG(string, - db_path, + database_path, "/var/osquery/osquery.db", "If using a disk-based backing store, specify a path"); +FLAG_ALIAS(std::string, db_path, database_path); -FLAG(bool, - use_in_memory_database, - false, - "Keep osquery backing-store in memory"); +FLAG(bool, database_in_memory, false, "Keep osquery backing-store in memory"); +FLAG_ALIAS(bool, use_in_memory_database, database_in_memory); ///////////////////////////////////////////////////////////////////////////// // constructors and destructors @@ -85,12 +84,12 @@ DBHandle::~DBHandle() { // getInstance methods ///////////////////////////////////////////////////////////////////////////// std::shared_ptr DBHandle::getInstance() { - return getInstance(FLAGS_db_path, FLAGS_use_in_memory_database); + return getInstance(FLAGS_database_path, FLAGS_database_in_memory); } bool DBHandle::checkDB() { try { - auto handle = DBHandle(FLAGS_db_path, FLAGS_use_in_memory_database); + auto handle = DBHandle(FLAGS_database_path, FLAGS_database_in_memory); } catch (const std::exception& e) { return false; } diff --git a/osquery/devtools/CMakeLists.txt b/osquery/devtools/CMakeLists.txt index c4f1ec0..42edd93 100644 --- a/osquery/devtools/CMakeLists.txt +++ b/osquery/devtools/CMakeLists.txt @@ -1,4 +1,3 @@ -ADD_OSQUERY_LIBRARY(FALSE osquery_devtools shell.cpp - printer.cpp) +ADD_OSQUERY_LIBRARY(FALSE osquery_devtools printer.cpp) ADD_OSQUERY_TEST(FALSE osquery_printer_tests printer_tests.cpp) diff --git a/osquery/distributed/distributed_tests.cpp b/osquery/distributed/distributed_tests.cpp index f85042b..d6d3412 100644 --- a/osquery/distributed/distributed_tests.cpp +++ b/osquery/distributed/distributed_tests.cpp @@ -221,6 +221,5 @@ TEST_F(DistributedTests, test_duplicate_request) { int main(int argc, char* argv[]) { testing::InitGoogleTest(&argc, argv); - osquery::initOsquery(argc, argv); return RUN_ALL_TESTS(); } diff --git a/osquery/events/events.cpp b/osquery/events/events.cpp index bb273fe..afeeeed 100644 --- a/osquery/events/events.cpp +++ b/osquery/events/events.cpp @@ -3,7 +3,7 @@ * All rights reserved. * * This source code is licensed under the BSD-style license found in the - * LICENSE file in the root directory of this source tree. An additional grant + * LICENSE file in the root directory of this source tree. An additional grant * of patent rights can be found in the PATENTS file in the same directory. * */ @@ -401,8 +401,7 @@ Status EventFactory::run(EventPublisherID& type_id) { EventPublisherRef publisher; try { publisher = EventFactory::getInstance().getEventPublisher(type_id); - } - catch (std::out_of_range& e) { + } catch (std::out_of_range& e) { return Status(1, "No event type found"); } @@ -418,7 +417,8 @@ Status EventFactory::run(EventPublisherID& type_id) { // The runloop status is not reflective of the event type's. publisher->tearDown(); - VLOG(1) << "Event publisher " << publisher->type() << " runloop terminated"; + VLOG(1) << "Event publisher " << publisher->type() + << " runloop terminated for reason: " << status.getMessage(); return Status(0, "OK"); } @@ -493,8 +493,7 @@ Status EventFactory::addSubscription(EventPublisherID& type_id, EventPublisherRef publisher; try { publisher = getInstance().getEventPublisher(type_id); - } - catch (std::out_of_range& e) { + } catch (std::out_of_range& e) { return Status(1, "No event type found"); } @@ -508,8 +507,7 @@ size_t EventFactory::numSubscriptions(EventPublisherID& type_id) { EventPublisherRef publisher; try { publisher = EventFactory::getInstance().getEventPublisher(type_id); - } - catch (std::out_of_range& e) { + } catch (std::out_of_range& e) { return 0; } return publisher->numSubscriptions(); @@ -539,8 +537,7 @@ Status EventFactory::deregisterEventPublisher(EventPublisherID& type_id) { EventPublisherRef publisher; try { publisher = ef.getEventPublisher(type_id); - } - catch (std::out_of_range& e) { + } catch (std::out_of_range& e) { return Status(1, "No event publisher to deregister"); } diff --git a/osquery/events/events_tests.cpp b/osquery/events/events_tests.cpp index aeeac41..ed95ea6 100644 --- a/osquery/events/events_tests.cpp +++ b/osquery/events/events_tests.cpp @@ -3,7 +3,7 @@ * All rights reserved. * * This source code is licensed under the BSD-style license found in the - * LICENSE file in the root directory of this source tree. An additional grant + * LICENSE file in the root directory of this source tree. An additional grant * of patent rights can be found in the PATENTS file in the same directory. * */ diff --git a/osquery/events/linux/inotify.cpp b/osquery/events/linux/inotify.cpp index 12ff672..9e6e026 100644 --- a/osquery/events/linux/inotify.cpp +++ b/osquery/events/linux/inotify.cpp @@ -63,6 +63,21 @@ void INotifyEventPublisher::tearDown() { inotify_handle_ = -1; } +Status INotifyEventPublisher::restartMonitoring(){ + if (last_restart_ != 0 && getUnixTime() - last_restart_ < 10) { + return Status(1, "Overflow"); + } + last_restart_ = getUnixTime(); + VLOG(1) << "Got an overflow, trying to restart..."; + for(const auto& desc : descriptors_){ + removeMonitor(desc, 1); + } + path_descriptors_.clear(); + descriptor_paths_.clear(); + configure(); + return Status(0, "OK"); +} + Status INotifyEventPublisher::run() { // Get a while wrapper for free. char buffer[BUFFER_SIZE]; @@ -92,7 +107,10 @@ Status INotifyEventPublisher::run() { auto event = reinterpret_cast(p); if (event->mask & IN_Q_OVERFLOW) { // The inotify queue was overflown (remove all paths). - return Status(1, "Overflow"); + Status stat = restartMonitoring(); + if(!stat.ok()){ + return stat; + } } if (event->mask & IN_IGNORED) { @@ -106,6 +124,9 @@ Status INotifyEventPublisher::run() { removeMonitor(event->wd, false); } else { auto ec = createEventContextFrom(event); + if(event->mask & IN_CREATE && isDirectory(ec->path).ok()){ + addMonitor(ec->path, 1); + } fire(ec); } // Continue to iterate @@ -129,8 +150,6 @@ INotifyEventContextRef INotifyEventPublisher::createEventContextFrom( path << "/" << event->name; } ec->path = path.str(); - - // Set the action (may be multiple) for (const auto& action : kMaskActions) { if (event->mask & action.first) { ec->action = action.second; @@ -179,15 +198,10 @@ bool INotifyEventPublisher::addMonitor(const std::string& path, if (recursive && isDirectory(path).ok()) { std::vector children; // Get a list of children of this directory (requesed recursive watches). - if (!listFilesInDirectory(path, children).ok()) { - return false; - } + listDirectoriesInDirectory(path, children); for (const auto& child : children) { - // Only watch child directories, a watch on the directory implies files. - if (isDirectory(child).ok()) { - addMonitor(child, recursive); - } + addMonitor(child, recursive); } } diff --git a/osquery/events/linux/inotify.h b/osquery/events/linux/inotify.h index 7325687..7d5a0d0 100644 --- a/osquery/events/linux/inotify.h +++ b/osquery/events/linux/inotify.h @@ -127,12 +127,15 @@ class INotifyEventPublisher int getHandle() { return inotify_handle_; } /// Get the number of actual INotify active descriptors. int numDescriptors() { return descriptors_.size(); } + /// If we overflow, try and restart the monitor + Status restartMonitoring(); // Consider an event queue if separating buffering from firing/servicing. DescriptorVector descriptors_; PathDescriptorMap path_descriptors_; DescriptorPathMap descriptor_paths_; int inotify_handle_; + int last_restart_; public: FRIEND_TEST(INotifyTests, test_inotify_optimization); diff --git a/osquery/examples/example_extension.cpp b/osquery/examples/example_extension.cpp index 1ba5536..ab66654 100644 --- a/osquery/examples/example_extension.cpp +++ b/osquery/examples/example_extension.cpp @@ -12,6 +12,14 @@ using namespace osquery; +class ExampleConfigPlugin : public ConfigPlugin { + public: + Status genConfig(std::map& config) { + config["data"] = "{\"options\": [], \"scheduledQueries\": []}"; + return Status(0, "OK"); + } +}; + class ExampleTable : public tables::TablePlugin { private: tables::TableColumns columns() const { @@ -30,10 +38,11 @@ class ExampleTable : public tables::TablePlugin { } }; -REGISTER(ExampleTable, "table", "example"); +REGISTER_EXTERNAL(ExampleConfigPlugin, "config", "example"); +REGISTER_EXTERNAL(ExampleTable, "table", "example"); int main(int argc, char* argv[]) { - initOsquery(argc, argv, OSQUERY_EXTENSION); + osquery::Initializer runner(argc, argv, OSQUERY_EXTENSION); auto status = startExtension("example", "0.0.1"); if (!status.ok()) { @@ -41,6 +50,6 @@ int main(int argc, char* argv[]) { } // Finally shutdown. - shutdownOsquery(); + runner.shutdown(); return 0; } diff --git a/osquery/examples/example_module.cpp b/osquery/examples/example_module.cpp new file mode 100644 index 0000000..4cdfad7 --- /dev/null +++ b/osquery/examples/example_module.cpp @@ -0,0 +1,44 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ + +#include + +using namespace osquery; + +class ExampleTable : public tables::TablePlugin { + private: + tables::TableColumns columns() const { + return {{"example_text", "TEXT"}, {"example_integer", "INTEGER"}}; + } + + QueryData generate(tables::QueryContext& request) { + QueryData results; + + Row r; + r["example_text"] = "example"; + r["example_integer"] = INTEGER(1); + + results.push_back(r); + return results; + } +}; + +// Create the module if the environment variable TESTFAIL1 is not defined. +// This allows the integration tests to, at run time, test the module +// loading workflow. +CREATE_MODULE_IF(getenv("TESTFAIL1") == nullptr, "example", "0.0.1", "0.0.0"); + +void initModule(void) { + // Register a plugin from a module if the environment variable TESTFAIL2 + // is not defined. + if (getenv("TESTFAIL2") == nullptr) { + REGISTER_MODULE(ExampleTable, "table", "example"); + } +} diff --git a/osquery/extensions/extensions.cpp b/osquery/extensions/extensions.cpp index a8b65eb..6a31571 100644 --- a/osquery/extensions/extensions.cpp +++ b/osquery/extensions/extensions.cpp @@ -10,6 +10,8 @@ #include +#include + #include #include #include @@ -17,20 +19,57 @@ #include #include "osquery/extensions/interface.h" +#include "osquery/core/watcher.h" using namespace osquery::extensions; -namespace osquery { - -const int kWatcherMLatency = 3000; - -FLAG(bool, disable_extensions, false, "Disable extension API"); +namespace fs = boost::filesystem; -FLAG(string, - extensions_socket, - "/var/osquery/osquery.em", - "Path to the extensions UNIX domain socket") +namespace osquery { +// Millisecond latency between initalizing manager pings. +const int kExtensionInitializeMLatency = 200; + +#ifdef __APPLE__ +const std::string kModuleExtension = ".dylib"; +#else +const std::string kModuleExtension = ".so"; +#endif + +CLI_FLAG(bool, disable_extensions, false, "Disable extension API"); + +CLI_FLAG(string, + extensions_socket, + "/var/osquery/osquery.em", + "Path to the extensions UNIX domain socket") + +CLI_FLAG(string, + extensions_autoload, + "/etc/osquery/extensions.load", + "Optional path to a list of autoloaded & managed extensions") + +CLI_FLAG(string, + extensions_timeout, + "3", + "Seconds to wait for autoloaded extensions"); + +CLI_FLAG(string, + extensions_interval, + "3", + "Seconds delay between connectivity checks") + +CLI_FLAG(string, + modules_autoload, + "/etc/osquery/modules.load", + "Optional path to a list of autoloaded registry modules") + +/// Alias the extensions_socket (used by core) to an alternate name reserved +/// for extension binaries +EXTENSION_FLAG_ALIAS(socket, extensions_socket); + +/// An extension manager may not be immediately available. +EXTENSION_FLAG_ALIAS(timeout, extensions_timeout); +EXTENSION_FLAG_ALIAS(interval, extensions_interval); void ExtensionWatcher::enter() { // Watch the manager, if the socket is removed then the extension will die. while (true) { @@ -84,6 +123,90 @@ void ExtensionManagerWatcher::watch() { } } +void loadExtensions() { + // Optionally autoload extensions + auto status = loadExtensions(FLAGS_extensions_autoload); + if (!status.ok()) { + LOG(WARNING) << "Could not autoload extensions: " << status.what(); + } +} + +void loadModules() { + auto status = loadModules(FLAGS_modules_autoload); + if (!status.ok()) { + LOG(WARNING) << "Modules autoload contains invalid paths"; + } +} + +Status loadExtensions(const std::string& loadfile) { + std::string autoload_paths; + if (readFile(loadfile, autoload_paths).ok()) { + for (auto& path : osquery::split(autoload_paths, "\n")) { + boost::trim(path); + if (path.size() > 0 && path[0] != '#' && path[0] != ';') { + Watcher::addExtensionPath(path); + } + } + return Status(0, "OK"); + } + return Status(1, "Cannot read extensions autoload file"); +} + +Status loadModuleFile(const std::string& path) { + fs::path module(path); + if (safePermissions(module.parent_path().string(), path)) { + if (module.extension().string() == kModuleExtension) { + // Silently allow module load failures to drop. + RegistryModuleLoader loader(module.string()); + loader.init(); + return Status(0, "OK"); + } + } + return Status(1, "Module check failed"); +} + +Status loadModules(const std::string& loadfile) { + // Split the search path for modules using a ':' delimiter. + std::string autoload_paths; + if (readFile(loadfile, autoload_paths).ok()) { + auto status = Status(0, "OK"); + for (auto& module_path : osquery::split(autoload_paths, "\n")) { + boost::trim(module_path); + auto path_status = loadModuleFile(module_path); + if (!path_status.ok()) { + status = path_status; + } + } + // Return an aggregate failure if any load fails (invalid search path). + return status; + } + return Status(1, "Cannot read modules autoload file"); +} + +Status extensionPathActive(const std::string& path, bool use_timeout = false) { + // Make sure the extension manager path exists, and is writable. + size_t delay = 0; + size_t timeout = atoi(FLAGS_extensions_timeout.c_str()); + do { + if (pathExists(path) && isWritable(path)) { + try { + auto client = EXManagerClient(path); + return Status(0, "OK"); + } catch (const std::exception& e) { + // Path might exist without a connected extension or extension manager. + } + } + // Only check active once if this check does not allow a timeout. + if (!use_timeout || timeout == 0) { + break; + } + // Increase the total wait detail. + delay += kExtensionInitializeMLatency; + ::usleep(kExtensionInitializeMLatency * 1000); + } while (delay < timeout * 1000); + return Status(1, "Extension socket not available: " + path); +} + Status startExtension(const std::string& name, const std::string& version) { return startExtension(name, version, "0.0.0"); } @@ -91,12 +214,8 @@ Status startExtension(const std::string& name, const std::string& version) { Status startExtension(const std::string& name, const std::string& version, const std::string& min_sdk_version) { - // No assumptions about how the extensions logs, the first action is to - // start the extension's registry. - Registry::setUp(); - - auto status = - startExtensionWatcher(FLAGS_extensions_socket, kWatcherMLatency, true); + auto latency = atoi(FLAGS_extensions_interval.c_str()) * 1000; + auto status = startExtensionWatcher(FLAGS_extensions_socket, latency, true); if (!status.ok()) { // If the threaded watcher fails to start, fail the extension. return status; @@ -113,6 +232,8 @@ Status startExtension(const std::string& name, } try { + // The extension does nothing but serve the thrift API. + // Join on both the thrift and extension manager watcher services. Dispatcher::joinServices(); } catch (const std::exception& e) { // The extension manager may shutdown without notifying the extension. @@ -120,7 +241,7 @@ Status startExtension(const std::string& name, } // An extension will only return on failure. - return Status(0, "OK"); + return Status(0, "Extension was shutdown"); } Status startExtension(const std::string& manager_path, @@ -129,35 +250,43 @@ Status startExtension(const std::string& manager_path, const std::string& min_sdk_version, const std::string& sdk_version) { // Make sure the extension manager path exists, and is writable. - if (!pathExists(manager_path) || !isWritable(manager_path)) { - return Status(1, "Extension manager socket not available: " + manager_path); + auto status = extensionPathActive(manager_path, true); + if (!status.ok()) { + printf("bad path active\n"); + return status; } // The Registry broadcast is used as the ExtensionRegistry. auto broadcast = Registry::getBroadcast(); - + // The extension will register and provide name, version, sdk details. InternalExtensionInfo info; info.name = name; info.version = version; info.sdk_version = sdk_version; info.min_sdk_version = min_sdk_version; + // If registration is successful, we will also request the manager's options. + InternalOptionList options; // Register the extension's registry broadcast with the manager. - ExtensionStatus status; + ExtensionStatus ext_status; try { auto client = EXManagerClient(manager_path); - client.get()->registerExtension(status, info, broadcast); - } - catch (const std::exception& e) { + client.get()->registerExtension(ext_status, info, broadcast); + // The main reason for a failed registry is a duplicate extension name + // (the extension process is already running), or the extension broadcasts + // a duplicate registry item. + if (ext_status.code != ExtensionCode::EXT_SUCCESS) { + return Status(ext_status.code, ext_status.message); + } + // Request the core options, mainly to set the active registry plugins for + // logger and config. + client.get()->options(options); + } catch (const std::exception& e) { return Status(1, "Extension register failed: " + std::string(e.what())); } - if (status.code != ExtensionCode::EXT_SUCCESS) { - return Status(status.code, status.message); - } - // Now that the uuid is known, try to clean up stale socket paths. - auto extension_path = getExtensionSocket(status.uuid, manager_path); + auto extension_path = getExtensionSocket(ext_status.uuid, manager_path); if (pathExists(extension_path).ok()) { if (!isWritable(extension_path).ok()) { return Status(1, "Cannot write extension socket: " + extension_path); @@ -168,20 +297,28 @@ Status startExtension(const std::string& manager_path, } } + // Set the active config and logger plugins. The core will arbitrate if the + // plugins are not available in the extension's local registry. + Registry::setActive("config", options["config_plugin"].value); + Registry::setActive("logger", options["logger_plugin"].value); + // Set up all lazy registry plugins and the active config/logger plugin. + Registry::setUp(); + // Start the extension's Thrift server Dispatcher::getInstance().addService( - std::make_shared(manager_path, status.uuid)); - VLOG(1) << "Extension (" << name << ", " << status.uuid << ", " << version + std::make_shared(manager_path, ext_status.uuid)); + VLOG(1) << "Extension (" << name << ", " << ext_status.uuid << ", " << version << ", " << sdk_version << ") registered"; - return Status(0, std::to_string(status.uuid)); + return Status(0, std::to_string(ext_status.uuid)); } Status queryExternal(const std::string& manager_path, const std::string& query, QueryData& results) { // Make sure the extension path exists, and is writable. - if (!pathExists(manager_path) || !isWritable(manager_path)) { - return Status(1, "Extension manager socket not available: " + manager_path); + auto status = extensionPathActive(manager_path); + if (!status.ok()) { + return status; } ExtensionResponse response; @@ -207,8 +344,9 @@ Status getQueryColumnsExternal(const std::string& manager_path, const std::string& query, tables::TableColumns& columns) { // Make sure the extension path exists, and is writable. - if (!pathExists(manager_path) || !isWritable(manager_path)) { - return Status(1, "Extension manager socket not available: " + manager_path); + auto status = extensionPathActive(manager_path); + if (!status.ok()) { + return status; } ExtensionResponse response; @@ -240,8 +378,9 @@ Status pingExtension(const std::string& path) { } // Make sure the extension path exists, and is writable. - if (!pathExists(path) || !isWritable(path)) { - return Status(1, "Extension socket not available: " + path); + auto status = extensionPathActive(path); + if (!status.ok()) { + return status; } ExtensionStatus ext_status; @@ -265,8 +404,9 @@ Status getExtensions(ExtensionList& extensions) { Status getExtensions(const std::string& manager_path, ExtensionList& extensions) { // Make sure the extension path exists, and is writable. - if (!pathExists(manager_path) || !isWritable(manager_path)) { - return Status(1, "Extension manager socket not available: " + manager_path); + auto status = extensionPathActive(manager_path); + if (!status.ok()) { + return status; } InternalExtensionList ext_list; @@ -309,8 +449,9 @@ Status callExtension(const std::string& extension_path, const PluginRequest& request, PluginResponse& response) { // Make sure the extension manager path exists, and is writable. - if (!pathExists(extension_path) || !isWritable(extension_path)) { - return Status(1, "Extension socket not available: " + extension_path); + auto status = extensionPathActive(extension_path); + if (!status.ok()) { + return status; } ExtensionResponse ext_response; @@ -335,8 +476,9 @@ Status startExtensionWatcher(const std::string& manager_path, size_t interval, bool fatal) { // Make sure the extension manager path exists, and is writable. - if (!pathExists(manager_path) || !isWritable(manager_path)) { - return Status(1, "Extension manager socket not available: " + manager_path); + auto status = extensionPathActive(manager_path, true); + if (!status.ok()) { + return status; } // Start a extension manager watcher, if the manager dies, so should we. @@ -364,10 +506,10 @@ Status startExtensionManager(const std::string& manager_path) { } } + auto latency = atoi(FLAGS_extensions_interval.c_str()) * 1000; // Start a extension manager watcher, if the manager dies, so should we. Dispatcher::getInstance().addService( - std::make_shared(manager_path, - kWatcherMLatency)); + std::make_shared(manager_path, latency)); // Start the extension manager thread. Dispatcher::getInstance().addService( diff --git a/osquery/extensions/extensions_tests.cpp b/osquery/extensions/extensions_tests.cpp index 2e0babc..507ab42 100644 --- a/osquery/extensions/extensions_tests.cpp +++ b/osquery/extensions/extensions_tests.cpp @@ -16,6 +16,7 @@ #include #include +#include "osquery/core/test_util.h" #include "osquery/extensions/interface.h" using namespace osquery::extensions; @@ -37,6 +38,7 @@ class ExtensionsTest : public testing::Test { void TearDown() { Dispatcher::getInstance().removeServices(); + Dispatcher::joinServices(); remove(kTestManagerSocket); } @@ -128,18 +130,11 @@ TEST_F(ExtensionsTest, test_extension_start) { Registry::allowDuplicates(true); status = startExtension(kTestManagerSocket, "test", "0.1", "0.0.0", "0.0.1"); // This will be false since we are registering duplicate items - EXPECT_TRUE(status.ok()); + ASSERT_TRUE(status.ok()); // The `startExtension` internal call (exposed for testing) returns the // uuid of the extension in the success status. - RouteUUID uuid; - try { - uuid = (RouteUUID)stoi(status.getMessage(), nullptr, 0); - } - catch (const std::exception& e) { - EXPECT_TRUE(false); - return; - } + RouteUUID uuid = (RouteUUID)stoi(status.getMessage(), nullptr, 0); // We can test-wait for the extensions's socket to open. EXPECT_TRUE(socketExists(kTestManagerSocket + "." + std::to_string(uuid))); @@ -155,7 +150,7 @@ class ExtensionPlugin : public Plugin { for (const auto& request_item : request) { response.push_back({{request_item.first, request_item.second}}); } - return Status(0, "Test sucess"); + return Status(0, "Test success"); } }; @@ -169,7 +164,7 @@ TEST_F(ExtensionsTest, test_extension_broadcast) { EXPECT_TRUE(socketExists(kTestManagerSocket)); // This time we're going to add a plugin to the extension_test registry. - REGISTER(TestExtensionPlugin, "extension_test", "test_item"); + Registry::add("extension_test", "test_item"); // Now we create a registry alias that will be broadcasted but NOT used for // internal call lookups. Aliasing was introduced for testing such that an @@ -235,6 +230,13 @@ TEST_F(ExtensionsTest, test_extension_broadcast) { Registry::removeBroadcast(uuid); Registry::allowDuplicates(false); } + +TEST_F(ExtensionsTest, test_extension_module_search) { + createMockFileStructure(); + EXPECT_TRUE(loadModules(kFakeDirectory)); + EXPECT_FALSE(loadModules("/dir/does/not/exist")); + tearDownMockFileStructure(); +} } int main(int argc, char* argv[]) { diff --git a/osquery/extensions/interface.cpp b/osquery/extensions/interface.cpp index 93aa882..7e4c1ee 100644 --- a/osquery/extensions/interface.cpp +++ b/osquery/extensions/interface.cpp @@ -36,6 +36,7 @@ void ExtensionHandler::call(ExtensionResponse& _return, PluginResponse response; PluginRequest plugin_request; for (const auto& request_item : request) { + // Create a PluginRequest from an ExtensionPluginRequest. plugin_request[request_item.first] = request_item.second; } @@ -46,11 +47,26 @@ void ExtensionHandler::call(ExtensionResponse& _return, if (status.ok()) { for (const auto& response_item : response) { + // Translate a PluginResponse to an ExtensionPluginResponse. _return.response.push_back(response_item); } } } +void ExtensionManagerHandler::extensions(InternalExtensionList& _return) { + refresh(); + _return = extensions_; +} + +void ExtensionManagerHandler::options(InternalOptionList& _return) { + auto flags = Flag::flags(); + for (const auto& flag : flags) { + _return[flag.first].value = flag.second.value; + _return[flag.first].default_value = flag.second.default_value; + _return[flag.first].type = flag.second.type; + } +} + void ExtensionManagerHandler::registerExtension( ExtensionStatus& _return, const InternalExtensionInfo& info, @@ -144,11 +160,6 @@ void ExtensionManagerHandler::refresh() { } } -void ExtensionManagerHandler::extensions(InternalExtensionList& _return) { - refresh(); - _return = extensions_; -} - bool ExtensionManagerHandler::exists(const std::string& name) { refresh(); diff --git a/osquery/extensions/interface.h b/osquery/extensions/interface.h index e3bc9c3..02f65c7 100644 --- a/osquery/extensions/interface.h +++ b/osquery/extensions/interface.h @@ -89,6 +89,21 @@ class ExtensionManagerHandler : virtual public ExtensionManagerIf, void extensions(InternalExtensionList& _return); /** + * @brief Return a map of osquery options (Flags, bootstrap CLI flags). + * + * osquery options are set via command line flags or overridden by a config + * options dictionary. There are some CLI-only flags that should never + * be overridden. If a bootstrap flag is changed there is undefined behavior + * since bootstrap candidates are settings needed before a configuration + * plugin is setUp. + * + * Extensions may broadcast config or logger plugins that need a snapshot + * of the current options. The best example is the `config_plugin` bootstrap + * flag. + */ + void options(InternalOptionList& _return); + + /** * @brief Request a Route UUID and advertise a set of Registry routes. * * When an Extension starts it must call registerExtension using a well known @@ -160,7 +175,9 @@ class ExtensionWatcher : public InternalRunnable { public: virtual ~ExtensionWatcher() {} ExtensionWatcher(const std::string& path, size_t interval, bool fatal) - : path_(path), interval_(interval), fatal_(fatal) {} + : path_(path), interval_(interval), fatal_(fatal) { + interval_ = (interval_ < 200) ? 200 : interval_; + } public: /// The Dispatcher thread entry point. diff --git a/osquery/filesystem/filesystem.cpp b/osquery/filesystem/filesystem.cpp index 0719738..1699bd5 100644 --- a/osquery/filesystem/filesystem.cpp +++ b/osquery/filesystem/filesystem.cpp @@ -10,7 +10,6 @@ #include #include -#include #include #include @@ -127,7 +126,10 @@ Status remove(const boost::filesystem::path& path) { } Status listFilesInDirectory(const boost::filesystem::path& path, - std::vector& results) { + std::vector& results, + bool ignore_error) { + boost::filesystem::directory_iterator begin_iter; + try { if (!boost::filesystem::exists(path)) { return Status(1, "Directory not found: " + path.string()); @@ -136,23 +138,31 @@ Status listFilesInDirectory(const boost::filesystem::path& path, if (!boost::filesystem::is_directory(path)) { return Status(1, "Supplied path is not a directory: " + path.string()); } + begin_iter = boost::filesystem::directory_iterator(path); + + } catch (const boost::filesystem::filesystem_error& e) { + return Status(1, e.what()); + } - boost::filesystem::directory_iterator begin_iter(path); - boost::filesystem::directory_iterator end_iter; - for (; begin_iter != end_iter; begin_iter++) { - if (!boost::filesystem::is_directory(begin_iter->path())) { + boost::filesystem::directory_iterator end_iter; + for (; begin_iter != end_iter; begin_iter++) { + try { + if (boost::filesystem::is_regular_file(begin_iter->path())) { results.push_back(begin_iter->path().string()); } + } catch (const boost::filesystem::filesystem_error& e) { + if (ignore_error == 0) { + return Status(1, e.what()); + } } - - return Status(0, "OK"); - } catch (const boost::filesystem::filesystem_error& e) { - return Status(1, e.what()); } + return Status(0, "OK"); } Status listDirectoriesInDirectory(const boost::filesystem::path& path, - std::vector& results) { + std::vector& results, + bool ignore_error) { + boost::filesystem::directory_iterator begin_iter; try { if (!boost::filesystem::exists(path)) { return Status(1, "Directory not found"); @@ -167,19 +177,24 @@ Status listDirectoriesInDirectory(const boost::filesystem::path& path, if (!stat.ok()) { return stat; } + begin_iter = boost::filesystem::directory_iterator(path); + } catch (const boost::filesystem::filesystem_error& e) { + return Status(1, e.what()); + } - boost::filesystem::directory_iterator begin_iter(path); - boost::filesystem::directory_iterator end_iter; - for (; begin_iter != end_iter; begin_iter++) { + boost::filesystem::directory_iterator end_iter; + for (; begin_iter != end_iter; begin_iter++) { + try { if (boost::filesystem::is_directory(begin_iter->path())) { results.push_back(begin_iter->path().string()); } + } catch (const boost::filesystem::filesystem_error& e) { + if (ignore_error == 0) { + return Status(1, e.what()); + } } - - return Status(0, "OK"); - } catch (const boost::filesystem::filesystem_error& e) { - return Status(1, e.what()); } + return Status(0, "OK"); } /** @@ -198,27 +213,33 @@ Status listDirectoriesInDirectory(const boost::filesystem::path& path, */ Status doubleStarTraversal(const boost::filesystem::path& fs_path, std::vector& results, + ReturnSetting setting, unsigned int rec_depth) { if (rec_depth >= kMaxDirectoryTraversalDepth) { return Status(2, fs_path.string().c_str()); } // List files first - Status stat = listFilesInDirectory(fs_path, results); - if (!stat.ok()) { - return Status(0, "OK"); + if (setting & REC_LIST_FILES) { + Status stat = listFilesInDirectory(fs_path, results); + if (!stat.ok()) { + return Status(0, "OK"); + } } std::vector folders; - stat = listDirectoriesInDirectory(fs_path, folders); + Status stat = listDirectoriesInDirectory(fs_path, folders); if (!stat.ok()) { return Status(0, "OK"); } - + if (setting & REC_LIST_FOLDERS) { + results.push_back(fs_path.string()); + } for (const auto& folder : folders) { boost::filesystem::path p(folder); if (boost::filesystem::is_symlink(p)) { continue; } - stat = doubleStarTraversal(folder, results, rec_depth + 1); + + stat = doubleStarTraversal(folder, results, setting, rec_depth + 1); if (!stat.ok() && stat.getCode() == 2) { return stat; } @@ -245,18 +266,26 @@ Status doubleStarTraversal(const boost::filesystem::path& fs_path, */ Status resolveLastPathComponent(const boost::filesystem::path& fs_path, std::vector& results, + ReturnSetting setting, const std::vector& components, unsigned int rec_depth) { + // Is the last component a double star? if (components[components.size() - 1] == kWildcardCharacterRecursive) { - Status stat = - doubleStarTraversal(fs_path.parent_path(), results, rec_depth); - return stat; + if (setting & REC_EVENT_OPT) { + results.push_back(fs_path.parent_path().string()); + return Status(0, "OK"); + } else { + Status stat = doubleStarTraversal( + fs_path.parent_path(), results, setting, rec_depth); + return stat; + } } // Is the path a file try { - if (boost::filesystem::is_regular_file(fs_path)) { + if (setting == REC_LIST_FILES && + boost::filesystem::is_regular_file(fs_path)) { results.push_back(fs_path.string()); return Status(0, "OK"); } @@ -266,15 +295,27 @@ Status resolveLastPathComponent(const boost::filesystem::path& fs_path, } std::vector files; - Status stat = listFilesInDirectory(fs_path.parent_path(), files); - if (!stat.ok()) { - return stat; - } + std::vector folders; + Status stat_file = listFilesInDirectory(fs_path.parent_path(), files); + Status stat_fold = listDirectoriesInDirectory(fs_path.parent_path(), folders); // Is the last component a wildcard? if (components[components.size() - 1] == kWildcardCharacter) { - for (const auto& file : files) { - results.push_back(file); + + if (setting & REC_EVENT_OPT) { + results.push_back(fs_path.parent_path().string()); + return Status(0, "OK"); + } + if (setting & REC_LIST_FOLDERS) { + results.push_back(fs_path.parent_path().string()); + for (const auto& fold : folders) { + results.push_back(fold); + } + } + if (setting & REC_LIST_FILES) { + for (const auto& file : files) { + results.push_back(file); + } } return Status(0, "OK"); } @@ -285,7 +326,7 @@ Status resolveLastPathComponent(const boost::filesystem::path& fs_path, std::vector(components.begin(), components.end() - 1), "/"); - // Is this a .*% type file match + // Is this a (.*)% type file match if (components[components.size() - 1].find(kWildcardCharacter, 1) != std::string::npos && components[components.size() - 1][0] != kWildcardCharacter[0]) { @@ -294,25 +335,49 @@ Status resolveLastPathComponent(const boost::filesystem::path& fs_path, processed_path + "/" + components[components.size() - 1].substr( 0, components[components.size() - 1].find(kWildcardCharacter, 1)); - for (const auto& file : files) { - if (file.find(prefix, 0) != 0) { - continue; + if (setting & REC_LIST_FOLDERS) { + for (const auto& fold : folders) { + if (fold.find(prefix, 0) != 0) { + continue; + } + results.push_back(fold); + } + } + if (setting & REC_LIST_FILES || setting & REC_EVENT_OPT) { + for (const auto& file : files) { + if (file.find(prefix, 0) != 0) { + continue; + } + results.push_back(file); } - results.push_back(file); } + // Should be a return here? + return Status(0, "OK"); } // Is this a %(.*) type file match if (components[components.size() - 1][0] == kWildcardCharacter[0]) { std::string suffix = components[components.size() - 1].substr(1); - - for (const auto& file : files) { - boost::filesystem::path p(file); - std::string file_name = p.filename().string(); - size_t pos = file_name.find(suffix); - if (pos != std::string::npos && - pos + suffix.length() == file_name.length()) { - results.push_back(file); + if (setting & REC_LIST_FOLDERS) { + for (const auto& fold : folders) { + std::string file_name = + boost::filesystem::path(fold).filename().string(); + size_t pos = file_name.find(suffix); + if (pos != std::string::npos && + pos + suffix.length() == file_name.length()) { + results.push_back(fold); + } + } + } + if (setting & REC_LIST_FILES || setting & REC_EVENT_OPT) { + for (const auto& file : files) { + boost::filesystem::path p(file); + std::string file_name = p.filename().string(); + size_t pos = file_name.find(suffix); + if (pos != std::string::npos && + pos + suffix.length() == file_name.length()) { + results.push_back(file); + } } } return Status(0, "OK"); @@ -325,9 +390,7 @@ Status resolveLastPathComponent(const boost::filesystem::path& fs_path, // Is the path a directory if (boost::filesystem::is_directory(fs_path)) { - for (auto& file : files) { - results.push_back(file); - } + results.push_back(fs_path.string()); return Status(0, "OK"); } @@ -351,6 +414,7 @@ Status resolveLastPathComponent(const boost::filesystem::path& fs_path, */ Status resolveFilePattern(std::vector components, std::vector& results, + ReturnSetting setting = REC_LIST_FILES, unsigned int processed_index = 0, unsigned int rec_depth = 0) { @@ -391,8 +455,8 @@ Status resolveFilePattern(std::vector components, for (const auto& dir : folders) { boost::filesystem::path p(dir); components[i] = p.filename().string(); - Status stat = - resolveFilePattern(components, results, i + 1, rec_depth + 1); + Status stat = resolveFilePattern( + components, results, setting, i + 1, rec_depth + 1); if (!stat.ok() && stat.getCode() == 2) { return stat; } @@ -412,8 +476,8 @@ Status resolveFilePattern(std::vector components, } boost::filesystem::path p(dir); components[i] = p.filename().string(); - Status stat = - resolveFilePattern(components, results, i + 1, rec_depth + 1); + Status stat = resolveFilePattern( + components, results, setting, i + 1, rec_depth + 1); if (!stat.ok() && stat.getCode() == 2) { return stat; } @@ -429,8 +493,8 @@ Status resolveFilePattern(std::vector components, if (pos != std::string::npos && pos + suffix.length() == folder_name.length()) { components[i] = p.filename().string(); - Status stat = - resolveFilePattern(components, results, i + 1, rec_depth + 1); + Status stat = resolveFilePattern( + components, results, setting, i + 1, rec_depth + 1); if (!stat.ok() && stat.getCode() == 2) { return stat; } @@ -445,15 +509,35 @@ Status resolveFilePattern(std::vector components, // list the files at this point or do our ** traversal return resolveLastPathComponent("/" + boost::algorithm::join(components, "/"), results, + setting, components, rec_depth); } Status resolveFilePattern(const boost::filesystem::path& fs_path, std::vector& results) { + if (fs_path.string()[0] != '/') { + return resolveFilePattern(split(boost::filesystem::current_path().string() + + "/" + fs_path.string(), + "/"), + results); + } return resolveFilePattern(split(fs_path.string(), "/"), results); } +Status resolveFilePattern(const boost::filesystem::path& fs_path, + std::vector& results, + ReturnSetting setting) { + if (fs_path.string()[0] != '/') { + return resolveFilePattern(split(boost::filesystem::current_path().string() + + "/" + fs_path.string(), + "/"), + results, + setting); + } + return resolveFilePattern(split(fs_path.string(), "/"), results, setting); +} + Status getDirectory(const boost::filesystem::path& path, boost::filesystem::path& dirpath) { if (!isDirectory(path).ok()) { @@ -465,27 +549,57 @@ Status getDirectory(const boost::filesystem::path& path, } Status isDirectory(const boost::filesystem::path& path) { - if (boost::filesystem::is_directory(path)) { - return Status(0, "OK"); + try { + if (boost::filesystem::is_directory(path)) { + return Status(0, "OK"); + } + return Status(1, "Path is not a directory: " + path.string()); + } catch (const boost::filesystem::filesystem_error& e) { + return Status(1, e.what()); } - return Status(1, "Path is not a directory: " + path.string()); } -std::vector getHomeDirectories() { - auto sql = SQL( - "SELECT DISTINCT directory FROM users WHERE directory != '/var/empty';"); - std::vector results; - if (sql.ok()) { - for (const auto& row : sql.rows()) { - results.push_back(row.at("directory")); +std::set getHomeDirectories() { + std::set results; + + auto users = SQL::selectAllFrom("users"); + for (const auto& user : users) { + if (user.at("directory").size() > 0) { + results.insert(user.at("directory")); } - } else { - LOG(ERROR) - << "Error executing query to return users: " << sql.getMessageString(); } + return results; } +bool safePermissions(const std::string& dir, + const std::string& path, + bool executable) { + struct stat file_stat, link_stat, dir_stat; + if (lstat(path.c_str(), &link_stat) < 0 || stat(path.c_str(), &file_stat) || + stat(dir.c_str(), &dir_stat)) { + // Path was not real, had too may links, or could not be accessed. + return false; + } + + if (dir_stat.st_mode & (1 << 9)) { + // Do not load modules from /tmp-like directories. + return false; + } else if (S_ISDIR(file_stat.st_mode)) { + // Only load file-like nodes (not directories). + return false; + } else if (file_stat.st_uid == getuid() || file_stat.st_uid == 0) { + // Otherwise, require matching or root file ownership. + if (executable && !file_stat.st_mode & S_IXUSR) { + // Require executable, implies by the owner. + return false; + } + return true; + } + // Do not load modules not owned by the user. + return false; +} + std::string lsperms(int mode) { static const char rwx[] = {'0', '1', '2', '3', '4', '5', '6', '7'}; std::string bits; diff --git a/osquery/filesystem/filesystem_tests.cpp b/osquery/filesystem/filesystem_tests.cpp index cde162a..3706894 100644 --- a/osquery/filesystem/filesystem_tests.cpp +++ b/osquery/filesystem/filesystem_tests.cpp @@ -53,10 +53,16 @@ TEST_F(FilesystemTests, test_list_files_in_directory_not_found) { EXPECT_FALSE(not_found.ok()); EXPECT_EQ(not_found.toString(), "Directory not found: /foo/bar"); } -TEST_F(FilesystemTests, test_wildcard_single_folder_list) { + +TEST_F(FilesystemTests, test_wildcard_single_file_list) { std::vector files; + std::vector files_flag; auto status = resolveFilePattern(kFakeDirectory + "/%", files); + auto status2 = + resolveFilePattern(kFakeDirectory + "/%", files_flag, REC_LIST_FILES); EXPECT_TRUE(status.ok()); + EXPECT_EQ(files.size(), 3); + EXPECT_EQ(files.size(), files_flag.size()); EXPECT_NE( std::find(files.begin(), files.end(), kFakeDirectory + "/roto.txt"), files.end()); @@ -132,6 +138,110 @@ TEST_F(FilesystemTests, test_list_files_in_directorty) { EXPECT_NE(std::find(results.begin(), results.end(), "/etc/hosts"), results.end()); } + +TEST_F(FilesystemTests, test_wildcard_single_folder_list) { + std::vector folders; + auto status = + resolveFilePattern(kFakeDirectory + "/%", folders, REC_LIST_FOLDERS); + EXPECT_TRUE(status.ok()); + EXPECT_EQ(folders.size(), 3); + EXPECT_NE( + std::find(folders.begin(), folders.end(), kFakeDirectory + "/deep11"), + folders.end()); +} + +TEST_F(FilesystemTests, test_wildcard_single_all_list) { + std::vector all; + auto status = resolveFilePattern(kFakeDirectory + "/%", all, REC_LIST_ALL); + EXPECT_TRUE(status.ok()); + EXPECT_EQ(all.size(), 6); + EXPECT_NE(std::find(all.begin(), all.end(), kFakeDirectory + "/roto.txt"), + all.end()); + EXPECT_NE(std::find(all.begin(), all.end(), kFakeDirectory + "/deep11"), + all.end()); +} + +TEST_F(FilesystemTests, test_wildcard_double_folders) { + std::vector all; + auto status = + resolveFilePattern(kFakeDirectory + "/%%", all, REC_LIST_FOLDERS); + EXPECT_TRUE(status.ok()); + EXPECT_EQ(all.size(), 6); + EXPECT_NE(std::find(all.begin(), all.end(), kFakeDirectory), all.end()); + EXPECT_NE( + std::find(all.begin(), all.end(), kFakeDirectory + "/deep11/deep2/deep3"), + all.end()); +} + +TEST_F(FilesystemTests, test_wildcard_double_all) { + std::vector all; + auto status = resolveFilePattern(kFakeDirectory + "/%%", all, REC_LIST_ALL); + EXPECT_TRUE(status.ok()); + EXPECT_EQ(all.size(), 15); + EXPECT_NE(std::find(all.begin(), all.end(), kFakeDirectory + "/roto.txt"), + all.end()); + EXPECT_NE( + std::find(all.begin(), all.end(), kFakeDirectory + "/deep11/deep2/deep3"), + all.end()); +} +TEST_F(FilesystemTests, test_double_wild_event_opt) { + std::vector all; + auto status = resolveFilePattern(kFakeDirectory + "/%%", all, + REC_LIST_FOLDERS | REC_EVENT_OPT); + EXPECT_TRUE(status.ok()); + EXPECT_EQ(all.size(), 1); + EXPECT_NE(std::find(all.begin(), all.end(), kFakeDirectory), all.end()); +} + +TEST_F(FilesystemTests, test_letter_wild_opt) { + std::vector all; + auto status = resolveFilePattern(kFakeDirectory + "/d%", all, + REC_LIST_FOLDERS | REC_EVENT_OPT); + EXPECT_TRUE(status.ok()); + EXPECT_EQ(all.size(), 3); + EXPECT_NE(std::find(all.begin(), all.end(), kFakeDirectory + "/deep1"), + all.end()); + EXPECT_NE(std::find(all.begin(), all.end(), kFakeDirectory + "/door.txt"), + all.end()); +} + +TEST_F(FilesystemTests, test_dotdot) { + std::vector all; + auto status = resolveFilePattern(kFakeDirectory + "/deep11/deep2/../../%", + all, REC_LIST_FILES); + EXPECT_TRUE(status.ok()); + EXPECT_EQ(all.size(), 3); + EXPECT_NE(std::find(all.begin(), all.end(), + kFakeDirectory + "/deep11/deep2/../../door.txt"), + all.end()); +} + +TEST_F(FilesystemTests, test_dotdot_relative) { + std::vector all; + auto status = + resolveFilePattern("../../../tools/tests/%", all, REC_LIST_ALL); + EXPECT_TRUE(status.ok()); + + bool found = false; + for (const auto& file : all) { + if (file.find("test.config")) { + found = true; + break; + } + } + EXPECT_TRUE(found); +} + +TEST_F(FilesystemTests, test_safe_permissions) { + // For testing we can request a different directory path. + EXPECT_TRUE(safePermissions("/", kFakeDirectory + "/door.txt")); + // A file with a directory.mode & 0x1000 fails. + EXPECT_FALSE(safePermissions("/tmp", kFakeDirectory + "/door.txt")); + // A directory for a file will fail. + EXPECT_FALSE(safePermissions("/", kFakeDirectory + "/deep11")); + // A root-owned file is appropriate + EXPECT_TRUE(safePermissions("/", "/dev/zero")); +} } int main(int argc, char* argv[]) { diff --git a/osquery/logger/logger.cpp b/osquery/logger/logger.cpp index 2d70dff..de91f14 100644 --- a/osquery/logger/logger.cpp +++ b/osquery/logger/logger.cpp @@ -27,7 +27,7 @@ FLAG_ALIAS(bool, debug, verbose); FLAG(bool, disable_logging, false, "Disable ERROR/INFO logging"); -FLAG(string, logger_plugin, "filesystem", "The default logger plugin"); +FLAG(string, logger_plugin, "filesystem", "Logger plugin name"); FLAG(bool, log_result_events, true, "Log scheduled results as events"); @@ -190,19 +190,21 @@ void initLogger(const std::string& name, bool forward_all) { BufferedLogSink::disable(); auto intermediate_logs = std::move(BufferedLogSink::dump()); + auto& logger_plugin = Registry::getActive("logger"); + if (!Registry::exists("logger", logger_plugin)) { + return; + } - if (Registry::exists("logger", FLAGS_logger_plugin)) { - // Start the custom status logging facilities, which may instruct glog as is - // the case with filesystem logging. - PluginRequest request = {{"init", name}}; - serializeIntermediateLog(intermediate_logs, request); - auto status = Registry::call("logger", FLAGS_logger_plugin, request); - if (status.ok() || forward_all) { - // When init returns success we reenabled the log sink in forwarding - // mode. Now, Glog status logs are buffered and sent to logStatus. - BufferedLogSink::forward(true); - BufferedLogSink::enable(); - } + // Start the custom status logging facilities, which may instruct glog as is + // the case with filesystem logging. + PluginRequest request = {{"init", name}}; + serializeIntermediateLog(intermediate_logs, request); + auto status = Registry::call("logger", request); + if (status.ok() || forward_all) { + // When init returns success we reenabled the log sink in forwarding + // mode. Now, Glog status logs are buffered and sent to logStatus. + BufferedLogSink::forward(true); + BufferedLogSink::enable(); } } @@ -223,7 +225,7 @@ void BufferedLogSink::send(google::LogSeverity severity, std::string(message, message_len)}); PluginRequest request = {{"status", "true"}}; serializeIntermediateLog(log, request); - Registry::call("logger", FLAGS_logger_plugin, request); + Registry::call("logger", request); } else { logs_.push_back({(StatusLogSeverity)severity, std::string(base_filename), @@ -249,7 +251,7 @@ Status LoggerPlugin::call(const PluginRequest& request, } Status logString(const std::string& s) { - return logString(s, FLAGS_logger_plugin); + return logString(s, Registry::getActive("logger")); } Status logString(const std::string& s, const std::string& receiver) { @@ -263,7 +265,7 @@ Status logString(const std::string& s, const std::string& receiver) { } Status logScheduledQueryLogItem(const osquery::ScheduledQueryLogItem& results) { - return logScheduledQueryLogItem(results, FLAGS_logger_plugin); + return logScheduledQueryLogItem(results, Registry::getActive("logger")); } Status logScheduledQueryLogItem(const osquery::ScheduledQueryLogItem& results, diff --git a/osquery/logger/logger_tests.cpp b/osquery/logger/logger_tests.cpp index c95718c..4371561 100644 --- a/osquery/logger/logger_tests.cpp +++ b/osquery/logger/logger_tests.cpp @@ -90,8 +90,8 @@ TEST_F(LoggerTests, test_plugin) { TEST_F(LoggerTests, test_logger_init) { // Expect the logger to have been registered from the first test. EXPECT_TRUE(Registry::exists("logger", "test")); + EXPECT_TRUE(Registry::setActive("logger", "test").ok()); - FLAGS_logger_plugin = "test"; initStatusLogger("logger_test"); // This will be printed to stdout. LOG(WARNING) << "Logger test is generating a warning status (1)"; diff --git a/osquery/main/daemon.cpp b/osquery/main/daemon.cpp index 5000ce8..cd01a2d 100644 --- a/osquery/main/daemon.cpp +++ b/osquery/main/daemon.cpp @@ -10,40 +10,31 @@ #include -#include #include -#include -#include -#include #include -#include "osquery/core/watcher.h" - const std::string kWatcherWorkerName = "osqueryd: worker"; int main(int argc, char* argv[]) { - osquery::initOsquery(argc, argv, osquery::OSQUERY_TOOL_DAEMON); + osquery::Initializer runner(argc, argv, osquery::OSQUERY_TOOL_DAEMON); - if (!osquery::isOsqueryWorker()) { - osquery::initOsqueryDaemon(); + if (!runner.isWorker()) { + runner.initDaemon(); } - if (!osquery::FLAGS_disable_watchdog) { - // When a watcher is used, the current watcher will fork into a worker. - osquery::initWorkerWatcher(kWatcherWorkerName, argc, argv); - } + // When a watchdog is used, the current daemon will fork/exec into a worker. + // In either case the watcher may start optionally loaded extensions. + runner.initWorkerWatcher(kWatcherWorkerName); - // Start event threads. - osquery::attachEvents(); - osquery::EventFactory::delay(); - osquery::startExtensionManager(); + // Start osquery work. + runner.start(); // Begin the schedule runloop. boost::thread scheduler_thread(osquery::initializeScheduler); scheduler_thread.join(); // Finally shutdown. - osquery::shutdownOsquery(); + runner.shutdown(); return 0; } diff --git a/osquery/main/shell.cpp b/osquery/main/shell.cpp index f7df57e..cc506e6 100644 --- a/osquery/main/shell.cpp +++ b/osquery/main/shell.cpp @@ -8,39 +8,19 @@ * */ -#include #include -#include #include -#include -#include -#include -#include - -const std::string kShellTemp = "/tmp/osquery"; int main(int argc, char *argv[]) { - // The shell is transient, rewrite config-loaded paths. - if (osquery::pathExists(kShellTemp).ok() || - boost::filesystem::create_directory(kShellTemp)) { - osquery::FLAGS_db_path = kShellTemp + "/shell.db"; - osquery::FLAGS_extensions_socket = kShellTemp + "/shell.em"; - FLAGS_log_dir = kShellTemp; - } - // Parse/apply flags, start registry, load logger/config plugins. - osquery::initOsquery(argc, argv, osquery::OSQUERY_TOOL_SHELL); - - // Start event threads. - osquery::attachEvents(); - osquery::EventFactory::delay(); - osquery::startExtensionManager(); + osquery::Initializer runner(argc, argv, osquery::OSQUERY_TOOL_SHELL); + runner.start(); // Virtual tables will be attached to the shell's in-memory SQLite DB. int retcode = osquery::launchIntoShell(argc, argv); // Finally shutdown. - osquery::shutdownOsquery(); + runner.shutdown(); return retcode; } diff --git a/osquery/registry/registry.cpp b/osquery/registry/registry.cpp index 37538d0..f6eda6c 100644 --- a/osquery/registry/registry.cpp +++ b/osquery/registry/registry.cpp @@ -11,6 +11,8 @@ #include #include +#include + #include #include @@ -38,11 +40,28 @@ void RegistryHelperCore::remove(const std::string& item_name) { } } +bool RegistryHelperCore::isInternal(const std::string& item_name) const { + if (std::find(internal_.begin(), internal_.end(), item_name) == + internal_.end()) { + return false; + } + return true; +} + +Status RegistryHelperCore::setActive(const std::string& item_name) { + if (items_.count(item_name) == 0 && external_.count(item_name) == 0) { + return Status(1, "Unknown registry item"); + } + active_ = item_name; + return Status(0, "OK"); +} + +const std::string& RegistryHelperCore::getActive() const { return active_; } + RegistryRoutes RegistryHelperCore::getRoutes() const { RegistryRoutes route_table; for (const auto& item : items_) { - if (std::find(internal_.begin(), internal_.end(), item.first) != - internal_.end()) { + if (isInternal(item.first)) { // This is an internal plugin, do not include the route. continue; } @@ -102,6 +121,13 @@ const std::string& RegistryHelperCore::getAlias( } void RegistryHelperCore::setUp() { + // If the registry is using a single 'active' plugin, setUp that plugin. + // For config and logger, only setUp the selected plugin. + if (active_.size() != 0 && exists(active_, true)) { + items_.at(active_)->setUp(); + return; + } + // If this registry does not auto-setup do NOT setup the registry items. if (!auto_setup_) { return; @@ -206,6 +232,11 @@ Status RegistryFactory::addBroadcast(const RouteUUID& uuid, // If any registry fails to add the set of external routes, stop. break; } + + for (const auto& plugin : registry.second) { + VLOG(1) << "Extension " << uuid << " registered " << registry.first + << " plugin " << plugin.first; + } } // If any registry failed, remove each (assume a broadcast is atomic). @@ -270,6 +301,32 @@ Status RegistryFactory::call(const std::string& registry_name, return call(registry_name, item_name, request, response); } +Status RegistryFactory::call(const std::string& registry_name, + const PluginRequest& request, + PluginResponse& response) { + auto& plugin = registry(registry_name)->getActive(); + return registry(registry_name)->call(plugin, request, response); +} + +Status RegistryFactory::call(const std::string& registry_name, + const PluginRequest& request) { + PluginResponse response; + return call(registry_name, request, response); +} + +Status RegistryFactory::setActive(const std::string& registry_name, + const std::string& item_name) { + if (!exists(registry_name, item_name)) { + return Status(1, "Registry plugin does not exist"); + } + return registry(registry_name)->setActive(item_name); +} + +const std::string& RegistryFactory::getActive( + const std::string& registry_name) { + return registry(registry_name)->getActive(); +} + void RegistryFactory::setUp() { for (const auto& registry : instance().all()) { registry.second->setUp(); @@ -287,6 +344,14 @@ bool RegistryFactory::exists(const std::string& registry_name, return registry(registry_name)->exists(item_name, local); } +std::vector RegistryFactory::names() { + std::vector names; + for (const auto& registry : all()) { + names.push_back(registry.second->getName()); + } + return names; +} + std::vector RegistryFactory::names( const std::string& registry_name) { if (instance().registries_.at(registry_name) == 0) { @@ -313,6 +378,116 @@ size_t RegistryFactory::count(const std::string& registry_name) { return instance().registry(registry_name)->count(); } +Status RegistryHelperCore::add(const std::string& item_name, bool internal) { + // The item can be listed as internal, meaning it does not broadcast. + if (internal) { + internal_.push_back(item_name); + } + + // The item may belong to a module. + if (RegistryFactory::usingModule()) { + modules_[item_name] = RegistryFactory::getModule(); + } + + return Status(0, "OK"); +} + +const std::map& RegistryFactory::getModules() { + return instance().modules_; +} + +RouteUUID RegistryFactory::getModule() { return instance().module_uuid_; } + +bool RegistryFactory::usingModule() { + // Check if the registry is allowing a module's registrations. + return (!instance().locked() && instance().module_uuid_ != 0); +} + +void RegistryFactory::shutdownModule() { + instance().locked(true); + instance().module_uuid_ = 0; +} + +void RegistryFactory::initModule(const std::string& path) { + // Begin a module initialization, lock until the module is determined + // appropriate by requesting a call to `declareModule`. + instance().module_uuid_ = (RouteUUID)rand(); + instance().modules_[getModule()].path = path; + instance().locked(true); +} + +void RegistryFactory::declareModule(const std::string& name, + const std::string& version, + const std::string& min_sdk_version, + const std::string& sdk_version) { + // Check the min_sdk_version against the Registry's SDK version. + auto& module = instance().modules_[instance().module_uuid_]; + module.name = name; + module.version = version; + module.sdk_version = sdk_version; + instance().locked(false); +} + +RegistryModuleLoader::RegistryModuleLoader(const std::string& path) + : handle_(nullptr), path_(path) { + // Tell the registry that we are attempting to construct a module. + // Locking the registry prevents the module's global initialization from + // adding or creating registry items. + RegistryFactory::initModule(path_); + handle_ = dlopen(path_.c_str(), RTLD_NOW | RTLD_LOCAL); + if (handle_ == nullptr) { + VLOG(1) << "Failed to load module: " << path_; + VLOG(1) << dlerror(); + return; + } + + // The module should have called RegistryFactory::declareModule and unlocked + // the registry for modification. The module should have done this using + // the SDK's CREATE_MODULE macro, which adds the global-scope constructor. + if (RegistryFactory::locked()) { + VLOG(1) << "Failed to declare module: " << path_; + dlclose(handle_); + handle_ = nullptr; + } +} + +void RegistryModuleLoader::init() { + if (handle_ == nullptr || RegistryFactory::locked()) { + handle_ = nullptr; + return; + } + + // Locate a well-known symbol in the module. + // This symbol name is protected against rewriting when the module uses the + // SDK's CREATE_MODULE macro. + auto initializer = (ModuleInitalizer)dlsym(handle_, "initModule"); + if (initializer != nullptr) { + initializer(); + VLOG(1) << "Initialized module: " << path_; + } else { + VLOG(1) << "Failed to initialize module: " << path_; + VLOG(1) << dlerror(); + dlclose(handle_); + handle_ = nullptr; + } +} + +RegistryModuleLoader::~RegistryModuleLoader() { + if (handle_ == nullptr) { + // The module was not loaded or did not initalize. + RegistryFactory::instance().modules_.erase(RegistryFactory::getModule()); + } + + // We do not close the module, and thus are OK with losing a reference to the + // module's handle. Attempting to close and clean up is very expensive for + // very little value/features. + if (!RegistryFactory::locked()) { + RegistryFactory::shutdownModule(); + } + // No need to clean this resource. + handle_ = nullptr; +} + void Plugin::getResponse(const std::string& key, const PluginResponse& response, boost::property_tree::ptree& tree) { diff --git a/osquery/registry/registry_tests.cpp b/osquery/registry/registry_tests.cpp index 443bd69..88e9ece 100644 --- a/osquery/registry/registry_tests.cpp +++ b/osquery/registry/registry_tests.cpp @@ -109,7 +109,7 @@ class BadDoge : public DogPlugin { Status setUp() { return Status(1, "Expect error... this is a bad dog"); } }; -auto AutoDogRegistry = TestCoreRegistry::create("dog"); +auto AutoDogRegistry = TestCoreRegistry::create("dog", true); TEST_F(RegistryTests, test_auto_registries) { TestCoreRegistry::add("dog", "doge"); @@ -228,6 +228,47 @@ TEST_F(RegistryTests, test_real_registry) { } EXPECT_TRUE(has_one_registered); } + +TEST_F(RegistryTests, test_registry_modules) { + // Test the registry's module loading state tracking. + RegistryFactory::locked(false); + EXPECT_FALSE(RegistryFactory::locked()); + RegistryFactory::locked(true); + EXPECT_TRUE(RegistryFactory::locked()); + RegistryFactory::locked(false); + + // Test initializing a module load and the module's registry modifications. + EXPECT_EQ(RegistryFactory::getModule(), 0); + RegistryFactory::initModule("/my/test/module"); + // The registry is locked, no modifications during module global ctors. + EXPECT_TRUE(RegistryFactory::locked()); + // The 'is the registry using a module' is not set during module ctors. + EXPECT_FALSE(RegistryFactory::usingModule()); + EXPECT_EQ(RegistryFactory::getModules().size(), 1); + // The unittest can introspect into the current module. + auto& module = RegistryFactory::getModules().at(RegistryFactory::getModule()); + EXPECT_EQ(module.path, "/my/test/module"); + EXPECT_EQ(module.name, ""); + RegistryFactory::declareModule("test", "0.1.1", "0.0.0", "0.0.1"); + // The registry is unlocked after the module is declared. + // This assures that module modifications happen with the correct information + // and state tracking (aka the SDK limits, name, and version). + EXPECT_FALSE(RegistryFactory::locked()); + // Now the 'is the registry using a module' is set for the duration of the + // modules loading. + EXPECT_TRUE(RegistryFactory::usingModule()); + EXPECT_EQ(module.name, "test"); + EXPECT_EQ(module.version, "0.1.1"); + EXPECT_EQ(module.sdk_version, "0.0.1"); + + // Finally, when the module load is complete, we clear state. + RegistryFactory::shutdownModule(); + // The registry is again locked. + EXPECT_TRUE(RegistryFactory::locked()); + // And the registry is no longer using a module. + EXPECT_FALSE(RegistryFactory::usingModule()); + EXPECT_EQ(RegistryFactory::getModule(), 0); +} } int main(int argc, char* argv[]) { diff --git a/osquery/scheduler/scheduler.cpp b/osquery/scheduler/scheduler.cpp index dd91fa6..55f8beb 100644 --- a/osquery/scheduler/scheduler.cpp +++ b/osquery/scheduler/scheduler.cpp @@ -25,7 +25,7 @@ namespace osquery { FLAG(string, host_identifier, "hostname", - "Field used to identify the host running osqueryd"); + "Field used to identify the host running osquery"); FLAG(int32, schedule_splay_percent, 10, "Percent to splay config times"); @@ -165,7 +165,7 @@ void initializeScheduler() { #endif // Iterate over scheduled queryies and add a splay to each. - auto schedule = Config::getInstance().getScheduledQueries(); + auto schedule = Config::getScheduledQueries(); for (auto& q : schedule) { auto old_interval = q.interval; auto new_interval = splayValue(old_interval, FLAGS_schedule_splay_percent); diff --git a/osquery/sql/sql_tests.cpp b/osquery/sql/sql_tests.cpp index 4ddef05..4c1344c 100644 --- a/osquery/sql/sql_tests.cpp +++ b/osquery/sql/sql_tests.cpp @@ -50,7 +50,7 @@ class TestTable : public tables::TablePlugin { }; TEST_F(SQLTests, test_raw_access_context) { - REGISTER(TestTable, "table", "test_table"); + Registry::add("table", "test_table"); auto results = SQL::selectAllFrom("test_table"); EXPECT_EQ(results.size(), 1); @@ -67,6 +67,5 @@ TEST_F(SQLTests, test_raw_access_context) { int main(int argc, char* argv[]) { testing::InitGoogleTest(&argc, argv); - osquery::initOsquery(argc, argv); return RUN_ALL_TESTS(); } diff --git a/osquery/sql/sqlite_util.cpp b/osquery/sql/sqlite_util.cpp index ba56688..3884ce5 100644 --- a/osquery/sql/sqlite_util.cpp +++ b/osquery/sql/sqlite_util.cpp @@ -153,7 +153,9 @@ int queryDataCallback(void* argument, int argc, char* argv[], char* column[]) { QueryData* qData = (QueryData*)argument; Row r; for (int i = 0; i < argc; i++) { - r[column[i]] = argv[i]; + if (column[i] != nullptr) { + r[column[i]] = (argv[i] != nullptr) ? argv[i] : ""; + } } (*qData).push_back(r); return 0; diff --git a/osquery/sql/virtual_table_tests.cpp b/osquery/sql/virtual_table_tests.cpp index d5f936e..972fc01 100644 --- a/osquery/sql/virtual_table_tests.cpp +++ b/osquery/sql/virtual_table_tests.cpp @@ -70,6 +70,5 @@ TEST_F(VirtualTableTests, test_sqlite3_attach_vtable) { int main(int argc, char* argv[]) { testing::InitGoogleTest(&argc, argv); - osquery::initOsquery(argc, argv); return RUN_ALL_TESTS(); } diff --git a/osquery/tables/CMakeLists.txt b/osquery/tables/CMakeLists.txt index 0620505..4f5a0fa 100644 --- a/osquery/tables/CMakeLists.txt +++ b/osquery/tables/CMakeLists.txt @@ -9,12 +9,14 @@ ADD_OSQUERY_LIBRARY(FALSE osquery_tables_linux events/linux/passwd_changes.cpp system/linux/process_open_files.cpp system/linux/shared_memory.cpp system/linux/smbios_tables.cpp + system/linux/sysctl_utils.cpp system/linux/users.cpp system/linux/groups.cpp system/linux/kernel_info.cpp system/linux/kernel_modules.cpp system/linux/memory_map.cpp system/linux/mounts.cpp + system/linux/os_version.cpp system/linux/pci_devices.cpp system/linux/usb_devices.cpp) @@ -31,6 +33,7 @@ ADD_OSQUERY_LIBRARY(TRUE osquery_tables networking/utils.cpp system/last.cpp system/shell_history.cpp system/suid_bin.cpp + system/system_controls.cpp system/logged_in_users.cpp) ADD_OSQUERY_TEST(FALSE osquery_etc_hosts_tests networking/etc_hosts_tests.cpp) diff --git a/osquery/tables/events/linux/file_changes.cpp b/osquery/tables/events/linux/file_changes.cpp index f3dc82b..a0d44fe 100644 --- a/osquery/tables/events/linux/file_changes.cpp +++ b/osquery/tables/events/linux/file_changes.cpp @@ -55,15 +55,17 @@ class FileChangesEventSubscriber REGISTER(FileChangesEventSubscriber, "event_subscriber", "file_changes"); void FileChangesEventSubscriber::init() { - const auto& file_map = Config::getInstance().getWatchedFiles(); + const auto& file_map = Config::getWatchedFiles(); for (const auto& element_kv : file_map) { for (const auto& file : element_kv.second) { + VLOG(1) << "Added listener to: " << file; auto mc = createSubscriptionContext(); + mc->recursive = 1; mc->path = file; mc->mask = IN_ATTRIB | IN_MODIFY | IN_DELETE | IN_CREATE; subscribe(&FileChangesEventSubscriber::Callback, mc, - (void*)(&element_kv.first)); + (void*)(&element_kv.first)); } } } @@ -74,7 +76,11 @@ Status FileChangesEventSubscriber::Callback(const INotifyEventContextRef& ec, r["action"] = ec->action; r["time"] = ec->time_string; r["target_path"] = ec->path; - r["category"] = *(std::string*)user_data; + if (user_data != nullptr) { + r["category"] = *(std::string*)user_data; + } else { + r["category"] = "Undefined"; + } r["transaction_id"] = INTEGER(ec->event->cookie); r["md5"] = hashFromFile(HASH_TYPE_MD5, ec->path); r["sha1"] = hashFromFile(HASH_TYPE_SHA1, ec->path); diff --git a/osquery/tables/specs/linux/process_memory_map.table b/osquery/tables/specs/linux/process_memory_map.table index ad08a23..2fafc9a 100644 --- a/osquery/tables/specs/linux/process_memory_map.table +++ b/osquery/tables/specs/linux/process_memory_map.table @@ -9,6 +9,6 @@ schema([ Column("device", TEXT, "MA:MI Major/minor device ID"), Column("inode", INTEGER, "Mapped path inode, 0 means uninitialized (BSS)"), Column("path", TEXT, "Path to mapped file or mapped type"), - Column("is_pseudo", INTEGER, "1 if path is a pseudo path, else 0"), + Column("pseudo", INTEGER, "1 if path is a pseudo path, else 0"), ]) implementation("processes@genProcessMemoryMap") diff --git a/osquery/tables/specs/x/file.table b/osquery/tables/specs/utility/file.table similarity index 100% rename from osquery/tables/specs/x/file.table rename to osquery/tables/specs/utility/file.table diff --git a/osquery/tables/specs/x/hash.table b/osquery/tables/specs/utility/hash.table similarity index 100% rename from osquery/tables/specs/x/hash.table rename to osquery/tables/specs/utility/hash.table diff --git a/osquery/tables/specs/utility/osquery_extensions.table b/osquery/tables/specs/utility/osquery_extensions.table new file mode 100644 index 0000000..3588a7e --- /dev/null +++ b/osquery/tables/specs/utility/osquery_extensions.table @@ -0,0 +1,12 @@ +table_name("osquery_extensions") +description("List of active osquery extensions.") +schema([ + Column("uuid", BIGINT, "The transient ID assigned for communication"), + Column("name", TEXT, "Extension's name"), + Column("version", TEXT, "Extenion's version"), + Column("sdk_version", TEXT, "osquery SDK version used to build the extension"), + Column("path", TEXT, "Path of the extenion's domain socket or library path"), + Column("type", TEXT, "SDK extension type: extension or module") +]) +attributes(utility=True) +implementation("osquery@genOsqueryExtensions") diff --git a/osquery/tables/specs/x/osquery_flags.table b/osquery/tables/specs/utility/osquery_flags.table similarity index 100% rename from osquery/tables/specs/x/osquery_flags.table rename to osquery/tables/specs/utility/osquery_flags.table diff --git a/osquery/tables/specs/x/osquery_info.table b/osquery/tables/specs/utility/osquery_info.table similarity index 100% rename from osquery/tables/specs/x/osquery_info.table rename to osquery/tables/specs/utility/osquery_info.table diff --git a/osquery/tables/specs/utility/osquery_registry.table b/osquery/tables/specs/utility/osquery_registry.table new file mode 100644 index 0000000..6cb87fc --- /dev/null +++ b/osquery/tables/specs/utility/osquery_registry.table @@ -0,0 +1,11 @@ +table_name("osquery_registry") +description("List the osquery registry plugins.") +schema([ + Column("registry", TEXT, "Name of the osquery registry"), + Column("name", TEXT, "Name of the plugin item"), + Column("owner_uuid", INTEGER, "Extension route UUID (0 for core)"), + Column("internal", INTEGER, "1 if the plugin is internal else 0"), + Column("active", INTEGER, "1 if this plugin is active else 0"), +]) +attributes(utility=True) +implementation("osquery@genOsqueryRegistry") diff --git a/osquery/tables/specs/x/time.table b/osquery/tables/specs/utility/time.table similarity index 100% rename from osquery/tables/specs/x/time.table rename to osquery/tables/specs/utility/time.table diff --git a/osquery/tables/specs/x/os_version.table b/osquery/tables/specs/x/os_version.table new file mode 100644 index 0000000..c436c50 --- /dev/null +++ b/osquery/tables/specs/x/os_version.table @@ -0,0 +1,8 @@ +table_name("os_version") +description("A single row containing the operating system version.") +schema([ + Column("major", INTEGER), + Column("minor", INTEGER), + Column("patch", INTEGER), +]) +implementation("system/os_version@genOSVersion") diff --git a/osquery/tables/specs/x/osquery_extensions.table b/osquery/tables/specs/x/osquery_extensions.table deleted file mode 100644 index 836ffd9..0000000 --- a/osquery/tables/specs/x/osquery_extensions.table +++ /dev/null @@ -1,11 +0,0 @@ -table_name("osquery_extensions") -description("List of active osquery extensions.") -schema([ - Column("uuid", BIGINT), - Column("name", TEXT), - Column("version", TEXT), - Column("sdk_version", TEXT), - Column("socket", TEXT), -]) -attributes(utility=True) -implementation("osquery@genOsqueryExtensions") diff --git a/osquery/tables/specs/x/system_controls.table b/osquery/tables/specs/x/system_controls.table new file mode 100644 index 0000000..ece3844 --- /dev/null +++ b/osquery/tables/specs/x/system_controls.table @@ -0,0 +1,11 @@ +table_name("system_controls") +description("sysctl names, values, and settings information.") +schema([ + Column("name", TEXT, "Full sysctl MIB name"), + Column("oid", TEXT, "Control MIB"), + Column("subsystem", TEXT, "Subsystem ID, control type"), + Column("current_value", TEXT, "Value of setting"), + Column("config_value", TEXT, "The MIB value set in /etc/sysctl.conf"), + Column("type", TEXT, "Data type"), +]) +implementation("system_controls@genSystemControls") diff --git a/osquery/tables/system/linux/os_version.cpp b/osquery/tables/system/linux/os_version.cpp new file mode 100644 index 0000000..af78875 --- /dev/null +++ b/osquery/tables/system/linux/os_version.cpp @@ -0,0 +1,61 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ + +#include + +#include + +#include +#include +#include + +#ifdef CENTOS +const std::string kLinuxOSRelease = "/etc/redhat-release"; +#define kLinuxOSRegex "CentOS release ([0-9]+).([0-9]+)" +#else +const std::string kLinuxOSRelease = "/etc/os-release"; +#define kLinuxOSRegex "VERSION=\"([0-9]+)\\.([0-9]+)[\\.]{0,1}([0-9]+)?" +#endif + +namespace osquery { +namespace tables { + +QueryData genOSVersion(QueryContext& context) { + std::string content; + if (!readFile(kLinuxOSRelease, content).ok()) { + return {}; + } + + std::vector version = {"0", "0", "0"}; + boost::regex rx(kLinuxOSRegex); + boost::smatch matches; + for (const auto& line : osquery::split(content, "\n")) { + if (boost::regex_search(line, matches, rx)) { + // Push the matches in reverse order. + version[0] = matches[1]; + version[1] = matches[2]; + if (matches.size() == 4) { + // Patch is optional for Ubuntu and not used for CentOS. + version[2] = matches[3]; + } + break; + } + } + + Row r; + if (version.size() == 3) { + r["major"] = INTEGER(version[0]); + r["minor"] = INTEGER(version[1]); + r["patch"] = INTEGER(version[2]); + } + return {r}; +} +} +} diff --git a/osquery/tables/system/linux/processes.cpp b/osquery/tables/system/linux/processes.cpp index 3ca594b..c2b1e8d 100644 --- a/osquery/tables/system/linux/processes.cpp +++ b/osquery/tables/system/linux/processes.cpp @@ -128,7 +128,7 @@ void genProcessMap(const proc_t* proc_info, QueryData& results) { } // BSS with name in pathname. - r["is_pseudo"] = (fields[4] == "0" && r["path"].size() > 0) ? "1" : "0"; + r["pseudo"] = (fields[4] == "0" && r["path"].size() > 0) ? "1" : "0"; results.push_back(r); } } diff --git a/osquery/tables/system/linux/sysctl_utils.cpp b/osquery/tables/system/linux/sysctl_utils.cpp new file mode 100644 index 0000000..f67a637 --- /dev/null +++ b/osquery/tables/system/linux/sysctl_utils.cpp @@ -0,0 +1,116 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ + +#include + +#include + +#include +#include + +#include "osquery/tables/system/sysctl_utils.h" + +namespace fs = boost::filesystem; + +namespace osquery { +namespace tables { + +const std::string kSystemControlPath = "/proc/sys/"; + +void genControlInfo(const std::string& mib_path, QueryData& results, + const std::map& config) { + if (isDirectory(mib_path).ok()) { + // Iterate through the subitems and items. + std::vector items; + if (listDirectoriesInDirectory(mib_path, items).ok()) { + for (const auto& item : items) { + genControlInfo(item, results, config); + } + } + + if (listFilesInDirectory(mib_path, items).ok()) { + for (const auto& item : items) { + genControlInfo(item, results, config); + } + } + return; + } + + // This is a file (leaf-control). + Row r; + r["name"] = mib_path.substr(kSystemControlPath.size()); + + std::replace(r["name"].begin(), r["name"].end(), '/', '.'); + // No known way to convert name MIB to int array. + r["subsystem"] = osquery::split(r.at("name"), ".")[0]; + + if (isReadable(mib_path).ok()) { + std::string content; + readFile(mib_path, content); + boost::trim(content); + r["current_value"] = content; + } + + if (config.count(r.at("name")) > 0) { + r["config_value"] = config.at(r.at("name")); + } + r["type"] = "string"; + results.push_back(r); +} + +void genControlInfo(int* oid, + size_t oid_size, + QueryData& results, + const std::map& config) { + // Get control size + size_t response_size = CTL_MAX_VALUE; + char response[CTL_MAX_VALUE + 1] = {0}; + if (sysctl(oid, oid_size, response, &response_size, 0, 0) != 0) { + // Cannot request MIB data. + return; + } + + // Data is output, but no way to determine type (long, int, string, struct). + Row r; + r["oid"] = stringFromMIB(oid, oid_size); + r["current_value"] = std::string(response); + r["type"] = "string"; + results.push_back(r); +} + +void genAllControls(QueryData& results, + const std::map& config, + const std::string& subsystem) { + // Linux sysctl subsystems are directories in /proc + std::vector subsystems; + if (!listDirectoriesInDirectory("/proc/sys", subsystems).ok()) { + return; + } + + for (const auto& sub : subsystems) { + if (subsystem.size() != 0 && fs::path(sub).filename().string() != subsystem) { + // Request is limiting subsystem. + continue; + } + genControlInfo(sub, results, config); + } +} + +void genControlInfoFromName(const std::string& name, QueryData& results, + const std::map& config) { + // Convert '.'-tokenized name to path. + std::string name_path = name; + std::replace(name_path.begin(), name_path.end(), '.', '/'); + auto mib_path = fs::path(kSystemControlPath) / name_path; + + genControlInfo(mib_path.string(), results, config); +} +} +} diff --git a/osquery/tables/system/sysctl_utils.h b/osquery/tables/system/sysctl_utils.h new file mode 100644 index 0000000..4346d71 --- /dev/null +++ b/osquery/tables/system/sysctl_utils.h @@ -0,0 +1,41 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ + +#include + +#include + +namespace osquery { +namespace tables { + +#define CTL_MAX_VALUE 128 + +#ifndef CTL_DEBUG_MAXID +#define CTL_DEBUG_MAXID (CTL_MAXNAME * 2) +#endif + +std::string stringFromMIB(const int* oid, size_t oid_size); + +/// Must be implemented by the platform. +void genAllControls(QueryData& results, + const std::map& config, + const std::string& subsystem); + +/// Must be implemented by the platform. +void genControlInfo(int* oid, + size_t oid_size, + QueryData& results, + const std::map& config); + +/// Must be implemented by the platform. +void genControlInfoFromName(const std::string& name, QueryData& results, + const std::map& config); +} +} diff --git a/osquery/tables/system/system_controls.cpp b/osquery/tables/system/system_controls.cpp new file mode 100644 index 0000000..e6dbeb7 --- /dev/null +++ b/osquery/tables/system/system_controls.cpp @@ -0,0 +1,126 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ + +#include + +#include +#include + +#include "osquery/tables/system/sysctl_utils.h" + +namespace osquery { +namespace tables { + +const std::vector kControlSettingsFiles = {"/etc/sysctl.conf"}; + +const std::vector kControlSettingsDirs = { + "/run/sysctl.d/%.conf", + "/etc/sysctl.d/%.conf", + "/usr/local/lib/sysctl.d/%.conf", + "/usr/lib/sysctl.d/%.conf", + "/lib/sysctl.d/%.conf", +}; + +std::string stringFromMIB(const int* oid, size_t oid_size) { + std::string result; + for (size_t i = 0; i < oid_size; ++i) { + // Walk an int-encoded MIB and return the string representation, '.'. + if (result.size() > 0) { + result += "."; + } + result += std::to_string(oid[i]); + } + return result; +} + +void genControlInfoFromOIDString(const std::string& oid_string, QueryData& results, + const std::map& config) { + int request[CTL_DEBUG_MAXID + 2] = {0}; + auto tokens = osquery::split(oid_string, "."); + if (tokens.size() > CTL_DEBUG_MAXID) { + // OID input string was too large. + return; + } + + // Convert the string into an int array. + for (size_t i = 0; i < tokens.size(); ++i) { + request[i] = atol(tokens.at(i).c_str()); + } + genControlInfo((int*)request, tokens.size(), results, config); +} + +void genControlConfigFromPath(const std::string& path, + std::map& config) { + std::string content; + if (!osquery::readFile(path, content).ok()) { + return; + } + + for (auto& line : split(content, "\n")) { + boost::trim(line); + if (line[0] == '#' || line[0] == ';') { + continue; + } + + // Try to tokenize the config line using '='. + auto detail = split(line, "="); + if (detail.size() == 2) { + boost::trim(detail[0]); + boost::trim(detail[1]); + config[detail[0]] = detail[1]; + } + } +} + +QueryData genSystemControls(QueryContext& context) { + QueryData results; + + // Read the sysctl.conf values. + std::map config; + for (const auto& path : kControlSettingsFiles) { + genControlConfigFromPath(path, config); + } + + for (const auto& dirs : kControlSettingsDirs) { + std::vector configs; + if (resolveFilePattern(dirs, configs).ok()) { + for (const auto& path : configs) { + genControlConfigFromPath(path, config); + } + } + } + + // Iterate through the sysctl-defined macro of control types. + if (context.constraints["name"].exists()) { + // Request MIB information by the description (name). + auto names = context.constraints["name"].getAll(EQUALS); + for (const auto& name : names) { + genControlInfoFromName(name, results, config); + } + } else if (context.constraints["oid"].exists()) { + // Request MIB by OID as a string, parse into set of INTs. + auto oids = context.constraints["oid"].getAll(EQUALS); + for (const auto& oid_string : oids) { + genControlInfoFromOIDString(oid_string, results, config); + } + } else if (context.constraints["subsystem"].exists()) { + // Limit the MIB search to a subsystem name (first find the INT). + auto subsystems = context.constraints["subsystem"].getAll(EQUALS); + for (const auto& subsystem : subsystems) { + genAllControls(results, config, subsystem); + } + } else { + genAllControls(results, config, ""); + } + + return results; +} +} +} diff --git a/osquery/tables/utility/file.cpp b/osquery/tables/utility/file.cpp index ea7abbf..80ee6db 100644 --- a/osquery/tables/utility/file.cpp +++ b/osquery/tables/utility/file.cpp @@ -82,12 +82,10 @@ QueryData genFile(QueryContext& context) { // Iterate over the directory and generate a hash for each regular file. boost::filesystem::directory_iterator begin(directory), end; for (; begin != end; ++begin) { - if (boost::filesystem::is_regular_file(begin->status())) { - genFileInfo(begin->path().string(), - begin->path().filename().string(), - directory_string, - results); - } + genFileInfo(begin->path().string(), + begin->path().filename().string(), + directory_string, + results); } } diff --git a/osquery/tables/utility/osquery.cpp b/osquery/tables/utility/osquery.cpp index ee5ecf5..e796f66 100644 --- a/osquery/tables/utility/osquery.cpp +++ b/osquery/tables/utility/osquery.cpp @@ -13,6 +13,7 @@ #include #include #include +#include #include #include @@ -43,21 +44,62 @@ QueryData genOsqueryFlags(QueryContext& context) { return results; } +QueryData genOsqueryRegistry(QueryContext& context) { + QueryData results; + + const auto& registries = RegistryFactory::all(); + for (const auto& registry : registries) { + const auto& plugins = registry.second->all(); + for (const auto& plugin : plugins) { + Row r; + r["registry"] = registry.first; + r["name"] = plugin.first; + r["owner_uuid"] = "0"; + r["internal"] = (registry.second->isInternal(plugin.first)) ? "1" : "0"; + r["active"] = "1"; + results.push_back(r); + } + + for (const auto& route : registry.second->getExternal()) { + Row r; + r["registry"] = registry.first; + r["name"] = route.first; + r["owner_uuid"] = INTEGER(route.second); + r["internal"] = "0"; + r["active"] = "1"; + results.push_back(r); + } + } + + return results; +} + QueryData genOsqueryExtensions(QueryContext& context) { QueryData results; ExtensionList extensions; - if (!getExtensions(extensions).ok()) { - return {}; + if (getExtensions(extensions).ok()) { + for (const auto& extenion : extensions) { + Row r; + r["uuid"] = TEXT(extenion.first); + r["name"] = extenion.second.name; + r["version"] = extenion.second.version; + r["sdk_version"] = extenion.second.sdk_version; + r["path"] = getExtensionSocket(extenion.first); + r["type"] = "extension"; + results.push_back(r); + } } - for (const auto& extenion : extensions) { + const auto& modules = RegistryFactory::getModules(); + for (const auto& module : modules) { Row r; - r["uuid"] = TEXT(extenion.first); - r["name"] = extenion.second.name; - r["version"] = extenion.second.version; - r["sdk_version"] = extenion.second.sdk_version; - r["socket"] = getExtensionSocket(extenion.first); + r["uuid"] = TEXT(module.first); + r["name"] = module.second.name; + r["version"] = module.second.version; + r["sdk_version"] = module.second.sdk_version; + r["path"] = module.second.path; + r["type"] = "module"; results.push_back(r); } @@ -72,7 +114,7 @@ QueryData genOsqueryInfo(QueryContext& context) { r["pid"] = INTEGER(getpid()); std::string hash_string; - auto s = Config::getInstance().getMD5(hash_string); + auto s = Config::getMD5(hash_string); if (s.ok()) { r["config_md5"] = TEXT(hash_string); } else { diff --git a/osquery/tables/utility/time.cpp b/osquery/tables/utility/time.cpp index 3be3f39..fbd84dc 100644 --- a/osquery/tables/utility/time.cpp +++ b/osquery/tables/utility/time.cpp @@ -15,8 +15,6 @@ namespace osquery { namespace tables { -const int kNumCols = 1; - QueryData genTime(QueryContext& context) { Row r; time_t _time = time(0); @@ -25,9 +23,7 @@ QueryData genTime(QueryContext& context) { r["minutes"] = INTEGER(now->tm_min); r["seconds"] = INTEGER(now->tm_sec); QueryData results; - for (int i = 0; i < kNumCols; ++i) { - results.push_back(r); - } + results.push_back(r); return results; } } diff --git a/packaging/osquery.spec b/packaging/osquery.spec index 163c53b..cf195ae 100644 --- a/packaging/osquery.spec +++ b/packaging/osquery.spec @@ -1,5 +1,5 @@ Name: osquery -Version: 1.4.2 +Version: 1.4.3 Release: 0 License: Apache-2.0 and GPLv2 Summary: A SQL powered operating system instrumentation, monitoring framework. diff --git a/tools/sync.sh b/tools/sync.sh index 362c956..23e2512 100755 --- a/tools/sync.sh +++ b/tools/sync.sh @@ -43,7 +43,8 @@ find "$SYNC_DIR" -type f -name "CMakeLists.txt" -exec rm -f {} \; # make the targets file mkdir -p "$SYNC_DIR/code-analysis" (cd "$SYNC_DIR/code-analysis" && SDK=True cmake -DCMAKE_EXPORT_COMPILE_COMMANDS=ON ../../../../) -python tools/codegen/gentargets.py -i "$SYNC_DIR/code-analysis/compile_commands.json" >$SYNC_DIR/osquery/TARGETS +python tools/codegen/gentargets.py -v $VERSION --sdk $VERSION \ + -i "$SYNC_DIR/code-analysis/compile_commands.json" >$SYNC_DIR/osquery/TARGETS # wrap it up in a tarball (cd "$SYNC_DIR" && tar -zcf osquery-sync-$VERSION.tar.gz osquery) diff --git a/tools/tests/test.config.d/extra_options.conf b/tools/tests/test.config.d/extra_options.conf new file mode 100644 index 0000000..e8ab9bd --- /dev/null +++ b/tools/tests/test.config.d/extra_options.conf @@ -0,0 +1,6 @@ +{ + "options":{ + "optionC" : "optionc-val", + "optionD" : "optiond-val" + } +} \ No newline at end of file diff --git a/tools/tests/test.config.d/osquery.conf b/tools/tests/test.config.d/osquery.conf new file mode 100644 index 0000000..6e89c41 --- /dev/null +++ b/tools/tests/test.config.d/osquery.conf @@ -0,0 +1,16 @@ +{ + "scheduledQueries": [ + { + "name": "time_again", + "query": "select * from time;", + "interval": 1 + } + ], + "options":{ + "optionA" : "optiona-val", + "optionB" : "optionb-val" + }, + "additional_monitoring" : { + "other_thing" : {"element" : "key"} + } +} -- 2.7.4