#ADD_DEFINITIONS("-pedantic-errors")
# TODO(sangwan.kwon): Get version from packing spec.
-SET(OSQUERY_BUILD_VERSION "1.4.2")
+SET(OSQUERY_BUILD_VERSION "1.4.3")
# Set various platform/platform-version/build version/etc defines.
ADD_DEFINITIONS(-DOSQUERY_BUILD_VERSION=${OSQUERY_BUILD_VERSION}
#include <memory>
#include <vector>
+#include <boost/property_tree/ptree.hpp>
+#include <boost/property_tree/json_parser.hpp>
+
#include <osquery/flags.h>
#include <osquery/registry.h>
#include <osquery/scheduler.h>
#include <osquery/status.h>
+namespace pt = boost::property_tree;
+
namespace osquery {
/// The builder or invoker may change the default config plugin.
std::vector<OsqueryScheduledQuery> scheduledQueries;
std::map<std::string, std::string> options;
std::map<std::string, std::vector<std::string> > eventFiles;
+ pt::ptree all_data;
};
/**
*
* This may perform a resource load such as TCP request or filesystem read.
*/
- Status load();
+ static Status load();
/**
* @brief Get a vector of all scheduled queries.
*
* @return A map all the files in the JSON blob organized by category
*/
- static std::map<std::string, std::vector<std::string> >& getWatchedFiles();
+ static std::map<std::string, std::vector<std::string> > getWatchedFiles();
+
+ /**
+ * @brief Return the configuration ptree
+ *
+ *
+ *
+ * @return Returns the unparsed, ptree representation of the given config
+ */
+ static pt::ptree getEntireConfiguration();
/**
* @brief Calculate the has of the osquery config
*
* @return The MD5 of the osquery config
*/
- Status getMD5(std::string& hashString);
+ static Status getMD5(std::string& hashString);
/**
* @brief Check to ensure that the config is accessible and properly
* @return an instance of osquery::Status, indicating the success or failure
* of the operation.
*/
- static osquery::Status genConfig(std::string& conf);
+ static osquery::Status genConfig(std::vector<std::string>& conf);
/// Prevent ConfigPlugins from implementing setUp.
osquery::Status setUp() { return Status(0, "Not used"); }
* indicates that config retrieval was successful, then the config data
* should be returned in pair.second.
*/
- virtual std::pair<osquery::Status, std::string> genConfig() = 0;
+ virtual Status genConfig(std::map<std::string, std::string>& config) = 0;
Status call(const PluginRequest& request, PluginResponse& response);
};
#endif
// clang-format on
+/// A configuration error is catastrophic and should exit the watcher.
+#define EXIT_CATASTROPHIC 78
+
namespace osquery {
/**
/**
* @brief A helpful tool type to report when logging, print help, or debugging.
*/
-enum osqueryTool {
+enum ToolType {
OSQUERY_TOOL_SHELL,
OSQUERY_TOOL_DAEMON,
OSQUERY_TOOL_TEST,
OSQUERY_EXTENSION,
};
-/**
- * @brief Sets up various aspects of osquery execution state.
- *
- * osquery needs a few things to happen as soon as the executable begins
- * executing. initOsquery takes care of setting up the relevant parameters.
- * initOsquery should be called in an executable's `main()` function.
- *
- * @param argc the number of elements in argv
- * @param argv the command-line arguments passed to `main()`
- */
-void initOsquery(int argc, char* argv[], int tool = OSQUERY_TOOL_TEST);
-
-/**
- * @brief Sets up a process as a osquery daemon.
- */
-void initOsqueryDaemon();
-
-/**
- * @brief Turns of various aspects of osquery such as event loops.
- *
- */
-void shutdownOsquery();
+class Initializer {
+ public:
+ /**
+ * @brief Sets up various aspects of osquery execution state.
+ *
+ * osquery needs a few things to happen as soon as the process begins
+ * executing. Initializer takes care of setting up the relevant parameters.
+ * Initializer should be called in an executable's `main()` function.
+ *
+ * @param argc the number of elements in argv
+ * @param argv the command-line arguments passed to `main()`
+ * @param tool the type of osquery main (daemon, shell, test, extension).
+ */
+ Initializer(int argc, char* argv[], ToolType tool = OSQUERY_TOOL_TEST);
+
+ /**
+ * @brief Sets up the process as an osquery daemon.
+ *
+ * A daemon has additional constraints, it can use a process mutext, check
+ * for sane/non-default configurations, etc.
+ */
+ void initDaemon();
+
+ /**
+ * @brief Daemon tools may want to continually spawn worker processes
+ * and monitor their utilization.
+ *
+ * A daemon may call initWorkerWatcher to begin watching child daemon
+ * processes until it-itself is unscheduled. The basic guarentee is that only
+ * workers will return from the function.
+ *
+ * The worker-watcher will implement performance bounds on CPU utilization
+ * and memory, as well as check for zombie/defunct workers and respawn them
+ * if appropriate. The appropriateness is determined from heuristics around
+ * how the worker exitted. Various exit states and velocities may cause the
+ * watcher to resign.
+ *
+ * @param name The name of the worker process.
+ */
+ void initWorkerWatcher(const std::string& name);
+
+ /// Assume initialization finished, start work.
+ void start();
+ /// Turns off various aspects of osquery such as event loops.
+ void shutdown();
+ /// Check if a process is an osquery worker.
+ bool isWorker();
+
+ private:
+ /// Initialize this process as an osquery daemon worker.
+ void initWorker(const std::string& name);
+ /// Initialize the osquery watcher, optionally spawn a worker.
+ void initWatcher();
+ /// Set the config and logger plugins, optionally depend on an extension.
+ void initConfigLogger();
+
+ private:
+ int argc_;
+ char** argv_;
+ int tool_;
+ std::string binary_;
+};
/**
* @brief Split a given string based on an optional delimiter.
*/
template <typename _Iterator1, typename _Iterator2>
inline size_t incUtf8StringIterator(_Iterator1& it, const _Iterator2& last) {
- if (it == last)
+ if (it == last) {
return 0;
+ }
+
unsigned char c;
size_t res = 1;
for (++it; last != it; ++it, ++res) {
c = *it;
- if (!(c & 0x80) || ((c & 0xC0) == 0xC0))
+ if (!(c & 0x80) || ((c & 0xC0) == 0xC0)) {
break;
+ }
}
return res;
inline size_t utf8StringSize(const std::string& str) {
size_t res = 0;
std::string::const_iterator it = str.begin();
- for (; it != str.end(); incUtf8StringIterator(it, str.end()))
+ for (; it != str.end(); incUtf8StringIterator(it, str.end())) {
res++;
+ }
return res;
}
namespace osquery {
-DECLARE_string(db_path);
+DECLARE_string(database_path);
/////////////////////////////////////////////////////////////////////////////
// Constants
void operator=(EventSubscriberPlugin const&);
private:
+ Status setUp() { return Status(0, "Setup never used"); }
+
+ private:
/// Do not respond to periodic/scheduled/triggered event expiration requests.
bool expire_events_;
/// Sleep in a boost::thread interruptable state.
void interruptableSleep(size_t milli);
-CREATE_LAZY_REGISTRY(EventPublisherPlugin, "event_publisher");
+CREATE_REGISTRY(EventPublisherPlugin, "event_publisher");
CREATE_REGISTRY(EventSubscriberPlugin, "event_subscriber");
}
DECLARE_int32(worker_threads);
DECLARE_string(extensions_socket);
+DECLARE_string(extensions_autoload);
+DECLARE_string(extensions_timeout);
+
+/// A millisecond internal applied to extension initialization.
+extern const int kExtensionInitializeMLatency;
/**
* @brief Helper struct for managing extenion metadata.
Status pingExtension(const std::string& path);
/**
+ * @brief Request the extensions API to autoload any appropriate extensions.
+ *
+ * Extensions may be 'autoloaded' using the `extensions_autoload` command line
+ * argument. loadExtensions should be called before any plugin or registry item
+ * is used. This allows appropriate extensions to expose plugin requirements.
+ *
+ * An 'appropriate' extension is one within the `extensions_autoload` search
+ * path with file ownership equivilent or greater (root) than the osquery
+ * process requesting autoload.
+ */
+void loadExtensions();
+
+/**
+ * @brief Load extensions from a delimited search path string.
+ *
+ * @param paths A colon-delimited path variable, e.g: '/path1:/path2'.
+ */
+Status loadExtensions(const std::string& loadfile);
+
+/**
+ * @brief Request the extensions API to autoload any appropriate modules.
+ *
+ * Extension modules are shared libraries that add Plugins to the osquery
+ * core's registry at runtime.
+ */
+void loadModules();
+
+/**
+ * @brief Load extenion modules from a delimited search path string.
+ *
+ * @param paths A colon-delimited path variable, e.g: '/path1:/path2'.
+ */
+Status loadModules(const std::string& loadfile);
+
+/// Load all modules in a direcotry.
+Status loadModuleFile(const std::string& path);
+
+/**
* @brief Call a Plugin exposed by an Extension Registry route.
*
* This is mostly a Registry%-internal method used to call an ExtensionHandler
#pragma once
#include <map>
+#include <set>
#include <string>
#include <vector>
* this many wildcards.
*/
const unsigned int kMaxDirectoryTraversalDepth = 40;
+typedef unsigned int ReturnSetting;
+enum {
+ REC_LIST_FILES = 0x1, // Return only files
+ REC_LIST_FOLDERS = 0x2, // Return only folders
+ REC_EVENT_OPT = 0x4, // Enable optimizations for file event resolutions
+ REC_LIST_ALL = REC_LIST_FILES | REC_LIST_FOLDERS
+};
const std::string kWildcardCharacter = "%";
const std::string kWildcardCharacterRecursive =
* of the operation.
*/
Status listFilesInDirectory(const boost::filesystem::path& path,
- std::vector<std::string>& results);
+ std::vector<std::string>& results,
+ bool ignore_error = 1);
/**
* @brief List all of the directories in a specific directory, non-recursively.
* of the operation.
*/
Status listDirectoriesInDirectory(const boost::filesystem::path& path,
- std::vector<std::string>& results);
+ std::vector<std::string>& results,
+ bool ignore_error = 1);
/**
* @brief Given a wildcard filesystem patten, resolve all possible paths
std::vector<std::string>& results);
/**
+ * @brief Given a wildcard filesystem patten, resolve all possible paths
+ *
+ * @code{.cpp}
+ * std::vector<std::string> results;
+ * auto s = resolveFilePattern("/Users/marpaia/Downloads/%", results);
+ * if (s.ok()) {
+ * for (const auto& result : results) {
+ * LOG(INFO) << result;
+ * }
+ * }
+ * @endcode
+ *
+ * @param fs_path The filesystem pattern
+ * @param results The vector in which all results will be returned
+ * @param setting Do you want files returned, folders or both?
+ *
+ * @return An instance of osquery::Status which indicates the success or
+ * failure of the operation
+ */
+Status resolveFilePattern(const boost::filesystem::path& fs_path,
+ std::vector<std::string>& results,
+ ReturnSetting setting);
+
+/**
* @brief Get directory portion of a path.
*
* @param path The input path, either a filename or directory.
*
* @return a vector of strings representing the path of all home directories
*/
-std::vector<boost::filesystem::path> getHomeDirectories();
+std::set<boost::filesystem::path> getHomeDirectories();
+
+/**
+ * @brief Check the permissions of a file and it's directory.
+ *
+ * 'Safe' implies the directory is not a /tmp-like directory in that users
+ * cannot control super-user-owner files. The file should be owned by the
+ * process's UID or the file should be owned by root.
+ *
+ * @param dir the directory to check /tmp mode
+ * @param path a path to a file to check
+ * @param executable the file must also be executable
+ *
+ * @return true if the file is 'safe' else false
+ */
+bool safePermissions(const std::string& dir,
+ const std::string& path,
+ bool executable = false);
/// Return bit-mask-style permissions.
std::string lsperms(int mode);
std::string description;
bool shell;
bool external;
+ bool cli;
};
struct FlagInfo {
static int create(const std::string& name, const FlagDetail& flag);
/// Create a Gflags alias to name, using the Flag::getValue accessor.
- static int createAlias(const std::string& alias, const std::string& name);
+ static int createAlias(const std::string& alias, const FlagDetail& flag);
static Flag& instance() {
static Flag f;
static std::string getType(const std::string& name);
/*
+ * @brief Get the description as a string of an osquery flag.
+ *
+ * @param name the flag name.
+ */
+ static std::string getDescription(const std::string& name);
+
+ /*
* @brief Print help-style output to stdout for a given flag set.
*
* @param shell Only print shell flags.
* @param external Only print external flags (from extensions).
*/
- static void printFlags(bool shell = false, bool external = false);
+ static void printFlags(bool shell = false,
+ bool external = false,
+ bool cli = false);
private:
std::map<std::string, FlagDetail> flags_;
- std::map<std::string, std::string> aliases_;
+ std::map<std::string, FlagDetail> aliases_;
};
/**
* @param value The default value, use a C++ literal.
* @param desc A string literal used for help display.
*/
-#define OSQUERY_FLAG(t, n, v, d, s, e) \
- DEFINE_##t(n, v, d); \
- namespace flags { \
- const int flag_##n = Flag::create(#n, {d, s, e}); \
+#define OSQUERY_FLAG(t, n, v, d, s, e, c) \
+ DEFINE_##t(n, v, d); \
+ namespace flags { \
+ const int flag_##n = Flag::create(#n, {d, s, e, c}); \
}
-#define FLAG(t, n, v, d) OSQUERY_FLAG(t, n, v, d, false, false)
-#define SHELL_FLAG(t, n, v, d) OSQUERY_FLAG(t, n, v, d, true, false)
-#define EXTENSION_FLAG(t, n, v, d) OSQUERY_FLAG(t, n, v, d, false, true)
-
-#define FLAG_ALIAS(t, a, n) \
- FlagAlias<t> FLAGS_##a(#a, #t, #n, &FLAGS_##n); \
- namespace flags { \
- static GFLAGS_NAMESPACE::FlagRegisterer oflag_##a( \
- #a, #t, #a, #a, &FLAGS_##n, &FLAGS_##n); \
- const int flag_alias_##a = Flag::createAlias(#a, #n); \
+#define FLAG(t, n, v, d) OSQUERY_FLAG(t, n, v, d, 0, 0, 0)
+#define SHELL_FLAG(t, n, v, d) OSQUERY_FLAG(t, n, v, d, 1, 0, 0)
+#define EXTENSION_FLAG(t, n, v, d) OSQUERY_FLAG(t, n, v, d, 0, 1, 0)
+#define CLI_FLAG(t, n, v, d) OSQUERY_FLAG(t, n, v, d, 0, 0, 1)
+
+#define OSQUERY_FLAG_ALIAS(t, a, n, s, e) \
+ FlagAlias<t> FLAGS_##a(#a, #t, #n, &FLAGS_##n); \
+ namespace flags { \
+ static GFLAGS_NAMESPACE::FlagRegisterer oflag_##a( \
+ #a, #t, #a, #a, &FLAGS_##n, &FLAGS_##n); \
+ const int flag_alias_##a = Flag::createAlias(#a, {#n, s, e, 0}); \
}
+
+#define FLAG_ALIAS(t, a, n) OSQUERY_FLAG_ALIAS(t, a, n, 0, 0)
+#define SHELL_FLAG_ALIAS(t, a, n) _OSQUERY_FLAG_ALIAS(t, a, n, 1, 0)
+#define EXTENSION_FLAG_ALIAS(a, n) OSQUERY_FLAG_ALIAS(std::string, a, n, 0, 1)
#include <glog/logging.h>
+#include <osquery/flags.h>
#include <osquery/registry.h>
#include <osquery/status.h>
#include <osquery/scheduler.h>
namespace osquery {
+DECLARE_bool(disable_logging);
+DECLARE_string(logger_plugin);
+
/**
* @breif An internal severity set mapping to Glog's LogSeverity levels.
*/
};
/**
+ * @brief Helper logging macro for table-generated verbose log lines.
+ *
+ * Since logging in tables does not always mean a critical warning or error
+ * but more likely a parsing or expected edge-case, we provide a TLOG.
+ *
+ * The tool user can set within config or via the CLI what level of logging
+ * to tolerate. It's the table developer's job to assume consistency in logging.
+ */
+#define TLOG VLOG(1)
+
+/**
+ * @brief Prepend a reference number to the log line.
+ *
+ * A reference number is an external-search helper for somewhat confusing or
+ * seeminly-critical log lines.
+ */
+#define RLOG(n) "[Ref #" #n "] "
+
+/**
* @brief Superclass for the pluggable logging facilities.
*
* In order to make the logging of osquery results and inline debug, warning,
#include <boost/noncopyable.hpp>
#include <boost/property_tree/ptree.hpp>
-#include <osquery/status.h>
+#include <osquery/core.h>
namespace osquery {
* @param type A typename that derives from Plugin.
* @param name A string identifier for the registry.
*/
-#define CREATE_REGISTRY(type, name) \
- namespace registry { \
- const auto type##Registry = Registry::create<type>(name); \
+#define CREATE_REGISTRY(type, name) \
+ namespace registry { \
+ __attribute__((constructor)) static void type##Registry() { \
+ Registry::create<type>(name); \
+ } \
}
/**
* @brief A boilerplate code helper to create a registry given a name and
- * plugin base class type. This 'lazy' registry does not automatically run
- * Plugin::setUp on all items.
+ * plugin base class type. This 'lazy' registry does not run
+ * Plugin::setUp on its items, so the registry will do it.
*
* @param type A typename that derives from Plugin.
* @param name A string identifier for the registry.
*/
-#define CREATE_LAZY_REGISTRY(type, name) \
- namespace registry { \
- const auto type##Registry = Registry::create<type>(name, false); \
+#define CREATE_LAZY_REGISTRY(type, name) \
+ namespace registry { \
+ __attribute__((constructor)) static void type##Registry() { \
+ Registry::create<type>(name, true); \
+ } \
}
/**
* @param registry The string name for the registry.
* @param name A string identifier for this registry item.
*/
-#define REGISTER(type, registry, name) \
- const auto type##RegistryItem = Registry::add<type>(registry, name);
+#define REGISTER(type, registry, name) \
+ __attribute__((constructor)) static void type##RegistryItem() { \
+ Registry::add<type>(registry, name); \
+ }
/// The same as REGISTER but prevents the plugin item from being broadcasted.
-#define REGISTER_INTERNAL(type, registry, name) \
- const auto type##RegistryItem = Registry::add<type>(registry, name, true);
+#define REGISTER_INTERNAL(type, registry, name) \
+ __attribute__((constructor)) static void type##RegistryItem() { \
+ Registry::add<type>(registry, name, true); \
+ }
/**
* @brief The request part of a plugin (registry item's) call.
AddExternalCallback;
typedef std::function<void(const std::string&)> RemoveExternalCallback;
+/// When a module is being initialized its information is kept in a transient
+/// RegistryFactory lookup location.
+struct ModuleInfo {
+ std::string path;
+ std::string name;
+ std::string version;
+ std::string sdk_version;
+};
+
+/// The call-in prototype for Registry modules.
+typedef void (*ModuleInitalizer)(void);
+
class Plugin {
public:
Plugin() { name_ = "unnamed"; }
class RegistryHelperCore {
public:
- explicit RegistryHelperCore(bool auto_setup = true)
+ explicit RegistryHelperCore(bool auto_setup = false)
: auto_setup_(auto_setup) {}
virtual ~RegistryHelperCore() {}
const PluginRequest& request,
PluginResponse& response);
+ Status add(const std::string& item_name, bool internal = false);
+
/**
* @brief Allow a plugin to perform some setup functions when osquery starts.
*
/// Allow the registry to introspect into the registered name (for logging).
void setName(const std::string& name);
+ /// Allow others to introspect into the registered name (for reporting).
+ const std::string& getName() const { return name_; }
+
+ /// Check if a given plugin name is considered internal.
+ bool isInternal(const std::string& item_name) const;
+
+ /// Allow others to introspect into the routes from extensions.
+ const std::map<std::string, RouteUUID>& getExternal() const {
+ return external_;
+ }
+
+ /// Set an 'active' plugin to receive registry calls when no item name given.
+ Status setActive(const std::string& item_name);
+
+ /// Get the 'active' plugin, return success with the active plugin name.
+ const std::string& getActive() const;
+
protected:
/// The identifier for this registry, used to register items.
std::string name_;
std::map<std::string, PluginResponse> routes_;
/// Keep a lookup of registry items that are blacklisted from broadcast.
std::vector<std::string> internal_;
+ /// Support an 'active' mode where calls without a specific item name will
+ /// be directed to the 'active' plugin.
+ std::string active_;
+ /// If a module was initialized/declared then store lookup information.
+ std::map<std::string, RouteUUID> modules_;
};
/**
typedef std::shared_ptr<RegistryType> RegistryTypeRef;
public:
- explicit RegistryHelper(bool auto_setup = true)
+ explicit RegistryHelper(bool auto_setup = false)
: RegistryHelperCore(auto_setup),
add_(&RegistryType::addExternal),
remove_(&RegistryType::removeExternal) {}
std::shared_ptr<RegistryType> item((RegistryType*)new Item());
item->setName(item_name);
items_[item_name] = item;
-
- // The item can be listed as internal, meaning it does not broadcast.
- if (internal) {
- internal_.push_back(item_name);
- }
-
- return Status(0, "OK");
+ return RegistryHelperCore::add(item_name, internal);
}
/**
RemoveExternalCallback remove_;
};
+/// Helper defintion for a shared pointer to a Plugin.
typedef std::shared_ptr<Plugin> PluginRef;
+/// Helper definition for a basic-templated Registry type using a base Plugin.
typedef RegistryHelper<Plugin> PluginRegistryHelper;
+/// Helper definitions for a shared pointer to the basic Registry type.
typedef std::shared_ptr<PluginRegistryHelper> PluginRegistryHelperRef;
+/**
+ * @basic A workflow manager for opening a module path and appending to the
+ * core registry.
+ *
+ * osquery Registry modules are part of the extensions API, in that they use
+ * the osquery SDK to expose additional features to the osquery core. Modules
+ * do not require the Thrift interface and may be compiled as shared objects
+ * and loaded late at run time once the core and internal registry has been
+ * initialized and setUp.
+ *
+ * A ModuleLoader interprets search paths, dynamically loads the modules,
+ * maintains identification within the RegistryFactory and any registries
+ * the module adds items into.
+ */
+class RegistryModuleLoader : private boost::noncopyable {
+ public:
+ /// Unlock the registry, open, construct, and allow the module to declare.
+ RegistryModuleLoader(const std::string& path);
+ /// Keep the symbol resolution/calling out of construction.
+ void init();
+
+ /// Clear module information, 'lock' the registry.
+ ~RegistryModuleLoader();
+
+ private:
+ // Keep the handle for symbol resolution/calling.
+ void* handle_;
+ // Keep the path for debugging/logging.
+ std::string path_;
+
+ private:
+ FRIEND_TEST(RegistryTests, test_registry_modules);
+};
+
class RegistryFactory : private boost::noncopyable {
public:
static RegistryFactory& instance() {
* @endcode
*
* @param registry_name The canonical name for this registry.
- * @param auto_setup Optionally set false if the registry handles setup
+ * @param auto_setup Set true if the registry does not setup itself
* @return A non-sense int that must be casted const.
*/
template <class Type>
- static int create(const std::string& registry_name, bool auto_setup = true) {
- if (instance().registries_.count(registry_name) > 0) {
+ static int create(const std::string& registry_name, bool auto_setup = false) {
+ if (locked() || instance().registries_.count(registry_name) > 0) {
return 0;
}
static Status add(const std::string& registry_name,
const std::string& item_name,
bool internal = false) {
- auto registry = instance().registry(registry_name);
- return registry->template add<Item>(item_name, internal);
+ if (!locked()) {
+ auto registry = instance().registry(registry_name);
+ return registry->template add<Item>(item_name, internal);
+ }
+ return Status(0, "Registry locked");
}
/// Direct access to all registries.
const std::string& item_name,
const PluginRequest& request);
+ /// A helper call that uses the active plugin (if the registry has one).
+ static Status call(const std::string& registry_name,
+ const PluginRequest& request,
+ PluginResponse& response);
+
+ /// A helper call that uses the active plugin (if the registry has one).
+ static Status call(const std::string& registry_name,
+ const PluginRequest& request);
+
+ /// Set a registry's active plugin.
+ static Status setActive(const std::string& registry_name,
+ const std::string& item_name);
+
+ /// Get a registry's active plugin.
+ static const std::string& getActive(const std::string& registry_nane);
+
/// Run `setUp` on every registry that is not marked 'lazy'.
static void setUp();
const std::string& item_name,
bool local = false);
+ /// Get a list of the registry names.
+ static std::vector<std::string> names();
+
/// Get a list of the registry item names for a given registry.
static std::vector<std::string> names(const std::string& registry_name);
/// Check if duplicate registry items using registry aliasing are allowed.
static bool allowDuplicates() { return instance().allow_duplicates_; }
+ /// Declare a module for initialization and subsequent registration attempts
+ static void declareModule(const std::string& name,
+ const std::string& version,
+ const std::string& min_sdk_version,
+ const std::string& sdk_version);
+
+ /// Access module metadata.
+ static const std::map<RouteUUID, ModuleInfo>& getModules();
+
+ private:
+ /// Access the current initializing module UUID.
+ static RouteUUID getModule();
+
+ /// Check if the registry is allowing module registrations.
+ static bool usingModule();
+
+ /// Initialize a module for lookup, resolution, and its registrations.
+ static void initModule(const std::string& path);
+
+ static void shutdownModule();
+
+ /// Check if the registries are locked.
+ static bool locked() { return instance().locked_; }
+
+ /// Set the registry locked status.
+ static void locked(bool locked) { instance().locked_ = locked; }
+
protected:
- RegistryFactory() : allow_duplicates_(false) {}
+ RegistryFactory() : allow_duplicates_(false), locked_(false) {}
RegistryFactory(RegistryFactory const&);
void operator=(RegistryFactory const&);
virtual ~RegistryFactory() {}
private:
+ /// Track duplicate registry item support, used for testing.
bool allow_duplicates_;
+ /// Track registry "locking", while locked a registry cannot add/create.
+ bool locked_;
+
+ /// The primary storage for constructed registries.
std::map<std::string, PluginRegistryHelperRef> registries_;
+ /**
+ * @brief The registry tracks the set of active extension routes.
+ *
+ * If an extension dies (the process ends or does not respond to a ping),
+ * the registry will be notified via the extension watcher.
+ * When an operation requests to use that extension route the extension
+ * manager will lazily check the registry for changes.
+ */
std::set<RouteUUID> extensions_;
+
+ /**
+ * @brief The registry tracks loaded extension module metadata/info.
+ *
+ * Each extension module is assigned a transient RouteUUID for identification
+ * those route IDs are passed to each registry to identify which plugin
+ * items belong to modules, similarly to extensions.
+ */
+ std::map<RouteUUID, ModuleInfo> modules_;
+
+ // During module initialization store the current-working module ID.
+ RouteUUID module_uuid_;
+
+ private:
+ friend class RegistryHelperCore;
+ friend class RegistryModuleLoader;
+ FRIEND_TEST(RegistryTests, test_registry_modules);
};
/**
#include <osquery/tables.h>
namespace osquery {
-/// Anything built with only libosquery.a (SDK) will not include an SQL
-/// provider (aka "sql" registry).
+/**
+ * @brief Create the external SQLite implementation wrapper.
+ *
+ * Anything built with only libosquery and not the 'additional' library will
+ * not include a native SQL implementation. This applies to extensions and
+ * separate applications built with the osquery SDK.
+ *
+ * The ExternalSQLPlugin is a wrapper around the SQLite API, which forwards
+ * calls to an osquery extension manager (core).
+ */
REGISTER_INTERNAL(ExternalSQLPlugin, "sql", "sql");
+
+/**
+ * @brief Mimic the REGISTER macro, extensions should use this helper.
+ *
+ * The SDK does not provide a REGISTER macro for modules or extensions.
+ * Tools built with the osquery SDK should use REGISTER_EXTERNAL to add to
+ * their own 'external' registry. This registry will broadcast to the osquery
+ * extension manager (core) in an extension.
+ *
+ * osquery 'modules' should not construct their plugin registrations in
+ * global scope (global construction time). Instead they should use the
+ * module call-in well defined symbol, declare their SDK constraints, then
+ * use the REGISTER_MODULE call within `initModule`.
+ */
+#define REGISTER_EXTERNAL(type, registry, name) \
+ __attribute__((constructor)) static void type##ExtensionRegistryItem() { \
+ Registry::add<type>(registry, name); \
+ }
+
+/// Helper macro to write the `initModule` symbol without rewrites.
+#define DECLARE_MODULE(name) \
+ extern "C" void initModule(void); \
+ __attribute__((constructor)) static void declareModule()
+
+/**
+ * @brief Create an osquery extension 'module'.
+ *
+ * This helper macro creates a constructor to declare an osquery module is
+ * loading. The osquery registry is set up when modules (shared objects) are
+ * discovered via search paths and opened. At that phase the registry is locked
+ * meaning no additional plugins can be registered. To unlock the registry
+ * for modifications a module must call Registry::declareModule. This declares
+ * and any plugins added will use the metadata in the declare to determine:
+ * - The name of the module adding the plugin
+ * - The SDK version the module was built with, to determine compatibility
+ * - The minimum SDK the module requires from osquery core
+ *
+ * The registry is again locked when the module load is complete and a well
+ * known module-exported symbol is called.
+ */
+#define CREATE_MODULE(name, version, min_sdk_version) \
+ DECLARE_MODULE(name) { \
+ Registry::declareModule( \
+ name, version, min_sdk_version, OSQUERY_SDK_VERSION); \
+ }
+
+/**
+ * @brief Create an osquery extension 'module', if an expression is true.
+ *
+ * This is a helper testing wrapper around CREATE_MODULE and DECLARE_MODULE.
+ * It allows unit and integration tests to generate global construction code
+ * that depends on data/variables available during global construction.
+ *
+ * And example use includes checking if a process environment variable is
+ * defined. If defined the module is declared.
+ */
+#define CREATE_MODULE_IF(expr, name, version, min_sdk_version) \
+ DECLARE_MODULE(name) { \
+ if ((expr)) { \
+ Registry::declareModule( \
+ name, version, min_sdk_version, OSQUERY_SDK_VERSION); \
+ } \
+ }
+
+/// Helper replacement for REGISTER, used within extension modules.
+#define REGISTER_MODULE(type, registry, name) \
+ auto type##ModuleRegistryItem = Registry::add<type>(registry, name)
+
+// Remove registry-helper macros from the SDK.
+#undef REGISTER
+#define REGISTER "Do not REGISTER in the osquery SDK"
+#undef REGISTER_INTERNAL
+#define REGISTER_INTERNAL "Do not REGISTER_INTERNAL in the osquery SDK"
+#undef CREATE_REGISTRY
+#define CREATE_REGISTRY "Do not CREATE_REGISTRY in the osquery SDK"
+#undef CREATE_LAZY_REGISTRY
+#define CREATE_LAZY_REGISTRY "Do not CREATE_LAZY_REGISTRY in the osquery SDK"
}
}
};
-CREATE_REGISTRY(SQLPlugin, "sql");
+CREATE_LAZY_REGISTRY(SQLPlugin, "sql");
}
std::string columnDefinition(const TableColumns& columns);
std::string columnDefinition(const PluginResponse& response);
-CREATE_REGISTRY(TablePlugin, "table");
+CREATE_LAZY_REGISTRY(TablePlugin, "table");
}
}
typedef map<string, string> ExtensionPluginRequest
typedef list<map<string, string>> ExtensionPluginResponse
+/// Extensions should request osquery options to set active registries and
+/// bootstrap any config/logger plugins.
+struct InternalOptionInfo {
+ 1:string value,
+ 2:string default_value,
+ 3:string type,
+}
+
+/// Each option (CLI flag) has a unique name.
+typedef map<string, InternalOptionInfo> InternalOptionList
+
/// When communicating extension metadata, use a thrift-internal structure.
struct InternalExtensionInfo {
1:string name,
service ExtensionManager extends Extension {
/// Return the list of active registered extensions.
InternalExtensionList extensions(),
+ /// Return the list of bootstrap or configuration options.
+ InternalOptionList options(),
/// The API endpoint used by an extension to register its plugins.
ExtensionStatus registerExtension(
1:InternalExtensionInfo info,
TARGET_LINK_LIBRARIES(${TEST_NAME} ${TARGET_OSQUERY_LIB})
TARGET_LINK_LIBRARIES(${TEST_NAME} gtest)
+# SET_TARGET_PROPERTIES(${TARGET} PROPERTIES COMPILE_FLAGS "-DGTEST_HAS_TR1_TUPLE=0")
ADD_TEST(${TEST_NAME} ${TEST_NAME})
INSTALL(TARGETS ${TEST_NAME}
DESTINATION ${CMAKE_INSTALL_BINDIR}
TARGET_LINK_LIBRARIES(${TARGET} "-Wl,-no-whole-archive")
ENDMACRO(TARGET_OSQUERY_LINK_WHOLE)
+MACRO(ADD_OSQUERY_MODULE TARGET)
+ ADD_LIBRARY(${TARGET} SHARED ${ARGN})
+ TARGET_LINK_LIBRARIES(${TARGET} dl)
+ ADD_DEPENDENCIES(${TARGET} ${TARGET_OSQUERY_LIB} glog)
+ SET_TARGET_PROPERTIES(${TARGET} PROPERTIES COMPILE_FLAGS "-fPIC")
+ SET_TARGET_PROPERTIES(${TARGET} PROPERTIES OUTPUT_NAME ${TARGET})
+ENDMACRO(ADD_OSQUERY_MODULE)
+
ADD_SUBDIRECTORY(core)
ADD_SUBDIRECTORY(config)
ADD_SUBDIRECTORY(dispatcher)
# DESTINATION ${CMAKE_INSTALL_INCLUDEDIR})
## osqueryi generation ##########################################################
-ADD_EXECUTABLE(${TARGET_OSQUERY_SHELL} main/shell.cpp)
+ADD_EXECUTABLE(${TARGET_OSQUERY_SHELL} devtools/shell.cpp main/shell.cpp)
TARGET_LINK_LIBRARIES(${TARGET_OSQUERY_SHELL} ${TARGET_OSQUERY_LIB})
INSTALL(TARGETS ${TARGET_OSQUERY_SHELL}
DESTINATION ${CMAKE_INSTALL_BINDIR}
## example extension with the SDK ##############################################
ADD_EXECUTABLE(example_extension examples/example_extension.cpp)
TARGET_LINK_LIBRARIES(example_extension ${TARGET_OSQUERY_LIB})
-SET_TARGET_PROPERTIES(example_extension PROPERTIES OUTPUT_NAME example_extension)
+SET_TARGET_PROPERTIES(example_extension PROPERTIES OUTPUT_NAME example_extension.ext)
+
+# Build the example extension module with the SDK
+ADD_OSQUERY_MODULE(modexample examples/example_module.cpp)
+
#include <mutex>
#include <sstream>
-#include <boost/property_tree/ptree.hpp>
-#include <boost/property_tree/json_parser.hpp>
#include <boost/thread/shared_mutex.hpp>
#include <osquery/config.h>
#include <osquery/filesystem.h>
#include <osquery/logger.h>
-#include "osquery/core/watcher.h"
-
namespace pt = boost::property_tree;
+
+typedef pt::ptree::value_type tree_node;
typedef std::map<std::string, std::vector<std::string> > EventFileMap_t;
namespace osquery {
-FLAG(string, config_plugin, "filesystem", "Config type (plugin)");
+CLI_FLAG(string, config_plugin, "filesystem", "Config plugin name");
// This lock is used to protect the entirety of the OSqueryConfig struct
// Is should be used when ever accessing the structs members, reading or
static boost::shared_mutex rw_lock;
Status Config::load() {
+ auto& config_plugin = Registry::getActive("config");
+ if (!Registry::exists("config", config_plugin)) {
+ return Status(1, "Missing config plugin " + config_plugin);
+ }
+
boost::unique_lock<boost::shared_mutex> lock(rw_lock);
- OsqueryConfig conf;
- auto s = Config::genConfig(conf);
- if (!s.ok()) {
+ OsqueryConfig conf;
+ if (!genConfig(conf).ok()) {
return Status(1, "Cannot generate config");
}
if (Flag::isDefault(option.first)) {
// Only override if option was NOT given as an argument.
Flag::updateValue(option.first, option.second);
- if (!osquery::isOsqueryWorker()) {
- VLOG(1) << "Setting flag option: " << option.first << "="
- << option.second;
- }
+ VLOG(1) << "Setting flag option: " << option.first << "="
+ << option.second;
}
}
- cfg_ = conf;
+ getInstance().cfg_ = conf;
return Status(0, "OK");
}
-Status Config::genConfig(std::string& conf) {
- if (!Registry::exists("config", FLAGS_config_plugin)) {
- LOG(ERROR) << "Config retriever " << FLAGS_config_plugin << " not found";
- return Status(1, "Config retriever not found");
+Status Config::genConfig(std::vector<std::string>& conf) {
+ auto& config_plugin = Registry::getActive("config");
+ if (!Registry::exists("config", config_plugin)) {
+ return Status(1, "Missing config plugin " + config_plugin);
}
PluginResponse response;
- auto status = Registry::call(
- "config", FLAGS_config_plugin, {{"action", "genConfig"}}, response);
-
+ auto status = Registry::call("config", {{"action", "genConfig"}}, response);
if (!status.ok()) {
return status;
}
- conf = response[0].at("data");
+ if (response.size() > 0) {
+ for (const auto& it : response[0]) {
+ conf.push_back(it.second);
+ }
+ }
return Status(0, "OK");
}
+inline void mergeOption(const tree_node& option, OsqueryConfig& conf) {
+ conf.options[option.first.data()] = option.second.data();
+ conf.all_data.add_child("options." + option.first, option.second);
+}
+
+inline void mergeAdditional(const tree_node& node, OsqueryConfig& conf) {
+ conf.all_data.add_child("additional_monitoring." + node.first, node.second);
+
+ // Support special merging of file paths.
+ if (node.first != "file_paths") {
+ return;
+ }
+
+ for (const auto& category : node.second) {
+ for (const auto& path : category.second) {
+ resolveFilePattern(path.second.data(),
+ conf.eventFiles[category.first],
+ REC_LIST_FOLDERS | REC_EVENT_OPT);
+ }
+ }
+}
+
+inline void mergeScheduledQuery(const tree_node& node, OsqueryConfig& conf) {
+ // Read tree/JSON into a query structure.
+ OsqueryScheduledQuery query;
+ query.name = node.second.get<std::string>("name", "");
+ query.query = node.second.get<std::string>("query", "");
+ query.interval = node.second.get<int>("interval", 0);
+ // Also store the raw node in the property tree list.
+ conf.scheduledQueries.push_back(query);
+ conf.all_data.add_child("scheduledQueries", node.second);
+}
+
Status Config::genConfig(OsqueryConfig& conf) {
- std::string config_string;
- auto s = genConfig(config_string);
+ std::vector<std::string> configs;
+ auto s = genConfig(configs);
if (!s.ok()) {
return s;
}
- std::stringstream json;
- pt::ptree tree;
- try {
- json << config_string;
- pt::read_json(json, tree);
- // Parse each scheduled query from the config.
- for (const pt::ptree::value_type& v : tree.get_child("scheduledQueries")) {
- osquery::OsqueryScheduledQuery q;
- q.name = (v.second).get<std::string>("name");
- q.query = (v.second).get<std::string>("query");
- q.interval = (v.second).get<int>("interval");
- conf.scheduledQueries.push_back(q);
- }
- // Flags may be set as 'options' within the config.
- if (tree.count("options") > 0) {
- for (const pt::ptree::value_type& v : tree.get_child("options")) {
- conf.options[v.first.data()] = v.second.data();
+ for (const auto& config_data : configs) {
+ std::stringstream json_data;
+ json_data << config_data;
+
+ pt::ptree tree;
+ pt::read_json(json_data, tree);
+
+ if (tree.count("scheduledQueries") > 0) {
+ for (const auto& node : tree.get_child("scheduledQueries")) {
+ mergeScheduledQuery(node, conf);
}
}
if (tree.count("additional_monitoring") > 0) {
- for (const pt::ptree::value_type& v :
- tree.get_child("additional_monitoring")) {
- if (v.first == "file_paths") {
- for (const pt::ptree::value_type& file_cat : v.second) {
- for (const pt::ptree::value_type& file : file_cat.second) {
- osquery::resolveFilePattern(file.second.get_value<std::string>(),
- conf.eventFiles[file_cat.first]);
- }
- }
- }
+ for (const auto& node : tree.get_child("additional_monitoring")) {
+ mergeAdditional(node, conf);
}
}
- } catch (const std::exception& e) {
- LOG(ERROR) << "Error parsing config JSON: " << e.what();
- return Status(1, e.what());
- }
+ if (tree.count("options") > 0) {
+ for (const auto& option : tree.get_child("options")) {
+ mergeOption(option, conf);
+ }
+ }
+ }
return Status(0, "OK");
}
return getInstance().cfg_.scheduledQueries;
}
-std::map<std::string, std::vector<std::string> >& Config::getWatchedFiles() {
+std::map<std::string, std::vector<std::string> > Config::getWatchedFiles() {
boost::shared_lock<boost::shared_mutex> lock(rw_lock);
return getInstance().cfg_.eventFiles;
}
+pt::ptree Config::getEntireConfiguration() {
+ boost::shared_lock<boost::shared_mutex> lock(rw_lock);
+ return getInstance().cfg_.all_data;
+}
+
Status Config::getMD5(std::string& hash_string) {
- std::string config_string;
- auto s = genConfig(config_string);
- if (!s.ok()) {
- return s;
- }
+ std::stringstream out;
+ write_json(out, getEntireConfiguration());
hash_string = osquery::hashFromBuffer(
- HASH_TYPE_MD5, (void*)config_string.c_str(), config_string.length());
+ HASH_TYPE_MD5, (void*)out.str().c_str(), out.str().length());
return Status(0, "OK");
}
}
if (request.at("action") == "genConfig") {
- auto config_data = genConfig();
- response.push_back({{"data", config_data.second}});
- return config_data.first;
+ std::map<std::string, std::string> config;
+ auto stat = genConfig(config);
+ response.push_back(config);
+ return stat;
}
return Status(1, "Config plugin action unknown: " + request.at("action"));
}
* of patent rights can be found in the PATENTS file in the same directory.
*
*/
+#include <vector>
#include <gtest/gtest.h>
class ConfigTests : public testing::Test {
public:
ConfigTests() {
- FLAGS_config_plugin = "filesystem";
+ Registry::setActive("config", "filesystem");
FLAGS_config_path = kTestDataPath + "test.config";
}
void SetUp() {
createMockFileStructure();
Registry::setUp();
- Config::getInstance().load();
+ Config::load();
}
void TearDown() { tearDownMockFileStructure(); }
class TestConfigPlugin : public ConfigPlugin {
public:
TestConfigPlugin() {}
-
- std::pair<Status, std::string> genConfig() {
- return std::make_pair(Status(0, "OK"), "foobar");
+ Status genConfig(std::map<std::string, std::string>& config) {
+ config["data"] = "foobar";
+ return Status(0, "OK");
+ ;
}
};
TEST_F(ConfigTests, test_plugin) {
Registry::add<TestConfigPlugin>("config", "test");
+ // Change the active config plugin.
+ EXPECT_TRUE(Registry::setActive("config", "test").ok());
+
PluginResponse response;
- auto status =
- Registry::call("config", "test", {{"action", "genConfig"}}, response);
+ auto status = Registry::call("config", {{"action", "genConfig"}}, response);
EXPECT_EQ(status.ok(), true);
EXPECT_EQ(status.toString(), "OK");
TEST_F(ConfigTests, test_queries_execute) {
auto queries = Config::getInstance().getScheduledQueries();
- EXPECT_EQ(queries.size(), 1);
+ EXPECT_EQ(queries.size(), 2);
}
TEST_F(ConfigTests, test_threatfiles_execute) {
- auto files = Config::getInstance().getWatchedFiles();
+ auto files = Config::getWatchedFiles();
EXPECT_EQ(files.size(), 2);
- EXPECT_EQ(files["downloads"].size(), 9);
- EXPECT_EQ(files["system_binaries"].size(), 5);
+ EXPECT_EQ(files["downloads"].size(), 1);
+ EXPECT_EQ(files["system_binaries"].size(), 2);
}
}
*
*/
-#include <fstream>
+#include <vector>
#include <boost/filesystem/operations.hpp>
#include <osquery/config.h>
#include <osquery/flags.h>
#include <osquery/logger.h>
+#include <osquery/filesystem.h>
namespace fs = boost::filesystem;
-using osquery::Status;
namespace osquery {
-FLAG(string, config_path, "/var/osquery/osquery.conf", "Path to config file");
+CLI_FLAG(string,
+ config_path,
+ "/var/osquery/osquery.conf",
+ "(filesystem) config plugin path to JSON config file");
class FilesystemConfigPlugin : public ConfigPlugin {
public:
- virtual std::pair<osquery::Status, std::string> genConfig();
+ Status genConfig(std::map<std::string, std::string>& config);
};
REGISTER(FilesystemConfigPlugin, "config", "filesystem");
-std::pair<osquery::Status, std::string> FilesystemConfigPlugin::genConfig() {
- std::string config;
+Status FilesystemConfigPlugin::genConfig(
+ std::map<std::string, std::string>& config) {
if (!fs::exists(FLAGS_config_path)) {
- return std::make_pair(Status(1, "config file does not exist"), config);
+ return Status(1, "config file does not exist");
}
- VLOG(1) << "Filesystem ConfigPlugin reading: " << FLAGS_config_path;
- std::ifstream config_stream(FLAGS_config_path);
+ std::vector<std::string> conf_files;
+ resolveFilePattern(FLAGS_config_path + ".d/%.conf", conf_files);
+ if (conf_files.size() > 0) {
+ VLOG(1) << "Discovered (" << conf_files.size() << ") additional configs";
+ }
- config_stream.seekg(0, std::ios::end);
- config.reserve(config_stream.tellg());
- config_stream.seekg(0, std::ios::beg);
+ std::sort(conf_files.begin(), conf_files.end());
+ conf_files.push_back(FLAGS_config_path);
- config.assign((std::istreambuf_iterator<char>(config_stream)),
- std::istreambuf_iterator<char>());
- return std::make_pair(Status(0, "OK"), config);
+ for (const auto& path : conf_files) {
+ std::string content;
+ if (readFile(path, content).ok()) {
+ config[path] = content;
+ }
+ }
+ return Status(0, "OK");
}
}
hash.cpp
watcher.cpp)
+# TODO(Sangwan): Detach from core
ADD_OSQUERY_LIBRARY(TRUE osquery_test_util test_util.cpp)
ADD_OSQUERY_TEST(TRUE osquery_flags_tests flags_tests.cpp)
return 0;
}
-int Flag::createAlias(const std::string& alias, const std::string& name) {
- instance().aliases_.insert(std::make_pair(alias, name));
+int Flag::createAlias(const std::string& alias, const FlagDetail& flag) {
+ instance().aliases_.insert(std::make_pair(alias, flag));
return 0;
}
return info.type;
}
+std::string Flag::getDescription(const std::string& name) {
+ if (instance().flags_.count(name)) {
+ return instance().flags_.at(name).description;
+ }
+
+ if (instance().aliases_.count(name)) {
+ return getDescription(instance().aliases_.at(name).description);
+ }
+ return "";
+}
+
Status Flag::updateValue(const std::string& name, const std::string& value) {
GFLAGS_NAMESPACE::SetCommandLineOption(name.c_str(), value.c_str());
return Status(0, "OK");
return flags;
}
-void Flag::printFlags(bool shell, bool external) {
+void Flag::printFlags(bool shell, bool external, bool cli) {
std::vector<GFLAGS_NAMESPACE::CommandLineFlagInfo> info;
GFLAGS_NAMESPACE::GetAllFlags(&info);
}
max += 7;
+ auto& aliases = instance().aliases_;
auto& details = instance().flags_;
for (const auto& flag : info) {
- if (details.count(flag.name) == 0) {
- // This flag was not defined within osquery code, skip.
- continue;
- }
-
- const auto& detail = details.at(flag.name);
- if ((shell && !detail.shell) || (!shell && detail.shell)) {
- continue;
- }
-
- if ((external && !detail.external) || (!external && detail.external)) {
+ if (details.count(flag.name) > 0) {
+ const auto& detail = details.at(flag.name);
+ if ((shell && !detail.shell) || (!shell && detail.shell) ||
+ (external && !detail.external) || (!external && detail.external) ||
+ (cli && !detail.cli) || (!cli && detail.cli)) {
+ continue;
+ }
+ } else if (aliases.count(flag.name) > 0) {
+ const auto& alias = aliases.at(flag.name);
+ // Aliases are only printed if this is an external tool and the alias
+ // is external.
+ if (!alias.external || !external) {
+ continue;
+ }
+ } else {
+ // This flag was not defined as an osquery flag or flag alias.
continue;
}
}
fprintf(stdout, "%s", std::string(pad - flag.name.size(), ' ').c_str());
- fprintf(stdout, "%s\n", detail.description.c_str());
+ fprintf(stdout, "%s\n", getDescription(flag.name).c_str());
}
}
}
int main(int argc, char* argv[]) {
testing::InitGoogleTest(&argc, argv);
- osquery::initOsquery(argc, argv);
return RUN_ALL_TESTS();
}
*
*/
+#include <pwd.h>
#include <syslog.h>
#include <time.h>
+#include <boost/algorithm/string/trim.hpp>
+#include <boost/filesystem.hpp>
+
#include <osquery/config.h>
#include <osquery/core.h>
#include <osquery/events.h>
+#include <osquery/extensions.h>
#include <osquery/flags.h>
#include <osquery/filesystem.h>
#include <osquery/logger.h>
#include <osquery/registry.h>
+#include "osquery/core/watcher.h"
+
namespace osquery {
-const std::string kDescription =
- "your operating system as a high-performance "
- "relational database";
-const std::string kEpilog = "osquery project page <http://osquery.io>.";
+#define DESCRIPTION \
+ "osquery %s, your OS as a high-performance relational database\n"
+#define EPILOG "\nosquery project page <http://osquery.io>.\n"
+#define OPTIONS \
+ "\nosquery configuration options (set by config or CLI flags):\n\n"
+#define OPTIONS_SHELL "\nosquery shell-only CLI flags:\n\n"
+#define OPTIONS_CLI "osquery%s command line flags:\n\n"
+#define USAGE "Usage: %s [OPTION]... %s\n\n"
+#define CONFIG_ERROR \
+ "You are using default configurations for osqueryd for one or more of the " \
+ "following\n" \
+ "flags: pidfile, db_path.\n\n" \
+ "These options create files in /var/osquery but it looks like that path " \
+ "has not\n" \
+ "been created. Please consider explicitly defining those " \
+ "options as a different \n" \
+ "path. Additionally, review the \"using osqueryd\" wiki page:\n" \
+ " - https://github.com/facebook/osquery/wiki/using-osqueryd\n\n";
-FLAG(bool, config_check, false, "Check the format of an osquery config");
+CLI_FLAG(bool,
+ config_check,
+ false,
+ "Check the format of an osquery config and exit");
#ifndef __APPLE__
-namespace osquery {
-FLAG(bool, daemonize, false, "Run as daemon (osqueryd only)");
-}
+CLI_FLAG(bool, daemonize, false, "Run as daemon (osqueryd only)");
#endif
namespace fs = boost::filesystem;
void printUsage(const std::string& binary, int tool) {
// Parse help options before gflags. Only display osquery-related options.
- fprintf(stdout, "osquery " OSQUERY_VERSION ", %s\n", kDescription.c_str());
+ fprintf(stdout, DESCRIPTION, OSQUERY_VERSION);
if (tool == OSQUERY_TOOL_SHELL) {
// The shell allows a caller to run a single SQL statement and exit.
- fprintf(
- stdout, "Usage: %s [OPTION]... [SQL STATEMENT]\n\n", binary.c_str());
+ fprintf(stdout, USAGE, binary.c_str(), "[SQL STATEMENT]");
} else {
- fprintf(stdout, "Usage: %s [OPTION]...\n\n", binary.c_str());
+ fprintf(stdout, USAGE, binary.c_str(), "");
}
- fprintf(stdout, "The following options control osquery.\n\n");
- // Print only the core/internal flags.
- Flag::printFlags();
+ if (tool == OSQUERY_EXTENSION) {
+ fprintf(stdout, OPTIONS_CLI, " extension");
+ Flag::printFlags(false, true);
+ } else {
+ fprintf(stdout, OPTIONS_CLI, "");
+ Flag::printFlags(false, false, true);
+ fprintf(stdout, OPTIONS);
+ Flag::printFlags();
+ }
if (tool == OSQUERY_TOOL_SHELL) {
// Print shell flags.
- fprintf(stdout, "\nThe following control the osquery shell.\n\n");
+ fprintf(stdout, OPTIONS_SHELL);
Flag::printFlags(true);
}
- fprintf(stdout, "\n%s\n", kEpilog.c_str());
-}
-
-void printConfigWarning() {
- std::cerr << "You are using default configurations for osqueryd for one or "
- "more of the following\n"
- << "flags: pidfile, db_path.\n\n"
- << "These options create files in /var/osquery but it looks like "
- "that path has not\n"
- << "been created. Please consider explicitly defining those "
- "options as a different \n"
- << "path. Additionally, review the \"using osqueryd\" wiki page:\n"
- << " - https://github.com/facebook/osquery/wiki/using-osqueryd\n\n";
+ fprintf(stdout, EPILOG);
}
-void announce() {
- syslog(LOG_NOTICE, "osqueryd started [version=" OSQUERY_VERSION "]");
-}
-
-void initOsquery(int argc, char* argv[], int tool) {
+Initializer::Initializer(int argc, char* argv[], ToolType tool)
+ : argc_(argc),
+ argv_((char**)argv),
+ tool_(tool),
+ binary_(fs::path(std::string(argv[0])).filename().string()) {
std::srand(time(nullptr));
- std::string binary(fs::path(std::string(argv[0])).filename().string());
- std::string first_arg = (argc > 1) ? std::string(argv[1]) : "";
// osquery implements a custom help/usage output.
+ std::string first_arg = (argc_ > 1) ? std::string(argv_[1]) : "";
if ((first_arg == "--help" || first_arg == "-h" || first_arg == "-help") &&
tool != OSQUERY_TOOL_TEST) {
- printUsage(binary, tool);
+ printUsage(binary_, tool_);
::exit(0);
}
FLAGS_logger_plugin = STR(OSQUERY_DEFAULT_LOGGER_PLUGIN);
#endif
+ if (tool == OSQUERY_TOOL_SHELL) {
+ // The shell is transient, rewrite config-loaded paths.
+ osquery::FLAGS_disable_logging = true;
+
+ // Get the caller's home dir for temporary storage/state management.
+ auto user = getpwuid(getuid());
+ std::string homedir;
+ if (getenv("HOME") != nullptr) {
+ homedir = std::string(getenv("HOME")) + "/.osquery";
+ } else if (user != nullptr || user->pw_dir != nullptr) {
+ homedir = std::string(user->pw_dir) + "/.osquery";
+ } else {
+ homedir = "/tmp/osquery";
+ }
+
+ if (osquery::pathExists(homedir).ok() ||
+ boost::filesystem::create_directory(homedir)) {
+ osquery::FLAGS_database_path = homedir + "/shell.db";
+ osquery::FLAGS_extensions_socket = homedir + "/shell.em";
+ }
+ }
+
// Set version string from CMake build
GFLAGS_NAMESPACE::SetVersionString(OSQUERY_VERSION);
// Let gflags parse the non-help options/flags.
- GFLAGS_NAMESPACE::ParseCommandLineFlags(&argc, &argv, false);
+ GFLAGS_NAMESPACE::ParseCommandLineFlags(&argc_, &argv_, false);
- // Initialize the status and results logger.
- initStatusLogger(binary);
- VLOG(1) << "osquery initialized [version=" OSQUERY_VERSION "]";
+ // If the caller is checking configuration, disable the watchdog/worker.
+ if (FLAGS_config_check) {
+ FLAGS_disable_watchdog = true;
+ }
+ // Initialize the status and results logger.
+ initStatusLogger(binary_);
if (tool != OSQUERY_EXTENSION) {
- // Load the osquery config using the default/active config plugin.
- Config::getInstance().load();
-
- if (FLAGS_config_check) {
- // The initiator requested an initialization and config check.
- auto s = Config::checkConfig();
- if (!s.ok()) {
- std::cerr << "Error reading config: " << s.toString() << "\n";
- }
- // A configuration check exits the application.
- ::exit(s.getCode());
- }
+ VLOG(1) << "osquery initialized [version=" << OSQUERY_VERSION << "]";
+ } else {
+ VLOG(1) << "osquery extension initialized [sdk=" << OSQUERY_SDK_VERSION
+ << "]";
}
-
- // Run the setup for all non-lazy registries.
- Registry::setUp();
- // Initialize the status and result plugin logger.
- initLogger(binary);
}
-void initOsqueryDaemon() {
+void Initializer::initDaemon() {
#ifndef __APPLE__
// OSX uses launchd to daemonize.
if (osquery::FLAGS_daemonize) {
#endif
// Print the version to SYSLOG.
- announce();
+ syslog(
+ LOG_NOTICE, "%s started [version=%s]", binary_.c_str(), OSQUERY_VERSION);
// check if /var/osquery exists
if ((Flag::isDefault("pidfile") || Flag::isDefault("db_path")) &&
!isDirectory("/var/osquery")) {
- printConfigWarning();
+ std::cerr << CONFIG_ERROR
}
// Create a process mutex around the daemon.
auto pid_status = createPidFile();
if (!pid_status.ok()) {
- LOG(ERROR) << "osqueryd initialize failed: " << pid_status.toString();
+ LOG(ERROR) << binary_ << " initialize failed: " << pid_status.toString();
::exit(EXIT_FAILURE);
}
+}
- // Check the backing store by allocating and exitting on error.
- if (!DBHandle::checkDB()) {
- LOG(ERROR) << "osqueryd initialize failed: Could not create DB handle";
+void Initializer::initWatcher() {
+ // The watcher takes a list of paths to autoload extensions from.
+ loadExtensions();
+
+ // Add a watcher service thread to start/watch an optional worker and set
+ // of optional extensions in the autoload paths.
+ if (Watcher::hasManagedExtensions() || !FLAGS_disable_watchdog) {
+ Dispatcher::getInstance().addService(
+ std::make_shared<WatcherRunner>(argc_, argv_, !FLAGS_disable_watchdog));
+ }
+
+ // If there are no autoloaded extensions, the watcher service will end,
+ // otherwise it will continue as a background thread and respawn them.
+ // If the watcher is also a worker watchdog it will do nothing but monitor
+ // the extensions and worker process.
+ if (!FLAGS_disable_watchdog) {
+ Dispatcher::joinServices();
+ // Executation should never reach this point.
::exit(EXIT_FAILURE);
}
}
-void shutdownOsquery() {
+void Initializer::initWorker(const std::string& name) {
+ // Set the worker's process name.
+ size_t name_size = strlen(argv_[0]);
+ for (int i = 0; i < argc_; i++) {
+ if (argv_[i] != nullptr) {
+ memset(argv_[i], 0, strlen(argv_[i]));
+ }
+ }
+ strncpy(argv_[0], name.c_str(), name_size);
+
+ // Start a watcher watcher thread to exit the process if the watcher exits.
+ Dispatcher::getInstance().addService(
+ std::make_shared<WatcherWatcherRunner>(getppid()));
+}
+
+void Initializer::initWorkerWatcher(const std::string& name) {
+ if (isWorker()) {
+ initWorker(name);
+ } else {
+ // The watcher will forever monitor and spawn additional workers.
+ initWatcher();
+ }
+}
+
+bool Initializer::isWorker() { return (getenv("OSQUERY_WORKER") != nullptr); }
+
+void Initializer::initConfigLogger() {
+ // Use a delay, meaning the amount of milliseconds waited for extensions.
+ size_t delay = 0;
+ // The timeout is the maximum time in seconds to wait for extensions.
+ size_t timeout = atoi(FLAGS_extensions_timeout.c_str());
+ while (!Registry::setActive("config", FLAGS_config_plugin)) {
+ // If there is at least 1 autoloaded extension, it may broadcast a route
+ // to the active config plugin.
+ if (!Watcher::hasManagedExtensions() || delay > timeout * 1000) {
+ LOG(ERROR) << "Config plugin not found: " << FLAGS_config_plugin;
+ ::exit(EXIT_CATASTROPHIC);
+ }
+ ::usleep(kExtensionInitializeMLatency * 1000);
+ delay += kExtensionInitializeMLatency;
+ }
+
+ // Try the same wait for a logger pluing too.
+ while (!Registry::setActive("logger", FLAGS_logger_plugin)) {
+ if (!Watcher::hasManagedExtensions() || delay > timeout * 1000) {
+ LOG(ERROR) << "Logger plugin not found: " << FLAGS_logger_plugin;
+ ::exit(EXIT_CATASTROPHIC);
+ }
+ ::usleep(kExtensionInitializeMLatency * 1000);
+ delay += kExtensionInitializeMLatency;
+ }
+}
+
+void Initializer::start() {
+ // Load registry/extension modules before extensions.
+ osquery::loadModules();
+
+ // Bind to an extensions socket and wait for registry additions.
+ osquery::startExtensionManager();
+
+ // Then set the config/logger plugins, which use a single/active plugin.
+ initConfigLogger();
+
+ // Run the setup for all lazy registries (tables, SQL).
+ Registry::setUp();
+
+ if (FLAGS_config_check) {
+ // The initiator requested an initialization and config check.
+ auto s = Config::checkConfig();
+ if (!s.ok()) {
+ std::cerr << "Error reading config: " << s.toString() << "\n";
+ }
+ // A configuration check exits the application.
+ ::exit(s.getCode());
+ }
+
+ // Load the osquery config using the default/active config plugin.
+ Config::load();
+
+ // Check the backing store by allocating and exiting on error.
+ if (!DBHandle::checkDB()) {
+ LOG(ERROR) << binary_ << " initialize failed: Could not create DB handle";
+ if (isWorker()) {
+ ::exit(EXIT_CATASTROPHIC);
+ } else {
+ ::exit(EXIT_FAILURE);
+ }
+ }
+
+ // Initialize the status and result plugin logger.
+ initLogger(binary_);
+
+ // Start event threads.
+ osquery::attachEvents();
+ osquery::EventFactory::delay();
+}
+
+void Initializer::shutdown() {
// End any event type run loops.
EventFactory::end();
namespace osquery {
/// The path to the pidfile for osqueryd
-FLAG(string,
- pidfile,
- "/var/osquery/osqueryd.pidfile",
- "The path to the pidfile for osqueryd");
+CLI_FLAG(string,
+ pidfile,
+ "/var/osquery/osqueryd.pidfile",
+ "Path to the daemon pidfile mutex");
/// Should the daemon force unload previously-running osqueryd daemons.
-FLAG(bool, force, false, "Force osqueryd to kill previously-running daemons");
+CLI_FLAG(bool,
+ force,
+ false,
+ "Force osqueryd to kill previously-running daemons");
std::string getHostname() {
char hostname[256]; // Linux max should be 64.
int main(int argc, char* argv[]) {
testing::InitGoogleTest(&argc, argv);
- osquery::initOsquery(argc, argv);
return RUN_ALL_TESTS();
}
"/deep11/deep2/deep3/");
boost::filesystem::create_directories(kFakeDirectory + "/deep1/deep2/");
writeTextFile(kFakeDirectory + "/root.txt", "root");
- writeTextFile(kFakeDirectory + "/toor.txt", "toor");
+ writeTextFile(kFakeDirectory + "/door.txt", "toor");
writeTextFile(kFakeDirectory + "/roto.txt", "roto");
writeTextFile(kFakeDirectory + "/deep1/level1.txt", "l1");
writeTextFile(kFakeDirectory + "/deep11/not_bash", "l1");
*/
#include <cstring>
-#include <sstream>
#include <sys/wait.h>
#include <signal.h>
#include <boost/filesystem.hpp>
-#include <osquery/core.h>
#include <osquery/events.h>
#include <osquery/filesystem.h>
#include <osquery/logger.h>
const std::map<WatchdogLimitType, std::vector<size_t> > kWatchdogLimits = {
// Maximum MB worker can privately allocate.
- {MEMORY_LIMIT, {50, 20, 10, 10}},
+ {MEMORY_LIMIT, {50, 30, 10, 10}},
// Percent of user or system CPU worker can utilize for LATENCY_LIMIT
// seconds.
- {UTILIZATION_LIMIT, {90, 70, 60, 50}},
+ {UTILIZATION_LIMIT, {90, 80, 60, 50}},
// Number of seconds the worker should run, else consider the exit fatal.
{RESPAWN_LIMIT, {20, 20, 20, 5}},
// If the worker respawns too quickly, backoff on creating additional.
{RESPAWN_DELAY, {5, 5, 5, 1}},
// Seconds of tolerable UTILIZATION_LIMIT sustained latency.
- {LATENCY_LIMIT, {5, 5, 3, 1}},
+ {LATENCY_LIMIT, {12, 6, 3, 1}},
// How often to poll for performance limit violations.
{INTERVAL, {3, 3, 3, 1}}, };
-FLAG(int32,
- watchdog_level,
- 1,
- "Performance limit level (0=loose, 1=normal, 2=restrictive, 3=debug)");
+const std::string kExtensionExtension = ".ext";
-FLAG(bool, disable_watchdog, false, "Disable userland watchdog process");
+CLI_FLAG(int32,
+ watchdog_level,
+ 1,
+ "Performance limit level (0=loose, 1=normal, 2=restrictive, 3=debug)");
-bool Watcher::ok() {
- ::sleep(getWorkerLimit(INTERVAL));
- return (worker_ >= 0);
+CLI_FLAG(bool, disable_watchdog, false, "Disable userland watchdog process");
+
+/// If the worker exits the watcher will inspect the return code.
+void childHandler(int signum) {
+ siginfo_t info;
+ // Make sure WNOWAIT is used to the wait information is not removed.
+ // Watcher::watch implements a thread to poll for this information.
+ waitid(P_ALL, 0, &info, WEXITED | WSTOPPED | WNOHANG | WNOWAIT);
+ if (info.si_code == CLD_EXITED && info.si_status == EXIT_CATASTROPHIC) {
+ // A child process had a catastrophic error, abort the watcher.
+ ::exit(EXIT_FAILURE);
+ }
+}
+
+void Watcher::resetWorkerCounters(size_t respawn_time) {
+ // Reset the monitoring counters for the watcher.
+ auto& state = instance().state_;
+ state.sustained_latency = 0;
+ state.user_time = 0;
+ state.system_time = 0;
+ state.last_respawn_time = respawn_time;
+}
+
+void Watcher::resetExtensionCounters(const std::string& extension,
+ size_t respawn_time) {
+ WatcherLocker locker;
+ auto& state = instance().extension_states_[extension];
+ state.sustained_latency = 0;
+ state.user_time = 0;
+ state.system_time = 0;
+ state.last_respawn_time = respawn_time;
+}
+
+std::string Watcher::getExtensionPath(pid_t child) {
+ for (const auto& extension : extensions()) {
+ if (extension.second == child) {
+ return extension.first;
+ }
+ }
+ return "";
+}
+
+void Watcher::removeExtensionPath(const std::string& extension) {
+ WatcherLocker locker;
+ instance().extensions_.erase(extension);
+ instance().extension_states_.erase(extension);
+}
+
+PerformanceState& Watcher::getState(pid_t child) {
+ if (child == instance().worker_) {
+ return instance().state_;
+ } else {
+ return instance().extension_states_[getExtensionPath(child)];
+ }
+}
+
+PerformanceState& Watcher::getState(const std::string& extension) {
+ return instance().extension_states_[extension];
}
-bool Watcher::watch() {
+void Watcher::setExtension(const std::string& extension, pid_t child) {
+ WatcherLocker locker;
+ instance().extensions_[extension] = child;
+}
+
+void Watcher::reset(pid_t child) {
+ if (child == instance().worker_) {
+ instance().worker_ = 0;
+ resetWorkerCounters(0);
+ return;
+ }
+
+ // If it was not the worker pid then find the extension name to reset.
+ for (const auto& extension : extensions()) {
+ if (extension.second == child) {
+ setExtension(extension.first, 0);
+ resetExtensionCounters(extension.first, 0);
+ }
+ }
+}
+
+void Watcher::addExtensionPath(const std::string& path) {
+ // Resolve acceptable extension binaries from autoload paths.
+ if (isDirectory(path).ok()) {
+ VLOG(1) << "Cannot autoload extension from directory: " << path;
+ return;
+ }
+
+ // Only autoload extensions which were safe at the time of discovery.
+ // If the extension binary later becomes unsafe (permissions change) then
+ // it will fail to reload if a reload is ever needed.
+ fs::path extension(path);
+ if (safePermissions(extension.parent_path().string(), path, true)) {
+ if (extension.extension().string() == kExtensionExtension) {
+ setExtension(extension.string(), 0);
+ resetExtensionCounters(extension.string(), 0);
+ VLOG(1) << "Found autoloadable extension: " << extension.string();
+ }
+ }
+}
+
+bool Watcher::hasManagedExtensions() {
+ if (instance().extensions_.size() > 0) {
+ return true;
+ }
+
+ // A watchdog process may hint to a worker the number of managed extensions.
+ // Setting this counter to 0 will prevent the worker from waiting for missing
+ // dependent config plugins. Otherwise, its existance, will cause a worker to
+ // wait for missing plugins to broadcast from managed extensions.
+ return (getenv("OSQUERY_EXTENSIONS") != nullptr);
+}
+
+bool WatcherRunner::ok() {
+ interruptableSleep(getWorkerLimit(INTERVAL) * 1000);
+ // Watcher is OK to run if a worker or at least one extension exists.
+ return (Watcher::getWorker() >= 0 || Watcher::hasManagedExtensions());
+}
+
+void WatcherRunner::enter() {
+ // Set worker performance counters to an initial state.
+ Watcher::resetWorkerCounters(0);
+ signal(SIGCHLD, childHandler);
+
+ // Enter the watch loop.
+ do {
+ if (use_worker_ && !watch(Watcher::getWorker())) {
+ // The watcher failed, create a worker.
+ createWorker();
+ }
+
+ // Loop over every managed extension and check sanity.
+ std::vector<std::string> failing_extensions;
+ for (const auto& extension : Watcher::extensions()) {
+ if (!watch(extension.second)) {
+ if (!createExtension(extension.first)) {
+ failing_extensions.push_back(extension.first);
+ }
+ }
+ }
+ // If any extension creations failed, stop managing them.
+ for (const auto& failed_extension : failing_extensions) {
+ Watcher::removeExtensionPath(failed_extension);
+ }
+ } while (ok());
+}
+
+bool WatcherRunner::watch(pid_t child) {
int status;
- pid_t result = waitpid(worker_, &status, WNOHANG);
- if (worker_ == 0 || result == worker_) {
+ pid_t result = waitpid(child, &status, WNOHANG);
+ if (child == 0 || result == child) {
// Worker does not exist or never existed.
return false;
} else if (result == 0) {
// If the inspect finds problems it will stop/restart the worker.
- if (!isWorkerSane()) {
- stopWorker();
+ if (!isChildSane(child)) {
+ stopChild(child);
return false;
}
}
return true;
}
-void Watcher::stopWorker() {
- kill(worker_, SIGKILL);
- worker_ = 0;
+void WatcherRunner::stopChild(pid_t child) {
+ kill(child, SIGKILL);
+ child = 0;
+
// Clean up the defunct (zombie) process.
- waitpid(-1, 0, 0);
+ waitpid(-1, 0, WNOHANG);
}
-bool Watcher::isWorkerSane() {
+bool WatcherRunner::isChildSane(pid_t child) {
auto rows =
- SQL::selectAllFrom("processes", "pid", tables::EQUALS, INTEGER(worker_));
+ SQL::selectAllFrom("processes", "pid", tables::EQUALS, INTEGER(child));
if (rows.size() == 0) {
// Could not find worker process?
return false;
}
+ // Get the performance state for the worker or extension.
+ size_t sustained_latency = 0;
// Compare CPU utilization since last check.
- BIGINT_LITERAL footprint, user_time, system_time;
+ BIGINT_LITERAL footprint, user_time, system_time, parent;
// IV is the check interval in seconds, and utilization is set per-second.
auto iv = getWorkerLimit(INTERVAL);
- try {
- user_time = AS_LITERAL(BIGINT_LITERAL, rows[0].at("user_time")) / iv;
- system_time = AS_LITERAL(BIGINT_LITERAL, rows[0].at("system_time")) / iv;
- footprint = AS_LITERAL(BIGINT_LITERAL, rows[0].at("phys_footprint"));
- } catch (const std::exception& e) {
- sustained_latency_ = 0;
- }
+ {
+ WatcherLocker locker;
+ auto state = Watcher::getState(child);
+ try {
+ parent = AS_LITERAL(BIGINT_LITERAL, rows[0].at("parent"));
+ user_time = AS_LITERAL(BIGINT_LITERAL, rows[0].at("user_time")) / iv;
+ system_time = AS_LITERAL(BIGINT_LITERAL, rows[0].at("system_time")) / iv;
+ footprint = AS_LITERAL(BIGINT_LITERAL, rows[0].at("phys_footprint"));
+ } catch (const std::exception& e) {
+ state.sustained_latency = 0;
+ }
- if (current_user_time_ + getWorkerLimit(UTILIZATION_LIMIT) < user_time ||
- current_system_time_ + getWorkerLimit(UTILIZATION_LIMIT) < system_time) {
- sustained_latency_++;
- } else {
- sustained_latency_ = 0;
+ // Check the different of CPU time used since last check.
+ if (state.user_time + getWorkerLimit(UTILIZATION_LIMIT) < user_time ||
+ state.system_time + getWorkerLimit(UTILIZATION_LIMIT) < system_time) {
+ state.sustained_latency++;
+ } else {
+ state.sustained_latency = 0;
+ }
+ // Update the current CPU time.
+ state.user_time = user_time;
+ state.system_time = system_time;
+
+ // Check if the sustained difference exceeded the acceptable latency limit.
+ sustained_latency = state.sustained_latency;
}
- current_user_time_ = user_time;
- current_system_time_ = system_time;
+ // Only make a decision about the child sanity if it is still the watcher's
+ // child. It's possible for the child to die, and its pid reused.
+ if (parent != getpid()) {
+ // The child's parent is not the watcher.
+ Watcher::reset(child);
+ // Do not stop or call the child insane, since it is not our child.
+ return true;
+ }
- if (sustained_latency_ * iv >= getWorkerLimit(LATENCY_LIMIT)) {
+ if (sustained_latency > 0 &&
+ sustained_latency * iv >= getWorkerLimit(LATENCY_LIMIT)) {
LOG(WARNING) << "osqueryd worker system performance limits exceeded";
return false;
}
- if (footprint > getWorkerLimit(MEMORY_LIMIT) * 1024 * 1024) {
- LOG(WARNING) << "osqueryd worker memory limits exceeded";
+ // Check if the private memory exceeds a memory limit.
+ if (footprint > 0 && footprint > getWorkerLimit(MEMORY_LIMIT) * 1024 * 1024) {
+ LOG(WARNING) << "osqueryd worker memory limits exceeded: " << footprint;
return false;
}
return true;
}
-void Watcher::createWorker() {
- if (last_respawn_time_ > getUnixTime() - getWorkerLimit(RESPAWN_LIMIT)) {
- LOG(WARNING) << "osqueryd worker respawning too quickly";
- ::sleep(getWorkerLimit(RESPAWN_DELAY));
+void WatcherRunner::createWorker() {
+ {
+ WatcherLocker locker;
+ if (Watcher::getState(Watcher::getWorker()).last_respawn_time >
+ getUnixTime() - getWorkerLimit(RESPAWN_LIMIT)) {
+ LOG(WARNING) << "osqueryd worker respawning too quickly";
+ interruptableSleep(getWorkerLimit(RESPAWN_DELAY) * 1000);
+ }
}
// Get the path of the current process.
::exit(EXIT_FAILURE);
}
- worker_ = fork();
- if (worker_ < 0) {
+ // Set an environment signaling to potential plugin-dependent workers to wait
+ // for extensions to broadcast.
+ if (Watcher::hasManagedExtensions()) {
+ setenv("OSQUERY_EXTENSIONS", "true", 1);
+ }
+
+ auto worker_pid = fork();
+ if (worker_pid < 0) {
// Unrecoverable error, cannot create a worker process.
LOG(ERROR) << "osqueryd could not create a worker process";
::exit(EXIT_FAILURE);
- } else if (worker_ == 0) {
+ } else if (worker_pid == 0) {
// This is the new worker process, no watching needed.
- setenv("OSQUERYD_WORKER", std::to_string(getpid()).c_str(), 1);
+ setenv("OSQUERY_WORKER", std::to_string(getpid()).c_str(), 1);
// Get the complete path of the osquery process binary.
auto exec_path = fs::system_complete(fs::path(qd[0]["path"]));
execve(exec_path.string().c_str(), argv_, environ);
- // Code will never reach this point.
- ::exit(EXIT_FAILURE);
+ // Code should never reach this point.
+ LOG(ERROR) << "osqueryd could not start worker process";
+ ::exit(EXIT_CATASTROPHIC);
}
+ Watcher::setWorker(worker_pid);
+ Watcher::resetWorkerCounters(getUnixTime());
VLOG(1) << "osqueryd watcher (" << getpid() << ") executing worker ("
- << worker_ << ")";
+ << worker_pid << ")";
}
-void Watcher::resetCounters() {
- // Reset the monitoring counters for the watcher.
- sustained_latency_ = 0;
- current_user_time_ = 0;
- current_system_time_ = 0;
- last_respawn_time_ = getUnixTime();
-}
-
-void Watcher::initWorker() {
- // Set the worker's process name.
- size_t name_size = strlen(argv_[0]);
- for (int i = 0; i < argc_; i++) {
- if (argv_[i] != nullptr) {
- memset(argv_[i], 0, strlen(argv_[i]));
+bool WatcherRunner::createExtension(const std::string& extension) {
+ {
+ WatcherLocker locker;
+ if (Watcher::getState(extension).last_respawn_time >
+ getUnixTime() - getWorkerLimit(RESPAWN_LIMIT)) {
+ LOG(WARNING) << "Extension respawning too quickly: " << extension;
+ // Unlike a worker, if an extension respawns to quickly we give up.
+ return false;
}
}
- strncpy(argv_[0], name_.c_str(), name_size);
- // Start a watcher watcher thread to exit the process if the watcher exits.
- Dispatcher::getInstance().addService(
- std::make_shared<WatcherWatcherRunner>(getppid()));
-}
+ // Check the path to the previously-discovered extension binary.
+ auto exec_path = fs::system_complete(fs::path(extension));
+ if (!safePermissions(
+ exec_path.parent_path().string(), exec_path.string(), true)) {
+ // Extension binary has become unsafe.
+ LOG(WARNING) << "Extension binary has unsafe permissions: " << extension;
+ return false;
+ }
+
+ auto ext_pid = fork();
+ if (ext_pid < 0) {
+ // Unrecoverable error, cannot create an extension process.
+ LOG(ERROR) << "Cannot create extension process: " << extension;
+ ::exit(EXIT_FAILURE);
+ } else if (ext_pid == 0) {
+ // Pass the current extension socket and a set timeout to the extension.
+ setenv("OSQUERY_EXTENSION", std::to_string(getpid()).c_str(), 1);
+ // Execute extension with very specific arguments.
+ execle(exec_path.string().c_str(),
+ ("osquery extension: " + extension).c_str(),
+ "--socket",
+ Flag::getValue("extensions_socket").c_str(),
+ "--timeout",
+ Flag::getValue("extensions_timeout").c_str(),
+ "--interval",
+ Flag::getValue("extensions_interval").c_str(),
+ (Flag::getValue("verbose") == "true") ? "--verbose" : (char*)nullptr,
+ (char*)nullptr,
+ environ);
+ // Code should never reach this point.
+ VLOG(1) << "Could not start extension process: " << extension;
+ ::exit(EXIT_FAILURE);
+ }
-bool isOsqueryWorker() {
- return (getenv("OSQUERYD_WORKER") != nullptr);
+ Watcher::setExtension(extension, ext_pid);
+ Watcher::resetExtensionCounters(extension, getUnixTime());
+ VLOG(1) << "Created and monitoring extension child (" << ext_pid << "): "
+ << extension;
+ return true;
}
void WatcherWatcherRunner::enter() {
}
return kWatchdogLimits.at(name).at(level);
}
-
-void initWorkerWatcher(const std::string& name, int argc, char* argv[]) {
- // The watcher will forever monitor and spawn additional workers.
- Watcher watcher(argc, argv);
- watcher.setWorkerName(name);
-
- if (isOsqueryWorker()) {
- // Do not start watching/spawning if this process is a worker.
- watcher.initWorker();
- } else {
- do {
- if (!watcher.watch()) {
- // The watcher failed, create a worker.
- watcher.createWorker();
- watcher.resetCounters();
- }
- } while (watcher.ok());
-
- // Executation should never reach this point.
- ::exit(EXIT_FAILURE);
- }
-}
}
#include <unistd.h>
+#include <boost/noncopyable.hpp>
+#include <boost/thread/mutex.hpp>
+
#include <osquery/flags.h>
#include "osquery/dispatcher/dispatcher.h"
DECLARE_bool(disable_watchdog);
+/**
+ * @brief Categories of process performance limitations.
+ *
+ * Performance limits are applied by a watcher thread on autoloaded extensions
+ * and optional a daemon worker process. The performance types are identified
+ * here, and organized into levels. Such that a caller may enforce rigor or
+ * relax the performance expectations of a osquery daemon.
+ */
enum WatchdogLimitType {
MEMORY_LIMIT,
UTILIZATION_LIMIT,
INTERVAL,
};
-class Watcher {
+/**
+ * @brief A performance state structure for an autoloaded extension or worker.
+ *
+ * A watcher thread will continue to check the performance state, and keep a
+ * last-checked snapshot for each autoloaded extension and worker process.
+ */
+struct PerformanceState {
+ /// A counter of how many intervals the process exceeded performance limits.
+ size_t sustained_latency;
+ /// The last checked user CPU time.
+ size_t user_time;
+ /// The last checked system CPU time.
+ size_t system_time;
+ /// A timestamp when the process/worker was last created.
+ size_t last_respawn_time;
+
+ PerformanceState() {
+ sustained_latency = 0;
+ user_time = 0;
+ system_time = 0;
+ last_respawn_time = 0;
+ }
+};
+
+/**
+ * @brief Thread-safe watched child process state manager.
+ *
+ * The Watcher instance is separated from the WatcherRunner thread to allow
+ * signals and osquery-introspection to monitor the autoloaded extensions
+ * and optional worker stats. A child-process change signal may indicate an
+ * autoloaded extension ended. Tables may also report on the historic worker
+ * or extension utilizations.
+ *
+ * Though not critical, it is perferred to remove the extension's broadcasted
+ * routes quickly. Locking access to the extensions list between signals and
+ * the WatcherRunner thread allows osquery to tearDown registry changes before
+ * attempting to respawn an extension process.
+ */
+class Watcher : private boost::noncopyable {
public:
- Watcher(int argc, char* argv[]) : worker_(0), argc_(argc), argv_(argv) {
- resetCounters();
- last_respawn_time_ = 0;
+ /// Instance accessor
+ static Watcher& instance() {
+ static Watcher instance;
+ return instance;
}
- void setWorkerName(const std::string& name) { name_ = name; }
- const std::string& getWorkerName() { return name_; }
+ /// Reset counters after a worker exits.
+ static void resetWorkerCounters(size_t respawn_time);
- /// Boilerplate function to sleep for some configured latency
- bool ok();
- /// Begin the worker-watcher process.
- bool watch();
- /// Fork a worker process.
- void createWorker();
- /// If the process is a worker, clean up identification.
- void initWorker();
- void resetCounters();
+ /// Reset counters for an extension path.
+ static void resetExtensionCounters(const std::string& extension,
+ size_t respawn_time);
+
+ /// Lock access to extensions.
+ static void lock() { instance().lock_.lock(); }
+
+ /// Unlock access to extensions.
+ static void unlock() { instance().lock_.unlock(); }
+
+ /// Accessor for autoloadable extension paths.
+ static const std::map<std::string, pid_t>& extensions() {
+ return instance().extensions_;
+ }
+
+ /// Lookup extension path from pid.
+ static std::string getExtensionPath(pid_t child);
+
+ /// Remove an autoloadable extension path.
+ static void removeExtensionPath(const std::string& extension);
+
+ /// Add extensions autoloadable paths.
+ static void addExtensionPath(const std::string& path);
+
+ /// Get state information for a worker or extension child.
+ static PerformanceState& getState(pid_t child);
+ static PerformanceState& getState(const std::string& extension);
+
+ /// Accessor for the worker process.
+ static pid_t getWorker() { return instance().worker_; }
+
+ /// Setter for worker process.
+ static void setWorker(pid_t child) { instance().worker_ = child; }
+
+ /// Setter for an extension process.
+ static void setExtension(const std::string& extension, pid_t child);
+
+ /// Reset pid and performance counters for a worker or extension process.
+ static void reset(pid_t child);
+
+ /// Return the number of autoloadable extensions.
+ static bool hasManagedExtensions();
private:
- /// Inspect into the memory, CPU, and other worker process states.
- bool isWorkerSane();
- /// If a worker as otherwise gone insane, stop it.
- void stopWorker();
+ /// Do not request the lock until extensions are used.
+ Watcher() : worker_(-1), lock_(mutex_, boost::defer_lock) {}
+ Watcher(Watcher const&);
+ void operator=(Watcher const&);
+ virtual ~Watcher() {}
private:
- size_t sustained_latency_;
- size_t current_user_time_;
- size_t current_system_time_;
- size_t last_respawn_time_;
+ /// Performance state for the worker process.
+ PerformanceState state_;
+ /// Performance states for each autoloadable extension binary.
+ std::map<std::string, PerformanceState> extension_states_;
private:
/// Keep the single worker process/thread ID for inspection.
pid_t worker_;
+ /// Keep a list of resolved extension paths and their managed pids.
+ std::map<std::string, pid_t> extensions_;
+ /// Path to autoload extensions from.
+ std::vector<std::string> extensions_paths_;
+
+ private:
+ /// Mutex and lock around extensions access.
+ boost::mutex mutex_;
+ /// Mutex and lock around extensions access.
+ boost::unique_lock<boost::mutex> lock_;
+};
+
+/**
+ * @brief A scoped locker for iterating over watcher extensions.
+ *
+ * A lock must be used if any part of osquery wants to enumerate the autoloaded
+ * extensions or autoloadable extension paths a Watcher may be monitoring.
+ * A signal or WatcherRunner thread may stop or start extensions.
+ */
+class WatcherLocker {
+ public:
+ /// Construct and gain watcher lock.
+ WatcherLocker() { Watcher::lock(); }
+ /// Destruct and release watcher lock.
+ ~WatcherLocker() { Watcher::unlock(); }
+};
+
+/**
+ * @brief The watchdog thread responsible for spawning/monitoring children.
+ *
+ * The WatcherRunner thread will spawn any autoloaded modules or optional
+ * osquery daemon worker processes. It will then poll for their performance
+ * state and kill/respawn osquery child processes.
+ */
+class WatcherRunner : public InternalRunnable {
+ public:
+ /**
+ * @brief Construct a watcher thread.
+ *
+ * @param argc The osquery process argc.
+ * @param argv The osquery process argv.
+ * @param use_worker True if the process should spawn and monitor a worker.
+ */
+ explicit WatcherRunner(int argc, char** argv, bool use_worker)
+ : argc_(argc), argv_(argv), use_worker_(use_worker) {
+ (void)argc_;
+ }
+
+ private:
+ void enter();
+ /// Boilerplate function to sleep for some configured latency
+ bool ok();
+ /// Begin the worker-watcher process.
+ bool watch(pid_t child);
+ /// Inspect into the memory, CPU, and other worker/extension process states.
+ bool isChildSane(pid_t child);
+
+ private:
+ /// Fork a worker process.
+ void createWorker();
+ /// Fork an extension process.
+ bool createExtension(const std::string& extension);
+ /// If a worker/extension has otherwise gone insane, stop it.
+ void stopChild(pid_t child);
+
+ private:
/// Keep the invocation daemon's argc to iterate through argv.
int argc_;
/// When a worker child is spawned the argv will be scrubed.
char** argv_;
- /// When a worker child is spawned the process name will be changed.
- std::string name_;
+ /// Spawn/monitor a worker process.
+ bool use_worker_;
};
/// The WatcherWatcher is spawned within the worker and watches the watcher.
pid_t watcher_;
};
-/// Check if the current process is already a worker.
-bool isOsqueryWorker();
-
/// Get a performance limit by name and optional level.
size_t getWorkerLimit(WatchdogLimitType limit, int level = -1);
-
-/**
- * @brief Daemon tools may want to continually spawn worker processes
- * and monitor their utilization.
- *
- * A daemon may call initWorkerWatcher to begin watching child daemon
- * processes until it-itself is unscheduled. The basic guarentee is that only
- * workers will return from the function.
- *
- * The worker-watcher will implement performance bounds on CPU utilization
- * and memory, as well as check for zombie/defunct workers and respawn them
- * if appropriate. The appropriateness is determined from heuristics around
- * how the worker exitted. Various exit states and velocities may cause the
- * watcher to resign.
- *
- * @param name The name of the worker process.
- * @param argc The daemon's argc.
- * @param argv The daemon's volitle argv.
- */
-void initWorkerWatcher(const std::string& name, int argc, char* argv[]);
}
const std::vector<std::string> kDomains = {kConfigurations, kQueries, kEvents};
FLAG(string,
- db_path,
+ database_path,
"/var/osquery/osquery.db",
"If using a disk-based backing store, specify a path");
+FLAG_ALIAS(std::string, db_path, database_path);
-FLAG(bool,
- use_in_memory_database,
- false,
- "Keep osquery backing-store in memory");
+FLAG(bool, database_in_memory, false, "Keep osquery backing-store in memory");
+FLAG_ALIAS(bool, use_in_memory_database, database_in_memory);
/////////////////////////////////////////////////////////////////////////////
// constructors and destructors
// getInstance methods
/////////////////////////////////////////////////////////////////////////////
std::shared_ptr<DBHandle> DBHandle::getInstance() {
- return getInstance(FLAGS_db_path, FLAGS_use_in_memory_database);
+ return getInstance(FLAGS_database_path, FLAGS_database_in_memory);
}
bool DBHandle::checkDB() {
try {
- auto handle = DBHandle(FLAGS_db_path, FLAGS_use_in_memory_database);
+ auto handle = DBHandle(FLAGS_database_path, FLAGS_database_in_memory);
} catch (const std::exception& e) {
return false;
}
-ADD_OSQUERY_LIBRARY(FALSE osquery_devtools shell.cpp
- printer.cpp)
+ADD_OSQUERY_LIBRARY(FALSE osquery_devtools printer.cpp)
ADD_OSQUERY_TEST(FALSE osquery_printer_tests printer_tests.cpp)
int main(int argc, char* argv[]) {
testing::InitGoogleTest(&argc, argv);
- osquery::initOsquery(argc, argv);
return RUN_ALL_TESTS();
}
* All rights reserved.
*
* This source code is licensed under the BSD-style license found in the
- * LICENSE file in the root directory of this source tree. An additional grant
+ * LICENSE file in the root directory of this source tree. An additional grant
* of patent rights can be found in the PATENTS file in the same directory.
*
*/
EventPublisherRef publisher;
try {
publisher = EventFactory::getInstance().getEventPublisher(type_id);
- }
- catch (std::out_of_range& e) {
+ } catch (std::out_of_range& e) {
return Status(1, "No event type found");
}
// The runloop status is not reflective of the event type's.
publisher->tearDown();
- VLOG(1) << "Event publisher " << publisher->type() << " runloop terminated";
+ VLOG(1) << "Event publisher " << publisher->type()
+ << " runloop terminated for reason: " << status.getMessage();
return Status(0, "OK");
}
EventPublisherRef publisher;
try {
publisher = getInstance().getEventPublisher(type_id);
- }
- catch (std::out_of_range& e) {
+ } catch (std::out_of_range& e) {
return Status(1, "No event type found");
}
EventPublisherRef publisher;
try {
publisher = EventFactory::getInstance().getEventPublisher(type_id);
- }
- catch (std::out_of_range& e) {
+ } catch (std::out_of_range& e) {
return 0;
}
return publisher->numSubscriptions();
EventPublisherRef publisher;
try {
publisher = ef.getEventPublisher(type_id);
- }
- catch (std::out_of_range& e) {
+ } catch (std::out_of_range& e) {
return Status(1, "No event publisher to deregister");
}
* All rights reserved.
*
* This source code is licensed under the BSD-style license found in the
- * LICENSE file in the root directory of this source tree. An additional grant
+ * LICENSE file in the root directory of this source tree. An additional grant
* of patent rights can be found in the PATENTS file in the same directory.
*
*/
inotify_handle_ = -1;
}
+Status INotifyEventPublisher::restartMonitoring(){
+ if (last_restart_ != 0 && getUnixTime() - last_restart_ < 10) {
+ return Status(1, "Overflow");
+ }
+ last_restart_ = getUnixTime();
+ VLOG(1) << "Got an overflow, trying to restart...";
+ for(const auto& desc : descriptors_){
+ removeMonitor(desc, 1);
+ }
+ path_descriptors_.clear();
+ descriptor_paths_.clear();
+ configure();
+ return Status(0, "OK");
+}
+
Status INotifyEventPublisher::run() {
// Get a while wrapper for free.
char buffer[BUFFER_SIZE];
auto event = reinterpret_cast<struct inotify_event*>(p);
if (event->mask & IN_Q_OVERFLOW) {
// The inotify queue was overflown (remove all paths).
- return Status(1, "Overflow");
+ Status stat = restartMonitoring();
+ if(!stat.ok()){
+ return stat;
+ }
}
if (event->mask & IN_IGNORED) {
removeMonitor(event->wd, false);
} else {
auto ec = createEventContextFrom(event);
+ if(event->mask & IN_CREATE && isDirectory(ec->path).ok()){
+ addMonitor(ec->path, 1);
+ }
fire(ec);
}
// Continue to iterate
path << "/" << event->name;
}
ec->path = path.str();
-
- // Set the action (may be multiple)
for (const auto& action : kMaskActions) {
if (event->mask & action.first) {
ec->action = action.second;
if (recursive && isDirectory(path).ok()) {
std::vector<std::string> children;
// Get a list of children of this directory (requesed recursive watches).
- if (!listFilesInDirectory(path, children).ok()) {
- return false;
- }
+ listDirectoriesInDirectory(path, children);
for (const auto& child : children) {
- // Only watch child directories, a watch on the directory implies files.
- if (isDirectory(child).ok()) {
- addMonitor(child, recursive);
- }
+ addMonitor(child, recursive);
}
}
int getHandle() { return inotify_handle_; }
/// Get the number of actual INotify active descriptors.
int numDescriptors() { return descriptors_.size(); }
+ /// If we overflow, try and restart the monitor
+ Status restartMonitoring();
// Consider an event queue if separating buffering from firing/servicing.
DescriptorVector descriptors_;
PathDescriptorMap path_descriptors_;
DescriptorPathMap descriptor_paths_;
int inotify_handle_;
+ int last_restart_;
public:
FRIEND_TEST(INotifyTests, test_inotify_optimization);
using namespace osquery;
+class ExampleConfigPlugin : public ConfigPlugin {
+ public:
+ Status genConfig(std::map<std::string, std::string>& config) {
+ config["data"] = "{\"options\": [], \"scheduledQueries\": []}";
+ return Status(0, "OK");
+ }
+};
+
class ExampleTable : public tables::TablePlugin {
private:
tables::TableColumns columns() const {
}
};
-REGISTER(ExampleTable, "table", "example");
+REGISTER_EXTERNAL(ExampleConfigPlugin, "config", "example");
+REGISTER_EXTERNAL(ExampleTable, "table", "example");
int main(int argc, char* argv[]) {
- initOsquery(argc, argv, OSQUERY_EXTENSION);
+ osquery::Initializer runner(argc, argv, OSQUERY_EXTENSION);
auto status = startExtension("example", "0.0.1");
if (!status.ok()) {
}
// Finally shutdown.
- shutdownOsquery();
+ runner.shutdown();
return 0;
}
--- /dev/null
+/*
+ * Copyright (c) 2014, Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree. An additional grant
+ * of patent rights can be found in the PATENTS file in the same directory.
+ *
+ */
+
+#include <osquery/sdk.h>
+
+using namespace osquery;
+
+class ExampleTable : public tables::TablePlugin {
+ private:
+ tables::TableColumns columns() const {
+ return {{"example_text", "TEXT"}, {"example_integer", "INTEGER"}};
+ }
+
+ QueryData generate(tables::QueryContext& request) {
+ QueryData results;
+
+ Row r;
+ r["example_text"] = "example";
+ r["example_integer"] = INTEGER(1);
+
+ results.push_back(r);
+ return results;
+ }
+};
+
+// Create the module if the environment variable TESTFAIL1 is not defined.
+// This allows the integration tests to, at run time, test the module
+// loading workflow.
+CREATE_MODULE_IF(getenv("TESTFAIL1") == nullptr, "example", "0.0.1", "0.0.0");
+
+void initModule(void) {
+ // Register a plugin from a module if the environment variable TESTFAIL2
+ // is not defined.
+ if (getenv("TESTFAIL2") == nullptr) {
+ REGISTER_MODULE(ExampleTable, "table", "example");
+ }
+}
#include <csignal>
+#include <boost/algorithm/string/trim.hpp>
+
#include <osquery/events.h>
#include <osquery/filesystem.h>
#include <osquery/logger.h>
#include <osquery/sql.h>
#include "osquery/extensions/interface.h"
+#include "osquery/core/watcher.h"
using namespace osquery::extensions;
-namespace osquery {
-
-const int kWatcherMLatency = 3000;
-
-FLAG(bool, disable_extensions, false, "Disable extension API");
+namespace fs = boost::filesystem;
-FLAG(string,
- extensions_socket,
- "/var/osquery/osquery.em",
- "Path to the extensions UNIX domain socket")
+namespace osquery {
+// Millisecond latency between initalizing manager pings.
+const int kExtensionInitializeMLatency = 200;
+
+#ifdef __APPLE__
+const std::string kModuleExtension = ".dylib";
+#else
+const std::string kModuleExtension = ".so";
+#endif
+
+CLI_FLAG(bool, disable_extensions, false, "Disable extension API");
+
+CLI_FLAG(string,
+ extensions_socket,
+ "/var/osquery/osquery.em",
+ "Path to the extensions UNIX domain socket")
+
+CLI_FLAG(string,
+ extensions_autoload,
+ "/etc/osquery/extensions.load",
+ "Optional path to a list of autoloaded & managed extensions")
+
+CLI_FLAG(string,
+ extensions_timeout,
+ "3",
+ "Seconds to wait for autoloaded extensions");
+
+CLI_FLAG(string,
+ extensions_interval,
+ "3",
+ "Seconds delay between connectivity checks")
+
+CLI_FLAG(string,
+ modules_autoload,
+ "/etc/osquery/modules.load",
+ "Optional path to a list of autoloaded registry modules")
+
+/// Alias the extensions_socket (used by core) to an alternate name reserved
+/// for extension binaries
+EXTENSION_FLAG_ALIAS(socket, extensions_socket);
+
+/// An extension manager may not be immediately available.
+EXTENSION_FLAG_ALIAS(timeout, extensions_timeout);
+EXTENSION_FLAG_ALIAS(interval, extensions_interval);
void ExtensionWatcher::enter() {
// Watch the manager, if the socket is removed then the extension will die.
while (true) {
}
}
+void loadExtensions() {
+ // Optionally autoload extensions
+ auto status = loadExtensions(FLAGS_extensions_autoload);
+ if (!status.ok()) {
+ LOG(WARNING) << "Could not autoload extensions: " << status.what();
+ }
+}
+
+void loadModules() {
+ auto status = loadModules(FLAGS_modules_autoload);
+ if (!status.ok()) {
+ LOG(WARNING) << "Modules autoload contains invalid paths";
+ }
+}
+
+Status loadExtensions(const std::string& loadfile) {
+ std::string autoload_paths;
+ if (readFile(loadfile, autoload_paths).ok()) {
+ for (auto& path : osquery::split(autoload_paths, "\n")) {
+ boost::trim(path);
+ if (path.size() > 0 && path[0] != '#' && path[0] != ';') {
+ Watcher::addExtensionPath(path);
+ }
+ }
+ return Status(0, "OK");
+ }
+ return Status(1, "Cannot read extensions autoload file");
+}
+
+Status loadModuleFile(const std::string& path) {
+ fs::path module(path);
+ if (safePermissions(module.parent_path().string(), path)) {
+ if (module.extension().string() == kModuleExtension) {
+ // Silently allow module load failures to drop.
+ RegistryModuleLoader loader(module.string());
+ loader.init();
+ return Status(0, "OK");
+ }
+ }
+ return Status(1, "Module check failed");
+}
+
+Status loadModules(const std::string& loadfile) {
+ // Split the search path for modules using a ':' delimiter.
+ std::string autoload_paths;
+ if (readFile(loadfile, autoload_paths).ok()) {
+ auto status = Status(0, "OK");
+ for (auto& module_path : osquery::split(autoload_paths, "\n")) {
+ boost::trim(module_path);
+ auto path_status = loadModuleFile(module_path);
+ if (!path_status.ok()) {
+ status = path_status;
+ }
+ }
+ // Return an aggregate failure if any load fails (invalid search path).
+ return status;
+ }
+ return Status(1, "Cannot read modules autoload file");
+}
+
+Status extensionPathActive(const std::string& path, bool use_timeout = false) {
+ // Make sure the extension manager path exists, and is writable.
+ size_t delay = 0;
+ size_t timeout = atoi(FLAGS_extensions_timeout.c_str());
+ do {
+ if (pathExists(path) && isWritable(path)) {
+ try {
+ auto client = EXManagerClient(path);
+ return Status(0, "OK");
+ } catch (const std::exception& e) {
+ // Path might exist without a connected extension or extension manager.
+ }
+ }
+ // Only check active once if this check does not allow a timeout.
+ if (!use_timeout || timeout == 0) {
+ break;
+ }
+ // Increase the total wait detail.
+ delay += kExtensionInitializeMLatency;
+ ::usleep(kExtensionInitializeMLatency * 1000);
+ } while (delay < timeout * 1000);
+ return Status(1, "Extension socket not available: " + path);
+}
+
Status startExtension(const std::string& name, const std::string& version) {
return startExtension(name, version, "0.0.0");
}
Status startExtension(const std::string& name,
const std::string& version,
const std::string& min_sdk_version) {
- // No assumptions about how the extensions logs, the first action is to
- // start the extension's registry.
- Registry::setUp();
-
- auto status =
- startExtensionWatcher(FLAGS_extensions_socket, kWatcherMLatency, true);
+ auto latency = atoi(FLAGS_extensions_interval.c_str()) * 1000;
+ auto status = startExtensionWatcher(FLAGS_extensions_socket, latency, true);
if (!status.ok()) {
// If the threaded watcher fails to start, fail the extension.
return status;
}
try {
+ // The extension does nothing but serve the thrift API.
+ // Join on both the thrift and extension manager watcher services.
Dispatcher::joinServices();
} catch (const std::exception& e) {
// The extension manager may shutdown without notifying the extension.
}
// An extension will only return on failure.
- return Status(0, "OK");
+ return Status(0, "Extension was shutdown");
}
Status startExtension(const std::string& manager_path,
const std::string& min_sdk_version,
const std::string& sdk_version) {
// Make sure the extension manager path exists, and is writable.
- if (!pathExists(manager_path) || !isWritable(manager_path)) {
- return Status(1, "Extension manager socket not available: " + manager_path);
+ auto status = extensionPathActive(manager_path, true);
+ if (!status.ok()) {
+ printf("bad path active\n");
+ return status;
}
// The Registry broadcast is used as the ExtensionRegistry.
auto broadcast = Registry::getBroadcast();
-
+ // The extension will register and provide name, version, sdk details.
InternalExtensionInfo info;
info.name = name;
info.version = version;
info.sdk_version = sdk_version;
info.min_sdk_version = min_sdk_version;
+ // If registration is successful, we will also request the manager's options.
+ InternalOptionList options;
// Register the extension's registry broadcast with the manager.
- ExtensionStatus status;
+ ExtensionStatus ext_status;
try {
auto client = EXManagerClient(manager_path);
- client.get()->registerExtension(status, info, broadcast);
- }
- catch (const std::exception& e) {
+ client.get()->registerExtension(ext_status, info, broadcast);
+ // The main reason for a failed registry is a duplicate extension name
+ // (the extension process is already running), or the extension broadcasts
+ // a duplicate registry item.
+ if (ext_status.code != ExtensionCode::EXT_SUCCESS) {
+ return Status(ext_status.code, ext_status.message);
+ }
+ // Request the core options, mainly to set the active registry plugins for
+ // logger and config.
+ client.get()->options(options);
+ } catch (const std::exception& e) {
return Status(1, "Extension register failed: " + std::string(e.what()));
}
- if (status.code != ExtensionCode::EXT_SUCCESS) {
- return Status(status.code, status.message);
- }
-
// Now that the uuid is known, try to clean up stale socket paths.
- auto extension_path = getExtensionSocket(status.uuid, manager_path);
+ auto extension_path = getExtensionSocket(ext_status.uuid, manager_path);
if (pathExists(extension_path).ok()) {
if (!isWritable(extension_path).ok()) {
return Status(1, "Cannot write extension socket: " + extension_path);
}
}
+ // Set the active config and logger plugins. The core will arbitrate if the
+ // plugins are not available in the extension's local registry.
+ Registry::setActive("config", options["config_plugin"].value);
+ Registry::setActive("logger", options["logger_plugin"].value);
+ // Set up all lazy registry plugins and the active config/logger plugin.
+ Registry::setUp();
+
// Start the extension's Thrift server
Dispatcher::getInstance().addService(
- std::make_shared<ExtensionRunner>(manager_path, status.uuid));
- VLOG(1) << "Extension (" << name << ", " << status.uuid << ", " << version
+ std::make_shared<ExtensionRunner>(manager_path, ext_status.uuid));
+ VLOG(1) << "Extension (" << name << ", " << ext_status.uuid << ", " << version
<< ", " << sdk_version << ") registered";
- return Status(0, std::to_string(status.uuid));
+ return Status(0, std::to_string(ext_status.uuid));
}
Status queryExternal(const std::string& manager_path,
const std::string& query,
QueryData& results) {
// Make sure the extension path exists, and is writable.
- if (!pathExists(manager_path) || !isWritable(manager_path)) {
- return Status(1, "Extension manager socket not available: " + manager_path);
+ auto status = extensionPathActive(manager_path);
+ if (!status.ok()) {
+ return status;
}
ExtensionResponse response;
const std::string& query,
tables::TableColumns& columns) {
// Make sure the extension path exists, and is writable.
- if (!pathExists(manager_path) || !isWritable(manager_path)) {
- return Status(1, "Extension manager socket not available: " + manager_path);
+ auto status = extensionPathActive(manager_path);
+ if (!status.ok()) {
+ return status;
}
ExtensionResponse response;
}
// Make sure the extension path exists, and is writable.
- if (!pathExists(path) || !isWritable(path)) {
- return Status(1, "Extension socket not available: " + path);
+ auto status = extensionPathActive(path);
+ if (!status.ok()) {
+ return status;
}
ExtensionStatus ext_status;
Status getExtensions(const std::string& manager_path,
ExtensionList& extensions) {
// Make sure the extension path exists, and is writable.
- if (!pathExists(manager_path) || !isWritable(manager_path)) {
- return Status(1, "Extension manager socket not available: " + manager_path);
+ auto status = extensionPathActive(manager_path);
+ if (!status.ok()) {
+ return status;
}
InternalExtensionList ext_list;
const PluginRequest& request,
PluginResponse& response) {
// Make sure the extension manager path exists, and is writable.
- if (!pathExists(extension_path) || !isWritable(extension_path)) {
- return Status(1, "Extension socket not available: " + extension_path);
+ auto status = extensionPathActive(extension_path);
+ if (!status.ok()) {
+ return status;
}
ExtensionResponse ext_response;
size_t interval,
bool fatal) {
// Make sure the extension manager path exists, and is writable.
- if (!pathExists(manager_path) || !isWritable(manager_path)) {
- return Status(1, "Extension manager socket not available: " + manager_path);
+ auto status = extensionPathActive(manager_path, true);
+ if (!status.ok()) {
+ return status;
}
// Start a extension manager watcher, if the manager dies, so should we.
}
}
+ auto latency = atoi(FLAGS_extensions_interval.c_str()) * 1000;
// Start a extension manager watcher, if the manager dies, so should we.
Dispatcher::getInstance().addService(
- std::make_shared<ExtensionManagerWatcher>(manager_path,
- kWatcherMLatency));
+ std::make_shared<ExtensionManagerWatcher>(manager_path, latency));
// Start the extension manager thread.
Dispatcher::getInstance().addService(
#include <osquery/filesystem.h>
#include <osquery/database.h>
+#include "osquery/core/test_util.h"
#include "osquery/extensions/interface.h"
using namespace osquery::extensions;
void TearDown() {
Dispatcher::getInstance().removeServices();
+ Dispatcher::joinServices();
remove(kTestManagerSocket);
}
Registry::allowDuplicates(true);
status = startExtension(kTestManagerSocket, "test", "0.1", "0.0.0", "0.0.1");
// This will be false since we are registering duplicate items
- EXPECT_TRUE(status.ok());
+ ASSERT_TRUE(status.ok());
// The `startExtension` internal call (exposed for testing) returns the
// uuid of the extension in the success status.
- RouteUUID uuid;
- try {
- uuid = (RouteUUID)stoi(status.getMessage(), nullptr, 0);
- }
- catch (const std::exception& e) {
- EXPECT_TRUE(false);
- return;
- }
+ RouteUUID uuid = (RouteUUID)stoi(status.getMessage(), nullptr, 0);
// We can test-wait for the extensions's socket to open.
EXPECT_TRUE(socketExists(kTestManagerSocket + "." + std::to_string(uuid)));
for (const auto& request_item : request) {
response.push_back({{request_item.first, request_item.second}});
}
- return Status(0, "Test sucess");
+ return Status(0, "Test success");
}
};
EXPECT_TRUE(socketExists(kTestManagerSocket));
// This time we're going to add a plugin to the extension_test registry.
- REGISTER(TestExtensionPlugin, "extension_test", "test_item");
+ Registry::add<TestExtensionPlugin>("extension_test", "test_item");
// Now we create a registry alias that will be broadcasted but NOT used for
// internal call lookups. Aliasing was introduced for testing such that an
Registry::removeBroadcast(uuid);
Registry::allowDuplicates(false);
}
+
+TEST_F(ExtensionsTest, test_extension_module_search) {
+ createMockFileStructure();
+ EXPECT_TRUE(loadModules(kFakeDirectory));
+ EXPECT_FALSE(loadModules("/dir/does/not/exist"));
+ tearDownMockFileStructure();
+}
}
int main(int argc, char* argv[]) {
PluginResponse response;
PluginRequest plugin_request;
for (const auto& request_item : request) {
+ // Create a PluginRequest from an ExtensionPluginRequest.
plugin_request[request_item.first] = request_item.second;
}
if (status.ok()) {
for (const auto& response_item : response) {
+ // Translate a PluginResponse to an ExtensionPluginResponse.
_return.response.push_back(response_item);
}
}
}
+void ExtensionManagerHandler::extensions(InternalExtensionList& _return) {
+ refresh();
+ _return = extensions_;
+}
+
+void ExtensionManagerHandler::options(InternalOptionList& _return) {
+ auto flags = Flag::flags();
+ for (const auto& flag : flags) {
+ _return[flag.first].value = flag.second.value;
+ _return[flag.first].default_value = flag.second.default_value;
+ _return[flag.first].type = flag.second.type;
+ }
+}
+
void ExtensionManagerHandler::registerExtension(
ExtensionStatus& _return,
const InternalExtensionInfo& info,
}
}
-void ExtensionManagerHandler::extensions(InternalExtensionList& _return) {
- refresh();
- _return = extensions_;
-}
-
bool ExtensionManagerHandler::exists(const std::string& name) {
refresh();
void extensions(InternalExtensionList& _return);
/**
+ * @brief Return a map of osquery options (Flags, bootstrap CLI flags).
+ *
+ * osquery options are set via command line flags or overridden by a config
+ * options dictionary. There are some CLI-only flags that should never
+ * be overridden. If a bootstrap flag is changed there is undefined behavior
+ * since bootstrap candidates are settings needed before a configuration
+ * plugin is setUp.
+ *
+ * Extensions may broadcast config or logger plugins that need a snapshot
+ * of the current options. The best example is the `config_plugin` bootstrap
+ * flag.
+ */
+ void options(InternalOptionList& _return);
+
+ /**
* @brief Request a Route UUID and advertise a set of Registry routes.
*
* When an Extension starts it must call registerExtension using a well known
public:
virtual ~ExtensionWatcher() {}
ExtensionWatcher(const std::string& path, size_t interval, bool fatal)
- : path_(path), interval_(interval), fatal_(fatal) {}
+ : path_(path), interval_(interval), fatal_(fatal) {
+ interval_ = (interval_ < 200) ? 200 : interval_;
+ }
public:
/// The Dispatcher thread entry point.
#include <exception>
#include <sstream>
-#include <iostream>
#include <fcntl.h>
#include <sys/stat.h>
}
Status listFilesInDirectory(const boost::filesystem::path& path,
- std::vector<std::string>& results) {
+ std::vector<std::string>& results,
+ bool ignore_error) {
+ boost::filesystem::directory_iterator begin_iter;
+
try {
if (!boost::filesystem::exists(path)) {
return Status(1, "Directory not found: " + path.string());
if (!boost::filesystem::is_directory(path)) {
return Status(1, "Supplied path is not a directory: " + path.string());
}
+ begin_iter = boost::filesystem::directory_iterator(path);
+
+ } catch (const boost::filesystem::filesystem_error& e) {
+ return Status(1, e.what());
+ }
- boost::filesystem::directory_iterator begin_iter(path);
- boost::filesystem::directory_iterator end_iter;
- for (; begin_iter != end_iter; begin_iter++) {
- if (!boost::filesystem::is_directory(begin_iter->path())) {
+ boost::filesystem::directory_iterator end_iter;
+ for (; begin_iter != end_iter; begin_iter++) {
+ try {
+ if (boost::filesystem::is_regular_file(begin_iter->path())) {
results.push_back(begin_iter->path().string());
}
+ } catch (const boost::filesystem::filesystem_error& e) {
+ if (ignore_error == 0) {
+ return Status(1, e.what());
+ }
}
-
- return Status(0, "OK");
- } catch (const boost::filesystem::filesystem_error& e) {
- return Status(1, e.what());
}
+ return Status(0, "OK");
}
Status listDirectoriesInDirectory(const boost::filesystem::path& path,
- std::vector<std::string>& results) {
+ std::vector<std::string>& results,
+ bool ignore_error) {
+ boost::filesystem::directory_iterator begin_iter;
try {
if (!boost::filesystem::exists(path)) {
return Status(1, "Directory not found");
if (!stat.ok()) {
return stat;
}
+ begin_iter = boost::filesystem::directory_iterator(path);
+ } catch (const boost::filesystem::filesystem_error& e) {
+ return Status(1, e.what());
+ }
- boost::filesystem::directory_iterator begin_iter(path);
- boost::filesystem::directory_iterator end_iter;
- for (; begin_iter != end_iter; begin_iter++) {
+ boost::filesystem::directory_iterator end_iter;
+ for (; begin_iter != end_iter; begin_iter++) {
+ try {
if (boost::filesystem::is_directory(begin_iter->path())) {
results.push_back(begin_iter->path().string());
}
+ } catch (const boost::filesystem::filesystem_error& e) {
+ if (ignore_error == 0) {
+ return Status(1, e.what());
+ }
}
-
- return Status(0, "OK");
- } catch (const boost::filesystem::filesystem_error& e) {
- return Status(1, e.what());
}
+ return Status(0, "OK");
}
/**
*/
Status doubleStarTraversal(const boost::filesystem::path& fs_path,
std::vector<std::string>& results,
+ ReturnSetting setting,
unsigned int rec_depth) {
if (rec_depth >= kMaxDirectoryTraversalDepth) {
return Status(2, fs_path.string().c_str());
}
// List files first
- Status stat = listFilesInDirectory(fs_path, results);
- if (!stat.ok()) {
- return Status(0, "OK");
+ if (setting & REC_LIST_FILES) {
+ Status stat = listFilesInDirectory(fs_path, results);
+ if (!stat.ok()) {
+ return Status(0, "OK");
+ }
}
std::vector<std::string> folders;
- stat = listDirectoriesInDirectory(fs_path, folders);
+ Status stat = listDirectoriesInDirectory(fs_path, folders);
if (!stat.ok()) {
return Status(0, "OK");
}
-
+ if (setting & REC_LIST_FOLDERS) {
+ results.push_back(fs_path.string());
+ }
for (const auto& folder : folders) {
boost::filesystem::path p(folder);
if (boost::filesystem::is_symlink(p)) {
continue;
}
- stat = doubleStarTraversal(folder, results, rec_depth + 1);
+
+ stat = doubleStarTraversal(folder, results, setting, rec_depth + 1);
if (!stat.ok() && stat.getCode() == 2) {
return stat;
}
*/
Status resolveLastPathComponent(const boost::filesystem::path& fs_path,
std::vector<std::string>& results,
+ ReturnSetting setting,
const std::vector<std::string>& components,
unsigned int rec_depth) {
+
// Is the last component a double star?
if (components[components.size() - 1] == kWildcardCharacterRecursive) {
- Status stat =
- doubleStarTraversal(fs_path.parent_path(), results, rec_depth);
- return stat;
+ if (setting & REC_EVENT_OPT) {
+ results.push_back(fs_path.parent_path().string());
+ return Status(0, "OK");
+ } else {
+ Status stat = doubleStarTraversal(
+ fs_path.parent_path(), results, setting, rec_depth);
+ return stat;
+ }
}
// Is the path a file
try {
- if (boost::filesystem::is_regular_file(fs_path)) {
+ if (setting == REC_LIST_FILES &&
+ boost::filesystem::is_regular_file(fs_path)) {
results.push_back(fs_path.string());
return Status(0, "OK");
}
}
std::vector<std::string> files;
- Status stat = listFilesInDirectory(fs_path.parent_path(), files);
- if (!stat.ok()) {
- return stat;
- }
+ std::vector<std::string> folders;
+ Status stat_file = listFilesInDirectory(fs_path.parent_path(), files);
+ Status stat_fold = listDirectoriesInDirectory(fs_path.parent_path(), folders);
// Is the last component a wildcard?
if (components[components.size() - 1] == kWildcardCharacter) {
- for (const auto& file : files) {
- results.push_back(file);
+
+ if (setting & REC_EVENT_OPT) {
+ results.push_back(fs_path.parent_path().string());
+ return Status(0, "OK");
+ }
+ if (setting & REC_LIST_FOLDERS) {
+ results.push_back(fs_path.parent_path().string());
+ for (const auto& fold : folders) {
+ results.push_back(fold);
+ }
+ }
+ if (setting & REC_LIST_FILES) {
+ for (const auto& file : files) {
+ results.push_back(file);
+ }
}
return Status(0, "OK");
}
std::vector<std::string>(components.begin(), components.end() - 1),
"/");
- // Is this a .*% type file match
+ // Is this a (.*)% type file match
if (components[components.size() - 1].find(kWildcardCharacter, 1) !=
std::string::npos &&
components[components.size() - 1][0] != kWildcardCharacter[0]) {
processed_path + "/" +
components[components.size() - 1].substr(
0, components[components.size() - 1].find(kWildcardCharacter, 1));
- for (const auto& file : files) {
- if (file.find(prefix, 0) != 0) {
- continue;
+ if (setting & REC_LIST_FOLDERS) {
+ for (const auto& fold : folders) {
+ if (fold.find(prefix, 0) != 0) {
+ continue;
+ }
+ results.push_back(fold);
+ }
+ }
+ if (setting & REC_LIST_FILES || setting & REC_EVENT_OPT) {
+ for (const auto& file : files) {
+ if (file.find(prefix, 0) != 0) {
+ continue;
+ }
+ results.push_back(file);
}
- results.push_back(file);
}
+ // Should be a return here?
+ return Status(0, "OK");
}
// Is this a %(.*) type file match
if (components[components.size() - 1][0] == kWildcardCharacter[0]) {
std::string suffix = components[components.size() - 1].substr(1);
-
- for (const auto& file : files) {
- boost::filesystem::path p(file);
- std::string file_name = p.filename().string();
- size_t pos = file_name.find(suffix);
- if (pos != std::string::npos &&
- pos + suffix.length() == file_name.length()) {
- results.push_back(file);
+ if (setting & REC_LIST_FOLDERS) {
+ for (const auto& fold : folders) {
+ std::string file_name =
+ boost::filesystem::path(fold).filename().string();
+ size_t pos = file_name.find(suffix);
+ if (pos != std::string::npos &&
+ pos + suffix.length() == file_name.length()) {
+ results.push_back(fold);
+ }
+ }
+ }
+ if (setting & REC_LIST_FILES || setting & REC_EVENT_OPT) {
+ for (const auto& file : files) {
+ boost::filesystem::path p(file);
+ std::string file_name = p.filename().string();
+ size_t pos = file_name.find(suffix);
+ if (pos != std::string::npos &&
+ pos + suffix.length() == file_name.length()) {
+ results.push_back(file);
+ }
}
}
return Status(0, "OK");
// Is the path a directory
if (boost::filesystem::is_directory(fs_path)) {
- for (auto& file : files) {
- results.push_back(file);
- }
+ results.push_back(fs_path.string());
return Status(0, "OK");
}
*/
Status resolveFilePattern(std::vector<std::string> components,
std::vector<std::string>& results,
+ ReturnSetting setting = REC_LIST_FILES,
unsigned int processed_index = 0,
unsigned int rec_depth = 0) {
for (const auto& dir : folders) {
boost::filesystem::path p(dir);
components[i] = p.filename().string();
- Status stat =
- resolveFilePattern(components, results, i + 1, rec_depth + 1);
+ Status stat = resolveFilePattern(
+ components, results, setting, i + 1, rec_depth + 1);
if (!stat.ok() && stat.getCode() == 2) {
return stat;
}
}
boost::filesystem::path p(dir);
components[i] = p.filename().string();
- Status stat =
- resolveFilePattern(components, results, i + 1, rec_depth + 1);
+ Status stat = resolveFilePattern(
+ components, results, setting, i + 1, rec_depth + 1);
if (!stat.ok() && stat.getCode() == 2) {
return stat;
}
if (pos != std::string::npos &&
pos + suffix.length() == folder_name.length()) {
components[i] = p.filename().string();
- Status stat =
- resolveFilePattern(components, results, i + 1, rec_depth + 1);
+ Status stat = resolveFilePattern(
+ components, results, setting, i + 1, rec_depth + 1);
if (!stat.ok() && stat.getCode() == 2) {
return stat;
}
// list the files at this point or do our ** traversal
return resolveLastPathComponent("/" + boost::algorithm::join(components, "/"),
results,
+ setting,
components,
rec_depth);
}
Status resolveFilePattern(const boost::filesystem::path& fs_path,
std::vector<std::string>& results) {
+ if (fs_path.string()[0] != '/') {
+ return resolveFilePattern(split(boost::filesystem::current_path().string() +
+ "/" + fs_path.string(),
+ "/"),
+ results);
+ }
return resolveFilePattern(split(fs_path.string(), "/"), results);
}
+Status resolveFilePattern(const boost::filesystem::path& fs_path,
+ std::vector<std::string>& results,
+ ReturnSetting setting) {
+ if (fs_path.string()[0] != '/') {
+ return resolveFilePattern(split(boost::filesystem::current_path().string() +
+ "/" + fs_path.string(),
+ "/"),
+ results,
+ setting);
+ }
+ return resolveFilePattern(split(fs_path.string(), "/"), results, setting);
+}
+
Status getDirectory(const boost::filesystem::path& path,
boost::filesystem::path& dirpath) {
if (!isDirectory(path).ok()) {
}
Status isDirectory(const boost::filesystem::path& path) {
- if (boost::filesystem::is_directory(path)) {
- return Status(0, "OK");
+ try {
+ if (boost::filesystem::is_directory(path)) {
+ return Status(0, "OK");
+ }
+ return Status(1, "Path is not a directory: " + path.string());
+ } catch (const boost::filesystem::filesystem_error& e) {
+ return Status(1, e.what());
}
- return Status(1, "Path is not a directory: " + path.string());
}
-std::vector<fs::path> getHomeDirectories() {
- auto sql = SQL(
- "SELECT DISTINCT directory FROM users WHERE directory != '/var/empty';");
- std::vector<fs::path> results;
- if (sql.ok()) {
- for (const auto& row : sql.rows()) {
- results.push_back(row.at("directory"));
+std::set<fs::path> getHomeDirectories() {
+ std::set<fs::path> results;
+
+ auto users = SQL::selectAllFrom("users");
+ for (const auto& user : users) {
+ if (user.at("directory").size() > 0) {
+ results.insert(user.at("directory"));
}
- } else {
- LOG(ERROR)
- << "Error executing query to return users: " << sql.getMessageString();
}
+
return results;
}
+bool safePermissions(const std::string& dir,
+ const std::string& path,
+ bool executable) {
+ struct stat file_stat, link_stat, dir_stat;
+ if (lstat(path.c_str(), &link_stat) < 0 || stat(path.c_str(), &file_stat) ||
+ stat(dir.c_str(), &dir_stat)) {
+ // Path was not real, had too may links, or could not be accessed.
+ return false;
+ }
+
+ if (dir_stat.st_mode & (1 << 9)) {
+ // Do not load modules from /tmp-like directories.
+ return false;
+ } else if (S_ISDIR(file_stat.st_mode)) {
+ // Only load file-like nodes (not directories).
+ return false;
+ } else if (file_stat.st_uid == getuid() || file_stat.st_uid == 0) {
+ // Otherwise, require matching or root file ownership.
+ if (executable && !file_stat.st_mode & S_IXUSR) {
+ // Require executable, implies by the owner.
+ return false;
+ }
+ return true;
+ }
+ // Do not load modules not owned by the user.
+ return false;
+}
+
std::string lsperms(int mode) {
static const char rwx[] = {'0', '1', '2', '3', '4', '5', '6', '7'};
std::string bits;
EXPECT_FALSE(not_found.ok());
EXPECT_EQ(not_found.toString(), "Directory not found: /foo/bar");
}
-TEST_F(FilesystemTests, test_wildcard_single_folder_list) {
+
+TEST_F(FilesystemTests, test_wildcard_single_file_list) {
std::vector<std::string> files;
+ std::vector<std::string> files_flag;
auto status = resolveFilePattern(kFakeDirectory + "/%", files);
+ auto status2 =
+ resolveFilePattern(kFakeDirectory + "/%", files_flag, REC_LIST_FILES);
EXPECT_TRUE(status.ok());
+ EXPECT_EQ(files.size(), 3);
+ EXPECT_EQ(files.size(), files_flag.size());
EXPECT_NE(
std::find(files.begin(), files.end(), kFakeDirectory + "/roto.txt"),
files.end());
EXPECT_NE(std::find(results.begin(), results.end(), "/etc/hosts"),
results.end());
}
+
+TEST_F(FilesystemTests, test_wildcard_single_folder_list) {
+ std::vector<std::string> folders;
+ auto status =
+ resolveFilePattern(kFakeDirectory + "/%", folders, REC_LIST_FOLDERS);
+ EXPECT_TRUE(status.ok());
+ EXPECT_EQ(folders.size(), 3);
+ EXPECT_NE(
+ std::find(folders.begin(), folders.end(), kFakeDirectory + "/deep11"),
+ folders.end());
+}
+
+TEST_F(FilesystemTests, test_wildcard_single_all_list) {
+ std::vector<std::string> all;
+ auto status = resolveFilePattern(kFakeDirectory + "/%", all, REC_LIST_ALL);
+ EXPECT_TRUE(status.ok());
+ EXPECT_EQ(all.size(), 6);
+ EXPECT_NE(std::find(all.begin(), all.end(), kFakeDirectory + "/roto.txt"),
+ all.end());
+ EXPECT_NE(std::find(all.begin(), all.end(), kFakeDirectory + "/deep11"),
+ all.end());
+}
+
+TEST_F(FilesystemTests, test_wildcard_double_folders) {
+ std::vector<std::string> all;
+ auto status =
+ resolveFilePattern(kFakeDirectory + "/%%", all, REC_LIST_FOLDERS);
+ EXPECT_TRUE(status.ok());
+ EXPECT_EQ(all.size(), 6);
+ EXPECT_NE(std::find(all.begin(), all.end(), kFakeDirectory), all.end());
+ EXPECT_NE(
+ std::find(all.begin(), all.end(), kFakeDirectory + "/deep11/deep2/deep3"),
+ all.end());
+}
+
+TEST_F(FilesystemTests, test_wildcard_double_all) {
+ std::vector<std::string> all;
+ auto status = resolveFilePattern(kFakeDirectory + "/%%", all, REC_LIST_ALL);
+ EXPECT_TRUE(status.ok());
+ EXPECT_EQ(all.size(), 15);
+ EXPECT_NE(std::find(all.begin(), all.end(), kFakeDirectory + "/roto.txt"),
+ all.end());
+ EXPECT_NE(
+ std::find(all.begin(), all.end(), kFakeDirectory + "/deep11/deep2/deep3"),
+ all.end());
+}
+TEST_F(FilesystemTests, test_double_wild_event_opt) {
+ std::vector<std::string> all;
+ auto status = resolveFilePattern(kFakeDirectory + "/%%", all,
+ REC_LIST_FOLDERS | REC_EVENT_OPT);
+ EXPECT_TRUE(status.ok());
+ EXPECT_EQ(all.size(), 1);
+ EXPECT_NE(std::find(all.begin(), all.end(), kFakeDirectory), all.end());
+}
+
+TEST_F(FilesystemTests, test_letter_wild_opt) {
+ std::vector<std::string> all;
+ auto status = resolveFilePattern(kFakeDirectory + "/d%", all,
+ REC_LIST_FOLDERS | REC_EVENT_OPT);
+ EXPECT_TRUE(status.ok());
+ EXPECT_EQ(all.size(), 3);
+ EXPECT_NE(std::find(all.begin(), all.end(), kFakeDirectory + "/deep1"),
+ all.end());
+ EXPECT_NE(std::find(all.begin(), all.end(), kFakeDirectory + "/door.txt"),
+ all.end());
+}
+
+TEST_F(FilesystemTests, test_dotdot) {
+ std::vector<std::string> all;
+ auto status = resolveFilePattern(kFakeDirectory + "/deep11/deep2/../../%",
+ all, REC_LIST_FILES);
+ EXPECT_TRUE(status.ok());
+ EXPECT_EQ(all.size(), 3);
+ EXPECT_NE(std::find(all.begin(), all.end(),
+ kFakeDirectory + "/deep11/deep2/../../door.txt"),
+ all.end());
+}
+
+TEST_F(FilesystemTests, test_dotdot_relative) {
+ std::vector<std::string> all;
+ auto status =
+ resolveFilePattern("../../../tools/tests/%", all, REC_LIST_ALL);
+ EXPECT_TRUE(status.ok());
+
+ bool found = false;
+ for (const auto& file : all) {
+ if (file.find("test.config")) {
+ found = true;
+ break;
+ }
+ }
+ EXPECT_TRUE(found);
+}
+
+TEST_F(FilesystemTests, test_safe_permissions) {
+ // For testing we can request a different directory path.
+ EXPECT_TRUE(safePermissions("/", kFakeDirectory + "/door.txt"));
+ // A file with a directory.mode & 0x1000 fails.
+ EXPECT_FALSE(safePermissions("/tmp", kFakeDirectory + "/door.txt"));
+ // A directory for a file will fail.
+ EXPECT_FALSE(safePermissions("/", kFakeDirectory + "/deep11"));
+ // A root-owned file is appropriate
+ EXPECT_TRUE(safePermissions("/", "/dev/zero"));
+}
}
int main(int argc, char* argv[]) {
FLAG(bool, disable_logging, false, "Disable ERROR/INFO logging");
-FLAG(string, logger_plugin, "filesystem", "The default logger plugin");
+FLAG(string, logger_plugin, "filesystem", "Logger plugin name");
FLAG(bool, log_result_events, true, "Log scheduled results as events");
BufferedLogSink::disable();
auto intermediate_logs = std::move(BufferedLogSink::dump());
+ auto& logger_plugin = Registry::getActive("logger");
+ if (!Registry::exists("logger", logger_plugin)) {
+ return;
+ }
- if (Registry::exists("logger", FLAGS_logger_plugin)) {
- // Start the custom status logging facilities, which may instruct glog as is
- // the case with filesystem logging.
- PluginRequest request = {{"init", name}};
- serializeIntermediateLog(intermediate_logs, request);
- auto status = Registry::call("logger", FLAGS_logger_plugin, request);
- if (status.ok() || forward_all) {
- // When init returns success we reenabled the log sink in forwarding
- // mode. Now, Glog status logs are buffered and sent to logStatus.
- BufferedLogSink::forward(true);
- BufferedLogSink::enable();
- }
+ // Start the custom status logging facilities, which may instruct glog as is
+ // the case with filesystem logging.
+ PluginRequest request = {{"init", name}};
+ serializeIntermediateLog(intermediate_logs, request);
+ auto status = Registry::call("logger", request);
+ if (status.ok() || forward_all) {
+ // When init returns success we reenabled the log sink in forwarding
+ // mode. Now, Glog status logs are buffered and sent to logStatus.
+ BufferedLogSink::forward(true);
+ BufferedLogSink::enable();
}
}
std::string(message, message_len)});
PluginRequest request = {{"status", "true"}};
serializeIntermediateLog(log, request);
- Registry::call("logger", FLAGS_logger_plugin, request);
+ Registry::call("logger", request);
} else {
logs_.push_back({(StatusLogSeverity)severity,
std::string(base_filename),
}
Status logString(const std::string& s) {
- return logString(s, FLAGS_logger_plugin);
+ return logString(s, Registry::getActive("logger"));
}
Status logString(const std::string& s, const std::string& receiver) {
}
Status logScheduledQueryLogItem(const osquery::ScheduledQueryLogItem& results) {
- return logScheduledQueryLogItem(results, FLAGS_logger_plugin);
+ return logScheduledQueryLogItem(results, Registry::getActive("logger"));
}
Status logScheduledQueryLogItem(const osquery::ScheduledQueryLogItem& results,
TEST_F(LoggerTests, test_logger_init) {
// Expect the logger to have been registered from the first test.
EXPECT_TRUE(Registry::exists("logger", "test"));
+ EXPECT_TRUE(Registry::setActive("logger", "test").ok());
- FLAGS_logger_plugin = "test";
initStatusLogger("logger_test");
// This will be printed to stdout.
LOG(WARNING) << "Logger test is generating a warning status (1)";
#include <boost/thread.hpp>
-#include <osquery/config.h>
#include <osquery/core.h>
-#include <osquery/events.h>
-#include <osquery/extensions.h>
-#include <osquery/logger.h>
#include <osquery/scheduler.h>
-#include "osquery/core/watcher.h"
-
const std::string kWatcherWorkerName = "osqueryd: worker";
int main(int argc, char* argv[]) {
- osquery::initOsquery(argc, argv, osquery::OSQUERY_TOOL_DAEMON);
+ osquery::Initializer runner(argc, argv, osquery::OSQUERY_TOOL_DAEMON);
- if (!osquery::isOsqueryWorker()) {
- osquery::initOsqueryDaemon();
+ if (!runner.isWorker()) {
+ runner.initDaemon();
}
- if (!osquery::FLAGS_disable_watchdog) {
- // When a watcher is used, the current watcher will fork into a worker.
- osquery::initWorkerWatcher(kWatcherWorkerName, argc, argv);
- }
+ // When a watchdog is used, the current daemon will fork/exec into a worker.
+ // In either case the watcher may start optionally loaded extensions.
+ runner.initWorkerWatcher(kWatcherWorkerName);
- // Start event threads.
- osquery::attachEvents();
- osquery::EventFactory::delay();
- osquery::startExtensionManager();
+ // Start osquery work.
+ runner.start();
// Begin the schedule runloop.
boost::thread scheduler_thread(osquery::initializeScheduler);
scheduler_thread.join();
// Finally shutdown.
- osquery::shutdownOsquery();
+ runner.shutdown();
return 0;
}
*
*/
-#include <boost/filesystem.hpp>
#include <osquery/core.h>
-#include <osquery/database.h>
#include <osquery/devtools.h>
-#include <osquery/events.h>
-#include <osquery/extensions.h>
-#include <osquery/filesystem.h>
-#include <osquery/logger.h>
-
-const std::string kShellTemp = "/tmp/osquery";
int main(int argc, char *argv[]) {
- // The shell is transient, rewrite config-loaded paths.
- if (osquery::pathExists(kShellTemp).ok() ||
- boost::filesystem::create_directory(kShellTemp)) {
- osquery::FLAGS_db_path = kShellTemp + "/shell.db";
- osquery::FLAGS_extensions_socket = kShellTemp + "/shell.em";
- FLAGS_log_dir = kShellTemp;
- }
-
// Parse/apply flags, start registry, load logger/config plugins.
- osquery::initOsquery(argc, argv, osquery::OSQUERY_TOOL_SHELL);
-
- // Start event threads.
- osquery::attachEvents();
- osquery::EventFactory::delay();
- osquery::startExtensionManager();
+ osquery::Initializer runner(argc, argv, osquery::OSQUERY_TOOL_SHELL);
+ runner.start();
// Virtual tables will be attached to the shell's in-memory SQLite DB.
int retcode = osquery::launchIntoShell(argc, argv);
// Finally shutdown.
- osquery::shutdownOsquery();
+ runner.shutdown();
return retcode;
}
#include <cstdlib>
#include <sstream>
+#include <dlfcn.h>
+
#include <boost/property_tree/json_parser.hpp>
#include <osquery/extensions.h>
}
}
+bool RegistryHelperCore::isInternal(const std::string& item_name) const {
+ if (std::find(internal_.begin(), internal_.end(), item_name) ==
+ internal_.end()) {
+ return false;
+ }
+ return true;
+}
+
+Status RegistryHelperCore::setActive(const std::string& item_name) {
+ if (items_.count(item_name) == 0 && external_.count(item_name) == 0) {
+ return Status(1, "Unknown registry item");
+ }
+ active_ = item_name;
+ return Status(0, "OK");
+}
+
+const std::string& RegistryHelperCore::getActive() const { return active_; }
+
RegistryRoutes RegistryHelperCore::getRoutes() const {
RegistryRoutes route_table;
for (const auto& item : items_) {
- if (std::find(internal_.begin(), internal_.end(), item.first) !=
- internal_.end()) {
+ if (isInternal(item.first)) {
// This is an internal plugin, do not include the route.
continue;
}
}
void RegistryHelperCore::setUp() {
+ // If the registry is using a single 'active' plugin, setUp that plugin.
+ // For config and logger, only setUp the selected plugin.
+ if (active_.size() != 0 && exists(active_, true)) {
+ items_.at(active_)->setUp();
+ return;
+ }
+
// If this registry does not auto-setup do NOT setup the registry items.
if (!auto_setup_) {
return;
// If any registry fails to add the set of external routes, stop.
break;
}
+
+ for (const auto& plugin : registry.second) {
+ VLOG(1) << "Extension " << uuid << " registered " << registry.first
+ << " plugin " << plugin.first;
+ }
}
// If any registry failed, remove each (assume a broadcast is atomic).
return call(registry_name, item_name, request, response);
}
+Status RegistryFactory::call(const std::string& registry_name,
+ const PluginRequest& request,
+ PluginResponse& response) {
+ auto& plugin = registry(registry_name)->getActive();
+ return registry(registry_name)->call(plugin, request, response);
+}
+
+Status RegistryFactory::call(const std::string& registry_name,
+ const PluginRequest& request) {
+ PluginResponse response;
+ return call(registry_name, request, response);
+}
+
+Status RegistryFactory::setActive(const std::string& registry_name,
+ const std::string& item_name) {
+ if (!exists(registry_name, item_name)) {
+ return Status(1, "Registry plugin does not exist");
+ }
+ return registry(registry_name)->setActive(item_name);
+}
+
+const std::string& RegistryFactory::getActive(
+ const std::string& registry_name) {
+ return registry(registry_name)->getActive();
+}
+
void RegistryFactory::setUp() {
for (const auto& registry : instance().all()) {
registry.second->setUp();
return registry(registry_name)->exists(item_name, local);
}
+std::vector<std::string> RegistryFactory::names() {
+ std::vector<std::string> names;
+ for (const auto& registry : all()) {
+ names.push_back(registry.second->getName());
+ }
+ return names;
+}
+
std::vector<std::string> RegistryFactory::names(
const std::string& registry_name) {
if (instance().registries_.at(registry_name) == 0) {
return instance().registry(registry_name)->count();
}
+Status RegistryHelperCore::add(const std::string& item_name, bool internal) {
+ // The item can be listed as internal, meaning it does not broadcast.
+ if (internal) {
+ internal_.push_back(item_name);
+ }
+
+ // The item may belong to a module.
+ if (RegistryFactory::usingModule()) {
+ modules_[item_name] = RegistryFactory::getModule();
+ }
+
+ return Status(0, "OK");
+}
+
+const std::map<RouteUUID, ModuleInfo>& RegistryFactory::getModules() {
+ return instance().modules_;
+}
+
+RouteUUID RegistryFactory::getModule() { return instance().module_uuid_; }
+
+bool RegistryFactory::usingModule() {
+ // Check if the registry is allowing a module's registrations.
+ return (!instance().locked() && instance().module_uuid_ != 0);
+}
+
+void RegistryFactory::shutdownModule() {
+ instance().locked(true);
+ instance().module_uuid_ = 0;
+}
+
+void RegistryFactory::initModule(const std::string& path) {
+ // Begin a module initialization, lock until the module is determined
+ // appropriate by requesting a call to `declareModule`.
+ instance().module_uuid_ = (RouteUUID)rand();
+ instance().modules_[getModule()].path = path;
+ instance().locked(true);
+}
+
+void RegistryFactory::declareModule(const std::string& name,
+ const std::string& version,
+ const std::string& min_sdk_version,
+ const std::string& sdk_version) {
+ // Check the min_sdk_version against the Registry's SDK version.
+ auto& module = instance().modules_[instance().module_uuid_];
+ module.name = name;
+ module.version = version;
+ module.sdk_version = sdk_version;
+ instance().locked(false);
+}
+
+RegistryModuleLoader::RegistryModuleLoader(const std::string& path)
+ : handle_(nullptr), path_(path) {
+ // Tell the registry that we are attempting to construct a module.
+ // Locking the registry prevents the module's global initialization from
+ // adding or creating registry items.
+ RegistryFactory::initModule(path_);
+ handle_ = dlopen(path_.c_str(), RTLD_NOW | RTLD_LOCAL);
+ if (handle_ == nullptr) {
+ VLOG(1) << "Failed to load module: " << path_;
+ VLOG(1) << dlerror();
+ return;
+ }
+
+ // The module should have called RegistryFactory::declareModule and unlocked
+ // the registry for modification. The module should have done this using
+ // the SDK's CREATE_MODULE macro, which adds the global-scope constructor.
+ if (RegistryFactory::locked()) {
+ VLOG(1) << "Failed to declare module: " << path_;
+ dlclose(handle_);
+ handle_ = nullptr;
+ }
+}
+
+void RegistryModuleLoader::init() {
+ if (handle_ == nullptr || RegistryFactory::locked()) {
+ handle_ = nullptr;
+ return;
+ }
+
+ // Locate a well-known symbol in the module.
+ // This symbol name is protected against rewriting when the module uses the
+ // SDK's CREATE_MODULE macro.
+ auto initializer = (ModuleInitalizer)dlsym(handle_, "initModule");
+ if (initializer != nullptr) {
+ initializer();
+ VLOG(1) << "Initialized module: " << path_;
+ } else {
+ VLOG(1) << "Failed to initialize module: " << path_;
+ VLOG(1) << dlerror();
+ dlclose(handle_);
+ handle_ = nullptr;
+ }
+}
+
+RegistryModuleLoader::~RegistryModuleLoader() {
+ if (handle_ == nullptr) {
+ // The module was not loaded or did not initalize.
+ RegistryFactory::instance().modules_.erase(RegistryFactory::getModule());
+ }
+
+ // We do not close the module, and thus are OK with losing a reference to the
+ // module's handle. Attempting to close and clean up is very expensive for
+ // very little value/features.
+ if (!RegistryFactory::locked()) {
+ RegistryFactory::shutdownModule();
+ }
+ // No need to clean this resource.
+ handle_ = nullptr;
+}
+
void Plugin::getResponse(const std::string& key,
const PluginResponse& response,
boost::property_tree::ptree& tree) {
Status setUp() { return Status(1, "Expect error... this is a bad dog"); }
};
-auto AutoDogRegistry = TestCoreRegistry::create<DogPlugin>("dog");
+auto AutoDogRegistry = TestCoreRegistry::create<DogPlugin>("dog", true);
TEST_F(RegistryTests, test_auto_registries) {
TestCoreRegistry::add<Doge>("dog", "doge");
}
EXPECT_TRUE(has_one_registered);
}
+
+TEST_F(RegistryTests, test_registry_modules) {
+ // Test the registry's module loading state tracking.
+ RegistryFactory::locked(false);
+ EXPECT_FALSE(RegistryFactory::locked());
+ RegistryFactory::locked(true);
+ EXPECT_TRUE(RegistryFactory::locked());
+ RegistryFactory::locked(false);
+
+ // Test initializing a module load and the module's registry modifications.
+ EXPECT_EQ(RegistryFactory::getModule(), 0);
+ RegistryFactory::initModule("/my/test/module");
+ // The registry is locked, no modifications during module global ctors.
+ EXPECT_TRUE(RegistryFactory::locked());
+ // The 'is the registry using a module' is not set during module ctors.
+ EXPECT_FALSE(RegistryFactory::usingModule());
+ EXPECT_EQ(RegistryFactory::getModules().size(), 1);
+ // The unittest can introspect into the current module.
+ auto& module = RegistryFactory::getModules().at(RegistryFactory::getModule());
+ EXPECT_EQ(module.path, "/my/test/module");
+ EXPECT_EQ(module.name, "");
+ RegistryFactory::declareModule("test", "0.1.1", "0.0.0", "0.0.1");
+ // The registry is unlocked after the module is declared.
+ // This assures that module modifications happen with the correct information
+ // and state tracking (aka the SDK limits, name, and version).
+ EXPECT_FALSE(RegistryFactory::locked());
+ // Now the 'is the registry using a module' is set for the duration of the
+ // modules loading.
+ EXPECT_TRUE(RegistryFactory::usingModule());
+ EXPECT_EQ(module.name, "test");
+ EXPECT_EQ(module.version, "0.1.1");
+ EXPECT_EQ(module.sdk_version, "0.0.1");
+
+ // Finally, when the module load is complete, we clear state.
+ RegistryFactory::shutdownModule();
+ // The registry is again locked.
+ EXPECT_TRUE(RegistryFactory::locked());
+ // And the registry is no longer using a module.
+ EXPECT_FALSE(RegistryFactory::usingModule());
+ EXPECT_EQ(RegistryFactory::getModule(), 0);
+}
}
int main(int argc, char* argv[]) {
FLAG(string,
host_identifier,
"hostname",
- "Field used to identify the host running osqueryd");
+ "Field used to identify the host running osquery");
FLAG(int32, schedule_splay_percent, 10, "Percent to splay config times");
#endif
// Iterate over scheduled queryies and add a splay to each.
- auto schedule = Config::getInstance().getScheduledQueries();
+ auto schedule = Config::getScheduledQueries();
for (auto& q : schedule) {
auto old_interval = q.interval;
auto new_interval = splayValue(old_interval, FLAGS_schedule_splay_percent);
};
TEST_F(SQLTests, test_raw_access_context) {
- REGISTER(TestTable, "table", "test_table");
+ Registry::add<TestTable>("table", "test_table");
auto results = SQL::selectAllFrom("test_table");
EXPECT_EQ(results.size(), 1);
int main(int argc, char* argv[]) {
testing::InitGoogleTest(&argc, argv);
- osquery::initOsquery(argc, argv);
return RUN_ALL_TESTS();
}
QueryData* qData = (QueryData*)argument;
Row r;
for (int i = 0; i < argc; i++) {
- r[column[i]] = argv[i];
+ if (column[i] != nullptr) {
+ r[column[i]] = (argv[i] != nullptr) ? argv[i] : "";
+ }
}
(*qData).push_back(r);
return 0;
int main(int argc, char* argv[]) {
testing::InitGoogleTest(&argc, argv);
- osquery::initOsquery(argc, argv);
return RUN_ALL_TESTS();
}
system/linux/process_open_files.cpp
system/linux/shared_memory.cpp
system/linux/smbios_tables.cpp
+ system/linux/sysctl_utils.cpp
system/linux/users.cpp
system/linux/groups.cpp
system/linux/kernel_info.cpp
system/linux/kernel_modules.cpp
system/linux/memory_map.cpp
system/linux/mounts.cpp
+ system/linux/os_version.cpp
system/linux/pci_devices.cpp
system/linux/usb_devices.cpp)
system/last.cpp
system/shell_history.cpp
system/suid_bin.cpp
+ system/system_controls.cpp
system/logged_in_users.cpp)
ADD_OSQUERY_TEST(FALSE osquery_etc_hosts_tests networking/etc_hosts_tests.cpp)
REGISTER(FileChangesEventSubscriber, "event_subscriber", "file_changes");
void FileChangesEventSubscriber::init() {
- const auto& file_map = Config::getInstance().getWatchedFiles();
+ const auto& file_map = Config::getWatchedFiles();
for (const auto& element_kv : file_map) {
for (const auto& file : element_kv.second) {
+ VLOG(1) << "Added listener to: " << file;
auto mc = createSubscriptionContext();
+ mc->recursive = 1;
mc->path = file;
mc->mask = IN_ATTRIB | IN_MODIFY | IN_DELETE | IN_CREATE;
subscribe(&FileChangesEventSubscriber::Callback, mc,
- (void*)(&element_kv.first));
+ (void*)(&element_kv.first));
}
}
}
r["action"] = ec->action;
r["time"] = ec->time_string;
r["target_path"] = ec->path;
- r["category"] = *(std::string*)user_data;
+ if (user_data != nullptr) {
+ r["category"] = *(std::string*)user_data;
+ } else {
+ r["category"] = "Undefined";
+ }
r["transaction_id"] = INTEGER(ec->event->cookie);
r["md5"] = hashFromFile(HASH_TYPE_MD5, ec->path);
r["sha1"] = hashFromFile(HASH_TYPE_SHA1, ec->path);
Column("device", TEXT, "MA:MI Major/minor device ID"),
Column("inode", INTEGER, "Mapped path inode, 0 means uninitialized (BSS)"),
Column("path", TEXT, "Path to mapped file or mapped type"),
- Column("is_pseudo", INTEGER, "1 if path is a pseudo path, else 0"),
+ Column("pseudo", INTEGER, "1 if path is a pseudo path, else 0"),
])
implementation("processes@genProcessMemoryMap")
--- /dev/null
+table_name("osquery_extensions")
+description("List of active osquery extensions.")
+schema([
+ Column("uuid", BIGINT, "The transient ID assigned for communication"),
+ Column("name", TEXT, "Extension's name"),
+ Column("version", TEXT, "Extenion's version"),
+ Column("sdk_version", TEXT, "osquery SDK version used to build the extension"),
+ Column("path", TEXT, "Path of the extenion's domain socket or library path"),
+ Column("type", TEXT, "SDK extension type: extension or module")
+])
+attributes(utility=True)
+implementation("osquery@genOsqueryExtensions")
--- /dev/null
+table_name("osquery_registry")
+description("List the osquery registry plugins.")
+schema([
+ Column("registry", TEXT, "Name of the osquery registry"),
+ Column("name", TEXT, "Name of the plugin item"),
+ Column("owner_uuid", INTEGER, "Extension route UUID (0 for core)"),
+ Column("internal", INTEGER, "1 if the plugin is internal else 0"),
+ Column("active", INTEGER, "1 if this plugin is active else 0"),
+])
+attributes(utility=True)
+implementation("osquery@genOsqueryRegistry")
--- /dev/null
+table_name("os_version")
+description("A single row containing the operating system version.")
+schema([
+ Column("major", INTEGER),
+ Column("minor", INTEGER),
+ Column("patch", INTEGER),
+])
+implementation("system/os_version@genOSVersion")
+++ /dev/null
-table_name("osquery_extensions")
-description("List of active osquery extensions.")
-schema([
- Column("uuid", BIGINT),
- Column("name", TEXT),
- Column("version", TEXT),
- Column("sdk_version", TEXT),
- Column("socket", TEXT),
-])
-attributes(utility=True)
-implementation("osquery@genOsqueryExtensions")
--- /dev/null
+table_name("system_controls")
+description("sysctl names, values, and settings information.")
+schema([
+ Column("name", TEXT, "Full sysctl MIB name"),
+ Column("oid", TEXT, "Control MIB"),
+ Column("subsystem", TEXT, "Subsystem ID, control type"),
+ Column("current_value", TEXT, "Value of setting"),
+ Column("config_value", TEXT, "The MIB value set in /etc/sysctl.conf"),
+ Column("type", TEXT, "Data type"),
+])
+implementation("system_controls@genSystemControls")
--- /dev/null
+/*
+ * Copyright (c) 2014, Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree. An additional grant
+ * of patent rights can be found in the PATENTS file in the same directory.
+ *
+ */
+
+#include <string>
+
+#include <boost/regex.hpp>
+
+#include <osquery/filesystem.h>
+#include <osquery/sql.h>
+#include <osquery/tables.h>
+
+#ifdef CENTOS
+const std::string kLinuxOSRelease = "/etc/redhat-release";
+#define kLinuxOSRegex "CentOS release ([0-9]+).([0-9]+)"
+#else
+const std::string kLinuxOSRelease = "/etc/os-release";
+#define kLinuxOSRegex "VERSION=\"([0-9]+)\\.([0-9]+)[\\.]{0,1}([0-9]+)?"
+#endif
+
+namespace osquery {
+namespace tables {
+
+QueryData genOSVersion(QueryContext& context) {
+ std::string content;
+ if (!readFile(kLinuxOSRelease, content).ok()) {
+ return {};
+ }
+
+ std::vector<std::string> version = {"0", "0", "0"};
+ boost::regex rx(kLinuxOSRegex);
+ boost::smatch matches;
+ for (const auto& line : osquery::split(content, "\n")) {
+ if (boost::regex_search(line, matches, rx)) {
+ // Push the matches in reverse order.
+ version[0] = matches[1];
+ version[1] = matches[2];
+ if (matches.size() == 4) {
+ // Patch is optional for Ubuntu and not used for CentOS.
+ version[2] = matches[3];
+ }
+ break;
+ }
+ }
+
+ Row r;
+ if (version.size() == 3) {
+ r["major"] = INTEGER(version[0]);
+ r["minor"] = INTEGER(version[1]);
+ r["patch"] = INTEGER(version[2]);
+ }
+ return {r};
+}
+}
+}
}
// BSS with name in pathname.
- r["is_pseudo"] = (fields[4] == "0" && r["path"].size() > 0) ? "1" : "0";
+ r["pseudo"] = (fields[4] == "0" && r["path"].size() > 0) ? "1" : "0";
results.push_back(r);
}
}
--- /dev/null
+/*
+ * Copyright (c) 2014, Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree. An additional grant
+ * of patent rights can be found in the PATENTS file in the same directory.
+ *
+ */
+
+#include <sys/sysctl.h>
+
+#include <boost/algorithm/string/trim.hpp>
+
+#include <osquery/filesystem.h>
+#include <osquery/tables.h>
+
+#include "osquery/tables/system/sysctl_utils.h"
+
+namespace fs = boost::filesystem;
+
+namespace osquery {
+namespace tables {
+
+const std::string kSystemControlPath = "/proc/sys/";
+
+void genControlInfo(const std::string& mib_path, QueryData& results,
+ const std::map<std::string, std::string>& config) {
+ if (isDirectory(mib_path).ok()) {
+ // Iterate through the subitems and items.
+ std::vector<std::string> items;
+ if (listDirectoriesInDirectory(mib_path, items).ok()) {
+ for (const auto& item : items) {
+ genControlInfo(item, results, config);
+ }
+ }
+
+ if (listFilesInDirectory(mib_path, items).ok()) {
+ for (const auto& item : items) {
+ genControlInfo(item, results, config);
+ }
+ }
+ return;
+ }
+
+ // This is a file (leaf-control).
+ Row r;
+ r["name"] = mib_path.substr(kSystemControlPath.size());
+
+ std::replace(r["name"].begin(), r["name"].end(), '/', '.');
+ // No known way to convert name MIB to int array.
+ r["subsystem"] = osquery::split(r.at("name"), ".")[0];
+
+ if (isReadable(mib_path).ok()) {
+ std::string content;
+ readFile(mib_path, content);
+ boost::trim(content);
+ r["current_value"] = content;
+ }
+
+ if (config.count(r.at("name")) > 0) {
+ r["config_value"] = config.at(r.at("name"));
+ }
+ r["type"] = "string";
+ results.push_back(r);
+}
+
+void genControlInfo(int* oid,
+ size_t oid_size,
+ QueryData& results,
+ const std::map<std::string, std::string>& config) {
+ // Get control size
+ size_t response_size = CTL_MAX_VALUE;
+ char response[CTL_MAX_VALUE + 1] = {0};
+ if (sysctl(oid, oid_size, response, &response_size, 0, 0) != 0) {
+ // Cannot request MIB data.
+ return;
+ }
+
+ // Data is output, but no way to determine type (long, int, string, struct).
+ Row r;
+ r["oid"] = stringFromMIB(oid, oid_size);
+ r["current_value"] = std::string(response);
+ r["type"] = "string";
+ results.push_back(r);
+}
+
+void genAllControls(QueryData& results,
+ const std::map<std::string, std::string>& config,
+ const std::string& subsystem) {
+ // Linux sysctl subsystems are directories in /proc
+ std::vector<std::string> subsystems;
+ if (!listDirectoriesInDirectory("/proc/sys", subsystems).ok()) {
+ return;
+ }
+
+ for (const auto& sub : subsystems) {
+ if (subsystem.size() != 0 && fs::path(sub).filename().string() != subsystem) {
+ // Request is limiting subsystem.
+ continue;
+ }
+ genControlInfo(sub, results, config);
+ }
+}
+
+void genControlInfoFromName(const std::string& name, QueryData& results,
+ const std::map<std::string, std::string>& config) {
+ // Convert '.'-tokenized name to path.
+ std::string name_path = name;
+ std::replace(name_path.begin(), name_path.end(), '.', '/');
+ auto mib_path = fs::path(kSystemControlPath) / name_path;
+
+ genControlInfo(mib_path.string(), results, config);
+}
+}
+}
--- /dev/null
+/*
+ * Copyright (c) 2014, Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree. An additional grant
+ * of patent rights can be found in the PATENTS file in the same directory.
+ *
+ */
+
+#include <sys/sysctl.h>
+
+#include <osquery/tables.h>
+
+namespace osquery {
+namespace tables {
+
+#define CTL_MAX_VALUE 128
+
+#ifndef CTL_DEBUG_MAXID
+#define CTL_DEBUG_MAXID (CTL_MAXNAME * 2)
+#endif
+
+std::string stringFromMIB(const int* oid, size_t oid_size);
+
+/// Must be implemented by the platform.
+void genAllControls(QueryData& results,
+ const std::map<std::string, std::string>& config,
+ const std::string& subsystem);
+
+/// Must be implemented by the platform.
+void genControlInfo(int* oid,
+ size_t oid_size,
+ QueryData& results,
+ const std::map<std::string, std::string>& config);
+
+/// Must be implemented by the platform.
+void genControlInfoFromName(const std::string& name, QueryData& results,
+ const std::map<std::string, std::string>& config);
+}
+}
--- /dev/null
+/*
+ * Copyright (c) 2014, Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree. An additional grant
+ * of patent rights can be found in the PATENTS file in the same directory.
+ *
+ */
+
+#include <boost/algorithm/string/trim.hpp>
+
+#include <osquery/filesystem.h>
+#include <osquery/tables.h>
+
+#include "osquery/tables/system/sysctl_utils.h"
+
+namespace osquery {
+namespace tables {
+
+const std::vector<std::string> kControlSettingsFiles = {"/etc/sysctl.conf"};
+
+const std::vector<std::string> kControlSettingsDirs = {
+ "/run/sysctl.d/%.conf",
+ "/etc/sysctl.d/%.conf",
+ "/usr/local/lib/sysctl.d/%.conf",
+ "/usr/lib/sysctl.d/%.conf",
+ "/lib/sysctl.d/%.conf",
+};
+
+std::string stringFromMIB(const int* oid, size_t oid_size) {
+ std::string result;
+ for (size_t i = 0; i < oid_size; ++i) {
+ // Walk an int-encoded MIB and return the string representation, '.'.
+ if (result.size() > 0) {
+ result += ".";
+ }
+ result += std::to_string(oid[i]);
+ }
+ return result;
+}
+
+void genControlInfoFromOIDString(const std::string& oid_string, QueryData& results,
+ const std::map<std::string, std::string>& config) {
+ int request[CTL_DEBUG_MAXID + 2] = {0};
+ auto tokens = osquery::split(oid_string, ".");
+ if (tokens.size() > CTL_DEBUG_MAXID) {
+ // OID input string was too large.
+ return;
+ }
+
+ // Convert the string into an int array.
+ for (size_t i = 0; i < tokens.size(); ++i) {
+ request[i] = atol(tokens.at(i).c_str());
+ }
+ genControlInfo((int*)request, tokens.size(), results, config);
+}
+
+void genControlConfigFromPath(const std::string& path,
+ std::map<std::string, std::string>& config) {
+ std::string content;
+ if (!osquery::readFile(path, content).ok()) {
+ return;
+ }
+
+ for (auto& line : split(content, "\n")) {
+ boost::trim(line);
+ if (line[0] == '#' || line[0] == ';') {
+ continue;
+ }
+
+ // Try to tokenize the config line using '='.
+ auto detail = split(line, "=");
+ if (detail.size() == 2) {
+ boost::trim(detail[0]);
+ boost::trim(detail[1]);
+ config[detail[0]] = detail[1];
+ }
+ }
+}
+
+QueryData genSystemControls(QueryContext& context) {
+ QueryData results;
+
+ // Read the sysctl.conf values.
+ std::map<std::string, std::string> config;
+ for (const auto& path : kControlSettingsFiles) {
+ genControlConfigFromPath(path, config);
+ }
+
+ for (const auto& dirs : kControlSettingsDirs) {
+ std::vector<std::string> configs;
+ if (resolveFilePattern(dirs, configs).ok()) {
+ for (const auto& path : configs) {
+ genControlConfigFromPath(path, config);
+ }
+ }
+ }
+
+ // Iterate through the sysctl-defined macro of control types.
+ if (context.constraints["name"].exists()) {
+ // Request MIB information by the description (name).
+ auto names = context.constraints["name"].getAll(EQUALS);
+ for (const auto& name : names) {
+ genControlInfoFromName(name, results, config);
+ }
+ } else if (context.constraints["oid"].exists()) {
+ // Request MIB by OID as a string, parse into set of INTs.
+ auto oids = context.constraints["oid"].getAll(EQUALS);
+ for (const auto& oid_string : oids) {
+ genControlInfoFromOIDString(oid_string, results, config);
+ }
+ } else if (context.constraints["subsystem"].exists()) {
+ // Limit the MIB search to a subsystem name (first find the INT).
+ auto subsystems = context.constraints["subsystem"].getAll(EQUALS);
+ for (const auto& subsystem : subsystems) {
+ genAllControls(results, config, subsystem);
+ }
+ } else {
+ genAllControls(results, config, "");
+ }
+
+ return results;
+}
+}
+}
// Iterate over the directory and generate a hash for each regular file.
boost::filesystem::directory_iterator begin(directory), end;
for (; begin != end; ++begin) {
- if (boost::filesystem::is_regular_file(begin->status())) {
- genFileInfo(begin->path().string(),
- begin->path().filename().string(),
- directory_string,
- results);
- }
+ genFileInfo(begin->path().string(),
+ begin->path().filename().string(),
+ directory_string,
+ results);
}
}
#include <osquery/extensions.h>
#include <osquery/flags.h>
#include <osquery/logger.h>
+#include <osquery/registry.h>
#include <osquery/sql.h>
#include <osquery/tables.h>
return results;
}
+QueryData genOsqueryRegistry(QueryContext& context) {
+ QueryData results;
+
+ const auto& registries = RegistryFactory::all();
+ for (const auto& registry : registries) {
+ const auto& plugins = registry.second->all();
+ for (const auto& plugin : plugins) {
+ Row r;
+ r["registry"] = registry.first;
+ r["name"] = plugin.first;
+ r["owner_uuid"] = "0";
+ r["internal"] = (registry.second->isInternal(plugin.first)) ? "1" : "0";
+ r["active"] = "1";
+ results.push_back(r);
+ }
+
+ for (const auto& route : registry.second->getExternal()) {
+ Row r;
+ r["registry"] = registry.first;
+ r["name"] = route.first;
+ r["owner_uuid"] = INTEGER(route.second);
+ r["internal"] = "0";
+ r["active"] = "1";
+ results.push_back(r);
+ }
+ }
+
+ return results;
+}
+
QueryData genOsqueryExtensions(QueryContext& context) {
QueryData results;
ExtensionList extensions;
- if (!getExtensions(extensions).ok()) {
- return {};
+ if (getExtensions(extensions).ok()) {
+ for (const auto& extenion : extensions) {
+ Row r;
+ r["uuid"] = TEXT(extenion.first);
+ r["name"] = extenion.second.name;
+ r["version"] = extenion.second.version;
+ r["sdk_version"] = extenion.second.sdk_version;
+ r["path"] = getExtensionSocket(extenion.first);
+ r["type"] = "extension";
+ results.push_back(r);
+ }
}
- for (const auto& extenion : extensions) {
+ const auto& modules = RegistryFactory::getModules();
+ for (const auto& module : modules) {
Row r;
- r["uuid"] = TEXT(extenion.first);
- r["name"] = extenion.second.name;
- r["version"] = extenion.second.version;
- r["sdk_version"] = extenion.second.sdk_version;
- r["socket"] = getExtensionSocket(extenion.first);
+ r["uuid"] = TEXT(module.first);
+ r["name"] = module.second.name;
+ r["version"] = module.second.version;
+ r["sdk_version"] = module.second.sdk_version;
+ r["path"] = module.second.path;
+ r["type"] = "module";
results.push_back(r);
}
r["pid"] = INTEGER(getpid());
std::string hash_string;
- auto s = Config::getInstance().getMD5(hash_string);
+ auto s = Config::getMD5(hash_string);
if (s.ok()) {
r["config_md5"] = TEXT(hash_string);
} else {
namespace osquery {
namespace tables {
-const int kNumCols = 1;
-
QueryData genTime(QueryContext& context) {
Row r;
time_t _time = time(0);
r["minutes"] = INTEGER(now->tm_min);
r["seconds"] = INTEGER(now->tm_sec);
QueryData results;
- for (int i = 0; i < kNumCols; ++i) {
- results.push_back(r);
- }
+ results.push_back(r);
return results;
}
}
Name: osquery
-Version: 1.4.2
+Version: 1.4.3
Release: 0
License: Apache-2.0 and GPLv2
Summary: A SQL powered operating system instrumentation, monitoring framework.
# make the targets file
mkdir -p "$SYNC_DIR/code-analysis"
(cd "$SYNC_DIR/code-analysis" && SDK=True cmake -DCMAKE_EXPORT_COMPILE_COMMANDS=ON ../../../../)
-python tools/codegen/gentargets.py -i "$SYNC_DIR/code-analysis/compile_commands.json" >$SYNC_DIR/osquery/TARGETS
+python tools/codegen/gentargets.py -v $VERSION --sdk $VERSION \
+ -i "$SYNC_DIR/code-analysis/compile_commands.json" >$SYNC_DIR/osquery/TARGETS
# wrap it up in a tarball
(cd "$SYNC_DIR" && tar -zcf osquery-sync-$VERSION.tar.gz osquery)
--- /dev/null
+{
+ "options":{
+ "optionC" : "optionc-val",
+ "optionD" : "optiond-val"
+ }
+}
\ No newline at end of file
--- /dev/null
+{
+ "scheduledQueries": [
+ {
+ "name": "time_again",
+ "query": "select * from time;",
+ "interval": 1
+ }
+ ],
+ "options":{
+ "optionA" : "optiona-val",
+ "optionB" : "optionb-val"
+ },
+ "additional_monitoring" : {
+ "other_thing" : {"element" : "key"}
+ }
+}
projects / platform / core / security / vist.git / commitdiff