From: sangwan.kwon Date: Thu, 22 Jan 2015 22:00:35 +0000 (-0800) Subject: Bump version to upstream-1.4.0 X-Git-Tag: accepted/tizen/unified/20200810.122954~249 X-Git-Url: http://review.tizen.org/git/?a=commitdiff_plain;h=48e70f2d66a3167074aa02e8f894fd1bf4dd5f5f;p=platform%2Fcore%2Fsecurity%2Fvist.git Bump version to upstream-1.4.0 - Add a watcher/worker model for osqueryd - Change to a new registry model - Add getQueryColumns function to core - Add extension API with thrift RPC Added: kernel_info, shared_memory, process_memory_map Signed-off-by: sangwan.kwon --- diff --git a/CMake/Thrift.cmake b/CMake/Thrift.cmake new file mode 100644 index 0000000..2e13097 --- /dev/null +++ b/CMake/Thrift.cmake @@ -0,0 +1,14 @@ +# Target for generating osquery thirft (extensions) code. +SET(OSQUERY_THRIFT_DIR "${CMAKE_BINARY_DIR}/generated/gen-cpp") +SET(OSQUERY_THRIFT_GENERATED_FILES ${OSQUERY_THRIFT_DIR}/Extension.cpp + ${OSQUERY_THRIFT_DIR}/Extension.h + ${OSQUERY_THRIFT_DIR}/ExtensionManager.cpp + ${OSQUERY_THRIFT_DIR}/ExtensionManager.h + ${OSQUERY_THRIFT_DIR}/osquery_types.cpp + ${OSQUERY_THRIFT_DIR}/osquery_types.h) + +# Allow targets to warn if the thrift interface code is not defined. +ADD_DEFINITIONS(-DOSQUERY_THRIFT="${OSQUERY_THRIFT_DIR}") + +# For the extensions targets, allow them to include thrift interface headers. +INCLUDE_DIRECTORIES("${OSQUERY_THRIFT_DIR}") diff --git a/CMakeLists.txt b/CMakeLists.txt index 617c2f7..c7ddb8f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -41,5 +41,7 @@ INCLUDE_DIRECTORIES("/usr/local/include") ENABLE_TESTING() +INCLUDE(CMake/Thrift.cmake) + ADD_SUBDIRECTORY(osquery) ADD_SUBDIRECTORY(sqlite3) diff --git a/Makefile b/Makefile index 2265a2d..78fe912 100644 --- a/Makefile +++ b/Makefile @@ -8,7 +8,3 @@ clean: docker_run: docker build --network=host --tag tizen-osquery ./docker docker run --rm -it --net=host --privileged -v $(shell pwd):/usr/src tizen-osquery - -%:: - mkdir -p build - cd build && cmake .. && make --no-print-directory $@ diff --git a/include/osquery/config.h b/include/osquery/config.h index f3360e7..8998bcb 100644 --- a/include/osquery/config.h +++ b/include/osquery/config.h @@ -15,13 +15,14 @@ #include #include +#include #include #include namespace osquery { /// The builder or invoker may change the default config plugin. -DECLARE_string(config_retriever); +DECLARE_string(config_plugin); /** * @brief A native representation of osquery configuration data. @@ -164,4 +165,56 @@ class Config { */ OsqueryConfig cfg_; }; + +/** + * @brief Superclass for the pluggable config component. + * + * In order to make the distribution of configurations to hosts running + * osquery, we take advantage of a plugin interface which allows you to + * integrate osquery with your internal configuration distribution mechanisms. + * You may use ZooKeeper, files on disk, a custom solution, etc. In order to + * use your specific configuration distribution system, one simply needs to + * create a custom subclass of ConfigPlugin. That subclass should implement + * the ConfigPlugin::genConfig method. + * + * Consider the following example: + * + * @code{.cpp} + * class TestConfigPlugin : public ConfigPlugin { + * public: + * virtual std::pair genConfig() { + * std::string config; + * auto status = getMyConfig(config); + * return std::make_pair(status, config); + * } + * }; + * + * REGISTER(TestConfigPlugin, "config", "test"); + * @endcode + */ +class ConfigPlugin : public Plugin { + public: + /** + * @brief Virtual method which should implemented custom config retrieval + * + * ConfigPlugin::genConfig should be implemented by a subclasses of + * ConfigPlugin which needs to retrieve config data in a custom way. + * + * @return a pair such that pair.first is an osquery::Status instance which + * indicates the success or failure of config retrieval. If pair.first + * indicates that config retrieval was successful, then the config data + * should be returned in pair.second. + */ + virtual std::pair genConfig() = 0; + Status call(const PluginRequest& request, PluginResponse& response); +}; + +/** + * @brief Config plugin registry. + * + * This creates an osquery registry for "config" which may implement + * ConfigPlugin. A ConfigPlugin's call API should make use of a genConfig + * after reading JSON data in the plugin implementation. + */ +CREATE_REGISTRY(ConfigPlugin, "config"); } diff --git a/include/osquery/config/plugin.h b/include/osquery/config/plugin.h deleted file mode 100644 index be04a0c..0000000 --- a/include/osquery/config/plugin.h +++ /dev/null @@ -1,75 +0,0 @@ -/* - * Copyright (c) 2014, Facebook, Inc. - * All rights reserved. - * - * This source code is licensed under the BSD-style license found in the - * LICENSE file in the root directory of this source tree. An additional grant - * of patent rights can be found in the PATENTS file in the same directory. - * - */ - -#pragma once - -#include -#include - -#include -#include - -namespace osquery { - -/** - * @brief Superclass for the pluggable config component. - * - * In order to make the distribution of configurations to hosts running - * osquery, we take advantage of a plugin interface which allows you to - * integrate osquery with your internal configuration distribution mechanisms. - * You may use ZooKeeper, files on disk, a custom solution, etc. In order to - * use your specific configuration distribution system, one simply needs to - * create a custom subclass of ConfigPlugin. That subclass should implement - * the ConfigPlugin::genConfig method. - * - * Consider the following example: - * - * @code{.cpp} - * class TestConfigPlugin : public ConfigPlugin { - * public: - * virtual std::pair genConfig() { - * std::string config; - * auto status = getMyConfig(config); - * return std::make_pair(status, config); - * } - * }; - * - * REGISTER_CONFIG_PLUGIN( - * "test", std::make_shared()); - * @endcode - */ -class ConfigPlugin { - public: - /** - * @brief Virtual method which should implemented custom config retrieval - * - * ConfigPlugin::genConfig should be implemented by a subclasses of - * ConfigPlugin which needs to retrieve config data in a custom way. - * - * @return a pair such that pair.first is an osquery::Status instance which - * indicates the success or failure of config retrieval. If pair.first - * indicates that config retrieval was successful, then the config data - * should be returned in pair.second. - */ - virtual std::pair genConfig() = 0; - - /// Virtual destructor - virtual ~ConfigPlugin() {} -}; -} - -DECLARE_REGISTRY(ConfigPlugins, - std::string, - std::shared_ptr) - -#define REGISTERED_CONFIG_PLUGINS REGISTRY(ConfigPlugins) - -#define REGISTER_CONFIG_PLUGIN(name, decorator) \ - REGISTER(ConfigPlugins, name, decorator) diff --git a/include/osquery/core.h b/include/osquery/core.h index a57080a..3651214 100644 --- a/include/osquery/core.h +++ b/include/osquery/core.h @@ -3,7 +3,7 @@ * All rights reserved. * * This source code is licensed under the BSD-style license found in the - * LICENSE file in the root directory of this source tree. An additional grant + * LICENSE file in the root directory of this source tree. An additional grant * of patent rights can be found in the PATENTS file in the same directory. * */ @@ -13,17 +13,18 @@ #include #include -#include - -#include - -#include +#include #ifndef STR #define STR_OF(x) #x #define STR(x) STR_OF(x) #endif +#ifndef FRIEND_TEST +#define FRIEND_TEST(test_case_name, test_name) \ + friend class test_case_name##_##test_name##_Test +#endif + namespace osquery { /** @@ -44,59 +45,6 @@ enum osqueryTool { }; /** - * @brief Execute a query - * - * This is a lower-level version of osquery::SQL. Prefer to use osquery::SQL. - * - * @code{.cpp} - * std::string q = "SELECT * FROM time;"; - * int i = 0; - * auto qd = query(q, i); - * if (i == 0) { - * for (const auto& each : qd) { - * for (const auto& it : each) { - * LOG(INFO) << it.first << ": " << it.second; - * } - * } - * } else { - * LOG(ERROR) << "Error: " << i; - * } - * @endcode - * - * @param q the query to execute - * @param error_return an int indicating the success or failure of the query - * - * @return the results of the query - */ -osquery::QueryData query(const std::string& q, int& error_return); - -/** - * @brief Execute a query on a specific database - * - * If you need to use a different database, other than the osquery default, - * use this method and pass along a pointer to a SQLite3 database. This is - * useful for testing. - * - * @param q the query to execute - * @param error_return an int indicating the success or failure of the query - * @param db the SQLite3 database the execute query q against - * - * @return the results of the query - */ -osquery::QueryData query(const std::string& q, int& error_return, sqlite3* db); - -/** - * @brief Return a fully configured sqlite3 database object - * - * An osquery database is basically just a SQLite3 database with several - * virtual tables attached. This method is the main abstraction for creating - * SQLite3 databases within osquery. - * - * @return a SQLite3 database with all virtual tables attached - */ -sqlite3* createDB(); - -/** * @brief Sets up various aspects of osquery execution state. * * osquery needs a few things to happen as soon as the executable begins @@ -109,9 +57,20 @@ sqlite3* createDB(); void initOsquery(int argc, char* argv[], int tool = OSQUERY_TOOL_TEST); /** - * @brief Split a given string based on an optional deliminator. + * @brief Sets up a process as a osquery daemon. + */ +void initOsqueryDaemon(); + +/** + * @brief Turns of various aspects of osquery such as event loops. * - * If no deliminator is supplied, the string will be split based on whitespace. + */ +void shutdownOsquery(); + +/** + * @brief Split a given string based on an optional delimiter. + * + * If no delimiter is supplied, the string will be split based on whitespace. * * @param s the string that you'd like to split * @param delim the delimiter which you'd like to split the string by @@ -151,13 +110,6 @@ std::string getAsciiTime(); int getUnixTime(); /** - * @brief Return a vector of all home directories on the system - * - * @return a vector of strings representing the path of all home directories - */ -std::vector getHomeDirectories(); - -/** * @brief Inline helper function for use with utf8StringSize */ template diff --git a/include/osquery/database/db_handle.h b/include/osquery/database/db_handle.h index 1189104..3d77118 100644 --- a/include/osquery/database/db_handle.h +++ b/include/osquery/database/db_handle.h @@ -88,6 +88,16 @@ class DBHandle { static std::shared_ptr getInstance(); /** + * @brief Check the sanity of the database configuration options + * + * Create a handle to the backing store using the database configuration. + * Catch any instance creation exceptions and release the handle immediately. + * + * @return Success if a handle was created without error. + */ + static bool checkDB(); + + /** * @brief Helper method which can be used to get a raw pointer to the * underlying RocksDB database handle * diff --git a/include/osquery/database/query.h b/include/osquery/database/query.h index 42106f5..dcfcaef 100644 --- a/include/osquery/database/query.h +++ b/include/osquery/database/query.h @@ -14,13 +14,16 @@ #include #include -#include - #include #include #include #include +#ifndef FRIEND_TEST +#define FRIEND_TEST(test_case_name, test_name) \ + friend class test_case_name##_##test_name##_Test +#endif + namespace osquery { /// Error message used when a query name isn't found in the database diff --git a/include/osquery/dispatcher.h b/include/osquery/dispatcher.h index 1f79d55..07c3eff 100644 --- a/include/osquery/dispatcher.h +++ b/include/osquery/dispatcher.h @@ -15,6 +15,8 @@ #include #include +#include + #include #include #include @@ -23,6 +25,9 @@ namespace osquery { +typedef apache::thrift::concurrency::ThreadManager InternalThreadManager; +typedef std::shared_ptr InternalThreadManagerRef; + /** * @brief Default number of threads in the thread pool. * @@ -31,6 +36,34 @@ namespace osquery { */ extern const int kDefaultThreadPoolSize; +class InternalRunnable : public apache::thrift::concurrency::Runnable { + public: + virtual ~InternalRunnable() {} + InternalRunnable() : run_(false) {} + + public: + /// The boost::thread entrypoint. + void run() { + run_ = true; + enter(); + } + + /// Check if the thread's entrypoint (run) executed, meaning thread context + /// was allocated. + bool hasRun() { return run_; } + /// Sleep in a boost::thread interruptable state. + void interruptableSleep(size_t milli); + + protected: + /// Require the runnable thread define an entrypoint. + virtual void enter() = 0; + + private: + bool run_; +}; + +typedef boost::shared_ptr InternalRunnableRef; + /** * @brief Singleton for queueing asynchronous tasks to be executed in parallel * @@ -80,7 +113,9 @@ class Dispatcher { * @return an instance of osquery::Status, indicating the success or failure * of the operation. */ - Status add(std::shared_ptr task); + Status add(std::shared_ptr task); + + Status addService(std::shared_ptr service); /** * @brief Getter for the underlying thread manager instance. @@ -100,8 +135,7 @@ class Dispatcher { * @return a shared pointer to the Apache Thrift `ThreadManager` instance * which is currently being used to orchestrate multi-threaded operations. */ - std::shared_ptr - getThreadManager(); + InternalThreadManagerRef getThreadManager(); /** * @brief Joins the thread manager. @@ -112,12 +146,16 @@ class Dispatcher { */ void join(); + void joinServices(); + + void removeServices(); + /** * @brief Get the current state of the thread manager. * * @return an Apache Thrift STATE enum. */ - apache::thrift::concurrency::ThreadManager::STATE state() const; + InternalThreadManager::STATE state() const; /** * @brief Add a worker thread. @@ -208,6 +246,8 @@ class Dispatcher { * * @see getThreadManager */ - std::shared_ptr thread_manager_; + InternalThreadManagerRef thread_manager_; + std::vector > service_threads_; + std::vector > services_; }; } diff --git a/include/osquery/events.h b/include/osquery/events.h index 59f0a09..e500905 100644 --- a/include/osquery/events.h +++ b/include/osquery/events.h @@ -95,7 +95,7 @@ extern const std::vector kEventTimeLists; */ #define DECLARE_PUBLISHER(TYPE) \ public: \ - EventPublisherID type() { return TYPE; } + EventPublisherID type() const { return TYPE; } /** * @brief DECLARE_SUBSCRIBER supplies needed boilerplate code that applies a @@ -103,7 +103,7 @@ extern const std::vector kEventTimeLists; */ #define DECLARE_SUBSCRIBER(NAME) \ public: \ - EventSubscriberID name() { return NAME; } + EventSubscriberID name() const { return NAME; } /** * @brief A Subscription is used to configure an EventPublisher and bind a @@ -146,7 +146,7 @@ struct Subscription { } }; -class EventPublisherCore { +class EventPublisherPlugin : public Plugin { public: /** * @brief A new Subscription was added, potentially change state based on all @@ -211,29 +211,31 @@ class EventPublisherCore { void fire(const EventContextRef& ec, EventTime time = 0); /// Number of Subscription%s watching this EventPublisher. - size_t numSubscriptions() { return subscriptions_.size(); } + size_t numSubscriptions() const { return subscriptions_.size(); } /** * @brief The number of events fired by this EventPublisher. * * @return The number of events. */ - size_t numEvents() { return next_ec_id_; } + size_t numEvents() const { return next_ec_id_; } /// Overriding the EventPublisher constructor is not recommended. - EventPublisherCore() : next_ec_id_(0), ending_(false){}; - virtual ~EventPublisherCore() {} + EventPublisherPlugin() : next_ec_id_(0), ending_(false), started_(false) {}; + virtual ~EventPublisherPlugin() {} /// Return a string identifier associated with this EventPublisher. - virtual EventPublisherID type() { return "publisher"; } + virtual EventPublisherID type() const { return "publisher"; } - void shouldEnd(bool should_end) { ending_ = should_end; } - bool isEnding() { return ending_; } + bool isEnding() const { return ending_; } + void isEnding(bool ending) { ending_ = ending; } + bool hasStarted() const { return started_; } + void hasStarted(bool started) { started_ = started; } protected: /// The internal fire method used by the typed EventPublisher. virtual void fireCallback(const SubscriptionRef& sub, - const EventContextRef& ec) = 0; + const EventContextRef& ec) const = 0; /// The EventPublisher will keep track of Subscription%s that contain callins. SubscriptionVector subscriptions_; @@ -243,8 +245,14 @@ class EventPublisherCore { EventContextID next_ec_id_; private: + EventPublisherPlugin(EventPublisherPlugin const&); + void operator=(EventPublisherPlugin const&); + + private: /// Set ending to True to cause event type run loops to finish. bool ending_; + /// Set to indicate whether the event run loop ever started. + bool started_; /// A lock for incrementing the next EventContextID. boost::mutex ec_id_lock_; @@ -287,7 +295,7 @@ class EventPublisherCore { * (thus event) matches. */ template -class EventPublisher : public EventPublisherCore { +class EventPublisher : public EventPublisherPlugin { public: /// A nested helper typename for the templated SubscriptionContextRef. typedef typename std::shared_ptr SCRef; @@ -329,7 +337,8 @@ class EventPublisher : public EventPublisherCore { * @param sub The SubscriptionContext and optional EventCallback. * @param ec The event that was fired. */ - void fireCallback(const SubscriptionRef& sub, const EventContextRef& ec) { + void fireCallback(const SubscriptionRef& sub, + const EventContextRef& ec) const { auto pub_sc = getSubscriptionContext(sub->context); auto pub_ec = getEventContext(ec); if (shouldFire(pub_sc, pub_ec) && sub->callback != nullptr) { @@ -347,7 +356,9 @@ class EventPublisher : public EventPublisherCore { * * @return should the Subscription%'s EventCallback be called for this event. */ - virtual bool shouldFire(const SCRef& sc, const ECRef& ec) { return true; } + virtual bool shouldFire(const SCRef& sc, const ECRef& ec) const { + return true; + } private: FRIEND_TEST(EventsTests, test_event_sub_subscribe); @@ -370,33 +381,6 @@ class EventFactory { /// Access to the EventFactory instance. static EventFactory& getInstance(); - /// A factory event publisher generator, simplify boilerplate code. - template - static EventPublisherRef createEventPublisher() { - auto pub = std::make_shared(); - auto base_pub = reinterpret_cast(pub); - return base_pub; - } - - /// A factory event subscriber generator, simplify boilerplate code. - template - static EventSubscriberRef createEventSubscriber() { - auto sub = std::make_shared(); - auto base_sub = reinterpret_cast(sub); - return base_sub; - } - - /** - * @brief Add an EventPublisher to the factory. - * - * The registration is mostly abstracted using osquery's registery. - */ - template - static Status registerEventPublisher() { - auto pub = std::make_shared(); - return registerEventPublisher(pub); - } - /** * @brief Add an EventPublisher to the factory. * @@ -409,8 +393,8 @@ class EventFactory { * EventFactory `getEventPublisher` accessor is encouraged. */ template - static Status registerEventPublisher(std::shared_ptr pub) { - auto base_pub = reinterpret_cast(pub); + static Status registerEventPublisher(const std::shared_ptr& pub) { + auto base_pub = reinterpret_cast(pub); return registerEventPublisher(base_pub); } @@ -504,15 +488,15 @@ class EventFactory { /// Deregister an EventPublisher by EventPublisherID. static Status deregisterEventPublisher(EventPublisherID& type_id); - /// Deregister all EventPublisher%s. - static Status deregisterEventPublishers(); - /// Return an instance to a registered EventPublisher. static EventPublisherRef getEventPublisher(EventPublisherID& pub); /// Return an instance to a registered EventSubscriber. static EventSubscriberRef getEventSubscriber(EventSubscriberID& pub); + static std::vector publisherTypes(); + static std::vector subscriberNames(); + public: /// The dispatched event thread's entrypoint (if needed). static Status run(EventPublisherID& type_id); @@ -520,7 +504,6 @@ class EventFactory { /// An initializer's entrypoint for spawning all event type run loops. static void delay(); - public: /// If a static EventPublisher callback wants to fire template static void fire(const EventContextRef& ec) { @@ -535,13 +518,14 @@ class EventFactory { * * @param should_end Reset the "is ending" state if False. */ - static void end(bool should_end = true); + static void end(bool join = false); private: /// An EventFactory will exist for the lifetime of the application. EventFactory() {} EventFactory(EventFactory const&); void operator=(EventFactory const&); + ~EventFactory() {} private: /// Set of registered EventPublisher instances. @@ -554,7 +538,7 @@ class EventFactory { std::vector > threads_; }; -class EventSubscriberCore { +class EventSubscriberPlugin : public Plugin { protected: /** * @brief Store parsed event data from an EventCallback in a backing store. @@ -663,11 +647,11 @@ class EventSubscriberCore { * EventPublisher instances will have run `setUp` and initialized their run * loops. */ - EventSubscriberCore() { + EventSubscriberPlugin() { expire_events_ = true; expire_time_ = 0; } - ~EventSubscriberCore() {} + virtual ~EventSubscriberPlugin() {} /** * @brief Suggested entrypoint for table generation. @@ -684,19 +668,23 @@ class EventSubscriberCore { } /// The string name identifying this EventSubscriber. - virtual EventSubscriberID name() { return "subscriber"; } + virtual EventSubscriberID name() const { return "subscriber"; } protected: /// Backing storage indexing namespace definition methods. - EventPublisherID dbNamespace() { return type() + "." + name(); } + EventPublisherID dbNamespace() const { return type() + "." + name(); } /// The string EventPublisher identifying this EventSubscriber. - virtual EventPublisherID type() = 0; + virtual EventPublisherID type() const = 0; /// Disable event expiration for this subscriber. void doNotExpire() { expire_events_ = false; } private: + EventSubscriberPlugin(EventSubscriberPlugin const&); + void operator=(EventSubscriberPlugin const&); + + private: /// Do not respond to periodic/scheduled/triggered event expiration requests. bool expire_events_; @@ -730,13 +718,12 @@ class EventSubscriberCore { * Small overheads exist that help query-time indexing and lookups. */ template -class EventSubscriber: public EventSubscriberCore { +class EventSubscriber : public EventSubscriberPlugin { protected: typedef typename PUB::SCRef SCRef; typedef typename PUB::ECRef ECRef; public: - /// Called after EventPublisher `setUp`. Add all Subscription%s here. /** * @brief Add Subscription%s to the EventPublisher this module will act on. * @@ -746,7 +733,9 @@ class EventSubscriber: public EventSubscriberCore { virtual void init() {} /// Helper function to call the publisher's templated subscription generator. - SCRef createSubscriptionContext() { return PUB::createSubscriptionContext(); } + SCRef createSubscriptionContext() const { + return PUB::createSubscriptionContext(); + } /** * @brief Bind a registered EventSubscriber member function to a Subscription. @@ -762,7 +751,7 @@ class EventSubscriber: public EventSubscriberCore { // Down-cast the pointer to the member function. auto base_entry = reinterpret_cast(entry); - // Create a callable to the member function using the instance of the + // Create a callable theo the member function using the instance of the // EventSubscriber and a single parameter placeholder (the EventContext). auto cb = std::bind(base_entry, self, _1); // Add a subscription using the callable and SubscriptionContext. @@ -770,45 +759,18 @@ class EventSubscriber: public EventSubscriberCore { } /// Helper EventPublisher string type accessor. - EventPublisherID type() { return BaseEventPublisher::getType(); } + EventPublisherID type() const { return BaseEventPublisher::getType(); } private: FRIEND_TEST(EventsTests, test_event_sub); FRIEND_TEST(EventsTests, test_event_sub_subscribe); FRIEND_TEST(EventsTests, test_event_sub_context); }; -} -/// Expose a Plugin-like Registry for EventPublisher instances. -DECLARE_REGISTRY(EventPublishers, std::string, EventPublisherRef) -#define REGISTERED_EVENTPUBLISHERS REGISTRY(EventPublishers) -#define REGISTER_EVENTPUBLISHER(PUB) \ - REGISTER(EventPublishers, #PUB, EventFactory::createEventPublisher()); +/// Iterate the event publisher registry and create run loops for each using +/// the event factory. +void attachEvents(); -/** - * @brief Expose a Plugin-link Registry for EventSubscriber instances. - * - * In most cases the EventSubscriber class will organize itself to include - * an generator entry point for query-time table generation too. - */ -DECLARE_REGISTRY(EventSubscribers, std::string, EventSubscriberRef) -#define REGISTERED_EVENTSUBSCRIBERS REGISTRY(EventSubscribers) -#define REGISTER_EVENTSUBSCRIBER(SUB) \ - REGISTER(EventSubscribers, #SUB, EventFactory::createEventSubscriber()); - -namespace osquery { -namespace registries { -/** - * @brief A utility method for moving EventPublisher%s and EventSubscriber%s - * (plugins) into the EventFactory. - * - * To handle run-time and compile-time EventPublisher and EventSubscriber - * additions as plugins or extensions, the osquery Registry workflow is used. - * During application launch (or within plugin load) the EventFactory faucet - * moves managed instances of these types to the EventFactory. The - * EventPublisher and EventSubscriber lifecycle/developer workflow is unknown - * to the Registry. - */ -void faucet(EventPublishers ets, EventSubscribers ems); -} +CREATE_LAZY_REGISTRY(EventPublisherPlugin, "event_publisher"); +CREATE_REGISTRY(EventSubscriberPlugin, "event_subscriber"); } diff --git a/include/osquery/extensions.h b/include/osquery/extensions.h new file mode 100644 index 0000000..37d9a27 --- /dev/null +++ b/include/osquery/extensions.h @@ -0,0 +1,243 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ + +#pragma once + +#include +#include +#include +#include + +#ifdef OSQUERY_THRIFT +#include "Extension.h" +#include "ExtensionManager.h" +#else +#error "Required -DOSQUERY_THRIFT=/path/to/thrift/gen-cpp" +#endif + +namespace osquery { + +DECLARE_string(extensions_socket); + +namespace extensions { + +/** + * @brief Helper struct for managing extenion metadata. + * + * This structure should match the members of Thrift's InternalExtensionInfo. + */ +struct ExtensionInfo { + std::string name; + std::string version; + std::string sdk_version; +}; + +/** + * @brief The Thrift API server used by an osquery Extension process. + * + * An extension will load and start a thread to serve the ExtensionHandler + * Thrift runloop. This handler is the implementation of the thrift IDL spec. + * It implements all the Extension API handlers. + * + */ +class ExtensionHandler : virtual public ExtensionIf { + public: + ExtensionHandler() {} + + /// Ping an Extension for status and metrics. + void ping(ExtensionStatus& _return); + + /** + * @brief The Thrift API used by Registry::call for an extension route. + * + * @param _return The return response (combo Status and PluginResponse). + * @param registry The name of the Extension registry. + * @param item The Extension plugin name. + * @param request The plugin request. + */ + void call(ExtensionResponse& _return, + const std::string& registry, + const std::string& item, + const ExtensionPluginRequest& request); +}; + +/** + * @brief The Thrift API server used by an osquery process. + * + * An extension will load and start a thread to serve the + * ExtensionManagerHandler. This listens for extensions and allows them to + * register their Registry route information. Calls to the registry may then + * match a route exposed by an extension. + * This handler is the implementation of the thrift IDL spec. + * It implements all the ExtensionManager API handlers. + * + */ +class ExtensionManagerHandler : virtual public ExtensionManagerIf, + public ExtensionHandler { + public: + ExtensionManagerHandler() {} + + /// Return a list of Route UUIDs and extension metadata. + void extensions(ExtensionList& _return) { _return = extensions_; } + + /** + * @brief Request a Route UUID and advertise a set of Registry routes. + * + * When an Extension starts it must call registerExtension using a well known + * ExtensionManager UNIX domain socket path. The ExtensionManager will check + * the broadcasted routes for duplicates as well as enforce SDK version + * compatibility checks. On success the Extension is returned a Route UUID and + * begins to serve the ExtensionHandler Thrift API. + * + * @param _return The output Status and optional assigned RouteUUID. + * @param info The osquery Thrift-internal Extension metadata container. + * @param registry The Extension's Registry::getBroadcast information. + */ + void registerExtension(ExtensionStatus& _return, + const InternalExtensionInfo& info, + const ExtensionRegistry& registry); + + /** + * @brief Request an Extension removal and removal of Registry routes. + * + * When an Extension process is gracefull killed it should deregister. + * Other priviledged tools may choose to deregister an Extension by + * the transient Extension's Route UUID, obtained using + * ExtensionManagerHandler::extensions. + * + * @param _return The output Status. + * @param uuid The assigned Route UUID to deregister. + */ + void deregisterExtension(ExtensionStatus& _return, + const ExtensionRouteUUID uuid); + + private: + /// Check if an extension exists by the name it registered. + bool exists(const std::string& name); + + /// Maintain a map of extension UUID to metadata for tracking deregistrations. + ExtensionList extensions_; +}; +} + +/// A Dispatcher service thread that watches an ExtensionManagerHandler. +class ExtensionWatcher : public InternalRunnable { + public: + virtual ~ExtensionWatcher() {} + ExtensionWatcher(const std::string& manager_path, + size_t interval, + bool fatal) { + manager_path_ = manager_path; + interval_ = interval; + fatal_ = fatal; + } + + public: + /// The Dispatcher thread entry point. + void enter(); + + private: + /// Exit the extension process with a fatal if the ExtensionManager dies. + void exitFatal(); + + private: + /// The UNIX domain socket path for the ExtensionManager. + std::string manager_path_; + /// The internal in milliseconds to ping the ExtensionManager. + size_t interval_; + /// If the ExtensionManager socket is closed, should the extension exit. + bool fatal_; +}; + +/// A Dispatcher service thread that starts ExtensionHandler. +class ExtensionRunner : public InternalRunnable { + public: + virtual ~ExtensionRunner(); + ExtensionRunner(const std::string& manager_path, RouteUUID uuid) { + path_ = manager_path + "." + std::to_string(uuid); + uuid_ = uuid; + } + + public: + /// The Dispatcher thread entry point. + void enter(); + + /// Access the UUID provided by the ExtensionManager. + RouteUUID getUUID() { return uuid_; } + + private: + /// The UNIX domain socket used for requests from the ExtensionManager. + std::string path_; + /// The unique and transient Extension UUID assigned by the ExtensionManager. + RouteUUID uuid_; +}; + +/// A Dispatcher service thread that starts ExtensionManagerHandler. +class ExtensionManagerRunner : public InternalRunnable { + public: + virtual ~ExtensionManagerRunner(); + ExtensionManagerRunner(const std::string& manager_path) { + path_ = manager_path; + } + + public: + void enter(); + + private: + std::string path_; +}; + +/** + * @brief Call a Plugin exposed by an Extension Registry route. + * + * This is mostly a Registry%-internal method used to call an ExtensionHandler + * call API if a Plugin is requested and had matched an Extension route. + * + * @param uuid Route UUID of the matched Extension + * @param registry The string name for the registry. + * @param item A string identifier for this registry item. + * @param request The plugin request input. + * @param response The plugin response output. + * @return Success indicates Extension API call success and Extension's + * Registry::call success. + */ +Status callExtension(const RouteUUID uuid, + const std::string& registry, + const std::string& item, + const PluginRequest& request, + PluginResponse& response); + +/// Internal callExtension implementation using a UNIX domain socket path. +Status callExtension(const std::string& extension_path, + const std::string& registry, + const std::string& item, + const PluginRequest& request, + PluginResponse& response); + +/// The main runloop entered by an Extension, start an ExtensionRunner thread. +Status startExtension(); + +/// Internal startExtension implementation using a UNIX domain socket path. +Status startExtension(const std::string& manager_path, + const std::string& name, + const std::string& version, + const std::string& sdk_version); + +/// Start an ExtensionWatcher thread. +Status startExtensionWatcher(const std::string& manager_path, + size_t interval, + bool fatal); + +/// Start an ExtensionManagerRunner thread. +Status startExtensionManager(); + +/// Internal startExtensionManager implementation. +Status startExtensionManager(const std::string& manager_path); +} diff --git a/include/osquery/filesystem.h b/include/osquery/filesystem.h index ceaccb4..2006744 100644 --- a/include/osquery/filesystem.h +++ b/include/osquery/filesystem.h @@ -3,7 +3,7 @@ * All rights reserved. * * This source code is licensed under the BSD-style license found in the - * LICENSE file in the root directory of this source tree. An additional grant + * LICENSE file in the root directory of this source tree. An additional grant * of patent rights can be found in the PATENTS file in the same directory. * */ @@ -22,6 +22,16 @@ namespace osquery { /** + * Our wildcard directory traversal function will not resolve more than + * this many wildcards. + */ +const unsigned int kMaxDirectoryTraversalDepth = 40; + +const std::string kWildcardCharacter = "%"; +const std::string kWildcardCharacterRecursive = + kWildcardCharacter + kWildcardCharacter; + +/** * @brief Read a file from disk. * * @param path the path of the file that you would like to read @@ -80,6 +90,42 @@ Status listFilesInDirectory(const boost::filesystem::path& path, std::vector& results); /** + * @brief List all of the directories in a specific directory, non-recursively. + * + * @param path the path which you would like to list. + * @param results a non-const reference to a vector which will be populated + * with the directory listing of the path param, assuming that all operations + * completed successfully. + * + * @return an instance of Status, indicating the success or failure + * of the operation. + */ +Status listDirectoriesInDirectory(const boost::filesystem::path& path, + std::vector& results); + +/** + * @brief Given a wildcard filesystem patten, resolve all possible paths + * + * @code{.cpp} + * std::vector results; + * auto s = resolveFilePattern("/Users/marpaia/Downloads/%", results); + * if (s.ok()) { + * for (const auto& result : results) { + * LOG(INFO) << result; + * } + * } + * @endcode + * + * @param fs_path The filesystem pattern + * @param results The vector in which all results will be returned + * + * @return An instance of osquery::Status which indicates the success or + * failure of the operation + */ +Status resolveFilePattern(const boost::filesystem::path& fs_path, + std::vector& results); + +/** * @brief Get directory portion of a path. * * @param path The input path, either a filename or directory. @@ -91,6 +137,8 @@ Status listFilesInDirectory(const boost::filesystem::path& path, Status getDirectory(const boost::filesystem::path& path, boost::filesystem::path& dirpath); +Status remove(const boost::filesystem::path& path); + /** * @brief Check if an input path is a directory. * @@ -101,34 +149,11 @@ Status getDirectory(const boost::filesystem::path& path, Status isDirectory(const boost::filesystem::path& path); /** - * @brief Parse the users out of a tomcat user config from disk - * - * @param path A string which represents the path of the tomcat user config - * @param a vector of pairs which represent all of the users which were found - * in the supplied file. pair.first is the username and pair.second is the - * password. + * @brief Return a vector of all home directories on the system * - * @return an instance of Status, indicating the success or failure - * of the operation - */ -Status parseTomcatUserConfigFromDisk( - const boost::filesystem::path& path, - std::vector >& credentials); - -/** - * @brief Parse the users out of a tomcat user config - * - * @param content A string which represents the content of the file to parse - * @param a vector of pairs which represent all of the users which were found - * in the supplied file. pair.first is the username and pair.second is the - * password. - * - * @return an instance of Status, indicating the success or failure - * of the operation + * @return a vector of strings representing the path of all home directories */ -Status parseTomcatUserConfig( - const std::string& content, - std::vector >& credentials); +std::vector getHomeDirectories(); #ifdef __APPLE__ /** diff --git a/include/osquery/flags.h b/include/osquery/flags.h index 7ec812a..b5a1ff7 100644 --- a/include/osquery/flags.h +++ b/include/osquery/flags.h @@ -10,10 +10,11 @@ #pragma once +#include + #define STRIP_FLAG_HELP 1 #include -#include #include #define __GFLAGS_NAMESPACE google diff --git a/include/osquery/logger.h b/include/osquery/logger.h index 041191c..4ab3690 100644 --- a/include/osquery/logger.h +++ b/include/osquery/logger.h @@ -10,12 +10,12 @@ #pragma once -#include #include #include #include +#include #include #include @@ -79,4 +79,60 @@ Status logScheduledQueryLogItem(const ScheduledQueryLogItem& item); */ Status logScheduledQueryLogItem(const ScheduledQueryLogItem& item, const std::string& receiver); + +/** + * @brief Superclass for the pluggable config component. + * + * In order to make the logging of osquery results easy to integrate into your + * environment, we take advantage of a plugin interface which allows you to + * integrate osquery with your internal large-scale logging infrastructure. + * You may use flume, splunk, syslog, scribe, etc. In order to use your + * specific upstream logging systems, one simply needs to create a custom + * subclass of LoggerPlugin. That subclass should implement the + * LoggerPlugin::logString method. + * + * Consider the following example: + * + * @code{.cpp} + * class TestLoggerPlugin : public LoggerPlugin { + * public: + * virtual osquery::Status logString(const std::string& s) { + * int i = 0; + * internal::logStringToFlume(s, i); + * std::string message; + * if (i == 0) { + * message = "OK"; + * } else { + * message = "Failed"; + * } + * return osquery::Status(i, message); + * } + * }; + * + * REGISTER(TestLoggerPlugin, "logger", "test"); + * @endcode + */ + +class LoggerPlugin : public Plugin { + public: + /** @brief Virtual method which should implement custom logging. + * + * LoggerPlugin::logString should be implemented by a subclass of + * LoggerPlugin which needs to log a string in a custom way. + * + * @return an instance of osquery::Status which indicates the success or + * failure of the operation. + */ + virtual Status logString(const std::string& s) = 0; + Status call(const PluginRequest& request, PluginResponse& response); +}; + +/** + * @brief Logger plugin registry. + * + * This creates an osquery registry for "logger" which may implement + * LoggerPlugin. Only strings are logged in practice, and LoggerPlugin provides + * a helper member for transforming PluginRequest%s to strings. + */ +CREATE_REGISTRY(LoggerPlugin, "logger"); } diff --git a/include/osquery/logger/plugin.h b/include/osquery/logger/plugin.h deleted file mode 100644 index a25a4b3..0000000 --- a/include/osquery/logger/plugin.h +++ /dev/null @@ -1,77 +0,0 @@ -/* - * Copyright (c) 2014, Facebook, Inc. - * All rights reserved. - * - * This source code is licensed under the BSD-style license found in the - * LICENSE file in the root directory of this source tree. An additional grant - * of patent rights can be found in the PATENTS file in the same directory. - * - */ - -#pragma once - -#include - -#include -#include - -namespace osquery { - -/** - * @brief Superclass for the pluggable config component. - * - * In order to make the logging of osquery results easy to integrate into your - * environment, we take advantage of a plugin interface which allows you to - * integrate osquery with your internal large-scale logging infrastructure. - * You may use flume, splunk, syslog, scribe, etc. In order to use your - * specific upstream logging systems, one simply needs to create a custom - * subclass of LoggerPlugin. That subclass should implement the - * LoggerPlugin::logString method. - * - * Consider the following example: - * - * @code{.cpp} - * class TestLoggerPlugin : public ConfigPlugin { - * public: - * virtual osquery::Status logString(const std::string& s) { - * int i = 0; - * internal::logStringToFlume(s, i); - * std::string message; - * if (i == 0) { - * message = "OK"; - * } else { - * message = "Failed"; - * } - * return osquery::Status(i, message); - * } - * }; - * - * REGISTER_LOGGER_PLUGIN( - * "test", std::make_shared()); - * @endcode - */ -class LoggerPlugin { - public: - /** @brief Virtual method which should implement custom logging. - * - * LoggerPlugin::logString should be implemented by a subclass of - * LoggerPlugin which needs to log a string in a custom way. - * - * @return an instance of osquery::Status which indicates the success or - * failure of the operation. - */ - virtual osquery::Status logString(const std::string& s) = 0; - - /// Virtual destructor - virtual ~LoggerPlugin() {} -}; -} - -DECLARE_REGISTRY(LoggerPlugins, - std::string, - std::shared_ptr) - -#define REGISTERED_LOGGER_PLUGINS REGISTRY(LoggerPlugins) - -#define REGISTER_LOGGER_PLUGIN(name, decorator) \ - REGISTER(LoggerPlugins, name, decorator) diff --git a/include/osquery/registry.h b/include/osquery/registry.h index 4a2461a..75f4b28 100644 --- a/include/osquery/registry.h +++ b/include/osquery/registry.h @@ -10,88 +10,410 @@ #pragma once -#include -#include -#include -#include +#include +#include +#include -#include -#include +#include +#include -#include "osquery/registry/init_registry.h" -#include "osquery/registry/singleton.h" +#include namespace osquery { /** - * @brief A simple registry system for making values available by key across - * components. + * @brief A boilerplate code helper to create a registry given a name and + * plugin base class type. * - * To use this registry, make a header like so: + * Registries are types of plugins, e.g., config, logger, table. They are + * defined with a string name and Plugin derived class. There is an expectation + * that any 'item' registered will inherit from the registry plugin-derived + * type. But there is NO type enforcement on that intermediate class. * - * @code{.cpp} - * #include + * This boilerplate macro puts the registry into a 'registry' namespace for + * organization and createa a global const int that may be instanciated + * in a header or implementation code without symbol duplication. + * The initialization is also boilerplate, whereas the Registry::create method + * (a whole-process-lived single instance object) creates and manages the + * registry instance. * - * DECLARE_REGISTRY(MathFuncs, int, std::function) - * #define REGISTERED_MATH_FUNCS REGISTRY(MathFuncs) - * #define REGISTER_MATH_FUNC(id, func) \ - * REGISTER(MathFuncs, id, func) - * @endcode + * @param type A typename that derives from Plugin. + * @param name A string identifier for the registry. + */ +#define CREATE_REGISTRY(type, name) \ + namespace registry { \ + const auto type##Registry = Registry::create(name); \ + } + +/** + * @brief A boilerplate code helper to create a registry given a name and + * plugin base class type. This 'lazy' registry does not automatically run + * Plugin::setUp on all items. + * + * @param type A typename that derives from Plugin. + * @param name A string identifier for the registry. + */ +#define CREATE_LAZY_REGISTRY(type, name) \ + namespace registry { \ + const auto type##Registry = Registry::create(name, false); \ + } + +/** + * @brief A boilerplate code helper to register a plugin. + * + * Like CREATE_REGISTRY, REGISTER creates a boilerplate global instance to + * create an instance of the plugin type within the whole-process-lived registry + * single instance. Registry items must derive from the `RegistryType` defined + * by the CREATE_REGISTRY and Registry::create call. + * + * @param type A typename that derives from the RegistryType. + * @param registry The string name for the registry. + * @param name A string identifier for this registry item. + */ +#define REGISTER(type, registry, name) \ + const auto type##RegistryItem = Registry::add(registry, name); + +/// A plugin (registry item) may return a custom key value map with its Route. +typedef std::map RouteInfo; +/// Registry routes are a map of item name to each optional RouteInfo. +typedef std::map RegistryRoutes; +/// An extension or core's broadcast includes routes from every Registry. +typedef std::map RegistryBroadcast; + +typedef uint32_t RouteUUID; + +/** + * @brief The request part of a plugin (registry item's) call. * - * Client code may then advertise an entry from a .cpp like so: + * To use a plugin use Registry::call with a request and response. + * The request portion is usually simple and normally includes an "action" + * key where the value is the action you want to perform on the plugin. + * Refer to the registry's documentation for the actions supported by + * each of its plugins. + */ +typedef std::map PluginRequest; +/** + * @brief The reponse part of a plugin (registry item's) call. * - * @code{.cpp} - * #include "my/registry/header.h" - * REGISTER_MATH_FUNC(1, sqrt); - * @endcode + * If a Registry::call succeeds it will fill in a PluginResponse. + * This reponse is a vector of key value maps. + */ +typedef std::vector PluginResponse; + +class Plugin { + public: + Plugin() { name_ = "unnamed"; } + virtual ~Plugin() {} + + public: + /// The plugin may perform some initialization, not required. + virtual Status setUp() { return Status(0, "Not used"); } + /// The plugin may perform some tear down, release, not required. + virtual void tearDown() {} + /// The plugin may publish route info (other than registry type and name). + virtual RouteInfo routeInfo() { + RouteInfo info; + return info; + } + /// The plugin will act on a serialized request, and if a response is needed + /// (response is set to true) then response should be a reference to a + /// string ready for a serialized response. + virtual Status call(const PluginRequest& request, PluginResponse& response) { + return Status(0, "Not used"); + } + + // Set the output request key to a serialized property tree. + // Used by the plugin to set a serialized PluginResponse. + static void setResponse(const std::string& key, + const boost::property_tree::ptree& tree, + PluginResponse& response); + + // Get a PluginResponse key as a property tree. + static void getResponse(const std::string& key, + const PluginResponse& response, + boost::property_tree::ptree& tree); + + /// Allow the plugin to introspect into the registered name (for logging). + void setName(const std::string& name) { name_ = name; } + + protected: + std::string name_; + + private: + Plugin(Plugin const&); + void operator=(Plugin const&); +}; + +class RegistryHelperCore { + protected: + virtual void type() const {} + + public: + RegistryHelperCore(bool auto_setup = true) : auto_setup_(auto_setup) {} + + /** + * @brief Remove a registry item by its identifier. + * + * @param item_name An identifier for this registry plugin. + */ + virtual void remove(const std::string& item_name); + + virtual RegistryRoutes getRoutes() const; + + /** + * @brief The only method a plugin user should call. + * + * Registry plugins are used internally and externally. They may belong + * to the process making the call or to an external process via a thrift + * transport. + * + * All plugin input and output must be serializable. The plugin types + * RegistryType usually exposes protected serialization methods for the + * data structures used by plugins (registry items). + * + * @param item_name The plugin identifier to call. + * @param request The plugin request, usually containing an action request. + * @param response If successful, the requested information. + * @return Success if the plugin was called, and response was filled. + */ + virtual Status call(const std::string& item_name, + const PluginRequest& request, + PluginResponse& response); + + /** + * @brief Allow a plugin to perform some setup functions when osquery starts. + * + * Doing work in a plugin constructor has unknown behavior. Plugins may + * be constructed at anytime during osquery's life, including global variable + * instanciation. To have a reliable state (aka, flags have been parsed, + * and logs are ready to stream), do construction work in Plugin::setUp. + * + * The registry `setUp` will iterate over all of its registry items and call + * their setup unless the registry is lazy (see CREATE_REGISTRY). + */ + virtual void setUp(); + + /// Facility method to check if a registry item exists. + virtual bool exists(const std::string& item_name) const; + + virtual Status addAlias(const std::string& item_name, + const std::string& alias); + virtual const std::string& getAlias(const std::string& alias) const; + + /// Facility method to list the registry item identifiers. + virtual std::vector names() const; + + /// Facility method to count the number of items in this registry. + virtual size_t count() const; + + /// Allow the registry to introspect into the registered name (for logging). + void setName(const std::string& name); + + protected: + /// The identifier for this registry, used to register items. + std::string name_; + /// Does this registry run setUp on each registry item at initialization. + bool auto_setup_; + + protected: + /// A map of registered plugin instances to their registered identifier. + std::map > items_; + std::map aliases_; +}; + +/** + * @brief The core interface for each registry type. * - * Server code may then access the set of registered values by using - * `REGISTERED_MATH_FUNCS` as a map, which will be populated after - * `osquery::InitRegistry::get().run()` has been called. + * The osquery Registry is partitioned into types. These are literal types + * but use a canonical string key for lookups and actions. + * Registries are created using Registry::create with a RegistryType and key. */ -template -class Registry : public std::unordered_map { +template +class RegistryHelper : public RegistryHelperCore { + protected: + typedef std::shared_ptr RegistryTypeRef; + public: + RegistryHelper(bool auto_setup = true) : RegistryHelperCore(auto_setup) {} + virtual ~RegistryHelper() {} + + /** + * @brief Add a plugin to this registry by allocating and indexing + * a type Item and a key identifier. + * + * @code{.cpp} + * /// Instead of calling RegistryFactory::add use: + * REGISTER(Type, "registry_name", "item_name"); + * @endcode + * + * @param item_name An identifier for this registry plugin. + * @return A success/failure status. + */ + template + Status add(const std::string& item_name) { + if (items_.count(item_name) > 0) { + return Status(1, "Duplicate registry item exists: " + item_name); + } + + // Run the item's constructor, the setUp call will happen later. + auto item = (RegistryType*)new Item(); + item->setName(item_name); + // Cast the specific registry-type derived item as the API typ the registry + // used when it was created using the registry factory. + std::shared_ptr shared_item(item); + items_[item_name] = shared_item; + return Status(0, "OK"); + } + /** - * @brief Register a value in the global registry + * @brief A raw accessor for a registry plugin. * - * This is used internally by the `DECLARE_REGISTRY` registration workflow. - * If you're calling this method directly, you're probably doing something - * incorrectly. + * If there is no plugin with an item_name identifier this will throw + * and out_of_range exception. + * + * @param item_name An identifier for this registry plugin. + * @return A std::shared_ptr of type RegistryType. */ - void registerValue(const Key& key, - const Value& value, - const char* displayName = "registry") { - if (this->insert(std::make_pair(key, value)).second) { - VLOG(1) << displayName << "[" << key << "]" - << " registered"; - } else { - LOG(ERROR) << displayName << "[" << key << "]" - << " already registered"; + RegistryTypeRef get(const std::string& item_name) { + return std::dynamic_pointer_cast(items_.at(item_name)); + } + + const std::map all() { + std::map ditems; + for (const auto& item : items_) { + ditems[item.first] = std::dynamic_pointer_cast(item.second); } + + return ditems; } + + private: + RegistryHelper(RegistryHelper const&); + void operator=(RegistryHelper const&); }; -} -#define DECLARE_REGISTRY(registryName, KeyType, ObjectType) \ - namespace osquery { \ - namespace registries { \ - class registryName : public Registry {}; \ - } \ +typedef std::shared_ptr PluginRef; +typedef RegistryHelper PluginRegistryHelper; +typedef std::shared_ptr PluginRegistryHelperRef; + +class RegistryFactory : private boost::noncopyable { + public: + static RegistryFactory& instance() { + static RegistryFactory instance; + return instance; + } + + /** + * @brief Create a registry using a plugin type and identifier. + * + * A short hard for allocating a new registry type a RegistryHelper and + * plugin derived class Type or RegistryType. This shorthard performs + * the allocation and initialization of the Type and keeps the instance + * identified by registry_name. + * + * @code{.cpp} + * /// Instead of calling RegistryFactory::create use: + * CREATE_REGISTRY(Type, "registry_name"); + * @endcode + * + * @param registry_name The canonical name for this registry. + * @param auto_setup Optionally set false if the registry handles setup + * @return A non-sense int that must be casted const. + */ + template + static int create(const std::string& registry_name, bool auto_setup = true) { + if (instance().registries_.count(registry_name) > 0) { + return 0; + } + + auto registry = (PluginRegistryHelper*)new RegistryHelper(auto_setup); + registry->setName(registry_name); + PluginRegistryHelperRef shared_registry(registry); + instance().registries_[registry_name] = shared_registry; + return 0; } -#define REGISTRY(registryName) \ - (osquery::Singleton::get()) + static PluginRegistryHelperRef registry(const std::string& registry_name); + + template + static Status add(const std::string& registry_name, + const std::string& item_name) { + auto registry = instance().registry(registry_name); + return registry->template add(item_name); + } + + static const std::map& all(); + + static const std::map all( + const std::string& registry_name); + + static PluginRef get(const std::string& registry_name, + const std::string& item_name); + + static RegistryBroadcast getBroadcast(); + + static Status addBroadcast(const RouteUUID& uuid, + const RegistryBroadcast& broadcast); + + static Status removeBroadcast(const RouteUUID& uuid); + + /// Adds an alias for an internal registry item. This registry will only + /// broadcast the alias name. + static Status addAlias(const std::string& registry_name, + const std::string& item_name, + const std::string& alias); + + /// Returns the item_name or the item alias if an alias exists. + static const std::string& getAlias(const std::string& registry_name, + const std::string& alias); + + static Status call(const std::string& registry_name, + const std::string& item_name, + const PluginRequest& request, + PluginResponse& response); -#ifndef UNIQUE_VAR -#define UNIQUE_VAR_CONCAT(_name_, _line_) _name_##_line_ -#define UNIQUE_VAR_LINENAME(_name_, _line_) UNIQUE_VAR_CONCAT(_name_, _line_) -#define UNIQUE_VAR(_name_) UNIQUE_VAR_LINENAME(_name_, __LINE__) -#endif + static Status call(const std::string& registry_name, + const std::string& item_name, + const PluginRequest& request); -#define REGISTER(registryName, key, value) \ - namespace {/* require global scope, don't pollute static namespace */ \ - static osquery::RegisterInitFunc UNIQUE_VAR(registryName)([] { \ - REGISTRY(registryName).registerValue((key), (value), #registryName); \ - }); \ + static void setUp(); + + static bool exists(const std::string& registry_name, + const std::string& item_name); + + static std::vector names(const std::string& registry_name); + + static size_t count(); + + static size_t count(const std::string& registry_name); + + static void allowDuplicates(bool allow) { + instance().allow_duplicates_ = allow; } + + static bool allowDuplicates() { return instance().allow_duplicates_; } + + protected: + RegistryFactory() : allow_duplicates_(false) {} + RegistryFactory(RegistryFactory const&); + void operator=(RegistryFactory const&); + virtual ~RegistryFactory() {} + + private: + bool allow_duplicates_; + std::map registries_; + std::map extensions_; +}; + +/** + * @brief The osquery Registry, refer to RegistryFactory for the caller API. + * + * The Registry class definition constructs the RegistryFactory behind the + * scenes using a class definition template API call Plugin. + * Each registry created by the RegistryFactory using RegistryFactory::create + * will provide a plugin type called RegistryType that inherits from Plugin. + * The actual plugins must add themselves to a registry type and should + * implement the Plugin and RegistryType interfaces. + */ +class Registry : public RegistryFactory {}; +} diff --git a/include/osquery/sql.h b/include/osquery/sql.h index 665b153..692937d 100644 --- a/include/osquery/sql.h +++ b/include/osquery/sql.h @@ -15,21 +15,9 @@ #include #include +#include namespace osquery { - -/** - * @brief A map of SQLite status codes to their corresponding message string - * - * Details of this map are defined at: http://www.sqlite.org/c3ref/c_abort.html - */ -extern const std::map kSQLiteReturnCodes; - -/** - * @brief Get a string representation of a SQLite return code - */ -std::string getStringForSQLiteReturnCode(int code); - /** * @brief The core interface to executing osquery SQL commands * @@ -88,10 +76,26 @@ class SQL { /** * @brief Get all, 'SELECT * ...', results given a virtual table name. * + * @param table The name of the virtual table. * @return A QueryData object of the 'SELECT *...' query results. */ static QueryData selectAllFrom(const std::string& table); + /** + * @brief Get all with constraint, 'SELECT * ... where', results given + * a virtual table name and single constraint + * + * @param table The name of the virtual table. + * @param column Table column name to apply constraint. + * @param op The SQL comparitive operator. + * @param expr The constraint expression. + * @return A QueryData object of the 'SELECT *...' query results. + */ + static QueryData selectAllFrom(const std::string& table, + const std::string& column, + tables::ConstraintOperator op, + const std::string& expr); + private: /** * @brief Private default constructor @@ -107,4 +111,45 @@ class SQL { /// the internal member which holds the status of the query Status status_; }; + +/** + * @brief Execute a query + * + * This is a lower-level version of osquery::SQL. Prefer to use osquery::SQL. + * + * @code{.cpp} + * std::string q = "SELECT * FROM time;"; + * QueryData results; + * auto status = query(q, results); + * if (status.ok()) { + * for (const auto& each : results) { + * for (const auto& it : each) { + * LOG(INFO) << it.first << ": " << it.second; + * } + * } + * } else { + * LOG(ERROR) << "Error: " << status.what(); + * } + * @endcode + * + * @param q the query to execute + * @param results A QueryData structure to emit result rows on success. + * @return A status indicating query success. + */ +Status query(const std::string& query, QueryData& results); + +/** + * @brief Analyze a query, providing information about the result columns + * + * This function asks SQLite to determine what the names and types are of the + * result columns of the provided query. Only table columns (not expressions or + * subqueries) can have their types determined. Types that are not determined + * are indicated with the string "UNKNOWN". + * + * @param q the query to analyze + * @param columns the vector to fill with column information + * + * @return status indicating success or failure of the operation + */ +Status getQueryColumns(const std::string& q, tables::TableColumns& columns); } diff --git a/include/osquery/status.h b/include/osquery/status.h index 53b5ade..185a14a 100644 --- a/include/osquery/status.h +++ b/include/osquery/status.h @@ -3,13 +3,14 @@ * All rights reserved. * * This source code is licensed under the BSD-style license found in the - * LICENSE file in the root directory of this source tree. An additional grant + * LICENSE file in the root directory of this source tree. An additional grant * of patent rights can be found in the PATENTS file in the same directory. * */ #pragma once +#include #include namespace osquery { @@ -93,6 +94,32 @@ class Status { std::string toString() const { return getMessage(); } std::string what() const { return getMessage(); } + /** + * @brief implicit conversion to bool + * + * Allows easy use of Status in an if statement, as below: + * + * @code{.cpp} + * if (doSomethingThatReturnsStatus()) { + * LOG(INFO) << "Success!"; + * } + * @endcode + */ + operator bool() const { return ok(); } + + // Below operator implementations useful for testing with gtest + + // Enables use of gtest (ASSERT|EXPECT)_EQ + bool operator==(const Status& rhs) const { + return (code_ == rhs.getCode()) && (message_ == rhs.getMessage()); + } + + // Enables use of gtest (ASSERT|EXPECT)_NE + bool operator!=(const Status& rhs) const { return !operator==(rhs); } + + // Enables pretty-printing in gtest (ASSERT|EXPECT)_(EQ|NE) + friend ::std::ostream& operator<<(::std::ostream& os, const Status& s); + private: /// the internal storage of the status code int code_; diff --git a/include/osquery/tables.h b/include/osquery/tables.h index 27313a6..4b96c44 100644 --- a/include/osquery/tables.h +++ b/include/osquery/tables.h @@ -3,7 +3,7 @@ * All rights reserved. * * This source code is licensed under the BSD-style license found in the - * LICENSE file in the root directory of this source tree. An additional grant + * LICENSE file in the root directory of this source tree. An additional grant * of patent rights can be found in the PATENTS file in the same directory. * */ @@ -13,19 +13,16 @@ #include #include #include +#include #include +#include -#include - +#include +#include #include #include -#ifndef FRIEND_TEST -#define FRIEND_TEST(test_case_name, test_name) \ - friend class test_case_name##_##test_name##_Test -#endif - namespace osquery { namespace tables { @@ -66,8 +63,8 @@ namespace tables { #define AS_LITERAL(literal, value) boost::lexical_cast(value) /// Helper alias for TablePlugin names. -typedef const std::string TableName; -typedef const std::vector > TableColumns; +typedef std::string TableName; +typedef std::vector > TableColumns; typedef std::map > TableData; /** @@ -147,7 +144,7 @@ struct ConstraintList { /** * @brief Check and return if there are any constraints on this column. * - * A ConstraintList is used in a ConstraintMap with a column name as the + * A ConstraintList is used in a ConstraintMap with a column name as the * map index. Tables that act on optional constraints should check if any * constraint was provided. * @@ -171,7 +168,7 @@ struct ConstraintList { /** * @brief Check if a constraint is missing or matches a type expression. * - * A ConstraintList is used in a ConstraintMap with a column name as the + * A ConstraintList is used in a ConstraintMap with a column name as the * map index. Tables that act on required constraints can make decisions * on missing constraints or a constraint match. * @@ -198,7 +195,17 @@ struct ConstraintList { * @param op the ConstraintOperator. * @return A list of TEXT%-represented types matching the operator. */ - std::vector getAll(ConstraintOperator op); + std::set getAll(ConstraintOperator op); + + template + std::set getAll(ConstraintOperator op) { + std::set literal_matches; + auto matches = getAll(op); + for (const auto& match : matches) { + literal_matches.insert(AS_LITERAL(T, match)); + } + return literal_matches; + } /** * @brief Add a new Constraint to the list of constraints. @@ -209,6 +216,20 @@ struct ConstraintList { constraints_.push_back(constraint); } + /** + * @brief Serialize a ConstraintList into a property tree. + * + * The property tree will use the format: + * { + * "affinity": affinity, + * "list": [ + * {"op": op, "expr": expr}, ... + * ] + * } + */ + void serialize(boost::property_tree::ptree& tree) const; + void unserialize(const boost::property_tree::ptree& tree); + ConstraintList() { affinity = "TEXT"; } private: @@ -232,6 +253,8 @@ struct QueryContext { ConstraintMap constraints; /// Support a limit to the number of results. int limit; + + QueryContext() : limit(0) {} }; typedef struct QueryContext QueryContext; @@ -243,30 +266,43 @@ typedef struct Constraint Constraint; * To attach a virtual table create a TablePlugin subclass and register the * virtual table name as the plugin ID. osquery will enumerate all registered * TablePlugins and attempt to attach them to SQLite at instanciation. + * + * Note: When updating this class, be sure to update the corresponding template + * in osquery/tables/templates/default.cpp.in */ -class TablePlugin { - public: - TableName name; - TableColumns columns; +class TablePlugin : public Plugin { + protected: /// Helper method to generate the virtual table CREATE statement. - std::string statement(TableName name, TableColumns columns); + virtual std::string statement(); + virtual std::string columnDefinition(); + virtual TableColumns columns() { + TableColumns columns; + return columns; + } + + virtual QueryData generate(QueryContext& request) { + QueryData data; + return data; + } public: - /// Part of the query state, number of rows generated. - int n; - /// Part of the query state, column data returned from a query. - TableData data; - /// Part of the query state, parsed set of query predicate constraints. - ConstraintSet constraints; + /// Public API methods. + Status call(const PluginRequest& request, PluginResponse& response); public: - virtual int attachVtable(sqlite3 *db) { return -1; } - virtual ~TablePlugin(){}; + /// Helper data structure transformation methods + static void setRequestFromContext(const QueryContext& context, + PluginRequest& request); + static void setResponseFromQueryData(const QueryData& data, + PluginResponse& response); + static void setContextFromRequest(const PluginRequest& request, + QueryContext& context); - protected: - TablePlugin() { n = 0; }; + private: + FRIEND_TEST(VirtualTableTests, test_tableplugin_columndefinition); + FRIEND_TEST(VirtualTableTests, test_tableplugin_statement); }; -typedef std::shared_ptr TablePluginRef; +CREATE_REGISTRY(TablePlugin, "table"); } } diff --git a/osquery.thrift b/osquery.thrift new file mode 100644 index 0000000..3e3fa31 --- /dev/null +++ b/osquery.thrift @@ -0,0 +1,59 @@ +namespace cpp osquery.extensions + +/// Registry operations use a registry name, plugin name, request/response. +typedef map ExtensionPluginRequest +typedef list> ExtensionPluginResponse + +struct InternalExtensionInfo { + 1:string name, + 2:string version, + 3:string sdk_version, +} + +typedef i64 ExtensionRouteUUID +typedef map ExtensionRoute +typedef map ExtensionRouteTable +typedef map ExtensionRegistry +typedef map ExtensionList + +enum ExtensionCode { + EXT_SUCCESS = 0, + EXT_FAILED = 1, + EXT_FATAL = 2, +} + +/// Most communication uses the Status return type. +struct ExtensionStatus { + 1:i32 code, + 2:string message, + 3:ExtensionRouteUUID uuid, +} + +struct ExtensionResponse { + 1:ExtensionStatus status, + 2:ExtensionPluginResponse response, +} + +exception ExtensionException { + 1:i32 code, + 2:string message, + 3:ExtensionRouteUUID uuid, +} + +service Extension { + ExtensionStatus ping(), + ExtensionResponse call( + 1:string registry, + 2:string item, + 3:ExtensionPluginRequest request), +} + +service ExtensionManager extends Extension { + ExtensionList extensions(), + ExtensionStatus registerExtension( + 1:InternalExtensionInfo info, + 2:ExtensionRegistry registry), + ExtensionStatus deregisterExtension( + 1:ExtensionRouteUUID uuid, + ), +} diff --git a/osquery/CMakeLists.txt b/osquery/CMakeLists.txt index 1a54d8f..4ad8a2d 100644 --- a/osquery/CMakeLists.txt +++ b/osquery/CMakeLists.txt @@ -13,6 +13,7 @@ # limitations under the License SET(TARGET_OSQUERY_LIB osquery) +SET(TARGET_OSQUERY_LIB_ADDITIONAL osquery_additional) SET(TARGET_OSQUERY_SHELL osqueryi) SET(TARGET_OSQUERY_DAEMON osqueryd) @@ -40,6 +41,7 @@ SET(${TARGET_OSQUERY_LIB}_DEP glog systemd udev) SET(${TARGET_OSQUERY_LIB}_SRCS "") +SET(${TARGET_OSQUERY_LIB_ADDITIONAL}_SRCS "") SET(OSQUERY_CODEGEN_PATH "${CMAKE_SOURCE_DIR}/tools/codegen") SET(OSQUERY_TABLES_PATH "${CMAKE_SOURCE_DIR}/osquery/tables") @@ -47,14 +49,21 @@ SET(OSQUERY_GENERATED_PATH "${CMAKE_BINARY_DIR}/generated") ADD_DEFINITIONS("-DOSQUERY_BUILD_VERSION=${OSQUERY_BUILD_VERSION}") -MACRO(ADD_OSQUERY_LIBRARY TARGET) +MACRO(ADD_OSQUERY_LIBRARY IS_CORE TARGET) ADD_LIBRARY(${TARGET} OBJECT ${ARGN}) - LIST(APPEND ${TARGET_OSQUERY_LIB}_SRCS $) - SET(${TARGET_OSQUERY_LIB}_SRCS ${${TARGET_OSQUERY_LIB}_SRCS} PARENT_SCOPE) + + IF(${IS_CORE}) + LIST(APPEND ${TARGET_OSQUERY_LIB}_SRCS $) + SET(${TARGET_OSQUERY_LIB}_SRCS ${${TARGET_OSQUERY_LIB}_SRCS} PARENT_SCOPE) + ELSE() + LIST(APPEND ${TARGET_OSQUERY_LIB_ADDITIONAL}_SRCS $) + SET(${TARGET_OSQUERY_LIB_ADDITIONAL}_SRCS ${${TARGET_OSQUERY_LIB_ADDITIONAL}_SRCS} PARENT_SCOPE) + ENDIF() ENDMACRO(ADD_OSQUERY_LIBRARY TARGET) -MACRO(ADD_OSQUERY_TEST TEST_NAME SOURCE) +MACRO(ADD_OSQUERY_TEST IS_CORE TEST_NAME SOURCE) ADD_EXECUTABLE(${TEST_NAME} ${SOURCE}) + TARGET_LINK_LIBRARIES(${TEST_NAME} ${TARGET_OSQUERY_LIB}) TARGET_LINK_LIBRARIES(${TEST_NAME} gtest) ADD_TEST(${TEST_NAME} ${TEST_NAME}) @@ -69,15 +78,24 @@ MACRO(ADD_OSQUERY_TEST TEST_NAME SOURCE) WORLD_EXECUTE) ENDMACRO(ADD_OSQUERY_TEST) +MACRO(TARGET_OSQUERY_LINK_WHOLE TARGET LIBRARY) + TARGET_LINK_LIBRARIES(${TARGET} "-Wl,-whole-archive") + TARGET_LINK_LIBRARIES(${TARGET} ${LIBRARY}) + TARGET_LINK_LIBRARIES(${TARGET} "-Wl,-no-whole-archive") +ENDMACRO(TARGET_OSQUERY_LINK_WHOLE) + ADD_SUBDIRECTORY(core) ADD_SUBDIRECTORY(config) ADD_SUBDIRECTORY(dispatcher) ADD_SUBDIRECTORY(devtools) ADD_SUBDIRECTORY(database) ADD_SUBDIRECTORY(events) +ADD_SUBDIRECTORY(extensions) ADD_SUBDIRECTORY(filesystem) ADD_SUBDIRECTORY(logger) +ADD_SUBDIRECTORY(registry) ADD_SUBDIRECTORY(scheduler) +ADD_SUBDIRECTORY(sql) ADD_SUBDIRECTORY(tables) ## Table generation ############################################################# @@ -124,17 +142,17 @@ ADD_CUSTOM_COMMAND( WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}") ## Library generation ########################################################### +# TODO(sangwan.kwon): Change amalgation files to additional ADD_LIBRARY(osquery_generated_tables OBJECT "${AMALGAMATION_FILE_GEN}") ADD_LIBRARY(osquery_static STATIC $ $ - ${${TARGET_OSQUERY_LIB}_SRCS}) + ${${TARGET_OSQUERY_LIB}_SRCS} + ${${TARGET_OSQUERY_LIB_ADDITIONAL}_SRCS}) SET_TARGET_PROPERTIES(osquery_static PROPERTIES OUTPUT_NAME osquery_static) # static_lib should include every object file in the archive in the link ADD_LIBRARY(${TARGET_OSQUERY_LIB} STATIC main/lib.cpp) -TARGET_LINK_LIBRARIES(${TARGET_OSQUERY_LIB} "-Wl,-whole-archive") -TARGET_LINK_LIBRARIES(${TARGET_OSQUERY_LIB} osquery_static) -TARGET_LINK_LIBRARIES(${TARGET_OSQUERY_LIB} "-Wl,-no-whole-archive") +TARGET_OSQUERY_LINK_WHOLE(${TARGET_OSQUERY_LIB} osquery_static) TARGET_LINK_LIBRARIES(${TARGET_OSQUERY_LIB} ${${TARGET_OSQUERY_LIB}_DEP}) #INSTALL(TARGETS ${TARGET_OSQUERY_LIB} diff --git a/osquery/config/CMakeLists.txt b/osquery/config/CMakeLists.txt index 24eed5e..f7698ab 100644 --- a/osquery/config/CMakeLists.txt +++ b/osquery/config/CMakeLists.txt @@ -1,4 +1,4 @@ -ADD_OSQUERY_LIBRARY(osquery_config config.cpp - plugins/filesystem.cpp) +ADD_OSQUERY_LIBRARY(TRUE osquery_config config.cpp) +ADD_OSQUERY_LIBRARY(FALSE osquery_config_plugins plugins/filesystem.cpp) -ADD_OSQUERY_TEST(osquery_config_tests config_tests.cpp) +ADD_OSQUERY_TEST(FALSE osquery_config_tests config_tests.cpp) diff --git a/osquery/config/config.cpp b/osquery/config/config.cpp index 8d16467..146a308 100644 --- a/osquery/config/config.cpp +++ b/osquery/config/config.cpp @@ -16,7 +16,6 @@ #include #include -#include #include #include #include @@ -26,9 +25,9 @@ namespace pt = boost::property_tree; namespace osquery { DEFINE_osquery_flag(string, - config_retriever, + config_plugin, "filesystem", - "Config type (plugin)."); + "Config type (plugin)"); static boost::shared_mutex rw_lock; @@ -61,25 +60,20 @@ Status Config::load() { } Status Config::genConfig(std::string& conf) { - if (REGISTERED_CONFIG_PLUGINS.find(FLAGS_config_retriever) == - REGISTERED_CONFIG_PLUGINS.end()) { - LOG(ERROR) << "Config retriever " << FLAGS_config_retriever << " not found"; + if (!Registry::exists("config", FLAGS_config_plugin)) { + LOG(ERROR) << "Config retriever " << FLAGS_config_plugin << " not found"; return Status(1, "Config retriever not found"); } - try { - auto config_data = - REGISTERED_CONFIG_PLUGINS.at(FLAGS_config_retriever)->genConfig(); - if (!config_data.first.ok()) { - return config_data.first; - } - conf = config_data.second; - } catch (std::exception& e) { - LOG(ERROR) << "Could not load ConfigPlugin " << FLAGS_config_retriever - << ": " << e.what(); - return Status(1, "Could not load config plugin"); + PluginResponse response; + auto status = Registry::call( + "config", FLAGS_config_plugin, {{"action", "genConfig"}}, response); + + if (!status.ok()) { + return status; } + conf = response[0].at("data"); return Status(0, "OK"); } @@ -141,4 +135,18 @@ Status Config::checkConfig() { OsqueryConfig c; return genConfig(c); } + +Status ConfigPlugin::call(const PluginRequest& request, + PluginResponse& response) { + if (request.count("action") == 0) { + return Status(1, "Config plugins require an action in PluginRequest"); + } + + if (request.at("action") == "genConfig") { + auto config_data = genConfig(); + response.push_back({{"data", config_data.second}}); + return config_data.first; + } + return Status(1, "Config plugin action unknown: " + request.at("action")); +} } diff --git a/osquery/config/config_tests.cpp b/osquery/config/config_tests.cpp index 84d1e04..7c48aab 100644 --- a/osquery/config/config_tests.cpp +++ b/osquery/config/config_tests.cpp @@ -11,10 +11,10 @@ #include #include -#include #include #include #include +#include #include "osquery/core/test_util.h" @@ -26,27 +26,15 @@ DECLARE_string(config_path); class ConfigTests : public testing::Test { public: ConfigTests() { - FLAGS_config_retriever = "filesystem"; + FLAGS_config_plugin = "filesystem"; FLAGS_config_path = kTestDataPath + "test.config"; - osquery::InitRegistry::get().run(); + Registry::setUp(); auto c = Config::getInstance(); c->load(); } }; -TEST_F(ConfigTests, test_queries_execute) { - auto c = Config::getInstance(); - auto queries = c->getScheduledQueries(); - - EXPECT_EQ(queries.size(), 1); - for (const auto& i : queries) { - int err; - auto r = query(i.query, err); - EXPECT_EQ(err, 0); - } -} - class TestConfigPlugin : public ConfigPlugin { public: TestConfigPlugin() {} @@ -54,17 +42,30 @@ class TestConfigPlugin : public ConfigPlugin { std::pair genConfig() { return std::make_pair(Status(0, "OK"), "foobar"); } - - virtual ~TestConfigPlugin() {} }; -REGISTER_CONFIG_PLUGIN("test", std::make_shared()) - TEST_F(ConfigTests, test_plugin) { - auto p = REGISTERED_CONFIG_PLUGINS.at("test")->genConfig(); - EXPECT_EQ(p.first.ok(), true); - EXPECT_EQ(p.first.toString(), "OK"); - EXPECT_EQ(p.second, "foobar"); + Registry::add("config", "test"); + + PluginResponse response; + auto status = + Registry::call("config", "test", {{"action", "genConfig"}}, response); + + EXPECT_EQ(status.ok(), true); + EXPECT_EQ(status.toString(), "OK"); + EXPECT_EQ(response[0].at("data"), "foobar"); +} + +TEST_F(ConfigTests, test_queries_execute) { + auto c = Config::getInstance(); + auto queries = c->getScheduledQueries(); + + EXPECT_EQ(queries.size(), 1); + for (const auto& i : queries) { + QueryData results; + auto status = query(i.query, results); + EXPECT_TRUE(status.ok()); + } } } diff --git a/osquery/config/plugins/filesystem.cpp b/osquery/config/plugins/filesystem.cpp index 72ca58c..0617558 100644 --- a/osquery/config/plugins/filesystem.cpp +++ b/osquery/config/plugins/filesystem.cpp @@ -12,7 +12,7 @@ #include -#include +#include #include #include @@ -28,25 +28,26 @@ DEFINE_osquery_flag(string, class FilesystemConfigPlugin : public ConfigPlugin { public: - virtual std::pair genConfig() { - std::string config; - if (!fs::exists(FLAGS_config_path)) { - return std::make_pair(Status(1, "config file does not exist"), config); - } - - VLOG(1) << "Filesystem ConfigPlugin reading: " << FLAGS_config_path; - std::ifstream config_stream(FLAGS_config_path); - - config_stream.seekg(0, std::ios::end); - config.reserve(config_stream.tellg()); - config_stream.seekg(0, std::ios::beg); - - config.assign((std::istreambuf_iterator(config_stream)), - std::istreambuf_iterator()); - return std::make_pair(Status(0, "OK"), config); - } + virtual std::pair genConfig(); }; -REGISTER_CONFIG_PLUGIN("filesystem", - std::make_shared()) +REGISTER(FilesystemConfigPlugin, "config", "filesystem"); + +std::pair FilesystemConfigPlugin::genConfig() { + std::string config; + if (!fs::exists(FLAGS_config_path)) { + return std::make_pair(Status(1, "config file does not exist"), config); + } + + VLOG(1) << "Filesystem ConfigPlugin reading: " << FLAGS_config_path; + std::ifstream config_stream(FLAGS_config_path); + + config_stream.seekg(0, std::ios::end); + config.reserve(config_stream.tellg()); + config_stream.seekg(0, std::ios::beg); + + config.assign((std::istreambuf_iterator(config_stream)), + std::istreambuf_iterator()); + return std::make_pair(Status(0, "OK"), config); +} } diff --git a/osquery/core/CMakeLists.txt b/osquery/core/CMakeLists.txt index f866f50..2c9dcc3 100644 --- a/osquery/core/CMakeLists.txt +++ b/osquery/core/CMakeLists.txt @@ -1,20 +1,17 @@ -ADD_OSQUERY_LIBRARY(osquery_core init.cpp - conversions.cpp - sql.cpp - sqlite_util.cpp - system.cpp - test_util.cpp - text.cpp - tables.cpp - virtual_table.cpp - flags.cpp - hash.cpp) +ADD_OSQUERY_LIBRARY(TRUE osquery_core init.cpp + conversions.cpp + system.cpp + text.cpp + tables.cpp + flags.cpp + hash.cpp + watcher.cpp) -ADD_OSQUERY_TEST(osquery_hash_tests hash_tests.cpp) -ADD_OSQUERY_TEST(osquery_status_tests status_tests.cpp) -ADD_OSQUERY_TEST(osquery_sql_tests sql_tests.cpp) -ADD_OSQUERY_TEST(osquery_sqlite_util_tests sqlite_util_tests.cpp) -ADD_OSQUERY_TEST(osquery_tables_tests tables_tests.cpp) -ADD_OSQUERY_TEST(osquery_test_util_tests test_util_tests.cpp) -ADD_OSQUERY_TEST(osquery_text_tests text_tests.cpp) -ADD_OSQUERY_TEST(osquery_conversions_tests conversions_tests.cpp) +ADD_OSQUERY_LIBRARY(TRUE osquery_test_util test_util.cpp) + +ADD_OSQUERY_TEST(TRUE osquery_hash_tests hash_tests.cpp) +ADD_OSQUERY_TEST(TRUE osquery_status_tests status_tests.cpp) +ADD_OSQUERY_TEST(TRUE osquery_tables_tests tables_tests.cpp) +ADD_OSQUERY_TEST(TRUE osquery_test_util_tests test_util_tests.cpp) +ADD_OSQUERY_TEST(TRUE osquery_text_tests text_tests.cpp) +ADD_OSQUERY_TEST(TRUE osquery_conversions_tests conversions_tests.cpp) diff --git a/osquery/core/flags.cpp b/osquery/core/flags.cpp index 9840be6..e715105 100644 --- a/osquery/core/flags.cpp +++ b/osquery/core/flags.cpp @@ -8,6 +8,8 @@ * */ +#include + #include namespace osquery { diff --git a/osquery/core/init.cpp b/osquery/core/init.cpp index 4a96a3a..291231f 100644 --- a/osquery/core/init.cpp +++ b/osquery/core/init.cpp @@ -12,6 +12,7 @@ #include #include +#include #include #include #include @@ -24,22 +25,30 @@ const std::string kDescription = "relational database"; const std::string kEpilog = "osquery project page ."; -DEFINE_osquery_flag(bool, debug, false, "Enable debug messages."); +DEFINE_osquery_flag(bool, debug, false, "Enable debug messages"); DEFINE_osquery_flag(bool, verbose_debug, false, - "Enable verbose debug messages."); + "Enable verbose debug messages"); -DEFINE_osquery_flag(bool, - disable_logging, - false, - "Disable ERROR/INFO logging."); +DEFINE_osquery_flag(bool, disable_logging, false, "Disable ERROR/INFO logging"); DEFINE_osquery_flag(string, osquery_log_dir, "/var/log/osquery/", - "Directory to store ERROR/INFO and results logging."); + "Directory for ERROR/INFO and results logging"); + +DEFINE_osquery_flag(bool, + config_check, + false, + "Check the format of an osquery config"); + +#ifndef __APPLE__ +namespace osquery { +DEFINE_osquery_flag(bool, daemonize, false, "Run as daemon (osqueryd only)"); +} +#endif namespace fs = boost::filesystem; @@ -68,7 +77,7 @@ void printUsage(const std::string& binary, int tool) { fprintf(stdout, "\n%s\n", kEpilog.c_str()); } -void announce(const std::string& basename) { +void announce() { syslog(LOG_NOTICE, "osqueryd started [version=" OSQUERY_VERSION "]"); } @@ -83,15 +92,16 @@ void initOsquery(int argc, char* argv[], int tool) { ::exit(0); } - // Print the version to SYSLOG. - if (tool == OSQUERY_TOOL_DAEMON) { - announce(binary); - } - FLAGS_alsologtostderr = true; FLAGS_logbufsecs = 0; // flush the log buffer immediately FLAGS_stop_logging_if_full_disk = true; FLAGS_max_log_size = 10; // max size for individual log file is 10MB + + // if you'd like to change the default logging plugin, compile osquery with + // -DOSQUERY_DEFAULT_CONFIG_PLUGIN= +#ifdef OSQUERY_DEFAULT_CONFIG_PLUGIN + FLAGS_config_plugin = STR(OSQUERY_DEFAULT_CONFIG_PLUGIN); +#endif // Set version string from CMake build __GFLAGS_NAMESPACE::SetVersionString(OSQUERY_VERSION); @@ -126,15 +136,57 @@ void initOsquery(int argc, char* argv[], int tool) { FLAGS_minloglevel = 2; // ERROR } + // Start the logging, and announce the daemon is starting. google::InitGoogleLogging(argv[0]); - VLOG(1) << "osquery starting [version=" OSQUERY_VERSION "]"; - osquery::InitRegistry::get().run(); + VLOG(1) << "osquery initializing [version=" OSQUERY_VERSION "]"; -// Once command line arguments are parsed load the osquery config. -#ifdef OSQUERY_DEFAULT_CONFIG_PLUGIN - FLAGS_config_retriever = STR(OSQUERY_DEFAULT_CONFIG_PLUGIN); -#endif + // Run the setup for all non-lazy registries. + Registry::setUp(); + // And finally load the config. auto config = Config::getInstance(); config->load(); + + if (FLAGS_config_check) { + auto s = Config::checkConfig(); + if (!s.ok()) { + std::cerr << "Error reading config: " << s.toString() << "\n"; + } + ::exit(s.getCode()); + } +} + +void initOsqueryDaemon() { +#ifndef __APPLE__ + // OSX uses launchd to daemonize. + if (osquery::FLAGS_daemonize) { + if (daemon(0, 0) == -1) { + ::exit(EXIT_FAILURE); + } + } +#endif + + // Print the version to SYSLOG. + announce(); + + // Create a process mutex around the daemon. + auto pid_status = createPidFile(); + if (!pid_status.ok()) { + LOG(ERROR) << "osqueryd initialize failed: " << pid_status.toString(); + ::exit(EXIT_FAILURE); + } + + // Check the backing store by allocating and exitting on error. + if (!DBHandle::checkDB()) { + LOG(ERROR) << "osqueryd initialize failed: Could not create DB handle"; + ::exit(EXIT_FAILURE); + } +} + +void shutdownOsquery() { + // End any event type run loops. + EventFactory::end(); + + // Hopefully release memory used by global string constructors in gflags. + __GFLAGS_NAMESPACE::ShutDownCommandLineFlags(); } } diff --git a/osquery/core/sql.cpp b/osquery/core/sql.cpp deleted file mode 100644 index e17a50e..0000000 --- a/osquery/core/sql.cpp +++ /dev/null @@ -1,90 +0,0 @@ -/* - * Copyright (c) 2014, Facebook, Inc. - * All rights reserved. - * - * This source code is licensed under the BSD-style license found in the - * LICENSE file in the root directory of this source tree. An additional grant - * of patent rights can be found in the PATENTS file in the same directory. - * - */ - -#include - -#include -#include -#include -#include - -#include "osquery/core/virtual_table.h" - -namespace osquery { - -const std::map kSQLiteReturnCodes = { - {0, "SQLITE_OK: Successful result"}, - {1, "SQLITE_ERROR: SQL error or missing database"}, - {2, "SQLITE_INTERNAL: Internal logic error in SQLite"}, - {3, "SQLITE_PERM: Access permission denied"}, - {4, "SQLITE_ABORT: Callback routine requested an abort"}, - {5, "SQLITE_BUSY: The database file is locked"}, - {6, "SQLITE_LOCKED: A table in the database is locked"}, - {7, "SQLITE_NOMEM: A malloc() failed"}, - {8, "SQLITE_READONLY: Attempt to write a readonly database"}, - {9, "SQLITE_INTERRUPT: Operation terminated by sqlite3_interrupt()"}, - {10, "SQLITE_IOERR: Some kind of disk I/O error occurred"}, - {11, "SQLITE_CORRUPT: The database disk image is malformed"}, - {12, "SQLITE_NOTFOUND: Unknown opcode in sqlite3_file_control()"}, - {13, "SQLITE_FULL: Insertion failed because database is full"}, - {14, "SQLITE_CANTOPEN: Unable to open the database file"}, - {15, "SQLITE_PROTOCOL: Database lock protocol error"}, - {16, "SQLITE_EMPTY: Database is empty"}, - {17, "SQLITE_SCHEMA: The database schema changed"}, - {18, "SQLITE_TOOBIG: String or BLOB exceeds size limit"}, - {19, "SQLITE_CONSTRAINT: Abort due to constraint violation"}, - {20, "SQLITE_MISMATCH: Data type mismatch"}, - {21, "SQLITE_MISUSE: Library used incorrectly"}, - {22, "SQLITE_NOLFS: Uses OS features not supported on host"}, - {23, "SQLITE_AUTH: Authorization denied"}, - {24, "SQLITE_FORMAT: Auxiliary database format error"}, - {25, "SQLITE_RANGE: 2nd parameter to sqlite3_bind out of range"}, - {26, "SQLITE_NOTADB: File opened that is not a database file"}, - {27, "SQLITE_NOTICE: Notifications from sqlite3_log()"}, - {28, "SQLITE_WARNING: Warnings from sqlite3_log()"}, - {100, "SQLITE_ROW: sqlite3_step() has another row ready"}, - {101, "SQLITE_DONE: sqlite3_step() has finished executing"}, -}; - -std::string getStringForSQLiteReturnCode(int code) { - if (kSQLiteReturnCodes.find(code) != kSQLiteReturnCodes.end()) { - return kSQLiteReturnCodes.at(code); - } else { - std::ostringstream s; - s << "Error: " << code << " is not a valid SQLite result code"; - return s.str(); - } -} - -SQL::SQL(const std::string& q) { - int code = 0; - results_ = query(q, code); - status_ = Status(code, getStringForSQLiteReturnCode(code)); -} - -QueryData SQL::rows() { return results_; } - -bool SQL::ok() { return status_.ok(); } - -std::string SQL::getMessageString() { return status_.toString(); } - -std::vector SQL::getTableNames() { - std::vector results; - for (const auto& it : REGISTERED_TABLES) { - results.push_back(it.first); - } - return results; -} - -QueryData SQL::selectAllFrom(const std::string& table) { - std::string query = "select * from " + table + ";"; - return SQL(query).rows(); -} -} diff --git a/osquery/core/sql_tests.cpp b/osquery/core/sql_tests.cpp deleted file mode 100644 index 5d5a946..0000000 --- a/osquery/core/sql_tests.cpp +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2014, Facebook, Inc. - * All rights reserved. - * - * This source code is licensed under the BSD-style license found in the - * LICENSE file in the root directory of this source tree. An additional grant - * of patent rights can be found in the PATENTS file in the same directory. - * - */ - -#include - -#include -#include - -namespace osquery { - -class SQLTests : public testing::Test {}; - -TEST_F(SQLTests, test_simple_query_execution) { - auto sql = SQL("SELECT * FROM time"); - EXPECT_TRUE(sql.ok()); - EXPECT_EQ(sql.getMessageString(), getStringForSQLiteReturnCode(0)); - EXPECT_EQ(sql.rows().size(), 1); -} - -TEST_F(SQLTests, test_get_tables) { - auto tables = SQL::getTableNames(); - EXPECT_TRUE(tables.size() > 0); -} -} - -int main(int argc, char* argv[]) { - testing::InitGoogleTest(&argc, argv); - osquery::initOsquery(argc, argv); - return RUN_ALL_TESTS(); -} diff --git a/osquery/core/sqlite_util.cpp b/osquery/core/sqlite_util.cpp deleted file mode 100644 index 572ee9a..0000000 --- a/osquery/core/sqlite_util.cpp +++ /dev/null @@ -1,66 +0,0 @@ -/* - * Copyright (c) 2014, Facebook, Inc. - * All rights reserved. - * - * This source code is licensed under the BSD-style license found in the - * LICENSE file in the root directory of this source tree. An additional grant - * of patent rights can be found in the PATENTS file in the same directory. - * - */ - -#include -#include -#include -#include - -#include "osquery/core/sqlite_util.h" -#include "osquery/core/virtual_table.h" - -namespace osquery { - -sqlite3* createDB() { - sqlite3* db = nullptr; - sqlite3_open(":memory:", &db); - osquery::tables::attachVirtualTables(db); - return db; -} - -QueryData query(const std::string& q, int& error_return) { - sqlite3* db = createDB(); - QueryData results = query(q, error_return, db); - sqlite3_close(db); - return results; -} - -QueryData query(const std::string& q, int& error_return, sqlite3* db) { - QueryData d; - char* err = nullptr; - sqlite3_exec(db, q.c_str(), query_data_callback, &d, &err); - if (err != nullptr) { - LOG(ERROR) << "Error launching query: " << err; - error_return = 1; - sqlite3_free(err); - } else { - error_return = 0; - } - - return d; -} - -int query_data_callback(void* argument, - int argc, - char* argv[], - char* column[]) { - if (argument == nullptr) { - LOG(ERROR) << "query_data_callback received nullptr as data argument"; - return SQLITE_MISUSE; - } - QueryData* qData = (QueryData*)argument; - Row r; - for (int i = 0; i < argc; i++) { - r[column[i]] = argv[i]; - } - (*qData).push_back(r); - return 0; -} -} diff --git a/osquery/core/sqlite_util.h b/osquery/core/sqlite_util.h deleted file mode 100644 index c1829f2..0000000 --- a/osquery/core/sqlite_util.h +++ /dev/null @@ -1,18 +0,0 @@ -/* - * Copyright (c) 2014, Facebook, Inc. - * All rights reserved. - * - * This source code is licensed under the BSD-style license found in the - * LICENSE file in the root directory of this source tree. An additional grant - * of patent rights can be found in the PATENTS file in the same directory. - * - */ - -#pragma once - -namespace osquery { - -// the callback for populating a std::vector set of results. "argument" -// should be a non-const reference to a std::vector -int query_data_callback(void *argument, int argc, char *argv[], char *column[]); -} diff --git a/osquery/core/sqlite_util_tests.cpp b/osquery/core/sqlite_util_tests.cpp deleted file mode 100644 index 40a187c..0000000 --- a/osquery/core/sqlite_util_tests.cpp +++ /dev/null @@ -1,56 +0,0 @@ -/* - * Copyright (c) 2014, Facebook, Inc. - * All rights reserved. - * - * This source code is licensed under the BSD-style license found in the - * LICENSE file in the root directory of this source tree. An additional grant - * of patent rights can be found in the PATENTS file in the same directory. - * - */ - -#include - -#include - -#include - -#include "osquery/core/sqlite_util.h" -#include "osquery/core/test_util.h" - -namespace osquery { - -class SQLiteUtilTests : public testing::Test {}; - -TEST_F(SQLiteUtilTests, test_simple_query_execution) { - int err; - auto db = createTestDB(); - auto results = query(kTestQuery, err, db); - sqlite3_close(db); - EXPECT_EQ(err, 0); - EXPECT_EQ(results, getTestDBExpectedResults()); -} - -TEST_F(SQLiteUtilTests, test_passing_callback_no_data_param) { - char* err = nullptr; - auto db = createTestDB(); - sqlite3_exec(db, kTestQuery.c_str(), query_data_callback, nullptr, &err); - sqlite3_close(db); - EXPECT_TRUE(err != nullptr); - if (err != nullptr) { - sqlite3_free(err); - } -} - -TEST_F(SQLiteUtilTests, test_aggregate_query) { - int err; - auto db = createTestDB(); - QueryData d = query(kTestQuery, err, db); - sqlite3_close(db); - EXPECT_EQ(err, 0); -} -} - -int main(int argc, char* argv[]) { - testing::InitGoogleTest(&argc, argv); - return RUN_ALL_TESTS(); -} diff --git a/osquery/core/system.cpp b/osquery/core/system.cpp index 1bdbf29..336b7a6 100644 --- a/osquery/core/system.cpp +++ b/osquery/core/system.cpp @@ -34,7 +34,13 @@ namespace osquery { DEFINE_osquery_flag(string, pidfile, "/var/osquery/osqueryd.pidfile", - "The path to the pidfile for osqueryd."); + "The path to the pidfile for osqueryd"); + +/// Should the daemon force unload previously-running osqueryd daemons. +DEFINE_osquery_flag(bool, + force, + false, + "Force osqueryd to kill previously-running daemons"); std::string getHostname() { char hostname[256]; // Linux max should be 64. @@ -53,8 +59,6 @@ std::string generateNewUuid() { std::string generateHostUuid() { #ifdef __APPLE__ // Use the hardware uuid available on OSX to identify this machine - char uuid[128]; - memset(uuid, 0, sizeof(uuid)); uuid_t id; // wait at most 5 seconds for gethostuuid to return const timespec wait = {5, 0}; @@ -86,27 +90,16 @@ int getUnixTime() { return result; } -std::vector getHomeDirectories() { - auto sql = SQL( - "SELECT DISTINCT directory FROM users WHERE directory != '/var/empty';"); - std::vector results; - if (sql.ok()) { - for (const auto& row : sql.rows()) { - results.push_back(row.at("directory")); - } - } else { - LOG(ERROR) - << "Error executing query to return users: " << sql.getMessageString(); - } - return results; -} - -Status checkStalePid(const std::string& pidfile_content) { +Status checkStalePid(const std::string& content) { int pid; try { - pid = stoi(pidfile_content); - } catch (const std::invalid_argument& e) { - return Status(1, std::string("Could not parse pidfile: ") + e.what()); + pid = boost::lexical_cast(content); + } catch (const boost::bad_lexical_cast& e) { + if (FLAGS_force) { + return Status(0, "Force loading and not parsing pidfile"); + } else { + return Status(1, "Could not parse pidfile"); + } } int status = kill(pid, 0); @@ -121,23 +114,21 @@ Status checkStalePid(const std::string& pidfile_content) { if (q.rows().size() >= 1 && q.rows().front()["name"] == "osqueryd") { // If the process really is osqueryd, return an "error" status. - return Status(1, - std::string("osqueryd (") + pidfile_content + - ") is already running"); + if (FLAGS_force) { + // The caller may choose to abort the existing daemon with --force. + status = kill(pid, SIGQUIT); + ::sleep(1); + + return Status(status, "Tried to force remove the existing osqueryd"); + } + + return Status(1, "osqueryd (" + content + ") is already running"); } else { - LOG(INFO) << "Found stale process for osqueryd (" << pidfile_content - << ") removing pidfile."; + LOG(INFO) << "Found stale process for osqueryd (" << content + << ") removing pidfile"; } } - // Now the pidfile is either the wrong pid or the pid is not running. - try { - boost::filesystem::remove(FLAGS_pidfile); - } catch (boost::filesystem::filesystem_error& e) { - // Unable to remove old pidfile. - LOG(WARNING) << "Unable to remove the osqueryd pidfile."; - } - return Status(0, "OK"); } @@ -158,6 +149,14 @@ Status createPidFile() { } } + // Now the pidfile is either the wrong pid or the pid is not running. + try { + boost::filesystem::remove(FLAGS_pidfile); + } catch (boost::filesystem::filesystem_error& e) { + // Unable to remove old pidfile. + LOG(WARNING) << "Unable to remove the osqueryd pidfile"; + } + // If no pidfile exists or the existing pid was stale, write, log, and run. auto pid = boost::lexical_cast(getpid()); LOG(INFO) << "Writing osqueryd pid (" << pid << ") to " << FLAGS_pidfile; diff --git a/osquery/core/tables.cpp b/osquery/core/tables.cpp index 3d0c3fa..910418b 100644 --- a/osquery/core/tables.cpp +++ b/osquery/core/tables.cpp @@ -3,17 +3,20 @@ * All rights reserved. * * This source code is licensed under the BSD-style license found in the - * LICENSE file in the root directory of this source tree. An additional grant + * LICENSE file in the root directory of this source tree. An additional grant * of patent rights can be found in the PATENTS file in the same directory. * */ +#include + #include #include namespace osquery { namespace tables { + bool ConstraintList::matches(const std::string& expr) { // Support each SQL affinity type casting. if (affinity == "TEXT") { @@ -60,15 +63,142 @@ bool ConstraintList::literal_matches(const T& base_expr) { return true; } -std::vector ConstraintList::getAll(ConstraintOperator op) { - std::vector set; +std::set ConstraintList::getAll(ConstraintOperator op) { + std::set set; for (size_t i = 0; i < constraints_.size(); ++i) { if (constraints_[i].op == op) { // TODO: this does not apply a distinct. - set.push_back(constraints_[i].expr); + set.insert(constraints_[i].expr); } } return set; } + +void ConstraintList::serialize(boost::property_tree::ptree& tree) const { + boost::property_tree::ptree expressions; + for (const auto& constraint : constraints_) { + boost::property_tree::ptree child; + child.put("op", constraint.op); + child.put("expr", constraint.expr); + expressions.push_back(std::make_pair("", child)); + } + tree.add_child("list", expressions); + tree.put("affinity", affinity); +} + +void ConstraintList::unserialize(const boost::property_tree::ptree& tree) { + // Iterate through the list of operand/expressions, then set the constraint + // type affinity. + for (const auto& list : tree.get_child("list")) { + Constraint constraint(list.second.get("op")); + constraint.expr = list.second.get("expr"); + constraints_.push_back(constraint); + } + affinity = tree.get("affinity"); +} + +void TablePlugin::setRequestFromContext(const QueryContext& context, + PluginRequest& request) { + boost::property_tree::ptree tree; + tree.put("limit", context.limit); + + // The QueryContext contains a constraint map from column to type information + // and the list of operand/expression constraints applied to that column from + // the query given. + boost::property_tree::ptree constraints; + for (const auto& constraint : context.constraints) { + boost::property_tree::ptree child; + child.put("name", constraint.first); + constraint.second.serialize(child); + constraints.push_back(std::make_pair("", child)); + } + tree.add_child("constraints", constraints); + + // Write the property tree as a JSON string into the PluginRequest. + std::ostringstream output; + boost::property_tree::write_json(output, tree, false); + request["context"] = output.str(); +} + +void TablePlugin::setResponseFromQueryData(const QueryData& data, + PluginResponse& response) { + for (const auto& row : data) { + response.push_back(row); + } +} + +void TablePlugin::setContextFromRequest(const PluginRequest& request, + QueryContext& context) { + if (request.count("context") == 0) { + return; + } + + // Read serialized context from PluginRequest. + std::stringstream input; + input << request.at("context"); + boost::property_tree::ptree tree; + boost::property_tree::read_json(input, tree); + + // Set the context limit and deserialize each column constraint list. + context.limit = tree.get("limit"); + for (const auto& constraint : tree.get_child("constraints")) { + auto column_name = constraint.second.get("name"); + context.constraints[column_name].unserialize(constraint.second); + } +} + +Status TablePlugin::call(const PluginRequest& request, + PluginResponse& response) { + response.clear(); + // TablePlugin API calling requires an action. + if (request.count("action") == 0) { + return Status(1, "Table plugins must include a request action"); + } + + if (request.at("action") == "statement") { + // The "statement" action generates an SQL create table statement. + response.push_back({{"statement", statement()}}); + } else if (request.at("action") == "generate") { + // "generate" runs the table implementation using a PluginRequest with + // optional serialized QueryContext and returns the QueryData results as + // the PluginRequest data. + QueryContext context; + if (request.count("context") > 0) { + setContextFromRequest(request, context); + } + setResponseFromQueryData(generate(context), response); + } else if (request.at("action") == "columns") { + // "columns" returns a PluginRequest filled with column information + // such as name and type. + auto column_list = columns(); + for (const auto& column : column_list) { + response.push_back({{"name", column.first}, {"type", column.second}}); + } + } else if (request.at("action") == "columns_definition") { + response.push_back({{"definition", columnDefinition()}}); + } else { + return Status(1, "Unknown table plugin action: " + request.at("action")); + } + + return Status(0, "OK"); +} + +std::string TablePlugin::columnDefinition() { + const auto& column_list = columns(); + std::string statement = "("; + for (size_t i = 0; i < column_list.size(); ++i) { + statement += column_list[i].first + " " + column_list.at(i).second; + if (i < column_list.size() - 1) { + statement += ", "; + } + } + statement += ")"; + return statement; +} + +std::string TablePlugin::statement() { + return "CREATE TABLE " + name_ + columnDefinition(); +} + } } diff --git a/osquery/core/tables_tests.cpp b/osquery/core/tables_tests.cpp index 8830bc3..d3f5f06 100644 --- a/osquery/core/tables_tests.cpp +++ b/osquery/core/tables_tests.cpp @@ -3,7 +3,7 @@ * All rights reserved. * * This source code is licensed under the BSD-style license found in the - * LICENSE file in the root directory of this source tree. An additional grant + * LICENSE file in the root directory of this source tree. An additional grant * of patent rights can be found in the PATENTS file in the same directory. * */ @@ -115,5 +115,6 @@ TEST_F(TablesTests, test_constraint_map) { int main(int argc, char* argv[]) { testing::InitGoogleTest(&argc, argv); + osquery::initOsquery(argc, argv); return RUN_ALL_TESTS(); } diff --git a/osquery/core/test_util.cpp b/osquery/core/test_util.cpp index fc662ff..14bd1b7 100644 --- a/osquery/core/test_util.cpp +++ b/osquery/core/test_util.cpp @@ -3,7 +3,7 @@ * All rights reserved. * * This source code is licensed under the BSD-style license found in the - * LICENSE file in the root directory of this source tree. An additional grant + * LICENSE file in the root directory of this source tree. An additional grant * of patent rights can be found in the PATENTS file in the same directory. * */ @@ -16,7 +16,6 @@ #include #include -#include "osquery/core/sqlite_util.h" #include "osquery/core/test_util.h" namespace pt = boost::property_tree; @@ -26,27 +25,6 @@ namespace osquery { const std::string kTestQuery = "SELECT * FROM test_table"; const std::string kTestDataPath = "../../../tools/tests/"; -sqlite3* createTestDB() { - sqlite3* db = createDB(); - char* err = nullptr; - std::vector queries = { - "CREATE TABLE test_table (" - "username varchar(30) primary key, " - "age int" - ")", - "INSERT INTO test_table VALUES (\"mike\", 23)", - "INSERT INTO test_table VALUES (\"matt\", 24)"}; - for (auto q : queries) { - sqlite3_exec(db, q.c_str(), nullptr, nullptr, &err); - if (err != nullptr) { - LOG(ERROR) << "Error creating test database: " << err; - return nullptr; - } - } - - return db; -} - QueryData getTestDBExpectedResults() { QueryData d; Row row1; @@ -260,4 +238,8 @@ osquery::QueryData getEtcHostsExpectedResults() { row4["hostnames"] = "localhost"; return {row1, row2, row3, row4}; } + +::std::ostream& operator<<(::std::ostream& os, const Status& s) { + return os << "Status(" << s.getCode() << ", \"" << s.getMessage() << "\")"; +} } diff --git a/osquery/core/test_util.h b/osquery/core/test_util.h index 8590d97..4634e96 100644 --- a/osquery/core/test_util.h +++ b/osquery/core/test_util.h @@ -31,10 +31,6 @@ namespace osquery { extern const std::string kTestQuery; extern const std::string kTestDataPath; -// createTestDB instantiates a sqlite3 struct and populates it with some test -// data -sqlite3* createTestDB(); - // getTestDBExpectedResults returns the results of kTestQuery of the table that // initially gets returned from createTestDB() osquery::QueryData getTestDBExpectedResults(); diff --git a/osquery/core/test_util_tests.cpp b/osquery/core/test_util_tests.cpp index f11f87a..10c79fa 100644 --- a/osquery/core/test_util_tests.cpp +++ b/osquery/core/test_util_tests.cpp @@ -21,32 +21,6 @@ namespace osquery { class TestUtilTests : public testing::Test {}; -TEST_F(TestUtilTests, test_expected_results) { - int err; - auto db = createTestDB(); - auto results = query(kTestQuery, err, db); - sqlite3_close(db); - EXPECT_EQ(err, 0); - EXPECT_EQ(results, getTestDBExpectedResults()); -} - -TEST_F(TestUtilTests, test_get_test_db_result_stream) { - auto db = createTestDB(); - auto results = getTestDBResultStream(); - for (auto r : results) { - char* err_char = nullptr; - sqlite3_exec(db, (r.first).c_str(), nullptr, nullptr, &err_char); - EXPECT_TRUE(err_char == nullptr); - if (err_char != nullptr) { - sqlite3_free(err_char); - ASSERT_TRUE(false); - } - int err_int; - auto expected = query(kTestQuery, err_int, db); - EXPECT_EQ(expected, r.second); - } - sqlite3_close(db); -} } int main(int argc, char* argv[]) { diff --git a/osquery/core/virtual_table.cpp b/osquery/core/virtual_table.cpp deleted file mode 100644 index 58dd7a7..0000000 --- a/osquery/core/virtual_table.cpp +++ /dev/null @@ -1,87 +0,0 @@ -/* - * Copyright (c) 2014, Facebook, Inc. - * All rights reserved. - * - * This source code is licensed under the BSD-style license found in the - * LICENSE file in the root directory of this source tree. An additional grant - * of patent rights can be found in the PATENTS file in the same directory. - * - */ - -#include - -#include "osquery/core/virtual_table.h" - -namespace osquery { -namespace tables { - -int xOpen(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCursor) { - int rc = SQLITE_NOMEM; - base_cursor *pCur; - - pCur = new base_cursor; - - if (pCur) { - memset(pCur, 0, sizeof(base_cursor)); - *ppCursor = (sqlite3_vtab_cursor *)pCur; - rc = SQLITE_OK; - } - - return rc; -} - -int xClose(sqlite3_vtab_cursor *cur) { - base_cursor *pCur = (base_cursor *)cur; - - delete pCur; - return SQLITE_OK; -} - -int xEof(sqlite3_vtab_cursor *cur) { - base_cursor *pCur = (base_cursor *)cur; - auto *pVtab = (x_vtab *)cur->pVtab; - return pCur->row >= pVtab->pContent->n; -} - -int xDestroy(sqlite3_vtab *p) { - auto *pVtab = (x_vtab *)p; - delete pVtab->pContent; - delete pVtab; - return SQLITE_OK; -} - -int xNext(sqlite3_vtab_cursor *cur) { - base_cursor *pCur = (base_cursor *)cur; - pCur->row++; - return SQLITE_OK; -} - -int xRowid(sqlite3_vtab_cursor *cur, sqlite_int64 *pRowid) { - base_cursor *pCur = (base_cursor *)cur; - *pRowid = pCur->row; - return SQLITE_OK; -} - -std::string TablePlugin::statement(TableName name, TableColumns columns) { - std::string statement = "CREATE TABLE " + name + "("; - for (size_t i = 0; i < columns.size(); ++i) { - statement += columns[i].first + " " + columns[i].second; - if (i < columns.size() - 1) { - statement += ", "; - } - } - statement += ")"; - return statement; -} - -void attachVirtualTables(sqlite3 *db) { - for (const auto &table : REGISTERED_TABLES) { - int s = table.second->attachVtable(db); - if (s != SQLITE_OK) { - LOG(ERROR) << "Error attaching virtual table: " << table.first << " (" - << s << ")"; - } - } -} -} -} diff --git a/osquery/core/virtual_table.h b/osquery/core/virtual_table.h deleted file mode 100644 index 332b1d8..0000000 --- a/osquery/core/virtual_table.h +++ /dev/null @@ -1,232 +0,0 @@ -/* - * Copyright (c) 2014, Facebook, Inc. - * All rights reserved. - * - * This source code is licensed under the BSD-style license found in the - * LICENSE file in the root directory of this source tree. An additional grant - * of patent rights can be found in the PATENTS file in the same directory. - * - */ - -#pragma once - -#include - -#include - -#include - -#include -#include - -namespace osquery { -namespace tables { - -/** - * @brief osquery cursor object. - * - * Only used in the SQLite virtual table module methods. - */ -struct base_cursor { - /// SQLite virtual table cursor. - sqlite3_vtab_cursor base; - /// Current cursor position. - int row; -}; - -/** - * @brief osquery virtual table object - * - * Only used in the SQLite virtual table module methods. - * This adds each table plugin class to the state tracking in SQLite. - */ -template -struct x_vtab { - sqlite3_vtab base; - /// To get custom functionality from SQLite virtual tables, add a struct. - TABLE_PLUGIN *pContent; -}; - -int xOpen(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCursor); - -int xClose(sqlite3_vtab_cursor *cur); - -int xNext(sqlite3_vtab_cursor *cur); - -int xRowid(sqlite3_vtab_cursor *cur, sqlite_int64 *pRowid); - -int xEof(sqlite3_vtab_cursor *cur); - -template -int xCreate(sqlite3 *db, - void *pAux, - int argc, - const char *const *argv, - sqlite3_vtab **ppVtab, - char **pzErr) { - auto *pVtab = new x_vtab; - - if (!pVtab) { - return SQLITE_NOMEM; - } - - memset(pVtab, 0, sizeof(x_vtab)); - auto *pContent = pVtab->pContent = new T; - auto create = pContent->statement(pContent->name, pContent->columns); - int rc = sqlite3_declare_vtab(db, create.c_str()); - - *ppVtab = (sqlite3_vtab *)pVtab; - return rc; -} - -int xDestroy(sqlite3_vtab *p); - -template -int xColumn(sqlite3_vtab_cursor *cur, sqlite3_context *ctx, int col) { - base_cursor *pCur = (base_cursor *)cur; - auto *pContent = ((x_vtab *)cur->pVtab)->pContent; - - if (col >= pContent->columns.size()) { - return SQLITE_ERROR; - } - - const auto &column_name = pContent->columns[col].first; - const auto &type = pContent->columns[col].second; - if (pCur->row >= pContent->data[column_name].size()) { - return SQLITE_ERROR; - } - - const auto &value = pContent->data[column_name][pCur->row]; - if (type == "TEXT") { - sqlite3_result_text(ctx, value.c_str(), -1, nullptr); - } else if (type == "INTEGER") { - int afinite; - try { - afinite = boost::lexical_cast(value); - } catch (const boost::bad_lexical_cast &e) { - afinite = -1; - LOG(WARNING) << "Error casting " << column_name << " (" << value - << ") to INTEGER"; - } - sqlite3_result_int(ctx, afinite); - } else if (type == "BIGINT") { - long long int afinite; - try { - afinite = boost::lexical_cast(value); - } catch (const boost::bad_lexical_cast &e) { - afinite = -1; - LOG(WARNING) << "Error casting " << column_name << " (" << value - << ") to BIGINT"; - } - sqlite3_result_int64(ctx, afinite); - } - - return SQLITE_OK; -} - -template -static int xBestIndex(sqlite3_vtab *tab, sqlite3_index_info *pIdxInfo) { - auto *pContent = ((x_vtab *)tab)->pContent; - pContent->constraints.clear(); - - int expr_index = 0; - int cost = 0; - for (size_t i = 0; i < pIdxInfo->nConstraint; ++i) { - if (!pIdxInfo->aConstraint[i].usable) { - // A higher cost less priority, prefer more usable query constraints. - cost += 10; - // TODO: OR is not usable. - continue; - } - - const auto& name = - pContent->columns[pIdxInfo->aConstraint[i].iColumn].first; - pContent->constraints.push_back( - std::make_pair(name, Constraint(pIdxInfo->aConstraint[i].op))); - pIdxInfo->aConstraintUsage[i].argvIndex = ++expr_index; - } - - pIdxInfo->estimatedCost = cost; - return SQLITE_OK; -} - -template -static int xFilter(sqlite3_vtab_cursor *pVtabCursor, - int idxNum, - const char *idxStr, - int argc, - sqlite3_value **argv) { - base_cursor *pCur = (base_cursor *)pVtabCursor; - auto *pContent = ((x_vtab *)pVtabCursor->pVtab)->pContent; - - pCur->row = 0; - pContent->n = 0; - QueryContext request; - - for (size_t i = 0; i < pContent->columns.size(); ++i) { - pContent->data[pContent->columns[i].first].clear(); - request.constraints[pContent->columns[i].first].affinity = - pContent->columns[i].second; - } - - for (size_t i = 0; i < argc; ++i) { - auto expr = (const char *)sqlite3_value_text(argv[i]); - // Set the expression from SQLite's now-populated argv. - pContent->constraints[i].second.expr = std::string(expr); - // Add the constraint to the column-sorted query request map. - request.constraints[pContent->constraints[i].first].add( - pContent->constraints[i].second); - } - - for (auto &row : pContent->generate(request)) { - for (const auto &column : pContent->columns) { - pContent->data[column.first].push_back(row[column.first]); - } - pContent->n++; - } - - return SQLITE_OK; -} - -template -int sqlite3_attach_vtable(sqlite3 *db, const std::string &name) { - int rc = SQLITE_OK; - - static sqlite3_module module = { - 0, - xCreate, - xCreate, - xBestIndex, - xDestroy, - xDestroy, - xOpen, - xClose, - xFilter, - xNext, - xEof, - xColumn, - xRowid, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - }; - - rc = sqlite3_create_module(db, name.c_str(), &module, 0); - if (rc == SQLITE_OK) { - auto format = "CREATE VIRTUAL TABLE temp." + name + " USING " + name; - rc = sqlite3_exec(db, format.c_str(), 0, 0, 0); - } - return rc; -} - -void attachVirtualTables(sqlite3 *db); -} -} - -DECLARE_REGISTRY(TablePlugins, std::string, osquery::tables::TablePluginRef); -#define REGISTERED_TABLES REGISTRY(TablePlugins) -#define REGISTER_TABLE(name, decorator) REGISTER(TablePlugins, name, decorator); diff --git a/osquery/core/watcher.cpp b/osquery/core/watcher.cpp new file mode 100644 index 0000000..ac4af4b --- /dev/null +++ b/osquery/core/watcher.cpp @@ -0,0 +1,228 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ + +#include +#include + +#include +#include + +#include + +#include +#include +#include +#include + +#include "osquery/core/watcher.h" + +extern char** environ; + +namespace fs = boost::filesystem; + +namespace osquery { + +const std::map > kWatchdogLimits = { + {MEMORY_LIMIT, {50, 20, 10}}, + {UTILIZATION_LIMIT, {90, 70, 60}}, + // Number of seconds the worker should run, else consider the exit fatal. + {RESPAWN_LIMIT, {20, 20, 20}}, + // If the worker respawns too quickly, backoff on creating additional. + {RESPAWN_DELAY, {5, 5, 5}}, + // Seconds of tolerable sustained latency. + {LATENCY_LIMIT, {5, 5, 3}}, + // How often to poll for performance limit violations. + {INTERVAL, {3, 3, 3}}, }; + +DEFINE_osquery_flag( + int32, + watchdog_level, + 1, + "Performance limit level (0=loose, 1=normal, 2=restrictive)"); + +DEFINE_osquery_flag(bool, + disable_watchdog, + false, + "Disable userland watchdog process"); + +bool Watcher::ok() { + ::sleep(getWorkerLimit(INTERVAL)); + return (worker_ >= 0); +} + +bool Watcher::watch() { + int status; + pid_t result = waitpid(worker_, &status, WNOHANG); + if (worker_ == 0 || result == worker_) { + // Worker does not exist or never existed. + return false; + } else if (result == 0) { + // If the inspect finds problems it will stop/restart the worker. + if (!isWorkerSane()) { + stopWorker(); + return false; + } + } + return true; +} + +void Watcher::stopWorker() { + kill(worker_, SIGKILL); + worker_ = 0; + // Clean up the defunct (zombie) process. + waitpid(-1, 0, 0); +} + +bool Watcher::isWorkerSane() { + auto rows = + SQL::selectAllFrom("processes", "pid", tables::EQUALS, INTEGER(worker_)); + if (rows.size() == 0) { + // Could not find worker process? + return false; + } + + // Compare CPU utilization since last check. + BIGINT_LITERAL footprint, user_time, system_time; + // IV is the check interval in seconds, and utilization is set per-second. + auto iv = getWorkerLimit(INTERVAL); + + try { + user_time = AS_LITERAL(BIGINT_LITERAL, rows[0].at("user_time")) / iv; + system_time = AS_LITERAL(BIGINT_LITERAL, rows[0].at("system_time")) / iv; + footprint = AS_LITERAL(BIGINT_LITERAL, rows[0].at("phys_footprint")); + } catch (const std::exception& e) { + sustained_latency_ = 0; + } + + if (current_user_time_ + getWorkerLimit(UTILIZATION_LIMIT) < user_time || + current_system_time_ + getWorkerLimit(UTILIZATION_LIMIT) < system_time) { + sustained_latency_++; + } else { + sustained_latency_ = 0; + } + + current_user_time_ = user_time; + current_system_time_ = system_time; + + if (sustained_latency_ * iv >= getWorkerLimit(LATENCY_LIMIT)) { + LOG(WARNING) << "osqueryd worker system performance limits exceeded"; + return false; + } + + if (footprint > getWorkerLimit(MEMORY_LIMIT) * 1024 * 1024) { + LOG(WARNING) << "osqueryd worker memory limits exceeded"; + return false; + } + + // The worker is sane, no action needed. + return true; +} + +void Watcher::createWorker() { + if (last_respawn_time_ > getUnixTime() - getWorkerLimit(RESPAWN_LIMIT)) { + LOG(WARNING) << "osqueryd worker respawning too quickly"; + ::sleep(getWorkerLimit(RESPAWN_DELAY)); + } + + worker_ = fork(); + if (worker_ < 0) { + // Unrecoverable error, cannot create a worker process. + LOG(ERROR) << "osqueryd could not create a worker process"; + ::exit(EXIT_FAILURE); + } else if (worker_ == 0) { + // This is the new worker process, no watching needed. + setenv("OSQUERYD_WORKER", std::to_string(getpid()).c_str(), 1); + fs::path exec_path(fs::initial_path()); + exec_path = fs::system_complete(fs::path(argv_[0])); + execve(exec_path.string().c_str(), argv_, environ); + // Code will never reach this point. + ::exit(EXIT_FAILURE); + } + + VLOG(1) << "osqueryd watcher (" << getpid() << ") executing worker (" + << worker_ << ")"; +} + +void Watcher::resetCounters() { + // Reset the monitoring counters for the watcher. + sustained_latency_ = 0; + current_user_time_ = 0; + current_system_time_ = 0; + last_respawn_time_ = getUnixTime(); +} + +void Watcher::initWorker() { + // Set the worker's process name. + size_t name_size = strlen(argv_[0]); + for (int i = 0; i < argc_; i++) { + if (argv_[i] != nullptr) { + memset(argv_[i], 0, strlen(argv_[i])); + } + } + strncpy(argv_[0], name_.c_str(), name_size); + + // Start a watcher watcher thread to exit the process if the watcher exits. + Dispatcher::getInstance().addService( + std::make_shared(getppid())); +} + +bool isOsqueryWorker() { + return (getenv("OSQUERYD_WORKER") != nullptr); +} + +void WatcherWatcherRunner::enter() { + while (true) { + if (getppid() != watcher_) { + // Watcher died, the worker must follow. + VLOG(1) << "osqueryd worker (" << getpid() + << ") detected killed watcher (" << watcher_ << ")"; + ::exit(EXIT_SUCCESS); + } + interruptableSleep(getWorkerLimit(INTERVAL) * 1000); + } +} + +size_t getWorkerLimit(WatchdogLimitType name, int level) { + if (kWatchdogLimits.count(name) == 0) { + return 0; + } + + // If no level was provided then use the default (config/switch). + if (level == -1) { + level = FLAGS_watchdog_level; + } + if (level > 3) { + return kWatchdogLimits.at(name).back(); + } + return kWatchdogLimits.at(name).at(level); +} + +void initWorkerWatcher(const std::string& name, int argc, char* argv[]) { + // The watcher will forever monitor and spawn additional workers. + Watcher watcher(argc, argv); + watcher.setWorkerName(name); + + if (isOsqueryWorker()) { + // Do not start watching/spawning if this process is a worker. + watcher.initWorker(); + } else { + do { + if (!watcher.watch()) { + // The watcher failed, create a worker. + watcher.createWorker(); + watcher.resetCounters(); + } + } while (watcher.ok()); + + // Executation should never reach this point. + ::exit(EXIT_FAILURE); + } +} +} diff --git a/osquery/core/watcher.h b/osquery/core/watcher.h new file mode 100644 index 0000000..984bb97 --- /dev/null +++ b/osquery/core/watcher.h @@ -0,0 +1,111 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ + +#pragma once + +#include + +#include + +#include +#include + +namespace osquery { + +DECLARE_bool(disable_watchdog); + +enum WatchdogLimitType { + MEMORY_LIMIT, + UTILIZATION_LIMIT, + RESPAWN_LIMIT, + RESPAWN_DELAY, + LATENCY_LIMIT, + INTERVAL, +}; + +class Watcher { + public: + Watcher(int argc, char* argv[]) : worker_(0), argc_(argc), argv_(argv) { + resetCounters(); + last_respawn_time_ = 0; + } + + void setWorkerName(const std::string& name) { name_ = name; } + const std::string& getWorkerName() { return name_; } + + /// Boilerplate function to sleep for some configured latency + bool ok(); + /// Begin the worker-watcher process. + bool watch(); + /// Fork a worker process. + void createWorker(); + /// If the process is a worker, clean up identification. + void initWorker(); + void resetCounters(); + + private: + /// Inspect into the memory, CPU, and other worker process states. + bool isWorkerSane(); + /// If a worker as otherwise gone insane, stop it. + void stopWorker(); + + private: + size_t sustained_latency_; + size_t current_user_time_; + size_t current_system_time_; + size_t last_respawn_time_; + + private: + /// Keep the single worker process/thread ID for inspection. + pid_t worker_; + /// Keep the invocation daemon's argc to iterate through argv. + int argc_; + /// When a worker child is spawned the argv will be scrubed. + char** argv_; + /// When a worker child is spawned the process name will be changed. + std::string name_; +}; + +/// The WatcherWatcher is spawned within the worker and watches the watcher. +class WatcherWatcherRunner : public InternalRunnable { + public: + WatcherWatcherRunner(pid_t watcher) : watcher_(watcher) {} + void enter(); + + private: + pid_t watcher_; +}; + +/// Check if the current process is already a worker. +bool isOsqueryWorker(); + +/// Get a performance limit by name and optional level. +size_t getWorkerLimit(WatchdogLimitType limit, int level = -1); + +/** + * @brief Daemon tools may want to continually spawn worker processes + * and monitor their utilization. + * + * A daemon may call initWorkerWatcher to begin watching child daemon + * processes until it-itself is unscheduled. The basic guarentee is that only + * workers will return from the function. + * + * The worker-watcher will implement performance bounds on CPU utilization + * and memory, as well as check for zombie/defunct workers and respawn them + * if appropriate. The appropriateness is determined from heuristics around + * how the worker exitted. Various exit states and velocities may cause the + * watcher to resign. + * + * @param name The name of the worker process. + * @param argc The daemon's argc. + * @param argv The daemon's volitle argv. + */ +void initWorkerWatcher(const std::string& name, int argc, char* argv[]); +} diff --git a/osquery/database/CMakeLists.txt b/osquery/database/CMakeLists.txt index 54a45b9..9471b47 100644 --- a/osquery/database/CMakeLists.txt +++ b/osquery/database/CMakeLists.txt @@ -1,7 +1,7 @@ -ADD_OSQUERY_LIBRARY(osquery_database db_handle.cpp +ADD_OSQUERY_LIBRARY(TRUE osquery_database db_handle.cpp query.cpp results.cpp) -ADD_OSQUERY_TEST(osquery_query_tests query_tests.cpp) -ADD_OSQUERY_TEST(osquery_db_handle_tests db_handle_tests.cpp) -ADD_OSQUERY_TEST(osquery_results_tests results_tests.cpp) +ADD_OSQUERY_TEST(TRUE osquery_query_tests query_tests.cpp) +ADD_OSQUERY_TEST(TRUE osquery_db_handle_tests db_handle_tests.cpp) +ADD_OSQUERY_TEST(TRUE osquery_results_tests results_tests.cpp) diff --git a/osquery/database/db_handle.cpp b/osquery/database/db_handle.cpp index 38bff27..9a08bd1 100644 --- a/osquery/database/db_handle.cpp +++ b/osquery/database/db_handle.cpp @@ -3,7 +3,7 @@ * All rights reserved. * * This source code is licensed under the BSD-style license found in the - * LICENSE file in the root directory of this source tree. An additional grant + * LICENSE file in the root directory of this source tree. An additional grant * of patent rights can be found in the PATENTS file in the same directory. * */ @@ -35,12 +35,12 @@ const std::vector kDomains = {kConfigurations, kQueries, kEvents}; DEFINE_osquery_flag(string, db_path, "/var/osquery/osquery.db", - "If using a disk-based backing store, specify a path."); + "If using a disk-based backing store, specify a path"); DEFINE_osquery_flag(bool, use_in_memory_database, false, - "Keep osquery backing-store in memory."); + "Keep osquery backing-store in memory"); ///////////////////////////////////////////////////////////////////////////// // constructors and destructors @@ -88,6 +88,15 @@ std::shared_ptr DBHandle::getInstance() { return getInstance(FLAGS_db_path, FLAGS_use_in_memory_database); } +bool DBHandle::checkDB() { + try { + auto handle = DBHandle(FLAGS_db_path, FLAGS_use_in_memory_database); + } catch (const std::exception& e) { + return false; + } + return true; +} + std::shared_ptr DBHandle::getInstanceInMemory() { return getInstance("", true); } diff --git a/osquery/database/db_handle_tests.cpp b/osquery/database/db_handle_tests.cpp index 80b9dea..913d9ec 100644 --- a/osquery/database/db_handle_tests.cpp +++ b/osquery/database/db_handle_tests.cpp @@ -15,6 +15,7 @@ #include #include +#include #include const std::string kTestingDBHandlePath = "/tmp/rocksdb-osquery-dbhandletests"; diff --git a/osquery/devtools/CMakeLists.txt b/osquery/devtools/CMakeLists.txt index ebe87ab..c4f1ec0 100644 --- a/osquery/devtools/CMakeLists.txt +++ b/osquery/devtools/CMakeLists.txt @@ -1,4 +1,4 @@ -ADD_OSQUERY_LIBRARY(osquery_devtools shell.cpp - printer.cpp) +ADD_OSQUERY_LIBRARY(FALSE osquery_devtools shell.cpp + printer.cpp) -ADD_OSQUERY_TEST(osquery_printer_tests printer_tests.cpp) +ADD_OSQUERY_TEST(FALSE osquery_printer_tests printer_tests.cpp) diff --git a/osquery/devtools/shell.cpp b/osquery/devtools/shell.cpp index 852de32..07fc62d 100644 --- a/osquery/devtools/shell.cpp +++ b/osquery/devtools/shell.cpp @@ -84,7 +84,7 @@ #include #include -#include "osquery/core/virtual_table.h" +#include "osquery/sql/virtual_table.h" // Json is a specific form of pretty printing. namespace osquery { diff --git a/osquery/dispatcher/CMakeLists.txt b/osquery/dispatcher/CMakeLists.txt index 27940c2..0a95aa8 100644 --- a/osquery/dispatcher/CMakeLists.txt +++ b/osquery/dispatcher/CMakeLists.txt @@ -1,3 +1,3 @@ -ADD_OSQUERY_LIBRARY(osquery_dispatcher dispatcher.cpp) +ADD_OSQUERY_LIBRARY(TRUE osquery_dispatcher dispatcher.cpp) -ADD_OSQUERY_TEST(osquery_dispatcher_tests dispatcher_tests.cpp) +ADD_OSQUERY_TEST(TRUE osquery_dispatcher_tests dispatcher_tests.cpp) diff --git a/osquery/dispatcher/dispatcher.cpp b/osquery/dispatcher/dispatcher.cpp index cf8044c..20d450d 100644 --- a/osquery/dispatcher/dispatcher.cpp +++ b/osquery/dispatcher/dispatcher.cpp @@ -8,6 +8,8 @@ * */ +#include + #include #include #include @@ -22,7 +24,11 @@ namespace osquery { DEFINE_osquery_flag(int32, worker_threads, 4, - "The number of threads to use for the work dispatcher"); + "Number of work dispatch threads"); + +void InternalRunnable::interruptableSleep(size_t milli) { + boost::this_thread::sleep(boost::posix_time::milliseconds(milli)); +} Dispatcher& Dispatcher::getInstance() { static Dispatcher d; @@ -30,15 +36,16 @@ Dispatcher& Dispatcher::getInstance() { } Dispatcher::Dispatcher() { - thread_manager_ = boost_to_std_shared_ptr( - ThreadManager::newSimpleThreadManager((size_t)FLAGS_worker_threads, 0)); + thread_manager_ = + boost_to_std_shared_ptr(InternalThreadManager::newSimpleThreadManager( + (size_t)FLAGS_worker_threads, 0)); auto threadFactory = boost::shared_ptr(new PosixThreadFactory()); thread_manager_->threadFactory(threadFactory); thread_manager_->start(); } -Status Dispatcher::add(std::shared_ptr task) { +Status Dispatcher::add(std::shared_ptr task) { try { thread_manager_->add(std_to_boost_shared_ptr(task), 0, 0); } catch (std::exception& e) { @@ -47,13 +54,54 @@ Status Dispatcher::add(std::shared_ptr task) { return Status(0, "OK"); } -std::shared_ptr Dispatcher::getThreadManager() { +Status Dispatcher::addService(std::shared_ptr service) { + if (service->hasRun()) { + return Status(1, "Cannot schedule a service twice"); + } + + auto thread = std::make_shared( + boost::bind(&InternalRunnable::run, &*service)); + service_threads_.push_back(thread); + services_.push_back(std::move(service)); + return Status(0, "OK"); +} + +InternalThreadManagerRef Dispatcher::getThreadManager() { return thread_manager_; } void Dispatcher::join() { thread_manager_->join(); } -ThreadManager::STATE Dispatcher::state() const { +void Dispatcher::joinServices() { + for (auto& thread : service_threads_) { + thread->join(); + } +} + +void Dispatcher::removeServices() { + for (const auto& service : services_) { + while (true) { + // Wait for each thread's entry point (enter) meaning the thread context + // was allocated and (run) was called by boost::thread started. + if (service->hasRun()) { + break; + } + // We only need to check if std::terminate is call very quickly after + // the boost::thread is created. + ::usleep(200); + } + } + + for (auto& thread : service_threads_) { + thread->interrupt(); + } + + // Deallocate services. + service_threads_.clear(); + services_.clear(); +} + +InternalThreadManager::STATE Dispatcher::state() const { return thread_manager_->state(); } diff --git a/osquery/dispatcher/dispatcher_tests.cpp b/osquery/dispatcher/dispatcher_tests.cpp index b319b68..409c8c7 100644 --- a/osquery/dispatcher/dispatcher_tests.cpp +++ b/osquery/dispatcher/dispatcher_tests.cpp @@ -24,11 +24,11 @@ TEST_F(DispatcherTests, test_singleton) { EXPECT_EQ(one.getThreadManager().get(), two.getThreadManager().get()); } -class TestRunnable : public apache::thrift::concurrency::Runnable { +class TestRunnable : public InternalRunnable { public: int* i; TestRunnable(int* i) : i(i) {} - virtual void run() { ++*i; } + virtual void enter() { ++*i; } }; TEST_F(DispatcherTests, test_add_work) { diff --git a/osquery/events/CMakeLists.txt b/osquery/events/CMakeLists.txt index 8dc154b..7f26f73 100644 --- a/osquery/events/CMakeLists.txt +++ b/osquery/events/CMakeLists.txt @@ -1,7 +1,8 @@ -ADD_OSQUERY_LIBRARY(osquery_events events.cpp - linux/inotify.cpp - linux/udev.cpp) +ADD_OSQUERY_LIBRARY(TRUE osquery_events events.cpp) +ADD_OSQUERY_LIBRARY(FALSE osquery_events_linux linux/inotify.cpp + linux/udev.cpp) + +ADD_OSQUERY_TEST(TRUE osquery_events_tests events_tests.cpp) +ADD_OSQUERY_TEST(TRUE osquery_events_database_tests events_database_tests.cpp) +ADD_OSQUERY_TEST(FALSE osquery_inotify_tests linux/inotify_tests.cpp) -ADD_OSQUERY_TEST(osquery_events_tests events_tests.cpp) -ADD_OSQUERY_TEST(osquery_events_database_tests events_database_tests.cpp) -ADD_OSQUERY_TEST(osquery_inotify_tests linux/inotify_tests.cpp) diff --git a/osquery/events/events.cpp b/osquery/events/events.cpp index 3d05ba0..e26e84a 100644 --- a/osquery/events/events.cpp +++ b/osquery/events/events.cpp @@ -25,14 +25,14 @@ namespace osquery { DEFINE_osquery_flag(bool, - event_pubsub, - true, - "Use (enable) the osquery eventing pub/sub."); + disable_events, + false, + "Disable osquery events pubsub"); DEFINE_osquery_flag(int32, - event_pubsub_expiry, + events_expiry, 86000, - "Expire (remove) recorded events after a timeout."); + "Timeout to expire event pubsub results"); const std::vector kEventTimeLists = { 1 * 60 * 60, // 1 hour @@ -40,7 +40,7 @@ const std::vector kEventTimeLists = { 10, // 10 seconds }; -void EventPublisherCore::fire(const EventContextRef& ec, EventTime time) { +void EventPublisherPlugin::fire(const EventContextRef& ec, EventTime time) { EventContextID ec_id; if (isEnding()) { @@ -73,9 +73,9 @@ void EventPublisherCore::fire(const EventContextRef& ec, EventTime time) { } } -std::vector EventSubscriberCore::getIndexes(EventTime start, - EventTime stop, - int list_key) { +std::vector EventSubscriberPlugin::getIndexes(EventTime start, + EventTime stop, + int list_key) { auto db = DBHandle::getInstance(); auto index_key = "indexes." + dbNamespace(); std::vector indexes; @@ -166,7 +166,7 @@ std::vector EventSubscriberCore::getIndexes(EventTime start, return indexes; } -Status EventSubscriberCore::expireIndexes( +Status EventSubscriberPlugin::expireIndexes( const std::string& list_type, const std::vector& indexes, const std::vector& expirations) { @@ -203,7 +203,7 @@ Status EventSubscriberCore::expireIndexes( return Status(0, "OK"); } -std::vector EventSubscriberCore::getRecords( +std::vector EventSubscriberPlugin::getRecords( const std::vector& indexes) { auto db = DBHandle::getInstance(); auto record_key = "records." + dbNamespace(); @@ -234,7 +234,7 @@ std::vector EventSubscriberCore::getRecords( return records; } -Status EventSubscriberCore::recordEvent(EventID& eid, EventTime time) { +Status EventSubscriberPlugin::recordEvent(EventID& eid, EventTime time) { Status status; auto db = DBHandle::getInstance(); std::string time_value = boost::lexical_cast(time); @@ -290,7 +290,7 @@ Status EventSubscriberCore::recordEvent(EventID& eid, EventTime time) { return Status(0, "OK"); } -EventID EventSubscriberCore::getEventID() { +EventID EventSubscriberPlugin::getEventID() { Status status; auto db = DBHandle::getInstance(); // First get an event ID from the meta key. @@ -317,7 +317,7 @@ EventID EventSubscriberCore::getEventID() { return eid_value; } -QueryData EventSubscriberCore::get(EventTime start, EventTime stop) { +QueryData EventSubscriberPlugin::get(EventTime start, EventTime stop) { QueryData results; Status status; @@ -359,7 +359,7 @@ QueryData EventSubscriberCore::get(EventTime start, EventTime stop) { return results; } -Status EventSubscriberCore::add(const Row& r, EventTime time) { +Status EventSubscriberPlugin::add(const Row& r, EventTime time) { Status status; std::shared_ptr db; @@ -402,39 +402,30 @@ Status EventFactory::run(EventPublisherID& type_id) { // Assume it can either make use of an entrypoint poller/selector or // take care of async callback registrations in setUp/configure/run // only once and handle event queueing/firing in callbacks. - auto event_pub = EventFactory::getInstance().getEventPublisher(type_id); - if (event_pub == nullptr) { - return Status(1, "No Event Type"); + EventPublisherRef publisher; + try { + publisher = EventFactory::getInstance().getEventPublisher(type_id); + } + catch (std::out_of_range& e) { + return Status(1, "No event type found"); } + VLOG(1) << "Starting event publisher runloop: " + type_id; + publisher->hasStarted(true); + auto status = Status(0, "OK"); - while (!event_pub->isEnding() && status.ok()) { + while (!publisher->isEnding() && status.ok()) { // Can optionally implement a global cooloff latency here. - status = event_pub->run(); + status = publisher->run(); ::usleep(20); } // The runloop status is not reflective of the event type's. - VLOG(1) << "Event publisher " << event_pub->type() << " has terminated"; + publisher->tearDown(); + VLOG(1) << "Event publisher " << publisher->type() << " runloop terminated"; return Status(0, "OK"); } -void EventFactory::end(bool should_end) { - auto& ef = EventFactory::getInstance(); - - for (const auto& publisher : ef.event_pubs_) { - publisher.second->shouldEnd(should_end); - } - - // Stop handling exceptions for the publisher threads. - for (const auto& thread : ef.threads_) { - thread->detach(); - } - - ::usleep(400); - ef.threads_.clear(); -} - // There's no reason for the event factory to keep multiple instances. EventFactory& EventFactory::getInstance() { static EventFactory ef; @@ -445,14 +436,14 @@ Status EventFactory::registerEventPublisher(const EventPublisherRef& pub) { auto& ef = EventFactory::getInstance(); auto type_id = pub->type(); - if (ef.getEventPublisher(type_id) != nullptr) { - // This is a duplicate type id? - return Status(1, "Duplicate Event Type"); + if (ef.event_pubs_.count(type_id) != 0) { + // This is a duplicate event publisher. + return Status(1, "Cannot register duplicate publisher type."); } if (!pub->setUp().ok()) { - // Only add the publisher if setUp was successful. - return Status(1, "SetUp failed."); + // Only start event loop if setUp succeeds. + return Status(1, "Event publisher setUp failed"); } ef.event_pubs_[type_id] = pub; @@ -477,44 +468,44 @@ Status EventFactory::addSubscription(EventPublisherID& type_id, Status EventFactory::addSubscription(EventPublisherID& type_id, const SubscriptionRef& subscription) { - auto event_pub = EventFactory::getInstance().getEventPublisher(type_id); - if (event_pub == nullptr) { - // Cannot create a Subscription for a missing type_id. - return Status(1, "No Event Type"); + EventPublisherRef publisher; + try { + publisher = getInstance().getEventPublisher(type_id); + } + catch (std::out_of_range& e) { + return Status(1, "No event type found"); } // The event factory is responsible for configuring the event types. - auto status = event_pub->addSubscription(subscription); - event_pub->configure(); + auto status = publisher->addSubscription(subscription); + publisher->configure(); return status; } size_t EventFactory::numSubscriptions(EventPublisherID& type_id) { - const auto& event_pub = - EventFactory::getInstance().getEventPublisher(type_id); - if (event_pub != nullptr) { - return event_pub->numSubscriptions(); + EventPublisherRef publisher; + try { + publisher = EventFactory::getInstance().getEventPublisher(type_id); } - return 0; + catch (std::out_of_range& e) { + return 0; + } + return publisher->numSubscriptions(); } EventPublisherRef EventFactory::getEventPublisher(EventPublisherID& type_id) { - auto& ef = EventFactory::getInstance(); - const auto& it = ef.event_pubs_.find(type_id); - if (it != ef.event_pubs_.end()) { - return ef.event_pubs_[type_id]; + if (getInstance().event_pubs_.count(type_id) == 0) { + LOG(ERROR) << "Requested unknown event publisher: " + type_id; } - return nullptr; + return getInstance().event_pubs_.at(type_id); } EventSubscriberRef EventFactory::getEventSubscriber( EventSubscriberID& name_id) { - auto& ef = EventFactory::getInstance(); - const auto& it = ef.event_subs_.find(name_id); - if (it != ef.event_subs_.end()) { - return ef.event_subs_[name_id]; + if (getInstance().event_subs_.count(name_id) == 0) { + LOG(ERROR) << "Requested unknown event subscriber: " + name_id; } - return nullptr; + return getInstance().event_subs_.at(name_id); } Status EventFactory::deregisterEventPublisher(const EventPublisherRef& pub) { @@ -523,43 +514,71 @@ Status EventFactory::deregisterEventPublisher(const EventPublisherRef& pub) { Status EventFactory::deregisterEventPublisher(EventPublisherID& type_id) { auto& ef = EventFactory::getInstance(); - const auto& it = ef.event_pubs_.find(type_id); - if (it == ef.event_pubs_.end()) { - return Status(1, "No Event Type registered"); + EventPublisherRef publisher; + try { + publisher = ef.getEventPublisher(type_id); + } + catch (std::out_of_range& e) { + return Status(1, "No event publisher to deregister."); } - ef.event_pubs_[type_id]->tearDown(); - ef.event_pubs_.erase(it); - return Status(0, "OK"); -} - -Status EventFactory::deregisterEventPublishers() { - auto& ef = EventFactory::getInstance(); - auto it = ef.event_pubs_.begin(); - for (; it != ef.event_pubs_.end(); it++) { - it->second->tearDown(); + publisher->isEnding(true); + if (!publisher->hasStarted()) { + // If a publisher's run loop was not started, call tearDown since + // the setUp happened at publisher registration time. + publisher->tearDown(); } - ef.event_pubs_.erase(ef.event_pubs_.begin(), ef.event_pubs_.end()); + ef.event_pubs_.erase(type_id); return Status(0, "OK"); } + +std::vector EventFactory::publisherTypes() { + std::vector types; + for (const auto& publisher : getInstance().event_pubs_) { + types.push_back(publisher.first); + } + return types; } -namespace osquery { -namespace registries { -void faucet(EventPublishers ets, EventSubscribers ems) { - if (!FLAGS_event_pubsub) { - // Invocation disabled eventing. - return; +std::vector EventFactory::subscriberNames() { + std::vector names; + for (const auto& subscriber : getInstance().event_subs_) { + names.push_back(subscriber.first); } - auto& ef = osquery::EventFactory::getInstance(); - for (const auto& event_pub : ets) { - ef.registerEventPublisher(event_pub.second); + return names; +} + +void EventFactory::end(bool join) { + auto& ef = EventFactory::getInstance(); + + // Call deregister on each publisher. + for (const auto& publisher : ef.publisherTypes()) { + deregisterEventPublisher(publisher); } - for (const auto& event_module : ems) { - ef.registerEventSubscriber(event_module.second); + // Stop handling exceptions for the publisher threads. + for (const auto& thread : ef.threads_) { + if (join) { + thread->join(); + } else { + thread->detach(); + } } + + ::usleep(400); + ef.threads_.clear(); } + +void attachEvents() { + const auto& publishers = Registry::all("event_publisher"); + for (const auto& publisher : publishers) { + EventFactory::registerEventPublisher(std::move(publisher.second)); + } + + const auto& subscribers = Registry::all("event_subscriber"); + for (const auto& subscriber : subscribers) { + EventFactory::registerEventSubscriber(std::move(subscriber.second)); + } } } diff --git a/osquery/events/events_tests.cpp b/osquery/events/events_tests.cpp index 7219ea3..d91f209 100644 --- a/osquery/events/events_tests.cpp +++ b/osquery/events/events_tests.cpp @@ -28,7 +28,7 @@ class EventsTests : public ::testing::Test { DBHandle::getInstanceAtPath(kTestingEventsDBPath); } - void TearDown() { EventFactory::deregisterEventPublishers(); } + void TearDown() { EventFactory::end(); } }; // The most basic event publisher uses useless Subscription/Event. @@ -70,22 +70,22 @@ TEST_F(EventsTests, test_event_pub) { } TEST_F(EventsTests, test_register_event_pub) { - // A caller may register an event type using the class template. - // This template class is equivilent to the reinterpret casting target. - auto status = EventFactory::registerEventPublisher(); - EXPECT_TRUE(status.ok()); + auto basic_pub = std::make_shared(); + auto status = EventFactory::registerEventPublisher(basic_pub); // This class is the SAME, there was no type override. - status = EventFactory::registerEventPublisher(); + auto another_basic_pub = std::make_shared(); + status = EventFactory::registerEventPublisher(another_basic_pub); EXPECT_FALSE(status.ok()); // This class is different but also uses different types! - status = EventFactory::registerEventPublisher(); + auto fake_pub = std::make_shared(); + status = EventFactory::registerEventPublisher(fake_pub); EXPECT_TRUE(status.ok()); // May also register the event_pub instance - auto pub = std::make_shared(); - status = EventFactory::registerEventPublisher(pub); + auto another_fake_pub = std::make_shared(); + status = EventFactory::registerEventPublisher(another_fake_pub); EXPECT_TRUE(status.ok()); } @@ -99,15 +99,41 @@ TEST_F(EventsTests, test_event_pub_types) { } TEST_F(EventsTests, test_create_event_pub) { - auto status = EventFactory::registerEventPublisher(); + auto pub = std::make_shared(); + auto status = EventFactory::registerEventPublisher(pub); EXPECT_TRUE(status.ok()); // Make sure only the first event type was recorded. EXPECT_EQ(EventFactory::numEventPublishers(), 1); } +class UniqueEventPublisher + : public EventPublisher { + DECLARE_PUBLISHER("unique"); +}; + +TEST_F(EventsTests, test_create_using_registry) { + // The events API uses attachEvents to move registry event publishers and + // subscribers into the events factory. + EXPECT_EQ(EventFactory::numEventPublishers(), 0); + attachEvents(); + + // Store the number of default event publishers (in core). + int default_publisher_count = EventFactory::numEventPublishers(); + + // Now add another registry item, but do not yet attach it. + auto UniqueEventPublisherRegistryItem = + Registry::add("event_publisher", "unique"); + EXPECT_EQ(EventFactory::numEventPublishers(), default_publisher_count); + + // Now attach and make sure it was added. + attachEvents(); + EXPECT_EQ(EventFactory::numEventPublishers(), default_publisher_count + 1); +} + TEST_F(EventsTests, test_create_subscription) { - EventFactory::registerEventPublisher(); + auto pub = std::make_shared(); + EventFactory::registerEventPublisher(pub); // Make sure a subscription cannot be added for a non-existent event type. // Note: It normally would not make sense to create a blank subscription. @@ -127,7 +153,8 @@ TEST_F(EventsTests, test_create_subscription) { TEST_F(EventsTests, test_multiple_subscriptions) { Status status; - EventFactory::registerEventPublisher(); + auto pub = std::make_shared(); + EventFactory::registerEventPublisher(pub); auto subscription = Subscription::create(); status = EventFactory::addSubscription("publisher", subscription); @@ -182,9 +209,10 @@ class TestEventPublisher }; TEST_F(EventsTests, test_create_custom_event_pub) { - auto status = EventFactory::registerEventPublisher(); + auto basic_pub = std::make_shared(); + EventFactory::registerEventPublisher(basic_pub); auto pub = std::make_shared(); - status = EventFactory::registerEventPublisher(pub); + auto status = EventFactory::registerEventPublisher(pub); // These event types have unique event type IDs EXPECT_TRUE(status.ok()); @@ -229,10 +257,7 @@ TEST_F(EventsTests, test_tear_down) { // Once more, now deregistering all event types. status = EventFactory::registerEventPublisher(pub); EXPECT_EQ(pub->getTestValue(), 3); - - status = EventFactory::deregisterEventPublishers(); - EXPECT_TRUE(status.ok()); - + EventFactory::end(); EXPECT_EQ(pub->getTestValue(), 4); // Make sure the factory state represented. diff --git a/osquery/events/linux/inotify.cpp b/osquery/events/linux/inotify.cpp index be25d45..f91a673 100644 --- a/osquery/events/linux/inotify.cpp +++ b/osquery/events/linux/inotify.cpp @@ -20,8 +20,6 @@ namespace osquery { -REGISTER_EVENTPUBLISHER(INotifyEventPublisher) - int kINotifyULatency = 200; static const uint32_t BUFFER_SIZE = (10 * ((sizeof(struct inotify_event)) + NAME_MAX + 1)); @@ -38,6 +36,8 @@ std::map kMaskActions = { {IN_OPEN, "OPENED"}, }; +REGISTER(INotifyEventPublisher, "event_publisher", "inotify"); + Status INotifyEventPublisher::setUp() { inotify_handle_ = ::inotify_init(); // If this does not work throw an exception. diff --git a/osquery/events/linux/inotify.h b/osquery/events/linux/inotify.h index 502f900..a0bbd32 100644 --- a/osquery/events/linux/inotify.h +++ b/osquery/events/linux/inotify.h @@ -96,7 +96,7 @@ typedef std::map DescriptorPathMap; */ class INotifyEventPublisher : public EventPublisher { - DECLARE_PUBLISHER("INotifyEventPublisher"); + DECLARE_PUBLISHER("inotify"); public: /// Create an `inotify` handle descriptor. diff --git a/osquery/events/linux/inotify_tests.cpp b/osquery/events/linux/inotify_tests.cpp index b51793e..d62e631 100644 --- a/osquery/events/linux/inotify_tests.cpp +++ b/osquery/events/linux/inotify_tests.cpp @@ -35,18 +35,26 @@ int kMaxEventLatency = 3000; class INotifyTests : public testing::Test { protected: void TearDown() { - EventFactory::deregisterEventPublishers(); + // End the event loops, and join on the threads. boost::filesystem::remove_all(kRealTestPath); boost::filesystem::remove_all(kRealTestDir); } void StartEventLoop() { event_pub_ = std::make_shared(); - EventFactory::registerEventPublisher(event_pub_); + auto status = EventFactory::registerEventPublisher(event_pub_); FILE* fd = fopen(kRealTestPath.c_str(), "w"); fclose(fd); + temp_thread_ = boost::thread(EventFactory::run, "inotify"); + } + + void StopEventLoop() { + while (!event_pub_->hasStarted()) { + ::usleep(20); + } - temp_thread_ = boost::thread(EventFactory::run, "INotifyEventPublisher"); + EventFactory::end(true); + temp_thread_.join(); } void SubscriptionAction(const std::string& path, @@ -56,7 +64,7 @@ class INotifyTests : public testing::Test { mc->path = path; mc->mask = mask; - EventFactory::addSubscription("INotifyEventPublisher", mc, ec); + EventFactory::addSubscription("inotify", mc, ec); } bool WaitForEvents(int max, int num_events = 0) { @@ -79,23 +87,20 @@ class INotifyTests : public testing::Test { fclose(fd); } - void EndEventLoop() { - EventFactory::end(); - event_pub_->tearDown(); - temp_thread_.join(); - EventFactory::end(false); - } - std::shared_ptr event_pub_; boost::thread temp_thread_; }; TEST_F(INotifyTests, test_register_event_pub) { - auto status = EventFactory::registerEventPublisher(); + auto pub = std::make_shared(); + auto status = EventFactory::registerEventPublisher(pub); EXPECT_TRUE(status.ok()); // Make sure only one event type exists EXPECT_EQ(EventFactory::numEventPublishers(), 1); + // And deregister + status = EventFactory::deregisterEventPublisher("inotify"); + EXPECT_TRUE(status.ok()); } TEST_F(INotifyTests, test_inotify_init) { @@ -104,44 +109,48 @@ TEST_F(INotifyTests, test_inotify_init) { EXPECT_FALSE(event_pub->isHandleOpen()); // Registering the event type initializes inotify. - EventFactory::registerEventPublisher(event_pub); + auto status = EventFactory::registerEventPublisher(event_pub); + EXPECT_TRUE(status.ok()); EXPECT_TRUE(event_pub->isHandleOpen()); // Similarly deregistering closes the handle. - EventFactory::deregisterEventPublishers(); + EventFactory::deregisterEventPublisher("inotify"); EXPECT_FALSE(event_pub->isHandleOpen()); } TEST_F(INotifyTests, test_inotify_add_subscription_missing_path) { - EventFactory::registerEventPublisher(); + auto pub = std::make_shared(); + EventFactory::registerEventPublisher(pub); // This subscription path is fake, and will succeed. auto mc = std::make_shared(); mc->path = "/this/path/is/fake"; auto subscription = Subscription::create(mc); - auto status = - EventFactory::addSubscription("INotifyEventPublisher", subscription); + auto status = EventFactory::addSubscription("inotify", subscription); EXPECT_TRUE(status.ok()); + EventFactory::deregisterEventPublisher("inotify"); } TEST_F(INotifyTests, test_inotify_add_subscription_success) { - EventFactory::registerEventPublisher(); + auto pub = std::make_shared(); + EventFactory::registerEventPublisher(pub); // This subscription path *should* be real. auto mc = std::make_shared(); mc->path = "/"; auto subscription = Subscription::create(mc); - auto status = - EventFactory::addSubscription("INotifyEventPublisher", subscription); + auto status = EventFactory::addSubscription("inotify", subscription); EXPECT_TRUE(status.ok()); + EventFactory::deregisterEventPublisher("inotify"); } TEST_F(INotifyTests, test_inotify_run) { // Assume event type is registered. event_pub_ = std::make_shared(); - EventFactory::registerEventPublisher(event_pub_); + auto status = EventFactory::registerEventPublisher(event_pub_); + EXPECT_TRUE(status.ok()); // Create a temporary file to watch, open writeable FILE* fd = fopen(kRealTestPath.c_str(), "w"); @@ -149,11 +158,11 @@ TEST_F(INotifyTests, test_inotify_run) { // Create a subscriptioning context auto mc = std::make_shared(); mc->path = kRealTestPath; - EventFactory::addSubscription("INotifyEventPublisher", - Subscription::create(mc)); + status = EventFactory::addSubscription("inotify", Subscription::create(mc)); + EXPECT_TRUE(status.ok()); // Create an event loop thread (similar to main) - boost::thread temp_thread(EventFactory::run, "INotifyEventPublisher"); + boost::thread temp_thread(EventFactory::run, "inotify"); EXPECT_TRUE(event_pub_->numEvents() == 0); // Cause an inotify event by writing to the watched path. @@ -166,12 +175,8 @@ TEST_F(INotifyTests, test_inotify_run) { // Wait for the thread's run loop to select. WaitForEvents(kMaxEventLatency); EXPECT_TRUE(event_pub_->numEvents() > 0); - - // Cause the thread to tear down. EventFactory::end(); temp_thread.join(); - // Reset the event factory state. - EventFactory::end(false); } class TestINotifyEventSubscriber @@ -239,9 +244,7 @@ TEST_F(INotifyTests, test_inotify_fire_event) { // Make sure our expected event fired (aka subscription callback was called). EXPECT_TRUE(sub->count() > 0); - - // Cause the thread to tear down. - EndEventLoop(); + StopEventLoop(); } TEST_F(INotifyTests, test_inotify_event_action) { @@ -259,15 +262,15 @@ TEST_F(INotifyTests, test_inotify_event_action) { // Make sure the inotify action was expected. EXPECT_TRUE(sub->actions().size() > 0); EXPECT_EQ(sub->actions()[0], "UPDATED"); - - // Cause the thread to tear down. - EndEventLoop(); +// EXPECT_EQ(sub->actions()[1], "OPENED"); +// EXPECT_EQ(sub->actions()[2], "UPDATED"); +// EXPECT_EQ(sub->actions()[3], "UPDATED"); + StopEventLoop(); } TEST_F(INotifyTests, test_inotify_optimization) { // Assume event type is registered. StartEventLoop(); - boost::filesystem::create_directory(kRealTestDir); // Adding a descriptor to a directory will monitor files within. @@ -278,9 +281,7 @@ TEST_F(INotifyTests, test_inotify_optimization) { // but this will NOT cause an additional INotify watch. SubscriptionAction(kRealTestDirPath); EXPECT_EQ(event_pub_->numDescriptors(), 1); - - // Cause the thread to tear down. - EndEventLoop(); + StopEventLoop(); } TEST_F(INotifyTests, test_inotify_recursion) { @@ -303,8 +304,7 @@ TEST_F(INotifyTests, test_inotify_recursion) { sub->WaitForEvents(kMaxEventLatency, 1); EXPECT_TRUE(sub->count() > 0); - - EndEventLoop(); + StopEventLoop(); } } diff --git a/osquery/events/linux/udev.cpp b/osquery/events/linux/udev.cpp index 81095f2..74d6e50 100644 --- a/osquery/events/linux/udev.cpp +++ b/osquery/events/linux/udev.cpp @@ -16,10 +16,10 @@ namespace osquery { -REGISTER_EVENTPUBLISHER(UdevEventPublisher); - int kUdevULatency = 200; +REGISTER(UdevEventPublisher, "event_publisher", "udev"); + Status UdevEventPublisher::setUp() { // Create the udev object. handle_ = udev_new(); @@ -37,6 +37,10 @@ Status UdevEventPublisher::setUp() { void UdevEventPublisher::configure() {} void UdevEventPublisher::tearDown() { + if (monitor_ != nullptr) { + udev_monitor_unref(monitor_); + } + if (handle_ != nullptr) { udev_unref(handle_); } diff --git a/osquery/events/linux/udev.h b/osquery/events/linux/udev.h index 1544a12..b289396 100644 --- a/osquery/events/linux/udev.h +++ b/osquery/events/linux/udev.h @@ -71,7 +71,7 @@ typedef std::shared_ptr UdevSubscriptionContextRef; */ class UdevEventPublisher : public EventPublisher { - DECLARE_PUBLISHER("UdevEventPublisher"); + DECLARE_PUBLISHER("udev"); public: Status setUp(); diff --git a/osquery/examples/example_extension.cpp b/osquery/examples/example_extension.cpp new file mode 100644 index 0000000..59be5f1 --- /dev/null +++ b/osquery/examples/example_extension.cpp @@ -0,0 +1,39 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ + +#include + +namespace osquery { + +class ExampleTable : public tables::TablePlugin { + private: + tables::TableColumns columns() { + return {{"example_text", "TEXT"}, {"example_integer", "INTEGER"}}; + } + + QueryData generate(tables::QueryContext& request) { + QueryData results; + + Row r; + r["example_text"] = "example"; + r["example_integer"] = INTEGER(1); + + results.push_back(r); + return results; + } +}; + +REGISTER(ExampleTable, "table", "example"); +} + +int main(int argc, char* argv[]) { + // Do some broadcast of the registry. + auto example = std::make_shared(); +} diff --git a/osquery/extensions/CMakeLists.txt b/osquery/extensions/CMakeLists.txt new file mode 100644 index 0000000..0521746 --- /dev/null +++ b/osquery/extensions/CMakeLists.txt @@ -0,0 +1,19 @@ +FIND_PROGRAM(THRIFT_COMPILER thrift /usr/local/bin + /usr/bin + NO_DEFAULT_PATH) + +# Generate the thrift intermediate/interface code. +ADD_CUSTOM_COMMAND( + COMMAND + ${THRIFT_COMPILER} --gen cpp:dense "${CMAKE_SOURCE_DIR}/osquery.thrift" + DEPENDS + "${CMAKE_SOURCE_DIR}/osquery.thrift" + WORKING_DIRECTORY + "${CMAKE_BINARY_DIR}/generated" + OUTPUT + ${OSQUERY_THRIFT_GENERATED_FILES}) + +ADD_OSQUERY_LIBRARY(TRUE osquery_extensions ${OSQUERY_THRIFT_GENERATED_FILES} + extensions.cpp) + +ADD_OSQUERY_TEST(TRUE osquery_extensions_test extensions_tests.cpp) diff --git a/osquery/extensions/extensions.cpp b/osquery/extensions/extensions.cpp new file mode 100644 index 0000000..afdaa9f --- /dev/null +++ b/osquery/extensions/extensions.cpp @@ -0,0 +1,352 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ + +#include +#include +#include +#include +#include + +#include +#include +#include + +using namespace apache::thrift; +using namespace apache::thrift::protocol; +using namespace apache::thrift::transport; +using namespace apache::thrift::server; + +using namespace osquery::extensions; + +namespace osquery { + +DEFINE_osquery_flag(bool, disable_extensions, false, "Disable extension API"); + +DEFINE_osquery_flag(string, + extensions_socket, + "/var/osquery/osquery.em", + "Path to the extensions UNIX domain socket") + +namespace extensions { + +void ExtensionHandler::ping(ExtensionStatus& _return) { + _return.code = ExtensionCode::EXT_SUCCESS; + _return.message = "pong"; +} + +void ExtensionHandler::call(ExtensionResponse& _return, + const std::string& registry, + const std::string& item, + const ExtensionPluginRequest& request) { + // Call will receive an extension or core's request to call the other's + // internal registry call. It is the ONLY actor that resolves registry + // item aliases. + auto local_item = Registry::getAlias(registry, item); + + PluginResponse response; + PluginRequest plugin_request; + for (const auto& request_item : request) { + plugin_request[request_item.first] = request_item.second; + } + + auto status = Registry::call(registry, local_item, request, response); + _return.status.code = status.getCode(); + _return.status.message = status.getMessage(); + + if (status.ok()) { + for (const auto& response_item : response) { + _return.response.push_back(response_item); + } + } +} + +void ExtensionManagerHandler::registerExtension( + ExtensionStatus& _return, + const InternalExtensionInfo& info, + const ExtensionRegistry& registry) { + if (exists(info.name)) { + LOG(WARNING) << "Refusing to register duplicate extension " << info.name; + _return.code = ExtensionCode::EXT_FAILED; + _return.message = "Duplicate extension registered"; + return; + } + + // Every call to registerExtension is assigned a new RouteUUID. + RouteUUID uuid = rand(); + LOG(INFO) << "Registering extension (" << info.name << ", " << uuid + << ", version=" << info.version << ", sdk=" << info.sdk_version + << ")"; + + if (!Registry::addBroadcast(uuid, registry).ok()) { + LOG(WARNING) << "Could not add extension (" << uuid + << ") broadcast to registry"; + _return.code = ExtensionCode::EXT_FAILED; + _return.message = "Failed adding registry broadcase"; + return; + } + + extensions_[uuid] = info; + _return.code = ExtensionCode::EXT_SUCCESS; + _return.message = "OK"; + _return.uuid = uuid; +} + +void ExtensionManagerHandler::deregisterExtension( + ExtensionStatus& _return, const ExtensionRouteUUID uuid) { + if (extensions_.count(uuid) == 0) { + _return.code = ExtensionCode::EXT_FAILED; + _return.message = "No extension UUID registered"; + return; + } + + Registry::removeBroadcast(uuid); + extensions_.erase(uuid); +} + +bool ExtensionManagerHandler::exists(const std::string& name) { + for (const auto& extension : extensions_) { + if (extension.second.name == name) { + return true; + } + } + return false; +} +} + +ExtensionRunner::~ExtensionRunner() { remove(path_); } + +void ExtensionRunner::enter() { + // Set the socket information for the extension manager. + auto socket_path = path_; + + // Create the thrift instances. + boost::shared_ptr handler(new ExtensionHandler()); + boost::shared_ptr processor(new ExtensionProcessor(handler)); + boost::shared_ptr serverTransport( + new TServerSocket(socket_path)); + boost::shared_ptr transportFactory( + new TBufferedTransportFactory()); + boost::shared_ptr protocolFactory( + new TBinaryProtocolFactory()); + + // Start the Thrift server's run loop. + try { + TSimpleServer server( + processor, serverTransport, transportFactory, protocolFactory); + server.serve(); + } catch (const std::exception& e) { + LOG(ERROR) << "Cannot start extension handler (" << socket_path << ") (" + << e.what() << ")"; + throw e; + } +} + +ExtensionManagerRunner::~ExtensionManagerRunner() { + // Remove the socket path. + remove(path_); +} + +void ExtensionManagerRunner::enter() { + // Set the socket information for the extension manager. + auto socket_path = path_; + + // Create the thrift instances. + boost::shared_ptr handler( + new ExtensionManagerHandler()); + boost::shared_ptr processor( + new ExtensionManagerProcessor(handler)); + boost::shared_ptr serverTransport( + new TServerSocket(socket_path)); + boost::shared_ptr transportFactory( + new TBufferedTransportFactory()); + boost::shared_ptr protocolFactory( + new TBinaryProtocolFactory()); + + // Start the Thrift server's run loop. + try { + TSimpleServer server( + processor, serverTransport, transportFactory, protocolFactory); + server.serve(); + } catch (const std::exception& e) { + LOG(WARNING) << "Extensions disabled: cannot start extension manager (" + << socket_path << ") (" << e.what() << ")"; + } +} + +void ExtensionWatcher::enter() { + // Watch the manager, if the socket is removed then the extension will die. + boost::shared_ptr socket(new TSocket(manager_path_)); + boost::shared_ptr transport(new TBufferedTransport(socket)); + boost::shared_ptr protocol(new TBinaryProtocol(transport)); + + // Open a long-lived client to the extension manager. + ExtensionManagerClient client(protocol); + transport->open(); + + ExtensionStatus status; + while (true) { + // Ping the extension manager until it goes down. + client.ping(status); + if (status.code != ExtensionCode::EXT_SUCCESS && fatal_) { + transport->close(); + exitFatal(); + } + interruptableSleep(interval_); + } + + // Code will never reach this socket close. + transport->close(); +} + +void ExtensionWatcher::exitFatal() { + // Exit the extension. + // Not yet implemented. +} + +#ifdef OSQUERY_EXTENSION_NAME +Status startExtension() { + // No assumptions about how the extensions logs, the first action is to + // start the extension's registry. + Registry::setUp(); + + auto status = startExtensionWatcher(FLAGS_extensions_socket, 3000, true); + if (status.ok()) { + status = startExtension(FLAGS_extensions_socket, + OSQUERY_EXTENSION_NAME, + OSQUERY_EXTENSION_VERSION, + OSQUERY_SDK_VERSION); + } + return status; +} +#endif + +Status startExtension(const std::string& manager_path, + const std::string& name, + const std::string& version, + const std::string& sdk_version) { + // Make sure the extension manager path exists, and is writable. + if (!pathExists(manager_path) || !isWritable(manager_path)) { + return Status(1, "Extension manager socket not availabe: " + manager_path); + } + + // Open a socket to the extension manager to register. + boost::shared_ptr socket(new TSocket(manager_path)); + boost::shared_ptr transport(new TBufferedTransport(socket)); + boost::shared_ptr protocol(new TBinaryProtocol(transport)); + + // The Registry broadcast is used as the ExtensionRegistry. + auto broadcast = Registry::getBroadcast(); + + InternalExtensionInfo info; + info.name = name; + info.version = version; + info.sdk_version = sdk_version; + + // Register the extension's registry broadcast with the manager. + ExtensionManagerClient client(protocol); + ExtensionStatus status; + try { + transport->open(); + client.registerExtension(status, info, broadcast); + transport->close(); + } + catch (const std::exception& e) { + return Status(1, "Extension register failed: " + std::string(e.what())); + } + + if (status.code != ExtensionCode::EXT_SUCCESS) { + return Status(status.code, status.message); + } + + // Start the extension's Thrift server + Dispatcher::getInstance().addService( + std::make_shared(manager_path, status.uuid)); + return Status(0, std::to_string(status.uuid)); +} + +Status callExtension(const RouteUUID uuid, + const std::string& registry, + const std::string& item, + const PluginRequest& request, + PluginResponse& response) { + // Not yet implemented. + return Status(0, "OK"); +} + +Status callExtension(const std::string& extension_path, + const std::string& registry, + const std::string& item, + const PluginRequest& request, + PluginResponse& response) { + // Make sure the extension manager path exists, and is writable. + if (!pathExists(extension_path) || !isWritable(extension_path)) { + return Status(1, "Extension socket not availabe: " + extension_path); + } + + // Open a socket to the extension manager to register. + boost::shared_ptr socket(new TSocket(extension_path)); + boost::shared_ptr transport(new TBufferedTransport(socket)); + boost::shared_ptr protocol(new TBinaryProtocol(transport)); + + ExtensionClient client(protocol); + ExtensionResponse ext_response; + try { + transport->open(); + client.call(ext_response, registry, item, request); + transport->close(); + } + catch (const std::exception& e) { + return Status(1, "Extension call failed: " + std::string(e.what())); + } + + if (ext_response.status.code == ExtensionCode::EXT_SUCCESS) { + for (const auto& item : ext_response.response) { + response.push_back(item); + } + } + return Status(ext_response.status.code, ext_response.status.message); +} + +Status startExtensionWatcher(const std::string& manager_path, + size_t interval, + bool fatal) { + // Make sure the extension manager path exists, and is writable. + if (!pathExists(manager_path) || !isWritable(manager_path)) { + return Status(1, "Extension manager socket not availabe: " + manager_path); + } + + // Start a extension manager watcher, if the manager dies, so should we. + Dispatcher::getInstance().addService( + std::make_shared(manager_path, interval, fatal)); + return Status(0, "OK"); +} + +Status startExtensionManager() { + return startExtensionManager(FLAGS_extensions_socket); +} + +Status startExtensionManager(const std::string& manager_path) { + // Check if the socket location exists. + if (pathExists(manager_path).ok()) { + if (!isWritable(manager_path).ok()) { + return Status(1, "Cannot write extension socket: " + manager_path); + } + + if (!remove(manager_path).ok()) { + return Status(1, "Cannot remove extension socket: " + manager_path); + } + } + + // Start the extension manager thread. + Dispatcher::getInstance().addService( + std::make_shared(manager_path)); + return Status(0, "OK"); +} +} diff --git a/osquery/extensions/extensions_tests.cpp b/osquery/extensions/extensions_tests.cpp new file mode 100644 index 0000000..6570fa2 --- /dev/null +++ b/osquery/extensions/extensions_tests.cpp @@ -0,0 +1,262 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ + +#include + +// GTest must come before the Thrift includes. +#include + +#include +#include +#include + +#include +#include + +using namespace apache::thrift; +using namespace apache::thrift::protocol; +using namespace apache::thrift::transport; + +using namespace osquery::extensions; + +namespace osquery { + +const int kDelayUS = 200; +const int kTimeoutUS = 10000; +const std::string kTestManagerSocket = "/tmp/osquery-em.socket"; + +class ExtensionsTest : public testing::Test { + protected: + void SetUp() { + remove(kTestManagerSocket); + if (pathExists(kTestManagerSocket).ok()) { + throw std::domain_error("Cannot test sockets: " + kTestManagerSocket); + } + } + + void TearDown() { + Dispatcher::getInstance().removeServices(); + remove(kTestManagerSocket); + } + + bool ping(int attempts = 3) { + // Open a socket to the test extension manager. + boost::shared_ptr socket(new TSocket(kTestManagerSocket)); + boost::shared_ptr transport(new TBufferedTransport(socket)); + boost::shared_ptr protocol(new TBinaryProtocol(transport)); + + ExtensionManagerClient client(protocol); + + // Calling open will except if the socket does not exist. + ExtensionStatus status; + for (int i = 0; i < attempts; ++i) { + try { + transport->open(); + client.ping(status); + transport->close(); + return (status.code == ExtensionCode::EXT_SUCCESS); + } + catch (const std::exception& e) { + ::usleep(kDelayUS); + } + } + + return false; + } + + ExtensionList registeredExtensions(int attempts = 3) { + // Open a socket to the test extension manager. + boost::shared_ptr socket(new TSocket(kTestManagerSocket)); + boost::shared_ptr transport(new TBufferedTransport(socket)); + boost::shared_ptr protocol(new TBinaryProtocol(transport)); + + ExtensionManagerClient client(protocol); + + // Calling open will except if the socket does not exist. + ExtensionList extensions; + for (int i = 0; i < attempts; ++i) { + try { + transport->open(); + client.extensions(extensions); + transport->close(); + } + catch (const std::exception& e) { + ::usleep(kDelayUS); + } + } + + return extensions; + } + + bool socketExists(const std::string& socket_path) { + // Wait until the runnable/thread created the socket. + int delay = 0; + while (delay < kTimeoutUS) { + if (pathExists(socket_path).ok() && isReadable(socket_path).ok()) { + return true; + } + ::usleep(kDelayUS); + delay += kDelayUS; + } + return false; + } +}; + +TEST_F(ExtensionsTest, test_manager_runnable) { + // Start a testing extension manager. + auto status = startExtensionManager(kTestManagerSocket); + EXPECT_TRUE(status.ok()); + // Call success if the Unix socket was created. + EXPECT_TRUE(socketExists(kTestManagerSocket)); +} + +TEST_F(ExtensionsTest, test_extension_runnable) { + auto status = startExtensionManager(kTestManagerSocket); + EXPECT_TRUE(status.ok()); + // Wait for the extension manager to start. + EXPECT_TRUE(socketExists(kTestManagerSocket)); + + // Test the extension manager API 'ping' call. + EXPECT_TRUE(ping()); +} + +TEST_F(ExtensionsTest, test_extension_start_failed) { + auto status = startExtensionManager(kTestManagerSocket); + EXPECT_TRUE(status.ok()); + // Wait for the extension manager to start. + EXPECT_TRUE(socketExists(kTestManagerSocket)); + + // Start an extension that does NOT fatal if the extension manager dies. + status = startExtension(kTestManagerSocket, "test", "0.1", "0.0.1"); + // This will be false since we are registering duplicate items + EXPECT_FALSE(status.ok()); +} + +TEST_F(ExtensionsTest, test_extension_start) { + auto status = startExtensionManager(kTestManagerSocket); + EXPECT_TRUE(status.ok()); + EXPECT_TRUE(socketExists(kTestManagerSocket)); + + // Now allow duplicates (for testing, since EM/E are the same). + Registry::allowDuplicates(true); + status = startExtension(kTestManagerSocket, "test", "0.1", "0.0.1"); + // This will be false since we are registering duplicate items + EXPECT_TRUE(status.ok()); + + // The `startExtension` internal call (exposed for testing) returns the + // uuid of the extension in the success status. + RouteUUID uuid; + try { + uuid = (RouteUUID)stoi(status.getMessage(), nullptr, 0); + } + catch (const std::exception& e) { + EXPECT_TRUE(false); + return; + } + + // We can test-wait for the extensions's socket to open. + EXPECT_TRUE(socketExists(kTestManagerSocket + "." + std::to_string(uuid))); + + // Then clean up the registry modifications. + Registry::removeBroadcast(uuid); + Registry::allowDuplicates(false); +} + +class ExtensionPlugin : public Plugin { + public: + Status call(const PluginRequest& request, PluginResponse& response) { + for (const auto& request_item : request) { + response.push_back({{request_item.first, request_item.second}}); + } + return Status(0, "Test sucess"); + } +}; + +class TestExtensionPlugin : public ExtensionPlugin {}; + +CREATE_REGISTRY(ExtensionPlugin, "extension_test"); + +TEST_F(ExtensionsTest, test_extension_broadcast) { + auto status = startExtensionManager(kTestManagerSocket); + EXPECT_TRUE(status.ok()); + EXPECT_TRUE(socketExists(kTestManagerSocket)); + + // This time we're going to add a plugin to the extension_test registry. + REGISTER(TestExtensionPlugin, "extension_test", "test_item"); + + // Now we create a registry alias that will be broadcasted but NOT used for + // internal call lookups. Aliasing was introduced for testing such that an + // EM/E could exist in the same process (the same registry) without having + // duplicate registry items in the internal registry list AND extension + // registry route table. + Registry::addAlias("extension_test", "test_item", "test_alias"); + Registry::allowDuplicates(true); + + // Before registering the extension there is NO route to "test_alias" since + // alias resolutions are performed by the EM. + EXPECT_TRUE(Registry::exists("extension_test", "test_item")); + EXPECT_FALSE(Registry::exists("extension_test", "test_alias")); + + status = startExtension(kTestManagerSocket, "test", "0.1", "0.0.1"); + EXPECT_TRUE(status.ok()); + + RouteUUID uuid; + try { + uuid = (RouteUUID)stoi(status.getMessage(), nullptr, 0); + } + catch (const std::exception& e) { + EXPECT_TRUE(false); + return; + } + + auto ext_socket = kTestManagerSocket + "." + std::to_string(uuid); + EXPECT_TRUE(socketExists(ext_socket)); + + // Make sure the EM registered the extension (called in start extension). + auto extensions = registeredExtensions(); + EXPECT_EQ(extensions.size(), 1); + EXPECT_EQ(extensions.count(uuid), 1); + EXPECT_EQ(extensions.at(uuid).name, "test"); + EXPECT_EQ(extensions.at(uuid).version, "0.1"); + EXPECT_EQ(extensions.at(uuid).sdk_version, "0.0.1"); + + // We are broadcasting to our own registry in the test, which internally has + // a "test_item" aliased to "test_alias", "test_item" is internally callable + // but "test_alias" can only be resolved by an EM call. + EXPECT_TRUE(Registry::exists("extension_test", "test_item")); + // Now "test_alias" exists since it is in the extensions route table. + EXPECT_TRUE(Registry::exists("extension_test", "test_alias")); + + PluginResponse response; + // This registry call will fail, since "test_alias" cannot be resolved using + // a local registry call. + status = Registry::call("extension_test", "test_alias", {{}}, response); + EXPECT_FALSE(status.ok()); + + // The following will be the result of a: + // Registry::call("extension_test", "test_alias", {{}}, response); + status = callExtension(ext_socket, + "extension_test", + "test_alias", + {{"test_key", "test_value"}}, + response); + EXPECT_TRUE(status.ok()); + EXPECT_EQ(response.size(), 1); + EXPECT_EQ(response[0]["test_key"], "test_value"); + + Registry::removeBroadcast(uuid); + Registry::allowDuplicates(false); +} +} + +int main(int argc, char* argv[]) { + testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/osquery/filesystem/CMakeLists.txt b/osquery/filesystem/CMakeLists.txt index c07d505..d971615 100644 --- a/osquery/filesystem/CMakeLists.txt +++ b/osquery/filesystem/CMakeLists.txt @@ -1,6 +1,6 @@ -ADD_OSQUERY_LIBRARY(osquery_filesystem filesystem.cpp) +ADD_OSQUERY_LIBRARY(TRUE osquery_filesystem filesystem.cpp) -ADD_OSQUERY_LIBRARY(osquery_filesystem_linux linux/proc.cpp - linux/mem.cpp) +ADD_OSQUERY_LIBRARY(TRUE osquery_filesystem_linux linux/proc.cpp + linux/mem.cpp) -ADD_OSQUERY_TEST(osquery_filesystem_tests filesystem_tests.cpp) +ADD_OSQUERY_TEST(TRUE osquery_filesystem_tests filesystem_tests.cpp) diff --git a/osquery/filesystem/filesystem.cpp b/osquery/filesystem/filesystem.cpp index f795963..585e341 100644 --- a/osquery/filesystem/filesystem.cpp +++ b/osquery/filesystem/filesystem.cpp @@ -3,25 +3,29 @@ * All rights reserved. * * This source code is licensed under the BSD-style license found in the - * LICENSE file in the root directory of this source tree. An additional grant + * LICENSE file in the root directory of this source tree. An additional grant * of patent rights can be found in the PATENTS file in the same directory. * */ #include #include +#include #include #include +#include #include #include #include #include #include +#include #include #include +#include namespace pt = boost::property_tree; namespace fs = boost::filesystem; @@ -36,20 +40,20 @@ Status writeTextFile(const boost::filesystem::path& path, int output_fd = open(path.c_str(), O_CREAT | O_APPEND | O_WRONLY, permissions); if (output_fd <= 0) { - return Status(1, "Could not create file"); + return Status(1, "Could not create file: " + path.string()); } // If the file existed with different permissions before our open // they must be restricted. if (chmod(path.c_str(), permissions) != 0) { // Could not change the file to the requested permissions. - return Status(1, "Failed to change permissions"); + return Status(1, "Failed to change permissions for file: " + path.string()); } auto bytes = write(output_fd, content.c_str(), content.size()); if (bytes != content.size()) { close(output_fd); - return Status(1, "Failed to write contents"); + return Status(1, "Failed to write contents to file: " + path.string()); } close(output_fd); @@ -62,25 +66,19 @@ Status readFile(const boost::filesystem::path& path, std::string& content) { return path_exists; } - int statusCode = 0; - std::string statusMessage = "OK"; std::stringstream buffer; - fs::ifstream file_h(path); if (file_h.is_open()) { buffer << file_h.rdbuf(); - if (file_h.bad()) { - statusCode = 1; - statusMessage = "Could not read file"; - } else - content.assign(std::move(buffer.str())); - + return Status(1, "Error reading file: " + path.string()); + } + content.assign(std::move(buffer.str())); } else { - statusCode = 1; - statusMessage = "Could not open file for reading"; + return Status(1, "Could not open file: " + path.string()); } - return Status(statusCode, statusMessage); + + return Status(0, "OK"); } Status isWritable(const boost::filesystem::path& path) { @@ -92,7 +90,7 @@ Status isWritable(const boost::filesystem::path& path) { if (access(path.c_str(), W_OK) == 0) { return Status(0, "OK"); } - return Status(1, "Path is not writable."); + return Status(1, "Path is not writable: " + path.string()); } Status isReadable(const boost::filesystem::path& path) { @@ -104,7 +102,7 @@ Status isReadable(const boost::filesystem::path& path) { if (access(path.c_str(), R_OK) == 0) { return Status(0, "OK"); } - return Status(1, "Path is not readable."); + return Status(1, "Path is not readable: " + path.string()); } Status pathExists(const boost::filesystem::path& path) { @@ -123,21 +121,59 @@ Status pathExists(const boost::filesystem::path& path) { return Status(0, "1"); } +Status remove(const boost::filesystem::path& path) { + auto status_code = std::remove(path.string().c_str()); + return Status(status_code, "N/A"); +} + Status listFilesInDirectory(const boost::filesystem::path& path, std::vector& results) { try { if (!boost::filesystem::exists(path)) { - return Status(1, "Directory not found"); + return Status(1, "Directory not found: " + path.string()); } if (!boost::filesystem::is_directory(path)) { - return Status(1, "Supplied path is not a directory"); + return Status(1, "Supplied path is not a directory: " + path.string()); + } + + boost::filesystem::directory_iterator begin_iter(path); + boost::filesystem::directory_iterator end_iter; + for (; begin_iter != end_iter; begin_iter++) { + if (!boost::filesystem::is_directory(begin_iter->path())) { + results.push_back(begin_iter->path().string()); + } + } + + return Status(0, "OK"); + } catch (const boost::filesystem::filesystem_error& e) { + return Status(1, e.what()); + } +} + +Status listDirectoriesInDirectory(const boost::filesystem::path& path, + std::vector& results) { + try { + if (!boost::filesystem::exists(path)) { + return Status(1, "Directory not found"); + } + + auto stat = pathExists(path); + if (!stat.ok()) { + return stat; + } + + stat = isDirectory(path); + if (!stat.ok()) { + return stat; } boost::filesystem::directory_iterator begin_iter(path); boost::filesystem::directory_iterator end_iter; for (; begin_iter != end_iter; begin_iter++) { - results.push_back(begin_iter->path().string()); + if (boost::filesystem::is_directory(begin_iter->path())) { + results.push_back(begin_iter->path().string()); + } } return Status(0, "OK"); @@ -146,6 +182,278 @@ Status listFilesInDirectory(const boost::filesystem::path& path, } } +/** + * @brief Drill down recursively and list all sub files + * + * This functions purpose is to take a path with no wildcards + * and it will recursively go through all files and and return + * them in the results vector. + * + * @param fs_path The entire resolved path + * @param results The vector where results will be returned + * @param rec_depth How many recursions deep the current execution is at + * + * @return An instance of osquery::Status indicating the success of failure of + * the operation + */ +Status doubleStarTraversal(const boost::filesystem::path& fs_path, + std::vector& results, + unsigned int rec_depth) { + if (rec_depth >= kMaxDirectoryTraversalDepth) { + return Status(2, fs_path.string().c_str()); + } + // List files first + Status stat = listFilesInDirectory(fs_path, results); + if (!stat.ok()) { + return Status(0, "OK"); + } + std::vector folders; + stat = listDirectoriesInDirectory(fs_path, folders); + if (!stat.ok()) { + return Status(0, "OK"); + } + + for (const auto& folder : folders) { + boost::filesystem::path p(folder); + if (boost::filesystem::is_symlink(p)) { + continue; + } + stat = doubleStarTraversal(folder, results, rec_depth + 1); + if (!stat.ok() && stat.getCode() == 2) { + return stat; + } + } + return Status(0, "OK"); +} + +/** + * @brief Resolve the last component of a file path + * + * This function exists because unlike the other parts of of a file + * path, which should only resolve to folder, a wildcard at the end + * means to list all files in that directory, as does just listing + * folder. Also, a double means to drill down recursively into that + * that folder and list all sub file. + * + * @param fs_path The entire resolved path (except last component) + * @param results The vector where results will be returned + * @param components A path, split by forward slashes + * @param rec_depth How many recursions deep the current execution is at + * + * @return An instance of osquery::Status indicating the success of failure of + * the operation + */ +Status resolveLastPathComponent(const boost::filesystem::path& fs_path, + std::vector& results, + const std::vector& components, + unsigned int rec_depth) { + // Is the last component a double star? + if (components[components.size() - 1] == kWildcardCharacterRecursive) { + Status stat = + doubleStarTraversal(fs_path.parent_path(), results, rec_depth); + return stat; + } + + // Is the path a file + try { + if (boost::filesystem::is_regular_file(fs_path)) { + results.push_back(fs_path.string()); + return Status(0, "OK"); + } + } catch (const boost::filesystem::filesystem_error& e) { + // This should catch permission problems + return Status(0, "OK"); + } + + std::vector files; + Status stat = listFilesInDirectory(fs_path.parent_path(), files); + if (!stat.ok()) { + return stat; + } + + // Is the last component a wildcard? + if (components[components.size() - 1] == kWildcardCharacter) { + for (const auto& file : files) { + results.push_back(file); + } + return Status(0, "OK"); + } + + std::string processed_path = + "/" + + boost::algorithm::join( + std::vector(components.begin(), components.end() - 1), + "/"); + + // Is this a .*% type file match + if (components[components.size() - 1].find(kWildcardCharacter, 1) != + std::string::npos && + components[components.size() - 1][0] != kWildcardCharacter[0]) { + + std::string prefix = + processed_path + "/" + + components[components.size() - 1].substr( + 0, components[components.size() - 1].find(kWildcardCharacter, 1)); + for (const auto& file : files) { + if (file.find(prefix, 0) != 0) { + continue; + } + results.push_back(file); + } + } + + // Is this a %(.*) type file match + if (components[components.size() - 1][0] == kWildcardCharacter[0]) { + std::string suffix = components[components.size() - 1].substr(1); + + for (const auto& file : files) { + boost::filesystem::path p(file); + std::string file_name = p.filename().string(); + size_t pos = file_name.find(suffix); + if (pos != std::string::npos && + pos + suffix.length() == file_name.length()) { + results.push_back(file); + } + } + return Status(0, "OK"); + } + + // Back out if this path doesn't exist due to invalid path + if (!(pathExists(fs_path).ok())) { + return Status(0, "OK"); + } + + // Is the path a directory + if (boost::filesystem::is_directory(fs_path)) { + for (auto& file : files) { + results.push_back(file); + } + return Status(0, "OK"); + } + + return Status(1, "UNKNOWN FILE TYPE"); +} + +/** + * @brief List all files in a directory recursively + * + * This is an overloaded version of the exported `resolveFilePattern`. This + * version is used internally to facilitate the tracking of the recursion + * depth. + * + * @param results The vector where results will be returned + * @param components A path, split by forward slashes + * @param processed_index What index of components has been resolved so far + * @param rec_depth How many recursions deep the current execution is at + * + * @return An instance of osquery::Status indicating the success of failure of + * the operation + */ +Status resolveFilePattern(std::vector components, + std::vector& results, + unsigned int processed_index = 0, + unsigned int rec_depth = 0) { + + // Stop recursing here if we've reached out max depth + if (rec_depth >= kMaxDirectoryTraversalDepth) { + return Status(2, "MAX_DEPTH"); + } + + // Handle all parts of the path except last because then we want to get files, + // not directories + for (auto i = processed_index; i < components.size() - 1; i++) { + + // If we encounter a full recursion, that is invalid because it is not + // the last component. So return. + if (components[i] == kWildcardCharacterRecursive) { + return Status(1, kWildcardCharacterRecursive + " NOT LAST COMPONENT"); + } + + // Create a vector to hold all the folders in the current folder + // Build the path we're at out of components + std::vector folders; + + std::string processed_path = + "/" + + boost::algorithm::join(std::vector(components.begin(), + components.begin() + i), + "/"); + Status stat = listDirectoriesInDirectory(processed_path, folders); + // If we couldn't list the directories it's probably because + // the path is invalid (or we don't have permission). Return + // here because this branch is no good. This is not an error + if (!stat.ok()) { + return Status(0, "OK"); + } + // If we just have a wildcard character then we will recurse though + // all folders we find + if (components[i] == kWildcardCharacter) { + for (const auto& dir : folders) { + boost::filesystem::path p(dir); + components[i] = p.filename().string(); + Status stat = + resolveFilePattern(components, results, i + 1, rec_depth + 1); + if (!stat.ok() && stat.getCode() == 2) { + return stat; + } + } + // Our subcalls that handle processing are now complete, return + return Status(0, "OK"); + + // The case of (.*)% + } else if (components[i].find(kWildcardCharacter, 1) != std::string::npos && + components[i][0] != kWildcardCharacter[0]) { + std::string prefix = + processed_path + "/" + + components[i].substr(0, components[i].find(kWildcardCharacter, 1)); + for (const auto& dir : folders) { + if (dir.find(prefix, 0) != 0) { + continue; + } + boost::filesystem::path p(dir); + components[i] = p.filename().string(); + Status stat = + resolveFilePattern(components, results, i + 1, rec_depth + 1); + if (!stat.ok() && stat.getCode() == 2) { + return stat; + } + } + return Status(0, "OK"); + // The case of %(.*) + } else if (components[i][0] == kWildcardCharacter[0]) { + std::string suffix = components[i].substr(1); + for (const auto& dir : folders) { + boost::filesystem::path p(dir); + std::string folder_name = p.filename().string(); + size_t pos = folder_name.find(suffix); + if (pos != std::string::npos && + pos + suffix.length() == folder_name.length()) { + components[i] = p.filename().string(); + Status stat = + resolveFilePattern(components, results, i + 1, rec_depth + 1); + if (!stat.ok() && stat.getCode() == 2) { + return stat; + } + } + } + return Status(0, "OK"); + } else { + } + } + + // At this point, all of our call paths have been resolved, so know we want to + // list the files at this point or do our ** traversal + return resolveLastPathComponent("/" + boost::algorithm::join(components, "/"), + results, + components, + rec_depth); +} + +Status resolveFilePattern(const boost::filesystem::path& fs_path, + std::vector& results) { + return resolveFilePattern(split(fs_path.string(), "/"), results); +} + Status getDirectory(const boost::filesystem::path& path, boost::filesystem::path& dirpath) { if (!isDirectory(path).ok()) { @@ -153,59 +461,28 @@ Status getDirectory(const boost::filesystem::path& path, return Status(0, "OK"); } dirpath = path; - return Status(1, "Path is a directory"); + return Status(1, "Path is a directory: " + path.string()); } Status isDirectory(const boost::filesystem::path& path) { if (boost::filesystem::is_directory(path)) { return Status(0, "OK"); } - return Status(1, "Path is not a directory"); -} - -Status parseTomcatUserConfigFromDisk( - const boost::filesystem::path& path, - std::vector >& credentials) { - std::string content; - auto s = readFile(path, content); - if (s.ok()) { - return parseTomcatUserConfig(content, credentials); - } else { - return s; - } + return Status(1, "Path is not a directory: " + path.string()); } -Status parseTomcatUserConfig( - const std::string& content, - std::vector >& credentials) { - std::stringstream ss; - ss << content; - pt::ptree tree; - try { - pt::xml_parser::read_xml(ss, tree); - } catch (const pt::xml_parser_error& e) { - return Status(1, e.what()); - } - try { - for (const auto& i : tree.get_child("tomcat-users")) { - if (i.first == "user") { - try { - std::pair user; - user.first = i.second.get(".username"); - user.second = i.second.get(".password"); - credentials.push_back(user); - } catch (const std::exception& e) { - LOG(ERROR) - << "An error occurred parsing the tomcat users xml: " << e.what(); - return Status(1, e.what()); - } - } +std::vector getHomeDirectories() { + auto sql = SQL( + "SELECT DISTINCT directory FROM users WHERE directory != '/var/empty';"); + std::vector results; + if (sql.ok()) { + for (const auto& row : sql.rows()) { + results.push_back(row.at("directory")); } - } catch (const std::exception& e) { - LOG(ERROR) << "An error occurred while trying to access the tomcat-users" - << " key in the XML content: " << e.what(); - return Status(1, e.what()); + } else { + LOG(ERROR) + << "Error executing query to return users: " << sql.getMessageString(); } - return Status(0, "OK"); + return results; } } diff --git a/osquery/filesystem/filesystem_tests.cpp b/osquery/filesystem/filesystem_tests.cpp index 2d8f47e..6516a8f 100644 --- a/osquery/filesystem/filesystem_tests.cpp +++ b/osquery/filesystem/filesystem_tests.cpp @@ -15,6 +15,7 @@ #include +#include #include #include @@ -24,34 +25,130 @@ namespace pt = boost::property_tree; namespace osquery { -class FilesystemTests : public testing::Test {}; +const std::string kFakeDirectory = "/tmp/osquery-fstests-pattern"; +const std::string kFakeFile = "/tmp/osquery-fstests-pattern/file0"; +const std::string kFakeSubFile = "/tmp/osquery-fstests-pattern/1/file1"; +const std::string kFakeSubSubFile = "/tmp/osquery-fstests-pattern/1/2/file2"; + +class FilesystemTests : public testing::Test { + + void createFileAt(const std::string loc, const std::string content) { + std::ofstream test_file(loc); + test_file.write(content.c_str(), sizeof("test123")); + test_file.close(); +} + + + protected: + void SetUp() { + boost::filesystem::create_directories(kFakeDirectory + "/deep11/deep2/deep3/"); + boost::filesystem::create_directories(kFakeDirectory + "/deep1/deep2/"); + + createFileAt(kFakeDirectory + "/root.txt", "root"); + createFileAt(kFakeDirectory + "/toor.txt", "toor"); + createFileAt(kFakeDirectory + "/roto.txt", "roto"); + createFileAt(kFakeDirectory + "/deep1/level1.txt", "l1"); + createFileAt(kFakeDirectory + "/deep11/not_bash", "l1"); + createFileAt(kFakeDirectory + "/deep1/deep2/level2.txt", "l2"); + + createFileAt(kFakeDirectory + "/deep11/level1.txt", "l1"); + createFileAt(kFakeDirectory + "/deep11/deep2/level2.txt", "l2"); + createFileAt(kFakeDirectory + "/deep11/deep2/deep3/level3.txt", "l3"); + } + + void TearDown() { boost::filesystem::remove_all(kFakeDirectory); } +}; TEST_F(FilesystemTests, test_plugin) { - std::ofstream test_file("/tmp/osquery-test-file"); + std::ofstream test_file("/tmp/osquery-fstests-file"); test_file.write("test123\n", sizeof("test123")); test_file.close(); std::string content; - auto s = readFile("/tmp/osquery-test-file", content); + auto s = readFile("/tmp/osquery-fstests-file", content); EXPECT_TRUE(s.ok()); EXPECT_EQ(s.toString(), "OK"); EXPECT_EQ(content, "test123\n"); - remove("/tmp/osquery-test-file"); + remove("/tmp/osquery-fstests-file"); } TEST_F(FilesystemTests, test_list_files_in_directory_not_found) { std::vector not_found_vector; auto not_found = listFilesInDirectory("/foo/bar", not_found_vector); EXPECT_FALSE(not_found.ok()); - EXPECT_EQ(not_found.toString(), "Directory not found"); + EXPECT_EQ(not_found.toString(), "Directory not found: /foo/bar"); +} +TEST_F(FilesystemTests, test_wildcard_single_folder_list) { + std::vector files; + auto status = resolveFilePattern(kFakeDirectory + "/%", files); + EXPECT_TRUE(status.ok()); + EXPECT_NE( + std::find(files.begin(), files.end(), kFakeDirectory + "/roto.txt"), + files.end()); +} + +TEST_F(FilesystemTests, test_wildcard_dual) { + std::vector files; + auto status = resolveFilePattern(kFakeDirectory + "/%/%", files); + EXPECT_TRUE(status.ok()); + EXPECT_NE(std::find(files.begin(), files.end(), + kFakeDirectory + "/deep1/level1.txt"), + files.end()); +} + +TEST_F(FilesystemTests, test_wildcard_full_recursion) { + std::vector files; + auto status = resolveFilePattern(kFakeDirectory + "/%%", files); + EXPECT_TRUE(status.ok()); + EXPECT_NE(std::find(files.begin(), files.end(), + kFakeDirectory + "/deep1/deep2/level2.txt"), + files.end()); +} + +TEST_F(FilesystemTests, test_wildcard_end_last_component) { + std::vector files; + auto status = resolveFilePattern(kFakeDirectory + "/%11/%sh", files); + EXPECT_TRUE(status.ok()); + EXPECT_NE(std::find(files.begin(), files.end(), + kFakeDirectory + "/deep11/not_bash"), + files.end()); +} + +TEST_F(FilesystemTests, test_wildcard_three_kinds) { + std::vector files; + auto status = resolveFilePattern(kFakeDirectory + "/%p11/%/%%", files); + EXPECT_TRUE(status.ok()); + EXPECT_NE(std::find(files.begin(), files.end(), + kFakeDirectory + "/deep11/deep2/deep3/level3.txt"), + files.end()); +} + +TEST_F(FilesystemTests, test_wildcard_invalid_path) { + std::vector files; + auto status = resolveFilePattern("/not_ther_abcdefz/%%", files); + EXPECT_TRUE(status.ok()); + EXPECT_EQ(files.size(), 0); +} + +TEST_F(FilesystemTests, test_wildcard_filewild) { + std::vector files; + auto status = resolveFilePattern(kFakeDirectory + "/deep1%/%", files); + EXPECT_TRUE(status.ok()); + EXPECT_NE(std::find(files.begin(), files.end(), + kFakeDirectory + "/deep1/level1.txt"), + files.end()); + EXPECT_NE(std::find(files.begin(), files.end(), + kFakeDirectory + "/deep11/level1.txt"), + files.end()); + boost::filesystem::remove_all(kFakeDirectory + ""); } TEST_F(FilesystemTests, test_list_files_in_directory_not_dir) { std::vector not_dir_vector; auto not_dir = listFilesInDirectory("/etc/hosts", not_dir_vector); EXPECT_FALSE(not_dir.ok()); - EXPECT_EQ(not_dir.toString(), "Supplied path is not a directory"); + EXPECT_EQ(not_dir.toString(), "Supplied path is not a directory: /etc/hosts"); } TEST_F(FilesystemTests, test_list_files_in_directorty) { @@ -62,50 +159,6 @@ TEST_F(FilesystemTests, test_list_files_in_directorty) { EXPECT_NE(std::find(results.begin(), results.end(), "/etc/hosts"), results.end()); } - -TEST_F(FilesystemTests, test_parse_tomcat_user_config) { - // clang-format off - std::string config_content = R"( - - - - - - - - -)"; - // clang-format on - - std::vector> credentials; - auto s = parseTomcatUserConfig(config_content, credentials); - EXPECT_TRUE(s.ok()); - EXPECT_EQ(s.toString(), "OK"); - EXPECT_EQ(credentials.size(), (size_t)1); - EXPECT_EQ(credentials[0].first, "tomcat"); - EXPECT_EQ(credentials[0].second, "tomcat"); -} } int main(int argc, char* argv[]) { diff --git a/osquery/filesystem/linux/mem.cpp b/osquery/filesystem/linux/mem.cpp index ce02e30..626c615 100644 --- a/osquery/filesystem/linux/mem.cpp +++ b/osquery/filesystem/linux/mem.cpp @@ -28,7 +28,7 @@ const std::string kLinuxMemPath = "/dev/mem"; DEFINE_osquery_flag(bool, disable_memory, false, - "Disable physical memory reads."); + "Disable physical memory reads"); Status readMem(int fd, size_t base, size_t length, uint8_t* buffer) { if (lseek(fd, base, SEEK_SET) == -1) { diff --git a/osquery/logger/CMakeLists.txt b/osquery/logger/CMakeLists.txt index 9402273..fbc7860 100644 --- a/osquery/logger/CMakeLists.txt +++ b/osquery/logger/CMakeLists.txt @@ -1,4 +1,4 @@ -ADD_OSQUERY_LIBRARY(osquery_logger logger.cpp - plugins/filesystem.cpp) +ADD_OSQUERY_LIBRARY(TRUE osquery_logger logger.cpp) +ADD_OSQUERY_LIBRARY(FALSE osquery_logger_plugins plugins/filesystem.cpp) -ADD_OSQUERY_TEST(osquery_logger_tests logger_tests.cpp) +ADD_OSQUERY_TEST(FALSE osquery_logger_tests logger_tests.cpp) diff --git a/osquery/logger/logger.cpp b/osquery/logger/logger.cpp index e34aae1..857a480 100644 --- a/osquery/logger/logger.cpp +++ b/osquery/logger/logger.cpp @@ -13,42 +13,46 @@ #include #include -#include - -using osquery::Status; namespace osquery { -/// `log_receiver` defines the default log receiver plugin name. +/// `logger` defines the default log receiver plugin name. DEFINE_osquery_flag(string, - log_receiver, + logger_plugin, "filesystem", - "The upstream log receiver to log messages to."); + "The default logger plugin"); DEFINE_osquery_flag(bool, log_result_events, true, - "Log scheduled results as events."); + "Log scheduled results as events"); + +Status LoggerPlugin::call(const PluginRequest& request, + PluginResponse& response) { + if (request.count("string") == 0) { + return Status(1, "Logger plugins only support a request string"); + } + + this->logString(request.at("string")); + return Status(0, "OK"); +} Status logString(const std::string& s) { - return logString(s, FLAGS_log_receiver); + return logString(s, FLAGS_logger_plugin); } Status logString(const std::string& s, const std::string& receiver) { - if (REGISTERED_LOGGER_PLUGINS.find(receiver) == - REGISTERED_LOGGER_PLUGINS.end()) { + if (!Registry::exists("logger", receiver)) { LOG(ERROR) << "Logger receiver " << receiver << " not found"; return Status(1, "Logger receiver not found"); } - auto log_status = REGISTERED_LOGGER_PLUGINS.at(receiver)->logString(s); - if (!log_status.ok()) { - return log_status; - } + + auto status = Registry::call("logger", receiver, {{"string", s}}); return Status(0, "OK"); } Status logScheduledQueryLogItem(const osquery::ScheduledQueryLogItem& results) { - return logScheduledQueryLogItem(results, FLAGS_log_receiver); + return logScheduledQueryLogItem(results, FLAGS_logger_plugin); } Status logScheduledQueryLogItem(const osquery::ScheduledQueryLogItem& results, diff --git a/osquery/logger/logger_tests.cpp b/osquery/logger/logger_tests.cpp index adaa945..3f4dd9d 100644 --- a/osquery/logger/logger_tests.cpp +++ b/osquery/logger/logger_tests.cpp @@ -12,15 +12,12 @@ #include #include -#include - -using osquery::Status; namespace osquery { class LoggerTests : public testing::Test { public: - LoggerTests() { osquery::InitRegistry::get().run(); } + LoggerTests() { Registry::setUp(); } }; class TestLoggerPlugin : public LoggerPlugin { @@ -32,12 +29,10 @@ class TestLoggerPlugin : public LoggerPlugin { virtual ~TestLoggerPlugin() {} }; -REGISTER_LOGGER_PLUGIN("test", std::make_shared()) - TEST_F(LoggerTests, test_plugin) { - auto s = REGISTERED_LOGGER_PLUGINS.at("test")->logString("foobar"); + Registry::add("logger", "test"); + auto s = Registry::call("logger", "test", {{"string", "foobar"}}); EXPECT_EQ(s.ok(), true); - EXPECT_EQ(s.toString(), "foobar"); } } diff --git a/osquery/logger/plugins/filesystem.cpp b/osquery/logger/plugins/filesystem.cpp index 8eeefdb..4e50ccb 100644 --- a/osquery/logger/plugins/filesystem.cpp +++ b/osquery/logger/plugins/filesystem.cpp @@ -14,7 +14,6 @@ #include #include #include -#include using osquery::Status; @@ -24,28 +23,42 @@ std::mutex filesystemLoggerPluginMutex; class FilesystemLoggerPlugin : public LoggerPlugin { public: - std::string log_path; - FilesystemLoggerPlugin() { - log_path = FLAGS_log_dir + "osqueryd.results.log"; - } + Status setUp(); + Status logString(const std::string& s); + + private: + std::string log_path_; +}; + +REGISTER(FilesystemLoggerPlugin, "logger", "filesystem"); + +Status FilesystemLoggerPlugin::setUp() { + log_path_ = FLAGS_log_dir + "osqueryd.results.log"; + return Status(0, "OK"); +} - virtual Status logString(const std::string& s) { - std::lock_guard lock(filesystemLoggerPluginMutex); - try { - VLOG(3) << "filesystem logger plugin: logging to " << log_path; - - // The results log may contain sensitive information if run as root. - auto status = writeTextFile(log_path, s, 0640, true); - if (!status.ok()) { - return status; - } - } catch (const std::exception& e) { - return Status(1, e.what()); +Status FilesystemLoggerPlugin::logString(const std::string& s) { + std::lock_guard lock(filesystemLoggerPluginMutex); + try { + VLOG(3) << "filesystem logger plugin: logging to " << log_path_; + + // The results log may contain sensitive information if run as root. + auto status = writeTextFile(log_path_, s, 0640, true); + if (!status.ok()) { + return status; } - return Status(0, "OK"); + } catch (const std::exception& e) { + return Status(1, e.what()); } -}; + return Status(0, "OK"); +} -REGISTER_LOGGER_PLUGIN("filesystem", - std::make_shared()) +class TestLoggerPlugin : public LoggerPlugin { + public: + TestLoggerPlugin() { test_ = "hello friend"; } + Status logString(const std::string& s) { return Status(0, "OK"); } + + private: + std::string test_; +}; } diff --git a/osquery/main/daemon.cpp b/osquery/main/daemon.cpp index c2aa818..5000ce8 100644 --- a/osquery/main/daemon.cpp +++ b/osquery/main/daemon.cpp @@ -11,92 +11,39 @@ #include #include -#include #include -#include #include +#include #include -#include #include -#ifndef __APPLE__ -namespace osquery { -DEFINE_osquery_flag(bool, daemonize, false, "Run as daemon (osqueryd only)."); -} -#endif +#include "osquery/core/watcher.h" -namespace osquery { -DEFINE_osquery_flag(bool, - config_check, - false, - "Check the format and accessibility of the daemon"); -} +const std::string kWatcherWorkerName = "osqueryd: worker"; int main(int argc, char* argv[]) { osquery::initOsquery(argc, argv, osquery::OSQUERY_TOOL_DAEMON); - if (osquery::FLAGS_config_check) { - auto s = osquery::Config::checkConfig(); - if (!s.ok()) { - std::cerr << "Error reading config: " << s.toString() << "\n"; - } - return s.getCode(); - } - -#ifndef __APPLE__ - // OSX uses launchd to daemonize. - if (osquery::FLAGS_daemonize) { - if (daemon(0, 0) == -1) { - ::exit(EXIT_FAILURE); - } - } -#endif - - auto pid_status = osquery::createPidFile(); - if (!pid_status.ok()) { - LOG(ERROR) << "Could not start osqueryd: " << pid_status.toString(); - ::exit(EXIT_FAILURE); - } - - try { - osquery::DBHandle::getInstance(); - } catch (std::exception& e) { - LOG(ERROR) << "osqueryd failed to start: " << e.what(); - ::exit(EXIT_FAILURE); - } - - LOG(INFO) << "Listing all plugins"; - - LOG(INFO) << "Logger plugins:"; - for (const auto& it : REGISTERED_LOGGER_PLUGINS) { - LOG(INFO) << " - " << it.first; - } - - LOG(INFO) << "Config plugins:"; - for (const auto& it : REGISTERED_CONFIG_PLUGINS) { - LOG(INFO) << " - " << it.first; - } - - LOG(INFO) << "Event Publishers:"; - for (const auto& it : REGISTERED_EVENTPUBLISHERS) { - LOG(INFO) << " - " << it.first; + if (!osquery::isOsqueryWorker()) { + osquery::initOsqueryDaemon(); } - LOG(INFO) << "Event Subscribers:"; - for (const auto& it : REGISTERED_EVENTSUBSCRIBERS) { - LOG(INFO) << " - " << it.first; + if (!osquery::FLAGS_disable_watchdog) { + // When a watcher is used, the current watcher will fork into a worker. + osquery::initWorkerWatcher(kWatcherWorkerName, argc, argv); } - // Start a thread for each appropriate event type - osquery::registries::faucet(REGISTERED_EVENTPUBLISHERS, - REGISTERED_EVENTSUBSCRIBERS); + // Start event threads. + osquery::attachEvents(); osquery::EventFactory::delay(); + osquery::startExtensionManager(); + // Begin the schedule runloop. boost::thread scheduler_thread(osquery::initializeScheduler); scheduler_thread.join(); - // End any event type run loops. - osquery::EventFactory::end(); + // Finally shutdown. + osquery::shutdownOsquery(); return 0; } diff --git a/osquery/main/run.cpp b/osquery/main/run.cpp index 0baa745..545d911 100644 --- a/osquery/main/run.cpp +++ b/osquery/main/run.cpp @@ -8,41 +8,57 @@ * */ +#include + #include #include +#include #include +#include DEFINE_string(query, "", "query to execute"); DEFINE_int32(iterations, 1, "times to run the query in question"); DEFINE_int32(delay, 0, "delay before and after the query"); int main(int argc, char* argv[]) { - osquery::initOsquery(argc, argv); + // Only log to stderr + FLAGS_logtostderr = true; - if (FLAGS_query != "") { - if (FLAGS_delay != 0) { - ::sleep(FLAGS_delay); - } + // Let gflags parse the non-help options/flags. + __GFLAGS_NAMESPACE::ParseCommandLineFlags(&argc, &argv, false); + __GFLAGS_NAMESPACE::InitGoogleLogging(argv[0]); - for (int i = 0; i < FLAGS_iterations; ++i) { - int err; - LOG(INFO) << "Executing: " << FLAGS_query; - osquery::query(FLAGS_query, err); - if (err != 0) { - LOG(ERROR) << "Query failed: " << err; - return 1; - } - LOG(INFO) << "Query succeeded"; - } + if (FLAGS_query == "") { + fprintf(stderr, "Usage: %s --query=\"query\"\n", argv[0]); + return 1; + } - if (FLAGS_delay != 0) { - ::sleep(FLAGS_delay); + osquery::Registry::setUp(); + osquery::attachEvents(); + + int result = 0; + if (FLAGS_delay != 0) { + ::sleep(FLAGS_delay); + } + + osquery::QueryData results; + for (int i = 0; i < FLAGS_iterations; ++i) { + printf("Executing: %s\n", FLAGS_query.c_str()); + auto status = osquery::query(FLAGS_query, results); + if (!status.ok()) { + fprintf(stderr, "Query failed: %d\n", status.getCode()); + break; + } else { + if (FLAGS_delay != 0) { + ::sleep(FLAGS_delay); + } } - } else { - LOG(ERROR) << "Usage: run --query=\"\""; - return 1; } - return 0; + // Instead of calling "shutdownOsquery" force the EF to join its threads. + osquery::EventFactory::end(true); + __GFLAGS_NAMESPACE::ShutDownCommandLineFlags(); + + return result; } diff --git a/osquery/main/shell.cpp b/osquery/main/shell.cpp index c6d3b61..f7df57e 100644 --- a/osquery/main/shell.cpp +++ b/osquery/main/shell.cpp @@ -7,24 +7,40 @@ * of patent rights can be found in the PATENTS file in the same directory. * */ - + +#include + #include #include #include #include +#include +#include +#include + +const std::string kShellTemp = "/tmp/osquery"; int main(int argc, char *argv[]) { - osquery::FLAGS_db_path = "/tmp/rocksdb-osquery-shell"; + // The shell is transient, rewrite config-loaded paths. + if (osquery::pathExists(kShellTemp).ok() || + boost::filesystem::create_directory(kShellTemp)) { + osquery::FLAGS_db_path = kShellTemp + "/shell.db"; + osquery::FLAGS_extensions_socket = kShellTemp + "/shell.em"; + FLAGS_log_dir = kShellTemp; + } + + // Parse/apply flags, start registry, load logger/config plugins. osquery::initOsquery(argc, argv, osquery::OSQUERY_TOOL_SHELL); - // Start a thread for each appropriate event type - osquery::registries::faucet(REGISTERED_EVENTPUBLISHERS, - REGISTERED_EVENTSUBSCRIBERS); + // Start event threads. + osquery::attachEvents(); osquery::EventFactory::delay(); + osquery::startExtensionManager(); + // Virtual tables will be attached to the shell's in-memory SQLite DB. int retcode = osquery::launchIntoShell(argc, argv); - // End any event type threads. - osquery::EventFactory::end(); + // Finally shutdown. + osquery::shutdownOsquery(); return retcode; } diff --git a/osquery/registry/CMakeLists.txt b/osquery/registry/CMakeLists.txt index d155dea..62ac9a8 100644 --- a/osquery/registry/CMakeLists.txt +++ b/osquery/registry/CMakeLists.txt @@ -1,3 +1,3 @@ -ADD_OSQUERY_LIBRARY(osquery_registry registry.cpp) +ADD_OSQUERY_LIBRARY(TRUE osquery_registry registry.cpp) -ADD_OSQUERY_TEST(osquery_registry_tests registry_tests.cpp) +ADD_OSQUERY_TEST(TRUE osquery_registry_tests registry_tests.cpp) diff --git a/osquery/registry/init_registry.h b/osquery/registry/init_registry.h deleted file mode 100644 index af3ba2a..0000000 --- a/osquery/registry/init_registry.h +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Copyright (c) 2014, Facebook, Inc. - * All rights reserved. - * - * This source code is licensed under the BSD-style license found in the - * LICENSE file in the root directory of this source tree. An additional grant - * of patent rights can be found in the PATENTS file in the same directory. - * - */ - -#pragma once - -#include - -#include "osquery/registry/registry_template.h" - -namespace osquery { - -class InitRegistry : public RegistryTemplate<> { - public: - /** - * Singleton. - */ - static InitRegistry& get() { - static InitRegistry instance; // Thread-safe. - return instance; - } - - private: - InitRegistry() {} -}; - -/** - * The most standard way to register an init func is to use this class - * as a static instance in some file to ensure the function gets called - * once main actually begins. - * - * Examples: - * - * static RegisterInitFunc reg1(&setupFancyTable); - * static RegisterInitFunc reg2(std::bind(&setupSomething, 10, 15)); - * static RegisterInitFunc reg3([] { setupSomethingElse(20, 25); }); - * - */ -struct RegisterInitFunc : private boost::noncopyable { - explicit RegisterInitFunc(InitRegistry::Func func, - int priority = InitRegistry::kDefaultPriority) { - InitRegistry::get().registerFunc(std::move(func), priority); - } -}; - -} // namespace osquery diff --git a/osquery/registry/registry.cpp b/osquery/registry/registry.cpp new file mode 100644 index 0000000..704ec4d --- /dev/null +++ b/osquery/registry/registry.cpp @@ -0,0 +1,286 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ + +#include +#include + +#include + +#include + +namespace osquery { + +void RegistryHelperCore::remove(const std::string& item_name) { + if (items_.count(item_name) > 0) { + items_[item_name]->tearDown(); + items_.erase(item_name); + } + + // Populate list of aliases to remove (those that mask item_name). + std::vector removed_aliases; + for (const auto& alias : aliases_) { + if (alias.second == item_name) { + removed_aliases.push_back(alias.first); + } + } + + for (const auto& alias : removed_aliases) { + aliases_.erase(alias); + } +} + +RegistryRoutes RegistryHelperCore::getRoutes() const { + RegistryRoutes route_table; + for (const auto& item : items_) { + bool has_alias = false; + for (const auto& alias : aliases_) { + if (alias.second == item.first) { + // If the item name is masked by at least one alias, it will not + // broadcast under the internal item name. + route_table[alias.first] = item.second->routeInfo(); + has_alias = true; + } + } + + if (!has_alias) { + route_table[item.first] = item.second->routeInfo(); + } + } + return route_table; +} + +Status RegistryHelperCore::call(const std::string& item_name, + const PluginRequest& request, + PluginResponse& response) { + if (items_.count(item_name) > 0) { + return items_[item_name]->call(request, response); + } + return Status(1, "Cannot call registry item: " + item_name); +} + +Status RegistryHelperCore::addAlias(const std::string& item_name, + const std::string& alias) { + if (aliases_.count(alias) > 0) { + return Status(1, "Duplicate alias: " + alias); + } + aliases_[alias] = item_name; + return Status(0, "OK"); +} + +const std::string& RegistryHelperCore::getAlias( + const std::string& alias) const { + if (aliases_.count(alias) == 0) { + return alias; + } + return aliases_.at(alias); +} + +void RegistryHelperCore::setUp() { + // If this registry does not auto-setup do NOT setup the registry items. + if (!auto_setup_) { + return; + } + + // Try to set up each of the registry items. + // If they fail, remove them from the registry. + std::vector failed; + for (auto& item : items_) { + if (!item.second->setUp().ok()) { + failed.push_back(item.first); + } + } + + for (const auto& failed_item : failed) { + remove(failed_item); + } +} + +/// Facility method to check if a registry item exists. +bool RegistryHelperCore::exists(const std::string& item_name) const { + return (items_.count(item_name) > 0); +} + +/// Facility method to list the registry item identifiers. +std::vector RegistryHelperCore::names() const { + std::vector names; + for (const auto& item : items_) { + names.push_back(item.first); + } + return names; +} + +/// Facility method to count the number of items in this registry. +size_t RegistryHelperCore::count() const { return items_.size(); } + +/// Allow the registry to introspect into the registered name (for logging). +void RegistryHelperCore::setName(const std::string& name) { name_ = name; } + +const std::map& RegistryFactory::all() { + return instance().registries_; +} + +PluginRegistryHelperRef RegistryFactory::registry( + const std::string& registry_name) { + return instance().registries_.at(registry_name); +} + +const std::map RegistryFactory::all( + const std::string& registry_name) { + return instance().registry(registry_name)->all(); +} + +PluginRef RegistryFactory::get(const std::string& registry_name, + const std::string& item_name) { + return instance().registry(registry_name)->get(item_name); +} + +RegistryBroadcast RegistryFactory::getBroadcast() { + RegistryBroadcast broadcast; + for (const auto& registry : instance().registries_) { + broadcast[registry.first] = registry.second->getRoutes(); + } + return broadcast; +} + +Status RegistryFactory::addBroadcast(const RouteUUID& uuid, + const RegistryBroadcast& broadcast) { + if (instance().extensions_.count(uuid) > 0) { + return Status(1, "Duplicate extension UUID: " + std::to_string(uuid)); + } + + // Make sure the extension does not broadcast conflicting registry items. + if (!Registry::allowDuplicates()) { + for (const auto& registry : broadcast) { + for (const auto& item : registry.second) { + if (Registry::exists(registry.first, item.first)) { + return Status(1, "Duplicate registry item: " + item.first); + } + } + } + } + + instance().extensions_[uuid] = broadcast; + return Status(0, "OK"); +} + +Status RegistryFactory::removeBroadcast(const RouteUUID& uuid) { + if (instance().extensions_.count(uuid) == 0) { + return Status(1, "Unknown extension UUID: " + std::to_string(uuid)); + } + + instance().extensions_.erase(uuid); + return Status(0, "OK"); +} + +/// Adds an alias for an internal registry item. This registry will only +/// broadcast the alias name. +Status RegistryFactory::addAlias(const std::string& registry_name, + const std::string& item_name, + const std::string& alias) { + if (instance().registries_.count(registry_name) == 0) { + return Status(1, "Unknown registry: " + registry_name); + } + return instance().registries_.at(registry_name)->addAlias(item_name, alias); +} + +/// Returns the item_name or the item alias if an alias exists. +const std::string& RegistryFactory::getAlias(const std::string& registry_name, + const std::string& alias) { + if (instance().registries_.count(registry_name) == 0) { + return alias; + } + return instance().registries_.at(registry_name)->getAlias(alias); +} + +Status RegistryFactory::call(const std::string& registry_name, + const std::string& item_name, + const PluginRequest& request, + PluginResponse& response) { + if (instance().registries_.count(registry_name) == 0) { + return Status(1, "Unknown registry: " + registry_name); + } + return instance().registries_[registry_name]->call( + item_name, request, response); +} + +Status RegistryFactory::call(const std::string& registry_name, + const std::string& item_name, + const PluginRequest& request) { + PluginResponse response; + return call(registry_name, item_name, request, response); +} + +void RegistryFactory::setUp() { + for (const auto& registry : instance().all()) { + registry.second->setUp(); + } +} + +bool RegistryFactory::exists(const std::string& registry_name, + const std::string& item_name) { + if (instance().registries_.count(registry_name) == 0) { + return false; + } + + // Check the local registry. + if (instance().registry(registry_name)->exists(item_name)) { + return true; + } + + // Check the known extension registries. + for (const auto& extension : instance().extensions_) { + if (extension.second.count(registry_name) > 0) { + if (extension.second.at(registry_name).count(item_name) > 0) { + return true; + } + } + } + + return false; +} + +std::vector RegistryFactory::names( + const std::string& registry_name) { + if (instance().registries_.at(registry_name) == 0) { + std::vector names; + return names; + } + return instance().registry(registry_name)->names(); +} + +size_t RegistryFactory::count() { return instance().registries_.size(); } + +size_t RegistryFactory::count(const std::string& registry_name) { + if (instance().registries_.count(registry_name) == 0) { + return 0; + } + return instance().registry(registry_name)->count(); +} + +void Plugin::getResponse(const std::string& key, + const PluginResponse& response, + boost::property_tree::ptree& tree) { + for (const auto& item : response) { + boost::property_tree::ptree child; + for (const auto& item_detail : item) { + child.put(item_detail.first, item_detail.second); + } + tree.add_child(key, child); + } +} + +void Plugin::setResponse(const std::string& key, + const boost::property_tree::ptree& tree, + PluginResponse& response) { + std::ostringstream output; + boost::property_tree::write_json(output, tree, false); + response.push_back({{key, output.str()}}); +} +} diff --git a/osquery/registry/registry_template.h b/osquery/registry/registry_template.h deleted file mode 100644 index de413b6..0000000 --- a/osquery/registry/registry_template.h +++ /dev/null @@ -1,77 +0,0 @@ -/* - * Copyright (c) 2014, Facebook, Inc. - * All rights reserved. - * - * This source code is licensed under the BSD-style license found in the - * LICENSE file in the root directory of this source tree. An additional grant - * of patent rights can be found in the PATENTS file in the same directory. - * - */ - -#pragma once - -#include -#include -#include -#include - -#include - -namespace osquery { - -template -class RegistryTemplate : private boost::noncopyable { - public: - typedef std::function Func; - static const int kDefaultPriority = 100000; - - RegistryTemplate() : alreadyRan_(false) {} - - /** - * Registers a function to be invoked when 'run()' is called; fails - * if run() has already been called. Functions will be run with lowest - * priority first, with FIFO order on ties. - * - * Function isn't allowed to throw, calling registerFunc() from within - * registered function will cause deadlock. - */ - bool registerFunc(Func func, int priority = kDefaultPriority) { - std::lock_guard guard(mutex_); - if (alreadyRan_) { - return false; - } - - funcMap_[priority].push_back(std::move(func)); - return true; - } - - /** - * Runs all functions already registered with registerFunc(), in - * order from lowest priority to highest (undefined order on ties). - */ - bool run(FuncArgs... args) noexcept { - std::lock_guard guard(mutex_); - if (alreadyRan_) { - return false; - } - - for (const auto& kv : funcMap_) { - for (const auto& func : kv.second) { - if (func) { - func(args...); - } - } - } - - alreadyRan_ = true; - return true; - } - - private: - std::mutex mutex_; - bool alreadyRan_; - - std::map > funcMap_; -}; - -} // namespace osquery diff --git a/osquery/registry/registry_tests.cpp b/osquery/registry/registry_tests.cpp index a132904..a6ee7b1 100644 --- a/osquery/registry/registry_tests.cpp +++ b/osquery/registry/registry_tests.cpp @@ -8,56 +8,225 @@ * */ -#include -#include - #include #include #include -class TestPlugin { +namespace osquery { + +class RegistryTests : public testing::Test {}; + +class CatPlugin : public Plugin { public: - virtual std::string getName() { return "test_base"; } - virtual ~TestPlugin() {} + CatPlugin() : some_value_(0) {} protected: - TestPlugin(){}; + int some_value_; +}; + +class HouseCat : public CatPlugin { + public: + Status setUp() { + // Make sure the Plugin implementation's init is called. + some_value_ = 9000; + return Status(0, "OK"); + } }; -DECLARE_REGISTRY(TestPlugins, std::string, std::shared_ptr) +/// This is a manual registry type without a name, so we cannot broadcast +/// this registry type and it does NOT need to conform to a registry API. +class CatRegistry : public RegistryHelper {}; + +TEST_F(RegistryTests, test_registry) { + CatRegistry cats; + + /// Add a CatRegistry item (a plugin) called "house". + cats.add("house"); + EXPECT_EQ(cats.count(), 1); + + /// Try to add the same plugin with the same name, this is meaningless. + cats.add("house"); + /// Now add the same plugin with a different name, a new plugin instance + /// will be created and registered. + cats.add("house2"); + EXPECT_EQ(cats.count(), 2); + + /// Request a plugin to call an API method. + auto cat = cats.get("house"); + cats.setUp(); + + /// Now let's iterate over every registered Cat plugin. + EXPECT_EQ(cats.all().size(), 2); +} + +/// Normally we have "Registry" that dictates the set of possible API methods +/// for all registry types. Here we use a "TestRegistry" instead. +class TestCoreRegistry : public RegistryFactory {}; + +/// We can automatically create a registry type as long as that type conforms +/// to the registry API defined in the "Registry". Here we use "TestRegistry". +/// The above "CatRegistry" was easier to understand, but using a auto +/// registry via the registry create method, we can assign a tracked name +/// and then broadcast that registry name to other plugins. +auto AutoCatRegistry = TestCoreRegistry::create("cat"); -#define REGISTERED_TEST_PLUGINS REGISTRY(TestPlugins) +TEST_F(RegistryTests, test_auto_factory) { + /// Using the registry, and a registry type by name, we can register a + /// plugin HouseCat called "house" like above. + TestCoreRegistry::registry("cat")->add("auto_house"); + TestCoreRegistry::add("cat", "auto_house2"); + TestCoreRegistry::registry("cat")->setUp(); -#define REGISTER_TEST_PLUGIN(name, decorator) \ - REGISTER(TestPlugins, name, decorator) + /// When acting on registries by name we can check the broadcasted + /// registry name of other plugin processes (via Thrift) as well as + /// internally registered plugins like HouseCat. + EXPECT_EQ(TestCoreRegistry::registry("cat")->count(), 2); + EXPECT_EQ(TestCoreRegistry::count("cat"), 2); -class TestPluginInstance : public TestPlugin { + /// And we can call an API method, since we guarantee CatPlugins conform + /// to the "TestCoreRegistry"'s "TestPluginAPI". + auto cat = TestCoreRegistry::get("cat", "auto_house"); + auto same_cat = TestCoreRegistry::get("cat", "auto_house"); + EXPECT_EQ(cat, same_cat); +} + +class DogPlugin : public Plugin { public: - TestPluginInstance(){}; + DogPlugin() : some_value_(10000) {} - std::string getName() { return std::string("test_1"); } + protected: + int some_value_; +}; - virtual ~TestPluginInstance() {} +class Doge : public DogPlugin { + public: + Doge() { some_value_ = 100000; } +}; + +class BadDoge : public DogPlugin { + public: + Status setUp() { return Status(1, "Expect error... this is a bad dog"); } }; -REGISTER_TEST_PLUGIN("test_1", std::make_shared()) +auto AutoDogRegistry = TestCoreRegistry::create("dog"); -class RegistryTests : public testing::Test { +TEST_F(RegistryTests, test_auto_registries) { + TestCoreRegistry::add("dog", "doge"); + TestCoreRegistry::registry("dog")->setUp(); + + EXPECT_EQ(TestCoreRegistry::count("dog"), 1); +} + +TEST_F(RegistryTests, test_persistant_registries) { + EXPECT_EQ(TestCoreRegistry::count("cat"), 2); +} + +TEST_F(RegistryTests, test_registry_exceptions) { + EXPECT_TRUE(TestCoreRegistry::add("dog", "duplicate_dog").ok()); + // Bad dog will be added fine, but when setup is run, it will be removed. + EXPECT_TRUE(TestCoreRegistry::add("dog", "bad_doge").ok()); + TestCoreRegistry::registry("dog")->setUp(); + // Make sure bad dog does not exist. + EXPECT_FALSE(TestCoreRegistry::exists("dog", "bad_doge")); + EXPECT_EQ(TestCoreRegistry::count("dog"), 2); + + int exception_count = 0; + try { + TestCoreRegistry::registry("does_not_exist"); + } catch (const std::out_of_range& e) { + exception_count++; + } + + try { + TestCoreRegistry::add("does_not_exist", "cat"); + } catch (const std::out_of_range& e) { + exception_count++; + } + + EXPECT_EQ(exception_count, 2); +} + +class WidgetPlugin : public Plugin { + public: + /// The route information will usually be provided by the plugin type. + /// The plugin/registry item will set some structures for the plugin + /// to parse and format. BUT a plugin/registry item can also fill this + /// information in if the plugin type/registry type exposes routeInfo as + /// a virtual method. + RouteInfo routeInfo() { + RouteInfo info; + info["name"] = name_; + return info; + } + + /// Plugin types should contain generic request/response formatters and + /// decorators. + std::string secretPower(const PluginRequest& request) { + if (request.count("secret_power") > 0) { + return request.at("secret_power"); + } + return "no_secret_power"; + } +}; + +class SpecialWidget : public WidgetPlugin { public: - RegistryTests() { osquery::InitRegistry::get().run(); } + Status call(const PluginRequest& request, PluginResponse& response); }; -TEST_F(RegistryTests, test_plugin_method) { - auto plugin = REGISTERED_TEST_PLUGINS.at("test_1"); - EXPECT_EQ(plugin->getName(), "test_1"); +Status SpecialWidget::call(const PluginRequest& request, + PluginResponse& response) { + response.push_back(request); + response[0]["from"] = name_; + response[0]["secret_power"] = secretPower(request); + return Status(0, "OK"); } -TEST_F(RegistryTests, test_plugin_map) { - EXPECT_EQ(REGISTERED_TEST_PLUGINS.size(), 1); +TEST_F(RegistryTests, test_registry_api) { + auto AutoWidgetRegistry = TestCoreRegistry::create("widgets"); + TestCoreRegistry::add("widgets", "special"); + + // Test route info propogation, from item to registry, to broadcast. + auto ri = TestCoreRegistry::get("widgets", "special")->routeInfo(); + EXPECT_EQ(ri.at("name"), "special"); + auto rr = TestCoreRegistry::registry("widgets")->getRoutes(); + EXPECT_EQ(rr.size(), 1); + EXPECT_EQ(rr.at("special").at("name"), "special"); + + // Broadcast will include all registries, and all their items. + auto broadcast_info = TestCoreRegistry::getBroadcast(); + EXPECT_TRUE(broadcast_info.size() >= 3); + EXPECT_EQ(broadcast_info.at("widgets").at("special").at("name"), "special"); + + PluginResponse response; + PluginRequest request; + auto status = TestCoreRegistry::call("widgets", "special", request, response); + EXPECT_TRUE(status.ok()); + EXPECT_EQ(response[0].at("from"), "special"); + EXPECT_EQ(response[0].at("secret_power"), "no_secret_power"); + + request["secret_power"] = "magic"; + status = TestCoreRegistry::call("widgets", "special", request, response); + EXPECT_EQ(response[0].at("secret_power"), "magic"); +} + +TEST_F(RegistryTests, test_real_registry) { + EXPECT_TRUE(Registry::count() > 0); + + bool has_one_registered = false; + for (const auto& registry : Registry::all()) { + if (Registry::count(registry.first) > 0) { + has_one_registered = true; + break; + } + } + EXPECT_TRUE(has_one_registered); +} } int main(int argc, char* argv[]) { testing::InitGoogleTest(&argc, argv); + google::InitGoogleLogging(argv[0]); return RUN_ALL_TESTS(); } diff --git a/osquery/registry/singleton.h b/osquery/registry/singleton.h deleted file mode 100644 index 16f53a3..0000000 --- a/osquery/registry/singleton.h +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Copyright (c) 2014, Facebook, Inc. - * All rights reserved. - * - * This source code is licensed under the BSD-style license found in the - * LICENSE file in the root directory of this source tree. An additional grant - * of patent rights can be found in the PATENTS file in the same directory. - * - */ - -#pragma once - -namespace osquery { - -// NOTE: T should have public or protected default ctor. -template -class Singleton : private T { - public: - static T& get() { - // C++11 guarantees that initialization of static local - // variables is thread-safe; Moreover, GCC started to - // provide the same guarantee long time ago. - // http://cppwisdom.quora.com/Singletons-are-easy - static Singleton instance; - return instance; - } - - private: - Singleton() {} - ~Singleton() {} -}; - -} // namespace osquery diff --git a/osquery/scheduler/CMakeLists.txt b/osquery/scheduler/CMakeLists.txt index 99581f7..0196b0c 100644 --- a/osquery/scheduler/CMakeLists.txt +++ b/osquery/scheduler/CMakeLists.txt @@ -1,3 +1,3 @@ -ADD_OSQUERY_LIBRARY(osquery_scheduler scheduler.cpp) +ADD_OSQUERY_LIBRARY(TRUE osquery_scheduler scheduler.cpp) -ADD_OSQUERY_TEST(osquery_scheduler_tests scheduler_tests.cpp) +ADD_OSQUERY_TEST(TRUE osquery_scheduler_tests scheduler_tests.cpp) diff --git a/osquery/scheduler/scheduler.cpp b/osquery/scheduler/scheduler.cpp index 6aa9142..376c5d7 100644 --- a/osquery/scheduler/scheduler.cpp +++ b/osquery/scheduler/scheduler.cpp @@ -30,7 +30,7 @@ DEFINE_osquery_flag(string, DEFINE_osquery_flag(int32, schedule_splay_percent, 10, - "Percent to splay config times."); + "Percent to splay config times"); Status getHostIdentifier(std::string& ident) { std::shared_ptr db; diff --git a/osquery/sql/CMakeLists.txt b/osquery/sql/CMakeLists.txt new file mode 100644 index 0000000..1157080 --- /dev/null +++ b/osquery/sql/CMakeLists.txt @@ -0,0 +1,7 @@ +ADD_OSQUERY_LIBRARY(TRUE osquery_sql sql.cpp + sqlite_util.cpp + virtual_table.cpp) + +ADD_OSQUERY_TEST(TRUE osquery_sql_test sql_tests.cpp) +ADD_OSQUERY_TEST(TRUE osquery_sqlite_util_tests sqlite_util_tests.cpp) +ADD_OSQUERY_TEST(TRUE osquery_virtual_table_tests virtual_table_tests.cpp) diff --git a/osquery/sql/sql.cpp b/osquery/sql/sql.cpp new file mode 100644 index 0000000..557915b --- /dev/null +++ b/osquery/sql/sql.cpp @@ -0,0 +1,77 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ + +#include + +#include +#include +#include +#include +#include + +#include "osquery/sql/sqlite_util.h" + +namespace osquery { + +const std::map kSQLOperatorRepr = { + {tables::EQUALS, "="}, + {tables::GREATER_THAN, ">"}, + {tables::LESS_THAN_OR_EQUALS, "<="}, + {tables::LESS_THAN, "<"}, + {tables::GREATER_THAN_OR_EQUALS, ">="}, +}; + +SQL::SQL(const std::string& q) { status_ = query(q, results_); } + +QueryData SQL::rows() { return results_; } + +bool SQL::ok() { return status_.ok(); } + +std::string SQL::getMessageString() { return status_.toString(); } + +std::vector SQL::getTableNames() { + std::vector results; + for (const auto& name : Registry::names("table")) { + results.push_back(name); + } + return results; +} + +QueryData SQL::selectAllFrom(const std::string& table) { + PluginResponse response; + PluginRequest request; + request["action"] = "generate"; + + Registry::call("table", table, request, response); + return response; +} + +QueryData SQL::selectAllFrom(const std::string& table, + const std::string& column, + tables::ConstraintOperator op, + const std::string& expr) { + PluginResponse response; + PluginRequest request = {{"action", "generate"}}; + tables::QueryContext ctx; + ctx.constraints[column].add(tables::Constraint(op, expr)); + + tables::TablePlugin::setRequestFromContext(ctx, request); + Registry::call("table", table, request, response); + return response; +} + +Status query(const std::string& q, QueryData& results) { + return queryInternal(q, results); +} + +Status getQueryColumns(const std::string& q, tables::TableColumns& columns) { + return getQueryColumnsInternal(q, columns); +} +} diff --git a/osquery/sql/sql_tests.cpp b/osquery/sql/sql_tests.cpp new file mode 100644 index 0000000..d681146 --- /dev/null +++ b/osquery/sql/sql_tests.cpp @@ -0,0 +1,81 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ + +#include + +#include +#include +#include +#include + +namespace osquery { + +class SQLTests : public testing::Test {}; + +TEST_F(SQLTests, test_simple_query_execution) { + auto sql = SQL("SELECT * FROM time"); + EXPECT_TRUE(sql.ok()); + EXPECT_EQ(sql.rows().size(), 1); +} + +TEST_F(SQLTests, test_get_tables) { + auto tables = SQL::getTableNames(); + EXPECT_TRUE(tables.size() > 0); +} + +TEST_F(SQLTests, test_raw_access) { + auto results = SQL::selectAllFrom("time"); + EXPECT_EQ(results.size(), 1); +} + +class TestTable : public tables::TablePlugin { + private: + tables::TableColumns columns() { + return {{"test_int", "INTEGER"}, {"test_text", "TEXT"}}; + } + + QueryData generate(tables::QueryContext& ctx) { + QueryData results; + if (ctx.constraints["test_int"].existsAndMatches("1")) { + results.push_back({{"test_int", "1"}, {"test_text", "0"}}); + } else { + results.push_back({{"test_int", "0"}, {"test_text", "1"}}); + } + + auto ints = ctx.constraints["test_int"].getAll(tables::EQUALS); + for (const auto& int_match : ints) { + results.push_back({{"test_int", INTEGER(int_match)}}); + } + + return results; + } +}; + +TEST_F(SQLTests, test_raw_access_context) { + REGISTER(TestTable, "table", "test_table"); + auto results = SQL::selectAllFrom("test_table"); + + EXPECT_EQ(results.size(), 1); + EXPECT_EQ(results[0]["test_text"], "1"); + + results = SQL::selectAllFrom("test_table", "test_int", tables::EQUALS, "1"); + EXPECT_EQ(results.size(), 2); + + results = SQL::selectAllFrom("test_table", "test_int", tables::EQUALS, "2"); + EXPECT_EQ(results.size(), 2); + EXPECT_EQ(results[0]["test_int"], "0"); +} +} + +int main(int argc, char* argv[]) { + testing::InitGoogleTest(&argc, argv); + osquery::initOsquery(argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/osquery/sql/sqlite_util.cpp b/osquery/sql/sqlite_util.cpp new file mode 100644 index 0000000..2fa271e --- /dev/null +++ b/osquery/sql/sqlite_util.cpp @@ -0,0 +1,155 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ + +#include +#include +#include +#include + +#include "osquery/sql/sqlite_util.h" +#include "osquery/sql/virtual_table.h" + +namespace osquery { + +const std::map kSQLiteReturnCodes = { + {0, "SQLITE_OK: Successful result"}, + {1, "SQLITE_ERROR: SQL error or missing database"}, + {2, "SQLITE_INTERNAL: Internal logic error in SQLite"}, + {3, "SQLITE_PERM: Access permission denied"}, + {4, "SQLITE_ABORT: Callback routine requested an abort"}, + {5, "SQLITE_BUSY: The database file is locked"}, + {6, "SQLITE_LOCKED: A table in the database is locked"}, + {7, "SQLITE_NOMEM: A malloc() failed"}, + {8, "SQLITE_READONLY: Attempt to write a readonly database"}, + {9, "SQLITE_INTERRUPT: Operation terminated by sqlite3_interrupt()"}, + {10, "SQLITE_IOERR: Some kind of disk I/O error occurred"}, + {11, "SQLITE_CORRUPT: The database disk image is malformed"}, + {12, "SQLITE_NOTFOUND: Unknown opcode in sqlite3_file_control()"}, + {13, "SQLITE_FULL: Insertion failed because database is full"}, + {14, "SQLITE_CANTOPEN: Unable to open the database file"}, + {15, "SQLITE_PROTOCOL: Database lock protocol error"}, + {16, "SQLITE_EMPTY: Database is empty"}, + {17, "SQLITE_SCHEMA: The database schema changed"}, + {18, "SQLITE_TOOBIG: String or BLOB exceeds size limit"}, + {19, "SQLITE_CONSTRAINT: Abort due to constraint violation"}, + {20, "SQLITE_MISMATCH: Data type mismatch"}, + {21, "SQLITE_MISUSE: Library used incorrectly"}, + {22, "SQLITE_NOLFS: Uses OS features not supported on host"}, + {23, "SQLITE_AUTH: Authorization denied"}, + {24, "SQLITE_FORMAT: Auxiliary database format error"}, + {25, "SQLITE_RANGE: 2nd parameter to sqlite3_bind out of range"}, + {26, "SQLITE_NOTADB: File opened that is not a database file"}, + {27, "SQLITE_NOTICE: Notifications from sqlite3_log()"}, + {28, "SQLITE_WARNING: Warnings from sqlite3_log()"}, + {100, "SQLITE_ROW: sqlite3_step() has another row ready"}, + {101, "SQLITE_DONE: sqlite3_step() has finished executing"}, +}; + +std::string getStringForSQLiteReturnCode(int code) { + if (kSQLiteReturnCodes.find(code) != kSQLiteReturnCodes.end()) { + return kSQLiteReturnCodes.at(code); + } else { + std::ostringstream s; + s << "Error: " << code << " is not a valid SQLite result code"; + return s.str(); + } +} + +sqlite3* createDB() { + sqlite3* db = nullptr; + sqlite3_open(":memory:", &db); + tables::attachVirtualTables(db); + return db; +} + +int queryDataCallback(void* argument, int argc, char* argv[], char* column[]) { + if (argument == nullptr) { + LOG(ERROR) << "queryDataCallback received nullptr as data argument"; + return SQLITE_MISUSE; + } + + QueryData* qData = (QueryData*)argument; + Row r; + for (int i = 0; i < argc; i++) { + r[column[i]] = argv[i]; + } + (*qData).push_back(r); + return 0; +} + +Status queryInternal(const std::string& q, QueryData& results) { + sqlite3* db = createDB(); + auto status = queryInternal(q, results, db); + sqlite3_close(db); + return status; +} + +Status queryInternal(const std::string& q, QueryData& results, sqlite3* db) { + char* err = nullptr; + sqlite3_exec(db, q.c_str(), queryDataCallback, &results, &err); + if (err != nullptr) { + sqlite3_free(err); + return Status(1, "Error running query: " + q); + } + + return Status(0, "OK"); +} + +Status getQueryColumnsInternal(const std::string& q, + tables::TableColumns& columns) { + sqlite3* db = createDB(); + Status status = getQueryColumnsInternal(q, columns, db); + sqlite3_close(db); + return status; +} + +Status getQueryColumnsInternal(const std::string& q, + tables::TableColumns& columns, + sqlite3* db) { + int rc; + + // Will automatically handle calling sqlite3_finalize on the prepared stmt + // (Note that sqlite3_finalize is explicitly a nop for nullptr) + std::unique_ptr stmt_managed( + nullptr, sqlite3_finalize); + sqlite3_stmt* stmt = stmt_managed.get(); + + // Turn the query into a prepared statement + rc = sqlite3_prepare_v2(db, q.c_str(), q.length() + 1, &stmt, nullptr); + if (rc != SQLITE_OK) { + return Status(1, sqlite3_errmsg(db)); + } + + // Get column count + int num_columns = sqlite3_column_count(stmt); + std::vector > results; + results.reserve(num_columns); + + // Get column names and types + for (int i = 0; i < num_columns; ++i) { + const char* col_name = sqlite3_column_name(stmt, i); + const char* col_type = sqlite3_column_decltype(stmt, i); + if (col_name == nullptr) { + return Status(1, "Got nullptr for column name"); + } + if (col_type == nullptr) { + // Types are only returned for table columns (not expressions or + // subqueries). See docs for column_decltype + // (https://www.sqlite.org/c3ref/column_decltype.html). + col_type = "UNKNOWN"; + } + results.push_back({col_name, col_type}); + } + + columns = std::move(results); + + return Status(0, "OK"); +} +} diff --git a/osquery/sql/sqlite_util.h b/osquery/sql/sqlite_util.h new file mode 100644 index 0000000..95775eb --- /dev/null +++ b/osquery/sql/sqlite_util.h @@ -0,0 +1,84 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ + +#pragma once + +#include + +namespace osquery { +/** + * @brief A map of SQLite status codes to their corresponding message string + * + * Details of this map are defined at: http://www.sqlite.org/c3ref/c_abort.html + */ +extern const std::map kSQLiteReturnCodes; + +/// Internal (core) SQL implementation of the osquery query API. +Status queryInternal(const std::string& q, QueryData& results); + +/** + * @brief Execute a query on a specific database + * + * If you need to use a different database, other than the osquery default, + * use this method and pass along a pointer to a SQLite3 database. This is + * useful for testing. + * + * @param q the query to execute + * @param results The QueryData struct to emit row on query success. + * @param db the SQLite3 database to execute query q against + * + * @return A status indicating SQL query results. + */ +Status queryInternal(const std::string& q, QueryData& results, sqlite3* db); + +/// Internal (core) SQL implementation of the osquery getQueryColumns API. +Status getQueryColumnsInternal(const std::string& q, tables::TableColumns& columns); + +/** + * @brief Analyze a query, providing information about the result columns + * + * This function asks SQLite to determine what the names and types are of the + * result columns of the provided query. Only table columns (not expressions or + * subqueries) can have their types determined. Types that are not determined + * are indicated with the string "UNKNOWN". + * + * @param q the query to analyze + * @param columns the vector to fill with column information + * @param db the SQLite3 database to perform the analysis on + * + * @return status indicating success or failure of the operation + */ +Status getQueryColumnsInternal(const std::string& q, + tables::TableColumns& columns, + sqlite3* db); + +/** + * @brief Return a fully configured sqlite3 database object + * + * An osquery database is basically just a SQLite3 database with several + * virtual tables attached. This method is the main abstraction for creating + * SQLite3 databases within osquery. + * + * Note: osquery::initOsquery must be called before calling createDB in order + * for virtual tables to be registered. + * + * @return a SQLite3 database with all virtual tables attached + */ +sqlite3* createDB(); + +/** + * @brief Get a string representation of a SQLite return code + */ +std::string getStringForSQLiteReturnCode(int code); + +// the callback for populating a std::vector set of results. "argument" +// should be a non-const reference to a std::vector +int queryDataCallback(void* argument, int argc, char* argv[], char* column[]); +} diff --git a/osquery/sql/sqlite_util_tests.cpp b/osquery/sql/sqlite_util_tests.cpp new file mode 100644 index 0000000..cc3331b --- /dev/null +++ b/osquery/sql/sqlite_util_tests.cpp @@ -0,0 +1,141 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ + +#include + +#include + +#include +#include + +#include "osquery/core/test_util.h" +#include "osquery/sql/sqlite_util.h" + +namespace osquery { + +class SQLiteUtilTests : public testing::Test {}; + +sqlite3* createTestDB() { + sqlite3* db = createDB(); + char* err = nullptr; + std::vector queries = { + "CREATE TABLE test_table (" + "username varchar(30) primary key, " + "age int" + ")", + "INSERT INTO test_table VALUES (\"mike\", 23)", + "INSERT INTO test_table VALUES (\"matt\", 24)"}; + for (auto q : queries) { + sqlite3_exec(db, q.c_str(), nullptr, nullptr, &err); + if (err != nullptr) { + return nullptr; + } + } + + return db; +} + +TEST_F(SQLiteUtilTests, test_simple_query_execution) { + auto db = createTestDB(); + QueryData results; + auto status = queryInternal(kTestQuery, results, db); + sqlite3_close(db); + EXPECT_TRUE(status.ok()); + EXPECT_EQ(results, getTestDBExpectedResults()); +} + +TEST_F(SQLiteUtilTests, test_passing_callback_no_data_param) { + char* err = nullptr; + auto db = createTestDB(); + sqlite3_exec(db, kTestQuery.c_str(), queryDataCallback, nullptr, &err); + sqlite3_close(db); + EXPECT_TRUE(err != nullptr); + if (err != nullptr) { + sqlite3_free(err); + } +} + +TEST_F(SQLiteUtilTests, test_aggregate_query) { + auto db = createTestDB(); + QueryData results; + auto status = queryInternal(kTestQuery, results, db); + sqlite3_close(db); + EXPECT_TRUE(status.ok()); + EXPECT_EQ(results, getTestDBExpectedResults()); +} + +TEST_F(SQLiteUtilTests, test_get_test_db_result_stream) { + auto db = createTestDB(); + auto results = getTestDBResultStream(); + for (auto r : results) { + char* err_char = nullptr; + sqlite3_exec(db, (r.first).c_str(), nullptr, nullptr, &err_char); + EXPECT_TRUE(err_char == nullptr); + if (err_char != nullptr) { + sqlite3_free(err_char); + ASSERT_TRUE(false); + } + + QueryData expected; + auto status = queryInternal(kTestQuery, expected, db); + EXPECT_EQ(expected, r.second); + } + sqlite3_close(db); +} + +TEST_F(SQLiteUtilTests, test_get_query_columns) { + std::unique_ptr db_managed(createDB(), + sqlite3_close); + sqlite3* db = db_managed.get(); + + std::string query; + Status status; + tables::TableColumns results; + + query = + "SELECT hour, minutes, seconds, version, config_md5, config_path, \ + pid FROM time JOIN osquery_info"; + status = getQueryColumnsInternal(query, results, db); + ASSERT_TRUE(status.ok()); + ASSERT_EQ(7, results.size()); + EXPECT_EQ(std::make_pair(std::string("hour"), std::string("INTEGER")), + results[0]); + EXPECT_EQ(std::make_pair(std::string("minutes"), std::string("INTEGER")), + results[1]); + EXPECT_EQ(std::make_pair(std::string("seconds"), std::string("INTEGER")), + results[2]); + EXPECT_EQ(std::make_pair(std::string("version"), std::string("TEXT")), + results[3]); + EXPECT_EQ(std::make_pair(std::string("config_md5"), std::string("TEXT")), + results[4]); + EXPECT_EQ(std::make_pair(std::string("config_path"), std::string("TEXT")), + results[5]); + EXPECT_EQ(std::make_pair(std::string("pid"), std::string("INTEGER")), + results[6]); + + query = "SELECT hour + 1 AS hour1, minutes + 1 FROM time"; + status = getQueryColumnsInternal(query, results, db); + ASSERT_TRUE(status.ok()); + ASSERT_EQ(2, results.size()); + EXPECT_EQ(std::make_pair(std::string("hour1"), std::string("UNKNOWN")), + results[0]); + EXPECT_EQ(std::make_pair(std::string("minutes + 1"), std::string("UNKNOWN")), + results[1]); + + query = "SELECT * FROM foo"; + status = getQueryColumnsInternal(query, results, db); + ASSERT_FALSE(status.ok()); +} +} + +int main(int argc, char* argv[]) { + testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/osquery/sql/virtual_table.cpp b/osquery/sql/virtual_table.cpp new file mode 100644 index 0000000..afe9706 --- /dev/null +++ b/osquery/sql/virtual_table.cpp @@ -0,0 +1,279 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ + +#include + +#include "osquery/sql/virtual_table.h" + +namespace osquery { +namespace tables { + +int xOpen(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCursor) { + int rc = SQLITE_NOMEM; + BaseCursor *pCur; + + pCur = new BaseCursor; + + if (pCur) { + memset(pCur, 0, sizeof(BaseCursor)); + *ppCursor = (sqlite3_vtab_cursor *)pCur; + rc = SQLITE_OK; + } + + return rc; +} + +int xClose(sqlite3_vtab_cursor *cur) { + BaseCursor *pCur = (BaseCursor *)cur; + + delete pCur; + return SQLITE_OK; +} + +int xEof(sqlite3_vtab_cursor *cur) { + BaseCursor *pCur = (BaseCursor *)cur; + auto *pVtab = (VirtualTable *)cur->pVtab; + return pCur->row >= pVtab->content->n; +} + +int xDestroy(sqlite3_vtab *p) { + auto *pVtab = (VirtualTable *)p; + delete pVtab->content; + delete pVtab; + return SQLITE_OK; +} + +int xNext(sqlite3_vtab_cursor *cur) { + BaseCursor *pCur = (BaseCursor *)cur; + pCur->row++; + return SQLITE_OK; +} + +int xRowid(sqlite3_vtab_cursor *cur, sqlite_int64 *pRowid) { + BaseCursor *pCur = (BaseCursor *)cur; + *pRowid = pCur->row; + return SQLITE_OK; +} + +int xCreate(sqlite3 *db, + void *pAux, + int argc, + const char *const *argv, + sqlite3_vtab **ppVtab, + char **pzErr) { + auto *pVtab = new VirtualTable; + + if (!pVtab || argc == 0 || argv[0] == nullptr) { + return SQLITE_NOMEM; + } + + memset(pVtab, 0, sizeof(VirtualTable)); + pVtab->content = new VirtualTableContent; + + PluginResponse response; + pVtab->content->name = std::string(argv[0]); + auto status = Registry::call( + "table", pVtab->content->name, {{"action", "statement"}}, response); + if (!status.ok() || response.size() == 0) { + return SQLITE_ERROR; + } + + int rc = sqlite3_declare_vtab(db, response[0].at("statement").c_str()); + if (rc != SQLITE_OK) { + return rc; + } + + // Also set the table column information. + status = Registry::call( + "table", pVtab->content->name, {{"action", "columns"}}, response); + if (!status.ok() || response.size() == 0) { + return SQLITE_ERROR; + } + + for (const auto &column : response) { + pVtab->content->columns.push_back( + std::make_pair(column.at("name"), column.at("type"))); + } + + *ppVtab = (sqlite3_vtab *)pVtab; + return rc; +} + +int xColumn(sqlite3_vtab_cursor *cur, sqlite3_context *ctx, int col) { + BaseCursor *pCur = (BaseCursor *)cur; + auto *pVtab = (VirtualTable *)cur->pVtab; + + if (col >= pVtab->content->columns.size()) { + return SQLITE_ERROR; + } + + const auto &column_name = pVtab->content->columns[col].first; + const auto &type = pVtab->content->columns[col].second; + if (pCur->row >= pVtab->content->data[column_name].size()) { + return SQLITE_ERROR; + } + + const auto &value = pVtab->content->data[column_name][pCur->row]; + if (type == "TEXT") { + sqlite3_result_text(ctx, value.c_str(), -1, nullptr); + } else if (type == "INTEGER") { + int afinite; + try { + afinite = boost::lexical_cast(value); + } catch (const boost::bad_lexical_cast &e) { + afinite = -1; + LOG(WARNING) << "Error casting " << column_name << " (" << value + << ") to INTEGER"; + } + sqlite3_result_int(ctx, afinite); + } else if (type == "BIGINT") { + long long int afinite; + try { + afinite = boost::lexical_cast(value); + } catch (const boost::bad_lexical_cast &e) { + afinite = -1; + LOG(WARNING) << "Error casting " << column_name << " (" << value + << ") to BIGINT"; + } + sqlite3_result_int64(ctx, afinite); + } + + return SQLITE_OK; +} + +static int xBestIndex(sqlite3_vtab *tab, sqlite3_index_info *pIdxInfo) { + auto *pVtab = (VirtualTable *)tab; + pVtab->content->constraints.clear(); + + int expr_index = 0; + int cost = 0; + for (size_t i = 0; i < pIdxInfo->nConstraint; ++i) { + if (!pIdxInfo->aConstraint[i].usable) { + // A higher cost less priority, prefer more usable query constraints. + cost += 10; + // TODO: OR is not usable. + continue; + } + + const auto &name = + pVtab->content->columns[pIdxInfo->aConstraint[i].iColumn].first; + pVtab->content->constraints.push_back( + std::make_pair(name, Constraint(pIdxInfo->aConstraint[i].op))); + pIdxInfo->aConstraintUsage[i].argvIndex = ++expr_index; + } + + pIdxInfo->estimatedCost = cost; + return SQLITE_OK; +} + +static int xFilter(sqlite3_vtab_cursor *pVtabCursor, + int idxNum, + const char *idxStr, + int argc, + sqlite3_value **argv) { + BaseCursor *pCur = (BaseCursor *)pVtabCursor; + auto *pVtab = (VirtualTable *)pVtabCursor->pVtab; + + pCur->row = 0; + pVtab->content->n = 0; + QueryContext context; + + for (size_t i = 0; i < pVtab->content->columns.size(); ++i) { + pVtab->content->data[pVtab->content->columns[i].first].clear(); + context.constraints[pVtab->content->columns[i].first].affinity = + pVtab->content->columns[i].second; + } + + for (size_t i = 0; i < argc; ++i) { + auto expr = (const char *)sqlite3_value_text(argv[i]); + // Set the expression from SQLite's now-populated argv. + pVtab->content->constraints[i].second.expr = std::string(expr); + // Add the constraint to the column-sorted query request map. + context.constraints[pVtab->content->constraints[i].first].add( + pVtab->content->constraints[i].second); + } + + PluginRequest request; + PluginResponse response; + request["action"] = "generate"; + TablePlugin::setRequestFromContext(context, request); + Registry::call("table", pVtab->content->name, request, response); + + // Now organize the response rows by column instead of row. + for (const auto &row : response) { + for (const auto &column : pVtab->content->columns) { + try { + pVtab->content->data[column.first].push_back(row.at(column.first)); + } catch (const std::out_of_range &e) { + VLOG(1) << "Table " << pVtab->content->name << " row " + << pVtab->content->n << " did not include column " + << column.first; + pVtab->content->data[column.first].push_back(""); + } + } + pVtab->content->n++; + } + + return SQLITE_OK; +} + +int attachTable(sqlite3 *db, const std::string &name) { + int rc = SQLITE_OK; + + static sqlite3_module module = { + 0, + xCreate, + xCreate, + xBestIndex, + xDestroy, + xDestroy, + xOpen, + xClose, + xFilter, + xNext, + xEof, + xColumn, + xRowid, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + }; + + // Column information is nice for virtual table create call. + PluginResponse response; + auto status = Registry::call( + "table", name, {{"action", "columns_definition"}}, response); + if (!status.ok() || response.size() == 0) { + return SQLITE_ERROR; + } + + rc = sqlite3_create_module(db, name.c_str(), &module, 0); + if (rc == SQLITE_OK) { + auto format = "CREATE VIRTUAL TABLE temp." + name + " USING " + name + + response[0].at("definition"); + rc = sqlite3_exec(db, format.c_str(), 0, 0, 0); + } + return rc; +} + +void attachVirtualTables(sqlite3 *db) { + for (const auto &name : Registry::names("table")) { + int status = attachTable(db, name); + if (status != SQLITE_OK) { + LOG(ERROR) << "Error attaching table: " << name << " (" << status << ")"; + } + } +} +} +} diff --git a/osquery/sql/virtual_table.h b/osquery/sql/virtual_table.h new file mode 100644 index 0000000..6106d2d --- /dev/null +++ b/osquery/sql/virtual_table.h @@ -0,0 +1,57 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ + +#pragma once + +#include + +#include "osquery/sql/sqlite_util.h" + +namespace osquery { +namespace tables { + +/** + * @brief osquery cursor object. + * + * Only used in the SQLite virtual table module methods. + */ +struct BaseCursor { + /// SQLite virtual table cursor. + sqlite3_vtab_cursor base; + /// Current cursor position. + int row; +}; + +struct VirtualTableContent { + TableName name; + TableColumns columns; + TableData data; + ConstraintSet constraints; + size_t n; +}; + +/** + * @brief osquery virtual table object + * + * Only used in the SQLite virtual table module methods. + * This adds each table plugin class to the state tracking in SQLite. + */ +struct VirtualTable { + sqlite3_vtab base; + VirtualTableContent *content; +}; + +/// Attach a table plugin name to an in-memory SQLite datable. +int attachTable(sqlite3 *db, const std::string &name); + +/// Attach all table plugins to an in-memory SQLite datable. +void attachVirtualTables(sqlite3 *db); +} +} diff --git a/osquery/sql/virtual_table_tests.cpp b/osquery/sql/virtual_table_tests.cpp new file mode 100644 index 0000000..7ae8e0b --- /dev/null +++ b/osquery/sql/virtual_table_tests.cpp @@ -0,0 +1,74 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ + +#include + +#include +#include +#include + +#include "osquery/sql/virtual_table.h" + +namespace osquery { +namespace tables { + +class VirtualTableTests : public testing::Test {}; + +// sample plugin used on tests +class sampleTablePlugin : public TablePlugin { + private: + TableColumns columns() { + return { + {"foo", "INTEGER"}, {"bar", "TEXT"}, + }; + } +}; + +TEST_F(VirtualTableTests, test_tableplugin_columndefinition) { + auto table = std::make_shared(); + EXPECT_EQ("(foo INTEGER, bar TEXT)", table->columnDefinition()); +} + +TEST_F(VirtualTableTests, test_tableplugin_statement) { + auto table = std::make_shared(); + table->setName("sample"); + EXPECT_EQ("CREATE TABLE sample(foo INTEGER, bar TEXT)", table->statement()); +} + +TEST_F(VirtualTableTests, test_sqlite3_attach_vtable) { + auto table = std::make_shared(); + table->setName("sample"); + sqlite3* db = nullptr; + sqlite3_open(":memory:", &db); + + // Virtual tables require the registry/plugin API to query tables. + int rc = osquery::tables::attachTable(db, "failed_sample"); + EXPECT_EQ(rc, SQLITE_ERROR); + + // The table attach will complete only when the table name is registered. + Registry::add("table", "sample"); + rc = osquery::tables::attachTable(db, "sample"); + EXPECT_EQ(rc, SQLITE_OK); + + std::string q = "SELECT sql FROM sqlite_temp_master WHERE tbl_name='sample';"; + QueryData results; + auto status = queryInternal(q, results, db); + EXPECT_EQ("CREATE VIRTUAL TABLE sample USING sample(foo INTEGER, bar TEXT)", + results[0]["sql"]); + sqlite3_close(db); +} +} +} + +int main(int argc, char* argv[]) { + testing::InitGoogleTest(&argc, argv); + osquery::initOsquery(argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/osquery/tables/CMakeLists.txt b/osquery/tables/CMakeLists.txt index 2433134..38429bf 100644 --- a/osquery/tables/CMakeLists.txt +++ b/osquery/tables/CMakeLists.txt @@ -1,32 +1,35 @@ -ADD_OSQUERY_LIBRARY(osquery_tables_linux events/linux/passwd_changes.cpp - events/linux/hardware_events.cpp - networking/linux/routes.cpp - networking/linux/process_open_sockets.cpp - networking/linux/arp_cache.cpp - system/linux/acpi_tables.cpp - system/linux/kernel_modules.cpp - system/linux/processes.cpp - system/linux/process_open_files.cpp - system/linux/smbios_tables.cpp - system/linux/users.cpp - system/linux/groups.cpp - system/linux/mounts.cpp - system/linux/pci_devices.cpp - system/linux/usb_devices.cpp) +ADD_OSQUERY_LIBRARY(FALSE osquery_tables_linux events/linux/passwd_changes.cpp + events/linux/hardware_events.cpp + networking/linux/routes.cpp + networking/linux/process_open_sockets.cpp + networking/linux/arp_cache.cpp + system/linux/acpi_tables.cpp + system/linux/processes.cpp + system/linux/process_open_files.cpp + system/linux/shared_memory.cpp + system/linux/smbios_tables.cpp + system/linux/users.cpp + system/linux/groups.cpp + system/linux/kernel_info.cpp + system/linux/kernel_modules.cpp + system/linux/memory_map.cpp + system/linux/mounts.cpp + system/linux/pci_devices.cpp + system/linux/usb_devices.cpp) -ADD_OSQUERY_LIBRARY(osquery_tables networking/utils.cpp - networking/etc_hosts.cpp - networking/etc_services.cpp - networking/listening_ports.cpp - utility/time.cpp - utility/hash.cpp - utility/file.cpp - utility/osquery.cpp - system/crontab.cpp - system/smbios_utils.cpp - system/last.cpp - system/shell_history.cpp - system/suid_bin.cpp - system/logged_in_users.cpp) +ADD_OSQUERY_LIBRARY(TRUE osquery_tables networking/utils.cpp + networking/etc_hosts.cpp # TODO(sangwan.kwon): Change to FALSE) + networking/etc_services.cpp + networking/listening_ports.cpp + utility/time.cpp + utility/hash.cpp + utility/file.cpp + utility/osquery.cpp + system/crontab.cpp + system/smbios_utils.cpp + system/last.cpp + system/shell_history.cpp + system/suid_bin.cpp + system/logged_in_users.cpp) -ADD_OSQUERY_TEST(osquery_etc_hosts_tests networking/etc_hosts_tests.cpp) +ADD_OSQUERY_TEST(FALSE osquery_etc_hosts_tests networking/etc_hosts_tests.cpp) diff --git a/osquery/tables/events/linux/hardware_events.cpp b/osquery/tables/events/linux/hardware_events.cpp index 5eedb20..c141e25 100644 --- a/osquery/tables/events/linux/hardware_events.cpp +++ b/osquery/tables/events/linux/hardware_events.cpp @@ -24,7 +24,7 @@ namespace tables { * @brief Track udev events in Linux */ class HardwareEventSubscriber : public EventSubscriber { - DECLARE_SUBSCRIBER("HardwareEventSubscriber"); + DECLARE_SUBSCRIBER("hardware_events"); public: void init(); @@ -32,7 +32,7 @@ class HardwareEventSubscriber : public EventSubscriber { Status Callback(const UdevEventContextRef& ec); }; -REGISTER_EVENTSUBSCRIBER(HardwareEventSubscriber); +REGISTER(HardwareEventSubscriber, "event_subscriber", "hardware_events"); void HardwareEventSubscriber::init() { auto subscription = createSubscriptionContext(); diff --git a/osquery/tables/events/linux/passwd_changes.cpp b/osquery/tables/events/linux/passwd_changes.cpp index 0cf5965..89d3171 100644 --- a/osquery/tables/events/linux/passwd_changes.cpp +++ b/osquery/tables/events/linux/passwd_changes.cpp @@ -27,7 +27,7 @@ namespace tables { */ class PasswdChangesEventSubscriber : public EventSubscriber { - DECLARE_SUBSCRIBER("PasswdChangesEventSubscriber"); + DECLARE_SUBSCRIBER("passwd_changes"); public: void init(); @@ -50,7 +50,7 @@ class PasswdChangesEventSubscriber * This registers PasswdChangesEventSubscriber into the osquery EventSubscriber * pseudo-plugin registry. */ -REGISTER_EVENTSUBSCRIBER(PasswdChangesEventSubscriber); +REGISTER(PasswdChangesEventSubscriber, "event_subscriber", "passwd_changes"); void PasswdChangesEventSubscriber::init() { auto mc = createSubscriptionContext(); diff --git a/osquery/tables/networking/linux/process_open_sockets.cpp b/osquery/tables/networking/linux/process_open_sockets.cpp index de083fa..575e6d7 100644 --- a/osquery/tables/networking/linux/process_open_sockets.cpp +++ b/osquery/tables/networking/linux/process_open_sockets.cpp @@ -11,6 +11,7 @@ #include #include +#include #include #include @@ -47,7 +48,7 @@ enum { #define TCPF_ALL 0xFFF #define SOCKET_BUFFER_SIZE (getpagesize() < 8192L ? getpagesize() : 8192L) -int send_diag_msg(int sockfd, int protocol, int family) { +int sendNLDiagMessage(int sockfd, int protocol, int family) { struct sockaddr_nl sa; memset(&sa, 0, sizeof(sa)); sa.nl_family = AF_NETLINK; @@ -90,12 +91,9 @@ int send_diag_msg(int sockfd, int protocol, int family) { return retval; } -Row getDiagMessage(struct inet_diag_msg *diag_msg, int protocol, int family) { - char local_addr_buf[INET6_ADDRSTRLEN]; - char remote_addr_buf[INET6_ADDRSTRLEN]; - - memset(local_addr_buf, 0, sizeof(local_addr_buf)); - memset(remote_addr_buf, 0, sizeof(remote_addr_buf)); +Row getNLDiagMessage(struct inet_diag_msg *diag_msg, int protocol, int family) { + char local_addr_buf[INET6_ADDRSTRLEN] = {0}; + char remote_addr_buf[INET6_ADDRSTRLEN] = {0}; // set up data structures depending on idiag_family type if (diag_msg->idiag_family == AF_INET) { @@ -130,6 +128,100 @@ Row getDiagMessage(struct inet_diag_msg *diag_msg, int protocol, int family) { return row; } +std::string addressFromHex(const std::string &encoded_address, int family) { + char addr_buffer[INET6_ADDRSTRLEN] = {0}; + if (family == AF_INET) { + struct in_addr decoded; + if (encoded_address.length() == 8) { + sscanf(encoded_address.c_str(), "%X", &(decoded.s_addr)); + inet_ntop(AF_INET, &decoded, addr_buffer, INET_ADDRSTRLEN); + } + } else if (family == AF_INET6) { + struct in6_addr decoded; + if (encoded_address.length() == 32) { + sscanf(encoded_address.c_str(), + "%8x%8x%8x%8x", + (unsigned int *)&(decoded.s6_addr[0]), + (unsigned int *)&(decoded.s6_addr[4]), + (unsigned int *)&(decoded.s6_addr[8]), + (unsigned int *)&(decoded.s6_addr[12])); + inet_ntop(AF_INET6, &decoded, addr_buffer, INET6_ADDRSTRLEN); + } + } + + return TEXT(addr_buffer); +} + +unsigned short portFromHex(const std::string &encoded_port) { + unsigned short decoded = 0; + if (encoded_port.length() == 4) { + sscanf(encoded_port.c_str(), "%hX", &decoded); + } + return decoded; +} + +/// A fallback method for generating socket information from /proc/net +void genSocketsFromProc(const std::map socket_inodes, + int protocol, + int family, + QueryData &results) { + std::string path = "/proc/net/"; + path += (protocol == IPPROTO_UDP) ? "udp" : "tcp"; + path += (family == AF_INET6) ? "6" : ""; + + std::string content; + if (!osquery::readFile(path, content).ok()) { + // Could not open socket information from /proc. + return; + } + + // The system's socket information is tokenized by line. + size_t index = 0; + for (const auto &line : osquery::split(content, "\n")) { + index += 1; + if (index == 1) { + // The first line is a textual header and will be ignored. + if (line.find("sl") != 0) { + // Header fields are unknown, stop parsing. + break; + } + continue; + } + + // The socket information is tokenized by spaces, each a field. + auto fields = osquery::split(line, " "); + if (fields.size() < 10) { + // Unknown/malformed socket information. + continue; + } + + // Two of the fields are the local/remote address/port pairs. + auto locals = osquery::split(fields[1], ":"); + auto remotes = osquery::split(fields[2], ":"); + if (locals.size() != 2 || remotes.size() != 2) { + // Unknown/malformed socket information. + continue; + } + + Row r; + r["socket"] = fields[9]; + r["family"] = INTEGER(family); + r["protocol"] = INTEGER(protocol); + r["local_address"] = addressFromHex(locals[0], family); + r["local_port"] = INTEGER(portFromHex(locals[1])); + r["remote_address"] = addressFromHex(remotes[0], family); + r["remote_port"] = INTEGER(portFromHex(remotes[1])); + + if (socket_inodes.count(r["socket"]) > 0) { + r["pid"] = socket_inodes.at(r["socket"]); + } else { + r["pid"] = "-1"; + } + + results.push_back(r); + } +} + void genSocketsForFamily(const std::map socket_inodes, int protocol, int family, @@ -141,7 +233,7 @@ void genSocketsForFamily(const std::map socket_inodes, } // send the inet_diag message - if (send_diag_msg(nl_sock, protocol, family) < 0) { + if (sendNLDiagMessage(nl_sock, protocol, family) < 0) { close(nl_sock); return; } @@ -161,14 +253,13 @@ void genSocketsForFamily(const std::map socket_inodes, } if (nlh->nlmsg_type == NLMSG_ERROR) { - auto error = (struct nlmsgerr *) NLMSG_DATA(nlh); - VLOG(1) << "NETLINK message error " << error->error; + genSocketsFromProc(socket_inodes, protocol, family, results); break; } // parse and process netlink message auto diag_msg = (struct inet_diag_msg *)NLMSG_DATA(nlh); - auto row = getDiagMessage(diag_msg, protocol, family); + auto row = getNLDiagMessage(diag_msg, protocol, family); if (socket_inodes.count(row["socket"]) > 0) { row["pid"] = socket_inodes.at(row["socket"]); diff --git a/osquery/tables/networking/linux/routes.cpp b/osquery/tables/networking/linux/routes.cpp index a0ab1bb..30b8402 100644 --- a/osquery/tables/networking/linux/routes.cpp +++ b/osquery/tables/networking/linux/routes.cpp @@ -184,7 +184,7 @@ QueryData genRoutes(QueryContext& context) { } // Wrap the read socket to support multi-netlink messages - size_t size; + size_t size = 0; if (!readNetlink(socket_fd, 1, (char*)netlink_msg, &size).ok()) { VLOG(1) << "Cannot read NETLINK response from socket"; close(socket_fd); diff --git a/osquery/tables/networking/utils.cpp b/osquery/tables/networking/utils.cpp index 0b92052..f44761d 100644 --- a/osquery/tables/networking/utils.cpp +++ b/osquery/tables/networking/utils.cpp @@ -106,7 +106,7 @@ std::string macAsString(const struct ifaddrs *addr) { int socket_fd = socket(AF_INET, SOCK_DGRAM, 0); ifr.ifr_addr.sa_family = AF_INET; - strcpy(ifr.ifr_name, addr->ifa_name); + strncpy(ifr.ifr_name, addr->ifa_name, IFNAMSIZ); ioctl(socket_fd, SIOCGIFHWADDR, &ifr); close(socket_fd); diff --git a/osquery/tables/specs/linux/kernel_modules.table b/osquery/tables/specs/linux/kernel_modules.table index 9ce31df..d4566fb 100644 --- a/osquery/tables/specs/linux/kernel_modules.table +++ b/osquery/tables/specs/linux/kernel_modules.table @@ -1,8 +1,9 @@ table_name("kernel_modules") +description("Linux kernel modules both loaded and within the load search path.") schema([ - Column("name", TEXT), - Column("size", TEXT), - Column("used_by", TEXT), + Column("name", TEXT, "Module name"), + Column("size", TEXT, "Size of module content"), + Column("used_by", TEXT, "Module reverse dependencies"), Column("status", TEXT), Column("address", TEXT), ]) diff --git a/osquery/tables/specs/linux/memory_map.table b/osquery/tables/specs/linux/memory_map.table new file mode 100644 index 0000000..08d51e7 --- /dev/null +++ b/osquery/tables/specs/linux/memory_map.table @@ -0,0 +1,9 @@ +table_name("memory_map") +description("OS memory region map.") +schema([ + Column("region", INTEGER, "Region index"), + Column("type", TEXT, "Textual description"), + Column("start", TEXT, "Start address of memory region"), + Column("end", TEXT, "End address of memory region"), +]) +implementation("memory_map@genMemoryMap") diff --git a/osquery/tables/specs/linux/process_memory_map.table b/osquery/tables/specs/linux/process_memory_map.table new file mode 100644 index 0000000..ee9718a --- /dev/null +++ b/osquery/tables/specs/linux/process_memory_map.table @@ -0,0 +1,14 @@ +table_name("process_memory_map") +description("Process memory mapped files and pseudo device/regions.") +schema([ + Column("pid", INTEGER), + Column("start", TEXT, "Virtual start address (hex)"), + Column("end", TEXT, "Virtual end address (hex)"), + Column("permissions", TEXT, "r=read, w=write, x=execute, p=private (cow)"), + Column("offset", BIGINT, "Offset into mapped path"), + Column("device", TEXT, "MA:MI Major/minor device ID"), + Column("inode", INTEGER, "Mapped path inode, 0 means uninitialized (BSS)"), + Column("path", TEXT, "Path to mapped file or mapped type"), + Column("is_pseudo", INTEGER, "1 if path is a pseudo path, else 0"), +]) +implementation("processes@genProcessMemoryMap") diff --git a/osquery/tables/specs/linux/shared_memory.table b/osquery/tables/specs/linux/shared_memory.table new file mode 100644 index 0000000..580e24d --- /dev/null +++ b/osquery/tables/specs/linux/shared_memory.table @@ -0,0 +1,18 @@ +table_name("shared_memory") +description("OS shared memory regions.") +schema([ + Column("shmid", INTEGER, "Shared memory segment ID"), + Column("owner_uid", BIGINT, "User ID of owning process"), + Column("creator_uid", BIGINT, "User ID of creator process"), + Column("pid", BIGINT, "Process ID to last use the segment"), + Column("creator_pid", BIGINT, "Process ID that created the segment"), + Column("atime", BIGINT, "Attached time"), + Column("dtime", BIGINT, "Detached time"), + Column("ctime", BIGINT, "Changed time"), + Column("permissions", TEXT, "Memory segment permissions"), + Column("size", BIGINT, "Size in bytes"), + Column("attached", INTEGER, "Number of attached processes"), + Column("status", TEXT, "Destination/attach status"), + Column("locked", INTEGER, "1 if segment is locked else 0"), +]) +implementation("shared_memory@genSharedMemory") diff --git a/osquery/tables/specs/x/acpi_tables.table b/osquery/tables/specs/x/acpi_tables.table index 1ed8290..1e257c7 100644 --- a/osquery/tables/specs/x/acpi_tables.table +++ b/osquery/tables/specs/x/acpi_tables.table @@ -1,7 +1,8 @@ table_name("acpi_tables") +description("Firmware ACPI functional table common metadata and content.") schema([ - Column("name", TEXT), - Column("size", INTEGER), - Column("md5", TEXT), + Column("name", TEXT, "ACPI table name"), + Column("size", INTEGER, "Size of compiled table data"), + Column("md5", TEXT, "MD5 hash of table content"), ]) implementation("system/acpi_tables@genACPITables") diff --git a/osquery/tables/specs/x/arp_cache.table b/osquery/tables/specs/x/arp_cache.table index ebe57b8..cb4bf74 100644 --- a/osquery/tables/specs/x/arp_cache.table +++ b/osquery/tables/specs/x/arp_cache.table @@ -1,8 +1,9 @@ table_name("arp_cache") +description("Address resolution cache, both static and dynamic (from ARP, NDP).") schema([ - Column("address", TEXT), - Column("mac", TEXT), - Column("interface", TEXT), + Column("address", TEXT, "IPv4 address target"), + Column("mac", TEXT, "MAC address of broadcasted address"), + Column("interface", TEXT, "Interface of the network for the MAC"), Column("permanent", TEXT, "1 for true, 0 for false"), ]) implementation("linux/arp_cache,darwin/routes@genArpCache") diff --git a/osquery/tables/specs/x/file.table b/osquery/tables/specs/x/file.table index 436b72d..59b5059 100644 --- a/osquery/tables/specs/x/file.table +++ b/osquery/tables/specs/x/file.table @@ -1,9 +1,23 @@ table_name("file") +description("Interactive filesystem attributes and metadata.") schema([ - Column("path", TEXT, "Must provide a path", required=True), - Column("filename", TEXT), - Column("is_file", INTEGER), - Column("is_dir", INTEGER), - Column("is_link", INTEGER), + Column("path", TEXT, "Absolute file path", required=True), + Column("filename", TEXT, "Name portion of file path"), + Column("inode", BIGINT), + Column("uid", BIGINT, "Owning user ID"), + Column("gid", BIGINT, "Owning group ID"), + Column("mode", TEXT, "Permission bits"), + Column("device", BIGINT, "Device ID (optional)"), + Column("size", BIGINT, "Size of file in bytes"), + Column("block_size", INTEGER, "Block size of filesystem"), + Column("atime", BIGINT, "Last access time"), + Column("mtime", BIGINT, "Last modification time"), + Column("ctime", BIGINT, "Creation time"), + Column("hard_links", INTEGER, "Number of hard links"), + Column("is_file", INTEGER, "1 if a file node else 0"), + Column("is_dir", INTEGER, "1 if a directory (not file) else 0"), + Column("is_link", INTEGER, "1 if a symlink else 0"), + Column("is_char", INTEGER, "1 if a character special device else 0"), + Column("is_block", INTEGER, "1 if a block special device else 0"), ]) implementation("utility/file@genFile") diff --git a/osquery/tables/specs/x/groups.table b/osquery/tables/specs/x/groups.table index 5a41884..8442051 100644 --- a/osquery/tables/specs/x/groups.table +++ b/osquery/tables/specs/x/groups.table @@ -1,7 +1,8 @@ table_name("groups") +description("Local system groups.") schema([ - Column("gid", BIGINT), - Column("gid_signed", BIGINT), - Column("groupname", TEXT), + Column("gid", BIGINT, "Unsigned int64 group ID"), + Column("gid_signed", BIGINT, "A signed int64 version of gid"), + Column("groupname", TEXT, "Canonical local group name"), ]) implementation("groups@genGroups") diff --git a/osquery/tables/specs/x/hardware_events.table b/osquery/tables/specs/x/hardware_events.table index 431e74d..3acfa39 100644 --- a/osquery/tables/specs/x/hardware_events.table +++ b/osquery/tables/specs/x/hardware_events.table @@ -1,15 +1,17 @@ table_name("hardware_events") +description("Hardware (PCI/USB/HID) events from UDEV or IOKit.") schema([ - Column("action", TEXT), - Column("path", TEXT), - Column("type", TEXT), - Column("driver", TEXT), - Column("model", TEXT), + Column("action", TEXT, "Remove, insert, change properties, etc"), + Column("path", TEXT, "Local device path assigned (optional)"), + Column("type", TEXT, "Type of hardware and hardware event"), + Column("driver", TEXT, "Driver claiming the device"), + Column("model", TEXT, "Hadware device model"), Column("model_id", INTEGER), - Column("vendor", TEXT), + Column("vendor", TEXT, "hardware device vendor"), Column("vendor_id", INTEGER), - Column("serial", TEXT), - Column("revision", INTEGER), - Column("time", INTEGER), + Column("serial", TEXT, "Device serial (optional)"), + Column("revision", INTEGER, "Device revision (optional)"), + Column("time", INTEGER, "Time of hardware event"), ]) -implementation("events/hardware_events@HardwareEventSubscriber::genTable") +attributes(event_subscriber=True) +implementation("events/hardware_events@hardware_events::genTable") diff --git a/osquery/tables/specs/x/hash.table b/osquery/tables/specs/x/hash.table index 6a8f38c..03e7096 100644 --- a/osquery/tables/specs/x/hash.table +++ b/osquery/tables/specs/x/hash.table @@ -1,4 +1,5 @@ table_name("hash") +description("Filesystem hash data.") schema([ Column("path", TEXT, "Must provide a path or directory", required=True), Column("directory", TEXT, "Must provide a path or directory", required=True), diff --git a/osquery/tables/specs/x/kernel_info.table b/osquery/tables/specs/x/kernel_info.table new file mode 100644 index 0000000..b938c68 --- /dev/null +++ b/osquery/tables/specs/x/kernel_info.table @@ -0,0 +1,10 @@ +table_name("kernel_info") +description("Basic active kernel information.") +schema([ + Column("version", TEXT), + Column("arguments", TEXT), + Column("path", TEXT), + Column("device", TEXT), + Column("md5", TEXT), +]) +implementation("system/kernel_info@genKernelInfo") diff --git a/osquery/tables/specs/x/last.table b/osquery/tables/specs/x/last.table index 1dea38f..0bf072f 100644 --- a/osquery/tables/specs/x/last.table +++ b/osquery/tables/specs/x/last.table @@ -1,4 +1,5 @@ table_name("last") +description("System logins and logouts.") schema([ Column("username", TEXT), Column("tty", TEXT), diff --git a/osquery/tables/specs/x/listening_ports.table b/osquery/tables/specs/x/listening_ports.table index c883ca4..0b2f0f2 100644 --- a/osquery/tables/specs/x/listening_ports.table +++ b/osquery/tables/specs/x/listening_ports.table @@ -1,9 +1,10 @@ table_name("listening_ports") +description("Processes with listening (bound) network sockets/ports.") schema([ - Column("pid", INTEGER), - Column("port", INTEGER), - Column("protocol", INTEGER), - Column("family", INTEGER), - Column("address", TEXT), + Column("pid", INTEGER, "Process (or thread) ID"), + Column("port", INTEGER, "Transport layer port"), + Column("protocol", INTEGER, "Transport protocol (TCP/UDP)"), + Column("family", INTEGER, "Network protocol (IPv4, IPv6)"), + Column("address", TEXT, "Specific address for bind"), ]) implementation("listening_ports@genListeningPorts") diff --git a/osquery/tables/specs/x/logged_in_users.table b/osquery/tables/specs/x/logged_in_users.table index a96e813..135a643 100644 --- a/osquery/tables/specs/x/logged_in_users.table +++ b/osquery/tables/specs/x/logged_in_users.table @@ -1,4 +1,5 @@ table_name("logged_in_users") +description("Users with an active shell on the system.") schema([ Column("user", TEXT), Column("tty", TEXT), diff --git a/osquery/tables/specs/x/mounts.table b/osquery/tables/specs/x/mounts.table index da5e274..ec055ee 100644 --- a/osquery/tables/specs/x/mounts.table +++ b/osquery/tables/specs/x/mounts.table @@ -1,4 +1,5 @@ table_name("mounts") +description("System mounted devices and filesystems (not process specific).") schema([ Column("device", TEXT), Column("device_alias", TEXT), diff --git a/osquery/tables/specs/x/osquery_flags.table b/osquery/tables/specs/x/osquery_flags.table index 18b907d..f60787e 100644 --- a/osquery/tables/specs/x/osquery_flags.table +++ b/osquery/tables/specs/x/osquery_flags.table @@ -1,4 +1,5 @@ table_name("osquery_flags") +description("Configurable flags that modify osquery's behavior.") schema([ Column("name", TEXT), Column("type", TEXT), diff --git a/osquery/tables/specs/x/osquery_info.table b/osquery/tables/specs/x/osquery_info.table index b55d0bb..3f1682a 100644 --- a/osquery/tables/specs/x/osquery_info.table +++ b/osquery/tables/specs/x/osquery_info.table @@ -1,4 +1,5 @@ table_name("osquery_info") +description("Top level information about the running version of osquery.") schema([ Column("version", TEXT), Column("config_md5", TEXT), diff --git a/osquery/tables/specs/x/passwd_changes.table b/osquery/tables/specs/x/passwd_changes.table index f6d89b0..e854c67 100644 --- a/osquery/tables/specs/x/passwd_changes.table +++ b/osquery/tables/specs/x/passwd_changes.table @@ -1,9 +1,10 @@ table_name("passwd_changes") -description("Mostly an example use of events.") +description("Track time, action changes to /etc/passwd.") schema([ Column("target_path", TEXT, "The path changed"), Column("time", TEXT), Column("action", TEXT, "Change action (UPDATE, REMOVE, etc)"), Column("transaction_id", BIGINT, "ID used during bulk update"), ]) -implementation("passwd_changes@PasswdChangesEventSubscriber::genTable") +attributes(event_subscriber=True) +implementation("passwd_changes@passwd_changes::genTable") diff --git a/osquery/tables/specs/x/pci_devices.table b/osquery/tables/specs/x/pci_devices.table index 472bd92..4856a5f 100644 --- a/osquery/tables/specs/x/pci_devices.table +++ b/osquery/tables/specs/x/pci_devices.table @@ -1,4 +1,5 @@ table_name("pci_devices") +description("PCI devices active on the host system.") schema([ Column("pci_slot", TEXT), Column("pci_class", TEXT), diff --git a/osquery/tables/specs/x/process_envs.table b/osquery/tables/specs/x/process_envs.table index 4b864a0..76e860c 100644 --- a/osquery/tables/specs/x/process_envs.table +++ b/osquery/tables/specs/x/process_envs.table @@ -2,8 +2,6 @@ table_name("process_envs") description("A key/value table of environment variables for each process.") schema([ Column("pid", INTEGER), - Column("name", TEXT, "Process name"), - Column("path", TEXT, "Process path"), Column("key", TEXT, "Environment variable name"), Column("value", TEXT, "Environment variable value"), ForeignKey(column="pid", table="processes"), diff --git a/osquery/tables/specs/x/process_open_files.table b/osquery/tables/specs/x/process_open_files.table index 8ab3625..d2c14bf 100644 --- a/osquery/tables/specs/x/process_open_files.table +++ b/osquery/tables/specs/x/process_open_files.table @@ -1,7 +1,8 @@ table_name("process_open_files") +description("File descriptors for each process.") schema([ - Column("pid", BIGINT), - Column("fd", BIGINT), - Column("path", TEXT), + Column("pid", BIGINT, "Process (or thread) ID"), + Column("fd", BIGINT, "Process-specific file descriptor number"), + Column("path", TEXT, "Filesystem path of descriptor"), ]) implementation("system/process_open_files@genOpenFiles") diff --git a/osquery/tables/specs/x/process_open_sockets.table b/osquery/tables/specs/x/process_open_sockets.table index 5983369..f3e9f97 100644 --- a/osquery/tables/specs/x/process_open_sockets.table +++ b/osquery/tables/specs/x/process_open_sockets.table @@ -1,4 +1,5 @@ table_name("process_open_sockets") +description("Processes which have open network sockets on the system.") schema([ Column("pid", INTEGER), Column("socket", INTEGER), diff --git a/osquery/tables/specs/x/processes.table b/osquery/tables/specs/x/processes.table index e99a267..ba18966 100644 --- a/osquery/tables/specs/x/processes.table +++ b/osquery/tables/specs/x/processes.table @@ -1,4 +1,5 @@ table_name("processes") +description("All running processes on the host system.") schema([ Column("pid", INTEGER), Column("name", TEXT, "The process path or shorthand argv[0]"), diff --git a/osquery/tables/specs/x/smbios_tables.table b/osquery/tables/specs/x/smbios_tables.table index 94c01ee..a7b1981 100644 --- a/osquery/tables/specs/x/smbios_tables.table +++ b/osquery/tables/specs/x/smbios_tables.table @@ -1,4 +1,5 @@ table_name("smbios_tables") +description("BIOS (DMI) structure common details and content.") schema([ Column("number", INTEGER), Column("type", INTEGER), diff --git a/osquery/tables/specs/x/suid_bin.table b/osquery/tables/specs/x/suid_bin.table index 0403abf..313c233 100644 --- a/osquery/tables/specs/x/suid_bin.table +++ b/osquery/tables/specs/x/suid_bin.table @@ -1,4 +1,5 @@ table_name("suid_bin") +description("suid binaries in common locations.") schema([ Column("path", TEXT), Column("username", TEXT), diff --git a/osquery/tables/specs/x/time.table b/osquery/tables/specs/x/time.table index 2a98ec2..63194f4 100644 --- a/osquery/tables/specs/x/time.table +++ b/osquery/tables/specs/x/time.table @@ -1,4 +1,5 @@ table_name("time") +description("Track current time in the system.") schema([ Column("hour", INTEGER), Column("minutes", INTEGER), diff --git a/osquery/tables/specs/x/usb_devices.table b/osquery/tables/specs/x/usb_devices.table index 58c2afb..a0692ea 100644 --- a/osquery/tables/specs/x/usb_devices.table +++ b/osquery/tables/specs/x/usb_devices.table @@ -1,4 +1,5 @@ table_name("usb_devices") +description("USB devices that are actively plugged into the host system.") schema([ Column("usb_address", INTEGER), Column("usb_port", INTEGER), diff --git a/osquery/tables/specs/x/users.table b/osquery/tables/specs/x/users.table index 636b839..74ecd60 100644 --- a/osquery/tables/specs/x/users.table +++ b/osquery/tables/specs/x/users.table @@ -1,12 +1,13 @@ table_name("users") +description("Local system users.") schema([ - Column("uid", BIGINT), - Column("gid", BIGINT), - Column("uid_signed", BIGINT), - Column("gid_signed", BIGINT), + Column("uid", BIGINT, "User ID"), + Column("gid", BIGINT, "Group ID (unsigned)"), + Column("uid_signed", BIGINT, "User ID as int64 signed (Apple)"), + Column("gid_signed", BIGINT, "Group ID as int64 signed (Apple)"), Column("username", TEXT), - Column("description", TEXT), - Column("directory", TEXT), - Column("shell", TEXT), + Column("description", TEXT, "Optional user description"), + Column("directory", TEXT, "User's home directory"), + Column("shell", TEXT, "User's configured default shell"), ]) implementation("users@genUsers") diff --git a/osquery/tables/system/linux/acpi_tables.cpp b/osquery/tables/system/linux/acpi_tables.cpp index 8ca85db..c7ce6b3 100644 --- a/osquery/tables/system/linux/acpi_tables.cpp +++ b/osquery/tables/system/linux/acpi_tables.cpp @@ -8,6 +8,8 @@ * */ +#include + #include #include #include diff --git a/osquery/tables/system/linux/kernel_info.cpp b/osquery/tables/system/linux/kernel_info.cpp new file mode 100644 index 0000000..d3cc2ff --- /dev/null +++ b/osquery/tables/system/linux/kernel_info.cpp @@ -0,0 +1,79 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ + +#include + +#include +#include +#include +#include +#include + +namespace osquery { +namespace tables { + +const std::string kKernelArgumentsPath = "/proc/cmdline"; +const std::string kKernelSignaturePath = "/proc/version"; + +QueryData genKernelInfo(QueryContext& context) { + QueryData results; + Row r; + + if (pathExists(kKernelArgumentsPath).ok()) { + std::string arguments_line; + // Grab the whole arguments string from proc. + if (readFile(kKernelArgumentsPath, arguments_line).ok()) { + auto arguments = split(arguments_line, " "); + std::string additional_arguments; + + // Iterate over each space-tokenized argument. + for (const auto& argument : arguments) { + if (argument.substr(0, 11) == "BOOT_IMAGE=") { + r["path"] = argument.substr(11); + } else if (argument.substr(0, 5) == "root=") { + r["device"] = argument.substr(5); + } else { + if (additional_arguments.size() > 0) { + additional_arguments += " "; + } + additional_arguments += argument; + } + } + r["arguments"] = additional_arguments; + } + } else { + VLOG(1) << "Cannot find kernel arguments file: " << kKernelArgumentsPath; + } + + if (pathExists(kKernelSignaturePath).ok()) { + std::string signature; + + // The signature includes the kernel version, build data, buildhost, + // GCC version used, and possibly build date. + if (readFile(kKernelSignaturePath, signature).ok()) { + auto details = split(signature, " "); + if (details.size() > 2 && details[1] == "version") { + r["version"] = details[2]; + } + } + } else { + VLOG(1) << "Cannot find kernel signature file: " << kKernelSignaturePath; + } + + // Using the path of the boot image, attempt to calculate a hash. + if (r.count("path") > 0) { + r["md5"] = hashFromFile(HASH_TYPE_MD5, r.at("path")); + } + + results.push_back(r); + return results; +} +} +} diff --git a/osquery/tables/system/linux/memory_map.cpp b/osquery/tables/system/linux/memory_map.cpp new file mode 100644 index 0000000..77dfe58 --- /dev/null +++ b/osquery/tables/system/linux/memory_map.cpp @@ -0,0 +1,61 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ + +#include + +#include +#include +#include +#include + +namespace fs = boost::filesystem; + +namespace osquery { +namespace tables { + +const std::string kMemoryMapLocation = "/sys/firmware/memmap"; + +QueryData genMemoryMap(QueryContext& context) { + QueryData results; + + // Linux memory map is exposed in /sys. + std::vector regions; + auto status = listDirectoriesInDirectory(kMemoryMapLocation, regions); + if (!status.ok()) { + return {}; + } + + for (const auto& index : regions) { + fs::path index_path(index); + Row r; + r["region"] = index_path.filename().string(); + + // The type is a textual description + std::string content; + readFile(index_path / "type", content); + boost::trim(content); + r["type"] = content; + + // Keep these in 0xFFFF (hex) form. + readFile(index_path / "start", content); + boost::trim(content); + r["start"] = content; + + readFile(index_path / "end", content); + boost::trim(content); + r["end"] = content; + + results.push_back(r); + } + + return results; +} +} +} diff --git a/osquery/tables/system/linux/processes.cpp b/osquery/tables/system/linux/processes.cpp index 0664210..8c97a05 100644 --- a/osquery/tables/system/linux/processes.cpp +++ b/osquery/tables/system/linux/processes.cpp @@ -35,22 +35,18 @@ namespace tables { PROC_FILLCOM | PROC_FILLMEM | PROC_FILLSTATUS | PROC_FILLSTAT #endif -std::string proc_name(const proc_t* proc_info) { +std::string getProcName(const proc_t* proc_info) { return std::string(proc_info->cmd); } -std::string proc_attr(const std::string& attr, const proc_t* proc_info) { - std::stringstream filename; - - filename << "/proc/" << proc_info->tid << "/" << attr; - return filename.str(); +std::string getProcAttr(const std::string& attr, const proc_t* proc_info) { + return "/proc/" + std::to_string(proc_info->tid) + "/" + attr; } -std::string proc_cmdline(const proc_t* proc_info) { - std::string attr; - std::string result; +std::string readProcCMDLine(const proc_t* proc_info) { + auto attr = getProcAttr("cmdline", proc_info); - attr = proc_attr("cmdline", proc_info); + std::string result; std::ifstream fd(attr, std::ios::in | std::ios::binary); if (fd) { result = std::string(std::istreambuf_iterator(fd), @@ -65,20 +61,15 @@ std::string proc_cmdline(const proc_t* proc_info) { return result; } -std::string proc_link(const proc_t* proc_info) { - std::string attr; - std::string result; - char* link_path; - long path_max; - int bytes; - +std::string readProcLink(const proc_t* proc_info) { // The exe is a symlink to the binary on-disk. - attr = proc_attr("exe", proc_info); - path_max = pathconf(attr.c_str(), _PC_PATH_MAX); - link_path = (char*)malloc(path_max); - + auto attr = getProcAttr("exe", proc_info); + long path_max = pathconf(attr.c_str(), _PC_PATH_MAX); + auto link_path = (char*)malloc(path_max); memset(link_path, 0, path_max); - bytes = readlink(attr.c_str(), link_path, path_max); + + std::string result; + int bytes = readlink(attr.c_str(), link_path, path_max); if (bytes >= 0) { result = std::string(link_path); } @@ -87,23 +78,61 @@ std::string proc_link(const proc_t* proc_info) { return result; } -std::map proc_env(const proc_t* proc_info) { - std::map env; - std::string attr = proc_attr("environ", proc_info); - std::string buf; +void genProcessEnvironment(const proc_t* proc_info, QueryData& results) { + auto attr = getProcAttr("environ", proc_info); std::ifstream fd(attr, std::ios::in | std::ios::binary); - + std::string buf; while (!(fd.fail() || fd.eof())) { std::getline(fd, buf, '\0'); size_t idx = buf.find_first_of("="); - std::string key = buf.substr(0, idx); - std::string value = buf.substr(idx + 1); + Row r; + r["pid"] = INTEGER(proc_info->tid); + r["key"] = buf.substr(0, idx); + r["value"] = buf.substr(idx + 1); + results.push_back(r); + } +} + +void genProcessMap(const proc_t* proc_info, QueryData& results) { + auto map = getProcAttr("maps", proc_info); + + std::ifstream fd(map, std::ios::in | std::ios::binary); + std::string line; + while (!(fd.fail() || fd.eof())) { + std::getline(fd, line, '\n'); + auto fields = osquery::split(line, " "); + + Row r; + r["pid"] = INTEGER(proc_info->tid); + + // If can't read address, not sure. + if (fields.size() < 5) { + continue; + } - env[key] = value; + if (fields[0].size() > 0) { + auto addresses = osquery::split(fields[0], "-"); + r["start"] = "0x" + addresses[0]; + r["end"] = "0x" + addresses[1]; + } + + r["permissions"] = fields[1]; + r["offset"] = BIGINT(std::stoll(fields[2], nullptr, 16)); + r["device"] = fields[3]; + r["inode"] = fields[4]; + + // Path name must be trimmed. + if (fields.size() > 5) { + boost::trim(fields[5]); + r["path"] = fields[5]; + } + + // BSS with name in pathname. + r["is_pseudo"] = (fields[4] == "0" && r["path"].size() > 0) ? "1" : "0"; + results.push_back(r); } - return env; } /** @@ -111,7 +140,7 @@ std::map proc_env(const proc_t* proc_info) { * * @param p The rbuf to free */ -void standard_freeproc(proc_t* p) { +void standardFreeproc(proc_t* p) { if (!p) { // in case p is NULL return; } @@ -140,18 +169,23 @@ QueryData genProcesses(QueryContext& context) { // Populate proc struc for each process. while ((proc_info = readproc(proc, NULL))) { - Row r; + if (!context.constraints["pid"].matches(proc_info->tid)) { + // Optimize by not searching when a pid is a constraint. + standardFreeproc(proc_info); + continue; + } + Row r; r["pid"] = INTEGER(proc_info->tid); r["uid"] = BIGINT((unsigned int)proc_info->ruid); r["gid"] = BIGINT((unsigned int)proc_info->rgid); r["euid"] = BIGINT((unsigned int)proc_info->euid); r["egid"] = BIGINT((unsigned int)proc_info->egid); - r["name"] = proc_name(proc_info); - std::string cmdline = proc_cmdline(proc_info); + r["name"] = getProcName(proc_info); + std::string cmdline = readProcCMDLine(proc_info); boost::algorithm::trim(cmdline); r["cmdline"] = cmdline; - r["path"] = proc_link(proc_info); + r["path"] = readProcLink(proc_info); r["on_disk"] = osquery::pathExists(r["path"]).toString(); r["resident_size"] = INTEGER(proc_info->vm_rss); @@ -162,7 +196,7 @@ QueryData genProcesses(QueryContext& context) { r["parent"] = INTEGER(proc_info->ppid); results.push_back(r); - standard_freeproc(proc_info); + standardFreeproc(proc_info); } closeproc(proc); @@ -177,20 +211,25 @@ QueryData genProcessEnvs(QueryContext& context) { PROCTAB* proc = openproc(PROC_SELECTS); // Populate proc struc for each process. - while ((proc_info = readproc(proc, NULL))) { - auto env = proc_env(proc_info); - for (auto itr = env.begin(); itr != env.end(); ++itr) { - Row r; - r["pid"] = INTEGER(proc_info->tid); - r["name"] = proc_name(proc_info); - r["path"] = proc_link(proc_info); - r["key"] = itr->first; - r["value"] = itr->second; - results.push_back(r); - } + genProcessEnvironment(proc_info, results); + standardFreeproc(proc_info); + } - standard_freeproc(proc_info); + closeproc(proc); + + return results; +} + +QueryData genProcessMemoryMap(QueryContext& context) { + QueryData results; + + proc_t* proc_info; + PROCTAB* proc = openproc(PROC_SELECTS); + + while ((proc_info = readproc(proc, NULL))) { + genProcessMap(proc_info, results); + standardFreeproc(proc_info); } closeproc(proc); diff --git a/osquery/tables/system/linux/shared_memory.cpp b/osquery/tables/system/linux/shared_memory.cpp new file mode 100644 index 0000000..1c06125 --- /dev/null +++ b/osquery/tables/system/linux/shared_memory.cpp @@ -0,0 +1,86 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ + +#include +#include + +#include +#include +#include + +namespace osquery { +namespace tables { + +struct shm_info { + int used_ids; + unsigned long shm_tot; + unsigned long shm_rss; + unsigned long shm_swp; + unsigned long swap_attempts; + unsigned long swap_successes; +}; + +QueryData genSharedMemory(QueryContext &context) { + QueryData results; + + // Use shared memory control (shmctl) to get the max SHMID. + struct shm_info shm_info; + int maxid = shmctl(0, SHM_INFO, (struct shmid_ds *)(void *)&shm_info); + if (maxid < 0) { + VLOG(1) << "Linux kernel not configured for shared memory"; + return {}; + } + + // Use a static pointer to access IPC permissions structure. + struct shmid_ds shmseg; + struct ipc_perm *ipcp = &shmseg.shm_perm; + + // Then iterate each shared memory ID up to the max. + for (int id = 0; id <= maxid; id++) { + int shmid = shmctl(id, SHM_STAT, &shmseg); + if (shmid < 0) { + continue; + } + + Row r; + r["shmid"] = INTEGER(shmid); + + struct passwd *pw = getpwuid(shmseg.shm_perm.uid); + if (pw != nullptr) { + r["owner_uid"] = BIGINT(pw->pw_uid); + } + + pw = getpwuid(shmseg.shm_perm.cuid); + if (pw != nullptr) { + r["creator_uid"] = BIGINT(pw->pw_uid); + } + + // Accessor, creator pids. + r["pid"] = BIGINT(shmseg.shm_lpid); + r["creator_pid"] = BIGINT(shmseg.shm_cpid); + + // Access, detached, creator times + r["atime"] = BIGINT(shmseg.shm_atime); + r["dtime"] = BIGINT(shmseg.shm_dtime); + r["ctime"] = BIGINT(shmseg.shm_ctime); + + r["permissions"] = INTEGER(ipcp->mode); + r["size"] = BIGINT(shmseg.shm_segsz); + r["attached"] = INTEGER(shmseg.shm_nattch); + r["status"] = (ipcp->mode & SHM_DEST) ? "dest" : ""; + r["locked"] = (ipcp->mode & SHM_LOCKED) ? "1" : "0"; + + results.push_back(r); + } + + return results; +} +} +} \ No newline at end of file diff --git a/osquery/tables/templates/amalgamation.cpp.in b/osquery/tables/templates/amalgamation.cpp.in index fddd400..1e34541 100644 --- a/osquery/tables/templates/amalgamation.cpp.in +++ b/osquery/tables/templates/amalgamation.cpp.in @@ -15,13 +15,8 @@ #include #include -#include "osquery/core/virtual_table.h" - namespace osquery { namespace tables { - {% for table in tables %} {{table}} - {% endfor %} - }} diff --git a/osquery/tables/templates/default.cpp.in b/osquery/tables/templates/default.cpp.in index 108103f..5255202 100644 --- a/osquery/tables/templates/default.cpp.in +++ b/osquery/tables/templates/default.cpp.in @@ -3,7 +3,7 @@ * All rights reserved. * * This source code is licensed under the BSD-style license found in the - * LICENSE file in the root directory of this source tree. An additional grant + * LICENSE file in the root directory of this source tree. An additional grant * of patent rights can be found in the PATENTS file in the same directory. * */ @@ -15,8 +15,6 @@ #include #include -#include "osquery/core/virtual_table.h" - namespace osquery { namespace tables { /// BEGIN[GENTABLE] @@ -30,14 +28,15 @@ class {{class_name}} { {% endif %}\ class {{table_name_cc}}TablePlugin : public TablePlugin { -public: - TableName name = "{{table_name}}"; - TableColumns columns = { + private: + TableColumns columns() { + return { {% for column in schema %}\ - std::make_pair("{{column.name}}", "{{column.type.affinity}}")\ + {"{{column.name}}", "{{column.type.affinity}}"}\ {% if not loop.last %}, {% endif %} {% endfor %}\ - }; + }; + } QueryData generate(QueryContext& request) { {% if class_name != "" %}\ @@ -47,16 +46,9 @@ public: return osquery::tables::{{function}}(request); {% endif %}\ } - -public: - {{table_name_cc}}TablePlugin() {} - int attachVtable(sqlite3 *db) { - return sqlite3_attach_vtable<{{table_name_cc}}TablePlugin>(db, name); - } - virtual ~{{table_name_cc}}TablePlugin() {} }; -REGISTER_TABLE("{{table_name}}", std::make_shared<{{table_name_cc}}TablePlugin>()); +REGISTER({{table_name_cc}}TablePlugin, "table", "{{table_name}}"); /// END[GENTABLE] }} diff --git a/osquery/tables/utility/file.cpp b/osquery/tables/utility/file.cpp index 73e84ae..65a1292 100644 --- a/osquery/tables/utility/file.cpp +++ b/osquery/tables/utility/file.cpp @@ -8,6 +8,8 @@ * */ +#include + #include #include @@ -16,6 +18,17 @@ namespace osquery { namespace tables { +inline std::string lsperms(int mode) { + static const char rwx[] = {'0', '1', '2', '3', '4', '5', '6', '7'}; + std::string bits; + + bits += rwx[(mode >> 9) & 7]; + bits += rwx[(mode >> 6) & 7]; + bits += rwx[(mode >> 3) & 7]; + bits += rwx[(mode >> 0) & 7]; + return bits; +} + QueryData genFile(QueryContext& context) { QueryData results; @@ -26,9 +39,34 @@ QueryData genFile(QueryContext& context) { Row r; r["path"] = path.string(); r["filename"] = path.filename().string(); - r["is_file"] = INTEGER(boost::filesystem::is_regular_file(path)); - r["is_dir"] = INTEGER(boost::filesystem::is_directory(path)); - r["is_link"] = INTEGER(boost::filesystem::is_symlink(path)); + + struct stat file_stat, link_stat; + if (lstat(path.string().c_str(), &link_stat) < 0 || + stat(path.string().c_str(), &file_stat)) { + // Path was not real, had too may links, or could not be accessed. + continue; + } + + r["inode"] = BIGINT(file_stat.st_ino); + r["uid"] = BIGINT(file_stat.st_uid); + r["gid"] = BIGINT(file_stat.st_gid); + r["mode"] = std::string(lsperms(file_stat.st_mode)); + r["device"] = BIGINT(file_stat.st_rdev); + r["size"] = BIGINT(file_stat.st_size); + r["block_size"] = INTEGER(file_stat.st_blksize); + r["hard_links"] = INTEGER(file_stat.st_nlink); + + // Times + r["atime"] = BIGINT(file_stat.st_atime); + r["mtime"] = BIGINT(file_stat.st_mtime); + r["ctime"] = BIGINT(file_stat.st_ctime); + + // Type booleans + r["is_file"] = (!S_ISDIR(file_stat.st_mode)) ? "1" : "0"; + r["is_dir"] = (S_ISDIR(file_stat.st_mode)) ? "1" : "0"; + r["is_link"] = (S_ISLNK(link_stat.st_mode)) ? "1" : "0"; + r["is_char"] = (S_ISCHR(file_stat.st_mode)) ? "1" : "0"; + r["is_block"] = (S_ISBLK(file_stat.st_mode)) ? "1" : "0"; results.push_back(r); } diff --git a/osquery/tables/utility/osquery.cpp b/osquery/tables/utility/osquery.cpp index 7eea475..c15e7ed 100644 --- a/osquery/tables/utility/osquery.cpp +++ b/osquery/tables/utility/osquery.cpp @@ -60,6 +60,7 @@ QueryData genOsqueryInfo(QueryContext& context) { if (s.ok()) { r["config_md5"] = TEXT(hash_string); } else { + r["config_md5"] = ""; VLOG(1) << "Could not retrieve config hash: " << s.toString(); } diff --git a/packaging/osquery.spec b/packaging/osquery.spec index 3a80eaa..c6e3939 100644 --- a/packaging/osquery.spec +++ b/packaging/osquery.spec @@ -1,5 +1,5 @@ Name: osquery -Version: 1.3.1 +Version: 1.4.0 Release: 0 License: Apache-2.0 and GPLv2 Summary: A SQL powered operating system instrumentation, monitoring framework. @@ -85,6 +85,7 @@ Testcases for osquery %{_bindir}/osquery_sqlite_util_tests %{_bindir}/osquery_scheduler_tests %{_bindir}/osquery_tables_tests +%{_bindir}/osquery_virtual_table_tests %{_bindir}/osquery_test_util_tests %{_bindir}/osquery_text_tests %{_bindir}/osquery_logger_tests @@ -95,3 +96,4 @@ Testcases for osquery %{_bindir}/osquery_inotify_tests %{_bindir}/osquery_etc_hosts_tests %{_bindir}/osquery_printer_tests +%{_bindir}/osquery_extensions_test diff --git a/tools/codegen/gentable.py b/tools/codegen/gentable.py index 09804c1..dc873c8 100755 --- a/tools/codegen/gentable.py +++ b/tools/codegen/gentable.py @@ -4,7 +4,7 @@ # All rights reserved. # # This source code is licensed under the BSD-style license found in the -# LICENSE file in the root directory of this source tree. An additional grant +# LICENSE file in the root directory of this source tree. An additional grant # of patent rights can be found in the PATENTS file in the same directory. from __future__ import absolute_import @@ -131,6 +131,7 @@ class TableState(Singleton): self.function = "" self.class_name = "" self.description = "" + self.attributes = {} def columns(self): return [i for i in self.schema if isinstance(i, Column)] @@ -148,7 +149,7 @@ class TableState(Singleton): header=self.header, impl=self.impl, function=self.function, - class_name=self.class_name + class_name=self.class_name, ) # Check for reserved column names @@ -214,6 +215,7 @@ def table_name(name): logging.debug(" - called with: %s" % name) table.table_name = name table.description = "" + table.attributes = {} def schema(schema_list): @@ -230,6 +232,15 @@ def schema(schema_list): table.schema = schema_list +def description(text): + table.description = text + + +def attributes(**kwargs): + for attr in kwargs: + table.attributes[attr] = kwargs[attr] + + def implementation(impl_string): """ define the path to the implementation file and the function which @@ -253,10 +264,6 @@ def implementation(impl_string): table.class_name = class_name -def description(text): - table.description = text - - def main(argc, argv): if DEVELOPING: logging.basicConfig(format=LOG_FORMAT, level=logging.DEBUG) @@ -270,18 +277,19 @@ def main(argc, argv): filename = argv[1] output = argv[2] - # Adding a 3rd parameter will enable the blacklist - disable_blacklist = argc > 3 - - setup_templates(filename) - with open(filename, "rU") as file_handle: - tree = ast.parse(file_handle.read()) - exec(compile(tree, "", "exec")) - blacklisted = is_blacklisted(table.table_name, path=filename) - if not disable_blacklist and blacklisted: - table.blacklist(output) - else: - table.generate(output) + if filename.endswith(".table"): + # Adding a 3rd parameter will enable the blacklist + disable_blacklist = argc > 3 + + setup_templates(filename) + with open(filename, "rU") as file_handle: + tree = ast.parse(file_handle.read()) + exec(compile(tree, "", "exec")) + blacklisted = is_blacklisted(table.table_name, path=filename) + if not disable_blacklist and blacklisted: + table.blacklist(output) + else: + table.generate(output) if __name__ == "__main__": main(len(sys.argv), sys.argv) diff --git a/tools/deployment/osquery.example.conf b/tools/deployment/osquery.example.conf index c326005..829bb00 100644 --- a/tools/deployment/osquery.example.conf +++ b/tools/deployment/osquery.example.conf @@ -2,10 +2,10 @@ /* Configure the daemon below */ "options": { // Select the osquery config plugin. - "config_retriever": "filesystem", + "config_plugin": "filesystem", // Select the osquery logging plugin. - "log_receiver": "filesystem", + "logger_plugin": "filesystem", // The log directory stores info, warning, and errors. // If the daemon uses the 'filesystem' logging retriever then the log_dir @@ -17,7 +17,7 @@ //"disable_logging": "false", // Query differential results are logged as change-events to assist log - // aggregation operatinos like searching and transactons. + // aggregation operations like searching and transactons. // Set 'log_results_events' to log differentials as transactions. //"log_result_events": "true", @@ -32,10 +32,6 @@ // Write the pid of the osqueryd process to a pidfile/mutex. //"pidfile": "/var/osquery/osquery.pidfile", - - // Disable the osquery event pubsub functionallity. - // Many tables depend on internal OS API events. - //"event_pubsub": "true", // Clear events from the osquery backing store after a number of seconds. "event_pubsub_expiry": "86000",