From: Sangwan Kwon Date: Thu, 25 Jun 2015 08:35:51 +0000 (-0700) Subject: Bump version to upstream-1.5.0 [stable] X-Git-Tag: accepted/tizen/unified/20200810.122954~211 X-Git-Url: http://review.tizen.org/git/?a=commitdiff_plain;h=4dc917c9401dbe1d608fafb0440fc55e4b71781d;p=platform%2Fcore%2Fsecurity%2Fvist.git Bump version to upstream-1.5.0 [stable] Signed-off-by: Sangwan Kwon --- diff --git a/CMakeLists.txt b/CMakeLists.txt index 8c07767..d799952 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -35,7 +35,7 @@ ADD_DEFINITIONS("-fPIC") #ADD_DEFINITIONS("-pedantic-errors") # TODO(sangwan.kwon): Get version from packing spec. -SET(OSQUERY_BUILD_VERSION "1.4.7") +SET(OSQUERY_BUILD_VERSION "1.5.0") # Set various platform/platform-version/build version/etc defines. ADD_DEFINITIONS(-DOSQUERY_BUILD_VERSION=${OSQUERY_BUILD_VERSION} diff --git a/include/osquery/config.h b/include/osquery/config.h index 8c65854..38c9002 100644 --- a/include/osquery/config.h +++ b/include/osquery/config.h @@ -209,7 +209,19 @@ class Config : private boost::noncopyable { bool force_merge_success_; private: + /** + * @brief A ConfigDataInstance requests read-only access to ConfigParser data. + * + * A ConfigParser plugin will receive several top-level-config keys and + * optionally parse and store information. That information is a property tree + * called ConfigParser::data_. Use ConfigDataInstance::getParsedData to + * retrieve read-only access to this data. + * + * @param parser The name of the config parser. + */ static const pt::ptree& getParsedData(const std::string& parser); + + /// See getParsedData but request access to the parser plugin. static const ConfigPluginRef getParser(const std::string& parser); /// A default, empty property tree used when a missing parser is requested. @@ -265,8 +277,26 @@ class ConfigDataInstance { const pt::ptree& data() const { return Config::getInstance().data_.all_data; } private: + /** + * @brief ConfigParser plugin's may update the internal config representation. + * + * If the config parser reads and calculates new information it should store + * that derived data itself and rely on ConfigDataInstance::getParsedData. + * This means another plugin is aware of the ConfigParser and knowns to make + * getParsedData calls. If the parser is augmenting/changing internal state, + * such as modifying the osquery schedule or options, then it must write + * changed back into the default data. + * + * Note that this returns the ConfigData instance, not the raw property tree. + */ + ConfigData& mutableConfigData() { return Config::getInstance().data_; } + + private: /// A read lock on the reader/writer config data accessor/update mutex. boost::shared_lock lock_; + + private: + friend class ConfigParserPlugin; }; /** @@ -366,6 +396,12 @@ class ConfigParserPlugin : public Plugin { virtual Status update(const ConfigTreeMap& config) = 0; protected: + /// Mutable config data accessor for ConfigParser%s. + ConfigData& mutableConfigData(ConfigDataInstance& cdi) { + return cdi.mutableConfigData(); + } + + protected: /// Allow the config parser to keep some global state. pt::ptree data_; diff --git a/include/osquery/core.h b/include/osquery/core.h index 4d83019..30260da 100644 --- a/include/osquery/core.h +++ b/include/osquery/core.h @@ -53,12 +53,16 @@ extern const std::string kSDKPlatform; * @brief A helpful tool type to report when logging, print help, or debugging. */ enum ToolType { + OSQUERY_TOOL_UNKNOWN = 0, OSQUERY_TOOL_SHELL, OSQUERY_TOOL_DAEMON, OSQUERY_TOOL_TEST, OSQUERY_EXTENSION, }; +/// The osquery tool type for runtime decisions. +extern ToolType kToolType; + class Initializer { public: /** @@ -77,7 +81,7 @@ class Initializer { /** * @brief Sets up the process as an osquery daemon. * - * A daemon has additional constraints, it can use a process mutext, check + * A daemon has additional constraints, it can use a process mutex, check * for sane/non-default configurations, etc. */ void initDaemon(); @@ -87,13 +91,13 @@ class Initializer { * and monitor their utilization. * * A daemon may call initWorkerWatcher to begin watching child daemon - * processes until it-itself is unscheduled. The basic guarentee is that only + * processes until it-itself is unscheduled. The basic guarantee is that only * workers will return from the function. * * The worker-watcher will implement performance bounds on CPU utilization * and memory, as well as check for zombie/defunct workers and respawn them * if appropriate. The appropriateness is determined from heuristics around - * how the worker exitted. Various exit states and velocities may cause the + * how the worker exited. Various exit states and velocities may cause the * watcher to resign. * * @param name The name of the worker process. @@ -104,8 +108,17 @@ class Initializer { void start(); /// Turns off various aspects of osquery such as event loops. void shutdown(); - /// Check if a process is an osquery worker. - bool isWorker(); + + /** + * @brief Check if a process is an osquery worker. + * + * By default an osqueryd process will fork/exec then set an environment + * variable: `OSQUERY_WORKER` while continually monitoring child I/O. + * The environment variable causes subsequent child processes to skip several + * initialization steps and jump into extension handling, registry setup, + * config/logger discovery and then the event publisher and scheduler. + */ + static bool isWorker(); private: /// Initialize this process as an osquery daemon worker. @@ -140,7 +153,7 @@ std::vector split(const std::string& s, * * @param s the string that you'd like to split. * @param delim the delimiter which you'd like to split the string by. - * @param occurences the number of times to split by delim. + * @param occurrences the number of times to split by delim. * * @return a vector of strings split by delim for occurrences. */ @@ -149,7 +162,7 @@ std::vector split(const std::string& s, size_t occurences); /** - * @brief Inline replace all instances of from with to. + * @brief In-line replace all instances of from with to. * * @param str The input/output mutable string. * @param from Search string @@ -201,14 +214,14 @@ std::string generateHostUuid(); std::string getAsciiTime(); /** - * @brief Getter for the current unix time. + * @brief Getter for the current UNIX time. * - * @return an int representing the amount of seconds since the unix epoch + * @return an int representing the amount of seconds since the UNIX epoch */ int getUnixTime(); /** - * @brief Inline helper function for use with utf8StringSize + * @brief In-line helper function for use with utf8StringSize */ template inline size_t incUtf8StringIterator(_Iterator1& it, const _Iterator2& last) { diff --git a/include/osquery/database.h b/include/osquery/database.h index 4a9b2c4..eaf35b5 100644 --- a/include/osquery/database.h +++ b/include/osquery/database.h @@ -263,7 +263,7 @@ struct ScheduledQuery { unsigned long long int system_time; /// Average memory differentials. This should be near 0. - unsigned long long int memory; + unsigned long long int average_memory; /// Total characters, bytes, generated by query. unsigned long long int output_size; @@ -278,7 +278,7 @@ struct ScheduledQuery { wall_time(0), user_time(0), system_time(0), - memory(0), + average_memory(0), output_size(0) {} /// equals operator diff --git a/include/osquery/events.h b/include/osquery/events.h index d52b8c9..1baabe6 100644 --- a/include/osquery/events.h +++ b/include/osquery/events.h @@ -3,7 +3,7 @@ * All rights reserved. * * This source code is licensed under the BSD-style license found in the - * LICENSE file in the root directory of this source tree. An additional grant + * LICENSE file in the root directory of this source tree. An additional grant * of patent rights can be found in the PATENTS file in the same directory. * */ @@ -47,7 +47,7 @@ typedef std::pair EventRecord; * EventPublisher will use to register OS API callbacks, create * subscriptioning/listening handles, etc. * - * Linux `inotify` should implement a SubscriptionContext that subscriptions + * Linux `inotify` should implement a SubscriptionContext that subscribes * filesystem events based on a filesystem path. `libpcap` will subscribe on * networking protocols at various stacks. Process creation may subscribe on * process name, parent pid, etc. @@ -65,10 +65,8 @@ struct SubscriptionContext {}; struct EventContext { /// An unique counting ID specific to the EventPublisher%'s fired events. EventContextID id; - /// The time the event occurred. + /// The time the event occurred, as determined by the publisher. EventTime time; - /// The string representation of the time, often used for indexing. - std::string time_string; EventContext() : id(0), time(0) {} }; @@ -93,8 +91,7 @@ typedef std::shared_ptr> EventSubscriberRef; * The supported states are: * - None: The default state, uninitialized. * - Running: Subscriber is ready for events. - * - Paused: Subscriber was successfully initialized but not currently accepting - * events. + * - Paused: Subscriber was initialized but is not currently accepting events. * - Failed: Subscriber failed to initialize or is otherwise offline. */ enum EventSubscriberState { @@ -197,8 +194,8 @@ class EventPublisherPlugin : public Plugin { * @brief Perform handle opening, OS API callback registration. * * `setUp` is the event framework's EventPublisher constructor equivalent. - * When `setUp` is called the EventPublisher is running in a dedicated thread - * and may manage/allocate/wait for resources. + * This is called in the main thread before the publisher's run loop has + * started, immediately following registration. */ virtual Status setUp() { return Status(0, "Not used"); } @@ -206,21 +203,31 @@ class EventPublisherPlugin : public Plugin { * @brief Perform handle closing, resource cleanup. * * osquery is about to end, the EventPublisher should close handle descriptors - * unblock resources, and prepare to exit. + * unblock resources, and prepare to exit. This will be called from the main + * thread after the run loop thread has exited. */ virtual void tearDown() {} /** - * @brief Implement a step of an optional run loop. + * @brief Implement a "step" of an optional run loop. * * @return A SUCCESS status will immediately call `run` again. A FAILED status * will exit the run loop and the thread. */ - virtual Status run() { return Status(1, "No runloop required"); } + virtual Status run() { return Status(1, "No run loop required"); } /** - * @brief A new EventSubscriber is subscriptioning events of this - * EventPublisher. + * @brief Allow the EventFactory to interrupt the run loop. + * + * Assume the main thread may ask the run loop to stop at anytime. + * Before end is called the publisher's `isEnding` is set and the EventFactory + * run loop manager will exit the stepping loop and fall through to a call + * to tearDown followed by a removal of the publisher. + */ + virtual void end() {} + + /** + * @brief A new EventSubscriber is subscribing events of this publisher type. * * @param subscription The Subscription context information and optional * EventCallback. @@ -232,17 +239,15 @@ class EventPublisherPlugin : public Plugin { return Status(0, "OK"); } - /** - * @brief The generic check loop to call SubscriptionContext callback methods. - * - * It is NOT recommended to override `fire`. The simple logic of enumerating - * the Subscription%s and using `shouldFire` is more appropriate. - * - * @param ec The EventContext created and fired by the EventPublisher. - * @param time The most accurate time associated with the event. - */ - void fire(const EventContextRef& ec, EventTime time = 0); + public: + /// Overriding the EventPublisher constructor is not recommended. + EventPublisherPlugin() : next_ec_id_(0), ending_(false), started_(false){}; + virtual ~EventPublisherPlugin() {} + /// Return a string identifier associated with this EventPublisher. + virtual EventPublisherID type() const { return "publisher"; } + + public: /// Number of Subscription%s watching this EventPublisher. size_t numSubscriptions() const { return subscriptions_.size(); } @@ -253,19 +258,30 @@ class EventPublisherPlugin : public Plugin { */ size_t numEvents() const { return next_ec_id_; } - /// Overriding the EventPublisher constructor is not recommended. - EventPublisherPlugin() : next_ec_id_(0), ending_(false), started_(false) {}; - virtual ~EventPublisherPlugin() {} - - /// Return a string identifier associated with this EventPublisher. - virtual EventPublisherID type() const { return "publisher"; } - + /// Check if the EventFactory is ending all publisher threads. bool isEnding() const { return ending_; } + + /// Set the ending status for this publisher. void isEnding(bool ending) { ending_ = ending; } + + /// Check if the publisher's run loop has started. bool hasStarted() const { return started_; } + + /// Set the run or started status for this publisher. void hasStarted(bool started) { started_ = started; } protected: + /** + * @brief The generic check loop to call SubscriptionContext callback methods. + * + * It is NOT recommended to override `fire`. The simple logic of enumerating + * the Subscription%s and using `shouldFire` is more appropriate. + * + * @param ec The EventContext created and fired by the EventPublisher. + * @param time The most accurate time associated with the event. + */ + virtual void fire(const EventContextRef& ec, EventTime time = 0) final; + /// The internal fire method used by the typed EventPublisher. virtual void fireCallback(const SubscriptionRef& sub, const EventContextRef& ec) const = 0; @@ -284,6 +300,7 @@ class EventPublisherPlugin : public Plugin { private: /// Set ending to True to cause event type run loops to finish. bool ending_; + /// Set to indicate whether the event run loop ever started. bool started_; @@ -291,6 +308,10 @@ class EventPublisherPlugin : public Plugin { boost::mutex ec_id_lock_; private: + /// Enable event factory "callins" through static publisher callbacks. + friend class EventFactory; + + private: FRIEND_TEST(EventsTests, test_event_pub); FRIEND_TEST(EventsTests, test_fire_event); }; @@ -301,7 +322,7 @@ class EventPublisherPlugin : public Plugin { * A 'class' of OS events is abstracted into an EventPublisher responsible for * remaining as agile as possible given a known-set of subscriptions. * - * The lifecycle of an EventPublisher may include, `setUp`, `configure`, `run`, + * The life cycle of an EventPublisher may include, `setUp`, `configure`, `run`, * `tearDown`, and `fire`. `setUp` and `tearDown` happen when osquery starts and * stops either as a daemon or interactive shell. `configure` is a pseudo-start * called every time a Subscription is added. EventPublisher%s can adjust their @@ -321,7 +342,7 @@ class EventPublisherPlugin : public Plugin { * Status run() { return Status(1, "Not Implemented"); } * @endcode * - * The final lifecycle component, `fire` will iterate over the EventPublisher + * The final life cycle component, `fire` will iterate over the EventPublisher * Subscription%s and call `shouldFire` for each, using the EventContext fired. * The `shouldFire` method should check the subscription-specific selectors and * only call the Subscription%'s callback function if the EventContext @@ -352,13 +373,6 @@ class EventPublisher : public EventPublisherPlugin { /// Create a SubscriptionContext based on the templated type. static SCRef createSubscriptionContext() { return std::make_shared(); } - /// A simple EventPublisher type accessor. - template - static EventPublisherID getType() { - auto pub = std::make_shared(); - return pub->type(); - } - protected: /** * @brief The internal `fire` phase of publishing. @@ -383,7 +397,7 @@ class EventPublisher : public EventPublisherPlugin { /** * @brief The generic `fire` will call `shouldFire` for each Subscription. * - * @param mc A SubscriptionContext with optional specifications for events + * @param sc A SubscriptionContext with optional specifications for events * details. * @param ec The event fired with event details. * @@ -416,7 +430,7 @@ class EventSubscriberPlugin : public Plugin { * * @return Was the element added to the backing store. */ - virtual Status add(const osquery::Row& r, EventTime time) final; + virtual Status add(Row& r, EventTime event_time) final; /** * @brief Return all events added by this EventSubscriber within start, stop. @@ -441,16 +455,16 @@ class EventSubscriberPlugin : public Plugin { * * @return List of EventID, EventTime%s */ - std::vector getRecords(const std::vector& indexes); + std::vector getRecords(const std::set& indexes); /** * @brief Get a unique storage-related EventID. * * An EventID is an index/element-identifier for the backing store. * Each EventPublisher maintains a fired EventContextID to identify the many - * events that may or may not be fired to 'subscriptioning' criteria for this + * events that may or may not be fired based on subscription criteria for this * EventSubscriber. This EventContextID is NOT the same as an EventID. - * EventSubscriber development should not require use of EventID%s, if this + * EventSubscriber development should not require use of EventID%s. If this * indexing is required within-EventCallback consider an * EventSubscriber%-unique indexing, counting mechanic. * @@ -467,9 +481,9 @@ class EventSubscriberPlugin : public Plugin { * * @return List of 'index.step' index strings. */ - std::vector getIndexes(EventTime start, - EventTime stop, - int list_key = 0); + std::set getIndexes(EventTime start, + EventTime stop, + int list_key = 0); /** * @brief Expire indexes and eventually records. @@ -477,12 +491,14 @@ class EventSubscriberPlugin : public Plugin { * @param list_type the string representation of list binning type. * @param indexes complete set of 'index.step' indexes for the list_type. * @param expirations of the indexes, the set to expire. - * - * @return status if the indexes and records were removed. */ - Status expireIndexes(const std::string& list_type, - const std::vector& indexes, - const std::vector& expirations); + void expireIndexes(const std::string& list_type, + const std::vector& indexes, + const std::vector& expirations); + /// Expire all datums within a bin. + void expireRecords(const std::string& list_type, + const std::string& index, + bool all); /** * @brief Add an EventID, EventTime pair to all matching list types. @@ -508,31 +524,31 @@ class EventSubscriberPlugin : public Plugin { * EventPublisher instances will have run `setUp` and initialized their run * loops. */ - EventSubscriberPlugin() { - expire_events_ = true; - expire_time_ = 0; - } + EventSubscriberPlugin() + : expire_events_(true), expire_time_(0), optimize_time_(0) {} virtual ~EventSubscriberPlugin() {} /** * @brief Suggested entrypoint for table generation. * * The EventSubscriber is a convention that removes a lot of boilerplate event - * 'subscriptioning' and acting. The `genTable` static entrypoint is the + * 'subscribing' and acting. The `genTable` static entrypoint is the * suggested method for table specs. * * @return The query-time table data, retrieved from a backing store. */ - virtual QueryData genTable(QueryContext& context) __attribute__((used)) { - return get(0, 0); - } + virtual QueryData genTable(QueryContext& context) __attribute__((used)); protected: - /// Backing storage indexing namespace definition methods. - EventPublisherID dbNamespace() const { return type() + "." + getName(); } - - /// The string EventPublisher identifying this EventSubscriber. - virtual EventPublisherID type() const = 0; + /** + * @brief Backing storage indexing namespace. + * + * The backing storage will accumulate events for this subscriber. A namespace + * is provided to prevent event indexing collisions between subscribers and + * publishers. The namespace is a combination of the publisher and subscriber + * registry plugin names. + */ + virtual EventPublisherID& dbNamespace() const = 0; /// Disable event expiration for this subscriber. void doNotExpire() { expire_events_ = false; } @@ -551,6 +567,15 @@ class EventSubscriberPlugin : public Plugin { /// Events before the expire_time_ are invalid and will be purged. EventTime expire_time_; + /** + * @brief Optimize subscriber selects by tracking the last select time. + * + * Event subscribers may optimize selects when used in a daemon schedule by + * requiring an event 'time' constraint and otherwise applying a minimum time + * as the last time the scheduled query ran. + */ + EventTime optimize_time_; + /// Lock used when incrementing the EventID database index. boost::mutex event_id_lock_; @@ -625,7 +650,7 @@ class EventFactory : private boost::noncopyable { * and add that Subscription to the EventPublisher associated identifier. * * @param type_id The string for an EventPublisher receiving the Subscription. - * @param mc A SubscriptionContext related to the EventPublisher. + * @param sc A SubscriptionContext related to the EventPublisher. * @param cb When the EventPublisher fires an event the SubscriptionContext * will be evaluated, if the event matches optional specifics in the context * this callback function will be called. It should belong to an @@ -635,18 +660,10 @@ class EventFactory : private boost::noncopyable { */ static Status addSubscription(EventPublisherID& type_id, EventSubscriberID& name_id, - const SubscriptionContextRef& mc, + const SubscriptionContextRef& sc, EventCallback cb = 0, void* user_data = nullptr); - /// Add a Subscription by templating the EventPublisher, using a - /// SubscriptionContext. - template - static Status addSubscription(const SubscriptionContextRef& mc, - EventCallback cb = 0) { - return addSubscription(BaseEventPublisher::getType(), mc, cb); - } - /// Add a Subscription using a caller Subscription instance. static Status addSubscription(EventPublisherID& type_id, const SubscriptionRef& subscription); @@ -660,11 +677,14 @@ class EventFactory : private boost::noncopyable { } /** - * @brief Halt the EventPublisher run loop and call its `tearDown`. + * @brief Halt the EventPublisher run loop. * * Any EventSubscriber%s with Subscription%s for this EventPublisher will * become useless. osquery callers MUST deregister events. * EventPublisher%s assume they can hook/trampoline, which requires cleanup. + * This will tear down and remove the publisher if the run loop did not start. + * Otherwise it will call end on the publisher and assume the run loop will + * tear down and remove. * * @param event_pub The string label for the EventPublisher. * @@ -680,9 +700,14 @@ class EventFactory : private boost::noncopyable { /// Return an instance to a registered EventSubscriber. static EventSubscriberRef getEventSubscriber(EventSubscriberID& sub); + + /// Check if an event subscriber exists. static bool exists(EventSubscriberID& sub); + /// Return a list of publisher types, these are their registry names. static std::vector publisherTypes(); + + /// Return a list of subscriber registry names, static std::vector subscriberNames(); public: @@ -695,14 +720,30 @@ class EventFactory : private boost::noncopyable { /// If a static EventPublisher callback wants to fire template static void fire(const EventContextRef& ec) { - auto event_pub = getEventPublisher(BaseEventPublisher::getType()); + auto event_pub = getEventPublisher(getType()); event_pub->fire(ec); } /** - * @brief End all EventPublisher run loops and call their `tearDown` methods. + * @brief Return the publisher registry name given a type. * - * End is NOT the same as deregistration. + * Subscriber initialization and runtime static callbacks can lookup the + * publisher type name, which is the registry plugin name. This allows static + * callbacks to fire into subscribers. + */ + template + static EventPublisherID getType() { + auto pub = std::make_shared(); + return pub->type(); + } + + /** + * @brief End all EventPublisher run loops and deregister. + * + * End is NOT the same as deregistration. End will call deregister on all + * publishers then either join or detach their run loop threads. + * See EventFactory::deregisterEventPublisher for actions taken during + * deregistration. * * @param should_end Reset the "is ending" state if False. */ @@ -754,6 +795,7 @@ class EventSubscriber : public EventSubscriberPlugin { */ virtual Status init() { return Status(0, "OK"); } + protected: /// Helper function to call the publisher's templated subscription generator. SCRef createSubscriptionContext() const { return PUB::createSubscriptionContext(); @@ -769,22 +811,38 @@ class EventSubscriber : public EventSubscriberPlugin { void subscribe(Status (T::*entry)(const std::shared_ptr&, const void*), const SubscriptionContextRef& sc, void* user_data) { - // Up-cast the CRTP-style EventSubscriber to the caller. - auto self = dynamic_cast(this); + // Up-cast the EventSubscriber to the caller. + auto sub = dynamic_cast(this); // Down-cast the pointer to the member function. auto base_entry = reinterpret_cast( entry); // Create a callable through the member function using the instance of the // EventSubscriber and a single parameter placeholder (the EventContext). - auto cb = std::bind(base_entry, self, _1, _2); + auto cb = std::bind(base_entry, sub, _1, _2); // Add a subscription using the callable and SubscriptionContext. - EventFactory::addSubscription(type(), self->getName(), sc, cb, user_data); + EventFactory::addSubscription(getType(), sub->getName(), sc, cb, user_data); + } + + /** + * @brief The registry plugin name for the subscriber's publisher. + * + * During event factory initialization the subscribers 'peek' at the registry + * plugin name assigned to publishers. The corresponding publisher name is + * interpreted as the subscriber's event 'type'. + */ + EventPublisherID& getType() const { + static EventPublisherID type = EventFactory::getType(); + return type; } - /// Helper EventPublisher string type accessor. - EventPublisherID type() const { return BaseEventPublisher::getType(); } + /// See getType for lookup rational. + EventPublisherID& dbNamespace() const { + static EventPublisherID _ns = getType() + '.' + getName(); + return _ns; + } + public: /** * @brief Request the subscriber's initialization state. * @@ -797,7 +855,6 @@ class EventSubscriber : public EventSubscriberPlugin { /// Set the subscriber state. void state(EventSubscriberState state) { state_ = state; } - public: EventSubscriber() : EventSubscriberPlugin(), state_(SUBSCRIBER_NONE) {} private: diff --git a/include/osquery/extensions.h b/include/osquery/extensions.h index 7df1e57..cd2e45c 100644 --- a/include/osquery/extensions.h +++ b/include/osquery/extensions.h @@ -23,7 +23,7 @@ DECLARE_string(extensions_timeout); DECLARE_bool(disable_extensions); /// A millisecond internal applied to extension initialization. -extern const int kExtensionInitializeMLatency; +extern const size_t kExtensionInitializeLatencyUS; /** * @brief Helper struct for managing extenion metadata. diff --git a/include/osquery/filesystem.h b/include/osquery/filesystem.h index fd791e6..8c4ef80 100644 --- a/include/osquery/filesystem.h +++ b/include/osquery/filesystem.h @@ -23,47 +23,52 @@ namespace osquery { /// Globbing directory traversal function recursive limit. -const unsigned int kMaxDirectoryTraversalDepth = 40; -typedef unsigned int ReturnSetting; +typedef unsigned short GlobLimits; enum { - /// Return only files - REC_LIST_FILES = 0x1, - /// Return only folders - REC_LIST_FOLDERS = 0x2, - /// Enable optimizations for file event resolutions - REC_EVENT_OPT = 0x4, - REC_LIST_ALL = REC_LIST_FILES | REC_LIST_FOLDERS + GLOB_FILES = 0x1, + GLOB_FOLDERS = 0x2, + GLOB_ALL = GLOB_FILES | GLOB_FOLDERS, }; /// Globbing wildcard character. -const std::string kWildcardCharacter = "%"; +const std::string kSQLGlobWildcard = "%"; /// Globbing wildcard recursive character (double wildcard). -const std::string kWildcardCharacterRecursive = - kWildcardCharacter + kWildcardCharacter; +const std::string kSQLGlobRecursive = kSQLGlobWildcard + kSQLGlobWildcard; /** * @brief Read a file from disk. * - * @param path the path of the file that you would like to read + * @param path the path of the file that you would like to read. * @param content a reference to a string which will be populated with the - * contents of the path indicated by the path parameter + * contents of the path indicated by the path parameter. + * @param dry_run do not actually read the file content. * - * @return an instance of Status, indicating the success or failure - * of the operation. + * @return an instance of Status, indicating success or failure. */ -Status readFile(const boost::filesystem::path& path, std::string& content); +Status readFile(const boost::filesystem::path& path, + std::string& content, + bool dry_run = false); + +/** + * @brief Return the status of an attempted file read. + * + * @param path the path of the file that you would like to read. + * + * @return success iff the file would have been read. On success the status + * message is the complete/absolute path. + */ +Status readFile(const boost::filesystem::path& path); /** * @brief Write text to disk. * - * @param path the path of the file that you would like to write - * @param content the text that should be written exactly to disk - * @param permissions the filesystem permissions to request when opening - * @param force_permissions always chmod the path after opening + * @param path the path of the file that you would like to write. + * @param content the text that should be written exactly to disk. + * @param permissions the filesystem permissions to request when opening. + * @param force_permissions always `chmod` the path after opening. * - * @return an instance of Status, indicating the success or failure - * of the operation. + * @return an instance of Status, indicating success or failure. */ Status writeTextFile(const boost::filesystem::path& path, const std::string& content, @@ -72,19 +77,19 @@ Status writeTextFile(const boost::filesystem::path& path, /// Check if a path is writable. Status isWritable(const boost::filesystem::path& path); + /// Check if a path is readable. Status isReadable(const boost::filesystem::path& path); /** * @brief A helper to check if a path exists on disk or not. * - * @param path the path on disk which you would like to check the existence of + * @param path Target path. * - * @return an instance of Status, indicating the success or failure - * of the operation. Specifically, the code of the Status instance - * will be -1 if no input was supplied, assuming the caller is not aware of how - * to check path-getter results. The code will be 0 if the path does not exist - * on disk and 1 if the path does exist on disk. + * @return The code of the Status instance will be -1 if no input was supplied, + * assuming the caller is not aware of how to check path-getter results. + * The code will be 0 if the path does not exist on disk and 1 if the path + * does exist on disk. */ Status pathExists(const boost::filesystem::path& path); @@ -96,8 +101,7 @@ Status pathExists(const boost::filesystem::path& path); * with the directory listing of the path param, assuming that all operations * completed successfully. * - * @return an instance of Status, indicating the success or failure - * of the operation. + * @return an instance of Status, indicating success or failure. */ Status listFilesInDirectory(const boost::filesystem::path& path, std::vector& results, @@ -106,20 +110,19 @@ Status listFilesInDirectory(const boost::filesystem::path& path, /** * @brief List all of the directories in a specific directory, non-recursively. * - * @param path the path which you would like to list. + * @param path the path which you would like to list * @param results a non-const reference to a vector which will be populated * with the directory listing of the path param, assuming that all operations * completed successfully. * - * @return an instance of Status, indicating the success or failure - * of the operation. + * @return an instance of Status, indicating success or failure. */ Status listDirectoriesInDirectory(const boost::filesystem::path& path, std::vector& results, bool ignore_error = 1); /** - * @brief Given a wildcard filesystem patten, resolve all possible paths + * @brief Given a filesystem globbing patten, resolve all matching paths. * * @code{.cpp} * std::vector results; @@ -131,44 +134,49 @@ Status listDirectoriesInDirectory(const boost::filesystem::path& path, * } * @endcode * - * @param fs_path The filesystem pattern - * @param results The vector in which all results will be returned + * @param pattern filesystem globbing pattern. + * @param results output vector of matching paths. * - * @return An instance of osquery::Status which indicates the success or - * failure of the operation + * @return an instance of Status, indicating success or failure. */ -Status resolveFilePattern(const boost::filesystem::path& fs_path, +Status resolveFilePattern(const boost::filesystem::path& pattern, std::vector& results); /** - * @brief Given a wildcard filesystem patten, resolve all possible paths + * @brief Given a filesystem globbing patten, resolve all matching paths. * - * @code{.cpp} - * std::vector results; - * auto s = resolveFilePattern("/Users/marpaia/Downloads/%", results); - * if (s.ok()) { - * for (const auto& result : results) { - * LOG(INFO) << result; - * } - * } - * @endcode + * See resolveFilePattern, but supply a limitation to request only directories + * or files that match the path. * - * @param fs_path The filesystem pattern - * @param results The vector in which all results will be returned - * @param setting Do you want files returned, folders or both? + * @param pattern filesystem globbing pattern. + * @param results output vector of matching paths. + * @param setting a bit list of match types, e.g., files, folders. * - * @return An instance of osquery::Status which indicates the success or - * failure of the operation + * @return an instance of Status, indicating success or failure. */ -Status resolveFilePattern(const boost::filesystem::path& fs_path, +Status resolveFilePattern(const boost::filesystem::path& pattern, std::vector& results, - ReturnSetting setting); + GlobLimits setting); + +/** + * @brief Transform a path with SQL wildcards to globbing wildcard. + * + * SQL uses '%' as a wildcard matching token, and filesystem globbing uses '*'. + * In osquery-internal methods the filesystem character is used. This helper + * method will perform the correct preg/escape and replace. + * + * This has a side effect of canonicalizing paths up to the first wildcard. + * For example: /tmp/% becomes /private/tmp/% on OS X systems. And /tmp/%. + * + * @param pattern the input and output filesystem glob pattern. + */ +void replaceGlobWildcards(std::string& pattern); /** * @brief Get directory portion of a path. * - * @param path The input path, either a filename or directory. - * @param dirpath a non-const reference to a resultant directory portion. + * @param path input path, either a filename or directory. + * @param dirpath output path set to the directory-only path. * * @return If the input path was a directory this will indicate failure. One * should use `isDirectory` before. @@ -176,42 +184,48 @@ Status resolveFilePattern(const boost::filesystem::path& fs_path, Status getDirectory(const boost::filesystem::path& path, boost::filesystem::path& dirpath); +/// Attempt to remove a directory path. Status remove(const boost::filesystem::path& path); /** * @brief Check if an input path is a directory. * - * @param path The input path, either a filename or directory. + * @param path input path, either a filename or directory. * * @return If the input path was a directory. */ Status isDirectory(const boost::filesystem::path& path); /** - * @brief Return a vector of all home directories on the system + * @brief Return a vector of all home directories on the system. * - * @return a vector of strings representing the path of all home directories + * @return a vector of string paths containing all home directories. */ std::set getHomeDirectories(); /** - * @brief Check the permissions of a file and it's directory. + * @brief Check the permissions of a file and its directory. * * 'Safe' implies the directory is not a /tmp-like directory in that users * cannot control super-user-owner files. The file should be owned by the * process's UID or the file should be owned by root. * - * @param dir the directory to check /tmp mode - * @param path a path to a file to check - * @param executable the file must also be executable + * @param dir the directory to check `/tmp` mode. + * @param path a path to a file to check. + * @param executable true if the file must also be executable. * - * @return true if the file is 'safe' else false + * @return true if the file is 'safe' else false. */ bool safePermissions(const std::string& dir, const std::string& path, bool executable = false); -/// The shell tooling may store local resources in an "osquery" home. +/** + * @brief osquery may use local storage in a user-protected "home". + * + * Return a standard path to an "osquery" home directory. This path may store + * a protected extensions socket, backing storage database, and debug logs. + */ const std::string& osqueryHomeDirectory(); /// Return bit-mask-style permissions. @@ -220,10 +234,10 @@ std::string lsperms(int mode); /** * @brief Parse a JSON file on disk into a property tree. * - * @param path the path of the JSON file - * @param tree output property tree + * @param path the path of the JSON file. + * @param tree output property tree. * - * @return an instance of Status, indicating the success or failure + * @return an instance of Status, indicating success or failure if malformed. */ Status parseJSON(const boost::filesystem::path& path, boost::property_tree::ptree& tree); @@ -231,10 +245,10 @@ Status parseJSON(const boost::filesystem::path& path, /** * @brief Parse JSON content into a property tree. * - * @param path JSON string data - * @param tree output property tree + * @param path JSON string data. + * @param tree output property tree. * - * @return an instance of Status, indicating the success or failure + * @return an instance of Status, indicating success or failure if malformed. */ Status parseJSONContent(const std::string& content, boost::property_tree::ptree& tree); @@ -243,11 +257,10 @@ Status parseJSONContent(const std::string& content, /** * @brief Parse a property list on disk into a property tree. * - * @param path the input path to a property list - * @param tree the output reference to a Boost property tree + * @param path the input path to a property list. + * @param tree the output property tree. * - * @return an instance of Status, indicating the success or failure - * of the operation. + * @return an instance of Status, indicating success or failure if malformed. */ Status parsePlist(const boost::filesystem::path& path, boost::property_tree::ptree& tree); @@ -255,11 +268,10 @@ Status parsePlist(const boost::filesystem::path& path, /** * @brief Parse property list content into a property tree. * - * @param content the input string-content of a property list - * @param tree the output reference to a Boost property tree + * @param content the input string-content of a property list. + * @param tree the output property tree. * - * @return an instance of Status, indicating the success or failure - * of the operation. + * @return an instance of Status, indicating success or failure if malformed. */ Status parsePlistContent(const std::string& content, boost::property_tree::ptree& tree); @@ -267,16 +279,16 @@ Status parsePlistContent(const std::string& content, #ifdef __linux__ /** - * @brief Iterate over proc process, returns a list of pids. + * @brief Iterate over `/proc` process, returns a list of pids. * * @param processes output list of process pids as strings (int paths in proc). * - * @return status of iteration. + * @return an instance of Status, indicating success or failure. */ Status procProcesses(std::set& processes); /** - * @brief Iterate over a proc process's descriptors, return a list of fds. + * @brief Iterate over a `/proc` process's descriptors, return a list of fds. * * @param process a string pid from proc. * @param descriptors output list of descriptor numbers as strings. diff --git a/include/osquery/flags.h b/include/osquery/flags.h index 62f9837..8eb17d9 100644 --- a/include/osquery/flags.h +++ b/include/osquery/flags.h @@ -38,6 +38,7 @@ struct FlagDetail { bool shell; bool external; bool cli; + bool hidden; }; struct FlagInfo { @@ -185,23 +186,24 @@ class FlagAlias { * @param value The default value, use a C++ literal. * @param desc A string literal used for help display. */ -#define OSQUERY_FLAG(t, n, v, d, s, e, c) \ - DEFINE_##t(n, v, d); \ - namespace flags { \ - const int flag_##n = Flag::create(#n, {d, s, e, c}); \ +#define OSQUERY_FLAG(t, n, v, d, s, e, c, h) \ + DEFINE_##t(n, v, d); \ + namespace flags { \ + const int flag_##n = Flag::create(#n, {d, s, e, c, h}); \ } -#define FLAG(t, n, v, d) OSQUERY_FLAG(t, n, v, d, 0, 0, 0) -#define SHELL_FLAG(t, n, v, d) OSQUERY_FLAG(t, n, v, d, 1, 0, 0) -#define EXTENSION_FLAG(t, n, v, d) OSQUERY_FLAG(t, n, v, d, 0, 1, 0) -#define CLI_FLAG(t, n, v, d) OSQUERY_FLAG(t, n, v, d, 0, 0, 1) - -#define OSQUERY_FLAG_ALIAS(t, a, n, s, e) \ - FlagAlias FLAGS_##a(#a, #t, #n, &FLAGS_##n); \ - namespace flags { \ - static GFLAGS_NAMESPACE::FlagRegisterer oflag_##a( \ - #a, #t, #a, &FLAGS_##n, &FLAGS_##n); \ - const int flag_alias_##a = Flag::createAlias(#a, {#n, s, e, 0}); \ +#define FLAG(t, n, v, d) OSQUERY_FLAG(t, n, v, d, 0, 0, 0, 0) +#define SHELL_FLAG(t, n, v, d) OSQUERY_FLAG(t, n, v, d, 1, 0, 0, 0) +#define EXTENSION_FLAG(t, n, v, d) OSQUERY_FLAG(t, n, v, d, 0, 1, 0, 0) +#define CLI_FLAG(t, n, v, d) OSQUERY_FLAG(t, n, v, d, 0, 0, 1, 0) +#define HIDDEN_FLAG(t, n, v, d) OSQUERY_FLAG(t, n, v, d, 0, 0, 0, 1) + +#define OSQUERY_FLAG_ALIAS(t, a, n, s, e) \ + FlagAlias FLAGS_##a(#a, #t, #n, &FLAGS_##n); \ + namespace flags { \ + static GFLAGS_NAMESPACE::FlagRegisterer oflag_##a( \ + #a, #t, #a, &FLAGS_##n, &FLAGS_##n); \ + const int flag_alias_##a = Flag::createAlias(#a, {#n, s, e, 0, 1}); \ } #define FLAG_ALIAS(t, a, n) OSQUERY_FLAG_ALIAS(t, a, n, 0, 0) diff --git a/include/osquery/logger.h b/include/osquery/logger.h index a3530a7..0ef8b15 100644 --- a/include/osquery/logger.h +++ b/include/osquery/logger.h @@ -271,6 +271,21 @@ Status logSnapshotQuery(const QueryLogItem& item); Status logHealthStatus(const QueryLogItem& item); /** + * @brief Sink a set of buffered status logs. + * + * When the osquery daemon uses a watcher/worker set, the watcher's status logs + * are accumulated in a buffered log sink. Well-performing workers should have + * the set of watcher status logs relayed and sent to the configured logger + * plugin. + * + * Status logs from extensions will be forwarded to the extension manager (core) + * normally, but the watcher does not receive or send registry requests. + * Extensions, the registry, configuration, and optional config/logger plugins + * are all protected as a monitored worker. + */ +void relayStatusLogs(); + +/** * @brief Logger plugin registry. * * This creates an osquery registry for "logger" which may implement diff --git a/include/osquery/registry.h b/include/osquery/registry.h index afcd6f7..1ea4587 100644 --- a/include/osquery/registry.h +++ b/include/osquery/registry.h @@ -137,16 +137,29 @@ class Plugin : private boost::noncopyable { public: /// The plugin may perform some initialization, not required. virtual Status setUp() { return Status(0, "Not used"); } + /// The plugin may perform some tear down, release, not required. virtual void tearDown() {} + /// The plugin may publish route info (other than registry type and name). virtual PluginResponse routeInfo() const { PluginResponse info; return info; } - /// The plugin will act on a serialized request, and if a response is needed - /// (response is set to true) then response should be a reference to a - /// string ready for a serialized response. + + /** + * @brief Plugins act by being called, using a request, returning a response. + * + * The plugin request is a thrift-serializable object. A response is optional + * but the API for using a plugin's call is defined by the registry. In most + * cases there are multiple supported call 'actions'. A registry type, or + * the plugin class, will define the action key and supported actions. + * + * @param request A plugin request input, including optional action. + * @param response A plugin response output. + * + * @return Status of the call, if the action was handled corrected. + */ virtual Status call(const PluginRequest& request, PluginResponse& response) { return Status(0, "Not used"); } diff --git a/include/osquery/tables.h b/include/osquery/tables.h index 45ab259..b7a6369 100644 --- a/include/osquery/tables.h +++ b/include/osquery/tables.h @@ -227,6 +227,7 @@ struct ConstraintList { */ std::set getAll(ConstraintOperator op) const; + /// See ConstraintList::getAll, but as a selected literal type. template std::set getAll(ConstraintOperator op) const { std::set literal_matches; @@ -237,6 +238,9 @@ struct ConstraintList { return literal_matches; } + /// Constraint list accessor, types and operator. + const std::vector getAll() const { return constraints_; } + /** * @brief Add a new Constraint to the list of constraints. * @@ -258,6 +262,8 @@ struct ConstraintList { * } */ void serialize(boost::property_tree::ptree& tree) const; + + /// See ConstraintList::unserialize. void unserialize(const boost::property_tree::ptree& tree); ConstraintList() : affinity("TEXT") {} diff --git a/osquery/config/config.cpp b/osquery/config/config.cpp index 9fda7d8..52b0c76 100644 --- a/osquery/config/config.cpp +++ b/osquery/config/config.cpp @@ -29,6 +29,7 @@ typedef pt::ptree::value_type tree_node; typedef std::map > EventFileMap_t; typedef std::chrono::high_resolution_clock chrono_clock; +/// The config plugin must be known before reading options. CLI_FLAG(string, config_plugin, "filesystem", "Config plugin name"); FLAG(int32, schedule_splay_percent, 10, "Percent to splay config times"); @@ -61,28 +62,31 @@ Status Config::update(const std::map& config) { } // Request a unique write lock when updating config. - boost::unique_lock unique_lock(getInstance().mutex_); + { + boost::unique_lock unique_lock(getInstance().mutex_); - ConfigData conf; - for (const auto& source : config) { - if (Registry::external()) { - VLOG(1) << "Updating extension config with source: " << source.first; - } else { - VLOG(1) << "Updating config with source: " << source.first; + for (const auto& source : config) { + if (Registry::external()) { + VLOG(1) << "Updating extension config with source: " << source.first; + } else { + VLOG(1) << "Updating config with source: " << source.first; + } + getInstance().raw_[source.first] = source.second; } - getInstance().raw_[source.first] = source.second; - } - // Now merge all sources together. - for (const auto& source : getInstance().raw_) { - auto status = mergeConfig(source.second, conf); - if (getInstance().force_merge_success_ && !status.ok()) { - return Status(1, status.what()); + // Now merge all sources together. + ConfigData conf; + for (const auto& source : getInstance().raw_) { + auto status = mergeConfig(source.second, conf); + if (getInstance().force_merge_success_ && !status.ok()) { + return Status(1, status.what()); + } } + + // Call each parser with the optionally-empty, requested, top level keys. + getInstance().data_ = std::move(conf); } - // Call each parser with the optionally-empty, requested, top level keys. - getInstance().data_ = conf; for (const auto& plugin : Registry::all("config_parser")) { auto parser = std::static_pointer_cast(plugin.second); if (parser == nullptr || parser.get() == nullptr) { @@ -92,12 +96,16 @@ Status Config::update(const std::map& config) { // For each key requested by the parser, add a property tree reference. std::map parser_config; for (const auto& key : parser->keys()) { - if (conf.all_data.count(key) > 0) { - parser_config[key] = conf.all_data.get_child(key); + if (getInstance().data_.all_data.count(key) > 0) { + parser_config[key] = getInstance().data_.all_data.get_child(key); } else { parser_config[key] = pt::ptree(); } } + + // The config parser plugin will receive a copy of each property tree for + // each top-level-config key. The parser may choose to update the config's + // internal state by requesting and modifying a ConfigDataInstance. parser->update(parser_config); } @@ -203,9 +211,10 @@ inline void mergeFilePath(const std::string& name, const tree_node& node, ConfigData& conf) { for (const auto& path : node.second) { - resolveFilePattern(path.second.data(), - conf.files[node.first], - REC_LIST_FOLDERS | REC_EVENT_OPT); + // Add the exact path after converting wildcards. + std::string pattern = path.second.data(); + replaceGlobWildcards(pattern); + conf.files[node.first].push_back(std::move(pattern)); } conf.all_data.add_child(name + "." + node.first, node.second); } @@ -291,7 +300,11 @@ Status Config::getMD5(std::string& hash_string) { ConfigDataInstance config; std::stringstream out; - pt::write_json(out, config.data()); + try { + pt::write_json(out, config.data(), false); + } catch (const pt::json_parser::json_parser_error& e) { + return Status(1, e.what()); + } hash_string = osquery::hashFromBuffer( HASH_TYPE_MD5, (void*)out.str().c_str(), out.str().length()); @@ -360,8 +373,8 @@ void Config::recordQueryPerformance(const std::string& name, AS_LITERAL(BIGINT_LITERAL, r0.at("resident_size")); if (diff > 0) { // Memory is stored as an average of RSS changes between query executions. - query.memory = (query.memory * query.executions) + diff; - query.memory = (query.memory / (query.executions + 1)); + query.average_memory = (query.average_memory * query.executions) + diff; + query.average_memory = (query.average_memory / (query.executions + 1)); } query.wall_time += delay; diff --git a/osquery/config/tests/config_tests.cpp b/osquery/config/tests/config_tests.cpp index 270e818..38a786b 100644 --- a/osquery/config/tests/config_tests.cpp +++ b/osquery/config/tests/config_tests.cpp @@ -78,8 +78,7 @@ TEST_F(ConfigTests, test_watched_files) { EXPECT_EQ(config.files().at("downloads").size(), 1); // From the new, recommended top-level "file_paths" collection. - EXPECT_EQ(config.files().at("downloads2").size(), 1); - EXPECT_EQ(config.files().at("system_binaries").size(), 1); + EXPECT_EQ(config.files().at("system_binaries").size(), 2); } */ @@ -157,6 +156,7 @@ TEST_F(ConfigTests, test_bad_config_update) { class TestConfigParserPlugin : public ConfigParserPlugin { public: std::vector keys() { + // This config parser requests the follow top-level-config keys. return {"dictionary", "dictionary2", "list"}; } @@ -169,10 +169,13 @@ class TestConfigParserPlugin : public ConfigParserPlugin { } // Set parser-rendered additional data. + // Other plugins may request this "rendered/derived" data using a + // ConfigDataInstance and the getParsedData method. data_.put("dictionary3.key2", "value2"); return Status(0, "OK"); } + // Flag tracking that the update method was called. static bool update_called; private: @@ -183,7 +186,7 @@ class TestConfigParserPlugin : public ConfigParserPlugin { bool TestConfigParserPlugin::update_called = false; TEST_F(ConfigTests, test_config_parser) { - // Register a config parser plugin. + // Register a config parser plugin, and call setup. Registry::add("config_parser", "test"); Registry::get("config_parser", "test")->setUp(); @@ -238,6 +241,50 @@ TEST_F(ConfigTests, test_config_parser) { } } +class TestConfigMutationParserPlugin : public ConfigParserPlugin { + public: + std::vector keys() { + // This config parser wants access to the well-known schedule key. + return {"schedule"}; + } + + Status update(const std::map& config) { + // The merged raw schedule is available as a property tree. + auto& schedule_data = config.at("schedule"); + (void)schedule_data; + + { + // But we want access to the parsed schedule structure. + ConfigDataInstance _config; + auto& data = mutableConfigData(_config); + + ScheduledQuery query; + query.query = "new query"; + query.interval = 1; + data.schedule["test_config_mutation"] = query; + } + + return Status(0, "OK"); + } + + private: + FRIEND_TEST(ConfigTests, test_config_mutaion_parser); +}; + +TEST_F(ConfigTests, test_config_mutaion_parser) { + Registry::add("config_parser", "mutable"); + Registry::get("config_parser", "mutable")->setUp(); + + // Update or load the config, expect the parser to be called. + Config::update({{"source1", "{\"schedule\": {}}"}}); + + { + ConfigDataInstance config; + // The config schedule should have been mutated. + EXPECT_EQ(config.schedule().count("test_config_mutation"), 1); + } +} + TEST_F(ConfigTests, test_splay) { auto val1 = splayValue(100, 10); EXPECT_GE(val1, 90); diff --git a/osquery/core/flags.cpp b/osquery/core/flags.cpp index 3194448..e1474a2 100644 --- a/osquery/core/flags.cpp +++ b/osquery/core/flags.cpp @@ -138,7 +138,7 @@ void Flag::printFlags(bool shell, bool external, bool cli) { const auto& detail = details.at(flag.name); if ((shell && !detail.shell) || (!shell && detail.shell) || (external && !detail.external) || (!external && detail.external) || - (cli && !detail.cli) || (!cli && detail.cli)) { + (cli && !detail.cli) || (!cli && detail.cli) || detail.hidden) { continue; } } else if (aliases.count(flag.name) > 0) { diff --git a/osquery/core/hash.cpp b/osquery/core/hash.cpp index 5b1b35e..fc7d4a9 100644 --- a/osquery/core/hash.cpp +++ b/osquery/core/hash.cpp @@ -11,6 +11,7 @@ #include #include +#include #include #include @@ -91,9 +92,15 @@ std::string hashFromBuffer(HashType hash_type, const void* buffer, size_t size) } std::string hashFromFile(HashType hash_type, const std::string& path) { - Hash hash(hash_type); + // Perform a dry-run of a file read without filling in any content. + auto status = readFile(path); + if (!status.ok()) { + return ""; + } - FILE* file = fopen(path.c_str(), "rb"); + Hash hash(hash_type); + // Use the canonicalized path returned from a successful readFile dry-run. + FILE* file = fopen(status.what().c_str(), "rb"); if (file == nullptr) { VLOG(1) << "Cannot hash/open file " << path; return ""; diff --git a/osquery/core/init.cpp b/osquery/core/init.cpp index 66234fd..655ab2d 100644 --- a/osquery/core/init.cpp +++ b/osquery/core/init.cpp @@ -97,6 +97,8 @@ CLI_FLAG(bool, CLI_FLAG(bool, daemonize, false, "Run as daemon (osqueryd only)"); #endif +ToolType kToolType = OSQUERY_TOOL_UNKNOWN; + void printUsage(const std::string& binary, int tool) { // Parse help options before gflags. Only display osquery-related options. fprintf(stdout, DESCRIPTION, kVersion.c_str()); @@ -163,6 +165,8 @@ Initializer::Initializer(int& argc, char**& argv, ToolType tool) GFLAGS_NAMESPACE::ParseCommandLineFlags( argc_, argv_, (tool == OSQUERY_TOOL_SHELL)); + // Set the tool type to allow runtime decisions based on daemon, shell, etc. + kToolType = tool; if (tool == OSQUERY_TOOL_SHELL) { // The shell is transient, rewrite config-loaded paths. FLAGS_disable_logging = true; @@ -206,7 +210,7 @@ void Initializer::initDaemon() { } #ifndef __APPLE__ - // OSX uses launchd to daemonize. + // OS X uses launchd to daemonize. if (osquery::FLAGS_daemonize) { if (daemon(0, 0) == -1) { ::exit(EXIT_FAILURE); @@ -235,7 +239,7 @@ void Initializer::initDaemon() { if (!FLAGS_disable_watchdog && FLAGS_watchdog_level >= WATCHDOG_LEVEL_DEFAULT && FLAGS_watchdog_level != WATCHDOG_LEVEL_DEBUG) { - // Set CPU scheduling IO limits. + // Set CPU scheduling I/O limits. setpriority(PRIO_PGRP, 0, 10); #ifdef __linux__ // Using: ioprio_set(IOPRIO_WHO_PGRP, 0, IOPRIO_CLASS_IDLE); @@ -306,15 +310,18 @@ void Initializer::initActivePlugin(const std::string& type, const std::string& name) { // Use a delay, meaning the amount of milliseconds waited for extensions. size_t delay = 0; - // The timeout is the maximum time in seconds to wait for extensions. - size_t timeout = atoi(FLAGS_extensions_timeout.c_str()); + // The timeout is the maximum microseconds in seconds to wait for extensions. + size_t timeout = atoi(FLAGS_extensions_timeout.c_str()) * 1000000; + if (timeout < kExtensionInitializeLatencyUS * 10) { + timeout = kExtensionInitializeLatencyUS * 10; + } while (!Registry::setActive(type, name)) { - if (!Watcher::hasManagedExtensions() || delay > timeout * 1000) { + if (!Watcher::hasManagedExtensions() || delay > timeout) { LOG(ERROR) << "Active " << type << " plugin not found: " << name; ::exit(EXIT_CATASTROPHIC); } - ::usleep(kExtensionInitializeMLatency * 1000); - delay += kExtensionInitializeMLatency; + delay += kExtensionInitializeLatencyUS; + ::usleep(kExtensionInitializeLatencyUS); } } diff --git a/osquery/core/tables.cpp b/osquery/core/tables.cpp index ce7d215..b946731 100644 --- a/osquery/core/tables.cpp +++ b/osquery/core/tables.cpp @@ -53,7 +53,11 @@ void TablePlugin::setRequestFromContext(const QueryContext& context, // Write the property tree as a JSON string into the PluginRequest. std::ostringstream output; - pt::write_json(output, tree, false); + try { + pt::write_json(output, tree, false); + } catch (const pt::json_parser::json_parser_error& e) { + // The content could not be represented as JSON. + } request["context"] = output.str(); } @@ -165,7 +169,7 @@ bool ConstraintList::matches(const std::string& expr) const { UNSIGNED_BIGINT_LITERAL lexpr = AS_LITERAL(UNSIGNED_BIGINT_LITERAL, expr); return literal_matches(lexpr); } else { - // Unsupprted affinity type. + // Unsupported affinity type. return false; } } diff --git a/osquery/core/test_util.cpp b/osquery/core/test_util.cpp index 63a2d4b..2ed7db2 100644 --- a/osquery/core/test_util.cpp +++ b/osquery/core/test_util.cpp @@ -19,6 +19,8 @@ #include "osquery/core/test_util.h" +namespace fs = boost::filesystem; + namespace osquery { /// Most tests will use binary or disk-backed content for parsing tests. @@ -251,9 +253,8 @@ QueryData getEtcProtocolsExpectedResults() { } void createMockFileStructure() { - boost::filesystem::create_directories(kFakeDirectory + - "/deep11/deep2/deep3/"); - boost::filesystem::create_directories(kFakeDirectory + "/deep1/deep2/"); + fs::create_directories(kFakeDirectory + "/deep11/deep2/deep3/"); + fs::create_directories(kFakeDirectory + "/deep1/deep2/"); writeTextFile(kFakeDirectory + "/root.txt", "root"); writeTextFile(kFakeDirectory + "/door.txt", "toor"); writeTextFile(kFakeDirectory + "/roto.txt", "roto"); @@ -264,6 +265,10 @@ void createMockFileStructure() { writeTextFile(kFakeDirectory + "/deep11/level1.txt", "l1"); writeTextFile(kFakeDirectory + "/deep11/deep2/level2.txt", "l2"); writeTextFile(kFakeDirectory + "/deep11/deep2/deep3/level3.txt", "l3"); + + boost::system::error_code ec; + fs::create_symlink( + kFakeDirectory + "/root.txt", kFakeDirectory + "/root2.txt", ec); } void tearDownMockFileStructure() { diff --git a/osquery/core/text.cpp b/osquery/core/text.cpp index 2c0e795..3ed8962 100644 --- a/osquery/core/text.cpp +++ b/osquery/core/text.cpp @@ -21,8 +21,10 @@ namespace osquery { std::vector split(const std::string& s, const std::string& delim) { std::vector elems; boost::split(elems, s, boost::is_any_of(delim)); - auto start = std::remove_if( - elems.begin(), elems.end(), [](const std::string& s) { return s == ""; }); + auto start = + std::remove_if(elems.begin(), elems.end(), [](const std::string& s) { + return s.size() == 0; + }); elems.erase(start, elems.end()); for (auto& each : elems) { boost::algorithm::trim(each); @@ -35,7 +37,7 @@ std::vector split(const std::string& s, size_t occurences) { // Split the string normally with the required delimiter. auto content = split(s, delim); - // While the result split exceeds the number of requested occurences, join. + // While the result split exceeds the number of requested occurrences, join. std::vector accumulator; std::vector elems; for (size_t i = 0; i < content.size(); i++) { diff --git a/osquery/core/watcher.cpp b/osquery/core/watcher.cpp index b27740a..fba34f2 100644 --- a/osquery/core/watcher.cpp +++ b/osquery/core/watcher.cpp @@ -161,7 +161,7 @@ bool Watcher::hasManagedExtensions() { // A watchdog process may hint to a worker the number of managed extensions. // Setting this counter to 0 will prevent the worker from waiting for missing - // dependent config plugins. Otherwise, its existance, will cause a worker to + // dependent config plugins. Otherwise, its existence, will cause a worker to // wait for missing plugins to broadcast from managed extensions. return (getenv("OSQUERY_EXTENSIONS") != nullptr); } @@ -289,16 +289,20 @@ bool WatcherRunner::isChildSane(pid_t child) { if (sustained_latency > 0 && sustained_latency * iv >= getWorkerLimit(LATENCY_LIMIT)) { - LOG(WARNING) << "osqueryd worker system performance limits exceeded"; + LOG(WARNING) << "osqueryd worker (" << child + << ") system performance limits exceeded"; return false; } // Check if the private memory exceeds a memory limit. if (footprint > 0 && footprint > getWorkerLimit(MEMORY_LIMIT) * 1024 * 1024) { - LOG(WARNING) << "osqueryd worker memory limits exceeded: " << footprint; + LOG(WARNING) << "osqueryd worker (" << child + << ") memory limits exceeded: " << footprint; return false; } // The worker is sane, no action needed. + // Attempt to flush status logs to the well-behaved worker. + relayStatusLogs(); return true; } @@ -307,7 +311,8 @@ void WatcherRunner::createWorker() { WatcherLocker locker; if (Watcher::getState(Watcher::getWorker()).last_respawn_time > getUnixTime() - getWorkerLimit(RESPAWN_LIMIT)) { - LOG(WARNING) << "osqueryd worker respawning too quickly"; + LOG(WARNING) << "osqueryd worker respawning too quickly: " + << Watcher::workerRestartCount() << " times"; Watcher::workerRestarted(); interruptableSleep(getWorkerLimit(RESPAWN_DELAY) * 1000); // Exponential back off for quickly-respawning clients. @@ -318,7 +323,7 @@ void WatcherRunner::createWorker() { // Get the path of the current process. auto qd = SQL::selectAllFrom("processes", "pid", EQUALS, INTEGER(getpid())); if (qd.size() != 1 || qd[0].count("path") == 0 || qd[0]["path"].size() == 0) { - LOG(ERROR) << "osquery watcher cannot determine process path"; + LOG(ERROR) << "osquery watcher cannot determine process path for worker"; ::exit(EXIT_FAILURE); } diff --git a/osquery/database/database.cpp b/osquery/database/database.cpp index 81dd9c6..96e9ec6 100644 --- a/osquery/database/database.cpp +++ b/osquery/database/database.cpp @@ -82,7 +82,12 @@ Status serializeRowJSON(const Row& r, std::string& json) { } std::ostringstream output; - pt::write_json(output, tree, false); + try { + pt::write_json(output, tree, false); + } catch (const pt::json_parser::json_parser_error& e) { + // The content could not be represented as JSON. + return Status(1, e.what()); + } json = output.str(); return Status(0, "OK"); } @@ -133,7 +138,12 @@ Status serializeQueryDataJSON(const QueryData& q, std::string& json) { } std::ostringstream output; - pt::write_json(output, tree, false); + try { + pt::write_json(output, tree, false); + } catch (const pt::json_parser::json_parser_error& e) { + // The content could not be represented as JSON. + return Status(1, e.what()); + } json = output.str(); return Status(0, "OK"); } @@ -210,7 +220,12 @@ Status serializeDiffResultsJSON(const DiffResults& d, std::string& json) { } std::ostringstream output; - pt::write_json(output, tree, false); + try { + pt::write_json(output, tree, false); + } catch (const pt::json_parser::json_parser_error& e) { + // The content could not be represented as JSON. + return Status(1, e.what()); + } json = output.str(); return Status(0, "OK"); } @@ -274,7 +289,12 @@ Status serializeQueryLogItemJSON(const QueryLogItem& i, std::string& json) { } std::ostringstream output; - pt::write_json(output, tree, false); + try { + pt::write_json(output, tree, false); + } catch (const pt::json_parser::json_parser_error& e) { + // The content could not be represented as JSON. + return Status(1, e.what()); + } json = output.str(); return Status(0, "OK"); } @@ -360,7 +380,11 @@ Status serializeQueryLogItemAsEventsJSON(const QueryLogItem& i, std::ostringstream output; for (auto& event : tree) { - pt::write_json(output, event.second, false); + try { + pt::write_json(output, event.second, false); + } catch (const pt::json_parser::json_parser_error& e) { + return Status(1, e.what()); + } } json = output.str(); return Status(0, "OK"); diff --git a/osquery/database/db_handle.cpp b/osquery/database/db_handle.cpp index 115720a..502af8e 100644 --- a/osquery/database/db_handle.cpp +++ b/osquery/database/db_handle.cpp @@ -16,6 +16,7 @@ #include #include +#include #include #include @@ -93,6 +94,10 @@ DBHandle::DBHandle(const std::string& path, bool in_memory) { options_.log_file_time_to_roll = 0; options_.keep_log_file_num = 10; options_.max_log_file_size = 1024 * 1024 * 1; + options_.compaction_style = rocksdb::kCompactionStyleLevel; + options_.write_buffer_size = 1 * 1024 * 1024; + options_.max_write_buffer_number = 2; + options_.max_background_compactions = 1; if (in_memory) { // Remove when MemEnv is included in librocksdb @@ -212,7 +217,9 @@ Status DBHandle::Delete(const std::string& domain, const std::string& key) { if (cfh == nullptr) { return Status(1, "Could not get column family for " + domain); } - auto s = getDB()->Delete(rocksdb::WriteOptions(), cfh, key); + auto options = rocksdb::WriteOptions(); + options.sync = true; + auto s = getDB()->Delete(options, cfh, key); return Status(s.code(), s.ToString()); } diff --git a/osquery/database/query.cpp b/osquery/database/query.cpp index 125c3e7..360a329 100644 --- a/osquery/database/query.cpp +++ b/osquery/database/query.cpp @@ -34,7 +34,7 @@ Status Query::getPreviousQueryResults(QueryData& results) { Status Query::getPreviousQueryResults(QueryData& results, DBHandleRef db) { if (!isQueryNameInDatabase()) { - return Status(1, "Query name not found in database"); + return Status(0, "Query name not found in database"); } std::string raw; @@ -89,6 +89,9 @@ Status Query::addNewResults(const QueryData& current_qd, // Get the rows from the last run of this query name. QueryData previous_qd; auto status = getPreviousQueryResults(previous_qd); + if (!status.ok()) { + return status; + } // Sanitize all non-ASCII characters from the query data values. QueryData escaped_current_qd; diff --git a/osquery/database/tests/query_tests.cpp b/osquery/database/tests/query_tests.cpp index aebd0c0..0142f27 100644 --- a/osquery/database/tests/query_tests.cpp +++ b/osquery/database/tests/query_tests.cpp @@ -108,7 +108,8 @@ TEST_F(QueryTests, test_query_name_not_found_in_db) { auto query = getOsqueryScheduledQuery(); auto cf = Query("not_a_real_query", query); auto status = cf.getPreviousQueryResults(previous_qd, db_); - EXPECT_FALSE(status.ok()); + EXPECT_TRUE(status.toString() == "Query name not found in database"); + EXPECT_TRUE(status.ok()); } TEST_F(QueryTests, test_is_query_name_in_database) { diff --git a/osquery/devtools/printer.cpp b/osquery/devtools/printer.cpp index c95f338..380d2ca 100644 --- a/osquery/devtools/printer.cpp +++ b/osquery/devtools/printer.cpp @@ -17,43 +17,68 @@ namespace osquery { +static std::vector kOffset = {0, 0}; +static std::string kToken = "|"; + std::string generateToken(const std::map& lengths, const std::vector& columns) { - std::string output = "+"; + std::string out = "+"; for (const auto& col : columns) { if (lengths.count(col) > 0) { - output += std::string(lengths.at(col) + 2, '-'); + if (getenv("ENHANCE") != nullptr) { + std::string e = "\xF0\x9F\x90\x8C"; + e[2] += kOffset[1]; + e[3] += kOffset[0]; + for (int i = 0; i < lengths.at(col) + 2; i++) { + e[3] = '\x8c' + kOffset[0]++; + if (e[3] == '\xbf') { + e[3] = '\x80'; + kOffset[1] = (kOffset[1] > 3 && kOffset[1] < 8) ? 9 : kOffset[1]; + e[2] = '\x90' + ++kOffset[1]; + kOffset[0] = 0; + } + if (kOffset[1] == ('\x97' - '\x8d')) { + kOffset = {0, 0}; + } + out += e.c_str(); + } + } else { + out += std::string(lengths.at(col) + 2, '-'); + } } - output += "+"; + out += "+"; } - output += "\n"; - return output; + out += "\n"; + return out; } std::string generateHeader(const std::map& lengths, const std::vector& columns) { - std::string output = "|"; + if (getenv("ENHANCE") != nullptr) { + kToken = "\xF0\x9F\x91\x8D"; + } + std::string out = kToken; for (const auto& col : columns) { - output += " " + col; + out += " " + col; if (lengths.count(col) > 0) { int buffer_size = lengths.at(col) - utf8StringSize(col) + 1; if (buffer_size > 0) { - output += std::string(buffer_size, ' '); + out += std::string(buffer_size, ' '); } else { - output += ' '; + out += ' '; } } - output += "|"; + out += kToken; } - output += "\n"; - return output; + out += "\n"; + return out; } std::string generateRow(const Row& r, const std::map& lengths, const std::vector& order) { - std::string output; + std::string out; for (const auto& column : order) { if (r.count(column) == 0 || lengths.count(column) == 0) { continue; @@ -62,16 +87,16 @@ std::string generateRow(const Row& r, int buffer_size = lengths.at(column) - utf8StringSize(r.at(column)) + 1; if (buffer_size > 0) { - output += "| " + r.at(column) + std::string(buffer_size, ' '); + out += kToken + " " + r.at(column) + std::string(buffer_size, ' '); } } - if (output.size() > 0) { + if (out.size() > 0) { // Only append if a row was added. - output += "|\n"; + out += kToken + "\n"; } - return output; + return out; } void prettyPrint(const QueryData& results, diff --git a/osquery/dispatcher/dispatcher.h b/osquery/dispatcher/dispatcher.h index e8bde8c..f4d23e5 100644 --- a/osquery/dispatcher/dispatcher.h +++ b/osquery/dispatcher/dispatcher.h @@ -77,7 +77,7 @@ typedef SHARED_PTR_IMPL ThriftInternalRunnableRef; typedef SHARED_PTR_IMPL ThriftThreadFactory; /** - * @brief Singleton for queueing asynchronous tasks to be executed in parallel + * @brief Singleton for queuing asynchronous tasks to be executed in parallel * * Dispatcher is a singleton which can be used to coordinate the parallel * execution of asynchronous tasks across an application. Internally, diff --git a/osquery/distributed/distributed.cpp b/osquery/distributed/distributed.cpp index 5f87b37..5647b1f 100644 --- a/osquery/distributed/distributed.cpp +++ b/osquery/distributed/distributed.cpp @@ -89,8 +89,7 @@ Status DistributedQueryHandler::serializeResults( return s; } } - } - catch (const std::exception& e) { + } catch (const std::exception& e) { return Status(1, std::string("Error serializing results: ") + e.what()); } return Status(); @@ -139,8 +138,7 @@ Status DistributedQueryHandler::doQueries() { std::ostringstream ss; pt::write_json(ss, serialized_results, false); json = ss.str(); - } - catch (const std::exception& e) { + } catch (const pt::json_parser::json_parser_error& e) { return Status(1, e.what()); } diff --git a/osquery/events/events.cpp b/osquery/events/events.cpp index 50dd2d5..5b0cd8b 100644 --- a/osquery/events/events.cpp +++ b/osquery/events/events.cpp @@ -27,9 +27,14 @@ namespace osquery { /// Helper cooloff (ms) macro to prevent thread failure thrashing. #define EVENTS_COOLOFF 20 -FLAG(bool, disable_events, false, "Disable osquery events pubsub"); +FLAG(bool, disable_events, false, "Disable osquery publish/subscribe system"); -FLAG(int32, events_expiry, 86000, "Timeout to expire event pubsub results"); +FLAG(bool, + events_optimize, + true, + "Optimize subscriber select queries (scheduler only)"); + +FLAG(int32, events_expiry, 86000, "Timeout to expire event subscriber results"); const std::vector kEventTimeLists = { 1 * 60 * 60, // 1 hour @@ -41,6 +46,40 @@ void publisherSleep(size_t milli) { boost::this_thread::sleep(boost::posix_time::milliseconds(milli)); } +QueryData EventSubscriberPlugin::genTable(QueryContext& context) { + EventTime start = 0, stop = -1; + if (context.constraints["time"].getAll().size() > 0) { + // Use the 'time' constraint to optimize backing-store lookups. + for (const auto& constraint : context.constraints["time"].getAll()) { + EventTime expr = 0; + try { + expr = boost::lexical_cast(constraint.expr); + } catch (const boost::bad_lexical_cast& e) { + expr = 0; + } + if (constraint.op == EQUALS) { + stop = start = expr; + break; + } else if (constraint.op == GREATER_THAN) { + start = std::max(start, expr + 1); + } else if (constraint.op == GREATER_THAN_OR_EQUALS) { + start = std::max(start, expr); + } else if (constraint.op == LESS_THAN) { + stop = std::min(stop, expr - 1); + } else if (constraint.op == LESS_THAN_OR_EQUALS) { + stop = std::min(stop, expr); + } + } + } else if (kToolType == OSQUERY_TOOL_DAEMON && FLAGS_events_optimize) { + // If the daemon is querying a subscriber without a 'time' constraint and + // allows optimization, only emit events since the last query. + start = optimize_time_; + optimize_time_ = getUnixTime() - 1; + } + + return get(start, stop); +} + void EventPublisherPlugin::fire(const EventContextRef& ec, EventTime time) { EventContextID ec_id; @@ -64,9 +103,6 @@ void EventPublisherPlugin::fire(const EventContextRef& ec, EventTime time) { // Todo: add a check to assure normalized (seconds) time. ec->time = time; } - - // Set the optional string-verion of the time for DB columns. - ec->time_string = std::to_string(ec->time); } for (const auto& subscription : subscriptions_) { @@ -77,12 +113,12 @@ void EventPublisherPlugin::fire(const EventContextRef& ec, EventTime time) { } } -std::vector EventSubscriberPlugin::getIndexes(EventTime start, - EventTime stop, - int list_key) { +std::set EventSubscriberPlugin::getIndexes(EventTime start, + EventTime stop, + int list_key) { auto db = DBHandle::getInstance(); auto index_key = "indexes." + dbNamespace(); - std::vector indexes; + std::set indexes; // Keep track of the tail/head of account time while bin searching. EventTime start_max = stop, stop_min = stop, local_start, local_stop; @@ -127,7 +163,7 @@ std::vector EventSubscriberPlugin::getIndexes(EventTime start, } // (1) The first iteration will have 1 range (start to start_max=stop). - // (2) Itermediate iterations will have 2 (start-start_max, stop-stop_min). + // (2) Intermediate iterations will have 2 (start-start_max, stop-stop_min). // For each iteration the range collapses based on the coverage using // the first bin's start time and the last bin's stop time. // (3) The last iteration's range includes relaxed bounds outside the @@ -139,18 +175,28 @@ std::vector EventSubscriberPlugin::getIndexes(EventTime start, auto step = boost::lexical_cast(bin); // Check if size * step -> size * (step + 1) is within a range. int bin_start = size * step, bin_stop = size * (step + 1); - if (expire_events_ && step * size < expire_time_) { - expirations.push_back(list_type + "." + bin); - } else if (bin_start >= start && bin_stop <= start_max) { + if (expire_events_ && expire_time_ > 0) { + if (bin_stop <= expire_time_) { + expirations.push_back(bin); + } else if (bin_start < expire_time_) { + expireRecords(list_type, bin, false); + } + } + + if (bin_start >= start && bin_stop <= start_max) { bins.push_back(bin); } else if ((bin_start >= stop_min && bin_stop <= stop) || stop == 0) { bins.push_back(bin); } } - expireIndexes(list_type, all_bins, expirations); + // Rewrite the index lists and delete each expired item. + if (expirations.size() > 0) { + expireIndexes(list_type, all_bins, expirations); + } + if (bins.size() != 0) { - // If more percision was acheived though this list's binning. + // If more precision was achieved though this list's binning. local_start = boost::lexical_cast(bins.front()) * size; start_max = (local_start < start_max) ? local_start : start_max; local_stop = (boost::lexical_cast(bins.back()) + 1) * size; @@ -158,7 +204,7 @@ std::vector EventSubscriberPlugin::getIndexes(EventTime start, } for (const auto& bin : bins) { - indexes.push_back(list_type + "." + bin); + indexes.insert(list_type + "." + bin); } if (start == start_max && stop == stop_min) { @@ -170,26 +216,47 @@ std::vector EventSubscriberPlugin::getIndexes(EventTime start, return indexes; } -Status EventSubscriberPlugin::expireIndexes( - const std::string& list_type, - const std::vector& indexes, - const std::vector& expirations) { +void EventSubscriberPlugin::expireRecords(const std::string& list_type, + const std::string& index, + bool all) { auto db = DBHandle::getInstance(); - auto index_key = "indexes." + dbNamespace(); auto record_key = "records." + dbNamespace(); auto data_key = "data." + dbNamespace(); - // Get the records list for the soon-to-be expired records. - std::vector record_indexes; - for (const auto& bin : expirations) { - record_indexes.push_back(list_type + "." + bin); + // If the expirations is not removing all records, rewrite the persisting. + std::vector persisting_records; + // Request all records within this list-size + bin offset. + auto expired_records = getRecords({list_type + "." + index}); + for (const auto& record : expired_records) { + if (all) { + db->Delete(kEvents, data_key + "." + record.first); + } else if (record.second > expire_time_) { + persisting_records.push_back(record.first + ":" + + std::to_string(record.second)); + } } - auto expired_records = getRecords(record_indexes); - // Remove the records using the list of expired indexes. + // Either drop or overwrite the record list. + if (all) { + db->Delete(kEvents, record_key + "." + list_type + "." + index); + } else { + auto new_records = boost::algorithm::join(persisting_records, ","); + db->Put(kEvents, record_key + "." + list_type + "." + index, new_records); + } +} + +void EventSubscriberPlugin::expireIndexes( + const std::string& list_type, + const std::vector& indexes, + const std::vector& expirations) { + auto db = DBHandle::getInstance(); + auto index_key = "indexes." + dbNamespace(); + + // Construct a mutable list of persisting indexes to rewrite as records. std::vector persisting_indexes = indexes; + // Remove the records using the list of expired indexes. for (const auto& bin : expirations) { - db->Delete(kEvents, record_key + "." + list_type + "." + bin); + expireRecords(list_type, bin, true); persisting_indexes.erase( std::remove(persisting_indexes.begin(), persisting_indexes.end(), bin), persisting_indexes.end()); @@ -198,21 +265,14 @@ Status EventSubscriberPlugin::expireIndexes( // Update the list of indexes with the non-expired indexes. auto new_indexes = boost::algorithm::join(persisting_indexes, ","); db->Put(kEvents, index_key + "." + list_type, new_indexes); - - // Delete record events. - for (const auto& record : expired_records) { - db->Delete(kEvents, data_key + "." + record.first); - } - - return Status(0, "OK"); } std::vector EventSubscriberPlugin::getRecords( - const std::vector& indexes) { + const std::set& indexes) { auto db = DBHandle::getInstance(); auto record_key = "records." + dbNamespace(); - std::vector records; + std::vector records; for (const auto& index : indexes) { std::string record_value; if (!db->Get(kEvents, record_key + "." + index, record_value).ok()) { @@ -345,6 +405,11 @@ QueryData EventSubscriberPlugin::get(EventTime start, EventTime stop) { } std::string events_key = "data." + dbNamespace(); + if (FLAGS_events_expiry > 0) { + // Set the expire time to NOW - "configured lifetime". + // Index retrieval will apply the constraints checking and auto-expire. + expire_time_ = getUnixTime() - FLAGS_events_expiry; + } // Select mapped_records using event_ids as keys. std::string data_value; @@ -363,10 +428,8 @@ QueryData EventSubscriberPlugin::get(EventTime start, EventTime stop) { return results; } -Status EventSubscriberPlugin::add(const Row& r, EventTime time) { - Status status; - - std::shared_ptr db; +Status EventSubscriberPlugin::add(Row& r, EventTime event_time) { + std::shared_ptr db = nullptr; try { db = DBHandle::getInstance(); } catch (const std::runtime_error& e) { @@ -375,19 +438,26 @@ Status EventSubscriberPlugin::add(const Row& r, EventTime time) { // Get and increment the EID for this module. EventID eid = getEventID(); + // Without encouraging a missing event time, do not support a 0-time. + auto index_time = getUnixTime(); + if (event_time == 0) { + r["time"] = std::to_string(index_time); + } else { + r["time"] = std::to_string(event_time); + } - std::string event_key = "data." + dbNamespace() + "." + eid; + // Serialize and store the row data, for query-time retrieval. std::string data; - - status = serializeRowJSON(r, data); + auto status = serializeRowJSON(r, data); if (!status.ok()) { return status; } // Store the event data. + std::string event_key = "data." + dbNamespace() + "." + eid; status = db->Put(kEvents, event_key, data); - // Record the event in the indexing bins. - recordEvent(eid, time); + // Record the event in the indexing bins, using the index time. + recordEvent(eid, event_time); return status; } @@ -407,19 +477,29 @@ void EventFactory::delay() { } Status EventFactory::run(EventPublisherID& type_id) { + auto& ef = EventFactory::getInstance(); + if (FLAGS_disable_events) { + return Status(0, "Events disabled"); + } + // An interesting take on an event dispatched entrypoint. // There is little introspection into the event type. // Assume it can either make use of an entrypoint poller/selector or // take care of async callback registrations in setUp/configure/run - // only once and handle event queueing/firing in callbacks. - EventPublisherRef publisher; + // only once and handle event queuing/firing in callbacks. + EventPublisherRef publisher = nullptr; try { - publisher = EventFactory::getInstance().getEventPublisher(type_id); + publisher = ef.getEventPublisher(type_id); } catch (std::out_of_range& e) { return Status(1, "No event type found"); } - VLOG(1) << "Starting event publisher runloop: " + type_id; + if (publisher == nullptr) { + return Status(1, "Event publisher is missing"); + } else if (publisher->hasStarted()) { + return Status(1, "Cannot restart an event publisher"); + } + VLOG(1) << "Starting event publisher run loop: " + type_id; publisher->hasStarted(true); auto status = Status(0, "OK"); @@ -428,11 +508,12 @@ Status EventFactory::run(EventPublisherID& type_id) { status = publisher->run(); osquery::publisherSleep(EVENTS_COOLOFF); } - // The runloop status is not reflective of the event type's. - publisher->tearDown(); VLOG(1) << "Event publisher " << publisher->type() - << " runloop terminated for reason: " << status.getMessage(); + << " run loop terminated for reason: " << status.getMessage(); + // Publishers auto tear down when their run loop stops. + publisher->tearDown(); + ef.event_pubs_.erase(type_id); return Status(0, "OK"); } @@ -463,9 +544,12 @@ Status EventFactory::registerEventPublisher(const PluginRef& pub) { return Status(1, "Duplicate publisher type"); } - if (!specialized_pub->setUp().ok()) { - // Only start event loop if setUp succeeds. - return Status(1, "Event publisher setUp failed"); + // Do not set up event publisher if events are disabled. + if (!FLAGS_disable_events) { + if (!specialized_pub->setUp().ok()) { + // Only start event loop if setUp succeeds. + return Status(1, "Event publisher setup failed"); + } } ef.event_pubs_[type_id] = specialized_pub; @@ -487,7 +571,10 @@ Status EventFactory::registerEventSubscriber(const PluginRef& sub) { } // Let the module initialize any Subscriptions. - auto status = specialized_sub->init(); + auto status = Status(0, "OK"); + if (!FLAGS_disable_events) { + status = specialized_sub->init(); + } auto& ef = EventFactory::getInstance(); ef.event_subs_[specialized_sub->getName()] = specialized_sub; @@ -520,7 +607,9 @@ Status EventFactory::addSubscription(EventPublisherID& type_id, // The event factory is responsible for configuring the event types. auto status = publisher->addSubscription(subscription); - publisher->configure(); + if (!FLAGS_disable_events) { + publisher->configure(); + } return status; } @@ -568,14 +657,20 @@ Status EventFactory::deregisterEventPublisher(EventPublisherID& type_id) { return Status(1, "No event publisher to deregister"); } - publisher->isEnding(true); - if (!publisher->hasStarted()) { - // If a publisher's run loop was not started, call tearDown since - // the setUp happened at publisher registration time. - publisher->tearDown(); + if (!FLAGS_disable_events) { + publisher->isEnding(true); + if (!publisher->hasStarted()) { + // If a publisher's run loop was not started, call tearDown since + // the setUp happened at publisher registration time. + publisher->tearDown(); + // If the run loop did run the tear down and erase will happen in the + // event + // thread wrapper when isEnding is next checked. + ef.event_pubs_.erase(type_id); + } else { + publisher->end(); + } } - - ef.event_pubs_.erase(type_id); return Status(0, "OK"); } @@ -612,18 +707,14 @@ void EventFactory::end(bool join) { } } - ::usleep(400); - ef.threads_.clear(); + // A small cool off helps OS API event publisher flushing. + if (!FLAGS_disable_events) { + ::usleep(400); + ef.threads_.clear(); + } } -typedef std::shared_ptr EventPublisherPluginRef; - void attachEvents() { - // Caller may disable events, do not setup any publishers or subscribers. - if (FLAGS_disable_events) { - return; - } - const auto& publishers = Registry::all("event_publisher"); for (const auto& publisher : publishers) { EventFactory::registerEventPublisher(publisher.second); diff --git a/osquery/events/linux/inotify.cpp b/osquery/events/linux/inotify.cpp index a28877d..69c0845 100644 --- a/osquery/events/linux/inotify.cpp +++ b/osquery/events/linux/inotify.cpp @@ -10,13 +10,18 @@ #include +#include #include +#include + #include #include #include "osquery/events/linux/inotify.h" +namespace fs = boost::filesystem; + namespace osquery { int kINotifyMLatency = 200; @@ -42,18 +47,49 @@ Status INotifyEventPublisher::setUp() { inotify_handle_ = ::inotify_init(); // If this does not work throw an exception. if (inotify_handle_ == -1) { - return Status(1, "Could not init inotify"); + return Status(1, "Could not start inotify: inotify_init failed"); } return Status(0, "OK"); } void INotifyEventPublisher::configure() { - for (const auto& sub : subscriptions_) { + for (auto& sub : subscriptions_) { // Anytime a configure is called, try to monitor all subscriptions. // Configure is called as a response to removing/adding subscriptions. // This means recalculating all monitored paths. auto sc = getSubscriptionContext(sub->context); - addMonitor(sc->path, sc->recursive); + if (sc->discovered_.size() > 0) { + continue; + } + + sc->discovered_ = sc->path; + if (sc->path.find("**") != std::string::npos) { + sc->recursive = true; + sc->discovered_ = sc->path.substr(0, sc->path.find("**")); + sc->path = sc->discovered_; + } + + if (sc->path.find('*') != std::string::npos) { + // If the wildcard exists within the file (leaf), remove and monitor the + // directory instead. Apply a fnmatch on fired events to filter leafs. + auto fullpath = fs::path(sc->path); + if (fullpath.filename().string().find('*') != std::string::npos) { + sc->discovered_ = fullpath.parent_path().string(); + } + + if (sc->discovered_.find('*') != std::string::npos) { + // If a wildcard exists within the tree (stem), resolve at configure + // time and monitor each path. + std::vector paths; + resolveFilePattern(sc->discovered_, paths); + for (const auto& _path : paths) { + addMonitor(_path, sc->recursive); + } + sc->recursive_match = sc->recursive; + continue; + } + } + addMonitor(sc->discovered_, sc->recursive); } } @@ -123,9 +159,6 @@ Status INotifyEventPublisher::run() { removeMonitor(event->wd, false); } else { auto ec = createEventContextFrom(event); - if(event->mask & IN_CREATE && isDirectory(ec->path).ok()){ - addMonitor(ec->path, 1); - } fire(ec); } // Continue to iterate @@ -143,12 +176,11 @@ INotifyEventContextRef INotifyEventPublisher::createEventContextFrom( ec->event = shared_event; // Get the pathname the watch fired on. - std::ostringstream path; - path << descriptor_paths_[event->wd]; + ec->path = descriptor_paths_[event->wd]; if (event->len > 1) { - path << "/" << event->name; + ec->path += event->name; } - ec->path = path.str(); + for (const auto& action : kMaskActions) { if (event->mask & action.first) { ec->action = action.second; @@ -160,20 +192,29 @@ INotifyEventContextRef INotifyEventPublisher::createEventContextFrom( bool INotifyEventPublisher::shouldFire(const INotifySubscriptionContextRef& sc, const INotifyEventContextRef& ec) const { - if (!sc->recursive && sc->path != ec->path) { - // Monitored path is not recursive and path is not an exact match. - return false; - } - - if (ec->path.find(sc->path) != 0) { - // The path does not exist as the base event path. + if (sc->recursive && !sc->recursive_match) { + ssize_t found = ec->path.find(sc->path); + if (found != 0) { + return false; + } + } else if (fnmatch((sc->path + "*").c_str(), + ec->path.c_str(), + FNM_PATHNAME | FNM_CASEFOLD | + ((sc->recursive_match) ? FNM_LEADING_DIR : 0)) != 0) { + // Only apply a leading-dir match if this is a recursive watch with a + // match requirement (an inline wildcard with ending recursive wildcard). return false; } - // The subscription may supply a required event mask. if (sc->mask != 0 && !(ec->event->mask & sc->mask)) { return false; } + + // inotify will not monitor recursively, new directories need watches. + if(sc->recursive && ec->action == "CREATED" && isDirectory(ec->path)){ + const_cast(this)->addMonitor(ec->path + '/', true); + } + return true; } @@ -196,7 +237,7 @@ bool INotifyEventPublisher::addMonitor(const std::string& path, if (recursive && isDirectory(path).ok()) { std::vector children; - // Get a list of children of this directory (requesed recursive watches). + // Get a list of children of this directory (requested recursive watches). listDirectoriesInDirectory(path, children); for (const auto& child : children) { diff --git a/osquery/events/linux/inotify.h b/osquery/events/linux/inotify.h index 32da7b6..50d07b8 100644 --- a/osquery/events/linux/inotify.h +++ b/osquery/events/linux/inotify.h @@ -23,13 +23,12 @@ namespace osquery { extern std::map kMaskActions; /** - * @brief Subscriptioning details for INotifyEventPublisher events. + * @brief Subscription details for INotifyEventPublisher events. * * This context is specific to INotifyEventPublisher. It allows the - *subscriptioning - * EventSubscriber to set a path (file or directory) and a limited action mask. - * Events are passed to the subscriptioning EventSubscriber if they match the - *context + * subscribing EventSubscriber to set a path (file or directory) and a + * limited action mask. + * Events are passed to the EventSubscriber if they match the context * path (or anything within a directory if the path is a directory) and if the * event action is part of the mask. If the mask is 0 then all actions are * passed to the EventSubscriber. @@ -37,12 +36,13 @@ extern std::map kMaskActions; struct INotifySubscriptionContext : public SubscriptionContext { /// Subscription the following filesystem path. std::string path; - /// Limit the `inotify` actions to the subscriptioned mask (if not 0). + /// Limit the `inotify` actions to the subscription mask (if not 0). uint32_t mask; /// Treat this path as a directory and subscription recursively. bool recursive; - INotifySubscriptionContext() : mask(0), recursive(false) {} + INotifySubscriptionContext() + : mask(0), recursive(false), recursive_match(false) {} /** * @brief Helper method to map a string action to `inotify` action mask bit. @@ -58,6 +58,15 @@ struct INotifySubscriptionContext : public SubscriptionContext { } } } + + private: + /// During configure the INotify publisher may modify/optimize the paths. + std::string discovered_; + /// A configure-time pattern was expanded to match absolute paths. + bool recursive_match; + + private: + friend class INotifyEventPublisher; }; /** @@ -117,28 +126,43 @@ class INotifyEventPublisher private: INotifyEventContextRef createEventContextFrom(struct inotify_event* event); + /// Check all added Subscription%s for a path. bool isPathMonitored(const std::string& path); + /// Add an INotify watch (monitor) on this path. bool addMonitor(const std::string& path, bool recursive); + /// Remove an INotify watch (monitor) from our tracking. bool removeMonitor(const std::string& path, bool force = false); bool removeMonitor(int watch, bool force = false); + /// Given a SubscriptionContext and INotifyEventContext match path and action. bool shouldFire(const INotifySubscriptionContextRef& mc, const INotifyEventContextRef& ec) const; + /// Get the INotify file descriptor. int getHandle() { return inotify_handle_; } + /// Get the number of actual INotify active descriptors. int numDescriptors() { return descriptors_.size(); } + /// If we overflow, try and restart the monitor Status restartMonitoring(); // Consider an event queue if separating buffering from firing/servicing. DescriptorVector descriptors_; + + /// Map of watched path string to inotify watch file descriptor. PathDescriptorMap path_descriptors_; + + /// Map of inotify watch file descriptor to watched path string. DescriptorPathMap descriptor_paths_; + + /// The inotify file descriptor handle. int inotify_handle_; + + /// Time in seconds of the last inotify restart. int last_restart_; public: diff --git a/osquery/events/linux/tests/inotify_tests.cpp b/osquery/events/linux/tests/inotify_tests.cpp index 8274f27..3557075 100644 --- a/osquery/events/linux/tests/inotify_tests.cpp +++ b/osquery/events/linux/tests/inotify_tests.cpp @@ -202,6 +202,12 @@ class TestINotifyEventSubscriber public: int callback_count_; std::vector actions_; + + private: + FRIEND_TEST(INotifyTests, test_inotify_fire_event); + FRIEND_TEST(INotifyTests, test_inotify_event_action); + FRIEND_TEST(INotifyTests, test_inotify_optimization); + FRIEND_TEST(INotifyTests, test_inotify_recursion); }; TEST_F(INotifyTests, test_inotify_run) { diff --git a/osquery/events/tests/events_database_tests.cpp b/osquery/events/tests/events_database_tests.cpp index 1250ce5..68314a7 100644 --- a/osquery/events/tests/events_database_tests.cpp +++ b/osquery/events/tests/events_database_tests.cpp @@ -20,20 +20,9 @@ namespace osquery { -const std::string kTestingEventsDBPath = "/tmp/rocksdb-osquery-testevents"; +//const std::string kTestingEventsDBPath = "/tmp/rocksdb-osquery-testevents"; -class EventsDatabaseTests : public ::testing::Test { - public: - void SetUp() { - // Setup a testing DB instance - DBHandle::getInstanceAtPath(kTestingEventsDBPath); - } - - void TearDown() { - // Todo: each test set should operate on a clear working directory. - boost::filesystem::remove_all(osquery::kTestingEventsDBPath); - } -}; +class EventsDatabaseTests : public ::testing::Test {}; class FakeEventPublisher : public EventPublisher { @@ -89,24 +78,24 @@ TEST_F(EventsDatabaseTests, test_record_indexing) { // Get a mix of indexes for the lower bounding. indexes = sub->getIndexes(2, (3 * 3600)); output = boost::algorithm::join(indexes, ", "); - EXPECT_EQ(output, "3600.1, 3600.2, 60.1, 10.0, 10.1"); + EXPECT_EQ(output, "10.0, 10.1, 3600.1, 3600.2, 60.1"); // Rare, but test ONLY intermediate indexes. indexes = sub->getIndexes(2, (3 * 3600), 1); output = boost::algorithm::join(indexes, ", "); - EXPECT_EQ(output, "60.0, 60.1, 60.60, 60.120"); + EXPECT_EQ(output, "60.0, 60.1, 60.120, 60.60"); // Add specific indexes to the upper bound. status = sub->testAdd((2 * 3600) + 11); status = sub->testAdd((2 * 3600) + 61); indexes = sub->getIndexes(2 * 3600, (2 * 3600) + 62); output = boost::algorithm::join(indexes, ", "); - EXPECT_EQ(output, "60.120, 10.726"); + EXPECT_EQ(output, "10.726, 60.120"); // Request specific lower and upper bounding. indexes = sub->getIndexes(2, (2 * 3600) + 62); output = boost::algorithm::join(indexes, ", "); - EXPECT_EQ(output, "3600.1, 60.1, 60.120, 10.0, 10.1, 10.726"); + EXPECT_EQ(output, "10.0, 10.1, 10.726, 3600.1, 60.1, 60.120"); } TEST_F(EventsDatabaseTests, test_record_range) { @@ -138,14 +127,32 @@ TEST_F(EventsDatabaseTests, test_record_expiration) { auto sub = std::make_shared(); // No expiration - auto indexes = sub->getIndexes(0, 60); + auto indexes = sub->getIndexes(0, 5000); auto records = sub->getRecords(indexes); - EXPECT_EQ(records.size(), 3); // 1, 2, 11 + EXPECT_EQ(records.size(), 5); // 1, 2, 11, 61, 3601 sub->expire_events_ = true; sub->expire_time_ = 10; - indexes = sub->getIndexes(0, 60); + indexes = sub->getIndexes(0, 5000); + records = sub->getRecords(indexes); + EXPECT_EQ(records.size(), 3); // 11, 61, 3601 + + indexes = sub->getIndexes(0, 5000, 0); + records = sub->getRecords(indexes); + EXPECT_EQ(records.size(), 3); // 11, 61, 3601 + + indexes = sub->getIndexes(0, 5000, 1); + records = sub->getRecords(indexes); + EXPECT_EQ(records.size(), 3); // 11, 61, 3601 + + indexes = sub->getIndexes(0, 5000, 2); + records = sub->getRecords(indexes); + EXPECT_EQ(records.size(), 3); // 11, 61, 3601 + + // Check that get/deletes did not act on cache. + sub->expire_time_ = 0; + indexes = sub->getIndexes(0, 5000); records = sub->getRecords(indexes); - EXPECT_EQ(records.size(), 1); // 11 + EXPECT_EQ(records.size(), 3); // 11, 61, 3601 } } diff --git a/osquery/events/tests/events_tests.cpp b/osquery/events/tests/events_tests.cpp index 343b199..5a983b1 100644 --- a/osquery/events/tests/events_tests.cpp +++ b/osquery/events/tests/events_tests.cpp @@ -51,7 +51,7 @@ struct FakeEventContext : EventContext { int required_value; }; -// Typdef the shared_ptr accessors. +// Typedef the shared_ptr accessors. typedef std::shared_ptr FakeSubscriptionContextRef; typedef std::shared_ptr FakeEventContextRef; @@ -318,7 +318,7 @@ class FakeEventSubscriber : public EventSubscriber { TEST_F(EventsTests, test_event_sub) { auto sub = std::make_shared(); - EXPECT_EQ(sub->type(), "FakePublisher"); + EXPECT_EQ(sub->getType(), "FakePublisher"); EXPECT_EQ(sub->getName(), "FakeSubscriber"); } diff --git a/osquery/extensions/extensions.cpp b/osquery/extensions/extensions.cpp index 4801c12..7ca2ea9 100644 --- a/osquery/extensions/extensions.cpp +++ b/osquery/extensions/extensions.cpp @@ -28,7 +28,7 @@ namespace fs = boost::filesystem; namespace osquery { // Millisecond latency between initalizing manager pings. -const int kExtensionInitializeMLatency = 200; +const size_t kExtensionInitializeLatencyUS = 20000; #ifdef __APPLE__ const std::string kModuleExtension = ".dylib"; @@ -114,17 +114,26 @@ void ExtensionManagerWatcher::watch() { for (const auto& uuid : uuids) { try { auto client = EXClient(getExtensionSocket(uuid)); - // Ping the extension until it goes down. client.get()->ping(status); } catch (const std::exception& e) { - LOG(INFO) << "Extension UUID " << uuid << " has gone away"; - Registry::removeBroadcast(uuid); + failures_[uuid] += 1; continue; } - if (status.code != ExtensionCode::EXT_SUCCESS && fatal_) { - Registry::removeBroadcast(uuid); + if (status.code != ExtensionCode::EXT_SUCCESS) { + LOG(INFO) << "Extension UUID " << uuid << " ping failed"; + failures_[uuid] += 1; + } else { + failures_[uuid] = 0; + } + } + + for (const auto& uuid : failures_) { + if (uuid.second >= 3) { + LOG(INFO) << "Extension UUID " << uuid.first << " has gone away"; + Registry::removeBroadcast(uuid.first); + failures_[uuid.first] = 0; } } } @@ -218,7 +227,11 @@ Status loadModules(const std::string& loadfile) { Status extensionPathActive(const std::string& path, bool use_timeout = false) { // Make sure the extension manager path exists, and is writable. size_t delay = 0; - size_t timeout = atoi(FLAGS_extensions_timeout.c_str()); + // The timeout is given in seconds, but checked interval is microseconds. + size_t timeout = atoi(FLAGS_extensions_timeout.c_str()) * 1000000; + if (timeout < kExtensionInitializeLatencyUS * 10) { + timeout = kExtensionInitializeLatencyUS * 10; + } do { if (pathExists(path) && isWritable(path)) { try { @@ -233,9 +246,9 @@ Status extensionPathActive(const std::string& path, bool use_timeout = false) { break; } // Increase the total wait detail. - delay += kExtensionInitializeMLatency; - ::usleep(kExtensionInitializeMLatency * 1000); - } while (delay < timeout * 1000); + delay += kExtensionInitializeLatencyUS; + ::usleep(kExtensionInitializeLatencyUS); + } while (delay < timeout); return Status(1, "Extension socket not available: " + path); } @@ -247,6 +260,7 @@ Status startExtension(const std::string& name, const std::string& version, const std::string& min_sdk_version) { Registry::setExternal(); + // Latency converted to milliseconds, used as a thread interruptible. auto latency = atoi(FLAGS_extensions_interval.c_str()) * 1000; auto status = startExtensionWatcher(FLAGS_extensions_socket, latency, true); if (!status.ok()) { @@ -528,6 +542,7 @@ Status startExtensionManager(const std::string& manager_path) { return status; } + // Seconds converted to milliseconds, used as a thread interruptible. auto latency = atoi(FLAGS_extensions_interval.c_str()) * 1000; // Start a extension manager watcher, if the manager dies, so should we. Dispatcher::addService( diff --git a/osquery/extensions/interface.h b/osquery/extensions/interface.h index f15d473..4c489db 100644 --- a/osquery/extensions/interface.h +++ b/osquery/extensions/interface.h @@ -195,6 +195,7 @@ class ExtensionWatcher : public InternalRunnable { virtual ~ExtensionWatcher() {} ExtensionWatcher(const std::string& path, size_t interval, bool fatal) : path_(path), interval_(interval), fatal_(fatal) { + // Set the interval to a minimum of 200 milliseconds. interval_ = (interval_ < 200) ? 200 : interval_; } @@ -227,6 +228,10 @@ class ExtensionManagerWatcher : public ExtensionWatcher { /// Start a specialized health check for an ExtensionManager. void watch(); + + private: + /// Allow extensions to fail for several intervals. + std::map failures_; }; class ExtensionRunnerCore : public InternalRunnable { @@ -313,7 +318,7 @@ class EXClient : public EXInternal { public: explicit EXClient(const std::string& path) : EXInternal(path) { client_ = std::make_shared(protocol_); - transport_->open(); + (void)transport_->open(); } const std::shared_ptr& get() { return client_; } @@ -328,7 +333,7 @@ class EXManagerClient : public EXInternal { explicit EXManagerClient(const std::string& manager_path) : EXInternal(manager_path) { client_ = std::make_shared(protocol_); - transport_->open(); + (void)transport_->open(); } const std::shared_ptr& get() { diff --git a/osquery/extensions/tests/extensions_tests.cpp b/osquery/extensions/tests/extensions_tests.cpp index 2f8234c..a76022c 100644 --- a/osquery/extensions/tests/extensions_tests.cpp +++ b/osquery/extensions/tests/extensions_tests.cpp @@ -22,8 +22,8 @@ using namespace osquery::extensions; namespace osquery { -const int kDelayUS = 200; -const int kTimeoutUS = 10000; +const int kDelayUS = 2000; +const int kTimeoutUS = 1000000; const std::string kTestManagerSocket = kTestWorkingDirectory + "test.em"; class ExtensionsTest : public testing::Test { @@ -236,7 +236,7 @@ TEST_F(ExtensionsTest, test_extension_broadcast) { TEST_F(ExtensionsTest, test_extension_module_search) { createMockFileStructure(); - EXPECT_TRUE(loadModules(kFakeDirectory)); + EXPECT_FALSE(loadModules(kFakeDirectory + "/root.txt")); EXPECT_FALSE(loadModules("/dir/does/not/exist")); tearDownMockFileStructure(); } diff --git a/osquery/filesystem/CMakeLists.txt b/osquery/filesystem/CMakeLists.txt index 91e6cd6..20e0c26 100644 --- a/osquery/filesystem/CMakeLists.txt +++ b/osquery/filesystem/CMakeLists.txt @@ -12,8 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License -ADD_OSQUERY_LIBRARY(osquery_filesystem filesystem.cpp - globbing.cpp) +ADD_OSQUERY_LIBRARY(osquery_filesystem filesystem.cpp) ADD_OSQUERY_LIBRARY(osquery_filesystem_linux linux/proc.cpp linux/mem.cpp) diff --git a/osquery/filesystem/filesystem.cpp b/osquery/filesystem/filesystem.cpp index 68f5d7b..b567c1d 100644 --- a/osquery/filesystem/filesystem.cpp +++ b/osquery/filesystem/filesystem.cpp @@ -11,9 +11,11 @@ #include #include +#include #include #include +#include #include #include #include @@ -28,6 +30,10 @@ namespace fs = boost::filesystem; namespace osquery { +FLAG(uint64, read_max, 50 * 1024 * 1024, "Maximum file read size"); +FLAG(uint64, read_user_max, 10 * 1024 * 1024, "Maximum non-su read size"); +FLAG(bool, read_user_links, true, "Read user-owned filesystem links"); + Status writeTextFile(const fs::path& path, const std::string& content, int permissions, @@ -56,27 +62,70 @@ Status writeTextFile(const fs::path& path, return Status(0, "OK"); } -Status readFile(const fs::path& path, std::string& content) { - auto path_exists = pathExists(path); - if (!path_exists.ok()) { - return path_exists; +Status readFile(const fs::path& path, std::string& content, bool dry_run) { + struct stat file; + if (lstat(path.string().c_str(), &file) == 0 && S_ISLNK(file.st_mode)) { + if (file.st_uid != 0 && !FLAGS_read_user_links) { + return Status(1, "User link reads disabled"); + } } - std::stringstream buffer; - fs::ifstream file_h(path); - if (file_h.is_open()) { - buffer << file_h.rdbuf(); - if (file_h.bad()) { + if (stat(path.string().c_str(), &file) < 0) { + return Status(1, "Cannot access path: " + path.string()); + } + + // Apply the max byte-read based on file/link target ownership. + size_t read_max = (file.st_uid == 0) + ? FLAGS_read_max + : std::min(FLAGS_read_max, FLAGS_read_user_max); + std::ifstream is(path.string(), std::ifstream::binary | std::ios::ate); + if (!is.is_open()) { + // Attempt to read without seeking to the end. + is.open(path.string(), std::ifstream::binary); + if (!is) { return Status(1, "Error reading file: " + path.string()); } + } + + // Attempt to read the file size. + ssize_t size = is.tellg(); + + // Erase/clear provided string buffer. + content.erase(); + if (size > read_max) { + VLOG(1) << "Cannot read " << path << " size exceeds limit: " << size + << " > " << read_max; + return Status(1, "File exceeds read limits"); + } + + if (dry_run) { + // The caller is only interested in performing file read checks. + boost::system::error_code ec; + return Status(0, fs::canonical(path, ec).string()); + } + + // Reset seek to the start of the stream. + is.seekg(0); + if (size == -1 || size == 0) { + // Size could not be determined. This may be a special device. + std::stringstream buffer; + buffer << is.rdbuf(); + if (is.bad()) { + return Status(1, "Error reading special file: " + path.string()); + } content.assign(std::move(buffer.str())); } else { - return Status(1, "Could not open file: " + path.string()); + content = std::string(size, '\0'); + is.read(&content[0], size); } - return Status(0, "OK"); } +Status readFile(const fs::path& path) { + std::string blank; + return readFile(path, blank, true); +} + Status isWritable(const fs::path& path) { auto path_exists = pathExists(path); if (!path_exists.ok()) { @@ -122,76 +171,109 @@ Status remove(const fs::path& path) { return Status(status_code, "N/A"); } -Status listFilesInDirectory(const fs::path& path, - std::vector& results, - bool ignore_error) { - fs::directory_iterator begin_iter; - try { - if (!fs::exists(path)) { - return Status(1, "Directory not found: " + path.string()); +static void genGlobs(std::string path, + std::vector& results, + GlobLimits limits) { + // Use our helped escape/replace for wildcards. + replaceGlobWildcards(path); + + // Generate a glob set and recurse for double star. + while (true) { + glob_t data; + glob(path.c_str(), GLOB_TILDE | GLOB_MARK | GLOB_BRACE, nullptr, &data); + size_t count = data.gl_pathc; + for (size_t index = 0; index < count; index++) { + results.push_back(data.gl_pathv[index]); } - - if (!fs::is_directory(path)) { - return Status(1, "Supplied path is not a directory: " + path.string()); + globfree(&data); + // The end state is a non-recursive ending or empty set of matches. + size_t wild = path.rfind("**"); + // Allow a trailing slash after the double wild indicator. + if (count == 0 || wild > path.size() || wild < path.size() - 3) { + break; } - begin_iter = fs::directory_iterator(path); - } catch (const fs::filesystem_error& e) { - return Status(1, e.what()); + path += "/**"; } - fs::directory_iterator end_iter; - for (; begin_iter != end_iter; begin_iter++) { - try { - if (fs::is_regular_file(begin_iter->path())) { - results.push_back(begin_iter->path().string()); - } - } catch (const fs::filesystem_error& e) { - if (ignore_error == 0) { - return Status(1, e.what()); + // Prune results based on settings/requested glob limitations. + auto end = std::remove_if( + results.begin(), results.end(), [limits](const std::string& found) { + return !((found[found.length() - 1] == '/' && limits & GLOB_FOLDERS) || + (found[found.length() - 1] != '/' && limits & GLOB_FILES)); + }); + results.erase(end, results.end()); +} + +Status resolveFilePattern(const fs::path& fs_path, + std::vector& results) { + return resolveFilePattern(fs_path, results, GLOB_ALL); +} + +Status resolveFilePattern(const fs::path& fs_path, + std::vector& results, + GlobLimits setting) { + genGlobs(fs_path.string(), results, setting); + return Status(0, "OK"); +} + +inline void replaceGlobWildcards(std::string& pattern) { + // Replace SQL-wildcard '%' with globbing wildcard '*'. + if (pattern.find("%") != std::string::npos) { + boost::replace_all(pattern, "%", "*"); + } + + // Relative paths are a bad idea, but we try to accommodate. + if ((pattern.size() == 0 || pattern[0] != '/') && pattern[0] != '~') { + pattern = (fs::initial_path() / pattern).string(); + } + + auto base = pattern.substr(0, pattern.find('*')); + if (base.size() > 0) { + boost::system::error_code ec; + auto canonicalized = fs::canonical(base, ec).string(); + if (canonicalized.size() > 0 && canonicalized != base) { + if (isDirectory(canonicalized)) { + // Canonicalized directory paths will not include a trailing '/'. + // However, if the wildcards are applied to files within a directory + // then the missing '/' changes the wildcard meaning. + canonicalized += '/'; } + // We are unable to canonicalize the meaning of post-wildcard limiters. + pattern = canonicalized + pattern.substr(base.size()); } } - return Status(0, "OK"); } -Status listDirectoriesInDirectory(const fs::path& path, - std::vector& results, - bool ignore_error) { - fs::directory_iterator begin_iter; +inline Status listInAbsoluteDirectory(const fs::path& path, + std::vector& results, + GlobLimits limits) { try { - if (!fs::exists(path)) { - return Status(1, "Directory not found"); - } - - auto stat = pathExists(path); - if (!stat.ok()) { - return stat; + if (path.filename() == "*" && !fs::exists(path.parent_path())) { + return Status(1, "Directory not found: " + path.parent_path().string()); } - stat = isDirectory(path); - if (!stat.ok()) { - return stat; + if (path.filename() == "*" && !fs::is_directory(path.parent_path())) { + return Status(1, "Path not a directory: " + path.parent_path().string()); } - begin_iter = fs::directory_iterator(path); } catch (const fs::filesystem_error& e) { return Status(1, e.what()); } - - fs::directory_iterator end_iter; - for (; begin_iter != end_iter; begin_iter++) { - try { - if (fs::is_directory(begin_iter->path())) { - results.push_back(begin_iter->path().string()); - } - } catch (const fs::filesystem_error& e) { - if (ignore_error == 0) { - return Status(1, e.what()); - } - } - } + genGlobs(path.string(), results, limits); return Status(0, "OK"); } +Status listFilesInDirectory(const fs::path& path, + std::vector& results, + bool ignore_error) { + return listInAbsoluteDirectory((path / "*"), results, GLOB_FILES); +} + +Status listDirectoriesInDirectory(const fs::path& path, + std::vector& results, + bool ignore_error) { + return listInAbsoluteDirectory((path / "*"), results, GLOB_FOLDERS); +} + Status getDirectory(const fs::path& path, fs::path& dirpath) { if (!isDirectory(path).ok()) { dirpath = fs::path(path).parent_path().string(); @@ -202,14 +284,14 @@ Status getDirectory(const fs::path& path, fs::path& dirpath) { } Status isDirectory(const fs::path& path) { - try { - if (fs::is_directory(path)) { - return Status(0, "OK"); - } + boost::system::error_code ec; + if (fs::is_directory(path, ec)) { + return Status(0, "OK"); + } + if (ec.value() == 0) { return Status(1, "Path is not a directory: " + path.string()); - } catch (const fs::filesystem_error& e) { - return Status(1, e.what()); } + return Status(ec.value(), ec.message()); } std::set getHomeDirectories() { @@ -263,7 +345,7 @@ const std::string& osqueryHomeDirectory() { } else if (user != nullptr && user->pw_dir != nullptr) { homedir = std::string(user->pw_dir) + "/.osquery"; } else { - // Failover to a temporary directory (used for the shell). + // Fail over to a temporary directory (used for the shell). homedir = "/tmp/osquery"; } } diff --git a/osquery/filesystem/globbing.cpp b/osquery/filesystem/globbing.cpp deleted file mode 100644 index 22bed6c..0000000 --- a/osquery/filesystem/globbing.cpp +++ /dev/null @@ -1,357 +0,0 @@ -/* - * Copyright (c) 2014, Facebook, Inc. - * All rights reserved. - * - * This source code is licensed under the BSD-style license found in the - * LICENSE file in the root directory of this source tree. An additional grant - * of patent rights can be found in the PATENTS file in the same directory. - * - */ - -#include -#include - -#include -#include - -namespace fs = boost::filesystem; - -namespace osquery { - -/** - * @brief Drill down recursively and list all sub files - * - * This functions purpose is to take a path with no wildcards - * and it will recursively go through all files and and return - * them in the results vector. - * - * @param fs_path The entire resolved path - * @param results The vector where results will be returned - * @param rec_depth How many recursions deep the current execution is at - * - * @return An instance of osquery::Status indicating the success of failure of - * the operation - */ -Status doubleStarTraversal(const fs::path& fs_path, - std::vector& results, - ReturnSetting setting, - unsigned int rec_depth) { - if (rec_depth >= kMaxDirectoryTraversalDepth) { - return Status(2, fs_path.string().c_str()); - } - // List files first - if (setting & REC_LIST_FILES) { - Status stat = listFilesInDirectory(fs_path, results); - if (!stat.ok()) { - return Status(0, "OK"); - } - } - std::vector folders; - Status stat = listDirectoriesInDirectory(fs_path, folders); - if (!stat.ok()) { - return Status(0, "OK"); - } - if (setting & REC_LIST_FOLDERS) { - results.push_back(fs_path.string()); - } - for (const auto& folder : folders) { - if (fs::is_symlink(folder)) { - continue; - } - - stat = doubleStarTraversal(folder, results, setting, rec_depth + 1); - if (!stat.ok() && stat.getCode() == 2) { - return stat; - } - } - return Status(0, "OK"); -} - -/** - * @brief Resolve the last component of a file path - * - * This function exists because unlike the other parts of of a file - * path, which should only resolve to folder, a wildcard at the end - * means to list all files in that directory, as does just listing - * folder. Also, a double means to drill down recursively into that - * that folder and list all sub file. - * - * @param fs_path The entire resolved path (except last component) - * @param results The vector where results will be returned - * @param components A path, split by forward slashes - * @param rec_depth How many recursions deep the current execution is at - * - * @return An instance of osquery::Status indicating the success of failure of - * the operation - */ -Status resolveLastPathComponent(const fs::path& fs_path, - std::vector& results, - ReturnSetting setting, - const std::vector& components, - unsigned int rec_depth) { - - // Is the last component a double star? - if (components[components.size() - 1] == kWildcardCharacterRecursive) { - if (setting & REC_EVENT_OPT) { - results.push_back(fs_path.parent_path().string()); - return Status(0, "OK"); - } else { - Status stat = doubleStarTraversal( - fs_path.parent_path(), results, setting, rec_depth); - return stat; - } - } - - try { - // Is the path a file - if ((setting & (REC_EVENT_OPT | REC_LIST_FILES)) > 0 && - fs::is_regular_file(fs_path)) { - results.push_back(fs_path.string()); - return Status(0, "OK"); - } - } catch (const fs::filesystem_error& e) { - return Status(0, "OK"); - } - - std::vector files; - std::vector folders; - Status stat_file = listFilesInDirectory(fs_path.parent_path(), files); - Status stat_fold = listDirectoriesInDirectory(fs_path.parent_path(), folders); - - // Is the last component a wildcard? - if (components[components.size() - 1] == kWildcardCharacter) { - - if (setting & REC_EVENT_OPT) { - results.push_back(fs_path.parent_path().string()); - return Status(0, "OK"); - } - if (setting & REC_LIST_FOLDERS) { - results.push_back(fs_path.parent_path().string()); - for (const auto& fold : folders) { - results.push_back(fold); - } - } - if (setting & REC_LIST_FILES) { - for (const auto& file : files) { - results.push_back(file); - } - } - return Status(0, "OK"); - } - - std::string processed_path = - "/" + - boost::algorithm::join( - std::vector(components.begin(), components.end() - 1), - "/"); - - // Is this a (.*)% type file match - if (components[components.size() - 1].find(kWildcardCharacter, 1) != - std::string::npos && - components[components.size() - 1][0] != kWildcardCharacter[0]) { - - std::string prefix = - processed_path + "/" + - components[components.size() - 1].substr( - 0, components[components.size() - 1].find(kWildcardCharacter, 1)); - if (setting & REC_LIST_FOLDERS) { - for (const auto& fold : folders) { - if (fold.find(prefix, 0) != 0) { - continue; - } - results.push_back(fold); - } - } - if (setting & REC_LIST_FILES || setting & REC_EVENT_OPT) { - for (const auto& file : files) { - if (file.find(prefix, 0) != 0) { - continue; - } - results.push_back(file); - } - } - // Should be a return here? - return Status(0, "OK"); - } - - // Is this a %(.*) type file match - if (components[components.size() - 1][0] == kWildcardCharacter[0]) { - std::string suffix = components[components.size() - 1].substr(1); - if (setting & REC_LIST_FOLDERS) { - for (const auto& fold : folders) { - std::string file_name = - boost::filesystem::path(fold).filename().string(); - size_t pos = file_name.find(suffix); - if (pos != std::string::npos && - pos + suffix.length() == file_name.length()) { - results.push_back(fold); - } - } - } - if (setting & REC_LIST_FILES || setting & REC_EVENT_OPT) { - for (const auto& file : files) { - boost::filesystem::path p(file); - std::string file_name = p.filename().string(); - size_t pos = file_name.find(suffix); - if (pos != std::string::npos && - pos + suffix.length() == file_name.length()) { - results.push_back(file); - } - } - } - return Status(0, "OK"); - } - - // Back out if this path doesn't exist due to invalid path - if (!(pathExists(fs_path).ok())) { - return Status(0, "OK"); - } - - // Is the path a directory - if (fs::is_directory(fs_path)) { - results.push_back(fs_path.string()); - return Status(0, "OK"); - } - - return Status(1, "UNKNOWN FILE TYPE"); -} - -/** - * @brief List all files in a directory recursively - * - * This is an overloaded version of the exported `resolveFilePattern`. This - * version is used internally to facilitate the tracking of the recursion - * depth. - * - * @param results The vector where results will be returned - * @param components A path, split by forward slashes - * @param processed_index What index of components has been resolved so far - * @param rec_depth How many recursions deep the current execution is at - * - * @return An instance of osquery::Status indicating the success of failure of - * the operation - */ -Status resolveFilePattern(std::vector components, - std::vector& results, - ReturnSetting setting = REC_LIST_FILES, - unsigned int processed_index = 0, - unsigned int rec_depth = 0) { - - // Stop recursing here if we've reached out max depth - if (rec_depth >= kMaxDirectoryTraversalDepth) { - return Status(2, "MAX_DEPTH"); - } - - // Handle all parts of the path except last because then we want to get files, - // not directories - for (auto i = processed_index; i < components.size() - 1; i++) { - - // If we encounter a full recursion, that is invalid because it is not - // the last component. So return. - if (components[i] == kWildcardCharacterRecursive) { - return Status(1, kWildcardCharacterRecursive + " NOT LAST COMPONENT"); - } - - // Create a vector to hold all the folders in the current folder - // Build the path we're at out of components - std::vector folders; - - std::string processed_path = - "/" + - boost::algorithm::join(std::vector(components.begin(), - components.begin() + i), - "/"); - Status stat = listDirectoriesInDirectory(processed_path, folders); - // If we couldn't list the directories it's probably because - // the path is invalid (or we don't have permission). Return - // here because this branch is no good. This is not an error - if (!stat.ok()) { - return Status(0, "OK"); - } - // If we just have a wildcard character then we will recurse though - // all folders we find - if (components[i] == kWildcardCharacter) { - for (const auto& dir : folders) { - boost::filesystem::path p(dir); - components[i] = p.filename().string(); - Status stat = resolveFilePattern( - components, results, setting, i + 1, rec_depth + 1); - if (!stat.ok() && stat.getCode() == 2) { - return stat; - } - } - // Our subcalls that handle processing are now complete, return - return Status(0, "OK"); - - // The case of (.*)% - } else if (components[i].find(kWildcardCharacter, 1) != std::string::npos && - components[i][0] != kWildcardCharacter[0]) { - std::string prefix = - processed_path + "/" + - components[i].substr(0, components[i].find(kWildcardCharacter, 1)); - for (const auto& dir : folders) { - if (dir.find(prefix, 0) != 0) { - continue; - } - boost::filesystem::path p(dir); - components[i] = p.filename().string(); - Status stat = resolveFilePattern( - components, results, setting, i + 1, rec_depth + 1); - if (!stat.ok() && stat.getCode() == 2) { - return stat; - } - } - return Status(0, "OK"); - // The case of %(.*) - } else if (components[i][0] == kWildcardCharacter[0]) { - std::string suffix = components[i].substr(1); - for (const auto& dir : folders) { - boost::filesystem::path p(dir); - std::string folder_name = p.filename().string(); - size_t pos = folder_name.find(suffix); - if (pos != std::string::npos && - pos + suffix.length() == folder_name.length()) { - components[i] = p.filename().string(); - Status stat = resolveFilePattern( - components, results, setting, i + 1, rec_depth + 1); - if (!stat.ok() && stat.getCode() == 2) { - return stat; - } - } - } - return Status(0, "OK"); - } else { - } - } - - // At this point, all of our call paths have been resolved, so know we want to - // list the files at this point or do our ** traversal - return resolveLastPathComponent("/" + boost::algorithm::join(components, "/"), - results, - setting, - components, - rec_depth); -} - -Status resolveFilePattern(const fs::path& fs_path, - std::vector& results) { - if (fs_path.string()[0] != '/') { - return resolveFilePattern( - split(fs::current_path().string() + "/" + fs_path.string(), "/"), - results); - } - return resolveFilePattern(split(fs_path.string(), "/"), results); -} - -Status resolveFilePattern(const fs::path& fs_path, - std::vector& results, - ReturnSetting setting) { - if (fs_path.string()[0] != '/') { - return resolveFilePattern( - split(fs::current_path().string() + "/" + fs_path.string(), "/"), - results, - setting); - } - return resolveFilePattern(split(fs_path.string(), "/"), results, setting); -} -} diff --git a/osquery/filesystem/tests/filesystem_tests.cpp b/osquery/filesystem/tests/filesystem_tests.cpp index 16e0796..85c87f7 100644 --- a/osquery/filesystem/tests/filesystem_tests.cpp +++ b/osquery/filesystem/tests/filesystem_tests.cpp @@ -24,15 +24,25 @@ namespace pt = boost::property_tree; namespace osquery { + +DECLARE_uint64(read_max); +DECLARE_uint64(read_user_max); +DECLARE_bool(read_user_links); + class FilesystemTests : public testing::Test { protected: void SetUp() { createMockFileStructure(); } void TearDown() { tearDownMockFileStructure(); } + + /// Helper method to check if a path was included in results. + bool contains(const std::vector& all, const std::string& n) { + return !(std::find(all.begin(), all.end(), n) == all.end()); + } }; -TEST_F(FilesystemTests, test_plugin) { +TEST_F(FilesystemTests, test_read_file) { std::ofstream test_file(kTestWorkingDirectory + "fstests-file"); test_file.write("test123\n", sizeof("test123")); test_file.close(); @@ -46,188 +56,210 @@ TEST_F(FilesystemTests, test_plugin) { remove(kTestWorkingDirectory + "fstests-file"); } -TEST_F(FilesystemTests, test_list_files_in_directory_not_found) { - std::vector not_found_vector; - auto not_found = listFilesInDirectory("/foo/bar", not_found_vector); - EXPECT_FALSE(not_found.ok()); - EXPECT_EQ(not_found.toString(), "Directory not found: /foo/bar"); +TEST_F(FilesystemTests, test_read_symlink) { + std::string content; + auto status = readFile(kFakeDirectory + "/root2.txt", content); + EXPECT_TRUE(status.ok()); + EXPECT_EQ(content, "root"); } -TEST_F(FilesystemTests, test_wildcard_single_file_list) { - std::vector files; - std::vector files_flag; - auto status = resolveFilePattern(kFakeDirectory + "/%", files); - auto status2 = - resolveFilePattern(kFakeDirectory + "/%", files_flag, REC_LIST_FILES); - EXPECT_TRUE(status.ok()); - EXPECT_EQ(files.size(), 3); - EXPECT_EQ(files.size(), files_flag.size()); - EXPECT_NE(std::find(files.begin(), files.end(), kFakeDirectory + "/roto.txt"), - files.end()); +TEST_F(FilesystemTests, test_read_limit) { + auto max = FLAGS_read_max; + auto user_max = FLAGS_read_user_max; + FLAGS_read_max = 3; + std::string content; + auto status = readFile(kFakeDirectory + "/root.txt", content); + EXPECT_FALSE(status.ok()); + FLAGS_read_max = max; + + if (getuid() != 0) { + content.erase(); + FLAGS_read_user_max = 2; + status = readFile(kFakeDirectory + "/root.txt", content); + EXPECT_FALSE(status.ok()); + FLAGS_read_user_max = user_max; + + // Test that user symlinks aren't followed if configured. + // 'root2.txt' is a symlink in this case. + FLAGS_read_user_links = false; + content.erase(); + status = readFile(kFakeDirectory + "/root2.txt", content); + EXPECT_FALSE(status.ok()); + + // Make sure non-link files are still readable. + content.erase(); + status = readFile(kFakeDirectory + "/root.txt", content); + EXPECT_TRUE(status.ok()); + + // Any the links are readable if enabled. + FLAGS_read_user_links = true; + status = readFile(kFakeDirectory + "/root2.txt", content); + EXPECT_TRUE(status.ok()); + } } -TEST_F(FilesystemTests, test_wildcard_dual) { - std::vector files; - auto status = resolveFilePattern(kFakeDirectory + "/%/%", files); - EXPECT_TRUE(status.ok()); - EXPECT_NE(std::find(files.begin(), - files.end(), - kFakeDirectory + "/deep1/level1.txt"), - files.end()); +TEST_F(FilesystemTests, test_list_files_missing_directory) { + std::vector results; + auto status = listFilesInDirectory("/foo/bar", results); + EXPECT_FALSE(status.ok()); } -TEST_F(FilesystemTests, test_wildcard_full_recursion) { - std::vector files; - auto status = resolveFilePattern(kFakeDirectory + "/%%", files); - EXPECT_TRUE(status.ok()); - EXPECT_NE(std::find(files.begin(), - files.end(), - kFakeDirectory + "/deep1/deep2/level2.txt"), - files.end()); +TEST_F(FilesystemTests, test_list_files_invalid_directory) { + std::vector results; + auto status = listFilesInDirectory("/etc/hosts", results); + EXPECT_FALSE(status.ok()); } -TEST_F(FilesystemTests, test_wildcard_end_last_component) { - std::vector files; - auto status = resolveFilePattern(kFakeDirectory + "/%11/%sh", files); - EXPECT_TRUE(status.ok()); - EXPECT_NE(std::find(files.begin(), - files.end(), - kFakeDirectory + "/deep11/not_bash"), - files.end()); +TEST_F(FilesystemTests, test_list_files_valid_directorty) { + std::vector results; + auto s = listFilesInDirectory("/etc", results); + // This directory may be different on OS X or Linux. + std::string hosts_path = "/etc/hosts"; + replaceGlobWildcards(hosts_path); + EXPECT_TRUE(s.ok()); + EXPECT_EQ(s.toString(), "OK"); + EXPECT_TRUE(contains(results, hosts_path)); } -TEST_F(FilesystemTests, test_wildcard_three_kinds) { - std::vector files; - auto status = resolveFilePattern(kFakeDirectory + "/%p11/%/%%", files); - EXPECT_TRUE(status.ok()); - EXPECT_NE(std::find(files.begin(), - files.end(), - kFakeDirectory + "/deep11/deep2/deep3/level3.txt"), - files.end()); +TEST_F(FilesystemTests, test_canonicalization) { + std::string complex = kFakeDirectory + "/deep1/../deep1/.."; + std::string simple = kFakeDirectory + "/"; + // Use the inline wildcard and canonicalization replacement. + // The 'simple' path contains a trailing '/', the replacement method will + // distinguish between file and directory paths. + replaceGlobWildcards(complex); + EXPECT_EQ(simple, complex); + // Now apply the same inline replacement on the simple directory and expect + // no change to the comparison. + replaceGlobWildcards(simple); + EXPECT_EQ(simple, complex); + + // Now add a wildcard within the complex pattern. The replacement method + // will not canonicalize past a '*' as the proceeding paths are limiters. + complex = kFakeDirectory + "/*/deep2/../deep2/"; + replaceGlobWildcards(complex); + EXPECT_EQ(complex, kFakeDirectory + "/*/deep2/../deep2/"); } -TEST_F(FilesystemTests, test_wildcard_invalid_path) { - std::vector files; - auto status = resolveFilePattern("/not_ther_abcdefz/%%", files); +TEST_F(FilesystemTests, test_simple_globs) { + std::vector results; + // Test the shell '*', we will support SQL's '%' too. + auto status = resolveFilePattern(kFakeDirectory + "/*", results); EXPECT_TRUE(status.ok()); - EXPECT_EQ(files.size(), 0); + EXPECT_EQ(results.size(), 6); + + // Test the csh-style bracket syntax: {}. + results.clear(); + resolveFilePattern(kFakeDirectory + "/{root,door}*", results); + EXPECT_EQ(results.size(), 3); + + // Test a tilde, home directory expansion, make no asserts about contents. + results.clear(); + resolveFilePattern("~", results); + if (results.size() == 0) { + LOG(WARNING) << "Tilde expansion failed."; + } } -TEST_F(FilesystemTests, test_wildcard_filewild) { - std::vector files; - auto status = resolveFilePattern(kFakeDirectory + "/deep1%/%", files); +TEST_F(FilesystemTests, test_wildcard_single_all) { + // Use '%' as a wild card to glob files within the temporarily-created dir. + std::vector results; + auto status = resolveFilePattern(kFakeDirectory + "/%", results, GLOB_ALL); EXPECT_TRUE(status.ok()); - EXPECT_NE(std::find(files.begin(), - files.end(), - kFakeDirectory + "/deep1/level1.txt"), - files.end()); - EXPECT_NE(std::find(files.begin(), - files.end(), - kFakeDirectory + "/deep11/level1.txt"), - files.end()); + EXPECT_EQ(results.size(), 6); + EXPECT_TRUE(contains(results, kFakeDirectory + "/roto.txt")); + EXPECT_TRUE(contains(results, kFakeDirectory + "/deep11/")); } -TEST_F(FilesystemTests, test_list_files_in_directory_not_dir) { - std::vector not_dir_vector; - auto not_dir = listFilesInDirectory("/etc/hosts", not_dir_vector); - EXPECT_FALSE(not_dir.ok()); - EXPECT_EQ(not_dir.toString(), "Supplied path is not a directory: /etc/hosts"); +TEST_F(FilesystemTests, test_wildcard_single_files) { + // Now list again with a restriction to only files. + std::vector results; + resolveFilePattern(kFakeDirectory + "/%", results, GLOB_FILES); + EXPECT_EQ(results.size(), 4); + EXPECT_TRUE(contains(results, kFakeDirectory + "/roto.txt")); } -TEST_F(FilesystemTests, test_list_files_in_directorty) { +TEST_F(FilesystemTests, test_wildcard_single_folders) { std::vector results; - auto s = listFilesInDirectory("/etc", results); - EXPECT_TRUE(s.ok()); - EXPECT_EQ(s.toString(), "OK"); - EXPECT_NE(std::find(results.begin(), results.end(), "/etc/hosts"), - results.end()); + resolveFilePattern(kFakeDirectory + "/%", results, GLOB_FOLDERS); + EXPECT_EQ(results.size(), 2); + EXPECT_TRUE(contains(results, kFakeDirectory + "/deep11/")); } -TEST_F(FilesystemTests, test_wildcard_single_folder_list) { - std::vector folders; - auto status = - resolveFilePattern(kFakeDirectory + "/%", folders, REC_LIST_FOLDERS); +TEST_F(FilesystemTests, test_wildcard_dual) { + // Now test two directories deep with a single wildcard for each. + std::vector results; + auto status = resolveFilePattern(kFakeDirectory + "/%/%", results); EXPECT_TRUE(status.ok()); - EXPECT_EQ(folders.size(), 3); - EXPECT_NE( - std::find(folders.begin(), folders.end(), kFakeDirectory + "/deep11"), - folders.end()); + EXPECT_TRUE(contains(results, kFakeDirectory + "/deep1/level1.txt")); } -TEST_F(FilesystemTests, test_wildcard_single_all_list) { - std::vector all; - auto status = resolveFilePattern(kFakeDirectory + "/%", all, REC_LIST_ALL); +TEST_F(FilesystemTests, test_wildcard_double) { + // TODO: this will fail. + std::vector results; + auto status = resolveFilePattern(kFakeDirectory + "/%%", results); EXPECT_TRUE(status.ok()); - EXPECT_EQ(all.size(), 6); - EXPECT_NE(std::find(all.begin(), all.end(), kFakeDirectory + "/roto.txt"), - all.end()); - EXPECT_NE(std::find(all.begin(), all.end(), kFakeDirectory + "/deep11"), - all.end()); + EXPECT_EQ(results.size(), 15); + EXPECT_TRUE(contains(results, kFakeDirectory + "/deep1/deep2/level2.txt")); } TEST_F(FilesystemTests, test_wildcard_double_folders) { - std::vector all; - auto status = - resolveFilePattern(kFakeDirectory + "/%%", all, REC_LIST_FOLDERS); + std::vector results; + resolveFilePattern(kFakeDirectory + "/%%", results, GLOB_FOLDERS); + EXPECT_EQ(results.size(), 5); + EXPECT_TRUE(contains(results, kFakeDirectory + "/deep11/deep2/deep3/")); +} + +TEST_F(FilesystemTests, test_wildcard_end_last_component) { + std::vector results; + auto status = resolveFilePattern(kFakeDirectory + "/%11/%sh", results); EXPECT_TRUE(status.ok()); - EXPECT_EQ(all.size(), 6); - EXPECT_NE(std::find(all.begin(), all.end(), kFakeDirectory), all.end()); - EXPECT_NE( - std::find(all.begin(), all.end(), kFakeDirectory + "/deep11/deep2/deep3"), - all.end()); + EXPECT_TRUE(contains(results, kFakeDirectory + "/deep11/not_bash")); } -TEST_F(FilesystemTests, test_wildcard_double_all) { - std::vector all; - auto status = resolveFilePattern(kFakeDirectory + "/%%", all, REC_LIST_ALL); +TEST_F(FilesystemTests, test_wildcard_middle_component) { + std::vector results; + auto status = resolveFilePattern(kFakeDirectory + "/deep1%/%", results); EXPECT_TRUE(status.ok()); - EXPECT_EQ(all.size(), 15); - EXPECT_NE(std::find(all.begin(), all.end(), kFakeDirectory + "/roto.txt"), - all.end()); - EXPECT_NE( - std::find(all.begin(), all.end(), kFakeDirectory + "/deep11/deep2/deep3"), - all.end()); + EXPECT_EQ(results.size(), 5); + EXPECT_TRUE(contains(results, kFakeDirectory + "/deep1/level1.txt")); + EXPECT_TRUE(contains(results, kFakeDirectory + "/deep11/level1.txt")); } -TEST_F(FilesystemTests, test_double_wild_event_opt) { - std::vector all; - auto status = resolveFilePattern( - kFakeDirectory + "/%%", all, REC_LIST_FOLDERS | REC_EVENT_OPT); + +TEST_F(FilesystemTests, test_wildcard_all_types) { + std::vector results; + auto status = resolveFilePattern(kFakeDirectory + "/%p11/%/%%", results); EXPECT_TRUE(status.ok()); - EXPECT_EQ(all.size(), 1); - EXPECT_NE(std::find(all.begin(), all.end(), kFakeDirectory), all.end()); + EXPECT_TRUE( + contains(results, kFakeDirectory + "/deep11/deep2/deep3/level3.txt")); } -TEST_F(FilesystemTests, test_letter_wild_opt) { - std::vector all; - auto status = resolveFilePattern( - kFakeDirectory + "/d%", all, REC_LIST_FOLDERS | REC_EVENT_OPT); +TEST_F(FilesystemTests, test_wildcard_invalid_path) { + std::vector results; + auto status = resolveFilePattern("/not_ther_abcdefz/%%", results); EXPECT_TRUE(status.ok()); - EXPECT_EQ(all.size(), 3); - EXPECT_NE(std::find(all.begin(), all.end(), kFakeDirectory + "/deep1"), - all.end()); - EXPECT_NE(std::find(all.begin(), all.end(), kFakeDirectory + "/door.txt"), - all.end()); + EXPECT_EQ(results.size(), 0); } -TEST_F(FilesystemTests, test_dotdot) { - std::vector all; +TEST_F(FilesystemTests, test_wildcard_dotdot_files) { + std::vector results; auto status = resolveFilePattern( - kFakeDirectory + "/deep11/deep2/../../%", all, REC_LIST_FILES); + kFakeDirectory + "/deep11/deep2/../../%", results, GLOB_FILES); EXPECT_TRUE(status.ok()); - EXPECT_EQ(all.size(), 3); - EXPECT_NE(std::find(all.begin(), - all.end(), - kFakeDirectory + "/deep11/deep2/../../door.txt"), - all.end()); + EXPECT_EQ(results.size(), 4); + // The response list will contain canonicalized versions: /tmp//... + std::string door_path = kFakeDirectory + "/deep11/deep2/../../door.txt"; + replaceGlobWildcards(door_path); + EXPECT_TRUE(contains(results, door_path)); } TEST_F(FilesystemTests, test_dotdot_relative) { - std::vector all; - auto status = resolveFilePattern(kTestDataPath + "%", all, REC_LIST_ALL); + std::vector results; + auto status = resolveFilePattern(kTestDataPath + "%", results); EXPECT_TRUE(status.ok()); bool found = false; - for (const auto& file : all) { + for (const auto& file : results) { if (file.find("test.config")) { found = true; break; @@ -237,13 +269,12 @@ TEST_F(FilesystemTests, test_dotdot_relative) { } TEST_F(FilesystemTests, test_no_wild) { - std::vector all; + std::vector results; auto status = - resolveFilePattern(kFakeDirectory + "/roto.txt", all, REC_LIST_FILES); + resolveFilePattern(kFakeDirectory + "/roto.txt", results, GLOB_FILES); EXPECT_TRUE(status.ok()); - EXPECT_EQ(all.size(), 1); - EXPECT_NE(std::find(all.begin(), all.end(), kFakeDirectory + "/roto.txt"), - all.end()); + EXPECT_EQ(results.size(), 1); + EXPECT_TRUE(contains(results, kFakeDirectory + "/roto.txt")); } TEST_F(FilesystemTests, test_safe_permissions) { diff --git a/osquery/logger/logger.cpp b/osquery/logger/logger.cpp index 04c0dd8..f0b420b 100644 --- a/osquery/logger/logger.cpp +++ b/osquery/logger/logger.cpp @@ -11,8 +11,10 @@ #include #include +#include #include +#include #include #include #include @@ -25,6 +27,7 @@ FLAG(bool, verbose, false, "Enable verbose informational messages"); FLAG_ALIAS(bool, verbose_debug, verbose); FLAG_ALIAS(bool, debug, verbose); +/// Despite being a configurable option, this is only read/used at load. FLAG(bool, disable_logging, false, "Disable ERROR/INFO logging"); FLAG(string, logger_plugin, "filesystem", "Logger plugin name"); @@ -47,7 +50,7 @@ FLAG(bool, log_result_events, true, "Log scheduled results as events"); * the active logger plugin is handling Glog status logs and (2) it must remove * itself as a Glog target. */ -class BufferedLogSink : google::LogSink { +class BufferedLogSink : public google::LogSink, private boost::noncopyable { public: /// We create this as a Singleton for proper disable/shutdown. static BufferedLogSink& instance() { @@ -103,8 +106,25 @@ class BufferedLogSink : google::LogSink { bool enabled_; }; -void serializeIntermediateLog(const std::vector& log, - PluginRequest& request) { +/// Scoped helper to perform logging actions without races. +class LoggerDisabler { + public: + LoggerDisabler() : stderr_status_(FLAGS_logtostderr) { + BufferedLogSink::disable(); + FLAGS_logtostderr = true; + } + + ~LoggerDisabler() { + BufferedLogSink::enable(); + FLAGS_logtostderr = stderr_status_; + } + + private: + bool stderr_status_; +}; + +static void serializeIntermediateLog(const std::vector& log, + PluginRequest& request) { pt::ptree tree; for (const auto& log_item : log) { pt::ptree child; @@ -121,8 +141,8 @@ void serializeIntermediateLog(const std::vector& log, request["log"] = output.str(); } -void deserializeIntermediateLog(const PluginRequest& request, - std::vector& log) { +static void deserializeIntermediateLog(const PluginRequest& request, + std::vector& log) { if (request.count("log") == 0) { return; } @@ -152,14 +172,14 @@ void setVerboseLevel() { // Turn verbosity up to 1. // Do log DEBUG, INFO, WARNING, ERROR to their log files. // Do log the above and verbose=1 to stderr. - FLAGS_minloglevel = 0; // WARNING - FLAGS_stderrthreshold = 0; + FLAGS_minloglevel = 0; // INFO + FLAGS_stderrthreshold = 0; // INFO FLAGS_v = 1; } else { // Do NOT log INFO, WARNING, ERROR to stderr. // Do log only WARNING, ERROR to log sinks. FLAGS_minloglevel = 1; // WARNING - FLAGS_stderrthreshold = 1; + FLAGS_stderrthreshold = 1; // WARNING } if (FLAGS_disable_logging) { @@ -167,7 +187,7 @@ void setVerboseLevel() { // Do NOT log INFO, WARNING, ERROR to their log files. FLAGS_logtostderr = true; if (!FLAGS_verbose) { - // verbose flag still will still emit logs to stderr. + // verbose flag will still emit logs to stderr. FLAGS_minloglevel = 2; // ERROR } } @@ -192,11 +212,12 @@ void initStatusLogger(const std::string& name) { } void initLogger(const std::string& name, bool forward_all) { - // Check if logging is disabled, if it is no need to shuttle intermediates. + // Check if logging is disabled, if so then no need to shuttle intermediates. if (FLAGS_disable_logging) { return; } + // Stop the buffering sink and store the intermediate logs. BufferedLogSink::disable(); auto intermediate_logs = std::move(BufferedLogSink::dump()); auto& logger_plugin = Registry::getActive("logger"); @@ -210,8 +231,8 @@ void initLogger(const std::string& name, bool forward_all) { serializeIntermediateLog(intermediate_logs, request); auto status = Registry::call("logger", request); if (status.ok() || forward_all) { - // When `init` returns success we re-enabled the log sink in forwarding - // mode. Now, Glog status logs are buffered and sent to logStatus. + // When LoggerPlugin::init returns success we enable the log sink in + // forwarding mode. Then Glog status logs are forwarded to logStatus. BufferedLogSink::forward(true); BufferedLogSink::enable(); } @@ -315,4 +336,26 @@ Status logHealthStatus(const QueryLogItem& item) { } return Registry::call("logger", {{"health", json}}); } + +void relayStatusLogs() { + // Prevent out dumping and registry calling from producing additional logs. + LoggerDisabler disabler; + + // Construct a status log plugin request. + PluginRequest req = {{"status", "true"}}; + auto& status_logs = BufferedLogSink::dump(); + if (status_logs.size() == 0) { + return; + } + + // Skip the registry's logic, and send directly to the core's logger. + PluginResponse resp; + serializeIntermediateLog(status_logs, req); + auto status = callExtension(0, "logger", FLAGS_logger_plugin, req, resp); + if (status.ok()) { + // Flush the buffered status logs. + // Otherwise the extension call failed and the buffering should continue. + status_logs.clear(); + } +} } diff --git a/osquery/main/run.cpp b/osquery/main/run.cpp index 0be4ff0..62a933a 100644 --- a/osquery/main/run.cpp +++ b/osquery/main/run.cpp @@ -21,6 +21,12 @@ DEFINE_string(query, "", "query to execute"); DEFINE_int32(iterations, 1, "times to run the query in question"); DEFINE_int32(delay, 0, "delay before and after the query"); +namespace osquery { + +DECLARE_bool(disable_events); +DECLARE_bool(registry_exceptions); +} + int main(int argc, char* argv[]) { // Only log to stderr FLAGS_logtostderr = true; @@ -35,6 +41,8 @@ int main(int argc, char* argv[]) { } osquery::Registry::setUp(); + osquery::FLAGS_disable_events = true; + osquery::FLAGS_registry_exceptions = true; osquery::attachEvents(); if (FLAGS_delay != 0) { @@ -56,7 +64,6 @@ int main(int argc, char* argv[]) { } // Instead of calling "shutdownOsquery" force the EF to join its threads. - osquery::EventFactory::end(true); GFLAGS_NAMESPACE::ShutDownCommandLineFlags(); return status.getCode(); diff --git a/osquery/registry/registry.cpp b/osquery/registry/registry.cpp index 3ae3232..a5acd7b 100644 --- a/osquery/registry/registry.cpp +++ b/osquery/registry/registry.cpp @@ -21,6 +21,8 @@ namespace osquery { +HIDDEN_FLAG(bool, registry_exceptions, false, "Allow plugin exceptions"); + void RegistryHelperCore::remove(const std::string& item_name) { if (items_.count(item_name) > 0) { items_[item_name]->tearDown(); @@ -299,10 +301,16 @@ Status RegistryFactory::call(const std::string& registry_name, } catch (const std::exception& e) { LOG(ERROR) << registry_name << " registry " << item_name << " plugin caused exception: " << e.what(); + if (FLAGS_registry_exceptions) { + throw e; + } return Status(1, e.what()); } catch (...) { LOG(ERROR) << registry_name << " registry " << item_name << " plugin caused unknown exception"; + if (FLAGS_registry_exceptions) { + throw std::runtime_error(registry_name + ": " + item_name + " failed"); + } return Status(2, "Unknown exception"); } } @@ -519,7 +527,11 @@ void Plugin::setResponse(const std::string& key, const boost::property_tree::ptree& tree, PluginResponse& response) { std::ostringstream output; - boost::property_tree::write_json(output, tree, false); + try { + boost::property_tree::write_json(output, tree, false); + } catch (const pt::json_parser::json_parser_error& e) { + // The plugin response could not be serialized. + } response.push_back({{key, output.str()}}); } } diff --git a/osquery/sql/sql.cpp b/osquery/sql/sql.cpp index 42a96f2..f7b5f87 100644 --- a/osquery/sql/sql.cpp +++ b/osquery/sql/sql.cpp @@ -56,9 +56,7 @@ std::vector SQL::getTableNames() { QueryData SQL::selectAllFrom(const std::string& table) { PluginResponse response; - PluginRequest request; - request["action"] = "generate"; - + PluginRequest request = {{"action", "generate"}}; Registry::call("table", table, request, response); return response; } diff --git a/osquery/sql/sqlite_util.cpp b/osquery/sql/sqlite_util.cpp index a8a83e3..162ad30 100644 --- a/osquery/sql/sqlite_util.cpp +++ b/osquery/sql/sqlite_util.cpp @@ -141,7 +141,7 @@ SQLiteDBInstance SQLiteDBManager::get() { if (!self.lock_.owns_lock() && self.lock_.try_lock()) { if (self.db_ == nullptr) { - // Create primary sqlite DB instance. + // Create primary SQLite DB instance. sqlite3_open(":memory:", &self.db_); attachVirtualTables(self.db_); } @@ -162,7 +162,7 @@ SQLiteDBManager::~SQLiteDBManager() { int queryDataCallback(void* argument, int argc, char* argv[], char* column[]) { if (argument == nullptr) { - LOG(ERROR) << "queryDataCallback received nullptr as data argument"; + VLOG(1) << "Query execution failed: received a bad callback argument"; return SQLITE_MISUSE; } @@ -173,16 +173,18 @@ int queryDataCallback(void* argument, int argc, char* argv[], char* column[]) { r[column[i]] = (argv[i] != nullptr) ? argv[i] : ""; } } - (*qData).push_back(r); + (*qData).push_back(std::move(r)); return 0; } Status queryInternal(const std::string& q, QueryData& results, sqlite3* db) { char* err = nullptr; sqlite3_exec(db, q.c_str(), queryDataCallback, &results, &err); + sqlite3_db_release_memory(db); if (err != nullptr) { + auto error_string = std::string(err); sqlite3_free(err); - return Status(1, "Error running query: " + q); + return Status(1, "Error running query: " + error_string); } return Status(0, "OK"); diff --git a/osquery/sql/sqlite_util.h b/osquery/sql/sqlite_util.h index 926a527..f903867 100644 --- a/osquery/sql/sqlite_util.h +++ b/osquery/sql/sqlite_util.h @@ -21,6 +21,8 @@ #include +#define SQLITE_SOFT_HEAP_LIMIT (5 * 1024 * 1024) + namespace osquery { /** @@ -107,6 +109,7 @@ class SQLiteDBManager : private boost::noncopyable { protected: SQLiteDBManager() : db_(nullptr), lock_(mutex_, boost::defer_lock) { + sqlite3_soft_heap_limit64(SQLITE_SOFT_HEAP_LIMIT); disabled_tables_ = parseDisableTablesFlag(Flag::getValue("disable_tables")); } SQLiteDBManager(SQLiteDBManager const&); @@ -184,7 +187,11 @@ class SQLiteSQLPlugin : SQLPlugin { */ std::string getStringForSQLiteReturnCode(int code); -// the callback for populating a std::vector set of results. "argument" -// should be a non-const reference to a std::vector +/** + * @brief Accumulate rows from an SQLite exec into a QueryData struct. + * + * The callback for populating a std::vector set of results. "argument" + * should be a non-const reference to a std::vector. + */ int queryDataCallback(void* argument, int argc, char* argv[], char* column[]); } diff --git a/osquery/sql/tests/virtual_table_tests.cpp b/osquery/sql/tests/virtual_table_tests.cpp index d6b32bb..d5ac553 100644 --- a/osquery/sql/tests/virtual_table_tests.cpp +++ b/osquery/sql/tests/virtual_table_tests.cpp @@ -64,4 +64,17 @@ TEST_F(VirtualTableTests, test_sqlite3_attach_vtable) { EXPECT_EQ("CREATE VIRTUAL TABLE sample USING sample(foo INTEGER, bar TEXT)", results[0]["sql"]); } + +TEST_F(VirtualTableTests, test_sqlite3_table_joins) { + // Get a database connection. + auto dbc = SQLiteDBManager::get(); + + QueryData results; + // Run a query with a join within. + std::string statement = + "SELECT p.pid FROM osquery_info oi, processes p WHERE oi.pid=p.pid"; + auto status = queryInternal(statement, results, dbc.db()); + EXPECT_TRUE(status.ok()); + EXPECT_EQ(results.size(), 1); +} } diff --git a/osquery/sql/virtual_table.cpp b/osquery/sql/virtual_table.cpp index 4590ccb..c8b62a0 100644 --- a/osquery/sql/virtual_table.cpp +++ b/osquery/sql/virtual_table.cpp @@ -115,15 +115,16 @@ int xColumn(sqlite3_vtab_cursor *cur, sqlite3_context *ctx, int col) { return SQLITE_ERROR; } - const auto &column_name = pVtab->content->columns[col].first; - const auto &type = pVtab->content->columns[col].second; + auto &column_name = pVtab->content->columns[col].first; + auto &type = pVtab->content->columns[col].second; if (pCur->row >= pVtab->content->data[column_name].size()) { return SQLITE_ERROR; } - const auto &value = pVtab->content->data[column_name][pCur->row]; + // Attempt to cast each xFilter-populated row/column to the SQLite type. + auto &value = pVtab->content->data[column_name][pCur->row]; if (type == "TEXT") { - sqlite3_result_text(ctx, value.c_str(), -1, nullptr); + sqlite3_result_text(ctx, value.c_str(), value.size(), SQLITE_STATIC); } else if (type == "INTEGER") { int afinite; try { @@ -197,11 +198,15 @@ static int xFilter(sqlite3_vtab_cursor *pVtabCursor, QueryContext context; for (size_t i = 0; i < pVtab->content->columns.size(); ++i) { + // Clear any data, this is the result container for each column + row. pVtab->content->data[pVtab->content->columns[i].first].clear(); + // Set the column affinity for each optional constraint list. + // There is a separate list for each column name. context.constraints[pVtab->content->columns[i].first].affinity = pVtab->content->columns[i].second; } + // Iterate over every argument to xFilter, filling in constraint values. for (size_t i = 0; i < argc; ++i) { auto expr = (const char *)sqlite3_value_text(argv[i]); if (expr == nullptr) { @@ -223,20 +228,22 @@ static int xFilter(sqlite3_vtab_cursor *pVtabCursor, // Now organize the response rows by column instead of row. auto &data = pVtab->content->data; - for (const auto &row : response) { + for (auto &row : response) { for (const auto &column : pVtab->content->columns) { - try { - auto &value = row.at(column.first); - if (value.size() > FLAGS_value_max) { - data[column.first].push_back(value.substr(0, FLAGS_value_max)); - } else { - data[column.first].push_back(value); - } - } catch (const std::out_of_range &e) { + if (row.count(column.first) == 0) { VLOG(1) << "Table " << pVtab->content->name << " row " << pVtab->content->n << " did not include column " << column.first; data[column.first].push_back(""); + continue; + } + + auto &value = row.at(column.first); + if (value.size() > FLAGS_value_max) { + data[column.first].push_back(value.substr(0, FLAGS_value_max)); + value.clear(); + } else { + data[column.first].push_back(std::move(value)); } } diff --git a/osquery/tables/events/linux/file_events.cpp b/osquery/tables/events/linux/file_events.cpp index ef1ea5f..91a014f 100644 --- a/osquery/tables/events/linux/file_events.cpp +++ b/osquery/tables/events/linux/file_events.cpp @@ -57,7 +57,8 @@ Status FileEventSubscriber::init() { for (const auto& file : element_kv.second) { VLOG(1) << "Added listener to: " << file; auto mc = createSubscriptionContext(); - mc->recursive = 1; + // Use the filesystem globbing pattern to determine recursiveness. + mc->recursive = 0; mc->path = file; mc->mask = IN_ATTRIB | IN_MODIFY | IN_DELETE | IN_CREATE; subscribe(&FileEventSubscriber::Callback, mc, @@ -72,7 +73,6 @@ Status FileEventSubscriber::Callback(const INotifyEventContextRef& ec, const void* user_data) { Row r; r["action"] = ec->action; - r["time"] = ec->time_string; r["target_path"] = ec->path; if (user_data != nullptr) { r["category"] = *(std::string*)user_data; @@ -80,13 +80,16 @@ Status FileEventSubscriber::Callback(const INotifyEventContextRef& ec, r["category"] = "Undefined"; } r["transaction_id"] = INTEGER(ec->event->cookie); - r["md5"] = hashFromFile(HASH_TYPE_MD5, ec->path); - r["sha1"] = hashFromFile(HASH_TYPE_SHA1, ec->path); - r["sha256"] = hashFromFile(HASH_TYPE_SHA256, ec->path); + + if (ec->action == "CREATED" || ec->action == "UPDATED") { + r["md5"] = hashFromFile(HASH_TYPE_MD5, ec->path); + r["sha1"] = hashFromFile(HASH_TYPE_SHA1, ec->path); + r["sha256"] = hashFromFile(HASH_TYPE_SHA256, ec->path); + } + if (ec->action != "" && ec->action != "OPENED") { // A callback is somewhat useless unless it changes the EventSubscriber - // state - // or calls `add` to store a marked up event. + // state or calls `add` to store a marked up event. add(r, ec->time); } return Status(0, "OK"); diff --git a/osquery/tables/events/linux/hardware_events.cpp b/osquery/tables/events/linux/hardware_events.cpp index d832ccc..813a095 100644 --- a/osquery/tables/events/linux/hardware_events.cpp +++ b/osquery/tables/events/linux/hardware_events.cpp @@ -70,8 +70,6 @@ Status HardwareEventSubscriber::Callback(const UdevEventContextRef& ec, r["serial"] = INTEGER(UdevEventPublisher::getValue(device, "ID_SERIAL_SHORT")); r["revision"] = INTEGER(UdevEventPublisher::getValue(device, "ID_REVISION")); - - r["time"] = INTEGER(ec->time); add(r, ec->time); return Status(0, "OK"); } diff --git a/osquery/tables/events/linux/passwd_changes.cpp b/osquery/tables/events/linux/passwd_changes.cpp index 8d85de4..6e36380 100644 --- a/osquery/tables/events/linux/passwd_changes.cpp +++ b/osquery/tables/events/linux/passwd_changes.cpp @@ -61,7 +61,6 @@ Status PasswdChangesEventSubscriber::Callback(const INotifyEventContextRef& ec, const void* user_data) { Row r; r["action"] = ec->action; - r["time"] = ec->time_string; r["target_path"] = ec->path; r["transaction_id"] = INTEGER(ec->event->cookie); if (ec->action != "" && ec->action != "OPENED") { diff --git a/osquery/tables/networking/linux/routes.cpp b/osquery/tables/networking/linux/routes.cpp index 30b8402..d32ad82 100644 --- a/osquery/tables/networking/linux/routes.cpp +++ b/osquery/tables/networking/linux/routes.cpp @@ -39,7 +39,7 @@ std::string getNetlinkIP(int family, const char* buffer) { } Status readNetlink(int socket_fd, int seq, char* output, size_t* size) { - struct nlmsghdr* nl_hdr; + struct nlmsghdr* nl_hdr = nullptr; size_t message_size = 0; do { @@ -162,13 +162,13 @@ QueryData genRoutes(QueryContext& context) { } // Create netlink message header - void* netlink_buffer = malloc(MAX_NETLINK_SIZE); - struct nlmsghdr* netlink_msg = (struct nlmsghdr*)netlink_buffer; - if (netlink_msg == nullptr) { + auto netlink_buffer = (void*)malloc(MAX_NETLINK_SIZE); + if (netlink_buffer == nullptr) { close(socket_fd); return {}; } + auto netlink_msg = (struct nlmsghdr*)netlink_buffer; netlink_msg->nlmsg_len = NLMSG_LENGTH(sizeof(struct rtmsg)); netlink_msg->nlmsg_type = RTM_GETROUTE; // routes from kernel routing table netlink_msg->nlmsg_flags = NLM_F_DUMP | NLM_F_REQUEST; @@ -177,7 +177,7 @@ QueryData genRoutes(QueryContext& context) { // Send the netlink request to the kernel if (send(socket_fd, netlink_msg, netlink_msg->nlmsg_len, 0) < 0) { - VLOG(1) << "Cannot write NETLINK request header to socket"; + TLOG << "Cannot write NETLINK request header to socket"; close(socket_fd); free(netlink_buffer); return {}; @@ -186,7 +186,7 @@ QueryData genRoutes(QueryContext& context) { // Wrap the read socket to support multi-netlink messages size_t size = 0; if (!readNetlink(socket_fd, 1, (char*)netlink_msg, &size).ok()) { - VLOG(1) << "Cannot read NETLINK response from socket"; + TLOG << "Cannot read NETLINK response from socket"; close(socket_fd); free(netlink_buffer); return {}; diff --git a/osquery/tables/networking/utils.cpp b/osquery/tables/networking/utils.cpp index 482a8de..6ece3f1 100644 --- a/osquery/tables/networking/utils.cpp +++ b/osquery/tables/networking/utils.cpp @@ -30,7 +30,6 @@ namespace tables { std::string ipAsString(const struct sockaddr *in) { char dst[INET6_ADDRSTRLEN] = {0}; - // memset(dst, 0, sizeof(dst)); void *in_addr = nullptr; if (in->sa_family == AF_INET) { @@ -71,13 +70,13 @@ int netmaskFromIP(const struct sockaddr *in) { int mask = 0; if (in->sa_family == AF_INET6) { - struct sockaddr_in6 *in6 = (struct sockaddr_in6 *)in; + auto in6 = (struct sockaddr_in6 *)in; for (size_t i = 0; i < 16; i++) { mask += addBits(in6->sin6_addr.s6_addr[i]); } } else { - struct sockaddr_in *in4 = (struct sockaddr_in *)in; - char *address = reinterpret_cast(&in4->sin_addr.s_addr); + auto in4 = (struct sockaddr_in *)in; + auto address = reinterpret_cast(&in4->sin_addr.s_addr); for (size_t i = 0; i < 4; i++) { mask += addBits(address[i]); } @@ -86,7 +85,7 @@ int netmaskFromIP(const struct sockaddr *in) { return mask; } -std::string macAsString(const char *addr) { +inline std::string macAsString(const char *addr) { std::stringstream mac; for (size_t i = 0; i < 6; i++) { @@ -101,50 +100,34 @@ std::string macAsString(const char *addr) { } std::string macAsString(const struct ifaddrs *addr) { - std::stringstream mac; - + static std::string blank_mac = "00:00:00:00:00:00"; if (addr->ifa_addr == nullptr) { // No link or MAC exists. - return ""; + return blank_mac; } #if defined(__linux__) struct ifreq ifr; - - int socket_fd = socket(AF_INET, SOCK_DGRAM, 0); - ifr.ifr_addr.sa_family = AF_INET; memcpy(ifr.ifr_name, addr->ifa_name, IFNAMSIZ); + + int socket_fd = socket(AF_INET, SOCK_DGRAM, 0); + if (socket_fd < 0) { + return blank_mac; + } ioctl(socket_fd, SIOCGIFHWADDR, &ifr); close(socket_fd); - for (size_t i = 0; i < 6; i++) { - mac << std::hex << std::setfill('0') << std::setw(2); - mac << (int)((uint8_t)ifr.ifr_hwaddr.sa_data[i]); - if (i != 5) { - mac << ":"; - } - } + return macAsString(ifr.ifr_hwaddr.sa_data); #else - struct sockaddr_dl *sdl = nullptr; - - sdl = (struct sockaddr_dl *)addr->ifa_addr; + auto sdl = (struct sockaddr_dl *)addr->ifa_addr; if (sdl->sdl_alen != 6) { // Do not support MAC address that are not 6 bytes... - return ""; + return blank_mac; } - for (size_t i = 0; i < sdl->sdl_alen; i++) { - mac << std::hex << std::setfill('0') << std::setw(2); - // Prevent char sign extension. - mac << (int)((uint8_t)sdl->sdl_data[i + sdl->sdl_nlen]); - if (i != 5) { - mac << ":"; - } - } + return macAsString(&sdl->sdl_data[sdl->sdl_nlen]); #endif - - return mac.str(); } } } diff --git a/osquery/tables/system/linux/os_version.cpp b/osquery/tables/system/linux/os_version.cpp index 0d960f8..e5ecad1 100644 --- a/osquery/tables/system/linux/os_version.cpp +++ b/osquery/tables/system/linux/os_version.cpp @@ -22,17 +22,10 @@ namespace xp = boost::xpressive; namespace osquery { namespace tables { -#ifdef CENTOS const std::string kLinuxOSRelease = "/etc/redhat-release"; const std::string kLinuxOSRegex = "(?P\\w+) .* " - "(?P[0-9]+).(?P[0-9]+)[\\.]{0,1}(?P[0-9]+)"; -#else -const std::string kLinuxOSRelease = "/etc/os-release"; -const std::string kLinuxOSRegex = - "VERSION=\"(?P[0-9]+)\\.(?P[0-9]+)[\\.]{0,1}(?P[0-9]+)" - "?.*, (?P[\\w ]*)\"$"; -#endif + "(?P[0-9]+)\\.(?P[0-9]+)[\\.]{0,1}(?P[0-9]+).*"; QueryData genOSVersion(QueryContext& context) { std::string content; diff --git a/osquery/tables/system/linux/processes.cpp b/osquery/tables/system/linux/processes.cpp index 7d30995..23641ec 100644 --- a/osquery/tables/system/linux/processes.cpp +++ b/osquery/tables/system/linux/processes.cpp @@ -56,12 +56,32 @@ inline std::string readProcLink(const std::string& attr, const std::string& pid) return result; } +std::set getProcList(const QueryContext& context) { + std::set pidlist; + if (context.constraints.count("pid") > 0 && + context.constraints.at("pid").exists(EQUALS)) { + for (const auto& pid : context.constraints.at("pid").getAll(EQUALS)) { + if (isDirectory("/proc/" + pid)) { + pidlist.insert(pid); + } + } + } else { + osquery::procProcesses(pidlist); + } + + return pidlist; +} + void genProcessEnvironment(const std::string& pid, QueryData& results) { auto attr = getProcAttr("environ", pid); std::string content; readFile(attr, content); - for (const auto& buf : osquery::split(content, "\n")) { + const char* variable = content.c_str(); + + // Stop at the end of nul-delimited string content. + while (*variable > 0) { + auto buf = std::string(variable); size_t idx = buf.find_first_of("="); Row r; @@ -69,6 +89,7 @@ void genProcessEnvironment(const std::string& pid, QueryData& results) { r["key"] = buf.substr(0, idx); r["value"] = buf.substr(idx + 1); results.push_back(r); + variable += buf.size() + 1; } } @@ -224,17 +245,8 @@ void genProcess(const std::string& pid, QueryData& results) { QueryData genProcesses(QueryContext& context) { QueryData results; - std::set pids; - if (context.constraints["pid"].exists(EQUALS)) { - pids = context.constraints["pid"].getAll(EQUALS); - } else { - osquery::procProcesses(pids); - } - - // Generate data for all pids in the vector. - // If there are comparison constraints this could apply the operator - // before generating the process structure. - for (const auto& pid : pids) { + auto pidlist = getProcList(context); + for (const auto& pid : pidlist) { genProcess(pid, results); } @@ -244,14 +256,8 @@ QueryData genProcesses(QueryContext& context) { QueryData genProcessEnvs(QueryContext& context) { QueryData results; - std::set pids; - if (context.constraints["pid"].exists(EQUALS)) { - pids = context.constraints["pid"].getAll(EQUALS); - } else { - osquery::procProcesses(pids); - } - - for (const auto& pid : pids) { + auto pidlist = getProcList(context); + for (const auto& pid : pidlist) { genProcessEnvironment(pid, results); } @@ -261,14 +267,8 @@ QueryData genProcessEnvs(QueryContext& context) { QueryData genProcessMemoryMap(QueryContext& context) { QueryData results; - std::set pids; - if (context.constraints["pid"].exists(EQUALS)) { - pids = context.constraints["pid"].getAll(EQUALS); - } else { - osquery::procProcesses(pids); - } - - for (const auto& pid : pids) { + auto pidlist = getProcList(context); + for (const auto& pid : pidlist) { genProcessMap(pid, results); } diff --git a/osquery/tables/system/uptime.cpp b/osquery/tables/system/uptime.cpp new file mode 100644 index 0000000..5528896 --- /dev/null +++ b/osquery/tables/system/uptime.cpp @@ -0,0 +1,69 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ + +#include + +#if defined(__APPLE__) + #include + #include + #include +#elif defined(__linux__) + #include +#endif + +namespace osquery { +namespace tables { + +long getUptime() { + #if defined(__APPLE__) + struct timeval boot_time; + size_t len = sizeof(boot_time); + int mib[2] = { + CTL_KERN, + KERN_BOOTTIME + }; + + if (sysctl(mib, 2, &boot_time, &len, NULL, 0) < 0) { + return -1; + } + + time_t seconds_since_boot = boot_time.tv_sec; + time_t current_seconds = time(NULL); + + return long(difftime(current_seconds, seconds_since_boot)); + #elif defined(__linux__) + struct sysinfo sys_info; + + if (sysinfo(&sys_info) != 0) { + return -1; + } + + return sys_info.uptime; + #endif +} + +QueryData genUptime(QueryContext& context) { + Row r; + QueryData results; + long uptime_in_seconds = getUptime(); + + if (uptime_in_seconds >= 0) { + r["days"] = INTEGER(uptime_in_seconds / 60 / 60 / 24); + r["hours"] = INTEGER((uptime_in_seconds / 60 / 60) % 24); + r["minutes"] = INTEGER((uptime_in_seconds / 60) % 60); + r["seconds"] = INTEGER(uptime_in_seconds % 60); + r["total_seconds"] = BIGINT(uptime_in_seconds); + results.push_back(r); + } + + return results; +} +} +} diff --git a/osquery/tables/utility/hash.cpp b/osquery/tables/utility/hash.cpp index 6b152d7..5baf307 100644 --- a/osquery/tables/utility/hash.cpp +++ b/osquery/tables/utility/hash.cpp @@ -36,7 +36,7 @@ void genHashForFile(const std::string& path, QueryData genHash(QueryContext& context) { QueryData results; - // The query must provide a predicate with constratins including path or + // The query must provide a predicate with constraints including path or // directory. We search for the parsed predicate constraints with the equals // operator. auto paths = context.constraints["path"].getAll(EQUALS); diff --git a/osquery/tables/utility/osquery.cpp b/osquery/tables/utility/osquery.cpp index 42d498c..46eb14d 100644 --- a/osquery/tables/utility/osquery.cpp +++ b/osquery/tables/utility/osquery.cpp @@ -221,7 +221,7 @@ QueryData genOsquerySchedule(QueryContext& context) { r["wall_time"] = BIGINT(query.second.wall_time); r["user_time"] = BIGINT(query.second.user_time); r["system_time"] = BIGINT(query.second.system_time); - r["average_memory"] = BIGINT(query.second.memory); + r["average_memory"] = BIGINT(query.second.average_memory); results.push_back(r); } diff --git a/osquery/tables/utility/time.cpp b/osquery/tables/utility/time.cpp index fbd84dc..1b68eff 100644 --- a/osquery/tables/utility/time.cpp +++ b/osquery/tables/utility/time.cpp @@ -9,6 +9,7 @@ */ #include +#include #include @@ -19,9 +20,30 @@ QueryData genTime(QueryContext& context) { Row r; time_t _time = time(0); struct tm* now = localtime(&_time); + struct tm* gmt = gmtime(&_time); + + char weekday[10] = {0}; + strftime(weekday, sizeof(weekday), "%A", now); + + std::string timestamp; + timestamp = asctime(gmt); + boost::algorithm::trim(timestamp); + timestamp += " UTC"; + + char iso_8601[21] = {0}; + strftime(iso_8601, sizeof(iso_8601), "%FT%TZ", gmt); + + r["weekday"] = TEXT(weekday); + r["year"] = INTEGER(now->tm_year + 1900); + r["month"] = INTEGER(now->tm_mon + 1); + r["day"] = INTEGER(now->tm_mday); r["hour"] = INTEGER(now->tm_hour); r["minutes"] = INTEGER(now->tm_min); r["seconds"] = INTEGER(now->tm_sec); + r["unix_time"] = INTEGER(_time); + r["timestamp"] = TEXT(timestamp); + r["iso_8601"] = TEXT(iso_8601); + QueryData results; results.push_back(r); return results; diff --git a/packaging/osquery.spec b/packaging/osquery.spec index d471bb8..30532e2 100644 --- a/packaging/osquery.spec +++ b/packaging/osquery.spec @@ -1,5 +1,5 @@ Name: osquery -Version: 1.4.7 +Version: 1.5.0 Release: 0 License: Apache-2.0 and GPLv2 Summary: A SQL powered operating system instrumentation, monitoring framework. diff --git a/specs/etc_hosts.table b/specs/etc_hosts.table index deb3a32..69f3637 100644 --- a/specs/etc_hosts.table +++ b/specs/etc_hosts.table @@ -1,7 +1,7 @@ table_name("etc_hosts") description("Line-parsed /etc/hosts.") schema([ - Column("address", TEXT), + Column("address", TEXT, "IP address mapping"), Column("hostnames", TEXT, "Raw hosts mapping"), ]) implementation("etc_hosts@genEtcHosts") diff --git a/specs/file_events.table b/specs/file_events.table index b9c7375..dfea56e 100644 --- a/specs/file_events.table +++ b/specs/file_events.table @@ -3,12 +3,12 @@ description("Track time/action changes to files specified in configuration data. schema([ Column("target_path", TEXT, "The path changed"), Column("category", TEXT, "The category of the file"), - Column("time", TEXT, "Time of the change"), Column("action", TEXT, "Change action (UPDATE, REMOVE, etc)"), Column("transaction_id", BIGINT, "ID used during bulk update"), Column("md5", TEXT, "The MD5 of the file after change"), Column("sha1", TEXT, "The SHA1 of the file after change"), Column("sha256", TEXT, "The SHA256 of the file after change"), + Column("time", BIGINT, "Time of file event"), ]) attributes(event_subscriber=True) implementation("file_events@file_events::genTable") diff --git a/specs/hardware_events.table b/specs/hardware_events.table index 3acfa39..ac63700 100644 --- a/specs/hardware_events.table +++ b/specs/hardware_events.table @@ -5,13 +5,13 @@ schema([ Column("path", TEXT, "Local device path assigned (optional)"), Column("type", TEXT, "Type of hardware and hardware event"), Column("driver", TEXT, "Driver claiming the device"), - Column("model", TEXT, "Hadware device model"), - Column("model_id", INTEGER), - Column("vendor", TEXT, "hardware device vendor"), - Column("vendor_id", INTEGER), + Column("model", TEXT, "Hardware device model"), + Column("model_id", INTEGER, "Hardware model identifier"), + Column("vendor", TEXT, "Hardware device vendor"), + Column("vendor_id", INTEGER, "Hardware vendor identifier"), Column("serial", TEXT, "Device serial (optional)"), Column("revision", INTEGER, "Device revision (optional)"), - Column("time", INTEGER, "Time of hardware event"), + Column("time", BIGINT, "Time of hardware event"), ]) attributes(event_subscriber=True) implementation("events/hardware_events@hardware_events::genTable") diff --git a/specs/kernel_info.table b/specs/kernel_info.table index b938c68..ae6b7a4 100644 --- a/specs/kernel_info.table +++ b/specs/kernel_info.table @@ -1,10 +1,10 @@ table_name("kernel_info") description("Basic active kernel information.") schema([ - Column("version", TEXT), - Column("arguments", TEXT), - Column("path", TEXT), - Column("device", TEXT), - Column("md5", TEXT), + Column("version", TEXT, "Kernel version"), + Column("arguments", TEXT, "Kernel arguments"), + Column("path", TEXT, "Kernel path"), + Column("device", TEXT, "Kernel device identifier"), + Column("md5", TEXT, "MD5 hash of Kernel"), ]) implementation("system/kernel_info@genKernelInfo") diff --git a/specs/last.table b/specs/last.table index c1252d0..d7ed96d 100644 --- a/specs/last.table +++ b/specs/last.table @@ -1,11 +1,11 @@ table_name("last") description("System logins and logouts.") schema([ - Column("username", TEXT), - Column("tty", TEXT), + Column("username", TEXT, "Entry username"), + Column("tty", TEXT, "Entry terminal"), Column("pid", INTEGER, "Process (or thread) ID"), - Column("type", INTEGER), - Column("time", INTEGER), - Column("host", TEXT), + Column("type", INTEGER, "Entry type, according to ut_type types (utmp.h)"), + Column("time", INTEGER, "Entry timestamp"), + Column("host", TEXT, "Entry hostname"), ]) implementation("last@genLastAccess") diff --git a/specs/linux/kernel_modules.table b/specs/linux/kernel_modules.table index d4566fb..654f63c 100644 --- a/specs/linux/kernel_modules.table +++ b/specs/linux/kernel_modules.table @@ -4,7 +4,7 @@ schema([ Column("name", TEXT, "Module name"), Column("size", TEXT, "Size of module content"), Column("used_by", TEXT, "Module reverse dependencies"), - Column("status", TEXT), - Column("address", TEXT), + Column("status", TEXT, "Kernel module status"), + Column("address", TEXT, "Kernel module address"), ]) implementation("kernel_modules@genKernelModules") diff --git a/specs/mounts.table b/specs/mounts.table index ec055ee..09ae840 100644 --- a/specs/mounts.table +++ b/specs/mounts.table @@ -1,16 +1,16 @@ table_name("mounts") description("System mounted devices and filesystems (not process specific).") schema([ - Column("device", TEXT), - Column("device_alias", TEXT), - Column("path", TEXT), - Column("type", TEXT), - Column("blocks_size", BIGINT), - Column("blocks", BIGINT), - Column("blocks_free", BIGINT), - Column("blocks_available", BIGINT), - Column("inodes", BIGINT), - Column("inodes_free", BIGINT), - Column("flags", TEXT), + Column("device", TEXT, "Mounted device"), + Column("device_alias", TEXT, "Mounted device alias"), + Column("path", TEXT, "Mounted device path"), + Column("type", TEXT, "Mounted device type"), + Column("blocks_size", BIGINT, "Block size in bytes"), + Column("blocks", BIGINT, "Mounted device used blocks"), + Column("blocks_free", BIGINT, "Mounted device free blocks"), + Column("blocks_available", BIGINT, "Mounted device available blocks"), + Column("inodes", BIGINT, "Mounted device used inodes"), + Column("inodes_free", BIGINT, "Mounted device free inodes"), + Column("flags", TEXT, "Mounted device flags"), ]) implementation("mounts@genMounts") diff --git a/specs/passwd_changes.table b/specs/passwd_changes.table index 56e1632..1a6c72d 100644 --- a/specs/passwd_changes.table +++ b/specs/passwd_changes.table @@ -2,9 +2,9 @@ table_name("passwd_changes") description("Track time, action changes to /etc/passwd.") schema([ Column("target_path", TEXT, "The path changed"), - Column("time", TEXT, "Time of the change"), Column("action", TEXT, "Change action (UPDATE, REMOVE, etc)"), Column("transaction_id", BIGINT, "ID used during bulk update"), + Column("time", BIGINT, "Time of the change"), ]) attributes(event_subscriber=True) implementation("passwd_changes@passwd_changes::genTable") diff --git a/specs/pci_devices.table b/specs/pci_devices.table index 4856a5f..cc5bb6f 100644 --- a/specs/pci_devices.table +++ b/specs/pci_devices.table @@ -1,18 +1,18 @@ table_name("pci_devices") description("PCI devices active on the host system.") schema([ - Column("pci_slot", TEXT), - Column("pci_class", TEXT), - Column("driver", TEXT), - Column("vendor", TEXT), - Column("vendor_id", TEXT), - Column("model", TEXT), - Column("model_id", TEXT), + Column("pci_slot", TEXT, "PCI Device used slot"), + Column("pci_class", TEXT, "PCI Device class"), + Column("driver", TEXT, "PCI Device used driver"), + Column("vendor", TEXT, "PCI Device vendor"), + Column("vendor_id", TEXT, "PCI Device vendor identifier"), + Column("model", TEXT, "PCI Device model"), + Column("model_id", TEXT, "PCI Device model identifier"), # Optional columns - #Column("subsystem", TEXT), - #Column("express", INTEGER), - #Column("thunderbolt", INTEGER), - #Column("removable", INTEGER), + #Column("subsystem", TEXT, "PCI Device subsystem"), + #Column("express", INTEGER, "1 If PCI device is express else 0"), + #Column("thunderbolt", INTEGER, "1 If PCI device is thunderbolt else 0"), + #Column("removable", INTEGER, "1 If PCI device is removable else 0"), ]) implementation("pci_devices@genPCIDevices") diff --git a/specs/process_memory_map.table b/specs/process_memory_map.table index 82bfc79..5e4294e 100644 --- a/specs/process_memory_map.table +++ b/specs/process_memory_map.table @@ -9,7 +9,7 @@ schema([ Column("device", TEXT, "MA:MI Major/minor device ID"), Column("inode", INTEGER, "Mapped path inode, 0 means uninitialized (BSS)"), Column("path", TEXT, "Path to mapped file or mapped type"), - Column("pseudo", INTEGER, "1 if path is a pseudo path, else 0"), + Column("pseudo", INTEGER, "1 If path is a pseudo path, else 0"), ]) implementation("processes@genProcessMemoryMap") examples([ diff --git a/specs/smbios_tables.table b/specs/smbios_tables.table index a7b1981..3f21b85 100644 --- a/specs/smbios_tables.table +++ b/specs/smbios_tables.table @@ -1,12 +1,12 @@ table_name("smbios_tables") description("BIOS (DMI) structure common details and content.") schema([ - Column("number", INTEGER), - Column("type", INTEGER), - Column("description", TEXT), - Column("handle", INTEGER), - Column("header_size", INTEGER), - Column("size", INTEGER), - Column("md5", TEXT), + Column("number", INTEGER, "Table entry number"), + Column("type", INTEGER, "Table entry type"), + Column("description", TEXT, "Table entry description"), + Column("handle", INTEGER, "Table entry handle"), + Column("header_size", INTEGER, "Header size in bytes"), + Column("size", INTEGER, "Table entry size in bytes"), + Column("md5", TEXT, "MD5 hash of table entry"), ]) implementation("system/smbios_tables@genSMBIOSTables") diff --git a/specs/suid_bin.table b/specs/suid_bin.table index 313c233..6f2d86b 100644 --- a/specs/suid_bin.table +++ b/specs/suid_bin.table @@ -1,9 +1,9 @@ table_name("suid_bin") description("suid binaries in common locations.") schema([ - Column("path", TEXT), - Column("username", TEXT), - Column("groupname", TEXT), - Column("permissions", TEXT), + Column("path", TEXT, "Binary path"), + Column("username", TEXT, "Binary owner username"), + Column("groupname", TEXT, "Binary owner group"), + Column("permissions", TEXT, "Binary permissions"), ]) implementation("suid_bin@genSuidBin") diff --git a/specs/uptime.table b/specs/uptime.table new file mode 100644 index 0000000..ba41677 --- /dev/null +++ b/specs/uptime.table @@ -0,0 +1,10 @@ +table_name("uptime") +description("Track time passed since last boot.") +schema([ + Column("days", INTEGER, "Days of uptime"), + Column("hours", INTEGER, "Hours of uptime"), + Column("minutes", INTEGER, "Minutes of uptime"), + Column("seconds", INTEGER, "Seconds of uptime"), + Column("total_seconds", BIGINT, "Total uptime seconds"), +]) +implementation("system/uptime@genUptime") diff --git a/specs/usb_devices.table b/specs/usb_devices.table index a0692ea..b848cf3 100644 --- a/specs/usb_devices.table +++ b/specs/usb_devices.table @@ -1,13 +1,13 @@ table_name("usb_devices") description("USB devices that are actively plugged into the host system.") schema([ - Column("usb_address", INTEGER), - Column("usb_port", INTEGER), - Column("vendor", TEXT), - Column("vendor_id", TEXT), - Column("model", TEXT), - Column("model_id", TEXT), - Column("serial", TEXT), - Column("removable", INTEGER), + Column("usb_address", INTEGER, "USB Device used address"), + Column("usb_port", INTEGER, "USB Device used port"), + Column("vendor", TEXT, "USB Device vendor string"), + Column("vendor_id", TEXT, "USB Device vendor identifier"), + Column("model", TEXT, "USB Device model string"), + Column("model_id", TEXT, "USB Device model identifier"), + Column("serial", TEXT, "USB Device serial connection"), + Column("removable", INTEGER, "1 If USB device is removable else 0"), ]) implementation("usb_devices@genUSBDevices") diff --git a/specs/users.table b/specs/users.table index b613798..99be598 100644 --- a/specs/users.table +++ b/specs/users.table @@ -5,7 +5,7 @@ schema([ Column("gid", BIGINT, "Group ID (unsigned)"), Column("uid_signed", BIGINT, "User ID as int64 signed (Apple)"), Column("gid_signed", BIGINT, "Default group ID as int64 signed (Apple)"), - Column("username", TEXT), + Column("username", TEXT, "Username"), Column("description", TEXT, "Optional user description"), Column("directory", TEXT, "User's home directory"), Column("shell", TEXT, "User's configured default shell"), diff --git a/specs/utility/file.table b/specs/utility/file.table index 87cd1d7..3ecd734 100644 --- a/specs/utility/file.table +++ b/specs/utility/file.table @@ -15,11 +15,11 @@ schema([ Column("mtime", BIGINT, "Last modification time"), Column("ctime", BIGINT, "Creation time"), Column("hard_links", INTEGER, "Number of hard links"), - Column("is_file", INTEGER, "1 if a file node else 0"), - Column("is_dir", INTEGER, "1 if a directory (not file) else 0"), - Column("is_link", INTEGER, "1 if a symlink else 0"), - Column("is_char", INTEGER, "1 if a character special device else 0"), - Column("is_block", INTEGER, "1 if a block special device else 0"), + Column("is_file", INTEGER, "1 If a file node else 0"), + Column("is_dir", INTEGER, "1 If a directory (not file) else 0"), + Column("is_link", INTEGER, "1 If a symlink else 0"), + Column("is_char", INTEGER, "1 If a character special device else 0"), + Column("is_block", INTEGER, "1 If a block special device else 0"), Column("pattern", TEXT, "A pattern which can be used to match file paths"), ]) attributes(utility=True) @@ -28,4 +28,4 @@ examples([ "select * from file where path = '/etc/passwd'", "select * from file where directory = '/etc/'", "select * from file where pattern = '/etc/%'", -]) \ No newline at end of file +]) diff --git a/specs/utility/osquery_flags.table b/specs/utility/osquery_flags.table index 6c43061..c72dcd7 100644 --- a/specs/utility/osquery_flags.table +++ b/specs/utility/osquery_flags.table @@ -1,12 +1,12 @@ table_name("osquery_flags") description("Configurable flags that modify osquery's behavior.") schema([ - Column("name", TEXT), - Column("type", TEXT), - Column("description", TEXT), - Column("default_value", TEXT), - Column("value", TEXT), - Column("shell_only", INTEGER), + Column("name", TEXT, "Flag name"), + Column("type", TEXT, "Flag type"), + Column("description", TEXT, "Flag description"), + Column("default_value", TEXT, "Flag default value"), + Column("value", TEXT, "Flag value"), + Column("shell_only", INTEGER, "Is the flag shell only?"), ]) attributes(utility=True) implementation("osquery@genOsqueryFlags") diff --git a/specs/utility/osquery_registry.table b/specs/utility/osquery_registry.table index 6cb87fc..e7990e9 100644 --- a/specs/utility/osquery_registry.table +++ b/specs/utility/osquery_registry.table @@ -4,8 +4,8 @@ schema([ Column("registry", TEXT, "Name of the osquery registry"), Column("name", TEXT, "Name of the plugin item"), Column("owner_uuid", INTEGER, "Extension route UUID (0 for core)"), - Column("internal", INTEGER, "1 if the plugin is internal else 0"), - Column("active", INTEGER, "1 if this plugin is active else 0"), + Column("internal", INTEGER, "1 If the plugin is internal else 0"), + Column("active", INTEGER, "1 If this plugin is active else 0"), ]) attributes(utility=True) implementation("osquery@genOsqueryRegistry") diff --git a/specs/utility/time.table b/specs/utility/time.table index a87638c..8c1b7c4 100644 --- a/specs/utility/time.table +++ b/specs/utility/time.table @@ -1,9 +1,16 @@ table_name("time") -description("Track current time in the system.") +description("Track current date and time in the system.") schema([ - Column("hour", INTEGER), - Column("minutes", INTEGER), - Column("seconds", INTEGER), + Column("weekday", TEXT, "Current weekday in the system"), + Column("year", INTEGER, "Current year in the system"), + Column("month", INTEGER, "Current month in the system"), + Column("day", INTEGER, "Current day in the system"), + Column("hour", INTEGER, "Current hour in the system"), + Column("minutes", INTEGER, "Current minutes in the system"), + Column("seconds", INTEGER, "Current seconds in the system"), + Column("unix_time", INTEGER, "Current unix time in the system"), + Column("timestamp", TEXT, "Current timestamp in the system"), + Column("iso_8601", TEXT, "Current time (iso format) in the system"), ]) attributes(utility=True) implementation("time@genTime") diff --git a/tools/codegen/gentable.py b/tools/codegen/gentable.py index 528dce1..b7fb75f 100755 --- a/tools/codegen/gentable.py +++ b/tools/codegen/gentable.py @@ -99,6 +99,7 @@ def is_blacklisted(table_name, path=None, blacklist=None): return table_name in blacklist if blacklist else False + def setup_templates(templates_path): if not os.path.exists(templates_path): templates_path = os.path.join(os.path.dirname(tables_path), "templates") @@ -292,6 +293,23 @@ def implementation(impl_string): table.function = function table.class_name = class_name + '''Check if the table has a subscriber attribute, if so, enforce time.''' + if "event_subscriber" in table.attributes: + columns = {} + # There is no dictionary comprehension on all supported platforms. + for column in table.schema: + if isinstance(column, Column): + columns[column.name] = column.type + if "time" not in columns: + print(lightred("Event subscriber: %s needs a 'time' column." % ( + table.table_name))) + sys.exit(1) + if columns["time"] is not BIGINT: + print(lightred( + "Event subscriber: %s, 'time' column must be a %s type" % ( + table.table_name, BIGINT))) + sys.exit(1) + def main(argc, argv): parser = argparse.ArgumentParser("Generate C++ Table Plugin from specfile.") diff --git a/tools/tests/test.config b/tools/tests/test.config index 15f0ab8..f54b16c 100644 --- a/tools/tests/test.config +++ b/tools/tests/test.config @@ -17,7 +17,7 @@ "additional_monitoring" : { "file_paths": { "downloads": [ - "/tmp/osquery-tests/fstests-pattern/%%" + "/tmp/osquery-tests/fstree/%%" ] } }, @@ -25,11 +25,11 @@ // New, recommended file monitoring (top-level) "file_paths": { "downloads2": [ - "/tmp/osquery-tests/fstests-pattern/%%" + "/tmp/osquery-tests/fstree/%%" ], "system_binaries": [ - "/tmp/osquery-tests/fstests-pattern/%", - "/tmp/osquery-tests/fstests-pattern/deep11/%" + "/tmp/osquery-tests/fstree/%", + "/tmp/osquery-tests/fstree/deep11/%" ] },