#ADD_DEFINITIONS("-pedantic-errors")
# TODO(sangwan.kwon): Get version from packing spec.
-SET(OSQUERY_BUILD_VERSION "1.4.7")
+SET(OSQUERY_BUILD_VERSION "1.5.0")
# Set various platform/platform-version/build version/etc defines.
ADD_DEFINITIONS(-DOSQUERY_BUILD_VERSION=${OSQUERY_BUILD_VERSION}
bool force_merge_success_;
private:
+ /**
+ * @brief A ConfigDataInstance requests read-only access to ConfigParser data.
+ *
+ * A ConfigParser plugin will receive several top-level-config keys and
+ * optionally parse and store information. That information is a property tree
+ * called ConfigParser::data_. Use ConfigDataInstance::getParsedData to
+ * retrieve read-only access to this data.
+ *
+ * @param parser The name of the config parser.
+ */
static const pt::ptree& getParsedData(const std::string& parser);
+
+ /// See getParsedData but request access to the parser plugin.
static const ConfigPluginRef getParser(const std::string& parser);
/// A default, empty property tree used when a missing parser is requested.
/// Helper accessor for Config::data_.all_data.
const pt::ptree& data() const { return Config::getInstance().data_.all_data; }
+ private:
+ /**
+ * @brief ConfigParser plugin's may update the internal config representation.
+ *
+ * If the config parser reads and calculates new information it should store
+ * that derived data itself and rely on ConfigDataInstance::getParsedData.
+ * This means another plugin is aware of the ConfigParser and knowns to make
+ * getParsedData calls. If the parser is augmenting/changing internal state,
+ * such as modifying the osquery schedule or options, then it must write
+ * changed back into the default data.
+ *
+ * Note that this returns the ConfigData instance, not the raw property tree.
+ */
+ ConfigData& mutableConfigData() { return Config::getInstance().data_; }
+
private:
/// A read lock on the reader/writer config data accessor/update mutex.
boost::shared_lock<boost::shared_mutex> lock_;
+
+ private:
+ friend class ConfigParserPlugin;
};
/**
*/
virtual Status update(const ConfigTreeMap& config) = 0;
+ protected:
+ /// Mutable config data accessor for ConfigParser%s.
+ ConfigData& mutableConfigData(ConfigDataInstance& cdi) {
+ return cdi.mutableConfigData();
+ }
+
protected:
/// Allow the config parser to keep some global state.
pt::ptree data_;
* @brief A helpful tool type to report when logging, print help, or debugging.
*/
enum ToolType {
+ OSQUERY_TOOL_UNKNOWN = 0,
OSQUERY_TOOL_SHELL,
OSQUERY_TOOL_DAEMON,
OSQUERY_TOOL_TEST,
OSQUERY_EXTENSION,
};
+/// The osquery tool type for runtime decisions.
+extern ToolType kToolType;
+
class Initializer {
public:
/**
/**
* @brief Sets up the process as an osquery daemon.
*
- * A daemon has additional constraints, it can use a process mutext, check
+ * A daemon has additional constraints, it can use a process mutex, check
* for sane/non-default configurations, etc.
*/
void initDaemon();
* and monitor their utilization.
*
* A daemon may call initWorkerWatcher to begin watching child daemon
- * processes until it-itself is unscheduled. The basic guarentee is that only
+ * processes until it-itself is unscheduled. The basic guarantee is that only
* workers will return from the function.
*
* The worker-watcher will implement performance bounds on CPU utilization
* and memory, as well as check for zombie/defunct workers and respawn them
* if appropriate. The appropriateness is determined from heuristics around
- * how the worker exitted. Various exit states and velocities may cause the
+ * how the worker exited. Various exit states and velocities may cause the
* watcher to resign.
*
* @param name The name of the worker process.
void start();
/// Turns off various aspects of osquery such as event loops.
void shutdown();
- /// Check if a process is an osquery worker.
- bool isWorker();
+
+ /**
+ * @brief Check if a process is an osquery worker.
+ *
+ * By default an osqueryd process will fork/exec then set an environment
+ * variable: `OSQUERY_WORKER` while continually monitoring child I/O.
+ * The environment variable causes subsequent child processes to skip several
+ * initialization steps and jump into extension handling, registry setup,
+ * config/logger discovery and then the event publisher and scheduler.
+ */
+ static bool isWorker();
private:
/// Initialize this process as an osquery daemon worker.
*
* @param s the string that you'd like to split.
* @param delim the delimiter which you'd like to split the string by.
- * @param occurences the number of times to split by delim.
+ * @param occurrences the number of times to split by delim.
*
* @return a vector of strings split by delim for occurrences.
*/
size_t occurences);
/**
- * @brief Inline replace all instances of from with to.
+ * @brief In-line replace all instances of from with to.
*
* @param str The input/output mutable string.
* @param from Search string
std::string getAsciiTime();
/**
- * @brief Getter for the current unix time.
+ * @brief Getter for the current UNIX time.
*
- * @return an int representing the amount of seconds since the unix epoch
+ * @return an int representing the amount of seconds since the UNIX epoch
*/
int getUnixTime();
/**
- * @brief Inline helper function for use with utf8StringSize
+ * @brief In-line helper function for use with utf8StringSize
*/
template <typename _Iterator1, typename _Iterator2>
inline size_t incUtf8StringIterator(_Iterator1& it, const _Iterator2& last) {
unsigned long long int system_time;
/// Average memory differentials. This should be near 0.
- unsigned long long int memory;
+ unsigned long long int average_memory;
/// Total characters, bytes, generated by query.
unsigned long long int output_size;
wall_time(0),
user_time(0),
system_time(0),
- memory(0),
+ average_memory(0),
output_size(0) {}
/// equals operator
* All rights reserved.
*
* This source code is licensed under the BSD-style license found in the
- * LICENSE file in the root directory of this source tree. An additional grant
+ * LICENSE file in the root directory of this source tree. An additional grant
* of patent rights can be found in the PATENTS file in the same directory.
*
*/
* EventPublisher will use to register OS API callbacks, create
* subscriptioning/listening handles, etc.
*
- * Linux `inotify` should implement a SubscriptionContext that subscriptions
+ * Linux `inotify` should implement a SubscriptionContext that subscribes
* filesystem events based on a filesystem path. `libpcap` will subscribe on
* networking protocols at various stacks. Process creation may subscribe on
* process name, parent pid, etc.
struct EventContext {
/// An unique counting ID specific to the EventPublisher%'s fired events.
EventContextID id;
- /// The time the event occurred.
+ /// The time the event occurred, as determined by the publisher.
EventTime time;
- /// The string representation of the time, often used for indexing.
- std::string time_string;
EventContext() : id(0), time(0) {}
};
* The supported states are:
* - None: The default state, uninitialized.
* - Running: Subscriber is ready for events.
- * - Paused: Subscriber was successfully initialized but not currently accepting
- * events.
+ * - Paused: Subscriber was initialized but is not currently accepting events.
* - Failed: Subscriber failed to initialize or is otherwise offline.
*/
enum EventSubscriberState {
* @brief Perform handle opening, OS API callback registration.
*
* `setUp` is the event framework's EventPublisher constructor equivalent.
- * When `setUp` is called the EventPublisher is running in a dedicated thread
- * and may manage/allocate/wait for resources.
+ * This is called in the main thread before the publisher's run loop has
+ * started, immediately following registration.
*/
virtual Status setUp() { return Status(0, "Not used"); }
* @brief Perform handle closing, resource cleanup.
*
* osquery is about to end, the EventPublisher should close handle descriptors
- * unblock resources, and prepare to exit.
+ * unblock resources, and prepare to exit. This will be called from the main
+ * thread after the run loop thread has exited.
*/
virtual void tearDown() {}
/**
- * @brief Implement a step of an optional run loop.
+ * @brief Implement a "step" of an optional run loop.
*
* @return A SUCCESS status will immediately call `run` again. A FAILED status
* will exit the run loop and the thread.
*/
- virtual Status run() { return Status(1, "No runloop required"); }
+ virtual Status run() { return Status(1, "No run loop required"); }
/**
- * @brief A new EventSubscriber is subscriptioning events of this
- * EventPublisher.
+ * @brief Allow the EventFactory to interrupt the run loop.
+ *
+ * Assume the main thread may ask the run loop to stop at anytime.
+ * Before end is called the publisher's `isEnding` is set and the EventFactory
+ * run loop manager will exit the stepping loop and fall through to a call
+ * to tearDown followed by a removal of the publisher.
+ */
+ virtual void end() {}
+
+ /**
+ * @brief A new EventSubscriber is subscribing events of this publisher type.
*
* @param subscription The Subscription context information and optional
* EventCallback.
return Status(0, "OK");
}
- /**
- * @brief The generic check loop to call SubscriptionContext callback methods.
- *
- * It is NOT recommended to override `fire`. The simple logic of enumerating
- * the Subscription%s and using `shouldFire` is more appropriate.
- *
- * @param ec The EventContext created and fired by the EventPublisher.
- * @param time The most accurate time associated with the event.
- */
- void fire(const EventContextRef& ec, EventTime time = 0);
+ public:
+ /// Overriding the EventPublisher constructor is not recommended.
+ EventPublisherPlugin() : next_ec_id_(0), ending_(false), started_(false){};
+ virtual ~EventPublisherPlugin() {}
+ /// Return a string identifier associated with this EventPublisher.
+ virtual EventPublisherID type() const { return "publisher"; }
+
+ public:
/// Number of Subscription%s watching this EventPublisher.
size_t numSubscriptions() const { return subscriptions_.size(); }
*/
size_t numEvents() const { return next_ec_id_; }
- /// Overriding the EventPublisher constructor is not recommended.
- EventPublisherPlugin() : next_ec_id_(0), ending_(false), started_(false) {};
- virtual ~EventPublisherPlugin() {}
-
- /// Return a string identifier associated with this EventPublisher.
- virtual EventPublisherID type() const { return "publisher"; }
-
+ /// Check if the EventFactory is ending all publisher threads.
bool isEnding() const { return ending_; }
+
+ /// Set the ending status for this publisher.
void isEnding(bool ending) { ending_ = ending; }
+
+ /// Check if the publisher's run loop has started.
bool hasStarted() const { return started_; }
+
+ /// Set the run or started status for this publisher.
void hasStarted(bool started) { started_ = started; }
protected:
+ /**
+ * @brief The generic check loop to call SubscriptionContext callback methods.
+ *
+ * It is NOT recommended to override `fire`. The simple logic of enumerating
+ * the Subscription%s and using `shouldFire` is more appropriate.
+ *
+ * @param ec The EventContext created and fired by the EventPublisher.
+ * @param time The most accurate time associated with the event.
+ */
+ virtual void fire(const EventContextRef& ec, EventTime time = 0) final;
+
/// The internal fire method used by the typed EventPublisher.
virtual void fireCallback(const SubscriptionRef& sub,
const EventContextRef& ec) const = 0;
private:
/// Set ending to True to cause event type run loops to finish.
bool ending_;
+
/// Set to indicate whether the event run loop ever started.
bool started_;
/// A lock for incrementing the next EventContextID.
boost::mutex ec_id_lock_;
+ private:
+ /// Enable event factory "callins" through static publisher callbacks.
+ friend class EventFactory;
+
private:
FRIEND_TEST(EventsTests, test_event_pub);
FRIEND_TEST(EventsTests, test_fire_event);
* A 'class' of OS events is abstracted into an EventPublisher responsible for
* remaining as agile as possible given a known-set of subscriptions.
*
- * The lifecycle of an EventPublisher may include, `setUp`, `configure`, `run`,
+ * The life cycle of an EventPublisher may include, `setUp`, `configure`, `run`,
* `tearDown`, and `fire`. `setUp` and `tearDown` happen when osquery starts and
* stops either as a daemon or interactive shell. `configure` is a pseudo-start
* called every time a Subscription is added. EventPublisher%s can adjust their
* Status run() { return Status(1, "Not Implemented"); }
* @endcode
*
- * The final lifecycle component, `fire` will iterate over the EventPublisher
+ * The final life cycle component, `fire` will iterate over the EventPublisher
* Subscription%s and call `shouldFire` for each, using the EventContext fired.
* The `shouldFire` method should check the subscription-specific selectors and
* only call the Subscription%'s callback function if the EventContext
/// Create a SubscriptionContext based on the templated type.
static SCRef createSubscriptionContext() { return std::make_shared<SC>(); }
- /// A simple EventPublisher type accessor.
- template <class PUB>
- static EventPublisherID getType() {
- auto pub = std::make_shared<PUB>();
- return pub->type();
- }
-
protected:
/**
* @brief The internal `fire` phase of publishing.
/**
* @brief The generic `fire` will call `shouldFire` for each Subscription.
*
- * @param mc A SubscriptionContext with optional specifications for events
+ * @param sc A SubscriptionContext with optional specifications for events
* details.
* @param ec The event fired with event details.
*
*
* @return Was the element added to the backing store.
*/
- virtual Status add(const osquery::Row& r, EventTime time) final;
+ virtual Status add(Row& r, EventTime event_time) final;
/**
* @brief Return all events added by this EventSubscriber within start, stop.
*
* @return List of EventID, EventTime%s
*/
- std::vector<EventRecord> getRecords(const std::vector<std::string>& indexes);
+ std::vector<EventRecord> getRecords(const std::set<std::string>& indexes);
/**
* @brief Get a unique storage-related EventID.
*
* An EventID is an index/element-identifier for the backing store.
* Each EventPublisher maintains a fired EventContextID to identify the many
- * events that may or may not be fired to 'subscriptioning' criteria for this
+ * events that may or may not be fired based on subscription criteria for this
* EventSubscriber. This EventContextID is NOT the same as an EventID.
- * EventSubscriber development should not require use of EventID%s, if this
+ * EventSubscriber development should not require use of EventID%s. If this
* indexing is required within-EventCallback consider an
* EventSubscriber%-unique indexing, counting mechanic.
*
*
* @return List of 'index.step' index strings.
*/
- std::vector<std::string> getIndexes(EventTime start,
- EventTime stop,
- int list_key = 0);
+ std::set<std::string> getIndexes(EventTime start,
+ EventTime stop,
+ int list_key = 0);
/**
* @brief Expire indexes and eventually records.
* @param list_type the string representation of list binning type.
* @param indexes complete set of 'index.step' indexes for the list_type.
* @param expirations of the indexes, the set to expire.
- *
- * @return status if the indexes and records were removed.
*/
- Status expireIndexes(const std::string& list_type,
- const std::vector<std::string>& indexes,
- const std::vector<std::string>& expirations);
+ void expireIndexes(const std::string& list_type,
+ const std::vector<std::string>& indexes,
+ const std::vector<std::string>& expirations);
+ /// Expire all datums within a bin.
+ void expireRecords(const std::string& list_type,
+ const std::string& index,
+ bool all);
/**
* @brief Add an EventID, EventTime pair to all matching list types.
* EventPublisher instances will have run `setUp` and initialized their run
* loops.
*/
- EventSubscriberPlugin() {
- expire_events_ = true;
- expire_time_ = 0;
- }
+ EventSubscriberPlugin()
+ : expire_events_(true), expire_time_(0), optimize_time_(0) {}
virtual ~EventSubscriberPlugin() {}
/**
* @brief Suggested entrypoint for table generation.
*
* The EventSubscriber is a convention that removes a lot of boilerplate event
- * 'subscriptioning' and acting. The `genTable` static entrypoint is the
+ * 'subscribing' and acting. The `genTable` static entrypoint is the
* suggested method for table specs.
*
* @return The query-time table data, retrieved from a backing store.
*/
- virtual QueryData genTable(QueryContext& context) __attribute__((used)) {
- return get(0, 0);
- }
+ virtual QueryData genTable(QueryContext& context) __attribute__((used));
protected:
- /// Backing storage indexing namespace definition methods.
- EventPublisherID dbNamespace() const { return type() + "." + getName(); }
-
- /// The string EventPublisher identifying this EventSubscriber.
- virtual EventPublisherID type() const = 0;
+ /**
+ * @brief Backing storage indexing namespace.
+ *
+ * The backing storage will accumulate events for this subscriber. A namespace
+ * is provided to prevent event indexing collisions between subscribers and
+ * publishers. The namespace is a combination of the publisher and subscriber
+ * registry plugin names.
+ */
+ virtual EventPublisherID& dbNamespace() const = 0;
/// Disable event expiration for this subscriber.
void doNotExpire() { expire_events_ = false; }
/// Events before the expire_time_ are invalid and will be purged.
EventTime expire_time_;
+ /**
+ * @brief Optimize subscriber selects by tracking the last select time.
+ *
+ * Event subscribers may optimize selects when used in a daemon schedule by
+ * requiring an event 'time' constraint and otherwise applying a minimum time
+ * as the last time the scheduled query ran.
+ */
+ EventTime optimize_time_;
+
/// Lock used when incrementing the EventID database index.
boost::mutex event_id_lock_;
* and add that Subscription to the EventPublisher associated identifier.
*
* @param type_id The string for an EventPublisher receiving the Subscription.
- * @param mc A SubscriptionContext related to the EventPublisher.
+ * @param sc A SubscriptionContext related to the EventPublisher.
* @param cb When the EventPublisher fires an event the SubscriptionContext
* will be evaluated, if the event matches optional specifics in the context
* this callback function will be called. It should belong to an
*/
static Status addSubscription(EventPublisherID& type_id,
EventSubscriberID& name_id,
- const SubscriptionContextRef& mc,
+ const SubscriptionContextRef& sc,
EventCallback cb = 0,
void* user_data = nullptr);
- /// Add a Subscription by templating the EventPublisher, using a
- /// SubscriptionContext.
- template <typename PUB>
- static Status addSubscription(const SubscriptionContextRef& mc,
- EventCallback cb = 0) {
- return addSubscription(BaseEventPublisher::getType<PUB>(), mc, cb);
- }
-
/// Add a Subscription using a caller Subscription instance.
static Status addSubscription(EventPublisherID& type_id,
const SubscriptionRef& subscription);
}
/**
- * @brief Halt the EventPublisher run loop and call its `tearDown`.
+ * @brief Halt the EventPublisher run loop.
*
* Any EventSubscriber%s with Subscription%s for this EventPublisher will
* become useless. osquery callers MUST deregister events.
* EventPublisher%s assume they can hook/trampoline, which requires cleanup.
+ * This will tear down and remove the publisher if the run loop did not start.
+ * Otherwise it will call end on the publisher and assume the run loop will
+ * tear down and remove.
*
* @param event_pub The string label for the EventPublisher.
*
/// Return an instance to a registered EventSubscriber.
static EventSubscriberRef getEventSubscriber(EventSubscriberID& sub);
+
+ /// Check if an event subscriber exists.
static bool exists(EventSubscriberID& sub);
+ /// Return a list of publisher types, these are their registry names.
static std::vector<std::string> publisherTypes();
+
+ /// Return a list of subscriber registry names,
static std::vector<std::string> subscriberNames();
public:
/// If a static EventPublisher callback wants to fire
template <typename PUB>
static void fire(const EventContextRef& ec) {
- auto event_pub = getEventPublisher(
BaseEventPublisher::getType<PUB>());
+ auto event_pub = getEventPublisher(getType<PUB>());
event_pub->fire(ec);
}
/**
- * @brief End all EventPublisher run loops and call their `tearDown` methods.
+ * @brief Return the publisher registry name given a type.
*
- * End is NOT the same as deregistration.
+ * Subscriber initialization and runtime static callbacks can lookup the
+ * publisher type name, which is the registry plugin name. This allows static
+ * callbacks to fire into subscribers.
+ */
+ template <class PUB>
+ static EventPublisherID getType() {
+ auto pub = std::make_shared<PUB>();
+ return pub->type();
+ }
+
+ /**
+ * @brief End all EventPublisher run loops and deregister.
+ *
+ * End is NOT the same as deregistration. End will call deregister on all
+ * publishers then either join or detach their run loop threads.
+ * See EventFactory::deregisterEventPublisher for actions taken during
+ * deregistration.
*
* @param should_end Reset the "is ending" state if False.
*/
*/
virtual Status init() { return Status(0, "OK"); }
+ protected:
/// Helper function to call the publisher's templated subscription generator.
SCRef createSubscriptionContext() const {
return PUB::createSubscriptionContext();
void subscribe(Status (T::*entry)(const std::shared_ptr<C>&, const void*),
const SubscriptionContextRef& sc,
void* user_data) {
- // Up-cast the CRTP-style EventSubscriber to the caller.
- auto self = dynamic_cast<T*>(this);
+ // Up-cast the EventSubscriber to the caller.
+ auto sub = dynamic_cast<T*>(this);
// Down-cast the pointer to the member function.
auto base_entry =
reinterpret_cast<Status (T::*)(const EventContextRef&, void const*)>(
entry);
// Create a callable through the member function using the instance of the
// EventSubscriber and a single parameter placeholder (the EventContext).
- auto cb = std::bind(base_entry, self, _1, _2);
+ auto cb = std::bind(base_entry, sub, _1, _2);
// Add a subscription using the callable and SubscriptionContext.
- EventFactory::addSubscription(type(), self->getName(), sc, cb, user_data);
+ EventFactory::addSubscription(getType(), sub->getName(), sc, cb, user_data);
+ }
+
+ /**
+ * @brief The registry plugin name for the subscriber's publisher.
+ *
+ * During event factory initialization the subscribers 'peek' at the registry
+ * plugin name assigned to publishers. The corresponding publisher name is
+ * interpreted as the subscriber's event 'type'.
+ */
+ EventPublisherID& getType() const {
+ static EventPublisherID type = EventFactory::getType<PUB>();
+ return type;
}
- /// Helper EventPublisher string type accessor.
- EventPublisherID type() const { return BaseEventPublisher::getType<PUB>(); }
+ /// See getType for lookup rational.
+ EventPublisherID& dbNamespace() const {
+ static EventPublisherID _ns = getType() + '.' + getName();
+ return _ns;
+ }
+ public:
/**
* @brief Request the subscriber's initialization state.
*
/// Set the subscriber state.
void state(EventSubscriberState state) { state_ = state; }
- public:
EventSubscriber() : EventSubscriberPlugin(), state_(SUBSCRIBER_NONE) {}
private:
DECLARE_bool(disable_extensions);
/// A millisecond internal applied to extension initialization.
-extern const int kExtensionInitializeMLatency;
+extern const size_t kExtensionInitializeLatencyUS;
/**
* @brief Helper struct for managing extenion metadata.
namespace osquery {
/// Globbing directory traversal function recursive limit.
-const unsigned int kMaxDirectoryTraversalDepth = 40;
-typedef unsigned int ReturnSetting;
+typedef unsigned short GlobLimits;
enum {
- /// Return only files
- REC_LIST_FILES = 0x1,
- /// Return only folders
- REC_LIST_FOLDERS = 0x2,
- /// Enable optimizations for file event resolutions
- REC_EVENT_OPT = 0x4,
- REC_LIST_ALL = REC_LIST_FILES | REC_LIST_FOLDERS
+ GLOB_FILES = 0x1,
+ GLOB_FOLDERS = 0x2,
+ GLOB_ALL = GLOB_FILES | GLOB_FOLDERS,
};
/// Globbing wildcard character.
-const std::string kWildcardCharacter = "%";
+const std::string kSQLGlobWildcard = "%";
/// Globbing wildcard recursive character (double wildcard).
-const std::string kWildcardCharacterRecursive =
- kWildcardCharacter + kWildcardCharacter;
+const std::string kSQLGlobRecursive = kSQLGlobWildcard + kSQLGlobWildcard;
/**
* @brief Read a file from disk.
*
- * @param path the path of the file that you would like to read
+ * @param path the path of the file that you would like to read.
* @param content a reference to a string which will be populated with the
- * contents of the path indicated by the path parameter
+ * contents of the path indicated by the path parameter.
+ * @param dry_run do not actually read the file content.
*
- * @return an instance of Status, indicating the success or failure
- * of the operation.
+ * @return an instance of Status, indicating success or failure.
*/
-Status readFile(const boost::filesystem::path& path, std::string& content);
+Status readFile(const boost::filesystem::path& path,
+ std::string& content,
+ bool dry_run = false);
+
+/**
+ * @brief Return the status of an attempted file read.
+ *
+ * @param path the path of the file that you would like to read.
+ *
+ * @return success iff the file would have been read. On success the status
+ * message is the complete/absolute path.
+ */
+Status readFile(const boost::filesystem::path& path);
/**
* @brief Write text to disk.
*
- * @param path the path of the file that you would like to write
- * @param content the text that should be written exactly to disk
- * @param permissions the filesystem permissions to request when opening
- * @param force_permissions always chmod the path after opening
+ * @param path the path of the file that you would like to write.
+ * @param content the text that should be written exactly to disk.
+ * @param permissions the filesystem permissions to request when opening.
+ * @param force_permissions always `chmod` the path after opening.
*
- * @return an instance of Status, indicating the success or failure
- * of the operation.
+ * @return an instance of Status, indicating success or failure.
*/
Status writeTextFile(const boost::filesystem::path& path,
const std::string& content,
/// Check if a path is writable.
Status isWritable(const boost::filesystem::path& path);
+
/// Check if a path is readable.
Status isReadable(const boost::filesystem::path& path);
/**
* @brief A helper to check if a path exists on disk or not.
*
- * @param path the path on disk which you would like to check the existence of
+ * @param path Target path.
*
- * @return an instance of Status, indicating the success or failure
- * of the operation. Specifically, the code of the Status instance
- * will be -1 if no input was supplied, assuming the caller is not aware of how
- * to check path-getter results. The code will be 0 if the path does not exist
- * on disk and 1 if the path does exist on disk.
+ * @return The code of the Status instance will be -1 if no input was supplied,
+ * assuming the caller is not aware of how to check path-getter results.
+ * The code will be 0 if the path does not exist on disk and 1 if the path
+ * does exist on disk.
*/
Status pathExists(const boost::filesystem::path& path);
* with the directory listing of the path param, assuming that all operations
* completed successfully.
*
- * @return an instance of Status, indicating the success or failure
- * of the operation.
+ * @return an instance of Status, indicating success or failure.
*/
Status listFilesInDirectory(const boost::filesystem::path& path,
std::vector<std::string>& results,
/**
* @brief List all of the directories in a specific directory, non-recursively.
*
- * @param path the path which you would like to list.
+ * @param path the path which you would like to list
* @param results a non-const reference to a vector which will be populated
* with the directory listing of the path param, assuming that all operations
* completed successfully.
*
- * @return an instance of Status, indicating the success or failure
- * of the operation.
+ * @return an instance of Status, indicating success or failure.
*/
Status listDirectoriesInDirectory(const boost::filesystem::path& path,
std::vector<std::string>& results,
bool ignore_error = 1);
/**
- * @brief Given a wildcard filesystem patten, resolve all possible paths
+ * @brief Given a filesystem globbing patten, resolve all matching paths.
*
* @code{.cpp}
* std::vector<std::string> results;
* }
* @endcode
*
- * @param fs_path The filesystem pattern
- * @param results The vector in which all results will be returned
+ * @param pattern filesystem globbing pattern.
+ * @param results output vector of matching paths.
*
- * @return An instance of osquery::Status which indicates the success or
- * failure of the operation
+ * @return an instance of Status, indicating success or failure.
*/
-Status resolveFilePattern(const boost::filesystem::path& fs_path,
+Status resolveFilePattern(const boost::filesystem::path& pattern,
std::vector<std::string>& results);
/**
- * @brief Given a wildcard filesystem patten, resolve all possible paths
+ * @brief Given a filesystem globbing patten, resolve all matching paths.
*
- * @code{.cpp}
- * std::vector<std::string> results;
- * auto s = resolveFilePattern("/Users/marpaia/Downloads/%", results);
- * if (s.ok()) {
- * for (const auto& result : results) {
- * LOG(INFO) << result;
- * }
- * }
- * @endcode
+ * See resolveFilePattern, but supply a limitation to request only directories
+ * or files that match the path.
*
- * @param fs_path The filesystem pattern
- * @param results The vector in which all results will be returned
- * @param setting Do you want files returned, folders or both?
+ * @param pattern filesystem globbing pattern.
+ * @param results output vector of matching paths.
+ * @param setting a bit list of match types, e.g., files, folders.
*
- * @return An instance of osquery::Status which indicates the success or
- * failure of the operation
+ * @return an instance of Status, indicating success or failure.
*/
-Status resolveFilePattern(const boost::filesystem::path& fs_path,
+Status resolveFilePattern(const boost::filesystem::path& pattern,
std::vector<std::string>& results,
- ReturnSetting setting);
+ GlobLimits setting);
+
+/**
+ * @brief Transform a path with SQL wildcards to globbing wildcard.
+ *
+ * SQL uses '%' as a wildcard matching token, and filesystem globbing uses '*'.
+ * In osquery-internal methods the filesystem character is used. This helper
+ * method will perform the correct preg/escape and replace.
+ *
+ * This has a side effect of canonicalizing paths up to the first wildcard.
+ * For example: /tmp/% becomes /private/tmp/% on OS X systems. And /tmp/%.
+ *
+ * @param pattern the input and output filesystem glob pattern.
+ */
+void replaceGlobWildcards(std::string& pattern);
/**
* @brief Get directory portion of a path.
*
- * @param path The input path, either a filename or directory.
- * @param dirpath a non-const reference to a resultant directory portion.
+ * @param path input path, either a filename or directory.
+ * @param dirpath output path set to the directory-only path.
*
* @return If the input path was a directory this will indicate failure. One
* should use `isDirectory` before.
Status getDirectory(const boost::filesystem::path& path,
boost::filesystem::path& dirpath);
+/// Attempt to remove a directory path.
Status remove(const boost::filesystem::path& path);
/**
* @brief Check if an input path is a directory.
*
- * @param path The input path, either a filename or directory.
+ * @param path input path, either a filename or directory.
*
* @return If the input path was a directory.
*/
Status isDirectory(const boost::filesystem::path& path);
/**
- * @brief Return a vector of all home directories on the system
+ * @brief Return a vector of all home directories on the system.
*
- * @return a vector of strings representing the path of all home directories
+ * @return a vector of string paths containing all home directories.
*/
std::set<boost::filesystem::path> getHomeDirectories();
/**
- * @brief Check the permissions of a file and it's directory.
+ * @brief Check the permissions of a file and its directory.
*
* 'Safe' implies the directory is not a /tmp-like directory in that users
* cannot control super-user-owner files. The file should be owned by the
* process's UID or the file should be owned by root.
*
- * @param dir the directory to check /tmp mode
- * @param path a path to a file to check
- * @param executable the file must also be executable
+ * @param dir the directory to check `/tmp` mode.
+ * @param path a path to a file to check.
+ * @param executable true if the file must also be executable.
*
- * @return true if the file is 'safe' else false
+ * @return true if the file is 'safe' else false.
*/
bool safePermissions(const std::string& dir,
const std::string& path,
bool executable = false);
-/// The shell tooling may store local resources in an "osquery" home.
+/**
+ * @brief osquery may use local storage in a user-protected "home".
+ *
+ * Return a standard path to an "osquery" home directory. This path may store
+ * a protected extensions socket, backing storage database, and debug logs.
+ */
const std::string& osqueryHomeDirectory();
/// Return bit-mask-style permissions.
/**
* @brief Parse a JSON file on disk into a property tree.
*
- * @param path the path of the JSON file
- * @param tree output property tree
+ * @param path the path of the JSON file.
+ * @param tree output property tree.
*
- * @return an instance of Status, indicating the success or failure
+ * @return an instance of Status, indicating success or failure if malformed.
*/
Status parseJSON(const boost::filesystem::path& path,
boost::property_tree::ptree& tree);
/**
* @brief Parse JSON content into a property tree.
*
- * @param path JSON string data
- * @param tree output property tree
+ * @param path JSON string data.
+ * @param tree output property tree.
*
- * @return an instance of Status, indicating the success or failure
+ * @return an instance of Status, indicating success or failure if malformed.
*/
Status parseJSONContent(const std::string& content,
boost::property_tree::ptree& tree);
/**
* @brief Parse a property list on disk into a property tree.
*
- * @param path the input path to a property list
- * @param tree the output reference to a Boost property tree
+ * @param path the input path to a property list.
+ * @param tree the output property tree.
*
- * @return an instance of Status, indicating the success or failure
- * of the operation.
+ * @return an instance of Status, indicating success or failure if malformed.
*/
Status parsePlist(const boost::filesystem::path& path,
boost::property_tree::ptree& tree);
/**
* @brief Parse property list content into a property tree.
*
- * @param content the input string-content of a property list
- * @param tree the output reference to a Boost property tree
+ * @param content the input string-content of a property list.
+ * @param tree the output property tree.
*
- * @return an instance of Status, indicating the success or failure
- * of the operation.
+ * @return an instance of Status, indicating success or failure if malformed.
*/
Status parsePlistContent(const std::string& content,
boost::property_tree::ptree& tree);
#ifdef __linux__
/**
- * @brief Iterate over proc process, returns a list of pids.
+ * @brief Iterate over `/proc` process, returns a list of pids.
*
* @param processes output list of process pids as strings (int paths in proc).
*
- * @return status of iteration.
+ * @return an instance of Status, indicating success or failure.
*/
Status procProcesses(std::set<std::string>& processes);
/**
- * @brief Iterate over a proc process's descriptors, return a list of fds.
+ * @brief Iterate over a `/proc` process's descriptors, return a list of fds.
*
* @param process a string pid from proc.
* @param descriptors output list of descriptor numbers as strings.
bool shell;
bool external;
bool cli;
+ bool hidden;
};
struct FlagInfo {
* @param value The default value, use a C++ literal.
* @param desc A string literal used for help display.
*/
-#define OSQUERY_FLAG(t, n, v, d, s, e, c) \
- DEFINE_##t(n, v, d); \
- namespace flags { \
- const int flag_##n = Flag::create(#n, {d, s, e, c}); \
+#define OSQUERY_FLAG(t, n, v, d, s, e, c, h) \
+ DEFINE_##t(n, v, d); \
+ namespace flags { \
+ const int flag_##n = Flag::create(#n, {d, s, e, c, h}); \
}
-#define FLAG(t, n, v, d) OSQUERY_FLAG(t, n, v, d, 0, 0, 0)
-#define SHELL_FLAG(t, n, v, d) OSQUERY_FLAG(t, n, v, d, 1, 0, 0)
-#define EXTENSION_FLAG(t, n, v, d) OSQUERY_FLAG(t, n, v, d, 0, 1, 0)
-#define CLI_FLAG(t, n, v, d) OSQUERY_FLAG(t, n, v, d, 0, 0, 1)
-
-#define OSQUERY_FLAG_ALIAS(t, a, n, s, e) \
- FlagAlias<t> FLAGS_##a(#a, #t, #n, &FLAGS_##n); \
- namespace flags { \
- static GFLAGS_NAMESPACE::FlagRegisterer oflag_##a( \
- #a, #t, #a, &FLAGS_##n, &FLAGS_##n); \
- const int flag_alias_##a = Flag::createAlias(#a, {#n, s, e, 0}); \
+#define FLAG(t, n, v, d) OSQUERY_FLAG(t, n, v, d, 0, 0, 0, 0)
+#define SHELL_FLAG(t, n, v, d) OSQUERY_FLAG(t, n, v, d, 1, 0, 0, 0)
+#define EXTENSION_FLAG(t, n, v, d) OSQUERY_FLAG(t, n, v, d, 0, 1, 0, 0)
+#define CLI_FLAG(t, n, v, d) OSQUERY_FLAG(t, n, v, d, 0, 0, 1, 0)
+#define HIDDEN_FLAG(t, n, v, d) OSQUERY_FLAG(t, n, v, d, 0, 0, 0, 1)
+
+#define OSQUERY_FLAG_ALIAS(t, a, n, s, e) \
+ FlagAlias<t> FLAGS_##a(#a, #t, #n, &FLAGS_##n); \
+ namespace flags { \
+ static GFLAGS_NAMESPACE::FlagRegisterer oflag_##a( \
+ #a, #t, #a, &FLAGS_##n, &FLAGS_##n); \
+ const int flag_alias_##a = Flag::createAlias(#a, {#n, s, e, 0, 1}); \
}
#define FLAG_ALIAS(t, a, n) OSQUERY_FLAG_ALIAS(t, a, n, 0, 0)
*/
Status logHealthStatus(const QueryLogItem& item);
+/**
+ * @brief Sink a set of buffered status logs.
+ *
+ * When the osquery daemon uses a watcher/worker set, the watcher's status logs
+ * are accumulated in a buffered log sink. Well-performing workers should have
+ * the set of watcher status logs relayed and sent to the configured logger
+ * plugin.
+ *
+ * Status logs from extensions will be forwarded to the extension manager (core)
+ * normally, but the watcher does not receive or send registry requests.
+ * Extensions, the registry, configuration, and optional config/logger plugins
+ * are all protected as a monitored worker.
+ */
+void relayStatusLogs();
+
/**
* @brief Logger plugin registry.
*
public:
/// The plugin may perform some initialization, not required.
virtual Status setUp() { return Status(0, "Not used"); }
+
/// The plugin may perform some tear down, release, not required.
virtual void tearDown() {}
+
/// The plugin may publish route info (other than registry type and name).
virtual PluginResponse routeInfo() const {
PluginResponse info;
return info;
}
- /// The plugin will act on a serialized request, and if a response is needed
- /// (response is set to true) then response should be a reference to a
- /// string ready for a serialized response.
+
+ /**
+ * @brief Plugins act by being called, using a request, returning a response.
+ *
+ * The plugin request is a thrift-serializable object. A response is optional
+ * but the API for using a plugin's call is defined by the registry. In most
+ * cases there are multiple supported call 'actions'. A registry type, or
+ * the plugin class, will define the action key and supported actions.
+ *
+ * @param request A plugin request input, including optional action.
+ * @param response A plugin response output.
+ *
+ * @return Status of the call, if the action was handled corrected.
+ */
virtual Status call(const PluginRequest& request, PluginResponse& response) {
return Status(0, "Not used");
}
*/
std::set<std::string> getAll(ConstraintOperator op) const;
+ /// See ConstraintList::getAll, but as a selected literal type.
template<typename T>
std::set<T> getAll(ConstraintOperator op) const {
std::set<T> literal_matches;
return literal_matches;
}
+ /// Constraint list accessor, types and operator.
+ const std::vector<struct Constraint> getAll() const { return constraints_; }
+
/**
* @brief Add a new Constraint to the list of constraints.
*
* }
*/
void serialize(boost::property_tree::ptree& tree) const;
+
+ /// See ConstraintList::unserialize.
void unserialize(const boost::property_tree::ptree& tree);
ConstraintList() : affinity("TEXT") {}
typedef std::map<std::string, std::vector<std::string> > EventFileMap_t;
typedef std::chrono::high_resolution_clock chrono_clock;
+/// The config plugin must be known before reading options.
CLI_FLAG(string, config_plugin, "filesystem", "Config plugin name");
FLAG(int32, schedule_splay_percent, 10, "Percent to splay config times");
}
// Request a unique write lock when updating config.
- boost::unique_lock<boost::shared_mutex> unique_lock(getInstance().mutex_);
+ {
+ boost::unique_lock<boost::shared_mutex> unique_lock(getInstance().mutex_);
- ConfigData conf;
- for (const auto& source : config) {
- if (Registry::external()) {
- VLOG(1) << "Updating extension config with source: " << source.first;
- } else {
- VLOG(1) << "Updating config with source: " << source.first;
+ for (const auto& source : config) {
+ if (Registry::external()) {
+ VLOG(1) << "Updating extension config with source: " << source.first;
+ } else {
+ VLOG(1) << "Updating config with source: " << source.first;
+ }
+ getInstance().raw_[source.first] = source.second;
}
- getInstance().raw_[source.first] = source.second;
- }
- // Now merge all sources together.
- for (const auto& source : getInstance().raw_) {
- auto status = mergeConfig(source.second, conf);
- if (getInstance().force_merge_success_ && !status.ok()) {
- return Status(1, status.what());
+ // Now merge all sources together.
+ ConfigData conf;
+ for (const auto& source : getInstance().raw_) {
+ auto status = mergeConfig(source.second, conf);
+ if (getInstance().force_merge_success_ && !status.ok()) {
+ return Status(1, status.what());
+ }
}
+
+ // Call each parser with the optionally-empty, requested, top level keys.
+ getInstance().data_ = std::move(conf);
}
- // Call each parser with the optionally-empty, requested, top level keys.
- getInstance().data_ = conf;
for (const auto& plugin : Registry::all("config_parser")) {
auto parser = std::static_pointer_cast<ConfigParserPlugin>(plugin.second);
if (parser == nullptr || parser.get() == nullptr) {
// For each key requested by the parser, add a property tree reference.
std::map<std::string, ConfigTree> parser_config;
for (const auto& key : parser->keys()) {
- if (conf.all_data.count(key) > 0) {
- parser_config[key] = conf.all_data.get_child(key);
+ if (getInstance().data_.all_data.count(key) > 0) {
+ parser_config[key] = getInstance().data_.all_data.get_child(key);
} else {
parser_config[key] = pt::ptree();
}
}
+
+ // The config parser plugin will receive a copy of each property tree for
+ // each top-level-config key. The parser may choose to update the config's
+ // internal state by requesting and modifying a ConfigDataInstance.
parser->update(parser_config);
}
const tree_node& node,
ConfigData& conf) {
for (const auto& path : node.second) {
- resolveFilePattern(path.second.data(),
- conf.files[node.first],
- REC_LIST_FOLDERS | REC_EVENT_OPT);
+ // Add the exact path after converting wildcards.
+ std::string pattern = path.second.data();
+ replaceGlobWildcards(pattern);
+ conf.files[node.first].push_back(std::move(pattern));
}
conf.all_data.add_child(name + "." + node.first, node.second);
}
ConfigDataInstance config;
std::stringstream out;
- pt::write_json(out, config.data());
+ try {
+ pt::write_json(out, config.data(), false);
+ } catch (const pt::json_parser::json_parser_error& e) {
+ return Status(1, e.what());
+ }
hash_string = osquery::hashFromBuffer(
HASH_TYPE_MD5, (void*)out.str().c_str(), out.str().length());
AS_LITERAL(BIGINT_LITERAL, r0.at("resident_size"));
if (diff > 0) {
// Memory is stored as an average of RSS changes between query executions.
- query.memory = (query.memory * query.executions) + diff;
- query.memory = (query.memory / (query.executions + 1));
+ query.average_memory = (query.average_memory * query.executions) + diff;
+ query.average_memory = (query.average_memory / (query.executions + 1));
}
query.wall_time += delay;
EXPECT_EQ(config.files().at("downloads").size(), 1);
// From the new, recommended top-level "file_paths" collection.
- EXPECT_EQ(config.files().at("downloads2").size(), 1);
- EXPECT_EQ(config.files().at("system_binaries").size(), 1);
+ EXPECT_EQ(config.files().at("system_binaries").size(), 2);
}
*/
class TestConfigParserPlugin : public ConfigParserPlugin {
public:
std::vector<std::string> keys() {
+ // This config parser requests the follow top-level-config keys.
return {"dictionary", "dictionary2", "list"};
}
}
// Set parser-rendered additional data.
+ // Other plugins may request this "rendered/derived" data using a
+ // ConfigDataInstance and the getParsedData method.
data_.put("dictionary3.key2", "value2");
return Status(0, "OK");
}
+ // Flag tracking that the update method was called.
static bool update_called;
private:
bool TestConfigParserPlugin::update_called = false;
TEST_F(ConfigTests, test_config_parser) {
- // Register a config parser plugin.
+ // Register a config parser plugin, and call setup.
Registry::add<TestConfigParserPlugin>("config_parser", "test");
Registry::get("config_parser", "test")->setUp();
}
}
+class TestConfigMutationParserPlugin : public ConfigParserPlugin {
+ public:
+ std::vector<std::string> keys() {
+ // This config parser wants access to the well-known schedule key.
+ return {"schedule"};
+ }
+
+ Status update(const std::map<std::string, ConfigTree>& config) {
+ // The merged raw schedule is available as a property tree.
+ auto& schedule_data = config.at("schedule");
+ (void)schedule_data;
+
+ {
+ // But we want access to the parsed schedule structure.
+ ConfigDataInstance _config;
+ auto& data = mutableConfigData(_config);
+
+ ScheduledQuery query;
+ query.query = "new query";
+ query.interval = 1;
+ data.schedule["test_config_mutation"] = query;
+ }
+
+ return Status(0, "OK");
+ }
+
+ private:
+ FRIEND_TEST(ConfigTests, test_config_mutaion_parser);
+};
+
+TEST_F(ConfigTests, test_config_mutaion_parser) {
+ Registry::add<TestConfigMutationParserPlugin>("config_parser", "mutable");
+ Registry::get("config_parser", "mutable")->setUp();
+
+ // Update or load the config, expect the parser to be called.
+ Config::update({{"source1", "{\"schedule\": {}}"}});
+
+ {
+ ConfigDataInstance config;
+ // The config schedule should have been mutated.
+ EXPECT_EQ(config.schedule().count("test_config_mutation"), 1);
+ }
+}
+
TEST_F(ConfigTests, test_splay) {
auto val1 = splayValue(100, 10);
EXPECT_GE(val1, 90);
const auto& detail = details.at(flag.name);
if ((shell && !detail.shell) || (!shell && detail.shell) ||
(external && !detail.external) || (!external && detail.external) ||
- (cli && !detail.cli) || (!cli && detail.cli)) {
+ (cli && !detail.cli) || (!cli && detail.cli) || detail.hidden) {
continue;
}
} else if (aliases.count(flag.name) > 0) {
#include <iomanip>
#include <sstream>
+#include <osquery/filesystem.h>
#include <osquery/hash.h>
#include <osquery/logger.h>
}
std::string hashFromFile(HashType hash_type, const std::string& path) {
- Hash hash(hash_type);
+ // Perform a dry-run of a file read without filling in any content.
+ auto status = readFile(path);
+ if (!status.ok()) {
+ return "";
+ }
- FILE* file = fopen(path.c_str(), "rb");
+ Hash hash(hash_type);
+ // Use the canonicalized path returned from a successful readFile dry-run.
+ FILE* file = fopen(status.what().c_str(), "rb");
if (file == nullptr) {
VLOG(1) << "Cannot hash/open file " << path;
return "";
CLI_FLAG(bool, daemonize, false, "Run as daemon (osqueryd only)");
#endif
+ToolType kToolType = OSQUERY_TOOL_UNKNOWN;
+
void printUsage(const std::string& binary, int tool) {
// Parse help options before gflags. Only display osquery-related options.
fprintf(stdout, DESCRIPTION, kVersion.c_str());
GFLAGS_NAMESPACE::ParseCommandLineFlags(
argc_, argv_, (tool == OSQUERY_TOOL_SHELL));
+ // Set the tool type to allow runtime decisions based on daemon, shell, etc.
+ kToolType = tool;
if (tool == OSQUERY_TOOL_SHELL) {
// The shell is transient, rewrite config-loaded paths.
FLAGS_disable_logging = true;
}
#ifndef __APPLE__
- // OSX uses launchd to daemonize.
+ // OS X uses launchd to daemonize.
if (osquery::FLAGS_daemonize) {
if (daemon(0, 0) == -1) {
::exit(EXIT_FAILURE);
if (!FLAGS_disable_watchdog &&
FLAGS_watchdog_level >= WATCHDOG_LEVEL_DEFAULT &&
FLAGS_watchdog_level != WATCHDOG_LEVEL_DEBUG) {
- // Set CPU scheduling IO limits.
+ // Set CPU scheduling I/O limits.
setpriority(PRIO_PGRP, 0, 10);
#ifdef __linux__
// Using: ioprio_set(IOPRIO_WHO_PGRP, 0, IOPRIO_CLASS_IDLE);
const std::string& name) {
// Use a delay, meaning the amount of milliseconds waited for extensions.
size_t delay = 0;
- // The timeout is the maximum time in seconds to wait for extensions.
- size_t timeout = atoi(FLAGS_extensions_timeout.c_str());
+ // The timeout is the maximum microseconds in seconds to wait for extensions.
+ size_t timeout = atoi(FLAGS_extensions_timeout.c_str()) * 1000000;
+ if (timeout < kExtensionInitializeLatencyUS * 10) {
+ timeout = kExtensionInitializeLatencyUS * 10;
+ }
while (!Registry::setActive(type, name)) {
- if (!Watcher::hasManagedExtensions() || delay > timeout * 1000) {
+ if (!Watcher::hasManagedExtensions() || delay > timeout) {
LOG(ERROR) << "Active " << type << " plugin not found: " << name;
::exit(EXIT_CATASTROPHIC);
}
- ::usleep(kExtensionInitializeMLatency * 1000);
- delay += kExtensionInitializeMLatency;
+ delay += kExtensionInitializeLatencyUS;
+ ::usleep(kExtensionInitializeLatencyUS);
}
}
// Write the property tree as a JSON string into the PluginRequest.
std::ostringstream output;
- pt::write_json(output, tree, false);
+ try {
+ pt::write_json(output, tree, false);
+ } catch (const pt::json_parser::json_parser_error& e) {
+ // The content could not be represented as JSON.
+ }
request["context"] = output.str();
}
UNSIGNED_BIGINT_LITERAL lexpr = AS_LITERAL(UNSIGNED_BIGINT_LITERAL, expr);
return literal_matches<UNSIGNED_BIGINT_LITERAL>(lexpr);
} else {
- // Unsupprted affinity type.
+ // Unsupported affinity type.
return false;
}
}
#include "osquery/core/test_util.h"
+namespace fs = boost::filesystem;
+
namespace osquery {
/// Most tests will use binary or disk-backed content for parsing tests.
}
void createMockFileStructure() {
- boost::filesystem::create_directories(kFakeDirectory +
- "/deep11/deep2/deep3/");
- boost::filesystem::create_directories(kFakeDirectory + "/deep1/deep2/");
+ fs::create_directories(kFakeDirectory + "/deep11/deep2/deep3/");
+ fs::create_directories(kFakeDirectory + "/deep1/deep2/");
writeTextFile(kFakeDirectory + "/root.txt", "root");
writeTextFile(kFakeDirectory + "/door.txt", "toor");
writeTextFile(kFakeDirectory + "/roto.txt", "roto");
writeTextFile(kFakeDirectory + "/deep11/level1.txt", "l1");
writeTextFile(kFakeDirectory + "/deep11/deep2/level2.txt", "l2");
writeTextFile(kFakeDirectory + "/deep11/deep2/deep3/level3.txt", "l3");
+
+ boost::system::error_code ec;
+ fs::create_symlink(
+ kFakeDirectory + "/root.txt", kFakeDirectory + "/root2.txt", ec);
}
void tearDownMockFileStructure() {
std::vector<std::string> split(const std::string& s, const std::string& delim) {
std::vector<std::string> elems;
boost::split(elems, s, boost::is_any_of(delim));
- auto start = std::remove_if(
- elems.begin(), elems.end(), [](const std::string& s) { return s == ""; });
+ auto start =
+ std::remove_if(elems.begin(), elems.end(), [](const std::string& s) {
+ return s.size() == 0;
+ });
elems.erase(start, elems.end());
for (auto& each : elems) {
boost::algorithm::trim(each);
size_t occurences) {
// Split the string normally with the required delimiter.
auto content = split(s, delim);
- // While the result split exceeds the number of requested occurences, join.
+ // While the result split exceeds the number of requested occurrences, join.
std::vector<std::string> accumulator;
std::vector<std::string> elems;
for (size_t i = 0; i < content.size(); i++) {
// A watchdog process may hint to a worker the number of managed extensions.
// Setting this counter to 0 will prevent the worker from waiting for missing
- // dependent config plugins. Otherwise, its existance, will cause a worker to
+ // dependent config plugins. Otherwise, its existence, will cause a worker to
// wait for missing plugins to broadcast from managed extensions.
return (getenv("OSQUERY_EXTENSIONS") != nullptr);
}
if (sustained_latency > 0 &&
sustained_latency * iv >= getWorkerLimit(LATENCY_LIMIT)) {
- LOG(WARNING) << "osqueryd worker system performance limits exceeded";
+ LOG(WARNING) << "osqueryd worker (" << child
+ << ") system performance limits exceeded";
return false;
}
// Check if the private memory exceeds a memory limit.
if (footprint > 0 && footprint > getWorkerLimit(MEMORY_LIMIT) * 1024 * 1024) {
- LOG(WARNING) << "osqueryd worker memory limits exceeded: " << footprint;
+ LOG(WARNING) << "osqueryd worker (" << child
+ << ") memory limits exceeded: " << footprint;
return false;
}
// The worker is sane, no action needed.
+ // Attempt to flush status logs to the well-behaved worker.
+ relayStatusLogs();
return true;
}
WatcherLocker locker;
if (Watcher::getState(Watcher::getWorker()).last_respawn_time >
getUnixTime() - getWorkerLimit(RESPAWN_LIMIT)) {
- LOG(WARNING) << "osqueryd worker respawning too quickly";
+ LOG(WARNING) << "osqueryd worker respawning too quickly: "
+ << Watcher::workerRestartCount() << " times";
Watcher::workerRestarted();
interruptableSleep(getWorkerLimit(RESPAWN_DELAY) * 1000);
// Exponential back off for quickly-respawning clients.
// Get the path of the current process.
auto qd = SQL::selectAllFrom("processes", "pid", EQUALS, INTEGER(getpid()));
if (qd.size() != 1 || qd[0].count("path") == 0 || qd[0]["path"].size() == 0) {
- LOG(ERROR) << "osquery watcher cannot determine process path";
+ LOG(ERROR) << "osquery watcher cannot determine process path for worker";
::exit(EXIT_FAILURE);
}
}
std::ostringstream output;
- pt::write_json(output, tree, false);
+ try {
+ pt::write_json(output, tree, false);
+ } catch (const pt::json_parser::json_parser_error& e) {
+ // The content could not be represented as JSON.
+ return Status(1, e.what());
+ }
json = output.str();
return Status(0, "OK");
}
}
std::ostringstream output;
- pt::write_json(output, tree, false);
+ try {
+ pt::write_json(output, tree, false);
+ } catch (const pt::json_parser::json_parser_error& e) {
+ // The content could not be represented as JSON.
+ return Status(1, e.what());
+ }
json = output.str();
return Status(0, "OK");
}
}
std::ostringstream output;
- pt::write_json(output, tree, false);
+ try {
+ pt::write_json(output, tree, false);
+ } catch (const pt::json_parser::json_parser_error& e) {
+ // The content could not be represented as JSON.
+ return Status(1, e.what());
+ }
json = output.str();
return Status(0, "OK");
}
}
std::ostringstream output;
- pt::write_json(output, tree, false);
+ try {
+ pt::write_json(output, tree, false);
+ } catch (const pt::json_parser::json_parser_error& e) {
+ // The content could not be represented as JSON.
+ return Status(1, e.what());
+ }
json = output.str();
return Status(0, "OK");
}
std::ostringstream output;
for (auto& event : tree) {
- pt::write_json(output, event.second, false);
+ try {
+ pt::write_json(output, event.second, false);
+ } catch (const pt::json_parser::json_parser_error& e) {
+ return Status(1, e.what());
+ }
}
json = output.str();
return Status(0, "OK");
#include <rocksdb/env.h>
#include <rocksdb/options.h>
+#include <snappy.h>
#include <osquery/database.h>
#include <osquery/filesystem.h>
options_.log_file_time_to_roll = 0;
options_.keep_log_file_num = 10;
options_.max_log_file_size = 1024 * 1024 * 1;
+ options_.compaction_style = rocksdb::kCompactionStyleLevel;
+ options_.write_buffer_size = 1 * 1024 * 1024;
+ options_.max_write_buffer_number = 2;
+ options_.max_background_compactions = 1;
if (in_memory) {
// Remove when MemEnv is included in librocksdb
if (cfh == nullptr) {
return Status(1, "Could not get column family for " + domain);
}
- auto s = getDB()->Delete(rocksdb::WriteOptions(), cfh, key);
+ auto options = rocksdb::WriteOptions();
+ options.sync = true;
+ auto s = getDB()->Delete(options, cfh, key);
return Status(s.code(), s.ToString());
}
Status Query::getPreviousQueryResults(QueryData& results, DBHandleRef db) {
if (!isQueryNameInDatabase()) {
- return Status(1, "Query name not found in database");
+ return Status(0, "Query name not found in database");
}
std::string raw;
// Get the rows from the last run of this query name.
QueryData previous_qd;
auto status = getPreviousQueryResults(previous_qd);
+ if (!status.ok()) {
+ return status;
+ }
// Sanitize all non-ASCII characters from the query data values.
QueryData escaped_current_qd;
auto query = getOsqueryScheduledQuery();
auto cf = Query("not_a_real_query", query);
auto status = cf.getPreviousQueryResults(previous_qd, db_);
- EXPECT_FALSE(status.ok());
+ EXPECT_TRUE(status.toString() == "Query name not found in database");
+ EXPECT_TRUE(status.ok());
}
TEST_F(QueryTests, test_is_query_name_in_database) {
namespace osquery {
+static std::vector<size_t> kOffset = {0, 0};
+static std::string kToken = "|";
+
std::string generateToken(const std::map<std::string, size_t>& lengths,
const std::vector<std::string>& columns) {
- std::string output = "+";
+ std::string out = "+";
for (const auto& col : columns) {
if (lengths.count(col) > 0) {
- output += std::string(lengths.at(col) + 2, '-');
+ if (getenv("ENHANCE") != nullptr) {
+ std::string e = "\xF0\x9F\x90\x8C";
+ e[2] += kOffset[1];
+ e[3] += kOffset[0];
+ for (int i = 0; i < lengths.at(col) + 2; i++) {
+ e[3] = '\x8c' + kOffset[0]++;
+ if (e[3] == '\xbf') {
+ e[3] = '\x80';
+ kOffset[1] = (kOffset[1] > 3 && kOffset[1] < 8) ? 9 : kOffset[1];
+ e[2] = '\x90' + ++kOffset[1];
+ kOffset[0] = 0;
+ }
+ if (kOffset[1] == ('\x97' - '\x8d')) {
+ kOffset = {0, 0};
+ }
+ out += e.c_str();
+ }
+ } else {
+ out += std::string(lengths.at(col) + 2, '-');
+ }
}
- output += "+";
+ out += "+";
}
- output += "\n";
- return output;
+ out += "\n";
+ return out;
}
std::string generateHeader(const std::map<std::string, size_t>& lengths,
const std::vector<std::string>& columns) {
- std::string output = "|";
+ if (getenv("ENHANCE") != nullptr) {
+ kToken = "\xF0\x9F\x91\x8D";
+ }
+ std::string out = kToken;
for (const auto& col : columns) {
- output += " " + col;
+ out += " " + col;
if (lengths.count(col) > 0) {
int buffer_size = lengths.at(col) - utf8StringSize(col) + 1;
if (buffer_size > 0) {
- output += std::string(buffer_size, ' ');
+ out += std::string(buffer_size, ' ');
} else {
- output += ' ';
+ out += ' ';
}
}
- output += "|";
+ out += kToken;
}
- output += "\n";
- return output;
+ out += "\n";
+ return out;
}
std::string generateRow(const Row& r,
const std::map<std::string, size_t>& lengths,
const std::vector<std::string>& order) {
- std::string output;
+ std::string out;
for (const auto& column : order) {
if (r.count(column) == 0 || lengths.count(column) == 0) {
continue;
int buffer_size = lengths.at(column) - utf8StringSize(r.at(column)) + 1;
if (buffer_size > 0) {
- output += "| " + r.at(column) + std::string(buffer_size, ' ');
+ out += kToken + " " + r.at(column) + std::string(buffer_size, ' ');
}
}
- if (output.size() > 0) {
+ if (out.size() > 0) {
// Only append if a row was added.
- output += "|\n";
+ out += kToken + "\n";
}
- return output;
+ return out;
}
void prettyPrint(const QueryData& results,
typedef SHARED_PTR_IMPL<PosixThreadFactory> ThriftThreadFactory;
/**
- * @brief Singleton for queueing asynchronous tasks to be executed in parallel
+ * @brief Singleton for queuing asynchronous tasks to be executed in parallel
*
* Dispatcher is a singleton which can be used to coordinate the parallel
* execution of asynchronous tasks across an application. Internally,
return s;
}
}
- }
- catch (const std::exception& e) {
+ } catch (const std::exception& e) {
return Status(1, std::string("Error serializing results: ") + e.what());
}
return Status();
std::ostringstream ss;
pt::write_json(ss, serialized_results, false);
json = ss.str();
- }
- catch (const std::exception& e) {
+ } catch (const pt::json_parser::json_parser_error& e) {
return Status(1, e.what());
}
/// Helper cooloff (ms) macro to prevent thread failure thrashing.
#define EVENTS_COOLOFF 20
-FLAG(bool, disable_events, false, "Disable osquery events pubsub");
+FLAG(bool, disable_events, false, "Disable osquery publish/subscribe system");
-FLAG(int32, events_expiry, 86000, "Timeout to expire event pubsub results");
+FLAG(bool,
+ events_optimize,
+ true,
+ "Optimize subscriber select queries (scheduler only)");
+
+FLAG(int32, events_expiry, 86000, "Timeout to expire event subscriber results");
const std::vector<size_t> kEventTimeLists = {
1 * 60 * 60, // 1 hour
boost::this_thread::sleep(boost::posix_time::milliseconds(milli));
}
+QueryData EventSubscriberPlugin::genTable(QueryContext& context) {
+ EventTime start = 0, stop = -1;
+ if (context.constraints["time"].getAll().size() > 0) {
+ // Use the 'time' constraint to optimize backing-store lookups.
+ for (const auto& constraint : context.constraints["time"].getAll()) {
+ EventTime expr = 0;
+ try {
+ expr = boost::lexical_cast<EventTime>(constraint.expr);
+ } catch (const boost::bad_lexical_cast& e) {
+ expr = 0;
+ }
+ if (constraint.op == EQUALS) {
+ stop = start = expr;
+ break;
+ } else if (constraint.op == GREATER_THAN) {
+ start = std::max(start, expr + 1);
+ } else if (constraint.op == GREATER_THAN_OR_EQUALS) {
+ start = std::max(start, expr);
+ } else if (constraint.op == LESS_THAN) {
+ stop = std::min(stop, expr - 1);
+ } else if (constraint.op == LESS_THAN_OR_EQUALS) {
+ stop = std::min(stop, expr);
+ }
+ }
+ } else if (kToolType == OSQUERY_TOOL_DAEMON && FLAGS_events_optimize) {
+ // If the daemon is querying a subscriber without a 'time' constraint and
+ // allows optimization, only emit events since the last query.
+ start = optimize_time_;
+ optimize_time_ = getUnixTime() - 1;
+ }
+
+ return get(start, stop);
+}
+
void EventPublisherPlugin::fire(const EventContextRef& ec, EventTime time) {
EventContextID ec_id;
// Todo: add a check to assure normalized (seconds) time.
ec->time = time;
}
-
- // Set the optional string-verion of the time for DB columns.
- ec->time_string = std::to_string(ec->time);
}
for (const auto& subscription : subscriptions_) {
}
}
-std::vector<std::string> EventSubscriberPlugin::getIndexes(EventTime start,
- EventTime stop,
- int list_key) {
+std::set<std::string> EventSubscriberPlugin::getIndexes(EventTime start,
+ EventTime stop,
+ int list_key) {
auto db = DBHandle::getInstance();
auto index_key = "indexes." + dbNamespace();
- std::vector<std::string> indexes;
+ std::set<std::string> indexes;
// Keep track of the tail/head of account time while bin searching.
EventTime start_max = stop, stop_min = stop, local_start, local_stop;
}
// (1) The first iteration will have 1 range (start to start_max=stop).
- // (2) Itermediate iterations will have 2 (start-start_max, stop-stop_min).
+ // (2) Intermediate iterations will have 2 (start-start_max, stop-stop_min).
// For each iteration the range collapses based on the coverage using
// the first bin's start time and the last bin's stop time.
// (3) The last iteration's range includes relaxed bounds outside the
auto step = boost::lexical_cast<EventTime>(bin);
// Check if size * step -> size * (step + 1) is within a range.
int bin_start = size * step, bin_stop = size * (step + 1);
- if (expire_events_ && step * size < expire_time_) {
- expirations.push_back(list_type + "." + bin);
- } else if (bin_start >= start && bin_stop <= start_max) {
+ if (expire_events_ && expire_time_ > 0) {
+ if (bin_stop <= expire_time_) {
+ expirations.push_back(bin);
+ } else if (bin_start < expire_time_) {
+ expireRecords(list_type, bin, false);
+ }
+ }
+
+ if (bin_start >= start && bin_stop <= start_max) {
bins.push_back(bin);
} else if ((bin_start >= stop_min && bin_stop <= stop) || stop == 0) {
bins.push_back(bin);
}
}
- expireIndexes(list_type, all_bins, expirations);
+ // Rewrite the index lists and delete each expired item.
+ if (expirations.size() > 0) {
+ expireIndexes(list_type, all_bins, expirations);
+ }
+
if (bins.size() != 0) {
- // If more percision was acheived though this list's binning.
+ // If more precision was achieved though this list's binning.
local_start = boost::lexical_cast<EventTime>(bins.front()) * size;
start_max = (local_start < start_max) ? local_start : start_max;
local_stop = (boost::lexical_cast<EventTime>(bins.back()) + 1) * size;
}
for (const auto& bin : bins) {
- indexes.push_back(list_type + "." + bin);
+ indexes.insert(list_type + "." + bin);
}
if (start == start_max && stop == stop_min) {
return indexes;
}
-Status EventSubscriberPlugin::expireIndexes(
- const std::string& list_type,
- const std::vector<std::string>& indexes,
- const std::vector<std::string>& expirations) {
+void EventSubscriberPlugin::expireRecords(const std::string& list_type,
+ const std::string& index,
+ bool all) {
auto db = DBHandle::getInstance();
- auto index_key = "indexes." + dbNamespace();
auto record_key = "records." + dbNamespace();
auto data_key = "data." + dbNamespace();
- // Get the records list for the soon-to-be expired records.
- std::vector<std::string> record_indexes;
- for (const auto& bin : expirations) {
- record_indexes.push_back(list_type + "." + bin);
+ // If the expirations is not removing all records, rewrite the persisting.
+ std::vector<std::string> persisting_records;
+ // Request all records within this list-size + bin offset.
+ auto expired_records = getRecords({list_type + "." + index});
+ for (const auto& record : expired_records) {
+ if (all) {
+ db->Delete(kEvents, data_key + "." + record.first);
+ } else if (record.second > expire_time_) {
+ persisting_records.push_back(record.first + ":" +
+ std::to_string(record.second));
+ }
}
- auto expired_records = getRecords(record_indexes);
- // Remove the records using the list of expired indexes.
+ // Either drop or overwrite the record list.
+ if (all) {
+ db->Delete(kEvents, record_key + "." + list_type + "." + index);
+ } else {
+ auto new_records = boost::algorithm::join(persisting_records, ",");
+ db->Put(kEvents, record_key + "." + list_type + "." + index, new_records);
+ }
+}
+
+void EventSubscriberPlugin::expireIndexes(
+ const std::string& list_type,
+ const std::vector<std::string>& indexes,
+ const std::vector<std::string>& expirations) {
+ auto db = DBHandle::getInstance();
+ auto index_key = "indexes." + dbNamespace();
+
+ // Construct a mutable list of persisting indexes to rewrite as records.
std::vector<std::string> persisting_indexes = indexes;
+ // Remove the records using the list of expired indexes.
for (const auto& bin : expirations) {
- db->Delete(kEvents, record_key + "." + list_type + "." + bin);
+ expireRecords(list_type, bin, true);
persisting_indexes.erase(
std::remove(persisting_indexes.begin(), persisting_indexes.end(), bin),
persisting_indexes.end());
// Update the list of indexes with the non-expired indexes.
auto new_indexes = boost::algorithm::join(persisting_indexes, ",");
db->Put(kEvents, index_key + "." + list_type, new_indexes);
-
- // Delete record events.
- for (const auto& record : expired_records) {
- db->Delete(kEvents, data_key + "." + record.first);
- }
-
- return Status(0, "OK");
}
std::vector<EventRecord> EventSubscriberPlugin::getRecords(
- const std::vector<std::string>& indexes) {
+ const std::set<std::string>& indexes) {
auto db = DBHandle::getInstance();
auto record_key = "records." + dbNamespace();
- std::vector<EventRecord> records;
+ std::vector<EventRecord> records;
for (const auto& index : indexes) {
std::string record_value;
if (!db->Get(kEvents, record_key + "." + index, record_value).ok()) {
}
std::string events_key = "data." + dbNamespace();
+ if (FLAGS_events_expiry > 0) {
+ // Set the expire time to NOW - "configured lifetime".
+ // Index retrieval will apply the constraints checking and auto-expire.
+ expire_time_ = getUnixTime() - FLAGS_events_expiry;
+ }
// Select mapped_records using event_ids as keys.
std::string data_value;
return results;
}
-Status EventSubscriberPlugin::add(const Row& r, EventTime time) {
- Status status;
-
- std::shared_ptr<DBHandle> db;
+Status EventSubscriberPlugin::add(Row& r, EventTime event_time) {
+ std::shared_ptr<DBHandle> db = nullptr;
try {
db = DBHandle::getInstance();
} catch (const std::runtime_error& e) {
// Get and increment the EID for this module.
EventID eid = getEventID();
+ // Without encouraging a missing event time, do not support a 0-time.
+ auto index_time = getUnixTime();
+ if (event_time == 0) {
+ r["time"] = std::to_string(index_time);
+ } else {
+ r["time"] = std::to_string(event_time);
+ }
- std::string event_key = "data." + dbNamespace() + "." + eid;
+ // Serialize and store the row data, for query-time retrieval.
std::string data;
-
- status = serializeRowJSON(r, data);
+ auto status = serializeRowJSON(r, data);
if (!status.ok()) {
return status;
}
// Store the event data.
+ std::string event_key = "data." + dbNamespace() + "." + eid;
status = db->Put(kEvents, event_key, data);
- // Record the event in the indexing bins.
- recordEvent(eid, time);
+ // Record the event in the indexing bins, using the index time.
+ recordEvent(eid, event_time);
return status;
}
}
Status EventFactory::run(EventPublisherID& type_id) {
+ auto& ef = EventFactory::getInstance();
+ if (FLAGS_disable_events) {
+ return Status(0, "Events disabled");
+ }
+
// An interesting take on an event dispatched entrypoint.
// There is little introspection into the event type.
// Assume it can either make use of an entrypoint poller/selector or
// take care of async callback registrations in setUp/configure/run
- // only once and handle event queueing/firing in callbacks.
- EventPublisherRef publisher;
+ // only once and handle event queuing/firing in callbacks.
+ EventPublisherRef publisher = nullptr;
try {
- publisher = EventFactory::getInstance().getEventPublisher(type_id);
+ publisher = ef.getEventPublisher(type_id);
} catch (std::out_of_range& e) {
return Status(1, "No event type found");
}
- VLOG(1) << "Starting event publisher runloop: " + type_id;
+ if (publisher == nullptr) {
+ return Status(1, "Event publisher is missing");
+ } else if (publisher->hasStarted()) {
+ return Status(1, "Cannot restart an event publisher");
+ }
+ VLOG(1) << "Starting event publisher run loop: " + type_id;
publisher->hasStarted(true);
auto status = Status(0, "OK");
status = publisher->run();
osquery::publisherSleep(EVENTS_COOLOFF);
}
-
// The runloop status is not reflective of the event type's.
- publisher->tearDown();
VLOG(1) << "Event publisher " << publisher->type()
- << " runloop terminated for reason: " << status.getMessage();
+ << " run loop terminated for reason: " << status.getMessage();
+ // Publishers auto tear down when their run loop stops.
+ publisher->tearDown();
+ ef.event_pubs_.erase(type_id);
return Status(0, "OK");
}
return Status(1, "Duplicate publisher type");
}
- if (!specialized_pub->setUp().ok()) {
- // Only start event loop if setUp succeeds.
- return Status(1, "Event publisher setUp failed");
+ // Do not set up event publisher if events are disabled.
+ if (!FLAGS_disable_events) {
+ if (!specialized_pub->setUp().ok()) {
+ // Only start event loop if setUp succeeds.
+ return Status(1, "Event publisher setup failed");
+ }
}
ef.event_pubs_[type_id] = specialized_pub;
}
// Let the module initialize any Subscriptions.
- auto status = specialized_sub->init();
+ auto status = Status(0, "OK");
+ if (!FLAGS_disable_events) {
+ status = specialized_sub->init();
+ }
auto& ef = EventFactory::getInstance();
ef.event_subs_[specialized_sub->getName()] = specialized_sub;
// The event factory is responsible for configuring the event types.
auto status = publisher->addSubscription(subscription);
- publisher->configure();
+ if (!FLAGS_disable_events) {
+ publisher->configure();
+ }
return status;
}
return Status(1, "No event publisher to deregister");
}
- publisher->isEnding(true);
- if (!publisher->hasStarted()) {
- // If a publisher's run loop was not started, call tearDown since
- // the setUp happened at publisher registration time.
- publisher->tearDown();
+ if (!FLAGS_disable_events) {
+ publisher->isEnding(true);
+ if (!publisher->hasStarted()) {
+ // If a publisher's run loop was not started, call tearDown since
+ // the setUp happened at publisher registration time.
+ publisher->tearDown();
+ // If the run loop did run the tear down and erase will happen in the
+ // event
+ // thread wrapper when isEnding is next checked.
+ ef.event_pubs_.erase(type_id);
+ } else {
+ publisher->end();
+ }
}
-
- ef.event_pubs_.erase(type_id);
return Status(0, "OK");
}
}
}
- ::usleep(400);
- ef.threads_.clear();
+ // A small cool off helps OS API event publisher flushing.
+ if (!FLAGS_disable_events) {
+ ::usleep(400);
+ ef.threads_.clear();
+ }
}
-typedef std::shared_ptr<EventPublisherPlugin> EventPublisherPluginRef;
-
void attachEvents() {
- // Caller may disable events, do not setup any publishers or subscribers.
- if (FLAGS_disable_events) {
- return;
- }
-
const auto& publishers = Registry::all("event_publisher");
for (const auto& publisher : publishers) {
EventFactory::registerEventPublisher(publisher.second);
#include <sstream>
+#include <fnmatch.h>
#include <linux/limits.h>
+#include <boost/filesystem.hpp>
+
#include <osquery/filesystem.h>
#include <osquery/logger.h>
#include "osquery/events/linux/inotify.h"
+namespace fs = boost::filesystem;
+
namespace osquery {
int kINotifyMLatency = 200;
inotify_handle_ = ::inotify_init();
// If this does not work throw an exception.
if (inotify_handle_ == -1) {
- return Status(1, "Could not init inotify");
+ return Status(1, "Could not start inotify: inotify_init failed");
}
return Status(0, "OK");
}
void INotifyEventPublisher::configure() {
- for (const auto& sub : subscriptions_) {
+ for (auto& sub : subscriptions_) {
// Anytime a configure is called, try to monitor all subscriptions.
// Configure is called as a response to removing/adding subscriptions.
// This means recalculating all monitored paths.
auto sc = getSubscriptionContext(sub->context);
- addMonitor(sc->path, sc->recursive);
+ if (sc->discovered_.size() > 0) {
+ continue;
+ }
+
+ sc->discovered_ = sc->path;
+ if (sc->path.find("**") != std::string::npos) {
+ sc->recursive = true;
+ sc->discovered_ = sc->path.substr(0, sc->path.find("**"));
+ sc->path = sc->discovered_;
+ }
+
+ if (sc->path.find('*') != std::string::npos) {
+ // If the wildcard exists within the file (leaf), remove and monitor the
+ // directory instead. Apply a fnmatch on fired events to filter leafs.
+ auto fullpath = fs::path(sc->path);
+ if (fullpath.filename().string().find('*') != std::string::npos) {
+ sc->discovered_ = fullpath.parent_path().string();
+ }
+
+ if (sc->discovered_.find('*') != std::string::npos) {
+ // If a wildcard exists within the tree (stem), resolve at configure
+ // time and monitor each path.
+ std::vector<std::string> paths;
+ resolveFilePattern(sc->discovered_, paths);
+ for (const auto& _path : paths) {
+ addMonitor(_path, sc->recursive);
+ }
+ sc->recursive_match = sc->recursive;
+ continue;
+ }
+ }
+ addMonitor(sc->discovered_, sc->recursive);
}
}
removeMonitor(event->wd, false);
} else {
auto ec = createEventContextFrom(event);
- if(event->mask & IN_CREATE && isDirectory(ec->path).ok()){
- addMonitor(ec->path, 1);
- }
fire(ec);
}
// Continue to iterate
ec->event = shared_event;
// Get the pathname the watch fired on.
- std::ostringstream path;
- path << descriptor_paths_[event->wd];
+ ec->path = descriptor_paths_[event->wd];
if (event->len > 1) {
- path << "/" << event->name;
+ ec->path += event->name;
}
- ec->path = path.str();
+
for (const auto& action : kMaskActions) {
if (event->mask & action.first) {
ec->action = action.second;
bool INotifyEventPublisher::shouldFire(const INotifySubscriptionContextRef& sc,
const INotifyEventContextRef& ec) const {
- if (!sc->recursive && sc->path != ec->path) {
- // Monitored path is not recursive and path is not an exact match.
- return false;
- }
-
- if (ec->path.find(sc->path) != 0) {
- // The path does not exist as the base event path.
+ if (sc->recursive && !sc->recursive_match) {
+ ssize_t found = ec->path.find(sc->path);
+ if (found != 0) {
+ return false;
+ }
+ } else if (fnmatch((sc->path + "*").c_str(),
+ ec->path.c_str(),
+ FNM_PATHNAME | FNM_CASEFOLD |
+ ((sc->recursive_match) ? FNM_LEADING_DIR : 0)) != 0) {
+ // Only apply a leading-dir match if this is a recursive watch with a
+ // match requirement (an inline wildcard with ending recursive wildcard).
return false;
}
-
// The subscription may supply a required event mask.
if (sc->mask != 0 && !(ec->event->mask & sc->mask)) {
return false;
}
+
+ // inotify will not monitor recursively, new directories need watches.
+ if(sc->recursive && ec->action == "CREATED" && isDirectory(ec->path)){
+ const_cast<INotifyEventPublisher*>(this)->addMonitor(ec->path + '/', true);
+ }
+
return true;
}
if (recursive && isDirectory(path).ok()) {
std::vector<std::string> children;
- // Get a list of children of this directory (requesed recursive watches).
+ // Get a list of children of this directory (requested recursive watches).
listDirectoriesInDirectory(path, children);
for (const auto& child : children) {
extern std::map<int, std::string> kMaskActions;
/**
- * @brief Subscriptioning details for INotifyEventPublisher events.
+ * @brief Subscription details for INotifyEventPublisher events.
*
* This context is specific to INotifyEventPublisher. It allows the
- *subscriptioning
- * EventSubscriber to set a path (file or directory) and a limited action mask.
- * Events are passed to the subscriptioning EventSubscriber if they match the
- *context
+ * subscribing EventSubscriber to set a path (file or directory) and a
+ * limited action mask.
+ * Events are passed to the EventSubscriber if they match the context
* path (or anything within a directory if the path is a directory) and if the
* event action is part of the mask. If the mask is 0 then all actions are
* passed to the EventSubscriber.
struct INotifySubscriptionContext : public SubscriptionContext {
/// Subscription the following filesystem path.
std::string path;
- /// Limit the `inotify` actions to the subscriptioned mask (if not 0).
+ /// Limit the `inotify` actions to the subscription mask (if not 0).
uint32_t mask;
/// Treat this path as a directory and subscription recursively.
bool recursive;
- INotifySubscriptionContext() : mask(0), recursive(false) {}
+ INotifySubscriptionContext()
+ : mask(0), recursive(false), recursive_match(false) {}
/**
* @brief Helper method to map a string action to `inotify` action mask bit.
}
}
}
+
+ private:
+ /// During configure the INotify publisher may modify/optimize the paths.
+ std::string discovered_;
+ /// A configure-time pattern was expanded to match absolute paths.
+ bool recursive_match;
+
+ private:
+ friend class INotifyEventPublisher;
};
/**
private:
INotifyEventContextRef createEventContextFrom(struct inotify_event* event);
+
/// Check all added Subscription%s for a path.
bool isPathMonitored(const std::string& path);
+
/// Add an INotify watch (monitor) on this path.
bool addMonitor(const std::string& path, bool recursive);
+
/// Remove an INotify watch (monitor) from our tracking.
bool removeMonitor(const std::string& path, bool force = false);
bool removeMonitor(int watch, bool force = false);
+
/// Given a SubscriptionContext and INotifyEventContext match path and action.
bool shouldFire(const INotifySubscriptionContextRef& mc,
const INotifyEventContextRef& ec) const;
+
/// Get the INotify file descriptor.
int getHandle() { return inotify_handle_; }
+
/// Get the number of actual INotify active descriptors.
int numDescriptors() { return descriptors_.size(); }
+
/// If we overflow, try and restart the monitor
Status restartMonitoring();
// Consider an event queue if separating buffering from firing/servicing.
DescriptorVector descriptors_;
+
+ /// Map of watched path string to inotify watch file descriptor.
PathDescriptorMap path_descriptors_;
+
+ /// Map of inotify watch file descriptor to watched path string.
DescriptorPathMap descriptor_paths_;
+
+ /// The inotify file descriptor handle.
int inotify_handle_;
+
+ /// Time in seconds of the last inotify restart.
int last_restart_;
public:
public:
int callback_count_;
std::vector<std::string> actions_;
+
+ private:
+ FRIEND_TEST(INotifyTests, test_inotify_fire_event);
+ FRIEND_TEST(INotifyTests, test_inotify_event_action);
+ FRIEND_TEST(INotifyTests, test_inotify_optimization);
+ FRIEND_TEST(INotifyTests, test_inotify_recursion);
};
TEST_F(INotifyTests, test_inotify_run) {
namespace osquery {
-const std::string kTestingEventsDBPath = "/tmp/rocksdb-osquery-testevents";
+//const std::string kTestingEventsDBPath = "/tmp/rocksdb-osquery-testevents";
-class EventsDatabaseTests : public ::testing::Test {
- public:
- void SetUp() {
- // Setup a testing DB instance
- DBHandle::getInstanceAtPath(kTestingEventsDBPath);
- }
-
- void TearDown() {
- // Todo: each test set should operate on a clear working directory.
- boost::filesystem::remove_all(osquery::kTestingEventsDBPath);
- }
-};
+class EventsDatabaseTests : public ::testing::Test {};
class FakeEventPublisher
: public EventPublisher<SubscriptionContext, EventContext> {
// Get a mix of indexes for the lower bounding.
indexes = sub->getIndexes(2, (3 * 3600));
output = boost::algorithm::join(indexes, ", ");
- EXPECT_EQ(output, "3600.1, 3600.2, 60.1, 10.0, 10.1");
+ EXPECT_EQ(output, "10.0, 10.1, 3600.1, 3600.2, 60.1");
// Rare, but test ONLY intermediate indexes.
indexes = sub->getIndexes(2, (3 * 3600), 1);
output = boost::algorithm::join(indexes, ", ");
- EXPECT_EQ(output, "60.0, 60.1, 60.60, 60.120");
+ EXPECT_EQ(output, "60.0, 60.1, 60.120, 60.60");
// Add specific indexes to the upper bound.
status = sub->testAdd((2 * 3600) + 11);
status = sub->testAdd((2 * 3600) + 61);
indexes = sub->getIndexes(2 * 3600, (2 * 3600) + 62);
output = boost::algorithm::join(indexes, ", ");
- EXPECT_EQ(output, "60.120, 10.726");
+ EXPECT_EQ(output, "10.726, 60.120");
// Request specific lower and upper bounding.
indexes = sub->getIndexes(2, (2 * 3600) + 62);
output = boost::algorithm::join(indexes, ", ");
- EXPECT_EQ(output, "3600.1, 60.1, 60.120, 10.0, 10.1, 10.726");
+ EXPECT_EQ(output, "10.0, 10.1, 10.726, 3600.1, 60.1, 60.120");
}
TEST_F(EventsDatabaseTests, test_record_range) {
auto sub = std::make_shared<FakeEventSubscriber>();
// No expiration
- auto indexes = sub->getIndexes(0, 60);
+ auto indexes = sub->getIndexes(0, 5000);
auto records = sub->getRecords(indexes);
- EXPECT_EQ(records.size(), 3); // 1, 2, 11
+ EXPECT_EQ(records.size(), 5); // 1, 2, 11, 61, 3601
sub->expire_events_ = true;
sub->expire_time_ = 10;
- indexes = sub->getIndexes(0, 60);
+ indexes = sub->getIndexes(0, 5000);
+ records = sub->getRecords(indexes);
+ EXPECT_EQ(records.size(), 3); // 11, 61, 3601
+
+ indexes = sub->getIndexes(0, 5000, 0);
+ records = sub->getRecords(indexes);
+ EXPECT_EQ(records.size(), 3); // 11, 61, 3601
+
+ indexes = sub->getIndexes(0, 5000, 1);
+ records = sub->getRecords(indexes);
+ EXPECT_EQ(records.size(), 3); // 11, 61, 3601
+
+ indexes = sub->getIndexes(0, 5000, 2);
+ records = sub->getRecords(indexes);
+ EXPECT_EQ(records.size(), 3); // 11, 61, 3601
+
+ // Check that get/deletes did not act on cache.
+ sub->expire_time_ = 0;
+ indexes = sub->getIndexes(0, 5000);
records = sub->getRecords(indexes);
- EXPECT_EQ(records.size(), 1); // 11
+ EXPECT_EQ(records.size(), 3); // 11, 61, 3601
}
}
int required_value;
};
-// Typdef the shared_ptr accessors.
+// Typedef the shared_ptr accessors.
typedef std::shared_ptr<FakeSubscriptionContext> FakeSubscriptionContextRef;
typedef std::shared_ptr<FakeEventContext> FakeEventContextRef;
TEST_F(EventsTests, test_event_sub) {
auto sub = std::make_shared<FakeEventSubscriber>();
- EXPECT_EQ(sub->type(), "FakePublisher");
+ EXPECT_EQ(sub->getType(), "FakePublisher");
EXPECT_EQ(sub->getName(), "FakeSubscriber");
}
namespace osquery {
// Millisecond latency between initalizing manager pings.
-const int kExtensionInitializeMLatency = 200;
+const size_t kExtensionInitializeLatencyUS = 20000;
#ifdef __APPLE__
const std::string kModuleExtension = ".dylib";
for (const auto& uuid : uuids) {
try {
auto client = EXClient(getExtensionSocket(uuid));
-
// Ping the extension until it goes down.
client.get()->ping(status);
} catch (const std::exception& e) {
- LOG(INFO) << "Extension UUID " << uuid << " has gone away";
- Registry::removeBroadcast(uuid);
+ failures_[uuid] += 1;
continue;
}
- if (status.code != ExtensionCode::EXT_SUCCESS && fatal_) {
- Registry::removeBroadcast(uuid);
+ if (status.code != ExtensionCode::EXT_SUCCESS) {
+ LOG(INFO) << "Extension UUID " << uuid << " ping failed";
+ failures_[uuid] += 1;
+ } else {
+ failures_[uuid] = 0;
+ }
+ }
+
+ for (const auto& uuid : failures_) {
+ if (uuid.second >= 3) {
+ LOG(INFO) << "Extension UUID " << uuid.first << " has gone away";
+ Registry::removeBroadcast(uuid.first);
+ failures_[uuid.first] = 0;
}
}
}
Status extensionPathActive(const std::string& path, bool use_timeout = false) {
// Make sure the extension manager path exists, and is writable.
size_t delay = 0;
- size_t timeout = atoi(FLAGS_extensions_timeout.c_str());
+ // The timeout is given in seconds, but checked interval is microseconds.
+ size_t timeout = atoi(FLAGS_extensions_timeout.c_str()) * 1000000;
+ if (timeout < kExtensionInitializeLatencyUS * 10) {
+ timeout = kExtensionInitializeLatencyUS * 10;
+ }
do {
if (pathExists(path) && isWritable(path)) {
try {
break;
}
// Increase the total wait detail.
- delay += kExtensionInitializeMLatency;
- ::usleep(kExtensionInitializeMLatency * 1000);
- } while (delay < timeout * 1000);
+ delay += kExtensionInitializeLatencyUS;
+ ::usleep(kExtensionInitializeLatencyUS);
+ } while (delay < timeout);
return Status(1, "Extension socket not available: " + path);
}
const std::string& version,
const std::string& min_sdk_version) {
Registry::setExternal();
+ // Latency converted to milliseconds, used as a thread interruptible.
auto latency = atoi(FLAGS_extensions_interval.c_str()) * 1000;
auto status = startExtensionWatcher(FLAGS_extensions_socket, latency, true);
if (!status.ok()) {
return status;
}
+ // Seconds converted to milliseconds, used as a thread interruptible.
auto latency = atoi(FLAGS_extensions_interval.c_str()) * 1000;
// Start a extension manager watcher, if the manager dies, so should we.
Dispatcher::addService(
virtual ~ExtensionWatcher() {}
ExtensionWatcher(const std::string& path, size_t interval, bool fatal)
: path_(path), interval_(interval), fatal_(fatal) {
+ // Set the interval to a minimum of 200 milliseconds.
interval_ = (interval_ < 200) ? 200 : interval_;
}
/// Start a specialized health check for an ExtensionManager.
void watch();
+
+ private:
+ /// Allow extensions to fail for several intervals.
+ std::map<RouteUUID, size_t> failures_;
};
class ExtensionRunnerCore : public InternalRunnable {
public:
explicit EXClient(const std::string& path) : EXInternal(path) {
client_ = std::make_shared<extensions::ExtensionClient>(protocol_);
- transport_->open();
+ (void)transport_->open();
}
const std::shared_ptr<extensions::ExtensionClient>& get() { return client_; }
explicit EXManagerClient(const std::string& manager_path)
: EXInternal(manager_path) {
client_ = std::make_shared<extensions::ExtensionManagerClient>(protocol_);
- transport_->open();
+ (void)transport_->open();
}
const std::shared_ptr<extensions::ExtensionManagerClient>& get() {
namespace osquery {
-const int kDelayUS = 200;
-const int kTimeoutUS = 10000;
+const int kDelayUS = 2000;
+const int kTimeoutUS = 1000000;
const std::string kTestManagerSocket = kTestWorkingDirectory + "test.em";
class ExtensionsTest : public testing::Test {
TEST_F(ExtensionsTest, test_extension_module_search) {
createMockFileStructure();
- EXPECT_TRUE(loadModules(kFakeDirectory));
+ EXPECT_FALSE(loadModules(kFakeDirectory + "/root.txt"));
EXPECT_FALSE(loadModules("/dir/does/not/exist"));
tearDownMockFileStructure();
}
# See the License for the specific language governing permissions and
# limitations under the License
-ADD_OSQUERY_LIBRARY(osquery_filesystem filesystem.cpp
- globbing.cpp)
+ADD_OSQUERY_LIBRARY(osquery_filesystem filesystem.cpp)
ADD_OSQUERY_LIBRARY(osquery_filesystem_linux linux/proc.cpp
linux/mem.cpp)
#include <sstream>
#include <fcntl.h>
+#include <glob.h>
#include <pwd.h>
#include <sys/stat.h>
+#include <boost/algorithm/string.hpp>
#include <boost/filesystem/fstream.hpp>
#include <boost/filesystem/operations.hpp>
#include <boost/property_tree/json_parser.hpp>
namespace osquery {
+FLAG(uint64, read_max, 50 * 1024 * 1024, "Maximum file read size");
+FLAG(uint64, read_user_max, 10 * 1024 * 1024, "Maximum non-su read size");
+FLAG(bool, read_user_links, true, "Read user-owned filesystem links");
+
Status writeTextFile(const fs::path& path,
const std::string& content,
int permissions,
return Status(0, "OK");
}
-Status readFile(const fs::path& path, std::string& content) {
- auto path_exists = pathExists(path);
- if (!path_exists.ok()) {
- return path_exists;
+Status readFile(const fs::path& path, std::string& content, bool dry_run) {
+ struct stat file;
+ if (lstat(path.string().c_str(), &file) == 0 && S_ISLNK(file.st_mode)) {
+ if (file.st_uid != 0 && !FLAGS_read_user_links) {
+ return Status(1, "User link reads disabled");
+ }
}
- std::stringstream buffer;
- fs::ifstream file_h(path);
- if (file_h.is_open()) {
- buffer << file_h.rdbuf();
- if (file_h.bad()) {
+ if (stat(path.string().c_str(), &file) < 0) {
+ return Status(1, "Cannot access path: " + path.string());
+ }
+
+ // Apply the max byte-read based on file/link target ownership.
+ size_t read_max = (file.st_uid == 0)
+ ? FLAGS_read_max
+ : std::min(FLAGS_read_max, FLAGS_read_user_max);
+ std::ifstream is(path.string(), std::ifstream::binary | std::ios::ate);
+ if (!is.is_open()) {
+ // Attempt to read without seeking to the end.
+ is.open(path.string(), std::ifstream::binary);
+ if (!is) {
return Status(1, "Error reading file: " + path.string());
}
+ }
+
+ // Attempt to read the file size.
+ ssize_t size = is.tellg();
+
+ // Erase/clear provided string buffer.
+ content.erase();
+ if (size > read_max) {
+ VLOG(1) << "Cannot read " << path << " size exceeds limit: " << size
+ << " > " << read_max;
+ return Status(1, "File exceeds read limits");
+ }
+
+ if (dry_run) {
+ // The caller is only interested in performing file read checks.
+ boost::system::error_code ec;
+ return Status(0, fs::canonical(path, ec).string());
+ }
+
+ // Reset seek to the start of the stream.
+ is.seekg(0);
+ if (size == -1 || size == 0) {
+ // Size could not be determined. This may be a special device.
+ std::stringstream buffer;
+ buffer << is.rdbuf();
+ if (is.bad()) {
+ return Status(1, "Error reading special file: " + path.string());
+ }
content.assign(std::move(buffer.str()));
} else {
- return Status(1, "Could not open file: " + path.string());
+ content = std::string(size, '\0');
+ is.read(&content[0], size);
}
-
return Status(0, "OK");
}
+Status readFile(const fs::path& path) {
+ std::string blank;
+ return readFile(path, blank, true);
+}
+
Status isWritable(const fs::path& path) {
auto path_exists = pathExists(path);
if (!path_exists.ok()) {
return Status(status_code, "N/A");
}
-Status listFilesInDirectory(const fs::path& path,
- std::vector<std::string>& results,
- bool ignore_error) {
- fs::directory_iterator begin_iter;
- try {
- if (!fs::exists(path)) {
- return Status(1, "Directory not found: " + path.string());
+static void genGlobs(std::string path,
+ std::vector<std::string>& results,
+ GlobLimits limits) {
+ // Use our helped escape/replace for wildcards.
+ replaceGlobWildcards(path);
+
+ // Generate a glob set and recurse for double star.
+ while (true) {
+ glob_t data;
+ glob(path.c_str(), GLOB_TILDE | GLOB_MARK | GLOB_BRACE, nullptr, &data);
+ size_t count = data.gl_pathc;
+ for (size_t index = 0; index < count; index++) {
+ results.push_back(data.gl_pathv[index]);
}
-
- if (!fs::is_directory(path)) {
- return Status(1, "Supplied path is not a directory: " + path.string());
+ globfree(&data);
+ // The end state is a non-recursive ending or empty set of matches.
+ size_t wild = path.rfind("**");
+ // Allow a trailing slash after the double wild indicator.
+ if (count == 0 || wild > path.size() || wild < path.size() - 3) {
+ break;
}
- begin_iter = fs::directory_iterator(path);
- } catch (const fs::filesystem_error& e) {
- return Status(1, e.what());
+ path += "/**";
}
- fs::directory_iterator end_iter;
- for (; begin_iter != end_iter; begin_iter++) {
- try {
- if (fs::is_regular_file(begin_iter->path())) {
- results.push_back(begin_iter->path().string());
- }
- } catch (const fs::filesystem_error& e) {
- if (ignore_error == 0) {
- return Status(1, e.what());
+ // Prune results based on settings/requested glob limitations.
+ auto end = std::remove_if(
+ results.begin(), results.end(), [limits](const std::string& found) {
+ return !((found[found.length() - 1] == '/' && limits & GLOB_FOLDERS) ||
+ (found[found.length() - 1] != '/' && limits & GLOB_FILES));
+ });
+ results.erase(end, results.end());
+}
+
+Status resolveFilePattern(const fs::path& fs_path,
+ std::vector<std::string>& results) {
+ return resolveFilePattern(fs_path, results, GLOB_ALL);
+}
+
+Status resolveFilePattern(const fs::path& fs_path,
+ std::vector<std::string>& results,
+ GlobLimits setting) {
+ genGlobs(fs_path.string(), results, setting);
+ return Status(0, "OK");
+}
+
+inline void replaceGlobWildcards(std::string& pattern) {
+ // Replace SQL-wildcard '%' with globbing wildcard '*'.
+ if (pattern.find("%") != std::string::npos) {
+ boost::replace_all(pattern, "%", "*");
+ }
+
+ // Relative paths are a bad idea, but we try to accommodate.
+ if ((pattern.size() == 0 || pattern[0] != '/') && pattern[0] != '~') {
+ pattern = (fs::initial_path() / pattern).string();
+ }
+
+ auto base = pattern.substr(0, pattern.find('*'));
+ if (base.size() > 0) {
+ boost::system::error_code ec;
+ auto canonicalized = fs::canonical(base, ec).string();
+ if (canonicalized.size() > 0 && canonicalized != base) {
+ if (isDirectory(canonicalized)) {
+ // Canonicalized directory paths will not include a trailing '/'.
+ // However, if the wildcards are applied to files within a directory
+ // then the missing '/' changes the wildcard meaning.
+ canonicalized += '/';
}
+ // We are unable to canonicalize the meaning of post-wildcard limiters.
+ pattern = canonicalized + pattern.substr(base.size());
}
}
- return Status(0, "OK");
}
-Status listDirectoriesInDirectory(const fs::path& path,
- std::vector<std::string>& results,
- bool ignore_error) {
- fs::directory_iterator begin_iter;
+inline Status listInAbsoluteDirectory(const fs::path& path,
+ std::vector<std::string>& results,
+ GlobLimits limits) {
try {
- if (!fs::exists(path)) {
- return Status(1, "Directory not found");
- }
-
- auto stat = pathExists(path);
- if (!stat.ok()) {
- return stat;
+ if (path.filename() == "*" && !fs::exists(path.parent_path())) {
+ return Status(1, "Directory not found: " + path.parent_path().string());
}
- stat = isDirectory(path);
- if (!stat.ok()) {
- return stat;
+ if (path.filename() == "*" && !fs::is_directory(path.parent_path())) {
+ return Status(1, "Path not a directory: " + path.parent_path().string());
}
- begin_iter = fs::directory_iterator(path);
} catch (const fs::filesystem_error& e) {
return Status(1, e.what());
}
-
- fs::directory_iterator end_iter;
- for (; begin_iter != end_iter; begin_iter++) {
- try {
- if (fs::is_directory(begin_iter->path())) {
- results.push_back(begin_iter->path().string());
- }
- } catch (const fs::filesystem_error& e) {
- if (ignore_error == 0) {
- return Status(1, e.what());
- }
- }
- }
+ genGlobs(path.string(), results, limits);
return Status(0, "OK");
}
+Status listFilesInDirectory(const fs::path& path,
+ std::vector<std::string>& results,
+ bool ignore_error) {
+ return listInAbsoluteDirectory((path / "*"), results, GLOB_FILES);
+}
+
+Status listDirectoriesInDirectory(const fs::path& path,
+ std::vector<std::string>& results,
+ bool ignore_error) {
+ return listInAbsoluteDirectory((path / "*"), results, GLOB_FOLDERS);
+}
+
Status getDirectory(const fs::path& path, fs::path& dirpath) {
if (!isDirectory(path).ok()) {
dirpath = fs::path(path).parent_path().string();
}
Status isDirectory(const fs::path& path) {
- try {
- if (fs::is_directory(path)) {
- return Status(0, "OK");
- }
+ boost::system::error_code ec;
+ if (fs::is_directory(path, ec)) {
+ return Status(0, "OK");
+ }
+ if (ec.value() == 0) {
return Status(1, "Path is not a directory: " + path.string());
- } catch (const fs::filesystem_error& e) {
- return Status(1, e.what());
}
+ return Status(ec.value(), ec.message());
}
std::set<fs::path> getHomeDirectories() {
} else if (user != nullptr && user->pw_dir != nullptr) {
homedir = std::string(user->pw_dir) + "/.osquery";
} else {
- // Failover to a temporary directory (used for the shell).
+ // Fail over to a temporary directory (used for the shell).
homedir = "/tmp/osquery";
}
}
+++ /dev/null
-/*
- * Copyright (c) 2014, Facebook, Inc.
- * All rights reserved.
- *
- * This source code is licensed under the BSD-style license found in the
- * LICENSE file in the root directory of this source tree. An additional grant
- * of patent rights can be found in the PATENTS file in the same directory.
- *
- */
-
-#include <boost/algorithm/string/join.hpp>
-#include <boost/filesystem/operations.hpp>
-
-#include <osquery/core.h>
-#include <osquery/filesystem.h>
-
-namespace fs = boost::filesystem;
-
-namespace osquery {
-
-/**
- * @brief Drill down recursively and list all sub files
- *
- * This functions purpose is to take a path with no wildcards
- * and it will recursively go through all files and and return
- * them in the results vector.
- *
- * @param fs_path The entire resolved path
- * @param results The vector where results will be returned
- * @param rec_depth How many recursions deep the current execution is at
- *
- * @return An instance of osquery::Status indicating the success of failure of
- * the operation
- */
-Status doubleStarTraversal(const fs::path& fs_path,
- std::vector<std::string>& results,
- ReturnSetting setting,
- unsigned int rec_depth) {
- if (rec_depth >= kMaxDirectoryTraversalDepth) {
- return Status(2, fs_path.string().c_str());
- }
- // List files first
- if (setting & REC_LIST_FILES) {
- Status stat = listFilesInDirectory(fs_path, results);
- if (!stat.ok()) {
- return Status(0, "OK");
- }
- }
- std::vector<std::string> folders;
- Status stat = listDirectoriesInDirectory(fs_path, folders);
- if (!stat.ok()) {
- return Status(0, "OK");
- }
- if (setting & REC_LIST_FOLDERS) {
- results.push_back(fs_path.string());
- }
- for (const auto& folder : folders) {
- if (fs::is_symlink(folder)) {
- continue;
- }
-
- stat = doubleStarTraversal(folder, results, setting, rec_depth + 1);
- if (!stat.ok() && stat.getCode() == 2) {
- return stat;
- }
- }
- return Status(0, "OK");
-}
-
-/**
- * @brief Resolve the last component of a file path
- *
- * This function exists because unlike the other parts of of a file
- * path, which should only resolve to folder, a wildcard at the end
- * means to list all files in that directory, as does just listing
- * folder. Also, a double means to drill down recursively into that
- * that folder and list all sub file.
- *
- * @param fs_path The entire resolved path (except last component)
- * @param results The vector where results will be returned
- * @param components A path, split by forward slashes
- * @param rec_depth How many recursions deep the current execution is at
- *
- * @return An instance of osquery::Status indicating the success of failure of
- * the operation
- */
-Status resolveLastPathComponent(const fs::path& fs_path,
- std::vector<std::string>& results,
- ReturnSetting setting,
- const std::vector<std::string>& components,
- unsigned int rec_depth) {
-
- // Is the last component a double star?
- if (components[components.size() - 1] == kWildcardCharacterRecursive) {
- if (setting & REC_EVENT_OPT) {
- results.push_back(fs_path.parent_path().string());
- return Status(0, "OK");
- } else {
- Status stat = doubleStarTraversal(
- fs_path.parent_path(), results, setting, rec_depth);
- return stat;
- }
- }
-
- try {
- // Is the path a file
- if ((setting & (REC_EVENT_OPT | REC_LIST_FILES)) > 0 &&
- fs::is_regular_file(fs_path)) {
- results.push_back(fs_path.string());
- return Status(0, "OK");
- }
- } catch (const fs::filesystem_error& e) {
- return Status(0, "OK");
- }
-
- std::vector<std::string> files;
- std::vector<std::string> folders;
- Status stat_file = listFilesInDirectory(fs_path.parent_path(), files);
- Status stat_fold = listDirectoriesInDirectory(fs_path.parent_path(), folders);
-
- // Is the last component a wildcard?
- if (components[components.size() - 1] == kWildcardCharacter) {
-
- if (setting & REC_EVENT_OPT) {
- results.push_back(fs_path.parent_path().string());
- return Status(0, "OK");
- }
- if (setting & REC_LIST_FOLDERS) {
- results.push_back(fs_path.parent_path().string());
- for (const auto& fold : folders) {
- results.push_back(fold);
- }
- }
- if (setting & REC_LIST_FILES) {
- for (const auto& file : files) {
- results.push_back(file);
- }
- }
- return Status(0, "OK");
- }
-
- std::string processed_path =
- "/" +
- boost::algorithm::join(
- std::vector<std::string>(components.begin(), components.end() - 1),
- "/");
-
- // Is this a (.*)% type file match
- if (components[components.size() - 1].find(kWildcardCharacter, 1) !=
- std::string::npos &&
- components[components.size() - 1][0] != kWildcardCharacter[0]) {
-
- std::string prefix =
- processed_path + "/" +
- components[components.size() - 1].substr(
- 0, components[components.size() - 1].find(kWildcardCharacter, 1));
- if (setting & REC_LIST_FOLDERS) {
- for (const auto& fold : folders) {
- if (fold.find(prefix, 0) != 0) {
- continue;
- }
- results.push_back(fold);
- }
- }
- if (setting & REC_LIST_FILES || setting & REC_EVENT_OPT) {
- for (const auto& file : files) {
- if (file.find(prefix, 0) != 0) {
- continue;
- }
- results.push_back(file);
- }
- }
- // Should be a return here?
- return Status(0, "OK");
- }
-
- // Is this a %(.*) type file match
- if (components[components.size() - 1][0] == kWildcardCharacter[0]) {
- std::string suffix = components[components.size() - 1].substr(1);
- if (setting & REC_LIST_FOLDERS) {
- for (const auto& fold : folders) {
- std::string file_name =
- boost::filesystem::path(fold).filename().string();
- size_t pos = file_name.find(suffix);
- if (pos != std::string::npos &&
- pos + suffix.length() == file_name.length()) {
- results.push_back(fold);
- }
- }
- }
- if (setting & REC_LIST_FILES || setting & REC_EVENT_OPT) {
- for (const auto& file : files) {
- boost::filesystem::path p(file);
- std::string file_name = p.filename().string();
- size_t pos = file_name.find(suffix);
- if (pos != std::string::npos &&
- pos + suffix.length() == file_name.length()) {
- results.push_back(file);
- }
- }
- }
- return Status(0, "OK");
- }
-
- // Back out if this path doesn't exist due to invalid path
- if (!(pathExists(fs_path).ok())) {
- return Status(0, "OK");
- }
-
- // Is the path a directory
- if (fs::is_directory(fs_path)) {
- results.push_back(fs_path.string());
- return Status(0, "OK");
- }
-
- return Status(1, "UNKNOWN FILE TYPE");
-}
-
-/**
- * @brief List all files in a directory recursively
- *
- * This is an overloaded version of the exported `resolveFilePattern`. This
- * version is used internally to facilitate the tracking of the recursion
- * depth.
- *
- * @param results The vector where results will be returned
- * @param components A path, split by forward slashes
- * @param processed_index What index of components has been resolved so far
- * @param rec_depth How many recursions deep the current execution is at
- *
- * @return An instance of osquery::Status indicating the success of failure of
- * the operation
- */
-Status resolveFilePattern(std::vector<std::string> components,
- std::vector<std::string>& results,
- ReturnSetting setting = REC_LIST_FILES,
- unsigned int processed_index = 0,
- unsigned int rec_depth = 0) {
-
- // Stop recursing here if we've reached out max depth
- if (rec_depth >= kMaxDirectoryTraversalDepth) {
- return Status(2, "MAX_DEPTH");
- }
-
- // Handle all parts of the path except last because then we want to get files,
- // not directories
- for (auto i = processed_index; i < components.size() - 1; i++) {
-
- // If we encounter a full recursion, that is invalid because it is not
- // the last component. So return.
- if (components[i] == kWildcardCharacterRecursive) {
- return Status(1, kWildcardCharacterRecursive + " NOT LAST COMPONENT");
- }
-
- // Create a vector to hold all the folders in the current folder
- // Build the path we're at out of components
- std::vector<std::string> folders;
-
- std::string processed_path =
- "/" +
- boost::algorithm::join(std::vector<std::string>(components.begin(),
- components.begin() + i),
- "/");
- Status stat = listDirectoriesInDirectory(processed_path, folders);
- // If we couldn't list the directories it's probably because
- // the path is invalid (or we don't have permission). Return
- // here because this branch is no good. This is not an error
- if (!stat.ok()) {
- return Status(0, "OK");
- }
- // If we just have a wildcard character then we will recurse though
- // all folders we find
- if (components[i] == kWildcardCharacter) {
- for (const auto& dir : folders) {
- boost::filesystem::path p(dir);
- components[i] = p.filename().string();
- Status stat = resolveFilePattern(
- components, results, setting, i + 1, rec_depth + 1);
- if (!stat.ok() && stat.getCode() == 2) {
- return stat;
- }
- }
- // Our subcalls that handle processing are now complete, return
- return Status(0, "OK");
-
- // The case of (.*)%
- } else if (components[i].find(kWildcardCharacter, 1) != std::string::npos &&
- components[i][0] != kWildcardCharacter[0]) {
- std::string prefix =
- processed_path + "/" +
- components[i].substr(0, components[i].find(kWildcardCharacter, 1));
- for (const auto& dir : folders) {
- if (dir.find(prefix, 0) != 0) {
- continue;
- }
- boost::filesystem::path p(dir);
- components[i] = p.filename().string();
- Status stat = resolveFilePattern(
- components, results, setting, i + 1, rec_depth + 1);
- if (!stat.ok() && stat.getCode() == 2) {
- return stat;
- }
- }
- return Status(0, "OK");
- // The case of %(.*)
- } else if (components[i][0] == kWildcardCharacter[0]) {
- std::string suffix = components[i].substr(1);
- for (const auto& dir : folders) {
- boost::filesystem::path p(dir);
- std::string folder_name = p.filename().string();
- size_t pos = folder_name.find(suffix);
- if (pos != std::string::npos &&
- pos + suffix.length() == folder_name.length()) {
- components[i] = p.filename().string();
- Status stat = resolveFilePattern(
- components, results, setting, i + 1, rec_depth + 1);
- if (!stat.ok() && stat.getCode() == 2) {
- return stat;
- }
- }
- }
- return Status(0, "OK");
- } else {
- }
- }
-
- // At this point, all of our call paths have been resolved, so know we want to
- // list the files at this point or do our ** traversal
- return resolveLastPathComponent("/" + boost::algorithm::join(components, "/"),
- results,
- setting,
- components,
- rec_depth);
-}
-
-Status resolveFilePattern(const fs::path& fs_path,
- std::vector<std::string>& results) {
- if (fs_path.string()[0] != '/') {
- return resolveFilePattern(
- split(fs::current_path().string() + "/" + fs_path.string(), "/"),
- results);
- }
- return resolveFilePattern(split(fs_path.string(), "/"), results);
-}
-
-Status resolveFilePattern(const fs::path& fs_path,
- std::vector<std::string>& results,
- ReturnSetting setting) {
- if (fs_path.string()[0] != '/') {
- return resolveFilePattern(
- split(fs::current_path().string() + "/" + fs_path.string(), "/"),
- results,
- setting);
- }
- return resolveFilePattern(split(fs_path.string(), "/"), results, setting);
-}
-}
namespace pt = boost::property_tree;
namespace osquery {
+
+DECLARE_uint64(read_max);
+DECLARE_uint64(read_user_max);
+DECLARE_bool(read_user_links);
+
class FilesystemTests : public testing::Test {
protected:
void SetUp() { createMockFileStructure(); }
void TearDown() { tearDownMockFileStructure(); }
+
+ /// Helper method to check if a path was included in results.
+ bool contains(const std::vector<std::string>& all, const std::string& n) {
+ return !(std::find(all.begin(), all.end(), n) == all.end());
+ }
};
-TEST_F(FilesystemTests, test_plugin) {
+TEST_F(FilesystemTests, test_read_file) {
std::ofstream test_file(kTestWorkingDirectory + "fstests-file");
test_file.write("test123\n", sizeof("test123"));
test_file.close();
remove(kTestWorkingDirectory + "fstests-file");
}
-TEST_F(FilesystemTests, test_list_files_in_directory_not_found) {
- std::vector<std::string> not_found_vector;
- auto not_found = listFilesInDirectory("/foo/bar", not_found_vector);
- EXPECT_FALSE(not_found.ok());
- EXPECT_EQ(not_found.toString(), "Directory not found: /foo/bar");
+TEST_F(FilesystemTests, test_read_symlink) {
+ std::string content;
+ auto status = readFile(kFakeDirectory + "/root2.txt", content);
+ EXPECT_TRUE(status.ok());
+ EXPECT_EQ(content, "root");
}
-TEST_F(FilesystemTests, test_wildcard_single_file_list) {
- std::vector<std::string> files;
- std::vector<std::string> files_flag;
- auto status = resolveFilePattern(kFakeDirectory + "/%", files);
- auto status2 =
- resolveFilePattern(kFakeDirectory + "/%", files_flag, REC_LIST_FILES);
- EXPECT_TRUE(status.ok());
- EXPECT_EQ(files.size(), 3);
- EXPECT_EQ(files.size(), files_flag.size());
- EXPECT_NE(std::find(files.begin(), files.end(), kFakeDirectory + "/roto.txt"),
- files.end());
+TEST_F(FilesystemTests, test_read_limit) {
+ auto max = FLAGS_read_max;
+ auto user_max = FLAGS_read_user_max;
+ FLAGS_read_max = 3;
+ std::string content;
+ auto status = readFile(kFakeDirectory + "/root.txt", content);
+ EXPECT_FALSE(status.ok());
+ FLAGS_read_max = max;
+
+ if (getuid() != 0) {
+ content.erase();
+ FLAGS_read_user_max = 2;
+ status = readFile(kFakeDirectory + "/root.txt", content);
+ EXPECT_FALSE(status.ok());
+ FLAGS_read_user_max = user_max;
+
+ // Test that user symlinks aren't followed if configured.
+ // 'root2.txt' is a symlink in this case.
+ FLAGS_read_user_links = false;
+ content.erase();
+ status = readFile(kFakeDirectory + "/root2.txt", content);
+ EXPECT_FALSE(status.ok());
+
+ // Make sure non-link files are still readable.
+ content.erase();
+ status = readFile(kFakeDirectory + "/root.txt", content);
+ EXPECT_TRUE(status.ok());
+
+ // Any the links are readable if enabled.
+ FLAGS_read_user_links = true;
+ status = readFile(kFakeDirectory + "/root2.txt", content);
+ EXPECT_TRUE(status.ok());
+ }
}
-TEST_F(FilesystemTests, test_wildcard_dual) {
- std::vector<std::string> files;
- auto status = resolveFilePattern(kFakeDirectory + "/%/%", files);
- EXPECT_TRUE(status.ok());
- EXPECT_NE(std::find(files.begin(),
- files.end(),
- kFakeDirectory + "/deep1/level1.txt"),
- files.end());
+TEST_F(FilesystemTests, test_list_files_missing_directory) {
+ std::vector<std::string> results;
+ auto status = listFilesInDirectory("/foo/bar", results);
+ EXPECT_FALSE(status.ok());
}
-TEST_F(FilesystemTests, test_wildcard_full_recursion) {
- std::vector<std::string> files;
- auto status = resolveFilePattern(kFakeDirectory + "/%%", files);
- EXPECT_TRUE(status.ok());
- EXPECT_NE(std::find(files.begin(),
- files.end(),
- kFakeDirectory + "/deep1/deep2/level2.txt"),
- files.end());
+TEST_F(FilesystemTests, test_list_files_invalid_directory) {
+ std::vector<std::string> results;
+ auto status = listFilesInDirectory("/etc/hosts", results);
+ EXPECT_FALSE(status.ok());
}
-TEST_F(FilesystemTests, test_wildcard_end_last_component) {
- std::vector<std::string> files;
- auto status = resolveFilePattern(kFakeDirectory + "/%11/%sh", files);
- EXPECT_TRUE(status.ok());
- EXPECT_NE(std::find(files.begin(),
- files.end(),
- kFakeDirectory + "/deep11/not_bash"),
- files.end());
+TEST_F(FilesystemTests, test_list_files_valid_directorty) {
+ std::vector<std::string> results;
+ auto s = listFilesInDirectory("/etc", results);
+ // This directory may be different on OS X or Linux.
+ std::string hosts_path = "/etc/hosts";
+ replaceGlobWildcards(hosts_path);
+ EXPECT_TRUE(s.ok());
+ EXPECT_EQ(s.toString(), "OK");
+ EXPECT_TRUE(contains(results, hosts_path));
}
-TEST_F(FilesystemTests, test_wildcard_three_kinds) {
- std::vector<std::string> files;
- auto status = resolveFilePattern(kFakeDirectory + "/%p11/%/%%", files);
- EXPECT_TRUE(status.ok());
- EXPECT_NE(std::find(files.begin(),
- files.end(),
- kFakeDirectory + "/deep11/deep2/deep3/level3.txt"),
- files.end());
+TEST_F(FilesystemTests, test_canonicalization) {
+ std::string complex = kFakeDirectory + "/deep1/../deep1/..";
+ std::string simple = kFakeDirectory + "/";
+ // Use the inline wildcard and canonicalization replacement.
+ // The 'simple' path contains a trailing '/', the replacement method will
+ // distinguish between file and directory paths.
+ replaceGlobWildcards(complex);
+ EXPECT_EQ(simple, complex);
+ // Now apply the same inline replacement on the simple directory and expect
+ // no change to the comparison.
+ replaceGlobWildcards(simple);
+ EXPECT_EQ(simple, complex);
+
+ // Now add a wildcard within the complex pattern. The replacement method
+ // will not canonicalize past a '*' as the proceeding paths are limiters.
+ complex = kFakeDirectory + "/*/deep2/../deep2/";
+ replaceGlobWildcards(complex);
+ EXPECT_EQ(complex, kFakeDirectory + "/*/deep2/../deep2/");
}
-TEST_F(FilesystemTests, test_wildcard_invalid_path) {
- std::vector<std::string> files;
- auto status = resolveFilePattern("/not_ther_abcdefz/%%", files);
+TEST_F(FilesystemTests, test_simple_globs) {
+ std::vector<std::string> results;
+ // Test the shell '*', we will support SQL's '%' too.
+ auto status = resolveFilePattern(kFakeDirectory + "/*", results);
EXPECT_TRUE(status.ok());
- EXPECT_EQ(files.size(), 0);
+ EXPECT_EQ(results.size(), 6);
+
+ // Test the csh-style bracket syntax: {}.
+ results.clear();
+ resolveFilePattern(kFakeDirectory + "/{root,door}*", results);
+ EXPECT_EQ(results.size(), 3);
+
+ // Test a tilde, home directory expansion, make no asserts about contents.
+ results.clear();
+ resolveFilePattern("~", results);
+ if (results.size() == 0) {
+ LOG(WARNING) << "Tilde expansion failed.";
+ }
}
-TEST_F(FilesystemTests, test_wildcard_filewild) {
- std::vector<std::string> files;
- auto status = resolveFilePattern(kFakeDirectory + "/deep1%/%", files);
+TEST_F(FilesystemTests, test_wildcard_single_all) {
+ // Use '%' as a wild card to glob files within the temporarily-created dir.
+ std::vector<std::string> results;
+ auto status = resolveFilePattern(kFakeDirectory + "/%", results, GLOB_ALL);
EXPECT_TRUE(status.ok());
- EXPECT_NE(std::find(files.begin(),
- files.end(),
- kFakeDirectory + "/deep1/level1.txt"),
- files.end());
- EXPECT_NE(std::find(files.begin(),
- files.end(),
- kFakeDirectory + "/deep11/level1.txt"),
- files.end());
+ EXPECT_EQ(results.size(), 6);
+ EXPECT_TRUE(contains(results, kFakeDirectory + "/roto.txt"));
+ EXPECT_TRUE(contains(results, kFakeDirectory + "/deep11/"));
}
-TEST_F(FilesystemTests, test_list_files_in_directory_not_dir) {
- std::vector<std::string> not_dir_vector;
- auto not_dir = listFilesInDirectory("/etc/hosts", not_dir_vector);
- EXPECT_FALSE(not_dir.ok());
- EXPECT_EQ(not_dir.toString(), "Supplied path is not a directory: /etc/hosts");
+TEST_F(FilesystemTests, test_wildcard_single_files) {
+ // Now list again with a restriction to only files.
+ std::vector<std::string> results;
+ resolveFilePattern(kFakeDirectory + "/%", results, GLOB_FILES);
+ EXPECT_EQ(results.size(), 4);
+ EXPECT_TRUE(contains(results, kFakeDirectory + "/roto.txt"));
}
-TEST_F(FilesystemTests, test_list_files_in_directorty) {
+TEST_F(FilesystemTests, test_wildcard_single_folders) {
std::vector<std::string> results;
- auto s = listFilesInDirectory("/etc", results);
- EXPECT_TRUE(s.ok());
- EXPECT_EQ(s.toString(), "OK");
- EXPECT_NE(std::find(results.begin(), results.end(), "/etc/hosts"),
- results.end());
+ resolveFilePattern(kFakeDirectory + "/%", results, GLOB_FOLDERS);
+ EXPECT_EQ(results.size(), 2);
+ EXPECT_TRUE(contains(results, kFakeDirectory + "/deep11/"));
}
-TEST_F(FilesystemTests, test_wildcard_single_folder_list) {
- std::vector<std::string> folders;
- auto status =
- resolveFilePattern(kFakeDirectory + "/%", folders, REC_LIST_FOLDERS);
+TEST_F(FilesystemTests, test_wildcard_dual) {
+ // Now test two directories deep with a single wildcard for each.
+ std::vector<std::string> results;
+ auto status = resolveFilePattern(kFakeDirectory + "/%/%", results);
EXPECT_TRUE(status.ok());
- EXPECT_EQ(folders.size(), 3);
- EXPECT_NE(
- std::find(folders.begin(), folders.end(), kFakeDirectory + "/deep11"),
- folders.end());
+ EXPECT_TRUE(contains(results, kFakeDirectory + "/deep1/level1.txt"));
}
-TEST_F(FilesystemTests, test_wildcard_single_all_list) {
- std::vector<std::string> all;
- auto status = resolveFilePattern(kFakeDirectory + "/%", all, REC_LIST_ALL);
+TEST_F(FilesystemTests, test_wildcard_double) {
+ // TODO: this will fail.
+ std::vector<std::string> results;
+ auto status = resolveFilePattern(kFakeDirectory + "/%%", results);
EXPECT_TRUE(status.ok());
- EXPECT_EQ(all.size(), 6);
- EXPECT_NE(std::find(all.begin(), all.end(), kFakeDirectory + "/roto.txt"),
- all.end());
- EXPECT_NE(std::find(all.begin(), all.end(), kFakeDirectory + "/deep11"),
- all.end());
+ EXPECT_EQ(results.size(), 15);
+ EXPECT_TRUE(contains(results, kFakeDirectory + "/deep1/deep2/level2.txt"));
}
TEST_F(FilesystemTests, test_wildcard_double_folders) {
- std::vector<std::string> all;
- auto status =
- resolveFilePattern(kFakeDirectory + "/%%", all, REC_LIST_FOLDERS);
+ std::vector<std::string> results;
+ resolveFilePattern(kFakeDirectory + "/%%", results, GLOB_FOLDERS);
+ EXPECT_EQ(results.size(), 5);
+ EXPECT_TRUE(contains(results, kFakeDirectory + "/deep11/deep2/deep3/"));
+}
+
+TEST_F(FilesystemTests, test_wildcard_end_last_component) {
+ std::vector<std::string> results;
+ auto status = resolveFilePattern(kFakeDirectory + "/%11/%sh", results);
EXPECT_TRUE(status.ok());
- EXPECT_EQ(all.size(), 6);
- EXPECT_NE(std::find(all.begin(), all.end(), kFakeDirectory), all.end());
- EXPECT_NE(
- std::find(all.begin(), all.end(), kFakeDirectory + "/deep11/deep2/deep3"),
- all.end());
+ EXPECT_TRUE(contains(results, kFakeDirectory + "/deep11/not_bash"));
}
-TEST_F(FilesystemTests, test_wildcard_double_all) {
- std::vector<std::string> all;
- auto status = resolveFilePattern(kFakeDirectory + "/%%", all, REC_LIST_ALL);
+TEST_F(FilesystemTests, test_wildcard_middle_component) {
+ std::vector<std::string> results;
+ auto status = resolveFilePattern(kFakeDirectory + "/deep1%/%", results);
EXPECT_TRUE(status.ok());
- EXPECT_EQ(all.size(), 15);
- EXPECT_NE(std::find(all.begin(), all.end(), kFakeDirectory + "/roto.txt"),
- all.end());
- EXPECT_NE(
- std::find(all.begin(), all.end(), kFakeDirectory + "/deep11/deep2/deep3"),
- all.end());
+ EXPECT_EQ(results.size(), 5);
+ EXPECT_TRUE(contains(results, kFakeDirectory + "/deep1/level1.txt"));
+ EXPECT_TRUE(contains(results, kFakeDirectory + "/deep11/level1.txt"));
}
-TEST_F(FilesystemTests, test_double_wild_event_opt) {
- std::vector<std::string> all;
- auto status = resolveFilePattern(
- kFakeDirectory + "/%%", all, REC_LIST_FOLDERS | REC_EVENT_OPT);
+
+TEST_F(FilesystemTests, test_wildcard_all_types) {
+ std::vector<std::string> results;
+ auto status = resolveFilePattern(kFakeDirectory + "/%p11/%/%%", results);
EXPECT_TRUE(status.ok());
- EXPECT_EQ(all.size(), 1);
- EXPECT_NE(std::find(all.begin(), all.end(), kFakeDirectory), all.end());
+ EXPECT_TRUE(
+ contains(results, kFakeDirectory + "/deep11/deep2/deep3/level3.txt"));
}
-TEST_F(FilesystemTests, test_letter_wild_opt) {
- std::vector<std::string> all;
- auto status = resolveFilePattern(
- kFakeDirectory + "/d%", all, REC_LIST_FOLDERS | REC_EVENT_OPT);
+TEST_F(FilesystemTests, test_wildcard_invalid_path) {
+ std::vector<std::string> results;
+ auto status = resolveFilePattern("/not_ther_abcdefz/%%", results);
EXPECT_TRUE(status.ok());
- EXPECT_EQ(all.size(), 3);
- EXPECT_NE(std::find(all.begin(), all.end(), kFakeDirectory + "/deep1"),
- all.end());
- EXPECT_NE(std::find(all.begin(), all.end(), kFakeDirectory + "/door.txt"),
- all.end());
+ EXPECT_EQ(results.size(), 0);
}
-TEST_F(FilesystemTests, test_dotdot) {
- std::vector<std::string> all;
+TEST_F(FilesystemTests, test_wildcard_dotdot_files) {
+ std::vector<std::string> results;
auto status = resolveFilePattern(
- kFakeDirectory + "/deep11/deep2/../../%", all, REC_LIST_FILES);
+ kFakeDirectory + "/deep11/deep2/../../%", results, GLOB_FILES);
EXPECT_TRUE(status.ok());
- EXPECT_EQ(all.size(), 3);
- EXPECT_NE(std::find(all.begin(),
- all.end(),
- kFakeDirectory + "/deep11/deep2/../../door.txt"),
- all.end());
+ EXPECT_EQ(results.size(), 4);
+ // The response list will contain canonicalized versions: /tmp/<tests>/...
+ std::string door_path = kFakeDirectory + "/deep11/deep2/../../door.txt";
+ replaceGlobWildcards(door_path);
+ EXPECT_TRUE(contains(results, door_path));
}
TEST_F(FilesystemTests, test_dotdot_relative) {
- std::vector<std::string> all;
- auto status = resolveFilePattern(kTestDataPath + "%", all, REC_LIST_ALL);
+ std::vector<std::string> results;
+ auto status = resolveFilePattern(kTestDataPath + "%", results);
EXPECT_TRUE(status.ok());
bool found = false;
- for (const auto& file : all) {
+ for (const auto& file : results) {
if (file.find("test.config")) {
found = true;
break;
}
TEST_F(FilesystemTests, test_no_wild) {
- std::vector<std::string> all;
+ std::vector<std::string> results;
auto status =
- resolveFilePattern(kFakeDirectory + "/roto.txt", all, REC_LIST_FILES);
+ resolveFilePattern(kFakeDirectory + "/roto.txt", results, GLOB_FILES);
EXPECT_TRUE(status.ok());
- EXPECT_EQ(all.size(), 1);
- EXPECT_NE(std::find(all.begin(), all.end(), kFakeDirectory + "/roto.txt"),
- all.end());
+ EXPECT_EQ(results.size(), 1);
+ EXPECT_TRUE(contains(results, kFakeDirectory + "/roto.txt"));
}
TEST_F(FilesystemTests, test_safe_permissions) {
#include <algorithm>
#include <thread>
+#include <boost/noncopyable.hpp>
#include <boost/property_tree/json_parser.hpp>
+#include <osquery/extensions.h>
#include <osquery/filesystem.h>
#include <osquery/flags.h>
#include <osquery/logger.h>
FLAG_ALIAS(bool, verbose_debug, verbose);
FLAG_ALIAS(bool, debug, verbose);
+/// Despite being a configurable option, this is only read/used at load.
FLAG(bool, disable_logging, false, "Disable ERROR/INFO logging");
FLAG(string, logger_plugin, "filesystem", "Logger plugin name");
* the active logger plugin is handling Glog status logs and (2) it must remove
* itself as a Glog target.
*/
-class BufferedLogSink : google::LogSink {
+class BufferedLogSink : public google::LogSink, private boost::noncopyable {
public:
/// We create this as a Singleton for proper disable/shutdown.
static BufferedLogSink& instance() {
bool enabled_;
};
-void serializeIntermediateLog(const std::vector<StatusLogLine>& log,
- PluginRequest& request) {
+/// Scoped helper to perform logging actions without races.
+class LoggerDisabler {
+ public:
+ LoggerDisabler() : stderr_status_(FLAGS_logtostderr) {
+ BufferedLogSink::disable();
+ FLAGS_logtostderr = true;
+ }
+
+ ~LoggerDisabler() {
+ BufferedLogSink::enable();
+ FLAGS_logtostderr = stderr_status_;
+ }
+
+ private:
+ bool stderr_status_;
+};
+
+static void serializeIntermediateLog(const std::vector<StatusLogLine>& log,
+ PluginRequest& request) {
pt::ptree tree;
for (const auto& log_item : log) {
pt::ptree child;
request["log"] = output.str();
}
-void deserializeIntermediateLog(const PluginRequest& request,
- std::vector<StatusLogLine>& log) {
+static void deserializeIntermediateLog(const PluginRequest& request,
+ std::vector<StatusLogLine>& log) {
if (request.count("log") == 0) {
return;
}
// Turn verbosity up to 1.
// Do log DEBUG, INFO, WARNING, ERROR to their log files.
// Do log the above and verbose=1 to stderr.
- FLAGS_minloglevel = 0; // WARNING
- FLAGS_stderrthreshold = 0;
+ FLAGS_minloglevel = 0; // INFO
+ FLAGS_stderrthreshold = 0; // INFO
FLAGS_v = 1;
} else {
// Do NOT log INFO, WARNING, ERROR to stderr.
// Do log only WARNING, ERROR to log sinks.
FLAGS_minloglevel = 1; // WARNING
- FLAGS_stderrthreshold = 1;
+ FLAGS_stderrthreshold = 1; // WARNING
}
if (FLAGS_disable_logging) {
// Do NOT log INFO, WARNING, ERROR to their log files.
FLAGS_logtostderr = true;
if (!FLAGS_verbose) {
- // verbose flag still will still emit logs to stderr.
+ // verbose flag will still emit logs to stderr.
FLAGS_minloglevel = 2; // ERROR
}
}
}
void initLogger(const std::string& name, bool forward_all) {
- // Check if logging is disabled, if it is no need to shuttle intermediates.
+ // Check if logging is disabled, if so then no need to shuttle intermediates.
if (FLAGS_disable_logging) {
return;
}
+ // Stop the buffering sink and store the intermediate logs.
BufferedLogSink::disable();
auto intermediate_logs = std::move(BufferedLogSink::dump());
auto& logger_plugin = Registry::getActive("logger");
serializeIntermediateLog(intermediate_logs, request);
auto status = Registry::call("logger", request);
if (status.ok() || forward_all) {
- // When `init` returns success we re-enabled the log sink in forwarding
- // mode. Now, Glog status logs are buffered and sent to logStatus.
+ // When LoggerPlugin::init returns success we enable the log sink in
+ // forwarding mode. Then Glog status logs are forwarded to logStatus.
BufferedLogSink::forward(true);
BufferedLogSink::enable();
}
}
return Registry::call("logger", {{"health", json}});
}
+
+void relayStatusLogs() {
+ // Prevent out dumping and registry calling from producing additional logs.
+ LoggerDisabler disabler;
+
+ // Construct a status log plugin request.
+ PluginRequest req = {{"status", "true"}};
+ auto& status_logs = BufferedLogSink::dump();
+ if (status_logs.size() == 0) {
+ return;
+ }
+
+ // Skip the registry's logic, and send directly to the core's logger.
+ PluginResponse resp;
+ serializeIntermediateLog(status_logs, req);
+ auto status = callExtension(0, "logger", FLAGS_logger_plugin, req, resp);
+ if (status.ok()) {
+ // Flush the buffered status logs.
+ // Otherwise the extension call failed and the buffering should continue.
+ status_logs.clear();
+ }
+}
}
DEFINE_int32(iterations, 1, "times to run the query in question");
DEFINE_int32(delay, 0, "delay before and after the query");
+namespace osquery {
+
+DECLARE_bool(disable_events);
+DECLARE_bool(registry_exceptions);
+}
+
int main(int argc, char* argv[]) {
// Only log to stderr
FLAGS_logtostderr = true;
}
osquery::Registry::setUp();
+ osquery::FLAGS_disable_events = true;
+ osquery::FLAGS_registry_exceptions = true;
osquery::attachEvents();
if (FLAGS_delay != 0) {
}
// Instead of calling "shutdownOsquery" force the EF to join its threads.
- osquery::EventFactory::end(true);
GFLAGS_NAMESPACE::ShutDownCommandLineFlags();
return status.getCode();
namespace osquery {
+HIDDEN_FLAG(bool, registry_exceptions, false, "Allow plugin exceptions");
+
void RegistryHelperCore::remove(const std::string& item_name) {
if (items_.count(item_name) > 0) {
items_[item_name]->tearDown();
} catch (const std::exception& e) {
LOG(ERROR) << registry_name << " registry " << item_name
<< " plugin caused exception: " << e.what();
+ if (FLAGS_registry_exceptions) {
+ throw e;
+ }
return Status(1, e.what());
} catch (...) {
LOG(ERROR) << registry_name << " registry " << item_name
<< " plugin caused unknown exception";
+ if (FLAGS_registry_exceptions) {
+ throw std::runtime_error(registry_name + ": " + item_name + " failed");
+ }
return Status(2, "Unknown exception");
}
}
const boost::property_tree::ptree& tree,
PluginResponse& response) {
std::ostringstream output;
- boost::property_tree::write_json(output, tree, false);
+ try {
+ boost::property_tree::write_json(output, tree, false);
+ } catch (const pt::json_parser::json_parser_error& e) {
+ // The plugin response could not be serialized.
+ }
response.push_back({{key, output.str()}});
}
}
QueryData SQL::selectAllFrom(const std::string& table) {
PluginResponse response;
- PluginRequest request;
- request["action"] = "generate";
-
+ PluginRequest request = {{"action", "generate"}};
Registry::call("table", table, request, response);
return response;
}
if (!self.lock_.owns_lock() && self.lock_.try_lock()) {
if (self.db_ == nullptr) {
- // Create primary sqlite DB instance.
+ // Create primary SQLite DB instance.
sqlite3_open(":memory:", &self.db_);
attachVirtualTables(self.db_);
}
int queryDataCallback(void* argument, int argc, char* argv[], char* column[]) {
if (argument == nullptr) {
- LOG(ERROR) << "queryDataCallback received nullptr as data argument";
+ VLOG(1) << "Query execution failed: received a bad callback argument";
return SQLITE_MISUSE;
}
r[column[i]] = (argv[i] != nullptr) ? argv[i] : "";
}
}
- (*qData).push_back(r);
+ (*qData).push_back(std::move(r));
return 0;
}
Status queryInternal(const std::string& q, QueryData& results, sqlite3* db) {
char* err = nullptr;
sqlite3_exec(db, q.c_str(), queryDataCallback, &results, &err);
+ sqlite3_db_release_memory(db);
if (err != nullptr) {
+ auto error_string = std::string(err);
sqlite3_free(err);
- return Status(1, "Error running query: " + q);
+ return Status(1, "Error running query: " + error_string);
}
return Status(0, "OK");
#include <osquery/sql.h>
+#define SQLITE_SOFT_HEAP_LIMIT (5 * 1024 * 1024)
+
namespace osquery {
/**
protected:
SQLiteDBManager() : db_(nullptr), lock_(mutex_, boost::defer_lock) {
+ sqlite3_soft_heap_limit64(SQLITE_SOFT_HEAP_LIMIT);
disabled_tables_ = parseDisableTablesFlag(Flag::getValue("disable_tables"));
}
SQLiteDBManager(SQLiteDBManager const&);
*/
std::string getStringForSQLiteReturnCode(int code);
-// the callback for populating a std::vector<row> set of results. "argument"
-// should be a non-const reference to a std::vector<row>
+/**
+ * @brief Accumulate rows from an SQLite exec into a QueryData struct.
+ *
+ * The callback for populating a std::vector<Row> set of results. "argument"
+ * should be a non-const reference to a std::vector<Row>.
+ */
int queryDataCallback(void* argument, int argc, char* argv[], char* column[]);
}
EXPECT_EQ("CREATE VIRTUAL TABLE sample USING sample(foo INTEGER, bar TEXT)",
results[0]["sql"]);
}
+
+TEST_F(VirtualTableTests, test_sqlite3_table_joins) {
+ // Get a database connection.
+ auto dbc = SQLiteDBManager::get();
+
+ QueryData results;
+ // Run a query with a join within.
+ std::string statement =
+ "SELECT p.pid FROM osquery_info oi, processes p WHERE oi.pid=p.pid";
+ auto status = queryInternal(statement, results, dbc.db());
+ EXPECT_TRUE(status.ok());
+ EXPECT_EQ(results.size(), 1);
+}
}
return SQLITE_ERROR;
}
- const auto &column_name = pVtab->content->columns[col].first;
- const auto &type = pVtab->content->columns[col].second;
+ auto &column_name = pVtab->content->columns[col].first;
+ auto &type = pVtab->content->columns[col].second;
if (pCur->row >= pVtab->content->data[column_name].size()) {
return SQLITE_ERROR;
}
- const auto &value = pVtab->content->data[column_name][pCur->row];
+ // Attempt to cast each xFilter-populated row/column to the SQLite type.
+ auto &value = pVtab->content->data[column_name][pCur->row];
if (type == "TEXT") {
- sqlite3_result_text(ctx, value.c_str(), -1, nullptr);
+ sqlite3_result_text(ctx, value.c_str(), value.size(), SQLITE_STATIC);
} else if (type == "INTEGER") {
int afinite;
try {
QueryContext context;
for (size_t i = 0; i < pVtab->content->columns.size(); ++i) {
+ // Clear any data, this is the result container for each column + row.
pVtab->content->data[pVtab->content->columns[i].first].clear();
+ // Set the column affinity for each optional constraint list.
+ // There is a separate list for each column name.
context.constraints[pVtab->content->columns[i].first].affinity =
pVtab->content->columns[i].second;
}
+ // Iterate over every argument to xFilter, filling in constraint values.
for (size_t i = 0; i < argc; ++i) {
auto expr = (const char *)sqlite3_value_text(argv[i]);
if (expr == nullptr) {
// Now organize the response rows by column instead of row.
auto &data = pVtab->content->data;
- for (const auto &row : response) {
+ for (auto &row : response) {
for (const auto &column : pVtab->content->columns) {
- try {
- auto &value = row.at(column.first);
- if (value.size() > FLAGS_value_max) {
- data[column.first].push_back(value.substr(0, FLAGS_value_max));
- } else {
- data[column.first].push_back(value);
- }
- } catch (const std::out_of_range &e) {
+ if (row.count(column.first) == 0) {
VLOG(1) << "Table " << pVtab->content->name << " row "
<< pVtab->content->n << " did not include column "
<< column.first;
data[column.first].push_back("");
+ continue;
+ }
+
+ auto &value = row.at(column.first);
+ if (value.size() > FLAGS_value_max) {
+ data[column.first].push_back(value.substr(0, FLAGS_value_max));
+ value.clear();
+ } else {
+ data[column.first].push_back(std::move(value));
}
}
for (const auto& file : element_kv.second) {
VLOG(1) << "Added listener to: " << file;
auto mc = createSubscriptionContext();
- mc->recursive = 1;
+ // Use the filesystem globbing pattern to determine recursiveness.
+ mc->recursive = 0;
mc->path = file;
mc->mask = IN_ATTRIB | IN_MODIFY | IN_DELETE | IN_CREATE;
subscribe(&FileEventSubscriber::Callback, mc,
const void* user_data) {
Row r;
r["action"] = ec->action;
- r["time"] = ec->time_string;
r["target_path"] = ec->path;
if (user_data != nullptr) {
r["category"] = *(std::string*)user_data;
r["category"] = "Undefined";
}
r["transaction_id"] = INTEGER(ec->event->cookie);
- r["md5"] = hashFromFile(HASH_TYPE_MD5, ec->path);
- r["sha1"] = hashFromFile(HASH_TYPE_SHA1, ec->path);
- r["sha256"] = hashFromFile(HASH_TYPE_SHA256, ec->path);
+
+ if (ec->action == "CREATED" || ec->action == "UPDATED") {
+ r["md5"] = hashFromFile(HASH_TYPE_MD5, ec->path);
+ r["sha1"] = hashFromFile(HASH_TYPE_SHA1, ec->path);
+ r["sha256"] = hashFromFile(HASH_TYPE_SHA256, ec->path);
+ }
+
if (ec->action != "" && ec->action != "OPENED") {
// A callback is somewhat useless unless it changes the EventSubscriber
- // state
- // or calls `add` to store a marked up event.
+ // state or calls `add` to store a marked up event.
add(r, ec->time);
}
return Status(0, "OK");
r["serial"] =
INTEGER(UdevEventPublisher::getValue(device, "ID_SERIAL_SHORT"));
r["revision"] = INTEGER(UdevEventPublisher::getValue(device, "ID_REVISION"));
-
- r["time"] = INTEGER(ec->time);
add(r, ec->time);
return Status(0, "OK");
}
const void* user_data) {
Row r;
r["action"] = ec->action;
- r["time"] = ec->time_string;
r["target_path"] = ec->path;
r["transaction_id"] = INTEGER(ec->event->cookie);
if (ec->action != "" && ec->action != "OPENED") {
}
Status readNetlink(int socket_fd, int seq, char* output, size_t* size) {
- struct nlmsghdr* nl_hdr;
+ struct nlmsghdr* nl_hdr = nullptr;
size_t message_size = 0;
do {
}
// Create netlink message header
- void* netlink_buffer = malloc(MAX_NETLINK_SIZE);
- struct nlmsghdr* netlink_msg = (struct nlmsghdr*)netlink_buffer;
- if (netlink_msg == nullptr) {
+ auto netlink_buffer = (void*)malloc(MAX_NETLINK_SIZE);
+ if (netlink_buffer == nullptr) {
close(socket_fd);
return {};
}
+ auto netlink_msg = (struct nlmsghdr*)netlink_buffer;
netlink_msg->nlmsg_len = NLMSG_LENGTH(sizeof(struct rtmsg));
netlink_msg->nlmsg_type = RTM_GETROUTE; // routes from kernel routing table
netlink_msg->nlmsg_flags = NLM_F_DUMP | NLM_F_REQUEST;
// Send the netlink request to the kernel
if (send(socket_fd, netlink_msg, netlink_msg->nlmsg_len, 0) < 0) {
- VLOG(1) << "Cannot write NETLINK request header to socket";
+ TLOG << "Cannot write NETLINK request header to socket";
close(socket_fd);
free(netlink_buffer);
return {};
// Wrap the read socket to support multi-netlink messages
size_t size = 0;
if (!readNetlink(socket_fd, 1, (char*)netlink_msg, &size).ok()) {
- VLOG(1) << "Cannot read NETLINK response from socket";
+ TLOG << "Cannot read NETLINK response from socket";
close(socket_fd);
free(netlink_buffer);
return {};
std::string ipAsString(const struct sockaddr *in) {
char dst[INET6_ADDRSTRLEN] = {0};
- // memset(dst, 0, sizeof(dst));
void *in_addr = nullptr;
if (in->sa_family == AF_INET) {
int mask = 0;
if (in->sa_family == AF_INET6) {
- struct sockaddr_in6 *in6 = (struct sockaddr_in6 *)in;
+ auto in6 = (struct sockaddr_in6 *)in;
for (size_t i = 0; i < 16; i++) {
mask += addBits(in6->sin6_addr.s6_addr[i]);
}
} else {
- struct sockaddr_in *in4 = (struct sockaddr_in *)in;
- char *address = reinterpret_cast<char *>(&in4->sin_addr.s_addr);
+ auto in4 = (struct sockaddr_in *)in;
+ auto address = reinterpret_cast<char *>(&in4->sin_addr.s_addr);
for (size_t i = 0; i < 4; i++) {
mask += addBits(address[i]);
}
return mask;
}
-std::string macAsString(const char *addr) {
+inline std::string macAsString(const char *addr) {
std::stringstream mac;
for (size_t i = 0; i < 6; i++) {
}
std::string macAsString(const struct ifaddrs *addr) {
- std::stringstream mac;
-
+ static std::string blank_mac = "00:00:00:00:00:00";
if (addr->ifa_addr == nullptr) {
// No link or MAC exists.
- return "";
+ return blank_mac;
}
#if defined(__linux__)
struct ifreq ifr;
-
- int socket_fd = socket(AF_INET, SOCK_DGRAM, 0);
-
ifr.ifr_addr.sa_family = AF_INET;
memcpy(ifr.ifr_name, addr->ifa_name, IFNAMSIZ);
+
+ int socket_fd = socket(AF_INET, SOCK_DGRAM, 0);
+ if (socket_fd < 0) {
+ return blank_mac;
+ }
ioctl(socket_fd, SIOCGIFHWADDR, &ifr);
close(socket_fd);
- for (size_t i = 0; i < 6; i++) {
- mac << std::hex << std::setfill('0') << std::setw(2);
- mac << (int)((uint8_t)ifr.ifr_hwaddr.sa_data[i]);
- if (i != 5) {
- mac << ":";
- }
- }
+ return macAsString(ifr.ifr_hwaddr.sa_data);
#else
- struct sockaddr_dl *sdl = nullptr;
-
- sdl = (struct sockaddr_dl *)addr->ifa_addr;
+ auto sdl = (struct sockaddr_dl *)addr->ifa_addr;
if (sdl->sdl_alen != 6) {
// Do not support MAC address that are not 6 bytes...
- return "";
+ return blank_mac;
}
- for (size_t i = 0; i < sdl->sdl_alen; i++) {
- mac << std::hex << std::setfill('0') << std::setw(2);
- // Prevent char sign extension.
- mac << (int)((uint8_t)sdl->sdl_data[i + sdl->sdl_nlen]);
- if (i != 5) {
- mac << ":";
- }
- }
+ return macAsString(&sdl->sdl_data[sdl->sdl_nlen]);
#endif
-
- return mac.str();
}
}
}
namespace osquery {
namespace tables {
-#ifdef CENTOS
const std::string kLinuxOSRelease = "/etc/redhat-release";
const std::string kLinuxOSRegex =
"(?P<name>\\w+) .* "
- "(?P<major>[0-9]+).(?P<minor>[0-9]+)[\\.]{0,1}(?P<patch>[0-9]+)";
-#else
-const std::string kLinuxOSRelease = "/etc/os-release";
-const std::string kLinuxOSRegex =
- "VERSION=\"(?P<major>[0-9]+)\\.(?P<minor>[0-9]+)[\\.]{0,1}(?P<patch>[0-9]+)"
- "?.*, (?P<name>[\\w ]*)\"$";
-#endif
+ "(?P<major>[0-9]+)\\.(?P<minor>[0-9]+)[\\.]{0,1}(?P<patch>[0-9]+).*";
QueryData genOSVersion(QueryContext& context) {
std::string content;
return result;
}
+std::set<std::string> getProcList(const QueryContext& context) {
+ std::set<std::string> pidlist;
+ if (context.constraints.count("pid") > 0 &&
+ context.constraints.at("pid").exists(EQUALS)) {
+ for (const auto& pid : context.constraints.at("pid").getAll(EQUALS)) {
+ if (isDirectory("/proc/" + pid)) {
+ pidlist.insert(pid);
+ }
+ }
+ } else {
+ osquery::procProcesses(pidlist);
+ }
+
+ return pidlist;
+}
+
void genProcessEnvironment(const std::string& pid, QueryData& results) {
auto attr = getProcAttr("environ", pid);
std::string content;
readFile(attr, content);
- for (const auto& buf : osquery::split(content, "\n")) {
+ const char* variable = content.c_str();
+
+ // Stop at the end of nul-delimited string content.
+ while (*variable > 0) {
+ auto buf = std::string(variable);
size_t idx = buf.find_first_of("=");
Row r;
r["key"] = buf.substr(0, idx);
r["value"] = buf.substr(idx + 1);
results.push_back(r);
+ variable += buf.size() + 1;
}
}
QueryData genProcesses(QueryContext& context) {
QueryData results;
- std::set<std::string> pids;
- if (context.constraints["pid"].exists(EQUALS)) {
- pids = context.constraints["pid"].getAll(EQUALS);
- } else {
- osquery::procProcesses(pids);
- }
-
- // Generate data for all pids in the vector.
- // If there are comparison constraints this could apply the operator
- // before generating the process structure.
- for (const auto& pid : pids) {
+ auto pidlist = getProcList(context);
+ for (const auto& pid : pidlist) {
genProcess(pid, results);
}
QueryData genProcessEnvs(QueryContext& context) {
QueryData results;
- std::set<std::string> pids;
- if (context.constraints["pid"].exists(EQUALS)) {
- pids = context.constraints["pid"].getAll(EQUALS);
- } else {
- osquery::procProcesses(pids);
- }
-
- for (const auto& pid : pids) {
+ auto pidlist = getProcList(context);
+ for (const auto& pid : pidlist) {
genProcessEnvironment(pid, results);
}
QueryData genProcessMemoryMap(QueryContext& context) {
QueryData results;
- std::set<std::string> pids;
- if (context.constraints["pid"].exists(EQUALS)) {
- pids = context.constraints["pid"].getAll(EQUALS);
- } else {
- osquery::procProcesses(pids);
- }
-
- for (const auto& pid : pids) {
+ auto pidlist = getProcList(context);
+ for (const auto& pid : pidlist) {
genProcessMap(pid, results);
}
--- /dev/null
+/*
+ * Copyright (c) 2014, Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree. An additional grant
+ * of patent rights can be found in the PATENTS file in the same directory.
+ *
+ */
+
+#include <osquery/tables.h>
+
+#if defined(__APPLE__)
+ #include <time.h>
+ #include <errno.h>
+ #include <sys/sysctl.h>
+#elif defined(__linux__)
+ #include <sys/sysinfo.h>
+#endif
+
+namespace osquery {
+namespace tables {
+
+long getUptime() {
+ #if defined(__APPLE__)
+ struct timeval boot_time;
+ size_t len = sizeof(boot_time);
+ int mib[2] = {
+ CTL_KERN,
+ KERN_BOOTTIME
+ };
+
+ if (sysctl(mib, 2, &boot_time, &len, NULL, 0) < 0) {
+ return -1;
+ }
+
+ time_t seconds_since_boot = boot_time.tv_sec;
+ time_t current_seconds = time(NULL);
+
+ return long(difftime(current_seconds, seconds_since_boot));
+ #elif defined(__linux__)
+ struct sysinfo sys_info;
+
+ if (sysinfo(&sys_info) != 0) {
+ return -1;
+ }
+
+ return sys_info.uptime;
+ #endif
+}
+
+QueryData genUptime(QueryContext& context) {
+ Row r;
+ QueryData results;
+ long uptime_in_seconds = getUptime();
+
+ if (uptime_in_seconds >= 0) {
+ r["days"] = INTEGER(uptime_in_seconds / 60 / 60 / 24);
+ r["hours"] = INTEGER((uptime_in_seconds / 60 / 60) % 24);
+ r["minutes"] = INTEGER((uptime_in_seconds / 60) % 60);
+ r["seconds"] = INTEGER(uptime_in_seconds % 60);
+ r["total_seconds"] = BIGINT(uptime_in_seconds);
+ results.push_back(r);
+ }
+
+ return results;
+}
+}
+}
QueryData genHash(QueryContext& context) {
QueryData results;
- // The query must provide a predicate with constratins including path or
+ // The query must provide a predicate with constraints including path or
// directory. We search for the parsed predicate constraints with the equals
// operator.
auto paths = context.constraints["path"].getAll(EQUALS);
r["wall_time"] = BIGINT(query.second.wall_time);
r["user_time"] = BIGINT(query.second.user_time);
r["system_time"] = BIGINT(query.second.system_time);
- r["average_memory"] = BIGINT(query.second.memory);
+ r["average_memory"] = BIGINT(query.second.average_memory);
results.push_back(r);
}
*/
#include <ctime>
+#include <boost/algorithm/string/trim.hpp>
#include <osquery/tables.h>
Row r;
time_t _time = time(0);
struct tm* now = localtime(&_time);
+ struct tm* gmt = gmtime(&_time);
+
+ char weekday[10] = {0};
+ strftime(weekday, sizeof(weekday), "%A", now);
+
+ std::string timestamp;
+ timestamp = asctime(gmt);
+ boost::algorithm::trim(timestamp);
+ timestamp += " UTC";
+
+ char iso_8601[21] = {0};
+ strftime(iso_8601, sizeof(iso_8601), "%FT%TZ", gmt);
+
+ r["weekday"] = TEXT(weekday);
+ r["year"] = INTEGER(now->tm_year + 1900);
+ r["month"] = INTEGER(now->tm_mon + 1);
+ r["day"] = INTEGER(now->tm_mday);
r["hour"] = INTEGER(now->tm_hour);
r["minutes"] = INTEGER(now->tm_min);
r["seconds"] = INTEGER(now->tm_sec);
+ r["unix_time"] = INTEGER(_time);
+ r["timestamp"] = TEXT(timestamp);
+ r["iso_8601"] = TEXT(iso_8601);
+
QueryData results;
results.push_back(r);
return results;
Name: osquery
-Version: 1.4.7
+Version: 1.5.0
Release: 0
License: Apache-2.0 and GPLv2
Summary: A SQL powered operating system instrumentation, monitoring framework.
table_name("etc_hosts")
description("Line-parsed /etc/hosts.")
schema([
- Column("address", TEXT),
+ Column("address", TEXT, "IP address mapping"),
Column("hostnames", TEXT, "Raw hosts mapping"),
])
implementation("etc_hosts@genEtcHosts")
schema([
Column("target_path", TEXT, "The path changed"),
Column("category", TEXT, "The category of the file"),
- Column("time", TEXT, "Time of the change"),
Column("action", TEXT, "Change action (UPDATE, REMOVE, etc)"),
Column("transaction_id", BIGINT, "ID used during bulk update"),
Column("md5", TEXT, "The MD5 of the file after change"),
Column("sha1", TEXT, "The SHA1 of the file after change"),
Column("sha256", TEXT, "The SHA256 of the file after change"),
+ Column("time", BIGINT, "Time of file event"),
])
attributes(event_subscriber=True)
implementation("file_events@file_events::genTable")
Column("path", TEXT, "Local device path assigned (optional)"),
Column("type", TEXT, "Type of hardware and hardware event"),
Column("driver", TEXT, "Driver claiming the device"),
- Column("model", TEXT, "Hadware device model"),
- Column("model_id", INTEGER),
- Column("vendor", TEXT, "hardware device vendor"),
- Column("vendor_id", INTEGER),
+ Column("model", TEXT, "Hardware device model"),
+ Column("model_id", INTEGER, "Hardware model identifier"),
+ Column("vendor", TEXT, "Hardware device vendor"),
+ Column("vendor_id", INTEGER, "Hardware vendor identifier"),
Column("serial", TEXT, "Device serial (optional)"),
Column("revision", INTEGER, "Device revision (optional)"),
- Column("time", INTEGER, "Time of hardware event"),
+ Column("time", BIGINT, "Time of hardware event"),
])
attributes(event_subscriber=True)
implementation("events/hardware_events@hardware_events::genTable")
table_name("kernel_info")
description("Basic active kernel information.")
schema([
- Column("version", TEXT),
- Column("arguments", TEXT),
- Column("path", TEXT),
- Column("device", TEXT),
- Column("md5", TEXT),
+ Column("version", TEXT, "Kernel version"),
+ Column("arguments", TEXT, "Kernel arguments"),
+ Column("path", TEXT, "Kernel path"),
+ Column("device", TEXT, "Kernel device identifier"),
+ Column("md5", TEXT, "MD5 hash of Kernel"),
])
implementation("system/kernel_info@genKernelInfo")
table_name("last")
description("System logins and logouts.")
schema([
- Column("username", TEXT),
- Column("tty", TEXT),
+ Column("username", TEXT, "Entry username"),
+ Column("tty", TEXT, "Entry terminal"),
Column("pid", INTEGER, "Process (or thread) ID"),
- Column("type", INTEGER),
- Column("time", INTEGER),
- Column("host", TEXT),
+ Column("type", INTEGER, "Entry type, according to ut_type types (utmp.h)"),
+ Column("time", INTEGER, "Entry timestamp"),
+ Column("host", TEXT, "Entry hostname"),
])
implementation("last@genLastAccess")
Column("name", TEXT, "Module name"),
Column("size", TEXT, "Size of module content"),
Column("used_by", TEXT, "Module reverse dependencies"),
- Column("status", TEXT),
- Column("address", TEXT),
+ Column("status", TEXT, "Kernel module status"),
+ Column("address", TEXT, "Kernel module address"),
])
implementation("kernel_modules@genKernelModules")
table_name("mounts")
description("System mounted devices and filesystems (not process specific).")
schema([
- Column("device", TEXT),
- Column("device_alias", TEXT),
- Column("path", TEXT),
- Column("type", TEXT),
- Column("blocks_size", BIGINT),
- Column("blocks", BIGINT),
- Column("blocks_free", BIGINT),
- Column("blocks_available", BIGINT),
- Column("inodes", BIGINT),
- Column("inodes_free", BIGINT),
- Column("flags", TEXT),
+ Column("device", TEXT, "Mounted device"),
+ Column("device_alias", TEXT, "Mounted device alias"),
+ Column("path", TEXT, "Mounted device path"),
+ Column("type", TEXT, "Mounted device type"),
+ Column("blocks_size", BIGINT, "Block size in bytes"),
+ Column("blocks", BIGINT, "Mounted device used blocks"),
+ Column("blocks_free", BIGINT, "Mounted device free blocks"),
+ Column("blocks_available", BIGINT, "Mounted device available blocks"),
+ Column("inodes", BIGINT, "Mounted device used inodes"),
+ Column("inodes_free", BIGINT, "Mounted device free inodes"),
+ Column("flags", TEXT, "Mounted device flags"),
])
implementation("mounts@genMounts")
description("Track time, action changes to /etc/passwd.")
schema([
Column("target_path", TEXT, "The path changed"),
- Column("time", TEXT, "Time of the change"),
Column("action", TEXT, "Change action (UPDATE, REMOVE, etc)"),
Column("transaction_id", BIGINT, "ID used during bulk update"),
+ Column("time", BIGINT, "Time of the change"),
])
attributes(event_subscriber=True)
implementation("passwd_changes@passwd_changes::genTable")
table_name("pci_devices")
description("PCI devices active on the host system.")
schema([
- Column("pci_slot", TEXT),
- Column("pci_class", TEXT),
- Column("driver", TEXT),
- Column("vendor", TEXT),
- Column("vendor_id", TEXT),
- Column("model", TEXT),
- Column("model_id", TEXT),
+ Column("pci_slot", TEXT, "PCI Device used slot"),
+ Column("pci_class", TEXT, "PCI Device class"),
+ Column("driver", TEXT, "PCI Device used driver"),
+ Column("vendor", TEXT, "PCI Device vendor"),
+ Column("vendor_id", TEXT, "PCI Device vendor identifier"),
+ Column("model", TEXT, "PCI Device model"),
+ Column("model_id", TEXT, "PCI Device model identifier"),
# Optional columns
- #Column("subsystem", TEXT),
- #Column("express", INTEGER),
- #Column("thunderbolt", INTEGER),
- #Column("removable", INTEGER),
+ #Column("subsystem", TEXT, "PCI Device subsystem"),
+ #Column("express", INTEGER, "1 If PCI device is express else 0"),
+ #Column("thunderbolt", INTEGER, "1 If PCI device is thunderbolt else 0"),
+ #Column("removable", INTEGER, "1 If PCI device is removable else 0"),
])
implementation("pci_devices@genPCIDevices")
Column("device", TEXT, "MA:MI Major/minor device ID"),
Column("inode", INTEGER, "Mapped path inode, 0 means uninitialized (BSS)"),
Column("path", TEXT, "Path to mapped file or mapped type"),
- Column("pseudo", INTEGER, "1 if path is a pseudo path, else 0"),
+ Column("pseudo", INTEGER, "1 If path is a pseudo path, else 0"),
])
implementation("processes@genProcessMemoryMap")
examples([
table_name("smbios_tables")
description("BIOS (DMI) structure common details and content.")
schema([
- Column("number", INTEGER),
- Column("type", INTEGER),
- Column("description", TEXT),
- Column("handle", INTEGER),
- Column("header_size", INTEGER),
- Column("size", INTEGER),
- Column("md5", TEXT),
+ Column("number", INTEGER, "Table entry number"),
+ Column("type", INTEGER, "Table entry type"),
+ Column("description", TEXT, "Table entry description"),
+ Column("handle", INTEGER, "Table entry handle"),
+ Column("header_size", INTEGER, "Header size in bytes"),
+ Column("size", INTEGER, "Table entry size in bytes"),
+ Column("md5", TEXT, "MD5 hash of table entry"),
])
implementation("system/smbios_tables@genSMBIOSTables")
table_name("suid_bin")
description("suid binaries in common locations.")
schema([
- Column("path", TEXT),
- Column("username", TEXT),
- Column("groupname", TEXT),
- Column("permissions", TEXT),
+ Column("path", TEXT, "Binary path"),
+ Column("username", TEXT, "Binary owner username"),
+ Column("groupname", TEXT, "Binary owner group"),
+ Column("permissions", TEXT, "Binary permissions"),
])
implementation("suid_bin@genSuidBin")
--- /dev/null
+table_name("uptime")
+description("Track time passed since last boot.")
+schema([
+ Column("days", INTEGER, "Days of uptime"),
+ Column("hours", INTEGER, "Hours of uptime"),
+ Column("minutes", INTEGER, "Minutes of uptime"),
+ Column("seconds", INTEGER, "Seconds of uptime"),
+ Column("total_seconds", BIGINT, "Total uptime seconds"),
+])
+implementation("system/uptime@genUptime")
table_name("usb_devices")
description("USB devices that are actively plugged into the host system.")
schema([
- Column("usb_address", INTEGER),
- Column("usb_port", INTEGER),
- Column("vendor", TEXT),
- Column("vendor_id", TEXT),
- Column("model", TEXT),
- Column("model_id", TEXT),
- Column("serial", TEXT),
- Column("removable", INTEGER),
+ Column("usb_address", INTEGER, "USB Device used address"),
+ Column("usb_port", INTEGER, "USB Device used port"),
+ Column("vendor", TEXT, "USB Device vendor string"),
+ Column("vendor_id", TEXT, "USB Device vendor identifier"),
+ Column("model", TEXT, "USB Device model string"),
+ Column("model_id", TEXT, "USB Device model identifier"),
+ Column("serial", TEXT, "USB Device serial connection"),
+ Column("removable", INTEGER, "1 If USB device is removable else 0"),
])
implementation("usb_devices@genUSBDevices")
Column("gid", BIGINT, "Group ID (unsigned)"),
Column("uid_signed", BIGINT, "User ID as int64 signed (Apple)"),
Column("gid_signed", BIGINT, "Default group ID as int64 signed (Apple)"),
- Column("username", TEXT),
+ Column("username", TEXT, "Username"),
Column("description", TEXT, "Optional user description"),
Column("directory", TEXT, "User's home directory"),
Column("shell", TEXT, "User's configured default shell"),
Column("mtime", BIGINT, "Last modification time"),
Column("ctime", BIGINT, "Creation time"),
Column("hard_links", INTEGER, "Number of hard links"),
- Column("is_file", INTEGER, "1 if a file node else 0"),
- Column("is_dir", INTEGER, "1 if a directory (not file) else 0"),
- Column("is_link", INTEGER, "1 if a symlink else 0"),
- Column("is_char", INTEGER, "1 if a character special device else 0"),
- Column("is_block", INTEGER, "1 if a block special device else 0"),
+ Column("is_file", INTEGER, "1 If a file node else 0"),
+ Column("is_dir", INTEGER, "1 If a directory (not file) else 0"),
+ Column("is_link", INTEGER, "1 If a symlink else 0"),
+ Column("is_char", INTEGER, "1 If a character special device else 0"),
+ Column("is_block", INTEGER, "1 If a block special device else 0"),
Column("pattern", TEXT, "A pattern which can be used to match file paths"),
])
attributes(utility=True)
"select * from file where path = '/etc/passwd'",
"select * from file where directory = '/etc/'",
"select * from file where pattern = '/etc/%'",
-])
\ No newline at end of file
+])
table_name("osquery_flags")
description("Configurable flags that modify osquery's behavior.")
schema([
- Column("name", TEXT),
- Column("type", TEXT),
- Column("description", TEXT),
- Column("default_value", TEXT),
- Column("value", TEXT),
- Column("shell_only", INTEGER),
+ Column("name", TEXT, "Flag name"),
+ Column("type", TEXT, "Flag type"),
+ Column("description", TEXT, "Flag description"),
+ Column("default_value", TEXT, "Flag default value"),
+ Column("value", TEXT, "Flag value"),
+ Column("shell_only", INTEGER, "Is the flag shell only?"),
])
attributes(utility=True)
implementation("osquery@genOsqueryFlags")
Column("registry", TEXT, "Name of the osquery registry"),
Column("name", TEXT, "Name of the plugin item"),
Column("owner_uuid", INTEGER, "Extension route UUID (0 for core)"),
- Column("internal", INTEGER, "1 if the plugin is internal else 0"),
- Column("active", INTEGER, "1 if this plugin is active else 0"),
+ Column("internal", INTEGER, "1 If the plugin is internal else 0"),
+ Column("active", INTEGER, "1 If this plugin is active else 0"),
])
attributes(utility=True)
implementation("osquery@genOsqueryRegistry")
table_name("time")
-description("Track current time in the system.")
+description("Track current date and time in the system.")
schema([
- Column("hour", INTEGER),
- Column("minutes", INTEGER),
- Column("seconds", INTEGER),
+ Column("weekday", TEXT, "Current weekday in the system"),
+ Column("year", INTEGER, "Current year in the system"),
+ Column("month", INTEGER, "Current month in the system"),
+ Column("day", INTEGER, "Current day in the system"),
+ Column("hour", INTEGER, "Current hour in the system"),
+ Column("minutes", INTEGER, "Current minutes in the system"),
+ Column("seconds", INTEGER, "Current seconds in the system"),
+ Column("unix_time", INTEGER, "Current unix time in the system"),
+ Column("timestamp", TEXT, "Current timestamp in the system"),
+ Column("iso_8601", TEXT, "Current time (iso format) in the system"),
])
attributes(utility=True)
implementation("time@genTime")
return table_name in blacklist if blacklist else False
+
def setup_templates(templates_path):
if not os.path.exists(templates_path):
templates_path = os.path.join(os.path.dirname(tables_path), "templates")
table.function = function
table.class_name = class_name
+ '''Check if the table has a subscriber attribute, if so, enforce time.'''
+ if "event_subscriber" in table.attributes:
+ columns = {}
+ # There is no dictionary comprehension on all supported platforms.
+ for column in table.schema:
+ if isinstance(column, Column):
+ columns[column.name] = column.type
+ if "time" not in columns:
+ print(lightred("Event subscriber: %s needs a 'time' column." % (
+ table.table_name)))
+ sys.exit(1)
+ if columns["time"] is not BIGINT:
+ print(lightred(
+ "Event subscriber: %s, 'time' column must be a %s type" % (
+ table.table_name, BIGINT)))
+ sys.exit(1)
+
def main(argc, argv):
parser = argparse.ArgumentParser("Generate C++ Table Plugin from specfile.")
"additional_monitoring" : {
"file_paths": {
"downloads": [
- "/tmp/osquery-tests/fstests-pattern/%%"
+ "/tmp/osquery-tests/fstree/%%"
]
}
},
// New, recommended file monitoring (top-level)
"file_paths": {
"downloads2": [
- "/tmp/osquery-tests/fstests-pattern/%%"
+ "/tmp/osquery-tests/fstree/%%"
],
"system_binaries": [
- "/tmp/osquery-tests/fstests-pattern/%",
- "/tmp/osquery-tests/fstests-pattern/deep11/%"
+ "/tmp/osquery-tests/fstree/%",
+ "/tmp/osquery-tests/fstree/deep11/%"
]
},
projects / platform / core / security / vist.git / commitdiff