From 0457d1eff74a1c9846ff3c94c8790ab02f5612a1 Mon Sep 17 00:00:00 2001 From: "sangwan.kwon" Date: Tue, 11 Nov 2014 11:17:28 -0500 Subject: [PATCH] Bump version to upstream-1.1.0 Added: arp Signed-off-by: sangwan.kwon --- include/osquery/config.h | 13 + include/osquery/core.h | 51 +++- include/osquery/database/db_handle.h | 12 - include/osquery/database/results.h | 22 +- include/osquery/flags.h | 44 +++- osquery/config/config.cpp | 32 +++ osquery/config/config_tests.cpp | 22 ++ osquery/core/init_osquery.cpp | 43 ++- osquery/core/system.cpp | 54 ++++ osquery/database/db_handle.cpp | 76 +++--- osquery/database/db_handle_tests.cpp | 9 - osquery/devtools/printer.cpp | 13 +- osquery/devtools/printer_tests.cpp | 34 +++ osquery/main/daemon.cpp | 6 + osquery/scheduler/scheduler.cpp | 1 - osquery/tables/CMakeLists.txt | 1 + osquery/tables/base.h | 2 +- osquery/tables/networking/linux/arp_cache.cpp | 70 +++++ osquery/tables/networking/linux/inet_diag.h | 137 ++++++++++ osquery/tables/networking/linux/port_inode.cpp | 32 ++- osquery/tables/networking/utils.cpp | 33 ++- osquery/tables/networking/utils.h | 12 +- osquery/tables/specs/blacklist | 2 + osquery/tables/specs/linux/kernel_modules.table | 10 +- osquery/tables/specs/linux/mounts.table | 17 -- osquery/tables/specs/linux/port_inode.table | 10 +- osquery/tables/specs/linux/socket_inode.table | 4 +- osquery/tables/specs/x/arp_cache.table | 8 + osquery/tables/specs/x/bash_history.table | 8 +- osquery/tables/specs/x/crontab.table | 21 +- osquery/tables/specs/x/etc_hosts.table | 5 +- osquery/tables/specs/x/groups.table | 4 +- osquery/tables/specs/x/last.table | 12 +- osquery/tables/specs/x/mounts.table | 15 ++ osquery/tables/specs/x/passwd_changes.table | 9 +- osquery/tables/specs/x/process_envs.table | 13 +- osquery/tables/specs/x/process_open_files.table | 18 +- osquery/tables/specs/x/processes.table | 32 +-- osquery/tables/specs/x/routes.table | 13 - osquery/tables/specs/x/suid_bin.table | 8 +- osquery/tables/specs/x/time.table | 6 +- osquery/tables/specs/x/users.table | 12 +- osquery/tables/system/last.cpp | 2 +- osquery/tables/system/linux/groups.cpp | 6 +- osquery/tables/system/linux/mounts.cpp | 20 +- osquery/tables/system/linux/processes.cpp | 36 +-- osquery/tables/system/suid_bin.cpp | 97 ++++--- osquery/tables/templates/blacklist.cpp.in | 7 + osquery/tables/templates/default.cpp.in | 197 ++++++++++++++ osquery/tables/utility/time.cpp | 8 +- packaging/osquery.spec | 2 +- tools/gentable.py | 332 +++++++----------------- 52 files changed, 1129 insertions(+), 524 deletions(-) create mode 100644 osquery/tables/networking/linux/arp_cache.cpp create mode 100644 osquery/tables/networking/linux/inet_diag.h delete mode 100644 osquery/tables/specs/linux/mounts.table create mode 100644 osquery/tables/specs/x/arp_cache.table create mode 100644 osquery/tables/specs/x/mounts.table delete mode 100644 osquery/tables/specs/x/routes.table create mode 100644 osquery/tables/templates/blacklist.cpp.in create mode 100644 osquery/tables/templates/default.cpp.in diff --git a/include/osquery/config.h b/include/osquery/config.h index e663ae2..0f1ac02 100644 --- a/include/osquery/config.h +++ b/include/osquery/config.h @@ -103,6 +103,19 @@ class Config { */ std::vector getScheduledQueries(); + /** + * @brief Calculate a splayed integer based on a variable splay percentage + * + * The value of splayPercent must be between 1 and 100. If it's not, the + * value of original will be returned. + * + * @param original The original value to be modified + * @param splayPercent The percent in which to splay the original value by + * + * @return The modified version of original + */ + static int splayValue(int original, int splayPercent); + private: /** * @brief Default constructor. diff --git a/include/osquery/core.h b/include/osquery/core.h index db68da1..116467a 100644 --- a/include/osquery/core.h +++ b/include/osquery/core.h @@ -11,17 +11,19 @@ #include "osquery/database/results.h" +#ifndef STR +#define STR_OF(x) #x +#define STR(x) STR_OF(x) +#endif + namespace osquery { /** * @brief The version of osquery */ extern const std::string kVersion; + /// Use a macro for the version literal, set the kVersion symbol in the library. -#ifndef STR -#define STR_OF(x) #x -#define STR(x) STR_OF(x) -#endif #define OSQUERY_VERSION STR(OSQUERY_BUILD_VERSION) /** @@ -146,4 +148,45 @@ int getUnixTime(); * @return a vector of strings representing the path of all home directories */ std::vector getHomeDirectories(); + +/** + * @brief Inline helper function for use with utf8StringSize + */ +template +inline size_t incUtf8StringIterator(_Iterator1& it, const _Iterator2& last) { + if (it == last) + return 0; + unsigned char c; + size_t res = 1; + for (++it; last != it; ++it, ++res) { + c = *it; + if (!(c & 0x80) || ((c & 0xC0) == 0xC0)) + break; + } + + return res; +} + +/** + * @brief Get the length of a UTF-8 string + * + * @param str The UTF-8 string + * + * @return the length of the string + */ +inline size_t utf8StringSize(const std::string& str) { + size_t res = 0; + std::string::const_iterator it = str.begin(); + for (; it != str.end(); incUtf8StringIterator(it, str.end())) + res++; + + return res; +} + +/** + * @brief Create a pid file + * + * @return A status object indicating the success or failure of the operation + */ +Status createPidFile(); } diff --git a/include/osquery/database/db_handle.h b/include/osquery/database/db_handle.h index 95d700c..dc460ed 100644 --- a/include/osquery/database/db_handle.h +++ b/include/osquery/database/db_handle.h @@ -80,15 +80,6 @@ class DBHandle { static std::shared_ptr getInstance(); /** - * @brief Getter for the status of the operations required to open the - * database - * - * @return an instance of osquery::Status which indicates the success or - * failure of connecting to RocksDB - */ - Status getStatus(); - - /** * @brief Helper method which can be used to get a raw pointer to the * underlying RocksDB database handle * @@ -231,9 +222,6 @@ class DBHandle { /// The database handle rocksdb::DB* db_; - /// The status code that is generated while attempting to connect to RocksDB - rocksdb::Status status_; - /// Column family descriptors which are used to connect to RocksDB std::vector column_families_; diff --git a/include/osquery/database/results.h b/include/osquery/database/results.h index 55f9b5e..db5af01 100644 --- a/include/osquery/database/results.h +++ b/include/osquery/database/results.h @@ -8,6 +8,7 @@ #include #include +#include #include "osquery/status.h" @@ -18,12 +19,31 @@ namespace osquery { ///////////////////////////////////////////////////////////////////////////// /** + * @brief The SQLite type affinities are available as macros + * + * Type affinities: TEXT, INTEGER, BIGINT + * + * You can represent any data that can be lexically casted to a string. + * Using the type affinity names helps table developers understand the data + * types they are storing, and more importantly how they are treated at query + * time. + */ +#define TEXT(x) std::string(x) +#define INTEGER(x) boost::lexical_cast(x) +#define BIGINT(x) boost::lexical_cast(x) + +/** + * @brief A variant type for the SQLite type affinities. + */ +typedef std::string RowData; + +/** * @brief A single row from a database query * * Row is a simple map where individual column names are keys, which map to * the Row's respective value */ -typedef std::map Row; +typedef std::map Row; /** * @brief Serialize a Row into a property tree diff --git a/include/osquery/flags.h b/include/osquery/flags.h index 92ad91e..e81d739 100644 --- a/include/osquery/flags.h +++ b/include/osquery/flags.h @@ -2,8 +2,6 @@ #pragma once -#include - #define STRIP_FLAG_HELP 1 #include @@ -24,9 +22,15 @@ class Flag { /* * @brief the instance accessor, but also can register flag data. * + * The accessor is mostly needless. The static instance and registration of + * flag data requires the accessor wrapper. + * * @param name The 'name' or the options switch data. * @param value The default value for this flag. * @param desc The description printed to the screen during help. + * @param shell_only Only print flag help when using `OSQUERY_TOOL_SHELL`. + * + * @return A mostly needless flag instance. */ static Flag& get(const std::string& name = "", const std::string& value = "", @@ -45,7 +49,7 @@ class Flag { * @param name The 'name' or the options switch data. * @param value The default value for this flag. * @param desc The description printed to the screen during help. - * @param shell_only Restrict this flag to the shell. + * @param shell_only Restrict this flag to the shell help output. */ void add(const std::string& name, const std::string& value, @@ -67,14 +71,20 @@ class Flag { std::map flags() { return flags_; } /// The public flags instance, usable when parsing `--help` for the shell. std::map shellFlags() { return shell_flags_; } - static void print_flags(const std::map flags) { + + /* + * @brief Print help-style output to stdout for a given flag set. + * + * @param flags A flag set (usually generated from Flag::flags). + */ + static void printFlags(const std::map flags) { for (const auto& flag : flags) { fprintf(stdout, - " --%s, --%s=VALUE\n %s (default: %s)\n", + " --%s, --%s=%s\n %s\n", flag.first.c_str(), flag.first.c_str(), - flag.second.second.c_str(), - flag.second.first.c_str()); + flag.second.first.c_str(), + flag.second.second.c_str()); } } @@ -97,12 +107,20 @@ class Flag { #define DEFINE_osquery_flag(type, name, value, desc) \ DEFINE_##type(name, value, desc); \ namespace flag_##name { \ - Flag flag = Flag::get(#name, #value, #desc); \ + Flag flag = Flag::get(#name, #value, desc); \ } -/// Wrapper to bypass osquery help output -#define DEFINE_shell_flag(type, name, value, desc) \ - DEFINE_##type(name, value, desc); \ - namespace flag_##name { \ - Flag flag = Flag::get(#name, #value, #desc, true); \ +/* + * @brief A duplicate of DEFINE_osquery_flag except the help output will only + * show when using OSQUERY_TOOL_SHELL (osqueryi). + * + * @param type The `_type` symbol portion of the gflags define. + * @param name The name symbol passed to gflags' `DEFINE_type`. + * @param value The default value, use a C++ literal. + * @param desc A string literal used for help display. + */ +#define DEFINE_shell_flag(type, name, value, desc) \ + DEFINE_##type(name, value, desc); \ + namespace flag_##name { \ + Flag flag = Flag::get(#name, #value, desc, true); \ } diff --git a/osquery/config/config.cpp b/osquery/config/config.cpp index a30ad6f..8ad4c95 100644 --- a/osquery/config/config.cpp +++ b/osquery/config/config.cpp @@ -28,6 +28,12 @@ DEFINE_osquery_flag(string, "filesystem", "The config mechanism to retrieve config content via."); +/// The percent to splay config times by +DEFINE_osquery_flag(int32, + schedule_splay_percent, + 10, + "The percent to splay config times by"); + boost::shared_mutex rw_lock; std::shared_ptr Config::getInstance() { @@ -41,6 +47,15 @@ Config::Config() { auto s = Config::genConfig(conf); if (!s.ok()) { LOG(ERROR) << "error retrieving config: " << s.toString(); + } else { + for (auto& q : conf.scheduledQueries) { + auto old_interval = q.interval; + auto new_interval = + splayValue(old_interval, FLAGS_schedule_splay_percent); + LOG(INFO) << "Changing the interval for " << q.name << " from " + << old_interval << " to " << new_interval; + q.interval = new_interval; + } } cfg_ = conf; } @@ -82,4 +97,21 @@ std::vector Config::getScheduledQueries() { boost::shared_lock lock(rw_lock); return cfg_.scheduledQueries; } + +int Config::splayValue(int original, int splayPercent) { + if (splayPercent <= 0 || splayPercent > 100) { + return original; + } + + float percent_to_modify_by = (float)splayPercent / 100; + int possible_difference = original * percent_to_modify_by; + int max_value = original + possible_difference; + int min_value = original - possible_difference; + + if (max_value == min_value) { + return max_value; + } + + return (rand() % (max_value - min_value)) + min_value; +} } diff --git a/osquery/config/config_tests.cpp b/osquery/config/config_tests.cpp index be45027..4b36026 100644 --- a/osquery/config/config_tests.cpp +++ b/osquery/config/config_tests.cpp @@ -49,6 +49,28 @@ TEST_F(ConfigTests, test_plugin) { EXPECT_EQ(p.first.toString(), "OK"); EXPECT_EQ(p.second, "foobar"); } + +TEST_F(ConfigTests, test_splay) { + auto val1 = Config::splayValue(100, 10); + EXPECT_GE(val1, 90); + EXPECT_LE(val1, 110); + + auto val2 = Config::splayValue(100, 10); + EXPECT_GE(val2, 90); + EXPECT_LE(val2, 110); + + EXPECT_NE(val1, val2); + + auto val3 = Config::splayValue(10, 0); + EXPECT_EQ(val3, 10); + + auto val4 = Config::splayValue(100, 1); + EXPECT_GE(val2, 99); + EXPECT_LE(val2, 101); + + auto val5 = Config::splayValue(1, 10); + EXPECT_EQ(val5, 1); +} } int main(int argc, char* argv[]) { diff --git a/osquery/core/init_osquery.cpp b/osquery/core/init_osquery.cpp index fee7f52..e8810be 100644 --- a/osquery/core/init_osquery.cpp +++ b/osquery/core/init_osquery.cpp @@ -16,10 +16,22 @@ const std::string kDescription = "relational database"; const std::string kEpilog = "osquery project page ."; +DEFINE_osquery_flag(bool, debug, false, "Enable debug messages."); + +DEFINE_osquery_flag(bool, + verbose_debug, + false, + "Enable verbose debug messages.") + +DEFINE_osquery_flag(bool, + disable_logging, + false, + "Disable ERROR/INFO logging."); + DEFINE_osquery_flag(string, osquery_log_dir, "/var/log/osquery/", - "Directory to store results logging."); + "Directory to store ERROR/INFO and results logging."); static const char* basename(const char* filename) { const char* sep = strrchr(filename, '/'); @@ -45,12 +57,12 @@ void initOsquery(int argc, char* argv[], int tool) { "The following options control the osquery " "daemon and shell.\n\n"); - Flag::print_flags(Flag::get().flags()); + Flag::printFlags(Flag::get().flags()); if (tool == OSQUERY_TOOL_SHELL) { // Print shell flags. fprintf(stdout, "\nThe following options control the osquery shell.\n\n"); - Flag::print_flags(Flag::get().shellFlags()); + Flag::printFlags(Flag::get().shellFlags()); } fprintf(stdout, "\n%s\n", kEpilog.c_str()); @@ -61,7 +73,7 @@ void initOsquery(int argc, char* argv[], int tool) { FLAGS_alsologtostderr = true; FLAGS_logbufsecs = 0; // flush the log buffer immediately FLAGS_stop_logging_if_full_disk = true; - FLAGS_max_log_size = 1024; // max size for individual log file is 1GB + FLAGS_max_log_size = 10; // max size for individual log file is 10MB // Set version string from CMake build __GFLAGS_NAMESPACE::SetVersionString(OSQUERY_VERSION); @@ -69,10 +81,33 @@ void initOsquery(int argc, char* argv[], int tool) { // Let gflags parse the non-help options/flags. __GFLAGS_NAMESPACE::ParseCommandLineFlags(&argc, &argv, false); + // The log dir is used for glogging and the filesystem results logs. if (isWritable(FLAGS_osquery_log_dir.c_str()).ok()) { FLAGS_log_dir = FLAGS_osquery_log_dir; } + if (FLAGS_verbose_debug) { + // Turn verbosity up to 1. + // Do log DEBUG, INFO, WARNING, ERROR to their log files. + // Do log the above and verbose=1 to stderr. + FLAGS_debug = true; + FLAGS_v = 1; + } + + if (!FLAGS_debug) { + // Do NOT log INFO, WARNING, ERROR to stderr. + // Do log to their log files. + FLAGS_minloglevel = 0; // INFO + FLAGS_alsologtostderr = false; + } + + if (FLAGS_disable_logging) { + // Do log ERROR to stderr. + // Do NOT log INFO, WARNING, ERROR to their log files. + FLAGS_logtostderr = true; + FLAGS_minloglevel = 2; // ERROR + } + google::InitGoogleLogging(argv[0]); osquery::InitRegistry::get().run(); } diff --git a/osquery/core/system.cpp b/osquery/core/system.cpp index 538c55c..dc928e1 100644 --- a/osquery/core/system.cpp +++ b/osquery/core/system.cpp @@ -3,6 +3,9 @@ #include "osquery/core.h" #include "osquery/database/db_handle.h" +#include +#include + #include #include #include @@ -11,12 +14,19 @@ #include +#include "osquery/filesystem.h" #include "osquery/sql.h" namespace fs = boost::filesystem; namespace osquery { +/// The path to the pidfile for osqueryd +DEFINE_osquery_flag(string, + pidfile, + "/var/osquery/osqueryd.pidfile", + "The path to the pidfile for osqueryd."); + std::string getHostname() { char hostname[256]; memset(hostname, 0, 255); @@ -79,4 +89,48 @@ std::vector getHomeDirectories() { } return results; } + +Status createPidFile() { + // check if pidfile exists + auto exists = pathExists(FLAGS_pidfile); + if (exists.ok()) { + // if it exists, check if that pid is running + std::string content; + auto read_status = readFile(FLAGS_pidfile, content); + if (!read_status.ok()) { + return Status(1, "Could not read pidfile: " + read_status.toString()); + } + int osqueryd_pid; + try { + osqueryd_pid = stoi(content); + } catch (const std::invalid_argument& e) { + return Status( + 1, + std::string("Could not convert pidfile content to an int: ") + + std::string(e.what())); + } + + if (kill(osqueryd_pid, 0) == 0) { + // if the pid is running, return an "error" status + return Status(1, "osqueryd is already running"); + } else if (errno == ESRCH) { + // if the pid isn't running, overwrite the pidfile + boost::filesystem::remove(FLAGS_pidfile); + goto write_new_pidfile; + } else { + return Status( + 1, + std::string( + "An unknown error occured checking if the pid is running: ") + + std::string(strerror(errno))); + } + } else { + // if it doesn't exist, write a pid file and return a "success" status + write_new_pidfile: + auto current_pid = boost::lexical_cast(getpid()); + LOG(INFO) << "Writing pid (" << current_pid << ") to " << FLAGS_pidfile; + auto write_status = writeTextFile(FLAGS_pidfile, current_pid, 0755); + return write_status; + } +} } diff --git a/osquery/database/db_handle.cpp b/osquery/database/db_handle.cpp index 481bc1f..57dbfd7 100644 --- a/osquery/database/db_handle.cpp +++ b/osquery/database/db_handle.cpp @@ -28,7 +28,7 @@ const std::vector kDomains = {kConfigurations, kQueries, kEvents}; DEFINE_osquery_flag(string, db_path, - "/tmp/rocksdb-osquery", + "/var/osquery/osquery.db", "If using a disk-based backing store, specify a path."); DEFINE_osquery_flag(bool, @@ -45,10 +45,13 @@ DBHandle::DBHandle(const std::string& path, bool in_memory) { options_.create_missing_column_families = true; if (in_memory) { - // Remove when upgrading to RocksDB 3.2 - // Replace with: + // Remove when MemEnv is included in librocksdb // options_.env = rocksdb::NewMemEnv(rocksdb::Env::Default()); - throw std::domain_error("Requires RocksDB 3.3 https://fburl.com/27350299"); + throw std::runtime_error("Requires MemEnv"); + } + + if (pathExists(path).ok() && !isWritable(path).ok()) { + throw std::runtime_error("Cannot write to RocksDB path: " + path); } column_families_.push_back(rocksdb::ColumnFamilyDescriptor( @@ -59,25 +62,17 @@ DBHandle::DBHandle(const std::string& path, bool in_memory) { cf_name, rocksdb::ColumnFamilyOptions())); } - if (pathExists(path).ok() && !isWritable(path).ok()) { - throw std::domain_error("Cannot write to RocksDB path: " + path); + auto s = rocksdb::DB::Open(options_, path, column_families_, &handles_, &db_); + if (!s.ok()) { + throw std::runtime_error(s.ToString()); } - - status_ = - rocksdb::DB::Open(options_, path, column_families_, &handles_, &db_); } DBHandle::~DBHandle() { for (auto handle : handles_) { - if (handle != nullptr) { - delete handle; - handle = nullptr; - } - } - if (db_ != nullptr) { - delete db_; - db_ = nullptr; + delete handle; } + delete db_; } ///////////////////////////////////////////////////////////////////////////// @@ -88,8 +83,6 @@ std::shared_ptr DBHandle::getInstance() { } std::shared_ptr DBHandle::getInstanceInMemory() { - // Remove when upgrading to RocksDB 3.3 - throw std::domain_error("Requires RocksDB 3.3 https://fburl.com/27350299"); return getInstance("", true); } @@ -108,18 +101,18 @@ std::shared_ptr DBHandle::getInstance(const std::string& path, // getters and setters ///////////////////////////////////////////////////////////////////////////// -osquery::Status DBHandle::getStatus() { - return Status(status_.code(), status_.ToString()); -} - rocksdb::DB* DBHandle::getDB() { return db_; } rocksdb::ColumnFamilyHandle* DBHandle::getHandleForColumnFamily( const std::string& cf) { - for (int i = 0; i < kDomains.size(); i++) { - if (kDomains[i] == cf) { - return handles_[i]; + try { + for (int i = 0; i < kDomains.size(); i++) { + if (kDomains[i] == cf) { + return handles_[i]; + } } + } catch (const std::exception& e) { + // pass through and return nullptr } return nullptr; } @@ -131,30 +124,45 @@ rocksdb::ColumnFamilyHandle* DBHandle::getHandleForColumnFamily( osquery::Status DBHandle::Get(const std::string& domain, const std::string& key, std::string& value) { - auto s = getDB()->Get( - rocksdb::ReadOptions(), getHandleForColumnFamily(domain), key, &value); + auto cfh = getHandleForColumnFamily(domain); + if (cfh == nullptr) { + return Status(1, "Could not get column family for " + domain); + } + auto s = getDB()->Get(rocksdb::ReadOptions(), cfh, key, &value); return Status(s.code(), s.ToString()); } osquery::Status DBHandle::Put(const std::string& domain, const std::string& key, const std::string& value) { - auto s = getDB()->Put( - rocksdb::WriteOptions(), getHandleForColumnFamily(domain), key, value); + auto cfh = getHandleForColumnFamily(domain); + if (cfh == nullptr) { + return Status(1, "Could not get column family for " + domain); + } + auto s = getDB()->Put(rocksdb::WriteOptions(), cfh, key, value); return Status(s.code(), s.ToString()); } osquery::Status DBHandle::Delete(const std::string& domain, const std::string& key) { - auto s = getDB()->Delete( - rocksdb::WriteOptions(), getHandleForColumnFamily(domain), key); + auto cfh = getHandleForColumnFamily(domain); + if (cfh == nullptr) { + return Status(1, "Could not get column family for " + domain); + } + auto s = getDB()->Delete(rocksdb::WriteOptions(), cfh, key); return Status(s.code(), s.ToString()); } osquery::Status DBHandle::Scan(const std::string& domain, std::vector& results) { - auto it = getDB()->NewIterator(rocksdb::ReadOptions(), - getHandleForColumnFamily(domain)); + auto cfh = getHandleForColumnFamily(domain); + if (cfh == nullptr) { + return Status(1, "Could not get column family for " + domain); + } + auto it = getDB()->NewIterator(rocksdb::ReadOptions(), cfh); + if (it == nullptr) { + return Status(1, "Could not get iterator for " + domain); + } for (it->SeekToFirst(); it->Valid(); it->Next()) { results.push_back(it->key().ToString()); } diff --git a/osquery/database/db_handle_tests.cpp b/osquery/database/db_handle_tests.cpp index b992624..b3fadea 100644 --- a/osquery/database/db_handle_tests.cpp +++ b/osquery/database/db_handle_tests.cpp @@ -31,18 +31,9 @@ class DBHandleTests : public testing::Test { std::shared_ptr db; }; -TEST_F(DBHandleTests, test_create_new_database_on_disk) { - EXPECT_TRUE(db->getStatus().ok()); - EXPECT_EQ(db->getStatus().toString(), "OK"); -} - TEST_F(DBHandleTests, test_singleton_on_disk) { auto db1 = DBHandle::getInstance(); - EXPECT_TRUE(db1->getStatus().ok()); - EXPECT_EQ(db1->getStatus().toString(), "OK"); auto db2 = DBHandle::getInstance(); - EXPECT_TRUE(db2->getStatus().ok()); - EXPECT_EQ(db2->getStatus().toString(), "OK"); EXPECT_EQ(db1, db2); } diff --git a/osquery/devtools/printer.cpp b/osquery/devtools/printer.cpp index 78d2827..c19447f 100644 --- a/osquery/devtools/printer.cpp +++ b/osquery/devtools/printer.cpp @@ -7,6 +7,8 @@ #include +#include "osquery/core.h" + namespace osquery { std::string beautify(const QueryData& q, @@ -62,7 +64,7 @@ std::string generateHeader(const std::map& lengths, header << " "; header << each; try { - for (int i = 0; i < (lengths.at(each) - each.size() + 1); ++i) { + for (int i = 0; i < (lengths.at(each) - utf8StringSize(each) + 1); ++i) { header << " "; } } catch (const std::out_of_range& e) { @@ -86,7 +88,7 @@ std::string generateRow(const Row& r, row << " "; try { row << r.at(each); - for (int i = 0; i < (lengths.at(each) - r.at(each).size() + 1); ++i) { + for (int i = 0; i < (lengths.at(each) - utf8StringSize(r.at(each)) + 1); ++i) { row << " "; } } catch (const std::out_of_range& e) { @@ -116,14 +118,15 @@ std::map computeQueryDataLengths(const QueryData& q) { } for (const auto& it : q.front()) { - results[it.first] = it.first.size(); + results[it.first] = utf8StringSize(it.first); } for (const auto& row : q) { for (const auto& it : row) { try { - if (it.second.size() > results[it.first]) { - results[it.first] = it.second.size(); + auto s = utf8StringSize(it.second); + if (s > results[it.first]) { + results[it.first] = s; } } catch (const std::out_of_range& e) { LOG(ERROR) << "Error retrieving the \"" << it.first diff --git a/osquery/devtools/printer_tests.cpp b/osquery/devtools/printer_tests.cpp index fdd59fd..f668119 100644 --- a/osquery/devtools/printer_tests.cpp +++ b/osquery/devtools/printer_tests.cpp @@ -105,6 +105,40 @@ TEST_F(PrinterTests, test_beautify) { )"; EXPECT_EQ(result, expected); } + +TEST_F(PrinterTests, test_unicode) { + QueryData augmented = { + { + {"name", "Mike Jones"}, + {"age", "39"}, + {"favorite_food", "mac and cheese"}, + {"lucky_number", "1"}, + }, + { + {"name", "Àlex Smith"}, + {"age", "44"}, + {"favorite_food", "peanut butter and jelly"}, + {"lucky_number", "2"}, + }, + { + {"name", "Doctor Who"}, + {"age", "2000"}, + {"favorite_food", "fish sticks and custard"}, + {"lucky_number", "11"}, + }, + }; + auto result = beautify(augmented, order); + std::string expected = R"( ++------------+------+-------------------------+--------------+ +| name | age | favorite_food | lucky_number | ++------------+------+-------------------------+--------------+ +| Mike Jones | 39 | mac and cheese | 1 | +| Àlex Smith | 44 | peanut butter and jelly | 2 | +| Doctor Who | 2000 | fish sticks and custard | 11 | ++------------+------+-------------------------+--------------+ +)"; + EXPECT_EQ(result, expected); +} } int main(int argc, char* argv[]) { diff --git a/osquery/main/daemon.cpp b/osquery/main/daemon.cpp index f28329a..1be4657 100644 --- a/osquery/main/daemon.cpp +++ b/osquery/main/daemon.cpp @@ -15,6 +15,12 @@ int main(int argc, char* argv[]) { osquery::initOsquery(argc, argv, osquery::OSQUERY_TOOL_DAEMON); + auto pid_status = osquery::createPidFile(); + if (!pid_status.ok()) { + LOG(ERROR) << "Could not create osquery pidfile: " << pid_status.toString(); + ::exit(-1); + } + try { osquery::DBHandle::getInstance(); } catch (std::exception& e) { diff --git a/osquery/scheduler/scheduler.cpp b/osquery/scheduler/scheduler.cpp index d06a588..0664a60 100644 --- a/osquery/scheduler/scheduler.cpp +++ b/osquery/scheduler/scheduler.cpp @@ -60,7 +60,6 @@ std::string getHostIdentifier(std::string hostIdFlag, void launchQueries(const std::vector& queries, const int64_t& second) { - LOG(INFO) << "launchQueries: " << second; for (const auto& q : queries) { if (second % q.interval == 0) { LOG(INFO) << "executing query: " << q.query; diff --git a/osquery/tables/CMakeLists.txt b/osquery/tables/CMakeLists.txt index c34979b..5fbc1fc 100644 --- a/osquery/tables/CMakeLists.txt +++ b/osquery/tables/CMakeLists.txt @@ -2,6 +2,7 @@ ADD_OSQUERY_LIBRARY(osquery_tables_linux events/linux/passwd_changes.cpp networking/linux/routes.cpp networking/linux/socket_inode.cpp networking/linux/port_inode.cpp + networking/linux/arp_cache.cpp system/linux/kernel_modules.cpp system/linux/processes.cpp system/linux/users.cpp diff --git a/osquery/tables/base.h b/osquery/tables/base.h index 76f2328..ffc6581 100644 --- a/osquery/tables/base.h +++ b/osquery/tables/base.h @@ -2,8 +2,8 @@ #pragma once -#include #include +#include namespace osquery { namespace tables { diff --git a/osquery/tables/networking/linux/arp_cache.cpp b/osquery/tables/networking/linux/arp_cache.cpp new file mode 100644 index 0000000..b2e94ff --- /dev/null +++ b/osquery/tables/networking/linux/arp_cache.cpp @@ -0,0 +1,70 @@ +// Copyright 2004-present Facebook. All Rights Reserved. + +#include + +#include +#include + +#include "osquery/database.h" +#include "osquery/filesystem.h" + +namespace osquery { +namespace tables { + +const std::string kLinuxArpTable = "/proc/net/arp"; + +QueryData genArpCache() { + QueryData results; + + boost::filesystem::path arp_path = kLinuxArpTable; + if (!osquery::isReadable(arp_path).ok()) { + VLOG(1) << "Cannot read arp table."; + return results; + } + + std::ifstream fd(arp_path.string(), std::ios::in | std::ios::binary); + std::string line; + + if (fd.fail() || fd.eof()) { + VLOG(1) << "Empty or failed arp table."; + return results; + } + + // Read the header line. + std::getline(fd, line, '\n'); + while (!(fd.fail() || fd.eof())) { + std::getline(fd, line, '\n'); + + // IP address, HW type, Flags, HW address, Mask Device + std::vector fields; + boost::split(fields, line, boost::is_any_of(" "), boost::token_compress_on); + for (auto& f : fields) { + // Inline trim each split. + boost::trim(f); + } + + if (fields.size() != 6) { + // An unhandled error case. + continue; + } + + Row r; + r["address"] = fields[0]; + r["mac"] = fields[3]; + r["interface"] = fields[5]; + + // Note: it's also possible to detect publish entries (ATF_PUB). + if (fields[2] == "0x6") { + // The string representation of ATF_COM | ATF_PERM. + r["permanent"] = "1"; + } else { + r["permanent"] = "0"; + } + + results.push_back(r); + } + + return results; +} +} +} diff --git a/osquery/tables/networking/linux/inet_diag.h b/osquery/tables/networking/linux/inet_diag.h new file mode 100644 index 0000000..bbde90f --- /dev/null +++ b/osquery/tables/networking/linux/inet_diag.h @@ -0,0 +1,137 @@ +#ifndef _UAPI_INET_DIAG_H_ +#define _UAPI_INET_DIAG_H_ + +#include + +/* Just some random number */ +#define TCPDIAG_GETSOCK 18 +#define DCCPDIAG_GETSOCK 19 + +#define INET_DIAG_GETSOCK_MAX 24 + +/* Socket identity */ +struct inet_diag_sockid { + __be16 idiag_sport; + __be16 idiag_dport; + __be32 idiag_src[4]; + __be32 idiag_dst[4]; + __u32 idiag_if; + __u32 idiag_cookie[2]; +#define INET_DIAG_NOCOOKIE (~0U) +}; + +/* Request structure */ + +struct inet_diag_req { + __u8 idiag_family; /* Family of addresses. */ + __u8 idiag_src_len; + __u8 idiag_dst_len; + __u8 idiag_ext; /* Query extended information */ + + struct inet_diag_sockid id; + + __u32 idiag_states; /* States to dump */ + __u32 idiag_dbs; /* Tables to dump (NI) */ +}; + +struct inet_diag_req_v2 { + __u8 sdiag_family; + __u8 sdiag_protocol; + __u8 idiag_ext; + __u8 pad; + __u32 idiag_states; + struct inet_diag_sockid id; +}; + +enum { + INET_DIAG_REQ_NONE, + INET_DIAG_REQ_BYTECODE, +}; + +#define INET_DIAG_REQ_MAX INET_DIAG_REQ_BYTECODE + +/* Bytecode is sequence of 4 byte commands followed by variable arguments. + * All the commands identified by "code" are conditional jumps forward: + * to offset cc+"yes" or to offset cc+"no". "yes" is supposed to be + * length of the command and its arguments. + */ + +struct inet_diag_bc_op { + unsigned char code; + unsigned char yes; + unsigned short no; +}; + +enum { + INET_DIAG_BC_NOP, + INET_DIAG_BC_JMP, + INET_DIAG_BC_S_GE, + INET_DIAG_BC_S_LE, + INET_DIAG_BC_D_GE, + INET_DIAG_BC_D_LE, + INET_DIAG_BC_AUTO, + INET_DIAG_BC_S_COND, + INET_DIAG_BC_D_COND, +}; + +struct inet_diag_hostcond { + __u8 family; + __u8 prefix_len; + int port; + __be32 addr[0]; +}; + +/* Base info structure. It contains socket identity (addrs/ports/cookie) + * and, alas, the information shown by netstat. */ +struct inet_diag_msg { + __u8 idiag_family; + __u8 idiag_state; + __u8 idiag_timer; + __u8 idiag_retrans; + + struct inet_diag_sockid id; + + __u32 idiag_expires; + __u32 idiag_rqueue; + __u32 idiag_wqueue; + __u32 idiag_uid; + __u32 idiag_inode; +}; + +/* Extensions */ + +enum { + INET_DIAG_NONE, + INET_DIAG_MEMINFO, + INET_DIAG_INFO, + INET_DIAG_VEGASINFO, + INET_DIAG_CONG, + INET_DIAG_TOS, + INET_DIAG_TCLASS, + INET_DIAG_SKMEMINFO, + INET_DIAG_SHUTDOWN, +}; + +#define INET_DIAG_MAX INET_DIAG_SHUTDOWN + + +/* INET_DIAG_MEM */ + +struct inet_diag_meminfo { + __u32 idiag_rmem; + __u32 idiag_wmem; + __u32 idiag_fmem; + __u32 idiag_tmem; +}; + +/* INET_DIAG_VEGASINFO */ + +struct tcpvegas_info { + __u32 tcpv_enabled; + __u32 tcpv_rttcnt; + __u32 tcpv_rtt; + __u32 tcpv_minrtt; +}; + + +#endif /* _UAPI_INET_DIAG_H_ */ diff --git a/osquery/tables/networking/linux/port_inode.cpp b/osquery/tables/networking/linux/port_inode.cpp index 524ce4b..b56a0f1 100644 --- a/osquery/tables/networking/linux/port_inode.cpp +++ b/osquery/tables/networking/linux/port_inode.cpp @@ -1,28 +1,38 @@ -#include -#include -#include -#include -#include +// Copyright 2004-present Facebook. All Rights Reserved. + +#include + +#include #include -#include #include #include -#include #include -#include -#include -#include +#include #include -#include +#include +#include +#include +#include +#include +#include #include #include #include + #include "osquery/core.h" #include "osquery/database.h" #include "osquery/logger.h" +// From uapi/linux/sock_diag.h +// From linux/sock_diag.h (<= 3.6) +#ifndef SOCK_DIAG_BY_FAMILY +#define SOCK_DIAG_BY_FAMILY 20 +#endif + +#include "inet_diag.h" + namespace osquery { namespace tables { diff --git a/osquery/tables/networking/utils.cpp b/osquery/tables/networking/utils.cpp index be1a09b..d7478a0 100644 --- a/osquery/tables/networking/utils.cpp +++ b/osquery/tables/networking/utils.cpp @@ -17,7 +17,10 @@ #include "osquery/tables/networking/utils.h" -std::string canonical_ip_address(const struct sockaddr *in) { +namespace osquery { +namespace tables { + +std::string ipAsString(const struct sockaddr *in) { char dst[INET6_ADDRSTRLEN]; memset(dst, 0, sizeof(dst)); void *in_addr; @@ -67,7 +70,21 @@ int netmaskFromIP(const struct sockaddr *in) { return mask; } -std::string canonical_mac_address(const struct ifaddrs *addr) { +std::string macAsString(const char *addr) { + std::stringstream mac; + + for (size_t i = 0; i < 6; i++) { + mac << std::hex << std::setfill('0') << std::setw(2); + mac << (int)((uint8_t)addr[i]); + if (i != 5) { + mac << ":"; + } + } + + return mac.str(); +} + +std::string macAsString(const struct ifaddrs *addr) { std::stringstream mac; if (addr->ifa_addr == NULL) { @@ -87,7 +104,10 @@ std::string canonical_mac_address(const struct ifaddrs *addr) { for (size_t i = 0; i < 6; i++) { mac << std::hex << std::setfill('0') << std::setw(2); - mac << (int)((uint8_t)ifr.ifr_hwaddr.sa_data[i]) << ":"; + mac << (int)((uint8_t)ifr.ifr_hwaddr.sa_data[i]); + if (i != 5) { + mac << ":"; + } } #else struct sockaddr_dl *sdl; @@ -101,9 +121,14 @@ std::string canonical_mac_address(const struct ifaddrs *addr) { for (size_t i = 0; i < sdl->sdl_alen; i++) { mac << std::hex << std::setfill('0') << std::setw(2); // Prevent char sign extension. - mac << (int)((uint8_t)sdl->sdl_data[i + sdl->sdl_nlen]) << ":"; + mac << (int)((uint8_t)sdl->sdl_data[i + sdl->sdl_nlen]); + if (i != 5) { + mac << ":"; + } } #endif return mac.str(); } +} +} diff --git a/osquery/tables/networking/utils.h b/osquery/tables/networking/utils.h index cf59128..6e6f2a7 100644 --- a/osquery/tables/networking/utils.h +++ b/osquery/tables/networking/utils.h @@ -7,7 +7,13 @@ #include #include +namespace osquery { +namespace tables { + // Return a string representation for an IPv4/IPv6 struct. -std::string canonical_ip_address(const struct sockaddr *in); -std::string canonical_mac_address(const struct ifaddrs *addr); -int netmaskFromIP(const struct sockaddr *in); \ No newline at end of file +std::string ipAsString(const struct sockaddr *in); +std::string macAsString(const struct ifaddrs *addr); +std::string macAsString(const char *addr); +int netmaskFromIP(const struct sockaddr *in); +} +} diff --git a/osquery/tables/specs/blacklist b/osquery/tables/specs/blacklist index 1027de0..c8fa3c8 100644 --- a/osquery/tables/specs/blacklist +++ b/osquery/tables/specs/blacklist @@ -4,3 +4,5 @@ quarantine suid_bin +port_inode +socket_inode diff --git a/osquery/tables/specs/linux/kernel_modules.table b/osquery/tables/specs/linux/kernel_modules.table index 40bdd59..9ce31df 100644 --- a/osquery/tables/specs/linux/kernel_modules.table +++ b/osquery/tables/specs/linux/kernel_modules.table @@ -1,9 +1,9 @@ table_name("kernel_modules") schema([ - Column(name="name", type="std::string"), - Column(name="size", type="std::string"), - Column(name="used_by", type="std::string"), - Column(name="status", type="std::string"), - Column(name="address", type="std::string"), + Column("name", TEXT), + Column("size", TEXT), + Column("used_by", TEXT), + Column("status", TEXT), + Column("address", TEXT), ]) implementation("kernel_modules@genKernelModules") diff --git a/osquery/tables/specs/linux/mounts.table b/osquery/tables/specs/linux/mounts.table deleted file mode 100644 index 6d3937d..0000000 --- a/osquery/tables/specs/linux/mounts.table +++ /dev/null @@ -1,17 +0,0 @@ -table_name("mounts") -schema([ - Column(name="fsname", type="std::string"), - Column(name="fsname_real", type="std::string"), - Column(name="path", type="std::string"), - Column(name="type", type="std::string"), - Column(name="opts", type="std::string"), - Column(name="freq", type="int"), - Column(name="passno", type="int"), - Column(name="block_size", type="long long int"), - Column(name="blocks", type="long long int"), - Column(name="blocks_free", type="long long int"), - Column(name="blocks_avail", type="long long int"), - Column(name="inodes", type="long long int"), - Column(name="inodes_free", type="long long int"), -]) -implementation("mounts@genMounts") diff --git a/osquery/tables/specs/linux/port_inode.table b/osquery/tables/specs/linux/port_inode.table index 98b6020..60bf1b8 100644 --- a/osquery/tables/specs/linux/port_inode.table +++ b/osquery/tables/specs/linux/port_inode.table @@ -1,10 +1,10 @@ table_name("port_inode") schema([ - Column(name="local_port", type="std::string"), - Column(name="remote_port", type="std::string"), - Column(name="local_ip", type="std::string"), - Column(name="remote_ip", type="std::string"), - Column(name="inode", type="std::string"), + Column("local_port", TEXT), + Column("remote_port", TEXT), + Column("local_ip", TEXT), + Column("remote_ip", TEXT), + Column("inode", TEXT), ]) implementation("osquery/tables/networking/linux/port_inode@genPortInode") diff --git a/osquery/tables/specs/linux/socket_inode.table b/osquery/tables/specs/linux/socket_inode.table index 8854a1d..92b156a 100644 --- a/osquery/tables/specs/linux/socket_inode.table +++ b/osquery/tables/specs/linux/socket_inode.table @@ -1,6 +1,6 @@ table_name("socket_inode") schema([ - Column(name="pid", type="std::string"), - Column(name="inode", type="std::string"), + Column("pid", TEXT), + Column("inode", TEXT), ]) implementation("osquery/tables/networking/linux/socket_inode@genSocketInode") diff --git a/osquery/tables/specs/x/arp_cache.table b/osquery/tables/specs/x/arp_cache.table new file mode 100644 index 0000000..ebe57b8 --- /dev/null +++ b/osquery/tables/specs/x/arp_cache.table @@ -0,0 +1,8 @@ +table_name("arp_cache") +schema([ + Column("address", TEXT), + Column("mac", TEXT), + Column("interface", TEXT), + Column("permanent", TEXT, "1 for true, 0 for false"), +]) +implementation("linux/arp_cache,darwin/routes@genArpCache") diff --git a/osquery/tables/specs/x/bash_history.table b/osquery/tables/specs/x/bash_history.table index 1138393..973ca04 100644 --- a/osquery/tables/specs/x/bash_history.table +++ b/osquery/tables/specs/x/bash_history.table @@ -1,7 +1,9 @@ table_name("bash_history") +description("A line-delimited (command) table of per-user .bash_history data.") schema([ - Column(name="username", type="std::string"), - Column(name="command", type="std::string"), - Column(name="history_file", type="std::string"), + Column("username", TEXT), + Column("command", TEXT), + Column("history_file", TEXT, "Path to the .bash_history for this user"), + ForeignKey(column="username", table="users"), ]) implementation("bash_history@genBashHistory") diff --git a/osquery/tables/specs/x/crontab.table b/osquery/tables/specs/x/crontab.table index c9177a6..e12278a 100644 --- a/osquery/tables/specs/x/crontab.table +++ b/osquery/tables/specs/x/crontab.table @@ -1,14 +1,13 @@ table_name("crontab") - +description("Line parsed values from system and user cron/tab.") schema([ - Column(name="event", type="std::string"), - Column(name="minute", type="std::string"), - Column(name="hour", type="std::string"), - Column(name="day_of_month", type="std::string"), - Column(name="month", type="std::string"), - Column(name="day_of_week", type="std::string"), - Column(name="command", type="std::string"), - Column(name="path", type="std::string"), + Column("event", TEXT, "The job @event name (rare)"), + Column("minute", TEXT), + Column("hour", TEXT), + Column("day_of_month", TEXT), + Column("month", TEXT), + Column("day_of_week", TEXT), + Column("command", TEXT, "Raw command string"), + Column("path", TEXT, "File parsed"), ]) - -implementation("crontab@genCronTab") \ No newline at end of file +implementation("crontab@genCronTab") diff --git a/osquery/tables/specs/x/etc_hosts.table b/osquery/tables/specs/x/etc_hosts.table index 24ad625..deb3a32 100644 --- a/osquery/tables/specs/x/etc_hosts.table +++ b/osquery/tables/specs/x/etc_hosts.table @@ -1,6 +1,7 @@ table_name("etc_hosts") +description("Line-parsed /etc/hosts.") schema([ - Column(name="address", type="std::string"), - Column(name="hostnames", type="std::string"), + Column("address", TEXT), + Column("hostnames", TEXT, "Raw hosts mapping"), ]) implementation("etc_hosts@genEtcHosts") diff --git a/osquery/tables/specs/x/groups.table b/osquery/tables/specs/x/groups.table index ed25f29..96fccd1 100644 --- a/osquery/tables/specs/x/groups.table +++ b/osquery/tables/specs/x/groups.table @@ -1,6 +1,6 @@ table_name("groups") schema([ - Column(name="gid", type="long long int"), - Column(name="name", type="std::string"), + Column("gid", BIGINT), + Column("groupname", TEXT), ]) implementation("groups@genGroups") diff --git a/osquery/tables/specs/x/last.table b/osquery/tables/specs/x/last.table index 838727e..1dea38f 100644 --- a/osquery/tables/specs/x/last.table +++ b/osquery/tables/specs/x/last.table @@ -1,10 +1,10 @@ table_name("last") schema([ - Column(name="login", type="std::string"), - Column(name="tty", type="std::string"), - Column(name="pid", type="int"), - Column(name="type", type="int"), - Column(name="time", type="int"), - Column(name="host", type="std::string"), + Column("username", TEXT), + Column("tty", TEXT), + Column("pid", INTEGER), + Column("type", INTEGER), + Column("time", INTEGER), + Column("host", TEXT), ]) implementation("last@genLastAccess") diff --git a/osquery/tables/specs/x/mounts.table b/osquery/tables/specs/x/mounts.table new file mode 100644 index 0000000..da5e274 --- /dev/null +++ b/osquery/tables/specs/x/mounts.table @@ -0,0 +1,15 @@ +table_name("mounts") +schema([ + Column("device", TEXT), + Column("device_alias", TEXT), + Column("path", TEXT), + Column("type", TEXT), + Column("blocks_size", BIGINT), + Column("blocks", BIGINT), + Column("blocks_free", BIGINT), + Column("blocks_available", BIGINT), + Column("inodes", BIGINT), + Column("inodes_free", BIGINT), + Column("flags", TEXT), +]) +implementation("mounts@genMounts") diff --git a/osquery/tables/specs/x/passwd_changes.table b/osquery/tables/specs/x/passwd_changes.table index 80e3810..f6d89b0 100644 --- a/osquery/tables/specs/x/passwd_changes.table +++ b/osquery/tables/specs/x/passwd_changes.table @@ -1,8 +1,9 @@ table_name("passwd_changes") +description("Mostly an example use of events.") schema([ - Column(name="target_path", type="std::string"), - Column(name="time", type="std::string"), - Column(name="action", type="std::string"), - Column(name="transaction_id", type="std::string"), + Column("target_path", TEXT, "The path changed"), + Column("time", TEXT), + Column("action", TEXT, "Change action (UPDATE, REMOVE, etc)"), + Column("transaction_id", BIGINT, "ID used during bulk update"), ]) implementation("passwd_changes@PasswdChangesEventSubscriber::genTable") diff --git a/osquery/tables/specs/x/process_envs.table b/osquery/tables/specs/x/process_envs.table index c70769e..4b864a0 100644 --- a/osquery/tables/specs/x/process_envs.table +++ b/osquery/tables/specs/x/process_envs.table @@ -1,9 +1,12 @@ table_name("process_envs") +description("A key/value table of environment variables for each process.") schema([ - Column(name="pid", type="int"), - Column(name="name", type="std::string"), - Column(name="path", type="std::string"), - Column(name="key", type="std::string"), - Column(name="value", type="std::string"), + Column("pid", INTEGER), + Column("name", TEXT, "Process name"), + Column("path", TEXT, "Process path"), + Column("key", TEXT, "Environment variable name"), + Column("value", TEXT, "Environment variable value"), + ForeignKey(column="pid", table="processes"), + ForeignKey(column="pid", table="process_open_files"), ]) implementation("system/processes@genProcessEnvs") diff --git a/osquery/tables/specs/x/process_open_files.table b/osquery/tables/specs/x/process_open_files.table index f891817..29414d3 100644 --- a/osquery/tables/specs/x/process_open_files.table +++ b/osquery/tables/specs/x/process_open_files.table @@ -1,13 +1,13 @@ table_name("process_open_files") schema([ - Column(name="pid", type="int"), - Column(name="name", type="std::string"), - Column(name="path", type="std::string"), - Column(name="file_type", type="std::string"), - Column(name="local_path", type="std::string"), - Column(name="local_host", type="std::string"), - Column(name="local_port", type="std::string"), - Column(name="remote_host", type="std::string"), - Column(name="remote_port", type="std::string"), + Column("pid", INTEGER), + Column("name", TEXT), + Column("path", TEXT), + Column("file_type", TEXT), + Column("local_path", TEXT), + Column("local_host", TEXT), + Column("local_port", TEXT), + Column("remote_host", TEXT), + Column("remote_port", TEXT), ]) implementation("system/processes@genProcessOpenFiles") diff --git a/osquery/tables/specs/x/processes.table b/osquery/tables/specs/x/processes.table index 077709a..cc61c65 100644 --- a/osquery/tables/specs/x/processes.table +++ b/osquery/tables/specs/x/processes.table @@ -1,20 +1,20 @@ table_name("processes") schema([ - Column(name="name", type="std::string"), - Column(name="path", type="std::string"), - Column(name="cmdline", type="std::string"), - Column(name="pid", type="int"), - Column(name="uid", type="long long int"), - Column(name="gid", type="long long int"), - Column(name="euid", type="long long int"), - Column(name="egid", type="long long int"), - Column(name="on_disk", type="std::string"), - Column(name="wired_size", type="std::string"), - Column(name="resident_size", type="std::string"), - Column(name="phys_footprint", type="std::string"), - Column(name="user_time", type="std::string"), - Column(name="system_time", type="std::string"), - Column(name="start_time", type="std::string"), - Column(name="parent", type="int"), + Column("name", TEXT), + Column("path", TEXT), + Column("cmdline", TEXT), + Column("pid", INTEGER), + Column("uid", BIGINT), + Column("gid", BIGINT), + Column("euid", BIGINT), + Column("egid", BIGINT), + Column("on_disk", TEXT), + Column("wired_size", TEXT), + Column("resident_size", TEXT), + Column("phys_footprint", TEXT), + Column("user_time", TEXT), + Column("system_time", TEXT), + Column("start_time", TEXT), + Column("parent", INTEGER), ]) implementation("system/processes@genProcesses") diff --git a/osquery/tables/specs/x/routes.table b/osquery/tables/specs/x/routes.table deleted file mode 100644 index 61dbd2b..0000000 --- a/osquery/tables/specs/x/routes.table +++ /dev/null @@ -1,13 +0,0 @@ -table_name("routes") -schema([ - Column(name="destination", type="std::string"), - Column(name="netmask", type="std::string"), - Column(name="gateway", type="std::string"), - Column(name="source", type="std::string"), - Column(name="flags", type="int"), - Column(name="interface", type="std::string"), - Column(name="mtu", type="int"), - Column(name="metric", type="int"), - Column(name="type", type="std::string"), -]) -implementation("networking/routes@genRoutes") diff --git a/osquery/tables/specs/x/suid_bin.table b/osquery/tables/specs/x/suid_bin.table index 4e1523d..0403abf 100644 --- a/osquery/tables/specs/x/suid_bin.table +++ b/osquery/tables/specs/x/suid_bin.table @@ -1,8 +1,8 @@ table_name("suid_bin") schema([ - Column(name="path", type="std::string"), - Column(name="unix_user", type="std::string"), - Column(name="unix_group", type="std::string"), - Column(name="permissions", type="std::string"), + Column("path", TEXT), + Column("username", TEXT), + Column("groupname", TEXT), + Column("permissions", TEXT), ]) implementation("suid_bin@genSuidBin") diff --git a/osquery/tables/specs/x/time.table b/osquery/tables/specs/x/time.table index 0ad3ebd..2a98ec2 100644 --- a/osquery/tables/specs/x/time.table +++ b/osquery/tables/specs/x/time.table @@ -1,7 +1,7 @@ table_name("time") schema([ - Column(name="hour", type="int"), - Column(name="minutes", type="int"), - Column(name="seconds", type="int"), + Column("hour", INTEGER), + Column("minutes", INTEGER), + Column("seconds", INTEGER), ]) implementation("time@genTime") diff --git a/osquery/tables/specs/x/users.table b/osquery/tables/specs/x/users.table index 8ebb7fb..dc1718e 100644 --- a/osquery/tables/specs/x/users.table +++ b/osquery/tables/specs/x/users.table @@ -1,10 +1,10 @@ table_name("users") schema([ - Column(name="uid", type="long long int"), - Column(name="gid", type="long long int"), - Column(name="username", type="std::string"), - Column(name="description", type="std::string"), - Column(name="directory", type="std::string"), - Column(name="shell", type="std::string"), + Column("uid", BIGINT), + Column("gid", BIGINT), + Column("username", TEXT), + Column("description", TEXT), + Column("directory", TEXT), + Column("shell", TEXT), ]) implementation("users@genUsers") diff --git a/osquery/tables/system/last.cpp b/osquery/tables/system/last.cpp index 75b86e6..f5daab7 100644 --- a/osquery/tables/system/last.cpp +++ b/osquery/tables/system/last.cpp @@ -29,7 +29,7 @@ QueryData genLastAccess() { #endif Row r; - r["login"] = std::string(ut->ut_user); + r["username"] = std::string(ut->ut_user); r["tty"] = std::string(ut->ut_line); r["pid"] = boost::lexical_cast(ut->ut_pid); r["type"] = boost::lexical_cast(ut->ut_type); diff --git a/osquery/tables/system/linux/groups.cpp b/osquery/tables/system/linux/groups.cpp index 19c3a3c..c764d9a 100644 --- a/osquery/tables/system/linux/groups.cpp +++ b/osquery/tables/system/linux/groups.cpp @@ -5,8 +5,6 @@ #include #include -#include - #include "osquery/core.h" #include "osquery/database.h" @@ -28,8 +26,8 @@ QueryData genGroups() { if (std::find(groups_in.begin(), groups_in.end(), grp->gr_gid) == groups_in.end()) { Row r; - r["gid"] = boost::lexical_cast(grp->gr_gid); - r["name"] = std::string(grp->gr_name); + r["gid"] = INTEGER(grp->gr_gid); + r["groupname"] = TEXT(grp->gr_name); results.push_back(r); groups_in.insert(grp->gr_gid); } diff --git a/osquery/tables/system/linux/mounts.cpp b/osquery/tables/system/linux/mounts.cpp index 31f71db..7ba6009 100644 --- a/osquery/tables/system/linux/mounts.cpp +++ b/osquery/tables/system/linux/mounts.cpp @@ -23,21 +23,19 @@ QueryData genMounts() { while ((ent = getmntent(mounts))) { Row r; - r["fsname"] = std::string(ent->mnt_fsname); - r["fsname_real"] = std::string( + r["device"] = std::string(ent->mnt_fsname); + r["device_alias"] = std::string( realpath(ent->mnt_fsname, real_path) ? real_path : ent->mnt_fsname); r["path"] = std::string(ent->mnt_dir); r["type"] = std::string(ent->mnt_type); - r["opts"] = std::string(ent->mnt_opts); - r["freq"] = boost::lexical_cast(ent->mnt_freq); - r["passno"] = boost::lexical_cast(ent->mnt_passno); + r["flags"] = std::string(ent->mnt_opts); if (!statfs(ent->mnt_dir, &st)) { - r["block_size"] = boost::lexical_cast(st.f_bsize); - r["blocks"] = boost::lexical_cast(st.f_blocks); - r["blocks_free"] = boost::lexical_cast(st.f_bfree); - r["blocks_avail"] = boost::lexical_cast(st.f_bavail); - r["inodes"] = boost::lexical_cast(st.f_files); - r["inodes_free"] = boost::lexical_cast(st.f_ffree); + r["blocks_size"] = BIGINT(st.f_bsize); + r["blocks"] = BIGINT(st.f_blocks); + r["blocks_free"] = BIGINT(st.f_bfree); + r["blocks_available"] = BIGINT(st.f_bavail); + r["inodes"] = BIGINT(st.f_files); + r["inodes_free"] = BIGINT(st.f_ffree); } results.push_back(r); diff --git a/osquery/tables/system/linux/processes.cpp b/osquery/tables/system/linux/processes.cpp index 2de6d6b..1a15ea6 100644 --- a/osquery/tables/system/linux/processes.cpp +++ b/osquery/tables/system/linux/processes.cpp @@ -82,7 +82,7 @@ std::string proc_link(const proc_t* proc_info) { std::map proc_env(const proc_t* proc_info) { std::map env; - std::string attr = osquery::tables::proc_attr("environ", proc_info); + std::string attr = proc_attr("environ", proc_info); std::string buf; std::ifstream fd(attr, std::ios::in | std::ios::binary); @@ -108,6 +108,12 @@ void standard_freeproc(proc_t* p) { if (!p) { // in case p is NULL return; } + +#ifdef PROC_EDITCMDLCVT + freeproc(p); + return; +#endif + // ptrs are after strings to avoid copying memory when building them. // so free is called on the address of the address of strvec[0]. if (p->cmdline) { @@ -129,29 +135,25 @@ QueryData genProcesses() { while ((proc_info = readproc(proc, NULL))) { Row r; - r["pid"] = boost::lexical_cast(proc_info->tid); - r["uid"] = boost::lexical_cast((unsigned int)proc_info->ruid); - r["gid"] = boost::lexical_cast((unsigned int)proc_info->rgid); - r["euid"] = boost::lexical_cast((unsigned int)proc_info->euid); - r["egid"] = boost::lexical_cast((unsigned int)proc_info->egid); + r["pid"] = INTEGER(proc_info->tid); + r["uid"] = BIGINT((unsigned int)proc_info->ruid); + r["gid"] = BIGINT((unsigned int)proc_info->rgid); + r["euid"] = BIGINT((unsigned int)proc_info->euid); + r["egid"] = BIGINT((unsigned int)proc_info->egid); r["name"] = proc_name(proc_info); r["cmdline"] = proc_cmdline(proc_info); r["path"] = proc_link(proc_info); r["on_disk"] = osquery::pathExists(r["path"]).toString(); - r["resident_size"] = boost::lexical_cast(proc_info->vm_rss); - r["phys_footprint"] = boost::lexical_cast(proc_info->vm_size); - r["user_time"] = boost::lexical_cast(proc_info->utime); - r["system_time"] = boost::lexical_cast(proc_info->stime); - r["start_time"] = boost::lexical_cast(proc_info->start_time); - r["parent"] = boost::lexical_cast(proc_info->ppid); + r["resident_size"] = INTEGER(proc_info->vm_rss); + r["phys_footprint"] = INTEGER(proc_info->vm_size); + r["user_time"] = INTEGER(proc_info->utime); + r["system_time"] = INTEGER(proc_info->stime); + r["start_time"] = INTEGER(proc_info->start_time); + r["parent"] = INTEGER(proc_info->ppid); results.push_back(r); -#ifdef PROC_EDITCMDLCVT - freeproc(proc_info); -#else standard_freeproc(proc_info); -#endif } closeproc(proc); @@ -171,7 +173,7 @@ QueryData genProcessEnvs() { auto env = proc_env(proc_info); for (auto itr = env.begin(); itr != env.end(); ++itr) { Row r; - r["pid"] = boost::lexical_cast(proc_info->tid); + r["pid"] = INTEGER(proc_info->tid); r["name"] = proc_name(proc_info); r["path"] = proc_link(proc_info); r["key"] = itr->first; diff --git a/osquery/tables/system/suid_bin.cpp b/osquery/tables/system/suid_bin.cpp index 55e7fd2..fa973b7 100644 --- a/osquery/tables/system/suid_bin.cpp +++ b/osquery/tables/system/suid_bin.cpp @@ -1,27 +1,80 @@ // Copyright 2004-present Facebook. All Rights Reserved. #include + #include #include #include -#include -#include + #include +#include +#include + #include -#include "osquery/database.h" -using std::string; -using boost::lexical_cast; +#include "osquery/database.h" namespace osquery { namespace tables { -QueryData genSuidBin() { +Status genBin(const boost::filesystem::path& path, + int perms, + QueryData& results) { + struct stat info; + // store user and group + if (stat(path.c_str(), &info) != 0) { + return Status(1, "stat failed"); + } + + // store path Row r; + r["path"] = path.string(); + + struct passwd *pw = getpwuid(info.st_uid); + struct group *gr = getgrgid(info.st_gid); + + // get user name + group + std::string user; + if (pw != nullptr) { + user = std::string(pw->pw_name); + } else { + user = boost::lexical_cast(info.st_uid); + } + + std::string group; + if (gr != nullptr) { + group = std::string(gr->gr_name); + } else { + group = boost::lexical_cast(info.st_gid); + } + + r["username"] = user; + r["groupname"] = group; + + r["permissions"] = ""; + if ((perms & 04000) == 04000) { + r["permissions"] += "S"; + } + + if ((perms & 02000) == 02000) { + r["permissions"] += "G"; + } + + results.push_back(r); + return Status(0, "OK"); +} + +QueryData genSuidBin() { QueryData results; - struct stat info; boost::system::error_code error; +#if defined(UBUNTU) + // When building on supported Ubuntu systems, boost may ABRT. + if (geteuid() != 0) { + return results; + } +#endif + boost::filesystem::recursive_directory_iterator it = boost::filesystem::recursive_directory_iterator( boost::filesystem::path("/"), error); @@ -35,32 +88,10 @@ QueryData genSuidBin() { while (it != end) { boost::filesystem::path path = *it; try { - if (boost::filesystem::is_regular_file(path) && - ((it.status().permissions() & 04000) == 04000 || - (it.status().permissions() & 02000) == 02000)) { - // store path - r["path"] = boost::lexical_cast(path); - - // store user and group - if (stat(path.c_str(), &info) == 0) { - struct passwd *pw = getpwuid(info.st_uid); - struct group *gr = getgrgid(info.st_gid); - // get user name - r["unix_user"] = pw ? boost::lexical_cast(pw->pw_name) - : boost::lexical_cast(info.st_uid); - // get group - r["unix_group"] = gr ? boost::lexical_cast(gr->gr_name) - : boost::lexical_cast(info.st_gid); - - // get permission - r["permissions"] = ""; - r["permissions"] += - (it.status().permissions() & 04000) == 04000 ? "S" : ""; - r["permissions"] += - (it.status().permissions() & 02000) == 02000 ? "G" : ""; - - results.push_back(r); - } + int perms = it.status().permissions(); + if (boost::filesystem::is_regular_file(path) && + ((perms & 04000) == 04000 || (perms & 02000) == 02000)) { + genBin(path, perms, results); } } catch (...) { // handle invalid files like /dev/fd/3 diff --git a/osquery/tables/templates/blacklist.cpp.in b/osquery/tables/templates/blacklist.cpp.in new file mode 100644 index 0000000..68a3f14 --- /dev/null +++ b/osquery/tables/templates/blacklist.cpp.in @@ -0,0 +1,7 @@ +// Copyright 2004-present Facebook. All Rights Reserved. + +/* +** This file is generated. Do not modify it manually! +*/ + +void __blacklisted_{{table_name}}() {} diff --git a/osquery/tables/templates/default.cpp.in b/osquery/tables/templates/default.cpp.in new file mode 100644 index 0000000..5014369 --- /dev/null +++ b/osquery/tables/templates/default.cpp.in @@ -0,0 +1,197 @@ +// Copyright 2004-present Facebook. All Rights Reserved. + +/* +** This file is generated. Do not modify it manually! +*/ + +#include +#include +#include + +#include + +#include "osquery/database.h" +#include "osquery/tables/base.h" +#include "osquery/registry/registry.h" + +namespace osquery { namespace tables { + +{% if class_name == "" %}\ +osquery::QueryData {{function}}(); +{% else %} +class {{class_name}} { + public: + static osquery::QueryData {{function}}(); +}; +{% endif %}\ + +struct sqlite3_{{table_name}} { + int n; +{% for col in schema %}\ + std::vector<{{col.type.type}}> xCol_{{col.name}}; +{% endfor %}\ +}; + +const std::string + sqlite3_{{table_name}}_create_table_statement = + "CREATE TABLE {{table_name}}(" + {% for col in schema %}\ + "{{col.name}} {{col.type.affinity}}\ +{% if not loop.last %}, {% endif %}" + {% endfor %}\ +")"; + +int {{table_name_cc}}Create( + sqlite3 *db, + void *pAux, + int argc, + const char *const *argv, + sqlite3_vtab **ppVtab, + char **pzErr +) { + return xCreate< + x_vtab, + sqlite3_{{table_name}} + >( + db, pAux, argc, argv, ppVtab, pzErr, + sqlite3_{{table_name}}_create_table_statement.c_str() + ); +} + +int {{table_name_cc}}Column( + sqlite3_vtab_cursor *cur, + sqlite3_context *ctx, + int col +) { + base_cursor *pCur = (base_cursor*)cur; + x_vtab *pVtab = + (x_vtab*)cur->pVtab; + + if(pCur->row >= 0 && pCur->row < pVtab->pContent->n) { + switch (col) { +{% for col in schema %}\ + // {{ col.name }} + case {{ loop.index0 }}: + if (pVtab->pContent->xCol_{{col.name}}.size() > pCur->row) { +{% if col.type.affinity == "TEXT" %}\ + sqlite3_result_text( + ctx, + (pVtab->pContent->xCol_{{col.name}}[pCur->row]).c_str(), + -1, + nullptr + ); +{% endif %}\ +{% if col.type.affinity == "INTEGER" %}\ + sqlite3_result_int( + ctx, + ({{col.type.type}})pVtab->pContent->xCol_{{col.name}}[pCur->row] + ); +{% endif %}\ +{% if col.type.affinity == "BIGINT" %}\ + sqlite3_result_int64( + ctx, + ({{col.type.type}})pVtab->pContent->xCol_{{col.name}}[pCur->row] + ); +{% endif %}\ + } + break; +{% endfor %}\ + } + } + return SQLITE_OK; +} + +int {{table_name_cc}}Filter( + sqlite3_vtab_cursor *pVtabCursor, + int idxNum, + const char *idxStr, + int argc, + sqlite3_value **argv +) { + base_cursor *pCur = (base_cursor *)pVtabCursor; + x_vtab *pVtab = + (x_vtab*)pVtabCursor->pVtab; + + pCur->row = 0; +{% for col in schema %}\ + pVtab->pContent->xCol_{{col.name}}.clear(); +{% endfor %}\ + +{% if class_name != "" %} + for (auto& row : osquery::tables::{{class_name}}::{{function}}()) { +{% else %} + for (auto& row : osquery::tables::{{function}}()) { +{% endif %} +{% for col in schema %}\ +{% if col.type.affinity == "TEXT" %}\ + pVtab->pContent->xCol_{{col.name}}.push_back(row["{{col.name}}"]); +{% endif %}\ +{% if col.type.affinity == "INTEGER" %}\ + try { + pVtab->pContent->xCol_{{col.name}}\ +.push_back(boost::lexical_cast<{{col.type.type}}>(row["{{col.name}}"])); + } catch (const boost::bad_lexical_cast& e) { + LOG(WARNING) << "Error casting {{col.name}} (" + << row["{{col.name}}"] << ") to {{col.type.type}}"; + pVtab->pContent->xCol_{{col.name}}.push_back(-1); + } +{% endif %}\ +{% if col.type.affinity == "BIGINT" %}\ + try { + pVtab->pContent->xCol_{{col.name}}\ +.push_back(boost::lexical_cast<{{col.type.type}}>(row["{{col.name}}"])); + } catch (const boost::bad_lexical_cast& e) { + LOG(WARNING) << "Error casting {{col.name}} (" + << row["{{col.name}}"] << ") to {{col.type.type}}"; + pVtab->pContent->xCol_{{col.name}}.push_back(-1); + } +{% endif %}\ +{% endfor %}\ + } + + pVtab->pContent->n = pVtab->pContent->xCol_{{schema[0].name}}.size(); + + return SQLITE_OK; +} + +static sqlite3_module {{table_name_cc}}Module = { + 0, + {{table_name_cc}}Create, + {{table_name_cc}}Create, + xBestIndex, + xDestroy>, + xDestroy>, + xOpen, + xClose, + {{table_name_cc}}Filter, + xNext, + xEof>, + {{table_name_cc}}Column, + xRowid, + 0, + 0, + 0, + 0, + 0, + 0, + 0, +}; + +class {{table_name_cc}}TablePlugin : public TablePlugin { +public: + {{table_name_cc}}TablePlugin() {} + + int attachVtable(sqlite3 *db) { + return sqlite3_attach_vtable( + db, "{{table_name}}", &{{table_name_cc}}Module); + } + + virtual ~{{table_name_cc}}TablePlugin() {} +}; + +REGISTER_TABLE( + "{{table_name}}", + std::make_shared<{{table_name_cc}}TablePlugin>() +); + +}} diff --git a/osquery/tables/utility/time.cpp b/osquery/tables/utility/time.cpp index d6b251a..1babd83 100644 --- a/osquery/tables/utility/time.cpp +++ b/osquery/tables/utility/time.cpp @@ -2,8 +2,6 @@ #include -#include - #include "osquery/database.h" namespace osquery { @@ -15,9 +13,9 @@ QueryData genTime() { Row r; time_t _time = time(0); struct tm* now = localtime(&_time); - r["hour"] = boost::lexical_cast(now->tm_hour); - r["minutes"] = boost::lexical_cast(now->tm_min); - r["seconds"] = boost::lexical_cast(now->tm_sec); + r["hour"] = INTEGER(now->tm_hour); + r["minutes"] = INTEGER(now->tm_min); + r["seconds"] = INTEGER(now->tm_sec); QueryData results; for (int i = 0; i < kNumCols; ++i) { results.push_back(r); diff --git a/packaging/osquery.spec b/packaging/osquery.spec index 84a9743..90b95eb 100644 --- a/packaging/osquery.spec +++ b/packaging/osquery.spec @@ -1,5 +1,5 @@ Name: osquery -Version: 1.0.4 +Version: 1.1.0 Release: 0 License: Apache-2.0 and GPLv2 Summary: A SQL powered operating system instrumentation, monitoring framework. diff --git a/tools/gentable.py b/tools/gentable.py index df66797..10081d3 100755 --- a/tools/gentable.py +++ b/tools/gentable.py @@ -18,232 +18,83 @@ DEVELOPING = False # the log format for the logging module LOG_FORMAT = "%(levelname)s [Line %(lineno)d]: %(message)s" -# BL_IMPL_TEMPLATE is the jinja template used to generate the virtual table -# implementation file when the table is blacklisted in ./osquery/tables/specs -BL_IMPL_TEMPLATE = """// Copyright 2004-present Facebook. All Rights Reserved. - -/* -** This file is generated. Do not modify it manually! -*/ - -void __blacklisted_{{table_name}}() {} - -""" - -# IMPL_TEMPLATE is the jinja template used to generate the virtual table -# implementation file -IMPL_TEMPLATE = """// Copyright 2004-present Facebook. All Rights Reserved. - -/* -** This file is generated. Do not modify it manually! -*/ - -#include -#include -#include - -#include - -#include "osquery/database.h" -#include "osquery/tables/base.h" -#include "osquery/registry/registry.h" - -namespace osquery { namespace tables { - -{% if class_name == "" %} -osquery::QueryData {{function}}(); -{% else %} -class {{class_name}} { - public: - static osquery::QueryData {{function}}(); -}; -{% endif %} - -struct sqlite3_{{table_name}} { - int n; -{% for col in schema %}\ - std::vector<{{col.type}}> {{col.name}}; -{% endfor %}\ -}; - -const std::string - sqlite3_{{table_name}}_create_table_statement = - "CREATE TABLE {{table_name}}(" - {% for col in schema %}\ - "{{col.name}} \ -{% if col.type == "std::string" %}VARCHAR{% endif %}\ -{% if col.type == "int" %}INTEGER{% endif %}\ -{% if col.type == "long long int" %}BIGINT{% endif %}\ -{% if not loop.last %}, {% endif %}" - {% endfor %}\ -")"; - -int {{table_name_cc}}Create( - sqlite3 *db, - void *pAux, - int argc, - const char *const *argv, - sqlite3_vtab **ppVtab, - char **pzErr -) { - return xCreate< - x_vtab, - sqlite3_{{table_name}} - >( - db, pAux, argc, argv, ppVtab, pzErr, - sqlite3_{{table_name}}_create_table_statement.c_str() - ); -} - -int {{table_name_cc}}Column( - sqlite3_vtab_cursor *cur, - sqlite3_context *ctx, - int col -) { - base_cursor *pCur = (base_cursor*)cur; - x_vtab *pVtab = - (x_vtab*)cur->pVtab; - - if(pCur->row >= 0 && pCur->row < pVtab->pContent->n) { - switch (col) { -{% for col in schema %}\ - // {{ col.name }} - case {{ loop.index0 }}: -{% if col.type == "std::string" %}\ - sqlite3_result_text( - ctx, - (pVtab->pContent->{{col.name}}[pCur->row]).c_str(), - -1, - nullptr - ); -{% endif %}\ -{% if col.type == "int" %}\ - sqlite3_result_int( - ctx, - (int)pVtab->pContent->{{col.name}}[pCur->row] - ); -{% endif %}\ -{% if col.type == "long long int" %}\ - sqlite3_result_int64( - ctx, - (long long int)pVtab->pContent->{{col.name}}[pCur->row] - ); -{% endif %}\ - break; -{% endfor %}\ - } - } - return SQLITE_OK; -} - -int {{table_name_cc}}Filter( - sqlite3_vtab_cursor *pVtabCursor, - int idxNum, - const char *idxStr, - int argc, - sqlite3_value **argv -) { - base_cursor *pCur = (base_cursor *)pVtabCursor; - x_vtab *pVtab = - (x_vtab*)pVtabCursor->pVtab; - - pCur->row = 0; -{% for col in schema %}\ - pVtab->pContent->{{col.name}}.clear(); -{% endfor %}\ - -{% if class_name != "" %} - for (auto& row : osquery::tables::{{class_name}}::{{function}}()) { -{% else %} - for (auto& row : osquery::tables::{{function}}()) { -{% endif %} -{% for col in schema %}\ -{% if col.type == "std::string" %}\ - pVtab->pContent->{{col.name}}.push_back(row["{{col.name}}"]); -{% endif %}\ -{% if col.type == "int" %}\ - try { - pVtab->pContent->{{col.name}}\ -.push_back(boost::lexical_cast(row["{{col.name}}"])); - } catch (const boost::bad_lexical_cast& e) { - LOG(WARNING) << "Error casting " << row["{{col.name}}"] << " to int"; - pVtab->pContent->{{col.name}}.push_back(-1); - } -{% endif %}\ -{% if col.type == "long long int" %}\ - try { - pVtab->pContent->{{col.name}}\ -.push_back(boost::lexical_cast(row["{{col.name}}"])); - } catch (const boost::bad_lexical_cast& e) { - LOG(WARNING) << "Error casting " << row["{{col.name}}"] << " to long long int"; - pVtab->pContent->{{col.name}}.push_back(-1); - } -{% endif %}\ -{% endfor %}\ - } - - pVtab->pContent->n = pVtab->pContent->{{schema[0].name}}.size(); - - return SQLITE_OK; -} - -static sqlite3_module {{table_name_cc}}Module = { - 0, - {{table_name_cc}}Create, - {{table_name_cc}}Create, - xBestIndex, - xDestroy>, - xDestroy>, - xOpen, - xClose, - {{table_name_cc}}Filter, - xNext, - xEof>, - {{table_name_cc}}Column, - xRowid, - 0, - 0, - 0, - 0, - 0, - 0, - 0, -}; - -class {{table_name_cc}}TablePlugin : public TablePlugin { -public: - {{table_name_cc}}TablePlugin() {} - - int attachVtable(sqlite3 *db) { - return sqlite3_attach_vtable( - db, "{{table_name}}", &{{table_name_cc}}Module); - } - - virtual ~{{table_name_cc}}TablePlugin() {} -}; - -REGISTER_TABLE( - "{{table_name}}", - std::make_shared<{{table_name_cc}}TablePlugin>() -) - -}} - -""" +# Read all implementation templates +TEMPLATES = {} + +# Temporary reserved column names +RESERVED = ["n"] + +# Supported SQL types for spec + + +class DataType(object): + + def __init__(self, affinity, cpp_type="std::string"): + '''A column datatype is a pair of a SQL affinity to C++ type.''' + self.affinity = affinity + self.type = cpp_type + + def __repr__(self): + return self.affinity + +# Define column-type MACROs for the table specs +TEXT = DataType("TEXT") +DATE = DataType("TEXT") +DATETIME = DataType("TEXT") +INTEGER = DataType("INTEGER", "int") +BIGINT = DataType("BIGINT", "long long int") + def usage(): """ print program usage """ - print("Usage: %s [disable_blacklist]" % sys.argv[0]) + print( + "Usage: %s [disable_blacklist]" % sys.argv[0]) + def to_camel_case(snake_case): """ convert a snake_case string to camelCase """ components = snake_case.split('_') return components[0] + "".join(x.title() for x in components[1:]) + def lightred(msg): return "\033[1;31m %s \033[0m" % str(msg) + +def is_blacklisted(table_name, path=None, blacklist=None): + """Allow blacklisting by tablename.""" + if blacklist is None: + specs_path = os.path.dirname(os.path.dirname(path)) + blacklist_path = os.path.join(specs_path, "blacklist") + if not os.path.exists(blacklist_path): + return False + try: + with open(blacklist_path, "r") as fh: + blacklist = [ + line.strip() for line in fh.read().split("\n") + if len(line.strip()) > 0 and line.strip()[0] != "#" + ] + except: + # Blacklist is not readable. + return False + # table_name based blacklisting! + return table_name in blacklist if blacklist else False + + +def setup_templates(path): + tables_path = os.path.dirname(os.path.dirname(os.path.dirname(path))) + templates_path = os.path.join(tables_path, "templates") + if not os.path.exists(templates_path): + print ("Cannot read templates path: %s" % (templates_path)) + exit(1) + for template in os.listdir(os.path.join(tables_path, "templates")): + template_name = template.split(".", 1)[0] + with open(os.path.join(templates_path, template), "rb") as fh: + TEMPLATES[template_name] = fh.read().replace("\\\n", "") + + class Singleton(object): + """ Make sure that anything that subclasses Singleton can only be instantiated once @@ -257,7 +108,9 @@ class Singleton(object): self, *args, **kwargs) return self._instance + class TableState(Singleton): + """ Maintain the state of of the table commands during the execution of the config file @@ -278,10 +131,10 @@ class TableState(Singleton): def foreign_keys(self): return [i for i in self.schema if isinstance(i, ForeignKey)] - def generate(self, path, template=IMPL_TEMPLATE): + def generate(self, path, template="default"): """Generate the virtual table files""" logging.debug("TableState.generate") - self.impl_content = jinja2.Template(template).render( + self.impl_content = jinja2.Template(TEMPLATES[template]).render( table_name=self.table_name, table_name_cc=to_camel_case(self.table_name), schema=self.columns(), @@ -291,6 +144,14 @@ class TableState(Singleton): class_name=self.class_name ) + # Check for reserved column names + for column in self.columns(): + if column.name in RESERVED: + print (lightred(("Cannot use column name: %s in table: %s " + "(the column name is reserved)" % ( + column.name, self.table_name)))) + exit(1) + path_bits = path.split("/") for i in range(1, len(path_bits)): dir_path = "" @@ -305,35 +166,44 @@ class TableState(Singleton): def blacklist(self, path): print (lightred("Blacklisting generated %s" % path)) logging.debug("blacklisting %s" % path) - self.generate(path, template=BL_IMPL_TEMPLATE) + self.generate(path, template="blacklist") table = TableState() + class Column(object): + """ Part of an osquery table schema. Define a column by name and type with an optional description to assist documentation generation and reference. """ - def __init__(self, **kwargs): - self.name = kwargs.get("name", "") - self.type = kwargs.get("type", "") - self.description = kwargs.get("description", "") + + def __init__(self, name, col_type, description="", **kwargs): + self.name = name + self.type = col_type + self.description = description + class ForeignKey(object): + """ Part of an osquery table schema. Loosely define a column in a table spec as a Foreign key in another table. """ + def __init__(self, **kwargs): self.column = kwargs.get("column", "") self.table = kwargs.get("table", "") + def table_name(name): """define the virtual table name""" logging.debug("- table_name") logging.debug(" - called with: %s" % name) table.table_name = name + table.description = "" + def schema(schema_list): """ @@ -348,6 +218,7 @@ def schema(schema_list): logging.debug(" - foreign_key: %s (%s)" % (it.column, it.table)) table.schema = schema_list + def implementation(impl_string): """ define the path to the implementation file and the function which @@ -370,25 +241,10 @@ def implementation(impl_string): table.function = function table.class_name = class_name + def description(text): table.description = text -def is_blacklisted(path, table_name): - """Allow blacklisting by tablename.""" - specs_path = os.path.dirname(os.path.dirname(path)) - blacklist_path = os.path.join(specs_path, "blacklist") - if not os.path.exists(blacklist_path): - return False - try: - with open(blacklist_path, "r") as fh: - blacklist = [line.strip() for line in fh.read().split("\n") - if len(line.strip()) > 0 and line.strip()[0] != "#"] - if table_name in blacklist: - return True - except: - # Blacklist is not readable. - pass - return False def main(argc, argv): if DEVELOPING: @@ -406,10 +262,12 @@ def main(argc, argv): # Adding a 3rd parameter will enable the blacklist disable_blacklist = argc > 3 + setup_templates(filename) with open(filename, "rU") as file_handle: tree = ast.parse(file_handle.read()) exec(compile(tree, "", "exec")) - if not disable_blacklist and is_blacklisted(filename, table.table_name): + blacklisted = is_blacklisted(table.table_name, path=filename) + if not disable_blacklist and blacklisted: table.blacklist(output) else: table.generate(output) -- 2.7.4