Import scheduler from upstream
authorsangwan.kwon <sangwan.kwon@samsung.com>
Thu, 16 May 2019 04:18:25 +0000 (13:18 +0900)
committerSangwan Kwon <sangwan.kwon@samsung.com>
Wed, 12 Jun 2019 00:04:42 +0000 (09:04 +0900)
osquery comes with a scheduler, which schedules a variety of things. This
is one of the core parts of the osqueryd daemon. To use this, simply use
this function as your entry point when creating a new thread.

TBD: Fix sql_tests failed cases. (It should be come with build-in tables.)

Signed-off-by: sangwan.kwon <sangwan.kwon@samsung.com>
include/osquery/scheduler.h [new file with mode: 0644]
include/osquery/sql.h [new file with mode: 0644]
osquery/CMakeLists.txt
osquery/core/CMakeLists.txt
osquery/core/sql.cpp [new file with mode: 0644]
osquery/core/sql_tests.cpp [new file with mode: 0644]
osquery/scheduler/CMakeLists.txt [new file with mode: 0644]
osquery/scheduler/scheduler.cpp [new file with mode: 0644]
osquery/scheduler/scheduler_tests.cpp [new file with mode: 0644]
packaging/osquery.spec

diff --git a/include/osquery/scheduler.h b/include/osquery/scheduler.h
new file mode 100644 (file)
index 0000000..f01d1b9
--- /dev/null
@@ -0,0 +1,21 @@
+// Copyright 2004-present Facebook. All Rights Reserved.
+
+#pragma once
+
+namespace osquery {
+
+/**
+ * @brief Launch the scheduler.
+ *
+ * osquery comes with a scheduler, which schedules a variety of things. This
+ * is one of the core parts of the osqueryd daemon. To use this, simply use
+ * this function as your entry point when creating a new thread.
+ *
+ * @code{.cpp}
+ *   boost::thread scheduler_thread(osquery::initializeScheduler);
+ *   // do some other stuff
+ *   scheduler_thread.join();
+ * @endcode
+ */
+void initializeScheduler();
+}
diff --git a/include/osquery/sql.h b/include/osquery/sql.h
new file mode 100644 (file)
index 0000000..0b4448d
--- /dev/null
@@ -0,0 +1,95 @@
+// Copyright 2004-present Facebook. All Rights Reserved.
+
+#pragma once
+
+#include <map>
+#include <string>
+#include <vector>
+
+#include "osquery/database/results.h"
+
+namespace osquery {
+
+/**
+ * @brief A map of SQLite status codes to their corresponding message string
+ *
+ * Details of this map are defined at: http://www.sqlite.org/c3ref/c_abort.html
+ */
+extern const std::map<int, std::string> kSQLiteReturnCodes;
+
+/**
+ * @brief Get a string representation of a SQLite return code
+ */
+std::string getStringForSQLiteReturnCode(int code);
+
+/**
+ * @brief The core interface to executing osquery SQL commands
+ *
+ * @code{.cpp}
+ *   auto sql = SQL("SELECT * FROM time");
+ *   if (sql.ok()) {
+ *     LOG(INFO) << "============================";
+ *     for (const auto& row : sql.rows()) {
+ *       for (const auto& it : row) {
+ *         LOG(INFO) << it.first << " => " << it.second;
+ *       }
+ *       LOG(INFO) << "============================";
+ *     }
+ *   } else {
+ *     LOG(ERROR) << sql.getMessageString();
+ *   }
+ * @endcode
+ */
+class SQL {
+ public:
+  /**
+   * @brief Instantiate an instance of the class with a query
+   *
+   * @param q An osquery SQL query
+   */
+  explicit SQL(const std::string& q);
+
+  /**
+   * @brief Accessor for the rows returned by the query
+   *
+   * @return A QueryData object of the query results
+   */
+  QueryData rows();
+
+  /**
+   * @brief Accessor to switch off of when checking the success of a query
+   *
+   * @return A bool indicating the success or failure of the operation
+   */
+  bool ok();
+
+  /**
+   * @brief Accessor for the message string indicating the status of the query
+   *
+   * @return The message string indicating the status of the query
+   */
+  std::string getMessageString();
+
+  /**
+   * @brief Accessor for the list of queryable tables
+   *
+   * @return A vector of table names
+   */
+  static std::vector<std::string> getTableNames();
+
+ private:
+  /**
+   * @brief Private default constructor
+   *
+   * The osquery::SQL class should only ever be instantiated with a query
+   */
+  SQL() {};
+
+ private:
+  /// the internal member which holds the results of the query
+  QueryData results_;
+
+  /// the internal member which holds the status of the query
+  Status status_;
+};
+}
index d99f224..a7ce2b8 100644 (file)
@@ -56,6 +56,7 @@ ADD_SUBDIRECTORY(database)
 ADD_SUBDIRECTORY(registry)
 ADD_SUBDIRECTORY(filesystem)
 ADD_SUBDIRECTORY(logger)
+ADD_SUBDIRECTORY(scheduler)
 
 ADD_LIBRARY(${TARGET_OSQUERY_LIB} STATIC main/lib.cpp ${${TARGET_OSQUERY_LIB}_SRCS})
 TARGET_LINK_LIBRARIES(${TARGET_OSQUERY_LIB} ${${TARGET_OSQUERY_LIB}_DEP})
index 8192d7c..036007e 100644 (file)
@@ -1,12 +1,12 @@
 ADD_OSQUERY_LIBRARY(osquery_core init_osquery.cpp
-#                                                               sql.cpp // scheduler
+                                                                sql.cpp
                                                                 sqlite_util.cpp
                                                                 system.cpp
                                                                 test_util.cpp
                                                                 text.cpp)
-#
-ADD_OSQUERY_TEST(osquery_status_test status_tests.cpp)
-#ADD_OSQUERY_TEST(sql_test sql_tests.cpp)
+
+ADD_OSQUERY_TEST(osquery_status_tests status_tests.cpp)
+ADD_OSQUERY_TEST(osquery_sql_tests sql_tests.cpp)
 ADD_OSQUERY_TEST(osquery_sqlite_util_tests sqlite_util_tests.cpp)
 ADD_OSQUERY_TEST(osquery_test_util_tests test_util_tests.cpp)
 ADD_OSQUERY_TEST(osquery_text_tests text_tests.cpp)
diff --git a/osquery/core/sql.cpp b/osquery/core/sql.cpp
new file mode 100644 (file)
index 0000000..a5673bb
--- /dev/null
@@ -0,0 +1,77 @@
+// Copyright 2004-present Facebook. All Rights Reserved.
+
+#include "osquery/sql.h"
+
+#include <sstream>
+
+#include <glog/logging.h>
+
+#include "osquery/core.h"
+#include "osquery/registry/registry.h"
+
+namespace osquery {
+
+const std::map<int, std::string> kSQLiteReturnCodes = {
+    {0, "SQLITE_OK: Successful result"},
+    {1, "SQLITE_ERROR: SQL error or missing database"},
+    {2, "SQLITE_INTERNAL: Internal logic error in SQLite"},
+    {3, "SQLITE_PERM: Access permission denied"},
+    {4, "SQLITE_ABORT: Callback routine requested an abort"},
+    {5, "SQLITE_BUSY: The database file is locked"},
+    {6, "SQLITE_LOCKED: A table in the database is locked"},
+    {7, "SQLITE_NOMEM: A malloc() failed"},
+    {8, "SQLITE_READONLY: Attempt to write a readonly database"},
+    {9, "SQLITE_INTERRUPT: Operation terminated by sqlite3_interrupt()"},
+    {10, "SQLITE_IOERR: Some kind of disk I/O error occurred"},
+    {11, "SQLITE_CORRUPT: The database disk image is malformed"},
+    {12, "SQLITE_NOTFOUND: Unknown opcode in sqlite3_file_control()"},
+    {13, "SQLITE_FULL: Insertion failed because database is full"},
+    {14, "SQLITE_CANTOPEN: Unable to open the database file"},
+    {15, "SQLITE_PROTOCOL: Database lock protocol error"},
+    {16, "SQLITE_EMPTY: Database is empty"},
+    {17, "SQLITE_SCHEMA: The database schema changed"},
+    {18, "SQLITE_TOOBIG: String or BLOB exceeds size limit"},
+    {19, "SQLITE_CONSTRAINT: Abort due to constraint violation"},
+    {20, "SQLITE_MISMATCH: Data type mismatch"},
+    {21, "SQLITE_MISUSE: Library used incorrectly"},
+    {22, "SQLITE_NOLFS: Uses OS features not supported on host"},
+    {23, "SQLITE_AUTH: Authorization denied"},
+    {24, "SQLITE_FORMAT: Auxiliary database format error"},
+    {25, "SQLITE_RANGE: 2nd parameter to sqlite3_bind out of range"},
+    {26, "SQLITE_NOTADB: File opened that is not a database file"},
+    {27, "SQLITE_NOTICE: Notifications from sqlite3_log()"},
+    {28, "SQLITE_WARNING: Warnings from sqlite3_log()"},
+    {100, "SQLITE_ROW: sqlite3_step() has another row ready"},
+    {101, "SQLITE_DONE: sqlite3_step() has finished executing"},
+};
+
+std::string getStringForSQLiteReturnCode(int code) {
+  if (kSQLiteReturnCodes.find(code) != kSQLiteReturnCodes.end()) {
+    return kSQLiteReturnCodes.at(code);
+  } else {
+    std::ostringstream s;
+    s << "Error: " << code << " is not a valid SQLite result code";
+    return s.str();
+  }
+}
+
+SQL::SQL(const std::string& q) {
+  int code = 0;
+  results_ = query(q, code);
+  status_ = Status(code, getStringForSQLiteReturnCode(code));
+}
+
+QueryData SQL::rows() { return results_; }
+
+bool SQL::ok() { return status_.ok(); }
+
+std::string SQL::getMessageString() { return status_.toString(); }
+
+std::vector<std::string> SQL::getTableNames() {
+  std::vector<std::string> results;
+  for (const auto& it : REGISTERED_TABLES) {
+    results.push_back(it.first);
+  }
+  return results;
+}
+}
diff --git a/osquery/core/sql_tests.cpp b/osquery/core/sql_tests.cpp
new file mode 100644 (file)
index 0000000..8557c16
--- /dev/null
@@ -0,0 +1,30 @@
+// Copyright 2004-present Facebook. All Rights Reserved.
+
+#include "osquery/sql.h"
+
+#include <gtest/gtest.h>
+
+#include "osquery/core.h"
+
+namespace osquery {
+
+class SQLTests : public testing::Test {};
+
+TEST_F(SQLTests, test_simple_query_execution) {
+  auto sql = SQL("SELECT * FROM time");
+  EXPECT_TRUE(sql.ok());
+  EXPECT_EQ(sql.getMessageString(), getStringForSQLiteReturnCode(0));
+  EXPECT_EQ(sql.rows().size(), 1);
+}
+
+TEST_F(SQLTests, test_get_tables) {
+  auto tables = SQL::getTableNames();
+  EXPECT_TRUE(tables.size() > 0);
+}
+}
+
+int main(int argc, char* argv[]) {
+  testing::InitGoogleTest(&argc, argv);
+  osquery::initOsquery(argc, argv);
+  return RUN_ALL_TESTS();
+}
diff --git a/osquery/scheduler/CMakeLists.txt b/osquery/scheduler/CMakeLists.txt
new file mode 100644 (file)
index 0000000..99581f7
--- /dev/null
@@ -0,0 +1,3 @@
+ADD_OSQUERY_LIBRARY(osquery_scheduler scheduler.cpp)
+
+ADD_OSQUERY_TEST(osquery_scheduler_tests scheduler_tests.cpp)
diff --git a/osquery/scheduler/scheduler.cpp b/osquery/scheduler/scheduler.cpp
new file mode 100644 (file)
index 0000000..d09a254
--- /dev/null
@@ -0,0 +1,78 @@
+// Copyright 2004-present Facebook. All Rights Reserved.
+
+#include "osquery/scheduler.h"
+
+#include <climits>
+#include <ctime>
+
+#include <glog/logging.h>
+
+#include "osquery/config.h"
+#include "osquery/core.h"
+#include "osquery/database.h"
+#include "osquery/logger.h"
+#include "osquery/sql.h"
+
+namespace osquery {
+
+void launchQueries(const std::vector<OsqueryScheduledQuery>& queries,
+                   const int64_t& second) {
+  LOG(INFO) << "launchQueries: " << second;
+  for (const auto& q : queries) {
+    if (second % q.interval == 0) {
+      LOG(INFO) << "executing query: " << q.query;
+      int unix_time = std::time(0);
+      int err;
+      auto sql = SQL(q.query);
+      if (!sql.ok()) {
+        LOG(ERROR) << "error executing query (" << q.query
+                   << "): " << sql.getMessageString();
+        continue;
+      }
+      auto dbQuery = Query(q);
+      DiffResults diff_results;
+      auto status = dbQuery.addNewResults(sql.rows(), diff_results, unix_time);
+      if (!status.ok()) {
+        LOG(ERROR)
+            << "error adding new results to database: " << status.toString();
+        continue;
+      }
+
+      if (diff_results.added.size() > 0 || diff_results.removed.size() > 0) {
+        VLOG(1) << "Results found for query: \"" << q.query << "\"";
+        ScheduledQueryLogItem item;
+        item.diffResults = diff_results;
+        item.name = q.name;
+        item.hostname = osquery::getHostname();
+        item.unixTime = osquery::getUnixTime();
+        item.calendarTime = osquery::getAsciiTime();
+        auto s = logScheduledQueryLogItem(item);
+        if (!s.ok()) {
+          LOG(ERROR) << "Error logging the results of query \"" << q.query
+                     << "\""
+                     << ": " << s.toString();
+        }
+      }
+    }
+  }
+}
+
+void initializeScheduler() {
+  DLOG(INFO) << "osquery::initializeScheduler";
+  time_t t = time(0);
+  struct tm* local = localtime(&t);
+  unsigned long int second = local->tm_sec;
+  auto cfg = Config::getInstance();
+#ifdef OSQUERY_TEST_DAEMON
+  // if we're testing the daemon, only iterate through 15 "seconds"
+  static unsigned long int stop_at = second + 15;
+#else
+  // if this is production, count forever
+  static unsigned long int stop_at = ULONG_MAX;
+#endif
+  for (; second <= stop_at; ++second) {
+    launchQueries(cfg->getScheduledQueries(), second);
+    sleep(1);
+  }
+}
+}
diff --git a/osquery/scheduler/scheduler_tests.cpp b/osquery/scheduler/scheduler_tests.cpp
new file mode 100644 (file)
index 0000000..c70c76d
--- /dev/null
@@ -0,0 +1,14 @@
+// Copyright 2004-present Facebook. All Rights Reserved.
+
+#include "osquery/scheduler.h"
+
+#include <gtest/gtest.h>
+
+class SchedulerTests : public testing::Test {};
+
+TEST_F(SchedulerTests, test) { EXPECT_EQ(true, true); }
+
+int main(int argc, char* argv[]) {
+  testing::InitGoogleTest(&argc, argv);
+  return RUN_ALL_TESTS();
+}
index 0d752ed..7fae3a3 100644 (file)
@@ -65,14 +65,16 @@ Testcases for osquery
 
 %files test
 %manifest %{name}.manifest
-%{_bindir}/osquery_status_test
+%{_bindir}/osquery_status_tests
 %{_bindir}/osquery_db_handle_tests
 %{_bindir}/osquery_results_tests
 %{_bindir}/osquery_registry_tests
 %{_bindir}/osquery_config_tests
 %{_bindir}/osquery_filesystem_tests
 %{_bindir}/osquery_query_tests
+%{_bindir}/osquery_sql_tests
 %{_bindir}/osquery_sqlite_util_tests
+%{_bindir}/osquery_scheduler_tests
 %{_bindir}/osquery_test_util_tests
 %{_bindir}/osquery_text_tests
 %{_bindir}/osquery_logger_tests