From 5f5263fa80c6a40ac1a3a5c298ad6b8401837e98 Mon Sep 17 00:00:00 2001 From: Sangwan Kwon Date: Tue, 14 Jan 2020 11:15:48 +0900 Subject: [PATCH] osquery: Remove hashing Signed-off-by: Sangwan Kwon --- src/osquery/CMakeLists.txt | 1 - src/osquery/config/config.cpp | 53 ----------- src/osquery/config/config.h | 19 ---- src/osquery/config/packs.cpp | 38 -------- src/osquery/config/tests/config_tests.cpp | 29 ------ src/osquery/config/tests/packs.cpp | 14 --- src/osquery/hashing/CMakeLists.txt | 15 ---- src/osquery/hashing/hashing.cpp | 141 ------------------------------ src/osquery/hashing/hashing.h | 130 --------------------------- src/osquery/sql/CMakeLists.txt | 1 - src/osquery/sql/sqlite_hashing.cpp | 89 ------------------- src/osquery/sql/sqlite_util.cpp | 1 - 12 files changed, 531 deletions(-) delete mode 100644 src/osquery/hashing/CMakeLists.txt delete mode 100644 src/osquery/hashing/hashing.cpp delete mode 100644 src/osquery/hashing/hashing.h delete mode 100644 src/osquery/sql/sqlite_hashing.cpp diff --git a/src/osquery/CMakeLists.txt b/src/osquery/CMakeLists.txt index 234be6e..0dbae5c 100644 --- a/src/osquery/CMakeLists.txt +++ b/src/osquery/CMakeLists.txt @@ -49,7 +49,6 @@ ADD_SUBDIRECTORY(database) ADD_SUBDIRECTORY(dispatcher) ADD_SUBDIRECTORY(events) ADD_SUBDIRECTORY(filesystem) -ADD_SUBDIRECTORY(hashing) ADD_SUBDIRECTORY(logger) ADD_SUBDIRECTORY(plugins) ADD_SUBDIRECTORY(process) diff --git a/src/osquery/config/config.cpp b/src/osquery/config/config.cpp index 2a679f9..0f84963 100644 --- a/src/osquery/config/config.cpp +++ b/src/osquery/config/config.cpp @@ -22,7 +22,6 @@ #include #include #include -#include #include #include #include @@ -589,13 +588,6 @@ void Config::backupConfig(const ConfigMap& config) { Status Config::updateSource(const std::string& source, const std::string& json) { - // Compute a 'synthesized' hash using the content before it is parsed. - if (!hashSource(source, json)) { - // This source did not change, the returned status allows the caller to - // choose to reconfigure if any sources had changed. - return Status(2); - } - { RecursiveLock lock(config_schedule_mutex_); // Remove all packs from this source. @@ -937,51 +929,6 @@ void Config::getPerformanceStats( } } -bool Config::hashSource(const std::string& source, const std::string& content) { - Hash hash(HASH_TYPE_SHA1); - hash.update(content.c_str(), content.size()); - auto new_hash = hash.digest(); - - WriteLock wlock(config_hash_mutex_); - if (hash_[source] == new_hash) { - return false; - } - hash_[source] = new_hash; - return true; -} - -Status Config::genHash(std::string& hash) const { - WriteLock lock(config_hash_mutex_); - if (!valid_) { - return Status(1, "Current config is not valid"); - } - - std::vector buffer; - buffer.reserve(hash_.size() * 32); - auto add = [&buffer](const std::string& text) { - for (const auto& c : text) { - buffer.push_back(c); - } - }; - for (const auto& it : hash_) { - add(it.second); - } - - Hash new_hash(HASH_TYPE_SHA1); - new_hash.update(buffer.data(), buffer.size()); - hash = new_hash.digest(); - - return Status::success(); -} - -std::string Config::getHash(const std::string& source) const { - WriteLock lock(config_hash_mutex_); - if (!hash_.count(source)) { - return std::string(); - } - return hash_.at(source); -} - const std::shared_ptr Config::getParser( const std::string& parser) { if (!RegistryFactory::get().exists("config_parser", parser, true)) { diff --git a/src/osquery/config/config.h b/src/osquery/config/config.h index c363e0e..327bae7 100644 --- a/src/osquery/config/config.h +++ b/src/osquery/config/config.h @@ -107,25 +107,6 @@ class Config : private boost::noncopyable { */ void recordQueryStart(const std::string& name); - /** - * @brief Calculate the hash of the osquery config - * - * @return The SHA1 hash of the osquery config - */ - Status genHash(std::string& hash) const; - - /// Retrieve the hash of a named source. - std::string getHash(const std::string& source) const; - - /** - * @brief Hash a source's config data - * - * @param source is the place where the config content came from - * @param content is the content of the config data for a given source - * @return false if the source did not change, otherwise true - */ - bool hashSource(const std::string& source, const std::string& content); - /// Whether or not the last loaded config was valid. bool isValid() const { return valid_; diff --git a/src/osquery/config/packs.cpp b/src/osquery/config/packs.cpp index 87d57a9..b878ad6 100644 --- a/src/osquery/config/packs.cpp +++ b/src/osquery/config/packs.cpp @@ -10,7 +10,6 @@ #include #include -#include #include #include #include @@ -63,28 +62,6 @@ size_t splayValue(size_t original, size_t splayPercent) { return distribution(generator); } -size_t getMachineShard(const std::string& hostname = "", bool force = false) { - static size_t shard = 0; - if (shard > 0 && !force) { - return shard; - } - - // An optional input hostname may override hostname detection for testing. - auto hn = (hostname.empty()) ? getHostname() : hostname; - - Hash hash(HASH_TYPE_SHA1); - hash.update(hn.c_str(), hn.size()); - auto hn_hash = hash.digest(); - - if (hn_hash.size() >= 2) { - auto const hn_num = tryTo(hn_hash.substr(0, 2), 16); - if (hn_num.isValue()) { - shard = (hn_num.get() * 100) / 255; - } - } - return shard; -} - size_t restoreSplayedValue(const std::string& name, size_t interval) { // Attempt to restore a previously-calculated splay. std::string content; @@ -141,14 +118,6 @@ void Pack::initialize(const std::string& name, oncall = "unknown"; } - // Apply the shard, platform, and version checking. - // It is important to set each value such that the packs meta-table can report - // each of the restrictions. - if ((shard_ > 0 && shard_ < getMachineShard()) || !checkPlatform() || - !checkVersion()) { - return; - } - discovery_queries_.clear(); if (obj.HasMember("discovery") && obj["discovery"].IsArray()) { for (const auto& item : obj["discovery"].GetArray()) { @@ -179,13 +148,6 @@ void Pack::initialize(const std::string& name, continue; } - if (q.value.HasMember("shard")) { - auto shard = JSON::valueToSize(q.value["shard"]); - if (shard > 0 && shard < getMachineShard()) { - continue; - } - } - if (q.value.HasMember("platform") && q.value["platform"].IsString()) { if (!checkPlatform(q.value["platform"].GetString())) { continue; diff --git a/src/osquery/config/tests/config_tests.cpp b/src/osquery/config/tests/config_tests.cpp index 1ab4eee..e64e1ff 100644 --- a/src/osquery/config/tests/config_tests.cpp +++ b/src/osquery/config/tests/config_tests.cpp @@ -285,35 +285,6 @@ TEST_F(ConfigTests, test_pack_removal) { EXPECT_EQ(pack_count, 0U); } -TEST_F(ConfigTests, test_content_update) { - const std::string source{"awesome"}; - - // Read config content manually. - std::string content; - readFile(getTestConfigDirectory() / "test_parse_items.conf", content); - - // Create the output of a `genConfig`. - std::map config_data; - config_data[source] = content; - - // Update, then clear, packs should have been cleared. - get().update(config_data); - auto source_hash = get().getHash(source); - EXPECT_EQ("fb0973b39c70db16655effbca532d4aa93381e59", source_hash); - - size_t count = 0; - auto packCounter = [&count](const Pack& pack) { count++; }; - get().packs(packCounter); - EXPECT_GT(count, 0U); - - // Now clear. - config_data[source] = ""; - get().update(config_data); - count = 0; - get().packs(packCounter); - EXPECT_EQ(count, 0U); -} - TEST_F(ConfigTests, test_get_scheduled_queries) { std::vector query_names; get().addPack("unrestricted_pack", "", getUnrestrictedPack().doc()); diff --git a/src/osquery/config/tests/packs.cpp b/src/osquery/config/tests/packs.cpp index 07764d8..b7deffc 100644 --- a/src/osquery/config/tests/packs.cpp +++ b/src/osquery/config/tests/packs.cpp @@ -31,9 +31,6 @@ namespace osquery { DECLARE_bool(disable_database); -extern size_t getMachineShard(const std::string& hostname = "", - bool force = false); - class PacksTests : public testing::Test { public: PacksTests() { @@ -79,17 +76,6 @@ TEST_F(PacksTests, test_version) { EXPECT_EQ(fpack.getVersion(), "1.5.0"); } -TEST_F(PacksTests, test_sharding) { - auto shard1 = getMachineShard("localhost.localdomain"); - auto shard2 = getMachineShard("not.localhost.localdomain"); - // Expect some static caching. - EXPECT_EQ(shard1, shard2); - - // Bypass the caching. - shard2 = getMachineShard("not.localhost.localdomain", true); - EXPECT_NE(shard1, shard2); -} - TEST_F(PacksTests, test_check_platform) { // First we exercise some basic functionality which should behave the same // regardless of the current build platform. diff --git a/src/osquery/hashing/CMakeLists.txt b/src/osquery/hashing/CMakeLists.txt deleted file mode 100644 index aadab34..0000000 --- a/src/osquery/hashing/CMakeLists.txt +++ /dev/null @@ -1,15 +0,0 @@ -# Copyright (c) 2019 Samsung Electronics Co., Ltd All Rights Reserved -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License - -ADD_OSQUERY_LIBRARY(osquery_hasing hashing.cpp) diff --git a/src/osquery/hashing/hashing.cpp b/src/osquery/hashing/hashing.cpp deleted file mode 100644 index c437023..0000000 --- a/src/osquery/hashing/hashing.cpp +++ /dev/null @@ -1,141 +0,0 @@ -/** - * Copyright (c) 2014-present, Facebook, Inc. - * All rights reserved. - * - * This source code is licensed in accordance with the terms specified in - * the LICENSE file found in the root directory of this source tree. - */ - -#include -#include -#include -#include -#include - -#include -#include - -#include -#include -#include -#include - -namespace osquery { - -/// The buffer read size from file IO to hashing structures. -const size_t kHashChunkSize{4096}; - -Hash::~Hash() { - if (ctx_ != nullptr) { - free(ctx_); - } -} - -Hash::Hash(HashType algorithm) : algorithm_(algorithm) { - if (algorithm_ == HASH_TYPE_MD5) { - length_ = MD5_DIGEST_LENGTH; - ctx_ = static_cast(malloc(sizeof(MD5_CTX))); - MD5_Init(static_cast(ctx_)); - } else if (algorithm_ == HASH_TYPE_SHA1) { - length_ = SHA_DIGEST_LENGTH; - ctx_ = static_cast(malloc(sizeof(SHA_CTX))); - SHA1_Init(static_cast(ctx_)); - } else if (algorithm_ == HASH_TYPE_SHA256) { - length_ = SHA256_DIGEST_LENGTH; - ctx_ = static_cast(malloc(sizeof(SHA256_CTX))); - SHA256_Init(static_cast(ctx_)); - } else { - throw std::domain_error("Unknown hash function"); - } -} - -void Hash::update(const void* buffer, size_t size) { - if (algorithm_ == HASH_TYPE_MD5) { - MD5_Update(static_cast(ctx_), buffer, size); - } else if (algorithm_ == HASH_TYPE_SHA1) { - SHA1_Update(static_cast(ctx_), buffer, size); - } else if (algorithm_ == HASH_TYPE_SHA256) { - SHA256_Update(static_cast(ctx_), buffer, size); - } -} - -std::string Hash::digest() { - std::vector hash; - hash.assign(length_, '\0'); - - if (algorithm_ == HASH_TYPE_MD5) { - MD5_Final(hash.data(), static_cast(ctx_)); - } else if (algorithm_ == HASH_TYPE_SHA1) { - SHA1_Final(hash.data(), static_cast(ctx_)); - } else if (algorithm_ == HASH_TYPE_SHA256) { - SHA256_Final(hash.data(), static_cast(ctx_)); - } - - // The hash value is only relevant as a hex digest. - std::stringstream digest; - for (size_t i = 0; i < length_; i++) { - digest << std::hex << std::setw(2) << std::setfill('0') << (int)hash[i]; - } - - return digest.str(); -} - -std::string hashFromBuffer(HashType hash_type, - const void* buffer, - size_t size) { - Hash hash(hash_type); - hash.update(buffer, size); - return hash.digest(); -} - -MultiHashes hashMultiFromFile(int mask, const std::string& path) { - std::map> hashes = { - {HASH_TYPE_MD5, std::make_shared(HASH_TYPE_MD5)}, - {HASH_TYPE_SHA1, std::make_shared(HASH_TYPE_SHA1)}, - {HASH_TYPE_SHA256, std::make_shared(HASH_TYPE_SHA256)}, - }; - - auto blocking = isPlatform(PlatformType::TYPE_WINDOWS); - auto s = readFile(path, - 0, - kHashChunkSize, - false, - true, - ([&hashes, &mask](std::string& buffer, size_t size) { - for (auto& hash : hashes) { - if (mask & hash.first) { - hash.second->update(&buffer[0], size); - } - } - }), - blocking); - - MultiHashes mh = {}; - if (!s.ok()) { - return mh; - } - - mh.mask = mask; - if (mask & HASH_TYPE_MD5) { - mh.md5 = hashes.at(HASH_TYPE_MD5)->digest(); - } - if (mask & HASH_TYPE_SHA1) { - mh.sha1 = hashes.at(HASH_TYPE_SHA1)->digest(); - } - if (mask & HASH_TYPE_SHA256) { - mh.sha256 = hashes.at(HASH_TYPE_SHA256)->digest(); - } - return mh; -} - -std::string hashFromFile(HashType hash_type, const std::string& path) { - auto hashes = hashMultiFromFile(hash_type, path); - if (hash_type == HASH_TYPE_MD5) { - return hashes.md5; - } else if (hash_type == HASH_TYPE_SHA1) { - return hashes.sha1; - } else { - return hashes.sha256; - } -} -} // namespace osquery diff --git a/src/osquery/hashing/hashing.h b/src/osquery/hashing/hashing.h deleted file mode 100644 index 7fc2aba..0000000 --- a/src/osquery/hashing/hashing.h +++ /dev/null @@ -1,130 +0,0 @@ -/** - * Copyright (c) 2014-present, Facebook, Inc. - * All rights reserved. - * - * This source code is licensed in accordance with the terms specified in - * the LICENSE file found in the root directory of this source tree. - */ - -#pragma once - -#include - -#include - -namespace osquery { - -/** - * @brief The supported hashing algorithms in osquery - * - * These are usually used as a constructor argument to osquery::Hash - */ -enum HashType { - HASH_TYPE_MD5 = 2, - HASH_TYPE_SHA1 = 4, - HASH_TYPE_SHA256 = 8, -}; - -/// A result structure for multiple hash requests. -struct MultiHashes { - int mask; - std::string md5; - std::string sha1; - std::string sha256; -}; - -/** - * @brief Hash is a general utility class for hashing content - * - * @code{.cpp} - * Hash my_hash(HASH_TYPE_SHA256); - * my_hash.update(my_buffer, my_buffer_size); - * std::cout << my_hash.digest(); - * @endcode - * - */ -class Hash : private boost::noncopyable { - public: - /** - * @brief Hash constructor - * - * The hash class should be initialized with one of osquery::HashType as a - * constructor argument. - * - * @param algorithm The hashing algorithm which will be used to compute the - * hash - */ - explicit Hash(HashType algorithm); - - /** - * @brief Hash destructor - */ - ~Hash(); - - /** - * @brief Update the internal context buffer with additional content - * - * This method allows you to chunk up large content so that it doesn't all - * have to be loaded into memory at the same time - * - * @param buffer The buffer to be hashed - * @param size The size of the buffer to be hashed - */ - void update(const void* buffer, size_t size); - - /** - * @brief Compute the final hash and return it's result - * - * @return The final hash value - */ - std::string digest(); - - private: - /** - * @brief Private default constructor - * - * The osquery::Hash class should only ever be instantiated with a HashType - */ - Hash(){}; - - private: - /// The hashing algorithm which is used to compute the hash - HashType algorithm_; - - /// The buffer used to maintain the context and state of the hashing - /// operations - void* ctx_; - - /// The length of the hash to be returned - size_t length_; -}; - -/** - * @brief Compute a hash digest from the file content at a path. - * - * @param hash_type The osquery-supported hash algorithm. - * @param path Filesystem path (the hash target). - * @return A string (hex) representation of the hash digest. - */ -std::string hashFromFile(HashType hash_type, const std::string& path); - -/** - * @brief Compute multiple hashes from a files contents simultaneously. - * - * @param mask Bitmask specifying target osquery-supported algorithms. - * @param path Filesystem path (the hash target). - * @return A struct containing string (hex) representations - * of the hash digests. - */ -MultiHashes hashMultiFromFile(int mask, const std::string& path); - -/** - * @brief Compute a hash digest from the contents of a buffer. - * - * @param hash_type The osquery-supported hash algorithm. - * @param buffer A caller-controlled buffer (already allocated). - * @param size The length of buffer in bytes. - * @return A string (hex) representation of the hash digest. - */ -std::string hashFromBuffer(HashType hash_type, const void* buffer, size_t size); -} // namespace osquery diff --git a/src/osquery/sql/CMakeLists.txt b/src/osquery/sql/CMakeLists.txt index 41d30de..7e5990f 100644 --- a/src/osquery/sql/CMakeLists.txt +++ b/src/osquery/sql/CMakeLists.txt @@ -14,7 +14,6 @@ ADD_OSQUERY_LIBRARY(osquery_sql dynamic_table_row.cpp sqlite_encoding.cpp - sqlite_hashing.cpp sqlite_util.cpp sql.cpp sqlite_filesystem.cpp diff --git a/src/osquery/sql/sqlite_hashing.cpp b/src/osquery/sql/sqlite_hashing.cpp deleted file mode 100644 index 1bee160..0000000 --- a/src/osquery/sql/sqlite_hashing.cpp +++ /dev/null @@ -1,89 +0,0 @@ -/** - * Copyright (c) 2014-present, Facebook, Inc. - * All rights reserved. - * - * This source code is licensed in accordance with the terms specified in - * the LICENSE file found in the root directory of this source tree. - */ - -#include -#include -#include - -#include - -#include - -namespace osquery { - -static void hashSqliteValue(sqlite3_context* ctx, - int argc, - sqlite3_value** argv, - HashType ht) { - if (argc == 0) { - return; - } - - if (SQLITE_NULL == sqlite3_value_type(argv[0])) { - sqlite3_result_null(ctx); - return; - } - - // Parse and verify the split input parameters. - const char* input = - reinterpret_cast(sqlite3_value_text(argv[0])); - if (input == nullptr) { - sqlite3_result_null(ctx); - return; - } - - auto result = hashFromBuffer(ht, input, strlen(input)); - sqlite3_result_text( - ctx, result.c_str(), static_cast(result.size()), SQLITE_TRANSIENT); -} - -static void sqliteMD5Func(sqlite3_context* context, - int argc, - sqlite3_value** argv) { - hashSqliteValue(context, argc, argv, HASH_TYPE_MD5); -} - -static void sqliteSHA1Func(sqlite3_context* context, - int argc, - sqlite3_value** argv) { - hashSqliteValue(context, argc, argv, HASH_TYPE_SHA1); -} - -static void sqliteSHA256Func(sqlite3_context* context, - int argc, - sqlite3_value** argv) { - hashSqliteValue(context, argc, argv, HASH_TYPE_SHA256); -} - -void registerHashingExtensions(sqlite3* db) { - sqlite3_create_function(db, - "md5", - 1, - SQLITE_UTF8 | SQLITE_DETERMINISTIC, - nullptr, - sqliteMD5Func, - nullptr, - nullptr); - sqlite3_create_function(db, - "sha1", - 1, - SQLITE_UTF8 | SQLITE_DETERMINISTIC, - nullptr, - sqliteSHA1Func, - nullptr, - nullptr); - sqlite3_create_function(db, - "sha256", - 1, - SQLITE_UTF8 | SQLITE_DETERMINISTIC, - nullptr, - sqliteSHA256Func, - nullptr, - nullptr); -} -} // namespace osquery diff --git a/src/osquery/sql/sqlite_util.cpp b/src/osquery/sql/sqlite_util.cpp index 10a6e03..6c4b4ce 100644 --- a/src/osquery/sql/sqlite_util.cpp +++ b/src/osquery/sql/sqlite_util.cpp @@ -279,7 +279,6 @@ static inline void openOptimized(sqlite3*& db) { registerStringExtensions(db); #endif registerFilesystemExtensions(db); - registerHashingExtensions(db); registerEncodingExtensions(db); } -- 2.7.4