From 25708252cdf8e04fe693ec22bf4ae720b52f8e97 Mon Sep 17 00:00:00 2001 From: Kyungwook Tak Date: Wed, 18 May 2016 11:57:52 +0900 Subject: [PATCH] Redesign db schema for app history handling Change-Id: I75e0ce4ada946b3c3ad4b39fdabb09c6a3bbcd48 Signed-off-by: Kyungwook Tak --- data/scripts/create_schema.sql | 79 +++++++++++--- data/scripts/drop_all.sql | 8 +- src/framework/db/manager.cpp | 131 ++++++++++++++++------- src/framework/db/manager.h | 23 +++-- src/framework/db/query.h | 65 +++++++----- src/framework/db/row.h | 1 + src/framework/db/statement.cpp | 10 +- src/framework/db/statement.h | 2 +- src/framework/service/cs-logic.cpp | 189 +++++++++++++++++++++++----------- src/framework/service/cs-logic.h | 3 +- src/framework/service/file-system.cpp | 17 +++ src/framework/service/file-system.h | 1 + test/internals/test-db.cpp | 37 ++++--- test/internals/test-wp-loader.cpp | 9 +- test/test-helper.cpp | 12 +-- 15 files changed, 415 insertions(+), 172 deletions(-) diff --git a/data/scripts/create_schema.sql b/data/scripts/create_schema.sql index 7b69e4c..c5c9c85 100644 --- a/data/scripts/create_schema.sql +++ b/data/scripts/create_schema.sql @@ -1,18 +1,69 @@ -CREATE TABLE IF NOT EXISTS SCHEMA_INFO(name TEXT PRIMARY KEY NOT NULL, - value TEXT); +CREATE TABLE IF NOT EXISTS SCHEMA_INFO( + name TEXT PRIMARY KEY NOT NULL, + value TEXT +); -CREATE TABLE IF NOT EXISTS ENGINE_STATE(id INTEGER PRIMARY KEY, - state INTEGER NOT NULL); +CREATE TABLE IF NOT EXISTS ENGINE_STATE( + id INTEGER PRIMARY KEY, + state INTEGER NOT NULL +); -CREATE TABLE IF NOT EXISTS SCAN_REQUEST(dir TEXT PRIMARY KEY, - last_scan INTEGER NOT NULL, - data_version TEXT NOT NULL); +CREATE TABLE IF NOT EXISTS SCAN_REQUEST( + dir TEXT PRIMARY KEY, + last_scan INTEGER NOT NULL, + data_version TEXT NOT NULL +); -CREATE TABLE IF NOT EXISTS DETECTED_MALWARE_FILE(path TEXT PRIMARY KEY NOT NULL, - data_version TEXT NOT NULL, - severity INTEGER NOT NULL, - malware_name TEXT NOT NULL, - detailed_url TEXT NOT NULL, - detected_time INTEGER NOT NULL, - ignored INTEGER NOT NULL); +CREATE TABLE IF NOT EXISTS NAMES( + name TEXT NOT NULL, + is_ignored INTEGER NOT NULL DEFAULT 0, + PRIMARY KEY(name) +); + +CREATE TABLE IF NOT EXISTS DETECTED_MALWARE( + file_path TEXT NOT NULL, + name TEXT NOT NULL, + data_version TEXT NOT NULL, + malware_name TEXT NOT NULL, + detailed_url TEXT NOT NULL, + severity INTEGER NOT NULL, + detected_time INTEGER NOT NULL, + + PRIMARY KEY(file_path), + FOREIGN KEY(name) REFERENCES NAMES(name) ON DELETE CASCADE +); + +CREATE TABLE IF NOT EXISTS PACKAGE_INFO( + pkg_id TEXT NOT NULL, + name TEXT NOT NULL, + worst TEXT NOT NULL, + + PRIMARY KEY(pkg_id), + FOREIGN KEY(worst) REFERENCES DETECTED_MALWARE(file_path) ON DELETE CASCADE, + FOREIGN KEY(name) REFERENCES NAMES(name) ON DELETE CASCADE +); + +CREATE VIEW IF NOT EXISTS [join_p_d] AS + SELECT P.name, D.file_path, D.data_version, D.malware_name, D.detailed_url, + D.severity, D.detected_time, P.pkg_id + FROM PACKAGE_INFO AS P INNER JOIN DETECTED_MALWARE AS D ON P.worst = D.file_path; + +CREATE VIEW IF NOT EXISTS [join_detecteds_by_name] AS + SELECT J.*, N.is_ignored + FROM + (SELECT D.name, D.file_path, D.data_version, D.malware_name, D.detailed_url, + D.severity, D.detected_time, P.pkg_id + FROM DETECTED_MALWARE AS D LEFT JOIN PACKAGE_INFO AS P ON D.name = P.name + WHERE NOT EXISTS (SELECT * FROM join_p_d WHERE name = D.name) + UNION + SELECT name, file_path, data_version, malware_name, detailed_url, + severity, detected_time, pkg_id + FROM join_p_d) + AS J INNER JOIN NAMES AS N ON J.name = N.name; + +CREATE VIEW IF NOT EXISTS [join_detecteds_by_file_path] AS + SELECT D.file_path, D.data_version, D.malware_name, D.detailed_url, D.severity, + D.detected_time, P.pkg_id, N.is_ignored + FROM (DETECTED_MALWARE AS D LEFT JOIN PACKAGE_INFO AS P ON D.name = P.name) + AS J INNER JOIN NAMES AS N ON J.name = N.name; diff --git a/data/scripts/drop_all.sql b/data/scripts/drop_all.sql index 1b66f96..abc9a26 100644 --- a/data/scripts/drop_all.sql +++ b/data/scripts/drop_all.sql @@ -1,4 +1,10 @@ -DROP TABLE IF EXISTS DETECTED_MALWARE_FILE; +DROP VIEW IF EXISTS join_detecteds_abstracted; +DROP VIEW IF EXISTS join_detecteds; +DROP VIEW IF EXISTS join_p_d; + +DROP TABLE IF EXISTS PACKAGE_INFO; +DROP TABLE IF EXISTS DETECTED_MALWARE; +DROP TABLE IF EXISTS NAMES; DROP TABLE IF EXISTS SCAN_REQUEST; diff --git a/src/framework/db/manager.cpp b/src/framework/db/manager.cpp index 0faf2c8..137abf0 100644 --- a/src/framework/db/manager.cpp +++ b/src/framework/db/manager.cpp @@ -47,6 +47,24 @@ const std::string SCRIPT_MIGRATE = "migrate_"; const std::string DB_VERSION_STR = "DB_VERSION"; const std::string SCHEMA_INFO_TABLE = "SCHEMA_INFO"; +RowShPtr extractRow(Statement &stmt) +{ + RowShPtr row = std::make_shared(); + + row->targetName = stmt.getText(); // name. + row->fileInAppPath = stmt.getText(); // file_path + row->dataVersion = stmt.getText(); // data_version + row->malwareName = stmt.getText(); // malware_name + row->detailedUrl = stmt.getText(); // detailed_url + row->severity = static_cast(stmt.getInt()); // severity + row->ts = static_cast(stmt.getInt64()); // detected_time + row->pkgId = stmt.getText(); // pkg_id + row->isApp = !row->pkgId.empty(); + row->isIgnored = static_cast(stmt.getInt()); + + return row; +} + } // namespace anonymous Manager::Manager(const std::string &dbfile, const std::string &scriptsDir) : @@ -237,91 +255,132 @@ void Manager::cleanLastScanTime() //=========================================================================== // DETECTED_MALWARE_FILE table //=========================================================================== -RowShPtrs Manager::getDetectedMalwares(const std::string &dir) +RowShPtr Manager::getDetectedByNameOnPath(const std::string &path) +{ + Statement stmt(m_conn, Query::SEL_DETECTED_BY_NAME_ON_PATH); + stmt.bind(path); + + if (!stmt.step()) + return nullptr; + + return extractRow(stmt); +} + +RowShPtrs Manager::getDetectedByNameOnDir(const std::string &dir) { - Statement stmt(m_conn, Query::SEL_DETECTED_BY_DIR); + Statement stmt(m_conn, Query::SEL_DETECTED_BY_NAME_ON_DIR); stmt.bind(dir); RowShPtrs rows; - while (stmt.step()) { - RowShPtr row = std::make_shared(); + while (stmt.step()) + rows.emplace_back(extractRow(stmt)); + + return rows; +} + +RowShPtrs Manager::getDetectedByFilepathOnDir(const std::string &dir) +{ + Statement stmt(m_conn, Query::SEL_DETECTED_BY_FILEPATH_ON_DIR); + stmt.bind(dir); - row->targetName = stmt.getText(); - row->dataVersion = stmt.getText(); - row->severity = static_cast(stmt.getInt()); - row->malwareName = stmt.getText(); - row->detailedUrl = stmt.getText(); - row->ts = static_cast(stmt.getInt64()); - row->isIgnored = static_cast(stmt.getInt()); + RowShPtrs rows; - rows.emplace_back(std::move(row)); - } + while (stmt.step()) + rows.emplace_back(extractRow(stmt)); return rows; } -RowShPtr Manager::getDetectedMalware(const std::string &path) +RowShPtr Manager::getWorstByPkgId(const std::string &pkgId) { - Statement stmt(m_conn, Query::SEL_DETECTED_BY_PATH); - stmt.bind(path); + Statement stmt(m_conn, Query::SEL_WORST_BY_PKGID); + stmt.bind(pkgId); if (!stmt.step()) return nullptr; RowShPtr row = std::make_shared(); - row->targetName = stmt.getText(); - row->dataVersion = stmt.getText(); - row->severity = static_cast(stmt.getInt()); - row->malwareName = stmt.getText(); - row->detailedUrl = stmt.getText(); - row->ts = static_cast(stmt.getInt64()); - row->isIgnored = static_cast(stmt.getInt()); + row->targetName = stmt.getText(); // name + row->fileInAppPath = stmt.getText(); // file_path + row->dataVersion = stmt.getText(); // data_version + row->malwareName = stmt.getText(); // malware_name + row->detailedUrl = stmt.getText(); // detailed_url + row->severity = static_cast(stmt.getInt()); // severity + row->ts = static_cast(stmt.getInt64()); // detected_time + row->pkgId = pkgId; + row->isApp = true; return row; } -void Manager::insertDetectedMalware(const CsDetected &d, const std::string &dataVersion) +void Manager::insertName(const std::string &name) +{ + Statement stmt(m_conn, Query::INS_NAME); + + stmt.bind(name); + stmt.exec(); +} + +void Manager::insertDetected(const CsDetected &d, const std::string &filepath, + const std::string &dataVersion) { Statement stmt(m_conn, Query::INS_DETECTED); + stmt.bind(filepath); stmt.bind(d.targetName); stmt.bind(dataVersion); - stmt.bind(static_cast(d.severity)); stmt.bind(d.malwareName); stmt.bind(d.detailedUrl); + stmt.bind(static_cast(d.severity)); stmt.bind(static_cast(d.ts)); - stmt.bind(static_cast(false)); stmt.exec(); } -void Manager::setDetectedMalwareIgnored(const std::string &path, - bool flag) +void Manager::insertWorst(const std::string &pkgId, const std::string &name, + const std::string &filepath) { - Statement stmt(m_conn, Query::UPD_DETECTED_INGNORED); + Statement stmt(m_conn, Query::INS_WORST); + + stmt.bind(pkgId); + stmt.bind(name); + stmt.bind(filepath); + stmt.exec(); +} + +void Manager::updateIgnoreFlag(const std::string &name, bool flag) +{ + Statement stmt(m_conn, Query::UPD_IGNORE); + + stmt.bind((flag ? 1 : 0)); + stmt.bind(name); + stmt.exec(); +} + +void Manager::deleteDetectedByNameOnPath(const std::string &path) +{ + Statement stmt(m_conn, Query::DEL_DETECTED_BY_NAME_ON_PATH); - stmt.bind(flag); stmt.bind(path); stmt.exec(); } -void Manager::deleteDetectedMalware(const std::string &path) +void Manager::deleteDetectedByFilepathOnPath(const std::string &path) { - Statement stmt(m_conn, Query::DEL_DETECTED_BY_PATH); + Statement stmt(m_conn, Query::DEL_DETECTED_BY_FILEPATH_ON_PATH); stmt.bind(path); stmt.exec(); } -void Manager::deleteDeprecatedDetectedMalwares(const std::string &dir, - const std::string &dataVersion) +void Manager::deleteDetectedDeprecatedOnDir(const std::string &dir, + const std::string &dataVersion) { - Statement stmt(m_conn, Query::DEL_DETECTED_DEPRECATED); + Statement stmt(m_conn, Query::DEL_DETECTED_DEPRECATED_ON_DIR); stmt.bind(dir); stmt.bind(dataVersion); - stmt.exec(); } diff --git a/src/framework/db/manager.h b/src/framework/db/manager.h index 2f8a568..cf39157 100644 --- a/src/framework/db/manager.h +++ b/src/framework/db/manager.h @@ -52,13 +52,22 @@ public: void cleanLastScanTime(); // DETECTED_MALWARE_FILE & USER_RESPONSE - RowShPtrs getDetectedMalwares(const std::string &dirpath); - RowShPtr getDetectedMalware(const std::string &filepath); - void insertDetectedMalware(const CsDetected &, const std::string &dataVersion); - void setDetectedMalwareIgnored(const std::string &path, bool flag); - void deleteDetectedMalware(const std::string &path); - void deleteDeprecatedDetectedMalwares(const std::string &dir, - const std::string &dataVersion); + RowShPtr getDetectedByNameOnPath(const std::string &path); + RowShPtrs getDetectedByNameOnDir(const std::string &dir); + RowShPtrs getDetectedByFilepathOnDir(const std::string &dir); + RowShPtr getWorstByPkgId(const std::string &pkgId); + + void insertName(const std::string &name); + void insertDetected(const CsDetected &, const std::string &filename, + const std::string &dataVersion); + void insertWorst(const std::string &pkgId, const std::string &name, + const std::string &filepath); + + void updateIgnoreFlag(const std::string &name, bool flag); + void deleteDetectedByNameOnPath(const std::string &path); + void deleteDetectedByFilepathOnPath(const std::string &path); + void deleteDetectedDeprecatedOnDir(const std::string &dir, + const std::string &dataVersion); private: void resetDatabase(); diff --git a/src/framework/db/query.h b/src/framework/db/query.h index 64d3969..5bce1f9 100644 --- a/src/framework/db/query.h +++ b/src/framework/db/query.h @@ -39,7 +39,7 @@ const std::string SEL_SCAN_REQUEST = const std::string INS_SCAN_REQUEST = "insert or replace into SCAN_REQUEST (dir, last_scan, data_version)" - "values (?, ?, ?)"; + " values (?, ?, ?)"; const std::string DEL_SCAN_REQUEST_BY_DIR = "delete from SCAN_REQUEST where dir = ?"; @@ -47,34 +47,53 @@ const std::string DEL_SCAN_REQUEST_BY_DIR = const std::string DEL_SCAN_REQUEST = "delete from SCAN_REQUEST"; +const std::string SEL_DETECTED_BY_NAME_ON_PATH = + "select name, file_path, data_version, malware_name, detailed_url, severity," + " detected_time, pkg_id, is_ignored" + " from join_detecteds_by_name" + " where name = ?"; + +const std::string SEL_DETECTED_BY_NAME_ON_DIR = + "select name, file_path, data_version, malware_name, detailed_url, severity," + " detected_time, pkg_id, is_ignored" + " from join_detecteds_by_name" + " where name like ? || '%'"; + +const std::string SEL_DETECTED_BY_FILEPATH_ON_DIR = + "select name, file_path, data_version, malware_name, detailed_url, severity," + " detected_time, pkg_id, is_ignored" + " from join_detecteds_by_file_path" + " where file_path like ? || '%'"; + +const std::string SEL_WORST_BY_PKGID = + "select name, file_path, data_version, malware_name, detailed_url, severity," + " detected_time" + " from join_p_d" + " where pkg_id = ?"; + +const std::string INS_NAME = + "insert or replace into NAMES(name) values(?)"; -const std::string SEL_DETECTED_BY_DIR = - "SELECT path, data_version, " - "severity, malware_name, " - "detailed_url, detected_time, ignored " - "FROM detected_malware_file where path like ? || '%'"; +const std::string INS_DETECTED = + "insert or replace into DETECTED_MALWARE(file_path, name, data_version, malware_name," + " detailed_url, severity, detected_time)" + " values(?, ?, ?, ?, ?, ?, ?)"; -const std::string SEL_DETECTED_BY_PATH = - "SELECT path, data_version, " - "severity, malware_name, " - "detailed_url, detected_time, ignored " - "FROM detected_malware_file where path = ?"; +const std::string INS_WORST = + "insert or replace into PACKAGE_INFO(pkg_id, name, worst) values(?, ?, ?)"; -const std::string INS_DETECTED = - "insert or replace into DETECTED_MALWARE_FILE " - "(path, data_version, severity, malware_name, " - "detailed_url, detected_time, ignored) " - "values (?, ?, ?, ?, ?, ?, ?)"; +const std::string UPD_IGNORE = + "update NAMES set is_ignored = ? where name = ?"; -const std::string UPD_DETECTED_INGNORED = - "update DETECTED_MALWARE_FILE set ignored = ? where path = ?"; +const std::string DEL_DETECTED_BY_NAME_ON_PATH = + "delete from NAMES where name = ?"; -const std::string DEL_DETECTED_BY_PATH = - "delete from DETECTED_MALWARE_FILE where path = ?"; +const std::string DEL_DETECTED_BY_FILEPATH_ON_PATH = + "delete from DETECTED_MALWARE where file_path = ?"; -const std::string DEL_DETECTED_DEPRECATED = - "delete from DETECTED_MALWARE_FILE where path like ? || '%' " - "and data_version != ?"; +const std::string DEL_DETECTED_DEPRECATED_ON_DIR = + "delete from DETECTED_MALWARE where file_path like ? || '%' " + " and data_version != ?"; } // namespace Query } // namespace Db diff --git a/src/framework/db/row.h b/src/framework/db/row.h index 2dd9f76..c6f23ab 100644 --- a/src/framework/db/row.h +++ b/src/framework/db/row.h @@ -33,6 +33,7 @@ using RowShPtr = std::shared_ptr; using RowShPtrs = std::vector; struct Row : public Csr::CsDetected { + std::string fileInAppPath; // for case of file in app std::string dataVersion; // engine's data version bool isIgnored; diff --git a/src/framework/db/statement.cpp b/src/framework/db/statement.cpp index d12c83c..7320f6f 100644 --- a/src/framework/db/statement.cpp +++ b/src/framework/db/statement.cpp @@ -47,7 +47,6 @@ Statement::Statement(const Connection &db, const std::string &query) : default: ThrowExc(DbFailed, db.getErrorMessage()); } - } Statement::~Statement() @@ -177,14 +176,17 @@ sqlite3_int64 Statement::getInt64() const return sqlite3_column_int64(m_stmt, ++m_columnIndex); } -const char *Statement::getText() const +std::string Statement::getText() const { if (!isColumnIndexValid()) ThrowExc(DbFailed, "index overflowed when getting text from row." " query: " << m_query << " index: " << m_columnIndex); - return reinterpret_cast(sqlite3_column_text(m_stmt, - ++m_columnIndex)); + const char *text = reinterpret_cast(sqlite3_column_text(m_stmt, + ++m_columnIndex)); + std::string str = (text ? text : std::string()); + + return str; } } // namespace Db diff --git a/src/framework/db/statement.h b/src/framework/db/statement.h index 0ac2825..e771715 100644 --- a/src/framework/db/statement.h +++ b/src/framework/db/statement.h @@ -46,7 +46,7 @@ public: // get column values. index of column auto-incremented int getInt() const; sqlite3_int64 getInt64() const; - const char *getText() const; + std::string getText() const; bool isNullColumn() const; // it's checking func. not auto incremented. diff --git a/src/framework/service/cs-logic.cpp b/src/framework/service/cs-logic.cpp index 5a0d543..49cf60c 100644 --- a/src/framework/service/cs-logic.cpp +++ b/src/framework/service/cs-logic.cpp @@ -114,14 +114,15 @@ RawBuffer CsLogic::scanAppOnCloud(const CsContext &context, return this->handleUserResponse(detected); } -CsDetectedPtr CsLogic::scanAppDelta(const std::string &pkgPath, const std::string &pkgId) +CsDetectedPtr CsLogic::scanAppDelta(const std::string &pkgPath, const std::string &pkgId, + std::string &riskiestPath) { auto starttime = time(nullptr); CsEngineContext engineContext(this->m_loader); auto &c = engineContext.get(); - // traverse files in app and take which is more danger than riskiest. + // traverse files in app and take which is more danger than riskiest auto visitor = FsVisitor::create( pkgPath, this->m_db.getLastScanTime(pkgPath, this->m_dataVersion)); @@ -132,29 +133,31 @@ CsDetectedPtr CsLogic::scanAppDelta(const std::string &pkgPath, const std::strin csre_cs_detected_h result; toException(this->m_loader.scanFile(c, file->getPath(), &result)); - if (!result) + if (!result) { + this->m_db.deleteDetectedByFilepathOnPath(file->getPath()); continue; + } INFO("New malware detected on file: " << file->getPath()); - auto candidate = this->convert(result, file->getPath()); + auto candidate = this->convert(result, pkgPath); candidate.isApp = true; candidate.pkgId = pkgId; - this->m_db.insertDetectedMalware(candidate, this->m_dataVersion); - if (!riskiest) + this->m_db.insertName(pkgPath); + this->m_db.insertDetected(candidate, file->getPath(), this->m_dataVersion); + + if (!riskiest) { riskiest.reset(new CsDetected(std::move(candidate))); - else if (*riskiest < candidate) + riskiestPath = file->getPath(); + } else if (*riskiest < candidate) { *riskiest = std::move(candidate); + riskiestPath = file->getPath(); + } } this->m_db.insertLastScanTime(pkgPath, starttime, this->m_dataVersion); - if (riskiest) { - INFO("Riskiest malware selected in pkg: " << pkgPath); - riskiest->targetName = pkgPath; - } - return riskiest; } @@ -174,39 +177,111 @@ RawBuffer CsLogic::scanApp(const CsContext &context, const std::string &path) if (context.isScanOnCloud) return this->scanAppOnCloud(context, pkgPath, pkgId); - auto riskiest = this->scanAppDelta(pkgPath, pkgId); - auto history = this->m_db.getDetectedMalware(pkgPath); + // old history + auto history = this->m_db.getWorstByPkgId(pkgId); + // riskiest detected among newly scanned files + std::string riskiestPath; + auto riskiest = this->scanAppDelta(pkgPath, pkgId, riskiestPath); + // history after delta scan. if worst file is changed, it's rescanned in scanAppDelta + // and deleted from db if it's cured. if history != nullptr && after == nullptr, + // it means worst detected item is cured anyway. + auto after = this->m_db.getWorstByPkgId(pkgId); + if (history && after && riskiest) { + if (*history < *riskiest) { + INFO("worst case is remained but the more worst newly detected. on pkg[" << + pkgPath << "]"); + if (history->isIgnored) + this->m_db.updateIgnoreFlag(pkgPath, false); + + this->m_db.insertWorst(pkgId, pkgPath, riskiestPath); - if (riskiest && history) { - if (*riskiest > *history) { - INFO("new malware found and more risky! history should be updated."); - this->m_db.insertDetectedMalware(*riskiest, this->m_dataVersion); + riskiest->response = this->getUserResponse(context, *riskiest); + return this->handleUserResponse(*riskiest); } else { - INFO("new malware is found but not riskier than history. history reusable."); + INFO("worst case is remained and can be re-used on pkg[" << pkgPath << "]"); if (history->isIgnored) return BinaryQueue::Serialize(CSR_ERROR_NONE).pop(); history->response = this->getUserResponse(context, *history); return this->handleUserResponse(*history); } - } else if (riskiest && !history) { - INFO("new malware found and no history exist! history should be inserted."); - this->m_db.insertDetectedMalware(*riskiest, this->m_dataVersion); - } else if (!riskiest && history) { - INFO("no malware found and history exist! history reusable."); + } else if (history && after && !riskiest) { + INFO("worst case is remained and NO new detected. history can be re-used. " + "on pkg[" << pkgPath << "]"); if (history->isIgnored) return BinaryQueue::Serialize(CSR_ERROR_NONE).pop(); history->response = this->getUserResponse(context, *history); return this->handleUserResponse(*history); + } else if (history && !after && riskiest) { + INFO("worst case is deleted but new detected. we have to find out " + "worse case in db and compare it with riskiest first. on pkg[" << pkgPath << + "]"); + Db::RowShPtr worse; + for (auto &row : this->m_db.getDetectedByFilepathOnDir(pkgPath)) + if (!worse || *worse < *row) + worse = std::move(row); + + if (*riskiest < *worse) { + INFO("worse case in db is worse than riskiest. on pkg[" << pkgPath << "]"); + riskiestPath = worse->fileInAppPath; + *riskiest = std::move(*worse); + } + + if (*history < *riskiest) { + INFO("worst case is deleted but the more worst newly detected. on pkg[" << + pkgPath << "]"); + if (history->isIgnored) + this->m_db.updateIgnoreFlag(pkgPath, false); + + this->m_db.insertWorst(pkgId, pkgPath, riskiestPath); + + riskiest->response = this->getUserResponse(context, *riskiest); + return this->handleUserResponse(*riskiest); + } else { + INFO("worst case is deleted but same or less level newly detected. on pkg[" << + pkgPath << "]"); + this->m_db.insertWorst(pkgId, pkgPath, riskiestPath); + + if (history->isIgnored) + return BinaryQueue::Serialize(CSR_ERROR_NONE).pop(); + + riskiest->response = this->getUserResponse(context, *riskiest); + return this->handleUserResponse(*riskiest); + } + } else if (history && !after && !riskiest) { + auto rows = this->m_db.getDetectedByFilepathOnDir(pkgPath); + if (rows.empty()) { + INFO("worst case is deleted cascadingly and NO new detected and " + "NO worse case. the pkg[" << pkgPath << "] is clean."); + this->m_db.deleteDetectedByNameOnPath(pkgPath); + return BinaryQueue::Serialize(CSR_ERROR_NONE).pop(); + } else { + INFO("worst case is deleted cascadingly and NO new detected and " + "worse case exist on pkg[" << pkgPath << "]. insert it to worst."); + Db::RowShPtr worse; + for (auto &row : rows) + if (!worse || *worse < *row) + worse = std::move(row); + + this->m_db.insertWorst(pkgId, pkgPath, worse->fileInAppPath); + + if (worse->isIgnored) + return BinaryQueue::Serialize(CSR_ERROR_NONE).pop(); + + worse->response = this->getUserResponse(context, *worse); + return this->handleUserResponse(*worse); + } + } else if (!history && riskiest) { + INFO("no history and new detected"); + this->m_db.insertWorst(pkgId, pkgPath, riskiestPath); + + riskiest->response = this->getUserResponse(context, *riskiest); + return this->handleUserResponse(*riskiest); } else { - INFO("no malware found and no history exist! it's clean!"); + DEBUG("no history and no new detected"); return BinaryQueue::Serialize(CSR_ERROR_NONE).pop(); } - - // new and more risky malware found and db updated case only left. - riskiest->response = this->getUserResponse(context, *riskiest); - return this->handleUserResponse(*riskiest); } RawBuffer CsLogic::scanFileWithoutDelta(const CsContext &context, @@ -224,7 +299,8 @@ RawBuffer CsLogic::scanFileWithoutDelta(const CsContext &context, auto d = this->convert(result, filepath); - this->m_db.insertDetectedMalware(d, this->m_dataVersion); + this->m_db.insertName(d.targetName); + this->m_db.insertDetected(d, d.targetName, this->m_dataVersion); d.response = this->getUserResponse(context, d); return this->handleUserResponse(d, std::forward(fileptr)); @@ -241,7 +317,7 @@ RawBuffer CsLogic::scanFile(const CsContext &context, const std::string &filepat DEBUG("Scan request on file: " << filepath); - auto history = this->m_db.getDetectedMalware(filepath); + auto history = this->m_db.getDetectedByNameOnPath(filepath); FilePtr fileptr; @@ -256,7 +332,7 @@ RawBuffer CsLogic::scanFile(const CsContext &context, const std::string &filepat // OR there's no history at all. if (fileptr) { if (history) - this->m_db.deleteDetectedMalware(filepath); + this->m_db.deleteDetectedByNameOnPath(filepath); if (fileptr->isDir()) ThrowExc(FileSystemError, "file type shouldn't be directory: " << filepath); @@ -330,18 +406,16 @@ RawBuffer CsLogic::getScannableFiles(const std::string &dir) if (lastScanTime != -1) { // for case: scan history exist and not modified. - for (auto &row : this->m_db.getDetectedMalwares(dir)) { + for (auto &row : this->m_db.getDetectedByNameOnDir(File::getPkgPath(dir))) { try { auto fileptr = File::create(row->targetName); - if (fileptr->isInApp()) - fileset.insert(fileptr->getAppPkgPath()); - else - fileset.insert(fileptr->getPath()); + fileset.insert(fileptr->isInApp() ? + fileptr->getAppPkgPath() : fileptr->getPath()); } catch (const FileDoNotExist &) { - this->m_db.deleteDetectedMalware(row->targetName); + this->m_db.deleteDetectedByNameOnPath(row->targetName); } catch (const FileSystemError &) { - this->m_db.deleteDetectedMalware(row->targetName); + this->m_db.deleteDetectedByNameOnPath(row->targetName); } } } @@ -371,14 +445,14 @@ RawBuffer CsLogic::judgeStatus(const std::string &filepath, csr_cs_action_e acti file = File::create(filepath); } catch (const Exception &e) { ERROR("file system related exception occured on file: " << filepath << - " let's refresh detected malware history..."); - - this->m_db.deleteDetectedMalware(filepath); + " This case might be file not exist or type invalid," + " file has changed anyway... Don't refresh detected history to know that" + " it's changed since the time."); throw; } - auto history = this->m_db.getDetectedMalware(filepath); + auto history = this->m_db.getDetectedByNameOnPath(File::getPkgPath(filepath)); if (!history) { ERROR("Target to be judged doesn't exist in db. name: " << filepath); @@ -387,26 +461,25 @@ RawBuffer CsLogic::judgeStatus(const std::string &filepath, csr_cs_action_e acti // TODO: make isModifiedSince member function to File class // not to regenerate like this. - if (File::create(filepath, static_cast(history->ts))) { - this->m_db.deleteDetectedMalware(filepath); - - ThrowExc(FileSystemError, "Target modified since db delta inserted. " - "name: " << filepath); - } + if (File::create(filepath, static_cast(history->ts))) + ThrowExc(FileSystemError, "File[" << filepath << "] modified since db delta " + "inserted. Don't refresh detected history to know that it's changed " + "since the time."); switch (action) { case CSR_CS_ACTION_REMOVE: file->remove(); - this->m_db.deleteDetectedMalware(filepath); + this->m_db.deleteDetectedByNameOnPath( + (file->isInApp() ? file->getAppPkgPath() : file->getPath())); break; case CSR_CS_ACTION_IGNORE: - this->m_db.setDetectedMalwareIgnored(filepath, true); + this->m_db.updateIgnoreFlag(File::getPkgPath(filepath), true); break; case CSR_CS_ACTION_UNIGNORE: - this->m_db.setDetectedMalwareIgnored(filepath, false); + this->m_db.updateIgnoreFlag(File::getPkgPath(filepath), false); break; default: @@ -427,7 +500,7 @@ RawBuffer CsLogic::getDetected(const std::string &filepath) { EXCEPTION_GUARD_START - auto row = this->m_db.getDetectedMalware(filepath); + auto row = this->m_db.getDetectedByNameOnPath(File::getPkgPath(filepath)); if (row && !row->isIgnored) return BinaryQueue::Serialize(CSR_ERROR_NONE, row).pop(); @@ -448,7 +521,7 @@ RawBuffer CsLogic::getDetectedList(const StrSet &dirSet) Db::RowShPtrs rows; std::for_each(dirSet.begin(), dirSet.end(), [this, &rows](const std::string & dir) { - for (auto &row : this->m_db.getDetectedMalwares(dir)) + for (auto &row : this->m_db.getDetectedByNameOnDir(File::getPkgPath(dir))) if (!row->isIgnored) rows.emplace_back(std::move(row)); }); @@ -470,7 +543,7 @@ RawBuffer CsLogic::getIgnored(const std::string &filepath) { EXCEPTION_GUARD_START - auto row = this->m_db.getDetectedMalware(filepath); + auto row = this->m_db.getDetectedByNameOnPath(File::getPkgPath(filepath)); if (row && row->isIgnored) return BinaryQueue::Serialize(CSR_ERROR_NONE, row).pop(); @@ -491,7 +564,7 @@ RawBuffer CsLogic::getIgnoredList(const StrSet &dirSet) Db::RowShPtrs rows; std::for_each(dirSet.begin(), dirSet.end(), [this, &rows](const std::string & dir) { - for (auto &row : this->m_db.getDetectedMalwares(dir)) + for (auto &row : this->m_db.getDetectedByNameOnDir(File::getPkgPath(dir))) if (row->isIgnored) rows.emplace_back(std::move(row)); }); @@ -512,7 +585,7 @@ RawBuffer CsLogic::handleUserResponse(const CsDetected &d, FilePtr &&fileptr) { switch (d.response) { case CSR_CS_IGNORE: - this->m_db.setDetectedMalwareIgnored(d.targetName, true); + this->m_db.updateIgnoreFlag(File::getPkgPath(d.targetName), true); break; case CSR_CS_REMOVE: @@ -531,7 +604,7 @@ RawBuffer CsLogic::handleUserResponse(const CsDetected &d, FilePtr &&fileptr) WARN("File type is changed, considered as different file: " << d.targetName); } - this->m_db.deleteDetectedMalware(d.targetName); + this->m_db.deleteDetectedByNameOnPath(File::getPkgPath(d.targetName)); break; case CSR_CS_SKIP: diff --git a/src/framework/service/cs-logic.h b/src/framework/service/cs-logic.h index 9b27275..efbe53a 100644 --- a/src/framework/service/cs-logic.h +++ b/src/framework/service/cs-logic.h @@ -53,7 +53,8 @@ private: RawBuffer scanApp(const CsContext &context, const std::string &path); RawBuffer scanAppOnCloud(const CsContext &context, const std::string &pkgPath, const std::string &pkgId); - CsDetectedPtr scanAppDelta(const std::string &pkgPath, const std::string &pkgId); + CsDetectedPtr scanAppDelta(const std::string &pkgPath, const std::string &pkgId, + std::string &riskiestPath); RawBuffer scanFileWithoutDelta(const CsContext &context, const std::string &filepath, FilePtr &&fileptr); diff --git a/src/framework/service/file-system.cpp b/src/framework/service/file-system.cpp index 3d9e234..e2d4c2a 100644 --- a/src/framework/service/file-system.cpp +++ b/src/framework/service/file-system.cpp @@ -76,6 +76,23 @@ bool File::isInApp(const std::string &path) return false; } +std::string File::getPkgPath(const std::string &path) +{ + std::smatch matched; + + for (const auto ®e : g_regexprs) { + if (!std::regex_search(path, matched, rege)) + continue; + + if (matched.size() == 3) + return matched[1]; + else if (matched.size() == 4) + return matched[1]; + } + + return path; +} + File::File(const std::string &fpath, bool isDir) : m_path(fpath), m_inApp(false), m_isDir(isDir) { std::smatch matched; diff --git a/src/framework/service/file-system.h b/src/framework/service/file-system.h index a64997a..d03ec76 100644 --- a/src/framework/service/file-system.h +++ b/src/framework/service/file-system.h @@ -53,6 +53,7 @@ public: static FilePtr create(const std::string &fpath, time_t modifiedSince = -1); static bool isInApp(const std::string &path); + static std::string getPkgPath(const std::string &path); private: explicit File(const std::string &fpath, bool isDir); diff --git a/test/internals/test-db.cpp b/test/internals/test-db.cpp index 30f4742..c9b818a 100644 --- a/test/internals/test-db.cpp +++ b/test/internals/test-db.cpp @@ -133,27 +133,29 @@ BOOST_AUTO_TEST_CASE(detected_malware_file) malware3.ts = 310; // select test with vacant data - auto detected = db.getDetectedMalware(malware1.targetName); + auto detected = db.getDetectedByNameOnPath(malware1.targetName); CHECK_IS_NULL(detected); - auto detectedList = db.getDetectedMalwares("/opt"); + auto detectedList = db.getDetectedByNameOnDir("/opt"); ASSERT_IF(detectedList.empty(), true); - db.insertDetectedMalware(malware1, initDataVersion); - detected = db.getDetectedMalware(malware1.targetName); + db.insertName(malware1.targetName); + db.insertDetected(malware1, malware1.targetName, initDataVersion); + detected = db.getDetectedByNameOnPath(malware1.targetName); checkSameMalware(malware1, *detected); ASSERT_IF(detected->dataVersion, initDataVersion); ASSERT_IF(detected->isIgnored, false); - db.insertDetectedMalware(malware2, initDataVersion); - db.setDetectedMalwareIgnored(malware2.targetName, true); - detected = db.getDetectedMalware(malware2.targetName); + db.insertName(malware2.targetName); + db.insertDetected(malware2, malware2.targetName, initDataVersion); + db.updateIgnoreFlag(malware2.targetName, true); + detected = db.getDetectedByNameOnPath(malware2.targetName); checkSameMalware(malware2, *detected); ASSERT_IF(detected->dataVersion, initDataVersion); ASSERT_IF(detected->isIgnored, true); // getDetectedMalwares test - detectedList = db.getDetectedMalwares("/opt"); + detectedList = db.getDetectedByNameOnDir("/opt"); ASSERT_IF(detectedList.size(), static_cast(2)); for (auto &item : detectedList) { @@ -166,25 +168,26 @@ BOOST_AUTO_TEST_CASE(detected_malware_file) } // setDetectedMalwareIgnored test - db.setDetectedMalwareIgnored(malware1.targetName, true); - detected = db.getDetectedMalware(malware1.targetName); + db.updateIgnoreFlag(malware1.targetName, true); + detected = db.getDetectedByNameOnPath(malware1.targetName); checkSameMalware(malware1, *detected); ASSERT_IF(detected->isIgnored, true); // deleteDeprecatedDetectedMalwares test - db.insertDetectedMalware(malware3, changedDataVersion); - db.deleteDeprecatedDetectedMalwares("/opt", changedDataVersion); - detected = db.getDetectedMalware(malware3.targetName); + db.insertName(malware3.targetName); + db.insertDetected(malware3, malware3.targetName, changedDataVersion); + db.deleteDetectedDeprecatedOnDir("/opt", changedDataVersion); + detected = db.getDetectedByNameOnPath(malware3.targetName); checkSameMalware(malware3, *detected); ASSERT_IF(detected->dataVersion, changedDataVersion); ASSERT_IF(detected->isIgnored, false); - CHECK_IS_NULL(db.getDetectedMalware(malware1.targetName)); - CHECK_IS_NULL(db.getDetectedMalware(malware2.targetName)); + CHECK_IS_NULL(db.getDetectedByNameOnPath(malware1.targetName)); + CHECK_IS_NULL(db.getDetectedByNameOnPath(malware2.targetName)); // deleteDetectedMalware test - db.deleteDetectedMalware(malware3.targetName); - CHECK_IS_NULL(db.getDetectedMalware(malware3.targetName)); + db.deleteDetectedByNameOnPath(malware3.targetName); + CHECK_IS_NULL(db.getDetectedByNameOnPath(malware3.targetName)); EXCEPTION_GUARD_END } diff --git a/test/internals/test-wp-loader.cpp b/test/internals/test-wp-loader.cpp index 8737e59..f345467 100755 --- a/test/internals/test-wp-loader.cpp +++ b/test/internals/test-wp-loader.cpp @@ -40,15 +40,16 @@ struct Result { csre_wp_risk_level_e risk_level; std::string detailed_url; - Result(csre_wp_risk_level_e r, const char *durl) : risk_level(r), - detailed_url(durl) {} + Result(csre_wp_risk_level_e r, const char *durl) : + risk_level(r), + detailed_url(durl ? durl : std::string()) {} }; std::unordered_map ExpectedResult = { - {"http://normal.test.com", Result(CSRE_WP_RISK_UNVERIFIED, "")}, + {"http://normal.test.com", Result(CSRE_WP_RISK_UNVERIFIED, nullptr)}, {"http://highrisky.test.com", Result(CSRE_WP_RISK_HIGH, "http://high.risky.com")}, {"http://mediumrisky.test.com", Result(CSRE_WP_RISK_MEDIUM, "http://medium.risky.com")}, - {"http://lowrisky.test.com", Result(CSRE_WP_RISK_LOW, "")} + {"http://lowrisky.test.com", Result(CSRE_WP_RISK_LOW, "http://low.risky.com")} }; inline void checkResult(Csr::WpLoader &loader, diff --git a/test/test-helper.cpp b/test/test-helper.cpp index d89c340..43544f7 100644 --- a/test/test-helper.cpp +++ b/test/test-helper.cpp @@ -62,12 +62,12 @@ void ASSERT_DETECTED_EXT(csr_cs_detected_h detected, time_t e_timestamp, ASSERT_IF(a_is_app, e_is_app); ASSERT_IF(a_pkg_id, e_pkg_id); - BOOST_CHECK_MESSAGE(e_timestamp <= a_timestamp, - "Actual detected item's time stamp is later than expected time " - "stamp (which is start time before scan_file maybe..). this case " - "should be the returned detected item comes from history which is " - "scanned in the past. actual time: " << a_timestamp << - " expected(started) time: " << e_timestamp); + BOOST_WARN_MESSAGE(e_timestamp <= a_timestamp, + "Actual detected item's time stamp is later than expected time " + "stamp (which is start time before scan_file maybe..). this case " + "should be the returned detected item comes from history which is " + "scanned in the past. actual time: " << a_timestamp << + " expected(started) time: " << e_timestamp); } void ASSERT_DETECTED_HANDLE(csr_cs_detected_h expected, csr_cs_detected_h actual) -- 2.7.4