From: Michal Michalski Date: Thu, 14 Nov 2019 16:54:50 +0000 (+0100) Subject: [common] JsonFilter prototype implementation. X-Git-Tag: submit/tizen/20191211.130748~25^2 X-Git-Url: http://review.tizen.org/git/?a=commitdiff_plain;h=refs%2Fchanges%2F28%2F217828%2F2;p=platform%2Fcore%2Fapi%2Fwebapi-plugins.git [common] JsonFilter prototype implementation. In present WebAPI implementation, several modules implement their own data filtering algorithms. This commit is an attempt to provide a common, unified way to perform data filtering across entire API. Design goal for this change was simplicity. I was not looking for ways to optimize the performance, as this will require in-depth performance testing in specific use cases. [Verification] Unit tests for JsonFilter class has been implemented which validate main logic. Signed-off-by: Michal Michalski Change-Id: I6cfb9fc98f6a1be8d6364a9dfff2bf07ba239dfa --- diff --git a/src/common/common.gyp b/src/common/common.gyp index 1baffe06..3e845a1e 100644 --- a/src/common/common.gyp +++ b/src/common/common.gyp @@ -21,6 +21,8 @@ 'picojson.h', 'json-utils.cc', 'json-utils.h', + 'json-filter.cc', + 'json-filter.h', 'utils.h', 'logger.cc', 'logger.h', diff --git a/src/common/common_ut.gyp b/src/common/common_ut.gyp index 61db8ad3..2956ca9d 100644 --- a/src/common/common_ut.gyp +++ b/src/common/common_ut.gyp @@ -20,6 +20,7 @@ '../googlemock/src/gmock-all.cc', 'ut/common_ut_extension.cc', 'ut/json-utils.cc', + 'ut/json-filter.cc', 'ut/main.cc' ], 'libraries': [ diff --git a/src/common/json-filter.cc b/src/common/json-filter.cc new file mode 100644 index 00000000..58dd98bc --- /dev/null +++ b/src/common/json-filter.cc @@ -0,0 +1,182 @@ +/* + * Copyright (c) 2019 Samsung Electronics Co., Ltd All Rights Reserved + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "common/json-filter.h" + +namespace internal { + +std::vector SplitString(std::string text, char delimiter) { + ScopeLogger(); + std::vector parts; + std::string word = ""; + for (char c : text) { + if (c == delimiter) { + if (!word.empty()) { + parts.push_back(word); + word.clear(); + } + } else { + word.push_back(c); + } + } + if (!word.empty()) { + parts.push_back(word); + } + return parts; +} + +picojson::value GetAttribute(std::string attribute, picojson::value json) { + ScopeLogger(); + for (auto node : SplitString(attribute, kAttributeSeparator)) { + if (!json.is()) { + throw "filtered attribute not found"; + } + + auto obj = json.get(); + if (obj.find(node) == obj.end()) { + throw "filtered attribute not found"; + } + + json = obj.at(node); + } + return json; +} + +} // namespace internal + +namespace common { + +JsonFilter::JsonFilter(picojson::value filter) : filter(filter) { + ScopeLogger(); + operators = {{"$AND", &JsonFilter::AndOperator}, + {"$OR", &JsonFilter::OrOperator}, + {"$EXACTLY", &JsonFilter::EqualsOperator}, + {"$CONTAINS", &JsonFilter::ContainsOperator}}; +} + +JsonFilter::~JsonFilter() { + ScopeLogger(); +} + +picojson::array JsonFilter::Filter(const picojson::array& records) const { + ScopeLogger(); + picojson::array filtered; + for (const auto& record : records) { + if (IsMatch(record)) { + filtered.push_back(record); + } + } + return filtered; +} + +bool JsonFilter::IsMatch(picojson::value record) const { + if (!filter.is()) { + throw "filter is not a json object"; + } + return EvaluateNode(filter.get(), record); +} + +bool JsonFilter::AndOperator(picojson::array args, picojson::value record) const { + ScopeLogger(); + bool result = true; + for (const auto& value : args) { + if (value.is()) { + result = result && EvaluateNode(value.get(), record); + } else if (value.is()) { + auto attribute = internal::GetAttribute(value.get(), record); + result = result && attribute.evaluate_as_boolean(); + } else { + result = result && value.evaluate_as_boolean(); + } + } + return result; +} + +bool JsonFilter::OrOperator(picojson::array args, picojson::value record) const { + ScopeLogger(); + bool result = false; + for (const auto& value : args) { + if (value.is()) { + result = result || EvaluateNode(value.get(), record); + } else if (value.is()) { + auto attribute = internal::GetAttribute(value.get(), record); + result = result || attribute.evaluate_as_boolean(); + } else { + result = result || value.evaluate_as_boolean(); + } + } + return result; +} + +bool JsonFilter::EqualsOperator(picojson::array args, picojson::value record) const { + ScopeLogger(); + if (args.size() != 2) { + throw "equals operator takes exactly 2 arguments"; + } + + if (!args[0].is()) { + throw "equals operator first argument must be a string (attribute path)"; + } + + auto attribute = internal::GetAttribute(args[0].get(), record); + return args[1].serialize() == attribute.serialize(); +} + +bool JsonFilter::ContainsOperator(picojson::array args, picojson::value record) const { + ScopeLogger(); + if (args.size() != 2) { + throw "contains operator takes exactly 2 arguments"; + } + + if (!args[0].is() || !args[1].is()) { + throw "contains operator arguments must be a string"; + } + + auto attribute = internal::GetAttribute(args[0].get(), record); + if (!attribute.is()) { + throw "attribute for contains operator must have a string value type"; + } + + return attribute.get().find(args[1].get()) != std::string::npos; +} + +bool JsonFilter::IsOperator(std::string identifier) const { + ScopeLogger(); + return operators.find(identifier) != operators.end(); +} + +bool JsonFilter::EvaluateOperator(std::string opType, picojson::value arguments, + picojson::value record) const { + ScopeLogger(); + if (!arguments.is()) { + throw "Operator requires an array of arguments"; + } + return (this->*(operators.at(opType)))(arguments.get(), record); +} + +bool JsonFilter::EvaluateNode(picojson::object node, picojson::value record) const { + ScopeLogger(); + bool result = true; + for (const auto attr : node) { + if (IsOperator(attr.first)) { + bool value = EvaluateOperator(attr.first, attr.second, record); + result = result && value; + } + } + return result; +} + +} // namespace common diff --git a/src/common/json-filter.h b/src/common/json-filter.h new file mode 100644 index 00000000..ea0a2489 --- /dev/null +++ b/src/common/json-filter.h @@ -0,0 +1,63 @@ +/* + * Copyright (c) 2019 Samsung Electronics Co., Ltd All Rights Reserved + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef COMMON_JSON_FILTER_H +#define COMMON_JSON_FILTER_H + +#include +#include +#include + +#include "picojson.h" + +namespace common { + +class JsonFilter { + using FilterOperator = bool (JsonFilter::*)(picojson::array, picojson::value) const; + + public: + JsonFilter(picojson::value filter = picojson::value()); + ~JsonFilter(); + + picojson::array Filter(const picojson::array& records) const; + bool IsMatch(picojson::value record) const; + + bool EvaluateNode(picojson::object node, picojson::value record) const; + bool IsOperator(std::string attr) const; + bool EvaluateOperator(std::string opType, picojson::value arguments, + picojson::value record) const; + + bool AndOperator(picojson::array args, picojson::value record) const; + bool OrOperator(picojson::array args, picojson::value record) const; + bool EqualsOperator(picojson::array args, picojson::value record) const; + bool ContainsOperator(picojson::array args, picojson::value record) const; + + private: + picojson::value filter; + std::map operators; +}; + +} // namespace common + +namespace internal { + +const char kAttributeSeparator = '.'; +std::vector SplitString(std::string text, char delimiter = kAttributeSeparator); +picojson::value GetAttribute(std::string path, picojson::value json); + +} // namespace internal + +#endif // COMMON_JSON_FILTER_H diff --git a/src/common/ut/json-filter.cc b/src/common/ut/json-filter.cc new file mode 100644 index 00000000..366788df --- /dev/null +++ b/src/common/ut/json-filter.cc @@ -0,0 +1,331 @@ +/* + * Copyright (c) 2019 Samsung Electronics Co., Ltd All Rights Reserved + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "gmock/gmock.h" +#include "gtest/gtest.h" + +#include "common/json-filter.h" +#include "common/ut/json-filter.h" + +#include +#include +#include +#include + +namespace { + +picojson::value jsonFromString(std::string json) { + picojson::value value; + picojson::parse(value, json); + return value; +} + +} // namespace + +class SplitStringTest : public testing::Test {}; + +TEST_F(SplitStringTest, BasicTest) { + std::string input = "attr1.attr2.attr3"; + std::vector expected = {"attr1", "attr2", "attr3"}; + auto output = internal::SplitString(input, '.'); + ASSERT_EQ(output.size(), expected.size()); + ASSERT_EQ(output[0], expected[0]); + ASSERT_EQ(output[1], expected[1]); + ASSERT_EQ(output[2], expected[2]); +} + +TEST_F(SplitStringTest, StartsWithDelimiter) { + std::string input = ".attr1.attr2"; + std::vector expected = {"attr1", "attr2"}; + auto output = internal::SplitString(input, '.'); + ASSERT_EQ(output.size(), expected.size()); + ASSERT_EQ(output[0], expected[0]); + ASSERT_EQ(output[1], expected[1]); +} + +TEST_F(SplitStringTest, EndsWithDelimiter) { + std::string input = "attr1.attr2."; + std::vector expected = {"attr1", "attr2"}; + auto output = internal::SplitString(input, '.'); + ASSERT_EQ(output.size(), expected.size()); + ASSERT_EQ(output[0], expected[0]); + ASSERT_EQ(output[1], expected[1]); +} + +TEST_F(SplitStringTest, DelimitersOnly) { + std::string input = "..."; + std::vector expected = {}; + auto output = internal::SplitString(input, '.'); + ASSERT_EQ(output.size(), expected.size()); +} + +TEST_F(SplitStringTest, SingleAttributeOnly) { + std::string input = "attr"; + std::vector expected = {"attr"}; + auto output = internal::SplitString(input, '.'); + ASSERT_EQ(output.size(), expected.size()); + ASSERT_EQ(output[0], expected[0]); +} + +class GetAttributeTest : public testing::Test {}; + +TEST_F(GetAttributeTest, BasicTest) { + std::string attribute = "attr1.attr2"; + std::string json = "{\"attr1\": {\"attr2\": \"value\"}}"; + picojson::value expected("value"); + auto output = internal::GetAttribute(attribute, jsonFromString(json)); + ASSERT_EQ(output.serialize(), expected.serialize()); +} + +TEST_F(GetAttributeTest, TopLevelAttribute) { + std::string attribute = "attr"; + std::string json = "{\"attr\": \"value\"}"; + picojson::value expected("value"); + auto output = internal::GetAttribute(attribute, jsonFromString(json)); + ASSERT_EQ(output.serialize(), expected.serialize()); + ASSERT_EQ(output.get(), expected.get()); +} + +TEST_F(GetAttributeTest, AttributeNotFound) { + std::string attribute = "bad_attr"; + std::string json = "{\"attr\": \"value\"}"; + ASSERT_THROW(auto output = internal::GetAttribute(attribute, jsonFromString(json)), const char*); +} + +TEST_F(GetAttributeTest, EmptyAttribute) { + std::string attribute = ""; + std::string json = "{\"attr\": \"value\"}"; + auto output = internal::GetAttribute(attribute, jsonFromString(json)); + ASSERT_EQ(output.serialize(), jsonFromString(json).serialize()); +} + +class EqualsOperatorTest : public testing::Test {}; + +TEST_F(EqualsOperatorTest, BasicPositiveTest) { + common::JsonFilter jf; + picojson::array arguments = {picojson::value("attr1.attr2"), + jsonFromString("{\"name\": \"value\"}")}; + auto record = jsonFromString("{\"attr1\": {\"attr2\": {\"name\": \"value\"}}}"); + bool output = jf.EqualsOperator(arguments, record); + ASSERT_TRUE(output); +} + +TEST_F(EqualsOperatorTest, BasicNegativeTest) { + common::JsonFilter jf; + picojson::array arguments = {picojson::value("attr1.attr2"), + jsonFromString("{\"name\": \"value\"}")}; + auto record = jsonFromString("{\"attr1\": {\"attr2\": {\"name\": \"different\"}}}"); + bool output = jf.EqualsOperator(arguments, record); + ASSERT_FALSE(output); +} + +TEST_F(EqualsOperatorTest, InvalidNumberOfArguments) { + common::JsonFilter jf; + picojson::array arguments = {picojson::value("attr1.attr2")}; + auto record = jsonFromString("{\"attr1\": {\"attr2\": {\"name\": \"different\"}}}"); + ASSERT_THROW(jf.EqualsOperator(arguments, record), const char*); +} + +TEST_F(EqualsOperatorTest, FirstArgumentNotString) { + common::JsonFilter jf; + picojson::array arguments = {picojson::value(false), jsonFromString("{}")}; + auto record = jsonFromString("{\"attr1\": {\"attr2\": {\"name\": \"different\"}}}"); + ASSERT_THROW(jf.EqualsOperator(arguments, record), const char*); +} + +class ContainsOperatorTest : public testing::Test {}; + +TEST_F(ContainsOperatorTest, BasicPositiveTest) { + common::JsonFilter jf; + picojson::array arguments = {picojson::value("attr1.attr2"), picojson::value("ello")}; + auto record = jsonFromString("{\"attr1\": {\"attr2\": \"Hello, world!\"}}"); + bool output = jf.ContainsOperator(arguments, record); + ASSERT_TRUE(output); +} + +TEST_F(ContainsOperatorTest, BasicNegativeTest) { + common::JsonFilter jf; + picojson::array arguments = {picojson::value("attr1.attr2"), picojson::value("tizen")}; + auto record = jsonFromString("{\"attr1\": {\"attr2\": \"Hello, world!\"}}"); + bool output = jf.ContainsOperator(arguments, record); + ASSERT_FALSE(output); +} + +TEST_F(ContainsOperatorTest, InvalidNumberOfArguments) { + common::JsonFilter jf; + picojson::array arguments = { + picojson::value("attr1.attr2"), + }; + auto record = jsonFromString("{}"); + ASSERT_THROW(jf.ContainsOperator(arguments, record), const char*); +} + +TEST_F(ContainsOperatorTest, InvalidArgumentsType) { + common::JsonFilter jf; + picojson::array arguments = {picojson::value(123.5), picojson::value("substring")}; + auto record = jsonFromString("{}"); + ASSERT_THROW(jf.ContainsOperator(arguments, record), const char*); +} + +TEST_F(ContainsOperatorTest, InvalidAttributeValueType) { + common::JsonFilter jf; + picojson::array arguments = {picojson::value("attr1.attr2"), picojson::value("substring")}; + auto record = jsonFromString("{\"attr1\": {\"attr2\": 123.5}}"); + ASSERT_THROW(jf.ContainsOperator(arguments, record), const char*); +} + +class AndOperatorTest : public ::testing::Test {}; + +TEST_F(AndOperatorTest, BasicPositiveTest) { + common::JsonFilter jf; + picojson::array arguments = { + picojson::value("attr1.attr2"), picojson::value("attr1.attr3"), + }; + auto record = jsonFromString("{\"attr1\": {\"attr2\": true, \"attr3\": true}}"); + bool output = jf.AndOperator(arguments, record); + ASSERT_TRUE(output); +} + +TEST_F(AndOperatorTest, BasicNegativeTest) { + common::JsonFilter jf; + picojson::array arguments = { + picojson::value("attr1.attr2"), picojson::value("attr1.attr3"), + }; + auto record = jsonFromString("{\"attr1\": {\"attr2\": true, \"attr3\": false}}"); + bool output = jf.AndOperator(arguments, record); + ASSERT_FALSE(output); +} + +TEST_F(AndOperatorTest, EvaluateWeirdArgumentsAsBooleans) { + common::JsonFilter jf; + picojson::array arguments = {picojson::value(127.5), picojson::value(picojson::array())}; + auto record = jsonFromString("{}"); + bool output = jf.AndOperator(arguments, record); + ASSERT_TRUE(output); +} + +class OrOperatorTest : public ::testing::Test {}; + +TEST_F(OrOperatorTest, BasicPositiveTest) { + common::JsonFilter jf; + picojson::array arguments = { + picojson::value("attr1.attr2"), picojson::value("attr1.attr3"), + }; + auto record = jsonFromString("{\"attr1\": {\"attr2\": true, \"attr3\": false}}"); + bool output = jf.OrOperator(arguments, record); + ASSERT_TRUE(output); +} + +TEST_F(OrOperatorTest, BasicNegativeTest) { + common::JsonFilter jf; + picojson::array arguments = { + picojson::value("attr1.attr2"), picojson::value("attr1.attr3"), + }; + auto record = jsonFromString("{\"attr1\": {\"attr2\": false, \"attr3\": false}}"); + bool output = jf.OrOperator(arguments, record); + ASSERT_FALSE(output); +} + +TEST_F(OrOperatorTest, EvaluateWeirdArgumentsAsBooleans) { + common::JsonFilter jf; + picojson::array arguments = {picojson::value(127.5), picojson::value()}; + auto record = jsonFromString("{}"); + bool output = jf.OrOperator(arguments, record); + ASSERT_TRUE(output); +} + +class EvaluateOperatorTest : public ::testing::Test {}; + +TEST_F(EvaluateOperatorTest, BasicTest) { + common::JsonFilter jf; + auto arguments = jsonFromString("[\"attr1.attr2\", \"attr1.attr3\"]"); + auto record = jsonFromString("{\"attr1\": {\"attr2\": true, \"attr3\": false}}"); + EXPECT_FALSE(jf.EvaluateOperator("$AND", arguments, record)); + EXPECT_TRUE(jf.EvaluateOperator("$OR", arguments, record)); +} + +TEST_F(EvaluateOperatorTest, ArgumentsNotAnArray) { + common::JsonFilter jf; + auto arguments = jsonFromString("{}"); + auto record = jsonFromString("{}"); + ASSERT_THROW(jf.EvaluateOperator("$AND", arguments, record), const char*); +} + +class EvaluateNodeTest : public ::testing::Test {}; + +TEST_F(EvaluateNodeTest, BasicPositiveTest) { + common::JsonFilter jf; + auto node = jsonFromString( + "{ " + " \"$AND\": [{ " + " \"$EXACTLY\": [ " + " \"attr1.attr2\"," + " \"tizen\" " + " ], " + " \"$CONTAINS\": [ " + " \"attr1.attr3\"," + " \"ello\" " + " ] " + " }] " + "} "); + auto record = jsonFromString( + "{ " + " \"attr1\": { " + " \"attr2\": \"tizen\", " + " \"attr3\": \"hello\" " + " } " + "} "); + ASSERT_TRUE(jf.EvaluateNode(node.get(), record)); +} + +class IsMatchTest : public ::testing::Test {}; + +TEST_F(IsMatchTest, BasicPositiveTest) { + common::JsonFilter jf( + jsonFromString("{ " + " \"$OR\": [{ " + " \"$EXACTLY\": [ " + " \"attr1.attr2\"," + " \"tizen\" " + " ]}, " + " {\"$EXACTLY\": [ " + " \"attr1.attr2\"," + " \"hello\" " + " ]} " + " }] " + "} ")); + auto record1 = jsonFromString( + "{ " + " \"attr1\": { " + " \"attr2\": \"tizen\", " + " } " + "} "); + auto record2 = jsonFromString( + "{ " + " \"attr1\": { " + " \"attr2\": \"hello\" " + " } " + "} "); + auto record3 = jsonFromString( + "{ " + " \"attr1\": { " + " \"attr2\": \"wrong\" " + " } " + "} "); + + EXPECT_TRUE(jf.IsMatch(record1)); + EXPECT_TRUE(jf.IsMatch(record2)); + EXPECT_FALSE(jf.IsMatch(record3)); +} diff --git a/src/common/ut/json-filter.h b/src/common/ut/json-filter.h new file mode 100644 index 00000000..405c22f2 --- /dev/null +++ b/src/common/ut/json-filter.h @@ -0,0 +1,30 @@ +/* + * Copyright (c) 2019 Samsung Electronics Co., Ltd All Rights Reserved + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef COMMON_UT_JSON_FILTER_H +#define COMMON_UT_JSON_FILTER_H + +class SplitStringTest; +class GetAttributeTest; +class EqualsOperatorTest; +class ContainsOperatorTest; +class AndOperatorTest; +class OrOperatorTest; +class EvaluateOperatorTest; +class EvaluateNodeTest; +class IsMatchTest; + +#endif // COMMON_UT_JSON_UTILS_H diff --git a/src/common/ut/main.cc b/src/common/ut/main.cc index f2a88986..17870e18 100644 --- a/src/common/ut/main.cc +++ b/src/common/ut/main.cc @@ -16,6 +16,7 @@ #include "gtest/gtest.h" +#include "common/ut/json-filter.h" #include "common/ut/json-utils.h" #include "tizen.h"