Merge "serialization: serialize send index" into tizen_5.5
authorsanghyeok oh <sanghyeok.oh@samsung.com>
Fri, 1 Nov 2019 08:08:01 +0000 (08:08 +0000)
committerGerrit Code Review <gerrit@review.ap-northeast-2.compute.internal>
Fri, 1 Nov 2019 08:08:01 +0000 (08:08 +0000)
src/internal/include/fb.fbs
src/internal/include/fb_generated.h
src/internal/serializer.cpp
src/internal/storage_backend_serialized.cpp

index 8fce719..efbf6b9 100644 (file)
@@ -53,6 +53,8 @@ table SendSet {
 
 table PolicySend {
     items:[ItemSend];
+    index:[NameScoresPair];            // version "LDP2" extension
+    prefix_index:[uint];               // version "LDP2" extension
 }
 
 table ItemSend {
@@ -107,6 +109,13 @@ table DecisionItem {
     privilege:string;
 }
 
+/* version LDP2 extension starts here - sendIndex added */
+table NameScoresPair {
+    name: string (key);
+    best_score: uint;
+    item_refs:[uint];
+}
+
 root_type File;
 
-file_identifier "LDP1";
+file_identifier "LDP2";
index ce371e4..335b3d9 100644 (file)
@@ -42,6 +42,8 @@ struct ItemAccess;
 
 struct DecisionItem;
 
+struct NameScoresPair;
+
 enum Decision {
   Decision_ANY = 0,
   Decision_ALLOW = 1,
@@ -723,16 +725,29 @@ inline flatbuffers::Offset<SendSet> CreateSendSetDirect(
 
 struct PolicySend FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table {
   enum FlatBuffersVTableOffset FLATBUFFERS_VTABLE_UNDERLYING_TYPE {
-    VT_ITEMS = 4
+    VT_ITEMS = 4,
+    VT_INDEX = 6,
+    VT_PREFIX_INDEX = 8
   };
   const flatbuffers::Vector<flatbuffers::Offset<ItemSend>> *items() const {
     return GetPointer<const flatbuffers::Vector<flatbuffers::Offset<ItemSend>> *>(VT_ITEMS);
   }
+  const flatbuffers::Vector<flatbuffers::Offset<NameScoresPair>> *index() const {
+    return GetPointer<const flatbuffers::Vector<flatbuffers::Offset<NameScoresPair>> *>(VT_INDEX);
+  }
+  const flatbuffers::Vector<uint32_t> *prefix_index() const {
+    return GetPointer<const flatbuffers::Vector<uint32_t> *>(VT_PREFIX_INDEX);
+  }
   bool Verify(flatbuffers::Verifier &verifier) const {
     return VerifyTableStart(verifier) &&
            VerifyOffset(verifier, VT_ITEMS) &&
            verifier.VerifyVector(items()) &&
            verifier.VerifyVectorOfTables(items()) &&
+           VerifyOffset(verifier, VT_INDEX) &&
+           verifier.VerifyVector(index()) &&
+           verifier.VerifyVectorOfTables(index()) &&
+           VerifyOffset(verifier, VT_PREFIX_INDEX) &&
+           verifier.VerifyVector(prefix_index()) &&
            verifier.EndTable();
   }
 };
@@ -743,6 +758,12 @@ struct PolicySendBuilder {
   void add_items(flatbuffers::Offset<flatbuffers::Vector<flatbuffers::Offset<ItemSend>>> items) {
     fbb_.AddOffset(PolicySend::VT_ITEMS, items);
   }
+  void add_index(flatbuffers::Offset<flatbuffers::Vector<flatbuffers::Offset<NameScoresPair>>> index) {
+    fbb_.AddOffset(PolicySend::VT_INDEX, index);
+  }
+  void add_prefix_index(flatbuffers::Offset<flatbuffers::Vector<uint32_t>> prefix_index) {
+    fbb_.AddOffset(PolicySend::VT_PREFIX_INDEX, prefix_index);
+  }
   explicit PolicySendBuilder(flatbuffers::FlatBufferBuilder &_fbb)
         : fbb_(_fbb) {
     start_ = fbb_.StartTable();
@@ -757,19 +778,29 @@ struct PolicySendBuilder {
 
 inline flatbuffers::Offset<PolicySend> CreatePolicySend(
     flatbuffers::FlatBufferBuilder &_fbb,
-    flatbuffers::Offset<flatbuffers::Vector<flatbuffers::Offset<ItemSend>>> items = 0) {
+    flatbuffers::Offset<flatbuffers::Vector<flatbuffers::Offset<ItemSend>>> items = 0,
+    flatbuffers::Offset<flatbuffers::Vector<flatbuffers::Offset<NameScoresPair>>> index = 0,
+    flatbuffers::Offset<flatbuffers::Vector<uint32_t>> prefix_index = 0) {
   PolicySendBuilder builder_(_fbb);
+  builder_.add_prefix_index(prefix_index);
+  builder_.add_index(index);
   builder_.add_items(items);
   return builder_.Finish();
 }
 
 inline flatbuffers::Offset<PolicySend> CreatePolicySendDirect(
     flatbuffers::FlatBufferBuilder &_fbb,
-    const std::vector<flatbuffers::Offset<ItemSend>> *items = nullptr) {
+    const std::vector<flatbuffers::Offset<ItemSend>> *items = nullptr,
+    const std::vector<flatbuffers::Offset<NameScoresPair>> *index = nullptr,
+    const std::vector<uint32_t> *prefix_index = nullptr) {
   auto items__ = items ? _fbb.CreateVector<flatbuffers::Offset<ItemSend>>(*items) : 0;
+  auto index__ = index ? _fbb.CreateVector<flatbuffers::Offset<NameScoresPair>>(*index) : 0;
+  auto prefix_index__ = prefix_index ? _fbb.CreateVector<uint32_t>(*prefix_index) : 0;
   return FB::CreatePolicySend(
       _fbb,
-      items__);
+      items__,
+      index__,
+      prefix_index__);
 }
 
 struct ItemSend FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table {
@@ -1409,6 +1440,89 @@ inline flatbuffers::Offset<DecisionItem> CreateDecisionItemDirect(
       privilege__);
 }
 
+struct NameScoresPair FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table {
+  enum FlatBuffersVTableOffset FLATBUFFERS_VTABLE_UNDERLYING_TYPE {
+    VT_NAME = 4,
+    VT_BEST_SCORE = 6,
+    VT_ITEM_REFS = 8
+  };
+  const flatbuffers::String *name() const {
+    return GetPointer<const flatbuffers::String *>(VT_NAME);
+  }
+  bool KeyCompareLessThan(const NameScoresPair *o) const {
+    return *name() < *o->name();
+  }
+  int KeyCompareWithValue(const char *val) const {
+    return strcmp(name()->c_str(), val);
+  }
+  uint32_t best_score() const {
+    return GetField<uint32_t>(VT_BEST_SCORE, 0);
+  }
+  const flatbuffers::Vector<uint32_t> *item_refs() const {
+    return GetPointer<const flatbuffers::Vector<uint32_t> *>(VT_ITEM_REFS);
+  }
+  bool Verify(flatbuffers::Verifier &verifier) const {
+    return VerifyTableStart(verifier) &&
+           VerifyOffsetRequired(verifier, VT_NAME) &&
+           verifier.VerifyString(name()) &&
+           VerifyField<uint32_t>(verifier, VT_BEST_SCORE) &&
+           VerifyOffset(verifier, VT_ITEM_REFS) &&
+           verifier.VerifyVector(item_refs()) &&
+           verifier.EndTable();
+  }
+};
+
+struct NameScoresPairBuilder {
+  flatbuffers::FlatBufferBuilder &fbb_;
+  flatbuffers::uoffset_t start_;
+  void add_name(flatbuffers::Offset<flatbuffers::String> name) {
+    fbb_.AddOffset(NameScoresPair::VT_NAME, name);
+  }
+  void add_best_score(uint32_t best_score) {
+    fbb_.AddElement<uint32_t>(NameScoresPair::VT_BEST_SCORE, best_score, 0);
+  }
+  void add_item_refs(flatbuffers::Offset<flatbuffers::Vector<uint32_t>> item_refs) {
+    fbb_.AddOffset(NameScoresPair::VT_ITEM_REFS, item_refs);
+  }
+  explicit NameScoresPairBuilder(flatbuffers::FlatBufferBuilder &_fbb)
+        : fbb_(_fbb) {
+    start_ = fbb_.StartTable();
+  }
+  NameScoresPairBuilder &operator=(const NameScoresPairBuilder &);
+  flatbuffers::Offset<NameScoresPair> Finish() {
+    const auto end = fbb_.EndTable(start_);
+    auto o = flatbuffers::Offset<NameScoresPair>(end);
+    fbb_.Required(o, NameScoresPair::VT_NAME);
+    return o;
+  }
+};
+
+inline flatbuffers::Offset<NameScoresPair> CreateNameScoresPair(
+    flatbuffers::FlatBufferBuilder &_fbb,
+    flatbuffers::Offset<flatbuffers::String> name = 0,
+    uint32_t best_score = 0,
+    flatbuffers::Offset<flatbuffers::Vector<uint32_t>> item_refs = 0) {
+  NameScoresPairBuilder builder_(_fbb);
+  builder_.add_item_refs(item_refs);
+  builder_.add_best_score(best_score);
+  builder_.add_name(name);
+  return builder_.Finish();
+}
+
+inline flatbuffers::Offset<NameScoresPair> CreateNameScoresPairDirect(
+    flatbuffers::FlatBufferBuilder &_fbb,
+    const char *name = nullptr,
+    uint32_t best_score = 0,
+    const std::vector<uint32_t> *item_refs = nullptr) {
+  auto name__ = name ? _fbb.CreateString(name) : 0;
+  auto item_refs__ = item_refs ? _fbb.CreateVector<uint32_t>(*item_refs) : 0;
+  return FB::CreateNameScoresPair(
+      _fbb,
+      name__,
+      best_score,
+      item_refs__);
+}
+
 inline const FB::File *GetFile(const void *buf) {
   return flatbuffers::GetRoot<FB::File>(buf);
 }
@@ -1418,7 +1532,7 @@ inline const FB::File *GetSizePrefixedFile(const void *buf) {
 }
 
 inline const char *FileIdentifier() {
-  return "LDP1";
+  return "LDP2";
 }
 
 inline bool FileBufferHasIdentifier(const void *buf) {
index eb185e8..f5cee57 100644 (file)
@@ -222,6 +222,48 @@ auto Serializer::serialize_policy(const PolicyOwn &policy)
        return serialize_tree(policy.getTree());
 }
 
+template <>
+auto Serializer::serialize_policy(const PolicySend &policy)
+                                   -> FbOff<FB::PolicySend> {
+       std::vector<FbOff<FB::ItemSend>> items;
+
+       // this maps a name to a pair (highest score + list of scores/ids for this name)
+       std::map<boost::string_ref, std::pair<uint32_t, std::vector<uint32_t>>> policyIndex;
+       // prefix index is just a list of ids
+       std::vector<uint32_t> prefixIndex;
+       uint32_t cnt = 1;
+
+       for (const auto &item : policy.getItems()) {
+               items.push_back(serialize_item<PolicySend>(item));
+
+               // create indexes
+               if (!item.isNamePrefix()) {
+                       auto &elem = policyIndex[item.getName()];   // create or get an entry
+                       elem.second.push_back(cnt);                 // add score/id to the list
+                       // we insert items in increasing score/id order, so we just update the highest score on each add
+                       elem.first = cnt;
+               } else {
+                       // just collect the prefix rules
+                       prefixIndex.push_back(cnt);
+               }
+               ++cnt;
+       }
+
+       // serialize main index
+       std::vector<FbOff<FB::NameScoresPair>> index;
+
+       for (auto &it: policyIndex)
+               index.push_back(FB::CreateNameScoresPairDirect(m_builder,
+                                                                                                               it.first.data(),     // name
+                                                                                                               it.second.first,     // best_score
+                                                                                                               &it.second.second)); // vector of scores/ids
+
+       return FB::CreatePolicySend(m_builder,
+                       m_builder.CreateVector(items),
+                       m_builder.CreateVector(index),
+                       m_builder.CreateVector(prefixIndex));
+}
+
 template <typename T>
 auto Serializer::serialize_policy(const T &policy) -> FbOff<typename type_helper<T>::policy> {
        std::vector<FbOff<typename type_helper<T>::item>> items;
index cf37c6b..5271adf 100644 (file)
@@ -24,6 +24,7 @@
 #include <cerrno>
 #include <fcntl.h>
 #include <map>
+#include <string>
 #include <sys/mman.h>
 #include <sys/stat.h>
 #include <sys/types.h>
@@ -133,27 +134,13 @@ class StorageBackendSerialized::StorageBackendSerializedImpl {
        void releaseMMap();
        void releaseFD();
 
-       typedef std::pair<const ItemSend *, unsigned> ItemWithScore;    // a pair: Item, index of Item
-       typedef std::vector<ItemWithScore> ItemsWithScore;                              // items
-       typedef std::pair<unsigned, ItemsWithScore> HighestScoreWithItems;      // value for index map: highest index for items, items
-       typedef std::map<boost::string_ref, HighestScoreWithItems> MapIndex;
-       typedef std::vector<ItemWithScore> SendPrefixIndex;
-
-       MapIndex sendIndex; // context default
-       SendPrefixIndex sendPrefixIndex; // context default prefix rules
-       std::map<uid_t, MapIndex> userSendIndex;
-       std::map<uid_t, SendPrefixIndex> userSendPrefixIndex;
-
 public:
        bool init(const char *filename, bool verify);
        bool init(const FB::File *f);
        bool initFromXML(const char *config_name);
        void release();
 
-       void createSendIndex();
-       ldp_xml_parser::DecisionItem getDecisionFromSendIndex(const MatchItemSend &item, const MapIndex &index, const SendPrefixIndex &prefixIndex);
-       ldp_xml_parser::DecisionItem getDecisionFromSendIndex(const MatchItemSend &item);
-       ldp_xml_parser::DecisionItem getDecisionFromSendIndex(const MatchItemSend &item, uid_t uid);
+       ldp_xml_parser::DecisionItem getDecisionFromSendIndex(const MatchItemSend &item, const FB::PolicySend *policy);
 
        void printContent(const bool xml_format = false) const;
 
@@ -162,90 +149,72 @@ public:
 };
 
 ldp_xml_parser::DecisionItem StorageBackendSerialized::StorageBackendSerializedImpl::getDecisionFromSendIndex(const MatchItemSend &item,
-               const MapIndex &index, const SendPrefixIndex &prefixIndex) {
-       ldp_xml_parser::DecisionItem decision(ldp_xml_parser::Decision::ANY);
+               const FB::PolicySend *policy) {
+
+       const auto *index = policy->index();
+
+       if (!index || index->size() == 0)           // this indicates version earlier than LDP2
+               return getDecisionItem(item, policy);   // make it old way for old databases
+
+       std::pair<const ItemSend*, uint32_t> currentBest(nullptr, 0);
+
+       auto updateCurrentBest = [&currentBest, &item, &policy](const flatbuffers::Vector<uint32_t> *vec) {
+               // look from the back, the rule is the same as for the full database
+               // we now only check among less elements, because the database is indexed to small lists
+               // item_scores are in increasing order in the index, and they serve also as ids of the policy rules
+               for (auto item_score_it = vec->rbegin(); item_score_it != vec->rend(); item_score_it++) {
+                       const auto *fb_item = policy->items()->Get(*item_score_it - 1); // rules are indexed/scored from 1
+                       if (*item_score_it > currentBest.second && match(item, fb_item)) {
+                               currentBest.first = fb_item;
+                               currentBest.second = *item_score_it;
+                               return;
+                       } else if (*item_score_it <= currentBest.second) {
+                               // there is no need to look further as we can't improve the score anymore
+                               return;
+                       }
+               }
+       };
 
-       ItemWithScore currentBest(nullptr, 0);
+       auto searchAndUpdateCurrentBest = [&currentBest, &index, &updateCurrentBest](boost::string_ref name_ref) {
+               // we need to create C-string for flatbuffers lookups
+               // boost::string_ref gives us correct start, but possibly NUL-terminated in a wrong place, as it does not modify
+               // input string and keeps only the length
+               std::string name(name_ref.data(), name_ref.size());
 
-       auto updateCurrentBest = [&currentBest, &item, &index](boost::string_ref name) {
                // check if there are any rules for the name
-               auto fit = index.find(name);
-               if (fit == index.end())
+               const auto *fit = index->LookupByKey(name.c_str());
+               if (!fit)
                        return;
 
                // check if there's any chance to get better score
-               // fit->second.first is the highest score from vector fit->second.second
-               if (fit->second.first <= currentBest.second)
+               if (fit->best_score() <= currentBest.second)
                        return;
 
                // look for better score
-               for (const auto &it: fit->second.second) {
-                       if (it.second > currentBest.second && match(item, it.first))
-                               currentBest = it;
-               }
+               updateCurrentBest(fit->item_refs());
        };
 
+       const auto *prefixIndex = policy->prefix_index();
+       assert(prefixIndex);
+
        // iterate over names
        for (const auto &name: item.getNames()) {
                // find and check the no-prefix rules
-               updateCurrentBest(name);
+               searchAndUpdateCurrentBest(name);
 
                // check the prefix rules
-               for (const auto &prefixItem: prefixIndex) {
-                       if (prefixItem.second > currentBest.second && match(item, prefixItem.first))
-                               currentBest = prefixItem;
-               }
+               updateCurrentBest(prefixIndex);
        }
 
        // check no-name rules
-       updateCurrentBest("");
+       searchAndUpdateCurrentBest("");
 
+       // if a matching rule was found, return the decision
        if (currentBest.first)
-               decision = makeDecisionItem(currentBest.first->decision());
-
-       return decision;
-}
+               return makeDecisionItem(currentBest.first->decision());
 
-ldp_xml_parser::DecisionItem StorageBackendSerialized::StorageBackendSerializedImpl::getDecisionFromSendIndex(const MatchItemSend &item) {
-       return getDecisionFromSendIndex(item, sendIndex, sendPrefixIndex);
-}
-
-ldp_xml_parser::DecisionItem StorageBackendSerialized::StorageBackendSerializedImpl::getDecisionFromSendIndex(const MatchItemSend &item, uid_t uid) {
-       return getDecisionFromSendIndex(item, userSendIndex[uid], userSendPrefixIndex[uid]);
-}
-
-void StorageBackendSerialized::StorageBackendSerializedImpl::createSendIndex() {
-       // helper for adding items to indexes
-       auto add = [](MapIndex &index, SendPrefixIndex &prefixIndex, boost::string_ref key, ItemWithScore value) {
-               if (value.first->is_name_prefix()) {
-                       prefixIndex.push_back(value);
-               } else {
-                       index[key].second.push_back(value);
-                       // we insert items in increasing score order, so we just update the highest score on each add
-                       index[key].first = value.second;
-               }
-       };
-
-       // context default index
-       const auto *policy = file->m_send_set()->context_default();
-       unsigned cnt = 1;
-
-       const auto *v = policy->items();
-       for (auto it = v->begin(); it != v->end(); ++it)
-               add(sendIndex, sendPrefixIndex,
-                               boost::string_ref(it->name()->c_str(), it->name()->size()),
-                               ItemWithScore(*it, cnt++));
-
-       // users index
-       for (const auto &uidPolicy: *file->m_send_set()->user()) {
-               cnt = 1;
-               for (const auto it: *uidPolicy->policy()->items()) {
-                       add(userSendIndex[uidPolicy->id()],
-                                       userSendPrefixIndex[uidPolicy->id()],
-                                       boost::string_ref(it->name()->c_str(), it->name()->size()),
-                                       ItemWithScore(it, cnt++));
-               }
-       }
+       // or if no matching rule was not found, return the default decision
+       return ldp_xml_parser::DecisionItem(ldp_xml_parser::Decision::ANY);
 }
 
 void StorageBackendSerialized::StorageBackendSerializedImpl::releaseMMap() {
@@ -324,17 +293,13 @@ bool StorageBackendSerialized::StorageBackendSerializedImpl::init(const char *fi
        mmapGuard.dismiss();
 
        file = GetFile(mem);
-       bool res = file != nullptr;
-       if (res)
-               createSendIndex();
-       return res;
+       return file != nullptr;
 }
 
 bool StorageBackendSerialized::StorageBackendSerializedImpl::init(const FB::File *f) {
        assert(nullptr == file);
 
        file = f;
-       createSendIndex();
        return true;
 }
 
@@ -403,8 +368,13 @@ ldp_xml_parser::DecisionItem StorageBackendSerialized::getDecisionItemContextMan
 }
 
 template <>
+ldp_xml_parser::DecisionItem StorageBackendSerialized::getDecisionItemContextMandatory(const MatchItemSend &item) const {
+       return pimpl->getDecisionFromSendIndex(item, pimpl->getPolicySet<MatchItemSend>()->context_mandatory());
+}
+
+template <>
 ldp_xml_parser::DecisionItem StorageBackendSerialized::getDecisionItemContextDefault(const MatchItemSend &item) const {
-       return pimpl->getDecisionFromSendIndex(item);
+       return pimpl->getDecisionFromSendIndex(item, pimpl->getPolicySet<MatchItemSend>()->context_default());
 }
 
 template <>
@@ -412,7 +382,7 @@ ldp_xml_parser::DecisionItem StorageBackendSerialized::getDecisionItemUser(uid_t
        auto *policyPair = pimpl->getPolicySet<MatchItemSend>()->user()->LookupByKey(uid);
        if (nullptr == policyPair)
                return ldp_xml_parser::Decision::ANY;
-       return pimpl->getDecisionFromSendIndex(item, uid);
+       return pimpl->getDecisionFromSendIndex(item, policyPair->policy());
 }
 
 template <typename T>