From: Adrian Szyndela Date: Fri, 25 Oct 2019 14:27:09 +0000 (+0200) Subject: serialization: serialize send index X-Git-Tag: accepted/tizen/5.5/unified/20191104.111320~1^2 X-Git-Url: http://review.tizen.org/git/?a=commitdiff_plain;h=e83f85abbe208d0c96c137ebc852ea4e459449ae;p=platform%2Fcore%2Fsystem%2Flibdbuspolicy.git serialization: serialize send index This removes creating send index after reading a serialized database. Instead, the send index is created while serializing the database, and written along with the database. Then, the index is used as before, but in a bit more generalized way, being a part of PolicySend now. Therefore, this extends the database with the new tables, making the database version up to LDP2. The previous version LDP1 is still compatible with new code, however the send index is not supported for old databases. This results in the following: - with serialized database, version LDP2, the send index is kept in serialized database file, and used accordingly for all lookups to "send" part of the database; - with serialized database, version LDP1, there is no send index, the lookups are performed in an old-fashined, linear manner; - with no serialized database, only XML files, the index is created in memory, along with the serialized database. Thus, running fast, but at the cost of memory. Change-Id: Ib918fa96eb60d216512faf60f8e20692c8a5d3a3 (cherry picked from commit 783b7e7315959b625a613b959c19b12a381e5831) --- diff --git a/src/internal/include/fb.fbs b/src/internal/include/fb.fbs index 8fce719..efbf6b9 100644 --- a/src/internal/include/fb.fbs +++ b/src/internal/include/fb.fbs @@ -53,6 +53,8 @@ table SendSet { table PolicySend { items:[ItemSend]; + index:[NameScoresPair]; // version "LDP2" extension + prefix_index:[uint]; // version "LDP2" extension } table ItemSend { @@ -107,6 +109,13 @@ table DecisionItem { privilege:string; } +/* version LDP2 extension starts here - sendIndex added */ +table NameScoresPair { + name: string (key); + best_score: uint; + item_refs:[uint]; +} + root_type File; -file_identifier "LDP1"; +file_identifier "LDP2"; diff --git a/src/internal/include/fb_generated.h b/src/internal/include/fb_generated.h index ce371e4..335b3d9 100644 --- a/src/internal/include/fb_generated.h +++ b/src/internal/include/fb_generated.h @@ -42,6 +42,8 @@ struct ItemAccess; struct DecisionItem; +struct NameScoresPair; + enum Decision { Decision_ANY = 0, Decision_ALLOW = 1, @@ -723,16 +725,29 @@ inline flatbuffers::Offset CreateSendSetDirect( struct PolicySend FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { enum FlatBuffersVTableOffset FLATBUFFERS_VTABLE_UNDERLYING_TYPE { - VT_ITEMS = 4 + VT_ITEMS = 4, + VT_INDEX = 6, + VT_PREFIX_INDEX = 8 }; const flatbuffers::Vector> *items() const { return GetPointer> *>(VT_ITEMS); } + const flatbuffers::Vector> *index() const { + return GetPointer> *>(VT_INDEX); + } + const flatbuffers::Vector *prefix_index() const { + return GetPointer *>(VT_PREFIX_INDEX); + } bool Verify(flatbuffers::Verifier &verifier) const { return VerifyTableStart(verifier) && VerifyOffset(verifier, VT_ITEMS) && verifier.VerifyVector(items()) && verifier.VerifyVectorOfTables(items()) && + VerifyOffset(verifier, VT_INDEX) && + verifier.VerifyVector(index()) && + verifier.VerifyVectorOfTables(index()) && + VerifyOffset(verifier, VT_PREFIX_INDEX) && + verifier.VerifyVector(prefix_index()) && verifier.EndTable(); } }; @@ -743,6 +758,12 @@ struct PolicySendBuilder { void add_items(flatbuffers::Offset>> items) { fbb_.AddOffset(PolicySend::VT_ITEMS, items); } + void add_index(flatbuffers::Offset>> index) { + fbb_.AddOffset(PolicySend::VT_INDEX, index); + } + void add_prefix_index(flatbuffers::Offset> prefix_index) { + fbb_.AddOffset(PolicySend::VT_PREFIX_INDEX, prefix_index); + } explicit PolicySendBuilder(flatbuffers::FlatBufferBuilder &_fbb) : fbb_(_fbb) { start_ = fbb_.StartTable(); @@ -757,19 +778,29 @@ struct PolicySendBuilder { inline flatbuffers::Offset CreatePolicySend( flatbuffers::FlatBufferBuilder &_fbb, - flatbuffers::Offset>> items = 0) { + flatbuffers::Offset>> items = 0, + flatbuffers::Offset>> index = 0, + flatbuffers::Offset> prefix_index = 0) { PolicySendBuilder builder_(_fbb); + builder_.add_prefix_index(prefix_index); + builder_.add_index(index); builder_.add_items(items); return builder_.Finish(); } inline flatbuffers::Offset CreatePolicySendDirect( flatbuffers::FlatBufferBuilder &_fbb, - const std::vector> *items = nullptr) { + const std::vector> *items = nullptr, + const std::vector> *index = nullptr, + const std::vector *prefix_index = nullptr) { auto items__ = items ? _fbb.CreateVector>(*items) : 0; + auto index__ = index ? _fbb.CreateVector>(*index) : 0; + auto prefix_index__ = prefix_index ? _fbb.CreateVector(*prefix_index) : 0; return FB::CreatePolicySend( _fbb, - items__); + items__, + index__, + prefix_index__); } struct ItemSend FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { @@ -1409,6 +1440,89 @@ inline flatbuffers::Offset CreateDecisionItemDirect( privilege__); } +struct NameScoresPair FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { + enum FlatBuffersVTableOffset FLATBUFFERS_VTABLE_UNDERLYING_TYPE { + VT_NAME = 4, + VT_BEST_SCORE = 6, + VT_ITEM_REFS = 8 + }; + const flatbuffers::String *name() const { + return GetPointer(VT_NAME); + } + bool KeyCompareLessThan(const NameScoresPair *o) const { + return *name() < *o->name(); + } + int KeyCompareWithValue(const char *val) const { + return strcmp(name()->c_str(), val); + } + uint32_t best_score() const { + return GetField(VT_BEST_SCORE, 0); + } + const flatbuffers::Vector *item_refs() const { + return GetPointer *>(VT_ITEM_REFS); + } + bool Verify(flatbuffers::Verifier &verifier) const { + return VerifyTableStart(verifier) && + VerifyOffsetRequired(verifier, VT_NAME) && + verifier.VerifyString(name()) && + VerifyField(verifier, VT_BEST_SCORE) && + VerifyOffset(verifier, VT_ITEM_REFS) && + verifier.VerifyVector(item_refs()) && + verifier.EndTable(); + } +}; + +struct NameScoresPairBuilder { + flatbuffers::FlatBufferBuilder &fbb_; + flatbuffers::uoffset_t start_; + void add_name(flatbuffers::Offset name) { + fbb_.AddOffset(NameScoresPair::VT_NAME, name); + } + void add_best_score(uint32_t best_score) { + fbb_.AddElement(NameScoresPair::VT_BEST_SCORE, best_score, 0); + } + void add_item_refs(flatbuffers::Offset> item_refs) { + fbb_.AddOffset(NameScoresPair::VT_ITEM_REFS, item_refs); + } + explicit NameScoresPairBuilder(flatbuffers::FlatBufferBuilder &_fbb) + : fbb_(_fbb) { + start_ = fbb_.StartTable(); + } + NameScoresPairBuilder &operator=(const NameScoresPairBuilder &); + flatbuffers::Offset Finish() { + const auto end = fbb_.EndTable(start_); + auto o = flatbuffers::Offset(end); + fbb_.Required(o, NameScoresPair::VT_NAME); + return o; + } +}; + +inline flatbuffers::Offset CreateNameScoresPair( + flatbuffers::FlatBufferBuilder &_fbb, + flatbuffers::Offset name = 0, + uint32_t best_score = 0, + flatbuffers::Offset> item_refs = 0) { + NameScoresPairBuilder builder_(_fbb); + builder_.add_item_refs(item_refs); + builder_.add_best_score(best_score); + builder_.add_name(name); + return builder_.Finish(); +} + +inline flatbuffers::Offset CreateNameScoresPairDirect( + flatbuffers::FlatBufferBuilder &_fbb, + const char *name = nullptr, + uint32_t best_score = 0, + const std::vector *item_refs = nullptr) { + auto name__ = name ? _fbb.CreateString(name) : 0; + auto item_refs__ = item_refs ? _fbb.CreateVector(*item_refs) : 0; + return FB::CreateNameScoresPair( + _fbb, + name__, + best_score, + item_refs__); +} + inline const FB::File *GetFile(const void *buf) { return flatbuffers::GetRoot(buf); } @@ -1418,7 +1532,7 @@ inline const FB::File *GetSizePrefixedFile(const void *buf) { } inline const char *FileIdentifier() { - return "LDP1"; + return "LDP2"; } inline bool FileBufferHasIdentifier(const void *buf) { diff --git a/src/internal/serializer.cpp b/src/internal/serializer.cpp index eb185e8..f5cee57 100644 --- a/src/internal/serializer.cpp +++ b/src/internal/serializer.cpp @@ -222,6 +222,48 @@ auto Serializer::serialize_policy(const PolicyOwn &policy) return serialize_tree(policy.getTree()); } +template <> +auto Serializer::serialize_policy(const PolicySend &policy) + -> FbOff { + std::vector> items; + + // this maps a name to a pair (highest score + list of scores/ids for this name) + std::map>> policyIndex; + // prefix index is just a list of ids + std::vector prefixIndex; + uint32_t cnt = 1; + + for (const auto &item : policy.getItems()) { + items.push_back(serialize_item(item)); + + // create indexes + if (!item.isNamePrefix()) { + auto &elem = policyIndex[item.getName()]; // create or get an entry + elem.second.push_back(cnt); // add score/id to the list + // we insert items in increasing score/id order, so we just update the highest score on each add + elem.first = cnt; + } else { + // just collect the prefix rules + prefixIndex.push_back(cnt); + } + ++cnt; + } + + // serialize main index + std::vector> index; + + for (auto &it: policyIndex) + index.push_back(FB::CreateNameScoresPairDirect(m_builder, + it.first.data(), // name + it.second.first, // best_score + &it.second.second)); // vector of scores/ids + + return FB::CreatePolicySend(m_builder, + m_builder.CreateVector(items), + m_builder.CreateVector(index), + m_builder.CreateVector(prefixIndex)); +} + template auto Serializer::serialize_policy(const T &policy) -> FbOff::policy> { std::vector::item>> items; diff --git a/src/internal/storage_backend_serialized.cpp b/src/internal/storage_backend_serialized.cpp index 4859951..8900779 100644 --- a/src/internal/storage_backend_serialized.cpp +++ b/src/internal/storage_backend_serialized.cpp @@ -24,6 +24,7 @@ #include #include #include +#include #include #include #include @@ -54,27 +55,13 @@ class StorageBackendSerialized::StorageBackendSerializedImpl { void releaseMMap(); void releaseFD(); - typedef std::pair ItemWithScore; // a pair: Item, index of Item - typedef std::vector ItemsWithScore; // items - typedef std::pair HighestScoreWithItems; // value for index map: highest index for items, items - typedef std::map MapIndex; - typedef std::vector SendPrefixIndex; - - MapIndex sendIndex; // context default - SendPrefixIndex sendPrefixIndex; // context default prefix rules - std::map userSendIndex; - std::map userSendPrefixIndex; - public: bool init(const char *filename, bool verify); bool init(const FB::File *f); bool initFromXML(const char *config_name); void release(); - void createSendIndex(); - ldp_xml_parser::DecisionItem getDecisionFromSendIndex(const MatchItemSend &item, const MapIndex &index, const SendPrefixIndex &prefixIndex); - ldp_xml_parser::DecisionItem getDecisionFromSendIndex(const MatchItemSend &item); - ldp_xml_parser::DecisionItem getDecisionFromSendIndex(const MatchItemSend &item, uid_t uid); + ldp_xml_parser::DecisionItem getDecisionFromSendIndex(const MatchItemSend &item, const FB::PolicySend *policy); void printContent(const bool xml_format = false) const; @@ -98,90 +85,72 @@ bool match(const T &match, const I *i) { } ldp_xml_parser::DecisionItem StorageBackendSerialized::StorageBackendSerializedImpl::getDecisionFromSendIndex(const MatchItemSend &item, - const MapIndex &index, const SendPrefixIndex &prefixIndex) { - ldp_xml_parser::DecisionItem decision(ldp_xml_parser::Decision::ANY); + const FB::PolicySend *policy) { + + const auto *index = policy->index(); + + if (!index || index->size() == 0) // this indicates version earlier than LDP2 + return getDecisionItem(item, policy); // make it old way for old databases + + std::pair currentBest(nullptr, 0); + + auto updateCurrentBest = [¤tBest, &item, &policy](const flatbuffers::Vector *vec) { + // look from the back, the rule is the same as for the full database + // we now only check among less elements, because the database is indexed to small lists + // item_scores are in increasing order in the index, and they serve also as ids of the policy rules + for (auto item_score_it = vec->rbegin(); item_score_it != vec->rend(); item_score_it++) { + const auto *fb_item = policy->items()->Get(*item_score_it - 1); // rules are indexed/scored from 1 + if (*item_score_it > currentBest.second && match(item, fb_item)) { + currentBest.first = fb_item; + currentBest.second = *item_score_it; + return; + } else if (*item_score_it <= currentBest.second) { + // there is no need to look further as we can't improve the score anymore + return; + } + } + }; - ItemWithScore currentBest(nullptr, 0); + auto searchAndUpdateCurrentBest = [¤tBest, &index, &updateCurrentBest](boost::string_ref name_ref) { + // we need to create C-string for flatbuffers lookups + // boost::string_ref gives us correct start, but possibly NUL-terminated in a wrong place, as it does not modify + // input string and keeps only the length + std::string name(name_ref.data(), name_ref.size()); - auto updateCurrentBest = [¤tBest, &item, &index](boost::string_ref name) { // check if there are any rules for the name - auto fit = index.find(name); - if (fit == index.end()) + const auto *fit = index->LookupByKey(name.c_str()); + if (!fit) return; // check if there's any chance to get better score - // fit->second.first is the highest score from vector fit->second.second - if (fit->second.first <= currentBest.second) + if (fit->best_score() <= currentBest.second) return; // look for better score - for (const auto &it: fit->second.second) { - if (it.second > currentBest.second && match(item, it.first)) - currentBest = it; - } + updateCurrentBest(fit->item_refs()); }; + const auto *prefixIndex = policy->prefix_index(); + assert(prefixIndex); + // iterate over names for (const auto &name: item.getNames()) { // find and check the no-prefix rules - updateCurrentBest(name); + searchAndUpdateCurrentBest(name); // check the prefix rules - for (const auto &prefixItem: prefixIndex) { - if (prefixItem.second > currentBest.second && match(item, prefixItem.first)) - currentBest = prefixItem; - } + updateCurrentBest(prefixIndex); } // check no-name rules - updateCurrentBest(""); + searchAndUpdateCurrentBest(""); + // if a matching rule was found, return the decision if (currentBest.first) - decision = makeDecisionItem(currentBest.first->decision()); - - return decision; -} + return makeDecisionItem(currentBest.first->decision()); -ldp_xml_parser::DecisionItem StorageBackendSerialized::StorageBackendSerializedImpl::getDecisionFromSendIndex(const MatchItemSend &item) { - return getDecisionFromSendIndex(item, sendIndex, sendPrefixIndex); -} - -ldp_xml_parser::DecisionItem StorageBackendSerialized::StorageBackendSerializedImpl::getDecisionFromSendIndex(const MatchItemSend &item, uid_t uid) { - return getDecisionFromSendIndex(item, userSendIndex[uid], userSendPrefixIndex[uid]); -} - -void StorageBackendSerialized::StorageBackendSerializedImpl::createSendIndex() { - // helper for adding items to indexes - auto add = [](MapIndex &index, SendPrefixIndex &prefixIndex, boost::string_ref key, ItemWithScore value) { - if (value.first->is_name_prefix()) { - prefixIndex.push_back(value); - } else { - index[key].second.push_back(value); - // we insert items in increasing score order, so we just update the highest score on each add - index[key].first = value.second; - } - }; - - // context default index - const auto *policy = file->m_send_set()->context_default(); - unsigned cnt = 1; - - const auto *v = policy->items(); - for (auto it = v->begin(); it != v->end(); ++it) - add(sendIndex, sendPrefixIndex, - boost::string_ref(it->name()->c_str(), it->name()->size()), - ItemWithScore(*it, cnt++)); - - // users index - for (const auto &uidPolicy: *file->m_send_set()->user()) { - cnt = 1; - for (const auto it: *uidPolicy->policy()->items()) { - add(userSendIndex[uidPolicy->id()], - userSendPrefixIndex[uidPolicy->id()], - boost::string_ref(it->name()->c_str(), it->name()->size()), - ItemWithScore(it, cnt++)); - } - } + // or if no matching rule was not found, return the default decision + return ldp_xml_parser::DecisionItem(ldp_xml_parser::Decision::ANY); } void StorageBackendSerialized::StorageBackendSerializedImpl::releaseMMap() { @@ -260,17 +229,13 @@ bool StorageBackendSerialized::StorageBackendSerializedImpl::init(const char *fi mmapGuard.dismiss(); file = GetFile(mem); - bool res = file != nullptr; - if (res) - createSendIndex(); - return res; + return file != nullptr; } bool StorageBackendSerialized::StorageBackendSerializedImpl::init(const FB::File *f) { assert(nullptr == file); file = f; - createSendIndex(); return true; } @@ -399,8 +364,13 @@ ldp_xml_parser::DecisionItem StorageBackendSerialized::getDecisionItemContextMan } template <> +ldp_xml_parser::DecisionItem StorageBackendSerialized::getDecisionItemContextMandatory(const MatchItemSend &item) const { + return pimpl->getDecisionFromSendIndex(item, pimpl->getPolicySet()->context_mandatory()); +} + +template <> ldp_xml_parser::DecisionItem StorageBackendSerialized::getDecisionItemContextDefault(const MatchItemSend &item) const { - return pimpl->getDecisionFromSendIndex(item); + return pimpl->getDecisionFromSendIndex(item, pimpl->getPolicySet()->context_default()); } template <> @@ -408,7 +378,7 @@ ldp_xml_parser::DecisionItem StorageBackendSerialized::getDecisionItemUser(uid_t auto *policyPair = pimpl->getPolicySet()->user()->LookupByKey(uid); if (nullptr == policyPair) return ldp_xml_parser::Decision::ANY; - return pimpl->getDecisionFromSendIndex(item, uid); + return pimpl->getDecisionFromSendIndex(item, policyPair->policy()); } template