1 // See www.openfst.org for extensive documentation on this weighted
2 // finite-state transducer library.
4 // Classes to allow matching labels leaving FST states.
6 #ifndef FST_LIB_MATCHER_H_
7 #define FST_LIB_MATCHER_H_
10 #include <unordered_map>
15 #include <fst/mutable-fst.h> // for all internal FST accessors.
20 // Matchers find and iterate through requested labels at FST states. In the
21 // simplest form, these are just some associative map or search keyed on labels.
22 // More generally, they may implement matching special labels that represent
23 // sets of labels such as sigma (all), rho (rest), or phi (fail). The Matcher
30 // using Arc = typename FST::Arc;
31 // using Label = typename Arc::Label;
32 // using StateId = typename Arc::StateId;
33 // using Weight = typename Arc::Weight;
35 // // Required constructors.
37 // Matcher(const FST &fst, MatchType type);
38 // Matcher(const Matcher &matcher, bool safe = false);
40 // // Standard copy method.
41 // Matcher<FST> *Copy(bool safe = false) const override;
43 // // Returns the match type that can be provided (depending on compatibility
44 // of the input FST). It is either the requested match type, MATCH_NONE, or
45 // MATCH_UNKNOWN. If test is false, a costly testing is avoided, but
46 // MATCH_UNKNOWN may be returned. If test is true, a definite answer is
47 // returned, but may involve more costly computation (e.g., visiting the FST).
48 // MatchType Type(bool test) const override;
50 // // Specifies the current state.
51 // void SetState(StateId s) final;
53 // // Finds matches to a label at the current state, returning true if a match
54 // // found. kNoLabel matches any non-consuming transitions, e.g., epsilon
55 // // transitions, which do not require a matching symbol.
56 // bool Find(Label label) final;
58 // // Iterator methods. Note that initially and after SetState() these have
59 // undefined behavior until Find() is called.
61 // bool Done() const final;
63 // const Arc &Value() const final;
67 // // Returns final weight of a state.
68 // Weight Final(StateId) const final;
70 // // Indicates preference for being the side used for matching in
71 // // composition. If the value is kRequirePriority, then it is
72 // // mandatory that it be used. Calling this method without passing the
73 // // current state of the matcher invalidates the state of the matcher.
74 // ssize_t Priority(StateId s) final;
76 // // This specifies the known FST properties as viewed from this matcher. It
77 // // takes as argument the input FST's known properties.
78 // uint64 Properties(uint64 props) const override;
80 // // Returns matcher flags.
81 // uint32 Flags() const override;
83 // // Returns matcher FST.
84 // const FST &GetFst() const override;
87 // Basic matcher flags.
89 // Matcher needs to be used as the matching side in composition for
90 // at least one state (has kRequirePriority).
91 constexpr uint32 kRequireMatch = 0x00000001;
93 // Flags used for basic matchers (see also lookahead.h).
94 constexpr uint32 kMatcherFlags = kRequireMatch;
96 // Matcher priority that is mandatory.
97 constexpr ssize_t kRequirePriority = -1;
99 // Matcher interface, templated on the Arc definition; used for matcher
100 // specializations that are returned by the InitMatcher FST method.
105 using Label = typename Arc::Label;
106 using StateId = typename Arc::StateId;
107 using Weight = typename Arc::Weight;
109 virtual ~MatcherBase() {}
111 // Virtual interface.
113 virtual MatcherBase<Arc> *Copy(bool safe = false) const = 0;
114 virtual MatchType Type(bool) const = 0;
115 virtual void SetState(StateId) = 0;
116 virtual bool Find(Label) = 0;
117 virtual bool Done() const = 0;
118 virtual const Arc &Value() const = 0;
119 virtual void Next() = 0;
120 virtual const Fst<Arc> &GetFst() const = 0;
121 virtual uint64 Properties(uint64) const = 0;
123 // Trivial implementations that can be used by derived classes. Full
124 // devirtualization is expected for any derived class marked final.
125 virtual uint32 Flags() const { return 0; }
127 virtual Weight Final(StateId s) const { return internal::Final(GetFst(), s); }
129 virtual ssize_t Priority(StateId s) { return internal::NumArcs(GetFst(), s); }
132 // A matcher that expects sorted labels on the side to be matched.
133 // If match_type == MATCH_INPUT, epsilons match the implicit self-loop
134 // Arc(kNoLabel, 0, Weight::One(), current_state) as well as any
135 // actual epsilon transitions. If match_type == MATCH_OUTPUT, then
136 // Arc(0, kNoLabel, Weight::One(), current_state) is instead matched.
138 class SortedMatcher : public MatcherBase<typename F::Arc> {
141 using Arc = typename FST::Arc;
142 using Label = typename Arc::Label;
143 using StateId = typename Arc::StateId;
144 using Weight = typename Arc::Weight;
146 using MatcherBase<Arc>::Flags;
147 using MatcherBase<Arc>::Properties;
149 // Labels >= binary_label will be searched for by binary search;
150 // o.w. linear search is used.
151 SortedMatcher(const FST &fst, MatchType match_type, Label binary_label = 1)
155 match_type_(match_type),
156 binary_label_(binary_label),
157 match_label_(kNoLabel),
159 loop_(kNoLabel, 0, Weight::One(), kNoStateId),
162 switch (match_type_) {
167 std::swap(loop_.ilabel, loop_.olabel);
170 FSTERROR() << "SortedMatcher: Bad match type";
171 match_type_ = MATCH_NONE;
176 SortedMatcher(const SortedMatcher<FST> &matcher, bool safe = false)
177 : fst_(matcher.fst_->Copy(safe)),
180 match_type_(matcher.match_type_),
181 binary_label_(matcher.binary_label_),
182 match_label_(kNoLabel),
184 loop_(matcher.loop_),
185 error_(matcher.error_),
188 ~SortedMatcher() override { Destroy(aiter_, &aiter_pool_); }
190 SortedMatcher<FST> *Copy(bool safe = false) const override {
191 return new SortedMatcher<FST>(*this, safe);
194 MatchType Type(bool test) const override {
195 if (match_type_ == MATCH_NONE) return match_type_;
196 const auto true_prop =
197 match_type_ == MATCH_INPUT ? kILabelSorted : kOLabelSorted;
198 const auto false_prop =
199 match_type_ == MATCH_INPUT ? kNotILabelSorted : kNotOLabelSorted;
200 const auto props = fst_->Properties(true_prop | false_prop, test);
201 if (props & true_prop) {
203 } else if (props & false_prop) {
206 return MATCH_UNKNOWN;
210 void SetState(StateId s) final {
211 if (state_ == s) return;
213 if (match_type_ == MATCH_NONE) {
214 FSTERROR() << "SortedMatcher: Bad match type";
217 Destroy(aiter_, &aiter_pool_);
218 aiter_ = new (&aiter_pool_) ArcIterator<FST>(*fst_, s);
219 aiter_->SetFlags(kArcNoCache, kArcNoCache);
220 narcs_ = internal::NumArcs(*fst_, s);
224 bool Find(Label match_label) final {
227 current_loop_ = false;
228 match_label_ = kNoLabel;
231 current_loop_ = match_label == 0;
232 match_label_ = match_label == kNoLabel ? 0 : match_label;
236 return current_loop_;
240 // Positions matcher to the first position where inserting match_label would
241 // maintain the sort order.
242 void LowerBound(Label label) {
243 exact_match_ = false;
244 current_loop_ = false;
246 match_label_ = kNoLabel;
249 match_label_ = label;
253 // After Find(), returns false if no more exact matches.
254 // After LowerBound(), returns false if no more arcs.
255 bool Done() const final {
256 if (current_loop_) return false;
257 if (aiter_->Done()) return true;
258 if (!exact_match_) return false;
260 match_type_ == MATCH_INPUT ? kArcILabelValue : kArcOLabelValue,
262 const auto label = match_type_ == MATCH_INPUT ? aiter_->Value().ilabel
263 : aiter_->Value().olabel;
264 return label != match_label_;
267 const Arc &Value() const final {
268 if (current_loop_) return loop_;
269 aiter_->SetFlags(kArcValueFlags, kArcValueFlags);
270 return aiter_->Value();
275 current_loop_ = false;
281 Weight Final(StateId s) const final {
282 return MatcherBase<Arc>::Final(s);
285 ssize_t Priority(StateId s) final {
286 return MatcherBase<Arc>::Priority(s);
289 const FST &GetFst() const override { return *fst_; }
291 uint64 Properties(uint64 inprops) const override {
292 return inprops | (error_ ? kError : 0);
295 size_t Position() const { return aiter_ ? aiter_->Position() : 0; }
300 std::unique_ptr<const FST> fst_;
301 StateId state_; // Matcher state.
302 ArcIterator<FST> *aiter_; // Iterator for current state.
303 MatchType match_type_; // Type of match to perform.
304 Label binary_label_; // Least label for binary search.
305 Label match_label_; // Current label to be matched.
306 size_t narcs_; // Current state arc count.
307 Arc loop_; // For non-consuming symbols.
308 bool current_loop_; // Current arc is the implicit loop.
309 bool exact_match_; // Exact match or lower bound?
310 bool error_; // Error encountered?
311 MemoryPool<ArcIterator<FST>> aiter_pool_; // Pool of arc iterators.
314 // Returns true iff match to match_label_, positioning arc iterator at lower
317 inline bool SortedMatcher<FST>::Search() {
319 match_type_ == MATCH_INPUT ? kArcILabelValue : kArcOLabelValue,
321 if (match_label_ >= binary_label_) {
322 // Binary search for match.
324 size_t high = narcs_;
326 const size_t mid = (low + high) / 2;
328 auto label = match_type_ == MATCH_INPUT ? aiter_->Value().ilabel
329 : aiter_->Value().olabel;
330 if (label > match_label_) {
332 } else if (label < match_label_) {
335 // Finds first matching label (when non-deterministic).
336 for (size_t i = mid; i > low; --i) {
338 label = match_type_ == MATCH_INPUT ? aiter_->Value().ilabel
339 : aiter_->Value().olabel;
340 if (label != match_label_) {
351 // Linear search for match.
352 for (aiter_->Reset(); !aiter_->Done(); aiter_->Next()) {
353 const auto label = match_type_ == MATCH_INPUT ? aiter_->Value().ilabel
354 : aiter_->Value().olabel;
355 if (label == match_label_) return true;
356 if (label > match_label_) break;
362 // A matcher that stores labels in a per-state hash table populated upon the
363 // first visit to that state. Sorting is not required. Treatment of
364 // epsilons are the same as with SortedMatcher.
366 class HashMatcher : public MatcherBase<typename F::Arc> {
369 using Arc = typename FST::Arc;
370 using Label = typename Arc::Label;
371 using StateId = typename Arc::StateId;
372 using Weight = typename Arc::Weight;
374 using MatcherBase<Arc>::Flags;
375 using MatcherBase<Arc>::Final;
376 using MatcherBase<Arc>::Priority;
378 HashMatcher(const FST &fst, MatchType match_type)
381 match_type_(match_type),
382 loop_(kNoLabel, 0, Weight::One(), kNoStateId) {
383 switch (match_type_) {
388 std::swap(loop_.ilabel, loop_.olabel);
391 FSTERROR() << "SortedMatcher: Bad match type";
392 match_type_ = MATCH_NONE;
397 HashMatcher(const HashMatcher<FST> &matcher, bool safe = false)
398 : fst_(matcher.fst_->Copy(safe)),
400 match_type_(matcher.match_type_),
401 loop_(matcher.loop_),
402 error_(matcher.error_) {}
404 HashMatcher<FST> *Copy(bool safe = false) const override {
405 return new HashMatcher<FST>(*this, safe);
408 // The argument is ignored as there are no relevant properties to test.
409 MatchType Type(bool test) const override { return match_type_; }
411 void SetState(StateId s) final;
413 bool Find(Label label) final {
414 current_loop_ = label == 0;
419 if (label == kNoLabel) label = 0;
420 return Search(label);
423 bool Done() const final {
424 if (current_loop_) return false;
425 return label_it_ == label_end_;
428 const Arc &Value() const final {
429 if (current_loop_) return loop_;
430 aiter_->Seek(label_it_->second);
431 return aiter_->Value();
436 current_loop_ = false;
442 const FST &GetFst() const override { return *fst_; }
444 uint64 Properties(uint64 inprops) const override {
445 return inprops | (error_ ? kError : 0);
449 bool Search(Label match_label);
451 using LabelTable = std::unordered_multimap<Label, size_t>;
452 using StateTable = std::unordered_map<StateId, LabelTable>;
454 std::unique_ptr<const FST> fst_;
455 StateId state_; // Matcher state.
456 MatchType match_type_;
457 Arc loop_; // The implicit loop itself.
458 bool current_loop_; // Is the current arc is the implicit loop?
459 bool error_; // Error encountered?
460 std::unique_ptr<ArcIterator<FST>> aiter_;
461 StateTable state_table_; // Table from states to label table.
462 LabelTable *label_table_; // Pointer to current state's label table.
463 typename LabelTable::iterator label_it_; // Position for label.
464 typename LabelTable::iterator label_end_; // Position for last label + 1.
468 void HashMatcher<FST>::SetState(typename FST::Arc::StateId s) {
469 if (state_ == s) return;
470 // Resets everything for the state.
472 loop_.nextstate = state_;
473 aiter_.reset(new ArcIterator<FST>(*fst_, state_));
474 if (match_type_ == MATCH_NONE) {
475 FSTERROR() << "HashMatcher: Bad match type";
478 // Attempts to insert a new label table; if it already exists,
479 // no additional work is done and we simply return.
480 auto it_and_success = state_table_.emplace(state_, LabelTable());
481 if (!it_and_success.second) return;
482 // Otherwise, populate this new table.
483 // Sets instance's pointer to the label table for this state.
484 label_table_ = &(it_and_success.first->second);
485 // Populates the label table.
486 label_table_->reserve(internal::NumArcs(*fst_, state_));
487 const auto aiter_flags =
488 (match_type_ == MATCH_INPUT ? kArcILabelValue : kArcOLabelValue) |
490 aiter_->SetFlags(aiter_flags, kArcFlags);
491 for (; !aiter_->Done(); aiter_->Next()) {
492 const auto label = (match_type_ == MATCH_INPUT) ? aiter_->Value().ilabel
493 : aiter_->Value().olabel;
494 label_table_->emplace(label, aiter_->Position());
496 aiter_->SetFlags(kArcValueFlags, kArcValueFlags);
500 inline bool HashMatcher<FST>::Search(typename FST::Arc::Label match_label) {
501 auto range = label_table_->equal_range(match_label);
502 if (range.first == range.second) return false;
503 label_it_ = range.first;
504 label_end_ = range.second;
505 aiter_->Seek(label_it_->second);
509 // Specifies whether we rewrite both the input and output sides during matching.
510 enum MatcherRewriteMode {
511 MATCHER_REWRITE_AUTO = 0, // Rewrites both sides iff acceptor.
512 MATCHER_REWRITE_ALWAYS,
513 MATCHER_REWRITE_NEVER
516 // For any requested label that doesn't match at a state, this matcher
517 // considers the *unique* transition that matches the label 'phi_label'
518 // (phi = 'fail'), and recursively looks for a match at its
519 // destination. When 'phi_loop' is true, if no match is found but a
520 // phi self-loop is found, then the phi transition found is returned
521 // with the phi_label rewritten as the requested label (both sides if
522 // an acceptor, or if 'rewrite_both' is true and both input and output
523 // labels of the found transition are 'phi_label'). If 'phi_label' is
524 // kNoLabel, this special matching is not done. PhiMatcher is
525 // templated itself on a matcher, which is used to perform the
526 // underlying matching. By default, the underlying matcher is
527 // constructed by PhiMatcher. The user can instead pass in this
528 // object; in that case, PhiMatcher takes its ownership.
529 // Phi non-determinism not supported. No non-consuming symbols other
530 // than epsilon supported with the underlying template argument matcher.
532 class PhiMatcher : public MatcherBase<typename M::Arc> {
534 using FST = typename M::FST;
535 using Arc = typename FST::Arc;
536 using Label = typename Arc::Label;
537 using StateId = typename Arc::StateId;
538 using Weight = typename Arc::Weight;
540 PhiMatcher(const FST &fst, MatchType match_type, Label phi_label = kNoLabel,
541 bool phi_loop = true,
542 MatcherRewriteMode rewrite_mode = MATCHER_REWRITE_AUTO,
543 M *matcher = nullptr)
544 : matcher_(matcher ? matcher : new M(fst, match_type)),
545 match_type_(match_type),
546 phi_label_(phi_label),
550 if (match_type == MATCH_BOTH) {
551 FSTERROR() << "PhiMatcher: Bad match type";
552 match_type_ = MATCH_NONE;
555 if (rewrite_mode == MATCHER_REWRITE_AUTO) {
556 rewrite_both_ = fst.Properties(kAcceptor, true);
557 } else if (rewrite_mode == MATCHER_REWRITE_ALWAYS) {
558 rewrite_both_ = true;
560 rewrite_both_ = false;
564 PhiMatcher(const PhiMatcher<M> &matcher, bool safe = false)
565 : matcher_(new M(*matcher.matcher_, safe)),
566 match_type_(matcher.match_type_),
567 phi_label_(matcher.phi_label_),
568 rewrite_both_(matcher.rewrite_both_),
570 phi_loop_(matcher.phi_loop_),
571 error_(matcher.error_) {}
573 PhiMatcher<M> *Copy(bool safe = false) const override {
574 return new PhiMatcher<M>(*this, safe);
577 MatchType Type(bool test) const override { return matcher_->Type(test); }
579 void SetState(StateId s) final {
580 if (state_ == s) return;
581 matcher_->SetState(s);
583 has_phi_ = phi_label_ != kNoLabel;
586 bool Find(Label match_label) final;
588 bool Done() const final { return matcher_->Done(); }
590 const Arc &Value() const final {
591 if ((phi_match_ == kNoLabel) && (phi_weight_ == Weight::One())) {
592 return matcher_->Value();
593 } else if (phi_match_ == 0) { // Virtual epsilon loop.
594 phi_arc_ = Arc(kNoLabel, 0, Weight::One(), state_);
595 if (match_type_ == MATCH_OUTPUT) {
596 std::swap(phi_arc_.ilabel, phi_arc_.olabel);
600 phi_arc_ = matcher_->Value();
601 phi_arc_.weight = Times(phi_weight_, phi_arc_.weight);
602 if (phi_match_ != kNoLabel) { // Phi loop match.
604 if (phi_arc_.ilabel == phi_label_) phi_arc_.ilabel = phi_match_;
605 if (phi_arc_.olabel == phi_label_) phi_arc_.olabel = phi_match_;
606 } else if (match_type_ == MATCH_INPUT) {
607 phi_arc_.ilabel = phi_match_;
609 phi_arc_.olabel = phi_match_;
616 void Next() final { matcher_->Next(); }
618 Weight Final(StateId s) const final {
619 auto weight = matcher_->Final(s);
620 if (phi_label_ == kNoLabel || weight != Weight::Zero()) {
623 weight = Weight::One();
624 matcher_->SetState(s);
625 while (matcher_->Final(s) == Weight::Zero()) {
626 if (!matcher_->Find(phi_label_ == 0 ? -1 : phi_label_)) break;
627 weight = Times(weight, matcher_->Value().weight);
628 if (s == matcher_->Value().nextstate) {
629 return Weight::Zero(); // Does not follow phi self-loops.
631 s = matcher_->Value().nextstate;
632 matcher_->SetState(s);
634 weight = Times(weight, matcher_->Final(s));
638 ssize_t Priority(StateId s) final {
639 if (phi_label_ != kNoLabel) {
640 matcher_->SetState(s);
641 const bool has_phi = matcher_->Find(phi_label_ == 0 ? -1 : phi_label_);
642 return has_phi ? kRequirePriority : matcher_->Priority(s);
644 return matcher_->Priority(s);
648 const FST &GetFst() const override { return matcher_->GetFst(); }
650 uint64 Properties(uint64 props) const override;
652 uint32 Flags() const override {
653 if (phi_label_ == kNoLabel || match_type_ == MATCH_NONE) {
654 return matcher_->Flags();
656 return matcher_->Flags() | kRequireMatch;
659 Label PhiLabel() const { return phi_label_; }
662 mutable std::unique_ptr<M> matcher_;
663 MatchType match_type_; // Type of match requested.
664 Label phi_label_; // Label that represents the phi transition.
665 bool rewrite_both_; // Rewrite both sides when both are phi_label_?
666 bool has_phi_; // Are there possibly phis at the current state?
667 Label phi_match_; // Current label that matches phi loop.
668 mutable Arc phi_arc_; // Arc to return.
669 StateId state_; // Matcher state.
670 Weight phi_weight_; // Product of the weights of phi transitions taken.
671 bool phi_loop_; // When true, phi self-loop are allowed and treated
672 // as rho (required for Aho-Corasick).
673 bool error_; // Error encountered?
675 PhiMatcher &operator=(const PhiMatcher &) = delete;
679 inline bool PhiMatcher<M>::Find(Label label) {
680 if (label == phi_label_ && phi_label_ != kNoLabel && phi_label_ != 0) {
681 FSTERROR() << "PhiMatcher::Find: bad label (phi): " << phi_label_;
685 matcher_->SetState(state_);
686 phi_match_ = kNoLabel;
687 phi_weight_ = Weight::One();
688 // If phi_label_ == 0, there are no more true epsilon arcs.
689 if (phi_label_ == 0) {
690 if (label == kNoLabel) {
693 if (label == 0) { // but a virtual epsilon loop needs to be returned.
694 if (!matcher_->Find(kNoLabel)) {
695 return matcher_->Find(0);
702 if (!has_phi_ || label == 0 || label == kNoLabel) {
703 return matcher_->Find(label);
706 while (!matcher_->Find(label)) {
707 // Look for phi transition (if phi_label_ == 0, we need to look
708 // for -1 to avoid getting the virtual self-loop)
709 if (!matcher_->Find(phi_label_ == 0 ? -1 : phi_label_)) return false;
710 if (phi_loop_ && matcher_->Value().nextstate == s) {
714 phi_weight_ = Times(phi_weight_, matcher_->Value().weight);
715 s = matcher_->Value().nextstate;
717 if (!matcher_->Done()) {
718 FSTERROR() << "PhiMatcher: Phi non-determinism not supported";
721 matcher_->SetState(s);
727 inline uint64 PhiMatcher<M>::Properties(uint64 inprops) const {
728 auto outprops = matcher_->Properties(inprops);
729 if (error_) outprops |= kError;
730 if (match_type_ == MATCH_NONE) {
732 } else if (match_type_ == MATCH_INPUT) {
733 if (phi_label_ == 0) {
734 outprops &= ~kEpsilons | ~kIEpsilons | ~kOEpsilons;
735 outprops |= kNoEpsilons | kNoIEpsilons;
739 ~(kODeterministic | kNonODeterministic | kString | kILabelSorted |
740 kNotILabelSorted | kOLabelSorted | kNotOLabelSorted);
743 ~(kODeterministic | kAcceptor | kString | kILabelSorted |
744 kNotILabelSorted | kOLabelSorted | kNotOLabelSorted);
746 } else if (match_type_ == MATCH_OUTPUT) {
747 if (phi_label_ == 0) {
748 outprops &= ~kEpsilons | ~kIEpsilons | ~kOEpsilons;
749 outprops |= kNoEpsilons | kNoOEpsilons;
753 ~(kIDeterministic | kNonIDeterministic | kString | kILabelSorted |
754 kNotILabelSorted | kOLabelSorted | kNotOLabelSorted);
757 ~(kIDeterministic | kAcceptor | kString | kILabelSorted |
758 kNotILabelSorted | kOLabelSorted | kNotOLabelSorted);
761 // Shouldn't ever get here.
762 FSTERROR() << "PhiMatcher: Bad match type: " << match_type_;
767 // For any requested label that doesn't match at a state, this matcher
768 // considers all transitions that match the label 'rho_label' (rho =
769 // 'rest'). Each such rho transition found is returned with the
770 // rho_label rewritten as the requested label (both sides if an
771 // acceptor, or if 'rewrite_both' is true and both input and output
772 // labels of the found transition are 'rho_label'). If 'rho_label' is
773 // kNoLabel, this special matching is not done. RhoMatcher is
774 // templated itself on a matcher, which is used to perform the
775 // underlying matching. By default, the underlying matcher is
776 // constructed by RhoMatcher. The user can instead pass in this
777 // object; in that case, RhoMatcher takes its ownership.
778 // No non-consuming symbols other than epsilon supported with
779 // the underlying template argument matcher.
781 class RhoMatcher : public MatcherBase<typename M::Arc> {
783 using FST = typename M::FST;
784 using Arc = typename FST::Arc;
785 using Label = typename Arc::Label;
786 using StateId = typename Arc::StateId;
787 using Weight = typename Arc::Weight;
789 RhoMatcher(const FST &fst, MatchType match_type, Label rho_label = kNoLabel,
790 MatcherRewriteMode rewrite_mode = MATCHER_REWRITE_AUTO,
791 M *matcher = nullptr)
792 : matcher_(matcher ? matcher : new M(fst, match_type)),
793 match_type_(match_type),
794 rho_label_(rho_label),
797 if (match_type == MATCH_BOTH) {
798 FSTERROR() << "RhoMatcher: Bad match type";
799 match_type_ = MATCH_NONE;
802 if (rho_label == 0) {
803 FSTERROR() << "RhoMatcher: 0 cannot be used as rho_label";
804 rho_label_ = kNoLabel;
807 if (rewrite_mode == MATCHER_REWRITE_AUTO) {
808 rewrite_both_ = fst.Properties(kAcceptor, true);
809 } else if (rewrite_mode == MATCHER_REWRITE_ALWAYS) {
810 rewrite_both_ = true;
812 rewrite_both_ = false;
816 RhoMatcher(const RhoMatcher<M> &matcher, bool safe = false)
817 : matcher_(new M(*matcher.matcher_, safe)),
818 match_type_(matcher.match_type_),
819 rho_label_(matcher.rho_label_),
820 rewrite_both_(matcher.rewrite_both_),
821 error_(matcher.error_),
822 state_(kNoStateId) {}
824 RhoMatcher<M> *Copy(bool safe = false) const override {
825 return new RhoMatcher<M>(*this, safe);
828 MatchType Type(bool test) const override { return matcher_->Type(test); }
830 void SetState(StateId s) final {
831 if (state_ == s) return;
833 matcher_->SetState(s);
834 has_rho_ = rho_label_ != kNoLabel;
837 bool Find(Label label) final {
838 if (label == rho_label_ && rho_label_ != kNoLabel) {
839 FSTERROR() << "RhoMatcher::Find: bad label (rho)";
843 if (matcher_->Find(label)) {
844 rho_match_ = kNoLabel;
846 } else if (has_rho_ && label != 0 && label != kNoLabel &&
847 (has_rho_ = matcher_->Find(rho_label_))) {
855 bool Done() const final { return matcher_->Done(); }
857 const Arc &Value() const final {
858 if (rho_match_ == kNoLabel) {
859 return matcher_->Value();
861 rho_arc_ = matcher_->Value();
863 if (rho_arc_.ilabel == rho_label_) rho_arc_.ilabel = rho_match_;
864 if (rho_arc_.olabel == rho_label_) rho_arc_.olabel = rho_match_;
865 } else if (match_type_ == MATCH_INPUT) {
866 rho_arc_.ilabel = rho_match_;
868 rho_arc_.olabel = rho_match_;
874 void Next() final { matcher_->Next(); }
876 Weight Final(StateId s) const final { return matcher_->Final(s); }
878 ssize_t Priority(StateId s) final {
880 matcher_->SetState(s);
881 has_rho_ = matcher_->Find(rho_label_);
883 return kRequirePriority;
885 return matcher_->Priority(s);
889 const FST &GetFst() const override { return matcher_->GetFst(); }
891 uint64 Properties(uint64 props) const override;
893 uint32 Flags() const override {
894 if (rho_label_ == kNoLabel || match_type_ == MATCH_NONE) {
895 return matcher_->Flags();
897 return matcher_->Flags() | kRequireMatch;
900 Label RhoLabel() const { return rho_label_; }
903 std::unique_ptr<M> matcher_;
904 MatchType match_type_; // Type of match requested.
905 Label rho_label_; // Label that represents the rho transition
906 bool rewrite_both_; // Rewrite both sides when both are rho_label_?
907 bool has_rho_; // Are there possibly rhos at the current state?
908 Label rho_match_; // Current label that matches rho transition.
909 mutable Arc rho_arc_; // Arc to return when rho match.
910 bool error_; // Error encountered?
911 StateId state_; // Matcher state.
915 inline uint64 RhoMatcher<M>::Properties(uint64 inprops) const {
916 auto outprops = matcher_->Properties(inprops);
917 if (error_) outprops |= kError;
918 if (match_type_ == MATCH_NONE) {
920 } else if (match_type_ == MATCH_INPUT) {
923 ~(kODeterministic | kNonODeterministic | kString | kILabelSorted |
924 kNotILabelSorted | kOLabelSorted | kNotOLabelSorted);
927 ~(kODeterministic | kAcceptor | kString | kILabelSorted |
930 } else if (match_type_ == MATCH_OUTPUT) {
933 ~(kIDeterministic | kNonIDeterministic | kString | kILabelSorted |
934 kNotILabelSorted | kOLabelSorted | kNotOLabelSorted);
937 ~(kIDeterministic | kAcceptor | kString | kOLabelSorted |
941 // Shouldn't ever get here.
942 FSTERROR() << "RhoMatcher: Bad match type: " << match_type_;
947 // For any requested label, this matcher considers all transitions
948 // that match the label 'sigma_label' (sigma = "any"), and this in
949 // additions to transitions with the requested label. Each such sigma
950 // transition found is returned with the sigma_label rewritten as the
951 // requested label (both sides if an acceptor, or if 'rewrite_both' is
952 // true and both input and output labels of the found transition are
953 // 'sigma_label'). If 'sigma_label' is kNoLabel, this special
954 // matching is not done. SigmaMatcher is templated itself on a
955 // matcher, which is used to perform the underlying matching. By
956 // default, the underlying matcher is constructed by SigmaMatcher.
957 // The user can instead pass in this object; in that case,
958 // SigmaMatcher takes its ownership. No non-consuming symbols other
959 // than epsilon supported with the underlying template argument matcher.
961 class SigmaMatcher : public MatcherBase<typename M::Arc> {
963 using FST = typename M::FST;
964 using Arc = typename FST::Arc;
965 using Label = typename Arc::Label;
966 using StateId = typename Arc::StateId;
967 using Weight = typename Arc::Weight;
969 SigmaMatcher(const FST &fst, MatchType match_type,
970 Label sigma_label = kNoLabel,
971 MatcherRewriteMode rewrite_mode = MATCHER_REWRITE_AUTO,
972 M *matcher = nullptr)
973 : matcher_(matcher ? matcher : new M(fst, match_type)),
974 match_type_(match_type),
975 sigma_label_(sigma_label),
978 if (match_type == MATCH_BOTH) {
979 FSTERROR() << "SigmaMatcher: Bad match type";
980 match_type_ = MATCH_NONE;
983 if (sigma_label == 0) {
984 FSTERROR() << "SigmaMatcher: 0 cannot be used as sigma_label";
985 sigma_label_ = kNoLabel;
988 if (rewrite_mode == MATCHER_REWRITE_AUTO) {
989 rewrite_both_ = fst.Properties(kAcceptor, true);
990 } else if (rewrite_mode == MATCHER_REWRITE_ALWAYS) {
991 rewrite_both_ = true;
993 rewrite_both_ = false;
997 SigmaMatcher(const SigmaMatcher<M> &matcher, bool safe = false)
998 : matcher_(new M(*matcher.matcher_, safe)),
999 match_type_(matcher.match_type_),
1000 sigma_label_(matcher.sigma_label_),
1001 rewrite_both_(matcher.rewrite_both_),
1002 error_(matcher.error_),
1003 state_(kNoStateId) {}
1005 SigmaMatcher<M> *Copy(bool safe = false) const override {
1006 return new SigmaMatcher<M>(*this, safe);
1009 MatchType Type(bool test) const override { return matcher_->Type(test); }
1011 void SetState(StateId s) final {
1012 if (state_ == s) return;
1014 matcher_->SetState(s);
1016 (sigma_label_ != kNoLabel) ? matcher_->Find(sigma_label_) : false;
1019 bool Find(Label match_label) final {
1020 match_label_ = match_label;
1021 if (match_label == sigma_label_ && sigma_label_ != kNoLabel) {
1022 FSTERROR() << "SigmaMatcher::Find: bad label (sigma)";
1026 if (matcher_->Find(match_label)) {
1027 sigma_match_ = kNoLabel;
1029 } else if (has_sigma_ && match_label != 0 && match_label != kNoLabel &&
1030 matcher_->Find(sigma_label_)) {
1031 sigma_match_ = match_label;
1038 bool Done() const final { return matcher_->Done(); }
1040 const Arc &Value() const final {
1041 if (sigma_match_ == kNoLabel) {
1042 return matcher_->Value();
1044 sigma_arc_ = matcher_->Value();
1045 if (rewrite_both_) {
1046 if (sigma_arc_.ilabel == sigma_label_) sigma_arc_.ilabel = sigma_match_;
1047 if (sigma_arc_.olabel == sigma_label_) sigma_arc_.olabel = sigma_match_;
1048 } else if (match_type_ == MATCH_INPUT) {
1049 sigma_arc_.ilabel = sigma_match_;
1051 sigma_arc_.olabel = sigma_match_;
1059 if (matcher_->Done() && has_sigma_ && (sigma_match_ == kNoLabel) &&
1060 (match_label_ > 0)) {
1061 matcher_->Find(sigma_label_);
1062 sigma_match_ = match_label_;
1066 Weight Final(StateId s) const final { return matcher_->Final(s); }
1068 ssize_t Priority(StateId s) final {
1069 if (sigma_label_ != kNoLabel) {
1071 return has_sigma_ ? kRequirePriority : matcher_->Priority(s);
1073 return matcher_->Priority(s);
1077 const FST &GetFst() const override { return matcher_->GetFst(); }
1079 uint64 Properties(uint64 props) const override;
1081 uint32 Flags() const override {
1082 if (sigma_label_ == kNoLabel || match_type_ == MATCH_NONE) {
1083 return matcher_->Flags();
1085 return matcher_->Flags() | kRequireMatch;
1088 Label SigmaLabel() const { return sigma_label_; }
1091 std::unique_ptr<M> matcher_;
1092 MatchType match_type_; // Type of match requested.
1093 Label sigma_label_; // Label that represents the sigma transition.
1094 bool rewrite_both_; // Rewrite both sides when both are sigma_label_?
1095 bool has_sigma_; // Are there sigmas at the current state?
1096 Label sigma_match_; // Current label that matches sigma transition.
1097 mutable Arc sigma_arc_; // Arc to return when sigma match.
1098 Label match_label_; // Label being matched.
1099 bool error_; // Error encountered?
1100 StateId state_; // Matcher state.
1104 inline uint64 SigmaMatcher<M>::Properties(uint64 inprops) const {
1105 auto outprops = matcher_->Properties(inprops);
1106 if (error_) outprops |= kError;
1107 if (match_type_ == MATCH_NONE) {
1109 } else if (rewrite_both_) {
1111 ~(kIDeterministic | kNonIDeterministic | kODeterministic |
1112 kNonODeterministic | kILabelSorted | kNotILabelSorted |
1113 kOLabelSorted | kNotOLabelSorted | kString);
1114 } else if (match_type_ == MATCH_INPUT) {
1116 ~(kIDeterministic | kNonIDeterministic | kODeterministic |
1117 kNonODeterministic | kILabelSorted | kNotILabelSorted | kString |
1119 } else if (match_type_ == MATCH_OUTPUT) {
1121 ~(kIDeterministic | kNonIDeterministic | kODeterministic |
1122 kNonODeterministic | kOLabelSorted | kNotOLabelSorted | kString |
1125 // Shouldn't ever get here.
1126 FSTERROR() << "SigmaMatcher: Bad match type: " << match_type_;
1131 // Flags for MultiEpsMatcher.
1133 // Return multi-epsilon arcs for Find(kNoLabel).
1134 const uint32 kMultiEpsList = 0x00000001;
1136 // Return a kNolabel loop for Find(multi_eps).
1137 const uint32 kMultiEpsLoop = 0x00000002;
1139 // MultiEpsMatcher: allows treating multiple non-0 labels as
1140 // non-consuming labels in addition to 0 that is always
1141 // non-consuming. Precise behavior controlled by 'flags' argument. By
1142 // default, the underlying matcher is constructed by
1143 // MultiEpsMatcher. The user can instead pass in this object; in that
1144 // case, MultiEpsMatcher takes its ownership iff 'own_matcher' is
1147 class MultiEpsMatcher {
1149 using FST = typename M::FST;
1150 using Arc = typename FST::Arc;
1151 using Label = typename Arc::Label;
1152 using StateId = typename Arc::StateId;
1153 using Weight = typename Arc::Weight;
1155 MultiEpsMatcher(const FST &fst, MatchType match_type,
1156 uint32 flags = (kMultiEpsLoop | kMultiEpsList),
1157 M *matcher = nullptr, bool own_matcher = true)
1158 : matcher_(matcher ? matcher : new M(fst, match_type)),
1160 own_matcher_(matcher ? own_matcher : true) {
1161 if (match_type == MATCH_INPUT) {
1162 loop_.ilabel = kNoLabel;
1166 loop_.olabel = kNoLabel;
1168 loop_.weight = Weight::One();
1169 loop_.nextstate = kNoStateId;
1172 MultiEpsMatcher(const MultiEpsMatcher<M> &matcher, bool safe = false)
1173 : matcher_(new M(*matcher.matcher_, safe)),
1174 flags_(matcher.flags_),
1176 multi_eps_labels_(matcher.multi_eps_labels_),
1177 loop_(matcher.loop_) {
1178 loop_.nextstate = kNoStateId;
1181 ~MultiEpsMatcher() {
1182 if (own_matcher_) delete matcher_;
1185 MultiEpsMatcher<M> *Copy(bool safe = false) const {
1186 return new MultiEpsMatcher<M>(*this, safe);
1189 MatchType Type(bool test) const { return matcher_->Type(test); }
1191 void SetState(StateId state) {
1192 matcher_->SetState(state);
1193 loop_.nextstate = state;
1196 bool Find(Label label);
1198 bool Done() const { return done_; }
1200 const Arc &Value() const { return current_loop_ ? loop_ : matcher_->Value(); }
1203 if (!current_loop_) {
1205 done_ = matcher_->Done();
1206 if (done_ && multi_eps_iter_ != multi_eps_labels_.End()) {
1208 while ((multi_eps_iter_ != multi_eps_labels_.End()) &&
1209 !matcher_->Find(*multi_eps_iter_)) {
1212 if (multi_eps_iter_ != multi_eps_labels_.End()) {
1215 done_ = !matcher_->Find(kNoLabel);
1223 const FST &GetFst() const { return matcher_->GetFst(); }
1225 uint64 Properties(uint64 props) const { return matcher_->Properties(props); }
1227 const M *GetMatcher() const { return matcher_; }
1229 Weight Final(StateId s) const { return matcher_->Final(s); }
1231 uint32 Flags() const { return matcher_->Flags(); }
1233 ssize_t Priority(StateId s) { return matcher_->Priority(s); }
1235 void AddMultiEpsLabel(Label label) {
1237 FSTERROR() << "MultiEpsMatcher: Bad multi-eps label: 0";
1239 multi_eps_labels_.Insert(label);
1243 void RemoveMultiEpsLabel(Label label) {
1245 FSTERROR() << "MultiEpsMatcher: Bad multi-eps label: 0";
1247 multi_eps_labels_.Erase(label);
1251 void ClearMultiEpsLabels() { multi_eps_labels_.Clear(); }
1256 bool own_matcher_; // Does this class delete the matcher?
1258 // Multi-eps label set.
1259 CompactSet<Label, kNoLabel> multi_eps_labels_;
1260 typename CompactSet<Label, kNoLabel>::const_iterator multi_eps_iter_;
1262 bool current_loop_; // Current arc is the implicit loop?
1263 mutable Arc loop_; // For non-consuming symbols.
1264 bool done_; // Matching done?
1266 MultiEpsMatcher &operator=(const MultiEpsMatcher &) = delete;
1270 inline bool MultiEpsMatcher<M>::Find(Label label) {
1271 multi_eps_iter_ = multi_eps_labels_.End();
1272 current_loop_ = false;
1275 ret = matcher_->Find(0);
1276 } else if (label == kNoLabel) {
1277 if (flags_ & kMultiEpsList) {
1278 // Returns all non-consuming arcs (including epsilon).
1279 multi_eps_iter_ = multi_eps_labels_.Begin();
1280 while ((multi_eps_iter_ != multi_eps_labels_.End()) &&
1281 !matcher_->Find(*multi_eps_iter_)) {
1284 if (multi_eps_iter_ != multi_eps_labels_.End()) {
1287 ret = matcher_->Find(kNoLabel);
1290 // Returns all epsilon arcs.
1291 ret = matcher_->Find(kNoLabel);
1293 } else if ((flags_ & kMultiEpsLoop) &&
1294 multi_eps_labels_.Find(label) != multi_eps_labels_.End()) {
1295 // Returns implicit loop.
1296 current_loop_ = true;
1299 ret = matcher_->Find(label);
1305 // This class discards any implicit matches (e.g., the implicit epsilon
1306 // self-loops in the SortedMatcher). Matchers are most often used in
1307 // composition/intersection where the implicit matches are needed
1308 // e.g. for epsilon processing. However, if a matcher is simply being
1309 // used to look-up explicit label matches, this class saves the user
1310 // from having to check for and discard the unwanted implicit matches
1313 class ExplicitMatcher : public MatcherBase<typename M::Arc> {
1315 using FST = typename M::FST;
1316 using Arc = typename FST::Arc;
1317 using Label = typename Arc::Label;
1318 using StateId = typename Arc::StateId;
1319 using Weight = typename Arc::Weight;
1321 ExplicitMatcher(const FST &fst, MatchType match_type, M *matcher = nullptr)
1322 : matcher_(matcher ? matcher : new M(fst, match_type)),
1323 match_type_(match_type),
1326 ExplicitMatcher(const ExplicitMatcher<M> &matcher, bool safe = false)
1327 : matcher_(new M(*matcher.matcher_, safe)),
1328 match_type_(matcher.match_type_),
1329 error_(matcher.error_) {}
1331 ExplicitMatcher<M> *Copy(bool safe = false) const override {
1332 return new ExplicitMatcher<M>(*this, safe);
1335 MatchType Type(bool test) const override { return matcher_->Type(test); }
1337 void SetState(StateId s) final { matcher_->SetState(s); }
1339 bool Find(Label label) final {
1340 matcher_->Find(label);
1345 bool Done() const final { return matcher_->Done(); }
1347 const Arc &Value() const final { return matcher_->Value(); }
1354 Weight Final(StateId s) const final { return matcher_->Final(s); }
1356 ssize_t Priority(StateId s) final { return matcher_->Priority(s); }
1358 const FST &GetFst() const final { return matcher_->GetFst(); }
1360 uint64 Properties(uint64 inprops) const override {
1361 return matcher_->Properties(inprops);
1364 const M *GetMatcher() const { return matcher_.get(); }
1366 uint32 Flags() const override { return matcher_->Flags(); }
1369 // Checks current arc if available and explicit. If not available, stops. If
1370 // not explicit, checks next ones.
1372 for (; !matcher_->Done(); matcher_->Next()) {
1373 const auto label = match_type_ == MATCH_INPUT ? matcher_->Value().ilabel
1374 : matcher_->Value().olabel;
1375 if (label != kNoLabel) return;
1379 std::unique_ptr<M> matcher_;
1380 MatchType match_type_; // Type of match requested.
1381 bool error_; // Error encountered?
1384 // Generic matcher, templated on the FST definition.
1386 // Here is a typical use:
1388 // Matcher<StdFst> matcher(fst, MATCH_INPUT);
1389 // matcher.SetState(state);
1390 // if (matcher.Find(label))
1391 // for (; !matcher.Done(); matcher.Next()) {
1392 // auto &arc = matcher.Value();
1399 using Arc = typename F::Arc;
1400 using Label = typename Arc::Label;
1401 using StateId = typename Arc::StateId;
1402 using Weight = typename Arc::Weight;
1404 Matcher(const FST &fst, MatchType match_type) {
1405 base_.reset(fst.InitMatcher(match_type));
1406 if (!base_) base_.reset(new SortedMatcher<FST>(fst, match_type));
1409 Matcher(const Matcher<FST> &matcher, bool safe = false) {
1410 base_.reset(matcher.base_->Copy(safe));
1413 // Takes ownership of the provided matcher.
1414 explicit Matcher(MatcherBase<Arc> *base_matcher) {
1415 base_.reset(base_matcher);
1418 Matcher<FST> *Copy(bool safe = false) const {
1419 return new Matcher<FST>(*this, safe);
1422 MatchType Type(bool test) const { return base_->Type(test); }
1424 void SetState(StateId s) { base_->SetState(s); }
1426 bool Find(Label label) { return base_->Find(label); }
1428 bool Done() const { return base_->Done(); }
1430 const Arc &Value() const { return base_->Value(); }
1432 void Next() { base_->Next(); }
1434 const FST &GetFst() const {
1435 return static_cast<const FST &>(base_->GetFst());
1438 uint64 Properties(uint64 props) const { return base_->Properties(props); }
1440 Weight Final(StateId s) const { return base_->Final(s); }
1442 uint32 Flags() const { return base_->Flags() & kMatcherFlags; }
1444 ssize_t Priority(StateId s) { return base_->Priority(s); }
1447 std::unique_ptr<MatcherBase<Arc>> base_;
1452 #endif // FST_LIB_MATCHER_H_