1 // See www.openfst.org for extensive documentation on this weighted
2 // finite-state transducer library.
4 // Class to map over/transform arcs e.g., change semirings or
5 // implement project/invert. Consider using when operation does
6 // not change the number of arcs (except possibly superfinal arcs).
8 #ifndef FST_LIB_ARC_MAP_H_
9 #define FST_LIB_ARC_MAP_H_
12 #include <unordered_map>
17 #include <fst/cache.h>
18 #include <fst/mutable-fst.h>
23 // Determines how final weights are mapped.
25 // A final weight is mapped into a final weight. An error is raised if this
28 // A final weight is mapped to an arc to the superfinal state when the result
29 // cannot be represented as a final weight. The superfinal state will be
30 // added only if it is needed.
32 // A final weight is mapped to an arc to the superfinal state unless the
33 // result can be represented as a final weight of weight Zero(). The
34 // superfinal state is always added (if the input is not the empty FST).
35 MAP_REQUIRE_SUPERFINAL
38 // Determines how symbol tables are mapped.
39 enum MapSymbolsAction {
40 // Symbols should be cleared in the result by the map.
42 // Symbols should be copied from the input FST by the map.
44 // Symbols should not be modified in the result by the map itself.
45 // (They may set by the mapper).
49 // The ArcMapper interfaces defines how arcs and final weights are mapped.
50 // This is useful for implementing operations that do not change the number of
51 // arcs (expect possibly superfinal arcs).
53 // template <class A, class B>
59 // // Maps an arc type FromArc to arc type ToArc.
60 // ToArc operator()(const FromArc &arc);
62 // // Specifies final action the mapper requires (see above).
63 // // The mapper will be passed final weights as arcs of the form
64 // // Arc(0, 0, weight, kNoStateId).
65 // MapFinalAction FinalAction() const;
67 // // Specifies input symbol table action the mapper requires (see above).
68 // MapSymbolsAction InputSymbolsAction() const;
70 // // Specifies output symbol table action the mapper requires (see above).
71 // MapSymbolsAction OutputSymbolsAction() const;
73 // // This specifies the known properties of an FST mapped by this mapper. It
74 // takes as argument the input FSTs's known properties.
75 // uint64 Properties(uint64 props) const;
78 // The ArcMap functions and classes below will use the FinalAction()
79 // method of the mapper to determine how to treat final weights, e.g., whether
80 // to add a superfinal state. They will use the Properties() method to set the
81 // result FST properties.
83 // We include a various map versions below. One dimension of variation is
84 // whether the mapping mutates its input, writes to a new result FST, or is an
85 // on-the-fly FST. Another dimension is how we pass the mapper. We allow passing
86 // the mapper by pointer for cases that we need to change the state of the
87 // user's mapper. This is the case with the EncodeMapper, which is reused
88 // during decoding. We also include map versions that pass the mapper by value
89 // or const reference when this suffices.
91 // Maps an arc type A using a mapper function object C, passed
92 // by pointer. This version modifies its Fst input.
93 template <class A, class C>
94 void ArcMap(MutableFst<A> *fst, C *mapper) {
97 using StateId = typename FromArc::StateId;
98 using Weight = typename FromArc::Weight;
99 if (mapper->InputSymbolsAction() == MAP_CLEAR_SYMBOLS) {
100 fst->SetInputSymbols(nullptr);
102 if (mapper->OutputSymbolsAction() == MAP_CLEAR_SYMBOLS) {
103 fst->SetOutputSymbols(nullptr);
105 if (fst->Start() == kNoStateId) return;
106 const auto props = fst->Properties(kFstProperties, false);
107 const auto final_action = mapper->FinalAction();
108 auto superfinal = kNoStateId;
109 if (final_action == MAP_REQUIRE_SUPERFINAL) {
110 superfinal = fst->AddState();
111 fst->SetFinal(superfinal, Weight::One());
113 for (StateIterator<MutableFst<FromArc>> siter(*fst); !siter.Done();
115 const auto state = siter.Value();
116 for (MutableArcIterator<MutableFst<FromArc>> aiter(fst, state);
117 !aiter.Done(); aiter.Next()) {
118 const auto &arc = aiter.Value();
119 aiter.SetValue((*mapper)(arc));
121 switch (final_action) {
122 case MAP_NO_SUPERFINAL:
124 const FromArc arc(0, 0, fst->Final(state), kNoStateId);
125 const auto final_arc = (*mapper)(arc);
126 if (final_arc.ilabel != 0 || final_arc.olabel != 0) {
127 FSTERROR() << "ArcMap: Non-zero arc labels for superfinal arc";
128 fst->SetProperties(kError, kError);
130 fst->SetFinal(state, final_arc.weight);
133 case MAP_ALLOW_SUPERFINAL: {
134 if (state != superfinal) {
135 const FromArc arc(0, 0, fst->Final(state), kNoStateId);
136 auto final_arc = (*mapper)(arc);
137 if (final_arc.ilabel != 0 || final_arc.olabel != 0) {
138 // Add a superfinal state if not already done.
139 if (superfinal == kNoStateId) {
140 superfinal = fst->AddState();
141 fst->SetFinal(superfinal, Weight::One());
143 final_arc.nextstate = superfinal;
144 fst->AddArc(state, final_arc);
145 fst->SetFinal(state, Weight::Zero());
147 fst->SetFinal(state, final_arc.weight);
152 case MAP_REQUIRE_SUPERFINAL: {
153 if (state != superfinal) {
154 const FromArc arc(0, 0, fst->Final(state), kNoStateId);
155 const auto final_arc = (*mapper)(arc);
156 if (final_arc.ilabel != 0 || final_arc.olabel != 0 ||
157 final_arc.weight != Weight::Zero()) {
158 fst->AddArc(state, ToArc(final_arc.ilabel, final_arc.olabel,
159 final_arc.weight, superfinal));
161 fst->SetFinal(state, Weight::Zero());
167 fst->SetProperties(mapper->Properties(props), kFstProperties);
170 // Maps an arc type A using a mapper function object C, passed by value. This
171 // version modifies its FST input.
172 template <class A, class C>
173 void ArcMap(MutableFst<A> *fst, C mapper) {
174 ArcMap(fst, &mapper);
177 // Maps an arc type A to an arc type B using mapper function object C,
178 // passed by pointer. This version writes the mapped input FST to an
179 // output MutableFst.
180 template <class A, class B, class C>
181 void ArcMap(const Fst<A> &ifst, MutableFst<B> *ofst, C *mapper) {
183 using StateId = typename FromArc::StateId;
184 using Weight = typename FromArc::Weight;
185 ofst->DeleteStates();
186 if (mapper->InputSymbolsAction() == MAP_COPY_SYMBOLS) {
187 ofst->SetInputSymbols(ifst.InputSymbols());
188 } else if (mapper->InputSymbolsAction() == MAP_CLEAR_SYMBOLS) {
189 ofst->SetInputSymbols(nullptr);
191 if (mapper->OutputSymbolsAction() == MAP_COPY_SYMBOLS) {
192 ofst->SetOutputSymbols(ifst.OutputSymbols());
193 } else if (mapper->OutputSymbolsAction() == MAP_CLEAR_SYMBOLS) {
194 ofst->SetOutputSymbols(nullptr);
196 const auto iprops = ifst.Properties(kCopyProperties, false);
197 if (ifst.Start() == kNoStateId) {
198 if (iprops & kError) ofst->SetProperties(kError, kError);
201 const auto final_action = mapper->FinalAction();
202 if (ifst.Properties(kExpanded, false)) {
204 CountStates(ifst) + final_action == MAP_NO_SUPERFINAL ? 0 : 1);
207 for (StateIterator<Fst<A>> siter(ifst); !siter.Done(); siter.Next()) {
210 StateId superfinal = kNoStateId;
211 if (final_action == MAP_REQUIRE_SUPERFINAL) {
212 superfinal = ofst->AddState();
213 ofst->SetFinal(superfinal, B::Weight::One());
215 for (StateIterator<Fst<A>> siter(ifst); !siter.Done(); siter.Next()) {
216 StateId s = siter.Value();
217 if (s == ifst.Start()) ofst->SetStart(s);
218 ofst->ReserveArcs(s, ifst.NumArcs(s));
219 for (ArcIterator<Fst<A>> aiter(ifst, s); !aiter.Done(); aiter.Next()) {
220 ofst->AddArc(s, (*mapper)(aiter.Value()));
222 switch (final_action) {
223 case MAP_NO_SUPERFINAL:
225 B final_arc = (*mapper)(A(0, 0, ifst.Final(s), kNoStateId));
226 if (final_arc.ilabel != 0 || final_arc.olabel != 0) {
227 FSTERROR() << "ArcMap: Non-zero arc labels for superfinal arc";
228 ofst->SetProperties(kError, kError);
230 ofst->SetFinal(s, final_arc.weight);
233 case MAP_ALLOW_SUPERFINAL: {
234 B final_arc = (*mapper)(A(0, 0, ifst.Final(s), kNoStateId));
235 if (final_arc.ilabel != 0 || final_arc.olabel != 0) {
236 // Add a superfinal state if not already done.
237 if (superfinal == kNoStateId) {
238 superfinal = ofst->AddState();
239 ofst->SetFinal(superfinal, B::Weight::One());
241 final_arc.nextstate = superfinal;
242 ofst->AddArc(s, final_arc);
243 ofst->SetFinal(s, B::Weight::Zero());
245 ofst->SetFinal(s, final_arc.weight);
249 case MAP_REQUIRE_SUPERFINAL: {
250 B final_arc = (*mapper)(A(0, 0, ifst.Final(s), kNoStateId));
251 if (final_arc.ilabel != 0 || final_arc.olabel != 0 ||
252 final_arc.weight != B::Weight::Zero()) {
253 ofst->AddArc(s, B(final_arc.ilabel, final_arc.olabel,
254 final_arc.weight, superfinal));
256 ofst->SetFinal(s, B::Weight::Zero());
261 const auto oprops = ofst->Properties(kFstProperties, false);
262 ofst->SetProperties(mapper->Properties(iprops) | oprops, kFstProperties);
265 // Maps an arc type A to an arc type B using mapper function
266 // object C, passed by value. This version writes the mapped input
267 // Fst to an output MutableFst.
268 template <class A, class B, class C>
269 void ArcMap(const Fst<A> &ifst, MutableFst<B> *ofst, C mapper) {
270 ArcMap(ifst, ofst, &mapper);
273 struct ArcMapFstOptions : public CacheOptions {
274 // ArcMapFst default caching behaviour is to do no caching. Most mappers are
275 // cheap and therefore we save memory by not doing caching.
276 ArcMapFstOptions() : CacheOptions(true, 0) {}
278 explicit ArcMapFstOptions(const CacheOptions &opts) : CacheOptions(opts) {}
281 template <class A, class B, class C>
286 // Implementation of delayed ArcMapFst.
287 template <class A, class B, class C>
288 class ArcMapFstImpl : public CacheImpl<B> {
291 using StateId = typename Arc::StateId;
292 using Weight = typename Arc::Weight;
294 using FstImpl<B>::SetType;
295 using FstImpl<B>::SetProperties;
296 using FstImpl<B>::SetInputSymbols;
297 using FstImpl<B>::SetOutputSymbols;
299 using CacheImpl<B>::PushArc;
300 using CacheImpl<B>::HasArcs;
301 using CacheImpl<B>::HasFinal;
302 using CacheImpl<B>::HasStart;
303 using CacheImpl<B>::SetArcs;
304 using CacheImpl<B>::SetFinal;
305 using CacheImpl<B>::SetStart;
307 friend class StateIterator<ArcMapFst<A, B, C>>;
309 ArcMapFstImpl(const Fst<A> &fst, const C &mapper,
310 const ArcMapFstOptions &opts)
311 : CacheImpl<B>(opts),
313 mapper_(new C(mapper)),
315 superfinal_(kNoStateId),
320 ArcMapFstImpl(const Fst<A> &fst, C *mapper, const ArcMapFstOptions &opts)
321 : CacheImpl<B>(opts),
325 superfinal_(kNoStateId),
330 ArcMapFstImpl(const ArcMapFstImpl<A, B, C> &impl)
331 : CacheImpl<B>(impl),
332 fst_(impl.fst_->Copy(true)),
333 mapper_(new C(*impl.mapper_)),
335 superfinal_(kNoStateId),
340 ~ArcMapFstImpl() override {
341 if (own_mapper_) delete mapper_;
345 if (!HasStart()) SetStart(FindOState(fst_->Start()));
346 return CacheImpl<B>::Start();
349 Weight Final(StateId s) {
351 switch (final_action_) {
352 case MAP_NO_SUPERFINAL:
354 const auto final_arc =
355 (*mapper_)(A(0, 0, fst_->Final(FindIState(s)), kNoStateId));
356 if (final_arc.ilabel != 0 || final_arc.olabel != 0) {
357 FSTERROR() << "ArcMapFst: Non-zero arc labels for superfinal arc";
358 SetProperties(kError, kError);
360 SetFinal(s, final_arc.weight);
363 case MAP_ALLOW_SUPERFINAL: {
364 if (s == superfinal_) {
365 SetFinal(s, Weight::One());
367 const auto final_arc =
368 (*mapper_)(A(0, 0, fst_->Final(FindIState(s)), kNoStateId));
369 if (final_arc.ilabel == 0 && final_arc.olabel == 0) {
370 SetFinal(s, final_arc.weight);
372 SetFinal(s, Weight::Zero());
377 case MAP_REQUIRE_SUPERFINAL: {
378 SetFinal(s, s == superfinal_ ? Weight::One() : Weight::Zero());
383 return CacheImpl<B>::Final(s);
386 size_t NumArcs(StateId s) {
387 if (!HasArcs(s)) Expand(s);
388 return CacheImpl<B>::NumArcs(s);
391 size_t NumInputEpsilons(StateId s) {
392 if (!HasArcs(s)) Expand(s);
393 return CacheImpl<B>::NumInputEpsilons(s);
396 size_t NumOutputEpsilons(StateId s) {
397 if (!HasArcs(s)) Expand(s);
398 return CacheImpl<B>::NumOutputEpsilons(s);
401 uint64 Properties() const override { return Properties(kFstProperties); }
403 // Sets error if found, and returns other FST impl properties.
404 uint64 Properties(uint64 mask) const override {
405 if ((mask & kError) && (fst_->Properties(kError, false) ||
406 (mapper_->Properties(0) & kError))) {
407 SetProperties(kError, kError);
409 return FstImpl<Arc>::Properties(mask);
412 void InitArcIterator(StateId s, ArcIteratorData<B> *data) {
413 if (!HasArcs(s)) Expand(s);
414 CacheImpl<B>::InitArcIterator(s, data);
417 void Expand(StateId s) {
419 if (s == superfinal_) {
423 for (ArcIterator<Fst<A>> aiter(*fst_, FindIState(s)); !aiter.Done();
425 auto aarc = aiter.Value();
426 aarc.nextstate = FindOState(aarc.nextstate);
427 const auto &barc = (*mapper_)(aarc);
431 // Check for superfinal arcs.
432 if (!HasFinal(s) || Final(s) == Weight::Zero()) {
433 switch (final_action_) {
434 case MAP_NO_SUPERFINAL:
437 case MAP_ALLOW_SUPERFINAL: {
439 (*mapper_)(A(0, 0, fst_->Final(FindIState(s)), kNoStateId));
440 if (final_arc.ilabel != 0 || final_arc.olabel != 0) {
441 if (superfinal_ == kNoStateId) superfinal_ = nstates_++;
442 final_arc.nextstate = superfinal_;
443 PushArc(s, final_arc);
447 case MAP_REQUIRE_SUPERFINAL: {
448 const auto final_arc =
449 (*mapper_)(A(0, 0, fst_->Final(FindIState(s)), kNoStateId));
450 if (final_arc.ilabel != 0 || final_arc.olabel != 0 ||
451 final_arc.weight != B::Weight::Zero()) {
452 PushArc(s, B(final_arc.ilabel, final_arc.olabel, final_arc.weight,
465 if (mapper_->InputSymbolsAction() == MAP_COPY_SYMBOLS) {
466 SetInputSymbols(fst_->InputSymbols());
467 } else if (mapper_->InputSymbolsAction() == MAP_CLEAR_SYMBOLS) {
468 SetInputSymbols(nullptr);
470 if (mapper_->OutputSymbolsAction() == MAP_COPY_SYMBOLS) {
471 SetOutputSymbols(fst_->OutputSymbols());
472 } else if (mapper_->OutputSymbolsAction() == MAP_CLEAR_SYMBOLS) {
473 SetOutputSymbols(nullptr);
475 if (fst_->Start() == kNoStateId) {
476 final_action_ = MAP_NO_SUPERFINAL;
477 SetProperties(kNullProperties);
479 final_action_ = mapper_->FinalAction();
480 uint64 props = fst_->Properties(kCopyProperties, false);
481 SetProperties(mapper_->Properties(props));
482 if (final_action_ == MAP_REQUIRE_SUPERFINAL) superfinal_ = 0;
486 // Maps from output state to input state.
487 StateId FindIState(StateId s) {
488 if (superfinal_ == kNoStateId || s < superfinal_) {
495 // Maps from input state to output state.
496 StateId FindOState(StateId is) {
498 if (!(superfinal_ == kNoStateId || is < superfinal_)) ++os;
499 if (os >= nstates_) nstates_ = os + 1;
503 std::unique_ptr<const Fst<A>> fst_;
505 const bool own_mapper_;
506 MapFinalAction final_action_;
511 } // namespace internal
513 // Maps an arc type A to an arc type B using Mapper function object
514 // C. This version is a delayed FST.
515 template <class A, class B, class C>
516 class ArcMapFst : public ImplToFst<internal::ArcMapFstImpl<A, B, C>> {
519 using StateId = typename Arc::StateId;
520 using Weight = typename Arc::Weight;
522 using Store = DefaultCacheStore<B>;
523 using State = typename Store::State;
524 using Impl = internal::ArcMapFstImpl<A, B, C>;
526 friend class ArcIterator<ArcMapFst<A, B, C>>;
527 friend class StateIterator<ArcMapFst<A, B, C>>;
529 ArcMapFst(const Fst<A> &fst, const C &mapper, const ArcMapFstOptions &opts)
530 : ImplToFst<Impl>(std::make_shared<Impl>(fst, mapper, opts)) {}
532 ArcMapFst(const Fst<A> &fst, C *mapper, const ArcMapFstOptions &opts)
533 : ImplToFst<Impl>(std::make_shared<Impl>(fst, mapper, opts)) {}
535 ArcMapFst(const Fst<A> &fst, const C &mapper)
537 std::make_shared<Impl>(fst, mapper, ArcMapFstOptions())) {}
539 ArcMapFst(const Fst<A> &fst, C *mapper)
541 std::make_shared<Impl>(fst, mapper, ArcMapFstOptions())) {}
543 // See Fst<>::Copy() for doc.
544 ArcMapFst(const ArcMapFst<A, B, C> &fst, bool safe = false)
545 : ImplToFst<Impl>(fst, safe) {}
547 // Get a copy of this ArcMapFst. See Fst<>::Copy() for further doc.
548 ArcMapFst<A, B, C> *Copy(bool safe = false) const override {
549 return new ArcMapFst<A, B, C>(*this, safe);
552 inline void InitStateIterator(StateIteratorData<B> *data) const override;
554 void InitArcIterator(StateId s, ArcIteratorData<B> *data) const override {
555 GetMutableImpl()->InitArcIterator(s, data);
559 using ImplToFst<Impl>::GetImpl;
560 using ImplToFst<Impl>::GetMutableImpl;
563 ArcMapFst &operator=(const ArcMapFst &) = delete;
566 // Specialization for ArcMapFst.
568 // This may be derived from.
569 template <class A, class B, class C>
570 class StateIterator<ArcMapFst<A, B, C>> : public StateIteratorBase<B> {
572 using StateId = typename B::StateId;
574 explicit StateIterator(const ArcMapFst<A, B, C> &fst)
575 : impl_(fst.GetImpl()),
576 siter_(*impl_->fst_),
578 superfinal_(impl_->final_action_ == MAP_REQUIRE_SUPERFINAL) {
582 bool Done() const final { return siter_.Done() && !superfinal_; }
584 StateId Value() const final { return s_; }
588 if (!siter_.Done()) {
591 } else if (superfinal_) {
599 superfinal_ = impl_->final_action_ == MAP_REQUIRE_SUPERFINAL;
604 void CheckSuperfinal() {
605 if (impl_->final_action_ != MAP_ALLOW_SUPERFINAL || superfinal_) return;
606 if (!siter_.Done()) {
607 const auto final_arc =
608 (*impl_->mapper_)(A(0, 0, impl_->fst_->Final(s_), kNoStateId));
609 if (final_arc.ilabel != 0 || final_arc.olabel != 0) superfinal_ = true;
613 const internal::ArcMapFstImpl<A, B, C> *impl_;
614 StateIterator<Fst<A>> siter_;
616 bool superfinal_; // True if there is a superfinal state and not done.
619 // Specialization for ArcMapFst.
620 template <class A, class B, class C>
621 class ArcIterator<ArcMapFst<A, B, C>>
622 : public CacheArcIterator<ArcMapFst<A, B, C>> {
624 using StateId = typename A::StateId;
626 ArcIterator(const ArcMapFst<A, B, C> &fst, StateId s)
627 : CacheArcIterator<ArcMapFst<A, B, C>>(fst.GetMutableImpl(), s) {
628 if (!fst.GetImpl()->HasArcs(s)) fst.GetMutableImpl()->Expand(s);
632 template <class A, class B, class C>
633 inline void ArcMapFst<A, B, C>::InitStateIterator(
634 StateIteratorData<B> *data) const {
635 data->base = new StateIterator<ArcMapFst<A, B, C>>(*this);
640 // Mapper that returns its input.
642 class IdentityArcMapper {
647 ToArc operator()(const FromArc &arc) const { return arc; }
649 constexpr MapFinalAction FinalAction() const { return MAP_NO_SUPERFINAL; }
651 constexpr MapSymbolsAction InputSymbolsAction() const {
652 return MAP_COPY_SYMBOLS;
655 constexpr MapSymbolsAction OutputSymbolsAction() const {
656 return MAP_COPY_SYMBOLS;
659 uint64 Properties(uint64 props) const { return props; }
662 // Mapper that converts all input symbols to epsilon.
664 class InputEpsilonMapper {
669 ToArc operator()(const FromArc &arc) const {
670 return ToArc(0, arc.olabel, arc.weight, arc.nextstate);
673 constexpr MapFinalAction FinalAction() const { return MAP_NO_SUPERFINAL; }
675 constexpr MapSymbolsAction InputSymbolsAction() const {
676 return MAP_CLEAR_SYMBOLS;
679 constexpr MapSymbolsAction OutputSymbolsAction() const {
680 return MAP_COPY_SYMBOLS;
683 uint64 Properties(uint64 props) const {
684 return (props & kSetArcProperties) | kIEpsilons;
688 // Mapper that converts all output symbols to epsilon.
690 class OutputEpsilonMapper {
695 ToArc operator()(const FromArc &arc) const {
696 return ToArc(arc.ilabel, 0, arc.weight, arc.nextstate);
699 constexpr MapFinalAction FinalAction() const { return MAP_NO_SUPERFINAL; }
701 constexpr MapSymbolsAction InputSymbolsAction() const {
702 return MAP_COPY_SYMBOLS;
705 constexpr MapSymbolsAction OutputSymbolsAction() const {
706 return MAP_CLEAR_SYMBOLS;
709 uint64 Properties(uint64 props) const {
710 return (props & kSetArcProperties) | kOEpsilons;
714 // Mapper that returns its input with final states redirected to a single
715 // super-final state.
717 class SuperFinalMapper {
721 using Label = typename FromArc::Label;
722 using Weight = typename FromArc::Weight;;
724 // Arg allows setting super-final label.
725 explicit SuperFinalMapper(Label final_label = 0)
726 : final_label_(final_label) {}
728 ToArc operator()(const FromArc &arc) const {
730 if (arc.nextstate == kNoStateId && arc.weight != Weight::Zero()) {
731 return ToArc(final_label_, final_label_, arc.weight, kNoStateId);
737 constexpr MapFinalAction FinalAction() const {
738 return MAP_REQUIRE_SUPERFINAL;
741 constexpr MapSymbolsAction InputSymbolsAction() const {
742 return MAP_COPY_SYMBOLS;
745 constexpr MapSymbolsAction OutputSymbolsAction() const {
746 return MAP_COPY_SYMBOLS;
749 uint64 Properties(uint64 props) const {
750 if (final_label_ == 0) {
751 return props & kAddSuperFinalProperties;
753 return props & kAddSuperFinalProperties &
754 kILabelInvariantProperties & kOLabelInvariantProperties;
762 // Mapper that leaves labels and nextstate unchanged and constructs a new weight
763 // from the underlying value of the arc weight. If no weight converter is
764 // explictly specified, requires that there is a WeightConvert class
765 // specialization that converts the weights.
766 template <class A, class B,
767 class C = WeightConvert<typename A::Weight, typename B::Weight>>
768 class WeightConvertMapper {
773 using FromWeight = typename FromArc::Weight;
774 using ToWeight = typename ToArc::Weight;
776 explicit WeightConvertMapper(const Converter &c = Converter())
777 : convert_weight_(c) {}
779 ToArc operator()(const FromArc &arc) const {
780 return ToArc(arc.ilabel, arc.olabel, convert_weight_(arc.weight),
784 constexpr MapFinalAction FinalAction() const { return MAP_NO_SUPERFINAL; }
786 constexpr MapSymbolsAction InputSymbolsAction() const {
787 return MAP_COPY_SYMBOLS;
790 constexpr MapSymbolsAction OutputSymbolsAction() const {
791 return MAP_COPY_SYMBOLS;
794 uint64 Properties(uint64 props) const { return props; }
797 Converter convert_weight_;
800 // Non-precision-changing weight conversions; consider using more efficient
801 // Cast method instead.
803 using StdToLogMapper = WeightConvertMapper<StdArc, LogArc>;
805 using LogToStdMapper = WeightConvertMapper<LogArc, StdArc>;
807 // Precision-changing weight conversions.
809 using StdToLog64Mapper = WeightConvertMapper<StdArc, Log64Arc>;
811 using LogToLog64Mapper = WeightConvertMapper<LogArc, Log64Arc>;
813 using Log64ToStdMapper = WeightConvertMapper<Log64Arc, StdArc>;
815 using Log64ToLogMapper = WeightConvertMapper<Log64Arc, LogArc>;
817 // Mapper from A to GallicArc<A>.
818 template <class A, GallicType G = GALLIC_LEFT>
819 class ToGallicMapper {
822 using ToArc = GallicArc<A, G>;
824 using SW = StringWeight<typename A::Label, GallicStringType(G)>;
825 using AW = typename FromArc::Weight;
826 using GW = typename ToArc::Weight;
828 ToArc operator()(const FromArc &arc) const {
830 if (arc.nextstate == kNoStateId && arc.weight != AW::Zero()) {
831 return ToArc(0, 0, GW(SW::One(), arc.weight), kNoStateId);
832 // Super-non-final arc.
833 } else if (arc.nextstate == kNoStateId) {
834 return ToArc(0, 0, GW::Zero(), kNoStateId);
836 } else if (arc.olabel == 0) {
837 return ToArc(arc.ilabel, arc.ilabel, GW(SW::One(), arc.weight),
841 return ToArc(arc.ilabel, arc.ilabel, GW(SW(arc.olabel), arc.weight),
846 constexpr MapFinalAction FinalAction() const { return MAP_NO_SUPERFINAL; }
848 constexpr MapSymbolsAction InputSymbolsAction() const {
849 return MAP_COPY_SYMBOLS;
852 constexpr MapSymbolsAction OutputSymbolsAction() const {
853 return MAP_CLEAR_SYMBOLS;
856 uint64 Properties(uint64 props) const {
857 return ProjectProperties(props, true) & kWeightInvariantProperties;
861 // Mapper from GallicArc<A> to A.
862 template <class A, GallicType G = GALLIC_LEFT>
863 class FromGallicMapper {
865 using FromArc = GallicArc<A, G>;
868 using Label = typename ToArc::Label;
869 using AW = typename ToArc::Weight;
870 using GW = typename FromArc::Weight;
872 explicit FromGallicMapper(Label superfinal_label = 0)
873 : superfinal_label_(superfinal_label), error_(false) {}
875 ToArc operator()(const FromArc &arc) const {
876 // 'Super-non-final' arc.
877 if (arc.nextstate == kNoStateId && arc.weight == GW::Zero()) {
878 return A(arc.ilabel, 0, AW::Zero(), kNoStateId);
882 if (!Extract(arc.weight, &weight, &l) || arc.ilabel != arc.olabel) {
883 FSTERROR() << "FromGallicMapper: Unrepresentable weight: " << arc.weight
884 << " for arc with ilabel = " << arc.ilabel
885 << ", olabel = " << arc.olabel
886 << ", nextstate = " << arc.nextstate;
889 if (arc.ilabel == 0 && l != 0 && arc.nextstate == kNoStateId) {
890 return ToArc(superfinal_label_, l, weight, arc.nextstate);
892 return ToArc(arc.ilabel, l, weight, arc.nextstate);
896 constexpr MapFinalAction FinalAction() const { return MAP_ALLOW_SUPERFINAL; }
898 constexpr MapSymbolsAction InputSymbolsAction() const {
899 return MAP_COPY_SYMBOLS;
902 constexpr MapSymbolsAction OutputSymbolsAction() const {
903 return MAP_CLEAR_SYMBOLS;
906 uint64 Properties(uint64 inprops) const {
907 uint64 outprops = inprops & kOLabelInvariantProperties &
908 kWeightInvariantProperties & kAddSuperFinalProperties;
909 if (error_) outprops |= kError;
914 template <GallicType GT>
915 static bool Extract(const GallicWeight<Label, AW, GT> &gallic_weight,
916 typename A::Weight *weight, typename A::Label *label) {
917 using GW = StringWeight<Label, GallicStringType(GT)>;
918 const GW &w1 = gallic_weight.Value1();
919 const AW &w2 = gallic_weight.Value2();
920 typename GW::Iterator iter1(w1);
921 const Label l = w1.Size() == 1 ? iter1.Value() : 0;
922 if (l == kStringInfinity || l == kStringBad || w1.Size() > 1) return false;
928 static bool Extract(const GallicWeight<Label, AW, GALLIC> &gallic_weight,
929 typename A::Weight *weight, typename A::Label *label) {
930 if (gallic_weight.Size() > 1) return false;
931 if (gallic_weight.Size() == 0) {
933 *weight = A::Weight::Zero();
936 return Extract<GALLIC_RESTRICT>(gallic_weight.Back(), weight, label);
939 const Label superfinal_label_;
943 // Mapper from GallicArc<A> to A.
944 template <class A, GallicType G = GALLIC_LEFT>
945 class GallicToNewSymbolsMapper {
947 using FromArc = GallicArc<A, G>;
950 using Label = typename ToArc::Label;
951 using StateId = typename ToArc::StateId;
952 using AW = typename ToArc::Weight;
953 using GW = typename FromArc::Weight;
954 using SW = StringWeight<Label, GallicStringType(G)>;
956 explicit GallicToNewSymbolsMapper(MutableFst<ToArc> *fst)
959 osymbols_(fst->OutputSymbols()),
962 fst_->DeleteStates();
963 state_ = fst_->AddState();
964 fst_->SetStart(state_);
965 fst_->SetFinal(state_, AW::One());
967 string name = osymbols_->Name() + "_from_gallic";
968 fst_->SetInputSymbols(new SymbolTable(name));
969 isymbols_ = fst_->MutableInputSymbols();
970 const int64 zero = 0;
971 isymbols_->AddSymbol(osymbols_->Find(zero), 0);
973 fst_->SetInputSymbols(nullptr);
977 ToArc operator()(const FromArc &arc) {
978 // Super-non-final arc.
979 if (arc.nextstate == kNoStateId && arc.weight == GW::Zero()) {
980 return ToArc(arc.ilabel, 0, AW::Zero(), kNoStateId);
982 SW w1 = arc.weight.Value1();
983 AW w2 = arc.weight.Value2();
985 if (w1.Size() == 0) {
988 auto insert_result = map_.insert(std::make_pair(w1, kNoLabel));
989 if (!insert_result.second) {
990 l = insert_result.first->second;
993 insert_result.first->second = l;
994 StringWeightIterator<SW> iter1(w1);
997 for (size_t i = 0, p = state_; i < w1.Size();
998 ++i, iter1.Next(), p = n) {
999 n = i == w1.Size() - 1 ? state_ : fst_->AddState();
1000 fst_->AddArc(p, ToArc(i ? 0 : l, iter1.Value(), AW::One(), n));
1003 s = s + osymbols_->Find(iter1.Value());
1006 if (isymbols_) isymbols_->AddSymbol(s, l);
1009 if (l == kStringInfinity || l == kStringBad || arc.ilabel != arc.olabel) {
1010 FSTERROR() << "GallicToNewSymbolMapper: Unrepresentable weight: " << l;
1013 return ToArc(arc.ilabel, l, w2, arc.nextstate);
1016 constexpr MapFinalAction FinalAction() const { return MAP_ALLOW_SUPERFINAL; }
1018 constexpr MapSymbolsAction InputSymbolsAction() const {
1019 return MAP_COPY_SYMBOLS;
1022 constexpr MapSymbolsAction OutputSymbolsAction() const {
1023 return MAP_CLEAR_SYMBOLS;
1026 uint64 Properties(uint64 inprops) const {
1027 uint64 outprops = inprops & kOLabelInvariantProperties &
1028 kWeightInvariantProperties & kAddSuperFinalProperties;
1029 if (error_) outprops |= kError;
1036 size_t operator()(const SW &x) const { return x.Hash(); }
1039 using Map = std::unordered_map<SW, Label, StringKey>;
1041 MutableFst<ToArc> *fst_;
1045 const SymbolTable *osymbols_;
1046 SymbolTable *isymbols_;
1047 mutable bool error_;
1050 // Mapper to add a constant to all weights.
1056 using Weight = typename FromArc::Weight;
1058 explicit PlusMapper(Weight weight) : weight_(std::move(weight)) {}
1060 ToArc operator()(const FromArc &arc) const {
1061 if (arc.weight == Weight::Zero()) return arc;
1062 return ToArc(arc.ilabel, arc.olabel, Plus(arc.weight, weight_),
1066 constexpr MapFinalAction FinalAction() const { return MAP_NO_SUPERFINAL; }
1068 constexpr MapSymbolsAction InputSymbolsAction() const {
1069 return MAP_COPY_SYMBOLS;
1072 constexpr MapSymbolsAction OutputSymbolsAction() const {
1073 return MAP_COPY_SYMBOLS;
1076 uint64 Properties(uint64 props) const {
1077 return props & kWeightInvariantProperties;
1081 const Weight weight_;
1084 // Mapper to (right) multiply a constant to all weights.
1090 using Weight = typename FromArc::Weight;
1092 explicit TimesMapper(Weight weight) : weight_(std::move(weight)) {}
1094 ToArc operator()(const FromArc &arc) const {
1095 if (arc.weight == Weight::Zero()) return arc;
1096 return ToArc(arc.ilabel, arc.olabel, Times(arc.weight, weight_),
1100 constexpr MapFinalAction FinalAction() const { return MAP_NO_SUPERFINAL; }
1102 constexpr MapSymbolsAction InputSymbolsAction() const {
1103 return MAP_COPY_SYMBOLS;
1106 constexpr MapSymbolsAction OutputSymbolsAction() const {
1107 return MAP_COPY_SYMBOLS;
1110 uint64 Properties(uint64 props) const {
1111 return props & kWeightInvariantProperties;
1115 const Weight weight_;
1118 // Mapper to take all arc-weights to a fixed power.
1124 using Weight = typename FromArc::Weight;
1126 explicit PowerMapper(size_t power) : power_(power) {}
1128 ToArc operator()(const FromArc &arc) const {
1129 return ToArc(arc.ilabel, arc.olabel, Power(arc.weight, power_),
1133 MapFinalAction FinalAction() const { return MAP_NO_SUPERFINAL; }
1135 MapSymbolsAction InputSymbolsAction() const { return MAP_COPY_SYMBOLS; }
1137 MapSymbolsAction OutputSymbolsAction() const { return MAP_COPY_SYMBOLS; }
1139 uint64 Properties(uint64 props) const {
1140 return props & kWeightInvariantProperties;
1147 // Mapper to reciprocate all non-Zero() weights.
1149 class InvertWeightMapper {
1153 using Weight = typename FromArc::Weight;
1155 ToArc operator()(const FromArc &arc) const {
1156 if (arc.weight == Weight::Zero()) return arc;
1157 return ToArc(arc.ilabel, arc.olabel, Divide(Weight::One(), arc.weight),
1161 constexpr MapFinalAction FinalAction() const { return MAP_NO_SUPERFINAL; }
1163 constexpr MapSymbolsAction InputSymbolsAction() const {
1164 return MAP_COPY_SYMBOLS;
1167 constexpr MapSymbolsAction OutputSymbolsAction() const {
1168 return MAP_COPY_SYMBOLS;
1171 uint64 Properties(uint64 props) const {
1172 return props & kWeightInvariantProperties;
1176 // Mapper to map all non-Zero() weights to One().
1177 template <class A, class B = A>
1178 class RmWeightMapper {
1182 using FromWeight = typename FromArc::Weight;
1183 using ToWeight = typename ToArc::Weight;
1185 ToArc operator()(const FromArc &arc) const {
1186 return ToArc(arc.ilabel, arc.olabel,
1187 arc.weight != FromWeight::Zero() ?
1188 ToWeight::One() : ToWeight::Zero(),
1192 constexpr MapFinalAction FinalAction() const { return MAP_NO_SUPERFINAL; }
1194 constexpr MapSymbolsAction InputSymbolsAction() const {
1195 return MAP_COPY_SYMBOLS;
1198 constexpr MapSymbolsAction OutputSymbolsAction() const {
1199 return MAP_COPY_SYMBOLS;
1202 uint64 Properties(uint64 props) const {
1203 return (props & kWeightInvariantProperties) | kUnweighted;
1207 // Mapper to quantize all weights.
1208 template <class A, class B = A>
1209 class QuantizeMapper {
1213 using FromWeight = typename FromArc::Weight;
1214 using ToWeight = typename ToArc::Weight;
1216 QuantizeMapper() : delta_(kDelta) {}
1218 explicit QuantizeMapper(float d) : delta_(d) {}
1220 ToArc operator()(const FromArc &arc) const {
1221 return ToArc(arc.ilabel, arc.olabel, arc.weight.Quantize(delta_),
1225 constexpr MapFinalAction FinalAction() const { return MAP_NO_SUPERFINAL; }
1227 constexpr MapSymbolsAction InputSymbolsAction() const {
1228 return MAP_COPY_SYMBOLS;
1231 constexpr MapSymbolsAction OutputSymbolsAction() const {
1232 return MAP_COPY_SYMBOLS;
1235 uint64 Properties(uint64 props) const {
1236 return props & kWeightInvariantProperties;
1243 // Mapper from A to B under the assumption:
1245 // B::Weight = A::Weight::ReverseWeight
1246 // B::Label == A::Label
1247 // B::StateId == A::StateId
1249 // The weight is reversed, while the label and nextstate are preserved.
1250 template <class A, class B>
1251 class ReverseWeightMapper {
1256 ToArc operator()(const FromArc &arc) const {
1257 return ToArc(arc.ilabel, arc.olabel, arc.weight.Reverse(), arc.nextstate);
1260 constexpr MapFinalAction FinalAction() const { return MAP_NO_SUPERFINAL; }
1262 constexpr MapSymbolsAction InputSymbolsAction() const {
1263 return MAP_COPY_SYMBOLS;
1266 constexpr MapSymbolsAction OutputSymbolsAction() const {
1267 return MAP_COPY_SYMBOLS;
1270 uint64 Properties(uint64 props) const { return props; }
1275 #endif // FST_LIB_ARC_MAP_H_