1 // See www.openfst.org for extensive documentation on this weighted
2 // finite-state transducer library.
4 // FST utility inline definitions.
16 #include <type_traits>
17 #include <unordered_map>
18 #include <unordered_set>
22 #include <fst/compat.h>
23 #include <fst/types.h>
27 #include <fst/flags.h>
30 // Utility for error handling.
32 DECLARE_bool(fst_error_fatal);
35 (FLAGS_fst_error_fatal ? LOG(FATAL) : LOG(ERROR))
39 // Utility for type I/O.
41 // Reads types from an input stream.
45 typename std::enable_if<std::is_class<T>::value, T>::type* = nullptr>
46 inline std::istream &ReadType(std::istream &strm, T *t) {
50 // Numeric (boolean, integral, floating-point) case.
52 typename std::enable_if<std::is_arithmetic<T>::value, T>::type* = nullptr>
53 inline std::istream &ReadType(std::istream &strm, T *t) {
54 return strm.read(reinterpret_cast<char *>(t), sizeof(T)); \
58 inline std::istream &ReadType(std::istream &strm, string *s) { // NOLINT
61 strm.read(reinterpret_cast<char *>(&ns), sizeof(ns));
62 for (int32 i = 0; i < ns; ++i) {
70 // Declares types that can be read from an input stream.
72 std::istream &ReadType(std::istream &strm, std::vector<T...> *c);
74 std::istream &ReadType(std::istream &strm, std::list<T...> *c);
76 std::istream &ReadType(std::istream &strm, std::set<T...> *c);
78 std::istream &ReadType(std::istream &strm, std::map<T...> *c);
80 std::istream &ReadType(std::istream &strm, std::unordered_map<T...> *c);
82 std::istream &ReadType(std::istream &strm, std::unordered_set<T...> *c);
85 template <typename S, typename T>
86 inline std::istream &ReadType(std::istream &strm, std::pair<S, T> *p) {
87 ReadType(strm, &p->first);
88 ReadType(strm, &p->second);
92 template <typename S, typename T>
93 inline std::istream &ReadType(std::istream &strm, std::pair<const S, T> *p) {
94 ReadType(strm, const_cast<S *>(&p->first));
95 ReadType(strm, &p->second);
100 template <class C, class ReserveFn>
101 std::istream &ReadContainerType(std::istream &strm, C *c, ReserveFn reserve) {
106 auto insert = std::inserter(*c, c->begin());
107 for (int64 i = 0; i < n; ++i) {
108 typename C::value_type value;
109 ReadType(strm, &value);
114 } // namespace internal
116 template <class... T>
117 std::istream &ReadType(std::istream &strm, std::vector<T...> *c) {
118 return internal::ReadContainerType(
119 strm, c, [](decltype(c) v, int n) { v->reserve(n); });
122 template <class... T>
123 std::istream &ReadType(std::istream &strm, std::list<T...> *c) {
124 return internal::ReadContainerType(strm, c, [](decltype(c) v, int n) {});
127 template <class... T>
128 std::istream &ReadType(std::istream &strm, std::set<T...> *c) {
129 return internal::ReadContainerType(strm, c, [](decltype(c) v, int n) {});
132 template <class... T>
133 std::istream &ReadType(std::istream &strm, std::map<T...> *c) {
134 return internal::ReadContainerType(strm, c, [](decltype(c) v, int n) {});
137 template <class... T>
138 std::istream &ReadType(std::istream &strm, std::unordered_set<T...> *c) {
139 return internal::ReadContainerType(
140 strm, c, [](decltype(c) v, int n) { v->reserve(n); });
143 template <class... T>
144 std::istream &ReadType(std::istream &strm, std::unordered_map<T...> *c) {
145 return internal::ReadContainerType(
146 strm, c, [](decltype(c) v, int n) { v->reserve(n); });
149 // Writes types to an output stream.
153 typename std::enable_if<std::is_class<T>::value, T>::type* = nullptr>
154 inline std::ostream &WriteType(std::ostream &strm, const T t) {
159 // Numeric (boolean, integral, floating-point) case.
161 typename std::enable_if<std::is_arithmetic<T>::value, T>::type* = nullptr>
162 inline std::ostream &WriteType(std::ostream &strm, const T t) {
163 return strm.write(reinterpret_cast<const char *>(&t), sizeof(T));
167 inline std::ostream &WriteType(std::ostream &strm, const string &s) { // NOLINT
169 strm.write(reinterpret_cast<const char *>(&ns), sizeof(ns));
170 return strm.write(s.data(), ns);
173 // Declares types that can be written to an output stream.
175 template <typename... T>
176 std::ostream &WriteType(std::ostream &strm, const std::vector<T...> &c);
177 template <typename... T>
178 std::ostream &WriteType(std::ostream &strm, const std::list<T...> &c);
179 template <typename... T>
180 std::ostream &WriteType(std::ostream &strm, const std::set<T...> &c);
181 template <typename... T>
182 std::ostream &WriteType(std::ostream &strm, const std::map<T...> &c);
183 template <typename... T>
184 std::ostream &WriteType(std::ostream &strm, const std::unordered_map<T...> &c);
185 template <typename... T>
186 std::ostream &WriteType(std::ostream &strm, const std::unordered_set<T...> &c);
189 template <typename S, typename T>
190 inline std::ostream &WriteType(std::ostream &strm,
191 const std::pair<S, T> &p) { // NOLINT
192 WriteType(strm, p.first);
193 WriteType(strm, p.second);
199 std::ostream &WriteContainer(std::ostream &strm, const C &c) {
200 const int64 n = c.size();
202 for (const auto &e : c) {
207 } // namespace internal
209 template <typename... T>
210 std::ostream &WriteType(std::ostream &strm, const std::vector<T...> &c) {
211 return internal::WriteContainer(strm, c);
214 template <typename... T>
215 std::ostream &WriteType(std::ostream &strm, const std::list<T...> &c) {
216 return internal::WriteContainer(strm, c);
219 template <typename... T>
220 std::ostream &WriteType(std::ostream &strm, const std::set<T...> &c) {
221 return internal::WriteContainer(strm, c);
224 template <typename... T>
225 std::ostream &WriteType(std::ostream &strm, const std::map<T...> &c) {
226 return internal::WriteContainer(strm, c);
229 template <typename... T>
230 std::ostream &WriteType(std::ostream &strm, const std::unordered_map<T...> &c) {
231 return internal::WriteContainer(strm, c);
234 template <typename... T>
235 std::ostream &WriteType(std::ostream &strm, const std::unordered_set<T...> &c) {
236 return internal::WriteContainer(strm, c);
239 // Utilities for converting between int64 or Weight and string.
241 int64 StrToInt64(const string &s, const string &src, size_t nline,
242 bool allow_negative, bool *error = nullptr);
244 template <typename Weight>
245 Weight StrToWeight(const string &s, const string &src, size_t nline) {
247 std::istringstream strm(s);
250 FSTERROR() << "StrToWeight: Bad weight = \"" << s << "\", source = " << src
251 << ", line = " << nline;
252 return Weight::NoWeight();
257 template <typename Weight>
258 void WeightToStr(Weight w, string *s) {
259 std::ostringstream strm;
262 s->append(strm.str().data(), strm.str().size());
265 // Utilities for reading/writing integer pairs (typically labels)
267 // Modifies line using a vector of pointers to a buffer beginning with line.
268 void SplitString(char *line, const char *delim, std::vector<char *> *vec,
269 bool omit_empty_strings);
271 template <typename I>
272 bool ReadIntPairs(const string &filename, std::vector<std::pair<I, I>> *pairs,
273 bool allow_negative = false) {
274 std::ifstream strm(filename, std::ios_base::in);
276 LOG(ERROR) << "ReadIntPairs: Can't open file: " << filename;
279 const int kLineLen = 8096;
283 while (strm.getline(line, kLineLen)) {
285 std::vector<char *> col;
286 SplitString(line, "\n\t ", &col, true);
287 // empty line or comment?
288 if (col.empty() || col[0][0] == '\0' || col[0][0] == '#') continue;
289 if (col.size() != 2) {
290 LOG(ERROR) << "ReadIntPairs: Bad number of columns, "
291 << "file = " << filename << ", line = " << nline;
295 I i1 = StrToInt64(col[0], filename, nline, allow_negative, &err);
296 if (err) return false;
297 I i2 = StrToInt64(col[1], filename, nline, allow_negative, &err);
298 if (err) return false;
299 pairs->push_back(std::make_pair(i1, i2));
304 template <typename I>
305 bool WriteIntPairs(const string &filename,
306 const std::vector<std::pair<I, I>> &pairs) {
307 std::ostream *strm = &std::cout;
308 if (!filename.empty()) {
309 strm = new std::ofstream(filename);
311 LOG(ERROR) << "WriteIntPairs: Can't open file: " << filename;
315 for (ssize_t n = 0; n < pairs.size(); ++n) {
316 *strm << pairs[n].first << "\t" << pairs[n].second << "\n";
319 LOG(ERROR) << "WriteIntPairs: Write failed: "
320 << (filename.empty() ? "standard output" : filename);
323 if (strm != &std::cout) delete strm;
327 // Utilities for reading/writing label pairs.
329 template <typename Label>
330 bool ReadLabelPairs(const string &filename,
331 std::vector<std::pair<Label, Label>> *pairs,
332 bool allow_negative = false) {
333 return ReadIntPairs(filename, pairs, allow_negative);
336 template <typename Label>
337 bool WriteLabelPairs(const string &filename,
338 const std::vector<std::pair<Label, Label>> &pairs) {
339 return WriteIntPairs(filename, pairs);
342 // Utilities for converting a type name to a legal C symbol.
344 void ConvertToLegalCSymbol(string *s);
346 // Utilities for stream I/O.
348 bool AlignInput(std::istream &strm);
349 bool AlignOutput(std::ostream &strm);
351 // An associative container for which testing membership is faster than an STL
352 // set if members are restricted to an interval that excludes most non-members.
353 // A Key must have ==, !=, and < operators defined. Element NoKey should be a
354 // key that marks an uninitialized key and is otherwise unused. Find() returns
355 // an STL const_iterator to the match found, otherwise it equals End().
356 template <class Key, Key NoKey>
359 using const_iterator = typename std::set<Key>::const_iterator;
361 CompactSet() : min_key_(NoKey), max_key_(NoKey) {}
363 CompactSet(const CompactSet<Key, NoKey> &compact_set)
364 : set_(compact_set.set_),
365 min_key_(compact_set.min_key_),
366 max_key_(compact_set.max_key_) {}
368 void Insert(Key key) {
370 if (min_key_ == NoKey || key < min_key_) min_key_ = key;
371 if (max_key_ == NoKey || max_key_ < key) max_key_ = key;
374 void Erase(Key key) {
377 min_key_ = max_key_ = NoKey;
378 } else if (key == min_key_) {
380 } else if (key == max_key_) {
387 min_key_ = max_key_ = NoKey;
390 const_iterator Find(Key key) const {
391 if (min_key_ == NoKey || key < min_key_ || max_key_ < key) {
394 return set_.find(key);
398 bool Member(Key key) const {
399 if (min_key_ == NoKey || key < min_key_ || max_key_ < key) {
400 return false; // out of range
401 } else if (min_key_ != NoKey && max_key_ + 1 == min_key_ + set_.size()) {
402 return true; // dense range
404 return set_.count(key);
408 const_iterator Begin() const { return set_.begin(); }
410 const_iterator End() const { return set_.end(); }
412 // All stored keys are greater than or equal to this value.
413 Key LowerBound() const { return min_key_; }
415 // All stored keys are less than or equal to this value.
416 Key UpperBound() const { return max_key_; }
423 void operator=(const CompactSet &) = delete;
428 #endif // FST_UTIL_H_