OpenFst: Release 1.7
- * Removes harmful constexpr specifications in FAR (1.7.4)
+ * Removes unnecessary template parameters in constructors (1.7.5)
+ * Converts RmEpsilonFstOptions from class to struct (1.7.5)
+ * Eliminates redundant checks in Minimize (1.7.5)
+ * CompactFst is now templated on Compactor rather than ArcCompactor (1.7.4)
+ * Removes harmful constexpr specifications in the FAR extension (1.7.4)
* Improved script API support for EncodeMapper (1.7.4)
* New header format for the EncodeMapper (1.7.4)
- * Imrproved C++17 compatibility shims (1.7.4)
- * Rewrite of the compactor interface (1.7.4)
+ * Many cleanups to the n-gram extension (1.7.4)
+ * Improved C++17 compatibility shims (1.7.4)
* Overloads Arc constructors with default weight argument (1.7.3)
* Fixes RmEpsilon and Union property checking bugs (1.7.3)
* Makes Isomorphic more robust to nondeterminism (1.7.3)
-OpenFst: Release 1.7.3.
-
OpenFst is a library for constructing, combining, optimizing, and searching
weighted finite-state transducers (FSTs).
#! /bin/sh
# Guess values for system-dependent variables and create Makefiles.
-# Generated by GNU Autoconf 2.69 for OpenFst 1.7.4.
+# Generated by GNU Autoconf 2.69 for OpenFst 1.7.5.
#
# Report bugs to <help@www.openfst.org>.
#
# Identity of this package.
PACKAGE_NAME='OpenFst'
PACKAGE_TARNAME='openfst'
-PACKAGE_VERSION='1.7.4'
-PACKAGE_STRING='OpenFst 1.7.4'
+PACKAGE_VERSION='1.7.5'
+PACKAGE_STRING='OpenFst 1.7.5'
PACKAGE_BUGREPORT='help@www.openfst.org'
PACKAGE_URL=''
# Omit some internal or obsolete options to make the list less imposing.
# This message is too long to be a string in the A/UX 3.1 sh.
cat <<_ACEOF
-\`configure' configures OpenFst 1.7.4 to adapt to many kinds of systems.
+\`configure' configures OpenFst 1.7.5 to adapt to many kinds of systems.
Usage: $0 [OPTION]... [VAR=VALUE]...
if test -n "$ac_init_help"; then
case $ac_init_help in
- short | recursive ) echo "Configuration of OpenFst 1.7.4:";;
+ short | recursive ) echo "Configuration of OpenFst 1.7.5:";;
esac
cat <<\_ACEOF
test -n "$ac_init_help" && exit $ac_status
if $ac_init_version; then
cat <<\_ACEOF
-OpenFst configure 1.7.4
+OpenFst configure 1.7.5
generated by GNU Autoconf 2.69
Copyright (C) 2012 Free Software Foundation, Inc.
This file contains any messages produced by compilers while
running configure, to aid debugging if configure makes a mistake.
-It was created by OpenFst $as_me 1.7.4, which was
+It was created by OpenFst $as_me 1.7.5, which was
generated by GNU Autoconf 2.69. Invocation command line was
$ $0 $@
# Define the identity of the package.
PACKAGE='openfst'
- VERSION='1.7.4'
+ VERSION='1.7.5'
cat >>confdefs.h <<_ACEOF
# report actual input values of CONFIG_FILES etc. instead of their
# values after options handling.
ac_log="
-This file was extended by OpenFst $as_me 1.7.4, which was
+This file was extended by OpenFst $as_me 1.7.5, which was
generated by GNU Autoconf 2.69. Invocation command line was
CONFIG_FILES = $CONFIG_FILES
cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1
ac_cs_config="`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`"
ac_cs_version="\\
-OpenFst config.status 1.7.4
+OpenFst config.status 1.7.5
configured by $0, generated by GNU Autoconf 2.69,
with options \\"\$ac_cs_config\\"
-AC_INIT([OpenFst], [1.7.4], [help@www.openfst.org])
+AC_INIT([OpenFst], [1.7.5], [help@www.openfst.org])
AM_INIT_AUTOMAKE([foreign nostdinc -Wall -Werror subdir-objects])
AM_PROG_AR
#include <string>
#include <fst/flags.h>
+#include <fst/types.h>
#include <fst/script/decode.h>
#include <fst/script/encode.h>
#include <fst/script/getters.h>
#include <string>
#include <fst/flags.h>
+#include <fst/types.h>
#include <fst/script/getters.h>
#include <fst/script/push.h>
#include <vector>
#include <fst/flags.h>
+#include <fst/types.h>
#include <fst/util.h>
#include <fst/script/relabel.h>
#include <fst/script/weight-class.h>
#include <vector>
#include <fst/flags.h>
+#include <fst/types.h>
#include <fst/script/getters.h>
#include <fst/script/replace.h>
#include <vector>
#include <fst/flags.h>
+#include <fst/types.h>
#include <fst/util.h>
#include <fst/script/fst-class.h>
#include <fst/script/verify.h>
lib_LTLIBRARIES = libfstcompact.la
libfstcompact_la_SOURCES = compact8_acceptor-fst.cc compact8_string-fst.cc compact8_unweighted-fst.cc compact8_unweighted_acceptor-fst.cc compact8_weighted_string-fst.cc compact16_acceptor-fst.cc compact16_string-fst.cc compact16_unweighted-fst.cc compact16_unweighted_acceptor-fst.cc compact16_weighted_string-fst.cc compact64_acceptor-fst.cc compact64_string-fst.cc compact64_unweighted-fst.cc compact64_unweighted_acceptor-fst.cc compact64_weighted_string-fst.cc
-libfstcompact_la_LDFLAGS = -version-info 17:0:0
+libfstcompact_la_LDFLAGS = -version-info 18:0:0
compact8_acceptor_fst_la_SOURCES = compact8_acceptor-fst.cc
compact8_acceptor_fst_la_LDFLAGS = -avoid-version -module
libfst_LTLIBRARIES = compact8_acceptor-fst.la compact8_string-fst.la compact8_unweighted-fst.la compact8_unweighted_acceptor-fst.la compact8_weighted_string-fst.la compact16_acceptor-fst.la compact16_string-fst.la compact16_unweighted-fst.la compact16_unweighted_acceptor-fst.la compact16_weighted_string-fst.la compact64_acceptor-fst.la compact64_string-fst.la compact64_unweighted-fst.la compact64_unweighted_acceptor-fst.la compact64_weighted_string-fst.la
lib_LTLIBRARIES = libfstcompact.la
libfstcompact_la_SOURCES = compact8_acceptor-fst.cc compact8_string-fst.cc compact8_unweighted-fst.cc compact8_unweighted_acceptor-fst.cc compact8_weighted_string-fst.cc compact16_acceptor-fst.cc compact16_string-fst.cc compact16_unweighted-fst.cc compact16_unweighted_acceptor-fst.cc compact16_weighted_string-fst.cc compact64_acceptor-fst.cc compact64_string-fst.cc compact64_unweighted-fst.cc compact64_unweighted_acceptor-fst.cc compact64_weighted_string-fst.cc
-libfstcompact_la_LDFLAGS = -version-info 17:0:0
+libfstcompact_la_LDFLAGS = -version-info 18:0:0
compact8_acceptor_fst_la_SOURCES = compact8_acceptor-fst.cc
compact8_acceptor_fst_la_LDFLAGS = -avoid-version -module
compact8_string_fst_la_SOURCES = compact8_string-fst.cc
if HAVE_SCRIPT
libfstcompressscript_la_SOURCES = compressscript.cc
-libfstcompressscript_la_LDFLAGS = -version-info 17:0:0
+libfstcompressscript_la_LDFLAGS = -version-info 18:0:0
libfstcompressscript_la_LIBADD = ../../script/libfstscript.la \
../../lib/libfst.la \
-lz -lm $(DL_LIBS)
@HAVE_BIN_TRUE@fstcompress_SOURCES = fstcompress.cc fstcompress-main.cc
@HAVE_SCRIPT_TRUE@libfstcompressscript_la_SOURCES = compressscript.cc
-@HAVE_SCRIPT_TRUE@libfstcompressscript_la_LDFLAGS = -version-info 17:0:0
+@HAVE_SCRIPT_TRUE@libfstcompressscript_la_LDFLAGS = -version-info 18:0:0
@HAVE_SCRIPT_TRUE@libfstcompressscript_la_LIBADD = ../../script/libfstscript.la \
@HAVE_SCRIPT_TRUE@ ../../lib/libfst.la \
@HAVE_SCRIPT_TRUE@ -lz -lm $(DL_LIBS)
lib_LTLIBRARIES = libfstconst.la
libfstconst_la_SOURCES = const8-fst.cc const16-fst.cc const64-fst.cc
-libfstconst_la_LDFLAGS = -version-info 17:0:0
+libfstconst_la_LDFLAGS = -version-info 18:0:0
const8_fst_la_SOURCES = const8-fst.cc
const8_fst_la_LDFLAGS = -avoid-version -module
libfst_LTLIBRARIES = const8-fst.la const16-fst.la const64-fst.la
lib_LTLIBRARIES = libfstconst.la
libfstconst_la_SOURCES = const8-fst.cc const16-fst.cc const64-fst.cc
-libfstconst_la_LDFLAGS = -version-info 17:0:0
+libfstconst_la_LDFLAGS = -version-info 18:0:0
const8_fst_la_SOURCES = const8-fst.cc
const8_fst_la_LDFLAGS = -avoid-version -module
const16_fst_la_SOURCES = const16-fst.cc
// See www.openfst.org for extensive documentation on this weighted
// finite-state transducer library.
-#include <fst/fst.h>
#include <fst/const-fst.h>
+#include <fst/fst.h>
namespace fst {
// See www.openfst.org for extensive documentation on this weighted
// finite-state transducer library.
-#include <fst/fst.h>
#include <fst/const-fst.h>
+#include <fst/fst.h>
namespace fst {
// See www.openfst.org for extensive documentation on this weighted
// finite-state transducer library.
-#include <fst/fst.h>
#include <fst/const-fst.h>
+#include <fst/fst.h>
namespace fst {
endif
libfstfar_la_SOURCES = sttable.cc stlist.cc
-libfstfar_la_LDFLAGS = -version-info 17:0:0
+libfstfar_la_LDFLAGS = -version-info 18:0:0
libfstfar_la_LIBADD = ../../lib/libfst.la -lm $(DL_LIBS)
if HAVE_SCRIPT
libfstfarscript_la_SOURCES = far-class.cc farscript.cc getters.cc script-impl.cc \
- strings.cc
-libfstfarscript_la_LDFLAGS = -version-info 17:0:0
+ strings.cc sttable.cc stlist.cc
+libfstfarscript_la_LDFLAGS = -version-info 18:0:0
libfstfarscript_la_LIBADD = \
libfstfar.la ../../script/libfstscript.la \
../../lib/libfst.la -lm $(DL_LIBS)
@HAVE_SCRIPT_TRUE@ ../../script/libfstscript.la \
@HAVE_SCRIPT_TRUE@ ../../lib/libfst.la $(am__DEPENDENCIES_1)
am__libfstfarscript_la_SOURCES_DIST = far-class.cc farscript.cc \
- getters.cc script-impl.cc strings.cc
+ getters.cc script-impl.cc strings.cc sttable.cc stlist.cc
@HAVE_SCRIPT_TRUE@am_libfstfarscript_la_OBJECTS = far-class.lo \
@HAVE_SCRIPT_TRUE@ farscript.lo getters.lo script-impl.lo \
-@HAVE_SCRIPT_TRUE@ strings.lo
+@HAVE_SCRIPT_TRUE@ strings.lo sttable.lo stlist.lo
libfstfarscript_la_OBJECTS = $(am_libfstfarscript_la_OBJECTS)
libfstfarscript_la_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \
$(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \
@HAVE_SCRIPT_FALSE@lib_LTLIBRARIES = libfstfar.la
@HAVE_SCRIPT_TRUE@lib_LTLIBRARIES = libfstfar.la libfstfarscript.la
libfstfar_la_SOURCES = sttable.cc stlist.cc
-libfstfar_la_LDFLAGS = -version-info 17:0:0
+libfstfar_la_LDFLAGS = -version-info 18:0:0
libfstfar_la_LIBADD = ../../lib/libfst.la -lm $(DL_LIBS)
@HAVE_SCRIPT_TRUE@libfstfarscript_la_SOURCES = far-class.cc farscript.cc getters.cc script-impl.cc \
-@HAVE_SCRIPT_TRUE@ strings.cc
+@HAVE_SCRIPT_TRUE@ strings.cc sttable.cc stlist.cc
-@HAVE_SCRIPT_TRUE@libfstfarscript_la_LDFLAGS = -version-info 17:0:0
+@HAVE_SCRIPT_TRUE@libfstfarscript_la_LDFLAGS = -version-info 18:0:0
@HAVE_SCRIPT_TRUE@libfstfarscript_la_LIBADD = \
@HAVE_SCRIPT_TRUE@ libfstfar.la ../../script/libfstscript.la \
@HAVE_SCRIPT_TRUE@ ../../lib/libfst.la -lm $(DL_LIBS)
lib_LTLIBRARIES = libfstlinearscript.la
libfstlinearscript_la_SOURCES = linearscript.cc
-libfstlinearscript_la_LDFLAGS = -version-info 17:0:0
+libfstlinearscript_la_LDFLAGS = -version-info 18:0:0
libfstlinearscript_la_LIBADD = ../../script/libfstscript.la
endif
@HAVE_BIN_TRUE@fstloglinearapply_LDADD = libfstlinearscript.la ../../script/libfstscript.la
@HAVE_SCRIPT_TRUE@lib_LTLIBRARIES = libfstlinearscript.la
@HAVE_SCRIPT_TRUE@libfstlinearscript_la_SOURCES = linearscript.cc
-@HAVE_SCRIPT_TRUE@libfstlinearscript_la_LDFLAGS = -version-info 17:0:0
+@HAVE_SCRIPT_TRUE@libfstlinearscript_la_LDFLAGS = -version-info 18:0:0
@HAVE_SCRIPT_TRUE@libfstlinearscript_la_LIBADD = ../../script/libfstscript.la
libfst_LTLIBRARIES = linear_tagger-fst.la linear_classifier-fst.la
linear_tagger_fst_la_SOURCES = linear-tagger-fst.cc
libfstlookahead_la_SOURCES = arc_lookahead-fst.cc ilabel_lookahead-fst.cc \
olabel_lookahead-fst.cc
-libfstlookahead_la_LDFLAGS = -version-info 17:0:0
+libfstlookahead_la_LDFLAGS = -version-info 18:0:0
arc_lookahead_fst_la_SOURCES = arc_lookahead-fst.cc
arc_lookahead_fst_la_LDFLAGS = -avoid-version -module
libfstlookahead_la_SOURCES = arc_lookahead-fst.cc ilabel_lookahead-fst.cc \
olabel_lookahead-fst.cc
-libfstlookahead_la_LDFLAGS = -version-info 17:0:0
+libfstlookahead_la_LDFLAGS = -version-info 18:0:0
arc_lookahead_fst_la_SOURCES = arc_lookahead-fst.cc
arc_lookahead_fst_la_LDFLAGS = -avoid-version -module
ilabel_lookahead_fst_la_SOURCES = ilabel_lookahead-fst.cc
if HAVE_SCRIPT
lib_LTLIBRARIES = libfstmpdtscript.la
libfstmpdtscript_la_SOURCES = mpdtscript.cc
-libfstmpdtscript_la_LDFLAGS = -version-info 17:0:0
+libfstmpdtscript_la_LDFLAGS = -version-info 18:0:0
libfstmpdtscript_la_LIBADD = ../../script/libfstscript.la \
../../lib/libfst.la -lm $(DL_LIBS)
endif
@HAVE_BIN_TRUE@mpdtreverse_SOURCES = mpdtreverse.cc mpdtreverse-main.cc
@HAVE_SCRIPT_TRUE@lib_LTLIBRARIES = libfstmpdtscript.la
@HAVE_SCRIPT_TRUE@libfstmpdtscript_la_SOURCES = mpdtscript.cc
-@HAVE_SCRIPT_TRUE@libfstmpdtscript_la_LDFLAGS = -version-info 17:0:0
+@HAVE_SCRIPT_TRUE@libfstmpdtscript_la_LDFLAGS = -version-info 18:0:0
@HAVE_SCRIPT_TRUE@libfstmpdtscript_la_LIBADD = ../../script/libfstscript.la \
@HAVE_SCRIPT_TRUE@ ../../lib/libfst.la -lm $(DL_LIBS)
#include <vector>
#include <fst/flags.h>
+#include <fst/types.h>
#include <fst/log.h>
#include <fst/extensions/mpdt/mpdtscript.h>
#include <fst/extensions/mpdt/read_write_utils.h>
#include <vector>
#include <fst/flags.h>
+#include <fst/types.h>
#include <fst/log.h>
#include <fst/extensions/mpdt/mpdtscript.h>
#include <fst/extensions/mpdt/read_write_utils.h>
#include <vector>
#include <fst/flags.h>
+#include <fst/types.h>
#include <fst/log.h>
#include <fst/extensions/mpdt/mpdtscript.h>
#include <fst/extensions/mpdt/read_write_utils.h>
#include <vector>
#include <fst/flags.h>
+#include <fst/types.h>
#include <fst/log.h>
#include <fst/extensions/mpdt/mpdtscript.h>
#include <fst/extensions/mpdt/read_write_utils.h>
#include <utility>
#include <vector>
+#include <fst/types.h>
#include <fst/extensions/mpdt/compose.h>
#include <fst/extensions/mpdt/expand.h>
#include <fst/extensions/mpdt/reverse.h>
ngram_fst_la_LDFLAGS = -avoid-version -module
libfstngram_la_SOURCES = bitmap-index.cc ngram-fst.cc nthbit.cc
-libfstngram_la_LDFLAGS = -version-info 17:0:0
+libfstngram_la_LDFLAGS = -version-info 18:0:0
ngram_fst_la_SOURCES = bitmap-index.cc ngram-fst.cc nthbit.cc
ngram_fst_la_LDFLAGS = -avoid-version -module
libfstngram_la_SOURCES = bitmap-index.cc ngram-fst.cc nthbit.cc
-libfstngram_la_LDFLAGS = -version-info 17:0:0
+libfstngram_la_LDFLAGS = -version-info 18:0:0
all: all-am
.SUFFIXES:
}
void BitmapIndex::BuildIndex(const uint64 *bits, size_t num_bits) {
+ // Primary counts are uint32s, so this is the most *set* bits we support
+ // for now. Just check the number of *input* bits is less than this
+ // to keep things simple.
+ DCHECK_LT(num_bits, uint64{1} << 32);
bits_ = bits;
num_bits_ = num_bits;
primary_index_.resize(primary_index_size());
#include <fst/extensions/ngram/nthbit.h>
-#if defined(__arm__) // 32-bit ARM
+#include <fst/types.h>
-// ARM64 is detected with __aarch64__.
+#if !defined(__BMI2__) // BMI2 has everything in the header
+#if SIZE_MAX == UINT32_MAX
-// ARM32 has a poor implementation of __builtin_popcount, so any nth_bit
-// based on that is slow. Use this table-based version instead.
+// 32-bit platforms will be slow when using 64-bit operations; use this
+// table-based version instead. This only contains constant shifts, which
+// have been benchmarked to be fast.
// These tables were generated using:
//
255, 255, 255, 7,
}};
-uint32 nth_bit(uint64 v, uint32 r) {
+uint32 nth_bit(const uint64 v, uint32 r) {
// nth_bit uses 1-origin for r, but code below is more natural with 0-origin.
r--;
- uint32 next_v = v & 255;
- uint32 v_in_byte = nth_bit_bit_count[next_v];
- if (r < v_in_byte) return nth_bit_bit_pos[r][next_v];
- r -= v_in_byte;
- next_v = (v >> 8) & 255;
- v_in_byte = nth_bit_bit_count[next_v];
- if (r < v_in_byte) return 8 + nth_bit_bit_pos[r][next_v];
- r -= v_in_byte;
- next_v = (v >> 16) & 255;
- v_in_byte = nth_bit_bit_count[next_v];
- if (r < v_in_byte) return 16 + nth_bit_bit_pos[r][next_v];
- r -= v_in_byte;
- next_v = (v >> 24) & 255;
- v_in_byte = nth_bit_bit_count[next_v];
- if (r < v_in_byte) return 24 + nth_bit_bit_pos[r][next_v];
- r -= v_in_byte;
- next_v = (v >> 32) & 255;
- v_in_byte = nth_bit_bit_count[next_v];
- if (r < v_in_byte) return 32 + nth_bit_bit_pos[r][next_v];
- r -= v_in_byte;
- next_v = (v >> 40) & 255;
- v_in_byte = nth_bit_bit_count[next_v];
- if (r < v_in_byte) return 40 + nth_bit_bit_pos[r][next_v];
- r -= v_in_byte;
- next_v = (v >> 48) & 255;
- v_in_byte = nth_bit_bit_count[next_v];
- if (r < v_in_byte) return 48 + nth_bit_bit_pos[r][next_v];
- r -= v_in_byte;
- next_v = (v >> 56) & 255;
- v_in_byte = nth_bit_bit_count[next_v];
- if (r < v_in_byte) return 56 + nth_bit_bit_pos[r][next_v];
+ uint32 next_byte = v & 255;
+ uint32 byte_popcount = nth_bit_bit_count[next_byte];
+ if (r < byte_popcount) return nth_bit_bit_pos[r][next_byte];
+ r -= byte_popcount;
+ next_byte = (v >> 8) & 255;
+ byte_popcount = nth_bit_bit_count[next_byte];
+ if (r < byte_popcount) return 8 + nth_bit_bit_pos[r][next_byte];
+ r -= byte_popcount;
+ next_byte = (v >> 16) & 255;
+ byte_popcount = nth_bit_bit_count[next_byte];
+ if (r < byte_popcount) return 16 + nth_bit_bit_pos[r][next_byte];
+ r -= byte_popcount;
+ next_byte = (v >> 24) & 255;
+ byte_popcount = nth_bit_bit_count[next_byte];
+ if (r < byte_popcount) return 24 + nth_bit_bit_pos[r][next_byte];
+ r -= byte_popcount;
+ next_byte = (v >> 32) & 255;
+ byte_popcount = nth_bit_bit_count[next_byte];
+ if (r < byte_popcount) return 32 + nth_bit_bit_pos[r][next_byte];
+ r -= byte_popcount;
+ next_byte = (v >> 40) & 255;
+ byte_popcount = nth_bit_bit_count[next_byte];
+ if (r < byte_popcount) return 40 + nth_bit_bit_pos[r][next_byte];
+ r -= byte_popcount;
+ next_byte = (v >> 48) & 255;
+ byte_popcount = nth_bit_bit_count[next_byte];
+ if (r < byte_popcount) return 48 + nth_bit_bit_pos[r][next_byte];
+ r -= byte_popcount;
+ next_byte = (v >> 56) & 255;
+ byte_popcount = nth_bit_bit_count[next_byte];
+ if (r < byte_popcount) return 56 + nth_bit_bit_pos[r][next_byte];
return -1;
}
-#elif !defined(__BMI2__)
-// This table is generated using:
+#elif SIZE_MAX == UINT64_MAX // 64-bit, non-BMI2
+// These tables are generated using:
//
-// for (size_t i = 0; i < 256; ++i) {
-// uint32 offsets = 0;
-// for (size_t b = 1; b <= 8; ++b) {
-// uint32 offset = min<uint32>(nth_bit_scan(i, b), 8);
-// offsets |= (offset << ((b - 1) << 2));
+// constexpr uint64 kOnesStep8 = 0x0101010101010101;
+// printf("const uint64 kPrefixSumOverflow[65] = {\n");
+// for (int k = 0; k <= 64; ++k) {
+// printf(" 0x%" FST_LL_FORMAT "x,\n", (0x80 - k) * kOnesStep8);
+// }
+// printf("};\n");
+//
+// printf("const uint8 kSelectInByte[8 * 256] = {\n");
+// for (int j = 0; j < 8; ++j) {
+// for (int i = 0; i < 256; ++i) {
+// if (i > 0) printf(" ");
+// if (i % 16 == 0) printf("\n ");
+// const int k = findbitn(i, j);
+// printf("%d,", k == -1 ? 0 : k);
// }
-// bit_offset = offsets;
-// printf("0x%x, ", bit_offset);
-// if (i % 4 == 3) printf("\n");
+// printf("\n");
// }
+// printf("};\n");
//
-const uint32 nth_bit_bit_offset[] = {
- 0x88888888, 0x88888880, 0x88888881, 0x88888810, 0x88888882, 0x88888820,
- 0x88888821, 0x88888210, 0x88888883, 0x88888830, 0x88888831, 0x88888310,
- 0x88888832, 0x88888320, 0x88888321, 0x88883210, 0x88888884, 0x88888840,
- 0x88888841, 0x88888410, 0x88888842, 0x88888420, 0x88888421, 0x88884210,
- 0x88888843, 0x88888430, 0x88888431, 0x88884310, 0x88888432, 0x88884320,
- 0x88884321, 0x88843210, 0x88888885, 0x88888850, 0x88888851, 0x88888510,
- 0x88888852, 0x88888520, 0x88888521, 0x88885210, 0x88888853, 0x88888530,
- 0x88888531, 0x88885310, 0x88888532, 0x88885320, 0x88885321, 0x88853210,
- 0x88888854, 0x88888540, 0x88888541, 0x88885410, 0x88888542, 0x88885420,
- 0x88885421, 0x88854210, 0x88888543, 0x88885430, 0x88885431, 0x88854310,
- 0x88885432, 0x88854320, 0x88854321, 0x88543210, 0x88888886, 0x88888860,
- 0x88888861, 0x88888610, 0x88888862, 0x88888620, 0x88888621, 0x88886210,
- 0x88888863, 0x88888630, 0x88888631, 0x88886310, 0x88888632, 0x88886320,
- 0x88886321, 0x88863210, 0x88888864, 0x88888640, 0x88888641, 0x88886410,
- 0x88888642, 0x88886420, 0x88886421, 0x88864210, 0x88888643, 0x88886430,
- 0x88886431, 0x88864310, 0x88886432, 0x88864320, 0x88864321, 0x88643210,
- 0x88888865, 0x88888650, 0x88888651, 0x88886510, 0x88888652, 0x88886520,
- 0x88886521, 0x88865210, 0x88888653, 0x88886530, 0x88886531, 0x88865310,
- 0x88886532, 0x88865320, 0x88865321, 0x88653210, 0x88888654, 0x88886540,
- 0x88886541, 0x88865410, 0x88886542, 0x88865420, 0x88865421, 0x88654210,
- 0x88886543, 0x88865430, 0x88865431, 0x88654310, 0x88865432, 0x88654320,
- 0x88654321, 0x86543210, 0x88888887, 0x88888870, 0x88888871, 0x88888710,
- 0x88888872, 0x88888720, 0x88888721, 0x88887210, 0x88888873, 0x88888730,
- 0x88888731, 0x88887310, 0x88888732, 0x88887320, 0x88887321, 0x88873210,
- 0x88888874, 0x88888740, 0x88888741, 0x88887410, 0x88888742, 0x88887420,
- 0x88887421, 0x88874210, 0x88888743, 0x88887430, 0x88887431, 0x88874310,
- 0x88887432, 0x88874320, 0x88874321, 0x88743210, 0x88888875, 0x88888750,
- 0x88888751, 0x88887510, 0x88888752, 0x88887520, 0x88887521, 0x88875210,
- 0x88888753, 0x88887530, 0x88887531, 0x88875310, 0x88887532, 0x88875320,
- 0x88875321, 0x88753210, 0x88888754, 0x88887540, 0x88887541, 0x88875410,
- 0x88887542, 0x88875420, 0x88875421, 0x88754210, 0x88887543, 0x88875430,
- 0x88875431, 0x88754310, 0x88875432, 0x88754320, 0x88754321, 0x87543210,
- 0x88888876, 0x88888760, 0x88888761, 0x88887610, 0x88888762, 0x88887620,
- 0x88887621, 0x88876210, 0x88888763, 0x88887630, 0x88887631, 0x88876310,
- 0x88887632, 0x88876320, 0x88876321, 0x88763210, 0x88888764, 0x88887640,
- 0x88887641, 0x88876410, 0x88887642, 0x88876420, 0x88876421, 0x88764210,
- 0x88887643, 0x88876430, 0x88876431, 0x88764310, 0x88876432, 0x88764320,
- 0x88764321, 0x87643210, 0x88888765, 0x88887650, 0x88887651, 0x88876510,
- 0x88887652, 0x88876520, 0x88876521, 0x88765210, 0x88887653, 0x88876530,
- 0x88876531, 0x88765310, 0x88876532, 0x88765320, 0x88765321, 0x87653210,
- 0x88887654, 0x88876540, 0x88876541, 0x88765410, 0x88876542, 0x88765420,
- 0x88765421, 0x87654210, 0x88876543, 0x88765430, 0x88765431, 0x87654310,
- 0x88765432, 0x87654320, 0x87654321, 0x76543210,
+namespace internal {
+
+// clang-format off
+const uint64 kPrefixSumOverflow[65] = {
+ 0x8080808080808080,
+ 0x7f7f7f7f7f7f7f7f,
+ 0x7e7e7e7e7e7e7e7e,
+ 0x7d7d7d7d7d7d7d7d,
+ 0x7c7c7c7c7c7c7c7c,
+ 0x7b7b7b7b7b7b7b7b,
+ 0x7a7a7a7a7a7a7a7a,
+ 0x7979797979797979,
+ 0x7878787878787878,
+ 0x7777777777777777,
+ 0x7676767676767676,
+ 0x7575757575757575,
+ 0x7474747474747474,
+ 0x7373737373737373,
+ 0x7272727272727272,
+ 0x7171717171717171,
+ 0x7070707070707070,
+ 0x6f6f6f6f6f6f6f6f,
+ 0x6e6e6e6e6e6e6e6e,
+ 0x6d6d6d6d6d6d6d6d,
+ 0x6c6c6c6c6c6c6c6c,
+ 0x6b6b6b6b6b6b6b6b,
+ 0x6a6a6a6a6a6a6a6a,
+ 0x6969696969696969,
+ 0x6868686868686868,
+ 0x6767676767676767,
+ 0x6666666666666666,
+ 0x6565656565656565,
+ 0x6464646464646464,
+ 0x6363636363636363,
+ 0x6262626262626262,
+ 0x6161616161616161,
+ 0x6060606060606060,
+ 0x5f5f5f5f5f5f5f5f,
+ 0x5e5e5e5e5e5e5e5e,
+ 0x5d5d5d5d5d5d5d5d,
+ 0x5c5c5c5c5c5c5c5c,
+ 0x5b5b5b5b5b5b5b5b,
+ 0x5a5a5a5a5a5a5a5a,
+ 0x5959595959595959,
+ 0x5858585858585858,
+ 0x5757575757575757,
+ 0x5656565656565656,
+ 0x5555555555555555,
+ 0x5454545454545454,
+ 0x5353535353535353,
+ 0x5252525252525252,
+ 0x5151515151515151,
+ 0x5050505050505050,
+ 0x4f4f4f4f4f4f4f4f,
+ 0x4e4e4e4e4e4e4e4e,
+ 0x4d4d4d4d4d4d4d4d,
+ 0x4c4c4c4c4c4c4c4c,
+ 0x4b4b4b4b4b4b4b4b,
+ 0x4a4a4a4a4a4a4a4a,
+ 0x4949494949494949,
+ 0x4848484848484848,
+ 0x4747474747474747,
+ 0x4646464646464646,
+ 0x4545454545454545,
+ 0x4444444444444444,
+ 0x4343434343434343,
+ 0x4242424242424242,
+ 0x4141414141414141,
+ 0x4040404040404040
+};
+
+const uint8 kSelectInByte[8 * 256] = {
+ 0, 0, 1, 0, 2, 0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0,
+ 4, 0, 1, 0, 2, 0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0,
+ 5, 0, 1, 0, 2, 0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0,
+ 4, 0, 1, 0, 2, 0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0,
+ 6, 0, 1, 0, 2, 0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0,
+ 4, 0, 1, 0, 2, 0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0,
+ 5, 0, 1, 0, 2, 0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0,
+ 4, 0, 1, 0, 2, 0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0,
+ 7, 0, 1, 0, 2, 0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0,
+ 4, 0, 1, 0, 2, 0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0,
+ 5, 0, 1, 0, 2, 0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0,
+ 4, 0, 1, 0, 2, 0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0,
+ 6, 0, 1, 0, 2, 0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0,
+ 4, 0, 1, 0, 2, 0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0,
+ 5, 0, 1, 0, 2, 0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0,
+ 4, 0, 1, 0, 2, 0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0,
+
+ 0, 0, 0, 1, 0, 2, 2, 1, 0, 3, 3, 1, 3, 2, 2, 1,
+ 0, 4, 4, 1, 4, 2, 2, 1, 4, 3, 3, 1, 3, 2, 2, 1,
+ 0, 5, 5, 1, 5, 2, 2, 1, 5, 3, 3, 1, 3, 2, 2, 1,
+ 5, 4, 4, 1, 4, 2, 2, 1, 4, 3, 3, 1, 3, 2, 2, 1,
+ 0, 6, 6, 1, 6, 2, 2, 1, 6, 3, 3, 1, 3, 2, 2, 1,
+ 6, 4, 4, 1, 4, 2, 2, 1, 4, 3, 3, 1, 3, 2, 2, 1,
+ 6, 5, 5, 1, 5, 2, 2, 1, 5, 3, 3, 1, 3, 2, 2, 1,
+ 5, 4, 4, 1, 4, 2, 2, 1, 4, 3, 3, 1, 3, 2, 2, 1,
+ 0, 7, 7, 1, 7, 2, 2, 1, 7, 3, 3, 1, 3, 2, 2, 1,
+ 7, 4, 4, 1, 4, 2, 2, 1, 4, 3, 3, 1, 3, 2, 2, 1,
+ 7, 5, 5, 1, 5, 2, 2, 1, 5, 3, 3, 1, 3, 2, 2, 1,
+ 5, 4, 4, 1, 4, 2, 2, 1, 4, 3, 3, 1, 3, 2, 2, 1,
+ 7, 6, 6, 1, 6, 2, 2, 1, 6, 3, 3, 1, 3, 2, 2, 1,
+ 6, 4, 4, 1, 4, 2, 2, 1, 4, 3, 3, 1, 3, 2, 2, 1,
+ 6, 5, 5, 1, 5, 2, 2, 1, 5, 3, 3, 1, 3, 2, 2, 1,
+ 5, 4, 4, 1, 4, 2, 2, 1, 4, 3, 3, 1, 3, 2, 2, 1,
+
+ 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 3, 0, 3, 3, 2,
+ 0, 0, 0, 4, 0, 4, 4, 2, 0, 4, 4, 3, 4, 3, 3, 2,
+ 0, 0, 0, 5, 0, 5, 5, 2, 0, 5, 5, 3, 5, 3, 3, 2,
+ 0, 5, 5, 4, 5, 4, 4, 2, 5, 4, 4, 3, 4, 3, 3, 2,
+ 0, 0, 0, 6, 0, 6, 6, 2, 0, 6, 6, 3, 6, 3, 3, 2,
+ 0, 6, 6, 4, 6, 4, 4, 2, 6, 4, 4, 3, 4, 3, 3, 2,
+ 0, 6, 6, 5, 6, 5, 5, 2, 6, 5, 5, 3, 5, 3, 3, 2,
+ 6, 5, 5, 4, 5, 4, 4, 2, 5, 4, 4, 3, 4, 3, 3, 2,
+ 0, 0, 0, 7, 0, 7, 7, 2, 0, 7, 7, 3, 7, 3, 3, 2,
+ 0, 7, 7, 4, 7, 4, 4, 2, 7, 4, 4, 3, 4, 3, 3, 2,
+ 0, 7, 7, 5, 7, 5, 5, 2, 7, 5, 5, 3, 5, 3, 3, 2,
+ 7, 5, 5, 4, 5, 4, 4, 2, 5, 4, 4, 3, 4, 3, 3, 2,
+ 0, 7, 7, 6, 7, 6, 6, 2, 7, 6, 6, 3, 6, 3, 3, 2,
+ 7, 6, 6, 4, 6, 4, 4, 2, 6, 4, 4, 3, 4, 3, 3, 2,
+ 7, 6, 6, 5, 6, 5, 5, 2, 6, 5, 5, 3, 5, 3, 3, 2,
+ 6, 5, 5, 4, 5, 4, 4, 2, 5, 4, 4, 3, 4, 3, 3, 2,
+
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3,
+ 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 4, 0, 4, 4, 3,
+ 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 5, 0, 5, 5, 3,
+ 0, 0, 0, 5, 0, 5, 5, 4, 0, 5, 5, 4, 5, 4, 4, 3,
+ 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 6, 0, 6, 6, 3,
+ 0, 0, 0, 6, 0, 6, 6, 4, 0, 6, 6, 4, 6, 4, 4, 3,
+ 0, 0, 0, 6, 0, 6, 6, 5, 0, 6, 6, 5, 6, 5, 5, 3,
+ 0, 6, 6, 5, 6, 5, 5, 4, 6, 5, 5, 4, 5, 4, 4, 3,
+ 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 7, 0, 7, 7, 3,
+ 0, 0, 0, 7, 0, 7, 7, 4, 0, 7, 7, 4, 7, 4, 4, 3,
+ 0, 0, 0, 7, 0, 7, 7, 5, 0, 7, 7, 5, 7, 5, 5, 3,
+ 0, 7, 7, 5, 7, 5, 5, 4, 7, 5, 5, 4, 5, 4, 4, 3,
+ 0, 0, 0, 7, 0, 7, 7, 6, 0, 7, 7, 6, 7, 6, 6, 3,
+ 0, 7, 7, 6, 7, 6, 6, 4, 7, 6, 6, 4, 6, 4, 4, 3,
+ 0, 7, 7, 6, 7, 6, 6, 5, 7, 6, 6, 5, 6, 5, 5, 3,
+ 7, 6, 6, 5, 6, 5, 5, 4, 6, 5, 5, 4, 5, 4, 4, 3,
+
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5,
+ 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 5, 0, 5, 5, 4,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6,
+ 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 6, 0, 6, 6, 4,
+ 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 6, 0, 6, 6, 5,
+ 0, 0, 0, 6, 0, 6, 6, 5, 0, 6, 6, 5, 6, 5, 5, 4,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7,
+ 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 7, 0, 7, 7, 4,
+ 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 7, 0, 7, 7, 5,
+ 0, 0, 0, 7, 0, 7, 7, 5, 0, 7, 7, 5, 7, 5, 5, 4,
+ 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 7, 0, 7, 7, 6,
+ 0, 0, 0, 7, 0, 7, 7, 6, 0, 7, 7, 6, 7, 6, 6, 4,
+ 0, 0, 0, 7, 0, 7, 7, 6, 0, 7, 7, 6, 7, 6, 6, 5,
+ 0, 7, 7, 6, 7, 6, 6, 5, 7, 6, 6, 5, 6, 5, 5, 4,
+
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6,
+ 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 6, 0, 6, 6, 5,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7,
+ 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 7, 0, 7, 7, 5,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7,
+ 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 7, 0, 7, 7, 6,
+ 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 7, 0, 7, 7, 6,
+ 0, 0, 0, 7, 0, 7, 7, 6, 0, 7, 7, 6, 7, 6, 6, 5,
+
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7,
+ 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 7, 0, 7, 7, 6,
+
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7
};
-#endif // !defined(__BMI2__) && !defined(__arm__)
+// clang-format on
+
+} // namespace internal
+#endif // 64-bit, non-BMI2
+#endif // !defined(__BMI2__)
if HAVE_SCRIPT
lib_LTLIBRARIES = libfstpdtscript.la
libfstpdtscript_la_SOURCES = getters.cc pdtscript.cc
-libfstpdtscript_la_LDFLAGS = -version-info 17:0:0
+libfstpdtscript_la_LDFLAGS = -version-info 18:0:0
libfstpdtscript_la_LIBADD = ../../script/libfstscript.la \
../../lib/libfst.la -lm $(DL_LIBS)
endif
@HAVE_BIN_TRUE@pdtshortestpath_SOURCES = pdtshortestpath.cc pdtshortestpath-main.cc
@HAVE_SCRIPT_TRUE@lib_LTLIBRARIES = libfstpdtscript.la
@HAVE_SCRIPT_TRUE@libfstpdtscript_la_SOURCES = getters.cc pdtscript.cc
-@HAVE_SCRIPT_TRUE@libfstpdtscript_la_LDFLAGS = -version-info 17:0:0
+@HAVE_SCRIPT_TRUE@libfstpdtscript_la_LDFLAGS = -version-info 18:0:0
@HAVE_SCRIPT_TRUE@libfstpdtscript_la_LIBADD = ../../script/libfstscript.la \
@HAVE_SCRIPT_TRUE@ ../../lib/libfst.la -lm $(DL_LIBS)
#include <vector>
#include <fst/flags.h>
+#include <fst/types.h>
#include <fst/log.h>
#include <fst/extensions/pdt/getters.h>
#include <fst/extensions/pdt/pdtscript.h>
#include <vector>
#include <fst/flags.h>
+#include <fst/types.h>
#include <fst/log.h>
#include <fst/extensions/pdt/pdtscript.h>
#include <fst/util.h>
#include <vector>
#include <fst/flags.h>
+#include <fst/types.h>
#include <fst/log.h>
#include <fst/extensions/pdt/pdtscript.h>
#include <fst/util.h>
#include <vector>
#include <fst/flags.h>
+#include <fst/types.h>
#include <fst/extensions/pdt/getters.h>
#include <fst/extensions/pdt/pdtscript.h>
#include <fst/util.h>
#include <vector>
#include <fst/flags.h>
+#include <fst/types.h>
#include <fst/log.h>
#include <fst/extensions/pdt/pdtscript.h>
#include <fst/util.h>
#include <utility>
#include <vector>
+#include <fst/types.h>
#include <fst/extensions/pdt/compose.h>
#include <fst/extensions/pdt/expand.h>
#include <fst/extensions/pdt/replace.h>
#include <vector>
#include <fst/flags.h>
+#include <fst/types.h>
#include <fst/log.h>
#include <fst/extensions/pdt/pdtscript.h>
#include <fst/util.h>
-lm $(DL_LIBS) $(PYTHON_LIBS)
# Exports the *.pxd/*.pxd source files.
-EXTRA_DIST = basictypes.pxd fst.pxd ios.pxd pywrapfst.pxd pywrapfst.pyx
+EXTRA_DIST = basictypes.pxd cpywrapfst.pxd ios.pxd pywrapfst.pxd pywrapfst.pyx
# Exports the *.pxd/*.pxd source files.
-EXTRA_DIST = basictypes.pxd fst.pxd ios.pxd pywrapfst.pxd pywrapfst.pyx
+EXTRA_DIST = basictypes.pxd cpywrapfst.pxd ios.pxd pywrapfst.pxd pywrapfst.pyx
all: all-am
.SUFFIXES:
from basictypes cimport *
-cimport fst as fst
+cimport cpywrapfst as fst
from ios cimport ostream
from ios cimport ofstream
lib_LTLIBRARIES = libfstspecial.la
libfstspecial_la_SOURCES = phi-fst.cc rho-fst.cc sigma-fst.cc
-libfstspecial_la_LDFLAGS = -version-info 17:0:0
+libfstspecial_la_LDFLAGS = -version-info 18:0:0
phi_fst_la_SOURCES = phi-fst.cc
phi_fst_la_LDFLAGS = -avoid-version -module
libfst_LTLIBRARIES = phi-fst.la rho-fst.la sigma-fst.la
lib_LTLIBRARIES = libfstspecial.la
libfstspecial_la_SOURCES = phi-fst.cc rho-fst.cc sigma-fst.cc
-libfstspecial_la_LDFLAGS = -version-info 17:0:0
+libfstspecial_la_LDFLAGS = -version-info 18:0:0
phi_fst_la_SOURCES = phi-fst.cc
phi_fst_la_LDFLAGS = -avoid-version -module
rho_fst_la_SOURCES = rho-fst.cc
state_weights_(nullptr),
error_(false) {}
- FastLogAccumulator(const FastLogAccumulator<Arc> &acc, bool safe = false)
+ FastLogAccumulator(const FastLogAccumulator &acc, bool safe = false)
: to_log_weight_(),
to_weight_(),
arc_limit_(acc.arc_limit_),
s_(kNoStateId),
error_(false) {}
- CacheLogAccumulator(const CacheLogAccumulator<Arc> &acc, bool safe = false)
+ CacheLogAccumulator(const CacheLogAccumulator &acc, bool safe = false)
: arc_limit_(acc.arc_limit_),
fst_(acc.fst_ ? acc.fst_->Copy() : nullptr),
data_(safe ? std::make_shared<CacheLogAccumulatorData<Arc>>(*acc.data_)
#include <string>
#include <utility>
+#include <fst/types.h>
#include <fst/log.h>
#include <fst/fst.h>
-
namespace fst {
// Identifies stream data as an add-on FST.
std::shared_ptr<A2> SharedSecond() const { return a2_; }
- static AddOnPair<A1, A2> *Read(std::istream &istrm,
- const FstReadOptions &opts) {
- A1 *a1 = nullptr;
+ static AddOnPair *Read(std::istream &istrm, const FstReadOptions &opts) {
bool have_addon1 = false;
ReadType(istrm, &have_addon1);
- if (have_addon1) a1 = A1::Read(istrm, opts);
+ std::unique_ptr<A1> a1;
+ if (have_addon1) a1 = fst::WrapUnique(A1::Read(istrm, opts));
- A2 *a2 = nullptr;
bool have_addon2 = false;
ReadType(istrm, &have_addon2);
- if (have_addon2) a2 = A2::Read(istrm, opts);
+ std::unique_ptr<A1> a2;
+ if (have_addon2) a2 = fst::WrapUnique(A2::Read(istrm, opts));
- return new AddOnPair<A1, A2>(std::shared_ptr<A1>(a1),
- std::shared_ptr<A2>(a2));
+ return new AddOnPair(std::move(a1), std::move(a2));
}
bool Write(std::ostream &ostrm, const FstWriteOptions &opts) const {
// We make a thread-safe copy of the FST by default since an FST
// implementation is expected to not share mutable data between objects.
AddOnImpl(const FST &fst, const std::string &type,
- std::shared_ptr<T> t = std::shared_ptr<T>())
+ std::shared_ptr<T> t = nullptr)
: fst_(fst, true), t_(std::move(t)) {
SetType(type);
SetProperties(fst_.Properties(kFstProperties, false));
// Conversion from const Fst<Arc> & to F always copies the underlying
// implementation.
AddOnImpl(const Fst<Arc> &fst, const std::string &type,
- std::shared_ptr<T> t = std::shared_ptr<T>())
+ std::shared_ptr<T> t = nullptr)
: fst_(fst), t_(std::move(t)) {
SetType(type);
SetProperties(fst_.Properties(kFstProperties, false));
// We make a thread-safe copy of the FST by default since an FST
// implementation is expected to not share mutable data between objects.
- AddOnImpl(const AddOnImpl<FST, T> &impl)
- : fst_(impl.fst_, true), t_(impl.t_) {
+ AddOnImpl(const AddOnImpl &impl) : fst_(impl.fst_, true), t_(impl.t_) {
SetType(impl.Type());
SetProperties(fst_.Properties(kCopyProperties, false));
SetInputSymbols(fst_.InputSymbols());
size_t NumStates() const { return fst_.NumStates(); }
- static AddOnImpl<FST, T> *Read(std::istream &strm,
- const FstReadOptions &opts) {
+ static AddOnImpl *Read(std::istream &strm, const FstReadOptions &opts) {
FstReadOptions nopts(opts);
FstHeader hdr;
if (!nopts.header) {
hdr.Read(strm, nopts.source);
nopts.header = &hdr;
}
- std::unique_ptr<AddOnImpl<FST, T>> impl(
- new AddOnImpl<FST, T>(nopts.header->FstType()));
+ std::unique_ptr<AddOnImpl> impl(new AddOnImpl(nopts.header->FstType()));
if (!impl->ReadHeader(strm, nopts, kMinFileVersion, &hdr)) return nullptr;
impl.reset();
int32 magic_number = 0;
t = std::shared_ptr<T>(T::Read(strm, fopts));
if (!t) return nullptr;
}
- return new AddOnImpl<FST, T>(*fst, nopts.header->FstType(), t);
+ return new AddOnImpl(*fst, nopts.header->FstType(), t);
}
bool Write(std::ostream &strm, const FstWriteOptions &opts) const {
#include <memory>
#include <utility>
+#include <fst/types.h>
#include <fst/fst.h>
#include <fst/memory.h>
#include <unordered_map>
class State {
public:
- Weight Final() const { return final_; }
+ Weight Final() const { return final_weight_; }
size_t NumInputEpsilons() const { return niepsilons_; }
int* MutableRefCount() const { return nullptr; }
private:
- State(Weight weight, int32 niepsilons, int32 noepsilons, int32 narcs,
+ State(Weight final_weight, int32 niepsilons, int32 noepsilons, int32 narcs,
const Arc *arcs)
- : final_(std::move(weight)),
+ : final_weight_(std::move(final_weight)),
niepsilons_(niepsilons),
noepsilons_(noepsilons),
narcs_(narcs),
arcs_(arcs) {}
- Weight final_;
+ Weight final_weight_;
size_t niepsilons_;
size_t noepsilons_;
size_t narcs_;
if (arcs[i].olabel == 0) ++noepsilons;
}
states_.emplace_back(
- State(builder.final_, niepsilons, noepsilons, narcs, arcs));
+ State(builder.final_weight_, niepsilons, noepsilons, narcs, arcs));
// Places it in the cache.
auto state = &states_.back();
it.first->second = state;
private:
class StateBuilder {
public:
- explicit StateBuilder(ArcArena<Arc>* arena)
- : arena_(arena), final_(Weight::Zero()), narcs_(0) {}
+ explicit StateBuilder(ArcArena<Arc> *arena)
+ : arena_(arena), final_weight_(Weight::Zero()), narcs_(0) {}
- void SetFinal(Weight weight) { final_ = std::move(weight); }
+ void SetFinal(Weight weight) { final_weight_ = std::move(weight); }
void ReserveArcs(size_t n) { arena_->ReserveArcs(n); }
friend class ArcArenaStateStore<Arc>;
ArcArena<Arc> *arena_;
- Weight final_;
+ Weight final_weight_;
size_t narcs_;
};
#include <unordered_map>
#include <utility>
+#include <fst/types.h>
#include <fst/log.h>
#include <fst/cache.h>
std::make_shared<Impl>(fst, mapper, ArcMapFstOptions())) {}
// See Fst<>::Copy() for doc.
- ArcMapFst(const ArcMapFst<A, B, C> &fst, bool safe = false)
+ ArcMapFst(const ArcMapFst &fst, bool safe = false)
: ImplToFst<Impl>(fst, safe) {}
// Get a copy of this ArcMapFst. See Fst<>::Copy() for further doc.
- ArcMapFst<A, B, C> *Copy(bool safe = false) const override {
- return new ArcMapFst<A, B, C>(*this, safe);
+ ArcMapFst *Copy(bool safe = false) const override {
+ return new ArcMapFst(*this, safe);
}
inline void InitStateIterator(StateIteratorData<B> *data) const override;
#include <string>
#include <vector>
+#include <fst/types.h>
+
#include <fst/cache.h>
#include <fst/state-map.h>
#include <fst/test-properties.h>
: StateMapFst<Arc, Arc, Mapper>(fst, Mapper(fst, comp), opts) {}
// See Fst<>::Copy() for doc.
- ArcSortFst(const ArcSortFst<Arc, Compare> &fst, bool safe = false)
+ ArcSortFst(const ArcSortFst &fst, bool safe = false)
: StateMapFst<Arc, Arc, Mapper>(fst, safe) {}
// Gets a copy of this ArcSortFst. See Fst<>::Copy() for further doc.
- ArcSortFst<Arc, Compare> *Copy(bool safe = false) const override {
+ ArcSortFst *Copy(bool safe = false) const override {
return new ArcSortFst(*this, safe);
}
#include <unordered_set>
#include <vector>
+#include <fst/types.h>
#include <fst/log.h>
#include <fst/memory.h>
#include <unordered_set>
#include <vector>
#include <fst/flags.h>
+#include <fst/types.h>
#include <fst/log.h>
#include <fst/vector-fst.h>
// Provides STL allocator for arcs.
explicit CacheState(const ArcAllocator &alloc)
- : final_(Weight::Zero()),
+ : final_weight_(Weight::Zero()),
niepsilons_(0),
noepsilons_(0),
arcs_(alloc),
ref_count_(0) {}
CacheState(const CacheState<A> &state, const ArcAllocator &alloc)
- : final_(state.Final()),
+ : final_weight_(state.Final()),
niepsilons_(state.NumInputEpsilons()),
noepsilons_(state.NumOutputEpsilons()),
arcs_(state.arcs_.begin(), state.arcs_.end(), alloc),
ref_count_(0) {}
void Reset() {
- final_ = Weight::Zero();
+ final_weight_ = Weight::Zero();
niepsilons_ = 0;
noepsilons_ = 0;
ref_count_ = 0;
arcs_.clear();
}
- Weight Final() const { return final_; }
+ Weight Final() const { return final_weight_; }
size_t NumInputEpsilons() const { return niepsilons_; }
// Accesses ref count; used by the caller.
int RefCount() const { return ref_count_; }
- void SetFinal(Weight weight = Weight::One()) { final_ = std::move(weight); }
+ void SetFinal(Weight weight = Weight::One()) {
+ final_weight_ = std::move(weight);
+ }
void ReserveArcs(size_t n) { arcs_.reserve(n); }
if (arc.olabel == 0) ++noepsilons_;
}
- Weight final_; // Final weight.
+ Weight final_weight_; // Final weight.
size_t niepsilons_; // # of input epsilons.
size_t noepsilons_; // # of output epsilons.
std::vector<Arc, ArcAllocator> arcs_; // Arcs representation.
~VectorCacheStore() { Clear(); }
- VectorCacheStore<State> &operator=(const VectorCacheStore<State> &store) {
+ VectorCacheStore &operator=(const VectorCacheStore &store) {
if (this != &store) {
CopyStates(store);
Reset();
~HashCacheStore() { Clear(); }
- HashCacheStore<State> &operator=(const HashCacheStore<State> &store) {
+ HashCacheStore &operator=(const HashCacheStore &store) {
if (this != &store) {
CopyStates(store);
Reset();
#include <algorithm>
#include <vector>
+#include <fst/types.h>
+
#include <fst/mutable-fst.h>
#include <fst/rational.h>
}
// See Fst<>::Copy() for doc.
- ClosureFst(const ClosureFst<Arc> &fst, bool safe = false)
+ ClosureFst(const ClosureFst &fst, bool safe = false)
: RationalFst<A>(fst, safe) {}
// Gets a copy of this ClosureFst. See Fst<>::Copy() for further doc.
- ClosureFst<A> *Copy(bool safe = false) const override {
- return new ClosureFst<A>(*this, safe);
+ ClosureFst *Copy(bool safe = false) const override {
+ return new ClosureFst(*this, safe);
}
private:
#include <utility>
#include <vector>
+#include <fst/types.h>
#include <fst/log.h>
#include <fst/cache.h>
#include <fst/test-properties.h>
#include <fst/util.h>
-
namespace fst {
struct CompactFstOptions : public CacheOptions {
// shared_ptr<Compactor> compactor = nullptr);
// // Copy constructor. Must make a thread-safe copy suitable for use by
// // by Fst::Copy(/*safe=*/true). Only thread-unsafe data structures
-// // need to be deeply copied.
+// // need to be deeply copied. Ideally, this constructor is O(1) and any
+// // large structures are thread-safe and shared, while small ones may
+// // need to be copied.
// Compactor(const Compactor &compactor);
// // Default constructor (optional, see comment below).
// Compactor();
// FSTERROR() << "Compactor: No default constructor";
// }
-// Default implementation data for DefaultCompactor. DefaultCompactStore
-// is thread-safe and can be shared between otherwise independent copies.
-// Only old-style ArcCompactors are supported because the DefaultCompactStore
-// constructors use the old API.
+// Default implementation data for DefaultCompactor. Only old-style
+// ArcCompactors are supported because the DefaultCompactStore constructors
+// use the old API.
+//
+// DefaultCompact store is thread-compatible, but not thread-safe.
+// The copy constructor makes a thread-safe copy.
//
// The implementation contains two arrays: 'states_' and 'compacts_'.
//
public:
DefaultCompactStore() = default;
+ // Makes a thread-safe copy. O(1).
+ DefaultCompactStore(const DefaultCompactStore &) = default;
+
template <class Arc, class ArcCompactor>
DefaultCompactStore(const Fst<Arc> &fst, const ArcCompactor &arc_compactor);
DefaultCompactStore(const Iterator begin, const Iterator end,
const ArcCompactor &arc_compactor);
- ~DefaultCompactStore() {
- if (!states_region_) delete[] states_;
- if (!compacts_region_) delete[] compacts_;
- }
+ ~DefaultCompactStore() = default;
template <class ArcCompactor>
static DefaultCompactStore *Read(std::istream &strm,
static const std::string &Type();
private:
- std::unique_ptr<MappedFile> states_region_;
- std::unique_ptr<MappedFile> compacts_region_;
+ std::shared_ptr<MappedFile> states_region_;
+ std::shared_ptr<MappedFile> compacts_region_;
+ // Unowned pointer into states_region_.
Unsigned *states_ = nullptr;
+ // Unowned pointer into compacts_region_.
Element *compacts_ = nullptr;
size_t nstates_ = 0;
size_t ncompacts_ = 0;
if (fst.Final(s) != Weight::Zero()) ++nfinals;
}
if (arc_compactor.Size() == -1) {
- states_ = new Unsigned[nstates_ + 1];
+ states_region_ = fst::WrapUnique(
+ MappedFile::Allocate(sizeof(states_[0]) * (nstates_ + 1)));
+ states_ = static_cast<Unsigned *>(states_region_->mutable_data());
ncompacts_ = narcs_ + nfinals;
- compacts_ = new Element[ncompacts_];
+ compacts_region_ = fst::WrapUnique(
+ MappedFile::Allocate(sizeof(compacts_[0]) * (ncompacts_)));
+ compacts_ = static_cast<Element *>(compacts_region_->mutable_data());
states_[nstates_] = ncompacts_;
} else {
states_ = nullptr;
error_ = true;
return;
}
- compacts_ = new Element[ncompacts_];
+ compacts_region_ = fst::WrapUnique(
+ MappedFile::Allocate(sizeof(compacts_[0]) * (ncompacts_)));
+ compacts_ = static_cast<Element *>(compacts_region_->mutable_data());
}
size_t pos = 0;
size_t fpos = 0;
if (ncompacts_ == 0) return;
start_ = 0;
nstates_ = ncompacts_ / arc_compactor.Size();
- compacts_ = new Element[ncompacts_];
+ compacts_region_ = fst::WrapUnique(
+ MappedFile::Allocate(sizeof(compacts_[0]) * (ncompacts_)));
+ compacts_ = static_cast<Element *>(compacts_region_->mutable_data());
size_t i = 0;
Iterator it = begin;
for (; it != end; ++it, ++i) {
}
}
start_ = 0;
- compacts_ = new Element[ncompacts_];
- states_ = new Unsigned[nstates_ + 1];
+ compacts_region_ = fst::WrapUnique(
+ MappedFile::Allocate(sizeof(compacts_[0]) * (ncompacts_)));
+ compacts_ = static_cast<Element *>(compacts_region_->mutable_data());
+ states_region_ = fst::WrapUnique(
+ MappedFile::Allocate(sizeof(states_[0]) * (nstates_ + 1)));
+ states_ = static_cast<Unsigned *>(states_region_->mutable_data());
states_[nstates_] = ncompacts_;
size_t i = 0;
size_t s = 0;
template <class C, class U, class S> class DefaultCompactState;
// Wraps an old-style arc compactor and a compact store as a new Fst compactor.
-// S must be thread-safe.
+// The copy constructors of AC and S must make thread-safe copies and should
+// be O(1).
template <class AC, class U,
class S /*= DefaultCompactStore<typename AC::Element, U>*/>
class DefaultCompactor {
: DefaultCompactor(b, e, std::make_shared<ArcCompactor>()) {}
// Copy constructor. This makes a thread-safe copy, so requires that
- // CompactStore is thread-safe.
+ // The ArcCompactor and CompactStore copy constructores make thread-safe
+ // copies.
DefaultCompactor(const DefaultCompactor &compactor)
: arc_compactor_(
- std::make_shared<ArcCompactor>(*compactor.GetArcCompactor())),
- compact_store_(compactor.SharedCompactStore()) {}
+ compactor.GetArcCompactor() == nullptr
+ ? nullptr
+ : std::make_shared<ArcCompactor>(*compactor.GetArcCompactor())),
+ compact_store_(compactor.GetCompactStore() == nullptr
+ ? nullptr
+ : std::make_shared<CompactStore>(
+ *compactor.GetCompactStore())) {}
template <class OtherC>
explicit DefaultCompactor(const DefaultCompactor<OtherC, U, S> &compactor)
: arc_compactor_(
- std::make_shared<ArcCompactor>(*compactor.GetArcCompactor())),
- compact_store_(compactor.SharedCompactStore()) {}
+ compactor.GetArcCompactor() == nullptr
+ ? nullptr
+ : std::make_shared<ArcCompactor>(*compactor.GetArcCompactor())),
+ compact_store_(compactor.GetCompactStore() == nullptr
+ ? nullptr
+ : std::make_shared<CompactStore>(
+ *compactor.GetCompactStore())) {}
StateId Start() const { return compact_store_->Start(); }
StateId NumStates() const { return compact_store_->NumStates(); }
}
const ArcCompactor *GetArcCompactor() const { return arc_compactor_.get(); }
- CompactStore *GetCompactStore() const { return compact_store_.get(); }
+ const CompactStore *GetCompactStore() const { return compact_store_.get(); }
std::shared_ptr<ArcCompactor> SharedArcCompactor() const {
return arc_compactor_;
#include <string>
#include <vector>
+#include <fst/types.h>
#include <fst/log.h>
#include <fst/fst.h>
}
// See Fst<>::Copy() for doc.
- ComplementFst(const ComplementFst<Arc> &fst, bool safe = false)
+ ComplementFst(const ComplementFst &fst, bool safe = false)
: ImplToFst<Impl>(fst, safe) {}
// Gets a copy of this FST. See Fst<>::Copy() for further doc.
- ComplementFst<Arc> *Copy(bool safe = false) const override {
- return new ComplementFst<Arc>(*this, safe);
+ ComplementFst *Copy(bool safe = false) const override {
+ return new ComplementFst(*this, safe);
}
inline void InitStateIterator(StateIteratorData<Arc> *data) const override;
#ifndef FST_COMPOSE_FILTER_H_
#define FST_COMPOSE_FILTER_H_
+#include <fst/types.h>
+
#include <fst/filter-state.h>
#include <fst/fst-decl.h> // For optional argument declarations
#include <fst/fst.h>
fst1_(matcher1_->GetFst()),
fst2_(matcher2_->GetFst()) {}
- NullComposeFilter(const NullComposeFilter<M1, M2> &filter, bool safe = false)
+ NullComposeFilter(const NullComposeFilter &filter, bool safe = false)
: matcher1_(filter.matcher1_->Copy(safe)),
matcher2_(filter.matcher2_->Copy(safe)),
fst1_(matcher1_->GetFst()),
: filter_(fst1, fst2, matcher1, matcher2),
keep_multi_eps_(keep_multi_eps) {}
- MultiEpsFilter(const MultiEpsFilter<Filter> &filter, bool safe = false)
+ MultiEpsFilter(const MultiEpsFilter &filter, bool safe = false)
: filter_(filter.filter_, safe),
keep_multi_eps_(filter.keep_multi_eps_) {}
#include <memory>
#include <utility>
+#include <fst/types.h>
#include <fst/log.h>
#include <fst/cache.h>
: ImplToFst<Impl>(CreateBase2(fst1, fst2, opts)) {}
// See Fst<>::Copy() for doc.
- ComposeFst(const ComposeFst<A, CacheStore> &fst, bool safe = false)
+ ComposeFst(const ComposeFst &fst, bool safe = false)
: ImplToFst<Impl>(safe ? std::shared_ptr<Impl>(fst.GetImpl()->Copy())
: fst.GetSharedImpl()) {}
// Get a copy of this ComposeFst. See Fst<>::Copy() for further doc.
- ComposeFst<A, CacheStore> *Copy(bool safe = false) const override {
- return new ComposeFst<A, CacheStore>(*this, safe);
+ ComposeFst *Copy(bool safe = false) const override {
+ return new ComposeFst(*this, safe);
}
inline void InitStateIterator(StateIteratorData<Arc> *data) const override;
if (match_type_ == MATCH_OUTPUT) std::swap(loop_.ilabel, loop_.olabel);
}
- ComposeFstMatcher<CacheStore, Filter, StateTable> *Copy(
- bool safe = false) const override {
- return new ComposeFstMatcher<CacheStore, Filter, StateTable>(*this, safe);
+ ComposeFstMatcher *Copy(bool safe = false) const override {
+ return new ComposeFstMatcher(*this, safe);
}
MatchType Type(bool test) const override {
#include <algorithm>
#include <vector>
+#include <fst/types.h>
+
#include <fst/expanded-fst.h>
#include <fst/mutable-fst.h>
#include <fst/rational.h>
}
// See Fst<>::Copy() for doc.
- ConcatFst(const ConcatFst<Arc> &fst, bool safe = false)
+ ConcatFst(const ConcatFst &fst, bool safe = false)
: RationalFst<Arc>(fst, safe) {}
// Get a copy of this ConcatFst. See Fst<>::Copy() for further doc.
- ConcatFst<Arc> *Copy(bool safe = false) const override {
- return new ConcatFst<Arc>(*this, safe);
+ ConcatFst *Copy(bool safe = false) const override {
+ return new ConcatFst(*this, safe);
}
private:
#include <memory>
#include <vector>
+#include <fst/types.h>
+
#include <fst/dfs-visit.h>
#include <fst/mutable-fst.h>
#include <fst/union-find.h>
// Google-only...
// ...Google-only
+#include <fst/types.h>
#include <fst/log.h>
#include <fst/expanded-fst.h>
StateId Start() const { return start_; }
- Weight Final(StateId s) const { return states_[s].weight; }
+ Weight Final(StateId s) const { return states_[s].final_weight; }
StateId NumStates() const { return nstates_; }
size_t NumOutputEpsilons(StateId s) const { return states_[s].noepsilons; }
- static ConstFstImpl<Arc, Unsigned> *Read(std::istream &strm,
- const FstReadOptions &opts);
+ static ConstFstImpl *Read(std::istream &strm, const FstReadOptions &opts);
const Arc *Arcs(StateId s) const { return arcs_ + states_[s].pos; }
// States implemented by array *states_ below, arcs by (single) *arcs_.
struct ConstState {
- Weight weight; // Final weight.
+ Weight final_weight; // Final weight.
Unsigned pos; // Start of state's arcs in *arcs_.
Unsigned narcs; // Number of arcs (per state).
Unsigned niepsilons; // Number of input epsilons.
Unsigned noepsilons; // Number of output epsilons.
- ConstState() : weight(Weight::Zero()) {}
+ ConstState() : final_weight(Weight::Zero()) {}
};
// Properties always true of this FST class.
++nstates_;
narcs_ += fst.NumArcs(siter.Value());
}
- states_region_.reset(MappedFile::Allocate(nstates_ * sizeof(*states_)));
- arcs_region_.reset(MappedFile::Allocate(narcs_ * sizeof(*arcs_)));
- states_ = reinterpret_cast<ConstState *>(states_region_->mutable_data());
- arcs_ =
reinterpret_cast<Arc *>(arcs_region_->mutable_data());
+ states_region_.reset(MappedFile::AllocateType<ConstState>(nstates_));
+ arcs_region_.reset(MappedFile::AllocateType<Arc>(narcs_));
+ states_ = static_cast<ConstState *>(states_region_->mutable_data());
+ arcs_ =
static_cast<Arc *>(arcs_region_->mutable_data());
size_t pos = 0;
for (StateId s = 0; s < nstates_; ++s) {
- states_[s].weight = fst.Final(s);
+ states_[s].final_weight = fst.Final(s);
states_[s].pos = pos;
states_[s].narcs = 0;
states_[s].niepsilons = 0;
template <class Arc, class Unsigned>
ConstFstImpl<Arc, Unsigned> *ConstFstImpl<Arc, Unsigned>::Read(
std::istream &strm, const FstReadOptions &opts) {
- using ConstState = typename ConstFstImpl<Arc, Unsigned>::ConstState;
- std::unique_ptr<ConstFstImpl<Arc, Unsigned>> impl(
- new ConstFstImpl<Arc, Unsigned>());
+ std::unique_ptr<ConstFstImpl> impl(new ConstFstImpl());
FstHeader hdr;
if (!impl->ReadHeader(strm, opts, kMinFileVersion, &hdr)) return nullptr;
impl->start_ = hdr.Start();
return nullptr;
}
impl->states_ =
- reinterpret_cast<ConstState *>(impl->states_region_->mutable_data());
+ static_cast<ConstState *>(impl->states_region_->mutable_data());
if ((hdr.GetFlags() & FstHeader::IS_ALIGNED) && !AlignInput(strm)) {
LOG(ERROR) << "ConstFst::Read: Alignment failed: " << opts.source;
return nullptr;
LOG(ERROR) << "ConstFst::Read: Read failed: " << opts.source;
return nullptr;
}
- impl->arcs_ =
reinterpret_cast<Arc *>(impl->arcs_region_->mutable_data());
+ impl->arcs_ =
static_cast<Arc *>(impl->arcs_region_->mutable_data());
return impl.release();
}
explicit ConstFst(const Fst<Arc> &fst)
: ImplToExpandedFst<Impl>(std::make_shared<Impl>(fst)) {}
- ConstFst(const ConstFst<A, Unsigned> &fst, bool safe = false)
+ ConstFst(const ConstFst &fst, bool safe = false)
: ImplToExpandedFst<Impl>(fst) {}
// Gets a copy of this ConstFst. See Fst<>::Copy() for further doc.
- ConstFst<A, Unsigned> *Copy(bool safe = false) const override {
- return new ConstFst<A, Unsigned>(*this, safe);
+ ConstFst *Copy(bool safe = false) const override {
+ return new ConstFst(*this, safe);
}
// Reads a ConstFst from an input stream, returning nullptr on error.
- static ConstFst<A, Unsigned> *Read(std::istream &strm,
- const FstReadOptions &opts) {
+ static ConstFst *Read(std::istream &strm, const FstReadOptions &opts) {
auto *impl = Impl::Read(strm, opts);
- return impl ? new ConstFst<A, Unsigned>(std::shared_ptr<Impl>(impl))
- : nullptr;
+ return impl ? new ConstFst(std::shared_ptr<Impl>(impl)) : nullptr;
}
// Read a ConstFst from a file; return nullptr on error; empty source reads
// from standard input.
- static ConstFst<A, Unsigned> *Read(const std::string &source) {
+ static ConstFst *Read(const std::string &source) {
auto *impl = ImplToExpandedFst<Impl>::Read(source);
- return impl ? new ConstFst<A, Unsigned>(std::shared_ptr<Impl>(impl))
- : nullptr;
+ return impl ? new ConstFst(std::shared_ptr<Impl>(impl)) : nullptr;
}
bool Write(std::ostream &strm, const FstWriteOptions &opts) const override {
}
size_t pos = 0;
size_t states = 0;
- typename ConstFst<Arc, Unsigned>::ConstState state;
+ ConstState state;
for (StateIterator<FST> siter(fst); !siter.Done(); siter.Next()) {
const auto s = siter.Value();
- state.weight = fst.Final(s);
+ state.final_weight = fst.Final(s);
state.pos = pos;
state.narcs = fst.NumArcs(s);
state.niepsilons = fst.NumInputEpsilons(s);
#include <utility>
#include <vector>
+#include <fst/types.h>
#include <fst/log.h>
#include <fst/arc-map.h>
SetOutputSymbols(fst.OutputSymbols());
}
- DeterminizeFstImplBase(const DeterminizeFstImplBase<Arc> &impl)
+ DeterminizeFstImplBase(const DeterminizeFstImplBase &impl)
: CacheImpl<Arc>(impl), fst_(impl.fst_->Copy(true)) {
SetType("determinize");
SetProperties(impl.Properties(), kCopyProperties);
SetOutputSymbols(impl.OutputSymbols());
}
- virtual DeterminizeFstImplBase<Arc> *Copy() const = 0;
+ virtual DeterminizeFstImplBase *Copy() const = 0;
StateId Start() {
if (!HasStart()) {
if (out_dist_) out_dist_->clear();
}
- DeterminizeFsaImpl(
- const DeterminizeFsaImpl<Arc, CommonDivisor, Filter, StateTable> &impl)
+ DeterminizeFsaImpl(const DeterminizeFsaImpl &impl)
: DeterminizeFstImplBase<Arc>(impl),
delta_(impl.delta_),
in_dist_(nullptr),
}
}
- DeterminizeFsaImpl<Arc, CommonDivisor, Filter, StateTable> *Copy()
- const override {
- return new DeterminizeFsaImpl<Arc, CommonDivisor, Filter, StateTable>(
- *this);
+ DeterminizeFsaImpl *Copy() const override {
+ return new DeterminizeFsaImpl(*this);
}
uint64 Properties() const override { return Properties(kFstProperties); }
Init(GetFst(), opts.filter);
}
- DeterminizeFstImpl(
- const DeterminizeFstImpl<Arc, G, CommonDivisor, Filter, StateTable> &impl)
+ DeterminizeFstImpl(const DeterminizeFstImpl &impl)
: DeterminizeFstImplBase<Arc>(impl),
delta_(impl.delta_),
subsequential_label_(impl.subsequential_label_),
Init(GetFst(), nullptr);
}
- DeterminizeFstImpl<Arc, G, CommonDivisor, Filter, StateTable> *Copy()
- const override {
- return new DeterminizeFstImpl<Arc, G, CommonDivisor, Filter, StateTable>(
- *this);
+ DeterminizeFstImpl *Copy() const override {
+ return new DeterminizeFstImpl(*this);
}
uint64 Properties() const override { return Properties(kFstProperties); }
}
// See Fst<>::Copy() for doc.
- DeterminizeFst(const DeterminizeFst<Arc> &fst, bool safe = false)
+ DeterminizeFst(const DeterminizeFst &fst, bool safe = false)
: ImplToFst<Impl>(safe ? std::shared_ptr<Impl>(fst.GetImpl()->Copy())
: fst.GetSharedImpl()) {}
// Get a copy of this DeterminizeFst. See Fst<>::Copy() for further doc.
- DeterminizeFst<Arc> *Copy(bool safe = false) const override {
- return new DeterminizeFst<Arc>(*this, safe);
+ DeterminizeFst *Copy(bool safe = false) const override {
+ return new DeterminizeFst(*this, safe);
}
inline void InitStateIterator(StateIteratorData<Arc> *data) const override;
}
// See Fst<>::Copy() for doc.
- DifferenceFst(const DifferenceFst<Arc> &fst, bool safe = false)
+ DifferenceFst(const DifferenceFst &fst, bool safe = false)
: ComposeFst<Arc>(fst, safe) {}
// Get a copy of this DifferenceFst. See Fst<>::Copy() for further doc.
- DifferenceFst<Arc> *Copy(bool safe = false) const override {
- return new DifferenceFst<Arc>(*this, safe);
+ DifferenceFst *Copy(bool safe = false) const override {
+ return new DifferenceFst(*this, safe);
}
private:
#include <utility>
#include <vector>
+#include <fst/types.h>
+
#include <fst/arcsort.h>
#include <fst/compose.h>
#include <fst/connect.h>
#include <unordered_map>
#include <vector>
+#include <fst/types.h>
#include <fst/log.h>
#include <fst/cache.h>
~EditFstData() {}
- static EditFstData<Arc, WrappedFstT, MutableFstT> *Read(
- std::istream &strm, const FstReadOptions &opts);
+ static EditFstData *Read(std::istream &strm, const FstReadOptions &opts);
bool Write(std::ostream &strm, const FstWriteOptions &opts) const {
// Serializes all private data members of this class.
EditFstData<A, WrappedFstT, MutableFstT> *
EditFstData<A, WrappedFstT, MutableFstT>::Read(std::istream &strm,
const FstReadOptions &opts) {
- auto *data = new EditFstData<A, WrappedFstT, MutableFstT>();
+ auto *data = new EditFstData;
// Next read in MutabelFstT machine that stores edits
FstReadOptions edits_opts(opts);
// Contained header was written out, so read it in.
return wrapped_->NumStates() + data_->NumNewStates();
}
- static EditFstImpl<Arc, WrappedFstT, MutableFstT> *Read(
- std::istream &strm, const FstReadOptions &opts);
+ static EditFstImpl *Read(std::istream &strm, const FstReadOptions &opts);
bool Write(std::ostream &strm, const FstWriteOptions &opts) const {
FstHeader hdr;
: ImplToMutableFst<Impl>(std::make_shared<Impl>(fst)) {}
// See Fst<>::Copy() for doc.
- EditFst(const EditFst<Arc, WrappedFstT, MutableFstT> &fst, bool safe = false)
+ EditFst(const EditFst &fst, bool safe = false)
: ImplToMutableFst<Impl>(fst, safe) {}
~EditFst() override {}
// Gets a copy of this EditFst. See Fst<>::Copy() for further doc.
- EditFst<Arc, WrappedFstT, MutableFstT> *Copy(
- bool safe = false) const override {
- return new EditFst<Arc, WrappedFstT, MutableFstT>(*this, safe);
+ EditFst *Copy(bool safe = false) const override {
+ return new EditFst(*this, safe);
}
- EditFst<Arc, WrappedFstT, MutableFstT> &operator=(
- const EditFst<Arc, WrappedFstT, MutableFstT> &fst) {
+ EditFst &operator=(const EditFst &fst) {
SetImpl(fst.GetSharedImpl());
return *this;
}
- EditFst<Arc, WrappedFstT, MutableFstT> &operator=(
- const Fst<Arc> &fst) override {
+ EditFst &operator=(const Fst<Arc> &fst) override {
SetImpl(std::make_shared<Impl>(fst));
return *this;
}
// Reads an EditFst from an input stream, returning nullptr on error.
- static EditFst<Arc, WrappedFstT, MutableFstT> *Read(
- std::istream &strm, const FstReadOptions &opts) {
+ static EditFst *Read(std::istream &strm, const FstReadOptions &opts) {
auto *impl = Impl::Read(strm, opts);
- return impl ? new EditFst<Arc>(std::shared_ptr<Impl>(impl)) : nullptr;
+ return impl ? new EditFst(std::shared_ptr<Impl>(impl)) : nullptr;
}
// Reads an EditFst from a file, returning nullptr on error. If the source
// argument is an empty string, it reads from standard input.
- static EditFst<Arc, WrappedFstT, MutableFstT> *Read(
- const std::string &source) {
+ static EditFst *Read(const std::string &source) {
auto *impl = ImplToExpandedFst<Impl, MutableFst<Arc>>::Read(source);
- return impl ? new EditFst<Arc, WrappedFstT, MutableFstT>(
- std::shared_ptr<Impl>(impl))
- : nullptr;
+ return impl ? new EditFst(std::shared_ptr<Impl>(impl)) : nullptr;
}
bool Write(std::ostream &strm, const FstWriteOptions &opts) const override {
#include <utility>
#include <vector>
+#include <fst/types.h>
#include <fst/log.h>
#include <fst/arc-map.h>
#include <fstream>
}
}
- // Given an arc, looks up its encoded label or returns kNoLabel if not found.
- Label GetLabel(const Arc &arc) const {
- const Triple triple(arc, flags_);
- auto it = triple2label_.find(&triple);
- return (it == triple2label_.end()) ? kNoLabel : it->second;
- }
-
// Given an encoded arc label, decodes back to input/output labels and costs.
const Triple *Decode(Label label) const {
if (label < 1 || label > triples_.size()) {
size_t Size() const { return triples_.size(); }
- static EncodeTable<Arc> *Read(std::istream &strm, const std::string &source);
+ static EncodeTable *Read(std::istream &strm, const std::string &source);
bool Write(std::ostream &strm, const std::string &source) const;
const auto size = hdr.Size();
auto table = fst::make_unique<EncodeTable>(flags);
for (int64 i = 0; i < size; ++i) {
- auto triple = fst::make_unique<Triple>();
- ReadType(strm, &triple->ilabel);
- ReadType(strm, &triple->olabel);
- ReadType(strm, &triple->weight);
- table->triples_.emplace_back(std::move(triple));
+ table->triples_.emplace_back(std::move(Triple::Read(strm)));
table->triple2label_[table->triples_.back().get()] = table->triples_.size();
}
if (flags & kEncodeHasISymbols) {
EncodeType Type() const { return type_; }
- static EncodeMapper<Arc> *Read(std::istream &strm, const std::string &source,
- EncodeType type = ENCODE) {
+ static EncodeMapper *Read(std::istream &strm, const std::string &source,
+ EncodeType type = ENCODE) {
auto *table = internal::EncodeTable<Arc>::Read(strm, source);
return table ? new EncodeMapper(table->Flags(), type, table) : nullptr;
}
- static EncodeMapper<Arc> *Read(const std::string &source,
- EncodeType type = ENCODE) {
+ static EncodeMapper *Read(const std::string &source,
+ EncodeType type = ENCODE) {
std::ifstream strm(source, std::ios_base::in | std::ios_base::binary);
if (!strm) {
LOG(ERROR) << "EncodeMapper: Can't open file: " << source;
: ArcMapFst<Arc, Arc, Mapper>(fst, encoder, ArcMapFstOptions()) {}
// See Fst<>::Copy() for doc.
- EncodeFst(const EncodeFst<Arc> &fst, bool copy = false)
+ EncodeFst(const EncodeFst &fst, bool copy = false)
: ArcMapFst<Arc, Arc, Mapper>(fst, copy) {}
// Makes a copy of this EncodeFst. See Fst<>::Copy() for further doc.
- EncodeFst<Arc> *Copy(bool safe = false) const override {
+ EncodeFst *Copy(bool safe = false) const override {
if (safe) {
FSTERROR() << "EncodeFst::Copy(true): Not allowed";
GetImpl()->SetProperties(kError, kError);
public:
using Mapper = EncodeMapper<Arc>;
using Impl = internal::ArcMapFstImpl<Arc, Arc, Mapper>;
- using ImplToFst<Impl>::GetImpl;
DecodeFst(const Fst<Arc> &fst, const Mapper &encoder)
: ArcMapFst<Arc, Arc, Mapper>(fst, Mapper(encoder, DECODE),
}
// See Fst<>::Copy() for doc.
- DecodeFst(const DecodeFst<Arc> &fst, bool safe = false)
+ DecodeFst(const DecodeFst &fst, bool safe = false)
: ArcMapFst<Arc, Arc, Mapper>(fst, safe) {}
// Makes a copy of this DecodeFst. See Fst<>::Copy() for further doc.
- DecodeFst<Arc> *Copy(bool safe = false) const override {
+ DecodeFst *Copy(bool safe = false) const override {
return new DecodeFst(*this, safe);
}
private:
+ using ImplToFst<Impl>::GetImpl;
using ImplToFst<Impl>::GetMutableImpl;
};
#include <unordered_map>
#include <vector>
+#include <fst/types.h>
#include <fst/log.h>
#include <fst/encode.h>
virtual StateId NumStates() const = 0; // State count
// Get a copy of this ExpandedFst. See Fst<>::Copy() for further doc.
- ExpandedFst<Arc> *Copy(bool safe = false) const override = 0;
+ ExpandedFst *Copy(bool safe = false) const override = 0;
// Read an ExpandedFst from an input stream; return NULL on error.
- static ExpandedFst<Arc> *Read(std::istream &strm,
- const FstReadOptions &opts) {
+ static ExpandedFst *Read(std::istream &strm, const FstReadOptions &opts) {
FstReadOptions ropts(opts);
FstHeader hdr;
if (ropts.header) {
}
auto *fst = reader(strm, ropts);
if (!fst) return nullptr;
- return static_cast<ExpandedFst<Arc> *>(fst);
+ return static_cast<ExpandedFst *>(fst);
}
// Read an ExpandedFst from a file; return NULL on error.
// Empty source reads from standard input.
- static ExpandedFst<Arc> *Read(const std::string &source) {
+ static ExpandedFst *Read(const std::string &source) {
if (!source.empty()) {
std::ifstream strm(source,
std::ios_base::in | std::ios_base::binary);
explicit ImplToExpandedFst(std::shared_ptr<Impl> impl)
: ImplToFst<Impl, FST>(impl) {}
- ImplToExpandedFst(const ImplToExpandedFst<Impl, FST> &fst, bool safe)
+ ImplToExpandedFst(const ImplToExpandedFst &fst, bool safe)
: ImplToFst<Impl, FST>(fst, safe) {}
static Impl *Read(std::istream &strm, const FstReadOptions &opts) {
using StateId = typename Arc::StateId;
void Reset() {
- final_ = Weight::Zero();
+ final_weight_ = Weight::Zero();
niepsilons_ = 0;
noepsilons_ = 0;
arcs_.clear();
}
- Weight Final() const { return final_; }
+ Weight Final() const { return final_weight_; }
size_t NumInputEpsilons() const { return niepsilons_; }
const Arc *Arcs() const { return arcs_.empty() ? nullptr : &arcs_[0]; }
- void SetFinal(Weight final) { final_ = final; }
+ void SetFinal(Weight weight) { final_weight_ = weight; }
void ReserveArcs(size_t n) { arcs_.reserve(n); }
int *MutableRefCount() const { return nullptr; }
private:
- Weight final_ = Weight::Zero();
+ Weight final_weight_ = Weight::Zero();
size_t niepsilons_ = 0; // Number of input epsilons.
size_t noepsilons_ = 0; // Number of output epsilons.
std::vector<Arc> arcs_;
#include <string>
+#include <fst/types.h>
#include <fst/log.h>
#include <fst/pair-weight.h>
}
PairWeight<X1, X2> Quantize(float delta = kDelta) const {
- return ExpectationWeight(PairWeight<X1, X2>::Quantize());
+ return ExpectationWeight(PairWeight<X1, X2>::Quantize(delta));
}
ReverseWeight Reverse() const {
#include <vector>
#include <fst/compat.h>
+#include <fst/types.h>
#include <fst/log.h>
#include <fst/extensions/compress/elias.h>
#include <fst/extensions/compress/gzfile.h>
#include <fst/extensions/far/create.h>
#include <fst/extensions/far/equal.h>
#include <fst/extensions/far/extract.h>
-#include <fst/extensions/far/far.h>
#include <fst/extensions/far/far-class.h>
+#include <fst/extensions/far/far.h>
#include <fst/extensions/far/info.h>
#include <fst/extensions/far/isomorphic.h>
#include <fst/extensions/far/print-strings.h>
#include <vector>
#include <fst/compat.h>
+#include <fst/types.h>
#include <fst/log.h>
#include <fst/extensions/linear/linear-fst-data.h>
#include <fst/fst.h>
SplitAndPush<Arc>(fields[0], delim, fsyms, input_labels);
SplitAndPush<Arc>(fields[1], delim, osyms, output_labels);
- *weight = StrToWeight<typename Arc::Weight>(fields[2], model, *num_line);
+ *weight = StrToWeight<typename Arc::Weight>(fields[2]);
GuessStartOrEnd<Arc>(input_labels, kNoLabel);
GuessStartOrEnd<Arc>(output_labels, kNoLabel);
#include <vector>
+#include <fst/types.h>
#include <fst/extensions/mpdt/mpdt.h>
#include <fst/extensions/pdt/paren.h>
#include <fst/cache.h>
#include <unordered_map>
#include <vector>
+#include <fst/types.h>
#include <fst/extensions/mpdt/mpdt.h>
#include <fst/fst.h>
#include <utility>
#include <vector>
+#include <fst/types.h>
#include <fst/log.h>
#include <fst/compose.h> // for ComposeOptions
#include <fst/util.h>
#include <vector>
#include <fst/compat.h>
+#include <fst/types.h>
// This class is a bitstring storage class with an index that allows
// seeking to the Nth set or clear bit in time O(Log(N)) where N is
#include <vector>
#include <fst/compat.h>
+#include <fst/types.h>
#include <fst/log.h>
#include <fst/extensions/ngram/bitmap-index.h>
#include <fstream>
strm.read(reinterpret_cast<char *>(&num_final), sizeof(num_final));
size_t size = Storage(num_states, num_futures, num_final);
MappedFile *data_region = MappedFile::Allocate(size);
- char *data = reinterpret_cast<char *>(data_region->mutable_data());
+ char *data = static_cast<char *>(data_region->mutable_data());
// Copy num_states, num_futures and num_final back into data.
memcpy(data, reinterpret_cast<char *>(&num_states), sizeof(num_states));
memcpy(data + sizeof(num_states), reinterpret_cast<char *>(&num_futures),
Label label = kNoLabel;
const size_t storage = Storage(num_states, num_futures, num_final);
MappedFile *data_region = MappedFile::Allocate(storage);
- char *data = reinterpret_cast<char *>(data_region->mutable_data());
+ char *data = static_cast<char *>(data_region->mutable_data());
memset(data, 0, storage);
size_t offset = 0;
memcpy(data + offset, reinterpret_cast<char *>(&num_states),
#ifndef FST_EXTENSIONS_NGRAM_NTHBIT_H_
#define FST_EXTENSIONS_NGRAM_NTHBIT_H_
-#include <fst/types.h>
-
-#if defined( __arm__) // 32-bit ARM
+#include <cstdint>
-// Returns the position (0-63) of the r-th 1 bit in v.
-// 1 <= r <= CountOnes(v) <= 64. Therefore, v must not be 0.
-uint32 nth_bit(uint64 v, uint32 r);
+#include <fst/types.h>
-#elif defined(__BMI2__) // Intel Bit Manipulation Instruction Set 2
-// PDEP requires BMI2.
+#if defined(__BMI2__) // Intel Bit Manipulation Instruction Set 2
+// PDEP requires BMI2; this is present starting with Haswell.
#include <immintrin.h>
return __builtin_ctzll(_pdep_u64(uint64{1} << (r - 1), v));
}
-#else // !defined(__BMI2__) && !defined(__arm__)
+#elif SIZE_MAX == UINT32_MAX
+// Detect 32-bit architectures via size_t.
+
+// Returns the position (0-63) of the r-th 1 bit in v.
+// 1 <= r <= CountOnes(v) <= 64. Therefore, v must not be 0.
+uint32 nth_bit(uint64 v, uint32 r);
+
+#elif SIZE_MAX == UINT64_MAX
+// Default 64-bit version, used by ARM64 and Intel < Haswell.
-extern const uint32 nth_bit_bit_offset[];
+namespace internal {
+extern const uint64 kPrefixSumOverflow[65];
+extern const uint8 kSelectInByte[2048];
+} // namespace internal
// Returns the position (0-63) of the r-th 1 bit in v.
// 1 <= r <= CountOnes(v) <= 64. Therefore, v must not be 0.
-inline uint32 nth_bit(uint64 v, uint32 r) {
- uint32 shift = 0;
- uint32 c = __builtin_popcount(v & 0xffffffff);
- uint32 mask = -(r > c);
- r -= c & mask;
- shift += (32 & mask);
-
- c = __builtin_popcount((v >> shift) & 0xffff);
- mask = -(r > c);
- r -= c & mask;
- shift += (16 & mask);
-
- c = __builtin_popcount((v >> shift) & 0xff);
- mask = -(r > c);
- r -= c & mask;
- shift += (8 & mask);
+//
+// This version is based on the paper "Broadword Implementation of
+// Rank/Select Queries" by Sebastiano Vigna, p. 5, Algorithm 2, with
+// improvements from "Optimized Succinct Data Structures for Massive Data"
+// by Gog & Petri, 2014.
+inline uint32 nth_bit(const uint64 v, const uint32 r) {
+ constexpr uint64 kOnesStep4 = 0x1111111111111111;
+ constexpr uint64 kOnesStep8 = 0x0101010101010101;
+ constexpr uint64 kMSBsStep8 = 0x80 * kOnesStep8;
+ uint64 s = v;
+ s = s - ((s >> 1) & (0x5 * kOnesStep4));
+ s = (s & (0x3 * kOnesStep4)) + ((s >> 2) & (0x3 * kOnesStep4));
+ s = (s + (s >> 4)) & (0xF * kOnesStep8);
+ // s now contains the byte-wise popcounts of v.
+
+ // byte_sums contains partial sums of the byte-wise popcounts.
+ // That is, byte i contains the popcounts of bytes <= i.
+ uint64 byte_sums = s * kOnesStep8;
+
+ // kPrefixSumOverflow[r] == (0x80 - r) * kOnesStep8, so the high bit is
+ // still set if byte_sums - r >= 0, or byte_sums >= r. The first one set
+ // is in the byte with the sum larger than or equal to r (since r is 1-based),
+ // so this is the byte we need.
+ const uint64 b = (byte_sums + internal::kPrefixSumOverflow[r]) & kMSBsStep8;
+ // The first bit set is the high bit in the byte, so
+ // num_trailing_zeros == 8 * byte_nr + 7 and the byte number is the
+ // number of trailing zeros divided by 8.
+ const int byte_nr = __builtin_ctzll(b) >> 3;
+ const int shift = byte_nr << 3;
+ // The top byte contains the whole-word popcount; we never need that.
+ byte_sums <<= 8;
+ // Paper uses reinterpret_cast<uint8 *>; use shift/mask instead.
+ // Adjust for fact that r is 1-based.
+ const int rank_in_byte = r - 1 - (byte_sums >> shift) & 0xFF;
return shift +
- ((nth_bit_bit_offset[(v >> shift) & 0xff] >> ((r - 1) << 2)) & 0xf);
+ internal::kSelectInByte[(rank_in_byte << 8) + ((v >> shift) & 0xFF)];
}
-#endif // !defined(__BMI2__)
+#else
+
+#error Unrecognized architecture size
+
+#endif
#endif // FST_EXTENSIONS_NGRAM_NTHBIT_H_
#include <forward_list>
#include <vector>
+#include <fst/types.h>
#include <fst/log.h>
#include <fst/extensions/pdt/paren.h>
#include <fst/extensions/pdt/pdt.h>
#include <unordered_set>
#include <vector>
+#include <fst/types.h>
#include <fst/extensions/pdt/pdt.h>
#include <fst/fst.h>
#include <utility>
#include <vector>
+#include <fst/types.h>
#include <fst/log.h>
#include <fst/extensions/pdt/compose.h>
#include <fst/extensions/pdt/expand.h>
#include <utility>
#include <vector>
+#include <fst/types.h>
#include <fst/log.h>
#include <fst/extensions/pdt/paren.h>
#include <fst/extensions/pdt/pdt.h>
#include <utility>
#include <vector>
+#include <fst/types.h>
#include <fst/log.h>
#include <fst/cache.h>
: ImplToFst<Impl>(std::make_shared<Impl>(fst, opts)) {}
// See Fst<>::Copy() for doc.
- FactorWeightFst(const FactorWeightFst<Arc, FactorIterator> &fst, bool copy)
+ FactorWeightFst(const FactorWeightFst &fst, bool copy)
: ImplToFst<Impl>(fst, copy) {}
// Get a copy of this FactorWeightFst. See Fst<>::Copy() for further doc.
- FactorWeightFst<Arc, FactorIterator> *Copy(bool copy = false) const override {
- return new FactorWeightFst<Arc, FactorIterator>(*this, copy);
+ FactorWeightFst *Copy(bool copy = false) const override {
+ return new FactorWeightFst(*this, copy);
}
inline void InitStateIterator(StateIteratorData<Arc> *data) const override;
#include <forward_list>
#include <utility>
+#include <fst/types.h>
+
#include <fst/fst-decl.h> // For optional argument declarations
#include <fst/fst.h>
#include <fst/matcher.h>
#include <string>
#include <type_traits>
+#include <fst/types.h>
+
#include <fst/util.h>
#include <fst/weight.h>
constexpr FloatWeightTpl(T f) : value_(f) {} // NOLINT
// TODO(mjansche): Leave implicit once Android NDK r18 is the default.
- FloatWeightTpl(const FloatWeightTpl<T> &) = default;
- FloatWeightTpl(FloatWeightTpl<T> &&) noexcept = default;
+ FloatWeightTpl(const FloatWeightTpl &) = default;
+ FloatWeightTpl(FloatWeightTpl &&) noexcept = default;
- FloatWeightTpl<T> &operator=(const FloatWeightTpl<T> &) = default;
- FloatWeightTpl<T> &operator=(FloatWeightTpl<T> &&) noexcept = default;
+ FloatWeightTpl &operator=(const FloatWeightTpl &) = default;
+ FloatWeightTpl &operator=(FloatWeightTpl &&) noexcept = default;
std::istream &Read(std::istream &strm) { return ReadType(strm, &value_); }
#include <fst/compat.h>
#include <fst/flags.h>
+#include <fst/types.h>
#include <fst/log.h>
#include <fstream>
// (3) If a MutableFst is copied and then mutated, then the original is
// unmodified and vice versa (often by a copy-on-write on the initial
// mutation, which may not be constant time).
- virtual Fst<Arc> *Copy(bool safe = false) const = 0;
+ virtual Fst *Copy(bool safe = false) const = 0;
// Reads an FST from an input stream; returns nullptr on error.
- static Fst<Arc> *Read(std::istream &strm, const FstReadOptions &opts) {
+ static Fst *Read(std::istream &strm, const FstReadOptions &opts) {
FstReadOptions ropts(opts);
FstHeader hdr;
if (ropts.header) {
// Reads an FST from a file; returns nullptr on error. An empty source
// results in reading from standard input.
- static Fst<Arc> *Read(const std::string &source) {
+ static Fst *Read(const std::string &source) {
if (!source.empty()) {
std::ifstream strm(source,
std::ios_base::in | std::ios_base::binary);
virtual ~FstImpl() {}
- FstImpl &operator=(const FstImpl<Arc> &impl) {
+ FstImpl &operator=(const FstImpl &impl) {
properties_ = impl.properties_;
type_ = impl.type_;
isymbols_ = impl.isymbols_ ? impl.isymbols_->Copy() : nullptr;
return *this;
}
- FstImpl &operator=(FstImpl<Arc> &&impl) noexcept;
+ FstImpl &operator=(FstImpl &&impl) noexcept;
const std::string &Type() const { return type_; }
// This constructor presumes there is a copy constructor for the
// implementation that produces a thread-safe copy.
- ImplToFst(const ImplToFst<Impl, FST> &fst, bool safe) {
+ ImplToFst(const ImplToFst &fst, bool safe) {
if (safe) {
impl_ = std::make_shared<Impl>(*(fst.impl_));
} else {
ImplToFst() = delete;
- ImplToFst(const ImplToFst<Impl, FST> &fst) : impl_(fst.impl_) {}
+ ImplToFst(const ImplToFst &fst) : impl_(fst.impl_) {}
- ImplToFst(ImplToFst<Impl, FST> &&fst) noexcept
- : impl_(std::move(fst.impl_)) {
+ ImplToFst(ImplToFst &&fst) noexcept : impl_(std::move(fst.impl_)) {
fst.impl_ = std::make_shared<Impl>();
}
- ImplToFst<Impl, FST> &operator=(const ImplToFst<Impl, FST> &fst) {
+ ImplToFst &operator=(const ImplToFst &fst) {
impl_ = fst.impl_;
return *this;
}
- ImplToFst<Impl, FST> &operator=(ImplToFst<Impl, FST> &&fst) noexcept {
+ ImplToFst &operator=(ImplToFst &&fst) noexcept {
if (this != &fst) {
impl_ = std::move(fst.impl_);
fst.impl_ = std::make_shared<Impl>();
#include <map>
#include <string>
-#include <fst/types.h>
#include <fst/log.h>
// Generic class representing a globally-stored correspondence between
#include <sstream>
#include <vector>
+#include <fst/types.h>
#include <fst/log.h>
namespace fst {
}
// See Fst<>::Copy() for doc.
- IntersectFst(const IntersectFst<Arc> &fst, bool safe = false)
+ IntersectFst(const IntersectFst &fst, bool safe = false)
: ComposeFst<Arc>(fst, safe) {}
// Get a copy of this IntersectFst. See Fst<>::Copy() for further doc.
- IntersectFst<Arc> *Copy(bool safe = false) const override {
- return new IntersectFst<Arc>(*this, safe);
+ IntersectFst *Copy(bool safe = false) const override {
+ return new IntersectFst(*this, safe);
}
private:
#ifndef FST_INVERT_H_
#define FST_INVERT_H_
+#include <fst/types.h>
+
#include <fst/arc-map.h>
#include <fst/mutable-fst.h>
}
// See Fst<>::Copy() for doc.
- InvertFst(const InvertFst<Arc> &fst, bool safe = false)
+ InvertFst(const InvertFst &fst, bool safe = false)
: ArcMapFst<Arc, Arc, Mapper>(fst, safe) {}
// Get a copy of this InvertFst. See Fst<>::Copy() for further doc.
- InvertFst<Arc> *Copy(bool safe = false) const override {
+ InvertFst *Copy(bool safe = false) const override {
return new InvertFst(*this, safe);
}
#include <utility>
#include <vector>
+#include <fst/types.h>
#include <fst/log.h>
#include <fst/accumulator.h>
Label FinalLabel() const { return final_label_; }
- static LabelReachableData<Label> *Read(std::istream &istrm,
- const FstReadOptions &opts) {
- auto *data = new LabelReachableData<Label>();
+ static LabelReachableData *Read(std::istream &istrm,
+ const FstReadOptions &opts) {
+ auto *data = new LabelReachableData();
ReadType(istrm, &data->reach_input_);
ReadType(istrm, &data->keep_relabel_data_);
data->have_relabel_data_ = data->keep_relabel_data_;
#include <cstdlib>
#include <string>
+#include <fst/types.h>
#include <fst/log.h>
#include <fst/pair-weight.h>
#include <vector>
+#include <fst/types.h>
#include <fst/log.h>
#include <fst/filter-state.h>
#include <vector>
#include <fst/flags.h>
+#include <fst/types.h>
#include <fst/log.h>
#include <fst/add-on.h>
//
// // If safe = true, the copy is thread-safe (except the lookahead FST is
// // preserved). See Fst<>::Copy() for further doc.
-// LookaheadMatcher<FST> *Copy(bool safe = false) const override;
+// LookaheadMatcher *Copy(bool safe = false) const override;
// // Below are methods for looking ahead for a match to a label and more
// // generally, to a rational set. Each returns false if there is definitely
: matcher_(fst, match_type) {}
// This makes a copy of the FST.
- TrivialLookAheadMatcher(const TrivialLookAheadMatcher<M> &lmatcher,
+ TrivialLookAheadMatcher(const TrivialLookAheadMatcher &lmatcher,
bool safe = false)
: matcher_(lmatcher.matcher_, safe) {}
- TrivialLookAheadMatcher<M> *Copy(bool safe = false) const override {
- return new TrivialLookAheadMatcher<M>(*this, safe);
+ TrivialLookAheadMatcher *Copy(bool safe = false) const override {
+ return new TrivialLookAheadMatcher(*this, safe);
}
MatchType Type(bool test) const override { return matcher_.Type(test); }
// This makes a copy of the FST.
ArcLookAheadMatcher(
const FST &fst, MatchType match_type,
- std::shared_ptr<MatcherData> data = std::shared_ptr<MatcherData>())
+ std::shared_ptr<MatcherData> data = nullptr)
: matcher_(fst, match_type),
fst_(matcher_.GetFst()),
lfst_(nullptr),
// This doesn't copy the FST.
ArcLookAheadMatcher(
const FST *fst, MatchType match_type,
- std::shared_ptr<MatcherData> data = std::shared_ptr<MatcherData>())
+ std::shared_ptr<MatcherData> data = nullptr)
: matcher_(fst, match_type),
fst_(matcher_.GetFst()),
lfst_(nullptr),
state_(kNoStateId) {}
// This makes a copy of the FST.
- ArcLookAheadMatcher(const ArcLookAheadMatcher<M, flags> &lmatcher,
- bool safe = false)
+ ArcLookAheadMatcher(const ArcLookAheadMatcher &lmatcher, bool safe = false)
: matcher_(lmatcher.matcher_, safe),
fst_(matcher_.GetFst()),
lfst_(lmatcher.lfst_),
state_(kNoStateId) {}
// General matcher methods.
- ArcLookAheadMatcher<M, flags> *Copy(bool safe = false) const override {
- return new ArcLookAheadMatcher<M, flags>(*this, safe);
+ ArcLookAheadMatcher *Copy(bool safe = false) const override {
+ return new ArcLookAheadMatcher(*this, safe);
}
MatchType Type(bool test) const override { return matcher_.Type(test); }
const MatcherData *GetData() const { return nullptr; }
- std::shared_ptr<MatcherData> GetSharedData() const {
- return std::shared_ptr<MatcherData>();
- }
+ std::shared_ptr<MatcherData> GetSharedData() const { return nullptr; }
// Look-ahead methods.
// This makes a copy of the FST.
LabelLookAheadMatcher(
const FST &fst, MatchType match_type,
- std::shared_ptr<MatcherData> data = std::shared_ptr<MatcherData>(),
+ std::shared_ptr<MatcherData> data = nullptr,
Accumulator *accumulator = nullptr)
: matcher_(fst, match_type),
lfst_(nullptr),
// This doesn't copy the FST.
LabelLookAheadMatcher(
const FST *fst, MatchType match_type,
- std::shared_ptr<MatcherData> data = std::shared_ptr<MatcherData>(),
+ std::shared_ptr<MatcherData> data = nullptr,
Accumulator *accumulator = nullptr)
: matcher_(fst, match_type),
lfst_(nullptr),
}
// This makes a copy of the FST.
- LabelLookAheadMatcher(
- const LabelLookAheadMatcher<M, flags, Accumulator, Reachable> &lmatcher,
- bool safe = false)
+ LabelLookAheadMatcher(const LabelLookAheadMatcher &lmatcher,
+ bool safe = false)
: matcher_(lmatcher.matcher_, safe),
lfst_(lmatcher.lfst_),
label_reachable_(lmatcher.label_reachable_
state_(kNoStateId),
error_(lmatcher.error_) {}
- LabelLookAheadMatcher<M, flags, Accumulator, Reachable> *Copy(
- bool safe = false) const override {
- return new LabelLookAheadMatcher<M, flags, Accumulator, Reachable>(*this,
- safe);
+ LabelLookAheadMatcher *Copy(bool safe = false) const override {
+ return new LabelLookAheadMatcher(*this, safe);
}
MatchType Type(bool test) const override { return matcher_.Type(test); }
std::shared_ptr<MatcherData> GetSharedData() const {
return label_reachable_ ? label_reachable_->GetSharedData()
- : std::shared_ptr<MatcherData>();
+ : nullptr;
}
// Checks if there is a matching (possibly super-final) transition at
// (state_, s).
}
// This makes a copy of the FST.
- LookAheadMatcher(const LookAheadMatcher<FST> &matcher, bool safe = false)
- : base_(matcher.base_->Copy(safe)),
- lookahead_(matcher.lookahead_) { }
+ LookAheadMatcher(const LookAheadMatcher &matcher, bool safe = false)
+ : base_(matcher.base_->Copy(safe)), lookahead_(matcher.lookahead_) {}
// Takes ownership of base.
explicit LookAheadMatcher(MatcherBase<Arc> *base)
: base_(base), lookahead_(false) {}
- LookAheadMatcher<FST> *Copy(bool safe = false) const {
- return new LookAheadMatcher<FST>(*this, safe);
+ LookAheadMatcher *Copy(bool safe = false) const {
+ return new LookAheadMatcher(*this, safe);
}
MatchType Type(bool test) const { return base_->Type(test); }
MapFst(const Fst<A> &fst, C *mapper) : ArcMapFst<A, B, C>(fst, mapper) {}
// See Fst<>::Copy() for doc.
- MapFst(const MapFst<A, B, C> &fst, bool safe = false)
+ MapFst(const MapFst &fst, bool safe = false)
: ArcMapFst<A, B, C>(fst, safe) {}
// Get a copy of this MapFst. See Fst<>::Copy() for further doc.
- MapFst<A, B, C> *Copy(bool safe = false) const override {
+ MapFst *Copy(bool safe = false) const override {
return new MapFst(*this, safe);
}
};
// factory function does not backoff to allocating and reading.
static MappedFile *MapFromFileDescriptor(int fd, size_t pos, size_t size);
- // Creates a MappedFile object with a new[]'ed block of memory of size. The
+ // Creates a MappedFile object with a new'ed block of memory of size. The
// align argument can be used to specify a desired block alignment.
// This is RECOMMENDED FOR INTERNAL USE ONLY as it may change in future
// releases.
static MappedFile *Allocate(size_t size, size_t align = kArchAlignment);
+ // Creates a MappedFile object with a new'ed block of memory with enough
+ // space for count elements of type T, correctly aligned for the type.
+ // This is RECOMMENDED FOR INTERNAL USE ONLY as it may change in future
+ // releases.
+ template <typename T>
+ static MappedFile *AllocateType(size_t count) {
+ return Allocate(sizeof(T) * count, alignof(T));
+ }
+
// Creates a MappedFile object pointing to a borrowed reference to data. This
// block of memory is not owned by the MappedFile object and will not be
// freed. This is RECOMMENDED FOR INTERNAL USE ONLY, may change in future
#include <memory>
#include <string>
+#include <fst/types.h>
+
#include <fst/add-on.h>
#include <fst/const-fst.h>
#include <fst/lookahead-matcher.h>
: ImplToExpandedFst<Impl>(CreateDataAndImpl(fst, Name)) {}
// See Fst<>::Copy() for doc.
- MatcherFst(const MatcherFst<FST, FstMatcher, Name, Init, Data> &fst,
- bool safe = false)
+ MatcherFst(const MatcherFst &fst, bool safe = false)
: ImplToExpandedFst<Impl>(fst, safe) {}
// Get a copy of this MatcherFst. See Fst<>::Copy() for further doc.
- MatcherFst<FST, FstMatcher, Name, Init, Data> *Copy(
- bool safe = false) const override {
- return new MatcherFst<FST, FstMatcher, Name, Init, Data>(*this, safe);
+ MatcherFst *Copy(bool safe = false) const override {
+ return new MatcherFst(*this, safe);
}
// Read a MatcherFst from an input stream; return nullptr on error
- static MatcherFst<FST, M, Name, Init, Data> *Read(
- std::istream &strm, const FstReadOptions &opts) {
+ static MatcherFst *Read(std::istream &strm, const FstReadOptions &opts) {
auto *impl = Impl::Read(strm, opts);
- return impl ? new MatcherFst<FST, FstMatcher, Name, Init, Data>(
- std::shared_ptr<Impl>(impl))
- : nullptr;
+ return impl ? new MatcherFst(std::shared_ptr<Impl>(impl)) : nullptr;
}
// Read a MatcherFst from a file; return nullptr on error
// Empty source reads from standard input
- static MatcherFst<FST, FstMatcher, Name, Init, Data> *Read(
- const std::string &source) {
+ static MatcherFst *Read(const std::string &source) {
auto *impl = ImplToExpandedFst<Impl>::Read(source);
- return impl ? new MatcherFst<FST, FstMatcher, Name, Init, Data>(
- std::shared_ptr<Impl>(impl))
- : nullptr;
+ return impl ? new MatcherFst(std::shared_ptr<Impl>(impl)) : nullptr;
}
bool Write(std::ostream &strm, const FstWriteOptions &opts) const override {
Matcher(const FST &fst, MatchType match_type)
: matcher_(fst.InitMatcher(match_type)) {}
- Matcher(const Matcher<FST> &matcher) : matcher_(matcher.matcher_->Copy()) {}
+ Matcher(const Matcher &matcher) : matcher_(matcher.matcher_->Copy()) {}
- Matcher<FST> *Copy() const { return new Matcher<FST>(*this); }
+ Matcher *Copy() const { return new Matcher(*this); }
MatchType Type(bool test) const { return matcher_->Type(test); }
LookAheadMatcher(const FST &fst, MatchType match_type)
: matcher_(fst.InitMatcher(match_type)) {}
- LookAheadMatcher(const LookAheadMatcher<FST> &matcher, bool safe = false)
+ LookAheadMatcher(const LookAheadMatcher &matcher, bool safe = false)
: matcher_(matcher.matcher_->Copy(safe)) {}
// General matcher methods.
- LookAheadMatcher<FST> *Copy(bool safe = false) const {
- return new LookAheadMatcher<FST>(*this, safe);
+ LookAheadMatcher *Copy(bool safe = false) const {
+ return new LookAheadMatcher(*this, safe);
}
MatchType Type(bool test) const { return matcher_->Type(test); }
#include <unordered_map>
#include <utility>
+#include <fst/types.h>
#include <fst/log.h>
#include <fst/mutable-fst.h> // for all internal FST accessors.
// This makes a copy of the FST.
- PhiMatcher(const PhiMatcher<M> &matcher, bool safe = false)
+ PhiMatcher(const PhiMatcher &matcher, bool safe = false)
: matcher_(new M(*matcher.matcher_, safe)),
match_type_(matcher.match_type_),
phi_label_(matcher.phi_label_),
phi_loop_(matcher.phi_loop_),
error_(matcher.error_) {}
- PhiMatcher<M> *Copy(bool safe = false) const override {
- return new PhiMatcher<M>(*this, safe);
+ PhiMatcher *Copy(bool safe = false) const override {
+ return new PhiMatcher(*this, safe);
}
MatchType Type(bool test) const override { return matcher_->Type(test); }
matcher ? matcher : new M(fst, match_type)) { }
// This makes a copy of the FST.
- RhoMatcher(const RhoMatcher<M> &matcher, bool safe = false)
+ RhoMatcher(const RhoMatcher &matcher, bool safe = false)
: matcher_(new M(*matcher.matcher_, safe)),
match_type_(matcher.match_type_),
rho_label_(matcher.rho_label_),
state_(kNoStateId),
has_rho_(false) {}
- RhoMatcher<M> *Copy(bool safe = false) const override {
- return new RhoMatcher<M>(*this, safe);
+ RhoMatcher *Copy(bool safe = false) const override {
+ return new RhoMatcher(*this, safe);
}
MatchType Type(bool test) const override { return matcher_->Type(test); }
matcher ? matcher : new M(fst, match_type)) { }
// This makes a copy of the FST.
- SigmaMatcher(const SigmaMatcher<M> &matcher, bool safe = false)
+ SigmaMatcher(const SigmaMatcher &matcher, bool safe = false)
: matcher_(new M(*matcher.matcher_, safe)),
match_type_(matcher.match_type_),
sigma_label_(matcher.sigma_label_),
error_(matcher.error_),
state_(kNoStateId) {}
- SigmaMatcher<M> *Copy(bool safe = false) const override {
- return new SigmaMatcher<M>(*this, safe);
+ SigmaMatcher *Copy(bool safe = false) const override {
+ return new SigmaMatcher(*this, safe);
}
MatchType Type(bool test) const override { return matcher_->Type(test); }
}
// This makes a copy of the FST.
- MultiEpsMatcher(const MultiEpsMatcher<M> &matcher, bool safe = false)
+ MultiEpsMatcher(const MultiEpsMatcher &matcher, bool safe = false)
: matcher_(new M(*matcher.matcher_, safe)),
flags_(matcher.flags_),
own_matcher_(true),
if (own_matcher_) delete matcher_;
}
- MultiEpsMatcher<M> *Copy(bool safe = false) const {
- return new MultiEpsMatcher<M>(*this, safe);
+ MultiEpsMatcher *Copy(bool safe = false) const {
+ return new MultiEpsMatcher(*this, safe);
}
MatchType Type(bool test) const { return matcher_->Type(test); }
error_(false) {}
// This makes a copy of the FST.
- ExplicitMatcher(const ExplicitMatcher<M> &matcher, bool safe = false)
+ ExplicitMatcher(const ExplicitMatcher &matcher, bool safe = false)
: matcher_(new M(*matcher.matcher_, safe)),
match_type_(matcher.match_type_),
error_(matcher.error_) {}
- ExplicitMatcher<M> *Copy(bool safe = false) const override {
- return new ExplicitMatcher<M>(*this, safe);
+ ExplicitMatcher *Copy(bool safe = false) const override {
+ return new ExplicitMatcher(*this, safe);
}
MatchType Type(bool test) const override { return matcher_->Type(test); }
#include <utility>
#include <vector>
-#include <fst/types.h>
#include <fst/log.h>
#include <fstream>
MemoryArenaCollection *arenas_;
- BlockAllocator<T> operator=(const BlockAllocator<T> &);
+ BlockAllocator<T> operator=(const BlockAllocator &);
};
template <typename T, typename U>
MemoryPoolCollection *pools_;
- PoolAllocator<T> operator=(const PoolAllocator<T> &);
+ PoolAllocator<T> operator=(const PoolAllocator &);
};
template <typename T, typename U>
#include <utility>
#include <vector>
+#include <fst/types.h>
#include <fst/log.h>
#include <fst/arcsort.h>
size_t result = p2;
size_t current_ilabel = kNoLabel;
for (ArcIterator<Fst<Arc>> aiter(fst_, s); !aiter.Done(); aiter.Next()) {
- Label this_ilabel = aiter.Value().ilabel;
+ const Label this_ilabel = aiter.Value().ilabel;
if (this_ilabel != current_ilabel) { // Ignores repeats.
result = p1 * result + this_ilabel;
current_ilabel = this_ilabel;
void Initialize(const ExpandedFst<Arc> &fst) {
// Constructs Tr.
Reverse(fst, &Tr_);
- ILabelCompare<RevArc> ilabel_comp;
- ArcSort(&Tr_, ilabel_comp);
+ static const ILabelCompare<RevArc> icomp;
+ ArcSort(&Tr_, icomp);
// Tells the partition how many elements to allocate. The first state in
// Tr_ is super-final state.
P_.Initialize(Tr_.NumStates() - 1);
// Prepares priority queue: opens arc iterator for each state in C, and
// inserts into priority queue.
for (PartitionIterator<StateId> siter(P_, C); !siter.Done(); siter.Next()) {
- StateId s = siter.Value();
+ const auto s = siter.Value();
if (Tr_.NumArcs(s + 1)) {
aiter_queue_->push(new ArcIterator<Fst<RevArc>>(Tr_, s + 1));
}
}
template <class Arc>
-void AcceptorMinimize(MutableFst<Arc> *fst,
- bool allow_acyclic_minimization = true) {
- if (!(fst->Properties(kAcceptor | kUnweighted, true) ==
- (kAcceptor | kUnweighted))) {
- FSTERROR() << "FST is not an unweighted acceptor";
- fst->SetProperties(kError, kError);
- return;
- }
+void AcceptorMinimize(MutableFst<Arc> *fst) {
// Connects FST before minimization, handles disconnected states.
Connect(fst);
- if (fst->NumStates() == 0) return;
- if (allow_acyclic_minimization && fst->Properties(kAcyclic, true)) {
+ if (fst->Start() == kNoStateId) return;
+ // The Revuz acyclic algorithm won't work for nondeterministic inputs, so if
+ // the input is nondeterministic, we force the use of the Hopcroft cyclic
+ // algorithm instead.
+ static constexpr auto revuz_props = kAcyclic | kIDeterministic;
+ if (fst->Properties(revuz_props, true) == revuz_props) {
// Acyclic minimization (Revuz).
VLOG(2) << "Acyclic minimization";
- ArcSort(fst, ILabelCompare<Arc>());
+ static const ILabelCompare<Arc> comp;
+ ArcSort(fst, comp);
AcyclicMinimizer<Arc> minimizer(*fst);
MergeStates(minimizer.GetPartition(), fst);
} else {
ArcUniqueMapper<Arc> mapper(*fst);
StateMap(fst, mapper);
}
-
} // namespace internal
-// In place minimization of deterministic weighted automata and transducers,
-// and also non-deterministic ones if they use an idempotent semiring.
-// For transducers, if the 'sfst' argument is not null, the algorithm
-// produces a compact factorization of the minimal transducer.
+// In place minimization of deterministic weighted automata and transducers, and
+// also non-deterministic ones if they use an idempotent semiring. For
+// transducers, if the 'sfst' argument is not null, the algorithm produces a
+// compact factorization of the minimal transducer.
//
-// In the acyclic deterministic case, we use an algorithm from Revuz that is
-// linear in the number of arcs (edges) in the machine.
+// In the acyclic deterministic case, we use an algorithm from Revuz; this has
+// complexity O(e).
//
-// In the cyclic or non-deterministic case, we use the classical Hopcroft
+// In cyclic and non-deterministic cases, we use the classical Hopcroft
// minimization (which was presented for the deterministic case but which
// also works for non-deterministic FSTs); this has complexity O(e log v).
-//
template <class Arc>
void Minimize(MutableFst<Arc> *fst, MutableFst<Arc> *sfst = nullptr,
float delta = kShortestDelta, bool allow_nondet = false) {
using Weight = typename Arc::Weight;
- const auto props = fst->Properties(
- kAcceptor | kIDeterministic | kWeighted | kUnweighted, true);
- bool allow_acyclic_minimization;
- if (props & kIDeterministic) {
- allow_acyclic_minimization = true;
- } else {
+ static constexpr auto minimize_props =
+ kAcceptor | kIDeterministic | kWeighted | kUnweighted;
+ const auto props = fst->Properties(minimize_props, true);
+ if (!(props & kIDeterministic)) {
// Our approach to minimization of non-deterministic FSTs will only work in
// idempotent semirings---for non-deterministic inputs, a state could have
// multiple transitions to states that will get merged, and we'd have to
<< "allow_nondet = false";
return;
}
- // The Revuz algorithm won't work for nondeterministic inputs, so if the
- // input is nondeterministic, we'll have to pass a bool saying not to use
- // that algorithm. We check at this level rather than in AcceptorMinimize(),
- // because it's possible that the FST at this level could be deterministic,
- // but a harmless type of non-determinism could be introduced by Encode()
- // (thanks to kEncodeWeights, if the FST has epsilons and has a final
- // weight with weights equal to some epsilon arc.)
- allow_acyclic_minimization = false;
}
- if (!(props & kAcceptor)) { // Weighted transducer.
+ if ((props & kAcceptor) != kAcceptor) { // Transducer.
VectorFst<GallicArc<Arc, GALLIC_LEFT>> gfst;
ArcMap(*fst, &gfst, ToGallicMapper<Arc, GALLIC_LEFT>());
fst->DeleteStates();
gfst.SetProperties(kAcceptor, kAcceptor);
Push(&gfst, REWEIGHT_TO_INITIAL, delta);
ArcMap(&gfst, QuantizeMapper<GallicArc<Arc, GALLIC_LEFT>>(delta));
- EncodeMapper<GallicArc<Arc, GALLIC_LEFT>> encoder(
- kEncodeLabels | kEncodeWeights, ENCODE);
+ EncodeMapper<GallicArc<Arc, GALLIC_LEFT>> encoder(kEncodeLabels |
+ kEncodeWeights);
Encode(&gfst, &encoder);
- internal::AcceptorMinimize(&gfst, allow_acyclic_minimization);
+ internal::AcceptorMinimize(&gfst);
Decode(&gfst, encoder);
if (!sfst) {
FactorWeightFst<GallicArc<Arc, GALLIC_LEFT>,
ArcMap(gfst, fst, &mapper);
fst->SetOutputSymbols(sfst->InputSymbols());
}
- } else if (props & kWeighted) { // Weighted acceptor.
+ } else if ((props & kWeighted) == kWeighted) { // Weighted acceptor.
Push(fst, REWEIGHT_TO_INITIAL, delta);
ArcMap(fst, QuantizeMapper<Arc>(delta));
- EncodeMapper<Arc> encoder(kEncodeLabels | kEncodeWeights, ENCODE);
+ // We encode labels even though this is already an acceptor because weight
+ // encoding gives us a transducer.
+ EncodeMapper<Arc> encoder(kEncodeLabels | kEncodeWeights);
Encode(fst, &encoder);
- internal::AcceptorMinimize(fst, allow_acyclic_minimization);
+ internal::AcceptorMinimize(fst);
Decode(fst, encoder);
} else { // Unweighted acceptor.
- internal::AcceptorMinimize(fst, allow_acyclic_minimization);
+ internal::AcceptorMinimize(fst);
}
}
#include <utility>
#include <vector>
+#include <fst/types.h>
#include <fst/log.h>
#include <fstream>
virtual MutableFst<Arc> &operator=(const Fst<Arc> &fst) = 0;
- MutableFst<Arc> &operator=(const MutableFst<Arc> &fst) {
+ MutableFst &operator=(const MutableFst &fst) {
return operator=(static_cast<const Fst<Arc> &>(fst));
}
virtual void SetOutputSymbols(const SymbolTable *osyms) = 0;
// Gets a copy of this MutableFst. See Fst<>::Copy() for further doc.
- MutableFst<A> *Copy(bool safe = false) const override = 0;
+ MutableFst *Copy(bool safe = false) const override = 0;
// Reads a MutableFst from an input stream, returning nullptr on error.
- static MutableFst<Arc> *Read(std::istream &strm, const FstReadOptions &opts) {
+ static MutableFst *Read(std::istream &strm, const FstReadOptions &opts) {
FstReadOptions ropts(opts);
FstHeader hdr;
if (ropts.header) {
}
auto *fst = reader(strm, ropts);
if (!fst) return nullptr;
- return static_cast<MutableFst<Arc> *>(fst);
+ return static_cast<MutableFst *>(fst);
}
// Reads a MutableFst from a file; returns nullptr on error. An empty
// source results in reading from standard input. If convert is true,
// convert to a mutable FST subclass (given by convert_type) in the case
// that the input FST is non-mutable.
- static MutableFst<Arc> *Read(const std::string &source, bool convert = false,
- const std::string &convert_type = "vector") {
+ static MutableFst *Read(const std::string &source, bool convert = false,
+ const std::string &convert_type = "vector") {
if (convert == false) {
if (!source.empty()) {
std::ifstream strm(source,
std::unique_ptr<Fst<Arc>> ifst(Fst<Arc>::Read(source));
if (!ifst) return nullptr;
if (ifst->Properties(kMutable, false)) {
- return static_cast<MutableFst<Arc> *>(ifst.release());
+ return static_cast<MutableFst *>(ifst.release());
} else {
std::unique_ptr<Fst<Arc>> ofst(Convert(*ifst, convert_type));
ifst.reset();
if (!ofst->Properties(kMutable, false)) {
LOG(ERROR) << "MutableFst: Bad convert type: " << convert_type;
}
- return static_cast<MutableFst<Arc> *>(ofst.release());
+ return static_cast<MutableFst *>(ofst.release());
}
}
}
explicit ImplToMutableFst(std::shared_ptr<Impl> impl)
: ImplToExpandedFst<Impl, FST>(impl) {}
- ImplToMutableFst(const ImplToMutableFst<Impl, FST> &fst, bool safe)
+ ImplToMutableFst(const ImplToMutableFst &fst, bool safe)
: ImplToExpandedFst<Impl, FST>(fst, safe) {}
void MutateCheck() {
#include <string>
+#include <fst/types.h>
+
#include <fst/tuple-weight.h>
#include <fst/weight.h>
#include <string>
#include <utility>
+#include <fst/types.h>
+
#include <fst/pair-weight.h>
#include <fst/weight.h>
#ifndef FST_PROJECT_H_
#define FST_PROJECT_H_
+#include <fst/types.h>
+
#include <fst/arc-map.h>
#include <fst/mutable-fst.h>
}
// See Fst<>::Copy() for doc.
- ProjectFst(const ProjectFst<A> &fst, bool safe = false)
+ ProjectFst(const ProjectFst &fst, bool safe = false)
: ArcMapFst<A, A, ProjectMapper<A>>(fst, safe) {}
// Gets a copy of this ProjectFst. See Fst<>::Copy() for further doc.
- ProjectFst<A> *Copy(bool safe = false) const override {
+ ProjectFst *Copy(bool safe = false) const override {
return new ProjectFst(*this, safe);
}
#include <vector>
#include <fst/compat.h>
+#include <fst/types.h>
namespace fst {
#include <vector>
+#include <fst/types.h>
#include <fst/log.h>
#include <fst/arc-map.h>
#include <utility>
#include <vector>
+#include <fst/types.h>
#include <fst/log.h>
#include <fst/arcfilter.h>
#include <utility>
#include <vector>
+#include <fst/types.h>
#include <fst/log.h>
#include <fst/accumulator.h>
: ImplToFst<Impl>(std::make_shared<Impl>(fst, opts)) {}
// See Fst<>::Copy() for doc.
- RandGenFst(const RandGenFst<FromArc, ToArc, Sampler> &fst, bool safe = false)
+ RandGenFst(const RandGenFst &fst, bool safe = false)
: ImplToFst<Impl>(fst, safe) {}
// Get a copy of this RandGenFst. See Fst<>::Copy() for further doc.
- RandGenFst<FromArc, ToArc, Sampler> *Copy(bool safe = false) const override {
- return new RandGenFst<FromArc, ToArc, Sampler>(*this, safe);
+ RandGenFst *Copy(bool safe = false) const override {
+ return new RandGenFst(*this, safe);
}
inline void InitStateIterator(StateIteratorData<ToArc> *data) const override;
#include <string>
#include <vector>
+#include <fst/types.h>
+
#include <fst/mutable-fst.h>
#include <fst/replace.h>
#include <fst/test-properties.h>
: ImplToFst<Impl>(std::make_shared<Impl>(opts)) {}
// See Fst<>::Copy() for doc.
- RationalFst(const RationalFst<Arc> &fst, bool safe = false)
+ RationalFst(const RationalFst &fst, bool safe = false)
: ImplToFst<Impl>(fst, safe) {}
private:
#include <fst/util.h>
-#include <fst/types.h>
#include <fst/log.h>
namespace fst {
#include <utility>
#include <vector>
+#include <fst/types.h>
#include <fst/log.h>
#include <fst/cache.h>
opts)) {}
// See Fst<>::Copy() for doc.
- RelabelFst(const RelabelFst<Arc> &fst, bool safe = false)
+ RelabelFst(const RelabelFst &fst, bool safe = false)
: ImplToFst<Impl>(fst, safe) {}
// Gets a copy of this RelabelFst. See Fst<>::Copy() for further doc.
- RelabelFst<Arc> *Copy(bool safe = false) const override {
- return new RelabelFst<Arc>(*this, safe);
+ RelabelFst *Copy(bool safe = false) const override {
+ return new RelabelFst(*this, safe);
}
void InitStateIterator(StateIteratorData<Arc> *data) const override;
#include <utility>
#include <vector>
+#include <fst/types.h>
#include <fst/log.h>
#include <fst/connect.h>
#include <utility>
#include <vector>
+#include <fst/types.h>
#include <fst/log.h>
#include <fst/cache.h>
: ImplToFst<Impl>(std::make_shared<Impl>(fst_array, opts)) {}
// See Fst<>::Copy() for doc.
- ReplaceFst(const ReplaceFst<Arc, StateTable, CacheStore> &fst,
- bool safe = false)
+ ReplaceFst(const ReplaceFst &fst, bool safe = false)
: ImplToFst<Impl>(fst, safe) {}
// Get a copy of this ReplaceFst. See Fst<>::Copy() for further doc.
- ReplaceFst<Arc, StateTable, CacheStore> *Copy(
- bool safe = false) const override {
- return new ReplaceFst<Arc, StateTable, CacheStore>(*this, safe);
+ ReplaceFst *Copy(bool safe = false) const override {
+ return new ReplaceFst(*this, safe);
}
inline void InitStateIterator(StateIteratorData<Arc> *data) const override;
}
// This makes a copy of the FST.
- ReplaceFstMatcher(
- const ReplaceFstMatcher<Arc, StateTable, CacheStore> &matcher,
- bool safe = false)
+ ReplaceFstMatcher(const ReplaceFstMatcher &matcher, bool safe = false)
: owned_fst_(matcher.fst_.Copy(safe)),
fst_(*owned_fst_),
impl_(fst_.GetMutableImpl()),
}
}
- ReplaceFstMatcher<Arc, StateTable, CacheStore> *Copy(
- bool safe = false) const override {
- return new ReplaceFstMatcher<Arc, StateTable, CacheStore>(*this, safe);
+ ReplaceFstMatcher *Copy(bool safe = false) const override {
+ return new ReplaceFstMatcher(*this, safe);
}
MatchType Type(bool test) const override {
#include <algorithm>
#include <vector>
+#include <fst/types.h>
+
#include <fst/cache.h>
#include <utility>
#include <vector>
+#include <fst/types.h>
#include <fst/log.h>
#include <fst/arcfilter.h>
namespace fst {
template <class Arc, class Queue>
-class RmEpsilonOptions
+struct RmEpsilonOptions
: public ShortestDistanceOptions<Arc, Queue, EpsilonArcFilter<Arc>> {
- public:
using StateId = typename Arc::StateId;
using Weight = typename Arc::Weight;
std::vector<Arc> &Arcs() { return arcs_; }
- const Weight &Final() const { return final_; }
+ const Weight &Final() const { return final_weight_; }
bool Error() const { return sd_state_.Error(); }
std::vector<bool> visited_; // True if the state has been visited.
std::forward_list<StateId> visited_states_; // List of visited states.
std::vector<Arc> arcs_; // Arcs of state being expanded.
- Weight final_; // Final weight of state being expanded.
- StateId expand_id_; // Unique ID for each call to Expand
+ Weight final_weight_; // Final weight of state being expanded.
+ StateId expand_id_; // Unique ID for each call to Expand
RmEpsilonState(const RmEpsilonState &) = delete;
RmEpsilonState &operator=(const RmEpsilonState &) = delete;
template <class Arc, class Queue>
void RmEpsilonState<Arc, Queue>::Expand(typename Arc::StateId source) {
- final_ = Weight::Zero();
+ final_weight_ = Weight::Zero();
arcs_.clear();
sd_state_.ShortestDistance(source);
if (sd_state_.Error()) return;
}
}
}
- final_ = Plus(final_, Times((*distance_)[state], fst_.Final(state)));
+ final_weight_ =
+ Plus(final_weight_, Times((*distance_)[state], fst_.Final(state)));
}
while (!visited_states_.empty()) {
visited_[visited_states_.front()] = false;
: ImplToFst<Impl>(std::make_shared<Impl>(fst, opts)) {}
// See Fst<>::Copy() for doc.
- RmEpsilonFst(const RmEpsilonFst<Arc> &fst, bool safe = false)
+ RmEpsilonFst(const RmEpsilonFst &fst, bool safe = false)
: ImplToFst<Impl>(fst, safe) {}
// Get a copy of this RmEpsilonFst. See Fst<>::Copy() for further doc.
- RmEpsilonFst<Arc> *Copy(bool safe = false) const override {
- return new RmEpsilonFst<Arc>(*this, safe);
+ RmEpsilonFst *Copy(bool safe = false) const override {
+ return new RmEpsilonFst(*this, safe);
}
inline void InitStateIterator(StateIteratorData<Arc> *data) const override;
#include <unordered_set>
#include <vector>
+#include <fst/types.h>
+
#include <fst/connect.h>
#include <fst/mutable-fst.h>
#include <memory>
#include <utility>
+#include <fst/types.h>
#include <fst/fstlib.h>
#include <fst/script/fst-class.h>
void SetFlags(uint8 flags, uint8 mask) final { aiter_.SetFlags(flags, mask); }
- void SetValue(const Arc &arc) { aiter_.SetValue(arc); }
-
- void SetValue(const ArcClass &ac) final { aiter_.SetValue(ac.GetArc<Arc>()); }
+ void SetValue(const ArcClass &ac) final { SetValue(ac.GetArc<Arc>()); }
// This is returned by value because it has not yet been constructed, and
// is likely to participate in return-value optimization.
~MutableArcIteratorClassImpl() override {}
private:
+ void SetValue(const Arc &arc) { aiter_.SetValue(arc); }
+
MutableArcIterator<MutableFst<Arc>> aiter_;
};
#include <memory>
#include <string>
+#include <fst/types.h>
#include <fst/encode.h>
#include <fst/generic-register.h>
#include <fst/script/arc-class.h>
~EncodeMapperClassImpl() override {}
- EncodeMapper<Arc> *GetImpl() const { return &mapper_; }
+
const EncodeMapper<Arc> *GetImpl() const { return &mapper_; }
EncodeMapper<Arc> *GetImpl() { return &mapper_; }
#define REGISTER_ENCODEMAPPER_CLASS(Arc) \
static EncodeMapperClassIORegistration::Registerer \
- Class##_##Arc##_registerer( \
+ EncodeMapperClass_##Arc##_registerer( \
Arc::Type(), \
EncodeMapperClassIORegistration::Entry( \
EncodeMapperClass::Read<Arc>, EncodeMapperClass::Create<Arc>));
REGISTER_FST_OPERATION(Info, Arc, InfoArgs);
REGISTER_FST_OPERATION(Intersect, Arc, IntersectArgs);
REGISTER_FST_OPERATION(Invert, Arc, MutableFstClass);
+ REGISTER_FST_OPERATION(Isomorphic, Arc, IsomorphicArgs);
}
void RegisterBatch2() {
#include <string>
+#include <fst/types.h>
#include <fst/log.h>
#include <fst/compose.h> // For ComposeFilter.
#include <fst/determinize.h> // For DeterminizeType.
#include <string>
#include <vector>
+#include <fst/types.h>
#include <fst/connect.h>
#include <fst/dfs-visit.h>
#include <fst/fst.h>
std::string arc_type_;
};
-void InfoImpl(const FstInfo &fstinfo);
-
} // namespace fst
#endif // FST_SCRIPT_INFO_IMPL_H_
#include <tuple>
+#include <fst/types.h>
#include <fst/push.h>
#include <fst/script/fst-class.h>
#include <ctime>
#include <tuple>
+#include <fst/types.h>
#include <fst/randequivalent.h>
#include <fst/script/arg-packs.h>
#include <fst/script/fst-class.h>
#include <utility>
#include <vector>
+#include <fst/types.h>
#include <fst/relabel.h>
#include <fst/script/fst-class.h>
#include <utility>
#include <vector>
+#include <fst/types.h>
#include <fst/replace.h>
#include <fst/script/fst-class.h>
#include <utility>
#include <vector>
+#include <fst/types.h>
#include <fst/queue.h>
#include <fst/rmepsilon.h>
#include <fst/script/fst-class.h>
#include <string>
#include <utility>
+#include <fst/types.h>
#include <fst/log.h>
#include <fst/generic-register.h>
#include <fst/script/fst-class.h>
OperationSignature,
GenericOperationRegister<OperationSignature>> {
public:
- void RegisterOperation(const std::string &operation_name,
- const std::string &arc_type, OperationSignature op) {
- this->SetEntry(std::make_pair(operation_name, arc_type), op);
- }
-
OperationSignature GetOperation(const std::string &operation_name,
const std::string &arc_type) {
return this->GetEntry(std::make_pair(operation_name, arc_type));
const auto op = OpReg::Register::GetRegister()->GetOperation(op_name,
arc_type);
if (!op) {
- FSTERROR() << "No operation found for " << op_name << " on "
- << "arc type " << arc_type;
+ FSTERROR() << op_name << ": No operation found on arc type " << arc_type;
return;
}
op(args);
template <class M, class N>
bool ArcTypesMatch(const M &m, const N &n, const std::string &op_name) {
if (m.ArcType() != n.ArcType()) {
- FSTERROR() << "Arguments with non-matching arc types passed to "
- << op_name << ":\t" << m.ArcType() << " and " << n.ArcType();
+ FSTERROR() << op_name << ": Arguments with non-matching arc types "
+ << m.ArcType() << " and " << n.ArcType();
return false;
}
return true;
#include <memory>
#include <vector>
+#include <fst/types.h>
#include <fst/shortest-path.h>
#include <fst/script/fst-class.h>
#include <fst/script/shortest-distance.h>
// Registration for generic weight types.
-using StrToWeightImplBaseT = WeightImplBase *(*)(const std::string &str,
- const std::string &src,
- size_t nline);
+using StrToWeightImplBaseT = WeightImplBase *(*)(const std::string &str);
template <class W>
-WeightImplBase *StrToWeightImplBase(const std::string &str,
- const std::string &src, size_t nline) {
+WeightImplBase *StrToWeightImplBase(const std::string &str) {
if (str == WeightClass::__ZERO__) {
return new WeightClassImpl<W>(W::Zero());
} else if (str == WeightClass::__ONE__) {
} else if (str == WeightClass::__NOWEIGHT__) {
return new WeightClassImpl<W>(W::NoWeight());
}
- return new WeightClassImpl<W>(StrToWeight<W>(str, src, nline));
+ return new WeightClassImpl<W>(StrToWeight<W>(str));
}
class WeightClassRegister
#include <string>
#include <vector>
+#include <fst/types.h>
+
#include <fst/union-weight.h>
#include <fst/weight.h>
#include <cstdlib>
+#include <fst/types.h>
+
#include <fst/float-weight.h>
#include <fst/pair-weight.h>
#include <fst/product-weight.h>
#include <climits>
#include <string>
+#include <fst/types.h>
+
#include <fst/sparse-tuple-weight.h>
#include <fst/weight.h>
#include <string>
#include <utility>
+#include <fst/types.h>
#include <fst/log.h>
#include <fst/arc-map.h>
std::make_shared<Impl>(fst, mapper, StateMapFstOptions())) {}
// See Fst<>::Copy() for doc.
- StateMapFst(const StateMapFst<A, B, C> &fst, bool safe = false)
+ StateMapFst(const StateMapFst &fst, bool safe = false)
: ImplToFst<Impl>(fst, safe) {}
// Get a copy of this StateMapFst. See Fst<>::Copy() for further doc.
- StateMapFst<A, B, C> *Copy(bool safe = false) const override {
- return new StateMapFst<A, B, C>(*this, safe);
+ StateMapFst *Copy(bool safe = false) const override {
+ return new StateMapFst(*this, safe);
}
void InitStateIterator(StateIteratorData<B> *data) const override {
#include <utility>
#include <vector>
+#include <fst/types.h>
#include <fst/log.h>
#include <fst/bi-table.h>
#include <algorithm>
#include <vector>
+#include <fst/types.h>
#include <fst/log.h>
#include <fst/mutable-fst.h>
#include <string>
#include <vector>
+#include <fst/types.h>
+
#include <fst/product-weight.h>
#include <fst/union-weight.h>
#include <fst/weight.h>
#include <vector>
#include <fst/flags.h>
+#include <fst/types.h>
#include <fst/log.h>
#include <fst/compact-fst.h>
#include <functional>
#include <ios>
#include <iostream>
+#include <iterator>
#include <memory>
#include <sstream>
#include <string>
#include <fst/compat.h>
#include <fst/flags.h>
+#include <fst/types.h>
#include <fst/log.h>
#include <fstream>
#include <map>
} // namespace internal
+class SymbolTableIterator;
+
// Symbol (string) to integer (and reverse) mapping.
//
// The SymbolTable implements the mappings of labels to strings and reverse.
class SymbolTable {
public:
using SymbolType = internal::SymbolTableImpl::SymbolType;
+ using iterator = SymbolTableIterator;
+ using const_iterator = iterator;
// Constructs symbol table with an optional name.
explicit SymbolTable(const std::string &name = "<unspecified>")
// Dumps a text representation of the symbol table.
bool WriteText(const std::string &source) const;
+ const_iterator begin() const;
+ const_iterator end() const;
+ const_iterator cbegin() const;
+ const_iterator cend() const;
+
protected:
explicit SymbolTable(std::shared_ptr<internal::SymbolTableImplBase> impl)
: impl_(impl) {}
std::shared_ptr<internal::SymbolTableImplBase> impl_;
};
+namespace internal {
+
+// TODO(wolfsonkin): Consider adding std::tie and structured bindings support.
+class SymbolTableIteratorItem {
+ public:
+ explicit SymbolTableIteratorItem(const SymbolTable &table, ssize_t pos)
+ : table_(table), key_(table.GetNthKey(pos)) {}
+
+ // Return the label of the current symbol.
+ int64 Label() const { return key_; }
+
+ // Return the string of the current symbol.
+ // TODO(wolfsonkin): Consider adding caching
+ std::string Symbol() const { return table_.Find(key_); }
+
+ private:
+ // Sets this item to the pos'th element in the symbol table
+ void SetPosition(ssize_t pos) { key_ = table_.GetNthKey(pos); }
+
+ friend class fst::SymbolTableIterator;
+ const SymbolTable &table_;
+ int64 key_;
+};
+
+} // namespace internal
+
// Iterator class for symbols in a symbol table.
class SymbolTableIterator {
public:
+ using iterator_category = std::input_iterator_tag;
+ using value_type = internal::SymbolTableIteratorItem;
+ using difference_type = std::ptrdiff_t;
+ using pointer = const value_type *const;
+ using reference = const value_type &;
+
explicit SymbolTableIterator(const SymbolTable &table)
- : table_(table),
- pos_(0),
- nsymbols_(table.NumSymbols()),
- key_(table.GetNthKey(0)) {}
+ : SymbolTableIterator(table, 0) {}
~SymbolTableIterator() {}
bool Done() const { return (pos_ == nsymbols_); }
// Return the key of the current symbol.
- int64 Value() const { return key_; }
+ int64 Value() const { return iter_item_.Label(); }
// Return the string of the current symbol.
- std::string Symbol() const { return table_.Find(key_); }
+ std::string Symbol() const { return iter_item_.Symbol(); }
// Advances iterator.
void Next() {
++pos_;
- if (pos_ < nsymbols_) key_ = table_.GetNthKey(pos_);
+ if (pos_ < nsymbols_) iter_item_.SetPosition(pos_);
+ }
+
+ SymbolTableIterator &operator++() {
+ Next();
+ return *this;
+ }
+
+ SymbolTableIterator operator++(int) {
+ SymbolTableIterator retval = *this;
+ ++(*this);
+ return retval;
+ }
+
+ bool operator==(const SymbolTableIterator &other) const {
+ return (pos_ == other.pos_);
+ }
+
+ bool operator!=(const SymbolTableIterator &other) const {
+ return !(*this == other);
}
+ reference operator*() { return iter_item_; }
+
+ pointer operator->() const { return &iter_item_; }
+
// Resets iterator.
void Reset() {
pos_ = 0;
- key_ = table_.GetNthKey(0);
+ iter_item_.SetPosition(pos_);
}
private:
- const SymbolTable &table_;
+ explicit SymbolTableIterator(const SymbolTable &table, ssize_t pos)
+ : pos_(pos), nsymbols_(table.NumSymbols()), iter_item_(table, pos) {}
+ friend class SymbolTable;
+
ssize_t pos_;
size_t nsymbols_;
- int64 key_;
+ value_type iter_item_;
};
// Relabels a symbol table as specified by the input vector of pairs
#include <unordered_set>
#include <vector>
+#include <fst/types.h>
+
#include <fst/cache.h>
#include <fst/test-properties.h>
: ImplToFst<Impl>(std::make_shared<Impl>(fst, opts)) {}
// See Fst<>::Copy() for doc.
- SynchronizeFst(const SynchronizeFst<Arc> &fst, bool safe = false)
+ SynchronizeFst(const SynchronizeFst &fst, bool safe = false)
: ImplToFst<Impl>(fst, safe) {}
// Gets a copy of this SynchronizeFst. See Fst<>::Copy() for further doc.
- SynchronizeFst<Arc> *Copy(bool safe = false) const override {
- return new SynchronizeFst<Arc>(*this, safe);
+ SynchronizeFst *Copy(bool safe = false) const override {
+ return new SynchronizeFst(*this, safe);
}
inline void InitStateIterator(StateIteratorData<Arc> *data) const override;
#include <unordered_set>
#include <fst/flags.h>
+#include <fst/types.h>
#include <fst/log.h>
#include <fst/connect.h>
#ifndef FST_TEST_ALGO_TEST_H_
#define FST_TEST_ALGO_TEST_H_
+#include <fst/types.h>
#include <fst/log.h>
#include <fst/fstlib.h>
#include <fst/test/rand-fst.h>
#include <type_traits>
+#include <fst/types.h>
#include <fst/arc.h>
#include <fst/fst.h>
#include <fst/vector-fst.h>
#include <vector>
#include <fst/flags.h>
+#include <fst/types.h>
#include <fst/log.h>
#include <fst/weight.h>
#include <string>
#include <utility>
+#include <fst/types.h>
+
#include <fst/weight.h>
#include <utility>
#include <vector>
+#include <fst/types.h>
+
#include <fst/expanded-fst.h>
#include <fst/mutable-fst.h>
#include <fst/rational.h>
}
// See Fst<>::Copy() for doc.
- UnionFst(const UnionFst<Arc> &fst, bool safe = false)
+ UnionFst(const UnionFst &fst, bool safe = false)
: RationalFst<Arc>(fst, safe) {}
// Gets a copy of this UnionFst. See Fst<>::Copy() for further doc.
- UnionFst<Arc> *Copy(bool safe = false) const override {
- return new UnionFst<Arc>(*this, safe);
+ UnionFst *Copy(bool safe = false) const override {
+ return new UnionFst(*this, safe);
}
private:
#ifndef FST_UTIL_H_
#define FST_UTIL_H_
+#include <array>
#include <iostream>
#include <iterator>
#include <list>
inline std::istream &ReadType(std::istream &strm, std::string *s) { // NOLINT
s->clear();
int32 ns = 0;
- strm.read(reinterpret_cast<char *>(&ns), sizeof(ns));
+ ReadType(strm, &ns);
for (int32 i = 0; i < ns; ++i) {
char c;
strm.read(&c, 1);
}
} // namespace internal
+template <class T, size_t N>
+std::istream &ReadType(std::istream &strm, std::array<T, N> *c) {
+ for (auto &v : *c) ReadType(strm, &v);
+ return strm;
+}
+
template <class... T>
std::istream &ReadType(std::istream &strm, std::vector<T...> *c) {
return internal::ReadContainerType(
inline std::ostream &WriteType(std::ostream &strm, // NOLINT
const std::string &s) {
int32 ns = s.size();
- strm.write(reinterpret_cast<const char *>(&ns), sizeof(ns));
+ WriteType(strm, ns);
return strm.write(s.data(), ns);
}
namespace internal {
template <class C>
-std::ostream &WriteContainer(std::ostream &strm, const C &c) {
- const int64 n = c.size();
- WriteType(strm, n);
+std::ostream &WriteSequence(std::ostream &strm, const C &c) {
for (const auto &e : c) {
WriteType(strm, e);
}
return strm;
}
+template <class C>
+std::ostream &WriteContainer(std::ostream &strm, const C &c) {
+ const int64 n = c.size();
+ WriteType(strm, n);
+ WriteSequence(strm, c);
+ return strm;
+}
} // namespace internal
+template <class T, size_t N>
+std::ostream &WriteType(std::ostream &strm, const std::array<T, N> &c) {
+ return internal::WriteSequence(strm, c);
+}
+
template <typename... T>
std::ostream &WriteType(std::ostream &strm, const std::vector<T...> &c) {
return internal::WriteContainer(strm, c);
bool allow_negative, bool *error = nullptr);
template <typename Weight>
-Weight StrToWeight(const std::string &s, const std::string &source,
- size_t nline) {
+Weight StrToWeight(const std::string &s) {
Weight w;
std::istringstream strm(s);
strm >> w;
if (!strm) {
- FSTERROR() << "StrToWeight: Bad weight = \"" << s
- << "\", source = " << source << ", line = " << nline;
+ FSTERROR() << "StrToWeight: Bad weight: " << s;
return Weight::NoWeight();
}
return w;
#include <utility>
#include <vector>
+#include <fst/types.h>
#include <fst/log.h>
#include <fst/fst-decl.h> // For optional argument declarations
// Provide STL allocator for arcs.
explicit VectorState(const ArcAllocator &alloc)
- : final_(Weight::Zero()), niepsilons_(0), noepsilons_(0), arcs_(alloc) {}
+ : final_weight_(Weight::Zero()),
+ niepsilons_(0),
+ noepsilons_(0),
+ arcs_(alloc) {}
VectorState(const VectorState<A, M> &state, const ArcAllocator &alloc)
- : final_(state.Final()),
+ : final_weight_(state.Final()),
niepsilons_(state.NumInputEpsilons()),
noepsilons_(state.NumOutputEpsilons()),
arcs_(state.arcs_.begin(), state.arcs_.end(), alloc) {}
void Reset() {
- final_ = Weight::Zero();
+ final_weight_ = Weight::Zero();
niepsilons_ = 0;
noepsilons_ = 0;
arcs_.clear();
}
- Weight Final() const { return final_; }
+ Weight Final() const { return final_weight_; }
size_t NumInputEpsilons() const { return niepsilons_; }
void ReserveArcs(size_t n) { arcs_.reserve(n); }
- void SetFinal(Weight weight) { final_ = std::move(weight); }
+ void SetFinal(Weight weight) { final_weight_ = std::move(weight); }
void SetNumInputEpsilons(size_t n) { niepsilons_ = n; }
if (arc.olabel == 0) ++noepsilons_;
}
- Weight final_; // Final weight.
+ Weight final_weight_; // Final weight.
size_t niepsilons_; // # of input epsilons
size_t noepsilons_; // # of output epsilons
std::vector<A, ArcAllocator> arcs_; // Arc container.
// Copying is not permitted.
VectorFstBaseImpl(const VectorFstBaseImpl<S> &) = delete;
- VectorFstBaseImpl<S> &operator=(const VectorFstBaseImpl<S> &) = delete;
+ VectorFstBaseImpl &operator=(const VectorFstBaseImpl &) = delete;
// Moving is permitted.
- VectorFstBaseImpl(VectorFstBaseImpl<S> &&impl) noexcept
+ VectorFstBaseImpl(VectorFstBaseImpl &&impl) noexcept
: FstImpl<typename S::Arc>(),
states_(std::move(impl.states_)),
start_(impl.start_) {
impl.start_ = kNoStateId;
}
- VectorFstBaseImpl<S> &operator=(VectorFstBaseImpl<S> &&impl) noexcept {
+ VectorFstBaseImpl &operator=(VectorFstBaseImpl &&impl) noexcept {
for (auto *state : states_) {
State::Destroy(state, &state_alloc_);
}
explicit VectorFstImpl(const Fst<Arc> &fst);
- static VectorFstImpl<S> *Read(std::istream &strm, const FstReadOptions &opts);
+ static VectorFstImpl *Read(std::istream &strm, const FstReadOptions &opts);
void SetStart(StateId state) {
BaseImpl::SetStart(state);
template <class S>
VectorFstImpl<S> *VectorFstImpl<S>::Read(std::istream &strm,
const FstReadOptions &opts) {
- std::unique_ptr<VectorFstImpl<S>> impl(new VectorFstImpl());
+ std::unique_ptr<VectorFstImpl> impl(new VectorFstImpl());
FstHeader hdr;
if (!impl->ReadHeader(strm, opts, kMinFileVersion, &hdr)) return nullptr;
impl->BaseImpl::SetStart(hdr.Start());
explicit VectorFst(const Fst<Arc> &fst)
: ImplToMutableFst<Impl>(std::make_shared<Impl>(fst)) {}
- VectorFst(const VectorFst<Arc, State> &fst, bool safe = false)
+ VectorFst(const VectorFst &fst, bool safe = false)
: ImplToMutableFst<Impl>(fst) {}
- VectorFst(VectorFst<Arc, State> &&) noexcept;
+ VectorFst(VectorFst &&) noexcept;
// Get a copy of this VectorFst. See Fst<>::Copy() for further doc.
- VectorFst<Arc, State> *Copy(bool safe = false) const override {
- return new VectorFst<Arc, State>(*this, safe);
+ VectorFst *Copy(bool safe = false) const override {
+ return new VectorFst(*this, safe);
}
- VectorFst<Arc, State> &operator=(const VectorFst<Arc, State> &) = default;
+ VectorFst &operator=(const VectorFst &) = default;
- VectorFst<Arc, State> &operator=(VectorFst<Arc, State> &&) noexcept;
+ VectorFst &operator=(VectorFst &&) noexcept;
- VectorFst
<Arc, State> &operator=(const Fst<Arc> &fst) override {
+ VectorFst &operator=(const Fst<Arc> &fst) override {
if (this != &fst) SetImpl(std::make_shared<Impl>(fst));
return *this;
}
}
// Reads a VectorFst from an input stream, returning nullptr on error.
- static VectorFst<Arc, State> *Read(std::istream &strm,
- const FstReadOptions &opts) {
+ static VectorFst *Read(std::istream &strm, const FstReadOptions &opts) {
auto *impl = Impl::Read(strm, opts);
- return impl ? new VectorFst<Arc, State>(std::shared_ptr<Impl>(impl))
- : nullptr;
+ return impl ? new VectorFst(std::shared_ptr<Impl>(impl)) : nullptr;
}
// Read a VectorFst from a file, returning nullptr on error; empty source
// reads from standard input.
- static VectorFst<Arc, State> *Read(const std::string &source) {
+ static VectorFst *Read(const std::string &source) {
auto *impl = ImplToExpandedFst<Impl, MutableFst<Arc>>::Read(source);
- return impl ? new VectorFst<Arc, State>(std::shared_ptr<Impl>(impl))
- : nullptr;
+ return impl ? new VectorFst(std::shared_ptr<Impl>(impl)) : nullptr;
}
bool Write(std::ostream &strm, const FstWriteOptions &opts) const override {
};
template <class Arc, class State>
-inline VectorFst<Arc, State>::VectorFst(
- VectorFst<Arc, State> &&fst) noexcept = default;
+inline VectorFst<Arc, State>::VectorFst(VectorFst &&fst) noexcept = default;
template <class Arc, class State>
inline VectorFst<Arc, State> &VectorFst<Arc, State>::operator=(
- VectorFst<Arc, State> &&fst) noexcept = default;
+ VectorFst &&fst) noexcept = default;
// Writes FST to file in Vector format, potentially with a pass over the machine
// before writing to compute number of states.
#ifndef FST_VERIFY_H_
#define FST_VERIFY_H_
+#include <fst/types.h>
#include <fst/log.h>
#include <fst/fst.h>
#include <utility>
#include <fst/compat.h>
+#include <fst/types.h>
#include <fst/log.h>
#include <fst/util.h>
#include <sstream>
#include <fst/flags.h>
+#include <fst/types.h>
#include <fst/log.h>
#include <fst/matcher-fst.h> // declarations of *_lookahead_fst_type
// Use std::aligned_alloc() when C++17 is allowed.
char *buffer = static_cast<char *>(operator new(size + align));
uintptr_t address = reinterpret_cast<uintptr_t>(buffer);
- region.offset = kArchAlignment - (address % align);
+ region.offset = align - (address % align);
region.data = buffer + region.offset;
}
region.mmap = nullptr;
#include <string>
#include <vector>
+#include <fst/types.h>
+
namespace fst {
// These functions determine the properties associated with the FST result of
#include <fst/symbol-table.h>
#include <fst/flags.h>
+#include <fst/types.h>
#include <fst/log.h>
#include <fstream>
}
}
+SymbolTable::const_iterator SymbolTable::begin() const {
+ return SymbolTable::const_iterator(*this, 0);
+}
+
+SymbolTable::const_iterator SymbolTable::end() const {
+ return SymbolTable::const_iterator(*this, this->NumSymbols());
+}
+
+SymbolTable::const_iterator SymbolTable::cbegin() const { return begin(); }
+
+SymbolTable::const_iterator SymbolTable::cend() const { return end(); }
+
bool CompatSymbols(const SymbolTable *syms1, const SymbolTable *syms2,
bool warning) {
// Flag can explicitly override this check.
#include <string>
#include <fst/flags.h>
+#include <fst/types.h>
#include <fst/log.h>
#include <fst/mapped-file.h>
text-io.cc topsort.cc union.cc weight-class.cc verify.cc
libfstscript_la_LIBADD = ../lib/libfst.la -lm $(DL_LIBS)
-libfstscript_la_LDFLAGS = -version-info 17:0:0
+libfstscript_la_LDFLAGS = -version-info 18:0:0
endif
@HAVE_SCRIPT_TRUE@text-io.cc topsort.cc union.cc weight-class.cc verify.cc
@HAVE_SCRIPT_TRUE@libfstscript_la_LIBADD = ../lib/libfst.la -lm $(DL_LIBS)
-@HAVE_SCRIPT_TRUE@libfstscript_la_LDFLAGS = -version-info 17:0:0
+@HAVE_SCRIPT_TRUE@libfstscript_la_LDFLAGS = -version-info 18:0:0
all: all-am
.SUFFIXES:
bool FstClass::WeightTypesMatch(const WeightClass &weight,
const std::string &op_name) const {
if (WeightType() != weight.Type()) {
- FSTERROR() << "FST and weight with non-matching weight types passed to "
- << op_name << ": " << WeightType() << " and " << weight.Type();
+ FSTERROR() << op_name << ": FST and weight with non-matching weight types: "
+ << WeightType() << " and " << weight.Type();
return false;
}
return true;
WeightClass::WeightClass(const std::string &weight_type,
const std::string &weight_str) {
- WeightClassRegister *reg = WeightClassRegister::GetRegister();
- StrToWeightImplBaseT stw = reg->GetEntry(weight_type);
+ static const auto *reg = WeightClassRegister::GetRegister();
+ const auto stw = reg->GetEntry(weight_type);
if (!stw) {
- FSTERROR() << "Unknown weight type: " << weight_type;
+ FSTERROR() << "WeightClass: Unknown weight type: " << weight_type;
impl_.reset();
return;
}
- impl_.reset(stw(weight_str, "WeightClass", 0));
+ impl_.reset(stw(weight_str));
}
constexpr char WeightClass::__ZERO__[];
const WeightClass &rhs,
const std::string &op_name) {
if (lhs.Type() != rhs.Type()) {
- FSTERROR() << "Weights with non-matching types passed to " << op_name
- << ": " << lhs.Type() << " and " << rhs.Type();
+ FSTERROR() << op_name << ": Weights with non-matching types: " << lhs.Type()
+ << " and " << rhs.Type();
return false;
}
return true;
#include <fst/test/fst_test.h>
#include <fst/flags.h>
+#include <fst/types.h>
#include <fst/log.h>
#include <fst/compact-fst.h>
#include <fst/const-fst.h>
projects / platform / upstream / openfst.git / commitdiff