int add_index(int phrase_length, /* in */ ucs4_t phrase[], /* in */ phrase_token_t token);
int remove_index(int phrase_length, /* in */ ucs4_t phrase[], /* in */ phrase_token_t token);
+
+ /* mask out method */
+ bool mask_out(phrase_token_t mask, phrase_token_t value);
};
int remove_index(int phrase_length, /* in */ ucs4_t phrase[], /* in */ phrase_token_t token) {
return m_bitmap_table.remove_index(phrase_length, phrase, token);
}
+
+ /* mask out method */
+ bool mask_out(phrase_token_t mask, phrase_token_t value) {
+ return m_bitmap_table.mask_out(mask, value);
+ }
};
-static inline int reduce_tokens(PhraseTokens tokens,
+static inline int reduce_tokens(const PhraseTokens tokens,
GArray * tokenarray) {
int num = 0;
+ g_array_set_size(tokenarray, 0);
for (size_t i = 0; i < PHRASE_INDEX_LIBRARY_COUNT; ++i) {
GArray * array = tokens[i];
num += array->len;
- for (size_t j = 0; j < array->len; ++j) {
- phrase_token_t token = g_array_index(array, phrase_token_t, j);
- g_array_append_val(tokenarray, token);
- }
+ g_array_append_vals(tokenarray, array->data, array->len);
}
/* the following line will be removed in future after code are verified. */
}
/* for compatibility. */
-static inline int get_first_token(PhraseTokens tokens,
+static inline int get_first_token(const PhraseTokens tokens,
/* out */ phrase_token_t & token){
- int num = 0; token = null_token;
-
- for (size_t i = 0; i < PHRASE_INDEX_LIBRARY_COUNT; ++i) {
- GArray * array = tokens[i];
- if (NULL == array || 0 == array->len)
- continue;
+ token = null_token;
- num += array->len;
-
- if (null_token == token) {
- token = g_array_index(array, phrase_token_t, 0);
- }
- }
-
- /* the following line will be removed in future after code are verified. */
- assert(0 == num || 1 == num);
+ GArray * tokenarray = g_array_new(FALSE, FALSE, sizeof(phrase_token_t));
+ int num = reduce_tokens(tokens, tokenarray);
+ if (num)
+ token = g_array_index(tokenarray, phrase_token_t, 0);
+ g_array_free(tokenarray, TRUE);
return num;
}