2 *******************************************************************************
3 * Copyright (C) 2013, International Business Machines
4 * Corporation and others. All Rights Reserved.
5 *******************************************************************************
8 * created on: 2012may31
9 * created by: Markus W. Scherer & Maxime Serrano
12 #ifndef __DICTIONARYDATA_H__
13 #define __DICTIONARYDATA_H__
15 #include "unicode/utypes.h"
17 #if !UCONFIG_NO_BREAK_ITERATION
19 #include "unicode/utext.h"
20 #include "unicode/udata.h"
22 #include "unicode/uobject.h"
23 #include "unicode/ustringtrie.h"
30 class U_COMMON_API DictionaryData : public UMemory {
32 static const int32_t TRIE_TYPE_BYTES; // = 0;
33 static const int32_t TRIE_TYPE_UCHARS; // = 1;
34 static const int32_t TRIE_TYPE_MASK; // = 7;
35 static const int32_t TRIE_HAS_VALUES; // = 8;
37 static const int32_t TRANSFORM_NONE; // = 0;
38 static const int32_t TRANSFORM_TYPE_OFFSET; // = 0x1000000;
39 static const int32_t TRANSFORM_TYPE_MASK; // = 0x7f000000;
40 static const int32_t TRANSFORM_OFFSET_MASK; // = 0x1fffff;
43 // Byte offsets from the start of the data, after the generic header.
44 IX_STRING_TRIE_OFFSET,
49 // Trie type: TRIE_HAS_VALUES | TRIE_TYPE_BYTES etc.
51 // Transform specification: TRANSFORM_TYPE_OFFSET | 0xe00 etc.
61 * Wrapper class around generic dictionaries, implementing matches().
62 * getType() should return a TRIE_TYPE_??? constant from DictionaryData.
64 * All implementations of this interface must be thread-safe if they are to be used inside of the
65 * dictionary-based break iteration code.
67 class U_COMMON_API DictionaryMatcher : public UMemory {
69 virtual ~DictionaryMatcher();
70 // this should emulate CompactTrieDictionary::matches()
71 virtual int32_t matches(UText *text, int32_t maxLength, int32_t *lengths, int32_t &count,
72 int32_t limit, int32_t *values = NULL) const = 0;
73 /** @return DictionaryData::TRIE_TYPE_XYZ */
74 virtual int32_t getType() const = 0;
77 // Implementation of the DictionaryMatcher interface for a UCharsTrie dictionary
78 class U_COMMON_API UCharsDictionaryMatcher : public DictionaryMatcher {
80 // constructs a new UCharsDictionaryMatcher.
81 // The UDataMemory * will be closed on this object's destruction.
82 UCharsDictionaryMatcher(const UChar *c, UDataMemory *f) : characters(c), file(f) { }
83 virtual ~UCharsDictionaryMatcher();
84 virtual int32_t matches(UText *text, int32_t maxLength, int32_t *lengths, int32_t &count,
85 int32_t limit, int32_t *values = NULL) const;
86 virtual int32_t getType() const;
88 const UChar *characters;
92 // Implementation of the DictionaryMatcher interface for a BytesTrie dictionary
93 class U_COMMON_API BytesDictionaryMatcher : public DictionaryMatcher {
95 // constructs a new BytesTrieDictionaryMatcher
96 // the transform constant should be the constant read from the file, not a masked version!
97 // the UDataMemory * fed in here will be closed on this object's destruction
98 BytesDictionaryMatcher(const char *c, int32_t t, UDataMemory *f)
99 : characters(c), transformConstant(t), file(f) { }
100 virtual ~BytesDictionaryMatcher();
101 virtual int32_t matches(UText *text, int32_t maxLength, int32_t *lengths, int32_t &count,
102 int32_t limit, int32_t *values = NULL) const;
103 virtual int32_t getType() const;
105 UChar32 transform(UChar32 c) const;
107 const char *characters;
108 int32_t transformConstant;
114 U_CAPI int32_t U_EXPORT2
115 udict_swap(const UDataSwapper *ds, const void *inData, int32_t length, void *outData, UErrorCode *pErrorCode);
118 * Format of dictionary .dict data files.
119 * Format version 1.0.
121 * A dictionary .dict data file contains a byte-serialized BytesTrie or
122 * a UChars-serialized UCharsTrie.
123 * Such files are used in dictionary-based break iteration (DBBI).
125 * For a BytesTrie, a transformation type is specified for
126 * transforming Unicode strings into byte sequences.
128 * A .dict file begins with a standard ICU data file header
129 * (DataHeader, see ucmndata.h and unicode/udata.h).
130 * The UDataInfo.dataVersion field is currently unused (set to 0.0.0.0).
132 * After the header, the file contains the following parts.
133 * Constants are defined in the DictionaryData class.
135 * For the data structure of BytesTrie & UCharsTrie see
136 * http://site.icu-project.org/design/struct/tries
137 * and the bytestrie.h and ucharstrie.h header files.
139 * int32_t indexes[indexesLength]; -- indexesLength=indexes[IX_STRING_TRIE_OFFSET]/4;
141 * The first four indexes are byte offsets in ascending order.
142 * Each byte offset marks the start of the next part in the data file,
143 * and the end of the previous one.
144 * When two consecutive byte offsets are the same, then the corresponding part is empty.
145 * Byte offsets are offsets from after the header,
146 * that is, from the beginning of the indexes[].
147 * Each part starts at an offset with proper alignment for its data.
148 * If necessary, the previous part may include padding bytes to achieve this alignment.
150 * trieType=indexes[IX_TRIE_TYPE] defines the trie type.
151 * transform=indexes[IX_TRANSFORM] defines the Unicode-to-bytes transformation.
152 * If the transformation type is TRANSFORM_TYPE_OFFSET,
153 * then the lower 21 bits contain the offset code point.
154 * Each code point c is mapped to byte b = (c - offset).
155 * Code points outside the range offset..(offset+0xff) cannot be mapped
156 * and do not occur in the dictionary.
158 * stringTrie; -- a serialized BytesTrie or UCharsTrie
160 * The dictionary maps strings to specific values (TRIE_HAS_VALUES bit set in trieType),
161 * or it maps all strings to 0 (TRIE_HAS_VALUES bit not set).
164 #endif /* !UCONFIG_NO_BREAK_ITERATION */
165 #endif /* __DICTIONARYDATA_H__ */