4 // Copyright (C) 2002-2016, International Business Machines Corporation and others.
5 // All Rights Reserved.
7 // This file contains declarations for class RBBIRuleScanner
14 #include "unicode/utypes.h"
15 #include "unicode/uobject.h"
16 #include "unicode/rbbi.h"
17 #include "unicode/uniset.h"
18 #include "unicode/parseerr.h"
21 #include "unicode/symtable.h"// For UnicodeSet parsing, is the interface that
22 // looks up references to $variables within a set.
28 class RBBIRuleBuilder;
29 class RBBISymbolTable;
32 //--------------------------------------------------------------------------------
34 // class RBBIRuleScanner does the lowest level, character-at-a-time
35 // scanning of break iterator rules.
37 // The output of the scanner is parse trees for
38 // the rule expressions and a list of all Unicode Sets
41 //--------------------------------------------------------------------------------
43 class RBBIRuleScanner : public UMemory {
47 kStackSize = 100 // The size of the state stack for
48 }; // rules parsing. Corresponds roughly
49 // to the depth of parentheses nesting
50 // that is allowed in the rules.
55 RBBIRuleChar() : fChar(0), fEscaped(FALSE) {};
58 RBBIRuleScanner(RBBIRuleBuilder *rb);
61 virtual ~RBBIRuleScanner();
63 void nextChar(RBBIRuleChar &c); // Get the next char from the input stream.
64 // Return false if at end.
66 UBool push(const RBBIRuleChar &c); // Push (unget) one character.
67 // Only a single character may be pushed.
69 void parse(); // Parse the rules, generating two parse
70 // trees, one each for the forward and
72 // and a list of UnicodeSets encountered.
75 * Return a rules string without unnecessary
78 static UnicodeString stripRules(const UnicodeString &rules);
81 UBool doParseActions(int32_t a);
82 void error(UErrorCode e); // error reporting convenience function.
83 void fixOpStack(RBBINode::OpPrecedence p);
85 void findSetFor(const UnicodeString &s, RBBINode *node, UnicodeSet *setToAdopt = NULL);
89 void printNodeStack(const char *title);
91 RBBINode *pushNewNode(RBBINode::NodeType t);
95 RBBIRuleBuilder *fRB; // The rule builder that we are part of.
97 int32_t fScanIndex; // Index of current character being processed
98 // in the rule input string.
99 int32_t fNextIndex; // Index of the next character, which
100 // is the first character not yet scanned.
101 UBool fQuoteMode; // Scan is in a 'quoted region'
102 int32_t fLineNum; // Line number in input file.
103 int32_t fCharNum; // Char position within the line.
104 UChar32 fLastChar; // Previous char, needed to count CR-LF
105 // as a single line, not two.
107 RBBIRuleChar fC; // Current char for parse state machine
109 UnicodeString fVarName; // $variableName, valid when we've just
112 RBBIRuleTableEl **fStateTable; // State Transition Table for RBBI Rule
113 // parsing. index by p[state][char-class]
115 uint16_t fStack[kStackSize]; // State stack, holds state pushes
116 int32_t fStackPtr; // and pops as specified in the state
119 RBBINode *fNodeStack[kStackSize]; // Node stack, holds nodes created
120 // during the parse of a rule
121 int32_t fNodeStackPtr;
124 UBool fReverseRule; // True if the rule currently being scanned
125 // is a reverse direction rule (if it
126 // starts with a '!')
128 UBool fLookAheadRule; // True if the rule includes a '/'
129 // somewhere within it.
131 UBool fNoChainInRule; // True if the current rule starts with a '^'.
133 RBBISymbolTable *fSymbolTable; // symbol table, holds definitions of
134 // $variable symbols.
136 UHashtable *fSetTable; // UnicocodeSet hash table, holds indexes to
137 // the sets created while parsing rules.
138 // The key is the string used for creating
141 UnicodeSet fRuleSets[10]; // Unicode Sets that are needed during
142 // the scanning of RBBI rules. The
143 // indicies for these are assigned by the
144 // perl script that builds the state tables.
147 int32_t fRuleNum; // Counts each rule as it is scanned.
149 int32_t fOptionStart; // Input index of start of a !!option
150 // keyword, while being scanned.
152 UnicodeSet *gRuleSet_rule_char;
153 UnicodeSet *gRuleSet_white_space;
154 UnicodeSet *gRuleSet_name_char;
155 UnicodeSet *gRuleSet_name_start_char;
157 RBBIRuleScanner(const RBBIRuleScanner &other); // forbid copying of this class
158 RBBIRuleScanner &operator=(const RBBIRuleScanner &other); // forbid copying of this class