1 // Tencent is pleased to support the open source community by making RapidJSON available.
3 // Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved.
5 // Licensed under the MIT License (the "License"); you may not use this file except
6 // in compliance with the License. You may obtain a copy of the License at
8 // http://opensource.org/licenses/MIT
10 // Unless required by applicable law or agreed to in writing, software distributed
11 // under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
12 // CONDITIONS OF ANY KIND, either express or implied. See the License for the
13 // specific language governing permissions and limitations under the License.
15 #ifndef RAPIDJSON_INTERNAL_REGEX_H_
16 #define RAPIDJSON_INTERNAL_REGEX_H_
18 #include "../allocators.h"
19 #include "../stream.h"
24 RAPIDJSON_DIAG_OFF(padded)
25 RAPIDJSON_DIAG_OFF(switch-enum)
26 RAPIDJSON_DIAG_OFF(implicit-fallthrough)
27 #elif defined(_MSC_VER)
29 RAPIDJSON_DIAG_OFF(4512) // assignment operator could not be generated
34 RAPIDJSON_DIAG_OFF(effc++)
36 RAPIDJSON_DIAG_OFF(implicit-fallthrough)
40 #ifndef RAPIDJSON_REGEX_VERBOSE
41 #define RAPIDJSON_REGEX_VERBOSE 0
44 RAPIDJSON_NAMESPACE_BEGIN
47 ///////////////////////////////////////////////////////////////////////////////
50 template <typename SourceStream, typename Encoding>
53 DecodedStream(SourceStream& ss) : ss_(ss), codepoint_() { Decode(); }
54 unsigned Peek() { return codepoint_; }
56 unsigned c = codepoint_;
57 if (c) // No further decoding when '\0'
64 if (!Encoding::Decode(ss_, &codepoint_))
72 ///////////////////////////////////////////////////////////////////////////////
75 static const SizeType kRegexInvalidState = ~SizeType(0); //!< Represents an invalid index in GenericRegex::State::out, out1
76 static const SizeType kRegexInvalidRange = ~SizeType(0);
78 template <typename Encoding, typename Allocator>
79 class GenericRegexSearch;
81 //! Regular expression engine with subset of ECMAscript grammar.
83 Supported regular expression syntax:
89 - \c a{3} Exactly 3 times
90 - \c a{3,} At least 3 times
91 - \c a{3,5} 3 to 5 times
93 - \c ^a At the beginning
96 - \c [abc] Character classes
97 - \c [a-c] Character class range
98 - \c [a-z0-9_] Character class combination
99 - \c [^abc] Negated character classes
100 - \c [^a-c] Negated character class range
101 - \c [\b] Backspace (U+0008)
102 - \c \\| \\\\ ... Escape characters
103 - \c \\f Form feed (U+000C)
104 - \c \\n Line feed (U+000A)
105 - \c \\r Carriage return (U+000D)
106 - \c \\t Tab (U+0009)
107 - \c \\v Vertical tab (U+000B)
109 \note This is a Thompson NFA engine, implemented with reference to
110 Cox, Russ. "Regular Expression Matching Can Be Simple And Fast (but is slow in Java, Perl, PHP, Python, Ruby,...).",
111 https://swtch.com/~rsc/regexp/regexp1.html
113 template <typename Encoding, typename Allocator = CrtAllocator>
116 typedef Encoding EncodingType;
117 typedef typename Encoding::Ch Ch;
118 template <typename, typename> friend class GenericRegexSearch;
120 GenericRegex(const Ch* source, Allocator* allocator = 0) :
121 ownAllocator_(allocator ? 0 : RAPIDJSON_NEW(Allocator)()), allocator_(allocator ? allocator : ownAllocator_),
122 states_(allocator_, 256), ranges_(allocator_, 256), root_(kRegexInvalidState), stateCount_(), rangeCount_(),
123 anchorBegin_(), anchorEnd_()
125 GenericStringStream<Encoding> ss(source);
126 DecodedStream<GenericStringStream<Encoding>, Encoding> ds(ss);
132 RAPIDJSON_DELETE(ownAllocator_);
135 bool IsValid() const {
136 return root_ != kRegexInvalidState;
149 static const unsigned kAnyCharacterClass = 0xFFFFFFFF; //!< For '.'
150 static const unsigned kRangeCharacterClass = 0xFFFFFFFE;
151 static const unsigned kRangeNegationFlag = 0x80000000;
160 SizeType out; //!< Equals to kInvalid for matching state
161 SizeType out1; //!< Equals to non-kInvalid for split
167 Frag(SizeType s, SizeType o, SizeType m) : start(s), out(o), minIndex(m) {}
169 SizeType out; //!< link-list of all output states
173 State& GetState(SizeType index) {
174 RAPIDJSON_ASSERT(index < stateCount_);
175 return states_.template Bottom<State>()[index];
178 const State& GetState(SizeType index) const {
179 RAPIDJSON_ASSERT(index < stateCount_);
180 return states_.template Bottom<State>()[index];
183 Range& GetRange(SizeType index) {
184 RAPIDJSON_ASSERT(index < rangeCount_);
185 return ranges_.template Bottom<Range>()[index];
188 const Range& GetRange(SizeType index) const {
189 RAPIDJSON_ASSERT(index < rangeCount_);
190 return ranges_.template Bottom<Range>()[index];
193 template <typename InputStream>
194 void Parse(DecodedStream<InputStream, Encoding>& ds) {
195 Stack<Allocator> operandStack(allocator_, 256); // Frag
196 Stack<Allocator> operatorStack(allocator_, 256); // Operator
197 Stack<Allocator> atomCountStack(allocator_, 256); // unsigned (Atom per parenthesis)
199 *atomCountStack.template Push<unsigned>() = 0;
202 while (ds.Peek() != 0) {
203 switch (codepoint = ds.Take()) {
213 while (!operatorStack.Empty() && *operatorStack.template Top<Operator>() < kAlternation)
214 if (!Eval(operandStack, *operatorStack.template Pop<Operator>(1)))
216 *operatorStack.template Push<Operator>() = kAlternation;
217 *atomCountStack.template Top<unsigned>() = 0;
221 *operatorStack.template Push<Operator>() = kLeftParenthesis;
222 *atomCountStack.template Push<unsigned>() = 0;
226 while (!operatorStack.Empty() && *operatorStack.template Top<Operator>() != kLeftParenthesis)
227 if (!Eval(operandStack, *operatorStack.template Pop<Operator>(1)))
229 if (operatorStack.Empty())
231 operatorStack.template Pop<Operator>(1);
232 atomCountStack.template Pop<unsigned>(1);
233 ImplicitConcatenation(atomCountStack, operatorStack);
237 if (!Eval(operandStack, kZeroOrOne))
242 if (!Eval(operandStack, kZeroOrMore))
247 if (!Eval(operandStack, kOneOrMore))
254 if (!ParseUnsigned(ds, &n))
257 if (ds.Peek() == ',') {
259 if (ds.Peek() == '}')
260 m = kInfinityQuantifier;
261 else if (!ParseUnsigned(ds, &m) || m < n)
267 if (!EvalQuantifier(operandStack, n, m) || ds.Peek() != '}')
274 PushOperand(operandStack, kAnyCharacterClass);
275 ImplicitConcatenation(atomCountStack, operatorStack);
281 if (!ParseRange(ds, &range))
283 SizeType s = NewState(kRegexInvalidState, kRegexInvalidState, kRangeCharacterClass);
284 GetState(s).rangeStart = range;
285 *operandStack.template Push<Frag>() = Frag(s, s, s);
287 ImplicitConcatenation(atomCountStack, operatorStack);
290 case '\\': // Escape character
291 if (!CharacterEscape(ds, &codepoint))
292 return; // Unsupported escape character
293 // fall through to default
295 default: // Pattern character
296 PushOperand(operandStack, codepoint);
297 ImplicitConcatenation(atomCountStack, operatorStack);
301 while (!operatorStack.Empty())
302 if (!Eval(operandStack, *operatorStack.template Pop<Operator>(1)))
305 // Link the operand to matching state.
306 if (operandStack.GetSize() == sizeof(Frag)) {
307 Frag* e = operandStack.template Pop<Frag>(1);
308 Patch(e->out, NewState(kRegexInvalidState, kRegexInvalidState, 0));
311 #if RAPIDJSON_REGEX_VERBOSE
312 printf("root: %d\n", root_);
313 for (SizeType i = 0; i < stateCount_ ; i++) {
314 State& s = GetState(i);
315 printf("[%2d] out: %2d out1: %2d c: '%c'\n", i, s.out, s.out1, (char)s.codepoint);
322 SizeType NewState(SizeType out, SizeType out1, unsigned codepoint) {
323 State* s = states_.template Push<State>();
326 s->codepoint = codepoint;
327 s->rangeStart = kRegexInvalidRange;
328 return stateCount_++;
331 void PushOperand(Stack<Allocator>& operandStack, unsigned codepoint) {
332 SizeType s = NewState(kRegexInvalidState, kRegexInvalidState, codepoint);
333 *operandStack.template Push<Frag>() = Frag(s, s, s);
336 void ImplicitConcatenation(Stack<Allocator>& atomCountStack, Stack<Allocator>& operatorStack) {
337 if (*atomCountStack.template Top<unsigned>())
338 *operatorStack.template Push<Operator>() = kConcatenation;
339 (*atomCountStack.template Top<unsigned>())++;
342 SizeType Append(SizeType l1, SizeType l2) {
344 while (GetState(l1).out != kRegexInvalidState)
345 l1 = GetState(l1).out;
346 GetState(l1).out = l2;
350 void Patch(SizeType l, SizeType s) {
351 for (SizeType next; l != kRegexInvalidState; l = next) {
352 next = GetState(l).out;
357 bool Eval(Stack<Allocator>& operandStack, Operator op) {
360 RAPIDJSON_ASSERT(operandStack.GetSize() >= sizeof(Frag) * 2);
362 Frag e2 = *operandStack.template Pop<Frag>(1);
363 Frag e1 = *operandStack.template Pop<Frag>(1);
364 Patch(e1.out, e2.start);
365 *operandStack.template Push<Frag>() = Frag(e1.start, e2.out, Min(e1.minIndex, e2.minIndex));
370 if (operandStack.GetSize() >= sizeof(Frag) * 2) {
371 Frag e2 = *operandStack.template Pop<Frag>(1);
372 Frag e1 = *operandStack.template Pop<Frag>(1);
373 SizeType s = NewState(e1.start, e2.start, 0);
374 *operandStack.template Push<Frag>() = Frag(s, Append(e1.out, e2.out), Min(e1.minIndex, e2.minIndex));
380 if (operandStack.GetSize() >= sizeof(Frag)) {
381 Frag e = *operandStack.template Pop<Frag>(1);
382 SizeType s = NewState(kRegexInvalidState, e.start, 0);
383 *operandStack.template Push<Frag>() = Frag(s, Append(e.out, s), e.minIndex);
389 if (operandStack.GetSize() >= sizeof(Frag)) {
390 Frag e = *operandStack.template Pop<Frag>(1);
391 SizeType s = NewState(kRegexInvalidState, e.start, 0);
393 *operandStack.template Push<Frag>() = Frag(s, s, e.minIndex);
399 RAPIDJSON_ASSERT(op == kOneOrMore);
400 if (operandStack.GetSize() >= sizeof(Frag)) {
401 Frag e = *operandStack.template Pop<Frag>(1);
402 SizeType s = NewState(kRegexInvalidState, e.start, 0);
404 *operandStack.template Push<Frag>() = Frag(e.start, s, e.minIndex);
411 bool EvalQuantifier(Stack<Allocator>& operandStack, unsigned n, unsigned m) {
412 RAPIDJSON_ASSERT(n <= m);
413 RAPIDJSON_ASSERT(operandStack.GetSize() >= sizeof(Frag));
416 if (m == 0) // a{0} not support
418 else if (m == kInfinityQuantifier)
419 Eval(operandStack, kZeroOrMore); // a{0,} -> a*
421 Eval(operandStack, kZeroOrOne); // a{0,5} -> a?
422 for (unsigned i = 0; i < m - 1; i++)
423 CloneTopOperand(operandStack); // a{0,5} -> a? a? a? a? a?
424 for (unsigned i = 0; i < m - 1; i++)
425 Eval(operandStack, kConcatenation); // a{0,5} -> a?a?a?a?a?
430 for (unsigned i = 0; i < n - 1; i++) // a{3} -> a a a
431 CloneTopOperand(operandStack);
433 if (m == kInfinityQuantifier)
434 Eval(operandStack, kOneOrMore); // a{3,} -> a a a+
436 CloneTopOperand(operandStack); // a{3,5} -> a a a a
437 Eval(operandStack, kZeroOrOne); // a{3,5} -> a a a a?
438 for (unsigned i = n; i < m - 1; i++)
439 CloneTopOperand(operandStack); // a{3,5} -> a a a a? a?
440 for (unsigned i = n; i < m; i++)
441 Eval(operandStack, kConcatenation); // a{3,5} -> a a aa?a?
444 for (unsigned i = 0; i < n - 1; i++)
445 Eval(operandStack, kConcatenation); // a{3} -> aaa, a{3,} -> aaa+, a{3.5} -> aaaa?a?
450 static SizeType Min(SizeType a, SizeType b) { return a < b ? a : b; }
452 void CloneTopOperand(Stack<Allocator>& operandStack) {
453 const Frag src = *operandStack.template Top<Frag>(); // Copy constructor to prevent invalidation
454 SizeType count = stateCount_ - src.minIndex; // Assumes top operand contains states in [src->minIndex, stateCount_)
455 State* s = states_.template Push<State>(count);
456 memcpy(s, &GetState(src.minIndex), count * sizeof(State));
457 for (SizeType j = 0; j < count; j++) {
458 if (s[j].out != kRegexInvalidState)
460 if (s[j].out1 != kRegexInvalidState)
463 *operandStack.template Push<Frag>() = Frag(src.start + count, src.out + count, src.minIndex + count);
464 stateCount_ += count;
467 template <typename InputStream>
468 bool ParseUnsigned(DecodedStream<InputStream, Encoding>& ds, unsigned* u) {
470 if (ds.Peek() < '0' || ds.Peek() > '9')
472 while (ds.Peek() >= '0' && ds.Peek() <= '9') {
473 if (r >= 429496729 && ds.Peek() > '5') // 2^32 - 1 = 4294967295
474 return false; // overflow
475 r = r * 10 + (ds.Take() - '0');
481 template <typename InputStream>
482 bool ParseRange(DecodedStream<InputStream, Encoding>& ds, SizeType* range) {
486 SizeType start = kRegexInvalidRange;
487 SizeType current = kRegexInvalidRange;
489 while ((codepoint = ds.Take()) != 0) {
492 if (codepoint == '^') {
500 if (start == kRegexInvalidRange)
501 return false; // Error: nothing inside []
502 if (step == 2) { // Add trailing '-'
503 SizeType r = NewRange('-');
504 RAPIDJSON_ASSERT(current != kRegexInvalidRange);
505 GetRange(current).next = r;
508 GetRange(start).start |= kRangeNegationFlag;
513 if (ds.Peek() == 'b') {
515 codepoint = 0x0008; // Escape backspace character
517 else if (!CharacterEscape(ds, &codepoint))
519 // fall through to default
524 if (codepoint == '-') {
528 // fall through to step 0 for other characters
532 SizeType r = NewRange(codepoint);
533 if (current != kRegexInvalidRange)
534 GetRange(current).next = r;
535 if (start == kRegexInvalidRange)
543 RAPIDJSON_ASSERT(step == 2);
544 GetRange(current).end = codepoint;
552 SizeType NewRange(unsigned codepoint) {
553 Range* r = ranges_.template Push<Range>();
554 r->start = r->end = codepoint;
555 r->next = kRegexInvalidRange;
556 return rangeCount_++;
559 template <typename InputStream>
560 bool CharacterEscape(DecodedStream<InputStream, Encoding>& ds, unsigned* escapedCodepoint) {
562 switch (codepoint = ds.Take()) {
577 *escapedCodepoint = codepoint; return true;
578 case 'f': *escapedCodepoint = 0x000C; return true;
579 case 'n': *escapedCodepoint = 0x000A; return true;
580 case 'r': *escapedCodepoint = 0x000D; return true;
581 case 't': *escapedCodepoint = 0x0009; return true;
582 case 'v': *escapedCodepoint = 0x000B; return true;
584 return false; // Unsupported escape character
588 Allocator* ownAllocator_;
589 Allocator* allocator_;
590 Stack<Allocator> states_;
591 Stack<Allocator> ranges_;
593 SizeType stateCount_;
594 SizeType rangeCount_;
596 static const unsigned kInfinityQuantifier = ~0u;
598 // For SearchWithAnchoring()
603 template <typename RegexType, typename Allocator = CrtAllocator>
604 class GenericRegexSearch {
606 typedef typename RegexType::EncodingType Encoding;
607 typedef typename Encoding::Ch Ch;
609 GenericRegexSearch(const RegexType& regex, Allocator* allocator = 0) :
610 regex_(regex), allocator_(allocator), ownAllocator_(0),
611 state0_(allocator, 0), state1_(allocator, 0), stateSet_()
613 RAPIDJSON_ASSERT(regex_.IsValid());
615 ownAllocator_ = allocator_ = RAPIDJSON_NEW(Allocator)();
616 stateSet_ = static_cast<unsigned*>(allocator_->Malloc(GetStateSetSize()));
617 state0_.template Reserve<SizeType>(regex_.stateCount_);
618 state1_.template Reserve<SizeType>(regex_.stateCount_);
621 ~GenericRegexSearch() {
622 Allocator::Free(stateSet_);
623 RAPIDJSON_DELETE(ownAllocator_);
626 template <typename InputStream>
627 bool Match(InputStream& is) {
628 return SearchWithAnchoring(is, true, true);
631 bool Match(const Ch* s) {
632 GenericStringStream<Encoding> is(s);
636 template <typename InputStream>
637 bool Search(InputStream& is) {
638 return SearchWithAnchoring(is, regex_.anchorBegin_, regex_.anchorEnd_);
641 bool Search(const Ch* s) {
642 GenericStringStream<Encoding> is(s);
647 typedef typename RegexType::State State;
648 typedef typename RegexType::Range Range;
650 template <typename InputStream>
651 bool SearchWithAnchoring(InputStream& is, bool anchorBegin, bool anchorEnd) {
652 DecodedStream<InputStream, Encoding> ds(is);
655 Stack<Allocator> *current = &state0_, *next = &state1_;
656 const size_t stateSetSize = GetStateSetSize();
657 std::memset(stateSet_, 0, stateSetSize);
659 bool matched = AddState(*current, regex_.root_);
661 while (!current->Empty() && (codepoint = ds.Take()) != 0) {
662 std::memset(stateSet_, 0, stateSetSize);
665 for (const SizeType* s = current->template Bottom<SizeType>(); s != current->template End<SizeType>(); ++s) {
666 const State& sr = regex_.GetState(*s);
667 if (sr.codepoint == codepoint ||
668 sr.codepoint == RegexType::kAnyCharacterClass ||
669 (sr.codepoint == RegexType::kRangeCharacterClass && MatchRange(sr.rangeStart, codepoint)))
671 matched = AddState(*next, sr.out) || matched;
672 if (!anchorEnd && matched)
676 AddState(*next, regex_.root_);
678 internal::Swap(current, next);
684 size_t GetStateSetSize() const {
685 return (regex_.stateCount_ + 31) / 32 * 4;
688 // Return whether the added states is a match state
689 bool AddState(Stack<Allocator>& l, SizeType index) {
690 RAPIDJSON_ASSERT(index != kRegexInvalidState);
692 const State& s = regex_.GetState(index);
693 if (s.out1 != kRegexInvalidState) { // Split
694 bool matched = AddState(l, s.out);
695 return AddState(l, s.out1) || matched;
697 else if (!(stateSet_[index >> 5] & (1u << (index & 31)))) {
698 stateSet_[index >> 5] |= (1u << (index & 31));
699 *l.template PushUnsafe<SizeType>() = index;
701 return s.out == kRegexInvalidState; // by using PushUnsafe() above, we can ensure s is not validated due to reallocation.
704 bool MatchRange(SizeType rangeIndex, unsigned codepoint) const {
705 bool yes = (regex_.GetRange(rangeIndex).start & RegexType::kRangeNegationFlag) == 0;
706 while (rangeIndex != kRegexInvalidRange) {
707 const Range& r = regex_.GetRange(rangeIndex);
708 if (codepoint >= (r.start & ~RegexType::kRangeNegationFlag) && codepoint <= r.end)
715 const RegexType& regex_;
716 Allocator* allocator_;
717 Allocator* ownAllocator_;
718 Stack<Allocator> state0_;
719 Stack<Allocator> state1_;
723 typedef GenericRegex<UTF8<> > Regex;
724 typedef GenericRegexSearch<Regex> RegexSearch;
726 } // namespace internal
727 RAPIDJSON_NAMESPACE_END
733 #if defined(__clang__) || defined(_MSC_VER)
737 #endif // RAPIDJSON_INTERNAL_REGEX_H_