From ae98608918f0b92985b47a5ac2987ebd9797be4c Mon Sep 17 00:00:00 2001 From: Karl Williamson Date: Mon, 25 Nov 2013 19:40:12 -0700 Subject: [PATCH] Convert regnode to a flag for [...] Prior to this commit, there were 3 types of ANYOF nodes; now there are two: regular, and one for the synthetic start class (ssc). This commit converted the third type dealing with warning about matching \p{} against non-Unicode code points, into using the spare flag bit for ANYOF nodes. This allows this bit to apply to ssc ANYOF nodes, whereas previously it couldn't. There is a bug in which the warning isn't raised if the match is rejected by the optimizer, because of this inability. This bug will be fixed in a later commit. Another option would have been to create a new node-type which was an ANYOF_SSC_WARN_SUPER node. But this adds extra complications to things; and we have a spare bit that we might as well use. The comments give better possibilities for freeing up 2 bits should they be needed. --- pod/perldebguts.pod | 281 +++++++++++++++++++++++------------------------ regcomp.c | 3 +- regcomp.h | 10 +- regcomp.sym | 1 - regexec.c | 5 +- regnodes.h | 309 ++++++++++++++++++++++++++-------------------------- t/lib/warnings/utf8 | 11 ++ 7 files changed, 308 insertions(+), 312 deletions(-) diff --git a/pod/perldebguts.pod b/pod/perldebguts.pod index 0ebd9fa..54fc478 100644 --- a/pod/perldebguts.pod +++ b/pod/perldebguts.pod @@ -556,61 +556,57 @@ will be lost. # Exit points - END no End of program. - SUCCEED no Return from a subroutine, basically. + END no End of program. + SUCCEED no Return from a subroutine, basically. # Anchors: - BOL no Match "" at beginning of line. - MBOL no Same, assuming multiline. - SBOL no Same, assuming singleline. - EOS no Match "" at end of string. - EOL no Match "" at end of line. - MEOL no Same, assuming multiline. - SEOL no Same, assuming singleline. - BOUND no Match "" at any word boundary using native - charset semantics for non-utf8 - BOUNDL no Match "" at any locale word boundary - BOUNDU no Match "" at any word boundary using Unicode - semantics - BOUNDA no Match "" at any word boundary using ASCII - semantics - NBOUND no Match "" at any word non-boundary using - native charset semantics for non-utf8 - NBOUNDL no Match "" at any locale word non-boundary - NBOUNDU no Match "" at any word non-boundary using - Unicode semantics - NBOUNDA no Match "" at any word non-boundary using - ASCII semantics - GPOS no Matches where last m//g left off. + BOL no Match "" at beginning of line. + MBOL no Same, assuming multiline. + SBOL no Same, assuming singleline. + EOS no Match "" at end of string. + EOL no Match "" at end of line. + MEOL no Same, assuming multiline. + SEOL no Same, assuming singleline. + BOUND no Match "" at any word boundary using native + charset semantics for non-utf8 + BOUNDL no Match "" at any locale word boundary + BOUNDU no Match "" at any word boundary using Unicode + semantics + BOUNDA no Match "" at any word boundary using ASCII + semantics + NBOUND no Match "" at any word non-boundary using + native charset semantics for non-utf8 + NBOUNDL no Match "" at any locale word non-boundary + NBOUNDU no Match "" at any word non-boundary using + Unicode semantics + NBOUNDA no Match "" at any word non-boundary using + ASCII semantics + GPOS no Matches where last m//g left off. # [Special] alternatives: - REG_ANY no Match any one character (except newline). - SANY no Match any one character. - CANY no Match any one byte. - ANYOF sv Match character in (or not in) this class, - single char match only - ANYOF_WARN_SUPER sv Match character in (or not in) this class, - warn (if enabled) upon matching a char - above Unicode max; - ANYOF_SYNTHETIC sv Synthetic start class - - POSIXD none Some [[:class:]] under /d; the FLAGS field - gives which one - POSIXL none Some [[:class:]] under /l; the FLAGS field - gives which one - POSIXU none Some [[:class:]] under /u; the FLAGS field - gives which one - POSIXA none Some [[:class:]] under /a; the FLAGS field - gives which one - NPOSIXD none complement of POSIXD, [[:^class:]] - NPOSIXL none complement of POSIXL, [[:^class:]] - NPOSIXU none complement of POSIXU, [[:^class:]] - NPOSIXA none complement of POSIXA, [[:^class:]] - - CLUMP no Match any extended grapheme cluster - sequence + REG_ANY no Match any one character (except newline). + SANY no Match any one character. + CANY no Match any one byte. + ANYOF sv Match character in (or not in) this class, + single char match only + ANYOF_SYNTHETIC sv Synthetic start class + + POSIXD none Some [[:class:]] under /d; the FLAGS field + gives which one + POSIXL none Some [[:class:]] under /l; the FLAGS field + gives which one + POSIXU none Some [[:class:]] under /u; the FLAGS field + gives which one + POSIXA none Some [[:class:]] under /a; the FLAGS field + gives which one + NPOSIXD none complement of POSIXD, [[:^class:]] + NPOSIXL none complement of POSIXL, [[:^class:]] + NPOSIXU none complement of POSIXU, [[:^class:]] + NPOSIXA none complement of POSIXA, [[:^class:]] + + CLUMP no Match any extended grapheme cluster sequence # Alternation @@ -623,40 +619,40 @@ will be lost. # pointer of each individual branch points; each branch # starts with the operand node of a BRANCH node. # - BRANCH node Match this alternative, or the next... + BRANCH node Match this alternative, or the next... # Back pointer # BACK Normal "next" pointers all implicitly point forward; # BACK exists to make loop structures possible. # not used - BACK no Match "", "next" ptr points backward. + BACK no Match "", "next" ptr points backward. # Literals - EXACT str Match this string (preceded by length). - EXACTF str Match this non-UTF-8 string (not guaranteed - to be folded) using /id rules (w/len). - EXACTFL str Match this string (not guaranteed to be - folded) using /il rules (w/len). - EXACTFU str Match this string (folded iff in UTF-8, - length in folding doesn't change if not in - UTF-8) using /iu rules (w/len). - EXACTFA str Match this string (not guaranteed to be - folded) using /iaa rules (w/len). - EXACTFU_SS str Match this string (folded iff in UTF-8, - length in folding may change even if not in - UTF-8) using /iu rules (w/len). - EXACTFA_NO_TRIE str Match this string (which is not trie-able; - not guaranteed to be folded) using /iaa - rules (w/len). + EXACT str Match this string (preceded by length). + EXACTF str Match this non-UTF-8 string (not guaranteed + to be folded) using /id rules (w/len). + EXACTFL str Match this string (not guaranteed to be + folded) using /il rules (w/len). + EXACTFU str Match this string (folded iff in UTF-8, + length in folding doesn't change if not in + UTF-8) using /iu rules (w/len). + EXACTFA str Match this string (not guaranteed to be + folded) using /iaa rules (w/len). + EXACTFU_SS str Match this string (folded iff in UTF-8, + length in folding may change even if not in + UTF-8) using /iu rules (w/len). + EXACTFA_NO_TRIE str Match this string (which is not trie-able; + not guaranteed to be folded) using /iaa + rules (w/len). # Do nothing types - NOTHING no Match empty string. + NOTHING no Match empty string. # A variant of above which delimits a group, thus stops optimizations - TAIL no Match empty string. Can jump here from - outside. + TAIL no Match empty string. Can jump here from + outside. # Loops @@ -665,71 +661,68 @@ will be lost. # (one character per match) are implemented with STAR # and PLUS for speed and to minimize recursive plunges. # - STAR node Match this (simple) thing 0 or more times. - PLUS node Match this (simple) thing 1 or more times. + STAR node Match this (simple) thing 0 or more times. + PLUS node Match this (simple) thing 1 or more times. - CURLY sv 2 Match this simple thing {n,m} times. - CURLYN no 2 Capture next-after-this simple thing - CURLYM no 2 Capture this medium-complex thing {n,m} - times. - CURLYX sv 2 Match this complex thing {n,m} times. + CURLY sv 2 Match this simple thing {n,m} times. + CURLYN no 2 Capture next-after-this simple thing + CURLYM no 2 Capture this medium-complex thing {n,m} + times. + CURLYX sv 2 Match this complex thing {n,m} times. # This terminator creates a loop structure for CURLYX - WHILEM no Do curly processing and see if rest - matches. + WHILEM no Do curly processing and see if rest matches. # Buffer related # OPEN,CLOSE,GROUPP ...are numbered at compile time. - OPEN num 1 Mark this point in input as start of #n. - CLOSE num 1 Analogous to OPEN. - - REF num 1 Match some already matched string - REFF num 1 Match already matched string, folded using - native charset semantics for non-utf8 - REFFL num 1 Match already matched string, folded in - loc. - REFFU num 1 Match already matched string, folded using - unicode semantics for non-utf8 - REFFA num 1 Match already matched string, folded using - unicode semantics for non-utf8, no mixing - ASCII, non-ASCII + OPEN num 1 Mark this point in input as start of #n. + CLOSE num 1 Analogous to OPEN. + + REF num 1 Match some already matched string + REFF num 1 Match already matched string, folded using + native charset semantics for non-utf8 + REFFL num 1 Match already matched string, folded in loc. + REFFU num 1 Match already matched string, folded using + unicode semantics for non-utf8 + REFFA num 1 Match already matched string, folded using + unicode semantics for non-utf8, no mixing + ASCII, non-ASCII # Named references. Code in regcomp.c assumes that these all are after # the numbered references - NREF no-sv 1 Match some already matched string - NREFF no-sv 1 Match already matched string, folded using - native charset semantics for non-utf8 - NREFFL no-sv 1 Match already matched string, folded in - loc. - NREFFU num 1 Match already matched string, folded using - unicode semantics for non-utf8 - NREFFA num 1 Match already matched string, folded using - unicode semantics for non-utf8, no mixing - ASCII, non-ASCII - - IFMATCH off 1 2 Succeeds if the following matches. - UNLESSM off 1 2 Fails if the following matches. - SUSPEND off 1 1 "Independent" sub-RE. - IFTHEN off 1 1 Switch, should be preceded by switcher. - GROUPP num 1 Whether the group matched. + NREF no-sv 1 Match some already matched string + NREFF no-sv 1 Match already matched string, folded using + native charset semantics for non-utf8 + NREFFL no-sv 1 Match already matched string, folded in loc. + NREFFU num 1 Match already matched string, folded using + unicode semantics for non-utf8 + NREFFA num 1 Match already matched string, folded using + unicode semantics for non-utf8, no mixing + ASCII, non-ASCII + + IFMATCH off 1 2 Succeeds if the following matches. + UNLESSM off 1 2 Fails if the following matches. + SUSPEND off 1 1 "Independent" sub-RE. + IFTHEN off 1 1 Switch, should be preceded by switcher. + GROUPP num 1 Whether the group matched. # Support for long RE - LONGJMP off 1 1 Jump far away. - BRANCHJ off 1 1 BRANCH with long offset. + LONGJMP off 1 1 Jump far away. + BRANCHJ off 1 1 BRANCH with long offset. # The heavy worker - EVAL evl 1 Execute some Perl code. + EVAL evl 1 Execute some Perl code. # Modifiers - MINMOD no Next operator is not greedy. - LOGICAL no Next opcode should set the flag only. + MINMOD no Next operator is not greedy. + LOGICAL no Next opcode should set the flag only. # This is not used yet - RENUM off 1 1 Group with independently numbered parens. + RENUM off 1 1 Group with independently numbered parens. # Trie Related @@ -737,60 +730,60 @@ will be lost. # have inline charclass data (ascii only), the 'C' store it in the # structure. - TRIE trie 1 Match many EXACT(F[ALU]?)? at once. - flags==type - TRIEC trie Same as TRIE, but with embedded charclass - charclass data + TRIE trie 1 Match many EXACT(F[ALU]?)? at once. + flags==type + TRIEC trie Same as TRIE, but with embedded charclass + charclass data - AHOCORASICK trie 1 Aho Corasick stclass. flags==type - AHOCORASICKC trie Same as AHOCORASICK, but with embedded - charclass charclass data + AHOCORASICK trie 1 Aho Corasick stclass. flags==type + AHOCORASICKC trie Same as AHOCORASICK, but with embedded + charclass charclass data # Regex Subroutines - GOSUB num/ofs 2L recurse to paren arg1 at (signed) ofs arg2 - GOSTART no recurse to start of pattern + GOSUB num/ofs 2L recurse to paren arg1 at (signed) ofs arg2 + GOSTART no recurse to start of pattern # Special conditionals - NGROUPP no-sv 1 Whether the group matched. - INSUBP num 1 Whether we are in a specific recurse. - DEFINEP none 1 Never execute directly. + NGROUPP no-sv 1 Whether the group matched. + INSUBP num 1 Whether we are in a specific recurse. + DEFINEP none 1 Never execute directly. # Backtracking Verbs - ENDLIKE none Used only for the type field of verbs - OPFAIL none Same as (?!) - ACCEPT parno 1 Accepts the current matched string. + ENDLIKE none Used only for the type field of verbs + OPFAIL none Same as (?!) + ACCEPT parno 1 Accepts the current matched string. # Verbs With Arguments - VERB no-sv 1 Used only for the type field of verbs - PRUNE no-sv 1 Pattern fails at this startpoint if no- - backtracking through this - MARKPOINT no-sv 1 Push the current location for rollback by - cut. - SKIP no-sv 1 On failure skip forward (to the mark) - before retrying - COMMIT no-sv 1 Pattern fails outright if backtracking - through this - CUTGROUP no-sv 1 On failure go to the next alternation in - the group + VERB no-sv 1 Used only for the type field of verbs + PRUNE no-sv 1 Pattern fails at this startpoint if no- + backtracking through this + MARKPOINT no-sv 1 Push the current location for rollback by + cut. + SKIP no-sv 1 On failure skip forward (to the mark) before + retrying + COMMIT no-sv 1 Pattern fails outright if backtracking + through this + CUTGROUP no-sv 1 On failure go to the next alternation in the + group # Control what to keep in $&. - KEEPS no $& begins here. + KEEPS no $& begins here. # New charclass like patterns - LNBREAK none generic newline pattern + LNBREAK none generic newline pattern # SPECIAL REGOPS # This is not really a node, but an optimized away piece of a "long" # node. To simplify debugging output, we mark it as if it were a node - OPTIMIZED off Placeholder for dump. + OPTIMIZED off Placeholder for dump. # Special opcode with the property that no opcode in a compiled program # will ever be of this type. Thus it can be used as a flag value that # no other opcode has been seen. END is used similarly, in that an END # node cant be optimized. So END implies "unoptimizable" and PSEUDO # mean "not seen anything to optimize yet". - PSEUDO off Pseudo opcode for internal use. + PSEUDO off Pseudo opcode for internal use. =for regcomp.pl end diff --git a/regcomp.c b/regcomp.c index eda2257..c24c7d0 100644 --- a/regcomp.c +++ b/regcomp.c @@ -4620,7 +4620,6 @@ PerlIO_printf(Perl_debug_log, "LHS=%"UVdf" RHS=%"UVdf"\n", } break; - case ANYOF_WARN_SUPER: case ANYOF: if (flags & SCF_DO_STCLASS_AND) ssc_and(pRExC_state, data->start_class, @@ -14407,7 +14406,7 @@ parseit: } if (warn_super) { - OP(ret) = ANYOF_WARN_SUPER; + ANYOF_FLAGS(ret) |= ANYOF_WARN_SUPER; } } diff --git a/regcomp.h b/regcomp.h index fb24579..d87865f 100644 --- a/regcomp.h +++ b/regcomp.h @@ -314,8 +314,8 @@ struct regnode_ssc { * ANYOF_NONBITMAP_NON_UTF8 bit is also set. */ #define ANYOF_NONBITMAP(node) (ARG(node) != ANYOF_NONBITMAP_EMPTY) -/* Flags for node->flags of ANYOF. These are in short supply, but there is one - * currently available. If more than this are needed, the ANYOF_LOCALE and +/* Flags for node->flags of ANYOF. These are in short supply, with none + * currently available. If more are needed, the ANYOF_LOCALE and * ANYOF_POSIXL bits could be shared, making a space penalty for all locale * nodes. Also, the ABOVE_LATIN1_ALL bit could be freed up by resorting to * creating a swash containing everything above 255. This introduces a @@ -351,7 +351,9 @@ struct regnode_ssc { #define ANYOF_CLASS ANYOF_POSIXL #define ANYOF_LARGE ANYOF_POSIXL -/* Unused: 0x10. When using, be sure to change ANYOF_FLAGS_ALL below */ +/* Should we raise a warning if matching against an above-Unicode code point? + * */ +#define ANYOF_WARN_SUPER 0x10 /* Can match something outside the bitmap that isn't in utf8 */ #define ANYOF_NONBITMAP_NON_UTF8 0x20 @@ -364,7 +366,7 @@ struct regnode_ssc { * in utf8. */ #define ANYOF_NON_UTF8_LATIN1_ALL 0x80 -#define ANYOF_FLAGS_ALL (0xff & ~0x10) +#define ANYOF_FLAGS_ALL (0xff) #define ANYOF_LOCALE_FLAGS (ANYOF_LOCALE \ |ANYOF_LOC_FOLD \ diff --git a/regcomp.sym b/regcomp.sym index 7d3cdc0..09962dd 100644 --- a/regcomp.sym +++ b/regcomp.sym @@ -55,7 +55,6 @@ REG_ANY REG_ANY, no 0 S ; Match any one character (except newline). SANY REG_ANY, no 0 S ; Match any one character. CANY REG_ANY, no 0 S ; Match any one byte. ANYOF ANYOF, sv 0 S ; Match character in (or not in) this class, single char match only -ANYOF_WARN_SUPER ANYOF, sv 0 S ; Match character in (or not in) this class, warn (if enabled) upon matching a char above Unicode max; ANYOF_SYNTHETIC ANYOF, sv 0 S ; Synthetic start class # Order of the below is important. See ordering comment above. diff --git a/regexec.c b/regexec.c index fab9009..af434cf 100644 --- a/regexec.c +++ b/regexec.c @@ -1489,7 +1489,6 @@ S_find_byclass(pTHX_ regexp * prog, const regnode *c, char *s, switch (OP(c)) { case ANYOF: case ANYOF_SYNTHETIC: - case ANYOF_WARN_SUPER: if (utf8_target) { REXEC_FBC_UTF8_CLASS_SCAN( reginclass(prog, c, (U8*)s, utf8_target)); @@ -4386,7 +4385,6 @@ S_regmatch(pTHX_ regmatch_info *reginfo, char *startpos, regnode *prog) break; case ANYOF: /* /[abc]/ */ - case ANYOF_WARN_SUPER: if (NEXTCHR_IS_EOS) sayNO; if (utf8_target) { @@ -7001,7 +6999,6 @@ S_regrepeat(pTHX_ regexp *prog, char **startposp, const regnode *p, break; } case ANYOF: - case ANYOF_WARN_SUPER: if (utf8_target) { while (hardcount < max && scan < loceol @@ -7544,7 +7541,7 @@ S_reginclass(pTHX_ regexp * const prog, const regnode * const n, const U8* const } if (UNICODE_IS_SUPER(c) - && OP(n) == ANYOF_WARN_SUPER + && (flags & ANYOF_WARN_SUPER) && ckWARN_d(WARN_NON_UNICODE)) { Perl_warner(aTHX_ packWARN(WARN_NON_UNICODE), diff --git a/regnodes.h b/regnodes.h index 2d55c8b..f1a5fe2 100644 --- a/regnodes.h +++ b/regnodes.h @@ -6,8 +6,8 @@ /* Regops and State definitions */ -#define REGNODE_MAX 95 -#define REGMATCH_STATE_MAX 135 +#define REGNODE_MAX 94 +#define REGMATCH_STATE_MAX 134 #define END 0 /* 0000 End of program. */ #define SUCCEED 1 /* 0x01 Return from a subroutine, basically. */ @@ -31,80 +31,79 @@ #define SANY 19 /* 0x13 Match any one character. */ #define CANY 20 /* 0x14 Match any one byte. */ #define ANYOF 21 /* 0x15 Match character in (or not in) this class, single char match only */ -#define ANYOF_WARN_SUPER 22 /* 0x16 Match character in (or not in) this class, warn (if enabled) upon matching a char above Unicode max; */ -#define ANYOF_SYNTHETIC 23 /* 0x17 Synthetic start class */ -#define POSIXD 24 /* 0x18 Some [[:class:]] under /d; the FLAGS field gives which one */ -#define POSIXL 25 /* 0x19 Some [[:class:]] under /l; the FLAGS field gives which one */ -#define POSIXU 26 /* 0x1a Some [[:class:]] under /u; the FLAGS field gives which one */ -#define POSIXA 27 /* 0x1b Some [[:class:]] under /a; the FLAGS field gives which one */ -#define NPOSIXD 28 /* 0x1c complement of POSIXD, [[:^class:]] */ -#define NPOSIXL 29 /* 0x1d complement of POSIXL, [[:^class:]] */ -#define NPOSIXU 30 /* 0x1e complement of POSIXU, [[:^class:]] */ -#define NPOSIXA 31 /* 0x1f complement of POSIXA, [[:^class:]] */ -#define CLUMP 32 /* 0x20 Match any extended grapheme cluster sequence */ -#define BRANCH 33 /* 0x21 Match this alternative, or the next... */ -#define BACK 34 /* 0x22 Match "", "next" ptr points backward. */ -#define EXACT 35 /* 0x23 Match this string (preceded by length). */ -#define EXACTF 36 /* 0x24 Match this non-UTF-8 string (not guaranteed to be folded) using /id rules (w/len). */ -#define EXACTFL 37 /* 0x25 Match this string (not guaranteed to be folded) using /il rules (w/len). */ -#define EXACTFU 38 /* 0x26 Match this string (folded iff in UTF-8, length in folding doesn't change if not in UTF-8) using /iu rules (w/len). */ -#define EXACTFA 39 /* 0x27 Match this string (not guaranteed to be folded) using /iaa rules (w/len). */ -#define EXACTFU_SS 40 /* 0x28 Match this string (folded iff in UTF-8, length in folding may change even if not in UTF-8) using /iu rules (w/len). */ -#define EXACTFA_NO_TRIE 41 /* 0x29 Match this string (which is not trie-able; not guaranteed to be folded) using /iaa rules (w/len). */ -#define NOTHING 42 /* 0x2a Match empty string. */ -#define TAIL 43 /* 0x2b Match empty string. Can jump here from outside. */ -#define STAR 44 /* 0x2c Match this (simple) thing 0 or more times. */ -#define PLUS 45 /* 0x2d Match this (simple) thing 1 or more times. */ -#define CURLY 46 /* 0x2e Match this simple thing {n,m} times. */ -#define CURLYN 47 /* 0x2f Capture next-after-this simple thing */ -#define CURLYM 48 /* 0x30 Capture this medium-complex thing {n,m} times. */ -#define CURLYX 49 /* 0x31 Match this complex thing {n,m} times. */ -#define WHILEM 50 /* 0x32 Do curly processing and see if rest matches. */ -#define OPEN 51 /* 0x33 Mark this point in input as start of #n. */ -#define CLOSE 52 /* 0x34 Analogous to OPEN. */ -#define REF 53 /* 0x35 Match some already matched string */ -#define REFF 54 /* 0x36 Match already matched string, folded using native charset semantics for non-utf8 */ -#define REFFL 55 /* 0x37 Match already matched string, folded in loc. */ -#define REFFU 56 /* 0x38 Match already matched string, folded using unicode semantics for non-utf8 */ -#define REFFA 57 /* 0x39 Match already matched string, folded using unicode semantics for non-utf8, no mixing ASCII, non-ASCII */ -#define NREF 58 /* 0x3a Match some already matched string */ -#define NREFF 59 /* 0x3b Match already matched string, folded using native charset semantics for non-utf8 */ -#define NREFFL 60 /* 0x3c Match already matched string, folded in loc. */ -#define NREFFU 61 /* 0x3d Match already matched string, folded using unicode semantics for non-utf8 */ -#define NREFFA 62 /* 0x3e Match already matched string, folded using unicode semantics for non-utf8, no mixing ASCII, non-ASCII */ -#define IFMATCH 63 /* 0x3f Succeeds if the following matches. */ -#define UNLESSM 64 /* 0x40 Fails if the following matches. */ -#define SUSPEND 65 /* 0x41 "Independent" sub-RE. */ -#define IFTHEN 66 /* 0x42 Switch, should be preceded by switcher. */ -#define GROUPP 67 /* 0x43 Whether the group matched. */ -#define LONGJMP 68 /* 0x44 Jump far away. */ -#define BRANCHJ 69 /* 0x45 BRANCH with long offset. */ -#define EVAL 70 /* 0x46 Execute some Perl code. */ -#define MINMOD 71 /* 0x47 Next operator is not greedy. */ -#define LOGICAL 72 /* 0x48 Next opcode should set the flag only. */ -#define RENUM 73 /* 0x49 Group with independently numbered parens. */ -#define TRIE 74 /* 0x4a Match many EXACT(F[ALU]?)? at once. flags==type */ -#define TRIEC 75 /* 0x4b Same as TRIE, but with embedded charclass data */ -#define AHOCORASICK 76 /* 0x4c Aho Corasick stclass. flags==type */ -#define AHOCORASICKC 77 /* 0x4d Same as AHOCORASICK, but with embedded charclass data */ -#define GOSUB 78 /* 0x4e recurse to paren arg1 at (signed) ofs arg2 */ -#define GOSTART 79 /* 0x4f recurse to start of pattern */ -#define NGROUPP 80 /* 0x50 Whether the group matched. */ -#define INSUBP 81 /* 0x51 Whether we are in a specific recurse. */ -#define DEFINEP 82 /* 0x52 Never execute directly. */ -#define ENDLIKE 83 /* 0x53 Used only for the type field of verbs */ -#define OPFAIL 84 /* 0x54 Same as (?!) */ -#define ACCEPT 85 /* 0x55 Accepts the current matched string. */ -#define VERB 86 /* 0x56 Used only for the type field of verbs */ -#define PRUNE 87 /* 0x57 Pattern fails at this startpoint if no-backtracking through this */ -#define MARKPOINT 88 /* 0x58 Push the current location for rollback by cut. */ -#define SKIP 89 /* 0x59 On failure skip forward (to the mark) before retrying */ -#define COMMIT 90 /* 0x5a Pattern fails outright if backtracking through this */ -#define CUTGROUP 91 /* 0x5b On failure go to the next alternation in the group */ -#define KEEPS 92 /* 0x5c $& begins here. */ -#define LNBREAK 93 /* 0x5d generic newline pattern */ -#define OPTIMIZED 94 /* 0x5e Placeholder for dump. */ -#define PSEUDO 95 /* 0x5f Pseudo opcode for internal use. */ +#define ANYOF_SYNTHETIC 22 /* 0x16 Synthetic start class */ +#define POSIXD 23 /* 0x17 Some [[:class:]] under /d; the FLAGS field gives which one */ +#define POSIXL 24 /* 0x18 Some [[:class:]] under /l; the FLAGS field gives which one */ +#define POSIXU 25 /* 0x19 Some [[:class:]] under /u; the FLAGS field gives which one */ +#define POSIXA 26 /* 0x1a Some [[:class:]] under /a; the FLAGS field gives which one */ +#define NPOSIXD 27 /* 0x1b complement of POSIXD, [[:^class:]] */ +#define NPOSIXL 28 /* 0x1c complement of POSIXL, [[:^class:]] */ +#define NPOSIXU 29 /* 0x1d complement of POSIXU, [[:^class:]] */ +#define NPOSIXA 30 /* 0x1e complement of POSIXA, [[:^class:]] */ +#define CLUMP 31 /* 0x1f Match any extended grapheme cluster sequence */ +#define BRANCH 32 /* 0x20 Match this alternative, or the next... */ +#define BACK 33 /* 0x21 Match "", "next" ptr points backward. */ +#define EXACT 34 /* 0x22 Match this string (preceded by length). */ +#define EXACTF 35 /* 0x23 Match this non-UTF-8 string (not guaranteed to be folded) using /id rules (w/len). */ +#define EXACTFL 36 /* 0x24 Match this string (not guaranteed to be folded) using /il rules (w/len). */ +#define EXACTFU 37 /* 0x25 Match this string (folded iff in UTF-8, length in folding doesn't change if not in UTF-8) using /iu rules (w/len). */ +#define EXACTFA 38 /* 0x26 Match this string (not guaranteed to be folded) using /iaa rules (w/len). */ +#define EXACTFU_SS 39 /* 0x27 Match this string (folded iff in UTF-8, length in folding may change even if not in UTF-8) using /iu rules (w/len). */ +#define EXACTFA_NO_TRIE 40 /* 0x28 Match this string (which is not trie-able; not guaranteed to be folded) using /iaa rules (w/len). */ +#define NOTHING 41 /* 0x29 Match empty string. */ +#define TAIL 42 /* 0x2a Match empty string. Can jump here from outside. */ +#define STAR 43 /* 0x2b Match this (simple) thing 0 or more times. */ +#define PLUS 44 /* 0x2c Match this (simple) thing 1 or more times. */ +#define CURLY 45 /* 0x2d Match this simple thing {n,m} times. */ +#define CURLYN 46 /* 0x2e Capture next-after-this simple thing */ +#define CURLYM 47 /* 0x2f Capture this medium-complex thing {n,m} times. */ +#define CURLYX 48 /* 0x30 Match this complex thing {n,m} times. */ +#define WHILEM 49 /* 0x31 Do curly processing and see if rest matches. */ +#define OPEN 50 /* 0x32 Mark this point in input as start of #n. */ +#define CLOSE 51 /* 0x33 Analogous to OPEN. */ +#define REF 52 /* 0x34 Match some already matched string */ +#define REFF 53 /* 0x35 Match already matched string, folded using native charset semantics for non-utf8 */ +#define REFFL 54 /* 0x36 Match already matched string, folded in loc. */ +#define REFFU 55 /* 0x37 Match already matched string, folded using unicode semantics for non-utf8 */ +#define REFFA 56 /* 0x38 Match already matched string, folded using unicode semantics for non-utf8, no mixing ASCII, non-ASCII */ +#define NREF 57 /* 0x39 Match some already matched string */ +#define NREFF 58 /* 0x3a Match already matched string, folded using native charset semantics for non-utf8 */ +#define NREFFL 59 /* 0x3b Match already matched string, folded in loc. */ +#define NREFFU 60 /* 0x3c Match already matched string, folded using unicode semantics for non-utf8 */ +#define NREFFA 61 /* 0x3d Match already matched string, folded using unicode semantics for non-utf8, no mixing ASCII, non-ASCII */ +#define IFMATCH 62 /* 0x3e Succeeds if the following matches. */ +#define UNLESSM 63 /* 0x3f Fails if the following matches. */ +#define SUSPEND 64 /* 0x40 "Independent" sub-RE. */ +#define IFTHEN 65 /* 0x41 Switch, should be preceded by switcher. */ +#define GROUPP 66 /* 0x42 Whether the group matched. */ +#define LONGJMP 67 /* 0x43 Jump far away. */ +#define BRANCHJ 68 /* 0x44 BRANCH with long offset. */ +#define EVAL 69 /* 0x45 Execute some Perl code. */ +#define MINMOD 70 /* 0x46 Next operator is not greedy. */ +#define LOGICAL 71 /* 0x47 Next opcode should set the flag only. */ +#define RENUM 72 /* 0x48 Group with independently numbered parens. */ +#define TRIE 73 /* 0x49 Match many EXACT(F[ALU]?)? at once. flags==type */ +#define TRIEC 74 /* 0x4a Same as TRIE, but with embedded charclass data */ +#define AHOCORASICK 75 /* 0x4b Aho Corasick stclass. flags==type */ +#define AHOCORASICKC 76 /* 0x4c Same as AHOCORASICK, but with embedded charclass data */ +#define GOSUB 77 /* 0x4d recurse to paren arg1 at (signed) ofs arg2 */ +#define GOSTART 78 /* 0x4e recurse to start of pattern */ +#define NGROUPP 79 /* 0x4f Whether the group matched. */ +#define INSUBP 80 /* 0x50 Whether we are in a specific recurse. */ +#define DEFINEP 81 /* 0x51 Never execute directly. */ +#define ENDLIKE 82 /* 0x52 Used only for the type field of verbs */ +#define OPFAIL 83 /* 0x53 Same as (?!) */ +#define ACCEPT 84 /* 0x54 Accepts the current matched string. */ +#define VERB 85 /* 0x55 Used only for the type field of verbs */ +#define PRUNE 86 /* 0x56 Pattern fails at this startpoint if no-backtracking through this */ +#define MARKPOINT 87 /* 0x57 Push the current location for rollback by cut. */ +#define SKIP 88 /* 0x58 On failure skip forward (to the mark) before retrying */ +#define COMMIT 89 /* 0x59 Pattern fails outright if backtracking through this */ +#define CUTGROUP 90 /* 0x5a On failure go to the next alternation in the group */ +#define KEEPS 91 /* 0x5b $& begins here. */ +#define LNBREAK 92 /* 0x5c generic newline pattern */ +#define OPTIMIZED 93 /* 0x5d Placeholder for dump. */ +#define PSEUDO 94 /* 0x5e Pseudo opcode for internal use. */ /* ------------ States ------------- */ #define TRIE_next (REGNODE_MAX + 1) /* state for TRIE */ #define TRIE_next_fail (REGNODE_MAX + 2) /* state for TRIE */ @@ -175,7 +174,6 @@ EXTCONST U8 PL_regkind[] = { REG_ANY, /* SANY */ REG_ANY, /* CANY */ ANYOF, /* ANYOF */ - ANYOF, /* ANYOF_WARN_SUPER */ ANYOF, /* ANYOF_SYNTHETIC */ POSIXD, /* POSIXD */ POSIXD, /* POSIXL */ @@ -319,7 +317,6 @@ static const U8 regarglen[] = { 0, /* SANY */ 0, /* CANY */ 0, /* ANYOF */ - 0, /* ANYOF_WARN_SUPER */ 0, /* ANYOF_SYNTHETIC */ 0, /* POSIXD */ 0, /* POSIXL */ @@ -420,7 +417,6 @@ static const char reg_off_by_arg[] = { 0, /* SANY */ 0, /* CANY */ 0, /* ANYOF */ - 0, /* ANYOF_WARN_SUPER */ 0, /* ANYOF_SYNTHETIC */ 0, /* POSIXD */ 0, /* POSIXL */ @@ -526,80 +522,79 @@ EXTCONST char * const PL_reg_name[] = { "SANY", /* 0x13 */ "CANY", /* 0x14 */ "ANYOF", /* 0x15 */ - "ANYOF_WARN_SUPER", /* 0x16 */ - "ANYOF_SYNTHETIC", /* 0x17 */ - "POSIXD", /* 0x18 */ - "POSIXL", /* 0x19 */ - "POSIXU", /* 0x1a */ - "POSIXA", /* 0x1b */ - "NPOSIXD", /* 0x1c */ - "NPOSIXL", /* 0x1d */ - "NPOSIXU", /* 0x1e */ - "NPOSIXA", /* 0x1f */ - "CLUMP", /* 0x20 */ - "BRANCH", /* 0x21 */ - "BACK", /* 0x22 */ - "EXACT", /* 0x23 */ - "EXACTF", /* 0x24 */ - "EXACTFL", /* 0x25 */ - "EXACTFU", /* 0x26 */ - "EXACTFA", /* 0x27 */ - "EXACTFU_SS", /* 0x28 */ - "EXACTFA_NO_TRIE", /* 0x29 */ - "NOTHING", /* 0x2a */ - "TAIL", /* 0x2b */ - "STAR", /* 0x2c */ - "PLUS", /* 0x2d */ - "CURLY", /* 0x2e */ - "CURLYN", /* 0x2f */ - "CURLYM", /* 0x30 */ - "CURLYX", /* 0x31 */ - "WHILEM", /* 0x32 */ - "OPEN", /* 0x33 */ - "CLOSE", /* 0x34 */ - "REF", /* 0x35 */ - "REFF", /* 0x36 */ - "REFFL", /* 0x37 */ - "REFFU", /* 0x38 */ - "REFFA", /* 0x39 */ - "NREF", /* 0x3a */ - "NREFF", /* 0x3b */ - "NREFFL", /* 0x3c */ - "NREFFU", /* 0x3d */ - "NREFFA", /* 0x3e */ - "IFMATCH", /* 0x3f */ - "UNLESSM", /* 0x40 */ - "SUSPEND", /* 0x41 */ - "IFTHEN", /* 0x42 */ - "GROUPP", /* 0x43 */ - "LONGJMP", /* 0x44 */ - "BRANCHJ", /* 0x45 */ - "EVAL", /* 0x46 */ - "MINMOD", /* 0x47 */ - "LOGICAL", /* 0x48 */ - "RENUM", /* 0x49 */ - "TRIE", /* 0x4a */ - "TRIEC", /* 0x4b */ - "AHOCORASICK", /* 0x4c */ - "AHOCORASICKC", /* 0x4d */ - "GOSUB", /* 0x4e */ - "GOSTART", /* 0x4f */ - "NGROUPP", /* 0x50 */ - "INSUBP", /* 0x51 */ - "DEFINEP", /* 0x52 */ - "ENDLIKE", /* 0x53 */ - "OPFAIL", /* 0x54 */ - "ACCEPT", /* 0x55 */ - "VERB", /* 0x56 */ - "PRUNE", /* 0x57 */ - "MARKPOINT", /* 0x58 */ - "SKIP", /* 0x59 */ - "COMMIT", /* 0x5a */ - "CUTGROUP", /* 0x5b */ - "KEEPS", /* 0x5c */ - "LNBREAK", /* 0x5d */ - "OPTIMIZED", /* 0x5e */ - "PSEUDO", /* 0x5f */ + "ANYOF_SYNTHETIC", /* 0x16 */ + "POSIXD", /* 0x17 */ + "POSIXL", /* 0x18 */ + "POSIXU", /* 0x19 */ + "POSIXA", /* 0x1a */ + "NPOSIXD", /* 0x1b */ + "NPOSIXL", /* 0x1c */ + "NPOSIXU", /* 0x1d */ + "NPOSIXA", /* 0x1e */ + "CLUMP", /* 0x1f */ + "BRANCH", /* 0x20 */ + "BACK", /* 0x21 */ + "EXACT", /* 0x22 */ + "EXACTF", /* 0x23 */ + "EXACTFL", /* 0x24 */ + "EXACTFU", /* 0x25 */ + "EXACTFA", /* 0x26 */ + "EXACTFU_SS", /* 0x27 */ + "EXACTFA_NO_TRIE", /* 0x28 */ + "NOTHING", /* 0x29 */ + "TAIL", /* 0x2a */ + "STAR", /* 0x2b */ + "PLUS", /* 0x2c */ + "CURLY", /* 0x2d */ + "CURLYN", /* 0x2e */ + "CURLYM", /* 0x2f */ + "CURLYX", /* 0x30 */ + "WHILEM", /* 0x31 */ + "OPEN", /* 0x32 */ + "CLOSE", /* 0x33 */ + "REF", /* 0x34 */ + "REFF", /* 0x35 */ + "REFFL", /* 0x36 */ + "REFFU", /* 0x37 */ + "REFFA", /* 0x38 */ + "NREF", /* 0x39 */ + "NREFF", /* 0x3a */ + "NREFFL", /* 0x3b */ + "NREFFU", /* 0x3c */ + "NREFFA", /* 0x3d */ + "IFMATCH", /* 0x3e */ + "UNLESSM", /* 0x3f */ + "SUSPEND", /* 0x40 */ + "IFTHEN", /* 0x41 */ + "GROUPP", /* 0x42 */ + "LONGJMP", /* 0x43 */ + "BRANCHJ", /* 0x44 */ + "EVAL", /* 0x45 */ + "MINMOD", /* 0x46 */ + "LOGICAL", /* 0x47 */ + "RENUM", /* 0x48 */ + "TRIE", /* 0x49 */ + "TRIEC", /* 0x4a */ + "AHOCORASICK", /* 0x4b */ + "AHOCORASICKC", /* 0x4c */ + "GOSUB", /* 0x4d */ + "GOSTART", /* 0x4e */ + "NGROUPP", /* 0x4f */ + "INSUBP", /* 0x50 */ + "DEFINEP", /* 0x51 */ + "ENDLIKE", /* 0x52 */ + "OPFAIL", /* 0x53 */ + "ACCEPT", /* 0x54 */ + "VERB", /* 0x55 */ + "PRUNE", /* 0x56 */ + "MARKPOINT", /* 0x57 */ + "SKIP", /* 0x58 */ + "COMMIT", /* 0x59 */ + "CUTGROUP", /* 0x5a */ + "KEEPS", /* 0x5b */ + "LNBREAK", /* 0x5c */ + "OPTIMIZED", /* 0x5d */ + "PSEUDO", /* 0x5e */ /* ------------ States ------------- */ "TRIE_next", /* REGNODE_MAX +0x01 */ "TRIE_next_fail", /* REGNODE_MAX +0x02 */ @@ -719,7 +714,7 @@ EXTCONST U8 PL_varies[] __attribute__deprecated__ = { EXTCONST U8 PL_varies_bitmask[]; #else EXTCONST U8 PL_varies_bitmask[] = { - 0x00, 0x00, 0x00, 0x00, 0x07, 0xF0, 0xE7, 0x7F, 0x26, 0x00, 0x00, 0x00 + 0x00, 0x00, 0x00, 0x80, 0x03, 0xF8, 0xF3, 0x3F, 0x13, 0x00, 0x00, 0x00 }; #endif /* DOINIT */ @@ -731,8 +726,8 @@ EXTCONST U8 PL_varies_bitmask[] = { EXTCONST U8 PL_simple[] __attribute__deprecated__; #else EXTCONST U8 PL_simple[] __attribute__deprecated__ = { - REG_ANY, SANY, CANY, ANYOF, ANYOF_WARN_SUPER, ANYOF_SYNTHETIC, POSIXD, - POSIXL, POSIXU, POSIXA, NPOSIXD, NPOSIXL, NPOSIXU, NPOSIXA, + REG_ANY, SANY, CANY, ANYOF, ANYOF_SYNTHETIC, POSIXD, POSIXL, POSIXU, + POSIXA, NPOSIXD, NPOSIXL, NPOSIXU, NPOSIXA, 0 }; #endif /* DOINIT */ @@ -741,7 +736,7 @@ EXTCONST U8 PL_simple[] __attribute__deprecated__ = { EXTCONST U8 PL_simple_bitmask[]; #else EXTCONST U8 PL_simple_bitmask[] = { - 0x00, 0x00, 0xFC, 0xFF, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 + 0x00, 0x00, 0xFC, 0x7F, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }; #endif /* DOINIT */ diff --git a/t/lib/warnings/utf8 b/t/lib/warnings/utf8 index 02507b3..aefa653 100644 --- a/t/lib/warnings/utf8 +++ b/t/lib/warnings/utf8 @@ -643,6 +643,17 @@ Code point 0x110048 is not Unicode, all \p{} matches fail; all \P{} matches succ Code point 0x110049 is not Unicode, all \p{} matches fail; all \P{} matches succeed at - line 72. Code point 0x110049 is not Unicode, all \p{} matches fail; all \P{} matches succeed at - line 72. ######## +# TODO Matching Unicode property against above-Unicode code point outputs a warning even if optimizer rejects the match (in synthetic start class) +use warnings 'non_unicode'; +"\x{110000}" =~ /b?\p{Space}/; +EXPECT +Code point 0x110000 is not Unicode, all \p{} matches fail; all \P{} matches succeed at - line 2. +######## +# NAME Matching POSIX class property against above-Unicode code point doesn't output a warning +use warnings 'non_unicode'; +"\x{110000}" =~ /b?[[:space:]]/; +EXPECT +######## use warnings 'utf8'; chr(0x110000) =~ /\p{Any}/; no warnings 'non_unicode'; -- 2.7.4