--- /dev/null
+/* Test for UTF-8 regular expression optimizations.
+ Copyright (C) 2003 Free Software Foundation, Inc.
+ This file is part of the GNU C Library.
+ Contributed by Jakub Jelinek <jakub@redhat.com>, 2003.
+
+ The GNU C Library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ The GNU C Library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with the GNU C Library; if not, write to the Free
+ Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA
+ 02111-1307 USA. */
+
+#include <sys/types.h>
+#include <mcheck.h>
+#include <regex.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <locale.h>
+
+#define RE_NO_INTERNAL_PROTOTYPES 1
+#include "regex_internal.h"
+
+static struct
+{
+ int syntax;
+ const char *pattern;
+ const char *string;
+ int res, optimize;
+} tests[] = {
+ /* \xc3\x84 LATIN CAPITAL LETTER A WITH DIAERESIS
+ \xc3\x96 LATIN CAPITAL LETTER O WITH DIAERESIS
+ \xc3\xa4 LATIN SMALL LETTER A WITH DIAERESIS
+ \xc3\xb6 LATIN SMALL LETTER O WITH DIAERESIS
+ \xe2\x80\x94 EM DASH */
+ /* Should be optimized. */
+ {RE_SYNTAX_POSIX_BASIC, "foo", "b\xc3\xa4rfoob\xc3\xa4z", 4, 1},
+ {RE_SYNTAX_POSIX_BASIC, "^x\\|xy*z$", "\xc3\xb6xyyz", 2, 1},
+ {RE_SYNTAX_POSIX_BASIC, "^x\\\\y\\{6\\}z\\+", "x\\yyyyyyzz\xc3\xb6", 0, 1},
+ {RE_SYNTAX_POSIX_BASIC, "^x\\\\y\\{2,36\\}z\\+", "x\\yzz\xc3\xb6", -1, 1},
+ {RE_SYNTAX_POSIX_BASIC, "^x\\\\y\\{,3\\}z\\+", "x\\yyyzz\xc3\xb6", 0, 1},
+ /* FIXME. This one should be optimizable, but is not ATM. */
+ {RE_SYNTAX_POSIX_BASIC, "x[ABC]y", "axCy", 1, 0},
+ {RE_SYNTAX_POSIX_BASIC, "\\`x\\|z\\'", "x\xe2\x80\x94", 0, 1},
+ {RE_SYNTAX_POSIX_BASIC, "\\(xy\\)z\\1a\\1", "\xe2\x80\x94xyzxyaxy\xc3\x84", 3, 1},
+ {RE_SYNTAX_POSIX_BASIC, "xy\\?z", "\xc3\x84xz\xc3\xb6", 2, 1},
+ {RE_SYNTAX_POSIX_EXTENDED, "foo", "b\xc3\xa4rfoob\xc3\xa4z", 4, 1},
+ {RE_SYNTAX_POSIX_EXTENDED, "^x|xy*z$", "\xc3\xb6xyyz", 2, 1},
+ {RE_SYNTAX_POSIX_EXTENDED, "^x\\\\y{6}z+", "x\\yyyyyyzz\xc3\xb6", 0, 1},
+ {RE_SYNTAX_POSIX_EXTENDED, "^x\\\\y{2,36}z+", "x\\yzz\xc3\xb6", -1, 1},
+ {RE_SYNTAX_POSIX_EXTENDED, "^x\\\\y{,3}z+", "x\\yyyzz\xc3\xb6", 0, 1},
+ /* FIXME. This one should be optimizable, but is not ATM. */
+ {RE_SYNTAX_POSIX_EXTENDED, "x[ABC]y", "axCy", 1, 0},
+ {RE_SYNTAX_POSIX_EXTENDED, "\\`x|z\\'", "x\xe2\x80\x94", 0, 1},
+ {RE_SYNTAX_POSIX_EXTENDED, "(xy)z\\1a\\1", "\xe2\x80\x94xyzxyaxy\xc3\x84", 3, 1},
+ {RE_SYNTAX_POSIX_EXTENDED, "xy?z", "\xc3\x84xz\xc3\xb6", 2, 1},
+ /* Should not be optimized. */
+ {RE_SYNTAX_POSIX_BASIC, "x.y", "ax\xe2\x80\x94yz", 1, 0},
+ {RE_SYNTAX_POSIX_BASIC, "x\xc3\x96*y", "ax\xc3\x96\xc3\x96yz", 1, 0},
+ {RE_SYNTAX_POSIX_BASIC, "x[\xc3\x84\xc3\xa4]y", "ax\xc3\xa4y", 1, 0},
+ {RE_SYNTAX_POSIX_BASIC, "x[A-Z,]y", "axCy", 1, 0},
+ {RE_SYNTAX_POSIX_BASIC, "x[^y]z", "ax\xe2\x80\x94z", 1, 0},
+ {RE_SYNTAX_POSIX_BASIC, "x[[:alnum:]]z", "ax\xc3\x96z", 1, 0},
+ {RE_SYNTAX_POSIX_BASIC, "\\<g", "\xe2\x80\x94g", 3, 0},
+ {RE_SYNTAX_POSIX_BASIC, "\\bg\\b", "\xe2\x80\x94g", 3, 0},
+ {RE_SYNTAX_POSIX_BASIC, "\\Bg\\B", "\xc3\xa4g\xc3\xa4", 2, 0},
+ {RE_SYNTAX_POSIX_BASIC, "a\\wz", "a\xc3\x84z", 0, 0},
+ {RE_SYNTAX_POSIX_BASIC, "x\\Wz", "\xc3\x96x\xe2\x80\x94z", 2, 0},
+ {RE_SYNTAX_POSIX_EXTENDED, "x.y", "ax\xe2\x80\x94yz", 1, 0},
+ {RE_SYNTAX_POSIX_EXTENDED, "x\xc3\x96*y", "ax\xc3\x96\xc3\x96yz", 1, 0},
+ {RE_SYNTAX_POSIX_EXTENDED, "x[\xc3\x84\xc3\xa4]y", "ax\xc3\xa4y", 1, 0},
+ {RE_SYNTAX_POSIX_EXTENDED, "x[A-Z,]y", "axCy", 1, 0},
+ {RE_SYNTAX_POSIX_EXTENDED, "x[^y]z", "ax\xe2\x80\x94z", 1, 0},
+ {RE_SYNTAX_POSIX_EXTENDED, "x[[:alnum:]]z", "ax\xc3\x96z", 1, 0},
+ {RE_SYNTAX_POSIX_EXTENDED, "\\<g", "\xe2\x80\x94g", 3, 0},
+ {RE_SYNTAX_POSIX_EXTENDED, "\\bg\\b", "\xe2\x80\x94g", 3, 0},
+ {RE_SYNTAX_POSIX_EXTENDED, "\\Bg\\B", "\xc3\xa4g\xc3\xa4", 2, 0},
+ {RE_SYNTAX_POSIX_EXTENDED, "a\\wz", "a\xc3\x84z", 0, 0},
+ {RE_SYNTAX_POSIX_EXTENDED, "x\\Wz", "\xc3\x96x\xe2\x80\x94z", 2, 0},
+};
+
+int
+main (void)
+{
+ struct re_pattern_buffer regbuf;
+ const char *err;
+ size_t i;
+ int ret = 0;
+
+ mtrace ();
+
+ setlocale (LC_ALL, "de_DE.UTF-8");
+ for (i = 0; i < sizeof (tests) / sizeof (tests[0]); ++i)
+ {
+ int res, optimized;
+ re_set_syntax (tests[i].syntax);
+ memset (®buf, '\0', sizeof (regbuf));
+ err = re_compile_pattern (tests[i].pattern, strlen (tests[i].pattern),
+ ®buf);
+ if (err != NULL)
+ {
+ printf ("re_compile_pattern failed: %s\n", err);
+ ret = 1;
+ continue;
+ }
+
+ /* Check if re_search will be done as multi-byte or single-byte. */
+ optimized = ((re_dfa_t *) regbuf.buffer)->mb_cur_max == 1;
+ if (optimized != tests[i].optimize)
+ {
+ printf ("pattern %zd %soptimized while it should%s be\n",
+ i, optimized ? "" : "not ", tests[i].optimize ? "" : " not");
+ ret = 1;
+ }
+
+ res = re_search (®buf, tests[i].string, strlen (tests[i].string), 0,
+ strlen (tests[i].string), NULL);
+ if (res != tests[i].res)
+ {
+ printf ("re_search %zd failed: %d\n", i, res);
+ ret = 1;
+ regfree (®buf);
+ continue;
+ }
+ regfree (®buf);
+ }
+
+ return ret;
+}
#endif /* RE_ENABLE_I18N */
static void free_workarea_compile (regex_t *preg);
static reg_errcode_t create_initial_state (re_dfa_t *dfa);
+#ifdef RE_ENABLE_I18N
+static void optimize_utf8 (re_dfa_t *dfa);
+#endif
static reg_errcode_t analyze (re_dfa_t *dfa);
static reg_errcode_t analyze_tree (re_dfa_t *dfa, bin_tree_t *node);
static void calc_first (re_dfa_t *dfa, bin_tree_t *node);
{
re_set_fastmap (fastmap, icase, dfa->nodes[node].opr.c);
#ifdef RE_ENABLE_I18N
- if ((bufp->syntax & RE_ICASE) && !icase)
+ if ((bufp->syntax & RE_ICASE) && dfa->mb_cur_max > 1)
{
unsigned char *buf = alloca (dfa->mb_cur_max), *p;
wchar_t wc;
memset (&state, '\0', sizeof (state));
__wcrtomb (buf, cset->mbchars[i], &state);
re_set_fastmap (fastmap, icase, *(unsigned char *) buf);
- if ((bufp->syntax & RE_ICASE) && !icase)
+ if ((bufp->syntax & RE_ICASE) && dfa->mb_cur_max > 1)
{
__wcrtomb (buf, towlower (cset->mbchars[i]), &state);
re_set_fastmap (fastmap, 0, *(unsigned char *) buf);
if (BE (dfa->str_tree == NULL, 0))
goto re_compile_internal_free_return;
+#ifdef RE_ENABLE_I18N
+ /* If possible, do searching in single byte encoding to speed things up. */
+ if (dfa->is_utf8 && !(syntax & RE_ICASE))
+ optimize_utf8 (dfa);
+#endif
+
/* Analyze the tree and collect information which is necessary to
create the dfa. */
err = analyze (dfa);
return REG_NOERROR;
}
\f
+#ifdef RE_ENABLE_I18N
+/* If it is possible to do searching in single byte encoding instead of UTF-8
+ to speed things up, set dfa->mb_cur_max to 1, clear is_utf8 and change
+ DFA nodes where needed. */
+
+static void
+optimize_utf8 (dfa)
+ re_dfa_t *dfa;
+{
+ int node;
+
+ for (node = 0; node < dfa->nodes_len; ++node)
+ switch (dfa->nodes[node].type)
+ {
+ case CHARACTER:
+ /* Chars >= 0x80 are optimizable in some cases (e.g. when not
+ followed by DUP operator, not in bracket etc.).
+ For now punt on them all. */
+ if (dfa->nodes[node].opr.c >= 0x80)
+ return;
+ break;
+ case ANCHOR:
+ switch (dfa->nodes[node].opr.idx)
+ {
+ case LINE_FIRST:
+ case LINE_LAST:
+ case BUF_FIRST:
+ case BUF_LAST:
+ break;
+ default:
+ /* Word anchors etc. cannot be handled. */
+ return;
+ }
+ break;
+ case OP_BACK_REF:
+ case OP_ALT:
+ case END_OF_RE:
+ case BACK_SLASH:
+ case OP_DUP_ASTERISK:
+ case OP_DUP_QUESTION:
+ case OP_DUP_PLUS:
+ case OP_OPEN_SUBEXP:
+ case OP_CLOSE_SUBEXP:
+ break;
+ default:
+ return;
+ }
+
+ /* The search can be in single byte locale. */
+ dfa->mb_cur_max = 1;
+ dfa->is_utf8 = 0;
+ dfa->has_mb_node = dfa->nbackref > 0;
+}
+#endif
+\f
/* Analyze the structure tree, and calculate "first", "next", "edest",
"eclosure", and "inveclosure". */