Make \N{unknown char} a syntax error

Previously, it was a warning with the REPLACEMENT CHARACTER substituted.
Unicode recommends that it be a syntax error, and any code that used
this had to be buggy since the REPLACEMENT CHARACTER has no other use in
Unicode.

diff --git a/lib/_charnames.pm b/lib/_charnames.pm
index 5431d0f..347ad27 100644 (file)
--- a/lib/_charnames.pm
+++ b/lib/_charnames.pm
@@ -7,7 +7,7 @@ package _charnames;
 use strict;
 use warnings;
 use File::Spec;
-our $VERSION = '1.32';
+our $VERSION = '1.33';
 use unicore::Name;    # mktables-generated algorithmically-defined names
 
 use bytes ();          # for $bytes::hint_bits
@@ -454,18 +454,8 @@ sub lookup_name ($$$) {
           }
 
           my $case = $name_has_uppercase ? "CAPITAL" : "SMALL";
-          if (! $scripts_trie
-              || $txt !~
-              /\t (?: $scripts_trie ) \ (?:$case\ )? LETTER \ \U$lookup_name $/xm)
-          {
-            # Here we still don't have it, give up.
-            return if $runtime;
-
-            # May have zapped input name, get it again.
-            $name = (defined $save_input) ? $save_input : $_[0];
-            carp "Unknown charname '$name'";
-            return ($wants_ord) ? 0xFFFD : pack("U", 0xFFFD);
-          }
+          return if (! $scripts_trie || $txt !~
+             /\t (?: $scripts_trie ) \ (?:$case\ )? LETTER \ \U$lookup_name $/xm);
 
           # Here have found the input name in the table.
           @off = ($-[0] + 1, $+[0]);  # The 1 is for the tab

diff --git a/lib/charnames.pm b/lib/charnames.pm
index bef3e1a..5c4ff3f 100644 (file)
--- a/lib/charnames.pm
+++ b/lib/charnames.pm
@@ -1,7 +1,7 @@
 package charnames;
 use strict;
 use warnings;
-our $VERSION = '1.32';
+our $VERSION = '1.33';
 use unicore::Name;    # mktables-generated algorithmically-defined names
 use _charnames ();    # The submodule for this where most of the work gets done
 
@@ -214,8 +214,7 @@ L<charnames::string_vianame()|/charnames::string_vianame(I<name>)>.
 Since Unicode 6.0, it is deprecated to use C<BELL>.  Instead use C<ALERT> (but
 C<BEL> will continue to work).
 
-If the input name is unknown, C<\N{NAME}> raises a warning and
-substitutes the Unicode REPLACEMENT CHARACTER (U+FFFD).
+It is a syntax error to use C<\N{NAME}> where C<NAME> is unknown.
 
 For C<\N{NAME}>, it is a fatal error if C<use bytes> is in effect and the
 input name is that of a character that won't fit into a byte (i.e., whose
@@ -338,8 +337,7 @@ L<script list, C<:short> option|/DESCRIPTION>, or L<custom aliases|/CUSTOM
 ALIASES> you may have defined.
 
 The only difference is that if the input name is unknown, C<string_vianame>
-returns C<undef> instead of the REPLACEMENT CHARACTER and does not raise a
-warning message.
+returns C<undef> instead of it being a syntax error.
 
 =head1 charnames::vianame(I<name>)
 

diff --git a/lib/charnames.t b/lib/charnames.t
index a158c9a..63cfc25 100644 (file)
--- a/lib/charnames.t
+++ b/lib/charnames.t
@@ -836,7 +836,8 @@ is("\N{U+1D0C5}", "\N{BYZANTINE MUSICAL SYMBOL FTHORA SKLIRON CHROMA VASIS}", 'V
         is("\N{mychar1}", "f", "Inner block: verify that \\N{mychar1} is redefined");
         is(charnames::vianame("mychar1"), ord("f"), "Inner block: verify that vianame(mychar1) is redefined");
         is(charnames::string_vianame("mychar1"), "f", "Inner block: verify that string_vianame(mychar1) is redefined");
-        is("\N{mychar2}", "\x{FFFD}", "Inner block: verify that \\N{mychar2} outer definition didn't leak");
+        eval '"\N{mychar2}"';
+        like($@, qr/Unknown charname 'mychar2'/, "Inner block: verify that \\N{mychar2} outer definition didn't leak");
         ok( ! defined charnames::vianame("mychar2"), "Inner block: verify that vianame(mychar2) outer definition didn't leak");
         ok( ! defined charnames::string_vianame("mychar2"), "Inner block: verify that string_vianame(mychar2) outer definition didn't leak");
         is("\N{myprivate1}", "\x{E8001}", "Inner block: verify that \\N{myprivate1} is redefined ");
@@ -844,38 +845,46 @@ is("\N{U+1D0C5}", "\N{BYZANTINE MUSICAL SYMBOL FTHORA SKLIRON CHROMA VASIS}", 'V
         is(charnames::string_vianame("myprivate1"), chr(0xE8001), "Inner block: verify that string_vianame(myprivate1) is redefined");
         is(charnames::viacode(0xE8001), "myprivate1", "Inner block: verify that myprivate1 viacode is redefined");
         ok(! defined charnames::viacode(0xE8000), "Inner block: verify that outer myprivate1 viacode didn't leak");
-        is("\N{myprivate2}", "\x{FFFD}", "Inner block: verify that \\N{myprivate2} outer definition didn't leak");
+        eval '"\N{myprivate2}"';
+        like($@, qr/Unknown charname 'myprivate2'/, "Inner block: verify that \\N{myprivate2} outer definition didn't leak");
         ok(! defined charnames::vianame("myprivate2"), "Inner block: verify that vianame(myprivate2) outer definition didn't leak");
         ok(! defined charnames::string_vianame("myprivate2"), "Inner block: verify that string_vianame(myprivate2) outer definition didn't leak");
         ok(! defined charnames::viacode(0x100000), "Inner block: verify that myprivate2 viacode outer definition didn't leak");
         is("\N{BE}", $hiragana_be, "Inner block: verify that \\N uses the correct script");
         cmp_ok(charnames::vianame("BE"), "==", ord($hiragana_be), "Inner block: verify that vianame uses the correct script");
         cmp_ok(charnames::string_vianame("BE"), "==", $hiragana_be, "Inner block: verify that string_vianame uses the correct script");
-        is("\N{Hiragana: BE}", "\x{FFFD}", "Inner block without :short: \\N with short doesn't work");
+        eval '"\N{Hiragana: BE}"';
+        like($@, qr/Unknown charname 'Hiragana: BE'/, "Inner block without :short: \\N with short doesn't work");
         ok(! defined charnames::vianame("Hiragana: BE"), "Inner block without :short: verify that vianame with short doesn't work");
         ok(! defined charnames::string_vianame("Hiragana: BE"), "Inner block without :short: verify that string_vianame with short doesn't work");
 
         {   # An inner block where only :short definitions are valid.
             use charnames ":short";
-            is("\N{mychar1}", "\x{FFFD}", "Inner inner block: verify that mychar1 outer definition didn't leak with \\N");
+            eval '"\N{mychar1}"';
+            like($@, qr/Unknown charname 'mychar1'/, "Inner inner block: verify that mychar1 outer definition didn't leak with \\N");
             ok( ! defined charnames::vianame("mychar1"), "Inner inner block: verify that mychar1 outer definition didn't leak with vianame");
             ok( ! defined charnames::string_vianame("mychar1"), "Inner inner block: verify that mychar1 outer definition didn't leak with string_vianame");
-            is("\N{mychar2}", "\x{FFFD}", "Inner inner block: verify that mychar2 outer definition didn't leak with \\N");
+            eval '"\N{mychar2}"';
+            like($@, qr/Unknown charname 'mychar2'/, "Inner inner block: verify that mychar2 outer definition didn't leak with \\N");
             ok( ! defined charnames::vianame("mychar2"), "Inner inner block: verify that mychar2 outer definition didn't leak with vianame");
             ok( ! defined charnames::string_vianame("mychar2"), "Inner inner block: verify that mychar2 outer definition didn't leak with string_vianame");
-            is("\N{myprivate1}", "\x{FFFD}", "Inner inner block: verify that myprivate1 outer definition didn't leak with \\N");
+            eval '"\N{myprivate1}"';
+            like($@, qr/Unknown charname 'myprivate1'/, "Inner inner block: verify that myprivate1 outer definition didn't leak with \\N");
             ok(! defined charnames::vianame("myprivate1"), "Inner inner block: verify that myprivate1 outer definition didn't leak with vianame");
             ok(! defined charnames::string_vianame("myprivate1"), "Inner inner block: verify that myprivate1 outer definition didn't leak with string_vianame");
-            is("\N{myprivate2}", "\x{FFFD}", "Inner inner block: verify that myprivate2 outer definition didn't leak with \\N");
+            eval '"\N{myprivate2}"';
+            like($@, qr/Unknown charname 'myprivate2'/, "Inner inner block: verify that myprivate2 outer definition didn't leak with \\N");
             ok(! defined charnames::vianame("myprivate2"), "Inner inner block: verify that myprivate2 outer definition didn't leak with vianame");
             ok(! defined charnames::string_vianame("myprivate2"), "Inner inner block: verify that myprivate2 outer definition didn't leak with string_vianame");
             ok(! defined charnames::viacode(0xE8000), "Inner inner block: verify that mychar1 outer outer definition didn't leak with viacode");
             ok(! defined charnames::viacode(0xE8001), "Inner inner block: verify that mychar1 outer definition didn't leak with viacode");
             ok(! defined charnames::viacode(0x100000), "Inner inner block: verify that mychar2 outer definition didn't leak with viacode");
-            is("\N{BE}", "\x{FFFD}", "Inner inner block without script: verify that outer :script didn't leak with \\N");
+            eval '"\N{BE}"';
+            like($@, qr/Unknown charname 'BE'/, "Inner inner block without script: verify that outer :script didn't leak with \\N");
             ok(! defined charnames::vianame("BE"), "Inner inner block without script: verify that outer :script didn't leak with vianames");
             ok(! defined charnames::string_vianame("BE"), "Inner inner block without script: verify that outer :script didn't leak with string_vianames");
-            is("\N{HIRAGANA LETTER BE}", "\x{FFFD}", "Inner inner block without :full: verify that outer :full didn't leak with \\N");
+            eval '"\N{HIRAGANA LETTER BE}"';
+            like($@, qr/Unknown charname 'HIRAGANA LETTER BE'/, "Inner inner block without :full: verify that outer :full didn't leak with \\N");
             is("\N{Hiragana: BE}", $hiragana_be, "Inner inner block with :short: verify that \\N works with :short");
             cmp_ok(charnames::vianame("Hiragana: BE"), "==", ord($hiragana_be), "Inner inner block with :short: verify that vianame works with :short");
             cmp_ok(charnames::string_vianame("Hiragana: BE"), "==", $hiragana_be, "Inner inner block with :short: verify that string_vianame works with :short");
@@ -885,7 +894,8 @@ is("\N{U+1D0C5}", "\N{BYZANTINE MUSICAL SYMBOL FTHORA SKLIRON CHROMA VASIS}", 'V
         is("\N{mychar1}", "f", "Inner block: verify that \\N{mychar1} is redefined");
         is(charnames::vianame("mychar1"), ord("f"), "Inner block: verify that vianame(mychar1) is redefined");
         is(charnames::string_vianame("mychar1"), "f", "Inner block: verify that string_vianame(mychar1) is redefined");
-        is("\N{mychar2}", "\x{FFFD}", "Inner block: verify that \\N{mychar2} outer definition didn't leak");
+        eval '"\N{mychar2}"';
+        like($@, qr/Unknown charname 'mychar2'/, "Inner block: verify that \\N{mychar2} outer definition didn't leak");
         ok( ! defined charnames::vianame("mychar2"), "Inner block: verify that vianame(mychar2) outer definition didn't leak");
         ok( ! defined charnames::string_vianame("mychar2"), "Inner block: verify that string_vianame(mychar2) outer definition didn't leak");
         is("\N{myprivate1}", "\x{E8001}", "Inner block: verify that \\N{myprivate1} is redefined ");
@@ -893,14 +903,16 @@ is("\N{U+1D0C5}", "\N{BYZANTINE MUSICAL SYMBOL FTHORA SKLIRON CHROMA VASIS}", 'V
         is(charnames::string_vianame("myprivate1"), chr(0xE8001), "Inner block: verify that string_vianame(myprivate1) is redefined");
         is(charnames::viacode(0xE8001), "myprivate1", "Inner block: verify that myprivate1 viacode is redefined");
         ok(! defined charnames::viacode(0xE8000), "Inner block: verify that outer myprivate1 viacode didn't leak");
-        is("\N{myprivate2}", "\x{FFFD}", "Inner block: verify that \\N{myprivate2} outer definition didn't leak");
+        eval '"\N{myprivate2}"';
+        like($@, qr/Unknown charname 'myprivate2'/, "Inner block: verify that \\N{myprivate2} outer definition didn't leak");
         ok(! defined charnames::vianame("myprivate2"), "Inner block: verify that vianame(myprivate2) outer definition didn't leak");
         ok(! defined charnames::string_vianame("myprivate2"), "Inner block: verify that string_vianame(myprivate2) outer definition didn't leak");
         ok(! defined charnames::viacode(0x100000), "Inner block: verify that myprivate2 viacode outer definition didn't leak");
         is("\N{BE}", $hiragana_be, "Inner block: verify that \\N uses the correct script");
         cmp_ok(charnames::vianame("BE"), "==", ord($hiragana_be), "Inner block: verify that vianame uses the correct script");
         cmp_ok(charnames::string_vianame("BE"), "==", $hiragana_be, "Inner block: verify that string_vianame uses the correct script");
-        is("\N{Hiragana: BE}", "\x{FFFD}", "Inner block without :short: \\N with short doesn't work");
+        eval '"\N{Hiragana: BE}"';
+        like($@, qr/Unknown charname 'Hiragana: BE'/, "Inner block without :short: \\N with short doesn't work");
         ok(! defined charnames::vianame("Hiragana: BE"), "Inner block without :short: verify that vianame with short doesn't work");
         ok(! defined charnames::string_vianame("Hiragana: BE"), "Inner block without :short: verify that string_vianame with short doesn't work");
     }
@@ -933,7 +945,8 @@ is("\N{U+1D0C5}", "\N{BYZANTINE MUSICAL SYMBOL FTHORA SKLIRON CHROMA VASIS}", 'V
         is(charnames::string_vianame("O-i"), chr(0x10426), "Verify that loose script list matching works with string_vianame");
         is(charnames::vianame("o i"), 0x1044E, "Verify that loose script list matching works with vianame");
     }
-    is ("\N{latincapitallettera}", "\x{FFFD}", "Verify that loose matching caching doesn't leak outside of scope");
+    eval '"\N{latincapitallettera}"';
+    like($@, qr/Unknown charname 'latincapitallettera'/, "Verify that loose matching caching doesn't leak outside of scope");
     {
         use charnames qw(:loose :short);
         cmp_ok("\N{co pt-ic:she-i}", "==", chr(0x3E3), "Verify that loose :short matching works");

diff --git a/pod/perldelta.pod b/pod/perldelta.pod
index 601da5e..81e75ec 100644 (file)
--- a/pod/perldelta.pod
+++ b/pod/perldelta.pod
@@ -45,6 +45,15 @@ XXX For a release on a stable branch, this section aspires to be:
 
 [ List each incompatible change as a =head2 entry ]
 
+=head2 An unknown character name in C<\N{...}> is now a syntax error
+
+Previously, it warned, and the Unicode REPLACEMENT CHARACTER was
+substituted.  Unicode now recommends that this situation be a syntax
+error.  Also, the previous behavior led to some confusing warnings and
+behaviors, and since the REPLACEMENT CHARACTER has no use other than as
+a stand-in for some unknown character, any code that has this problem is
+buggy.
+
 =head1 Deprecations
 
 XXX Any deprecated features, syntax, modules etc. should be listed here.  In

diff --git a/t/lib/charnames/alias b/t/lib/charnames/alias
index 75280be..5e8608e 100644 (file)
--- a/t/lib/charnames/alias
+++ b/t/lib/charnames/alias
@@ -36,8 +36,9 @@ no warnings;
 use charnames "alias";
 "Here: \N{e_ACUTE}!\n";
 EXPECT
-OPTIONS regex
-Unknown charname 'e_ACUTE' at
+OPTIONS random fatal
+Unknown charname 'e_ACUTE' at - line 3, within string
+Execution of - aborted due to compilation errors.
 ########
 # NAME alias without an argument
 use warnings;
@@ -60,8 +61,10 @@ use warnings;
 use charnames ":alias" => { e_ACUTE => "LATIN SMALL LETTER E WITH ACUTE" };
 "Here: \N{e_ACUTE}!\n";
 EXPECT
-OPTIONS regex
-Unknown charname 'LATIN SMALL LETTER E WITH ACUTE' at
+OPTIONS random fatal
+Use of uninitialized value at - line 3.
+Unknown charname 'e_ACUTE' at - line 3, within string
+Execution of - aborted due to compilation errors.
 ########
 # NAME alias with hashref but with :short
 use warnings;
@@ -69,8 +72,10 @@ no warnings 'void';
 use charnames ":short", ":alias" => { e_ACUTE => "LATIN SMALL LETTER E WITH ACUTE" };
 "Here: \N{e_ACUTE}!\n";
 EXPECT
-OPTIONS regex
-Unknown charname 'LATIN SMALL LETTER E WITH ACUTE' at
+OPTIONS random fatal
+Use of uninitialized value at - line 4.
+Unknown charname 'e_ACUTE' at - line 4, within string
+Execution of - aborted due to compilation errors.
 ########
 # NAME alias with hashref to :full OK
 use warnings;
@@ -96,8 +101,10 @@ no warnings 'void';
 use charnames ":loose", ":alias" => { e_ACUTE => "latin SMALL LETTER E WITH ACUTE" };
 "Here: \N{e_ACUTE}!\n";
 EXPECT
-OPTIONS regex
-Unknown charname 'latin SMALL LETTER E WITH ACUTE' at
+OPTIONS random fatal
+Use of uninitialized value at - line 4.
+Unknown charname 'e_ACUTE' at - line 4, within string
+Execution of - aborted due to compilation errors.
 ########
 # NAME alias with hashref to :short but using :full
 use warnings;
@@ -105,8 +112,10 @@ no warnings 'void';
 use charnames ":full", ":alias" => { e_ACUTE => "LATIN:e WITH ACUTE" };
 "Here: \N{e_ACUTE}!\n";
 EXPECT
-OPTIONS regex
-Unknown charname 'LATIN:e WITH ACUTE' at
+OPTIONS random fatal
+Use of uninitialized value at - line 4.
+Unknown charname 'e_ACUTE' at - line 4, within string
+Execution of - aborted due to compilation errors.
 ########
 # NAME alias with hashref to :short OK
 use warnings;
@@ -152,8 +161,10 @@ use charnames ":short", ":alias" => {
     };
 "Here: \N{e_ACUTE}\N{a_ACUTE}!\n";
 EXPECT
-OPTIONS regex
-Unknown charname '' at
+OPTIONS random fatal
+Use of uninitialized value at - line 7.
+Unknown charname 'a_ACUTE' at - line 7, within string
+Execution of - aborted due to compilation errors.
 ########
 # NAME alias with hashref two aliases
 use warnings;
@@ -175,8 +186,10 @@ use charnames ":short", ":alias" => {
     };
 "Here: \N{e_ACUTE}\N{a_ACUTE}!\n";
 EXPECT
-OPTIONS regex
-Unknown charname 'LATIN SMALL LETTER A WITH ACUT' at
+OPTIONS random fatal
+Use of uninitialized value at - line 6.
+Unknown charname 'a_ACUTE' at - line 6, within string
+Execution of - aborted due to compilation errors.
 ########
 # NAME alias with hashref using mixed aliases
 use warnings;
@@ -186,8 +199,10 @@ use charnames ":short", ":alias" => {
     };
 "Here: \N{e_ACUTE}\N{a_ACUTE}!\n";
 EXPECT
-OPTIONS regex
-Unknown charname 'LATIN SMALL LETTER A WITH ACUTE' at
+OPTIONS random fatal
+Use of uninitialized value at - line 6.
+Unknown charname 'a_ACUTE' at - line 6, within string
+Execution of - aborted due to compilation errors.
 ########
 # NAME alias with hashref using mixed aliases
 use warnings;
@@ -198,8 +213,11 @@ use charnames ":full", ":alias" => {
     };
 "Here: \N{e_ACUTE}\N{a_ACUTE}!\n";
 EXPECT
-OPTIONS regex
-Unknown charname 'LATIN:e WITH ACUTE' at
+OPTIONS random fatal
+Use of uninitialized value at - line 7.
+Use of uninitialized value at - line 7.
+Unknown charname 'e_ACUTE' at - line 7, within string
+Execution of - aborted due to compilation errors.
 ########
 # NAME alias with nonexisting file
 use warnings;
@@ -251,8 +269,11 @@ no warnings 'void';
 use charnames ":full", ":alias" => "xyzzy";
 "Here: \N{e_ACUTE}\N{a_ACUTE}!\n";
 EXPECT
-OPTIONS regex
-Unknown charname 'e_ACUTE' at
+OPTIONS random fatal
+Use of uninitialized value at - line 4.
+Use of uninitialized value at - line 4.
+Unknown charname 'e_ACUTE' at - line 4, within string
+Execution of - aborted due to compilation errors.
 ########
 # alias with file OK but file has :short aliases
 --FILE-- ../../lib/unicore/xyzzy_alias.pl
@@ -266,8 +287,10 @@ no warnings 'void';
 use charnames ":full", ":alias" => "xyzzy";
 "Here: \N{e_ACUTE}\N{a_ACUTE}!\n";
 EXPECT
-OPTIONS regex
-Unknown charname 'LATIN:e WITH ACUTE' at
+Use of uninitialized value at - line 4.
+Use of uninitialized value at - line 4.
+Unknown charname 'e_ACUTE' at - line 4, within string
+Execution of - aborted due to compilation errors.
 ########
 # alias with :short and file OK
 --FILE-- ../../lib/unicore/xyzzy_alias.pl
@@ -296,8 +319,11 @@ no warnings 'void';
 use charnames ":short", ":alias" => "xyzzy";
 "Here: \N{e_ACUTE}\N{a_ACUTE}!\n";
 EXPECT
-OPTIONS regex
-Unknown charname 'LATIN SMALL LETTER E WITH ACUTE' at
+OPTIONS random fatal
+Use of uninitialized value at - line 4.
+Use of uninitialized value at - line 4.
+Unknown charname 'e_ACUTE' at - line 4, within string
+Execution of - aborted due to compilation errors.
 ########
 # alias with file implicit :full but file has :short aliases
 --FILE-- ../../lib/unicore/xyzzy_alias.pl
@@ -311,8 +337,10 @@ no warnings 'void';
 use charnames ":alias" => ":xyzzy";
 "Here: \N{e_ACUTE}\N{a_ACUTE}!\n";
 EXPECT
-OPTIONS regex
-Unknown charname 'LATIN:e WITH ACUTE' at
+Use of uninitialized value at - line 4.
+Use of uninitialized value at - line 4.
+Unknown charname 'e_ACUTE' at - line 4, within string
+Execution of - aborted due to compilation errors.
 ########
 # alias with file implicit :full and file has :long aliases
 --FILE-- ../../lib/unicore/xyzzy_alias.pl

diff --git a/toke.c b/toke.c
index 63b6d02..97b2170 100644 (file)
--- a/toke.c
+++ b/toke.c
@@ -8930,8 +8930,15 @@ S_new_constant(pTHX_ const char *s, STRLEN len, const char *key, STRLEN keylen,
             why2 = key;
             why3 = "} is not defined";
         report:
-            msg = Perl_newSVpvf(aTHX_ "Constant(%s): %s%s%s",
+            if (strEQ(key,"charnames")) {
+                msg = Perl_newSVpvf(aTHX_
+                        /* The +3 is for '\N{'; -4 for that, plus '}' */
+                        "Unknown charname '%.*s'", typelen - 4, type + 3);
+            }
+            else {
+                msg = Perl_newSVpvf(aTHX_ "Constant(%s): %s%s%s",
                                 (type ? type: "undef"), why1, why2, why3);
+            }
         }
        yyerror(SvPVX_const(msg));
        SvREFCNT_dec(msg);