libs/locale/test/test_codepage_converter.cpp

   1 //
   2 //  Copyright (c) 2009-2011 Artyom Beilis (Tonkikh)
   3 //
   4 //  Distributed under the Boost Software License, Version 1.0. (See
   5 //  accompanying file LICENSE_1_0.txt or copy at
   6 //  http://www.boost.org/LICENSE_1_0.txt)
   7 //
   8 #include "test_locale.hpp"
   9 #include "test_locale_tools.hpp"
  10 #include <boost/locale/util.hpp>
  11 #ifdef BOOST_LOCALE_WITH_ICU
  12 #include "../src/icu/codecvt.hpp"
  13 #endif
  14 #if defined(BOOST_LOCALE_WITH_ICONV) && !defined(BOOST_LOCALE_NO_POSIX_BACKEND)
  15 #include "../src/posix/codecvt.hpp"
  16 #endif
  17
  18 #include <string.h>
  19
  20 char *make2(unsigned v)
  21 {
  22     static unsigned char buf[3] = {0};
  23     buf[0] = 0xC0 | (v >> 6);
  24     buf[1] = 0x80 | (v & 0x3F );
  25     return reinterpret_cast<char*>(buf);
  26 }
  27
  28 char *make3(unsigned v)
  29 {
  30     static unsigned char buf[4] = {0};
  31     buf[0] = 0xE0 | ((v >> 12) ) ;
  32     buf[1] = 0x80 | ((v >>  6) & 0x3F );
  33     buf[2] = 0x80 | ((v >>  0) & 0x3F );
  34     return reinterpret_cast<char*>(buf);
  35 }
  36
  37 char *make4(unsigned v)
  38 {
  39     static unsigned char buf[5] = {0};
  40     buf[0] = 0xF0 | ((v >> 18) ) ;
  41     buf[1] = 0x80 | ((v >> 12) & 0x3F );
  42     buf[2] = 0x80 | ((v >>  6) & 0x3F );
  43     buf[3] = 0x80 | ((v >>  0) & 0x3F );
  44     return reinterpret_cast<char*>(buf);
  45 }
  46
  47 static const unsigned illegal=0xFFFFFFFF;
  48 static const unsigned incomplete=0xFFFFFFFE;
  49
  50
  51 bool test_to(boost::locale::util::base_converter &cvt,char const *s,unsigned codepoint)
  52 {
  53     size_t len = strlen(s);
  54     char const *end = s + len;
  55     return cvt.to_unicode(s,end) == codepoint;
  56 }
  57
  58 bool test_from(boost::locale::util::base_converter &cvt,unsigned codepoint,char const *str)
  59 {
  60     char buf[32];
  61     unsigned res = cvt.from_unicode(codepoint,buf,buf+sizeof(buf));
  62     if(res == boost::locale::util::base_converter::illegal) {
  63         return str == 0;
  64     }
  65     else {
  66         return str!=0 && strlen(str) == res && memcmp(str,buf,res) == 0;
  67     }
  68 }
  69
  70 bool test_incomplete(boost::locale::util::base_converter &cvt,unsigned codepoint,int len)
  71 {
  72     char buf[32];
  73     unsigned res = cvt.from_unicode(codepoint,buf,buf+len);
  74     return res == incomplete;
  75 }
  76
  77
  78
  79 #define TEST_TO(str,codepoint) TEST(test_to(*cvt,str,codepoint))
  80 #define TEST_FROM(str,codepoint) TEST(test_from(*cvt,codepoint,str))
  81 #define TEST_INC(codepoint,len) TEST(test_incomplete(*cvt,codepoint,len))
  82
  83 void test_shiftjis(std::auto_ptr<boost::locale::util::base_converter> cvt)
  84 {
  85         std::cout << "- Correct" << std::endl;
  86         TEST_TO("a",'a');
  87         TEST_TO("X",'X');
  88         TEST_TO("\xCB",0xFF8b); // half width katakana Hi ヒ
  89         TEST_TO("\x83\x71",0x30d2); // Full width katakana Hi ヒ
  90         TEST_TO("\x82\xd0",0x3072); // Full width hiragana Hi ひ
  91
  92         TEST_FROM("a",'a');
  93         TEST_FROM("X",'X');
  94         TEST_FROM("\xCB",0xFF8b); // half width katakana Hi ヒ
  95         TEST_FROM("\x83\x71",0x30d2); // Full width katakana Hi ヒ
  96         TEST_FROM("\x82\xd0",0x3072); // Full width hiragana Hi ひ
  97
  98         std::cout << "- Illegal/incomplete" << std::endl;
  99
 100         TEST_TO("\xa0",illegal);
 101         TEST_TO("\x82",incomplete);
 102         TEST_TO("\x83\xf0",illegal);
 103
 104         TEST_INC(0x30d2,1); // Full width katakana Hi ヒ
 105         TEST_INC(0x3072,1); // Full width hiragana Hi ひ
 106
 107         TEST_FROM(0,0x5e9); // Hebrew ש not in ShiftJIS
 108 }
 109
 110
 111 int main()
 112 {
 113     try {
 114         using namespace boost::locale::util;
 115
 116         std::auto_ptr<base_converter> cvt;
 117
 118         std::cout << "Test UTF-8" << std::endl;
 119         std::cout << "- From UTF-8" << std::endl;
 120
 121
 122         cvt = create_utf8_converter();
 123         TEST(cvt.get());
 124         TEST(cvt->is_thread_safe());
 125         TEST(cvt->max_len() == 4);
 126
 127         std::cout << "-- Correct" << std::endl;
 128
 129         TEST_TO("\x7f",0x7f);
 130         TEST_TO("\xC2\x80",0x80);
 131         TEST_TO("\xdf\xBF",0x7FF);
 132         TEST_TO("\xe0\xa0\x80",0x800);
 133         TEST_TO("\xef\xbf\xbf",0xFFFF);
 134         TEST_TO("\xf0\x90\x80\x80",0x10000);
 135         TEST_TO("\xf4\x8f\xbf\xbf",0x10FFFF);
 136
 137         std::cout << "-- Too big" << std::endl;
 138         TEST_TO("\xf4\x9f\x80\x80",illegal); //  11 0000
 139         TEST_TO("\xfb\xbf\xbf\xbf",illegal); // 3FF FFFF
 140         TEST_TO("\xf8\x90\x80\x80\x80",illegal);  // 400 0000
 141         TEST_TO("\xfd\xbf\xbf\xbf\xbf\xbf",illegal);  // 7fff ffff
 142
 143         std::cout << "-- Invalid trail" << std::endl;
 144         TEST_TO("\xC2\x7F",illegal);
 145         TEST_TO("\xdf\x7F",illegal);
 146         TEST_TO("\xe0\x7F\x80",illegal);
 147         TEST_TO("\xef\xbf\x7F",illegal);
 148         TEST_TO("\xe0\x7F\x80",illegal);
 149         TEST_TO("\xef\xbf\x7F",illegal);
 150         TEST_TO("\xf0\x7F\x80\x80",illegal);
 151         TEST_TO("\xf4\x7f\xbf\xbf",illegal);
 152         TEST_TO("\xf0\x90\x7F\x80",illegal);
 153         TEST_TO("\xf4\x8f\x7F\xbf",illegal);
 154         TEST_TO("\xf0\x90\x80\x7F",illegal);
 155         TEST_TO("\xf4\x8f\xbf\x7F",illegal);
 156
 157         std::cout << "-- Invalid length" << std::endl;
 158
 159         /// Test that this actually works
 160         TEST_TO(make2(0x80),0x80);
 161         TEST_TO(make2(0x7ff),0x7ff);
 162
 163         TEST_TO(make3(0x800),0x800);
 164         TEST_TO(make3(0xffff),0xffff);
 165
 166         TEST_TO(make4(0x10000),0x10000);
 167         TEST_TO(make4(0x10ffff),0x10ffff);
 168
 169         TEST_TO(make4(0x110000),illegal);
 170         TEST_TO(make4(0x1fffff),illegal);
 171
 172         TEST_TO(make2(0),illegal);
 173         TEST_TO(make3(0),illegal);
 174         TEST_TO(make4(0),illegal);
 175         TEST_TO(make2(0x7f),illegal);
 176         TEST_TO(make3(0x7f),illegal);
 177         TEST_TO(make4(0x7f),illegal);
 178
 179         TEST_TO(make3(0x80),illegal);
 180         TEST_TO(make4(0x80),illegal);
 181         TEST_TO(make3(0x7ff),illegal);
 182         TEST_TO(make4(0x7ff),illegal);
 183
 184         TEST_TO(make4(0x8000),illegal);
 185         TEST_TO(make4(0xffff),illegal);
 186
 187         std::cout << "-- Invalid surrogate" << std::endl;
 188
 189         TEST_TO(make3(0xD800),illegal);
 190         TEST_TO(make3(0xDBFF),illegal);
 191         TEST_TO(make3(0xDC00),illegal);
 192         TEST_TO(make3(0xDFFF),illegal);
 193
 194         TEST_TO(make4(0xD800),illegal);
 195         TEST_TO(make4(0xDBFF),illegal);
 196         TEST_TO(make4(0xDC00),illegal);
 197         TEST_TO(make4(0xDFFF),illegal);
 198
 199         std::cout <<"-- Incomplete" << std::endl;
 200
 201         TEST_TO("\x80",illegal);
 202         TEST_TO("\xC2",incomplete);
 203
 204         TEST_TO("\xdf",incomplete);
 205
 206         TEST_TO("\xe0",incomplete);
 207         TEST_TO("\xe0\xa0",incomplete);
 208
 209         TEST_TO("\xef\xbf",incomplete);
 210         TEST_TO("\xef",incomplete);
 211
 212         TEST_TO("\xf0\x90\x80",incomplete);
 213         TEST_TO("\xf0\x90",incomplete);
 214         TEST_TO("\xf0",incomplete);
 215
 216         TEST_TO("\xf4\x8f\xbf",incomplete);
 217         TEST_TO("\xf4\x8f",incomplete);
 218         TEST_TO("\xf4",incomplete);
 219
 220         std::cout << "- To UTF-8" << std::endl;
 221
 222         std::cout << "-- Test correct" << std::endl;
 223
 224         TEST_FROM("\x7f",0x7f);
 225         TEST_FROM("\xC2\x80",0x80);
 226         TEST_FROM("\xdf\xBF",0x7FF);
 227         TEST_INC(0x7FF,1);
 228         TEST_FROM("\xe0\xa0\x80",0x800);
 229         TEST_INC(0x800,2);
 230         TEST_INC(0x800,1);
 231         TEST_FROM("\xef\xbf\xbf",0xFFFF);
 232         TEST_INC(0x10000,3);
 233         TEST_INC(0x10000,2);
 234         TEST_INC(0x10000,1);
 235         TEST_FROM("\xf0\x90\x80\x80",0x10000);
 236         TEST_FROM("\xf4\x8f\xbf\xbf",0x10FFFF);
 237
 238         std::cout << "-- Test no surrogate " << std::endl;
 239
 240         TEST_FROM(0,0xD800);
 241         TEST_FROM(0,0xDBFF);
 242         TEST_FROM(0,0xDC00);
 243         TEST_FROM(0,0xDFFF);
 244
 245         std::cout << "-- Test invalid " << std::endl;
 246
 247         TEST_FROM(0,0x110000);
 248         TEST_FROM(0,0x1FFFFF);
 249
 250
 251         std::cout << "Test windows-1255" << std::endl;
 252
 253         cvt = create_simple_converter("windows-1255");
 254
 255         TEST(cvt.get());
 256         TEST(cvt->is_thread_safe());
 257         TEST(cvt->max_len() == 1);
 258
 259         std::cout << "- From 1255" << std::endl;
 260
 261         TEST_TO("\xa4",0x20aa);
 262         TEST_TO("\xe0",0x05d0);
 263         TEST_TO("\xc4",0x5b4);
 264         TEST_TO("\xfb",illegal);
 265         TEST_TO("\xdd",illegal);
 266         TEST_TO("\xff",illegal);
 267         TEST_TO("\xfe",0x200f);
 268
 269         std::cout << "- To 1255" << std::endl;
 270
 271         TEST_FROM("\xa4",0x20aa);
 272         TEST_FROM("\xe0",0x05d0);
 273         TEST_FROM("\xc4",0x5b4);
 274         TEST_FROM("\xfe",0x200f);
 275
 276         TEST_FROM(0,0xe4);
 277         TEST_FROM(0,0xd0);
 278
 279         #ifdef BOOST_LOCALE_WITH_ICU
 280         std::cout << "Testing Shift-JIS using ICU/uconv" << std::endl;
 281
 282         cvt = boost::locale::impl_icu::create_uconv_converter("Shift-JIS");
 283         TEST(cvt.get());
 284         test_shiftjis(cvt);
 285         #endif
 286
 287         #if defined(BOOST_LOCALE_WITH_ICONV) && !defined(BOOST_LOCALE_NO_POSIX_BACKEND)
 288         std::cout << "Testing Shift-JIS using POSIX/iconv" << std::endl;
 289
 290         cvt = boost::locale::impl_posix::create_iconv_converter("Shift-JIS");
 291         TEST(cvt.get());
 292         test_shiftjis(cvt);
 293         #endif
 294
 295     }
 296     catch(std::exception const &e) {
 297         std::cerr << "Failed " << e.what() << std::endl;
 298         return EXIT_FAILURE;
 299     }
 300     FINALIZE();
 301 }
 302
 303 // vim: tabstop=4 expandtab shiftwidth=4 softtabstop=4
 304 // boostinspect:noascii