source/test/cintltst/ncnvtst.c

   1 // Copyright (C) 2016 and later: Unicode, Inc. and others.
   2 // License & terms of use: http://www.unicode.org/copyright.html
   3 /********************************************************************
   4  * COPYRIGHT:
   5  * Copyright (c) 1997-2016, International Business Machines Corporation and
   6  * others. All Rights Reserved.
   7  ********************************************************************/
   8 /*****************************************************************************
   9 *
  10 * File ncnvtst.c
  11 *
  12 * Modification History:
  13 *        Name                     Description
  14 *   Madhu Katragadda              7/7/2000        Converter Tests for extended code coverage
  15 ******************************************************************************
  16 */
  17 #include <stdio.h>
  18 #include <stdlib.h>
  19 #include <string.h>
  20 #include "unicode/uloc.h"
  21 #include "unicode/ucnv.h"
  22 #include "unicode/utypes.h"
  23 #include "unicode/ustring.h"
  24 #include "unicode/uset.h"
  25 #include "cintltst.h"
  26 #include "cmemory.h"
  27
  28 #define MAX_LENGTH 999
  29
  30 #define UNICODE_LIMIT 0x10FFFF
  31 #define SURROGATE_HIGH_START    0xD800
  32 #define SURROGATE_LOW_END       0xDFFF
  33
  34 static int32_t  gInBufferSize = 0;
  35 static int32_t  gOutBufferSize = 0;
  36 static char     gNuConvTestName[1024];
  37
  38 #define nct_min(x,y)  ((x<y) ? x : y)
  39
  40 static void printSeq(const unsigned char* a, int len);
  41 static void printSeqErr(const unsigned char* a, int len);
  42 static void printUSeq(const UChar* a, int len);
  43 static void printUSeqErr(const UChar* a, int len);
  44 static UBool convertFromU( const UChar *source, int sourceLen,  const uint8_t *expect, int expectLen,
  45                 const char *codepage, const int32_t *expectOffsets, UBool doFlush, UErrorCode expectedStatus);
  46 static UBool convertToU( const uint8_t *source, int sourceLen, const UChar *expect, int expectLen,
  47                const char *codepage, const int32_t *expectOffsets, UBool doFlush, UErrorCode expectedStatus);
  48
  49 static UBool testConvertFromU( const UChar *source, int sourceLen,  const uint8_t *expect, int expectLen,
  50                 const char *codepage, UConverterFromUCallback callback, const int32_t *expectOffsets, UBool testReset);
  51 static UBool testConvertToU( const uint8_t *source, int sourcelen, const UChar *expect, int expectlen,
  52                const char *codepage, UConverterToUCallback callback, const int32_t *expectOffsets, UBool testReset);
  53
  54 static void setNuConvTestName(const char *codepage, const char *direction)
  55 {
  56     sprintf(gNuConvTestName, "[Testing %s %s Unicode, InputBufSiz=%d, OutputBufSiz=%d]",
  57         codepage,
  58         direction,
  59         (int)gInBufferSize,
  60         (int)gOutBufferSize);
  61 }
  62
  63
  64 static void TestSurrogateBehaviour(void);
  65 static void TestErrorBehaviour(void);
  66
  67 #if !UCONFIG_NO_LEGACY_CONVERSION
  68 static void TestToUnicodeErrorBehaviour(void);
  69 static void TestGetNextErrorBehaviour(void);
  70 #endif
  71
  72 static void TestRegressionUTF8(void);
  73 static void TestRegressionUTF32(void);
  74 static void TestAvailableConverters(void);
  75 static void TestFlushInternalBuffer(void);  /*for improved code coverage in ucnv_cnv.c*/
  76 static void TestResetBehaviour(void);
  77 static void TestTruncated(void);
  78 static void TestUnicodeSet(void);
  79
  80 static void TestWithBufferSize(int32_t osize, int32_t isize);
  81
  82
  83 static void printSeq(const unsigned char* a, int len)
  84 {
  85     int i=0;
  86     log_verbose("\n{");
  87     while (i<len)
  88         log_verbose("0x%02X ", a[i++]);
  89     log_verbose("}\n");
  90 }
  91
  92 static void printUSeq(const UChar* a, int len)
  93 {
  94     int i=0;
  95     log_verbose("\n{");
  96     while (i<len)
  97         log_verbose("%0x04X ", a[i++]);
  98     log_verbose("}\n");
  99 }
 100
 101 static void printSeqErr(const unsigned char* a, int len)
 102 {
 103     int i=0;
 104     fprintf(stderr, "\n{");
 105     while (i<len)  fprintf(stderr, "0x%02X ", a[i++]);
 106     fprintf(stderr, "}\n");
 107 }
 108
 109 static void printUSeqErr(const UChar* a, int len)
 110 {
 111     int i=0;
 112     fprintf(stderr, "\n{");
 113     while (i<len)
 114         fprintf(stderr, "0x%04X ", a[i++]);
 115     fprintf(stderr,"}\n");
 116 }
 117
 118 void addExtraTests(TestNode** root);
 119
 120 void addExtraTests(TestNode** root)
 121 {
 122      addTest(root, &TestSurrogateBehaviour,         "tsconv/ncnvtst/TestSurrogateBehaviour");
 123      addTest(root, &TestErrorBehaviour,             "tsconv/ncnvtst/TestErrorBehaviour");
 124
 125 #if !UCONFIG_NO_LEGACY_CONVERSION
 126      addTest(root, &TestToUnicodeErrorBehaviour,    "tsconv/ncnvtst/ToUnicodeErrorBehaviour");
 127      addTest(root, &TestGetNextErrorBehaviour,      "tsconv/ncnvtst/TestGetNextErrorBehaviour");
 128 #endif
 129
 130      addTest(root, &TestAvailableConverters,        "tsconv/ncnvtst/TestAvailableConverters");
 131      addTest(root, &TestFlushInternalBuffer,        "tsconv/ncnvtst/TestFlushInternalBuffer");
 132      addTest(root, &TestResetBehaviour,             "tsconv/ncnvtst/TestResetBehaviour");
 133      addTest(root, &TestRegressionUTF8,             "tsconv/ncnvtst/TestRegressionUTF8");
 134      addTest(root, &TestRegressionUTF32,            "tsconv/ncnvtst/TestRegressionUTF32");
 135      addTest(root, &TestTruncated,                  "tsconv/ncnvtst/TestTruncated");
 136      addTest(root, &TestUnicodeSet,                 "tsconv/ncnvtst/TestUnicodeSet");
 137 }
 138
 139 /*test surrogate behaviour*/
 140 static void TestSurrogateBehaviour(){
 141     log_verbose("Testing for SBCS and LATIN_1\n");
 142     {
 143         UChar sampleText[] = {0x0031, 0xd801, 0xdc01, 0x0032};
 144         const uint8_t expected[] = {0x31, 0x1a, 0x32};
 145
 146 #if !UCONFIG_NO_LEGACY_CONVERSION
 147         /*SBCS*/
 148         if(!convertFromU(sampleText, UPRV_LENGTHOF(sampleText),
 149                 expected, sizeof(expected), "ibm-920", 0 , TRUE, U_ZERO_ERROR))
 150             log_err("u-> ibm-920 [UCNV_SBCS] not match.\n");
 151 #endif
 152
 153         /*LATIN_1*/
 154         if(!convertFromU(sampleText, UPRV_LENGTHOF(sampleText),
 155                 expected, sizeof(expected), "LATIN_1", 0, TRUE, U_ZERO_ERROR ))
 156             log_err("u-> LATIN_1 not match.\n");
 157
 158     }
 159
 160 #if !UCONFIG_NO_LEGACY_CONVERSION
 161     log_verbose("Testing for DBCS and MBCS\n");
 162     {
 163         UChar sampleText[]       = {0x00a1, 0xd801, 0xdc01, 0x00a4};
 164         const uint8_t expected[] = {0xa2, 0xae, 0xa1, 0xe0, 0xa2, 0xb4};
 165         int32_t offsets[]        = {0x00, 0x00, 0x01, 0x01, 0x03, 0x03 };
 166
 167         /*DBCS*/
 168         if(!convertFromU(sampleText, UPRV_LENGTHOF(sampleText),
 169                 expected, sizeof(expected), "ibm-1363", 0 , TRUE, U_ZERO_ERROR))
 170             log_err("u-> ibm-1363 [UCNV_DBCS portion] not match.\n");
 171         if(!convertFromU(sampleText, UPRV_LENGTHOF(sampleText),
 172                 expected, sizeof(expected), "ibm-1363", offsets , TRUE, U_ZERO_ERROR))
 173             log_err("u-> ibm-1363 [UCNV_DBCS portion] not match.\n");
 174         /*MBCS*/
 175         if(!convertFromU(sampleText, UPRV_LENGTHOF(sampleText),
 176                 expected, sizeof(expected), "ibm-1363", 0 , TRUE, U_ZERO_ERROR))
 177             log_err("u-> ibm-1363 [UCNV_MBCS] not match.\n");
 178         if(!convertFromU(sampleText, UPRV_LENGTHOF(sampleText),
 179                 expected, sizeof(expected), "ibm-1363", offsets, TRUE, U_ZERO_ERROR))
 180             log_err("u-> ibm-1363 [UCNV_MBCS] not match.\n");
 181     }
 182
 183     log_verbose("Testing for ISO-2022-jp\n");
 184     {
 185         UChar    sampleText[] =   { 0x4e00, 0x04e01, 0x0031, 0xd801, 0xdc01, 0x0032};
 186
 187         const uint8_t expected[] = {0x1b, 0x24, 0x42,0x30,0x6c,0x43,0x7a,0x1b,0x28,0x42,
 188                                     0x31,0x1A, 0x32};
 189
 190
 191         int32_t offsets[] = {0,0,0,0,0,1,1,2,2,2,2,3,5 };
 192
 193         /*iso-2022-jp*/
 194         if(!convertFromU(sampleText, UPRV_LENGTHOF(sampleText),
 195                 expected, sizeof(expected), "iso-2022-jp", 0 , TRUE, U_ZERO_ERROR))
 196             log_err("u-> not match.\n");
 197         if(!convertFromU(sampleText, UPRV_LENGTHOF(sampleText),
 198                 expected, sizeof(expected), "iso-2022-jp", offsets , TRUE, U_ZERO_ERROR))
 199             log_err("u->  not match.\n");
 200     }
 201
 202     log_verbose("Testing for ISO-2022-cn\n");
 203     {
 204         static const UChar    sampleText[] =   { 0x4e00, 0x04e01, 0x0031, 0xd801, 0xdc01, 0x0032};
 205
 206         static const uint8_t expected[] = {
 207                                     0x1B, 0x24, 0x29, 0x41, 0x0E, 0x52, 0x3B,
 208                                     0x36, 0x21,
 209                                     0x0F, 0x31,
 210                                     0x1A,
 211                                     0x32
 212                                     };
 213
 214
 215
 216         static const int32_t offsets[] = {
 217                                     0,    0,    0,    0,    0,    0,    0,
 218                                     1,    1,
 219                                     2,    2,
 220                                     3,
 221                                     5,  };
 222
 223         /*iso-2022-CN*/
 224         if(!convertFromU(sampleText, UPRV_LENGTHOF(sampleText),
 225                 expected, sizeof(expected), "iso-2022-cn", 0 , TRUE, U_ZERO_ERROR))
 226             log_err("u-> not match.\n");
 227         if(!convertFromU(sampleText, UPRV_LENGTHOF(sampleText),
 228                 expected, sizeof(expected), "iso-2022-cn", offsets , TRUE, U_ZERO_ERROR))
 229             log_err("u-> not match.\n");
 230     }
 231
 232         log_verbose("Testing for ISO-2022-kr\n");
 233     {
 234         static const UChar    sampleText[] =   { 0x4e00,0xd801, 0xdc01, 0x04e01, 0x0031, 0xd801, 0xdc01, 0x0032};
 235
 236         static const uint8_t expected[] = {0x1B, 0x24, 0x29, 0x43,
 237                                     0x0E, 0x6C, 0x69,
 238                                     0x0f, 0x1A,
 239                                     0x0e, 0x6F, 0x4B,
 240                                     0x0F, 0x31,
 241                                     0x1A,
 242                                     0x32 };
 243
 244         static const int32_t offsets[] = {-1, -1, -1, -1,
 245                               0, 0, 0,
 246                               1, 1,
 247                               3, 3, 3,
 248                               4, 4,
 249                               5,
 250                               7,
 251                             };
 252
 253         /*iso-2022-kr*/
 254         if(!convertFromU(sampleText, UPRV_LENGTHOF(sampleText),
 255                 expected, sizeof(expected), "iso-2022-kr", 0 , TRUE, U_ZERO_ERROR))
 256             log_err("u-> iso-2022-kr [UCNV_DBCS] not match.\n");
 257         if(!convertFromU(sampleText, UPRV_LENGTHOF(sampleText),
 258                 expected, sizeof(expected), "iso-2022-kr", offsets , TRUE, U_ZERO_ERROR))
 259             log_err("u-> iso-2022-kr [UCNV_DBCS] not match.\n");
 260     }
 261
 262         log_verbose("Testing for HZ\n");
 263     {
 264         static const UChar    sampleText[] =   { 0x4e00, 0xd801, 0xdc01, 0x04e01, 0x0031, 0xd801, 0xdc01, 0x0032};
 265
 266         static const uint8_t expected[] = {0x7E, 0x7B, 0x52, 0x3B,
 267                                     0x7E, 0x7D, 0x1A,
 268                                     0x7E, 0x7B, 0x36, 0x21,
 269                                     0x7E, 0x7D, 0x31,
 270                                     0x1A,
 271                                     0x32 };
 272
 273
 274         static const int32_t offsets[] = {0,0,0,0,
 275                              1,1,1,
 276                              3,3,3,3,
 277                              4,4,4,
 278                              5,
 279                              7,};
 280
 281         /*hz*/
 282         if(!convertFromU(sampleText, UPRV_LENGTHOF(sampleText),
 283                 expected, sizeof(expected), "HZ", 0 , TRUE, U_ZERO_ERROR))
 284             log_err("u-> HZ not match.\n");
 285         if(!convertFromU(sampleText, UPRV_LENGTHOF(sampleText),
 286                 expected, sizeof(expected), "HZ", offsets , TRUE, U_ZERO_ERROR))
 287             log_err("u-> HZ not match.\n");
 288     }
 289 #endif
 290
 291     /*UTF-8*/
 292      log_verbose("Testing for UTF8\n");
 293     {
 294         static const UChar    sampleText[] =   { 0x4e00, 0x0701, 0x0031, 0xbfc1, 0xd801, 0xdc01, 0x0032};
 295         static const int32_t offsets[]={0x00, 0x00, 0x00, 0x01, 0x01, 0x02,
 296                            0x03, 0x03, 0x03, 0x04, 0x04, 0x04,
 297                            0x04, 0x06 };
 298         static const uint8_t expected[] = {0xe4, 0xb8, 0x80, 0xdc, 0x81, 0x31,
 299             0xeb, 0xbf, 0x81, 0xF0, 0x90, 0x90, 0x81, 0x32};
 300
 301
 302         static const int32_t fromOffsets[] = { 0x0000, 0x0003, 0x0005, 0x0006, 0x0009, 0x0009, 0x000D };
 303         /*UTF-8*/
 304         if(!convertFromU(sampleText, UPRV_LENGTHOF(sampleText),
 305             expected, sizeof(expected), "UTF8", offsets, TRUE, U_ZERO_ERROR ))
 306             log_err("u-> UTF8 with offsets and flush true did not match.\n");
 307         if(!convertFromU(sampleText, UPRV_LENGTHOF(sampleText),
 308             expected, sizeof(expected), "UTF8", 0, TRUE, U_ZERO_ERROR ))
 309             log_err("u-> UTF8 with offsets and flush true did not match.\n");
 310         if(!convertFromU(sampleText, UPRV_LENGTHOF(sampleText),
 311             expected, sizeof(expected), "UTF8", offsets, FALSE, U_ZERO_ERROR ))
 312             log_err("u-> UTF8 with offsets and flush true did not match.\n");
 313         if(!convertFromU(sampleText, UPRV_LENGTHOF(sampleText),
 314             expected, sizeof(expected), "UTF8", 0, FALSE, U_ZERO_ERROR ))
 315             log_err("u-> UTF8 with offsets and flush true did not match.\n");
 316
 317         if(!convertToU(expected, sizeof(expected),
 318             sampleText, UPRV_LENGTHOF(sampleText), "UTF8", 0, TRUE, U_ZERO_ERROR ))
 319             log_err("UTF8 -> u did not match.\n");
 320         if(!convertToU(expected, sizeof(expected),
 321             sampleText, UPRV_LENGTHOF(sampleText), "UTF8", 0, FALSE, U_ZERO_ERROR ))
 322             log_err("UTF8 -> u did not match.\n");
 323         if(!convertToU(expected, sizeof(expected),
 324             sampleText, UPRV_LENGTHOF(sampleText), "UTF8", fromOffsets, TRUE, U_ZERO_ERROR ))
 325             log_err("UTF8 ->u  did not match.\n");
 326         if(!convertToU(expected, sizeof(expected),
 327             sampleText, UPRV_LENGTHOF(sampleText), "UTF8", fromOffsets, FALSE, U_ZERO_ERROR ))
 328             log_err("UTF8 -> u did not match.\n");
 329
 330     }
 331 }
 332
 333 /*test various error behaviours*/
 334 static void TestErrorBehaviour(){
 335     log_verbose("Testing for SBCS and LATIN_1\n");
 336     {
 337         static const UChar    sampleText[] =   { 0x0031, 0xd801};
 338         static const UChar    sampleText2[] =   { 0x0031, 0xd801, 0x0032};
 339         static const uint8_t expected0[] =          { 0x31};
 340         static const uint8_t expected[] =          { 0x31, 0x1a};
 341         static const uint8_t expected2[] =         { 0x31, 0x1a, 0x32};
 342
 343 #if !UCONFIG_NO_LEGACY_CONVERSION
 344         /*SBCS*/
 345         if(!convertFromU(sampleText, UPRV_LENGTHOF(sampleText),
 346                 expected, sizeof(expected), "ibm-920", 0, TRUE, U_ZERO_ERROR))
 347             log_err("u-> ibm-920 [UCNV_SBCS] \n");
 348         if(!convertFromU(sampleText, UPRV_LENGTHOF(sampleText),
 349                 expected0, sizeof(expected0), "ibm-920", 0, FALSE, U_ZERO_ERROR))
 350             log_err("u-> ibm-920 [UCNV_SBCS] \n");
 351         if(!convertFromU(sampleText2, UPRV_LENGTHOF(sampleText2),
 352                 expected2, sizeof(expected2), "ibm-920", 0, TRUE, U_ZERO_ERROR))
 353             log_err("u-> ibm-920 [UCNV_SBCS] did not match\n");
 354 #endif
 355
 356         /*LATIN_1*/
 357         if(!convertFromU(sampleText, UPRV_LENGTHOF(sampleText),
 358                 expected, sizeof(expected), "LATIN_1", 0, TRUE, U_ZERO_ERROR))
 359             log_err("u-> LATIN_1 is supposed to fail\n");
 360         if(!convertFromU(sampleText, UPRV_LENGTHOF(sampleText),
 361                 expected0, sizeof(expected0), "LATIN_1", 0, FALSE, U_ZERO_ERROR))
 362             log_err("u-> LATIN_1 is supposed to fail\n");
 363
 364         if(!convertFromU(sampleText2, UPRV_LENGTHOF(sampleText2),
 365                 expected2, sizeof(expected2), "LATIN_1", 0, TRUE, U_ZERO_ERROR))
 366             log_err("u-> LATIN_1 did not match\n");
 367     }
 368
 369 #if !UCONFIG_NO_LEGACY_CONVERSION
 370     log_verbose("Testing for DBCS and MBCS\n");
 371     {
 372         static const UChar    sampleText[]    = { 0x00a1, 0xd801};
 373         static const uint8_t expected[] = { 0xa2, 0xae};
 374         static const int32_t offsets[]        = { 0x00, 0x00};
 375         static const uint8_t expectedSUB[] = { 0xa2, 0xae, 0xa1, 0xe0};
 376         static const int32_t offsetsSUB[]        = { 0x00, 0x00, 0x01, 0x01};
 377
 378         static const UChar       sampleText2[] = { 0x00a1, 0xd801, 0x00a4};
 379         static const uint8_t expected2[] = { 0xa2, 0xae, 0xa1, 0xe0, 0xa2, 0xb4};
 380         static const int32_t offsets2[]        = { 0x00, 0x00, 0x01, 0x01, 0x02, 0x02};
 381
 382         static const UChar       sampleText3MBCS[] = { 0x0001, 0x00a4, 0xdc01};
 383         static const uint8_t expected3MBCS[] = { 0x01, 0xa2, 0xb4, 0xa1, 0xe0};
 384         static const int32_t offsets3MBCS[]        = { 0x00, 0x01, 0x01, 0x02, 0x02};
 385
 386         static const UChar       sampleText4MBCS[] = { 0x0061, 0xFFE4, 0xdc01};
 387         static const uint8_t expected4MBCS[] = { 0x61, 0x8f, 0xa2, 0xc3, 0xf4, 0xfe};
 388         static const int32_t offsets4MBCS[]        = { 0x00, 0x01, 0x01, 0x01, 0x02, 0x02 };
 389
 390         /*DBCS*/
 391         if(!convertFromU(sampleText, UPRV_LENGTHOF(sampleText),
 392                 expectedSUB, sizeof(expectedSUB), "ibm-1363", 0, TRUE, U_ZERO_ERROR))
 393             log_err("u-> ibm-1363 [UCNV_DBCS portion] is supposed to fail\n");
 394         if(!convertFromU(sampleText, UPRV_LENGTHOF(sampleText),
 395                 expected, sizeof(expected), "ibm-1363", 0, FALSE, U_AMBIGUOUS_ALIAS_WARNING))
 396             log_err("u-> ibm-1363 [UCNV_DBCS portion] is supposed to fail\n");
 397
 398         if(!convertFromU(sampleText, UPRV_LENGTHOF(sampleText),
 399                 expectedSUB, sizeof(expectedSUB), "ibm-1363", offsetsSUB, TRUE, U_ZERO_ERROR))
 400             log_err("u-> ibm-1363 [UCNV_DBCS portion] is supposed to fail\n");
 401         if(!convertFromU(sampleText, UPRV_LENGTHOF(sampleText),
 402                 expected, sizeof(expected), "ibm-1363", offsets, FALSE, U_AMBIGUOUS_ALIAS_WARNING))
 403             log_err("u-> ibm-1363 [UCNV_DBCS portion] is supposed to fail\n");
 404
 405
 406         if(!convertFromU(sampleText2, UPRV_LENGTHOF(sampleText2),
 407                 expected2, sizeof(expected2), "ibm-1363", 0, TRUE, U_ZERO_ERROR))
 408             log_err("u-> ibm-1363 [UCNV_DBCS portion] did not match \n");
 409         if(!convertFromU(sampleText2, UPRV_LENGTHOF(sampleText2),
 410                 expected2, sizeof(expected2), "ibm-1363", offsets2, TRUE, U_ZERO_ERROR))
 411             log_err("u-> ibm-1363 [UCNV_DBCS portion] did not match \n");
 412
 413         /*MBCS*/
 414         if(!convertFromU(sampleText, UPRV_LENGTHOF(sampleText),
 415                 expectedSUB, sizeof(expectedSUB), "ibm-1363", 0, TRUE, U_ZERO_ERROR))
 416             log_err("u-> ibm-1363 [UCNV_MBCS] \n");
 417         if(!convertFromU(sampleText, UPRV_LENGTHOF(sampleText),
 418                 expected, sizeof(expected), "ibm-1363", 0, FALSE, U_AMBIGUOUS_ALIAS_WARNING))
 419             log_err("u-> ibm-1363 [UCNV_MBCS] \n");
 420
 421         if(!convertFromU(sampleText2, UPRV_LENGTHOF(sampleText2),
 422                 expected2, sizeof(expected2), "ibm-1363", 0, TRUE, U_ZERO_ERROR))
 423             log_err("u-> ibm-1363 [UCNV_DBCS] did not match\n");
 424         if(!convertFromU(sampleText2, UPRV_LENGTHOF(sampleText2),
 425                 expected2, sizeof(expected2), "ibm-1363", 0, FALSE, U_ZERO_ERROR))
 426             log_err("u-> ibm-1363 [UCNV_DBCS] did not match\n");
 427         if(!convertFromU(sampleText2, UPRV_LENGTHOF(sampleText2),
 428                 expected2, sizeof(expected2), "ibm-1363", offsets2, FALSE, U_ZERO_ERROR))
 429             log_err("u-> ibm-1363 [UCNV_DBCS] did not match\n");
 430
 431         if(!convertFromU(sampleText3MBCS, UPRV_LENGTHOF(sampleText3MBCS),
 432                 expected3MBCS, sizeof(expected3MBCS), "ibm-1363", offsets3MBCS, TRUE, U_ZERO_ERROR))
 433             log_err("u-> ibm-1363 [UCNV_MBCS] \n");
 434         if(!convertFromU(sampleText3MBCS, UPRV_LENGTHOF(sampleText3MBCS),
 435                 expected3MBCS, sizeof(expected3MBCS), "ibm-1363", offsets3MBCS, FALSE, U_ZERO_ERROR))
 436             log_err("u-> ibm-1363 [UCNV_MBCS] \n");
 437
 438         if(!convertFromU(sampleText4MBCS, UPRV_LENGTHOF(sampleText4MBCS),
 439                 expected4MBCS, sizeof(expected4MBCS), "IBM-eucJP", offsets4MBCS, TRUE, U_ZERO_ERROR))
 440             log_err("u-> euc-jp [UCNV_MBCS] \n");
 441         if(!convertFromU(sampleText4MBCS, UPRV_LENGTHOF(sampleText4MBCS),
 442                 expected4MBCS, sizeof(expected4MBCS), "IBM-eucJP", offsets4MBCS, FALSE, U_ZERO_ERROR))
 443             log_err("u-> euc-jp [UCNV_MBCS] \n");
 444     }
 445
 446     /*iso-2022-jp*/
 447     log_verbose("Testing for iso-2022-jp\n");
 448     {
 449         static const UChar    sampleText[]    = { 0x0031, 0xd801};
 450         static const uint8_t expected[] = {  0x31};
 451         static const uint8_t expectedSUB[] = {  0x31, 0x1a};
 452         static const int32_t offsets[]        = { 0x00, 1};
 453
 454         static const UChar       sampleText2[] = { 0x0031, 0xd801, 0x0032};
 455         static const uint8_t expected2[] = {  0x31,0x1A,0x32};
 456         static const int32_t offsets2[]        = { 0x00,0x01,0x02};
 457
 458         static const UChar       sampleText4MBCS[] = { 0x0061, 0x4e00, 0xdc01};
 459         static const uint8_t expected4MBCS[] = { 0x61, 0x1b, 0x24, 0x42, 0x30, 0x6c,0x1b,0x28,0x42,0x1a};
 460         static const int32_t offsets4MBCS[]        = { 0x00, 0x01, 0x01 ,0x01, 0x01, 0x01,0x02,0x02,0x02,0x02 };
 461         if(!convertFromU(sampleText, UPRV_LENGTHOF(sampleText),
 462                 expectedSUB, sizeof(expectedSUB), "iso-2022-jp", offsets, TRUE, U_ZERO_ERROR))
 463             log_err("u-> iso-2022-jp [UCNV_MBCS] \n");
 464         if(!convertFromU(sampleText, UPRV_LENGTHOF(sampleText),
 465                 expected, sizeof(expected), "iso-2022-jp", offsets, FALSE, U_AMBIGUOUS_ALIAS_WARNING))
 466             log_err("u-> iso-2022-jp [UCNV_MBCS] \n");
 467
 468         if(!convertFromU(sampleText2, UPRV_LENGTHOF(sampleText2),
 469                 expected2, sizeof(expected2), "iso-2022-jp", offsets2, TRUE, U_ZERO_ERROR))
 470             log_err("u->iso-2022-jp[UCNV_DBCS] did not match\n");
 471         if(!convertFromU(sampleText2, UPRV_LENGTHOF(sampleText2),
 472                 expected2, sizeof(expected2), "iso-2022-jp", offsets2, FALSE, U_ZERO_ERROR))
 473             log_err("u-> iso-2022-jp [UCNV_DBCS] did not match\n");
 474         if(!convertFromU(sampleText2, UPRV_LENGTHOF(sampleText2),
 475                 expected2, sizeof(expected2), "iso-2022-jp", offsets2, FALSE, U_ZERO_ERROR))
 476             log_err("u-> iso-2022-jp [UCNV_DBCS] did not match\n");
 477
 478         if(!convertFromU(sampleText4MBCS, UPRV_LENGTHOF(sampleText4MBCS),
 479                 expected4MBCS, sizeof(expected4MBCS), "iso-2022-jp", offsets4MBCS, TRUE, U_ZERO_ERROR))
 480             log_err("u-> iso-2022-jp [UCNV_MBCS] \n");
 481         if(!convertFromU(sampleText4MBCS, UPRV_LENGTHOF(sampleText4MBCS),
 482                 expected4MBCS, sizeof(expected4MBCS), "iso-2022-jp", offsets4MBCS, FALSE, U_ZERO_ERROR))
 483             log_err("u-> iso-2022-jp [UCNV_MBCS] \n");
 484     }
 485
 486     /*iso-2022-cn*/
 487     log_verbose("Testing for iso-2022-cn\n");
 488     {
 489         static const UChar    sampleText[]    = { 0x0031, 0xd801};
 490         static const uint8_t expected[] = { 0x31};
 491         static const uint8_t expectedSUB[] = { 0x31, 0x1A};
 492         static const int32_t offsets[]        = { 0x00, 1};
 493
 494         static const UChar       sampleText2[] = { 0x0031, 0xd801, 0x0032};
 495         static const uint8_t expected2[] = { 0x31, 0x1A,0x32};
 496         static const int32_t offsets2[]        = { 0x00, 0x01,0x02};
 497
 498         static const UChar       sampleText3MBCS[] = { 0x0051, 0x0050, 0xdc01};
 499         static const uint8_t expected3MBCS[] = {0x51, 0x50, 0x1A};
 500         static const int32_t offsets3MBCS[]        = { 0x00, 0x01, 0x02 };
 501
 502         static const UChar       sampleText4MBCS[] = { 0x0061, 0x4e00, 0xdc01};
 503         static const uint8_t expected4MBCS[] = { 0x61, 0x1b, 0x24, 0x29, 0x41, 0x0e, 0x52, 0x3b, 0x0f, 0x1a };
 504         static const int32_t offsets4MBCS[]        = { 0x00, 0x01, 0x01 ,0x01, 0x01, 0x01, 0x01, 0x01, 0x02, 0x02 };
 505         if(!convertFromU(sampleText, UPRV_LENGTHOF(sampleText),
 506                 expectedSUB, sizeof(expectedSUB), "iso-2022-cn", offsets, TRUE, U_ZERO_ERROR))
 507             log_err("u-> iso-2022-cn [UCNV_MBCS] \n");
 508         if(!convertFromU(sampleText, UPRV_LENGTHOF(sampleText),
 509                 expected, sizeof(expected), "iso-2022-cn", offsets, FALSE, U_ZERO_ERROR))
 510             log_err("u-> ibm-1363 [UCNV_MBCS] \n");
 511
 512         if(!convertFromU(sampleText2, UPRV_LENGTHOF(sampleText2),
 513                 expected2, sizeof(expected2), "iso-2022-cn", offsets2, TRUE, U_ZERO_ERROR))
 514             log_err("u->iso-2022-cn[UCNV_DBCS] did not match\n");
 515         if(!convertFromU(sampleText2, UPRV_LENGTHOF(sampleText2),
 516                 expected2, sizeof(expected2), "iso-2022-cn", offsets2, FALSE, U_ZERO_ERROR))
 517             log_err("u-> iso-2022-cn [UCNV_DBCS] did not match\n");
 518         if(!convertFromU(sampleText2, UPRV_LENGTHOF(sampleText2),
 519                 expected2, sizeof(expected2), "iso-2022-cn", offsets2, FALSE, U_ZERO_ERROR))
 520             log_err("u-> iso-2022-cn [UCNV_DBCS] did not match\n");
 521
 522         if(!convertFromU(sampleText3MBCS, UPRV_LENGTHOF(sampleText3MBCS),
 523                 expected3MBCS, sizeof(expected3MBCS), "iso-2022-cn", offsets3MBCS, TRUE, U_ZERO_ERROR))
 524             log_err("u->iso-2022-cn [UCNV_MBCS] \n");
 525         if(!convertFromU(sampleText3MBCS, UPRV_LENGTHOF(sampleText3MBCS),
 526                 expected3MBCS, sizeof(expected3MBCS), "iso-2022-cn", offsets3MBCS, FALSE, U_ZERO_ERROR))
 527             log_err("u-> iso-2022-cn[UCNV_MBCS] \n");
 528
 529         if(!convertFromU(sampleText4MBCS, UPRV_LENGTHOF(sampleText4MBCS),
 530                 expected4MBCS, sizeof(expected4MBCS), "iso-2022-cn", offsets4MBCS, TRUE, U_ZERO_ERROR))
 531             log_err("u-> iso-2022-cn [UCNV_MBCS] \n");
 532         if(!convertFromU(sampleText4MBCS, UPRV_LENGTHOF(sampleText4MBCS),
 533                 expected4MBCS, sizeof(expected4MBCS), "iso-2022-cn", offsets4MBCS, FALSE, U_ZERO_ERROR))
 534             log_err("u-> iso-2022-cn [UCNV_MBCS] \n");
 535     }
 536
 537     /*iso-2022-kr*/
 538     log_verbose("Testing for iso-2022-kr\n");
 539     {
 540         static const UChar    sampleText[]    = { 0x0031, 0xd801};
 541         static const uint8_t expected[] = { 0x1b, 0x24, 0x29, 0x43, 0x31};
 542         static const uint8_t expectedSUB[] = { 0x1b, 0x24, 0x29, 0x43, 0x31, 0x1A};
 543         static const int32_t offsets[]        = { -1,   -1,   -1,   -1,   0x00, 1};
 544
 545         static const UChar       sampleText2[] = { 0x0031, 0xd801, 0x0032};
 546         static const uint8_t expected2[] = { 0x1b, 0x24, 0x29, 0x43, 0x31, 0x1A, 0x32};
 547         static const int32_t offsets2[]        = { -1,   -1,   -1,   -1,   0x00, 0x01, 0x02};
 548
 549         static const UChar       sampleText3MBCS[] = { 0x0051, 0x0050, 0xdc01};
 550         static const uint8_t expected3MBCS[] = { 0x1b, 0x24, 0x29, 0x43,  0x51, 0x50, 0x1A };
 551         static const int32_t offsets3MBCS[]        = { -1,   -1,   -1,   -1,    0x00, 0x01, 0x02, 0x02 };
 552
 553         if(!convertFromU(sampleText, UPRV_LENGTHOF(sampleText),
 554                 expectedSUB, sizeof(expectedSUB), "iso-2022-kr", offsets, TRUE, U_ZERO_ERROR))
 555             log_err("u-> iso-2022-kr [UCNV_MBCS] \n");
 556         if(!convertFromU(sampleText, UPRV_LENGTHOF(sampleText),
 557                 expected, sizeof(expected), "iso-2022-kr", offsets, FALSE, U_ZERO_ERROR))
 558             log_err("u-> ibm-1363 [UCNV_MBCS] \n");
 559
 560         if(!convertFromU(sampleText2, UPRV_LENGTHOF(sampleText2),
 561                 expected2, sizeof(expected2), "iso-2022-kr", offsets2, TRUE, U_ZERO_ERROR))
 562             log_err("u->iso-2022-kr[UCNV_DBCS] did not match\n");
 563         if(!convertFromU(sampleText2, UPRV_LENGTHOF(sampleText2),
 564                 expected2, sizeof(expected2), "iso-2022-kr", offsets2, FALSE, U_ZERO_ERROR))
 565             log_err("u-> iso-2022-kr [UCNV_DBCS] did not match\n");
 566         if(!convertFromU(sampleText2, UPRV_LENGTHOF(sampleText2),
 567                 expected2, sizeof(expected2), "iso-2022-kr", offsets2, FALSE, U_ZERO_ERROR))
 568             log_err("u-> iso-2022-kr [UCNV_DBCS] did not match\n");
 569
 570         if(!convertFromU(sampleText3MBCS, UPRV_LENGTHOF(sampleText3MBCS),
 571                 expected3MBCS, sizeof(expected3MBCS), "iso-2022-kr", offsets3MBCS, TRUE, U_ZERO_ERROR))
 572             log_err("u->iso-2022-kr [UCNV_MBCS] \n");
 573         if(!convertFromU(sampleText3MBCS, UPRV_LENGTHOF(sampleText3MBCS),
 574                 expected3MBCS, sizeof(expected3MBCS), "iso-2022-kr", offsets3MBCS, FALSE, U_ZERO_ERROR))
 575             log_err("u-> iso-2022-kr[UCNV_MBCS] \n");
 576     }
 577
 578     /*HZ*/
 579     log_verbose("Testing for HZ\n");
 580     {
 581         static const UChar    sampleText[]    = { 0x0031, 0xd801};
 582         static const uint8_t expected[] = { 0x7e, 0x7d, 0x31};
 583         static const uint8_t expectedSUB[] = { 0x7e, 0x7d, 0x31, 0x1A};
 584         static const int32_t offsets[]        = { 0x00, 0x00, 0x00, 1};
 585
 586         static const UChar       sampleText2[] = { 0x0031, 0xd801, 0x0032};
 587         static const uint8_t expected2[] = { 0x7e, 0x7d, 0x31,  0x1A,  0x32 };
 588         static const int32_t offsets2[]        = { 0x00, 0x00, 0x00, 0x01,  0x02 };
 589
 590         static const UChar       sampleText3MBCS[] = { 0x0051, 0x0050, 0xdc01};
 591         static const uint8_t expected3MBCS[] = { 0x7e, 0x7d, 0x51, 0x50,  0x1A };
 592         static const int32_t offsets3MBCS[]        = { 0x00, 0x00, 0x00, 0x01, 0x02};
 593
 594         static const UChar       sampleText4MBCS[] = { 0x0061, 0x4e00, 0xdc01};
 595         static const uint8_t expected4MBCS[] = { 0x7e, 0x7d, 0x61, 0x7e, 0x7b, 0x52, 0x3b, 0x7e, 0x7d, 0x1a };
 596         static const int32_t offsets4MBCS[]        = { 0x00, 0x00, 0x00, 0x01, 0x01, 0x01 ,0x01, 0x02, 0x02, 0x02 };
 597         if(!convertFromU(sampleText, UPRV_LENGTHOF(sampleText),
 598                 expectedSUB, sizeof(expectedSUB), "HZ", offsets, TRUE, U_ZERO_ERROR))
 599             log_err("u-> HZ [UCNV_MBCS] \n");
 600         if(!convertFromU(sampleText, UPRV_LENGTHOF(sampleText),
 601                 expected, sizeof(expected), "HZ", offsets, FALSE, U_ZERO_ERROR))
 602             log_err("u-> ibm-1363 [UCNV_MBCS] \n");
 603
 604         if(!convertFromU(sampleText2, UPRV_LENGTHOF(sampleText2),
 605                 expected2, sizeof(expected2), "HZ", offsets2, TRUE, U_ZERO_ERROR))
 606             log_err("u->HZ[UCNV_DBCS] did not match\n");
 607         if(!convertFromU(sampleText2, UPRV_LENGTHOF(sampleText2),
 608                 expected2, sizeof(expected2), "HZ", offsets2, FALSE, U_ZERO_ERROR))
 609             log_err("u-> HZ [UCNV_DBCS] did not match\n");
 610         if(!convertFromU(sampleText2, UPRV_LENGTHOF(sampleText2),
 611                 expected2, sizeof(expected2), "HZ", offsets2, FALSE, U_ZERO_ERROR))
 612             log_err("u-> HZ [UCNV_DBCS] did not match\n");
 613
 614         if(!convertFromU(sampleText3MBCS, UPRV_LENGTHOF(sampleText3MBCS),
 615                 expected3MBCS, sizeof(expected3MBCS), "HZ", offsets3MBCS, TRUE, U_ZERO_ERROR))
 616             log_err("u->HZ [UCNV_MBCS] \n");
 617         if(!convertFromU(sampleText3MBCS, UPRV_LENGTHOF(sampleText3MBCS),
 618                 expected3MBCS, sizeof(expected3MBCS), "HZ", offsets3MBCS, FALSE, U_ZERO_ERROR))
 619             log_err("u-> HZ[UCNV_MBCS] \n");
 620
 621         if(!convertFromU(sampleText4MBCS, UPRV_LENGTHOF(sampleText4MBCS),
 622                 expected4MBCS, sizeof(expected4MBCS), "HZ", offsets4MBCS, TRUE, U_ZERO_ERROR))
 623             log_err("u-> HZ [UCNV_MBCS] \n");
 624         if(!convertFromU(sampleText4MBCS, UPRV_LENGTHOF(sampleText4MBCS),
 625                 expected4MBCS, sizeof(expected4MBCS), "HZ", offsets4MBCS, FALSE, U_ZERO_ERROR))
 626             log_err("u-> HZ [UCNV_MBCS] \n");
 627     }
 628 #endif
 629 }
 630
 631 #if !UCONFIG_NO_LEGACY_CONVERSION
 632 /*test different convertToUnicode error behaviours*/
 633 static void TestToUnicodeErrorBehaviour()
 634 {
 635     log_verbose("Testing error conditions for DBCS\n");
 636     {
 637         uint8_t sampleText[] = { 0xa2, 0xae, 0x03, 0x04};
 638         const UChar expected[] = { 0x00a1 };
 639
 640         if(!convertToU(sampleText, sizeof(sampleText),
 641                 expected, UPRV_LENGTHOF(expected), "ibm-1363", 0, TRUE, U_AMBIGUOUS_ALIAS_WARNING ))
 642             log_err("DBCS (ibm-1363)->Unicode  did not match.\n");
 643         if(!convertToU(sampleText, sizeof(sampleText),
 644                 expected, UPRV_LENGTHOF(expected), "ibm-1363", 0, FALSE, U_AMBIGUOUS_ALIAS_WARNING ))
 645             log_err("DBCS (ibm-1363)->Unicode  with flush = false did not match.\n");
 646     }
 647     log_verbose("Testing error conditions for SBCS\n");
 648     {
 649         uint8_t sampleText[] = { 0xa2, 0xFF};
 650         const UChar expected[] = { 0x00c2 };
 651
 652       /*  uint8_t sampleText2[] = { 0xa2, 0x70 };
 653         const UChar expected2[] = { 0x0073 };*/
 654
 655         if(!convertToU(sampleText, sizeof(sampleText),
 656                 expected, UPRV_LENGTHOF(expected), "ibm-1051", 0, TRUE, U_ZERO_ERROR ))
 657             log_err("SBCS (ibm-1051)->Unicode  did not match.\n");
 658         if(!convertToU(sampleText, sizeof(sampleText),
 659                 expected, UPRV_LENGTHOF(expected), "ibm-1051", 0, FALSE, U_ZERO_ERROR ))
 660             log_err("SBCS (ibm-1051)->Unicode  with flush = false did not match.\n");
 661
 662     }
 663 }
 664
 665 static void TestGetNextErrorBehaviour(){
 666    /*Test for unassigned character*/
 667 #define INPUT_SIZE 1
 668     static const char input1[INPUT_SIZE]={ 0x70 };
 669     const char* source=(const char*)input1;
 670     UErrorCode err=U_ZERO_ERROR;
 671     UChar32 c=0;
 672     UConverter *cnv=ucnv_open("ibm-424", &err);
 673     if(U_FAILURE(err)) {
 674         log_data_err("Unable to open a SBCS(ibm-424) converter: %s\n", u_errorName(err));
 675         return;
 676     }
 677     c=ucnv_getNextUChar(cnv, &source, source + INPUT_SIZE, &err);
 678     if(err != U_INVALID_CHAR_FOUND && c!=0xfffd){
 679         log_err("FAIL in TestGetNextErrorBehaviour(unassigned): Expected: U_INVALID_CHAR_ERROR or 0xfffd ----Got:%s and 0x%lx\n",  myErrorName(err), c);
 680     }
 681     ucnv_close(cnv);
 682 }
 683 #endif
 684
 685 #define MAX_UTF16_LEN 2
 686 #define MAX_UTF8_LEN 4
 687
 688 /*Regression test for utf8 converter*/
 689 static void TestRegressionUTF8(){
 690     UChar32 currCh = 0;
 691     int32_t offset8;
 692     int32_t offset16;
 693     UChar *standardForm = (UChar*)malloc(MAX_LENGTH*sizeof(UChar));
 694     uint8_t *utf8 = (uint8_t*)malloc(MAX_LENGTH);
 695
 696     while (currCh <= UNICODE_LIMIT) {
 697         offset16 = 0;
 698         offset8 = 0;
 699         while(currCh <= UNICODE_LIMIT
 700             && offset16 < (MAX_LENGTH/sizeof(UChar) - MAX_UTF16_LEN)
 701             && offset8 < (MAX_LENGTH - MAX_UTF8_LEN))
 702         {
 703             if (currCh == SURROGATE_HIGH_START) {
 704                 currCh = SURROGATE_LOW_END + 1; /* Skip surrogate range */
 705             }
 706             UTF16_APPEND_CHAR_SAFE(standardForm, offset16, MAX_LENGTH, currCh);
 707             UTF8_APPEND_CHAR_SAFE(utf8, offset8, MAX_LENGTH, currCh);
 708             currCh++;
 709         }
 710         if(!convertFromU(standardForm, offset16,
 711             utf8, offset8, "UTF8", 0, TRUE, U_ZERO_ERROR )) {
 712             log_err("Unicode->UTF8 did not match.\n");
 713         }
 714         if(!convertToU(utf8, offset8,
 715             standardForm, offset16, "UTF8", 0, TRUE, U_ZERO_ERROR )) {
 716             log_err("UTF8->Unicode did not match.\n");
 717         }
 718     }
 719
 720     free(standardForm);
 721     free(utf8);
 722
 723     {
 724         static const char src8[] = { (char)0xCC, (char)0x81, (char)0xCC, (char)0x80 };
 725         static const UChar expected[] = { 0x0301, 0x0300 };
 726         UConverter *conv8;
 727         UErrorCode err = U_ZERO_ERROR;
 728         UChar pivotBuffer[100];
 729         const UChar* const pivEnd = pivotBuffer + 100;
 730         const char* srcBeg;
 731         const char* srcEnd;
 732         UChar* pivBeg;
 733
 734         conv8 = ucnv_open("UTF-8", &err);
 735
 736         srcBeg = src8;
 737         pivBeg = pivotBuffer;
 738         srcEnd = src8 + 3;
 739         ucnv_toUnicode(conv8, &pivBeg, pivEnd, &srcBeg, srcEnd, 0, FALSE, &err);
 740         if (srcBeg != srcEnd) {
 741             log_err("Did not consume whole buffer on first call.\n");
 742         }
 743
 744         srcEnd = src8 + 4;
 745         ucnv_toUnicode(conv8, &pivBeg, pivEnd, &srcBeg, srcEnd, 0, TRUE, &err);
 746         if (srcBeg != srcEnd) {
 747             log_err("Did not consume whole buffer on second call.\n");
 748         }
 749
 750         if (U_FAILURE(err) || (int32_t)(pivBeg - pivotBuffer) != 2 || u_strncmp(pivotBuffer, expected, 2) != 0) {
 751             log_err("Did not get expected results for UTF-8.\n");
 752         }
 753         ucnv_close(conv8);
 754     }
 755 }
 756
 757 #define MAX_UTF32_LEN 1
 758
 759 static void TestRegressionUTF32(){
 760 #if !UCONFIG_ONLY_HTML_CONVERSION
 761     UChar32 currCh = 0;
 762     int32_t offset32;
 763     int32_t offset16;
 764     UChar *standardForm = (UChar*)malloc(MAX_LENGTH*sizeof(UChar));
 765     UChar32 *utf32 = (UChar32*)malloc(MAX_LENGTH*sizeof(UChar32));
 766
 767     while (currCh <= UNICODE_LIMIT) {
 768         offset16 = 0;
 769         offset32 = 0;
 770         while(currCh <= UNICODE_LIMIT
 771             && offset16 < (MAX_LENGTH/sizeof(UChar) - MAX_UTF16_LEN)
 772             && offset32 < (MAX_LENGTH/sizeof(UChar32) - MAX_UTF32_LEN))
 773         {
 774             if (currCh == SURROGATE_HIGH_START) {
 775                 currCh = SURROGATE_LOW_END + 1; /* Skip surrogate range */
 776             }
 777             UTF16_APPEND_CHAR_SAFE(standardForm, offset16, MAX_LENGTH, currCh);
 778             UTF32_APPEND_CHAR_SAFE(utf32, offset32, MAX_LENGTH, currCh);
 779             currCh++;
 780         }
 781         if(!convertFromU(standardForm, offset16,
 782             (const uint8_t *)utf32, offset32*sizeof(UChar32), "UTF32_PlatformEndian", 0, TRUE, U_ZERO_ERROR )) {
 783             log_err("Unicode->UTF32 did not match.\n");
 784         }
 785         if(!convertToU((const uint8_t *)utf32, offset32*sizeof(UChar32),
 786             standardForm, offset16, "UTF32_PlatformEndian", 0, TRUE, U_ZERO_ERROR )) {
 787             log_err("UTF32->Unicode did not match.\n");
 788         }
 789     }
 790     free(standardForm);
 791     free(utf32);
 792
 793     {
 794         /* Check for lone surrogate error handling. */
 795         static const UChar   sampleBadStartSurrogate[] = { 0x0031, 0xD800, 0x0032 };
 796         static const UChar   sampleBadEndSurrogate[] = { 0x0031, 0xDC00, 0x0032 };
 797         static const uint8_t expectedUTF32BE[] = {
 798             0x00, 0x00, 0x00, 0x31,
 799             0x00, 0x00, 0xff, 0xfd,
 800             0x00, 0x00, 0x00, 0x32
 801         };
 802         static const uint8_t expectedUTF32LE[] = {
 803             0x31, 0x00, 0x00, 0x00,
 804             0xfd, 0xff, 0x00, 0x00,
 805             0x32, 0x00, 0x00, 0x00
 806         };
 807         static const int32_t offsetsUTF32[] = {
 808             0x00, 0x00, 0x00, 0x00,
 809             0x01, 0x01, 0x01, 0x01,
 810             0x02, 0x02, 0x02, 0x02
 811         };
 812
 813         if(!convertFromU(sampleBadStartSurrogate, UPRV_LENGTHOF(sampleBadStartSurrogate),
 814                 expectedUTF32BE, sizeof(expectedUTF32BE), "UTF-32BE", offsetsUTF32, TRUE, U_ZERO_ERROR))
 815             log_err("u->UTF-32BE\n");
 816         if(!convertFromU(sampleBadEndSurrogate, UPRV_LENGTHOF(sampleBadEndSurrogate),
 817                 expectedUTF32BE, sizeof(expectedUTF32BE), "UTF-32BE", offsetsUTF32, TRUE, U_ZERO_ERROR))
 818             log_err("u->UTF-32BE\n");
 819
 820         if(!convertFromU(sampleBadStartSurrogate, UPRV_LENGTHOF(sampleBadStartSurrogate),
 821                 expectedUTF32LE, sizeof(expectedUTF32LE), "UTF-32LE", offsetsUTF32, TRUE, U_ZERO_ERROR))
 822             log_err("u->UTF-32LE\n");
 823         if(!convertFromU(sampleBadEndSurrogate, UPRV_LENGTHOF(sampleBadEndSurrogate),
 824                 expectedUTF32LE, sizeof(expectedUTF32LE), "UTF-32LE", offsetsUTF32, TRUE, U_ZERO_ERROR))
 825             log_err("u->UTF-32LE\n");
 826     }
 827
 828     {
 829         static const char srcBE[] = { 0, 0, 0, 0x31, 0, 0, 0, 0x30 };
 830         static const UChar expected[] = { 0x0031, 0x0030 };
 831         UConverter *convBE;
 832         UErrorCode err = U_ZERO_ERROR;
 833         UChar pivotBuffer[100];
 834         const UChar* const pivEnd = pivotBuffer + 100;
 835         const char* srcBeg;
 836         const char* srcEnd;
 837         UChar* pivBeg;
 838
 839         convBE = ucnv_open("UTF-32BE", &err);
 840
 841         srcBeg = srcBE;
 842         pivBeg = pivotBuffer;
 843         srcEnd = srcBE + 5;
 844         ucnv_toUnicode(convBE, &pivBeg, pivEnd, &srcBeg, srcEnd, 0, FALSE, &err);
 845         if (srcBeg != srcEnd) {
 846             log_err("Did not consume whole buffer on first call.\n");
 847         }
 848
 849         srcEnd = srcBE + 8;
 850         ucnv_toUnicode(convBE, &pivBeg, pivEnd, &srcBeg, srcEnd, 0, TRUE, &err);
 851         if (srcBeg != srcEnd) {
 852             log_err("Did not consume whole buffer on second call.\n");
 853         }
 854
 855         if (U_FAILURE(err) || (int32_t)(pivBeg - pivotBuffer) != 2 || u_strncmp(pivotBuffer, expected, 2) != 0) {
 856             log_err("Did not get expected results for UTF-32BE.\n");
 857         }
 858         ucnv_close(convBE);
 859     }
 860     {
 861         static const char srcLE[] = { 0x31, 0, 0, 0, 0x30, 0, 0, 0 };
 862         static const UChar expected[] = { 0x0031, 0x0030 };
 863         UConverter *convLE;
 864         UErrorCode err = U_ZERO_ERROR;
 865         UChar pivotBuffer[100];
 866         const UChar* const pivEnd = pivotBuffer + 100;
 867         const char* srcBeg;
 868         const char* srcEnd;
 869         UChar* pivBeg;
 870
 871         convLE = ucnv_open("UTF-32LE", &err);
 872
 873         srcBeg = srcLE;
 874         pivBeg = pivotBuffer;
 875         srcEnd = srcLE + 5;
 876         ucnv_toUnicode(convLE, &pivBeg, pivEnd, &srcBeg, srcEnd, 0, FALSE, &err);
 877         if (srcBeg != srcEnd) {
 878             log_err("Did not consume whole buffer on first call.\n");
 879         }
 880
 881         srcEnd = srcLE + 8;
 882         ucnv_toUnicode(convLE, &pivBeg, pivEnd, &srcBeg, srcEnd, 0, TRUE, &err);
 883         if (srcBeg != srcEnd) {
 884             log_err("Did not consume whole buffer on second call.\n");
 885         }
 886
 887         if (U_FAILURE(err) || (int32_t)(pivBeg - pivotBuffer) != 2 || u_strncmp(pivotBuffer, expected, 2) != 0) {
 888             log_err("Did not get expected results for UTF-32LE.\n");
 889         }
 890         ucnv_close(convLE);
 891     }
 892 #endif
 893 }
 894
 895 /*Walk through the available converters*/
 896 static void TestAvailableConverters(){
 897     UErrorCode status=U_ZERO_ERROR;
 898     UConverter *conv=NULL;
 899     int32_t i=0;
 900     for(i=0; i < ucnv_countAvailable(); i++){
 901         status=U_ZERO_ERROR;
 902         conv=ucnv_open(ucnv_getAvailableName(i), &status);
 903         if(U_FAILURE(status)){
 904             log_err("ERROR: converter creation failed. Failure in alias table or the data table for \n converter=%s. Error=%s\n",
 905                         ucnv_getAvailableName(i), myErrorName(status));
 906             continue;
 907         }
 908         ucnv_close(conv);
 909     }
 910
 911 }
 912
 913 static void TestFlushInternalBuffer(){
 914     TestWithBufferSize(MAX_LENGTH, 1);
 915     TestWithBufferSize(1, 1);
 916     TestWithBufferSize(1, MAX_LENGTH);
 917     TestWithBufferSize(MAX_LENGTH, MAX_LENGTH);
 918 }
 919
 920 static void TestWithBufferSize(int32_t insize, int32_t outsize){
 921
 922     gInBufferSize =insize;
 923     gOutBufferSize = outsize;
 924
 925      log_verbose("Testing fromUnicode for UTF-8 with UCNV_TO_U_CALLBACK_SUBSTITUTE \n");
 926     {
 927         UChar    sampleText[] =
 928             { 0x0031, 0x0032, 0x0033, 0x0000, 0x4e00, 0x4e8c, 0x4e09,  0x002E  };
 929         const uint8_t expectedUTF8[] =
 930             { 0x31, 0x32, 0x33, 0x00, 0xe4, 0xb8, 0x80, 0xe4, 0xba, 0x8c, 0xe4, 0xb8, 0x89, 0x2E };
 931         int32_t  toUTF8Offs[] =
 932             { 0x00, 0x01, 0x02, 0x03, 0x04, 0x04, 0x04, 0x05, 0x05, 0x05, 0x06, 0x06, 0x06, 0x07};
 933        /* int32_t fmUTF8Offs[] =
 934             { 0x0000, 0x0001, 0x0002, 0x0003, 0x0004, 0x0007, 0x000a, 0x000d };*/
 935
 936         /*UTF-8*/
 937         if(!testConvertFromU(sampleText, UPRV_LENGTHOF(sampleText),
 938             expectedUTF8, sizeof(expectedUTF8), "UTF8", UCNV_FROM_U_CALLBACK_SUBSTITUTE, toUTF8Offs ,FALSE))
 939              log_err("u-> UTF8 did not match.\n");
 940     }
 941
 942 #if !UCONFIG_NO_LEGACY_CONVERSION
 943      log_verbose("Testing fromUnicode with UCNV_FROM_U_CALLBACK_ESCAPE  \n");
 944     {
 945         UChar inputTest[] = { 0x0061, 0xd801, 0xdc01, 0xd801, 0x0061 };
 946         const uint8_t toIBM943[]= { 0x61,
 947             0x25, 0x55, 0x44, 0x38, 0x30, 0x31,
 948             0x25, 0x55, 0x44, 0x43, 0x30, 0x31,
 949             0x25, 0x55, 0x44, 0x38, 0x30, 0x31,
 950             0x61 };
 951         int32_t offset[]= {0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 3, 3, 3, 3, 3, 3, 4};
 952
 953         if(!testConvertFromU(inputTest, UPRV_LENGTHOF(inputTest),
 954                 toIBM943, sizeof(toIBM943), "ibm-943",
 955                 (UConverterFromUCallback)UCNV_FROM_U_CALLBACK_ESCAPE, offset,FALSE))
 956             log_err("u-> ibm-943 with subst with value did not match.\n");
 957     }
 958 #endif
 959
 960      log_verbose("Testing fromUnicode for UTF-8 with UCNV_TO_U_CALLBACK_SUBSTITUTE \n");
 961     {
 962         const uint8_t sampleText1[] = { 0x31, 0xe4, 0xba, 0x8c,
 963             0xe0, 0x80,  0x61};
 964         UChar    expected1[] = {  0x0031, 0x4e8c, 0xfffd, 0x0061};
 965         int32_t offsets1[] = {   0x0000, 0x0001, 0x0004, 0x0006};
 966
 967         if(!testConvertToU(sampleText1, sizeof(sampleText1),
 968                  expected1, UPRV_LENGTHOF(expected1),"utf8", UCNV_TO_U_CALLBACK_SUBSTITUTE, offsets1,FALSE))
 969             log_err("utf8->u with substitute did not match.\n");;
 970     }
 971
 972 #if !UCONFIG_NO_LEGACY_CONVERSION
 973     log_verbose("Testing toUnicode with UCNV_TO_U_CALLBACK_ESCAPE \n");
 974     /*to Unicode*/
 975     {
 976         const uint8_t sampleTxtToU[]= { 0x00, 0x9f, 0xaf,
 977             0x81, 0xad, /*unassigned*/
 978             0x89, 0xd3 };
 979         UChar IBM_943toUnicode[] = { 0x0000, 0x6D63,
 980             0x25, 0x58, 0x38, 0x31, 0x25, 0x58, 0x41, 0x44,
 981             0x7B87};
 982         int32_t  fromIBM943Offs [] =    { 0, 1, 3, 3, 3, 3, 3, 3, 3, 3, 5};
 983
 984         if(!testConvertToU(sampleTxtToU, sizeof(sampleTxtToU),
 985                  IBM_943toUnicode, UPRV_LENGTHOF(IBM_943toUnicode),"ibm-943",
 986                 (UConverterToUCallback)UCNV_TO_U_CALLBACK_ESCAPE, fromIBM943Offs,FALSE))
 987             log_err("ibm-943->u with substitute with value did not match.\n");
 988
 989     }
 990 #endif
 991 }
 992
 993 static UBool convertFromU( const UChar *source, int sourceLen,  const uint8_t *expect, int expectLen,
 994                 const char *codepage, const int32_t *expectOffsets, UBool doFlush, UErrorCode expectedStatus)
 995 {
 996
 997     int32_t i=0;
 998     char *p=0;
 999     const UChar *src;
1000     char buffer[MAX_LENGTH];
1001     int32_t offsetBuffer[MAX_LENGTH];
1002     int32_t *offs=0;
1003     char *targ;
1004     char *targetLimit;
1005     UChar *sourceLimit=0;
1006     UErrorCode status = U_ZERO_ERROR;
1007     UConverter *conv = 0;
1008     conv = ucnv_open(codepage, &status);
1009     if(U_FAILURE(status))
1010     {
1011         log_data_err("Couldn't open converter %s\n",codepage);
1012         return TRUE;
1013     }
1014     log_verbose("Converter %s opened..\n", ucnv_getName(conv, &status));
1015
1016     for(i=0; i<MAX_LENGTH; i++){
1017         buffer[i]=(char)0xF0;
1018         offsetBuffer[i]=0xFF;
1019     }
1020
1021     src=source;
1022     sourceLimit=(UChar*)src+(sourceLen);
1023     targ=buffer;
1024     targetLimit=targ+MAX_LENGTH;
1025     offs=offsetBuffer;
1026     ucnv_fromUnicode (conv,
1027                   (char **)&targ,
1028                   (const char *)targetLimit,
1029                   &src,
1030                   sourceLimit,
1031                   expectOffsets ? offs : NULL,
1032                   doFlush,
1033                   &status);
1034     ucnv_close(conv);
1035     if(status != expectedStatus){
1036           log_err("ucnv_fromUnicode() failed for codepage=%s. Error =%s Expected=%s\n", codepage, myErrorName(status), myErrorName(expectedStatus));
1037           return FALSE;
1038     }
1039
1040     log_verbose("\nConversion done [%d uchars in -> %d chars out]. \nResult :",
1041         sourceLen, targ-buffer);
1042
1043     if(expectLen != targ-buffer)
1044     {
1045         log_err("Expected %d chars out, got %d FROM Unicode to %s\n", expectLen, targ-buffer, codepage);
1046         log_verbose("Expected %d chars out, got %d FROM Unicode to %s\n", expectLen, targ-buffer, codepage);
1047         printSeqErr((const unsigned char *)buffer, (int32_t)(targ-buffer));
1048         printSeqErr((const unsigned char*)expect, expectLen);
1049         return FALSE;
1050     }
1051
1052     if(memcmp(buffer, expect, expectLen)){
1053         log_err("String does not match. FROM Unicode to codePage%s\n", codepage);
1054         log_info("\nGot:");
1055         printSeqErr((const unsigned char *)buffer, expectLen);
1056         log_info("\nExpected:");
1057         printSeqErr((const unsigned char *)expect, expectLen);
1058         return FALSE;
1059     }
1060     else {
1061         log_verbose("Matches!\n");
1062     }
1063
1064     if (expectOffsets != 0){
1065         log_verbose("comparing %d offsets..\n", targ-buffer);
1066         if(memcmp(offsetBuffer,expectOffsets,(targ-buffer) * sizeof(int32_t) )){
1067             log_err("did not get the expected offsets. for FROM Unicode to %s\n", codepage);
1068             log_info("\nGot  : ");
1069             printSeqErr((const unsigned char*)buffer, (int32_t)(targ-buffer));
1070             for(p=buffer;p<targ;p++)
1071                 log_info("%d, ", offsetBuffer[p-buffer]);
1072             log_info("\nExpected: ");
1073             for(i=0; i< (targ-buffer); i++)
1074                 log_info("%d,", expectOffsets[i]);
1075         }
1076     }
1077
1078     return TRUE;
1079 }
1080
1081
1082 static UBool convertToU( const uint8_t *source, int sourceLen, const UChar *expect, int expectLen,
1083                const char *codepage, const int32_t *expectOffsets, UBool doFlush, UErrorCode expectedStatus)
1084 {
1085     UErrorCode status = U_ZERO_ERROR;
1086     UConverter *conv = 0;
1087     int32_t i=0;
1088     UChar *p=0;
1089     const char* src;
1090     UChar buffer[MAX_LENGTH];
1091     int32_t offsetBuffer[MAX_LENGTH];
1092     int32_t *offs=0;
1093     UChar *targ;
1094     UChar *targetLimit;
1095     uint8_t *sourceLimit=0;
1096
1097
1098
1099     conv = ucnv_open(codepage, &status);
1100     if(U_FAILURE(status))
1101     {
1102         log_data_err("Couldn't open converter %s\n",codepage);
1103         return TRUE;
1104     }
1105     log_verbose("Converter %s opened..\n", ucnv_getName(conv, &status));
1106
1107
1108
1109     for(i=0; i<MAX_LENGTH; i++){
1110         buffer[i]=0xFFFE;
1111         offsetBuffer[i]=-1;
1112     }
1113
1114     src=(const char *)source;
1115     sourceLimit=(uint8_t*)(src+(sourceLen));
1116     targ=buffer;
1117     targetLimit=targ+MAX_LENGTH;
1118     offs=offsetBuffer;
1119
1120
1121
1122     ucnv_toUnicode (conv,
1123                 &targ,
1124                 targetLimit,
1125                 (const char **)&src,
1126                 (const char *)sourceLimit,
1127                 expectOffsets ? offs : NULL,
1128                 doFlush,
1129                 &status);
1130
1131     ucnv_close(conv);
1132     if(status != expectedStatus){
1133           log_err("ucnv_fromUnicode() failed for codepage=%s. Error =%s Expected=%s\n", codepage, myErrorName(status), myErrorName(expectedStatus));
1134           return FALSE;
1135     }
1136     log_verbose("\nConversion done [%d uchars in -> %d chars out]. \nResult :",
1137         sourceLen, targ-buffer);
1138
1139
1140
1141
1142     log_verbose("comparing %d uchars (%d bytes)..\n",expectLen,expectLen*2);
1143
1144     if (expectOffsets != 0) {
1145         if(memcmp(offsetBuffer, expectOffsets, (targ-buffer) * sizeof(int32_t))){
1146
1147             log_err("did not get the expected offsets from %s To UNICODE\n", codepage);
1148             log_info("\nGot : ");
1149             for(p=buffer;p<targ;p++)
1150                 log_info("%d, ", offsetBuffer[p-buffer]);
1151             log_info("\nExpected: ");
1152             for(i=0; i<(targ-buffer); i++)
1153                 log_info("%d, ", expectOffsets[i]);
1154             log_info("\nGot result:");
1155             for(i=0; i<(targ-buffer); i++)
1156                 log_info("0x%04X,", buffer[i]);
1157             log_info("\nFrom Input:");
1158             for(i=0; i<(src-(const char *)source); i++)
1159                 log_info("0x%02X,", (unsigned char)source[i]);
1160             log_info("\n");
1161         }
1162     }
1163     if(memcmp(buffer, expect, expectLen*2)){
1164         log_err("String does not match. from codePage %s TO Unicode\n", codepage);
1165         log_info("\nGot:");
1166         printUSeqErr(buffer, expectLen);
1167         log_info("\nExpected:");
1168         printUSeqErr(expect, expectLen);
1169         return FALSE;
1170     }
1171     else {
1172         log_verbose("Matches!\n");
1173     }
1174
1175     return TRUE;
1176 }
1177
1178
1179 static UBool testConvertFromU( const UChar *source, int sourceLen,  const uint8_t *expect, int expectLen,
1180                 const char *codepage, UConverterFromUCallback callback , const int32_t *expectOffsets, UBool testReset)
1181 {
1182     UErrorCode status = U_ZERO_ERROR;
1183     UConverter *conv = 0;
1184     char    junkout[MAX_LENGTH]; /* FIX */
1185     int32_t    junokout[MAX_LENGTH]; /* FIX */
1186     char *p;
1187     const UChar *src;
1188     char *end;
1189     char *targ;
1190     int32_t *offs;
1191     int i;
1192     int32_t   realBufferSize;
1193     char *realBufferEnd;
1194     const UChar *realSourceEnd;
1195     const UChar *sourceLimit;
1196     UBool checkOffsets = TRUE;
1197     UBool doFlush;
1198
1199     UConverterFromUCallback oldAction = NULL;
1200     const void* oldContext = NULL;
1201
1202     for(i=0;i<MAX_LENGTH;i++)
1203         junkout[i] = (char)0xF0;
1204     for(i=0;i<MAX_LENGTH;i++)
1205         junokout[i] = 0xFF;
1206
1207     setNuConvTestName(codepage, "FROM");
1208
1209     log_verbose("\n=========  %s\n", gNuConvTestName);
1210
1211     conv = ucnv_open(codepage, &status);
1212     if(U_FAILURE(status))
1213     {
1214         log_data_err("Couldn't open converter %s\n",codepage);
1215         return TRUE;
1216     }
1217
1218     log_verbose("Converter opened..\n");
1219     /*----setting the callback routine----*/
1220     ucnv_setFromUCallBack (conv, callback, NULL, &oldAction, &oldContext, &status);
1221     if (U_FAILURE(status)) {
1222         log_err("FAILURE in setting the callback Function! %s\n", myErrorName(status));
1223     }
1224     /*------------------------*/
1225
1226     src = source;
1227     targ = junkout;
1228     offs = junokout;
1229
1230     realBufferSize = UPRV_LENGTHOF(junkout);
1231     realBufferEnd = junkout + realBufferSize;
1232     realSourceEnd = source + sourceLen;
1233
1234     if ( gOutBufferSize != realBufferSize )
1235       checkOffsets = FALSE;
1236
1237     if( gInBufferSize != MAX_LENGTH )
1238       checkOffsets = FALSE;
1239
1240     do
1241     {
1242         end = nct_min(targ + gOutBufferSize, realBufferEnd);
1243         sourceLimit = nct_min(src + gInBufferSize, realSourceEnd);
1244
1245         doFlush = (UBool)(sourceLimit == realSourceEnd);
1246
1247         if(targ == realBufferEnd)
1248           {
1249         log_err("Error, overflowed the real buffer while about to call fromUnicode! targ=%08lx %s", targ, gNuConvTestName);
1250         return FALSE;
1251           }
1252         log_verbose("calling fromUnicode @ SOURCE:%08lx to %08lx  TARGET: %08lx to %08lx, flush=%s\n", src,sourceLimit, targ,end, doFlush?"TRUE":"FALSE");
1253
1254
1255         status = U_ZERO_ERROR;
1256         if(gInBufferSize ==999 && gOutBufferSize==999)
1257             doFlush = FALSE;
1258         ucnv_fromUnicode (conv,
1259                   (char **)&targ,
1260                   (const char *)end,
1261                   &src,
1262                   sourceLimit,
1263                   offs,
1264                   doFlush, /* flush if we're at the end of the input data */
1265                   &status);
1266         if(testReset)
1267             ucnv_resetToUnicode(conv);
1268         if(gInBufferSize ==999 && gOutBufferSize==999)
1269             ucnv_resetToUnicode(conv);
1270
1271       } while ( (status == U_BUFFER_OVERFLOW_ERROR) || (U_SUCCESS(status) && sourceLimit < realSourceEnd) );
1272
1273     if(U_FAILURE(status)) {
1274         log_err("Problem doing fromUnicode to %s, errcode %s %s\n", codepage, myErrorName(status), gNuConvTestName);
1275         return FALSE;
1276       }
1277
1278     log_verbose("\nConversion done [%d uchars in -> %d chars out]. \nResult :",
1279         sourceLen, targ-junkout);
1280     if(getTestOption(VERBOSITY_OPTION))
1281     {
1282         char junk[999];
1283         char offset_str[999];
1284         char *ptr;
1285
1286         junk[0] = 0;
1287         offset_str[0] = 0;
1288         for(ptr = junkout;ptr<targ;ptr++)
1289         {
1290             sprintf(junk + strlen(junk), "0x%02x, ", (0xFF) & (unsigned int)*ptr);
1291             sprintf(offset_str + strlen(offset_str), "0x%02x, ", (0xFF) & (unsigned int)junokout[ptr-junkout]);
1292         }
1293
1294         log_verbose(junk);
1295         printSeq((const unsigned char *)expect, expectLen);
1296         if ( checkOffsets )
1297           {
1298             log_verbose("\nOffsets:");
1299             log_verbose(offset_str);
1300           }
1301         log_verbose("\n");
1302     }
1303     ucnv_close(conv);
1304
1305
1306     if(expectLen != targ-junkout)
1307     {
1308         log_err("Expected %d chars out, got %d %s\n", expectLen, targ-junkout, gNuConvTestName);
1309         log_verbose("Expected %d chars out, got %d %s\n", expectLen, targ-junkout, gNuConvTestName);
1310         log_info("\nGot:");
1311         printSeqErr((const unsigned char*)junkout, (int32_t)(targ-junkout));
1312         log_info("\nExpected:");
1313         printSeqErr((const unsigned char*)expect, expectLen);
1314         return FALSE;
1315     }
1316
1317     if (checkOffsets && (expectOffsets != 0) )
1318     {
1319         log_verbose("comparing %d offsets..\n", targ-junkout);
1320         if(memcmp(junokout,expectOffsets,(targ-junkout) * sizeof(int32_t) )){
1321             log_err("did not get the expected offsets. %s", gNuConvTestName);
1322             log_err("Got  : ");
1323             printSeqErr((const unsigned char*)junkout, (int32_t)(targ-junkout));
1324             for(p=junkout;p<targ;p++)
1325                 log_err("%d, ", junokout[p-junkout]);
1326             log_err("\nExpected: ");
1327             for(i=0; i<(targ-junkout); i++)
1328                 log_err("%d,", expectOffsets[i]);
1329         }
1330     }
1331
1332     log_verbose("comparing..\n");
1333     if(!memcmp(junkout, expect, expectLen))
1334     {
1335         log_verbose("Matches!\n");
1336         return TRUE;
1337     }
1338     else
1339     {
1340         log_err("String does not match. %s\n", gNuConvTestName);
1341         printUSeqErr(source, sourceLen);
1342         log_info("\nGot:");
1343         printSeqErr((const unsigned char *)junkout, expectLen);
1344         log_info("\nExpected:");
1345         printSeqErr((const unsigned char *)expect, expectLen);
1346
1347         return FALSE;
1348     }
1349 }
1350
1351 static UBool testConvertToU( const uint8_t *source, int sourcelen, const UChar *expect, int expectlen,
1352                const char *codepage, UConverterToUCallback callback, const int32_t *expectOffsets, UBool testReset)
1353 {
1354     UErrorCode status = U_ZERO_ERROR;
1355     UConverter *conv = 0;
1356     UChar    junkout[MAX_LENGTH]; /* FIX */
1357     int32_t    junokout[MAX_LENGTH]; /* FIX */
1358     const char *src;
1359     const char *realSourceEnd;
1360     const char *srcLimit;
1361     UChar *p;
1362     UChar *targ;
1363     UChar *end;
1364     int32_t *offs;
1365     int i;
1366     UBool   checkOffsets = TRUE;
1367     int32_t   realBufferSize;
1368     UChar *realBufferEnd;
1369     UBool doFlush;
1370
1371     UConverterToUCallback oldAction = NULL;
1372     const void* oldContext = NULL;
1373
1374
1375     for(i=0;i<MAX_LENGTH;i++)
1376         junkout[i] = 0xFFFE;
1377
1378     for(i=0;i<MAX_LENGTH;i++)
1379         junokout[i] = -1;
1380
1381     setNuConvTestName(codepage, "TO");
1382
1383     log_verbose("\n=========  %s\n", gNuConvTestName);
1384
1385     conv = ucnv_open(codepage, &status);
1386     if(U_FAILURE(status))
1387     {
1388         log_data_err("Couldn't open converter %s\n",gNuConvTestName);
1389         return TRUE;
1390     }
1391
1392     log_verbose("Converter opened..\n");
1393      /*----setting the callback routine----*/
1394     ucnv_setToUCallBack (conv, callback, NULL, &oldAction, &oldContext, &status);
1395     if (U_FAILURE(status)) {
1396         log_err("FAILURE in setting the callback Function! %s\n", myErrorName(status));
1397     }
1398     /*-------------------------------------*/
1399
1400     src = (const char *)source;
1401     targ = junkout;
1402     offs = junokout;
1403
1404     realBufferSize = UPRV_LENGTHOF(junkout);
1405     realBufferEnd = junkout + realBufferSize;
1406     realSourceEnd = src + sourcelen;
1407
1408     if ( gOutBufferSize != realBufferSize )
1409       checkOffsets = FALSE;
1410
1411     if( gInBufferSize != MAX_LENGTH )
1412       checkOffsets = FALSE;
1413
1414     do
1415       {
1416         end = nct_min( targ + gOutBufferSize, realBufferEnd);
1417         srcLimit = nct_min(realSourceEnd, src + gInBufferSize);
1418
1419         if(targ == realBufferEnd)
1420         {
1421             log_err("Error, the end would overflow the real output buffer while about to call toUnicode! tarjey=%08lx %s",targ,gNuConvTestName);
1422             return FALSE;
1423         }
1424         log_verbose("calling toUnicode @ %08lx to %08lx\n", targ,end);
1425
1426         /* oldTarg = targ; */
1427
1428         status = U_ZERO_ERROR;
1429         doFlush=(UBool)((gInBufferSize ==999 && gOutBufferSize==999)?(srcLimit == realSourceEnd) : FALSE);
1430
1431         ucnv_toUnicode (conv,
1432                 &targ,
1433                 end,
1434                 (const char **)&src,
1435                 (const char *)srcLimit,
1436                 offs,
1437                 doFlush, /* flush if we're at the end of hte source data */
1438                 &status);
1439         if(testReset)
1440             ucnv_resetFromUnicode(conv);
1441         if(gInBufferSize ==999 && gOutBufferSize==999)
1442             ucnv_resetToUnicode(conv);
1443         /*        offs += (targ-oldTarg); */
1444
1445       } while ( (status == U_BUFFER_OVERFLOW_ERROR) || (U_SUCCESS(status) && (srcLimit < realSourceEnd)) ); /* while we just need another buffer */
1446
1447     if(U_FAILURE(status))
1448     {
1449         log_err("Problem doing %s toUnicode, errcode %s %s\n", codepage, myErrorName(status), gNuConvTestName);
1450         return FALSE;
1451     }
1452
1453     log_verbose("\nConversion done. %d bytes -> %d chars.\nResult :",
1454         sourcelen, targ-junkout);
1455     if(getTestOption(VERBOSITY_OPTION))
1456     {
1457         char junk[999];
1458         char offset_str[999];
1459
1460         UChar *ptr;
1461
1462         junk[0] = 0;
1463         offset_str[0] = 0;
1464
1465         for(ptr = junkout;ptr<targ;ptr++)
1466         {
1467             sprintf(junk + strlen(junk), "0x%04x, ", (0xFFFF) & (unsigned int)*ptr);
1468             sprintf(offset_str + strlen(offset_str), "0x%04x, ", (0xFFFF) & (unsigned int)junokout[ptr-junkout]);
1469         }
1470
1471         log_verbose(junk);
1472
1473         if ( checkOffsets )
1474           {
1475             log_verbose("\nOffsets:");
1476             log_verbose(offset_str);
1477           }
1478         log_verbose("\n");
1479     }
1480     ucnv_close(conv);
1481
1482     log_verbose("comparing %d uchars (%d bytes)..\n",expectlen,expectlen*2);
1483
1484     if (checkOffsets && (expectOffsets != 0))
1485     {
1486         if(memcmp(junokout,expectOffsets,(targ-junkout) * sizeof(int32_t))){
1487
1488             log_err("did not get the expected offsets. %s",gNuConvTestName);
1489             for(p=junkout;p<targ;p++)
1490                 log_err("%d, ", junokout[p-junkout]);
1491             log_err("\nExpected: ");
1492             for(i=0; i<(targ-junkout); i++)
1493                 log_err("%d,", expectOffsets[i]);
1494             log_err("");
1495             for(i=0; i<(targ-junkout); i++)
1496                 log_err("%X,", junkout[i]);
1497             log_err("");
1498             for(i=0; i<(src-(const char *)source); i++)
1499                 log_err("%X,", (unsigned char)source[i]);
1500         }
1501     }
1502
1503     if(!memcmp(junkout, expect, expectlen*2))
1504     {
1505         log_verbose("Matches!\n");
1506         return TRUE;
1507     }
1508     else
1509     {
1510         log_err("String does not match. %s\n", gNuConvTestName);
1511         log_verbose("String does not match. %s\n", gNuConvTestName);
1512         log_info("\nGot:");
1513         printUSeq(junkout, expectlen);
1514         log_info("\nExpected:");
1515         printUSeq(expect, expectlen);
1516         return FALSE;
1517     }
1518 }
1519
1520
1521 static void TestResetBehaviour(void){
1522 #if !UCONFIG_NO_LEGACY_CONVERSION
1523     log_verbose("Testing Reset for DBCS and MBCS\n");
1524     {
1525         static const UChar sampleText[]       = {0x00a1, 0xd801, 0xdc01, 0x00a4};
1526         static const uint8_t expected[] = {0xa2, 0xae, 0xa1, 0xe0, 0xa2, 0xb4};
1527         static const int32_t offsets[]        = {0x00, 0x00, 0x01, 0x01, 0x03, 0x03 };
1528
1529
1530         static const UChar sampleText1[] = {0x00a1, 0x00a4, 0x00a7, 0x00a8};
1531         static const uint8_t expected1[] = {0xa2, 0xae,0xA2,0xB4,0xA1,0xD7,0xA1,0xA7};
1532         static const int32_t offsets1[] =  { 0,2,4,6};
1533
1534         /*DBCS*/
1535         if(!testConvertFromU(sampleText, UPRV_LENGTHOF(sampleText),
1536                 expected, sizeof(expected), "ibm-1363", UCNV_FROM_U_CALLBACK_SUBSTITUTE , NULL, TRUE))
1537             log_err("u-> ibm-1363 [UCNV_DBCS portion] not match.\n");
1538         if(!testConvertFromU(sampleText, UPRV_LENGTHOF(sampleText),
1539                 expected, sizeof(expected), "ibm-1363", UCNV_FROM_U_CALLBACK_SUBSTITUTE,offsets , TRUE))
1540             log_err("u-> ibm-1363 [UCNV_DBCS portion] not match.\n");
1541
1542         if(!testConvertToU(expected1, sizeof(expected1),
1543                 sampleText1, UPRV_LENGTHOF(sampleText1), "ibm-1363",UCNV_TO_U_CALLBACK_SUBSTITUTE ,
1544                 offsets1, TRUE))
1545            log_err("ibm-1363 -> did not match.\n");
1546         /*MBCS*/
1547         if(!testConvertFromU(sampleText, UPRV_LENGTHOF(sampleText),
1548                 expected, sizeof(expected), "ibm-1363", UCNV_FROM_U_CALLBACK_SUBSTITUTE , NULL, TRUE))
1549             log_err("u-> ibm-1363 [UCNV_MBCS] not match.\n");
1550         if(!testConvertFromU(sampleText, UPRV_LENGTHOF(sampleText),
1551                 expected, sizeof(expected), "ibm-1363", UCNV_FROM_U_CALLBACK_SUBSTITUTE,offsets , TRUE))
1552             log_err("u-> ibm-1363 [UCNV_MBCS] not match.\n");
1553
1554         if(!testConvertToU(expected1, sizeof(expected1),
1555                 sampleText1, UPRV_LENGTHOF(sampleText1), "ibm-1363",UCNV_TO_U_CALLBACK_SUBSTITUTE ,
1556                 offsets1, TRUE))
1557            log_err("ibm-1363 -> did not match.\n");
1558
1559     }
1560
1561     log_verbose("Testing Reset for ISO-2022-jp\n");
1562     {
1563         static const UChar    sampleText[] =   { 0x4e00, 0x04e01, 0x0031, 0xd801, 0xdc01, 0x0032};
1564
1565         static const uint8_t expected[] = {0x1b, 0x24, 0x42,0x30,0x6c,0x43,0x7a,0x1b,0x28,0x42,
1566                                     0x31,0x1A, 0x32};
1567
1568
1569         static const int32_t offsets[] = {0,0,0,0,0,1,1,2,2,2,2,3,5 };
1570
1571
1572         static const UChar sampleText1[] = {0x4e00, 0x04e01, 0x0031,0x001A, 0x0032};
1573         static const uint8_t expected1[] = {0x1b, 0x24, 0x42,0x30,0x6c,0x43,0x7a,0x1b,0x28,0x42,
1574                                     0x31,0x1A, 0x32};
1575         static const int32_t offsets1[] =  { 3,5,10,11,12};
1576
1577         /*iso-2022-jp*/
1578         if(!testConvertFromU(sampleText, UPRV_LENGTHOF(sampleText),
1579                 expected, sizeof(expected), "iso-2022-jp",  UCNV_FROM_U_CALLBACK_SUBSTITUTE , NULL, TRUE))
1580             log_err("u-> not match.\n");
1581         if(!testConvertFromU(sampleText, UPRV_LENGTHOF(sampleText),
1582                 expected, sizeof(expected), "iso-2022-jp", UCNV_FROM_U_CALLBACK_SUBSTITUTE,offsets , TRUE))
1583             log_err("u->  not match.\n");
1584
1585         if(!testConvertToU(expected1, sizeof(expected1),
1586                 sampleText1, UPRV_LENGTHOF(sampleText1), "iso-2022-jp",UCNV_TO_U_CALLBACK_SUBSTITUTE ,
1587                 offsets1, TRUE))
1588            log_err("iso-2022-jp -> did not match.\n");
1589
1590     }
1591
1592     log_verbose("Testing Reset for ISO-2022-cn\n");
1593     {
1594         static const UChar    sampleText[] =   { 0x4e00, 0x04e01, 0x0031, 0xd801, 0xdc01, 0x0032};
1595
1596         static const uint8_t expected[] = {
1597                                     0x1B, 0x24, 0x29, 0x41, 0x0E, 0x52, 0x3B,
1598                                     0x36, 0x21,
1599                                     0x0f, 0x31,
1600                                     0x1A,
1601                                     0x32
1602                                     };
1603
1604
1605         static const int32_t offsets[] = {
1606                                     0,    0,    0,    0,    0,    0,    0,
1607                                     1,    1,
1608                                     2,    2,
1609                                     3,
1610                                     5,  };
1611
1612         UChar sampleText1[] = {0x4e00, 0x04e01, 0x0031,0x001A, 0x0032};
1613         static const uint8_t expected1[] = {
1614                                     0x1B, 0x24, 0x29, 0x41, 0x0E, 0x52, 0x3B,
1615                                     0x36, 0x21,
1616                                     0x1B, 0x24, 0x29, 0x47, 0x24, 0x22,
1617                                     0x0f, 0x1A,
1618                                     0x32
1619                                     };
1620         static const int32_t offsets1[] =  { 5,7,13,16,17};
1621
1622         /*iso-2022-CN*/
1623         if(!testConvertFromU(sampleText, UPRV_LENGTHOF(sampleText),
1624                 expected, sizeof(expected), "iso-2022-cn", UCNV_FROM_U_CALLBACK_SUBSTITUTE , NULL, TRUE))
1625             log_err("u-> not match.\n");
1626         if(!testConvertFromU(sampleText, UPRV_LENGTHOF(sampleText),
1627                 expected, sizeof(expected), "iso-2022-cn", UCNV_FROM_U_CALLBACK_SUBSTITUTE,offsets , TRUE))
1628             log_err("u-> not match.\n");
1629
1630         if(!testConvertToU(expected1, sizeof(expected1),
1631                 sampleText1, UPRV_LENGTHOF(sampleText1), "iso-2022-cn",UCNV_TO_U_CALLBACK_SUBSTITUTE ,
1632                 offsets1, TRUE))
1633            log_err("iso-2022-cn -> did not match.\n");
1634     }
1635
1636         log_verbose("Testing Reset for ISO-2022-kr\n");
1637     {
1638         UChar    sampleText[] =   { 0x4e00,0xd801, 0xdc01, 0x04e01, 0x0031, 0xd801, 0xdc01, 0x0032};
1639
1640         static const uint8_t expected[] = {0x1B, 0x24, 0x29, 0x43,
1641                                     0x0E, 0x6C, 0x69,
1642                                     0x0f, 0x1A,
1643                                     0x0e, 0x6F, 0x4B,
1644                                     0x0F, 0x31,
1645                                     0x1A,
1646                                     0x32 };
1647
1648         static const int32_t offsets[] = {-1, -1, -1, -1,
1649                               0, 0, 0,
1650                               1, 1,
1651                               3, 3, 3,
1652                               4, 4,
1653                               5,
1654                               7,
1655                             };
1656         static const UChar    sampleText1[] =   { 0x4e00,0x0041, 0x04e01, 0x0031, 0x0042, 0x0032};
1657
1658         static const uint8_t expected1[] = {0x1B, 0x24, 0x29, 0x43,
1659                                     0x0E, 0x6C, 0x69,
1660                                     0x0f, 0x41,
1661                                     0x0e, 0x6F, 0x4B,
1662                                     0x0F, 0x31,
1663                                     0x42,
1664                                     0x32 };
1665
1666         static const int32_t offsets1[] = {
1667                               5, 8, 10,
1668                               13, 14, 15
1669
1670                             };
1671         /*iso-2022-kr*/
1672         if(!testConvertFromU(sampleText, UPRV_LENGTHOF(sampleText),
1673                 expected, sizeof(expected), "iso-2022-kr",  UCNV_FROM_U_CALLBACK_SUBSTITUTE , NULL, TRUE))
1674             log_err("u-> iso-2022-kr [UCNV_DBCS] not match.\n");
1675         if(!testConvertFromU(sampleText, UPRV_LENGTHOF(sampleText),
1676                 expected, sizeof(expected), "iso-2022-kr",  UCNV_FROM_U_CALLBACK_SUBSTITUTE,offsets , TRUE))
1677             log_err("u-> iso-2022-kr [UCNV_DBCS] not match.\n");
1678         if(!testConvertToU(expected1, sizeof(expected1),
1679                 sampleText1, UPRV_LENGTHOF(sampleText1), "iso-2022-kr",UCNV_TO_U_CALLBACK_SUBSTITUTE ,
1680                 offsets1, TRUE))
1681            log_err("iso-2022-kr -> did not match.\n");
1682     }
1683
1684         log_verbose("Testing Reset for HZ\n");
1685     {
1686         static const UChar    sampleText[] =   { 0x4e00, 0xd801, 0xdc01, 0x04e01, 0x0031, 0xd801, 0xdc01, 0x0032};
1687
1688         static const uint8_t expected[] = {0x7E, 0x7B, 0x52, 0x3B,
1689                                     0x7E, 0x7D, 0x1A,
1690                                     0x7E, 0x7B, 0x36, 0x21,
1691                                     0x7E, 0x7D, 0x31,
1692                                     0x1A,
1693                                     0x32 };
1694
1695
1696         static const int32_t offsets[] = {0,0,0,0,
1697                              1,1,1,
1698                              3,3,3,3,
1699                              4,4,4,
1700                              5,
1701                              7,};
1702         static const UChar    sampleText1[] =   { 0x4e00, 0x0035, 0x04e01, 0x0031, 0x0041, 0x0032};
1703
1704         static const uint8_t expected1[] = {0x7E, 0x7B, 0x52, 0x3B,
1705                                     0x7E, 0x7D, 0x35,
1706                                     0x7E, 0x7B, 0x36, 0x21,
1707                                     0x7E, 0x7D, 0x31,
1708                                     0x41,
1709                                     0x32 };
1710
1711
1712         static const int32_t offsets1[] = {2,6,9,13,14,15
1713                             };
1714
1715         /*hz*/
1716         if(!testConvertFromU(sampleText, UPRV_LENGTHOF(sampleText),
1717                 expected, sizeof(expected), "HZ", UCNV_FROM_U_CALLBACK_SUBSTITUTE,NULL , TRUE))
1718             log_err("u->  not match.\n");
1719         if(!testConvertFromU(sampleText, UPRV_LENGTHOF(sampleText),
1720                 expected, sizeof(expected), "HZ", UCNV_FROM_U_CALLBACK_SUBSTITUTE,offsets , TRUE))
1721             log_err("u->  not match.\n");
1722         if(!testConvertToU(expected1, sizeof(expected1),
1723                 sampleText1, UPRV_LENGTHOF(sampleText1), "hz",UCNV_TO_U_CALLBACK_SUBSTITUTE ,
1724                 offsets1, TRUE))
1725            log_err("hz -> did not match.\n");
1726     }
1727 #endif
1728
1729     /*UTF-8*/
1730      log_verbose("Testing for UTF8\n");
1731     {
1732         static const UChar    sampleText[] =   { 0x4e00, 0x0701, 0x0031, 0xbfc1, 0xd801, 0xdc01, 0x0032};
1733         int32_t offsets[]={0x00, 0x00, 0x00, 0x01, 0x01, 0x02,
1734                            0x03, 0x03, 0x03, 0x04, 0x04, 0x04,
1735                            0x04, 0x06 };
1736         static const uint8_t expected[] = {0xe4, 0xb8, 0x80, 0xdc, 0x81, 0x31,
1737             0xeb, 0xbf, 0x81, 0xF0, 0x90, 0x90, 0x81, 0x32};
1738
1739
1740         static const int32_t fromOffsets[] = { 0x0000, 0x0003, 0x0005, 0x0006, 0x0009, 0x0009, 0x000D };
1741         /*UTF-8*/
1742         if(!testConvertFromU(sampleText, UPRV_LENGTHOF(sampleText),
1743             expected, sizeof(expected), "UTF8", UCNV_FROM_U_CALLBACK_SUBSTITUTE,offsets , TRUE))
1744             log_err("u-> UTF8 with offsets and flush true did not match.\n");
1745         if(!testConvertFromU(sampleText, UPRV_LENGTHOF(sampleText),
1746             expected, sizeof(expected), "UTF8",  UCNV_FROM_U_CALLBACK_SUBSTITUTE,NULL , TRUE))
1747             log_err("u-> UTF8 with offsets and flush true did not match.\n");
1748         if(!testConvertFromU(sampleText, UPRV_LENGTHOF(sampleText),
1749             expected, sizeof(expected), "UTF8", UCNV_FROM_U_CALLBACK_SUBSTITUTE,offsets , TRUE))
1750             log_err("u-> UTF8 with offsets and flush true did not match.\n");
1751         if(!testConvertFromU(sampleText, UPRV_LENGTHOF(sampleText),
1752             expected, sizeof(expected), "UTF8",  UCNV_FROM_U_CALLBACK_SUBSTITUTE,NULL , TRUE))
1753             log_err("u-> UTF8 with offsets and flush true did not match.\n");
1754         if(!testConvertToU(expected, sizeof(expected),
1755             sampleText, UPRV_LENGTHOF(sampleText), "UTF8",UCNV_TO_U_CALLBACK_SUBSTITUTE , NULL, TRUE))
1756             log_err("UTF8 -> did not match.\n");
1757         if(!testConvertToU(expected, sizeof(expected),
1758             sampleText, UPRV_LENGTHOF(sampleText), "UTF8", UCNV_TO_U_CALLBACK_SUBSTITUTE , NULL, TRUE))
1759             log_err("UTF8 -> did not match.\n");
1760         if(!testConvertToU(expected, sizeof(expected),
1761             sampleText, UPRV_LENGTHOF(sampleText), "UTF8",UCNV_TO_U_CALLBACK_SUBSTITUTE , fromOffsets, TRUE))
1762             log_err("UTF8 -> did not match.\n");
1763         if(!testConvertToU(expected, sizeof(expected),
1764             sampleText, UPRV_LENGTHOF(sampleText), "UTF8", UCNV_TO_U_CALLBACK_SUBSTITUTE , fromOffsets, TRUE))
1765             log_err("UTF8 -> did not match.\n");
1766
1767     }
1768
1769 }
1770
1771 /* Test that U_TRUNCATED_CHAR_FOUND is set. */
1772 static void
1773 doTestTruncated(const char *cnvName, const uint8_t *bytes, int32_t length) {
1774     UConverter *cnv;
1775
1776     UChar buffer[2];
1777     UChar *target, *targetLimit;
1778     const char *source, *sourceLimit;
1779
1780     UErrorCode errorCode;
1781
1782     errorCode=U_ZERO_ERROR;
1783     cnv=ucnv_open(cnvName, &errorCode);
1784     if(U_FAILURE(errorCode)) {
1785         log_data_err("error TestTruncated: unable to open \"%s\" - %s\n", cnvName, u_errorName(errorCode));
1786         return;
1787     }
1788     ucnv_setToUCallBack(cnv, UCNV_TO_U_CALLBACK_STOP, NULL, NULL, NULL, &errorCode);
1789     if(U_FAILURE(errorCode)) {
1790         log_data_err("error TestTruncated: unable to set the stop callback on \"%s\" - %s\n",
1791                     cnvName, u_errorName(errorCode));
1792         ucnv_close(cnv);
1793         return;
1794     }
1795
1796     source=(const char *)bytes;
1797     sourceLimit=source+length;
1798     target=buffer;
1799     targetLimit=buffer+UPRV_LENGTHOF(buffer);
1800
1801     /* 1. input bytes with flush=FALSE, then input nothing with flush=TRUE */
1802     ucnv_toUnicode(cnv, &target, targetLimit, &source, sourceLimit, NULL, FALSE, &errorCode);
1803     if(U_FAILURE(errorCode) || source!=sourceLimit || target!=buffer) {
1804         log_err("error TestTruncated(%s, 1a): input bytes[%d], flush=FALSE: %s, input left %d, output %d\n",
1805                 cnvName, length, u_errorName(errorCode), (int)(sourceLimit-source), (int)(target-buffer));
1806     }
1807
1808     errorCode=U_ZERO_ERROR;
1809     source=sourceLimit;
1810     target=buffer;
1811     ucnv_toUnicode(cnv, &target, targetLimit, &source, sourceLimit, NULL, TRUE, &errorCode);
1812     if(errorCode!=U_TRUNCATED_CHAR_FOUND || target!=buffer) {
1813         log_err("error TestTruncated(%s, 1b): no input (previously %d), flush=TRUE: %s (should be U_TRUNCATED_CHAR_FOUND), output %d\n",
1814                 cnvName, (int)length, u_errorName(errorCode), (int)(target-buffer));
1815     }
1816
1817     /* 2. input bytes with flush=TRUE */
1818     ucnv_resetToUnicode(cnv);
1819
1820     errorCode=U_ZERO_ERROR;
1821     source=(const char *)bytes;
1822     target=buffer;
1823     ucnv_toUnicode(cnv, &target, targetLimit, &source, sourceLimit, NULL, TRUE, &errorCode);
1824     if(errorCode!=U_TRUNCATED_CHAR_FOUND || source!=sourceLimit || target!=buffer) {
1825         log_err("error TestTruncated(%s, 2): input bytes[%d], flush=TRUE: %s (should be U_TRUNCATED_CHAR_FOUND), input left %d, output %d\n",
1826                 cnvName, length, u_errorName(errorCode), (int)(sourceLimit-source), (int)(target-buffer));
1827     }
1828
1829
1830     ucnv_close(cnv);
1831 }
1832
1833 static void
1834 TestTruncated() {
1835     static const struct {
1836         const char *cnvName;
1837         uint8_t bytes[8]; /* partial input bytes resulting in no output */
1838         int32_t length;
1839     } testCases[]={
1840         { "IMAP-mailbox-name",  { 0x26 }, 1 }, /* & */
1841         { "IMAP-mailbox-name",  { 0x26, 0x42 }, 2 }, /* &B */
1842         { "IMAP-mailbox-name",  { 0x26, 0x42, 0x42 }, 3 }, /* &BB */
1843         { "IMAP-mailbox-name",  { 0x26, 0x41, 0x41 }, 3 }, /* &AA */
1844
1845         { "UTF-7",      { 0x2b, 0x42 }, 2 }, /* +B */
1846         { "UTF-8",      { 0xd1 }, 1 },
1847
1848         { "UTF-16BE",   { 0x4e }, 1 },
1849         { "UTF-16LE",   { 0x4e }, 1 },
1850         { "UTF-16",     { 0x4e }, 1 },
1851         { "UTF-16",     { 0xff }, 1 },
1852         { "UTF-16",     { 0xfe, 0xff, 0x4e }, 3 },
1853
1854         { "UTF-32BE",   { 0, 0, 0x4e }, 3 },
1855         { "UTF-32LE",   { 0x4e }, 1 },
1856         { "UTF-32",     { 0, 0, 0x4e }, 3 },
1857         { "UTF-32",     { 0xff }, 1 },
1858         { "UTF-32",     { 0, 0, 0xfe, 0xff, 0 }, 5 },
1859         { "SCSU",       { 0x0e, 0x4e }, 2 }, /* SQU 0x4e */
1860
1861 #if !UCONFIG_NO_LEGACY_CONVERSION
1862         { "BOCU-1",     { 0xd5 }, 1 },
1863
1864         { "Shift-JIS",  { 0xe0 }, 1 },
1865
1866         { "ibm-939",    { 0x0e, 0x41 }, 2 } /* SO 0x41 */
1867 #else
1868         { "BOCU-1",     { 0xd5 }, 1 ,}
1869 #endif
1870     };
1871     int32_t i;
1872
1873     for(i=0; i<UPRV_LENGTHOF(testCases); ++i) {
1874         doTestTruncated(testCases[i].cnvName, testCases[i].bytes, testCases[i].length);
1875     }
1876 }
1877
1878 typedef struct NameRange {
1879     const char *name;
1880     UChar32 start, end, start2, end2, notStart, notEnd;
1881 } NameRange;
1882
1883 static void
1884 TestUnicodeSet() {
1885     UErrorCode errorCode;
1886     UConverter *cnv;
1887     USet *set;
1888     const char *name;
1889     int32_t i, count;
1890
1891     static const char *const completeSetNames[]={
1892         "UTF-7",
1893         "UTF-8",
1894         "UTF-16",
1895         "UTF-16BE",
1896         "UTF-16LE",
1897         "UTF-32",
1898         "UTF-32BE",
1899         "UTF-32LE",
1900         "SCSU",
1901         "BOCU-1",
1902         "CESU-8",
1903 #if !UCONFIG_NO_LEGACY_CONVERSION
1904         "gb18030",
1905 #endif
1906         "IMAP-mailbox-name"
1907     };
1908 #if !UCONFIG_NO_LEGACY_CONVERSION
1909     static const char *const lmbcsNames[]={
1910         "LMBCS-1",
1911         "LMBCS-2",
1912         "LMBCS-3",
1913         "LMBCS-4",
1914         "LMBCS-5",
1915         "LMBCS-6",
1916         "LMBCS-8",
1917         "LMBCS-11",
1918         "LMBCS-16",
1919         "LMBCS-17",
1920         "LMBCS-18",
1921         "LMBCS-19"
1922     };
1923 #endif
1924
1925     static const NameRange nameRanges[]={
1926         { "US-ASCII", 0, 0x7f, -1, -1, 0x80, 0x10ffff },
1927 #if !UCONFIG_NO_LEGACY_CONVERSION
1928         { "ibm-367", 0, 0x7f, -1, -1, 0x80, 0x10ffff },
1929 #endif
1930         { "ISO-8859-1", 0, 0x7f, -1, -1, 0x100, 0x10ffff },
1931 #if !UCONFIG_NO_LEGACY_CONVERSION
1932         { "UTF-8", 0, 0xd7ff, 0xe000, 0x10ffff, 0xd800, 0xdfff },
1933         { "windows-1251", 0, 0x7f, 0x410, 0x44f, 0x3000, 0xd7ff },
1934         /* HZ test case fixed and moved to intltest's conversion.txt, ticket #6002 */
1935         { "shift-jis", 0x3041, 0x3093, 0x30a1, 0x30f3, 0x900, 0x1cff }
1936 #else
1937         { "UTF-8", 0, 0xd7ff, 0xe000, 0x10ffff, 0xd800, 0xdfff }
1938 #endif
1939     };
1940
1941     /* open an empty set */
1942     set=uset_open(1, 0);
1943
1944     count=ucnv_countAvailable();
1945     for(i=0; i<count; ++i) {
1946         errorCode=U_ZERO_ERROR;
1947         name=ucnv_getAvailableName(i);
1948         cnv=ucnv_open(name, &errorCode);
1949         if(U_FAILURE(errorCode)) {
1950             log_data_err("error: unable to open converter %s - %s\n",
1951                     name, u_errorName(errorCode));
1952             continue;
1953         }
1954
1955         uset_clear(set);
1956         ucnv_getUnicodeSet(cnv, set, UCNV_ROUNDTRIP_SET, &errorCode);
1957         if(U_FAILURE(errorCode)) {
1958             log_err("error: ucnv_getUnicodeSet(%s) failed - %s\n",
1959                     name, u_errorName(errorCode));
1960         } else if(uset_size(set)==0) {
1961             log_err("error: ucnv_getUnicodeSet(%s) returns an empty set\n", name);
1962         }
1963
1964         ucnv_close(cnv);
1965     }
1966
1967     /* test converters that are known to convert all of Unicode (except maybe for surrogates) */
1968     for(i=0; i<UPRV_LENGTHOF(completeSetNames); ++i) {
1969         errorCode=U_ZERO_ERROR;
1970         name=completeSetNames[i];
1971         cnv=ucnv_open(name, &errorCode);
1972         if(U_FAILURE(errorCode)) {
1973             log_data_err("error: unable to open converter %s - %s\n",
1974                     name, u_errorName(errorCode));
1975             continue;
1976         }
1977
1978         uset_clear(set);
1979         ucnv_getUnicodeSet(cnv, set, UCNV_ROUNDTRIP_SET, &errorCode);
1980         if(U_FAILURE(errorCode)) {
1981             log_err("error: ucnv_getUnicodeSet(%s) failed - %s\n",
1982                     name, u_errorName(errorCode));
1983         } else if(!uset_containsRange(set, 0, 0xd7ff) || !uset_containsRange(set, 0xe000, 0x10ffff)) {
1984             log_err("error: ucnv_getUnicodeSet(%s) does not return an all-Unicode set\n", name);
1985         }
1986
1987         ucnv_close(cnv);
1988     }
1989
1990 #if !UCONFIG_NO_LEGACY_CONVERSION
1991     /* test LMBCS variants which convert all of Unicode except for U+F6xx */
1992     for(i=0; i<UPRV_LENGTHOF(lmbcsNames); ++i) {
1993         errorCode=U_ZERO_ERROR;
1994         name=lmbcsNames[i];
1995         cnv=ucnv_open(name, &errorCode);
1996         if(U_FAILURE(errorCode)) {
1997             log_data_err("error: unable to open converter %s - %s\n",
1998                     name, u_errorName(errorCode));
1999             continue;
2000         }
2001
2002         uset_clear(set);
2003         ucnv_getUnicodeSet(cnv, set, UCNV_ROUNDTRIP_SET, &errorCode);
2004         if(U_FAILURE(errorCode)) {
2005             log_err("error: ucnv_getUnicodeSet(%s) failed - %s\n",
2006                     name, u_errorName(errorCode));
2007         } else if(!uset_containsRange(set, 0, 0xf5ff) || !uset_containsRange(set, 0xf700, 0x10ffff)) {
2008             log_err("error: ucnv_getUnicodeSet(%s) does not return an all-Unicode set (minus U+F6xx)\n", name);
2009         }
2010
2011         ucnv_close(cnv);
2012     }
2013 #endif
2014
2015     /* test specific sets */
2016     for(i=0; i<UPRV_LENGTHOF(nameRanges); ++i) {
2017         errorCode=U_ZERO_ERROR;
2018         name=nameRanges[i].name;
2019         cnv=ucnv_open(name, &errorCode);
2020         if(U_FAILURE(errorCode)) {
2021             log_data_err("error: unable to open converter %s - %s\n",
2022                          name, u_errorName(errorCode));
2023             continue;
2024         }
2025
2026         uset_clear(set);
2027         ucnv_getUnicodeSet(cnv, set, UCNV_ROUNDTRIP_SET, &errorCode);
2028         if(U_FAILURE(errorCode)) {
2029             log_err("error: ucnv_getUnicodeSet(%s) failed - %s\n",
2030                     name, u_errorName(errorCode));
2031         } else if(
2032             !uset_containsRange(set, nameRanges[i].start, nameRanges[i].end) ||
2033             (nameRanges[i].start2>=0 && !uset_containsRange(set, nameRanges[i].start2, nameRanges[i].end2))
2034         ) {
2035             log_err("error: ucnv_getUnicodeSet(%s) does not contain the expected ranges\n", name);
2036         } else if(nameRanges[i].notStart>=0) {
2037             /* simulate containsAny() with the C API */
2038             uset_complement(set);
2039             if(!uset_containsRange(set, nameRanges[i].notStart, nameRanges[i].notEnd)) {
2040                 log_err("error: ucnv_getUnicodeSet(%s) contains part of the unexpected range\n", name);
2041             }
2042         }
2043
2044         ucnv_close(cnv);
2045     }
2046
2047     errorCode = U_ZERO_ERROR;
2048     ucnv_getUnicodeSet(NULL, set, UCNV_ROUNDTRIP_SET, &errorCode);
2049     if (errorCode != U_ILLEGAL_ARGUMENT_ERROR) {
2050         log_err("error: ucnv_getUnicodeSet(NULL) returned wrong status code %s\n", u_errorName(errorCode));
2051     }
2052     errorCode = U_PARSE_ERROR;
2053     /* Make sure that it does nothing if an error is passed in. Difficult to proper test for. */
2054     ucnv_getUnicodeSet(NULL, NULL, UCNV_ROUNDTRIP_SET, &errorCode);
2055     if (errorCode != U_PARSE_ERROR) {
2056         log_err("error: ucnv_getUnicodeSet(NULL) returned wrong status code %s\n", u_errorName(errorCode));
2057     }
2058
2059     uset_close(set);
2060 }