source/test/intltest/alphaindextst.cpp

   1 // Copyright (C) 2016 and later: Unicode, Inc. and others.
   2 // License & terms of use: http://www.unicode.org/copyright.html
   3 /********************************************************************
   4  * COPYRIGHT:
   5  * Copyright (c) 2012-2016, International Business Machines Corporation
   6  * and others. All Rights Reserved.
   7  ********************************************************************/
   8 //
   9 //   file:  alphaindextst.cpp
  10 //          Alphabetic Index Tests.
  11 //
  12 //   Note: please... no character literals cast to UChars.. use (UChar)0xZZZZ
  13
  14 #include <stdio.h>  // for sprintf
  15
  16 #include "intltest.h"
  17 #include "alphaindextst.h"
  18 #include "cmemory.h"
  19
  20 #include "unicode/alphaindex.h"
  21 #include "unicode/coll.h"
  22 #include "unicode/localpointer.h"
  23 #include "unicode/tblcoll.h"
  24 #include "unicode/uniset.h"
  25
  26 #if !UCONFIG_NO_COLLATION && !UCONFIG_NO_NORMALIZATION
  27
  28 // #include <string>
  29 // #include <iostream>
  30
  31 namespace {
  32
  33 UnicodeString joinLabelsAndAppend(AlphabeticIndex::ImmutableIndex &index, UnicodeString &dest) {
  34     int32_t oldLength = dest.length();
  35     const AlphabeticIndex::Bucket *bucket;
  36     for (int32_t i = 0; (bucket = index.getBucket(i)) != NULL; ++i) {
  37         if (dest.length() > oldLength) {
  38             dest.append((UChar)0x3A);  // ':'
  39         }
  40         dest.append(bucket->getLabel());
  41     }
  42     return dest;
  43 }
  44
  45 }  // namespace
  46
  47 AlphabeticIndexTest::AlphabeticIndexTest() {
  48 }
  49
  50 AlphabeticIndexTest::~AlphabeticIndexTest() {
  51 }
  52
  53 void AlphabeticIndexTest::runIndexedTest( int32_t index, UBool exec, const char* &name, char* /*par*/ )
  54 {
  55     if (exec) logln("TestSuite AlphabeticIndex: ");
  56     TESTCASE_AUTO_BEGIN;
  57     TESTCASE_AUTO(APITest);
  58     TESTCASE_AUTO(ManyLocalesTest);
  59     TESTCASE_AUTO(HackPinyinTest);
  60     TESTCASE_AUTO(TestBug9009);
  61     TESTCASE_AUTO(TestIndexCharactersList);
  62     TESTCASE_AUTO(TestHaniFirst);
  63     TESTCASE_AUTO(TestPinyinFirst);
  64     TESTCASE_AUTO(TestSchSt);
  65     TESTCASE_AUTO(TestNoLabels);
  66     TESTCASE_AUTO(TestChineseZhuyin);
  67     TESTCASE_AUTO(TestJapaneseKanji);
  68     TESTCASE_AUTO(TestChineseUnihan);
  69     TESTCASE_AUTO_END;
  70 }
  71
  72 #define TEST_CHECK_STATUS {if (U_FAILURE(status)) {dataerrln("%s:%d: Test failure.  status=%s", \
  73                                                               __FILE__, __LINE__, u_errorName(status)); return;}}
  74
  75 #define TEST_ASSERT(expr) {if ((expr)==FALSE) {errln("%s:%d: Test failure \n", __FILE__, __LINE__);};}
  76
  77 //
  78 //  APITest.   Invoke every function at least once, and check that it does something.
  79 //             Does not attempt to check complete functionality.
  80 //
  81 void AlphabeticIndexTest::APITest() {
  82     //
  83     //  Simple constructor and destructor,  getBucketCount()
  84     //
  85     UErrorCode status = U_ZERO_ERROR;
  86     int32_t lc = 0;
  87     int32_t i  = 0;
  88     AlphabeticIndex *index = new AlphabeticIndex(Locale::getEnglish(), status);
  89     TEST_CHECK_STATUS;
  90     lc = index->getBucketCount(status);
  91     TEST_CHECK_STATUS;
  92     TEST_ASSERT(28 == lc);    // 26 letters plus two under/overflow labels.
  93     //printf("getBucketCount() == %d\n", lc);
  94     delete index;
  95
  96     // Constructor from a Collator
  97     //
  98     status = U_ZERO_ERROR;
  99     RuleBasedCollator *coll = dynamic_cast<RuleBasedCollator *>(
 100         Collator::createInstance(Locale::getGerman(), status));
 101     TEST_CHECK_STATUS;
 102     TEST_ASSERT(coll != NULL);
 103     index = new AlphabeticIndex(coll, status);
 104     TEST_CHECK_STATUS;
 105     TEST_ASSERT(coll == &index->getCollator());
 106     assertEquals("only the underflow label in an index built from a collator",
 107                  1, index->getBucketCount(status));
 108     TEST_CHECK_STATUS;
 109     delete index;
 110
 111
 112     // addLabels()
 113
 114     status = U_ZERO_ERROR;
 115     index = new AlphabeticIndex(Locale::getEnglish(), status);
 116     TEST_CHECK_STATUS;
 117     UnicodeSet additions;
 118     additions.add((UChar32)0x410).add((UChar32)0x415);   // A couple of Cyrillic letters
 119     index->addLabels(additions, status);
 120     TEST_CHECK_STATUS;
 121     lc = index->getBucketCount(status);
 122     TEST_CHECK_STATUS;
 123     assertEquals("underflow, A-Z, inflow, 2 Cyrillic, overflow",
 124                  31, index->getBucketCount(status));
 125     // std::cout << lc << std::endl;
 126     delete index;
 127
 128
 129     // addLabels(Locale)
 130
 131     status = U_ZERO_ERROR;
 132     index = new AlphabeticIndex(Locale::getEnglish(), status);
 133     TEST_CHECK_STATUS;
 134     AlphabeticIndex &aip = index->addLabels(Locale::getJapanese(), status);
 135     TEST_ASSERT(&aip == index);
 136     TEST_CHECK_STATUS;
 137     lc = index->getBucketCount(status);
 138     TEST_CHECK_STATUS;
 139     TEST_ASSERT(35 < lc);  // Japanese should add a bunch.  Don't rely on the exact value.
 140     delete index;
 141
 142     // GetCollator(),  Get under/in/over flow labels
 143
 144     status = U_ZERO_ERROR;
 145     index = new AlphabeticIndex(Locale::getGerman(), status);
 146     TEST_CHECK_STATUS;
 147     Collator *germanCol = Collator::createInstance(Locale::getGerman(), status);
 148     TEST_CHECK_STATUS;
 149     const RuleBasedCollator &indexCol = index->getCollator();
 150     TEST_ASSERT(*germanCol == indexCol);
 151     delete germanCol;
 152
 153     UnicodeString ELLIPSIS;  ELLIPSIS.append((UChar32)0x2026);
 154     UnicodeString s = index->getUnderflowLabel();
 155     TEST_ASSERT(ELLIPSIS == s);
 156     s = index->getOverflowLabel();
 157     TEST_ASSERT(ELLIPSIS == s);
 158     s = index->getInflowLabel();
 159     TEST_ASSERT(ELLIPSIS == s);
 160     index->setOverflowLabel(UNICODE_STRING_SIMPLE("O"), status);
 161     index->setUnderflowLabel(UNICODE_STRING_SIMPLE("U"), status).setInflowLabel(UNICODE_STRING_SIMPLE("I"), status);
 162     s = index->getUnderflowLabel();
 163     TEST_ASSERT(UNICODE_STRING_SIMPLE("U") == s);
 164     s = index->getOverflowLabel();
 165     TEST_ASSERT(UNICODE_STRING_SIMPLE("O") == s);
 166     s = index->getInflowLabel();
 167     TEST_ASSERT(UNICODE_STRING_SIMPLE("I") == s);
 168
 169
 170
 171
 172     delete index;
 173
 174
 175
 176     const UnicodeString adam = UNICODE_STRING_SIMPLE("Adam");
 177     const UnicodeString baker = UNICODE_STRING_SIMPLE("Baker");
 178     const UnicodeString charlie = UNICODE_STRING_SIMPLE("Charlie");
 179     const UnicodeString chad = UNICODE_STRING_SIMPLE("Chad");
 180     const UnicodeString zed  = UNICODE_STRING_SIMPLE("Zed");
 181     const UnicodeString Cyrillic = UNICODE_STRING_SIMPLE("\\u0410\\u0443\\u0435").unescape();
 182
 183     // addRecord(), verify that it comes back out.
 184     //
 185     status = U_ZERO_ERROR;
 186     index = new AlphabeticIndex(Locale::getEnglish(), status);
 187     TEST_CHECK_STATUS;
 188     index->addRecord(UnicodeString("Adam"), this, status);
 189     UBool   b;
 190     TEST_CHECK_STATUS;
 191     index->resetBucketIterator(status);
 192     TEST_CHECK_STATUS;
 193     index->nextBucket(status);  // Move to underflow label
 194     index->nextBucket(status);  // Move to "A"
 195     TEST_CHECK_STATUS;
 196     const UnicodeString &label2 = index->getBucketLabel();
 197     UnicodeString A_STR = UNICODE_STRING_SIMPLE("A");
 198     TEST_ASSERT(A_STR == label2);
 199
 200     b = index->nextRecord(status);
 201     TEST_CHECK_STATUS;
 202     TEST_ASSERT(b);
 203     const UnicodeString &itemName = index->getRecordName();
 204     TEST_ASSERT(adam == itemName);
 205
 206     const void *itemContext = index->getRecordData();
 207     TEST_ASSERT(itemContext == this);
 208
 209     delete index;
 210
 211     // clearRecords, addRecord(), Iteration
 212
 213     status = U_ZERO_ERROR;
 214     index = new AlphabeticIndex(Locale::getEnglish(), status);
 215     TEST_CHECK_STATUS;
 216     while (index->nextBucket(status)) {
 217         TEST_CHECK_STATUS;
 218         while (index->nextRecord(status)) {
 219             TEST_CHECK_STATUS;
 220             TEST_ASSERT(FALSE);   // No items have been added.
 221         }
 222         TEST_CHECK_STATUS;
 223     }
 224
 225     index->addRecord(adam, NULL, status);
 226     index->addRecord(baker, NULL, status);
 227     index->addRecord(charlie, NULL, status);
 228     index->addRecord(chad, NULL, status);
 229     TEST_CHECK_STATUS;
 230     int itemCount = 0;
 231     index->resetBucketIterator(status);
 232     while (index->nextBucket(status)) {
 233         TEST_CHECK_STATUS;
 234         while (index->nextRecord(status)) {
 235             TEST_CHECK_STATUS;
 236             ++itemCount;
 237         }
 238     }
 239     TEST_CHECK_STATUS;
 240     TEST_ASSERT(itemCount == 4);
 241
 242     TEST_ASSERT(index->nextBucket(status) == FALSE);
 243     index->resetBucketIterator(status);
 244     TEST_CHECK_STATUS;
 245     TEST_ASSERT(index->nextBucket(status) == TRUE);
 246
 247     index->clearRecords(status);
 248     TEST_CHECK_STATUS;
 249     index->resetBucketIterator(status);
 250     while (index->nextBucket(status)) {
 251         TEST_CHECK_STATUS;
 252         while (index->nextRecord(status)) {
 253             TEST_ASSERT(FALSE);   // No items have been added.
 254         }
 255     }
 256     TEST_CHECK_STATUS;
 257     delete index;
 258
 259     // getBucketLabel(), getBucketType()
 260
 261     status = U_ZERO_ERROR;
 262     index = new AlphabeticIndex(Locale::getEnglish(), status);
 263     TEST_CHECK_STATUS;
 264     index->setUnderflowLabel(adam, status).setOverflowLabel(charlie, status);
 265     TEST_CHECK_STATUS;
 266     for (i=0; index->nextBucket(status); i++) {
 267         TEST_CHECK_STATUS;
 268         UnicodeString label = index->getBucketLabel();
 269         UAlphabeticIndexLabelType type = index->getBucketLabelType();
 270         if (i == 0) {
 271             TEST_ASSERT(type == U_ALPHAINDEX_UNDERFLOW);
 272             TEST_ASSERT(label == adam);
 273         } else if (i <= 26) {
 274             // Labels A - Z for English locale
 275             TEST_ASSERT(type == U_ALPHAINDEX_NORMAL);
 276             UnicodeString expectedLabel((UChar)(0x40 + i));
 277             TEST_ASSERT(expectedLabel == label);
 278         } else if (i == 27) {
 279             TEST_ASSERT(type == U_ALPHAINDEX_OVERFLOW);
 280             TEST_ASSERT(label == charlie);
 281         } else {
 282             TEST_ASSERT(FALSE);
 283         }
 284     }
 285     TEST_ASSERT(i==28);
 286     delete index;
 287
 288     // getBucketIndex()
 289
 290     status = U_ZERO_ERROR;
 291     index = new AlphabeticIndex(Locale::getEnglish(), status);
 292     TEST_CHECK_STATUS;
 293     int32_t n = index->getBucketIndex(adam, status);
 294     TEST_CHECK_STATUS;
 295     TEST_ASSERT(n == 1);    /*  Label #0 is underflow, 1 is A, etc. */
 296     n = index->getBucketIndex(baker, status);
 297     TEST_ASSERT(n == 2);
 298     n = index->getBucketIndex(Cyrillic, status);
 299     TEST_ASSERT(n == 27);   // Overflow label
 300     n = index->getBucketIndex(zed, status);
 301     TEST_ASSERT(n == 26);
 302
 303     for (i=0; index->nextBucket(status); i++) {
 304         n = index->getBucketIndex();
 305         TEST_ASSERT(n == i);
 306         UnicodeString label = index->getBucketLabel();
 307         TEST_ASSERT(n == i);
 308     }
 309     TEST_ASSERT(i == 28);
 310
 311     delete index;
 312     index = new AlphabeticIndex(Locale::createFromName("ru"), status);
 313     TEST_CHECK_STATUS;
 314     assertEquals("Russian index.getBucketCount()", 32, index->getBucketCount(status));
 315     // Latin-script names should go into the underflow label (0)
 316     // if the Russian collation does not use script reordering,
 317     // but into the overflow label (getBucketCount()-1)
 318     // if Russian sorts Cyrillic first.
 319     int32_t reorderCodes[20];
 320     int32_t expectedLatinIndex = 0;
 321     if (index->getCollator().getReorderCodes(reorderCodes, UPRV_LENGTHOF(reorderCodes), status) > 0) {
 322         expectedLatinIndex = index->getBucketCount(status) - 1;
 323     }
 324     n = index->getBucketIndex(adam, status);
 325     TEST_CHECK_STATUS;
 326     assertEquals("Russian index.getBucketIndex(adam)", expectedLatinIndex, n);
 327     n = index->getBucketIndex(baker, status);
 328     assertEquals("Russian index.getBucketIndex(baker)", expectedLatinIndex, n);
 329     n = index->getBucketIndex(Cyrillic, status);
 330     assertEquals("Russian index.getBucketIndex(Cyrillic)", 1, n);
 331     n = index->getBucketIndex(zed, status);
 332     assertEquals("Russian index.getBucketIndex(zed)", expectedLatinIndex, n);
 333
 334     delete index;
 335
 336 }
 337
 338
 339 static const char * KEY_LOCALES[] = {
 340             "en", "es", "de", "fr", "ja", "it", "tr", "pt", "zh", "nl",
 341             "pl", "ar", "ru", "zh_Hant", "ko", "th", "sv", "fi", "da",
 342             "he", "nb", "el", "hr", "bg", "sk", "lt", "vi", "lv", "sr",
 343             "pt_PT", "ro", "hu", "cs", "id", "sl", "fil", "fa", "uk",
 344             "ca", "hi", "et", "eu", "is", "sw", "ms", "bn", "am", "ta",
 345             "te", "mr", "ur", "ml", "kn", "gu", "or", ""};
 346
 347
 348 void AlphabeticIndexTest::ManyLocalesTest() {
 349     UErrorCode status = U_ZERO_ERROR;
 350     int32_t  lc = 0;
 351
 352     for (int i=0; ; ++i) {
 353         status = U_ZERO_ERROR;
 354         const char *localeName = KEY_LOCALES[i];
 355         if (localeName[0] == 0) {
 356             break;
 357         }
 358         // std::cout <<  localeName << "  ";
 359         Locale loc = Locale::createFromName(localeName);
 360         AlphabeticIndex index(loc, status);
 361         TEST_CHECK_STATUS;
 362         lc = index.getBucketCount(status);
 363         TEST_CHECK_STATUS;
 364         // std::cout << "getBucketCount() == " << lc << std::endl;
 365
 366         LocalPointer<AlphabeticIndex::ImmutableIndex> immIndex(index.buildImmutableIndex(status));
 367         TEST_CHECK_STATUS;
 368         TEST_ASSERT(lc == immIndex->getBucketCount());
 369
 370         assertEquals("initial bucket index", -1, index.getBucketIndex());
 371         int32_t bucketIndex = 0;
 372         while (index.nextBucket(status)) {
 373             TEST_CHECK_STATUS;
 374             assertEquals("bucket index", bucketIndex, index.getBucketIndex());
 375             const UnicodeString &label = index.getBucketLabel();
 376             TEST_ASSERT(label.length()>0);
 377             // std::string ss;
 378             // std::cout << ":" << label.toUTF8String(ss);
 379             const AlphabeticIndex::Bucket *bucket = immIndex->getBucket(bucketIndex);
 380             TEST_ASSERT(bucket != NULL);
 381             assertEquals("bucket label vs. immutable: locale=" + UnicodeString(localeName) +
 382                          " index=" + bucketIndex,
 383                          label, bucket->getLabel());
 384             TEST_ASSERT(&label != &bucket->getLabel());  // not the same pointers
 385             UAlphabeticIndexLabelType labelType = index.getBucketLabelType();
 386             TEST_ASSERT(labelType == bucket->getLabelType());
 387             ++bucketIndex;
 388         }
 389         // std::cout << ":" << std::endl;
 390
 391         TEST_ASSERT(immIndex->getBucketCount() == bucketIndex);
 392         TEST_ASSERT(immIndex->getBucket(-1) == NULL);
 393         TEST_ASSERT(immIndex->getBucket(bucketIndex) == NULL);
 394     }
 395 }
 396
 397
 398 // Test data for Pinyin based indexes.
 399 //  The Chinese characters should be distributed under latin labels in
 400 //  an index.
 401
 402 static const char *pinyinTestData[] = {
 403         "\\u0101", "\\u5416", "\\u58ba", //
 404         "b", "\\u516b", "\\u62d4", "\\u8500", //
 405         "c", "\\u5693", "\\u7938", "\\u9e7e", //
 406         "d", "\\u5491", "\\u8fcf", "\\u964a", //
 407         "\\u0113","\\u59b8", "\\u92e8", "\\u834b", //
 408         "f", "\\u53d1", "\\u9197", "\\u99a5", //
 409         "g", "\\u7324", "\\u91d3", "\\u8142", //
 410         "h", "\\u598e", "\\u927f", "\\u593b", //
 411         "j", "\\u4e0c", "\\u6785", "\\u9d58", //
 412         "k", "\\u5494", "\\u958b", "\\u7a52", //
 413         "l", "\\u5783", "\\u62c9", "\\u9ba5", //
 414         "m", "\\u5638", "\\u9ebb", "\\u65c0", //
 415         "n", "\\u62ff", "\\u80ad", "\\u685b", //
 416         "\\u014D", "\\u5662", "\\u6bee", "\\u8bb4", //
 417         "p", "\\u5991", "\\u8019", "\\u8c31", //
 418         "q", "\\u4e03", "\\u6053", "\\u7f56", //
 419         "r", "\\u5465", "\\u72aa", "\\u6e03", //
 420         "s", "\\u4ee8", "\\u9491", "\\u93c1", //
 421         "t", "\\u4ed6", "\\u9248", "\\u67dd", //
 422         "w", "\\u5c72", "\\u5558", "\\u5a7a", //
 423         "x", "\\u5915", "\\u5438", "\\u6bbe", //
 424         "y", "\\u4e2b", "\\u82bd", "\\u8574", //
 425         "z", "\\u5e00", "\\u707d", "\\u5c0a",
 426         NULL
 427     };
 428
 429 void AlphabeticIndexTest::HackPinyinTest() {
 430     UErrorCode status = U_ZERO_ERROR;
 431     AlphabeticIndex aindex(Locale::createFromName("zh"), status);
 432     TEST_CHECK_STATUS;
 433
 434     UnicodeString names[UPRV_LENGTHOF(pinyinTestData)];
 435     int32_t  nameCount;
 436     for (nameCount=0; pinyinTestData[nameCount] != NULL; nameCount++) {
 437         names[nameCount] = UnicodeString(pinyinTestData[nameCount], -1, UnicodeString::kInvariant).unescape();
 438         aindex.addRecord(names[nameCount], &names[nameCount], status);
 439         TEST_CHECK_STATUS;
 440         if (U_FAILURE(status)) {
 441             return;
 442         }
 443     }
 444     TEST_ASSERT(nameCount == aindex.getRecordCount(status));
 445
 446     // Weak checking:  make sure that none of the Chinese names landed in the overflow bucket
 447     //   of the index, and that the names are distributed among several buckets.
 448     //   (Exact expected data would be subject to change with evolution of the collation rules.)
 449
 450     int32_t bucketCount = 0;
 451     int32_t filledBucketCount = 0;
 452     while (aindex.nextBucket(status)) {
 453         bucketCount++;
 454         UnicodeString label = aindex.getBucketLabel();
 455         // std::string s;
 456         // std::cout << label.toUTF8String(s) << ":  ";
 457
 458         UBool  bucketHasContents = FALSE;
 459         while (aindex.nextRecord(status)) {
 460             bucketHasContents = TRUE;
 461             UnicodeString name = aindex.getRecordName();
 462             if (aindex.getBucketLabelType() != U_ALPHAINDEX_NORMAL) {
 463                 errln("File %s, Line %d, Name \"\\u%x\" is in an under or overflow bucket.",
 464                     __FILE__, __LINE__, name.char32At(0));
 465             }
 466             // s.clear();
 467             // std::cout << aindex.getRecordName().toUTF8String(s) << " ";
 468         }
 469         if (bucketHasContents) {
 470             filledBucketCount++;
 471         }
 472         // std::cout << std::endl;
 473     }
 474     TEST_ASSERT(bucketCount > 25);
 475     TEST_ASSERT(filledBucketCount > 15);
 476 }
 477
 478
 479 void AlphabeticIndexTest::TestBug9009() {
 480     UErrorCode status = U_ZERO_ERROR;
 481     Locale loc("root");
 482     AlphabeticIndex aindex(loc, status);
 483     TEST_CHECK_STATUS;
 484     aindex.nextBucket(status);  // Crash here before bug was fixed.
 485     TEST_CHECK_STATUS;
 486 }
 487
 488 static const char *localeAndIndexCharactersLists[][2] = {
 489     /* Arabic*/ {"ar", "\\u0627:\\u0628:\\u062A:\\u062B:\\u062C:\\u062D:\\u062E:\\u062F:\\u0630:\\u0631:\\u0632:\\u0633:\\u0634:\\u0635:\\u0636:\\u0637:\\u0638:\\u0639:\\u063A:\\u0641:\\u0642:\\u0643:\\u0644:\\u0645:\\u0646:\\u0647:\\u0648:\\u064A"},
 490     /* Bulgarian*/  {"bg", "\\u0410:\\u0411:\\u0412:\\u0413:\\u0414:\\u0415:\\u0416:\\u0417:\\u0418:\\u0419:\\u041A:\\u041B:\\u041C:\\u041D:\\u041E:\\u041F:\\u0420:\\u0421:\\u0422:\\u0423:\\u0424:\\u0425:\\u0426:\\u0427:\\u0428:\\u0429:\\u042E:\\u042F"},
 491     /* Catalan*/    {"ca", "A:B:C:D:E:F:G:H:I:J:K:L:M:N:O:P:Q:R:S:T:U:V:W:X:Y:Z"},
 492     /* Czech*/  {"cs", "A:B:C:\\u010C:D:E:F:G:H:CH:I:J:K:L:M:N:O:P:Q:R:\\u0158:S:\\u0160:T:U:V:W:X:Y:Z:\\u017D"},
 493     /* Danish*/ {"da", "A:B:C:D:E:F:G:H:I:J:K:L:M:N:O:P:Q:R:S:T:U:V:W:X:Y:Z:\\u00C6:\\u00D8:\\u00C5"},
 494     /* German*/ {"de", "A:B:C:D:E:F:G:H:I:J:K:L:M:N:O:P:Q:R:S:T:U:V:W:X:Y:Z"},
 495     /* Greek*/  {"el", "\\u0391:\\u0392:\\u0393:\\u0394:\\u0395:\\u0396:\\u0397:\\u0398:\\u0399:\\u039A:\\u039B:\\u039C:\\u039D:\\u039E:\\u039F:\\u03A0:\\u03A1:\\u03A3:\\u03A4:\\u03A5:\\u03A6:\\u03A7:\\u03A8:\\u03A9"},
 496     /* English*/    {"en", "A:B:C:D:E:F:G:H:I:J:K:L:M:N:O:P:Q:R:S:T:U:V:W:X:Y:Z"},
 497     /* Spanish*/    {"es", "A:B:C:D:E:F:G:H:I:J:K:L:M:N:\\u00D1:O:P:Q:R:S:T:U:V:W:X:Y:Z"},
 498     /* Estonian*/   {"et", "A:B:C:D:E:F:G:H:I:J:K:L:M:N:O:P:Q:R:S:\\u0160:Z:\\u017D:T:U:V:\\u00D5:\\u00C4:\\u00D6:\\u00DC:X:Y"},
 499     /* Basque*/ {"eu", "A:B:C:D:E:F:G:H:I:J:K:L:M:N:O:P:Q:R:S:T:U:V:W:X:Y:Z"},
 500     /* Finnish*/    {"fi", "A:B:C:D:E:F:G:H:I:J:K:L:M:N:O:P:Q:R:S:T:U:V:W:X:Y:Z:\\u00C5:\\u00C4:\\u00D6"},
 501     /* Filipino*/   {"fil", "A:B:C:D:E:F:G:H:I:J:K:L:M:N:\\u00D1:Ng:O:P:Q:R:S:T:U:V:W:X:Y:Z"},
 502     /* French*/ {"fr", "A:B:C:D:E:F:G:H:I:J:K:L:M:N:O:P:Q:R:S:T:U:V:W:X:Y:Z"},
 503     /* Hebrew*/ {"he", "\\u05D0:\\u05D1:\\u05D2:\\u05D3:\\u05D4:\\u05D5:\\u05D6:\\u05D7:\\u05D8:\\u05D9:\\u05DB:\\u05DC:\\u05DE:\\u05E0:\\u05E1:\\u05E2:\\u05E4:\\u05E6:\\u05E7:\\u05E8:\\u05E9:\\u05EA"},
 504     /* Icelandic*/  {"is", "A:\\u00C1:B:C:D:\\u00D0:E:\\u00C9:F:G:H:I:\\u00CD:J:K:L:M:N:O:\\u00D3:P:Q:R:S:T:U:\\u00DA:V:W:X:Y:\\u00DD:Z:\\u00DE:\\u00C6:\\u00D6"},
 505     /* Italian*/    {"it", "A:B:C:D:E:F:G:H:I:J:K:L:M:N:O:P:Q:R:S:T:U:V:W:X:Y:Z"},
 506     /* Japanese*/   {"ja", "\\u3042:\\u304B:\\u3055:\\u305F:\\u306A:\\u306F:\\u307E:\\u3084:\\u3089:\\u308F"},
 507     /* Korean*/ {"ko", "\\u3131:\\u3134:\\u3137:\\u3139:\\u3141:\\u3142:\\u3145:\\u3147:\\u3148:\\u314A:\\u314B:\\u314C:\\u314D:\\u314E"},
 508     /* Lithuanian*/ {"lt", "A:B:C:\\u010C:D:E:F:G:H:I:J:K:L:M:N:O:P:R:S:\\u0160:T:U:V:Z:\\u017D"},
 509     /* Latvian*/    {"lv", "A:B:C:\\u010C:D:E:F:G:\\u0122:H:I:J:K:\\u0136:L:\\u013B:M:N:\\u0145:O:P:Q:R:S:\\u0160:T:U:V:W:X:Z:\\u017D"},
 510     /* Norwegian Bokm\\u00E5l*/  {"nb", "A:B:C:D:E:F:G:H:I:J:K:L:M:N:O:P:Q:R:S:T:U:V:W:X:Y:Z:\\u00C6:\\u00D8:\\u00C5"},
 511     /* Dutch*/  {"nl", "A:B:C:D:E:F:G:H:I:J:K:L:M:N:O:P:Q:R:S:T:U:V:W:X:Y:Z"},
 512     /* Polish*/ {"pl", "A:\\u0104:B:C:\\u0106:D:E:\\u0118:F:G:H:I:J:K:L:\\u0141:M:N:\\u0143:O:\\u00D3:P:Q:R:S:\\u015A:T:U:V:W:X:Y:Z:\\u0179:\\u017B"},
 513     /* Portuguese*/ {"pt", "A:B:C:D:E:F:G:H:I:J:K:L:M:N:O:P:Q:R:S:T:U:V:W:X:Y:Z"},
 514     /* Romanian*/   {"ro", "A:\\u0102:\\u00C2:B:C:D:E:F:G:H:I:\\u00CE:J:K:L:M:N:O:P:Q:R:S:\\u0218:T:\\u021A:U:V:W:X:Y:Z"},
 515     /* Russian*/    {"ru", "\\u0410:\\u0411:\\u0412:\\u0413:\\u0414:\\u0415:\\u0416:\\u0417:\\u0418:\\u0419:\\u041A:\\u041B:\\u041C:\\u041D:\\u041E:\\u041F:\\u0420:\\u0421:\\u0422:\\u0423:\\u0424:\\u0425:\\u0426:\\u0427:\\u0428:\\u0429:\\u042B:\\u042D:\\u042E:\\u042F"},
 516     /* Slovak*/ {"sk", "A:\\u00C4:B:C:\\u010C:D:E:F:G:H:CH:I:J:K:L:M:N:O:\\u00D4:P:Q:R:S:\\u0160:T:U:V:W:X:Y:Z:\\u017D"},
 517     /* Slovenian*/  {"sl", "A:B:C:\\u010C:\\u0106:D:\\u0110:E:F:G:H:I:J:K:L:M:N:O:P:Q:R:S:\\u0160:T:U:V:W:X:Y:Z:\\u017D"},
 518     /* Serbian*/    {"sr", "\\u0410:\\u0411:\\u0412:\\u0413:\\u0414:\\u0402:\\u0415:\\u0416:\\u0417:\\u0418:\\u0408:\\u041A:\\u041B:\\u0409:\\u041C:\\u041D:\\u040A:\\u041E:\\u041F:\\u0420:\\u0421:\\u0422:\\u040B:\\u0423:\\u0424:\\u0425:\\u0426:\\u0427:\\u040F:\\u0428"},
 519     /* Swedish*/    {"sv", "A:B:C:D:E:F:G:H:I:J:K:L:M:N:O:P:Q:R:S:T:U:V:W:X:Y:Z:\\u00C5:\\u00C4:\\u00D6"},
 520     /* Turkish*/    {"tr", "A:B:C:\\u00C7:D:E:F:G:H:I:\\u0130:J:K:L:M:N:O:\\u00D6:P:Q:R:S:\\u015E:T:U:\\u00DC:V:W:X:Y:Z"},
 521     /* Ukrainian*/  {"uk", "\\u0410:\\u0411:\\u0412:\\u0413:\\u0490:\\u0414:\\u0415:\\u0404:\\u0416:\\u0417:\\u0418:\\u0406:\\u0407:\\u0419:\\u041A:\\u041B:\\u041C:\\u041D:\\u041E:\\u041F:\\u0420:\\u0421:\\u0422:\\u0423:\\u0424:\\u0425:\\u0426:\\u0427:\\u0428:\\u0429:\\u042E:\\u042F"},
 522     /* Vietnamese*/ {"vi", "A:\\u0102:\\u00C2:B:C:D:\\u0110:E:\\u00CA:F:G:H:I:J:K:L:M:N:O:\\u00D4:\\u01A0:P:Q:R:S:T:U:\\u01AF:V:W:X:Y:Z"},
 523     /* Chinese*/    {"zh", "A:B:C:D:E:F:G:H:I:J:K:L:M:N:O:P:Q:R:S:T:U:V:W:X:Y:Z"},
 524     /* Chinese (Traditional Han)*/  {"zh_Hant", "1\\u5283:2\\u5283:3\\u5283:4\\u5283:5\\u5283:6\\u5283:7\\u5283:8\\u5283:9\\u5283:10\\u5283:11\\u5283:12\\u5283:13\\u5283:14\\u5283:15\\u5283:16\\u5283:17\\u5283:18\\u5283:19\\u5283:20\\u5283:21\\u5283:22\\u5283:23\\u5283:24\\u5283:25\\u5283:26\\u5283:27\\u5283:28\\u5283:29\\u5283:30\\u5283:31\\u5283:32\\u5283:33\\u5283:35\\u5283:36\\u5283:39\\u5283:48\\u5283"},
 525 };
 526
 527 void AlphabeticIndexTest::TestIndexCharactersList() {
 528     UErrorCode status = U_ZERO_ERROR;
 529     for (int32_t i = 0; i < UPRV_LENGTHOF(localeAndIndexCharactersLists); ++i) {
 530         const char *(&localeAndIndexCharacters)[2] = localeAndIndexCharactersLists[i];
 531         const char *locale = localeAndIndexCharacters[0];
 532         UnicodeString expectedIndexCharacters
 533             = (UnicodeString("\\u2026:") + localeAndIndexCharacters[1] + ":\\u2026").unescape();
 534         AlphabeticIndex index(locale, status);
 535         TEST_CHECK_STATUS;
 536         LocalPointer<AlphabeticIndex::ImmutableIndex> immIndex(index.buildImmutableIndex(status));
 537         TEST_CHECK_STATUS;
 538
 539         // Join the elements of the list to a string with delimiter ":"
 540         UnicodeString actualIndexCharacters;
 541         assertEquals(locale,
 542                      expectedIndexCharacters,
 543                      joinLabelsAndAppend(*immIndex, actualIndexCharacters));
 544         logln(locale + UnicodeString(": ") + actualIndexCharacters);
 545     }
 546 }
 547
 548 void AlphabeticIndexTest::TestHaniFirst() {
 549     UErrorCode status = U_ZERO_ERROR;
 550     LocalPointer<RuleBasedCollator> coll(
 551         static_cast<RuleBasedCollator *>(Collator::createInstance(Locale::getRoot(), status)));
 552
 553     if (U_FAILURE(status)) {
 554         dataerrln("Failed Collator::createInstance call - %s", u_errorName(status));
 555         return;
 556     }
 557     int32_t reorderCodes[] = { USCRIPT_HAN };
 558     coll->setReorderCodes(reorderCodes, UPRV_LENGTHOF(reorderCodes), status);
 559     TEST_CHECK_STATUS;
 560     AlphabeticIndex index(coll.orphan(), status);
 561     TEST_CHECK_STATUS;
 562     assertEquals("getBucketCount()", 1, index.getBucketCount(status));   // ... (underflow only)
 563     index.addLabels(Locale::getEnglish(), status);
 564     assertEquals("getBucketCount()", 28, index.getBucketCount(status));  // ... A-Z ...
 565     int32_t bucketIndex = index.getBucketIndex(UnicodeString((UChar)0x897f), status);
 566     assertEquals("getBucketIndex(U+897F)", 0, bucketIndex);  // underflow bucket
 567     bucketIndex = index.getBucketIndex("i", status);
 568     assertEquals("getBucketIndex(i)", 9, bucketIndex);
 569     bucketIndex = index.getBucketIndex(UnicodeString((UChar)0x03B1), status);
 570     assertEquals("getBucketIndex(Greek alpha)", 27, bucketIndex);
 571     // U+50005 is an unassigned code point which sorts at the end, independent of the Hani group.
 572     bucketIndex = index.getBucketIndex(UnicodeString((UChar32)0x50005), status);
 573     assertEquals("getBucketIndex(U+50005)", 27, bucketIndex);
 574     bucketIndex = index.getBucketIndex(UnicodeString((UChar)0xFFFF), status);
 575     assertEquals("getBucketIndex(U+FFFF)", 27, bucketIndex);
 576 }
 577
 578 void AlphabeticIndexTest::TestPinyinFirst() {
 579     UErrorCode status = U_ZERO_ERROR;
 580     LocalPointer<RuleBasedCollator> coll(
 581         static_cast<RuleBasedCollator *>(Collator::createInstance(Locale::getChinese(), status)));
 582     if (U_FAILURE(status)) {
 583         dataerrln("Failed Collator::createInstance call - %s", u_errorName(status));
 584         return;
 585     }
 586     int32_t reorderCodes[] = { USCRIPT_HAN };
 587     coll->setReorderCodes(reorderCodes, UPRV_LENGTHOF(reorderCodes), status);
 588     TEST_CHECK_STATUS;
 589     AlphabeticIndex index(coll.orphan(), status);
 590     TEST_CHECK_STATUS;
 591     assertEquals("getBucketCount()", 28, index.getBucketCount(status));   // ... A-Z ...
 592     index.addLabels(Locale::getChinese(), status);
 593     assertEquals("getBucketCount()", 28, index.getBucketCount(status));  // ... A-Z ...
 594     int32_t bucketIndex = index.getBucketIndex(UnicodeString((UChar)0x897f), status);
 595     assertEquals("getBucketIndex(U+897F)", (int32_t)((UChar)0x0058/*X*/ - (UChar)0x0041/*A*/ + 1), bucketIndex);
 596     bucketIndex = index.getBucketIndex("i", status);
 597     assertEquals("getBucketIndex(i)", 9, bucketIndex);
 598     bucketIndex = index.getBucketIndex(UnicodeString((UChar)0x03B1), status);
 599     assertEquals("getBucketIndex(Greek alpha)", (int32_t)27, bucketIndex);
 600     // U+50005 is an unassigned code point which sorts at the end, independent of the Hani group.
 601     bucketIndex = index.getBucketIndex(UnicodeString((UChar32)0x50005), status);
 602     assertEquals("getBucketIndex(U+50005)", 27, bucketIndex);
 603     bucketIndex = index.getBucketIndex(UnicodeString((UChar)0xFFFF), status);
 604     assertEquals("getBucketIndex(U+FFFF)", 27, bucketIndex);
 605 }
 606
 607 void AlphabeticIndexTest::TestSchSt() {
 608     UErrorCode status = U_ZERO_ERROR;
 609     AlphabeticIndex index(Locale::getGerman(), status);
 610     index.addLabels(UnicodeSet("[\\u00C6{Sch*}{St*}]", status), status);
 611     TEST_CHECK_STATUS;
 612     // ... A AE-ligature B-R S Sch St T-Z ...
 613     LocalPointer<AlphabeticIndex::ImmutableIndex> immIndex(index.buildImmutableIndex(status));
 614     TEST_CHECK_STATUS;
 615     assertEquals("getBucketCount()", 31, index.getBucketCount(status));
 616     assertEquals("immutable getBucketCount()", 31, immIndex->getBucketCount());
 617     static const struct TestCase {
 618         const char *name;
 619         int32_t bucketIndex;
 620         const char *bucketLabel;
 621     } testCases[] = {
 622         // name, bucket index, bucket label
 623         { "Adelbert", 1, "A" },
 624         { "Afrika", 1, "A" },
 625         { "\\u00C6sculap", 2, "\\u00C6" },
 626         { "Aesthet", 2, "\\u00C6" },
 627         { "Berlin", 3, "B" },
 628         { "Rilke", 19, "R" },
 629         { "Sacher", 20, "S" },
 630         { "Seiler", 20, "S" },
 631         { "Sultan", 20, "S" },
 632         { "Schiller", 21, "Sch" },
 633         { "Steiff", 22, "St" },
 634         { "Thomas", 23, "T" }
 635     };
 636     for (int32_t i = 0; i < UPRV_LENGTHOF(testCases); ++i) {
 637         const TestCase &testCase = testCases[i];
 638         UnicodeString name = UnicodeString(testCase.name).unescape();
 639         UnicodeString label = UnicodeString(testCase.bucketLabel).unescape();
 640         char msg[100];
 641         sprintf(msg, "getBucketIndex(%s)", testCase.name);
 642         assertEquals(msg, testCase.bucketIndex, index.getBucketIndex(name, status));
 643         sprintf(msg, "immutable getBucketIndex(%s)", testCase.name);
 644         assertEquals(msg, testCase.bucketIndex, immIndex->getBucketIndex(name, status));
 645         sprintf(msg, "immutable bucket label (%s)", testCase.name);
 646         assertEquals(msg, label, immIndex->getBucket(testCase.bucketIndex)->getLabel());
 647     }
 648 }
 649
 650 void AlphabeticIndexTest::TestNoLabels() {
 651     UErrorCode status = U_ZERO_ERROR;
 652     LocalPointer<RuleBasedCollator> coll(
 653         static_cast<RuleBasedCollator *>(Collator::createInstance(Locale::getRoot(), status)));
 654     TEST_CHECK_STATUS;
 655     AlphabeticIndex index(coll.orphan(), status);
 656     TEST_CHECK_STATUS;
 657     index.addRecord(UnicodeString((UChar)0x897f), NULL, status);
 658     index.addRecord("i", NULL, status);
 659     index.addRecord(UnicodeString((UChar)0x03B1), NULL, status);
 660     assertEquals("getBucketCount()", 1, index.getBucketCount(status));  // ...
 661     TEST_ASSERT(index.nextBucket(status));
 662     assertEquals("underflow label type", (int32_t)U_ALPHAINDEX_UNDERFLOW, index.getBucketLabelType());
 663     assertEquals("all records in the underflow bucket", (int32_t)3, index.getBucketRecordCount());
 664 }
 665
 666 void AlphabeticIndexTest::TestChineseZhuyin() {
 667     UErrorCode status = U_ZERO_ERROR;
 668     char loc[100];
 669     uloc_forLanguageTag("zh-u-co-zhuyin", loc, UPRV_LENGTHOF(loc), NULL, &status);
 670     AlphabeticIndex index(loc, status);
 671     LocalPointer<AlphabeticIndex::ImmutableIndex> immIndex(index.buildImmutableIndex(status));
 672     TEST_CHECK_STATUS;
 673     assertEquals("getBucketCount()", 38, immIndex->getBucketCount());
 674     assertEquals("label 1", UnicodeString((UChar)0x3105), immIndex->getBucket(1)->getLabel());
 675     assertEquals("label 2", UnicodeString((UChar)0x3106), immIndex->getBucket(2)->getLabel());
 676     assertEquals("label 3", UnicodeString((UChar)0x3107), immIndex->getBucket(3)->getLabel());
 677     assertEquals("label 4", UnicodeString((UChar)0x3108), immIndex->getBucket(4)->getLabel());
 678     assertEquals("label 5", UnicodeString((UChar)0x3109), immIndex->getBucket(5)->getLabel());
 679 }
 680
 681 void AlphabeticIndexTest::TestJapaneseKanji() {
 682     UErrorCode status = U_ZERO_ERROR;
 683     AlphabeticIndex index(Locale::getJapanese(), status);
 684     LocalPointer<AlphabeticIndex::ImmutableIndex> immIndex(index.buildImmutableIndex(status));
 685     TEST_CHECK_STATUS;
 686     // There are no index characters for Kanji in the Japanese standard collator.
 687     // They should all go into the overflow bucket.
 688     static const UChar32 kanji[] = { 0x4E9C, 0x95C7, 0x4E00, 0x58F1 };
 689     int32_t overflowIndex = immIndex->getBucketCount() - 1;
 690     for(int32_t i = 0; i < UPRV_LENGTHOF(kanji); ++i) {
 691         char msg[40];
 692         sprintf(msg, "kanji[%d]=U+%04lX in overflow bucket", (int)i, (long)kanji[i]);
 693         assertEquals(msg, overflowIndex, immIndex->getBucketIndex(UnicodeString(kanji[i]), status));
 694         TEST_CHECK_STATUS;
 695     }
 696 }
 697
 698 void AlphabeticIndexTest::TestChineseUnihan() {
 699     UErrorCode status = U_ZERO_ERROR;
 700     AlphabeticIndex index("zh-u-co-unihan", status);
 701     if(U_FAILURE(status)) {
 702         dataerrln("unable create an AlphabeticIndex for Chinese/unihan: %s", u_errorName(status));
 703         return;
 704     }
 705     index.setMaxLabelCount(500, status);  // ICU 54 default is 99.
 706     LocalPointer<AlphabeticIndex::ImmutableIndex> immIndex(index.buildImmutableIndex(status));
 707     TEST_CHECK_STATUS;
 708     int32_t bucketCount = immIndex->getBucketCount();
 709     if(bucketCount < 216) {
 710         // There should be at least an underflow and overflow label,
 711         // and one for each of 214 radicals,
 712         // and maybe additional labels for simplified radicals.
 713         dataerrln("too few buckets/labels for Chinese/unihan: %d (is zh/unihan data available?)",
 714                   bucketCount);
 715         return;
 716     } else {
 717         logln("Chinese/unihan has %d buckets/labels", bucketCount);
 718     }
 719     // bucketIndex = radical number, adjusted for simplified radicals in lower buckets.
 720     int32_t bucketIndex = index.getBucketIndex(UnicodeString((UChar)0x4e5d), status);
 721     assertEquals("getBucketIndex(U+4E5D)", 5, bucketIndex);
 722     // radical 100, and there is a 90' since Unicode 8
 723     bucketIndex = index.getBucketIndex(UnicodeString((UChar)0x7527), status);
 724     assertEquals("getBucketIndex(U+7527)", 101, bucketIndex);
 725 }
 726
 727 #endif