add support for non-BMP ligatures

> http://www.unicode.org/versions/Unicode5.2.0/
D. Character Additions:
There are three new characters in the newly-encoded Kaithi script that will
require changes in implementations which make hard-coded assumptions about
composition during normalization. Most new characters added to the standard
with decompositions cannot be generated by the operations toNFC() or toNFKC),
but these three can. Implementers should check their code carefully
to ensure that it handles these three characters correctly.
U+1109A KAITHI LETTER DDDHA
U+1109C KAITHI LETTER RHA
U+110AB KAITHI LETTER VA

UCD 6.1 adds two more of them:
U+1112E CHAKMA VOWEL SIGN O
U+1112F CHAKMA VOWEL SIGN AU

Change-Id: I781a26848078d8b83a182b0fd4e681be2a6d9a27
Reviewed-by: Lars Knoll <lars.knoll@nokia.com>

diff --git a/src/corelib/tools/qchar.cpp b/src/corelib/tools/qchar.cpp
index 89017fe..358653a 100644 (file)
--- a/src/corelib/tools/qchar.cpp
+++ b/src/corelib/tools/qchar.cpp
@@ -1446,7 +1446,17 @@ inline bool operator<(ushort u1, const UCS2Pair &ligature)
 inline bool operator<(const UCS2Pair &ligature, ushort u1)
 { return ligature.u1 < u1; }
 
-static ushort ligatureHelper(ushort u1, ushort u2)
+struct UCS2SurrogatePair {
+    UCS2Pair p1;
+    UCS2Pair p2;
+};
+
+inline bool operator<(uint u1, const UCS2SurrogatePair &ligature)
+{ return u1 < QChar::surrogateToUcs4(ligature.p1.u1, ligature.p1.u2); }
+inline bool operator<(const UCS2SurrogatePair &ligature, uint u1)
+{ return QChar::surrogateToUcs4(ligature.p1.u1, ligature.p1.u2) < u1; }
+
+static uint inline ligatureHelper(uint u1, uint u2)
 {
     if (u1 >= Hangul_LBase && u1 <= Hangul_SBase + Hangul_SCount) {
         // compute Hangul syllable composition as per UAX #15
@@ -1471,9 +1481,14 @@ static ushort ligatureHelper(ushort u1, ushort u2)
         return 0;
     const unsigned short *ligatures = uc_ligature_map+index;
     ushort length = *ligatures++;
-    {
+    if (QChar::requiresSurrogates(u1)) {
+        const UCS2SurrogatePair *data = reinterpret_cast<const UCS2SurrogatePair *>(ligatures);
+        const UCS2SurrogatePair *r = qBinaryFind(data, data + length, u1);
+        if (r != data + length)
+            return QChar::surrogateToUcs4(r->p2.u1, r->p2.u2);
+    } else {
         const UCS2Pair *data = reinterpret_cast<const UCS2Pair *>(ligatures);
-        const UCS2Pair *r = qBinaryFind(data, data + length, u1);
+        const UCS2Pair *r = qBinaryFind(data, data + length, ushort(u1));
         if (r != data + length)
             return r->u2;
     }
@@ -1485,14 +1500,17 @@ static void composeHelper(QString *str, QChar::UnicodeVersion version, int from)
 {
     QString &s = *str;
 
-    if (s.length() - from < 2)
+    if (from < 0 || s.length() - from < 2)
         return;
 
-    // the loop can partly ignore high Unicode as all ligatures are in the BMP
-    int starter = 0;
+    int starter = 0; // starter position
+    uint stcode = 0; // starter code point
+    int next = -1;
     int lastCombining = 0;
+
     int pos = from;
     while (pos < s.length()) {
+        int i = pos;
         uint uc = s.at(pos).unicode();
         if (QChar(uc).isHighSurrogate() && pos < s.length()-1) {
             ushort low = s.at(pos+1).unicode();
@@ -1501,26 +1519,43 @@ static void composeHelper(QString *str, QChar::UnicodeVersion version, int from)
                 ++pos;
             }
         }
+
         const QUnicodeTables::Properties *p = qGetProp(uc);
         if (p->unicodeVersion > version || p->unicodeVersion == QChar::Unicode_Unassigned) {
-            starter = -1; // to prevent starter == pos - 1
-            lastCombining = 0;
+            starter = -1;
+            next = -1; // to prevent i == next
+            lastCombining = 255; // to prevent combining > lastCombining
             ++pos;
             continue;
         }
+
         int combining = p->combiningClass;
-        if (starter == pos - 1 || combining > lastCombining) {
+        if (i == next || combining > lastCombining) {
+            Q_ASSERT(starter >= from);
             // allowed to form ligature with S
-            QChar ligature = ligatureHelper(s.at(starter).unicode(), uc);
-            if (ligature.unicode()) {
-                s[starter] = ligature;
-                s.remove(pos, 1);
+            uint ligature = ligatureHelper(stcode, uc);
+            if (ligature) {
+                stcode = ligature;
+                QChar *d = s.data();
+                // ligatureHelper() never changes planes
+                if (QChar::requiresSurrogates(ligature)) {
+                    d[starter] = QChar::highSurrogate(ligature);
+                    d[starter + 1] = QChar::lowSurrogate(ligature);
+                    s.remove(i, 2);
+                } else {
+                    d[starter] = ligature;
+                    s.remove(i, 1);
+                }
                 continue;
             }
         }
-        if (!combining)
-            starter = pos;
+        if (combining == 0) {
+            starter = i;
+            stcode = uc;
+            next = pos + 1;
+        }
         lastCombining = combining;
+
         ++pos;
     }
 }

diff --git a/src/corelib/tools/qunicodetables.cpp b/src/corelib/tools/qunicodetables.cpp
index ea61d20..0403125 100644 (file)
--- a/src/corelib/tools/qunicodetables.cpp
+++ b/src/corelib/tools/qunicodetables.cpp
@@ -7616,58 +7616,92 @@ static const unsigned short uc_decomposition_map[] = {
 static const unsigned short uc_ligature_trie[] = {
     // 0 - 0x3100
 
-    392, 392, 392, 392, 392, 392, 392, 392,
-    392, 392, 392, 392, 392, 392, 392, 392,
-    392, 392, 392, 392, 392, 392, 392, 392,
-    424, 456, 488, 392, 392, 392, 392, 392,
-    392, 392, 392, 392, 392, 392, 392, 392,
-    392, 392, 392, 392, 392, 392, 392, 392,
-    392, 392, 520, 392, 392, 392, 392, 392,
-    392, 392, 392, 392, 392, 392, 392, 392,
-    392, 392, 392, 392, 392, 392, 392, 392,
-    392, 552, 392, 392, 392, 584, 616, 392,
-    392, 392, 392, 392, 392, 392, 392, 392,
-    392, 648, 680, 392, 392, 712, 744, 392,
-    392, 392, 776, 392, 392, 392, 808, 392,
-    392, 840, 872, 392, 392, 392, 904, 392,
-    392, 392, 392, 392, 392, 392, 392, 392,
-    392, 392, 392, 392, 392, 392, 392, 392,
-
-    392, 936, 392, 392, 392, 392, 392, 392,
-    392, 392, 392, 392, 392, 392, 392, 392,
-    392, 392, 392, 392, 392, 392, 392, 392,
-    392, 392, 392, 392, 392, 392, 392, 392,
-    392, 392, 392, 392, 392, 392, 392, 392,
-    392, 392, 392, 392, 392, 392, 392, 392,
-    392, 392, 392, 392, 392, 392, 392, 392,
-    392, 392, 392, 392, 392, 392, 392, 392,
-    392, 392, 392, 392, 392, 392, 392, 392,
-    392, 392, 392, 392, 392, 392, 392, 392,
-    392, 392, 392, 392, 392, 392, 392, 392,
-    392, 968, 392, 392, 392, 392, 392, 392,
-    392, 392, 392, 392, 392, 392, 392, 392,
-    392, 392, 392, 392, 392, 392, 392, 392,
-    392, 392, 392, 392, 392, 392, 392, 392,
-    392, 392, 392, 392, 392, 392, 392, 392,
-
-    392, 392, 392, 392, 392, 392, 392, 392,
-    392, 392, 392, 392, 392, 392, 392, 392,
-    392, 392, 392, 392, 392, 392, 392, 392,
-    392, 392, 392, 392, 392, 392, 392, 392,
-    392, 392, 392, 392, 392, 392, 392, 392,
-    392, 392, 392, 392, 392, 392, 392, 392,
-    392, 392, 392, 392, 392, 392, 392, 392,
-    392, 392, 392, 392, 392, 392, 392, 392,
-    392, 392, 392, 392, 392, 392, 392, 392,
-    392, 392, 392, 392, 392, 392, 392, 392,
-    392, 392, 392, 392, 392, 392, 392, 392,
-    392, 392, 392, 392, 392, 392, 392, 392,
-    392, 392, 392, 392, 392, 392, 392, 392,
-    392, 392, 392, 392, 392, 392, 392, 392,
-    392, 392, 392, 392, 392, 392, 392, 392,
-    392, 392, 392, 392, 392, 392, 392, 392,
-
-    392, 392, 392, 392, 1000, 392, 392, 392,
+    631, 631, 631, 631, 631, 631, 631, 631,
+    631, 631, 631, 631, 631, 631, 631, 631,
+    631, 631, 631, 631, 631, 631, 631, 631,
+    663, 695, 727, 631, 631, 631, 631, 631,
+    631, 631, 631, 631, 631, 631, 631, 631,
+    631, 631, 631, 631, 631, 631, 631, 631,
+    631, 631, 759, 631, 631, 631, 631, 631,
+    631, 631, 631, 631, 631, 631, 631, 631,
+    631, 631, 631, 631, 631, 631, 631, 631,
+    631, 791, 631, 631, 631, 823, 855, 631,
+    631, 631, 631, 631, 631, 631, 631, 631,
+    631, 887, 919, 631, 631, 951, 983, 631,
+    631, 631, 1015, 631, 631, 631, 1047, 631,
+    631, 1079, 1111, 631, 631, 631, 1143, 631,
+    631, 631, 631, 631, 631, 631, 631, 631,
+    631, 631, 631, 631, 631, 631, 631, 631,
+
+    631, 1175, 631, 631, 631, 631, 631, 631,
+    631, 631, 631, 631, 631, 631, 631, 631,
+    631, 631, 631, 631, 631, 631, 631, 631,
+    631, 631, 631, 631, 631, 631, 631, 631,
+    631, 631, 631, 631, 631, 631, 631, 631,
+    631, 631, 631, 631, 631, 631, 631, 631,
+    631, 631, 631, 631, 631, 631, 631, 631,
+    631, 631, 631, 631, 631, 631, 631, 631,
+    631, 631, 631, 631, 631, 631, 631, 631,
+    631, 631, 631, 631, 631, 631, 631, 631,
+    631, 631, 631, 631, 631, 631, 631, 631,
+    631, 1207, 631, 631, 631, 631, 631, 631,
+    631, 631, 631, 631, 631, 631, 631, 631,
+    631, 631, 631, 631, 631, 631, 631, 631,
+    631, 631, 631, 631, 631, 631, 631, 631,
+    631, 631, 631, 631, 631, 631, 631, 631,
+
+    631, 631, 631, 631, 631, 631, 631, 631,
+    631, 631, 631, 631, 631, 631, 631, 631,
+    631, 631, 631, 631, 631, 631, 631, 631,
+    631, 631, 631, 631, 631, 631, 631, 631,
+    631, 631, 631, 631, 631, 631, 631, 631,
+    631, 631, 631, 631, 631, 631, 631, 631,
+    631, 631, 631, 631, 631, 631, 631, 631,
+    631, 631, 631, 631, 631, 631, 631, 631,
+    631, 631, 631, 631, 631, 631, 631, 631,
+    631, 631, 631, 631, 631, 631, 631, 631,
+    631, 631, 631, 631, 631, 631, 631, 631,
+    631, 631, 631, 631, 631, 631, 631, 631,
+    631, 631, 631, 631, 631, 631, 631, 631,
+    631, 631, 631, 631, 631, 631, 631, 631,
+    631, 631, 631, 631, 631, 631, 631, 631,
+    631, 631, 631, 631, 631, 631, 631, 631,
+
+    631, 631, 631, 631, 1239, 631, 631, 631,
+
+    // 0x3100 - 0x12000
+
+    1271, 1271, 1271, 1271, 1271, 1271, 1271, 1271,
+    1271, 1271, 1271, 1271, 1271, 1271, 1271, 1271,
+    1271, 1271, 1271, 1271, 1271, 1271, 1271, 1271,
+    1271, 1271, 1271, 1271, 1271, 1271, 1271, 1271,
+    1271, 1271, 1271, 1271, 1271, 1271, 1271, 1271,
+    1271, 1271, 1271, 1271, 1271, 1271, 1271, 1271,
+    1271, 1271, 1271, 1271, 1271, 1271, 1271, 1271,
+    1271, 1271, 1271, 1271, 1271, 1271, 1271, 1271,
+    1271, 1271, 1271, 1271, 1271, 1271, 1271, 1271,
+    1271, 1271, 1271, 1271, 1271, 1271, 1271, 1271,
+    1271, 1271, 1271, 1271, 1271, 1271, 1271, 1271,
+    1271, 1271, 1271, 1271, 1271, 1271, 1271, 1271,
+    1271, 1271, 1271, 1271, 1271, 1271, 1271, 1271,
+    1271, 1271, 1271, 1271, 1271, 1271, 1271, 1271,
+    1271, 1271, 1271, 1271, 1271, 1271, 1271, 1271,
+
+    1271, 1271, 1271, 1271, 1271, 1271, 1271, 1271,
+    1271, 1271, 1271, 1271, 1271, 1271, 1271, 1271,
+    1271, 1271, 1271, 1271, 1271, 1271, 1271, 1271,
+    1271, 1271, 1271, 1271, 1271, 1271, 1271, 1271,
+    1271, 1271, 1271, 1271, 1271, 1271, 1271, 1271,
+    1271, 1271, 1271, 1271, 1271, 1271, 1271, 1271,
+    1271, 1271, 1271, 1271, 1271, 1271, 1271, 1271,
+    1271, 1271, 1271, 1271, 1271, 1271, 1271, 1271,
+    1271, 1271, 1271, 1271, 1271, 1271, 1271, 1271,
+    1271, 1271, 1271, 1271, 1271, 1271, 1271, 1271,
+    1271, 1271, 1271, 1271, 1271, 1271, 1271, 1271,
+    1271, 1271, 1271, 1271, 1271, 1271, 1271, 1271,
+    1271, 1271, 1271, 1271, 1271, 1271, 1271, 1271,
+    1271, 1271, 1271, 1271, 1271, 1271, 1271, 1271,
+    1271, 1271, 1271, 1271, 1271, 1271, 1271,
 
 
     0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff,
@@ -7768,11 +7802,48 @@ static const unsigned short uc_ligature_trie[] = {
     0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff,
     0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff,
     0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff,
-    0xffff, 0x700, 0x761, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff
+    0xffff, 0x700, 0x761, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff,
+
+    0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff,
+    0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff,
+    0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff,
+    0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff,
+    0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff,
+    0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff,
+    0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff,
+    0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff,
+    0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff,
+    0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff,
+    0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff,
+    0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff,
+    0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff,
+    0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff,
+    0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff,
+    0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff,
+    0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff,
+    0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff,
+    0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff,
+    0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff,
+    0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff,
+    0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff,
+    0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff,
+    0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff,
+    0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff,
+    0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff,
+    0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff,
+    0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff,
+    0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff,
+    0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff,
+    0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff,
+    0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff
 };
 
-#define GET_LIGATURE_INDEX(u2) \
-       (u2 < 0x3100 ? uc_ligature_trie[uc_ligature_trie[u2>>5] + (u2 & 0x1f)] : 0xffff);
+#define GET_LIGATURE_INDEX(ucs4) \
+       (ucs4 < 0x3100 \
+        ? (uc_ligature_trie[uc_ligature_trie[ucs4>>5] + (ucs4 & 0x1f)]) \
+        : (ucs4 < 0x12000\
+           ? uc_ligature_trie[uc_ligature_trie[((ucs4 - 0x3100)>>8) + 0x188] + (ucs4 & 0xff)]\
+           : 0xffff))
 
 static const unsigned short uc_ligature_map[] = {
     0x54, 0x41, 0xc0, 0x45, 0xc8, 0x49, 0xcc, 0x4e,

diff --git a/tests/auto/corelib/tools/qchar/tst_qchar.cpp b/tests/auto/corelib/tools/qchar/tst_qchar.cpp
index 1a56807..570ecb1 100644 (file)
--- a/tests/auto/corelib/tools/qchar/tst_qchar.cpp
+++ b/tests/auto/corelib/tools/qchar/tst_qchar.cpp
@@ -791,6 +791,18 @@ void tst_QChar::normalization()
 void tst_QChar::normalization_manual()
 {
     {
+        QString decomposed;
+        decomposed += QChar(0x41);
+        decomposed += QChar(0x0221); // assigned in 4.0
+        decomposed += QChar(0x300);
+
+        QVERIFY(decomposed.normalized(QString::NormalizationForm_C, QChar::Unicode_3_2) == decomposed);
+
+        decomposed[1] = QChar(0x037f); // unassigned in 6.1
+
+        QVERIFY(decomposed.normalized(QString::NormalizationForm_C) == decomposed);
+    }
+    {
         QString composed;
         composed += QChar(0xc0);
         QString decomposed;

diff --git a/util/unicode/main.cpp b/util/unicode/main.cpp
index 89e4406..b9245ba 100644 (file)
--- a/util/unicode/main.cpp
+++ b/util/unicode/main.cpp
@@ -714,15 +714,15 @@ static int numLigatures = 0;
 static int highestLigature = 0;
 
 struct Ligature {
-    ushort u1;
-    ushort u2;
-    ushort ligature;
+    int u1;
+    int u2;
+    int ligature;
 };
 // we need them sorted after the first component for fast lookup
 bool operator < (const Ligature &l1, const Ligature &l2)
 { return l1.u1 < l2.u1; }
 
-static QHash<ushort, QList<Ligature> > ligatureHashes;
+static QHash<int, QList<Ligature> > ligatureHashes;
 
 static QHash<int, int> combiningClassUsage;
 
@@ -1086,7 +1086,7 @@ static void readDerivedNormalizationProps()
 
             ++numLigatures;
             highestLigature = qMax(highestLigature, part1);
-            Ligature l = {(ushort)part1, (ushort)part2, (ushort)codepoint};
+            Ligature l = { part1, part2, codepoint };
             ligatureHashes[part2].append(l);
         }
     }
@@ -2522,17 +2522,28 @@ static QByteArray createLigatureInfo()
 {
     qDebug("createLigatureInfo: numLigatures=%d, highestLigature=0x%x", numLigatures, highestLigature);
 
-    QList<DecompositionBlock> blocks;
-    QList<int> blockMap;
-    QList<unsigned short> ligatures;
+    foreach (const QList<Ligature> &l, ligatureHashes) {
+        for (int j = 0; j < l.size(); ++j) {
+            // if the condition below doesn't hold anymore we need to modify our ligatureHelper code
+            Q_ASSERT(QChar::requiresSurrogates(l.at(j).u2) == QChar::requiresSurrogates(l.at(j).ligature) &&
+                     QChar::requiresSurrogates(l.at(j).u2) == QChar::requiresSurrogates(l.at(j).u1));
+        }
+    }
 
     const int BMP_BLOCKSIZE = 32;
     const int BMP_SHIFT = 5;
     const int BMP_END = 0x3100;
+    const int SMP_END = 0x12000;
+    const int SMP_BLOCKSIZE = 256;
+    const int SMP_SHIFT = 8;
 
-    if (BMP_END <= highestLigature)
+    if (SMP_END <= highestLigature)
         qFatal("end of table smaller than highest ligature character 0x%x", highestLigature);
 
+    QList<DecompositionBlock> blocks;
+    QList<int> blockMap;
+    QList<unsigned short> ligatures;
+
     int used = 0;
     int tableIndex = 0;
 
@@ -2569,6 +2580,38 @@ static QByteArray createLigatureInfo()
     int bmp_blocks = blocks.size();
     Q_ASSERT(blockMap.size() == BMP_END/BMP_BLOCKSIZE);
 
+    for (int block = BMP_END/SMP_BLOCKSIZE; block < SMP_END/SMP_BLOCKSIZE; ++block) {
+        DecompositionBlock b;
+        for (int i = 0; i < SMP_BLOCKSIZE; ++i) {
+            int uc = block*SMP_BLOCKSIZE + i;
+            QList<Ligature> l = ligatureHashes.value(uc);
+            if (!l.isEmpty()) {
+                Q_ASSERT(QChar::requiresSurrogates(uc));
+                qSort(l); // needed for bsearch in ligatureHelper code
+
+                ligatures.append(l.size());
+                for (int j = 0; j < l.size(); ++j) {
+                    ligatures.append(QChar::highSurrogate(l.at(j).u1));
+                    ligatures.append(QChar::lowSurrogate(l.at(j).u1));
+                    ligatures.append(QChar::highSurrogate(l.at(j).ligature));
+                    ligatures.append(QChar::lowSurrogate(l.at(j).ligature));
+                }
+                b.decompositionPositions.append(tableIndex);
+                tableIndex += 4*l.size() + 1;
+            } else {
+                b.decompositionPositions.append(0xffff);
+            }
+        }
+        int index = blocks.indexOf(b);
+        if (index == -1) {
+            index = blocks.size();
+            b.index = used;
+            used += SMP_BLOCKSIZE;
+            blocks.append(b);
+        }
+        blockMap.append(blocks.at(index).index);
+    }
+
     // if the condition below doesn't hold anymore we need to modify our composition code
     Q_ASSERT(tableIndex < 0xffff);
 
@@ -2579,8 +2622,16 @@ static QByteArray createLigatureInfo()
     qDebug("        block data uses: %d bytes", bmp_block_data);
     qDebug("        trie data uses : %d bytes", bmp_trie);
     qDebug("        memory usage: %d bytes", bmp_mem);
+
+    int smp_block_data = (blocks.size() - bmp_blocks)*SMP_BLOCKSIZE*2;
+    int smp_trie = (SMP_END-BMP_END)/SMP_BLOCKSIZE*2;
+    int smp_mem = smp_block_data + smp_trie;
+    qDebug("    %d unique blocks in SMP.", blocks.size()-bmp_blocks);
+    qDebug("        block data uses: %d bytes", smp_block_data);
+    qDebug("        trie data uses : %d bytes", smp_trie);
+
     qDebug("\n        ligature data uses : %d bytes", ligatures.size()*2);
-    qDebug("    memory usage: %d bytes", bmp_mem + ligatures.size() * 2);
+    qDebug("    memory usage: %d bytes", bmp_mem + smp_mem + ligatures.size() * 2);
 
     QByteArray out;
 
@@ -2601,6 +2652,20 @@ static QByteArray createLigatureInfo()
     }
     if (out.endsWith(' '))
         out.chop(1);
+    out += "\n\n    // 0x" + QByteArray::number(BMP_END, 16) + " - 0x" + QByteArray::number(SMP_END, 16) + "\n";
+    for (int i = BMP_END/BMP_BLOCKSIZE; i < blockMap.size(); ++i) {
+        if (!(i % 8)) {
+            if (out.endsWith(' '))
+                out.chop(1);
+            if (!(i % (0x10000/SMP_BLOCKSIZE)))
+                out += "\n";
+            out += "\n    ";
+        }
+        out += QByteArray::number(blockMap.at(i) + blockMap.size());
+        out += ", ";
+    }
+    if (out.endsWith(' '))
+        out.chop(1);
     out += "\n";
     // write the data
     for (int i = 0; i < blocks.size(); ++i) {
@@ -2622,10 +2687,15 @@ static QByteArray createLigatureInfo()
         out.chop(2);
     out += "\n};\n\n"
 
-           "#define GET_LIGATURE_INDEX(u2) \\\n"
-           "       (u2 < 0x" + QByteArray::number(BMP_END, 16) + " ? "
-           "uc_ligature_trie[uc_ligature_trie[u2>>" + QByteArray::number(BMP_SHIFT) +
-           "] + (u2 & 0x" + QByteArray::number(BMP_BLOCKSIZE-1, 16)+ ")] : 0xffff);\n\n"
+           "#define GET_LIGATURE_INDEX(ucs4) \\\n"
+           "       (ucs4 < 0x" + QByteArray::number(BMP_END, 16) + " \\\n"
+           "        ? (uc_ligature_trie[uc_ligature_trie[ucs4>>" + QByteArray::number(BMP_SHIFT) +
+           "] + (ucs4 & 0x" + QByteArray::number(BMP_BLOCKSIZE-1, 16)+ ")]) \\\n"
+           "        : (ucs4 < 0x" + QByteArray::number(SMP_END, 16) + "\\\n"
+           "           ? uc_ligature_trie[uc_ligature_trie[((ucs4 - 0x" + QByteArray::number(BMP_END, 16) +
+           ")>>" + QByteArray::number(SMP_SHIFT) + ") + 0x" + QByteArray::number(BMP_END/BMP_BLOCKSIZE, 16) + "]"
+           " + (ucs4 & 0x" + QByteArray::number(SMP_BLOCKSIZE-1, 16) + ")]\\\n"
+           "           : 0xffff))\n\n"
 
            "static const unsigned short uc_ligature_map[] = {";