Remove temporary string allocations when reading prepared statement.

Instead, we use the binary MySQL encoding and copy the data directly
into the QVariant of the desired type. This gets rid of the temporary
string allocations and greatly improves the performance of the added
benchmark. On my machine, the results are:

Before:
     0.562 msecs per iteration (total: 563, iterations: 1000)
     1,922,479.330 instructions per iteration (total: 1,922,479,330, iterations: 1000)

After:
     0.381 msecs per iteration (total: 381, iterations: 1000)
     774,132.957 instructions per iteration (total: 774,132,958, iterations: 1000)

Note that the same could be applied to floating point data types in
the future. Additionally, special support for MYSQL_TIME structure
coult be added to get rid of the string conversions there.

To ensure everything keeps working, a new auto test is added as well
that verifies the select statements and insertions of integral data
into a MySql table works as intended.

[ChangeLog][QtSql] Improve performance when reading integer values
from MySQL databases via prepared statements.

Change-Id: I21dd9277661971ded934546f09535014b63f8eb8
Reviewed-by: Olivier Goffart (Woboq GmbH) <ogoffart@woboq.com>

diff --git a/src/sql/drivers/mysql/qsql_mysql.cpp b/src/sql/drivers/mysql/qsql_mysql.cpp
index 086a232746ae0ec3d7cd1a48eb9d7743665cfa49..ea8a918dcf637eb434d5541dfaafcfd994afc6fd 100644 (file)
--- a/src/sql/drivers/mysql/qsql_mysql.cpp
+++ b/src/sql/drivers/mysql/qsql_mysql.cpp
@@ -237,7 +237,11 @@ static QVariant::Type qDecodeMYSQLType(int mysqltype, uint flags)
     QVariant::Type type;
     switch (mysqltype) {
     case FIELD_TYPE_TINY :
+        type = static_cast<QVariant::Type>((flags & UNSIGNED_FLAG) ? QMetaType::UChar : QMetaType::Char);
+        break;
     case FIELD_TYPE_SHORT :
+        type = static_cast<QVariant::Type>((flags & UNSIGNED_FLAG) ? QMetaType::UShort : QMetaType::Short);
+        break;
     case FIELD_TYPE_LONG :
     case FIELD_TYPE_INT24 :
         type = (flags & UNSIGNED_FLAG) ? QVariant::UInt : QVariant::Int;
@@ -316,14 +320,12 @@ static bool qIsBlob(int t)
 
 static bool qIsInteger(int t)
 {
-    return t == MYSQL_TYPE_TINY
-           || t == MYSQL_TYPE_SHORT
-           || t == MYSQL_TYPE_LONG
-           || t == MYSQL_TYPE_LONGLONG
-           || t == MYSQL_TYPE_INT24;
+    return t == QMetaType::Char || t == QMetaType::UChar
+        || t == QMetaType::Short || t == QMetaType::UShort
+        || t == QMetaType::Int || t == QMetaType::UInt
+        || t == QMetaType::LongLong || t == QMetaType::ULongLong;
 }
 
-
 void QMYSQLResultPrivate::bindBlobs()
 {
     int i;
@@ -371,14 +373,9 @@ bool QMYSQLResultPrivate::bindInValues()
             // after mysql_stmt_exec() in QMYSQLResult::exec()
             fieldInfo->length = 0;
             hasBlobs = true;
+        } else if (qIsInteger(f.type)) {
+            fieldInfo->length = 8;
         } else {
-            // fieldInfo->length specifies the display width, which may be too
-            // small to hold valid integer values (see
-            // http://dev.mysql.com/doc/refman/5.0/en/numeric-types.html ), so
-            // always use the MAX_BIGINT_WIDTH for integer types
-            if (qIsInteger(fieldInfo->type)) {
-                fieldInfo->length = MAX_BIGINT_WIDTH;
-            }
             fieldInfo->type = MYSQL_TYPE_STRING;
         }
         bind = &inBinds[i];
@@ -598,6 +595,9 @@ QVariant QMYSQLResult::data(int field)
         if (f.nullIndicator)
             return QVariant(f.type);
 
+        if (qIsInteger(f.type))
+            return QVariant(f.type, f.outField);
+
         if (f.type != QVariant::ByteArray)
             val = toUnicode(d->driver->d_func()->tc, f.outField, f.bufLength);
     } else {
@@ -605,18 +605,24 @@ QVariant QMYSQLResult::data(int field)
             // NULL value
             return QVariant(f.type);
         }
+
         fieldLength = mysql_fetch_lengths(d->result)[field];
+
         if (f.type != QVariant::ByteArray)
             val = toUnicode(d->driver->d_func()->tc, d->row[field], fieldLength);
     }
 
-    switch(f.type) {
+    switch (static_cast<int>(f.type)) {
     case QVariant::LongLong:
         return QVariant(val.toLongLong());
     case QVariant::ULongLong:
         return QVariant(val.toULongLong());
+    case QMetaType::Char:
+    case QMetaType::Short:
     case QVariant::Int:
         return QVariant(val.toInt());
+    case QMetaType::UChar:
+    case QMetaType::UShort:
     case QVariant::UInt:
         return QVariant(val.toUInt());
     case QVariant::Double: {

diff --git a/tests/auto/sql/kernel/qsqlquery/tst_qsqlquery.cpp b/tests/auto/sql/kernel/qsqlquery/tst_qsqlquery.cpp
index 61586eb8416a2275f56a2f431ae903f7c7232325..b98ab68ae9fae55bb992accb79c49bf8258b7c8b 100644 (file)
--- a/tests/auto/sql/kernel/qsqlquery/tst_qsqlquery.cpp
+++ b/tests/auto/sql/kernel/qsqlquery/tst_qsqlquery.cpp
@@ -241,6 +241,10 @@ private slots:
 
     void aggregateFunctionTypes_data() { generic_data(); }
     void aggregateFunctionTypes();
+
+    void integralTypesMysql_data() { generic_data("QMYSQL"); }
+    void integralTypesMysql();
+
 private:
     // returns all database connections
     void generic_data(const QString &engine=QString());
@@ -3974,5 +3978,65 @@ void tst_QSqlQuery::aggregateFunctionTypes()
     }
 }
 
+template<typename T>
+void runIntegralTypesMysqlTest(QSqlDatabase &db, const QString &tableName, const QString &type, const bool withPreparedStatement,
+                               const T min = std::numeric_limits<T>::min(), const T max = std::numeric_limits<T>::max())
+{
+    QSqlQuery q(db);
+    QVERIFY_SQL(q, exec("DROP TABLE IF EXISTS " + tableName));
+    QVERIFY_SQL(q, exec("CREATE TABLE " + tableName + " (id " + type + ")"));
+
+    const int steps = 20;
+    const T increment = max / steps - min / steps;
+
+    // insert some values
+    QVector<T> values;
+    values.resize(steps);
+    T v = min;
+    if (withPreparedStatement) {
+        QVERIFY_SQL(q, prepare("INSERT INTO " + tableName + " (id) VALUES (?)"));
+    }
+    for (int i = 0; i < values.size(); ++i) {
+        if (withPreparedStatement) {
+            q.bindValue(0, v);
+            QVERIFY_SQL(q, exec());
+        } else {
+            QVERIFY_SQL(q, exec("INSERT INTO " + tableName + " (id) VALUES (" + QString::number(v) + ")"));
+        }
+        values[i] = v;
+        v += increment;
+    }
+
+    // ensure we can read them back properly
+    QVERIFY_SQL(q, exec("SELECT id FROM " + tableName));
+    QVector<T> actualValues;
+    actualValues.reserve(values.size());
+    while (q.next()) {
+        actualValues << q.value(0).value<T>();
+    }
+    QCOMPARE(actualValues, values);
+}
+
+void tst_QSqlQuery::integralTypesMysql()
+{
+    QFETCH(QString, dbName);
+    QSqlDatabase db = QSqlDatabase::database(dbName);
+    CHECK_DATABASE(db);
+
+    for (int i = 0; i < 2; ++i) {
+        const bool withPreparedStatement = (i == 1);
+        runIntegralTypesMysqlTest<char>(db, "tinyIntTest", "TINYINT", withPreparedStatement);
+        runIntegralTypesMysqlTest<unsigned char>(db, "unsignedTinyIntTest", "TINYINT UNSIGNED", withPreparedStatement);
+        runIntegralTypesMysqlTest<char>(db, "smallIntTest", "SMALLINT", withPreparedStatement);
+        runIntegralTypesMysqlTest<unsigned char>(db, "unsignedSmallIntTest", "SMALLINT UNSIGNED", withPreparedStatement);
+        runIntegralTypesMysqlTest<int>(db, "mediumIntTest", "MEDIUMINT", withPreparedStatement, -(1 << 23), (1 << 23) - 1);
+        runIntegralTypesMysqlTest<unsigned int>(db, "unsignedMediumIntTest", "MEDIUMINT UNSIGNED", withPreparedStatement, 0, (1 << 24) - 1);
+        runIntegralTypesMysqlTest<int>(db, "intTest", "INT", withPreparedStatement);
+        runIntegralTypesMysqlTest<unsigned int>(db, "unsignedIntTest", "INT UNSIGNED", withPreparedStatement);
+        runIntegralTypesMysqlTest<long long>(db, "bigIntTest", "BIGINT", withPreparedStatement);
+        runIntegralTypesMysqlTest<unsigned long long>(db, "unsignedBigIntTest", "BIGINT UNSIGNED", withPreparedStatement);
+    }
+}
+
 QTEST_MAIN( tst_QSqlQuery )
 #include "tst_qsqlquery.moc"

diff --git a/tests/benchmarks/sql/kernel/qsqlquery/main.cpp b/tests/benchmarks/sql/kernel/qsqlquery/main.cpp
index 63aa6f7250e72a23f9855b22944ad2564e9d272c..b5937b76b0542a2229e8ad3d7dc366b3018f1976 100644 (file)
--- a/tests/benchmarks/sql/kernel/qsqlquery/main.cpp
+++ b/tests/benchmarks/sql/kernel/qsqlquery/main.cpp
@@ -55,6 +55,8 @@ public slots:
 private slots:
     void benchmark_data() { generic_data(); }
     void benchmark();
+    void benchmarkSelectPrepared_data() { generic_data(); }
+    void benchmarkSelectPrepared();
 
 private:
     // returns all database connections
@@ -264,4 +266,42 @@ void tst_QSqlQuery::benchmark()
     tst_Databases::safeDropTable( db, tableName );
 }
 
+void tst_QSqlQuery::benchmarkSelectPrepared()
+{
+    QFETCH( QString, dbName );
+    QSqlDatabase db = QSqlDatabase::database(dbName);
+    CHECK_DATABASE(db);
+    if (tst_Databases::getMySqlVersion(db).section(QChar('.'), 0, 0).toInt() < 5)
+        QSKIP("Test requires MySQL >= 5.0");
+
+    QSqlQuery q(db);
+    const QString tableName(qTableName("benchmark", __FILE__, db));
+
+    tst_Databases::safeDropTable(db, tableName);
+
+    QVERIFY_SQL(q, exec("CREATE TABLE " + tableName + "(id INT NOT NULL)"));
+
+    const int NUM_ROWS = 1000;
+    int expectedSum = 0;
+    QString fillQuery = "INSERT INTO " + tableName + " VALUES (0)";
+    for (int i = 1; i < NUM_ROWS; ++i) {
+        fillQuery += ", (" + QString::number(i) + ")";
+        expectedSum += i;
+    }
+    QVERIFY_SQL(q, exec(fillQuery));
+
+    QVERIFY_SQL(q, prepare("SELECT id FROM "+tableName));
+    QBENCHMARK {
+        QVERIFY_SQL(q, exec());
+        int sum = 0;
+
+        while (q.next())
+            sum += q.value(0).toInt();
+
+        QCOMPARE(sum, expectedSum);
+    }
+
+    tst_Databases::safeDropTable(db, tableName);
+}
+
 #include "main.moc"