addressbook/libedata-book/e-book-sqlite.c

   1 /*-*- Mode: C; tab-width: 8; indent-tabs-mode: t; c-basic-offset: 8 -*- */
   2 /* e-book-sqlite.c
   3  *
   4  * Copyright (C) 2013 Intel Corporation
   5  *
   6  * Authors:
   7  *     Tristan Van Berkom <tristanvb@openismus.com>
   8  *
   9  * This library is free software you can redistribute it and/or modify it
  10  * under the terms of the GNU Lesser General Public License as published by
  11  * the Free Software Foundation.
  12  *
  13  * This library is distributed in the hope that it will be useful, but
  14  * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
  15  * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  16  *for more details.
  17  *
  18  * You should have received a copy of the GNU Lesser General Public License
  19  * along with this program; if not, see <http://www.gnu.org/licenses/>.
  20  */
  21
  22 /**
  23  * SECTION: e-book-sqlite
  24  * @include: libedata-book/libedata-book.h
  25  * @short_description: An SQLite storage facility for addressbooks
  26  *
  27  * The #EBookSqlite is an API for storing and looking up #EContacts
  28  * in an SQLite database. It also supports a lean index mode via
  29  * the #EbSqlVCardCallback, if you are in a situation where it is
  30  * not convenient to store the vCards directly in the SQLite. It is
  31  * however recommended to avoid storing contacts in separate storage
  32  * if at all possible, as this will decrease performance of searches
  33  * an also contribute to flash wear.
  34  *
  35  * The API is thread safe, with special considerations to be made
  36  * around e_book_sqlite_lock() and e_book_sqlite_unlock() for
  37  * the sake of isolating transactions across threads.
  38  *
  39  * Any operations which can take a lot of time to complete (depending
  40  * on the size of your addressbook) can be cancelled using a #GCancellable.
  41  *
  42  * Depending on your summary configuration, your mileage will vary. Refer
  43  * to the #ESourceBackendSummarySetup for configuring your addressbook
  44  * for the type of usage you mean to make of it.
  45  */
  46
  47 #include "e-book-sqlite.h"
  48
  49 #include <locale.h>
  50 #include <string.h>
  51 #include <errno.h>
  52
  53 #include <glib/gi18n.h>
  54 #include <glib/gstdio.h>
  55
  56 #include <sqlite3.h>
  57
  58 /* For e_sqlite3_vfs_init() */
  59 #include <libebackend/libebackend.h>
  60
  61 #include "e-book-backend-sexp.h"
  62
  63 #define E_BOOK_SQLITE_GET_PRIVATE(obj) \
  64         (G_TYPE_INSTANCE_GET_PRIVATE \
  65         ((obj), E_TYPE_BOOK_SQLITE, EBookSqlitePrivate))
  66
  67 /******************************************************
  68  *                 Debugging Macros                   *
  69  ******************************************************
  70  * Run EDS with EBSQL_DEBUG=statements:explain to print
  71  * all statements and explain query plans.
  72  *
  73  * Use any of the values below to select which debug
  74  * to enable.
  75  */
  76 #define EBSQL_ENV_DEBUG   "EBSQL_DEBUG"
  77
  78 typedef enum {
  79         EBSQL_DEBUG_STATEMENTS    = 1 << 0,  /* Output all executed statements */
  80         EBSQL_DEBUG_EXPLAIN       = 1 << 1,  /* Output SQLite's query plan for SELECT statements */
  81         EBSQL_DEBUG_LOCKS         = 1 << 2,  /* Print which function locks and unlocks the mutex */
  82         EBSQL_DEBUG_ERRORS        = 1 << 3,  /* Print all errors which are set */
  83         EBSQL_DEBUG_SCHEMA        = 1 << 4,  /* Debugging the schema building / upgrading */
  84         EBSQL_DEBUG_INSERT        = 1 << 5,  /* Debugging contact insertions */
  85         EBSQL_DEBUG_FETCH_VCARD   = 1 << 6,  /* Print invocations of the EbSqlVCardCallback fallback */
  86         EBSQL_DEBUG_CURSOR        = 1 << 7,  /* Print information about EbSqlCursor operations */
  87         EBSQL_DEBUG_CONVERT_E164  = 1 << 8,  /* Print information e164 phone number conversions in vcards */
  88         EBSQL_DEBUG_REF_COUNTS    = 1 << 9,  /* Print about shared EBookSqlite instances, print when finalized */
  89         EBSQL_DEBUG_CANCEL        = 1 << 10, /* Print information about GCancellable cancellations */
  90         EBSQL_DEBUG_PREFLIGHT     = 1 << 11, /* Print information about query preflighting */
  91 } EbSqlDebugFlag;
  92
  93 static const GDebugKey ebsql_debug_keys[] = {
  94         { "statements",     EBSQL_DEBUG_STATEMENTS   },
  95         { "explain",        EBSQL_DEBUG_EXPLAIN      },
  96         { "locks",          EBSQL_DEBUG_LOCKS        },
  97         { "errors",         EBSQL_DEBUG_ERRORS       },
  98         { "schema",         EBSQL_DEBUG_SCHEMA       },
  99         { "insert",         EBSQL_DEBUG_INSERT       },
 100         { "fetch-vcard",    EBSQL_DEBUG_FETCH_VCARD  },
 101         { "cursor",         EBSQL_DEBUG_CURSOR       },
 102         { "e164",           EBSQL_DEBUG_CONVERT_E164 },
 103         { "ref-counts",     EBSQL_DEBUG_REF_COUNTS   },
 104         { "cancel",         EBSQL_DEBUG_CANCEL       },
 105         { "preflight",      EBSQL_DEBUG_PREFLIGHT    },
 106 };
 107
 108 static EbSqlDebugFlag ebsql_debug_flags = 0;
 109
 110 static void
 111 ebsql_init_debug (void)
 112 {
 113         static gboolean initialized = FALSE;
 114
 115         if (G_UNLIKELY (!initialized)) {
 116                 const gchar *env_string;
 117
 118                 env_string = g_getenv (EBSQL_ENV_DEBUG);
 119
 120                 if (env_string != NULL)
 121                         ebsql_debug_flags =
 122                                 g_parse_debug_string (
 123                                         env_string,
 124                                         ebsql_debug_keys,
 125                                         G_N_ELEMENTS (ebsql_debug_keys));
 126         }
 127 }
 128
 129 #define EBSQL_NOTE(type,action) \
 130         G_STMT_START { \
 131                 if (ebsql_debug_flags & EBSQL_DEBUG_##type) \
 132                         { action; }; \
 133         } G_STMT_END
 134
 135 #define EBSQL_ERROR_STR(code) \
 136         ((code) == E_BOOK_SQLITE_ERROR_ENGINE ? "engine" : \
 137          (code) == E_BOOK_SQLITE_ERROR_CONSTRAINT ? "constraint" : \
 138          (code) == E_BOOK_SQLITE_ERROR_CONTACT_NOT_FOUND ? "contact not found" : \
 139          (code) == E_BOOK_SQLITE_ERROR_INVALID_QUERY ? "invalid query" : \
 140          (code) == E_BOOK_SQLITE_ERROR_UNSUPPORTED_QUERY ? "unsupported query" : \
 141          (code) == E_BOOK_SQLITE_ERROR_UNSUPPORTED_FIELD ? "unsupported field" : \
 142          (code) == E_BOOK_SQLITE_ERROR_END_OF_LIST ? "end of list" : \
 143          (code) == E_BOOK_SQLITE_ERROR_LOAD ? "load" : "(unknown)")
 144
 145 #define EBSQL_ORIGIN_STR(origin) \
 146         ((origin) == EBSQL_CURSOR_ORIGIN_CURRENT ? "current" : \
 147          (origin) == EBSQL_CURSOR_ORIGIN_BEGIN ? "begin" : \
 148          (origin) == EBSQL_CURSOR_ORIGIN_END ? "end" : "(invalid)")
 149
 150 #define EBSQL_LOCK_MUTEX(mutex) \
 151         G_STMT_START { \
 152                 if (ebsql_debug_flags & EBSQL_DEBUG_LOCKS) { \
 153                         g_printerr ("%s: Locking %s\n", G_STRFUNC, #mutex); \
 154                         g_mutex_lock (mutex); \
 155                         g_printerr ("%s: Locked %s\n", G_STRFUNC, #mutex); \
 156                 } else { \
 157                         g_mutex_lock (mutex); \
 158                 } \
 159         } G_STMT_END
 160
 161 #define EBSQL_UNLOCK_MUTEX(mutex) \
 162         G_STMT_START { \
 163                 if (ebsql_debug_flags & EBSQL_DEBUG_LOCKS) { \
 164                         g_printerr ("%s: Unlocking %s\n", G_STRFUNC, #mutex); \
 165                         g_mutex_unlock (mutex); \
 166                         g_printerr ("%s: Unlocked %s\n", G_STRFUNC, #mutex); \
 167                 } else { \
 168                         g_mutex_unlock (mutex); \
 169                 } \
 170         } G_STMT_END
 171
 172 /* Format strings are passed through dgettext(), need to be reformatted */
 173 #define EBSQL_SET_ERROR(error, code, fmt, args...) \
 174         G_STMT_START { \
 175                 if (ebsql_debug_flags & EBSQL_DEBUG_ERRORS) { \
 176                         gchar *format = g_strdup_printf ( \
 177                                 "ERR [%%s]: Set error code '%%s': %s\n", fmt); \
 178                         g_printerr (format, G_STRFUNC, \
 179                                     EBSQL_ERROR_STR (code), ## args); \
 180                         g_free (format); \
 181                 } \
 182                 g_set_error (error, E_BOOK_SQLITE_ERROR, code, fmt, ## args); \
 183         } G_STMT_END
 184
 185 #define EBSQL_SET_ERROR_LITERAL(error, code, detail) \
 186         G_STMT_START { \
 187                 if (ebsql_debug_flags & EBSQL_DEBUG_ERRORS) { \
 188                         g_printerr ("ERR [%s]: " \
 189                                     "Set error code %s: %s\n", \
 190                                     G_STRFUNC, \
 191                                     EBSQL_ERROR_STR (code), detail); \
 192                 } \
 193                 g_set_error_literal (error, E_BOOK_SQLITE_ERROR, code, detail); \
 194         } G_STMT_END
 195
 196 /* EBSQL_LOCK_OR_RETURN:
 197  * @ebsql: The #EBookSqlite
 198  * @cancellable: A #GCancellable passed into an API
 199  * @val: Value to return if this check fails
 200  *
 201  * This will first lock the mutex and then check if
 202  * the passed cancellable is valid or invalid, it can
 203  * be invalid if it differs from a cancellable passed
 204  * to a toplevel transaction via e_book_sqlite_lock().
 205  *
 206  * If the check fails, the lock is released and then
 207  * @val is returned.
 208  */
 209 #define EBSQL_LOCK_OR_RETURN(ebsql, cancellable, val) \
 210         G_STMT_START { \
 211                 EBSQL_LOCK_MUTEX (&(ebsql)->priv->lock); \
 212                 if (cancellable != NULL && \
 213                     (ebsql)->priv->cancel != cancellable) { \
 214                         g_warning ("The GCancellable passed to `%s' " \
 215                                    "is not the same as the cancel object " \
 216                                    "passed to e_book_sqlite_lock()", \
 217                                    G_STRFUNC); \
 218                         EBSQL_UNLOCK_MUTEX (&(ebsql)->priv->lock); \
 219                         return val; \
 220                 } \
 221         } G_STMT_END
 222
 223 /* Set an error code from an sqlite_exec() or sqlite_step() return value & error message */
 224 #define EBSQL_SET_ERROR_FROM_SQLITE(error, code, message) \
 225         G_STMT_START { \
 226                 if (code == SQLITE_CONSTRAINT) { \
 227                         EBSQL_SET_ERROR_LITERAL (error, \
 228                                                  E_BOOK_SQLITE_ERROR_CONSTRAINT, \
 229                                                  errmsg); \
 230                 } else if (code == SQLITE_ABORT) { \
 231                         if (ebsql_debug_flags & EBSQL_DEBUG_ERRORS) { \
 232                                 g_printerr ("ERR [%s]: Set cancelled error\n", \
 233                                             G_STRFUNC); \
 234                         } \
 235                         g_set_error (error, \
 236                                      G_IO_ERROR, \
 237                                      G_IO_ERROR_CANCELLED, \
 238                                      "Operation cancelled: %s", errmsg); \
 239                 } else { \
 240                         EBSQL_SET_ERROR (error, \
 241                                          E_BOOK_SQLITE_ERROR_ENGINE, \
 242                                          "SQLite error code `%d': %s", \
 243                                          code, errmsg); \
 244                 } \
 245         } G_STMT_END
 246
 247 #define FOLDER_VERSION                8
 248 #define INSERT_MULTI_STMT_BYTES       128
 249 #define COLUMN_DEFINITION_BYTES       32
 250 #define GENERATED_QUERY_BYTES         1024
 251
 252 #define DEFAULT_FOLDER_ID            "folder_id"
 253
 254 /* We use a 64 bitmask to track which auxiliary tables
 255  * are needed to satisfy a query, it's doubtful that
 256  * anyone will need an addressbook with 64 fields configured
 257  * in the summary.
 258  */
 259 #define EBSQL_MAX_SUMMARY_FIELDS      64
 260
 261 /* The number of SQLite virtual machine instructions that are
 262  * evaluated at a time, the user passed GCancellable is
 263  * checked between each batch of evaluated instructions.
 264  */
 265 #define EBSQL_CANCEL_BATCH_SIZE       200
 266
 267 /* Number of contacts to relocalize at a time
 268  * while relocalizing the whole database
 269  */
 270 #define EBSQL_UPGRADE_BATCH_SIZE      20
 271
 272 #define EBSQL_ESCAPE_SEQUENCE        "ESCAPE '^'"
 273
 274 /* Names for custom functions */
 275 #define EBSQL_FUNC_COMPARE_VCARD     "compare_vcard"
 276 #define EBSQL_FUNC_FETCH_VCARD       "fetch_vcard"
 277 #define EBSQL_FUNC_EQPHONE_EXACT     "eqphone_exact"
 278 #define EBSQL_FUNC_EQPHONE_NATIONAL  "eqphone_national"
 279 #define EBSQL_FUNC_EQPHONE_SHORT     "eqphone_short"
 280
 281 /* Fallback collations are generated as with a prefix and an EContactField name */
 282 #define EBSQL_COLLATE_PREFIX         "ebsql_"
 283
 284 /* A special vcard attribute that we use only for private vcards */
 285 #define EBSQL_VCARD_SORT_KEY         "X-EVOLUTION-SORT-KEY"
 286
 287 /* Suffixes for column names used to store specialized data */
 288 #define EBSQL_SUFFIX_REVERSE         "reverse"
 289 #define EBSQL_SUFFIX_SORT_KEY        "localized"
 290 #define EBSQL_SUFFIX_PHONE           "phone"
 291 #define EBSQL_SUFFIX_COUNTRY         "country"
 292 #define EBSQL_SUFFIX_TRANSLIT        "translit"
 293
 294 /* Track EBookIndexType's in a bit mask  */
 295 #define INDEX_FLAG(type)  (1 << E_BOOK_INDEX_##type)
 296
 297 /* This macro is used to reffer to vcards in statements */
 298 #define EBSQL_VCARD_FRAGMENT(ebsql) \
 299         ((ebsql)->priv->vcard_callback ? \
 300          EBSQL_FUNC_FETCH_VCARD " (summary.uid, summary.bdata)" : \
 301          "summary.vcard")
 302
 303 /* Signatures for some of the SQLite callbacks which we pass around */
 304 typedef void    (*EbSqlCustomFunc)              (sqlite3_context *context,
 305                                                  gint argc,
 306                                                  sqlite3_value **argv);
 307 typedef gint    (*EbSqlRowFunc)                 (gpointer ref,
 308                                                  gint n_cols,
 309                                                  gchar **cols,
 310                                                  gchar **names);
 311
 312 /* Some forward declarations */
 313 static gboolean         ebsql_init_statements   (EBookSqlite *ebsql,
 314                                                  GError **error);
 315 static gboolean         ebsql_insert_contact    (EBookSqlite *ebsql,
 316                                                  EbSqlChangeType change_type,
 317                                                  EContact *contact,
 318                                                  const gchar *original_vcard,
 319                                                  const gchar *extra,
 320                                                  gboolean replace,
 321                                                  GError **error);
 322 static gboolean         ebsql_exec              (EBookSqlite *ebsql,
 323                                                  const gchar *stmt,
 324                                                  EbSqlRowFunc callback,
 325                                                  gpointer data,
 326                                                  GCancellable *cancellable,
 327                                                  GError **error);
 328
 329 typedef struct {
 330         EContactField field_id;           /* The EContact field */
 331         GType         type;               /* The GType (only support string or gboolean) */
 332         const gchar  *dbname;             /* The key for this field in the sqlite3 table */
 333         gint          index;              /* Types of searches this field should support (see EBookIndexType) */
 334         gchar        *aux_table;          /* Name of auxiliary table for this field, for multivalued fields only */
 335         gchar        *aux_table_symbolic; /* Symolic name of auxiliary table used in queries */
 336 } SummaryField;
 337
 338 struct _EBookSqlitePrivate {
 339
 340         /* Parameters and settings */
 341         gchar          *path;            /* Full file name of the file we're operating on (used for hash table entries) */
 342         gchar          *locale;          /* The current locale */
 343         gchar          *region_code;     /* Region code (for phone number parsing) */
 344         gchar          *folderid;        /* The summary table name (configurable, for support of legacy
 345                                           * databases created by EBookSqliteDB) */
 346
 347         EbSqlVCardCallback  vcard_callback;     /* User callback to fetch vcards instead of storing them */
 348         EbSqlChangeCallback change_callback;    /* User callback to catch change notifications  */
 349         gpointer            user_data;          /* Data & Destroy notifier for the above callbacks */
 350         GDestroyNotify      user_data_destroy;
 351
 352         /* Summary configuration */
 353         SummaryField   *summary_fields;
 354         gint            n_summary_fields;
 355
 356         GMutex          lock;            /* Main API lock */
 357         GMutex          updates_lock;    /* Lock used for calls to e_book_sqlite_lock_updates () */
 358         guint32         in_transaction;  /* Nested transaction counter */
 359         EbSqlLockType   lock_type;       /* The lock type acquired for the current transaction */
 360         GCancellable   *cancel;          /* User passed GCancellable, we abort an operation if cancelled */
 361
 362         ECollator      *collator;        /* The ECollator to create sort keys for any sortable fields */
 363         ETransliterator *transliterator; /* For transliterated queries */
 364
 365         /* SQLite resources  */
 366         sqlite3        *db;
 367         sqlite3_stmt   *insert_stmt;     /* Insert statement for main summary table */
 368         sqlite3_stmt   *replace_stmt;    /* Replace statement for main summary table */
 369         GHashTable     *multi_deletes;   /* Delete statement for each auxiliary table */
 370         GHashTable     *multi_inserts;   /* Insert statement for each auxiliary table */
 371
 372         ESource        *source;
 373 };
 374
 375 enum {
 376         BEFORE_INSERT_CONTACT,
 377         BEFORE_REMOVE_CONTACT,
 378         LAST_SIGNAL
 379 };
 380
 381 static guint signals[LAST_SIGNAL];
 382
 383 G_DEFINE_TYPE_WITH_CODE (EBookSqlite, e_book_sqlite, G_TYPE_OBJECT,
 384                          G_IMPLEMENT_INTERFACE (E_TYPE_EXTENSIBLE, NULL))
 385 G_DEFINE_QUARK (e-book-backend-sqlite-error-quark,
 386                 e_book_sqlite_error)
 387
 388 /* The ColumnInfo struct is used to constant data
 389  * and dynamically allocated data, the 'type' and
 390  * 'extra' members are however always constant.
 391  */
 392 typedef struct {
 393         gchar       *name;
 394         const gchar *type;
 395         const gchar *extra;
 396         gchar       *index;
 397 } ColumnInfo;
 398
 399 static ColumnInfo main_table_columns[] = {
 400         { (gchar *) "folder_id",       "TEXT",      "PRIMARY KEY", NULL },
 401         { (gchar *) "version",         "INTEGER",    NULL,         NULL },
 402         { (gchar *) "multivalues",     "TEXT",       NULL,         NULL },
 403         { (gchar *) "lc_collate",      "TEXT",       NULL,         NULL },
 404         { (gchar *) "countrycode",     "VARCHAR(2)", NULL,         NULL },
 405 };
 406
 407 /* Default summary configuration */
 408 static EContactField default_summary_fields[] = {
 409         E_CONTACT_UID,
 410         E_CONTACT_REV,
 411         E_CONTACT_FILE_AS,
 412         E_CONTACT_NICKNAME,
 413         E_CONTACT_FULL_NAME,
 414         E_CONTACT_GIVEN_NAME,
 415         E_CONTACT_FAMILY_NAME,
 416         E_CONTACT_EMAIL,
 417         E_CONTACT_TEL,
 418         E_CONTACT_IS_LIST,
 419         E_CONTACT_LIST_SHOW_ADDRESSES,
 420         E_CONTACT_WANTS_HTML
 421 };
 422
 423 /* Create indexes on full_name and email fields as autocompletion
 424  * queries would mainly rely on this.
 425  *
 426  * Add sort keys for name fields as those are likely targets for
 427  * cursor usage.
 428  */
 429 static EContactField default_indexed_fields[] = {
 430         E_CONTACT_FULL_NAME,
 431         E_CONTACT_EMAIL,
 432         E_CONTACT_FILE_AS,
 433         E_CONTACT_FAMILY_NAME,
 434         E_CONTACT_GIVEN_NAME
 435 };
 436
 437 static EBookIndexType default_index_types[] = {
 438         E_BOOK_INDEX_PREFIX,
 439         E_BOOK_INDEX_PREFIX,
 440         E_BOOK_INDEX_SORT_KEY,
 441         E_BOOK_INDEX_SORT_KEY,
 442         E_BOOK_INDEX_SORT_KEY
 443 };
 444
 445 /******************************************************
 446  *                  Summary Fields                    *
 447  ******************************************************/
 448 static ColumnInfo *
 449 column_info_new (SummaryField *field,
 450                  const gchar *folderid,
 451                  const gchar *column_suffix,
 452                  const gchar *column_type,
 453                  const gchar *column_extra,
 454                  const gchar *idx_prefix)
 455 {
 456         ColumnInfo *info;
 457
 458         info = g_slice_new0 (ColumnInfo);
 459         info->type = column_type;
 460         info->extra = column_extra;
 461
 462         if (!info->type) {
 463                 if (field->type == G_TYPE_STRING)
 464                         info->type = "TEXT";
 465                 else if (field->type == G_TYPE_BOOLEAN)
 466                         info->type = "INTEGER";
 467                 else if (field->type == E_TYPE_CONTACT_ATTR_LIST)
 468                         info->type = "TEXT";
 469                 else
 470                         g_warn_if_reached ();
 471         }
 472
 473         if (field->type == E_TYPE_CONTACT_ATTR_LIST)
 474                 /* Attribute lists are on their own table  */
 475                 info->name = g_strconcat (
 476                         "value",
 477                         column_suffix ? "_" : NULL,
 478                         column_suffix,
 479                         NULL);
 480         else
 481                 /* Regular fields are named by their 'dbname' */
 482                 info->name = g_strconcat (
 483                         field->dbname,
 484                         column_suffix ? "_" : NULL,
 485                         column_suffix,
 486                         NULL);
 487
 488         if (idx_prefix)
 489                 info->index = g_strconcat (
 490                         idx_prefix,
 491                         "_", field->dbname,
 492                         "_", folderid,
 493                         NULL);
 494
 495         return info;
 496 }
 497
 498 static void
 499 column_info_free (ColumnInfo *info)
 500 {
 501         if (info) {
 502                 g_free (info->name);
 503                 g_free (info->index);
 504                 g_slice_free (ColumnInfo, info);
 505         }
 506 }
 507
 508 static gint
 509 summary_field_array_index (GArray *array,
 510                            EContactField field)
 511 {
 512         gint i;
 513
 514         for (i = 0; i < array->len; i++) {
 515                 SummaryField *iter = &g_array_index (array, SummaryField, i);
 516                 if (field == iter->field_id)
 517                         return i;
 518         }
 519
 520         return -1;
 521 }
 522
 523 static SummaryField *
 524 summary_field_append (GArray *array,
 525                       const gchar *folderid,
 526                       EContactField field_id,
 527                       GError **error)
 528 {
 529         const gchar *dbname = NULL;
 530         GType        type = G_TYPE_INVALID;
 531         gint         idx;
 532         SummaryField new_field = { 0, };
 533
 534         if (field_id < 1 || field_id >= E_CONTACT_FIELD_LAST) {
 535                 EBSQL_SET_ERROR (
 536                         error, E_BOOK_SQLITE_ERROR_UNSUPPORTED_FIELD,
 537                         _("Unsupported contact field '%d' specified in summary"),
 538                         field_id);
 539                 return NULL;
 540         }
 541
 542         /* Avoid including the same field twice in the summary */
 543         idx = summary_field_array_index (array, field_id);
 544         if (idx >= 0)
 545                 return &g_array_index (array, SummaryField, idx);
 546
 547         /* Resolve some exceptions, we store these
 548          * specific contact fields with different names
 549          * than those found in the EContactField table
 550          */
 551         switch (field_id) {
 552         case E_CONTACT_UID:
 553                 dbname = "uid";
 554                 break;
 555         case E_CONTACT_IS_LIST:
 556                 dbname = "is_list";
 557                 break;
 558         default:
 559                 dbname = e_contact_field_name (field_id);
 560                 break;
 561         }
 562
 563         type = e_contact_field_type (field_id);
 564
 565         if (type != G_TYPE_STRING &&
 566             type != G_TYPE_BOOLEAN &&
 567             type != E_TYPE_CONTACT_ATTR_LIST) {
 568                 EBSQL_SET_ERROR (
 569                         error, E_BOOK_SQLITE_ERROR_UNSUPPORTED_FIELD,
 570                         _("Contact field '%s' of type '%s' specified in summary, "
 571                         "but only boolean, string and string list field types are supported"),
 572                         e_contact_pretty_name (field_id), g_type_name (type));
 573                 return NULL;
 574         }
 575
 576         if (type == E_TYPE_CONTACT_ATTR_LIST) {
 577                 new_field.aux_table = g_strconcat (folderid, "_", dbname, "_list", NULL);
 578                 new_field.aux_table_symbolic = g_strconcat (dbname, "_list", NULL);
 579         }
 580
 581         new_field.field_id = field_id;
 582         new_field.dbname = dbname;
 583         new_field.type = type;
 584         new_field.index = 0;
 585         g_array_append_val (array, new_field);
 586
 587         return &g_array_index (array, SummaryField, array->len - 1);
 588 }
 589
 590 static gboolean
 591 summary_field_remove (GArray *array,
 592                       EContactField field)
 593 {
 594         gint idx;
 595
 596         idx = summary_field_array_index (array, field);
 597         if (idx < 0)
 598                 return FALSE;
 599
 600         g_array_remove_index_fast (array, idx);
 601         return TRUE;
 602 }
 603
 604 static void
 605 summary_fields_add_indexes (GArray *array,
 606                             EContactField *indexes,
 607                             EBookIndexType *index_types,
 608                             gint n_indexes)
 609 {
 610         gint i, j;
 611
 612         for (i = 0; i < array->len; i++) {
 613                 SummaryField *sfield = &g_array_index (array, SummaryField, i);
 614
 615                 for (j = 0; j < n_indexes; j++) {
 616                         if (sfield->field_id == indexes[j])
 617                                 sfield->index |= (1 << index_types[j]);
 618
 619                 }
 620         }
 621 }
 622
 623 static inline gint
 624 summary_field_get_index (EBookSqlite *ebsql,
 625                          EContactField field_id)
 626 {
 627         gint i;
 628
 629         for (i = 0; i < ebsql->priv->n_summary_fields; i++) {
 630                 if (ebsql->priv->summary_fields[i].field_id == field_id)
 631                         return i;
 632         }
 633
 634         return -1;
 635 }
 636
 637 static inline SummaryField *
 638 summary_field_get (EBookSqlite *ebsql,
 639                    EContactField field_id)
 640 {
 641         gint index;
 642
 643         index = summary_field_get_index (ebsql, field_id);
 644         if (index >= 0)
 645                 return &(ebsql->priv->summary_fields[index]);
 646
 647         return NULL;
 648 }
 649
 650 static GSList *
 651 summary_field_list_columns (SummaryField *field,
 652                             const gchar *folderid)
 653 {
 654         GSList *columns = NULL;
 655         ColumnInfo *info;
 656
 657         /* Doesn't hurt to verify a bit more here, this shouldn't happen though */
 658         g_return_val_if_fail (
 659                 field->type == G_TYPE_STRING ||
 660                 field->type == G_TYPE_BOOLEAN ||
 661                 field->type == E_TYPE_CONTACT_ATTR_LIST,
 662                 NULL);
 663
 664         /* Normal / default column */
 665         info = column_info_new (
 666                 field, folderid, NULL, NULL,
 667                 (field->field_id == E_CONTACT_UID) ? "PRIMARY KEY" : NULL,
 668                 (field->index & INDEX_FLAG (PREFIX)) != 0 ? "INDEX" : NULL);
 669         columns = g_slist_prepend (columns, info);
 670
 671         /* Localized column, for storing sort keys */
 672         if (field->type == G_TYPE_STRING && (field->index & INDEX_FLAG (SORT_KEY))) {
 673                 info = column_info_new (field, folderid, EBSQL_SUFFIX_SORT_KEY, "TEXT", NULL, "SINDEX");
 674                 columns = g_slist_prepend (columns, info);
 675         }
 676
 677         /* Suffix match column */
 678         if (field->type != G_TYPE_BOOLEAN && (field->index & INDEX_FLAG (SUFFIX)) != 0) {
 679                 info = column_info_new (field, folderid, EBSQL_SUFFIX_REVERSE, "TEXT", NULL, "RINDEX");
 680                 columns = g_slist_prepend (columns, info);
 681         }
 682
 683         /* Phone match columns */
 684         if (field->type != G_TYPE_BOOLEAN && (field->index & INDEX_FLAG (PHONE)) != 0) {
 685
 686                 /* One indexed column for storing the national number */
 687                 info = column_info_new (field, folderid, EBSQL_SUFFIX_PHONE, "TEXT", NULL, "PINDEX");
 688                 columns = g_slist_prepend (columns, info);
 689
 690                 /* One integer column for storing the country code */
 691                 info = column_info_new (field, folderid, EBSQL_SUFFIX_COUNTRY, "INTEGER", "DEFAULT 0", NULL);
 692                 columns = g_slist_prepend (columns, info);
 693         }
 694
 695         /* Transliterated value column */
 696         if (field->type != G_TYPE_BOOLEAN && (field->index & INDEX_FLAG (TRANSLIT)) != 0) {
 697                 info    = column_info_new (field, folderid, EBSQL_SUFFIX_TRANSLIT, "TEXT", NULL, "TINDEX");
 698                 columns = g_slist_prepend (columns, info);
 699         }
 700
 701         return g_slist_reverse (columns);
 702 }
 703
 704 static void
 705 summary_fields_array_free (SummaryField *fields,
 706                            gint n_fields)
 707 {
 708         gint i;
 709
 710         for (i = 0; i < n_fields; i++) {
 711                 g_free (fields[i].aux_table);
 712                 g_free (fields[i].aux_table_symbolic);
 713         }
 714
 715         g_free (fields);
 716 }
 717
 718 /******************************************************
 719  *        Sharing EBookSqlite instances        *
 720  ******************************************************/
 721 static GHashTable *db_connections = NULL;
 722 static GMutex dbcon_lock;
 723
 724 static EBookSqlite *
 725 ebsql_ref_from_hash (const gchar *path)
 726 {
 727         EBookSqlite *ebsql = NULL;
 728
 729         if (db_connections != NULL) {
 730                 ebsql = g_hash_table_lookup (db_connections, path);
 731         }
 732
 733         if (ebsql) {
 734                 EBSQL_NOTE (REF_COUNTS, g_printerr ("EBookSqlite ref count increased from hash table reference\n"));
 735                 g_object_ref (ebsql);
 736         }
 737
 738         return NULL;
 739 }
 740
 741 static void
 742 ebsql_register_to_hash (EBookSqlite *ebsql,
 743                         const gchar *path)
 744 {
 745         if (db_connections == NULL)
 746                 db_connections = g_hash_table_new_full (
 747                         (GHashFunc) g_str_hash,
 748                         (GEqualFunc) g_str_equal,
 749                         (GDestroyNotify) g_free,
 750                         (GDestroyNotify) NULL);
 751         g_hash_table_insert (db_connections, g_strdup (path), ebsql);
 752 }
 753
 754 static void
 755 ebsql_unregister_from_hash (EBookSqlite *ebsql)
 756 {
 757         EBookSqlitePrivate *priv = ebsql->priv;
 758
 759         EBSQL_LOCK_MUTEX (&dbcon_lock);
 760         if (db_connections != NULL) {
 761                 if (priv->path != NULL) {
 762                         g_hash_table_remove (db_connections, priv->path);
 763
 764                         if (g_hash_table_size (db_connections) == 0) {
 765                                 g_hash_table_destroy (db_connections);
 766                                 db_connections = NULL;
 767                         }
 768
 769                 }
 770         }
 771         EBSQL_UNLOCK_MUTEX (&dbcon_lock);
 772 }
 773
 774 /************************************************************
 775  *                SQLite helper functions                   *
 776  ************************************************************/
 777
 778 /* For EBSQL_DEBUG_EXPLAIN */
 779 static gint
 780 ebsql_debug_query_plan_cb (gpointer ref,
 781                            gint n_cols,
 782                            gchar **cols,
 783                            gchar **name)
 784 {
 785         gint i;
 786
 787         for (i = 0; i < n_cols; i++) {
 788                 if (strcmp (name[i], "detail") == 0) {
 789                         g_printerr ("  PLAN: %s\n", cols[i]);
 790                         break;
 791                 }
 792         }
 793
 794         return 0;
 795 }
 796
 797 /* Collect a GList of column names in the main summary table */
 798 static gint
 799 get_columns_cb (gpointer ref,
 800                 gint col,
 801                 gchar **cols,
 802                 gchar **name)
 803 {
 804         GSList **columns = (GSList **) ref;
 805         gint i;
 806
 807         for (i = 0; i < col; i++) {
 808                 if (strcmp (name[i], "name") == 0) {
 809
 810                         /* Keep comparing for the legacy 'bdata' column */
 811                         if (strcmp (cols[i], "vcard") != 0 &&
 812                             strcmp (cols[i], "bdata") != 0) {
 813                                 gchar *column = g_strdup (cols[i]);
 814
 815                                 *columns = g_slist_prepend (*columns, column);
 816                         }
 817                         break;
 818                 }
 819         }
 820         return 0;
 821 }
 822
 823 /* Collect the first string result */
 824 static gint
 825 get_string_cb (gpointer ref,
 826                gint col,
 827                gchar **cols,
 828                gchar **name)
 829 {
 830         gchar **ret = ref;
 831
 832         *ret = g_strdup (cols [0]);
 833
 834         return 0;
 835 }
 836
 837 /* Collect the first integer result */
 838 static gint
 839 get_int_cb (gpointer ref,
 840             gint col,
 841             gchar **cols,
 842             gchar **name)
 843 {
 844         gint *ret = ref;
 845
 846         *ret = cols [0] ? g_ascii_strtoll (cols[0], NULL, 10) : 0;
 847
 848         return 0;
 849 }
 850
 851 /* Collect the result of a SELECT count(*) statement */
 852 static gint
 853 get_count_cb (gpointer ref,
 854               gint n_cols,
 855               gchar **cols,
 856               gchar **name)
 857 {
 858         gint64 count = 0;
 859         gint *ret = ref;
 860         gint i;
 861
 862         for (i = 0; i < n_cols; i++) {
 863                 if (name[i] && strncmp (name[i], "count", 5) == 0) {
 864                         count = g_ascii_strtoll (cols[i], NULL, 10);
 865
 866                         break;
 867                 }
 868         }
 869
 870         *ret = count;
 871
 872         return 0;
 873 }
 874
 875 /* Report if there was at least one result */
 876 static gint
 877 get_exists_cb (gpointer ref,
 878                gint col,
 879                gchar **cols,
 880                gchar **name)
 881 {
 882         gboolean *exists = ref;
 883
 884         *exists = TRUE;
 885
 886         return 0;
 887 }
 888
 889 static EbSqlSearchData *
 890 search_data_from_results (gint ncol,
 891                           gchar **cols,
 892                           gchar **names)
 893 {
 894         EbSqlSearchData *data = g_slice_new0 (EbSqlSearchData);
 895         gint i;
 896
 897         for (i = 0; i < ncol; i++) {
 898
 899                 if (!names[i] || !cols[i])
 900                         continue;
 901
 902                 /* These come through differently depending on the configuration,
 903                  * search within text is good enough
 904                  */
 905                 if (!g_ascii_strcasecmp (names[i], "uid")) {
 906                         data->uid = g_strdup (cols[i]);
 907                 } else if (!g_ascii_strcasecmp (names[i], "vcard") ||
 908                            !g_ascii_strncasecmp (names[i], "fetch_vcard", 11)) {
 909                         data->vcard = g_strdup (cols[i]);
 910                 } else if (!g_ascii_strcasecmp (names[i], "bdata")) {
 911                         data->extra = g_strdup (cols[i]);
 912                 }
 913         }
 914
 915         return data;
 916 }
 917
 918 static gint
 919 collect_full_results_cb (gpointer ref,
 920                          gint ncol,
 921                          gchar **cols,
 922                          gchar **names)
 923 {
 924         EbSqlSearchData *data;
 925         GSList **vcard_data = ref;
 926
 927         data = search_data_from_results (ncol, cols, names);
 928
 929         *vcard_data = g_slist_prepend (*vcard_data, data);
 930
 931         return 0;
 932 }
 933
 934 static gint
 935 collect_uid_results_cb (gpointer ref,
 936                         gint ncol,
 937                         gchar **cols,
 938                         gchar **names)
 939 {
 940         GSList **uids = ref;
 941
 942         if (cols[0])
 943                 *uids = g_slist_prepend (*uids, g_strdup (cols [0]));
 944
 945         return 0;
 946 }
 947
 948 static gint
 949 collect_lean_results_cb (gpointer ref,
 950                          gint ncol,
 951                          gchar **cols,
 952                          gchar **names)
 953 {
 954         GSList **vcard_data = ref;
 955         EbSqlSearchData *search_data = g_slice_new0 (EbSqlSearchData);
 956         EContact *contact = e_contact_new ();
 957         gchar *vcard;
 958         gint i;
 959
 960         /* parse through cols, this will be useful if the api starts supporting field restrictions */
 961         for (i = 0; i < ncol; i++) {
 962                 if (!names[i] || !cols[i])
 963                         continue;
 964
 965                 /* Only UID & REV can be used to create contacts from the summary columns */
 966                 if (!g_ascii_strcasecmp (names[i], "uid")) {
 967                         e_contact_set (contact, E_CONTACT_UID, cols[i]);
 968                         search_data->uid = g_strdup (cols[i]);
 969                 } else if (!g_ascii_strcasecmp (names[i], "Rev")) {
 970                         e_contact_set (contact, E_CONTACT_REV, cols[i]);
 971                 } else if (!g_ascii_strcasecmp (names[i], "bdata")) {
 972                         search_data->extra = g_strdup (cols[i]);
 973                 }
 974         }
 975
 976         vcard = e_vcard_to_string (E_VCARD (contact), EVC_FORMAT_VCARD_30);
 977         search_data->vcard = vcard;
 978         *vcard_data = g_slist_prepend (*vcard_data, search_data);
 979
 980         g_object_unref (contact);
 981         return 0;
 982 }
 983
 984 static void
 985 ebsql_string_append_vprintf (GString *string,
 986                              const gchar *fmt,
 987                              va_list args)
 988 {
 989         gchar *stmt;
 990
 991         /* Unfortunately, sqlite3_vsnprintf() doesnt tell us
 992          * how many bytes it would have needed if it doesnt fit
 993          * into the target buffer, so we can't avoid this
 994          * really disgusting memory dup.
 995          */
 996         stmt = sqlite3_vmprintf (fmt, args);
 997         g_string_append (string, stmt);
 998         sqlite3_free (stmt);
 999 }
1000
1001 static void
1002 ebsql_string_append_printf (GString *string,
1003                             const gchar *fmt,
1004                             ...)
1005 {
1006         va_list args;
1007
1008         va_start (args, fmt);
1009         ebsql_string_append_vprintf (string, fmt, args);
1010         va_end (args);
1011 }
1012
1013 /* Appends an identifier suitable to identify the
1014  * column to test in the context of a query.
1015  *
1016  * The suffix is for special indexed columns (such as
1017  * reverse values, sort keys, phone numbers, etc).
1018  */
1019 static void
1020 ebsql_string_append_column (GString *string,
1021                             SummaryField *field,
1022                             const gchar *suffix)
1023 {
1024         if (field->aux_table) {
1025                 g_string_append (string, field->aux_table_symbolic);
1026                 g_string_append (string, ".value");
1027         } else {
1028                 g_string_append (string, "summary.");
1029                 g_string_append (string, field->dbname);
1030         }
1031
1032         if (suffix) {
1033                 g_string_append_c (string, '_');
1034                 g_string_append (string, suffix);
1035         }
1036 }
1037
1038 static gboolean
1039 ebsql_exec_vprintf (EBookSqlite *ebsql,
1040                     const gchar *fmt,
1041                     EbSqlRowFunc callback,
1042                     gpointer data,
1043                     GCancellable *cancellable,
1044                     GError **error,
1045                     va_list args)
1046 {
1047         gboolean success;
1048         gchar *stmt;
1049
1050         stmt = sqlite3_vmprintf (fmt, args);
1051         success = ebsql_exec (ebsql, stmt, callback, data, cancellable, error);
1052         sqlite3_free (stmt);
1053
1054         return success;
1055 }
1056
1057 static gboolean
1058 ebsql_exec_printf (EBookSqlite *ebsql,
1059                    const gchar *fmt,
1060                    EbSqlRowFunc callback,
1061                    gpointer data,
1062                    GCancellable *cancellable,
1063                    GError **error,
1064                    ...)
1065 {
1066         gboolean success;
1067         va_list args;
1068
1069         va_start (args, error);
1070         success = ebsql_exec_vprintf (ebsql, fmt, callback, data, cancellable, error, args);
1071         va_end (args);
1072
1073         return success;
1074 }
1075
1076 static inline void
1077 ebsql_exec_maybe_debug (EBookSqlite *ebsql,
1078                         const gchar *stmt)
1079 {
1080         if (ebsql_debug_flags & EBSQL_DEBUG_EXPLAIN &&
1081             strncmp (stmt, "SELECT", 6) == 0) {
1082                     g_printerr ("EXPLAIN BEGIN\n  STMT: %s\n", stmt);
1083                     ebsql_exec_printf (ebsql, "EXPLAIN QUERY PLAN %s",
1084                                        ebsql_debug_query_plan_cb,
1085                                        NULL, NULL, NULL, stmt);
1086                     g_printerr ("EXPLAIN END\n");
1087         } else {
1088                 EBSQL_NOTE (STATEMENTS, g_printerr ("STMT: %s\n", stmt));
1089         }
1090 }
1091
1092 static gboolean
1093 ebsql_exec (EBookSqlite *ebsql,
1094             const gchar *stmt,
1095             EbSqlRowFunc callback,
1096             gpointer data,
1097             GCancellable *cancellable,
1098             GError **error)
1099 {
1100         gboolean had_cancel;
1101         gchar *errmsg = NULL;
1102         gint ret = -1;
1103
1104         /* Debug output for statements and query plans */
1105         ebsql_exec_maybe_debug (ebsql, stmt);
1106
1107         /* Just convenience to set the cancellable on an execution
1108          * without a transaction, error checking on the cancellable
1109          * is done with EBSQL_LOCK_OR_RETURN()
1110          */
1111         if (ebsql->priv->cancel) {
1112                 had_cancel = TRUE;
1113         } else {
1114                 ebsql->priv->cancel = cancellable;
1115                 had_cancel = TRUE;
1116         }
1117
1118         ret = sqlite3_exec (ebsql->priv->db, stmt, callback, data, &errmsg);
1119
1120         while (ret == SQLITE_BUSY || ret == SQLITE_LOCKED || ret == -1) {
1121                 if (errmsg) {
1122                         sqlite3_free (errmsg);
1123                         errmsg = NULL;
1124                 }
1125                 g_thread_yield ();
1126                 ret = sqlite3_exec (ebsql->priv->db, stmt, callback, data, &errmsg);
1127         }
1128
1129         if (!had_cancel)
1130                 ebsql->priv->cancel = NULL;
1131
1132         if (ret != SQLITE_OK) {
1133                 EBSQL_SET_ERROR_FROM_SQLITE (error, ret, errmsg);
1134                 sqlite3_free (errmsg);
1135                 return FALSE;
1136         }
1137
1138         if (errmsg)
1139                 sqlite3_free (errmsg);
1140
1141         return TRUE;
1142 }
1143
1144 static gboolean
1145 ebsql_start_transaction (EBookSqlite *ebsql,
1146                          EbSqlLockType lock_type,
1147                          GCancellable *cancel,
1148                          GError **error)
1149 {
1150         gboolean success = TRUE;
1151
1152         g_return_val_if_fail (ebsql != NULL, FALSE);
1153         g_return_val_if_fail (ebsql->priv != NULL, FALSE);
1154         g_return_val_if_fail (ebsql->priv->db != NULL, FALSE);
1155
1156         ebsql->priv->in_transaction++;
1157         g_return_val_if_fail (ebsql->priv->in_transaction > 0, FALSE);
1158
1159         if (ebsql->priv->in_transaction == 1) {
1160
1161                 /* No cancellable should be set at transaction start time */
1162                 if (ebsql->priv->cancel) {
1163                         g_warning (
1164                                 "Starting a transaction with a cancellable already set. "
1165                                 "Clearing previously set cancellable");
1166                         g_clear_object (&ebsql->priv->cancel);
1167                 }
1168
1169                 /* Hold on to the cancel object until the end of the transaction */
1170                 if (cancel)
1171                         ebsql->priv->cancel = g_object_ref (cancel);
1172
1173                 /* It's important to make the distinction between a
1174                  * transaction which will read or one which will write.
1175                  *
1176                  * While it's not well documented, when receiving the SQLITE_BUSY
1177                  * error status, one can only safely retry at the beginning of
1178                  * the transaction.
1179                  *
1180                  * If a transaction is 'upgraded' to require a writer lock
1181                  * half way through the transaction and SQLITE_BUSY is returned,
1182                  * the whole transaction would need to be retried from the beginning.
1183                  */
1184                 ebsql->priv->lock_type = lock_type;
1185
1186                 switch (lock_type) {
1187                 case EBSQL_LOCK_READ:
1188                         success = ebsql_exec (ebsql, "BEGIN", NULL, NULL, NULL, error);
1189                         break;
1190                 case EBSQL_LOCK_WRITE:
1191                         success = ebsql_exec (ebsql, "BEGIN IMMEDIATE", NULL, NULL, NULL, error);
1192                         break;
1193                 }
1194
1195         } else {
1196
1197                 /* Warn about cases where where a read transaction might be upgraded */
1198                 if (lock_type == EBSQL_LOCK_WRITE && ebsql->priv->lock_type == EBSQL_LOCK_READ)
1199                         g_warning (
1200                                 "A nested transaction wants to write, "
1201                                 "but the outermost transaction was started "
1202                                 "without a writer lock.");
1203         }
1204
1205         return success;
1206 }
1207
1208 static gboolean
1209 ebsql_commit_transaction (EBookSqlite *ebsql,
1210                           GError **error)
1211 {
1212         gboolean success = TRUE;
1213
1214         g_return_val_if_fail (ebsql != NULL, FALSE);
1215         g_return_val_if_fail (ebsql->priv != NULL, FALSE);
1216         g_return_val_if_fail (ebsql->priv->db != NULL, FALSE);
1217
1218         g_return_val_if_fail (ebsql->priv->in_transaction > 0, FALSE);
1219
1220         ebsql->priv->in_transaction--;
1221
1222         if (ebsql->priv->in_transaction == 0) {
1223                 success = ebsql_exec (ebsql, "COMMIT", NULL, NULL, NULL, error);
1224
1225                 /* The outermost transaction is finished, let's release
1226                  * our reference to the user's cancel object here */
1227                 g_clear_object (&ebsql->priv->cancel);
1228         }
1229
1230         return success;
1231 }
1232
1233 static gboolean
1234 ebsql_rollback_transaction (EBookSqlite *ebsql,
1235                             GError **error)
1236 {
1237         gboolean success = TRUE;
1238
1239         g_return_val_if_fail (ebsql != NULL, FALSE);
1240         g_return_val_if_fail (ebsql->priv != NULL, FALSE);
1241         g_return_val_if_fail (ebsql->priv->db != NULL, FALSE);
1242
1243         g_return_val_if_fail (ebsql->priv->in_transaction > 0, FALSE);
1244
1245         ebsql->priv->in_transaction--;
1246
1247         if (ebsql->priv->in_transaction == 0) {
1248                 success = ebsql_exec (ebsql, "ROLLBACK", NULL, NULL, NULL, error);
1249
1250                 /* The outermost transaction is finished, let's release
1251                  * our reference to the user's cancel object here */
1252                 g_clear_object (&ebsql->priv->cancel);
1253         }
1254         return success;
1255 }
1256
1257 static sqlite3_stmt *
1258 ebsql_prepare_statement (EBookSqlite *ebsql,
1259                          const gchar *stmt_str,
1260                          GError **error)
1261 {
1262         sqlite3_stmt *stmt;
1263         const gchar *stmt_tail = NULL;
1264         gint ret;
1265
1266         ret = sqlite3_prepare_v2 (ebsql->priv->db, stmt_str, strlen (stmt_str), &stmt, &stmt_tail);
1267
1268         if (ret != SQLITE_OK) {
1269                 const gchar *errmsg = sqlite3_errmsg (ebsql->priv->db);
1270                 EBSQL_SET_ERROR_LITERAL (
1271                         error,
1272                         E_BOOK_SQLITE_ERROR_ENGINE,
1273                         errmsg);
1274         } else if (stmt == NULL) {
1275                 EBSQL_SET_ERROR_LITERAL (
1276                         error,
1277                         E_BOOK_SQLITE_ERROR_ENGINE,
1278                         "Unknown error preparing SQL statement");
1279         }
1280
1281         if (stmt_tail && stmt_tail[0])
1282                 g_warning ("Part of this statement was not parsed: %s", stmt_tail);
1283
1284         return stmt;
1285 }
1286
1287 /* Convenience for running statements. After successfully
1288  * binding all parameters, just return with this.
1289  */
1290 static gboolean
1291 ebsql_complete_statement (EBookSqlite *ebsql,
1292                           sqlite3_stmt *stmt,
1293                           gint ret,
1294                           GError **error)
1295 {
1296         if (ret == SQLITE_OK)
1297                 ret = sqlite3_step (stmt);
1298
1299         if (ret != SQLITE_OK && ret != SQLITE_DONE) {
1300                 const gchar *errmsg = sqlite3_errmsg (ebsql->priv->db);
1301                 EBSQL_SET_ERROR_FROM_SQLITE (error, ret, errmsg);
1302         }
1303
1304         /* Reset / Clear at the end, regardless of error state */
1305         sqlite3_reset (stmt);
1306         sqlite3_clear_bindings (stmt);
1307
1308         return (ret == SQLITE_OK || ret == SQLITE_DONE);
1309 }
1310
1311 /******************************************************
1312  *       Functions installed into the SQLite          *
1313  ******************************************************/
1314
1315 /* Implementation for REGEXP keyword */
1316 static void
1317 ebsql_regexp (sqlite3_context *context,
1318               gint argc,
1319               sqlite3_value **argv)
1320 {
1321         GRegex *regex;
1322         const gchar *expression;
1323         const gchar *text;
1324
1325         /* Reuse the same GRegex for all REGEXP queries with the same expression */
1326         regex = sqlite3_get_auxdata (context, 0);
1327         if (!regex) {
1328                 GError *error = NULL;
1329
1330                 expression = (const gchar *) sqlite3_value_text (argv[0]);
1331
1332                 regex = g_regex_new (expression, 0, 0, &error);
1333
1334                 if (!regex) {
1335                         sqlite3_result_error (
1336                                 context,
1337                                 error ? error->message :
1338                                 _("Error parsing regular expression"),
1339                                 -1);
1340                         g_clear_error (&error);
1341                         return;
1342                 }
1343
1344                 /* SQLite will take care of freeing the GRegex when we're done with the query */
1345                 sqlite3_set_auxdata (context, 0, regex, (GDestroyNotify) g_regex_unref);
1346         }
1347
1348         /* Now perform the comparison */
1349         text = (const gchar *) sqlite3_value_text (argv[1]);
1350         if (text != NULL) {
1351                 gboolean match;
1352
1353                 match = g_regex_match (regex, text, 0, NULL);
1354                 sqlite3_result_int (context, match ? 1 : 0);
1355         }
1356 }
1357
1358 /* Implementation of EBSQL_FUNC_COMPARE_VCARD (fallback for non-summary queries) */
1359 static void
1360 ebsql_compare_vcard (sqlite3_context *context,
1361                      gint argc,
1362                      sqlite3_value **argv)
1363 {
1364         EBookBackendSExp *sexp = NULL;
1365         const gchar *text;
1366         const gchar *vcard;
1367
1368         /* Reuse the same sexp for all queries with the same search expression */
1369         sexp = sqlite3_get_auxdata (context, 0);
1370         if (!sexp) {
1371
1372                 /* The first argument will be reused for many rows */
1373                 text = (const gchar *) sqlite3_value_text (argv[0]);
1374                 if (text) {
1375                         sexp = e_book_backend_sexp_new (text);
1376                         sqlite3_set_auxdata (
1377                                 context, 0,
1378                                 sexp,
1379                                 g_object_unref);
1380                 }
1381
1382                 /* This shouldn't happen, catch invalid sexp in preflight */
1383                 if (!sexp) {
1384                         sqlite3_result_int (context, 0);
1385                         return;
1386                 }
1387
1388         }
1389
1390         /* Reuse the same vcard as much as possible (it can be referred to more than
1391          * once in the query, so it can be reused for multiple comparisons on the same row)
1392          *
1393          * This may look extensive, but as the vcard might be resolved by calling a
1394          * EbSqlVCardCallback, it's important to reuse this string as much as possible.
1395          *
1396          * See ebsql_fetch_vcard() for details.
1397          */
1398         vcard = sqlite3_get_auxdata (context, 1);
1399         if (!vcard) {
1400                 vcard = (const gchar *) sqlite3_value_text (argv[1]);
1401
1402                 if (vcard)
1403                         sqlite3_set_auxdata (context, 1, g_strdup (vcard), g_free);
1404         }
1405
1406         /* A NULL vcard can never match */
1407         if (vcard == NULL || *vcard == '\0') {
1408                 sqlite3_result_int (context, 0);
1409                 return;
1410         }
1411
1412         /* Compare this vcard */
1413         if (e_book_backend_sexp_match_vcard (sexp, vcard))
1414                 sqlite3_result_int (context, 1);
1415         else
1416                 sqlite3_result_int (context, 0);
1417 }
1418
1419 static void
1420 ebsql_eqphone (sqlite3_context *context,
1421                gint argc,
1422                sqlite3_value **argv,
1423                EPhoneNumberMatch requested_match)
1424 {
1425         EBookSqlite *ebsql = sqlite3_user_data (context);
1426         EPhoneNumber *input_phone = NULL, *row_phone = NULL;
1427         EPhoneNumberMatch match = E_PHONE_NUMBER_MATCH_NONE;
1428         const gchar *text;
1429
1430         /* Reuse the same phone number for all queries with the same phone number argument */
1431         input_phone = sqlite3_get_auxdata (context, 0);
1432         if (!input_phone) {
1433
1434                 /* The first argument will be reused for many rows */
1435                 text = (const gchar *) sqlite3_value_text (argv[0]);
1436                 if (text) {
1437
1438                         /* Ignore errors, they are fine for phone numbers */
1439                         input_phone = e_phone_number_from_string (text, ebsql->priv->region_code, NULL);
1440
1441                         /* SQLite will take care of freeing the EPhoneNumber when we're done with the expression */
1442                         if (input_phone)
1443                                 sqlite3_set_auxdata (
1444                                         context, 0,
1445                                         input_phone,
1446                                         (GDestroyNotify) e_phone_number_free);
1447                 }
1448         }
1449
1450         /* This shouldn't happen, as we catch invalid phone number queries in preflight
1451          */
1452         if (!input_phone) {
1453                 sqlite3_result_int (context, 0);
1454                 return;
1455         }
1456
1457         /* Parse the phone number for this row */
1458         text = (const gchar *) sqlite3_value_text (argv[1]);
1459         if (text != NULL) {
1460                 row_phone = e_phone_number_from_string (text, ebsql->priv->region_code, NULL);
1461
1462                 /* And perform the comparison */
1463                 if (row_phone) {
1464                         match = e_phone_number_compare (input_phone, row_phone);
1465
1466                         e_phone_number_free (row_phone);
1467                 }
1468         }
1469
1470         /* Now report the result */
1471         if (match != E_PHONE_NUMBER_MATCH_NONE &&
1472             match <= requested_match)
1473                 sqlite3_result_int (context, 1);
1474         else
1475                 sqlite3_result_int (context, 0);
1476 }
1477
1478 /* Exact phone number match function: EBSQL_FUNC_EQPHONE_EXACT */
1479 static void
1480 ebsql_eqphone_exact (sqlite3_context *context,
1481                      gint argc,
1482                      sqlite3_value **argv)
1483 {
1484         ebsql_eqphone (context, argc, argv, E_PHONE_NUMBER_MATCH_EXACT);
1485 }
1486
1487 /* National phone number match function: EBSQL_FUNC_EQPHONE_NATIONAL */
1488 static void
1489 ebsql_eqphone_national (sqlite3_context *context,
1490                         gint argc,
1491                         sqlite3_value **argv)
1492 {
1493         ebsql_eqphone (context, argc, argv, E_PHONE_NUMBER_MATCH_NATIONAL);
1494 }
1495
1496 /* Short phone number match function: EBSQL_FUNC_EQPHONE_SHORT */
1497 static void
1498 ebsql_eqphone_short (sqlite3_context *context,
1499                      gint argc,
1500                      sqlite3_value **argv)
1501 {
1502         ebsql_eqphone (context, argc, argv, E_PHONE_NUMBER_MATCH_SHORT);
1503 }
1504
1505 /* Implementation of EBSQL_FUNC_FETCH_VCARD (fallback for shallow addressbooks) */
1506 static void
1507 ebsql_fetch_vcard (sqlite3_context *context,
1508                    gint argc,
1509                    sqlite3_value **argv)
1510 {
1511         EBookSqlite *ebsql = sqlite3_user_data (context);
1512         const gchar *uid;
1513         const gchar *extra;
1514         gchar *vcard = NULL;
1515
1516         uid = (const gchar *) sqlite3_value_text (argv[0]);
1517         extra = (const gchar *) sqlite3_value_text (argv[1]);
1518
1519         /* Call our delegate to generate the vcard */
1520         if (ebsql->priv->vcard_callback)
1521                 vcard = ebsql->priv->vcard_callback (
1522                         uid, extra, ebsql->priv->user_data);
1523
1524         EBSQL_NOTE (
1525                 FETCH_VCARD,
1526                 g_printerr (
1527                         "fetch_vcard (%s, %s) %s",
1528                         uid, extra, vcard ? "Got VCard" : "No VCard"));
1529
1530         sqlite3_result_text (context, vcard, -1, g_free);
1531 }
1532
1533 typedef struct {
1534         const gchar     *name;
1535         EbSqlCustomFunc  func;
1536         gint             arguments;
1537 } EbSqlCustomFuncTab;
1538
1539 static EbSqlCustomFuncTab ebsql_custom_functions[] = {
1540         { "regexp",                    ebsql_regexp,           2 }, /* regexp (expression, column_data) */
1541         { EBSQL_FUNC_COMPARE_VCARD,    ebsql_compare_vcard,    2 }, /* compare_vcard (sexp, vcard) */
1542         { EBSQL_FUNC_FETCH_VCARD,      ebsql_fetch_vcard,      2 }, /* fetch_vcard (uid, extra) */
1543         { EBSQL_FUNC_EQPHONE_EXACT,    ebsql_eqphone_exact,    2 }, /* eqphone_exact (search_input, column_data) */
1544         { EBSQL_FUNC_EQPHONE_NATIONAL, ebsql_eqphone_national, 2 }, /* eqphone_national (search_input, column_data) */
1545         { EBSQL_FUNC_EQPHONE_SHORT,    ebsql_eqphone_short,    2 }, /* eqphone_national (search_input, column_data) */
1546 };
1547
1548 /******************************************************
1549  *            Fallback Collation Sequences            *
1550  ******************************************************
1551  *
1552  * The fallback simply compares vcards, vcards which have been
1553  * stored on the cursor will have a preencoded key (these
1554  * utilities encode & decode that key).
1555  */
1556 static gchar *
1557 ebsql_encode_vcard_sort_key (const gchar *sort_key)
1558 {
1559         EVCard *vcard = e_vcard_new ();
1560         gchar *base64;
1561         gchar *encoded;
1562
1563         /* Encode this otherwise e-vcard messes it up */
1564         base64 = g_base64_encode ((const guchar *) sort_key, strlen (sort_key));
1565         e_vcard_append_attribute_with_value (
1566                 vcard,
1567                 e_vcard_attribute_new (NULL, EBSQL_VCARD_SORT_KEY),
1568                 base64);
1569         encoded = e_vcard_to_string (vcard, EVC_FORMAT_VCARD_30);
1570
1571         g_free (base64);
1572         g_object_unref (vcard);
1573
1574         return encoded;
1575 }
1576
1577 static gchar *
1578 ebsql_decode_vcard_sort_key_from_vcard (EVCard *vcard)
1579 {
1580         EVCardAttribute *attr;
1581         GList *values = NULL;
1582         gchar *sort_key = NULL;
1583         gchar *base64 = NULL;
1584
1585         attr = e_vcard_get_attribute (vcard, EBSQL_VCARD_SORT_KEY);
1586         if (attr)
1587                 values = e_vcard_attribute_get_values (attr);
1588
1589         if (values && values->data) {
1590                 gsize len;
1591
1592                 base64 = g_strdup (values->data);
1593
1594                 sort_key = (gchar *) g_base64_decode (base64, &len);
1595                 g_free (base64);
1596         }
1597
1598         return sort_key;
1599 }
1600
1601 static gchar *
1602 ebsql_decode_vcard_sort_key (const gchar *encoded)
1603 {
1604         EVCard *vcard;
1605         gchar *sort_key;
1606
1607         vcard = e_vcard_new_from_string (encoded);
1608         sort_key = ebsql_decode_vcard_sort_key_from_vcard (vcard);
1609         g_object_unref (vcard);
1610
1611         return sort_key;
1612 }
1613
1614 typedef struct {
1615         EBookSqlite *ebsql;
1616         EContactField field;
1617 } EbSqlCollData;
1618
1619 static gint
1620 ebsql_fallback_collator (gpointer ref,
1621                          gint len1,
1622                          gconstpointer data1,
1623                          gint len2,
1624                          gconstpointer data2)
1625 {
1626         EbSqlCollData *data = (EbSqlCollData *) ref;
1627         EBookSqlitePrivate *priv;
1628         EContact *contact1, *contact2;
1629         const gchar *str1, *str2;
1630         gchar *key1, *key2;
1631         gchar *tmp;
1632         gint result = 0;
1633
1634         priv = data->ebsql->priv;
1635
1636         str1 = (const gchar *) data1;
1637         str2 = (const gchar *) data2;
1638
1639         /* Construct 2 contacts (we're comparing vcards) */
1640         contact1 = e_contact_new ();
1641         contact2 = e_contact_new ();
1642         e_vcard_construct_full (E_VCARD (contact1), str1, len1, NULL);
1643         e_vcard_construct_full (E_VCARD (contact2), str2, len2, NULL);
1644
1645         /* Extract first key */
1646         key1 = ebsql_decode_vcard_sort_key_from_vcard (E_VCARD (contact1));
1647         if (!key1) {
1648                 tmp = e_contact_get (contact1, data->field);
1649                 if (tmp)
1650                         key1 = e_collator_generate_key (priv->collator, tmp, NULL);
1651                 g_free (tmp);
1652         }
1653         if (!key1)
1654                 key1 = g_strdup ("");
1655
1656         /* Extract second key */
1657         key2 = ebsql_decode_vcard_sort_key_from_vcard (E_VCARD (contact2));
1658         if (!key2) {
1659                 tmp = e_contact_get (contact2, data->field);
1660                 if (tmp)
1661                         key2 = e_collator_generate_key (priv->collator, tmp, NULL);
1662                 g_free (tmp);
1663         }
1664         if (!key2)
1665                 key2 = g_strdup ("");
1666
1667         result = strcmp (key1, key2);
1668
1669         g_free (key1);
1670         g_free (key2);
1671         g_object_unref (contact1);
1672         g_object_unref (contact2);
1673
1674         return result;
1675 }
1676
1677 static EbSqlCollData *
1678 ebsql_coll_data_new (EBookSqlite *ebsql,
1679                      EContactField field)
1680 {
1681         EbSqlCollData *data = g_slice_new (EbSqlCollData);
1682
1683         data->ebsql = ebsql;
1684         data->field = field;
1685
1686         return data;
1687 }
1688
1689 static void
1690 ebsql_coll_data_free (EbSqlCollData *data)
1691 {
1692         if (data)
1693                 g_slice_free (EbSqlCollData, data);
1694 }
1695
1696 /* COLLATE functions are generated on demand only */
1697 static void
1698 ebsql_generate_collator (gpointer ref,
1699                          sqlite3 *db,
1700                          gint eTextRep,
1701                          const gchar *coll_name)
1702 {
1703         EBookSqlite *ebsql = (EBookSqlite *) ref;
1704         EbSqlCollData *data;
1705         EContactField field;
1706         const gchar *field_name;
1707
1708         field_name = coll_name + strlen (EBSQL_COLLATE_PREFIX);
1709         field = e_contact_field_id (field_name);
1710
1711         /* This should be caught before reaching here, just an extra check */
1712         if (field == 0 || field >= E_CONTACT_FIELD_LAST ||
1713             e_contact_field_type (field) != G_TYPE_STRING) {
1714                 g_warning ("Specified collation on invalid contact field");
1715                 return;
1716         }
1717
1718         data = ebsql_coll_data_new (ebsql, field);
1719         sqlite3_create_collation_v2 (
1720                 db, coll_name, SQLITE_UTF8,
1721                 data, ebsql_fallback_collator,
1722                 (GDestroyNotify) ebsql_coll_data_free);
1723 }
1724
1725 /**********************************************************
1726  *        Cancel long operations with GCancellable        *
1727  **********************************************************/
1728 static gint
1729 ebsql_check_cancel (gpointer ref)
1730 {
1731         EBookSqlite *ebsql = (EBookSqlite *) ref;
1732
1733         if (ebsql->priv->cancel &&
1734             g_cancellable_is_cancelled (ebsql->priv->cancel)) {
1735                 EBSQL_NOTE (
1736                         CANCEL,
1737                         g_printerr ("CANCEL: An operation was canceled\n"));
1738                 return -1;
1739         }
1740
1741         return 0;
1742 }
1743
1744 /**********************************************************
1745  *                  Database Initialization               *
1746  **********************************************************/
1747 static inline gint
1748 main_table_index_by_name (const gchar *name)
1749 {
1750         gint i;
1751
1752         for (i = 0; i < G_N_ELEMENTS (main_table_columns); i++) {
1753                 if (g_strcmp0 (name, main_table_columns[i].name) == 0)
1754                         return i;
1755         }
1756
1757         return -1;
1758 }
1759
1760 static gint
1761 check_main_table_columns (gpointer data,
1762                           gint n_cols,
1763                           gchar **cols,
1764                           gchar **name)
1765 {
1766         guint *columns_mask = (guint *) data;
1767         gint i;
1768
1769         for (i = 0; i < n_cols; i++) {
1770
1771                 if (g_strcmp0 (name[i], "name") == 0) {
1772                         gint idx = main_table_index_by_name (cols[i]);
1773
1774                         if (idx >= 0)
1775                                 *columns_mask |= (1 << idx);
1776
1777                         break;
1778                 }
1779         }
1780
1781         return 0;
1782 }
1783
1784 static gboolean
1785 ebsql_init_sqlite (EBookSqlite *ebsql,
1786                    const gchar *filename,
1787                    GError **error)
1788 {
1789         gint ret, i;
1790
1791         e_sqlite3_vfs_init ();
1792
1793         ret = sqlite3_open (filename, &ebsql->priv->db);
1794
1795         /* Handle GCancellable */
1796         sqlite3_progress_handler (
1797                 ebsql->priv->db,
1798                 EBSQL_CANCEL_BATCH_SIZE,
1799                 ebsql_check_cancel,
1800                 ebsql);
1801
1802         /* Install our custom functions */
1803         for (i = 0; ret == SQLITE_OK && i < G_N_ELEMENTS (ebsql_custom_functions); i++)
1804                 ret = sqlite3_create_function (
1805                         ebsql->priv->db,
1806                         ebsql_custom_functions[i].name,
1807                         ebsql_custom_functions[i].arguments,
1808                         SQLITE_UTF8, ebsql,
1809                         ebsql_custom_functions[i].func,
1810                         NULL, NULL);
1811
1812         /* Fallback COLLATE implementations generated on demand */
1813         if (ret == SQLITE_OK)
1814                 ret = sqlite3_collation_needed (
1815                         ebsql->priv->db, ebsql, ebsql_generate_collator);
1816
1817         if (ret != SQLITE_OK) {
1818                 if (!ebsql->priv->db) {
1819                         EBSQL_SET_ERROR_LITERAL (
1820                                 error,
1821                                 E_BOOK_SQLITE_ERROR_LOAD,
1822                                 _("Insufficient memory"));
1823                 } else {
1824                         const gchar *errmsg = sqlite3_errmsg (ebsql->priv->db);
1825
1826                         EBSQL_SET_ERROR (
1827                                 error,
1828                                 E_BOOK_SQLITE_ERROR_ENGINE,
1829                                 "Can't open database %s: %s\n",
1830                                 filename, errmsg);
1831                         sqlite3_close (ebsql->priv->db);
1832                 }
1833                 return FALSE;
1834         }
1835
1836         ebsql_exec (ebsql, "ATTACH DATABASE ':memory:' AS mem", NULL, NULL, NULL, NULL);
1837         ebsql_exec (ebsql, "PRAGMA foreign_keys = ON",          NULL, NULL, NULL, NULL);
1838         ebsql_exec (ebsql, "PRAGMA case_sensitive_like = ON",   NULL, NULL, NULL, NULL);
1839
1840         return TRUE;
1841 }
1842
1843 static inline void
1844 format_column_declaration (GString *string,
1845                            ColumnInfo *info)
1846 {
1847         g_string_append (string, info->name);
1848         g_string_append_c (string, ' ');
1849
1850         g_string_append (string, info->type);
1851
1852         if (info->extra) {
1853                 g_string_append_c (string, ' ');
1854                 g_string_append (string, info->extra);
1855         }
1856 }
1857
1858 static inline gboolean
1859 ensure_column_index (EBookSqlite *ebsql,
1860                      const gchar *table,
1861                      ColumnInfo *info,
1862                      GError **error)
1863 {
1864         if (!info->index)
1865                 return TRUE;
1866
1867         return ebsql_exec_printf (
1868                 ebsql,
1869                 "CREATE INDEX IF NOT EXISTS %Q ON %Q (%s)",
1870                 NULL, NULL, NULL, error,
1871                 info->index, table, info->name);
1872 }
1873
1874 /* Called with the lock held and inside a transaction */
1875 static gboolean
1876 ebsql_resolve_folderid (EBookSqlite *ebsql,
1877                         gint *previous_schema,
1878                         gint *already_exists,
1879                         GError **error)
1880 {
1881         gint n_folders = 0;
1882         gint version = 0;
1883         gchar *loaded_folder_id = NULL;
1884         gboolean success;
1885
1886         success = ebsql_exec (
1887                 ebsql, "SELECT count(*) FROM sqlite_master "
1888                 "WHERE type='table' AND name='folders';",
1889                 get_count_cb, &n_folders, NULL, error);
1890
1891         if (success && n_folders > 1) {
1892                 EBSQL_SET_ERROR_LITERAL (
1893                         error,
1894                         E_BOOK_SQLITE_ERROR_LOAD,
1895                         _("Cannot upgrade contacts database from a legacy "
1896                         "database with more than one addressbook. "
1897                         "Delete one of the entries in the 'folders' table first."));
1898                 success = FALSE;
1899         }
1900
1901         if (success && n_folders == 1)
1902                 success = ebsql_exec (
1903                         ebsql, "SELECT folder_id FROM folders LIMIT 1",
1904                         get_string_cb, &loaded_folder_id, NULL, error);
1905
1906         if (success && n_folders == 1)
1907                 success = ebsql_exec (
1908                         ebsql, "SELECT version FROM folders LIMIT 1",
1909                         get_int_cb, &version, NULL, error);
1910
1911         if (success && n_folders == 1) {
1912                 g_free (ebsql->priv->folderid);
1913                 ebsql->priv->folderid = loaded_folder_id;
1914         } else {
1915                 g_free (loaded_folder_id);
1916         }
1917
1918         if (n_folders == 1)
1919                 *already_exists = TRUE;
1920         else
1921                 *already_exists = FALSE;
1922
1923         EBSQL_NOTE (
1924                 SCHEMA,
1925                 g_printerr (
1926                         "SCHEMA: main folder id resolved as '%s', "
1927                         "already existing tables: %d loaded version: %d (%s)\n",
1928                         ebsql->priv->folderid, n_folders, version,
1929                         success ? "success" : "failed"));
1930
1931         *previous_schema = version;
1932
1933         return success;
1934 }
1935
1936 /* Called with the lock held and inside a transaction */
1937 static gboolean
1938 ebsql_init_folders (EBookSqlite *ebsql,
1939                     gint previous_schema,
1940                     GError **error)
1941 {
1942         GString *string;
1943         guint existing_columns_mask = 0, i;
1944         gboolean success;
1945
1946         string = g_string_sized_new (COLUMN_DEFINITION_BYTES * G_N_ELEMENTS (main_table_columns));
1947         g_string_append (string, "CREATE TABLE IF NOT EXISTS folders (");
1948         for (i = 0; i < G_N_ELEMENTS (main_table_columns); i++) {
1949
1950                 if (i > 0)
1951                         g_string_append (string, ", ");
1952
1953                 format_column_declaration (string, &(main_table_columns[i]));
1954         }
1955         g_string_append_c (string, ')');
1956
1957         /* Create main folders table */
1958         success = ebsql_exec (ebsql, string->str, NULL, NULL, NULL, error);
1959         g_string_free (string, TRUE);
1960
1961         /* Check which columns in the main table already exist */
1962         if (success)
1963                 success = ebsql_exec (
1964                         ebsql, "PRAGMA table_info (folders)",
1965                         check_main_table_columns, &existing_columns_mask,
1966                         NULL, error);
1967
1968         /* Add columns which may be missing */
1969         for (i = 0; success && i < G_N_ELEMENTS (main_table_columns); i++) {
1970                 ColumnInfo *info = &(main_table_columns[i]);
1971
1972                 if ((existing_columns_mask & (1 << i)) != 0)
1973                         continue;
1974
1975                 success = ebsql_exec_printf (
1976                         ebsql, "ALTER TABLE folders ADD COLUMN %s %s %s",
1977                         NULL, NULL, NULL, error, info->name, info->type,
1978                         info->extra ? info->extra : "");
1979         }
1980
1981         /* Special case upgrade for schema versions 3 & 4.
1982          *
1983          * Drops the reverse_multivalues column.
1984          */
1985         if (success && previous_schema >= 3 && previous_schema < 5) {
1986
1987                 success = ebsql_exec (
1988                         ebsql,
1989                         "UPDATE folders SET "
1990                                 "multivalues = REPLACE(RTRIM(REPLACE("
1991                                         "multivalues || ':', ':', "
1992                                         "CASE reverse_multivalues "
1993                                                 "WHEN 0 THEN ';prefix ' "
1994                                                 "ELSE ';prefix;suffix ' "
1995                                         "END)), ' ', ':'), "
1996                                 "reverse_multivalues = NULL",
1997                         NULL, NULL, NULL, error);
1998         }
1999
2000         /* Finish the eventual upgrade by storing the current schema version.
2001          */
2002         if (success && previous_schema >= 1 && previous_schema < FOLDER_VERSION)
2003                 success = ebsql_exec_printf (
2004                         ebsql, "UPDATE folders SET version = %d",
2005                         NULL, NULL, NULL, error, FOLDER_VERSION);
2006
2007         EBSQL_NOTE (
2008                 SCHEMA,
2009                 g_printerr (
2010                         "SCHEMA: Initialized main folders table (%s)\n",
2011                         success ? "success" : "failed"));
2012
2013         return success;
2014 }
2015
2016 /* Called with the lock held and inside a transaction */
2017 static gboolean
2018 ebsql_init_keys (EBookSqlite *ebsql,
2019                  GError **error)
2020 {
2021         gboolean success;
2022
2023         /* Create a child table to store key/value pairs for a folder. */
2024         success = ebsql_exec (
2025                 ebsql,
2026                 "CREATE TABLE IF NOT EXISTS keys ("
2027                 " key TEXT PRIMARY KEY,"
2028                 " value TEXT,"
2029                 " folder_id TEXT REFERENCES folders)",
2030                 NULL, NULL, NULL, error);
2031
2032         /* Add an index on the keys */
2033         if (success)
2034                 success = ebsql_exec (
2035                         ebsql,
2036                         "CREATE INDEX IF NOT EXISTS keysindex ON keys (folder_id)",
2037                         NULL, NULL, NULL, error);
2038
2039         EBSQL_NOTE (
2040                 SCHEMA,
2041                 g_printerr (
2042                         "SCHEMA: Initialized keys table (%s)\n",
2043                         success ? "success" : "failed"));
2044
2045         return success;
2046 }
2047
2048 static gchar *
2049 format_multivalues (EBookSqlite *ebsql)
2050 {
2051         gint i;
2052         GString *string;
2053         gboolean first = TRUE;
2054
2055         string = g_string_new (NULL);
2056
2057         for (i = 0; i < ebsql->priv->n_summary_fields; i++) {
2058                 if (ebsql->priv->summary_fields[i].type == E_TYPE_CONTACT_ATTR_LIST) {
2059                         if (first)
2060                                 first = FALSE;
2061                         else
2062                                 g_string_append_c (string, ':');
2063
2064                         g_string_append (string, ebsql->priv->summary_fields[i].dbname);
2065
2066                         /* E_BOOK_INDEX_SORT_KEY is not supported in the multivalue fields */
2067                         if ((ebsql->priv->summary_fields[i].index & INDEX_FLAG (PREFIX)) != 0)
2068                                 g_string_append (string, ";prefix");
2069                         if ((ebsql->priv->summary_fields[i].index & INDEX_FLAG (SUFFIX)) != 0)
2070                                 g_string_append (string, ";suffix");
2071                         if ((ebsql->priv->summary_fields[i].index & INDEX_FLAG (PHONE)) != 0)
2072                                 g_string_append (string, ";phone");
2073                         if ((ebsql->priv->summary_fields[i].index & INDEX_FLAG (TRANSLIT)) != 0)
2074                                 g_string_append (string, ";translit");
2075                 }
2076         }
2077
2078         return g_string_free (string, FALSE);
2079 }
2080
2081 /* Called with the lock held and inside a transaction */
2082 static gboolean
2083 ebsql_add_folder (EBookSqlite *ebsql,
2084                   GError **error)
2085 {
2086         gboolean success;
2087         gchar *multivalues;
2088         const gchar *lc_collate;
2089
2090         multivalues = format_multivalues (ebsql);
2091         lc_collate = setlocale (LC_COLLATE, NULL);
2092
2093         success = ebsql_exec_printf (
2094                 ebsql,
2095                 "INSERT OR IGNORE INTO folders"
2096                 " ( folder_id, version, multivalues, lc_collate ) "
2097                 "VALUES ( %Q, %d, %Q, %Q ) ",
2098                 NULL, NULL, NULL, error,
2099                 ebsql->priv->folderid, FOLDER_VERSION, multivalues, lc_collate);
2100
2101         g_free (multivalues);
2102
2103         EBSQL_NOTE (
2104                 SCHEMA,
2105                 g_printerr (
2106                         "SCHEMA: Added '%s' entry to main folder (%s)\n",
2107                         ebsql->priv->folderid, success ? "success" : "failed"));
2108
2109         return success;
2110 }
2111
2112 /* Called with the lock held and inside a transaction */
2113 static gboolean
2114 ebsql_introspect_summary (EBookSqlite *ebsql,
2115                           gint previous_schema,
2116                           GSList **introspected_columns,
2117                           GError **error)
2118 {
2119         gboolean success;
2120         GSList *summary_columns = NULL, *l;
2121         GArray *summary_fields = NULL;
2122         gchar *multivalues = NULL;
2123         gint i, j;
2124
2125         success = ebsql_exec_printf (
2126                 ebsql, "PRAGMA table_info (%Q);",
2127                 get_columns_cb, &summary_columns, NULL, error,
2128                 ebsql->priv->folderid);
2129
2130         if (!success)
2131                 goto introspect_summary_finish;
2132
2133         summary_columns = g_slist_reverse (summary_columns);
2134         summary_fields = g_array_new (FALSE, FALSE, sizeof (SummaryField));
2135
2136         /* Introspect the normal summary fields */
2137         for (l = summary_columns; l; l = l->next) {
2138                 EContactField field_id;
2139                 const gchar *col = l->data;
2140                 gchar *p;
2141                 gint computed = 0;
2142                 gchar *freeme = NULL;
2143
2144                 /* Note that we don't have any way to introspect
2145                  * E_BOOK_INDEX_PREFIX, this is not important because if
2146                  * the prefix index is specified, it will be created
2147                  * the first time the SQLite tables are created, so
2148                  * it's not important to ensure prefix indexes after
2149                  * introspecting the summary.
2150                  */
2151
2152                 /* Check if we're parsing a reverse field */
2153                 if ((p = strstr (col, "_" EBSQL_SUFFIX_REVERSE)) != NULL) {
2154                         computed = INDEX_FLAG (SUFFIX);
2155                         freeme = g_strndup (col, p - col);
2156                         col = freeme;
2157                 } else if ((p = strstr (col, "_" EBSQL_SUFFIX_PHONE)) != NULL) {
2158                         computed = INDEX_FLAG (PHONE);
2159                         freeme = g_strndup (col, p - col);
2160                         col = freeme;
2161                 } else if ((p = strstr (col, "_" EBSQL_SUFFIX_COUNTRY)) != NULL) {
2162                         computed = INDEX_FLAG (PHONE);
2163                         freeme = g_strndup (col, p - col);
2164                         col = freeme;
2165                 } else if ((p = strstr (col, "_" EBSQL_SUFFIX_SORT_KEY)) != NULL) {
2166                         computed = INDEX_FLAG (SORT_KEY);
2167                         freeme = g_strndup (col, p - col);
2168                         col = freeme;
2169                 } else if ((p = strstr (col, "_" EBSQL_SUFFIX_TRANSLIT)) != NULL) {
2170                         computed = INDEX_FLAG (TRANSLIT);
2171                         freeme = g_strndup (col, p - col);
2172                         col = freeme;
2173                 }
2174
2175                 /* First check exception fields */
2176                 if (g_ascii_strcasecmp (col, "uid") == 0)
2177                         field_id = E_CONTACT_UID;
2178                 else if (g_ascii_strcasecmp (col, "is_list") == 0)
2179                         field_id = E_CONTACT_IS_LIST;
2180                 else
2181                         field_id = e_contact_field_id (col);
2182
2183                 /* Check for parse error */
2184                 if (field_id == 0) {
2185                         EBSQL_SET_ERROR (
2186                                 error,
2187                                 E_BOOK_SQLITE_ERROR_UNSUPPORTED_FIELD,
2188                                 _("Error introspecting unknown summary field '%s'"),
2189                                 col);
2190                         success = FALSE;
2191                         g_free (freeme);
2192                         break;
2193                 }
2194
2195                 /* Computed columns are always declared after the normal columns,
2196                  * if a reverse field is encountered we need to set the suffix
2197                  * index on the coresponding summary field
2198                  */
2199                 if (computed) {
2200                         gint field_idx;
2201                         SummaryField *iter;
2202
2203                         field_idx = summary_field_array_index (summary_fields, field_id);
2204                         if (field_idx >= 0) {
2205                                 iter = &g_array_index (summary_fields, SummaryField, field_idx);
2206                                 iter->index |= computed;
2207                         }
2208
2209                 } else {
2210                         summary_field_append (
2211                                 summary_fields, ebsql->priv->folderid,
2212                                 field_id, NULL);
2213                 }
2214
2215                 g_free (freeme);
2216         }
2217
2218         if (!success)
2219                 goto introspect_summary_finish;
2220
2221         /* Introspect the multivalied summary fields */
2222         success = ebsql_exec_printf (
2223                 ebsql,
2224                 "SELECT multivalues FROM folders "
2225                 "WHERE folder_id = %Q",
2226                 get_string_cb, &multivalues, NULL, error,
2227                 ebsql->priv->folderid);
2228
2229         if (!success)
2230                 goto introspect_summary_finish;
2231
2232         if (multivalues) {
2233                 gchar **fields = g_strsplit (multivalues, ":", 0);
2234
2235                 for (i = 0; fields[i] != NULL; i++) {
2236                         EContactField field_id;
2237                         SummaryField *iter;
2238                         gchar **params;
2239
2240                         params = g_strsplit (fields[i], ";", 0);
2241                         field_id = e_contact_field_id (params[0]);
2242                         iter = summary_field_append (
2243                                 summary_fields,
2244                                 ebsql->priv->folderid,
2245                                 field_id, NULL);
2246
2247                         if (iter) {
2248                                 for (j = 1; params[j]; ++j) {
2249                                         /* Sort keys not supported for multivalued fields */
2250                                         if (strcmp (params[j], "prefix") == 0) {
2251                                                 iter->index |= INDEX_FLAG (PREFIX);
2252                                         } else if (strcmp (params[j], "suffix") == 0) {
2253                                                 iter->index |= INDEX_FLAG (SUFFIX);
2254                                         } else if (strcmp (params[j], "phone") == 0) {
2255                                                 iter->index |= INDEX_FLAG (PHONE);
2256                                         } else if (strcmp (params[j], "translit") == 0) {
2257                                                 iter->index |= INDEX_FLAG (TRANSLIT);
2258                                         }
2259                                 }
2260                         }
2261
2262                         g_strfreev (params);
2263                 }
2264
2265                 g_strfreev (fields);
2266         }
2267
2268         /* HARD CODE UP AHEAD
2269          *
2270          * Now we're finished introspecting, if the summary is from a previous version,
2271          * we need to add any summary fields which we're added to the default summary
2272          * since the schema version which was introduced here
2273          */
2274         if (previous_schema >= 1) {
2275                 SummaryField *summary_field;
2276
2277                 if (previous_schema < 8) {
2278
2279                         /* We used to keep 4 email fields in the summary, before we supported
2280                          * the multivaliued E_CONTACT_EMAIL... convert the old summary to use
2281                          * the multivaliued field instead.
2282                          */
2283                         if (summary_field_array_index (summary_fields, E_CONTACT_EMAIL_1) >= 0 &&
2284                             summary_field_array_index (summary_fields, E_CONTACT_EMAIL_2) >= 0 &&
2285                             summary_field_array_index (summary_fields, E_CONTACT_EMAIL_3) >= 0 &&
2286                             summary_field_array_index (summary_fields, E_CONTACT_EMAIL_4) >= 0) {
2287
2288                                 summary_field_remove (summary_fields, E_CONTACT_EMAIL_1);
2289                                 summary_field_remove (summary_fields, E_CONTACT_EMAIL_2);
2290                                 summary_field_remove (summary_fields, E_CONTACT_EMAIL_3);
2291                                 summary_field_remove (summary_fields, E_CONTACT_EMAIL_4);
2292
2293                                 summary_field = summary_field_append (
2294                                         summary_fields,
2295                                         ebsql->priv->folderid,
2296                                         E_CONTACT_EMAIL, NULL);
2297                                 summary_field->index |= INDEX_FLAG (PREFIX);
2298                         }
2299
2300                         /* Regardless of whether it was a default summary or not, add the sort
2301                          * keys to anything less than Schema 8 (as long as those fields are at least
2302                          * in the summary)
2303                          */
2304                         if ((i = summary_field_array_index (summary_fields, E_CONTACT_FILE_AS)) >= 0) {
2305                                 summary_field = &g_array_index (summary_fields, SummaryField, i);
2306                                 summary_field->index |= INDEX_FLAG (SORT_KEY);
2307                         }
2308
2309                         if ((i = summary_field_array_index (summary_fields, E_CONTACT_GIVEN_NAME)) >= 0) {
2310                                 summary_field = &g_array_index (summary_fields, SummaryField, i);
2311                                 summary_field->index |= INDEX_FLAG (SORT_KEY);
2312                         }
2313
2314                         if ((i = summary_field_array_index (summary_fields, E_CONTACT_FAMILY_NAME)) >= 0) {
2315                                 summary_field = &g_array_index (summary_fields, SummaryField, i);
2316                                 summary_field->index |= INDEX_FLAG (SORT_KEY);
2317                         }
2318                 }
2319         }
2320
2321  introspect_summary_finish:
2322
2323         /* Apply the introspected summary fields */
2324         if (success) {
2325                 summary_fields_array_free (
2326                         ebsql->priv->summary_fields,
2327                         ebsql->priv->n_summary_fields);
2328
2329                 ebsql->priv->n_summary_fields = summary_fields->len;
2330                 ebsql->priv->summary_fields = (SummaryField *) g_array_free (summary_fields, FALSE);
2331
2332                 *introspected_columns = summary_columns;
2333         } else if (summary_fields) {
2334                 gint n_fields;
2335                 SummaryField *fields;
2336
2337                 /* Properly free the array */
2338                 n_fields = summary_fields->len;
2339                 fields = (SummaryField *) g_array_free (summary_fields, FALSE);
2340                 summary_fields_array_free (fields, n_fields);
2341
2342                 g_slist_free_full (summary_columns, (GDestroyNotify) g_free);
2343         }
2344
2345         g_free (multivalues);
2346
2347         EBSQL_NOTE (
2348                 SCHEMA,
2349                 g_printerr (
2350                         "SCHEMA: Introspected summary (%s)\n",
2351                         success ? "success" : "failed"));
2352
2353         return success;
2354 }
2355
2356 /* Called with the lock held and inside a transaction */
2357 static gboolean
2358 ebsql_init_contacts (EBookSqlite *ebsql,
2359                      GSList *introspected_columns,
2360                      GError **error)
2361 {
2362         gint i;
2363         gboolean success = TRUE;
2364         GString *string;
2365         GSList *summary_columns = NULL, *l;
2366
2367         /* Get a list of all columns and indexes which should be present
2368          * in the main summary table */
2369         for (i = 0; i < ebsql->priv->n_summary_fields; i++) {
2370                 SummaryField *field = &(ebsql->priv->summary_fields[i]);
2371
2372                 if (field->type != E_TYPE_CONTACT_ATTR_LIST) {
2373                         l = summary_field_list_columns (field, ebsql->priv->folderid);
2374                         summary_columns = g_slist_concat (summary_columns, l);
2375                 }
2376         }
2377
2378         /* Create the main contacts table for this folder
2379          */
2380         string = g_string_sized_new (32 * g_slist_length (summary_columns));
2381         g_string_append (string, "CREATE TABLE IF NOT EXISTS %Q (");
2382
2383         for (l = summary_columns; l; l = l->next) {
2384                 ColumnInfo *info = l->data;
2385
2386                 if (l != summary_columns)
2387                         g_string_append (string, ", ");
2388
2389                 format_column_declaration (string, info);
2390         }
2391         g_string_append (string, ", vcard TEXT, bdata TEXT)");
2392
2393         success = ebsql_exec_printf (
2394                 ebsql, string->str,
2395                 NULL, NULL, NULL, error,
2396                 ebsql->priv->folderid);
2397
2398         g_string_free (string, TRUE);
2399
2400         /* If we introspected something, let's first adjust the contacts table
2401          * so that it includes the right columns */
2402         if (introspected_columns) {
2403
2404                 /* Add any missing columns which are in the summary fields but
2405                  * not found in the contacts table
2406                  */
2407                 for (l = summary_columns; success && l; l = l->next) {
2408                         ColumnInfo *info = l->data;
2409
2410                         if (g_slist_find_custom (introspected_columns,
2411                                                  info->name, (GCompareFunc) g_ascii_strcasecmp))
2412                                 continue;
2413
2414                         success = ebsql_exec_printf (
2415                                 ebsql,
2416                                 "ALTER TABLE %Q ADD COLUMN %s %s %s",
2417                                 NULL, NULL, NULL, error,
2418                                 ebsql->priv->folderid,
2419                                 info->name, info->type,
2420                                 info->extra ? info->extra : "");
2421                 }
2422         }
2423
2424         /* Add indexes to columns in the main contacts table
2425          */
2426         for (l = summary_columns; success && l; l = l->next) {
2427                 ColumnInfo *info = l->data;
2428
2429                 success = ensure_column_index (ebsql, ebsql->priv->folderid, info, error);
2430         }
2431
2432         g_slist_free_full (summary_columns, (GDestroyNotify) column_info_free);
2433
2434         EBSQL_NOTE (
2435                 SCHEMA,
2436                 g_printerr (
2437                         "SCHEMA: Initialized summary table '%s' (%s)\n",
2438                         ebsql->priv->folderid, success ? "success" : "failed"));
2439
2440         return success;
2441 }
2442
2443 /* Called with the lock held and inside a transaction */
2444 static gboolean
2445 ebsql_init_aux_tables (EBookSqlite *ebsql,
2446                        gint previous_schema,
2447                        GError **error)
2448 {
2449         GString *string;
2450         gboolean success = TRUE;
2451         GSList *aux_columns = NULL, *l;
2452         gchar *tmp;
2453         gint i;
2454
2455         /* Drop the general 'folder_id_lists' table which was used prior to
2456          * version 8 of the schema
2457          */
2458         if (previous_schema >= 1 && previous_schema < 8) {
2459                 tmp = g_strconcat (ebsql->priv->folderid, "_lists", NULL);
2460                 success = ebsql_exec_printf (
2461                         ebsql, "DROP TABLE IF EXISTS %Q",
2462                         NULL, NULL, NULL, error, tmp);
2463                 g_free (tmp);
2464         }
2465
2466         for (i = 0; success && i < ebsql->priv->n_summary_fields; i++) {
2467                 SummaryField *field = &(ebsql->priv->summary_fields[i]);
2468
2469                 if (field->type != E_TYPE_CONTACT_ATTR_LIST)
2470                         continue;
2471
2472                 aux_columns = summary_field_list_columns (field, ebsql->priv->folderid);
2473
2474                 /* Create the auxiliary table for this multi valued field */
2475                 string = g_string_sized_new (
2476                         COLUMN_DEFINITION_BYTES * 3 +
2477                         COLUMN_DEFINITION_BYTES * g_slist_length (aux_columns));
2478
2479                 g_string_append (string, "CREATE TABLE IF NOT EXISTS %Q (uid TEXT NOT NULL REFERENCES %Q (uid)");
2480                 for (l = aux_columns; l; l = l->next) {
2481                         ColumnInfo *info = l->data;
2482
2483                         g_string_append (string, ", ");
2484                         format_column_declaration (string, info);
2485                 }
2486                 g_string_append_c (string, ')');
2487
2488                 success = ebsql_exec_printf (
2489                         ebsql, string->str, NULL, NULL, NULL, error,
2490                         field->aux_table, ebsql->priv->folderid);
2491                 g_string_free (string, TRUE);
2492
2493                 if (success) {
2494
2495                         /* Create an index on the implied 'uid' column, this is important
2496                          * when replacing (modifying) contacts, since we need to remove
2497                          * all rows in an auxiliary table which matches a given UID.
2498                          *
2499                          * This index speeds up the constraint in a statement such as:
2500                          *
2501                          *   DELETE from email_list WHERE email_list.uid = 'contact uid'
2502                          */
2503                         tmp = g_strconcat (
2504                                 "UID_INDEX",
2505                                 "_", field->dbname,
2506                                 "_", ebsql->priv->folderid,
2507                                 NULL);
2508                         ebsql_exec_printf (
2509                                 ebsql,
2510                                 "CREATE INDEX IF NOT EXISTS %Q ON %Q (%s)",
2511                                 NULL, NULL, NULL, error,
2512                                 tmp, field->aux_table, "uid");
2513                         g_free (tmp);
2514                 }
2515
2516                 /* Add indexes to columns in this auxiliary table
2517                  */
2518                 for (l = aux_columns; success && l; l = l->next) {
2519                         ColumnInfo *info = l->data;
2520
2521                         success = ensure_column_index (ebsql, field->aux_table, info, error);
2522                 }
2523
2524                 g_slist_free_full (aux_columns, (GDestroyNotify) column_info_free);
2525
2526                 EBSQL_NOTE (
2527                         SCHEMA,
2528                         g_printerr (
2529                                 "SCHEMA: Initialized auxiliary table '%s'\n",
2530                                 field->aux_table));
2531         }
2532
2533         EBSQL_NOTE (
2534                 SCHEMA,
2535                 g_printerr (
2536                         "SCHEMA: Initialized auxiliary tables (%s)\n",
2537                         success ? "success" : "failed"));
2538
2539         return success;
2540 }
2541
2542 static gboolean
2543 ebsql_upgrade_one (EBookSqlite *ebsql,
2544                    EbSqlChangeType change_type,
2545                    EbSqlSearchData *result,
2546                    GError **error)
2547 {
2548         EContact *contact = NULL;
2549         gboolean success;
2550
2551         /* It can be we're opening a light summary which was created without
2552          * storing the vcards, such as was used in EDS versions 3.2 to 3.6.
2553          *
2554          * In this case we just want to skip the contacts we can't load
2555          * and leave them as is in the SQLite, they will be added from
2556          * the old BDB in the case of a migration anyway.
2557          */
2558         if (result->vcard)
2559                 contact = e_contact_new_from_vcard_with_uid (result->vcard, result->uid);
2560
2561         if (contact == NULL)
2562                 return TRUE;
2563
2564         success = ebsql_insert_contact (
2565                 ebsql, change_type, contact,
2566                 result->vcard, result->extra,
2567                 TRUE, error);
2568
2569         g_object_unref (contact);
2570
2571         return success;
2572 }
2573
2574 /* Called with the lock held and inside a transaction */
2575 static gboolean
2576 ebsql_upgrade (EBookSqlite *ebsql,
2577                EbSqlChangeType change_type,
2578                GError **error)
2579 {
2580         gchar *uid = NULL;
2581         gint n_results;
2582         gboolean success = TRUE;
2583
2584         do {
2585                 GSList *batch = NULL, *l;
2586                 EbSqlSearchData *result = NULL;
2587
2588                 if (uid == NULL) {
2589                         success = ebsql_exec_printf (
2590                                 ebsql,
2591                                 "SELECT summary.uid, %s, summary.bdata FROM %Q AS summary "
2592                                 "ORDER BY summary.uid ASC LIMIT %d",
2593                                 collect_full_results_cb, &batch, NULL, error,
2594                                 EBSQL_VCARD_FRAGMENT (ebsql),
2595                                 ebsql->priv->folderid, EBSQL_UPGRADE_BATCH_SIZE);
2596                 } else {
2597                         success = ebsql_exec_printf (
2598                                 ebsql,
2599                                 "SELECT summary.uid, %s, summary.bdata FROM %Q AS summary "
2600                                 "WHERE summary.uid > %Q "
2601                                 "ORDER BY summary.uid ASC LIMIT %d",
2602                                 collect_full_results_cb, &batch, NULL, error,
2603                                 EBSQL_VCARD_FRAGMENT (ebsql),
2604                                 ebsql->priv->folderid, uid, EBSQL_UPGRADE_BATCH_SIZE);
2605                 }
2606
2607                 /* Reverse the list, we want to walk through it forwards */
2608                 batch = g_slist_reverse (batch);
2609                 for (l = batch; success && l; l = l->next) {
2610                         result = l->data;
2611                         success = ebsql_upgrade_one (
2612                                 ebsql,
2613                                 change_type,
2614                                 result,
2615                                 error);
2616                 }
2617
2618                 /* result is now the last one in the list */
2619                 if (result) {
2620                         g_free (uid);
2621                         uid = result->uid;
2622                         result->uid = NULL;
2623                 }
2624
2625                 n_results = g_slist_length (batch);
2626                 g_slist_free_full (batch, (GDestroyNotify) e_book_sqlite_search_data_free);
2627
2628         } while (success && n_results == EBSQL_UPGRADE_BATCH_SIZE);
2629
2630         g_free (uid);
2631
2632         /* Store the new locale & country code */
2633         if (success)
2634                 success = ebsql_exec_printf (
2635                         ebsql, "UPDATE folders SET countrycode = %Q WHERE folder_id = %Q",
2636                         NULL, NULL, NULL, error,
2637                         ebsql->priv->region_code, ebsql->priv->folderid);
2638
2639         if (success)
2640                 success = ebsql_exec_printf (
2641                         ebsql, "UPDATE folders SET lc_collate = %Q WHERE folder_id = %Q",
2642                         NULL, NULL, NULL, error,
2643                         ebsql->priv->locale, ebsql->priv->folderid);
2644
2645         return success;
2646 }
2647
2648 static gboolean
2649 ebsql_set_locale_internal (EBookSqlite *ebsql,
2650                            const gchar *locale,
2651                            GError **error)
2652 {
2653         EBookSqlitePrivate *priv = ebsql->priv;
2654         ECollator *collator;
2655
2656         g_return_val_if_fail (locale && locale[0], FALSE);
2657
2658         if (g_strcmp0 (priv->locale, locale) != 0) {
2659                 gchar *country_code = NULL;
2660
2661                 collator = e_collator_new_interpret_country (
2662                         locale, &country_code, error);
2663                 if (collator == NULL)
2664                         return FALSE;
2665
2666                 /* Assign region code parsed from the locale by ICU */
2667                 g_free (priv->region_code);
2668                 priv->region_code = country_code;
2669
2670                 /* Assign locale */
2671                 g_free (priv->locale);
2672                 priv->locale = g_strdup (locale);
2673
2674                 /* Assign collator */
2675                 if (ebsql->priv->collator)
2676                         e_collator_unref (ebsql->priv->collator);
2677                 ebsql->priv->collator = collator;
2678         }
2679
2680         return TRUE;
2681 }
2682
2683 /* Called with the lock held and inside a transaction */
2684 static gboolean
2685 ebsql_init_is_populated (EBookSqlite *ebsql,
2686                          gint previous_schema,
2687                          GError **error)
2688 {
2689         gboolean success = TRUE;
2690
2691         /* Schema 8 is when we moved from EBookSqlite */
2692         if (previous_schema >= 1 && previous_schema < 8) {
2693                 gint is_populated = 0;
2694
2695                 /* We need to hold on to the value of any previously set 'is_populated' flag */
2696                 success = ebsql_exec_printf (
2697                         ebsql, "SELECT is_populated FROM folders WHERE folder_id = %Q",
2698                         get_int_cb, &is_populated, NULL, error, ebsql->priv->folderid);
2699
2700                 if (success) {
2701                         /* We can't use e_book_sqlite_set_key_value_int() at this
2702                          * point as that would hold the access locks
2703                          */
2704                         success = ebsql_exec_printf (
2705                                 ebsql, "INSERT or REPLACE INTO keys (key, value, folder_id) values (%Q, %Q, %Q)",
2706                                 NULL, NULL, NULL, error,
2707                                 E_BOOK_SQL_IS_POPULATED_KEY,
2708                                 is_populated ? "1" : "0",
2709                                 ebsql->priv->folderid);
2710                 }
2711         }
2712
2713         return success;
2714 }
2715
2716 /* Called with the lock held and inside a transaction */
2717 static gboolean
2718 ebsql_init_locale (EBookSqlite *ebsql,
2719                    gint previous_schema,
2720                    gboolean already_exists,
2721                    GError **error)
2722 {
2723         gchar *stored_lc_collate = NULL;
2724         gchar *stored_region_code = NULL;
2725         const gchar *lc_collate = NULL;
2726         gboolean success = TRUE;
2727         gboolean relocalize_needed = FALSE;
2728
2729         /* Get the locale setting for this addressbook */
2730         if (already_exists) {
2731                 success = ebsql_exec_printf (
2732                         ebsql, "SELECT lc_collate FROM folders WHERE folder_id = %Q",
2733                         get_string_cb, &stored_lc_collate, NULL, error, ebsql->priv->folderid);
2734
2735                 if (success)
2736                         success = ebsql_exec_printf (
2737                                 ebsql, "SELECT countrycode FROM folders WHERE folder_id = %Q",
2738                                 get_string_cb, &stored_region_code, NULL, error, ebsql->priv->folderid);
2739
2740                 lc_collate = stored_lc_collate;
2741         }
2742
2743         /* When creating a new addressbook, or upgrading from a version
2744          * where we did not have any locale setting; default to system locale,
2745          * we must absolutely always have a locale set.
2746          */
2747         if (!lc_collate || !lc_collate[0])
2748                 lc_collate = setlocale (LC_COLLATE, NULL);
2749         if (!lc_collate || !lc_collate[0])
2750                 lc_collate = setlocale (LC_ALL, NULL);
2751         if (!lc_collate || !lc_collate[0])
2752                 lc_collate = "en_US.utf8";
2753
2754         /* Before touching any data, make sure we have a valid ECollator,
2755          * this will also resolve our region code
2756          */
2757         if (success)
2758                 success = ebsql_set_locale_internal (ebsql, lc_collate, error);
2759
2760         /* Check if we need to relocalize */
2761         if (success) {
2762                 /* Need to relocalize the whole thing if the schema has been upgraded to version 7 */
2763                 if (previous_schema >= 1 && previous_schema < 7)
2764                         relocalize_needed = TRUE;
2765
2766                 /* We may need to relocalize for a country code change */
2767                 else if (g_strcmp0 (ebsql->priv->region_code, stored_region_code) != 0)
2768                         relocalize_needed = TRUE;
2769         }
2770
2771         /* Reinsert all contacts with new locale & country code */
2772         if (success && relocalize_needed)
2773                 success = ebsql_upgrade (ebsql, EBSQL_CHANGE_LAST, error);
2774
2775         EBSQL_NOTE (
2776                 SCHEMA,
2777                 g_printerr (
2778                         "SCHEMA: Initialized locale as '%s' (%s)\n",
2779                         ebsql->priv->locale, success ? "success" : "failed"));
2780
2781         g_free (stored_region_code);
2782         g_free (stored_lc_collate);
2783
2784         return success;
2785 }
2786
2787 static EBookSqlite *
2788 ebsql_new_internal (const gchar *path,
2789                     ESource *source,
2790                     EbSqlVCardCallback vcard_callback,
2791                     EbSqlChangeCallback change_callback,
2792                     gpointer user_data,
2793                     GDestroyNotify user_data_destroy,
2794                     SummaryField *fields,
2795                     gint n_fields,
2796                     GCancellable *cancellable,
2797                     GError **error)
2798 {
2799         EBookSqlite *ebsql;
2800         gchar *dirname = NULL;
2801         gint previous_schema = 0;
2802         gboolean already_exists = FALSE;
2803         gboolean success = TRUE;
2804         GSList *introspected_columns = NULL;
2805
2806         g_return_val_if_fail (path != NULL, NULL);
2807
2808         EBSQL_LOCK_MUTEX (&dbcon_lock);
2809
2810         EBSQL_NOTE (
2811                 SCHEMA,
2812                 g_printerr ("SCHEMA: Creating new EBookSqlite at path '%s'\n", path));
2813
2814         ebsql = ebsql_ref_from_hash (path);
2815         if (ebsql) {
2816                 EBSQL_NOTE (SCHEMA, g_printerr ("SCHEMA: An EBookSqlite already existed\n"));
2817                 goto exit;
2818         }
2819
2820         ebsql = g_object_new (E_TYPE_BOOK_SQLITE, NULL);
2821         ebsql->priv->path = g_strdup (path);
2822         ebsql->priv->folderid = g_strdup (DEFAULT_FOLDER_ID);
2823         ebsql->priv->summary_fields = fields;
2824         ebsql->priv->n_summary_fields = n_fields;
2825         ebsql->priv->vcard_callback = vcard_callback;
2826         ebsql->priv->change_callback = change_callback;
2827         ebsql->priv->user_data = user_data;
2828         ebsql->priv->user_data_destroy = user_data_destroy;
2829         if (source != NULL)
2830                 ebsql->priv->source = g_object_ref (source);
2831         else
2832                 ebsql->priv->source = NULL;
2833
2834         EBSQL_NOTE (REF_COUNTS, g_printerr ("EBookSqlite initially created\n"));
2835
2836         /* Ensure existance of the directories leading up to 'path' */
2837         dirname = g_path_get_dirname (path);
2838         if (g_mkdir_with_parents (dirname, 0777) < 0) {
2839                 EBSQL_SET_ERROR (
2840                         error,
2841                         E_BOOK_SQLITE_ERROR_LOAD,
2842                         "Can not make parent directory: %s",
2843                         g_strerror (errno));
2844                 success = FALSE;
2845                 goto exit;
2846         }
2847
2848         /* The additional instance lock is unneccesarry because of the global
2849          * lock held here, but let's keep it locked because we hold it while
2850          * executing any SQLite code throughout this code
2851          */
2852         EBSQL_LOCK_MUTEX (&ebsql->priv->lock);
2853
2854         /* Initialize the SQLite (set some parameters and add some custom hooks) */
2855         if (!ebsql_init_sqlite (ebsql, path, error)) {
2856                 EBSQL_UNLOCK_MUTEX (&ebsql->priv->lock);
2857                 success = FALSE;
2858                 goto exit;
2859         }
2860
2861         /* Lets do it all atomically inside a single transaction */
2862         if (!ebsql_start_transaction (ebsql, EBSQL_LOCK_WRITE, cancellable, error)) {
2863                 EBSQL_UNLOCK_MUTEX (&ebsql->priv->lock);
2864                 success = FALSE;
2865                 goto exit;
2866         }
2867
2868         /* When loading addressbooks created by EBookBackendSqlite, we
2869          * need to fetch the 'folderid' which was in use for that existing
2870          * addressbook before introspecting it's summary and upgrading
2871          * the schema.
2872          */
2873         if (success)
2874                 success = ebsql_resolve_folderid (
2875                         ebsql,
2876                         &previous_schema,
2877                         &already_exists,
2878                         error);
2879
2880         /* Initialize main folders table, also retrieve the current
2881          * schema version if the table already exists
2882          */
2883         if (success)
2884                 success = ebsql_init_folders (ebsql, previous_schema, error);
2885
2886         /* Initialize the key/value table */
2887         if (success)
2888                 success = ebsql_init_keys (ebsql, error);
2889
2890         /* Determine if the addressbook already existed, and fill out
2891          * some information in the main folder table
2892          */
2893         if (success && !already_exists)
2894                 success = ebsql_add_folder (ebsql, error);
2895
2896         /* If the addressbook did exist, then check how it's configured.
2897          *
2898          * Let the existing summary information override the current
2899          * one asked for by our callers.
2900          *
2901          * Some summary fields are also adjusted for schema upgrades
2902          */
2903         if (success && already_exists)
2904                 success = ebsql_introspect_summary (
2905                         ebsql,
2906                         previous_schema,
2907                         &introspected_columns,
2908                         error);
2909
2910         /* Add the contacts table, ensure the right columns are defined
2911          * to handle our summary configuration
2912          */
2913         if (success)
2914                 success = ebsql_init_contacts (
2915                         ebsql,
2916                         introspected_columns,
2917                         error);
2918
2919         /* Add any auxiliary tables which we might need to support our
2920          * summary configuration.
2921          *
2922          * Any fields which represent a 'list-of-strings' require an
2923          * auxiliary table to store them in.
2924          */
2925         if (success)
2926                 success = ebsql_init_aux_tables (ebsql, previous_schema, error);
2927
2928         /* At this point we have resolved our schema, let's build our
2929          * precompiled statements, we might use them to re-insert contacts
2930          * in the next step
2931          */
2932         if (success)
2933                 success = ebsql_init_statements (ebsql, error);
2934
2935         /* When porting from older schemas, we need to port the old 'is-populated' flag */
2936         if (success)
2937                 success = ebsql_init_is_populated (ebsql, previous_schema, error);
2938
2939         /* Load / resolve the current locale setting
2940          *
2941          * Also perform the overall upgrade in this step
2942          * in the case that an upgrade happened, or a locale
2943          * change is detected... all rows need to be renormalized
2944          * for this.
2945          */
2946         if (success)
2947                 success = ebsql_init_locale (
2948                         ebsql, previous_schema,
2949                         already_exists, error);
2950
2951         if (success)
2952                 success = ebsql_commit_transaction (ebsql, error);
2953         else
2954                 /* The GError is already set. */
2955                 ebsql_rollback_transaction (ebsql, NULL);
2956
2957         /* Release the instance lock and register to the global hash */
2958         EBSQL_UNLOCK_MUTEX (&ebsql->priv->lock);
2959
2960         if (success)
2961                 ebsql_register_to_hash (ebsql, path);
2962
2963  exit:
2964
2965         /* Cleanup and exit */
2966         EBSQL_UNLOCK_MUTEX (&dbcon_lock);
2967
2968         /* If we failed somewhere, give up on creating the 'ebsql',
2969          * otherwise add it to the hash table
2970          */
2971         if (!success)
2972                 g_clear_object (&ebsql);
2973
2974         EBSQL_NOTE (
2975                 SCHEMA,
2976                 g_printerr (
2977                         "SCHEMA: %s the new EBookSqlite\n",
2978                         success ? "Successfully created" : "Failed to create"));
2979
2980         g_slist_free_full (introspected_columns, (GDestroyNotify) g_free);
2981         g_free (dirname);
2982
2983         return ebsql;
2984 }
2985
2986 /**********************************************************
2987  *                   Inserting Contacts                   *
2988  **********************************************************/
2989 static gchar *
2990 convert_phone (const gchar *normal,
2991                const gchar *region_code,
2992                gint *out_country_code)
2993 {
2994         EPhoneNumber *number = NULL;
2995         gchar *national_number = NULL;
2996         gint country_code = 0;
2997
2998         /* Don't warn about erronous phone number strings, it's a perfectly normal
2999          * use case for users to enter notes instead of phone numbers in the phone
3000          * number contact fields, such as "Ask Jenny for Lisa's phone number"
3001          */
3002         if (normal && e_phone_number_is_supported ())
3003                 number = e_phone_number_from_string (normal, region_code, NULL);
3004
3005         if (number) {
3006                 EPhoneNumberCountrySource source;
3007
3008                 national_number = e_phone_number_get_national_number (number);
3009                 country_code = e_phone_number_get_country_code (number, &source);
3010                 e_phone_number_free (number);
3011
3012                 if (source == E_PHONE_NUMBER_COUNTRY_FROM_DEFAULT)
3013                         country_code = 0;
3014         }
3015
3016         if (out_country_code)
3017                 *out_country_code = country_code;
3018
3019         return national_number;
3020 }
3021
3022 typedef struct {
3023         gint country_code;
3024         gchar *national;
3025 } E164Number;
3026
3027 static E164Number *
3028 ebsql_e164_number_new (gint country_code,
3029                        gchar *national)
3030 {
3031         E164Number *number = g_slice_new (E164Number);
3032
3033         number->country_code = country_code;
3034         number->national = g_strdup (national);
3035
3036         return number;
3037 }
3038
3039 static void
3040 ebsql_e164_number_free (E164Number *number)
3041 {
3042         if (number) {
3043                 g_free (number->national);
3044                 g_slice_free (E164Number, number);
3045         }
3046 }
3047
3048 static gint
3049 ebsql_e164_number_find (E164Number *number_a,
3050                         E164Number *number_b)
3051 {
3052         gint ret;
3053
3054         ret = number_a->country_code - number_b->country_code;
3055
3056         if (ret == 0)
3057                 ret = g_strcmp0 (
3058                         number_a->national,
3059                         number_b->national);
3060
3061         return ret;
3062 }
3063
3064 static GList *
3065 extract_e164_attribute_params (EContact *contact)
3066 {
3067         EVCard *vcard = E_VCARD (contact);
3068         GList *extracted = NULL;
3069         GList *attr_list;
3070
3071         for (attr_list = e_vcard_get_attributes (vcard); attr_list; attr_list = attr_list->next) {
3072                 EVCardAttribute *const attr = attr_list->data;
3073                 EVCardAttributeParam *param = NULL;
3074                 GList *param_list, *values, *l;
3075                 gchar *this_national = NULL;
3076                 gint this_country = 0;
3077
3078                 /* We only attach E164 parameters to TEL attributes. */
3079                 if (strcmp (e_vcard_attribute_get_name (attr), EVC_TEL) != 0)
3080                         continue;
3081
3082                 /* Find already exisiting parameter, so that we can reuse it. */
3083                 for (param_list = e_vcard_attribute_get_params (attr); param_list; param_list = param_list->next) {
3084                         if (strcmp (e_vcard_attribute_param_get_name (param_list->data), EVC_X_E164) == 0) {
3085                                 param = param_list->data;
3086                                 break;
3087                         }
3088                 }
3089
3090                 if (!param)
3091                         continue;
3092
3093                 values = e_vcard_attribute_param_get_values (param);
3094                 for (l = values; l; l = l->next) {
3095                         const gchar *value = l->data;
3096
3097                         if (value[0] == '+')
3098                                 this_country = g_ascii_strtoll (&value[1], NULL, 10);
3099                         else if (this_national == NULL)
3100                                 this_national = g_strdup (value);
3101                 }
3102
3103                 if (this_national) {
3104                         E164Number *number;
3105
3106                         EBSQL_NOTE (
3107                                 CONVERT_E164,
3108                                 g_printerr (
3109                                         "Extracted e164 number from '%s' with "
3110                                         "country = %d national = %s\n",
3111                                         (gchar *) e_contact_get_const (contact, E_CONTACT_UID),
3112                                         this_country, this_national));
3113
3114                         number = ebsql_e164_number_new (
3115                                 this_country, this_national);
3116                         extracted = g_list_prepend (extracted, number);
3117                 }
3118
3119                 g_free (this_national);
3120
3121                 /* Clear the values, we'll insert new ones */
3122                 e_vcard_attribute_param_remove_values (param);
3123                 e_vcard_attribute_remove_param (attr, EVC_X_E164);
3124         }
3125
3126         EBSQL_NOTE (
3127                 CONVERT_E164,
3128                 g_printerr (
3129                         "Extracted %d numbers from '%s'\n",
3130                         g_list_length (extracted),
3131                         (gchar *) e_contact_get_const (contact, E_CONTACT_UID)));
3132
3133         return extracted;
3134 }
3135
3136 static gboolean
3137 update_e164_attribute_params (EBookSqlite *ebsql,
3138                               EContact *contact,
3139                               const gchar *default_region)
3140 {
3141         GList *original_numbers = NULL;
3142         GList *attr_list;
3143         gboolean changed = FALSE;
3144         gint n_numbers = 0;
3145         EVCard *vcard = E_VCARD (contact);
3146
3147         original_numbers = extract_e164_attribute_params (contact);
3148
3149         for (attr_list = e_vcard_get_attributes (vcard); attr_list; attr_list = attr_list->next) {
3150                 EVCardAttribute *const attr = attr_list->data;
3151                 EVCardAttributeParam *param = NULL;
3152                 const gchar *original_number = NULL;
3153                 gchar *country_string;
3154                 GList *values;
3155                 E164Number number = { 0, NULL };
3156
3157                 /* We only attach E164 parameters to TEL attributes. */
3158                 if (strcmp (e_vcard_attribute_get_name (attr), EVC_TEL) != 0)
3159                         continue;
3160
3161                 /* Fetch the TEL value */
3162                 values = e_vcard_attribute_get_values (attr);
3163
3164                 /* Compute E164 number based on the TEL value */
3165                 if (values && values->data) {
3166                         original_number = (const gchar *) values->data;
3167                         number.national = convert_phone (
3168                                 original_number,
3169                                 ebsql->priv->region_code,
3170                                 &(number.country_code));
3171                 }
3172
3173                 if (number.national == NULL)
3174                         continue;
3175
3176                 /* Count how many we successfully parsed in this region code */
3177                 n_numbers++;
3178
3179                 /* Check if we have a differing e164 number, if there is no match
3180                  * in the old existing values then the vcard changed
3181                  */
3182                 if (!g_list_find_custom (original_numbers, &number,
3183                                          (GCompareFunc) ebsql_e164_number_find))
3184                         changed = TRUE;
3185
3186                 if (number.country_code != 0)
3187                         country_string = g_strdup_printf ("+%d", number.country_code);
3188                 else
3189                         country_string = g_strdup ("");
3190
3191                 param = e_vcard_attribute_param_new (EVC_X_E164);
3192                 e_vcard_attribute_add_param (attr, param);
3193
3194                 /* Assign the parameter values. It seems odd that we revert
3195                  * the order of NN and CC, but at least EVCard's parser doesn't
3196                  * permit an empty first param value. Which of course could be
3197                  * fixed - in order to create a nice potential IOP problem with
3198                  ** other vCard parsers. */
3199                 e_vcard_attribute_param_add_values (param, number.national, country_string, NULL);
3200
3201                 EBSQL_NOTE (
3202                         CONVERT_E164,
3203                         g_printerr (
3204                                 "Converted '%s' to e164 number with country = %d "
3205                                 "national = %s for '%s' (changed %s)\n",
3206                                 original_number, number.country_code, number.national,
3207                                 (gchar *) e_contact_get_const (contact, E_CONTACT_UID),
3208                                 changed ? "yes" : "no"));
3209
3210                 g_free (number.national);
3211                 g_free (country_string);
3212         }
3213
3214         if (!changed &&
3215             n_numbers != g_list_length (original_numbers))
3216                 changed = TRUE;
3217
3218         EBSQL_NOTE (
3219                 CONVERT_E164,
3220                 g_printerr (
3221                         "Converted %d e164 numbers for '%s' which previously had %d e164 numbers\n",
3222                         n_numbers,
3223                         (gchar *) e_contact_get_const (contact, E_CONTACT_UID),
3224                         g_list_length (original_numbers)));
3225
3226         g_list_free_full (original_numbers, (GDestroyNotify) ebsql_e164_number_free);
3227
3228         return changed;
3229 }
3230
3231 static sqlite3_stmt *
3232 ebsql_prepare_multi_delete (EBookSqlite *ebsql,
3233                             SummaryField *field,
3234                             GError **error)
3235 {
3236         sqlite3_stmt *stmt = NULL;
3237         gchar *stmt_str;
3238
3239         stmt_str = sqlite3_mprintf ("DELETE FROM %Q WHERE uid = :uid", field->aux_table);
3240         stmt = ebsql_prepare_statement (ebsql, stmt_str, error);
3241         sqlite3_free (stmt_str);
3242
3243         return stmt;
3244 }
3245
3246 static gboolean
3247 ebsql_run_multi_delete (EBookSqlite *ebsql,
3248                         SummaryField *field,
3249                         const gchar *uid,
3250                         GError **error)
3251 {
3252         sqlite3_stmt *stmt;
3253         gint ret;
3254
3255         stmt = g_hash_table_lookup (ebsql->priv->multi_deletes, GUINT_TO_POINTER (field->field_id));
3256
3257         /* This can return an error if a previous call to sqlite3_step() had errors,
3258          * so let's just ignore any error in this case
3259          */
3260         sqlite3_reset (stmt);
3261
3262         /* Clear all previously set values */
3263         ret = sqlite3_clear_bindings (stmt);
3264
3265         /* Set the UID host parameter statically */
3266         if (ret == SQLITE_OK)
3267                 ret = sqlite3_bind_text (stmt, 1, uid, -1, SQLITE_STATIC);
3268
3269         /* Run the statement */
3270         return ebsql_complete_statement (ebsql, stmt, ret, error);
3271 }
3272
3273 static sqlite3_stmt *
3274 ebsql_prepare_multi_insert (EBookSqlite *ebsql,
3275                             SummaryField *field,
3276                             GError **error)
3277 {
3278         sqlite3_stmt *stmt = NULL;
3279         GString *string;
3280
3281         string = g_string_sized_new (INSERT_MULTI_STMT_BYTES);
3282         ebsql_string_append_printf (string, "INSERT INTO %Q (uid, value", field->aux_table);
3283
3284         if ((field->index & INDEX_FLAG (SUFFIX)) != 0)
3285                 g_string_append (string, ", value_" EBSQL_SUFFIX_REVERSE);
3286
3287         if ((field->index & INDEX_FLAG (PHONE)) != 0) {
3288                 g_string_append (string, ", value_" EBSQL_SUFFIX_PHONE);
3289                 g_string_append (string, ", value_" EBSQL_SUFFIX_COUNTRY);
3290         }
3291
3292         if ((field->index & INDEX_FLAG (TRANSLIT)) != 0)
3293                 g_string_append (string, ", value_" EBSQL_SUFFIX_TRANSLIT);
3294
3295         g_string_append (string, ") VALUES (:uid, :value");
3296
3297         if ((field->index & INDEX_FLAG (SUFFIX)) != 0)
3298                 g_string_append (string, ", :value_" EBSQL_SUFFIX_REVERSE);
3299
3300         if ((field->index & INDEX_FLAG (PHONE)) != 0) {
3301                 g_string_append (string, ", :value_" EBSQL_SUFFIX_PHONE);
3302                 g_string_append (string, ", :value_" EBSQL_SUFFIX_COUNTRY);
3303         }
3304
3305         if ((field->index & INDEX_FLAG (TRANSLIT)) != 0)
3306                 g_string_append (string, ", :value_" EBSQL_SUFFIX_TRANSLIT);
3307
3308         g_string_append_c (string, ')');
3309
3310         stmt = ebsql_prepare_statement (ebsql, string->str, error);
3311         g_string_free (string, TRUE);
3312
3313         return stmt;
3314 }
3315
3316 static gboolean
3317 ebsql_run_multi_insert_one (EBookSqlite *ebsql,
3318                             sqlite3_stmt *stmt,
3319                             SummaryField *field,
3320                             const gchar *uid,
3321                             const gchar *value,
3322                             GError **error)
3323 {
3324         gchar *normal = e_util_utf8_normalize (value);
3325         gchar *tmp, *str;
3326         gint ret, param_idx = 1;
3327
3328         /* :uid */
3329         ret = sqlite3_bind_text (stmt, param_idx++, uid, -1, SQLITE_STATIC);
3330
3331         if (ret == SQLITE_OK)  /* :value */
3332                 ret = sqlite3_bind_text (stmt, param_idx++, normal, -1, g_free);
3333
3334         if (ret == SQLITE_OK && (field->index & INDEX_FLAG (SUFFIX)) != 0) {
3335                 if (normal)
3336                         str = g_utf8_strreverse (normal, -1);
3337                 else
3338                         str = NULL;
3339
3340                 /* :value_reverse */
3341                 ret = sqlite3_bind_text (stmt, param_idx++, str, -1, g_free);
3342         }
3343
3344         if (ret == SQLITE_OK && (field->index & INDEX_FLAG (PHONE)) != 0) {
3345                 gint country_code;
3346
3347                 str = convert_phone (
3348                         normal, ebsql->priv->region_code,
3349                         &country_code);
3350
3351                 /* :value_phone */
3352                 ret = sqlite3_bind_text (stmt, param_idx++, str, -1, g_free);
3353
3354                 /* :value_country */
3355                 if (ret == SQLITE_OK)
3356                         sqlite3_bind_int (stmt, param_idx++, country_code);
3357
3358         }
3359
3360         if (ret == SQLITE_OK && (field->index & INDEX_FLAG (TRANSLIT)) != 0) {
3361
3362                 if (value) {
3363                         tmp = e_transliterator_transliterate (ebsql->priv->transliterator, value);
3364                         str = e_util_utf8_normalize (tmp);
3365                         g_free (tmp);
3366                 } else {
3367                         str = NULL;
3368                 }
3369
3370                 /* :value_translit */
3371                 ret = sqlite3_bind_text (stmt, param_idx++, str, -1, g_free);
3372         }
3373
3374         /* Run the statement */
3375         return ebsql_complete_statement (ebsql, stmt, ret, error);
3376 }
3377
3378 static gboolean
3379 ebsql_run_multi_insert (EBookSqlite *ebsql,
3380                         SummaryField *field,
3381                         const gchar *uid,
3382                         EContact *contact,
3383                         GError **error)
3384 {
3385         sqlite3_stmt *stmt;
3386         GList *values, *l;
3387         gboolean success = TRUE;
3388
3389         stmt = g_hash_table_lookup (ebsql->priv->multi_inserts, GUINT_TO_POINTER (field->field_id));
3390         values = e_contact_get (contact, field->field_id);
3391
3392         for (l = values; success && l != NULL; l = l->next) {
3393                 gchar *value = (gchar *) l->data;
3394
3395                 success = ebsql_run_multi_insert_one (
3396                         ebsql, stmt, field, uid, value, error);
3397         }
3398
3399         /* Free the list of allocated strings */
3400         e_contact_attr_list_free (values);
3401
3402         return success;
3403 }
3404
3405 static sqlite3_stmt *
3406 ebsql_prepare_insert (EBookSqlite *ebsql,
3407                       gboolean replace_existing,
3408                       GError **error)
3409 {
3410         sqlite3_stmt *stmt;
3411         GString *string;
3412         gint i;
3413
3414         string = g_string_new ("");
3415         if (replace_existing)
3416                 ebsql_string_append_printf (
3417                         string, "INSERT or REPLACE INTO %Q (",
3418                         ebsql->priv->folderid);
3419         else
3420                 ebsql_string_append_printf (
3421                         string, "INSERT or FAIL INTO %Q (",
3422                         ebsql->priv->folderid);
3423
3424         /*
3425          * First specify the column names for the insert, since it's possible we
3426          * upgraded the DB and cannot be sure the order of the columns are ordered
3427          * just how we like them to be.
3428          */
3429         for (i = 0; i < ebsql->priv->n_summary_fields; i++) {
3430                 SummaryField *field = &(ebsql->priv->summary_fields[i]);
3431
3432                 /* Multi values go into a separate table/statement */
3433                 if (field->type != E_TYPE_CONTACT_ATTR_LIST) {
3434
3435                         /* Only add a ", " before every field except the first,
3436                          * this will not break because the first 2 fields (UID & REV)
3437                          * are string fields.
3438                          */
3439                         if (i > 0)
3440                                 g_string_append (string, ", ");
3441
3442                         g_string_append (string, field->dbname);
3443                 }
3444
3445                 if (field->type == G_TYPE_STRING) {
3446
3447                         if ((field->index & INDEX_FLAG (SORT_KEY)) != 0) {
3448                                 g_string_append (string, ", ");
3449                                 g_string_append (string, field->dbname);
3450                                 g_string_append (string, "_" EBSQL_SUFFIX_SORT_KEY);
3451                         }
3452
3453                         if ((field->index & INDEX_FLAG (SUFFIX)) != 0) {
3454                                 g_string_append (string, ", ");
3455                                 g_string_append (string, field->dbname);
3456                                 g_string_append (string, "_" EBSQL_SUFFIX_REVERSE);
3457                         }
3458
3459                         if ((field->index & INDEX_FLAG (PHONE)) != 0) {
3460
3461                                 g_string_append (string, ", ");
3462                                 g_string_append (string, field->dbname);
3463                                 g_string_append (string, "_" EBSQL_SUFFIX_PHONE);
3464
3465                                 g_string_append (string, ", ");
3466                                 g_string_append (string, field->dbname);
3467                                 g_string_append (string, "_" EBSQL_SUFFIX_COUNTRY);
3468                         }
3469
3470                         if ((field->index & INDEX_FLAG (TRANSLIT)) != 0) {
3471                                 g_string_append (string, ", ");
3472                                 g_string_append (string, field->dbname);
3473                                 g_string_append (string, "_" EBSQL_SUFFIX_TRANSLIT);
3474                         }
3475                 }
3476         }
3477         g_string_append (string, ", vcard, bdata)");
3478
3479         /*
3480          * Now specify values for all of the column names we specified.
3481          */
3482         g_string_append (string, " VALUES (");
3483         for (i = 0; i < ebsql->priv->n_summary_fields; i++) {
3484                 SummaryField *field = &(ebsql->priv->summary_fields[i]);
3485
3486                 if (field->type != E_TYPE_CONTACT_ATTR_LIST) {
3487                         /* Only add a ", " before every field except the first,
3488                          * this will not break because the first 2 fields (UID & REV)
3489                          * are string fields.
3490                          */
3491                         if (i > 0)
3492                                 g_string_append (string, ", ");
3493                 }
3494
3495                 if (field->type == G_TYPE_STRING || field->type == G_TYPE_BOOLEAN) {
3496
3497                         g_string_append_c (string, ':');
3498                         g_string_append (string, field->dbname);
3499
3500                         if ((field->index & INDEX_FLAG (SORT_KEY)) != 0)
3501                                 g_string_append_printf (string, ", :%s_" EBSQL_SUFFIX_SORT_KEY, field->dbname);
3502
3503                         if ((field->index & INDEX_FLAG (SUFFIX)) != 0)
3504                                 g_string_append_printf (string, ", :%s_" EBSQL_SUFFIX_REVERSE, field->dbname);
3505
3506                         if ((field->index & INDEX_FLAG (PHONE)) != 0) {
3507                                 g_string_append_printf (string, ", :%s_" EBSQL_SUFFIX_PHONE, field->dbname);
3508                                 g_string_append_printf (string, ", :%s_" EBSQL_SUFFIX_COUNTRY, field->dbname);
3509                         }
3510
3511                         if ((field->index & INDEX_FLAG (TRANSLIT)) != 0)
3512                                 g_string_append_printf (string, ", :%s_" EBSQL_SUFFIX_TRANSLIT, field->dbname);
3513
3514                 } else if (field->type != E_TYPE_CONTACT_ATTR_LIST)
3515                         g_warn_if_reached ();
3516         }
3517
3518         g_string_append (string, ", :vcard, :bdata)");
3519
3520         stmt = ebsql_prepare_statement (ebsql, string->str, error);
3521         g_string_free (string, TRUE);
3522
3523         return stmt;
3524 }
3525
3526 static gboolean
3527 ebsql_init_statements (EBookSqlite *ebsql,
3528                        GError **error)
3529 {
3530         sqlite3_stmt *stmt;
3531         gint i;
3532
3533         ebsql->priv->insert_stmt = ebsql_prepare_insert (ebsql, FALSE, error);
3534         if (!ebsql->priv->insert_stmt)
3535                 goto preparation_failed;
3536
3537         ebsql->priv->replace_stmt = ebsql_prepare_insert (ebsql, TRUE, error);
3538         if (!ebsql->priv->replace_stmt)
3539                 goto preparation_failed;
3540
3541         ebsql->priv->multi_deletes =
3542                 g_hash_table_new_full (
3543                         g_direct_hash, g_direct_equal,
3544                         NULL,
3545                         (GDestroyNotify) sqlite3_finalize);
3546         ebsql->priv->multi_inserts =
3547                 g_hash_table_new_full (
3548                         g_direct_hash, g_direct_equal,
3549                         NULL,
3550                         (GDestroyNotify) sqlite3_finalize);
3551
3552         for (i = 0; i < ebsql->priv->n_summary_fields; i++) {
3553                 SummaryField *field = &(ebsql->priv->summary_fields[i]);
3554
3555                 if (field->type != E_TYPE_CONTACT_ATTR_LIST)
3556                         continue;
3557
3558                 stmt = ebsql_prepare_multi_insert (ebsql, field, error);
3559                 if (!stmt)
3560                         goto preparation_failed;
3561
3562                 g_hash_table_insert (
3563                         ebsql->priv->multi_inserts,
3564                         GUINT_TO_POINTER (field->field_id),
3565                         stmt);
3566
3567                 stmt = ebsql_prepare_multi_delete (ebsql, field, error);
3568                 if (!stmt)
3569                         goto preparation_failed;
3570
3571                 g_hash_table_insert (
3572                         ebsql->priv->multi_deletes,
3573                         GUINT_TO_POINTER (field->field_id),
3574                         stmt);
3575         }
3576
3577         return TRUE;
3578
3579  preparation_failed:
3580
3581         return FALSE;
3582 }
3583
3584 static gboolean
3585 ebsql_run_insert (EBookSqlite *ebsql,
3586                   gboolean replace,
3587                   EContact *contact,
3588                   gchar *vcard,
3589                   const gchar *extra,
3590                   GError **error)
3591 {
3592         EBookSqlitePrivate *priv;
3593         sqlite3_stmt *stmt;
3594         gint i, param_idx;
3595         gint ret;
3596         gboolean success;
3597         GError *local_error = NULL;
3598
3599         priv = ebsql->priv;
3600
3601         if (replace)
3602                 stmt = ebsql->priv->replace_stmt;
3603         else
3604                 stmt = ebsql->priv->insert_stmt;
3605
3606         /* This can return an error if a previous call to sqlite3_step() had errors,
3607          * so let's just ignore any error in this case
3608          */
3609         sqlite3_reset (stmt);
3610
3611         /* Clear all previously set values */
3612         ret = sqlite3_clear_bindings (stmt);
3613
3614         for (i = 0, param_idx = 1; ret == SQLITE_OK && i < ebsql->priv->n_summary_fields; i++) {
3615                 SummaryField *field = &(ebsql->priv->summary_fields[i]);
3616
3617                 if (field->type == G_TYPE_STRING) {
3618                         gchar *val;
3619                         gchar *normal;
3620                         gchar *str;
3621
3622                         val = e_contact_get (contact, field->field_id);
3623
3624                         /* Special exception, never normalize/localize the UID or REV string */
3625                         if (field->field_id != E_CONTACT_UID &&
3626                             field->field_id != E_CONTACT_REV) {
3627                                 normal = e_util_utf8_normalize (val);
3628                         } else
3629                                 normal = g_strdup (val);
3630
3631                         /* Takes ownership of 'normal' */
3632                         ret = sqlite3_bind_text (stmt, param_idx++, normal, -1, g_free);
3633
3634                         if (ret == SQLITE_OK &&
3635                             (field->index & INDEX_FLAG (SORT_KEY)) != 0) {
3636                                 if (val)
3637                                         str = e_collator_generate_key (ebsql->priv->collator, val, NULL);
3638                                 else
3639                                         str = g_strdup ("");
3640
3641                                 ret = sqlite3_bind_text (stmt, param_idx++, str, -1, g_free);
3642                         }
3643
3644                         if (ret == SQLITE_OK &&
3645                             (field->index & INDEX_FLAG (SUFFIX)) != 0) {
3646                                 if (normal)
3647                                         str = g_utf8_strreverse (normal, -1);
3648                                 else
3649                                         str = NULL;
3650
3651                                 ret = sqlite3_bind_text (stmt, param_idx++, str, -1, g_free);
3652                         }
3653
3654                         if (ret == SQLITE_OK &&
3655                             (field->index & INDEX_FLAG (PHONE)) != 0) {
3656                                 gint country_code;
3657
3658                                 str = convert_phone (
3659                                         normal, ebsql->priv->region_code,
3660                                         &country_code);
3661
3662                                 ret = sqlite3_bind_text (stmt, param_idx++, str, -1, g_free);
3663                                 if (ret == SQLITE_OK)
3664                                         sqlite3_bind_int (stmt, param_idx++, country_code);
3665                         }
3666
3667                         if (ret == SQLITE_OK &&
3668                             (field->index & INDEX_FLAG (TRANSLIT)) != 0) {
3669
3670                                 if (val) {
3671                                         gchar *tmp = e_transliterator_transliterate (ebsql->priv->transliterator, val);
3672                                         str = e_util_utf8_normalize (tmp);
3673                                         g_free (tmp);
3674                                 } else {
3675                                         str = NULL;
3676                                 }
3677
3678                                 ret = sqlite3_bind_text (stmt, param_idx++, str, -1, g_free);
3679                         }
3680
3681                         g_free (val);
3682                 } else if (field->type == G_TYPE_BOOLEAN) {
3683                         gboolean val;
3684
3685                         val = e_contact_get (contact, field->field_id) ? TRUE : FALSE;
3686
3687                         ret = sqlite3_bind_int (stmt, param_idx++, val ? 1 : 0);
3688                 } else if (field->type != E_TYPE_CONTACT_ATTR_LIST)
3689                         g_warn_if_reached ();
3690         }
3691
3692         if (ret == SQLITE_OK) {
3693
3694                 EBSQL_NOTE (
3695                         INSERT,
3696                         g_printerr (
3697                                 "Inserting vcard for contact with UID '%s'\n%s\n",
3698                                 (gchar *) e_contact_get_const (contact, E_CONTACT_UID),
3699                                 vcard ? vcard : "(no vcard)"));
3700
3701                 /* If we have a priv->vcard_callback, then it's a shallow addressbook
3702                  * and we don't populate the vcard column, need to free it anyway
3703                  */
3704                 if (priv->vcard_callback != NULL) {
3705                         g_free (vcard);
3706                         vcard = NULL;
3707                 }
3708
3709                 ret = sqlite3_bind_text (stmt, param_idx++, vcard, -1, g_free);
3710         }
3711
3712         /* The extra data */
3713         if (ret == SQLITE_OK)
3714                 ret = sqlite3_bind_text (stmt, param_idx++, g_strdup (extra), -1, g_free);
3715
3716         /* Run the statement */
3717         success = ebsql_complete_statement (ebsql, stmt, ret, &local_error);
3718
3719         EBSQL_NOTE (
3720                 INSERT,
3721                 g_printerr (
3722                         "%s contact with UID '%s' and extra data '%s' vcard: %s (error: %s)\n",
3723                         success ? "Succesfully inserted" : "Failed to insert",
3724                         (gchar *) e_contact_get_const (contact, E_CONTACT_UID), extra,
3725                         vcard ? "yes" : "no",
3726                         local_error ? local_error->message : "(none)"));
3727
3728         if (!success)
3729                 g_propagate_error (error, local_error);
3730
3731         return success;
3732 }
3733
3734 static gboolean
3735 ebsql_insert_contact (EBookSqlite *ebsql,
3736                       EbSqlChangeType change_type,
3737                       EContact *contact,
3738                       const gchar *original_vcard,
3739                       const gchar *extra,
3740                       gboolean replace,
3741                       GError **error)
3742 {
3743         EBookSqlitePrivate *priv;
3744         gboolean e164_changed = FALSE;
3745         gboolean success;
3746         gchar *uid, *vcard = NULL;
3747
3748         priv = ebsql->priv;
3749         uid = e_contact_get (contact, E_CONTACT_UID);
3750
3751         /* Update E.164 parameters in vcard if needed */
3752         e164_changed = update_e164_attribute_params (
3753                 ebsql, contact, priv->region_code);
3754
3755         if (e164_changed || original_vcard == NULL) {
3756
3757                 /* Generate a new one if it changed (or if we don't have one) */
3758                 vcard = e_vcard_to_string (E_VCARD (contact), EVC_FORMAT_VCARD_30);
3759
3760                 if (e164_changed &&
3761                     change_type != EBSQL_CHANGE_LAST &&
3762                     ebsql->priv->change_callback)
3763                         ebsql->priv->change_callback (change_type,
3764                                                       uid, extra, vcard,
3765                                                       ebsql->priv->user_data);
3766         } else {
3767
3768                 vcard = g_strdup (original_vcard);
3769         }
3770
3771         /* This actually consumes 'vcard' */
3772         success = ebsql_run_insert (ebsql, replace, contact, vcard, extra, error);
3773
3774         /* Update attribute list table */
3775         if (success) {
3776                 gint i;
3777
3778                 for (i = 0; success && i < priv->n_summary_fields; i++) {
3779                         SummaryField *field = &(ebsql->priv->summary_fields[i]);
3780
3781                         if (field->type != E_TYPE_CONTACT_ATTR_LIST)
3782                                 continue;
3783
3784                         success = ebsql_run_multi_delete (
3785                                 ebsql, field, uid, error);
3786
3787                         if (success)
3788                                 success = ebsql_run_multi_insert (
3789                                         ebsql, field, uid, contact, error);
3790                 }
3791         }
3792
3793         g_free (uid);
3794
3795         return success;
3796 }
3797
3798 /***************************************************************
3799  * Structures and utilities for preflight and query generation *
3800  ***************************************************************/
3801
3802 /* This enumeration is ordered by severity, higher values
3803  * of PreflightStatus take precedence in error reporting.
3804  */
3805 typedef enum {
3806         PREFLIGHT_OK = 0,
3807         PREFLIGHT_LIST_ALL,
3808         PREFLIGHT_NOT_SUMMARIZED,
3809         PREFLIGHT_INVALID,
3810         PREFLIGHT_UNSUPPORTED,
3811 } PreflightStatus;
3812
3813 #define EBSQL_STATUS_STR(status) \
3814         ((status) == PREFLIGHT_OK ? "Ok" : \
3815          (status) == PREFLIGHT_LIST_ALL ? "List all" : \
3816          (status) == PREFLIGHT_NOT_SUMMARIZED ? "Not Summarized" : \
3817          (status) == PREFLIGHT_INVALID ? "Invalid" : \
3818          (status) == PREFLIGHT_UNSUPPORTED ? "Unsupported" : "(unknown status)")
3819
3820 /* Whether we can satisfy the constraints or whether we
3821  * need to do a fallback, we still need to call
3822  * ebsql_generate_constraints()
3823  */
3824 #define EBSQL_STATUS_GEN_CONSTRAINTS(status) \
3825         ((status) == PREFLIGHT_OK || \
3826          (status) == PREFLIGHT_NOT_SUMMARIZED)
3827
3828 /* Internal extension of the EBookQueryTest enumeration */
3829 enum {
3830         /* 'exists' is a supported query on a field, but not part of EBookQueryTest */
3831         BOOK_QUERY_EXISTS = E_BOOK_QUERY_LAST,
3832
3833         /* From here the compound types start */
3834         BOOK_QUERY_SUB_AND,
3835         BOOK_QUERY_SUB_OR,
3836         BOOK_QUERY_SUB_NOT,
3837         BOOK_QUERY_SUB_END,
3838
3839         BOOK_QUERY_SUB_FIRST = BOOK_QUERY_SUB_AND,
3840 };
3841
3842 #define EBSQL_QUERY_TYPE_STR(query) \
3843         ((query) == BOOK_QUERY_EXISTS ? "exists" : \
3844          (query) == BOOK_QUERY_SUB_AND ? "AND" : \
3845          (query) == BOOK_QUERY_SUB_OR ? "OR" : \
3846          (query) == BOOK_QUERY_SUB_NOT ? "NOT" : \
3847          (query) == BOOK_QUERY_SUB_END ? "END" : \
3848          (query) == E_BOOK_QUERY_IS ? "is" : \
3849          (query) == E_BOOK_QUERY_CONTAINS ? "contains" : \
3850          (query) == E_BOOK_QUERY_BEGINS_WITH ? "begins-with" : \
3851          (query) == E_BOOK_QUERY_ENDS_WITH ? "ends-with" : \
3852          (query) == E_BOOK_QUERY_EQUALS_PHONE_NUMBER ? "eqphone" : \
3853          (query) == E_BOOK_QUERY_EQUALS_NATIONAL_PHONE_NUMBER ? "eqphone-national" : \
3854          (query) == E_BOOK_QUERY_EQUALS_SHORT_PHONE_NUMBER ? "eqphone-short" : \
3855          (query) == E_BOOK_QUERY_REGEX_NORMAL ? "regex-normal" : \
3856          (query) == E_BOOK_QUERY_REGEX_RAW ? "regex-raw" : \
3857          (query) == E_BOOK_QUERY_TRANSLIT_IS    ? "translit-is" : \
3858          (query) == E_BOOK_QUERY_TRANSLIT_CONTAINS ? "translit-contains" : \
3859          (query) == E_BOOK_QUERY_TRANSLIT_BEGINS_WITH ? "translit-begins-with" : \
3860          (query) == E_BOOK_QUERY_TRANSLIT_ENDS_WITH ? "translit-ends-with" : "unknown")
3861
3862 #define EBSQL_FIELD_ID_STR(field_id) \
3863         ((field_id) == E_CONTACT_FIELD_LAST ? "x-evolution-any-field" : \
3864          (field_id) == 0 ? "(not an EContactField)" : \
3865          e_contact_field_name (field_id))
3866
3867 #define IS_QUERY_PHONE(query) \
3868         ((query) == E_BOOK_QUERY_EQUALS_PHONE_NUMBER || \
3869          (query) == E_BOOK_QUERY_EQUALS_NATIONAL_PHONE_NUMBER || \
3870          (query) == E_BOOK_QUERY_EQUALS_SHORT_PHONE_NUMBER)
3871
3872 typedef struct {
3873         guint          query; /* EBookQueryTest (extended) */
3874 } QueryElement;
3875
3876 typedef struct {
3877         guint          query; /* EBookQueryTest (extended) */
3878 } QueryDelimiter;
3879
3880 typedef struct {
3881         guint          query;          /* EBookQueryTest (extended) */
3882
3883         EContactField  field_id;       /* The EContactField to compare */
3884         SummaryField  *field;          /* The summary field for 'field' */
3885         gchar         *value;          /* The value to compare with */
3886
3887 } QueryFieldTest;
3888
3889 typedef struct {
3890         guint          query;          /* EBookQueryTest (extended) */
3891
3892         /* Common fields from QueryFieldTest */
3893         EContactField  field_id;       /* The EContactField to compare */
3894         SummaryField  *field;          /* The summary field for 'field' */
3895         gchar         *value;          /* The value to compare with */
3896
3897         /* Extension */
3898         gchar         *region;   /* Region code from the query input */
3899         gchar         *national; /* Parsed national number */
3900         gint           country;  /* Parsed country code */
3901 } QueryPhoneTest;
3902
3903 /* Stack initializer for the PreflightContext struct below */
3904 #define PREFLIGHT_CONTEXT_INIT { PREFLIGHT_OK, NULL, 0 }
3905
3906 typedef struct {
3907         PreflightStatus  status;         /* result status */
3908         GPtrArray       *constraints;    /* main query */
3909         guint64          aux_mask;       /* Bitmask of which auxiliary tables are needed in the query */
3910 } PreflightContext;
3911
3912 static QueryElement *
3913 query_delimiter_new (guint query)
3914 {
3915         QueryDelimiter *delim;
3916
3917         g_return_val_if_fail (query >= BOOK_QUERY_SUB_FIRST, NULL);
3918
3919         delim = g_slice_new (QueryDelimiter);
3920         delim->query = query;
3921
3922         return (QueryElement *) delim;
3923 }
3924
3925 static QueryFieldTest *
3926 query_field_test_new (guint query,
3927                       EContactField field)
3928 {
3929         QueryFieldTest *test;
3930
3931         g_return_val_if_fail (query < BOOK_QUERY_SUB_FIRST, NULL);
3932         g_return_val_if_fail (IS_QUERY_PHONE (query) == FALSE, NULL);
3933
3934         test = g_slice_new (QueryFieldTest);
3935         test->query = query;
3936         test->field_id = field;
3937
3938         /* Instead of g_slice_new0, NULL them out manually */
3939         test->field = NULL;
3940         test->value = NULL;
3941
3942         return test;
3943 }
3944
3945 static QueryPhoneTest *
3946 query_phone_test_new (guint query,
3947                       EContactField field)
3948 {
3949         QueryPhoneTest *test;
3950
3951         g_return_val_if_fail (IS_QUERY_PHONE (query), NULL);
3952
3953         test = g_slice_new (QueryPhoneTest);
3954         test->query = query;
3955         test->field_id = field;
3956
3957         /* Instead of g_slice_new0, NULL them out manually */
3958         test->field = NULL;
3959         test->value = NULL;
3960
3961         /* Extra QueryPhoneTest fields */
3962         test->region = NULL;
3963         test->national = NULL;
3964         test->country = 0;
3965
3966         return test;
3967 }
3968
3969 static void
3970 query_element_free (QueryElement *element)
3971 {
3972         if (element) {
3973
3974                 if (element->query >= BOOK_QUERY_SUB_FIRST) {
3975                         QueryDelimiter *delim = (QueryDelimiter *) element;
3976
3977                         g_slice_free (QueryDelimiter, delim);
3978                 } else if (IS_QUERY_PHONE (element->query)) {
3979                         QueryPhoneTest *test = (QueryPhoneTest *) element;
3980
3981                         g_free (test->value);
3982                         g_free (test->region);
3983                         g_free (test->national);
3984                         g_slice_free (QueryPhoneTest, test);
3985                 } else {
3986                         QueryFieldTest *test = (QueryFieldTest *) element;
3987
3988                         g_free (test->value);
3989                         g_slice_free (QueryFieldTest, test);
3990                 }
3991         }
3992 }
3993
3994 /* We use ptr arrays for the QueryElement vectors */
3995 static inline void
3996 constraints_insert (GPtrArray *array,
3997                     gint idx,
3998                     gpointer data)
3999 {
4000 #if 0
4001         g_ptr_array_insert (array, idx, data);
4002 #else
4003         g_return_if_fail ((idx >= -1) && (idx < (gint) array->len + 1));
4004
4005         if (idx < 0)
4006                 idx = array->len;
4007
4008         g_ptr_array_add (array, NULL);
4009
4010         if (idx != (array->len - 1))
4011                 memmove (
4012                         &(array->pdata[idx + 1]),
4013                         &(array->pdata[idx]),
4014                         ((array->len - 1) - idx) * sizeof (gpointer));
4015
4016         array->pdata[idx] = data;
4017 #endif
4018 }
4019
4020 static inline QueryElement *
4021 constraints_take (GPtrArray *array,
4022                   gint idx)
4023 {
4024         QueryElement *element;
4025
4026         g_return_val_if_fail (idx >= 0 && idx < (gint) array->len, NULL);
4027
4028         element = array->pdata[idx];
4029         array->pdata[idx] = NULL;
4030         g_ptr_array_remove_index (array, idx);
4031
4032         return element;
4033 }
4034
4035 static inline void
4036 constraints_insert_delimiter (GPtrArray *array,
4037                               gint idx,
4038                               guint query)
4039 {
4040         QueryElement *delim;
4041
4042         delim = query_delimiter_new (query);
4043         constraints_insert (array, idx, delim);
4044 }
4045
4046 static inline void
4047 constraints_insert_field_test (GPtrArray *array,
4048                                gint idx,
4049                                SummaryField *field,
4050                                guint query,
4051                                const gchar *value)
4052 {
4053         QueryFieldTest *test;
4054
4055         test = query_field_test_new (query, field->field_id);
4056         test->field = field;
4057         test->value = g_strdup (value);
4058
4059         constraints_insert (array, idx, test);
4060 }
4061
4062 static void
4063 preflight_context_clear (PreflightContext *context)
4064 {
4065         if (context) {
4066                 /* Free any allocated data, but leave the context values in place */
4067                 if (context->constraints)
4068                         g_ptr_array_free (context->constraints, TRUE);
4069                 context->constraints = NULL;
4070         }
4071 }
4072
4073 /* A small API to track the current sub-query context.
4074  *
4075  * I.e. sub contexts can be OR, AND, or NOT, in which
4076  * field tests or other sub contexts are nested.
4077  */
4078 typedef GQueue SubQueryContext;
4079
4080 typedef struct {
4081         guint sub_type; /* The type of this sub context */
4082         guint count;    /* The number of field tests so far in this context */
4083 } SubQueryData;
4084
4085 #define sub_query_context_new g_queue_new
4086 #define sub_query_context_free(ctx) g_queue_free (ctx)
4087
4088 static inline void
4089 sub_query_context_push (SubQueryContext *ctx,
4090                         guint sub_type)
4091 {
4092         SubQueryData *data;
4093
4094         data = g_slice_new (SubQueryData);
4095         data->sub_type = sub_type;
4096         data->count = 0;
4097
4098         g_queue_push_tail (ctx, data);
4099 }
4100
4101 static inline void
4102 sub_query_context_pop (SubQueryContext *ctx)
4103 {
4104         SubQueryData *data;
4105
4106         data = g_queue_pop_tail (ctx);
4107         g_slice_free (SubQueryData, data);
4108 }
4109
4110 static inline guint
4111 sub_query_context_peek_type (SubQueryContext *ctx)
4112 {
4113         SubQueryData *data;
4114
4115         data = g_queue_peek_tail (ctx);
4116
4117         return data->sub_type;
4118 }
4119
4120 /* Returns the context field test count before incrementing */
4121 static inline guint
4122 sub_query_context_increment (SubQueryContext *ctx)
4123 {
4124         SubQueryData *data;
4125
4126         data = g_queue_peek_tail (ctx);
4127
4128         if (data) {
4129                 data->count++;
4130
4131                 return (data->count - 1);
4132         }
4133
4134         /* If we're not in a sub context, just return 0 */
4135         return 0;
4136 }
4137
4138 /**********************************************************
4139  *                  Querying preflighting                 *
4140  **********************************************************
4141  *
4142  * The preflight checks are performed before a query might
4143  * take place in order to evaluate whether the given query
4144  * can be performed with the current summary configuration.
4145  *
4146  * After preflighting, all relevant data has been extracted
4147  * from the search expression and the search expression need
4148  * not be parsed again.
4149  */
4150
4151 /* The PreflightSubCallback is expected to return TRUE
4152  * to keep iterating and FALSE to abort iteration.
4153  *
4154  * The sub_level is the counter of how deep the 'element'
4155  * is nested in sub elements, the offset is the real offset
4156  * of 'element' in the array passed to query_preflight_foreach_sub().
4157  */
4158 typedef gboolean (* PreflightSubCallback) (QueryElement *element,
4159                                            gint          sub_level,
4160                                            gint          offset,
4161                                            gpointer      user_data);
4162
4163 static void
4164 query_preflight_foreach_sub (QueryElement **elements,
4165                              gint n_elements,
4166                              gint offset,
4167                              gboolean include_delim,
4168                              PreflightSubCallback callback,
4169                              gpointer user_data)
4170 {
4171         gint sub_counter = 1, i;
4172
4173         g_return_if_fail (offset >= 0 && offset < n_elements);
4174         g_return_if_fail (elements[offset]->query >= BOOK_QUERY_SUB_FIRST);
4175         g_return_if_fail (callback != NULL);
4176
4177         if (include_delim && !callback (elements[offset], 0, offset, user_data))
4178                 return;
4179
4180         for (i = (offset + 1); sub_counter > 0 && i < n_elements; i++) {
4181
4182                 if (elements[i]->query >= BOOK_QUERY_SUB_FIRST) {
4183
4184                         if (elements[i]->query == BOOK_QUERY_SUB_END)
4185                                 sub_counter--;
4186                         else
4187                                 sub_counter++;
4188
4189                         if (include_delim &&
4190                             !callback (elements[i], sub_counter, i, user_data))
4191                                 break;
4192                 } else {
4193
4194                         if (!callback (elements[i], sub_counter, i, user_data))
4195                                 break;
4196                 }
4197         }
4198 }
4199
4200 /* Table used in ESExp parsing below */
4201 static const struct {
4202         const gchar *name;    /* Name of the symbol to match for this parse phase */
4203         gboolean     subset;  /* TRUE for the subset ESExpIFunc, otherwise the field check ESExpFunc */
4204         guint        test;    /* Extended EBookQueryTest value */
4205 } check_symbols[] = {
4206         { "and",              TRUE, BOOK_QUERY_SUB_AND },
4207         { "or",               TRUE, BOOK_QUERY_SUB_OR },
4208         { "not",              TRUE, BOOK_QUERY_SUB_NOT },
4209         { "contains",         FALSE, E_BOOK_QUERY_CONTAINS },
4210         { "is",               FALSE, E_BOOK_QUERY_IS },
4211         { "beginswith",       FALSE, E_BOOK_QUERY_BEGINS_WITH },
4212         { "endswith",         FALSE, E_BOOK_QUERY_ENDS_WITH },
4213         { "eqphone",          FALSE, E_BOOK_QUERY_EQUALS_PHONE_NUMBER },
4214         { "eqphone_national", FALSE, E_BOOK_QUERY_EQUALS_NATIONAL_PHONE_NUMBER },
4215         { "eqphone_short",    FALSE, E_BOOK_QUERY_EQUALS_SHORT_PHONE_NUMBER },
4216         { "regex_normal",     FALSE, E_BOOK_QUERY_REGEX_NORMAL },
4217         { "regex_raw",        FALSE, E_BOOK_QUERY_REGEX_RAW },
4218         { "translit_is",         FALSE, E_BOOK_QUERY_TRANSLIT_IS },
4219         { "translit_contains",   FALSE, E_BOOK_QUERY_TRANSLIT_CONTAINS },
4220         { "translit_beginswith", FALSE, E_BOOK_QUERY_TRANSLIT_BEGINS_WITH },
4221         { "translit_endswith",   FALSE, E_BOOK_QUERY_TRANSLIT_ENDS_WITH },
4222         { "exists",           FALSE, BOOK_QUERY_EXISTS },
4223 };
4224
4225 /* Cheat our way into passing mode data to these funcs */
4226 static ESExpResult *
4227 func_check_subset (ESExp *f,
4228                    gint argc,
4229                    struct _ESExpTerm **argv,
4230                    gpointer data)
4231 {
4232         ESExpResult *result, *sub_result;
4233         GPtrArray *result_array;
4234         QueryElement *element, **sub_elements;
4235         gint i, j, len;
4236         guint query_type;
4237
4238         query_type = GPOINTER_TO_UINT (data);
4239
4240         /* The compound query delimiter is the first element in this return array */
4241         result_array = g_ptr_array_new_with_free_func ((GDestroyNotify) query_element_free);
4242         element = query_delimiter_new (query_type);
4243         g_ptr_array_add (result_array, element);
4244
4245         EBSQL_NOTE (
4246                 PREFLIGHT,
4247                 g_printerr (
4248                         "PREFLIGHT INIT: Open sub: %s\n",
4249                         EBSQL_QUERY_TYPE_STR (query_type)));
4250
4251         for (i = 0; i < argc; i++) {
4252                 sub_result = e_sexp_term_eval (f, argv[i]);
4253
4254                 if (sub_result->type == ESEXP_RES_ARRAY_PTR) {
4255                         /* Steal the elements directly from the sub result */
4256                         sub_elements = (QueryElement **) sub_result->value.ptrarray->pdata;
4257                         len = sub_result->value.ptrarray->len;
4258
4259                         for (j = 0; j < len; j++) {
4260                                 element = sub_elements[j];
4261                                 sub_elements[j] = NULL;
4262
4263                                 g_ptr_array_add (result_array, element);
4264                         }
4265                 }
4266                 e_sexp_result_free (f, sub_result);
4267         }
4268
4269         EBSQL_NOTE (
4270                 PREFLIGHT,
4271                 g_printerr (
4272                         "PREFLIGHT INIT: Close sub: %s\n",
4273                         EBSQL_QUERY_TYPE_STR (query_type)));
4274
4275         /* The last element in this return array is the sub end delimiter */
4276         element = query_delimiter_new (BOOK_QUERY_SUB_END);
4277         g_ptr_array_add (result_array, element);
4278
4279         result = e_sexp_result_new (f, ESEXP_RES_ARRAY_PTR);
4280         result->value.ptrarray = result_array;
4281
4282         return result;
4283 }
4284
4285 static ESExpResult *
4286 func_check (struct _ESExp *f,
4287             gint argc,
4288             struct _ESExpResult **argv,
4289             gpointer data)
4290 {
4291         ESExpResult *result;
4292         GPtrArray *result_array;
4293         QueryElement *element = NULL;
4294         EContactField field_id = 0;
4295         const gchar *query_name = NULL;
4296         const gchar *query_value = NULL;
4297         const gchar *query_extra = NULL;
4298         guint query_type;
4299
4300         query_type = GPOINTER_TO_UINT (data);
4301
4302         if (argc == 2 &&
4303             argv[0]->type == ESEXP_RES_STRING &&
4304             argv[1]->type == ESEXP_RES_STRING) {
4305                 query_name = argv[0]->value.string;
4306                 query_value = argv[1]->value.string;
4307
4308                 /* We use E_CONTACT_FIELD_LAST to hold the special case of "x-evolution-any-field" */
4309                 if (g_strcmp0 (query_name, "x-evolution-any-field") == 0)
4310                         field_id = E_CONTACT_FIELD_LAST;
4311                 else
4312                         field_id = e_contact_field_id (query_name);
4313
4314         } else if (argc == 3 &&
4315                    argv[0]->type == ESEXP_RES_STRING &&
4316                    argv[1]->type == ESEXP_RES_STRING &&
4317                    argv[2]->type == ESEXP_RES_STRING) {
4318                 query_name = argv[0]->value.string;
4319                 query_value = argv[1]->value.string;
4320                 query_extra = argv[2]->value.string;
4321
4322                 field_id = e_contact_field_id (query_name);
4323         }
4324
4325         if (IS_QUERY_PHONE (query_type)) {
4326                 QueryPhoneTest *test;
4327
4328                 /* Collect data from this field test */
4329                 test = query_phone_test_new (query_type, field_id);
4330                 test->value = g_strdup (query_value);
4331                 test->region = g_strdup (query_extra);
4332
4333                 element = (QueryElement *) test;
4334         } else {
4335                 QueryFieldTest *test;
4336
4337                 /* Collect data from this field test */
4338                 test = query_field_test_new (query_type, field_id);
4339                 test->value = g_strdup (query_value);
4340
4341                 element = (QueryElement *) test;
4342         }
4343
4344         EBSQL_NOTE (
4345                 PREFLIGHT,
4346                 g_printerr (
4347                         "PREFLIGHT INIT: Adding field test: `%s' on field `%s' "
4348                         "(field name: %s query value: %s query extra: %s)\n",
4349                         EBSQL_QUERY_TYPE_STR (query_type),
4350                         EBSQL_FIELD_ID_STR (field_id),
4351                         query_name, query_value, query_extra));
4352
4353         /* Return an array with only one element, for lack of a pointer type ESExpResult */
4354         result_array = g_ptr_array_new_with_free_func ((GDestroyNotify) query_element_free);
4355         g_ptr_array_add (result_array, element);
4356
4357         result = e_sexp_result_new (f, ESEXP_RES_ARRAY_PTR);
4358         result->value.ptrarray = result_array;
4359
4360         return result;
4361 }
4362
4363 /* Initial stage of preflighting:
4364  *
4365  *  o Parse the search expression and generate our array of QueryElements
4366  *  o Collect lengths of query terms
4367  */
4368 static void
4369 query_preflight_initialize (PreflightContext *context,
4370                             const gchar *sexp)
4371 {
4372         ESExp *sexp_parser;
4373         ESExpResult *result;
4374         gint esexp_error, i;
4375
4376         if (sexp == NULL || *sexp == '\0') {
4377                 context->status = PREFLIGHT_LIST_ALL;
4378                 return;
4379         }
4380
4381         sexp_parser = e_sexp_new ();
4382
4383         for (i = 0; i < G_N_ELEMENTS (check_symbols); i++) {
4384                 if (check_symbols[i].subset) {
4385                         e_sexp_add_ifunction (
4386                                 sexp_parser, 0, check_symbols[i].name,
4387                                 func_check_subset,
4388                                 GUINT_TO_POINTER (check_symbols[i].test));
4389                 } else {
4390                         e_sexp_add_function (
4391                                 sexp_parser, 0, check_symbols[i].name,
4392                                 func_check,
4393                                 GUINT_TO_POINTER (check_symbols[i].test));
4394                 }
4395         }
4396
4397         e_sexp_input_text (sexp_parser, sexp, strlen (sexp));
4398         esexp_error = e_sexp_parse (sexp_parser);
4399
4400         if (esexp_error == -1) {
4401                 context->status = PREFLIGHT_INVALID;
4402
4403                 EBSQL_NOTE (
4404                         PREFLIGHT,
4405                         g_printerr ("PREFLIGHT INIT: Sexp parse error\n"));
4406         } else {
4407
4408                 result = e_sexp_eval (sexp_parser);
4409                 if (result) {
4410
4411                         if (result->type == ESEXP_RES_ARRAY_PTR) {
4412
4413                                 /* Just steal the array away from the ESexpResult */
4414                                 context->constraints = result->value.ptrarray;
4415                                 result->value.ptrarray = NULL;
4416
4417                         } else {
4418                                 context->status = PREFLIGHT_INVALID;
4419
4420                                 EBSQL_NOTE (
4421                                         PREFLIGHT,
4422                                         g_printerr ("PREFLIGHT INIT: ERROR, Did not get GPtrArray\n"));
4423                         }
4424                 }
4425
4426                 e_sexp_result_free (sexp_parser, result);
4427         }
4428
4429         e_sexp_unref (sexp_parser);
4430
4431         EBSQL_NOTE (
4432                 PREFLIGHT,
4433                 g_printerr (
4434                         "PREFLIGHT INIT: Completed with status %s\n",
4435                         EBSQL_STATUS_STR (context->status)));
4436 }
4437
4438 typedef struct {
4439         EBookSqlite   *ebsql;
4440         gboolean       has_attr_list;
4441 } AttrListCheckData;
4442
4443 static gboolean
4444 check_has_attr_list_cb (QueryElement *element,
4445                         gint sub_level,
4446                         gint offset,
4447                         gpointer user_data)
4448 {
4449         QueryFieldTest *test = (QueryFieldTest *) element;
4450         AttrListCheckData *data = (AttrListCheckData *) user_data;
4451
4452         /* We havent resolved all the fields at this stage yet */
4453         if (!test->field)
4454                 test->field = summary_field_get (data->ebsql, test->field_id);
4455
4456         if (test->field && test->field->type == E_TYPE_CONTACT_ATTR_LIST)
4457                 data->has_attr_list = TRUE;
4458
4459         /* Keep looping until we find one */
4460         return (data->has_attr_list == FALSE);
4461 }
4462
4463 /* What is done in this pass:
4464  *  o Viability of the query is analyzed, i.e. can it be done with the summary columns.
4465  *  o Phone numbers are parsed and loaded onto QueryPhoneTests
4466  *  o Bitmask of auxiliary tables is collected
4467  */
4468 static void
4469 query_preflight_check (PreflightContext *context,
4470                        EBookSqlite *ebsql)
4471 {
4472         gint i, n_elements;
4473         QueryElement **elements;
4474
4475         context->status = PREFLIGHT_OK;
4476
4477         elements = (QueryElement **) context->constraints->pdata;
4478         n_elements = context->constraints->len;
4479
4480         for (i = 0; i < n_elements; i++) {
4481                 QueryFieldTest *test;
4482                 guint           field_test;
4483
4484                 EBSQL_NOTE (
4485                         PREFLIGHT,
4486                         g_printerr (
4487                                 "PREFLIGHT CHECK: Encountered: %s\n",
4488                                 EBSQL_QUERY_TYPE_STR (elements[i]->query)));
4489
4490                 if (elements[i]->query >= BOOK_QUERY_SUB_FIRST) {
4491                         /* It's too complicated to properly perform
4492                          * the unary NOT operator on a constraint which
4493                          * accesses attribute lists.
4494                          *
4495                          * Hint, if the contact has a "%.com" email address
4496                          * and a "%.org" email address, what do we return
4497                          * for (not (endswith "email" ".com") ?
4498                          *
4499                          * Currently we rely on DISTINCT to sort out
4500                          * muliple results from the attribute list tables,
4501                          * this breaks down with NOT.
4502                          */
4503                         if (elements[i]->query == BOOK_QUERY_SUB_NOT) {
4504                                 AttrListCheckData data = { ebsql, FALSE };
4505
4506                                 query_preflight_foreach_sub (elements,
4507                                                              n_elements,
4508                                                              i, FALSE,
4509                                                              check_has_attr_list_cb,
4510                                                              &data);
4511
4512                                 if (data.has_attr_list) {
4513                                         context->status = MAX (
4514                                                 context->status,
4515                                                 PREFLIGHT_NOT_SUMMARIZED);
4516                                         EBSQL_NOTE (
4517                                                 PREFLIGHT,
4518                                                 g_printerr (
4519                                                         "PREFLIGHT CHECK: "
4520                                                         "Setting invalid for NOT (mutli-attribute), "
4521                                                         "new status: %s\n",
4522                                                         EBSQL_STATUS_STR (context->status)));
4523                                 }
4524                         }
4525                         continue;
4526                 }
4527
4528                 test = (QueryFieldTest *) elements[i];
4529                 field_test = (EBookQueryTest) test->query;
4530
4531                 if (!test->field)
4532                         test->field = summary_field_get (ebsql, test->field_id);
4533
4534                 /* Even if the field is not in the summary, we need to
4535                  * retport unsupported errors if phone number queries are
4536                  * issued while libphonenumber is unavailable
4537                  */
4538                 if (!test->field) {
4539
4540                         /* Special case for e_book_query_any_field_contains().
4541                          *
4542                          * We interpret 'x-evolution-any-field' as E_CONTACT_FIELD_LAST
4543                          */
4544                         if (test->field_id == E_CONTACT_FIELD_LAST) {
4545
4546                                 /* If we search for a NULL or zero length string, it
4547                                  * means 'get all contacts', that is considered a summary
4548                                  * query but is handled differently (i.e. we just drop the
4549                                  * field tests and run a regular query).
4550                                  *
4551                                  * This is only true if the 'any field contains' query is
4552                                  * the only test in the constraints, however.
4553                                  */
4554                                 if (n_elements == 1 && (!test->value || !test->value[0])) {
4555
4556                                         context->status = MAX (context->status, PREFLIGHT_LIST_ALL);
4557                                         EBSQL_NOTE (
4558                                                 PREFLIGHT,
4559                                                 g_printerr (
4560                                                         "PREFLIGHT CHECK: "
4561                                                         "Encountered lonesome 'x-evolution-any-field' with empty value, "
4562                                                         "new status: %s\n",
4563                                                         EBSQL_STATUS_STR (context->status)));
4564                                 } else {
4565
4566                                         /* Searching for a value with 'x-evolution-any-field' is
4567                                          * not a summary query.
4568                                          */
4569                                         context->status = MAX (context->status, PREFLIGHT_NOT_SUMMARIZED);
4570                                         EBSQL_NOTE (
4571                                                 PREFLIGHT,
4572                                                 g_printerr (
4573                                                         "PREFLIGHT CHECK: "
4574                                                         "Encountered 'x-evolution-any-field', "
4575                                                         "new status: %s\n",
4576                                                         EBSQL_STATUS_STR (context->status)));
4577                                 }
4578
4579                         } else {
4580
4581                                 /* Couldnt resolve the field, it's not a summary query */
4582                                 context->status = MAX (context->status, PREFLIGHT_NOT_SUMMARIZED);
4583                                 EBSQL_NOTE (
4584                                         PREFLIGHT,
4585                                         g_printerr (
4586                                                 "PREFLIGHT CHECK: "
4587                                                 "Field `%s' not in the summary, new status: %s\n",
4588                                                 EBSQL_FIELD_ID_STR (test->field_id),
4589                                                 EBSQL_STATUS_STR (context->status)));
4590                         }
4591                 }
4592
4593                 switch (field_test) {
4594                 case E_BOOK_QUERY_IS:
4595                         break;
4596
4597                 case BOOK_QUERY_EXISTS:
4598                 case E_BOOK_QUERY_CONTAINS:
4599                 case E_BOOK_QUERY_BEGINS_WITH:
4600                 case E_BOOK_QUERY_ENDS_WITH:
4601                 case E_BOOK_QUERY_REGEX_NORMAL:
4602
4603                         /* All of these queries can only apply to string fields,
4604                          * or fields which hold multiple strings
4605                          */
4606                         if (test->field) {
4607                                 if (test->field->type != G_TYPE_STRING &&
4608                                     test->field->type != E_TYPE_CONTACT_ATTR_LIST) {
4609                                         context->status = MAX (context->status, PREFLIGHT_INVALID);
4610                                         EBSQL_NOTE (
4611                                                 PREFLIGHT,
4612                                                 g_printerr (
4613                                                         "PREFLIGHT CHECK: "
4614                                                         "Refusing pattern match on boolean field `%s', new status: %s\n",
4615                                                         EBSQL_FIELD_ID_STR (test->field_id),
4616                                                         EBSQL_STATUS_STR (context->status)));
4617                                 }
4618                         }
4619
4620                         break;
4621
4622                 case E_BOOK_QUERY_REGEX_RAW:
4623                         /* Raw regex queries only supported in the fallback */
4624                         context->status = MAX (context->status, PREFLIGHT_NOT_SUMMARIZED);
4625                         EBSQL_NOTE (
4626                                 PREFLIGHT,
4627                                 g_printerr (
4628                                         "PREFLIGHT CHECK: "
4629                                         "Raw regexp requires full data, new status: %s\n",
4630                                         EBSQL_STATUS_STR (context->status)));
4631                         break;
4632
4633                 case E_BOOK_QUERY_EQUALS_PHONE_NUMBER:
4634                 case E_BOOK_QUERY_EQUALS_NATIONAL_PHONE_NUMBER:
4635                 case E_BOOK_QUERY_EQUALS_SHORT_PHONE_NUMBER:
4636
4637                         /* Phone number queries are supported so long as they are in the summary,
4638                          * libphonenumber is available, and the phone number string is a valid one
4639                          */
4640                         if (!e_phone_number_is_supported ()) {
4641
4642                                 context->status = MAX (context->status, PREFLIGHT_UNSUPPORTED);
4643                                 EBSQL_NOTE (
4644                                         PREFLIGHT,
4645                                         g_printerr (
4646                                                 "PREFLIGHT CHECK: "
4647                                                 "Usupported phone number query, new status: %s\n",
4648                                                 EBSQL_STATUS_STR (context->status)));
4649                         } else {
4650                                 QueryPhoneTest *phone_test = (QueryPhoneTest *) test;
4651                                 EPhoneNumberCountrySource source;
4652                                 EPhoneNumber *number;
4653                                 const gchar *region_code;
4654
4655                                 if (phone_test->region)
4656                                         region_code = phone_test->region;
4657                                 else
4658                                         region_code = ebsql->priv->region_code;
4659
4660                                 number = e_phone_number_from_string (
4661                                         phone_test->value,
4662                                         region_code, NULL);
4663
4664                                 if (number == NULL) {
4665
4666                                         context->status = MAX (context->status, PREFLIGHT_INVALID);
4667                                         EBSQL_NOTE (
4668                                                 PREFLIGHT,
4669                                                 g_printerr (
4670                                                         "PREFLIGHT CHECK: "
4671                                                         "Invalid phone number `%s', new status: %s\n",
4672                                                         phone_test->value,
4673                                                         EBSQL_STATUS_STR (context->status)));
4674                                 } else {
4675                                         /* Collect values we'll need later while generating field
4676                                          * tests, no need to parse the phone number more than once
4677                                          */
4678                                         phone_test->national = e_phone_number_get_national_number (number);
4679                                         phone_test->country = e_phone_number_get_country_code (number, &source);
4680
4681                                         if (source == E_PHONE_NUMBER_COUNTRY_FROM_DEFAULT)
4682                                                 phone_test->country = 0;
4683
4684                                         e_phone_number_free (number);
4685                                 }
4686                         }
4687                         break;
4688
4689                 case E_BOOK_QUERY_TRANSLIT_IS:
4690                 case E_BOOK_QUERY_TRANSLIT_CONTAINS:
4691                 case E_BOOK_QUERY_TRANSLIT_BEGINS_WITH:
4692                 case E_BOOK_QUERY_TRANSLIT_ENDS_WITH:
4693
4694                         /* These only only searchable in the summary with the E_BOOK_INDEX_TRANSLIT index */
4695                         if (test->field == NULL ||
4696                             (test->field->index & INDEX_FLAG (TRANSLIT)) == 0) {
4697
4698                                 context->status = MAX (context->status, PREFLIGHT_NOT_SUMMARIZED);
4699                                 EBSQL_NOTE (PREFLIGHT,
4700                                             g_printerr ("PREFLIGHT CHECK: "
4701                                                         "Query `%s' needs fallback search, new status: %s\n",
4702                                                         EBSQL_QUERY_TYPE_STR (field_test),
4703                                                         EBSQL_STATUS_STR (context->status)));
4704                         }
4705                         break;
4706                 }
4707
4708                 if (test->field &&
4709                     test->field->type == E_TYPE_CONTACT_ATTR_LIST) {
4710                         gint aux_index = summary_field_get_index (ebsql, test->field_id);
4711
4712                         /* It's really improbable that we ever get 64 fields in the summary
4713                          * In any case we warn about this in e_book_sqlite_new_full().
4714                          */
4715                         g_warn_if_fail (aux_index < EBSQL_MAX_SUMMARY_FIELDS);
4716                         context->aux_mask |= (1 << aux_index);
4717                         EBSQL_NOTE (
4718                                 PREFLIGHT,
4719                                 g_printerr (
4720                                         "PREFLIGHT CHECK: "
4721                                         "Adding auxiliary field `%s' to the mask\n",
4722                                         EBSQL_FIELD_ID_STR (test->field_id)));
4723                 }
4724         }
4725 }
4726
4727 /* Handle special case of E_CONTACT_FULL_NAME
4728  *
4729  * For any query which accesses the full name field,
4730  * we need to also OR it with any of the related name
4731  * fields, IF those are found in the summary as well.
4732  */
4733 static void
4734 query_preflight_substitute_full_name (PreflightContext *context,
4735                                       EBookSqlite *ebsql)
4736 {
4737         gint i, j;
4738
4739         for (i = 0; i < context->constraints->len; i++) {
4740                 SummaryField *family_name, *given_name, *nickname;
4741                 QueryElement *element;
4742                 QueryFieldTest *test;
4743                 gboolean need_translit = FALSE;
4744
4745                 element = g_ptr_array_index (context->constraints, i);
4746
4747                 if (element->query >= BOOK_QUERY_SUB_FIRST)
4748                         continue;
4749
4750                 test = (QueryFieldTest *) element;
4751                 if (test->field_id != E_CONTACT_FULL_NAME)
4752                         continue;
4753
4754                 /* If it's transliterated, we won't do this */
4755                 if (element->query >= E_BOOK_QUERY_TRANSLIT_IS &&
4756                     element->query <= E_BOOK_QUERY_TRANSLIT_ENDS_WITH)
4757                         need_translit = TRUE;
4758
4759                 family_name = summary_field_get (ebsql, E_CONTACT_FAMILY_NAME);
4760                 given_name = summary_field_get (ebsql, E_CONTACT_GIVEN_NAME);
4761                 nickname = summary_field_get (ebsql, E_CONTACT_NICKNAME);
4762
4763                 /* Only OR them in if they are also indexed for transliteration */
4764                 if (need_translit) {
4765                         if (family_name &&
4766                             (family_name->index & INDEX_FLAG (TRANSLIT)) == 0)
4767                                 family_name = NULL;
4768
4769                         if (given_name &&
4770                             (given_name->index & INDEX_FLAG (TRANSLIT)) == 0)
4771                                 given_name = NULL;
4772
4773                         if (nickname &&
4774                             (nickname->index & INDEX_FLAG (TRANSLIT)) == 0)
4775                                 nickname = NULL;
4776                 }
4777
4778                 /* If any of these are in the summary, then we'll construct
4779                  * a grouped OR statment for this E_CONTACT_FULL_NAME test */
4780                 if (family_name || given_name || nickname) {
4781                         /* Add the OR directly before the E_CONTACT_FULL_NAME test */
4782                         constraints_insert_delimiter (context->constraints, i, BOOK_QUERY_SUB_OR);
4783
4784                         j = i + 2;
4785
4786                         if (family_name)
4787                                 constraints_insert_field_test (
4788                                         context->constraints, j++,
4789                                         family_name, test->query,
4790                                         test->value);
4791
4792                         if (given_name)
4793                                 constraints_insert_field_test (
4794                                         context->constraints, j++,
4795                                         given_name, test->query,
4796                                         test->value);
4797
4798                         if (nickname)
4799                                 constraints_insert_field_test (
4800                                         context->constraints, j++,
4801                                         nickname, test->query,
4802                                         test->value);
4803
4804                         constraints_insert_delimiter (context->constraints, j, BOOK_QUERY_SUB_END);
4805
4806                         i = j;
4807                 }
4808         }
4809 }
4810
4811 static void
4812 query_preflight (PreflightContext *context,
4813                  EBookSqlite *ebsql,
4814                  const gchar *sexp)
4815 {
4816         EBSQL_NOTE (PREFLIGHT, g_printerr ("PREFLIGHT BEGIN\n"));
4817         query_preflight_initialize (context, sexp);
4818
4819         if (context->status == PREFLIGHT_OK) {
4820
4821                 query_preflight_check (context, ebsql);
4822
4823                 /* No need to change the constraints if we're not
4824                  * going to generate statements with it
4825                  */
4826                 if (context->status == PREFLIGHT_OK) {
4827                         EBSQL_NOTE (
4828                                 PREFLIGHT,
4829                                 g_printerr ("PREFLIGHT: Substituting full name\n"));
4830
4831                         /* Handle E_CONTACT_FULL_NAME substitutions */
4832                         query_preflight_substitute_full_name (context, ebsql);
4833
4834                 } else {
4835                         EBSQL_NOTE (PREFLIGHT, g_printerr ("PREFLIGHT: Clearing context\n"));
4836
4837                         /* We might use this context to perform a fallback query,
4838                          * so let's clear out all the constraints now
4839                          */
4840                         preflight_context_clear (context);
4841                 }
4842         }
4843
4844         EBSQL_NOTE (
4845                 PREFLIGHT,
4846                 g_printerr (
4847                         "PREFLIGHT END (status: %s)\n",
4848                         EBSQL_STATUS_STR (context->status)));
4849 }
4850
4851 /**********************************************************
4852  *                 Field Test Generators                  *
4853  **********************************************************
4854  *
4855  * This section contains the field test generators for
4856  * various EBookQueryTest types. When implementing new
4857  * query types, a new GenerateFieldTest needs to be created
4858  * and added to the table below.
4859  */
4860
4861 typedef void (* GenerateFieldTest) (EBookSqlite      *ebsql,
4862                                     GString          *string,
4863                                     QueryFieldTest   *test);
4864
4865 /* This function escapes characters which need escaping
4866  * for LIKE statements as well as the single quotes.
4867  *
4868  * The return value is not suitable to be formatted
4869  * with %Q or %q
4870  */
4871 static gchar *
4872 ebsql_normalize_for_like (EBookSqlite *ebsql,
4873                           QueryFieldTest *test,
4874                           gboolean reverse_string,
4875                           gboolean *escape_needed)
4876 {
4877         GString *str;
4878         size_t len;
4879         gchar c;
4880         gboolean escape_modifier_needed = FALSE;
4881         const gchar *normal = NULL;
4882         const gchar *ptr;
4883         const gchar *str_to_escape;
4884         gchar *reverse = NULL;
4885         gchar *freeme = NULL;
4886
4887         if (test->field_id == E_CONTACT_UID ||
4888             test->field_id == E_CONTACT_REV) {
4889                 normal = test->value;
4890         } else if (test->query >= E_BOOK_QUERY_TRANSLIT_CONTAINS &&
4891                    test->query <= E_BOOK_QUERY_TRANSLIT_ENDS_WITH) {
4892                 gchar *tmp = e_transliterator_transliterate (ebsql->priv->transliterator, test->value);
4893                 freeme = e_util_utf8_normalize (tmp);
4894                 g_free (tmp);
4895
4896                 normal = freeme;
4897         } else {
4898                 freeme = e_util_utf8_normalize (test->value);
4899                 normal = freeme;
4900         }
4901
4902         if (reverse_string) {
4903                 reverse = g_utf8_strreverse (normal, -1);
4904                 str_to_escape = reverse;
4905         } else
4906                 str_to_escape = normal;
4907
4908         /* Just assume each character must be escaped. The result of this function
4909          * is discarded shortly after calling this function. Therefore it's
4910          * acceptable to possibly allocate twice the memory needed.
4911          */
4912         len = strlen (str_to_escape);
4913         str = g_string_sized_new (2 * len + 4 + strlen (EBSQL_ESCAPE_SEQUENCE) - 1);
4914
4915         ptr = str_to_escape;
4916         while ((c = *ptr++)) {
4917                 if (c == '\'') {
4918                         g_string_append_c (str, '\'');
4919                 } else if (c == '%' || c == '_' || c == '^') {
4920                         g_string_append_c (str, '^');
4921                         escape_modifier_needed = TRUE;
4922                 }
4923
4924                 g_string_append_c (str, c);
4925         }
4926
4927         if (escape_needed)
4928                 *escape_needed = escape_modifier_needed;
4929
4930         g_free (freeme);
4931         g_free (reverse);
4932
4933         return g_string_free (str, FALSE);
4934 }
4935
4936 static void
4937 field_test_query_is (EBookSqlite *ebsql,
4938                      GString *string,
4939                      QueryFieldTest *test)
4940 {
4941         SummaryField *field = test->field;
4942         gchar *normal;
4943
4944         ebsql_string_append_column (string, field, NULL);
4945
4946         if (test->field_id == E_CONTACT_UID ||
4947             test->field_id == E_CONTACT_REV) {
4948                 /* UID & REV fields are not normalized in the summary */
4949                 ebsql_string_append_printf (string, " = %Q", test->value);
4950         } else {
4951                 normal = e_util_utf8_normalize (test->value);
4952                 ebsql_string_append_printf (string, " = %Q", normal);
4953                 g_free (normal);
4954         }
4955 }
4956
4957 static void
4958 field_test_query_contains (EBookSqlite *ebsql,
4959                            GString *string,
4960                            QueryFieldTest *test)
4961 {
4962         SummaryField *field = test->field;
4963         gboolean need_escape;
4964         gchar *escaped;
4965
4966         escaped = ebsql_normalize_for_like (ebsql, test, FALSE, &need_escape);
4967
4968         g_string_append_c (string, '(');
4969
4970         ebsql_string_append_column (string, field, NULL);
4971         g_string_append (string, " IS NOT NULL AND ");
4972         ebsql_string_append_column (string, field, NULL);
4973         g_string_append (string, " LIKE '%");
4974         g_string_append (string, escaped);
4975         g_string_append (string, "%'");
4976
4977         if (need_escape)
4978                 g_string_append (string, EBSQL_ESCAPE_SEQUENCE);
4979
4980         g_string_append_c (string, ')');
4981
4982         g_free (escaped);
4983 }
4984
4985 static void
4986 field_test_query_begins_with (EBookSqlite *ebsql,
4987                               GString *string,
4988                               QueryFieldTest *test)
4989 {
4990         SummaryField *field = test->field;
4991         gboolean need_escape;
4992         gchar *escaped;
4993
4994         escaped = ebsql_normalize_for_like (ebsql, test, FALSE, &need_escape);
4995
4996         g_string_append_c (string, '(');
4997         ebsql_string_append_column (string, field, NULL);
4998         g_string_append (string, " IS NOT NULL AND ");
4999
5000         ebsql_string_append_column (string, field, NULL);
5001         g_string_append (string, " LIKE \'");
5002         g_string_append (string, escaped);
5003         g_string_append (string, "%\'");
5004
5005         if (need_escape)
5006                 g_string_append (string, EBSQL_ESCAPE_SEQUENCE);
5007         g_string_append_c (string, ')');
5008
5009         g_free (escaped);
5010 }
5011
5012 static void
5013 field_test_query_ends_with (EBookSqlite *ebsql,
5014                             GString *string,
5015                             QueryFieldTest *test)
5016 {
5017         SummaryField *field = test->field;
5018         gboolean need_escape;
5019         gchar *escaped;
5020
5021         if ((field->index & INDEX_FLAG (SUFFIX)) != 0) {
5022
5023                 escaped = ebsql_normalize_for_like (ebsql, test, TRUE, &need_escape);
5024
5025                 g_string_append_c (string, '(');
5026                 ebsql_string_append_column (string, field, EBSQL_SUFFIX_REVERSE);
5027                 g_string_append (string, " IS NOT NULL AND ");
5028
5029                 ebsql_string_append_column (string, field, EBSQL_SUFFIX_REVERSE);
5030                 g_string_append (string, " LIKE \'");
5031                 g_string_append (string, escaped);
5032                 g_string_append (string, "%\'");
5033
5034         } else {
5035
5036                 escaped = ebsql_normalize_for_like (ebsql, test, FALSE, &need_escape);
5037                 g_string_append_c (string, '(');
5038
5039                 ebsql_string_append_column (string, field, NULL);
5040                 g_string_append (string, " IS NOT NULL AND ");
5041
5042                 ebsql_string_append_column (string, field, NULL);
5043                 g_string_append (string, " LIKE \'%");
5044                 g_string_append (string, escaped);
5045                 g_string_append (string, "\'");
5046         }
5047
5048         if (need_escape)
5049                 g_string_append (string, EBSQL_ESCAPE_SEQUENCE);
5050
5051         g_string_append_c (string, ')');
5052         g_free (escaped);
5053 }
5054
5055 static void
5056 field_test_query_eqphone (EBookSqlite *ebsql,
5057                           GString *string,
5058                           QueryFieldTest *test)
5059 {
5060         SummaryField *field = test->field;
5061         QueryPhoneTest *phone_test = (QueryPhoneTest *) test;
5062
5063         if ((field->index & INDEX_FLAG (PHONE)) != 0) {
5064
5065                 g_string_append_c (string, '(');
5066                 ebsql_string_append_column (string, field, EBSQL_SUFFIX_PHONE);
5067                 ebsql_string_append_printf (string, " = %Q AND ", phone_test->national);
5068
5069                 /* For exact matches, a country code qualifier is required by both
5070                  * query input and row input
5071                  */
5072                 ebsql_string_append_column (string, field, EBSQL_SUFFIX_COUNTRY);
5073                 g_string_append (string, " != 0 AND ");
5074
5075                 ebsql_string_append_column (string, field, EBSQL_SUFFIX_COUNTRY);
5076                 ebsql_string_append_printf (string, " = %d", phone_test->country);
5077                 g_string_append_c (string, ')');
5078
5079         } else {
5080
5081                 /* No indexed columns available, perform the fallback */
5082                 g_string_append (string, EBSQL_FUNC_EQPHONE_EXACT " (");
5083                 ebsql_string_append_column (string, field, NULL);
5084                 ebsql_string_append_printf (string, ", %Q)", test->value);
5085         }
5086 }
5087
5088 static void
5089 field_test_query_eqphone_national (EBookSqlite *ebsql,
5090                                    GString *string,
5091                                    QueryFieldTest *test)
5092 {
5093
5094         SummaryField *field = test->field;
5095         QueryPhoneTest *phone_test = (QueryPhoneTest *) test;
5096
5097         if ((field->index & INDEX_FLAG (PHONE)) != 0) {
5098
5099                 /* Only a compound expression if there is a country code */
5100                 if (phone_test->country)
5101                         g_string_append_c (string, '(');
5102
5103                 /* Generate: phone = %Q */
5104                 ebsql_string_append_column (string, field, EBSQL_SUFFIX_PHONE);
5105                 ebsql_string_append_printf (string, " = %Q", phone_test->national);
5106
5107                 /* When doing a national search, no need to check country
5108                  * code unless the query number also has a country code
5109                  */
5110                 if (phone_test->country) {
5111                         /* Generate: (phone = %Q AND (country = 0 OR country = %d)) */
5112                         g_string_append (string, " AND (");
5113                         ebsql_string_append_column (string, field, EBSQL_SUFFIX_COUNTRY);
5114                         g_string_append (string, " = 0 OR ");
5115                         ebsql_string_append_column (string, field, EBSQL_SUFFIX_COUNTRY);
5116                         ebsql_string_append_printf (string, " = %d))", phone_test->country);
5117
5118                 }
5119
5120         } else {
5121
5122                 /* No indexed columns available, perform the fallback */
5123                 g_string_append (string, EBSQL_FUNC_EQPHONE_NATIONAL " (");
5124                 ebsql_string_append_column (string, field, NULL);
5125                 ebsql_string_append_printf (string, ", %Q)", test->value);
5126         }
5127 }
5128
5129 static void
5130 field_test_query_eqphone_short (EBookSqlite *ebsql,
5131                                 GString *string,
5132                                 QueryFieldTest *test)
5133 {
5134         SummaryField *field = test->field;
5135
5136         /* No quick way to do the short match */
5137         g_string_append (string, EBSQL_FUNC_EQPHONE_SHORT " (");
5138         ebsql_string_append_column (string, field, NULL);
5139         ebsql_string_append_printf (string, ", %Q)", test->value);
5140 }
5141
5142 static void
5143 field_test_query_regex_normal (EBookSqlite *ebsql,
5144                                GString *string,
5145                                QueryFieldTest *test)
5146 {
5147         SummaryField *field = test->field;
5148         gchar *normal;
5149
5150         normal = e_util_utf8_normalize (test->value);
5151
5152         if (field->aux_table)
5153                 ebsql_string_append_printf (
5154                         string, "%s.value REGEXP %Q",
5155                         field->aux_table_symbolic,
5156                         normal);
5157         else
5158                 ebsql_string_append_printf (
5159                         string, "summary.%s REGEXP %Q",
5160                         field->dbname,
5161                         normal);
5162
5163         g_free (normal);
5164 }
5165
5166 static void
5167 field_test_query_translit_is (EBookSqlite *ebsql,
5168                               GString *string,
5169                               QueryFieldTest *test)
5170 {
5171         SummaryField *field = test->field;
5172         gchar *normal, *translit;
5173
5174         translit = e_transliterator_transliterate (ebsql->priv->transliterator, test->value);
5175         normal = e_util_utf8_normalize (translit);
5176
5177         ebsql_string_append_column (string, field, EBSQL_SUFFIX_TRANSLIT);
5178         ebsql_string_append_printf (string, " = %Q", normal);
5179
5180         g_free (translit);
5181         g_free (normal);
5182 }
5183
5184 static void
5185 field_test_query_translit_contains (EBookSqlite *ebsql,
5186                                     GString *string,
5187                                     QueryFieldTest *test)
5188 {
5189         SummaryField *field = test->field;
5190         gboolean need_escape;
5191         gchar *escaped;
5192
5193         escaped = ebsql_normalize_for_like (ebsql, test, FALSE, &need_escape);
5194
5195         g_string_append_c (string, '(');
5196
5197         ebsql_string_append_column (string, field, EBSQL_SUFFIX_TRANSLIT);
5198         g_string_append (string, " IS NOT NULL AND ");
5199         ebsql_string_append_column (string, field, EBSQL_SUFFIX_TRANSLIT);
5200         g_string_append (string, " LIKE '%");
5201         g_string_append (string, escaped);
5202         g_string_append (string, "%'");
5203
5204         if (need_escape)
5205                 g_string_append (string, EBSQL_ESCAPE_SEQUENCE);
5206
5207         g_string_append_c (string, ')');
5208
5209         g_free (escaped);
5210 }
5211
5212 static void
5213 field_test_query_translit_begins_with (EBookSqlite *ebsql,
5214                                                                    GString *string,
5215                                                                    QueryFieldTest *test)
5216 {
5217         SummaryField *field = test->field;
5218         gboolean need_escape;
5219         gchar *escaped;
5220
5221         escaped = ebsql_normalize_for_like (ebsql, test, FALSE, &need_escape);
5222
5223         g_string_append_c (string, '(');
5224         ebsql_string_append_column (string, field, EBSQL_SUFFIX_TRANSLIT);
5225         g_string_append (string, " IS NOT NULL AND ");
5226
5227         ebsql_string_append_column (string, field, EBSQL_SUFFIX_TRANSLIT);
5228         g_string_append (string, " LIKE \'");
5229         g_string_append (string, escaped);
5230         g_string_append (string, "%\'");
5231
5232         if (need_escape)
5233                 g_string_append (string, EBSQL_ESCAPE_SEQUENCE);
5234         g_string_append_c (string, ')');
5235
5236         g_free (escaped);
5237 }
5238
5239 static void
5240 field_test_query_translit_ends_with (EBookSqlite *ebsql,
5241                                                                      GString *string,
5242                                                                      QueryFieldTest *test)
5243 {
5244         SummaryField *field = test->field;
5245         gboolean need_escape;
5246         gchar *escaped;
5247
5248         escaped = ebsql_normalize_for_like (ebsql, test, FALSE, &need_escape);
5249         g_string_append_c (string, '(');
5250
5251         ebsql_string_append_column (string, field, EBSQL_SUFFIX_TRANSLIT);
5252         g_string_append (string, " IS NOT NULL AND ");
5253
5254         ebsql_string_append_column (string, field, EBSQL_SUFFIX_TRANSLIT);
5255         g_string_append (string, " LIKE \'%");
5256         g_string_append (string, escaped);
5257         g_string_append (string, "\'");
5258
5259         if (need_escape)
5260                 g_string_append (string, EBSQL_ESCAPE_SEQUENCE);
5261
5262         g_string_append_c (string, ')');
5263         g_free (escaped);
5264 }
5265
5266 static void
5267 field_test_query_exists (EBookSqlite *ebsql,
5268                                                  GString *string,
5269                                                  QueryFieldTest *test)
5270 {
5271         SummaryField *field = test->field;
5272
5273         ebsql_string_append_column (string, field, NULL);
5274         ebsql_string_append_printf (string, " IS NOT NULL");
5275 }
5276
5277 /* Lookup table for field test generators per EBookQueryTest,
5278  *
5279  * WARNING: This must stay in line with the EBookQueryTest definition.
5280  */
5281 static const GenerateFieldTest field_test_func_table[] = {
5282         field_test_query_is,               /* E_BOOK_QUERY_IS */
5283         field_test_query_contains,         /* E_BOOK_QUERY_CONTAINS */
5284         field_test_query_begins_with,      /* E_BOOK_QUERY_BEGINS_WITH */
5285         field_test_query_ends_with,        /* E_BOOK_QUERY_ENDS_WITH */
5286         field_test_query_eqphone,          /* E_BOOK_QUERY_EQUALS_PHONE_NUMBER */
5287         field_test_query_eqphone_national, /* E_BOOK_QUERY_EQUALS_NATIONAL_PHONE_NUMBER */
5288         field_test_query_eqphone_short,    /* E_BOOK_QUERY_EQUALS_SHORT_PHONE_NUMBER */
5289         field_test_query_regex_normal,     /* E_BOOK_QUERY_REGEX_NORMAL */
5290         NULL /* Requires fallback */,      /* E_BOOK_QUERY_REGEX_RAW  */
5291         field_test_query_translit_is,      /* E_BOOK_QUERY_TRANSLIT_IS  */
5292         field_test_query_translit_contains,/* E_BOOK_QUERY_TRANSLIT_CONTAINS  */
5293         field_test_query_translit_begins_with,/* E_BOOK_QUERY_TRANSLIT_BEGINS_WITH  */
5294         field_test_query_translit_ends_with,/* E_BOOK_QUERY_TRANSLIT_ENDS_WITH  */
5295         field_test_query_exists,           /* BOOK_QUERY_EXISTS */
5296 };
5297
5298 /**********************************************************
5299  *                   Querying Contacts                    *
5300  **********************************************************/
5301
5302 /* The various search types indicate what should be fetched
5303  */
5304 typedef enum {
5305         SEARCH_FULL,          /* Get a list of EbSqlSearchData */
5306         SEARCH_UID_AND_REV,   /* Get a list of EbSqlSearchData, with shallow vcards only containing UID & REV */
5307         SEARCH_UID,           /* Get a list of UID strings */
5308         SEARCH_COUNT,         /* Get the number of matching rows */
5309 } SearchType;
5310
5311 static void
5312 ebsql_generate_constraints (EBookSqlite *ebsql,
5313                             GString *string,
5314                             GPtrArray *constraints,
5315                             const gchar *sexp)
5316 {
5317         SubQueryContext *ctx;
5318         QueryDelimiter *delim;
5319         QueryFieldTest *test;
5320         QueryElement **elements;
5321         gint n_elements, i;
5322
5323         /* If there are no constraints, we generate the fallback constraint for 'sexp' */
5324         if (constraints == NULL) {
5325                 ebsql_string_append_printf (
5326                         string,
5327                         EBSQL_FUNC_COMPARE_VCARD " (%Q, %s)",
5328                         sexp, EBSQL_VCARD_FRAGMENT (ebsql));
5329                 return;
5330         }
5331
5332         elements = (QueryElement **) constraints->pdata;
5333         n_elements = constraints->len;
5334
5335         ctx = sub_query_context_new ();
5336
5337         for (i = 0; i < n_elements; i++) {
5338                 GenerateFieldTest generate_test_func = NULL;
5339
5340                 /* Seperate field tests with the appropriate grouping */
5341                 if (elements[i]->query != BOOK_QUERY_SUB_END &&
5342                     sub_query_context_increment (ctx) > 0) {
5343                         guint delim_type = sub_query_context_peek_type (ctx);
5344
5345                         switch (delim_type) {
5346                         case BOOK_QUERY_SUB_AND:
5347
5348                                 g_string_append (string, " AND ");
5349                                 break;
5350
5351                         case BOOK_QUERY_SUB_OR:
5352
5353                                 g_string_append (string, " OR ");
5354                                 break;
5355
5356                         case BOOK_QUERY_SUB_NOT:
5357
5358                                 /* Nothing to do between children of NOT,
5359                                  * there should only ever be one child of NOT anyway
5360                                  */
5361                                 break;
5362
5363                         case BOOK_QUERY_SUB_END:
5364                         default:
5365                                 g_warn_if_reached ();
5366                         }
5367                 }
5368
5369                 if (elements[i]->query >= BOOK_QUERY_SUB_FIRST) {
5370                         delim = (QueryDelimiter *) elements[i];
5371
5372                         switch (delim->query) {
5373
5374                         case BOOK_QUERY_SUB_NOT:
5375
5376                                 /* NOT is a unary operator and as such
5377                                  * comes before the opening parenthesis
5378                                  */
5379                                 g_string_append (string, "NOT ");
5380
5381                                 /* Fall through */
5382
5383                         case BOOK_QUERY_SUB_AND:
5384                         case BOOK_QUERY_SUB_OR:
5385
5386                                 /* Open a grouped statement and push the context */
5387                                 sub_query_context_push (ctx, delim->query);
5388                                 g_string_append_c (string, '(');
5389                                 break;
5390
5391                         case BOOK_QUERY_SUB_END:
5392                                 /* Close a grouped statement and pop the context */
5393                                 g_string_append_c (string, ')');
5394                                 sub_query_context_pop (ctx);
5395                                 break;
5396                         default:
5397                                 g_warn_if_reached ();
5398                         }
5399
5400                         continue;
5401                 }
5402
5403                 /* Find the appropriate field test generator */
5404                 test = (QueryFieldTest *) elements[i];
5405                 if (test->query < G_N_ELEMENTS (field_test_func_table))
5406                         generate_test_func = field_test_func_table[test->query];
5407
5408                 /* These should never happen, if it does it should be
5409                  * fixed in the preflight checks
5410                  */
5411                 g_warn_if_fail (generate_test_func != NULL);
5412                 g_warn_if_fail (test->field != NULL);
5413
5414                 /* Generate the field test */
5415                 generate_test_func (ebsql, string, test);
5416         }
5417
5418         sub_query_context_free (ctx);
5419 }
5420
5421 /* Generates the SELECT portion of the query, this will take care of
5422  * preparing the context of the query, and add the needed JOIN statements
5423  * based on which fields are referenced in the query expression.
5424  *
5425  * This also handles getting the correct callback and asking for the
5426  * right data depending on the 'search_type'
5427  */
5428 static EbSqlRowFunc
5429 ebsql_generate_select (EBookSqlite *ebsql,
5430                        GString *string,
5431                        SearchType search_type,
5432                        PreflightContext *context,
5433                        GError **error)
5434 {
5435         EbSqlRowFunc callback = NULL;
5436         gboolean add_auxiliary_tables = FALSE;
5437         gint i;
5438
5439         if (context->status == PREFLIGHT_OK &&
5440             context->aux_mask != 0)
5441                 add_auxiliary_tables = TRUE;
5442
5443         g_string_append (string, "SELECT ");
5444         if (add_auxiliary_tables)
5445                 g_string_append (string, "DISTINCT ");
5446
5447         switch (search_type) {
5448         case SEARCH_FULL:
5449                 callback = collect_full_results_cb;
5450                 g_string_append (string, "summary.uid, ");
5451                 g_string_append (string, EBSQL_VCARD_FRAGMENT (ebsql));
5452                 g_string_append (string, ", summary.bdata ");
5453                 break;
5454         case SEARCH_UID_AND_REV:
5455                 callback = collect_lean_results_cb;
5456                 g_string_append (string, "summary.uid, summary.Rev, summary.bdata ");
5457                 break;
5458         case SEARCH_UID:
5459                 callback = collect_uid_results_cb;
5460                 g_string_append (string, "summary.uid ");
5461                 break;
5462         case SEARCH_COUNT:
5463                 callback = get_count_cb;
5464                 if (context->aux_mask != 0)
5465                         g_string_append (string, "count (DISTINCT summary.uid) ");
5466                 else
5467                         g_string_append (string, "count (*) ");
5468                 break;
5469         }
5470
5471         ebsql_string_append_printf (string, "FROM %Q AS summary", ebsql->priv->folderid);
5472
5473         /* Add any required auxiliary tables into the query context */
5474         if (add_auxiliary_tables) {
5475                 for (i = 0; i < ebsql->priv->n_summary_fields; i++) {
5476
5477                         /* We cap this at EBSQL_MAX_SUMMARY_FIELDS (64 bits) at creation time */
5478                         if ((context->aux_mask & (1 << i)) != 0) {
5479                                 SummaryField *field = &(ebsql->priv->summary_fields[i]);
5480
5481                                 /* Note the '+' in the JOIN statement.
5482                                  *
5483                                  * This plus makes the uid's index ineligable to participate
5484                                  * in any indexing.
5485                                  *
5486                                  * Without this, the indexes which we prefer for prefix or
5487                                  * suffix matching in the auxiliary tables are ignored and
5488                                  * only considered on exact matches.
5489                                  *
5490                                  * This is crucial to ensure that the uid index does not
5491                                  * compete with the value index in constraints such as:
5492                                  *
5493                                  *     WHERE email_list.value LIKE "boogieman%"
5494                                  */
5495                                 ebsql_string_append_printf (
5496                                         string, " JOIN %Q AS %s ON +%s.uid = summary.uid",
5497                                         field->aux_table,
5498                                         field->aux_table_symbolic,
5499                                         field->aux_table_symbolic);
5500                         }
5501                 }
5502         }
5503
5504         return callback;
5505 }
5506
5507 static gboolean
5508 ebsql_do_search_query (EBookSqlite *ebsql,
5509                        PreflightContext *context,
5510                        const gchar *sexp,
5511                        SearchType search_type,
5512                        GSList **return_data,
5513                        GCancellable *cancellable,
5514                        GError **error)
5515 {
5516         GString *string;
5517         EbSqlRowFunc callback = NULL;
5518         gboolean success = FALSE;
5519
5520         /* We might calculate a reasonable estimation of bytes
5521          * during the preflight checks */
5522         string = g_string_sized_new (GENERATED_QUERY_BYTES);
5523
5524         /* Generate the leading SELECT statement */
5525         callback = ebsql_generate_select (
5526                 ebsql, string, search_type, context, error);
5527
5528         if (callback &&
5529             EBSQL_STATUS_GEN_CONSTRAINTS (context->status)) {
5530                 /*
5531                  * Now generate the search expression on the main contacts table
5532                  */
5533                 g_string_append (string, " WHERE ");
5534                 ebsql_generate_constraints (
5535                         ebsql, string, context->constraints, sexp);
5536         }
5537
5538         if (callback)
5539                 success = ebsql_exec (
5540                         ebsql, string->str,
5541                         callback, return_data,
5542                         cancellable, error);
5543
5544         g_string_free (string, TRUE);
5545
5546         return success;
5547 }
5548
5549 /* ebsql_search_query:
5550  * @ebsql: An EBookSqlite
5551  * @sexp: The search expression, or NULL for all contacts
5552  * @search_type: Indicates what kind of data should be returned
5553  * @return_data: A list of data fetched from the DB, as specified by 'search_type'
5554  * @error: Location to store any error which may have occurred
5555  *
5556  * This is the main common entry point for querying contacts.
5557  *
5558  * If the query cannot be satisfied with the summary, then
5559  * a fallback will automatically be used.
5560  */
5561 static gboolean
5562 ebsql_search_query (EBookSqlite *ebsql,
5563                     const gchar *sexp,
5564                     SearchType search_type,
5565                     GSList **return_data,
5566                     GCancellable *cancellable,
5567                     GError **error)
5568 {
5569         PreflightContext context = PREFLIGHT_CONTEXT_INIT;
5570         gboolean success = FALSE;
5571
5572         /* Now start with the query preflighting */
5573         query_preflight (&context, ebsql, sexp);
5574
5575         switch (context.status) {
5576         case PREFLIGHT_OK:
5577         case PREFLIGHT_LIST_ALL:
5578         case PREFLIGHT_NOT_SUMMARIZED:
5579                 /* No errors, let's really search */
5580                 success = ebsql_do_search_query (
5581                         ebsql, &context, sexp,
5582                         search_type, return_data,
5583                         cancellable, error);
5584                 break;
5585
5586         case PREFLIGHT_INVALID:
5587                 EBSQL_SET_ERROR (
5588                         error,
5589                         E_BOOK_SQLITE_ERROR_INVALID_QUERY,
5590                         _("Invalid query: %s"), sexp);
5591                 break;
5592
5593         case PREFLIGHT_UNSUPPORTED:
5594                 EBSQL_SET_ERROR_LITERAL (
5595                         error,
5596                         E_BOOK_SQLITE_ERROR_UNSUPPORTED_QUERY,
5597                         _("Query contained unsupported elements"));
5598                 break;
5599         }
5600
5601         preflight_context_clear (&context);
5602
5603         return success;
5604 }
5605
5606 /******************************************************************
5607  *                    EbSqlCursor Implementation                  *
5608  ******************************************************************/
5609 typedef struct _CursorState CursorState;
5610
5611 struct _CursorState {
5612         gchar            **values;    /* The current cursor position, results will be returned after this position */
5613         gchar             *last_uid;  /* The current cursor contact UID position, used as a tie breaker */
5614         EbSqlCursorOrigin  position;  /* The position is updated with the cursor state and is used to distinguish
5615                                        * between the beginning and the ending of the cursor's contact list.
5616                                        * While the cursor is in a non-null state, the position will be
5617                                        * EBSQL_CURSOR_ORIGIN_CURRENT.
5618                                        */
5619 };
5620
5621 struct _EbSqlCursor {
5622         EBookBackendSExp *sexp;       /* An EBookBackendSExp based on the query, used by e_book_sqlite_cursor_compare () */
5623         gchar         *select_vcards; /* The first fragment when querying results */
5624         gchar         *select_count;  /* The first fragment when querying contact counts */
5625         gchar         *query;         /* The SQL query expression derived from the passed search expression */
5626         gchar         *order;         /* The normal order SQL query fragment to append at the end, containing ORDER BY etc */
5627         gchar         *reverse_order; /* The reverse order SQL query fragment to append at the end, containing ORDER BY etc */
5628
5629         EContactField       *sort_fields;   /* The fields to sort in a query in the order or sort priority */
5630         EBookCursorSortType *sort_types;    /* The sort method to use for each field */
5631         gint                 n_sort_fields; /* The amound of sort fields */
5632
5633         CursorState          state;
5634 };
5635
5636 static CursorState *cursor_state_copy             (EbSqlCursor          *cursor,
5637                                                    CursorState          *state);
5638 static void         cursor_state_free             (EbSqlCursor          *cursor,
5639                                                    CursorState          *state);
5640 static void         cursor_state_clear            (EbSqlCursor          *cursor,
5641                                                    CursorState          *state,
5642                                                    EbSqlCursorOrigin     position);
5643 static void         cursor_state_set_from_contact (EBookSqlite          *ebsql,
5644                                                    EbSqlCursor          *cursor,
5645                                                    CursorState          *state,
5646                                                    EContact             *contact);
5647 static void         cursor_state_set_from_vcard   (EBookSqlite          *ebsql,
5648                                                    EbSqlCursor          *cursor,
5649                                                    CursorState          *state,
5650                                                    const gchar          *vcard);
5651
5652 static CursorState *
5653 cursor_state_copy (EbSqlCursor *cursor,
5654                    CursorState *state)
5655 {
5656         CursorState *copy;
5657         gint i;
5658
5659         copy = g_slice_new0 (CursorState);
5660         copy->values = g_new0 (gchar *, cursor->n_sort_fields);
5661
5662         for (i = 0; i < cursor->n_sort_fields; i++)
5663                 copy->values[i] = g_strdup (state->values[i]);
5664
5665         copy->last_uid = g_strdup (state->last_uid);
5666         copy->position = state->position;
5667
5668         return copy;
5669 }
5670
5671 static void
5672 cursor_state_free (EbSqlCursor *cursor,
5673                    CursorState *state)
5674 {
5675         if (state) {
5676                 cursor_state_clear (cursor, state, EBSQL_CURSOR_ORIGIN_BEGIN);
5677                 g_free (state->values);
5678                 g_slice_free (CursorState, state);
5679         }
5680 }
5681
5682 static void
5683 cursor_state_clear (EbSqlCursor *cursor,
5684                     CursorState *state,
5685                     EbSqlCursorOrigin position)
5686 {
5687         gint i;
5688
5689         for (i = 0; i < cursor->n_sort_fields; i++) {
5690                 g_free (state->values[i]);
5691                 state->values[i] = NULL;
5692         }
5693
5694         g_free (state->last_uid);
5695         state->last_uid = NULL;
5696         state->position = position;
5697 }
5698
5699 static void
5700 cursor_state_set_from_contact (EBookSqlite *ebsql,
5701                                EbSqlCursor *cursor,
5702                                CursorState *state,
5703                                EContact *contact)
5704 {
5705         gint i;
5706
5707         cursor_state_clear (cursor, state, EBSQL_CURSOR_ORIGIN_BEGIN);
5708
5709         for (i = 0; i < cursor->n_sort_fields; i++) {
5710                 const gchar *string = e_contact_get_const (contact, cursor->sort_fields[i]);
5711                 SummaryField *field;
5712                 gchar *sort_key;
5713
5714                 if (string)
5715                         sort_key = e_collator_generate_key (
5716                                 ebsql->priv->collator,
5717                                 string, NULL);
5718                 else
5719                         sort_key = g_strdup ("");
5720
5721                 field = summary_field_get (ebsql, cursor->sort_fields[i]);
5722
5723                 if (field && (field->index & INDEX_FLAG (SORT_KEY)) != 0) {
5724                         state->values[i] = sort_key;
5725                 } else {
5726                         state->values[i] = ebsql_encode_vcard_sort_key (sort_key);
5727                         g_free (sort_key);
5728                 }
5729         }
5730
5731         state->last_uid = e_contact_get (contact, E_CONTACT_UID);
5732         state->position = EBSQL_CURSOR_ORIGIN_CURRENT;
5733 }
5734
5735 static void
5736 cursor_state_set_from_vcard (EBookSqlite *ebsql,
5737                              EbSqlCursor *cursor,
5738                              CursorState *state,
5739                              const gchar *vcard)
5740 {
5741         EContact *contact;
5742
5743         contact = e_contact_new_from_vcard (vcard);
5744         cursor_state_set_from_contact (ebsql, cursor, state, contact);
5745         g_object_unref (contact);
5746 }
5747
5748 static gboolean
5749 ebsql_cursor_setup_query (EBookSqlite *ebsql,
5750                           EbSqlCursor *cursor,
5751                           const gchar *sexp,
5752                           GError **error)
5753 {
5754         PreflightContext context = PREFLIGHT_CONTEXT_INIT;
5755         GString *string;
5756
5757         /* Preflighting and error checking */
5758         if (sexp) {
5759                 query_preflight (&context, ebsql, sexp);
5760
5761                 if (context.status > PREFLIGHT_NOT_SUMMARIZED) {
5762                         EBSQL_SET_ERROR_LITERAL (
5763                                 error,
5764                                 E_BOOK_SQLITE_ERROR_INVALID_QUERY,
5765                                 _("Invalid query for EbSqlCursor"));
5766
5767                         preflight_context_clear (&context);
5768                         return FALSE;
5769
5770                 }
5771         }
5772
5773         /* Now we caught the errors, let's generate our queries and get out of here ... */
5774         g_free (cursor->select_vcards);
5775         g_free (cursor->select_count);
5776         g_free (cursor->query);
5777         g_clear_object (&(cursor->sexp));
5778
5779         /* Generate the leading SELECT portions that we need */
5780         string = g_string_new ("");
5781         ebsql_generate_select (ebsql, string, SEARCH_FULL, &context, NULL);
5782         cursor->select_vcards = g_string_free (string, FALSE);
5783
5784         string = g_string_new ("");
5785         ebsql_generate_select (ebsql, string, SEARCH_COUNT, &context, NULL);
5786         cursor->select_count = g_string_free (string, FALSE);
5787
5788         if (sexp == NULL || context.status == PREFLIGHT_LIST_ALL) {
5789                 cursor->query = NULL;
5790                 cursor->sexp = NULL;
5791         } else {
5792                 /* Generate the constraints for our queries
5793                  */
5794                 string = g_string_new (NULL);
5795                 ebsql_generate_constraints (
5796                         ebsql, string, context.constraints, sexp);
5797                 cursor->query = g_string_free (string, FALSE);
5798                 cursor->sexp = e_book_backend_sexp_new (sexp);
5799         }
5800
5801         preflight_context_clear (&context);
5802
5803         return TRUE;
5804 }
5805
5806 static gchar *
5807 ebsql_cursor_order_by_fragment (EBookSqlite *ebsql,
5808                                 const EContactField *sort_fields,
5809                                 const EBookCursorSortType *sort_types,
5810                                 guint n_sort_fields,
5811                                 gboolean reverse)
5812 {
5813         GString *string;
5814         gint i;
5815
5816         string = g_string_new ("ORDER BY ");
5817
5818         for (i = 0; i < n_sort_fields; i++) {
5819                 SummaryField *field = summary_field_get (ebsql, sort_fields[i]);
5820
5821                 if (i > 0)
5822                         g_string_append (string, ", ");
5823
5824                 if (field &&
5825                     (field->index & INDEX_FLAG (SORT_KEY)) != 0) {
5826                         g_string_append (string, "summary.");
5827                         g_string_append (string, field->dbname);
5828                         g_string_append (string, "_" EBSQL_SUFFIX_SORT_KEY " ");
5829                 } else {
5830                         g_string_append (string, EBSQL_VCARD_FRAGMENT (ebsql));
5831                         g_string_append (string, " COLLATE ");
5832                         g_string_append (string, EBSQL_COLLATE_PREFIX);
5833                         g_string_append (string, e_contact_field_name (sort_fields[i]));
5834                         g_string_append_c (string, ' ');
5835                 }
5836
5837                 if (reverse)
5838                         g_string_append (string, (sort_types[i] == E_BOOK_CURSOR_SORT_ASCENDING ? "DESC" : "ASC"));
5839                 else
5840                         g_string_append (string, (sort_types[i] == E_BOOK_CURSOR_SORT_ASCENDING ? "ASC" : "DESC"));
5841         }
5842
5843         /* Also order the UID, since it's our tie breaker */
5844         if (n_sort_fields > 0)
5845                 g_string_append (string, ", ");
5846
5847         g_string_append (string, "summary.uid ");
5848         g_string_append (string, reverse ? "DESC" : "ASC");
5849
5850         return g_string_free (string, FALSE);
5851 }
5852
5853 static EbSqlCursor *
5854 ebsql_cursor_new (EBookSqlite *ebsql,
5855                   const gchar *sexp,
5856                   const EContactField *sort_fields,
5857                   const EBookCursorSortType *sort_types,
5858                   guint n_sort_fields)
5859 {
5860         EbSqlCursor *cursor = g_slice_new0 (EbSqlCursor);
5861
5862         cursor->order = ebsql_cursor_order_by_fragment (
5863                 ebsql, sort_fields, sort_types, n_sort_fields, FALSE);
5864         cursor->reverse_order = ebsql_cursor_order_by_fragment (
5865                 ebsql, sort_fields, sort_types, n_sort_fields, TRUE);
5866
5867         /* Sort parameters */
5868         cursor->n_sort_fields = n_sort_fields;
5869         cursor->sort_fields = g_memdup (sort_fields, sizeof (EContactField) * n_sort_fields);
5870         cursor->sort_types = g_memdup (sort_types,  sizeof (EBookCursorSortType) * n_sort_fields);
5871
5872         /* Cursor state */
5873         cursor->state.values = g_new0 (gchar *, n_sort_fields);
5874         cursor->state.last_uid = NULL;
5875         cursor->state.position = EBSQL_CURSOR_ORIGIN_BEGIN;
5876
5877         return cursor;
5878 }
5879
5880 static void
5881 ebsql_cursor_free (EbSqlCursor *cursor)
5882 {
5883         if (cursor) {
5884                 cursor_state_clear (cursor, &(cursor->state), EBSQL_CURSOR_ORIGIN_BEGIN);
5885                 g_free (cursor->state.values);
5886
5887                 g_clear_object (&(cursor->sexp));
5888                 g_free (cursor->select_vcards);
5889                 g_free (cursor->select_count);
5890                 g_free (cursor->query);
5891                 g_free (cursor->order);
5892                 g_free (cursor->reverse_order);
5893                 g_free (cursor->sort_fields);
5894                 g_free (cursor->sort_types);
5895
5896                 g_slice_free (EbSqlCursor, cursor);
5897         }
5898 }
5899
5900 #define GREATER_OR_LESS(cursor, idx, reverse) \
5901         (reverse ? \
5902          (((EbSqlCursor *) cursor)->sort_types[idx] == E_BOOK_CURSOR_SORT_ASCENDING ? '<' : '>') : \
5903          (((EbSqlCursor *) cursor)->sort_types[idx] == E_BOOK_CURSOR_SORT_ASCENDING ? '>' : '<'))
5904
5905 static inline void
5906 ebsql_cursor_format_equality (EBookSqlite *ebsql,
5907                               GString *string,
5908                               EContactField field_id,
5909                               const gchar *value,
5910                               gchar equality)
5911 {
5912         SummaryField *field = summary_field_get (ebsql, field_id);
5913
5914         if (field &&
5915             (field->index & INDEX_FLAG (SORT_KEY)) != 0) {
5916
5917                 g_string_append (string, "summary.");
5918                 g_string_append (string, field->dbname);
5919                 g_string_append (string, "_" EBSQL_SUFFIX_SORT_KEY " ");
5920
5921                 ebsql_string_append_printf (string, "%c %Q", equality, value);
5922
5923         } else {
5924                 ebsql_string_append_printf (
5925                         string, "(%s %c %Q ",
5926                         EBSQL_VCARD_FRAGMENT (ebsql),
5927                         equality, value);
5928
5929                 g_string_append (string, "COLLATE " EBSQL_COLLATE_PREFIX);
5930                 g_string_append (string, e_contact_field_name (field_id));
5931                 g_string_append_c (string, ')');
5932         }
5933 }
5934
5935 static gchar *
5936 ebsql_cursor_constraints (EBookSqlite *ebsql,
5937                           EbSqlCursor *cursor,
5938                           CursorState *state,
5939                           gboolean reverse,
5940                           gboolean include_current_uid)
5941 {
5942         GString *string;
5943         gint i, j;
5944
5945         /* Example for:
5946          *    ORDER BY family_name ASC, given_name DESC
5947          *
5948          * Where current cursor values are:
5949          *    family_name = Jackson
5950          *    given_name  = Micheal
5951          *
5952          * With reverse = FALSE
5953          *
5954          *    (summary.family_name > 'Jackson') OR
5955          *    (summary.family_name = 'Jackson' AND summary.given_name < 'Micheal') OR
5956          *    (summary.family_name = 'Jackson' AND summary.given_name = 'Micheal' AND summary.uid > 'last-uid')
5957          *
5958          * With reverse = TRUE (needed for moving the cursor backwards through results)
5959          *
5960          *    (summary.family_name < 'Jackson') OR
5961          *    (summary.family_name = 'Jackson' AND summary.given_name > 'Micheal') OR
5962          *    (summary.family_name = 'Jackson' AND summary.given_name = 'Micheal' AND summary.uid < 'last-uid')
5963          *
5964          */
5965         string = g_string_new (NULL);
5966
5967         for (i = 0; i <= cursor->n_sort_fields; i++) {
5968
5969                 /* Break once we hit a NULL value */
5970                 if ((i < cursor->n_sort_fields && state->values[i] == NULL) ||
5971                     (i == cursor->n_sort_fields && state->last_uid == NULL))
5972                         break;
5973
5974                 /* Between each qualifier, add an 'OR' */
5975                 if (i > 0)
5976                         g_string_append (string, " OR ");
5977
5978                 /* Begin qualifier */
5979                 g_string_append_c (string, '(');
5980
5981                 /* Create the '=' statements leading up to the current tie breaker */
5982                 for (j = 0; j < i; j++) {
5983                         ebsql_cursor_format_equality (ebsql, string,
5984                                                       cursor->sort_fields[j],
5985                                                       state->values[j], '=');
5986                         g_string_append (string, " AND ");
5987                 }
5988
5989                 if (i == cursor->n_sort_fields) {
5990
5991                         /* The 'include_current_uid' clause is used for calculating
5992                          * the current position of the cursor, inclusive of the
5993                          * current position.
5994                          */
5995                         if (include_current_uid)
5996                                 g_string_append_c (string, '(');
5997
5998                         /* Append the UID tie breaker */
5999                         ebsql_string_append_printf (
6000                                 string,
6001                                 "summary.uid %c %Q",
6002                                 reverse ? '<' : '>',
6003                                 state->last_uid);
6004
6005                         if (include_current_uid)
6006                                 ebsql_string_append_printf (
6007                                         string,
6008                                         " OR summary.uid = %Q)",
6009                                         state->last_uid);
6010
6011                 } else {
6012
6013                         /* SPECIAL CASE: If we have a parially set cursor state, then we must
6014                          * report next results that are inclusive of the final qualifier.
6015                          *
6016                          * This allows one to set the cursor with the family name set to 'J'
6017                          * and include the results for contact's Mr & Miss 'J'.
6018                          */
6019                         gboolean include_exact_match =
6020                                 (reverse == FALSE &&
6021                                  ((i + 1 < cursor->n_sort_fields && state->values[i + 1] == NULL) ||
6022                                   (i + 1 == cursor->n_sort_fields && state->last_uid == NULL)));
6023
6024                         if (include_exact_match)
6025                                 g_string_append_c (string, '(');
6026
6027                         /* Append the final qualifier for this field */
6028                         ebsql_cursor_format_equality (ebsql, string,
6029                                                       cursor->sort_fields[i],
6030                                                       state->values[i],
6031                                                       GREATER_OR_LESS (cursor, i, reverse));
6032
6033                         if (include_exact_match) {
6034                                 g_string_append (string, " OR ");
6035                                 ebsql_cursor_format_equality (ebsql, string,
6036                                                               cursor->sort_fields[i],
6037                                                               state->values[i], '=');
6038                                 g_string_append_c (string, ')');
6039                         }
6040                 }
6041
6042                 /* End qualifier */
6043                 g_string_append_c (string, ')');
6044         }
6045
6046         return g_string_free (string, FALSE);
6047 }
6048
6049 static gboolean
6050 cursor_count_total_locked (EBookSqlite *ebsql,
6051                            EbSqlCursor *cursor,
6052                            gint *total,
6053                            GError **error)
6054 {
6055         GString *query;
6056         gboolean success;
6057
6058         query = g_string_new (cursor->select_count);
6059
6060         /* Add the filter constraints (if any) */
6061         if (cursor->query) {
6062                 g_string_append (query, " WHERE ");
6063
6064                 g_string_append_c (query, '(');
6065                 g_string_append (query, cursor->query);
6066                 g_string_append_c (query, ')');
6067         }
6068
6069         /* Execute the query */
6070         success = ebsql_exec (ebsql, query->str, get_count_cb, total, NULL, error);
6071
6072         g_string_free (query, TRUE);
6073
6074         return success;
6075 }
6076
6077 static gboolean
6078 cursor_count_position_locked (EBookSqlite *ebsql,
6079                               EbSqlCursor *cursor,
6080                               gint *position,
6081                               GError **error)
6082 {
6083         GString *query;
6084         gboolean success;
6085
6086         query = g_string_new (cursor->select_count);
6087
6088         /* Add the filter constraints (if any) */
6089         if (cursor->query) {
6090                 g_string_append (query, " WHERE ");
6091
6092                 g_string_append_c (query, '(');
6093                 g_string_append (query, cursor->query);
6094                 g_string_append_c (query, ')');
6095         }
6096
6097         /* Add the cursor constraints (if any) */
6098         if (cursor->state.values[0] != NULL) {
6099                 gchar *constraints = NULL;
6100
6101                 if (!cursor->query)
6102                         g_string_append (query, " WHERE ");
6103                 else
6104                         g_string_append (query, " AND ");
6105
6106                 /* Here we do a reverse query, we're looking for all the
6107                  * results leading up to the current cursor value, including
6108                  * the cursor value
6109                  */
6110                 constraints = ebsql_cursor_constraints (
6111                         ebsql, cursor, &(cursor->state), TRUE, TRUE);
6112
6113                 g_string_append_c (query, '(');
6114                 g_string_append (query, constraints);
6115                 g_string_append_c (query, ')');
6116
6117                 g_free (constraints);
6118         }
6119
6120         /* Execute the query */
6121         success = ebsql_exec (ebsql, query->str, get_count_cb, position, NULL, error);
6122
6123         g_string_free (query, TRUE);
6124
6125         return success;
6126 }
6127
6128 /**********************************************************
6129  *                     GObjectClass                       *
6130  **********************************************************/
6131 static void
6132 e_book_sqlite_dispose (GObject *object)
6133 {
6134         EBookSqlite *ebsql = E_BOOK_SQLITE (object);
6135
6136         ebsql_unregister_from_hash (ebsql);
6137
6138         /* Chain up to parent's dispose() method. */
6139         G_OBJECT_CLASS (e_book_sqlite_parent_class)->dispose (object);
6140 }
6141
6142 static void
6143 e_book_sqlite_finalize (GObject *object)
6144 {
6145         EBookSqlite *ebsql = E_BOOK_SQLITE (object);
6146         EBookSqlitePrivate *priv = ebsql->priv;
6147
6148         summary_fields_array_free (
6149                 priv->summary_fields,
6150                 priv->n_summary_fields);
6151
6152         g_free (priv->folderid);
6153         g_free (priv->path);
6154         g_free (priv->locale);
6155         g_free (priv->region_code);
6156
6157         if (priv->collator)
6158                 e_collator_unref (priv->collator);
6159
6160         if (ebsql->priv->transliterator)
6161                 e_transliterator_unref (ebsql->priv->transliterator);
6162
6163         g_clear_object (&priv->source);
6164
6165         g_mutex_clear (&priv->lock);
6166         g_mutex_clear (&priv->updates_lock);
6167
6168         if (priv->multi_deletes)
6169                 g_hash_table_destroy (priv->multi_deletes);
6170
6171         if (priv->multi_inserts)
6172                 g_hash_table_destroy (priv->multi_inserts);
6173
6174         if (priv->user_data && priv->user_data_destroy)
6175                 priv->user_data_destroy (priv->user_data);
6176
6177         sqlite3_finalize (priv->insert_stmt);
6178         sqlite3_finalize (priv->replace_stmt);
6179         sqlite3_close (priv->db);
6180
6181         EBSQL_NOTE (REF_COUNTS, g_printerr ("EBookSqlite finalized\n"));
6182
6183         /* Chain up to parent's finalize() method. */
6184         G_OBJECT_CLASS (e_book_sqlite_parent_class)->finalize (object);
6185 }
6186
6187 static void
6188 e_book_sqlite_constructed (GObject *object)
6189 {
6190         /* Chain up to parent's constructed() method. */
6191         G_OBJECT_CLASS (e_book_sqlite_parent_class)->constructed (object);
6192
6193         e_extensible_load_extensions (E_EXTENSIBLE (object));
6194 }
6195
6196 static gboolean
6197 ebsql_signals_accumulator (GSignalInvocationHint *ihint,
6198                            GValue *return_accu,
6199                            const GValue *handler_return,
6200                            gpointer data)
6201 {
6202         gboolean handler_result;
6203
6204         handler_result = g_value_get_boolean (handler_return);
6205         g_value_set_boolean (return_accu, handler_result);
6206
6207         return handler_result;
6208 }
6209
6210 static gboolean
6211 ebsql_before_insert_contact_default (EBookSqlite *ebsql,
6212                                      gpointer db,
6213                                      EContact *contact,
6214                                      const gchar *extra,
6215                                      gboolean replace,
6216                                      GCancellable *cancellable,
6217                                      GError **error)
6218 {
6219         return TRUE;
6220 }
6221
6222 static gboolean
6223 ebsql_before_remove_contact_default (EBookSqlite *ebsql,
6224                                      gpointer db,
6225                                      const gchar *contact_uid,
6226                                      GCancellable *cancellable,
6227                                      GError **error)
6228 {
6229         return TRUE;
6230 }
6231
6232 static void
6233 e_book_sqlite_class_init (EBookSqliteClass *class)
6234 {
6235         GObjectClass *object_class;
6236
6237         g_type_class_add_private (class, sizeof (EBookSqlitePrivate));
6238
6239         object_class = G_OBJECT_CLASS (class);
6240         object_class->dispose = e_book_sqlite_dispose;
6241         object_class->finalize = e_book_sqlite_finalize;
6242         object_class->constructed = e_book_sqlite_constructed;
6243
6244         class->before_insert_contact = ebsql_before_insert_contact_default;
6245         class->before_remove_contact = ebsql_before_remove_contact_default;
6246
6247         /* Parse the EBSQL_DEBUG environment variable */
6248         ebsql_init_debug ();
6249
6250         signals[BEFORE_INSERT_CONTACT] = g_signal_new (
6251                 "before-insert-contact",
6252                 G_OBJECT_CLASS_TYPE (class),
6253                 G_SIGNAL_RUN_LAST,
6254                 G_STRUCT_OFFSET (EBookSqliteClass, before_insert_contact),
6255                 ebsql_signals_accumulator,
6256                 NULL,
6257                 g_cclosure_marshal_generic,
6258                 G_TYPE_BOOLEAN, 6,
6259                 G_TYPE_POINTER,
6260                 G_TYPE_OBJECT,
6261                 G_TYPE_STRING,
6262                 G_TYPE_BOOLEAN,
6263                 G_TYPE_OBJECT,
6264                 G_TYPE_POINTER);
6265
6266         signals[BEFORE_REMOVE_CONTACT] = g_signal_new (
6267                 "before-remove-contact",
6268                 G_OBJECT_CLASS_TYPE (class),
6269                 G_SIGNAL_RUN_LAST,
6270                 G_STRUCT_OFFSET (EBookSqliteClass, before_remove_contact),
6271                 ebsql_signals_accumulator,
6272                 NULL,
6273                 g_cclosure_marshal_generic,
6274                 G_TYPE_BOOLEAN, 4,
6275                 G_TYPE_POINTER,
6276                 G_TYPE_STRING,
6277                 G_TYPE_OBJECT,
6278                 G_TYPE_POINTER);
6279 }
6280
6281 static void
6282 e_book_sqlite_init (EBookSqlite *ebsql)
6283 {
6284         ebsql->priv = E_BOOK_SQLITE_GET_PRIVATE (ebsql);
6285
6286         g_mutex_init (&ebsql->priv->lock);
6287         g_mutex_init (&ebsql->priv->updates_lock);
6288
6289         ebsql->priv->transliterator = e_transliterator_new ("Any-Latin");
6290 }
6291
6292 /**********************************************************
6293  *                          API                           *
6294  **********************************************************/
6295 static EBookSqlite *
6296 ebsql_new_default (const gchar *path,
6297                    ESource *source,
6298                    EbSqlVCardCallback vcard_callback,
6299                    EbSqlChangeCallback change_callback,
6300                    gpointer user_data,
6301                    GDestroyNotify user_data_destroy,
6302                    GCancellable *cancellable,
6303                    GError **error)
6304 {
6305         EBookSqlite *ebsql;
6306         GArray *summary_fields;
6307         gint i;
6308
6309         /* Create the default summary structs */
6310         summary_fields = g_array_new (FALSE, FALSE, sizeof (SummaryField));
6311         for (i = 0; i < G_N_ELEMENTS (default_summary_fields); i++)
6312                 summary_field_append (summary_fields, DEFAULT_FOLDER_ID, default_summary_fields[i], NULL);
6313
6314         /* Add the default index flags */
6315         summary_fields_add_indexes (
6316                 summary_fields,
6317                 default_indexed_fields,
6318                 default_index_types,
6319                 G_N_ELEMENTS (default_indexed_fields));
6320
6321         ebsql = ebsql_new_internal (
6322                 path, source,
6323                 vcard_callback, change_callback,
6324                 user_data, user_data_destroy,
6325                 (SummaryField *) summary_fields->data,
6326                 summary_fields->len,
6327                 cancellable, error);
6328
6329         g_array_free (summary_fields, FALSE);
6330
6331         return ebsql;
6332 }
6333
6334 /**
6335  * e_book_sqlite_new:
6336  * @path: location to load or create the new database
6337  * @cancellable: (allow-none): A #GCancellable
6338  * @error: (allow-none): A location to store any error that may have occurred.
6339  *
6340  * Creates a new #EBookSqlite with the default summary configuration.
6341  *
6342  * Aside from the manditory fields %E_CONTACT_UID, %E_CONTACT_REV,
6343  * the default configuration stores the following fields for quick
6344  * performance of searches: %E_CONTACT_FILE_AS, %E_CONTACT_NICKNAME,
6345  * %E_CONTACT_FULL_NAME, %E_CONTACT_GIVEN_NAME, %E_CONTACT_FAMILY_NAME,
6346  * %E_CONTACT_EMAIL, %E_CONTACT_TEL, %E_CONTACT_IS_LIST, %E_CONTACT_LIST_SHOW_ADDRESSES,
6347  * and %E_CONTACT_WANTS_HTML.
6348  *
6349  * The fields %E_CONTACT_FULL_NAME and %E_CONTACT_EMAIL are configured
6350  * to respond extra quickly with the %E_BOOK_INDEX_PREFIX index flag.
6351  *
6352  * The fields %E_CONTACT_FILE_AS, %E_CONTACT_FAMILY_NAME and
6353  * %E_CONTACT_GIVEN_NAME are configured to perform well with
6354  * the #EbSqlCursor interface, using the %E_BOOK_INDEX_SORT_KEY
6355  * index flag.
6356  *
6357  * Returns: (transfer full): A reference to a #EBookSqlite
6358  *
6359  * Since: 3.12
6360  **/
6361 EBookSqlite *
6362 e_book_sqlite_new (const gchar *path,
6363                    ESource *source,
6364                    GCancellable *cancellable,
6365                    GError **error)
6366 {
6367         g_return_val_if_fail (path && path[0], NULL);
6368
6369         return ebsql_new_default (path, source, NULL, NULL, NULL, NULL, cancellable, error);
6370 }
6371
6372 /**
6373  * e_book_sqlite_new_full:
6374  * @path: location to load or create the new database
6375  * @setup: (allow-none): an #ESourceBackendSummarySetup describing how the summary should be setup, or %NULL to use the default
6376  * @vcard_callback: (allow-none) (scope async) (closure user_data): A function to resolve vcards
6377  * @change_callback: (allow-none) (scope async) (closure user_data): A function to catch notifications of vcard changes
6378  * @user_data: (allow-none): callback user data
6379  * @user_data_destroy: (allow-none): A function to free @user_data automatically when the created #EBookSqlite is destroyed.
6380  * @cancellable: (allow-none): A #GCancellable
6381  * @error: (allow-none): A location to store any error that may have occurred.
6382  *
6383  * Opens or creates a new addressbook at @path.
6384  *
6385  * Like e_book_sqlite_new(), but allows configuration of which contact fields
6386  * will be stored for quick reference in the summary. The configuration indicated by
6387  * @setup will only be taken into account when initially creating the underlying table,
6388  * further configurations will be ignored.
6389  *
6390  * The fields %E_CONTACT_UID and %E_CONTACT_REV are not optional,
6391  * they will be stored in the summary regardless of this function's parameters.
6392  * Only #EContactFields with the type #G_TYPE_STRING, #G_TYPE_BOOLEAN or
6393  * #E_TYPE_CONTACT_ATTR_LIST are currently supported.
6394  *
6395  * If @vcard_callback is specified, then vcards will not be stored by functions
6396  * such as e_book_sqlitedb_add_contact(). Instead @vcard_callback will be invoked
6397  * at any time the created #EBookSqlite requires a vcard, either as a fallback
6398  * for querying search expressions which cannot be satisfied with the summary
6399  * fields, or when reporting results from searches.
6400  *
6401  * If any error occurs and %NULL is returned, then the passed @user_data will
6402  * be automatically freed using the @user_data_destroy function, if specified.
6403  *
6404  * It is recommended to store all contact vcards in the #EBookSqlite addressbook
6405  * if at all possible, however in some cases the vcards must be stored in some
6406  * other storage.
6407  *
6408  * Returns: (transfer full): The newly created #EBookSqlite, or %NULL if opening or creating the addressbook failed.
6409  *
6410  * Since: 3.12
6411  **/
6412 EBookSqlite *
6413 e_book_sqlite_new_full (const gchar *path,
6414                         ESource *source,
6415                         ESourceBackendSummarySetup *setup,
6416                         EbSqlVCardCallback vcard_callback,
6417                         EbSqlChangeCallback change_callback,
6418                         gpointer user_data,
6419                         GDestroyNotify user_data_destroy,
6420                         GCancellable *cancellable,
6421                         GError **error)
6422 {
6423         EBookSqlite *ebsql = NULL;
6424         EContactField *fields;
6425         EContactField *indexed_fields;
6426         EBookIndexType *index_types = NULL;
6427         gboolean had_error = FALSE;
6428         GArray *summary_fields;
6429         gint n_fields = 0, n_indexed_fields = 0, i;
6430
6431         g_return_val_if_fail (path && path[0], NULL);
6432         g_return_val_if_fail (setup == NULL || E_IS_SOURCE_BACKEND_SUMMARY_SETUP (setup), NULL);
6433
6434         if (!setup)
6435                 return ebsql_new_default (
6436                         path,
6437                         source,
6438                         vcard_callback,
6439                         change_callback,
6440                         user_data,
6441                         user_data_destroy,
6442                         cancellable, error);
6443
6444         fields = e_source_backend_summary_setup_get_summary_fields (setup, &n_fields);
6445         indexed_fields = e_source_backend_summary_setup_get_indexed_fields (setup, &index_types, &n_indexed_fields);
6446
6447         /* No specified summary fields indicates the default summary configuration should be used */
6448         if (n_fields <= 0 || n_fields >= EBSQL_MAX_SUMMARY_FIELDS) {
6449
6450                 if (n_fields)
6451                         g_warning (
6452                                 "EBookSqlite refused to create addressbook with over %d summary fields",
6453                                 EBSQL_MAX_SUMMARY_FIELDS);
6454
6455                 ebsql = ebsql_new_default (
6456                         path,
6457                         source,
6458                         vcard_callback,
6459                         change_callback,
6460                         user_data,
6461                         user_data_destroy,
6462                         cancellable, error);
6463                 g_free (fields);
6464                 g_free (index_types);
6465                 g_free (indexed_fields);
6466
6467                 return ebsql;
6468         }
6469
6470         summary_fields = g_array_new (FALSE, FALSE, sizeof (SummaryField));
6471
6472         /* Ensure the non-optional fields first */
6473         summary_field_append (summary_fields, DEFAULT_FOLDER_ID, E_CONTACT_UID, error);
6474         summary_field_append (summary_fields, DEFAULT_FOLDER_ID, E_CONTACT_REV, error);
6475
6476         for (i = 0; i < n_fields; i++) {
6477                 if (!summary_field_append (summary_fields, DEFAULT_FOLDER_ID, fields[i], error)) {
6478                         had_error = TRUE;
6479                         break;
6480                 }
6481         }
6482
6483         if (had_error) {
6484                 gint n_sfields;
6485                 SummaryField *sfields;
6486
6487                 /* Properly free the array */
6488                 n_sfields = summary_fields->len;
6489                 sfields = (SummaryField *) g_array_free (summary_fields, FALSE);
6490                 summary_fields_array_free (sfields, n_sfields);
6491
6492                 g_free (fields);
6493                 g_free (index_types);
6494                 g_free (indexed_fields);
6495
6496                 if (user_data && user_data_destroy)
6497                         user_data_destroy (user_data);
6498
6499                 return NULL;
6500         }
6501
6502         /* Add the 'indexed' flag to the SummaryField structs */
6503         summary_fields_add_indexes (
6504                 summary_fields, indexed_fields, index_types, n_indexed_fields);
6505
6506         ebsql = ebsql_new_internal (
6507                 path, source,
6508                 vcard_callback, change_callback,
6509                 user_data, user_data_destroy,
6510                 (SummaryField *) summary_fields->data,
6511                 summary_fields->len,
6512                 cancellable, error);
6513
6514         g_free (fields);
6515         g_free (index_types);
6516         g_free (indexed_fields);
6517         g_array_free (summary_fields, FALSE);
6518
6519         return ebsql;
6520 }
6521
6522 /**
6523  * e_book_sqlite_lock:
6524  * @ebsql: An #EBookSqlite
6525  * @lock_type: The #EbSqlLockType to acquire
6526  * @cancellable: (allow-none): A #GCancellable
6527  * @error: (allow-none): A location to store any error that may have occurred.
6528  *
6529  * Obtains an exclusive lock on @ebsql and starts a transaction.
6530  *
6531  * This should be called if you need to access @ebsql multiple times while
6532  * ensuring an atomic transaction. End this transaction with e_book_sqlite_unlock().
6533  *
6534  * If @cancellable is specified, then @ebsql will retain a reference to it until
6535  * e_book_sqlite_unlock() is called. Any accesses to @ebsql with the lock held
6536  * are expected to have the same @cancellable specified, or %NULL.
6537  *
6538  * <note><para>Aside from ensuring atomicity of transactions, this function will hold a mutex
6539  * which will cause further calls to e_book_sqlite_lock() to block. If you are accessing
6540  * @ebsql from multiple threads, then any interactions with @ebsql should be nested in calls
6541  * to e_book_sqlite_lock() and e_book_sqlite_unlock().</para></note>
6542  *
6543  * Returns: %TRUE on success, otherwise %FALSE is returned and @error is set appropriately.
6544  *
6545  * Since: 3.12
6546  **/
6547 gboolean
6548 e_book_sqlite_lock (EBookSqlite *ebsql,
6549                     EbSqlLockType lock_type,
6550                     GCancellable *cancellable,
6551                     GError **error)
6552 {
6553         gboolean success;
6554
6555         g_return_val_if_fail (E_IS_BOOK_SQLITE (ebsql), FALSE);
6556
6557         EBSQL_LOCK_MUTEX (&ebsql->priv->updates_lock);
6558
6559         /* Here, after obtaining the outer facing transaction lock, we need
6560          * to assert that there is no cancellable already set */
6561         if (ebsql->priv->cancel != NULL) {
6562                 /* This should never happen, if it does it's a bug
6563                  * in this code, not the calling code
6564                  */
6565                 g_warn_if_reached ();
6566                 EBSQL_UNLOCK_MUTEX (&ebsql->priv->updates_lock);
6567                 return FALSE;
6568         }
6569
6570         EBSQL_LOCK_MUTEX (&ebsql->priv->lock);
6571
6572         /* Here, after obtaining the regular lock, we need to assert that we are
6573          * the toplevel transaction */
6574         if (ebsql->priv->in_transaction != 0) {
6575                 g_warn_if_reached ();
6576                 EBSQL_LOCK_MUTEX (&ebsql->priv->lock);
6577                 EBSQL_UNLOCK_MUTEX (&ebsql->priv->updates_lock);
6578                 return FALSE;
6579         }
6580
6581         success = ebsql_start_transaction (ebsql, lock_type, cancellable, error);
6582         EBSQL_UNLOCK_MUTEX (&ebsql->priv->lock);
6583
6584         /* If we failed to start the transaction, we don't hold the lock */
6585         if (!success)
6586                 EBSQL_UNLOCK_MUTEX (&ebsql->priv->updates_lock);
6587
6588         return success;
6589 }
6590
6591 /**
6592  * e_book_sqlite_unlock:
6593  * @ebsql: An #EBookSqlite
6594  * @action: Which #EbSqlUnlockAction to take while unlocking
6595  * @error: (allow-none): A location to store any error that may have occurred.
6596  *
6597  * Releases an exclusive on @ebsql and finishes a transaction previously
6598  * started with e_book_sqlite_lock_updates().
6599  *
6600  * <note><para>If this fails, the lock on @ebsql is still released and @error will
6601  * be set to indicate why the transaction or rollback failed.</para></note>
6602  *
6603  * Returns: %TRUE on success, otherwise %FALSE is returned and @error is set appropriately.
6604  *
6605  * Since: 3.12
6606  **/
6607 gboolean
6608 e_book_sqlite_unlock (EBookSqlite *ebsql,
6609                       EbSqlUnlockAction action,
6610                       GError **error)
6611 {
6612         gboolean success = FALSE;
6613
6614         g_return_val_if_fail (E_IS_BOOK_SQLITE (ebsql), FALSE);
6615
6616         EBSQL_LOCK_MUTEX (&ebsql->priv->lock);
6617
6618         switch (action) {
6619         case EBSQL_UNLOCK_NONE:
6620         case EBSQL_UNLOCK_COMMIT:
6621                 success = ebsql_commit_transaction (ebsql, error);
6622                 break;
6623         case EBSQL_UNLOCK_ROLLBACK:
6624                 success = ebsql_rollback_transaction (ebsql, error);
6625                 break;
6626         }
6627
6628         EBSQL_UNLOCK_MUTEX (&ebsql->priv->lock);
6629
6630         EBSQL_UNLOCK_MUTEX (&ebsql->priv->updates_lock);
6631
6632         return success;
6633 }
6634
6635 /**
6636  * e_book_sqlite_ref_collator:
6637  * @ebsql: An #EBookSqlite
6638  *
6639  * References the currently active #ECollator for @ebsql,
6640  * use e_collator_unref() when finished using the returned collator.
6641  *
6642  * Note that the active collator will change with the active locale setting.
6643  *
6644  * Returns: (transfer full): A reference to the active collator.
6645  *
6646  * Since: 3.12
6647  */
6648 ECollator *
6649 e_book_sqlite_ref_collator (EBookSqlite *ebsql)
6650 {
6651         g_return_val_if_fail (E_IS_BOOK_SQLITE (ebsql), NULL);
6652
6653         return e_collator_ref (ebsql->priv->collator);
6654 }
6655
6656 /**
6657  * e_book_sqlite_ref_source:
6658  * @ebsql: An #EBookSqlite
6659  *
6660  * References the #ESource to which @ebsql is paired,
6661  * use g_object_unref() when finished using the source.
6662  * It can be %NULL in some cases, like when running tests.
6663  *
6664  * Returns: (transfer-full): A reference to the #ESource to which @ebsql
6665  * is paired, or %NULL.
6666  *
6667  * Since: 3.14
6668 */
6669 ESource *
6670 e_book_sqlite_ref_source (EBookSqlite *ebsql)
6671 {
6672         g_return_val_if_fail (E_IS_BOOK_SQLITE (ebsql), NULL);
6673
6674         if (!ebsql->priv->source)
6675                 return NULL;
6676
6677         return g_object_ref (ebsql->priv->source);
6678 }
6679
6680 /**
6681  * e_book_sqlitedb_add_contact:
6682  * @ebsql: An #EBookSqlite
6683  * @contact: EContact to be added
6684  * @extra: Extra data to store in association with this contact
6685  * @replace: Whether this contact should replace another contact with the same UID.
6686  * @cancellable: (allow-none): A #GCancellable
6687  * @error: (allow-none): A location to store any error that may have occurred.
6688  *
6689  * This is a convenience wrapper for e_book_sqlite_add_contacts(),
6690  * which is the preferred means to add or modify multiple contacts when possible.
6691  *
6692  * Returns: %TRUE on success, otherwise %FALSE is returned and @error is set appropriately.
6693  *
6694  * Since: 3.12
6695  **/
6696 gboolean
6697 e_book_sqlite_add_contact (EBookSqlite *ebsql,
6698                            EContact *contact,
6699                            const gchar *extra,
6700                            gboolean replace,
6701                            GCancellable *cancellable,
6702                            GError **error)
6703 {
6704         GSList l;
6705         GSList el;
6706
6707         g_return_val_if_fail (E_IS_BOOK_SQLITE (ebsql), FALSE);
6708         g_return_val_if_fail (E_IS_CONTACT (contact), FALSE);
6709
6710         l.data = contact;
6711         l.next = NULL;
6712
6713         el.data = (gpointer) extra;
6714         el.next = NULL;
6715
6716         return e_book_sqlite_add_contacts (ebsql, &l, &el, replace, cancellable, error);
6717 }
6718
6719 /**
6720  * e_book_sqlite_new_contacts:
6721  * @ebsql: An #EBookSqlite
6722  * @contacts: (element-type EContact): A list of contacts to add to @ebsql
6723  * @extra: (allow-none) (element-type utf8): A list of extra data to store in association with this contact
6724  * @replace: Whether this contact should replace another contact with the same UID.
6725  * @cancellable: (allow-none): A #GCancellable
6726  * @error: (allow-none): A location to store any error that may have occurred.
6727  *
6728  * Adds or replaces contacts in @ebsql. If @replace_existing is specified then existing
6729  * contacts with the same UID will be replaced, otherwise adding an existing contact
6730  * will return an error.
6731  *
6732  * If @extra is specified, it must have an equal length as the @contacts list. Each element
6733  * from the @extra list will be stored in association with it's corresponding contact
6734  * in the @contacts list.
6735  *
6736  * Returns: %TRUE on success, otherwise %FALSE is returned and @error is set appropriately.
6737  *
6738  * Since: 3.12
6739  **/
6740 gboolean
6741 e_book_sqlite_add_contacts (EBookSqlite *ebsql,
6742                             GSList *contacts,
6743                             GSList *extra,
6744                             gboolean replace,
6745                             GCancellable *cancellable,
6746                             GError **error)
6747 {
6748         GSList *l, *ll;
6749         gboolean success = TRUE;
6750
6751         g_return_val_if_fail (E_IS_BOOK_SQLITE (ebsql), FALSE);
6752         g_return_val_if_fail (contacts != NULL, FALSE);
6753         g_return_val_if_fail (extra == NULL ||
6754                               g_slist_length (extra) == g_slist_length (contacts), FALSE);
6755
6756         EBSQL_LOCK_OR_RETURN (ebsql, cancellable, FALSE);
6757
6758         if (!ebsql_start_transaction (ebsql, EBSQL_LOCK_WRITE, cancellable, error)) {
6759                 EBSQL_UNLOCK_MUTEX (&ebsql->priv->lock);
6760                 return FALSE;
6761         }
6762
6763         for (l = contacts, ll = extra;
6764              success && l != NULL;
6765              l = l->next, ll = ll ? ll->next : NULL) {
6766                 EContact *contact = (EContact *) l->data;
6767                 const gchar *extra_data = NULL;
6768
6769                 if (ll)
6770                         extra_data = (const gchar *) ll->data;
6771
6772                 g_signal_emit (ebsql,
6773                                signals[BEFORE_INSERT_CONTACT],
6774                                0,
6775                                ebsql->priv->db,
6776                                contact, extra_data,
6777                                replace,
6778                                cancellable, error,
6779                                &success);
6780                 if (!success)
6781                         break;
6782
6783                 success = ebsql_insert_contact (
6784                         ebsql,
6785                         EBSQL_CHANGE_CONTACT_ADDED,
6786                         contact, NULL, extra_data,
6787                         replace, error);
6788         }
6789
6790         if (success)
6791                 success = ebsql_commit_transaction (ebsql, error);
6792         else
6793                 /* The GError is already set. */
6794                 ebsql_rollback_transaction (ebsql, NULL);
6795
6796         EBSQL_UNLOCK_MUTEX (&ebsql->priv->lock);
6797
6798         return success;
6799 }
6800
6801 /**
6802  * e_book_sqlite_remove_contact:
6803  * @ebsql: An #EBookSqlite
6804  * @uid: the uid of the contact to remove
6805  * @cancellable: (allow-none): A #GCancellable
6806  * @error: (allow-none): A location to store any error that may have occurred.
6807  *
6808  * Removes the contact indicated by @uid from @ebsql.
6809  *
6810  * Returns: %TRUE on success, otherwise %FALSE is returned and @error is set appropriately.
6811  *
6812  * Since: 3.12
6813  **/
6814 gboolean
6815 e_book_sqlite_remove_contact (EBookSqlite *ebsql,
6816                               const gchar *uid,
6817                               GCancellable *cancellable,
6818                               GError **error)
6819 {
6820         GSList l;
6821
6822         g_return_val_if_fail (E_IS_BOOK_SQLITE (ebsql), FALSE);
6823         g_return_val_if_fail (uid != NULL, FALSE);
6824
6825         l.data = (gchar *) uid; /* Won't modify it, I promise :) */
6826         l.next = NULL;
6827
6828         return e_book_sqlite_remove_contacts (
6829                 ebsql, &l, cancellable, error);
6830 }
6831
6832 static gchar *
6833 generate_delete_stmt (const gchar *table,
6834                       GSList *uids)
6835 {
6836         GString *str = g_string_new (NULL);
6837         GSList  *l;
6838
6839         ebsql_string_append_printf (str, "DELETE FROM %Q WHERE uid IN (", table);
6840
6841         for (l = uids; l; l = l->next) {
6842                 const gchar *uid = (const gchar *) l->data;
6843
6844                 /* First uid with no comma */
6845                 if (l != uids)
6846                         g_string_append_printf (str, ", ");
6847
6848                 ebsql_string_append_printf (str, "%Q", uid);
6849         }
6850
6851         g_string_append_c (str, ')');
6852
6853         return g_string_free (str, FALSE);
6854 }
6855
6856 /**
6857  * e_book_sqlite_remove_contacts:
6858  * @ebsql: An #EBookSqlite
6859  * @uids: a #GSList of uids indicating which contacts to remove
6860  * @cancellable: (allow-none): A #GCancellable
6861  * @error: (allow-none): A location to store any error that may have occurred.
6862  *
6863  * Removes the contacts indicated by @uids from @ebsql.
6864  *
6865  * Returns: %TRUE on success, otherwise %FALSE is returned and @error is set appropriately.
6866  *
6867  * Since: 3.12
6868  **/
6869 gboolean
6870 e_book_sqlite_remove_contacts (EBookSqlite *ebsql,
6871                                GSList *uids,
6872                                GCancellable *cancellable,
6873                                GError **error)
6874 {
6875         gboolean success = TRUE;
6876         gint i;
6877         gchar *stmt;
6878         const gchar *contact_uid;
6879         GSList *l = NULL;
6880
6881         g_return_val_if_fail (E_IS_BOOK_SQLITE (ebsql), FALSE);
6882         g_return_val_if_fail (uids != NULL, FALSE);
6883
6884         EBSQL_LOCK_OR_RETURN (ebsql, cancellable, FALSE);
6885
6886         if (!ebsql_start_transaction (ebsql, EBSQL_LOCK_WRITE, cancellable, error)) {
6887                 EBSQL_UNLOCK_MUTEX (&ebsql->priv->lock);
6888                 return FALSE;
6889         }
6890
6891         for (l = uids; success && l; l = l->next) {
6892                 contact_uid = (const gchar *) l->data;
6893                 g_signal_emit (ebsql,
6894                                signals[BEFORE_REMOVE_CONTACT],
6895                                0,
6896                                ebsql->priv->db,
6897                                contact_uid,
6898                                cancellable, error,
6899                                &success);
6900         }
6901
6902         /* Delete data from the auxiliary tables first */
6903         for (i = 0; success && i < ebsql->priv->n_summary_fields; i++) {
6904                 SummaryField *field = &(ebsql->priv->summary_fields[i]);
6905
6906                 if (field->type != E_TYPE_CONTACT_ATTR_LIST)
6907                         continue;
6908
6909                 stmt = generate_delete_stmt (field->aux_table, uids);
6910                 success = ebsql_exec (ebsql, stmt, NULL, NULL, NULL, error);
6911                 g_free (stmt);
6912         }
6913
6914         /* Now delete the entry from the main contacts */
6915         if (success) {
6916                 stmt = generate_delete_stmt (ebsql->priv->folderid, uids);
6917                 success = ebsql_exec (ebsql, stmt, NULL, NULL, NULL, error);
6918                 g_free (stmt);
6919         }
6920
6921         if (success)
6922                 success = ebsql_commit_transaction (ebsql, error);
6923         else
6924                 /* The GError is already set. */
6925                 ebsql_rollback_transaction (ebsql, NULL);
6926
6927         EBSQL_UNLOCK_MUTEX (&ebsql->priv->lock);
6928
6929         return success;
6930 }
6931
6932 /**
6933  * e_book_sqlite_has_contact:
6934  * @ebsql: An #EBookSqlite
6935  * @uid: The uid of the contact to check for
6936  * @exists: (out): Return location to store whether the contact exists.
6937  * @error: (allow-none): A location to store any error that may have occurred.
6938  *
6939  * Checks if a contact bearing the UID indicated by @uid is stored in @ebsql.
6940  *
6941  * Returns: %TRUE on success, otherwise %FALSE is returned and @error is set appropriately.
6942  *
6943  * Since: 3.12
6944  **/
6945 gboolean
6946 e_book_sqlite_has_contact (EBookSqlite *ebsql,
6947                            const gchar *uid,
6948                            gboolean *exists,
6949                            GError **error)
6950 {
6951         gboolean local_exists = FALSE;
6952         gboolean success;
6953
6954         g_return_val_if_fail (E_IS_BOOK_SQLITE (ebsql), FALSE);
6955         g_return_val_if_fail (uid != NULL, FALSE);
6956         g_return_val_if_fail (exists != NULL, FALSE);
6957
6958         EBSQL_LOCK_MUTEX (&ebsql->priv->lock);
6959         success = ebsql_exec_printf (
6960                 ebsql,
6961                 "SELECT uid FROM %Q WHERE uid = %Q",
6962                 get_exists_cb, &local_exists, NULL, error,
6963                 ebsql->priv->folderid, uid);
6964         EBSQL_UNLOCK_MUTEX (&ebsql->priv->lock);
6965
6966         *exists = local_exists;
6967
6968         return success;
6969 }
6970
6971 /**
6972  * e_book_sqlite_get_contact:
6973  * @ebsql: An #EBookSqlite
6974  * @uid: The uid of the contact to fetch
6975  * @meta_contact: Whether an entire contact is desired, or only the metadata
6976  * @ret_contact: (out) (transfer full): Return location to store the fetched contact
6977  * @error: (allow-none): A location to store any error that may have occurred.
6978  *
6979  * Fetch the #EContact specified by @uid in @ebsql.
6980  *
6981  * If @meta_contact is specified, then a shallow #EContact will be created
6982  * holding only the %E_CONTACT_UID and %E_CONTACT_REV fields.
6983  *
6984  * Returns: %TRUE on success, otherwise %FALSE is returned and @error is set appropriately.
6985  *
6986  * Since: 3.12
6987  **/
6988 gboolean
6989 e_book_sqlite_get_contact (EBookSqlite *ebsql,
6990                            const gchar *uid,
6991                            gboolean meta_contact,
6992                            EContact **ret_contact,
6993                            GError **error)
6994 {
6995         gboolean success = FALSE;
6996         gchar *vcard = NULL;
6997
6998         g_return_val_if_fail (E_IS_BOOK_SQLITE (ebsql), FALSE);
6999         g_return_val_if_fail (uid != NULL, FALSE);
7000         g_return_val_if_fail (ret_contact != NULL && *ret_contact == NULL, FALSE);
7001
7002         success = e_book_sqlite_get_vcard (
7003                 ebsql, uid, meta_contact, &vcard, error);
7004
7005         if (success && vcard) {
7006                 *ret_contact = e_contact_new_from_vcard_with_uid (vcard, uid);
7007                 g_free (vcard);
7008         }
7009
7010         return success;
7011 }
7012
7013 /**
7014  * ebsql_get_contact_unlocked:
7015  * @ebsql: An #EBookSqlite
7016  * @uid: The uid of the contact to fetch
7017  * @meta_contact: Whether an entire contact is desired, or only the metadata
7018  * @contact: (out) (transfer full): Return location to store the fetched contact
7019  * @error: (allow-none): A location to store any error that may have occurred.
7020  *
7021  * Fetch the #EContact specified by @uid in @ebsql without locking internal mutex.
7022  *
7023  * If @meta_contact is specified, then a shallow #EContact will be created
7024  * holding only the %E_CONTACT_UID and %E_CONTACT_REV fields.
7025  *
7026  * Returns: %TRUE on success, otherwise %FALSE is returned and @error is set appropriately.
7027  *
7028  * Since: 3.14
7029  **/
7030 gboolean
7031 ebsql_get_contact_unlocked (EBookSqlite *ebsql,
7032                             const gchar *uid,
7033                             gboolean meta_contact,
7034                             EContact **contact,
7035                             GError **error)
7036 {
7037         gboolean success = FALSE;
7038         gchar *vcard = NULL;
7039
7040         g_return_val_if_fail (E_IS_BOOK_SQLITE (ebsql), FALSE);
7041         g_return_val_if_fail (uid != NULL, FALSE);
7042         g_return_val_if_fail (contact != NULL && *contact == NULL, FALSE);
7043
7044         success = ebsql_get_vcard_unlocked (ebsql,
7045                                             uid,
7046                                             meta_contact,
7047                                             &vcard,
7048                                             error);
7049
7050         if (success && vcard) {
7051                 *contact = e_contact_new_from_vcard_with_uid (vcard, uid);
7052                 g_free (vcard);
7053         }
7054
7055         return success;
7056 }
7057
7058 /**
7059  * e_book_sqlite_get_vcard:
7060  * @ebsql: An #EBookSqlite
7061  * @uid: The uid of the contact to fetch
7062  * @meta_contact: Whether an entire contact is desired, or only the metadata
7063  * @ret_vcard: (out) (transfer full): Return location to store the fetched vcard string
7064  * @error: (allow-none): A location to store any error that may have occurred.
7065  *
7066  * Fetch a vcard string for @uid in @ebsql.
7067  *
7068  * If @meta_contact is specified, then a shallow vcard representation will be
7069  * created holding only the %E_CONTACT_UID and %E_CONTACT_REV fields.
7070  *
7071  * Returns: %TRUE on success, otherwise %FALSE is returned and @error is set appropriately.
7072  *
7073  * Since: 3.12
7074  **/
7075 gboolean
7076 e_book_sqlite_get_vcard (EBookSqlite *ebsql,
7077                          const gchar *uid,
7078                          gboolean meta_contact,
7079                          gchar **ret_vcard,
7080                          GError **error)
7081 {
7082         gboolean success = FALSE;
7083         gchar *vcard = NULL;
7084
7085         g_return_val_if_fail (E_IS_BOOK_SQLITE (ebsql), FALSE);
7086         g_return_val_if_fail (uid != NULL, FALSE);
7087         g_return_val_if_fail (ret_vcard != NULL && *ret_vcard == NULL, FALSE);
7088
7089         EBSQL_LOCK_MUTEX (&ebsql->priv->lock);
7090
7091         /* Try constructing contacts from only UID/REV first if that's requested */
7092         if (meta_contact) {
7093                 GSList *vcards = NULL;
7094
7095                 success = ebsql_exec_printf (
7096                         ebsql, "SELECT summary.uid, summary.Rev FROM %Q AS summary WHERE uid = %Q",
7097                         collect_lean_results_cb, &vcards, NULL, error,
7098                         ebsql->priv->folderid, uid);
7099
7100                 if (vcards) {
7101                         EbSqlSearchData *search_data = (EbSqlSearchData *) vcards->data;
7102
7103                         vcard = search_data->vcard;
7104                         search_data->vcard = NULL;
7105
7106                         g_slist_free_full (vcards, (GDestroyNotify) e_book_sqlite_search_data_free);
7107                         vcards = NULL;
7108                 }
7109
7110         } else {
7111                 success = ebsql_exec_printf (
7112                         ebsql, "SELECT %s FROM %Q AS summary WHERE summary.uid = %Q",
7113                         get_string_cb, &vcard, NULL, error,
7114                         EBSQL_VCARD_FRAGMENT (ebsql), ebsql->priv->folderid, uid);
7115         }
7116
7117         EBSQL_UNLOCK_MUTEX (&ebsql->priv->lock);
7118
7119         *ret_vcard = vcard;
7120
7121         if (success && !vcard) {
7122                 EBSQL_SET_ERROR (
7123                         error,
7124                         E_BOOK_SQLITE_ERROR_CONTACT_NOT_FOUND,
7125                         _("Contact '%s' not found"), uid);
7126                 success = FALSE;
7127         }
7128
7129         return success;
7130 }
7131
7132 /**
7133  * ebsql_get_vcard_unlocked:
7134  * @ebsql: An #EBookSqlite
7135  * @uid: The uid of the contact to fetch
7136  * @meta_contact: Whether an entire contact is desired, or only the metadata
7137  * @ret_vcard: (out) (transfer full): Return location to store the fetched vcard string
7138  * @error: (allow-none): A location to store any error that may have occurred.
7139  *
7140  * Fetch a vcard string for @uid in @ebsql without locking internal mutex.
7141  *
7142  * If @meta_contact is specified, then a shallow vcard representation will be
7143  * created holding only the %E_CONTACT_UID and %E_CONTACT_REV fields.
7144  *
7145  * Returns: %TRUE on success, otherwise %FALSE is returned and @error is set appropriately.
7146  *
7147  * Since: 3.14
7148  **/
7149 gboolean
7150 ebsql_get_vcard_unlocked (EBookSqlite *ebsql,
7151                          const gchar *uid,
7152                          gboolean meta_contact,
7153                          gchar **ret_vcard,
7154                          GError **error)
7155 {
7156         gboolean success = FALSE;
7157         gchar *vcard = NULL;
7158
7159         g_return_val_if_fail (E_IS_BOOK_SQLITE (ebsql), FALSE);
7160         g_return_val_if_fail (uid != NULL, FALSE);
7161         g_return_val_if_fail (ret_vcard != NULL && *ret_vcard == NULL, FALSE);
7162
7163         /* Try constructing contacts from only UID/REV first if that's requested */
7164         if (meta_contact) {
7165                 GSList *vcards = NULL;
7166
7167                 success = ebsql_exec_printf (
7168                         ebsql, "SELECT summary.uid, summary.Rev FROM %Q AS summary WHERE uid = %Q",
7169                         collect_lean_results_cb, &vcards, NULL, error,
7170                         ebsql->priv->folderid, uid);
7171
7172                 if (vcards) {
7173                         EbSqlSearchData *search_data = (EbSqlSearchData *) vcards->data;
7174
7175                         vcard = search_data->vcard;
7176                         search_data->vcard = NULL;
7177
7178                         g_slist_free_full (vcards, (GDestroyNotify) e_book_sqlite_search_data_free);
7179                         vcards = NULL;
7180                 }
7181
7182        } else {
7183                success = ebsql_exec_printf (
7184                        ebsql, "SELECT %s FROM %Q AS summary WHERE summary.uid = %Q",
7185                        get_string_cb, &vcard, NULL, error,
7186                        EBSQL_VCARD_FRAGMENT (ebsql), ebsql->priv->folderid, uid);
7187        }
7188
7189         *ret_vcard = vcard;
7190
7191         if (success && !vcard) {
7192                 EBSQL_SET_ERROR (error,
7193                                  E_BOOK_SQLITE_ERROR_CONTACT_NOT_FOUND,
7194                                  _("Contact '%s' not found"), uid);
7195                 success = FALSE;
7196         }
7197
7198         return success;
7199 }
7200
7201 /**
7202  * e_book_sqlite_set_contact_extra:
7203  * @ebsql: An #EBookSqlite
7204  * @uid: The uid of the contact to set the extra data for
7205  * @extra: (allow-none): The extra data to set
7206  * @error: (allow-none): A location to store any error that may have occurred.
7207  *
7208  * Sets or replaces the extra data associated with @uid.
7209  *
7210  * Returns: %TRUE on success, otherwise %FALSE is returned and @error is set appropriately.
7211  *
7212  * Since: 3.12
7213  **/
7214 gboolean
7215 e_book_sqlite_set_contact_extra (EBookSqlite *ebsql,
7216                                  const gchar *uid,
7217                                  const gchar *extra,
7218                                  GError **error)
7219 {
7220         gboolean success;
7221
7222         g_return_val_if_fail (E_IS_BOOK_SQLITE (ebsql), FALSE);
7223         g_return_val_if_fail (uid != NULL, FALSE);
7224
7225         EBSQL_LOCK_MUTEX (&ebsql->priv->lock);
7226         success = ebsql_exec_printf (
7227                 ebsql, "UPDATE %Q SET bdata = %Q WHERE uid = %Q",
7228                 NULL, NULL, NULL, error,
7229                 ebsql->priv->folderid, uid);
7230         EBSQL_UNLOCK_MUTEX (&ebsql->priv->lock);
7231
7232         return success;
7233 }
7234
7235 /**
7236  * e_book_sqlite_get_contact_extra:
7237  * @ebsql: An #EBookSqlite
7238  * @uid: The uid of the contact to fetch the extra data for
7239  * @ret_extra: (out) (transfer full): Return location to store the extra data
7240  * @error: (allow-none): A location to store any error that may have occurred.
7241  *
7242  * Fetches the extra data previously set for @uid, either with
7243  * e_book_sqlite_set_contact_extra() or when adding contacts.
7244  *
7245  * Returns: %TRUE on success, otherwise %FALSE is returned and @error is set appropriately.
7246  *
7247  * Since: 3.12
7248  **/
7249 gboolean
7250 e_book_sqlite_get_contact_extra (EBookSqlite *ebsql,
7251                                  const gchar *uid,
7252                                  gchar **ret_extra,
7253                                  GError **error)
7254 {
7255         gboolean success;
7256
7257         g_return_val_if_fail (E_IS_BOOK_SQLITE (ebsql), FALSE);
7258         g_return_val_if_fail (uid != NULL, FALSE);
7259         g_return_val_if_fail (ret_extra != NULL && *ret_extra == NULL, FALSE);
7260
7261         EBSQL_LOCK_MUTEX (&ebsql->priv->lock);
7262         success = ebsql_exec_printf (
7263                 ebsql, "SELECT bdata FROM %Q WHERE uid = %Q",
7264                 get_string_cb, ret_extra, NULL, error,
7265                 ebsql->priv->folderid, uid);
7266         EBSQL_UNLOCK_MUTEX (&ebsql->priv->lock);
7267
7268         return success;
7269 }
7270
7271 /**
7272  * ebsql_get_contact_extra_unlocked:
7273  * @ebsql: An #EBookSqlite
7274  * @uid: The uid of the contact to fetch the extra data for
7275  * @ret_extra: (out) (transfer full): Return location to store the extra data
7276  * @error: (allow-none): A location to store any error that may have occurred.
7277  *
7278  * Fetches the extra data previously set for @uid, either with
7279  * e_book_sqlite_set_contact_extra() or when adding contacts,
7280  * without locking internal mutex.
7281  *
7282  * Returns: %TRUE on success, otherwise %FALSE is returned and @error is set appropriately.
7283  *
7284  * Since: 3.14
7285  **/
7286 gboolean
7287 ebsql_get_contact_extra_unlocked (EBookSqlite *ebsql,
7288                                   const gchar *uid,
7289                                   gchar **ret_extra,
7290                                   GError **error)
7291 {
7292         gboolean success;
7293
7294         g_return_val_if_fail (E_IS_BOOK_SQLITE (ebsql), FALSE);
7295         g_return_val_if_fail (uid != NULL, FALSE);
7296         g_return_val_if_fail (ret_extra != NULL && *ret_extra == NULL, FALSE);
7297
7298         success = ebsql_exec_printf (
7299                 ebsql, "SELECT bdata FROM %Q WHERE uid = %Q",
7300                 get_string_cb, ret_extra, NULL, error,
7301                 ebsql->priv->folderid, uid);
7302
7303         return success;
7304 }
7305
7306 /**
7307  * e_book_sqlite_search:
7308  * @ebsql: An #EBookSqlite
7309  * @sexp: (allow-none): search expression; use %NULL or an empty string to list all stored contacts.
7310  * @meta_contacts: Whether entire contacts are desired, or only the metadata
7311  * @ret_list: (out) (transfer full) (element-type EbSqlSearchData): Return location
7312  * to store a #GSList of #EbSqlSearchData structures
7313  * @cancellable: (allow-none): A #GCancellable
7314  * @error: (allow-none): A location to store any error that may have occurred.
7315  *
7316  * Searches @ebsql for contacts matching the search expression indicated by @sexp.
7317  *
7318  * When @sexp refers only to #EContactFields configured in the summary of @ebsql,
7319  * the search should always be quick, when searching for other #EContactFields
7320  * a fallback will be used, possibly invoking any #EbSqlVCardCallback which
7321  * may have been passed to e_book_sqlite_new_full().
7322  *
7323  * The returned @ret_list list should be freed with g_slist_free()
7324  * and all elements freed with e_book_sqlite_search_data_free().
7325  *
7326  * If @meta_contact is specified, then shallow vcard representations will be
7327  * created holding only the %E_CONTACT_UID and %E_CONTACT_REV fields.
7328  *
7329  * Returns: %TRUE on success, otherwise %FALSE is returned and @error is set appropriately.
7330  *
7331  * Since: 3.12
7332  **/
7333 gboolean
7334 e_book_sqlite_search (EBookSqlite *ebsql,
7335                       const gchar *sexp,
7336                       gboolean meta_contacts,
7337                       GSList **ret_list,
7338                       GCancellable *cancellable,
7339                       GError **error)
7340 {
7341         gboolean success;
7342
7343         g_return_val_if_fail (E_IS_BOOK_SQLITE (ebsql), FALSE);
7344         g_return_val_if_fail (ret_list != NULL && *ret_list == NULL, FALSE);
7345
7346         EBSQL_LOCK_OR_RETURN (ebsql, cancellable, FALSE);
7347         success = ebsql_search_query (
7348                 ebsql, sexp,
7349                 meta_contacts ?
7350                 SEARCH_UID_AND_REV : SEARCH_FULL,
7351                 ret_list,
7352                 cancellable,
7353                 error);
7354         EBSQL_UNLOCK_MUTEX (&ebsql->priv->lock);
7355
7356         return success;
7357 }
7358
7359 /**
7360  * e_book_sqlite_search_uids:
7361  * @ebsql: An #EBookSqlite
7362  * @sexp: (allow-none): search expression; use %NULL or an empty string to get all stored contacts.
7363  * @ret_list: (out) (transfer full): Return location to store a #GSList of contact uids
7364  * @cancellable: (allow-none): A #GCancellable
7365  * @error: (allow-none): A location to store any error that may have occurred.
7366  *
7367  * Similar to e_book_sqlitedb_search(), but fetches only a list of contact UIDs.
7368  *
7369  * The returned @ret_list list should be freed with g_slist_free() and all
7370  * elements freed with g_free().
7371  *
7372  * Returns: %TRUE on success, otherwise %FALSE is returned and @error is set appropriately.
7373  *
7374  * Since: 3.12
7375  **/
7376 gboolean
7377 e_book_sqlite_search_uids (EBookSqlite *ebsql,
7378                            const gchar *sexp,
7379                            GSList **ret_list,
7380                            GCancellable *cancellable,
7381                            GError **error)
7382 {
7383         gboolean success;
7384
7385         g_return_val_if_fail (E_IS_BOOK_SQLITE (ebsql), FALSE);
7386         g_return_val_if_fail (ret_list != NULL && *ret_list == NULL, FALSE);
7387
7388         EBSQL_LOCK_OR_RETURN (ebsql, cancellable, FALSE);
7389         success = ebsql_search_query (ebsql, sexp, SEARCH_UID, ret_list, cancellable, error);
7390         EBSQL_UNLOCK_MUTEX (&ebsql->priv->lock);
7391
7392         return success;
7393 }
7394
7395 /**
7396  * e_book_sqlite_get_key_value:
7397  * @ebsql: An #EBookSqlite
7398  * @key: The key to fetch a value for
7399  * @value: (out) (transfer full): A return location to store the value for @key
7400  * @error: (allow-none): A location to store any error that may have occurred.
7401  *
7402  * Fetches the value for @key and stores it in @value
7403  *
7404  * Returns: %TRUE on success, otherwise %FALSE is returned and @error is set appropriately.
7405  *
7406  * Since: 3.12
7407  **/
7408 gboolean
7409 e_book_sqlite_get_key_value (EBookSqlite *ebsql,
7410                              const gchar *key,
7411                              gchar **value,
7412                              GError **error)
7413 {
7414         gboolean success;
7415
7416         g_return_val_if_fail (E_IS_BOOK_SQLITE (ebsql), FALSE);
7417         g_return_val_if_fail (key != NULL, FALSE);
7418         g_return_val_if_fail (value != NULL && *value == NULL, FALSE);
7419
7420         EBSQL_LOCK_MUTEX (&ebsql->priv->lock);
7421         success = ebsql_exec_printf (
7422                 ebsql,
7423                 "SELECT value FROM keys WHERE folder_id = %Q AND key = %Q",
7424                 get_string_cb, value, NULL, error,
7425                 ebsql->priv->folderid, key);
7426         EBSQL_UNLOCK_MUTEX (&ebsql->priv->lock);
7427
7428         return success;
7429 }
7430
7431 /**
7432  * e_book_sqlite_set_key_value:
7433  * @ebsql: An #EBookSqlite
7434  * @key: The key to fetch a value for
7435  * @value: The new value for @key
7436  * @error: (allow-none): A location to store any error that may have occurred.
7437  *
7438  * Sets the value for @key to be @value
7439  *
7440  * Returns: %TRUE on success, otherwise %FALSE is returned and @error is set appropriately.
7441  *
7442  * Since: 3.12
7443  **/
7444 gboolean
7445 e_book_sqlite_set_key_value (EBookSqlite *ebsql,
7446                              const gchar *key,
7447                              const gchar *value,
7448                              GError **error)
7449 {
7450         gboolean success;
7451
7452         g_return_val_if_fail (E_IS_BOOK_SQLITE (ebsql), FALSE);
7453         g_return_val_if_fail (key != NULL, FALSE);
7454         g_return_val_if_fail (value != NULL, FALSE);
7455
7456         EBSQL_LOCK_MUTEX (&ebsql->priv->lock);
7457         success = ebsql_exec_printf (
7458                 ebsql, "INSERT or REPLACE INTO keys (key, value, folder_id) values (%Q, %Q, %Q)",
7459                 NULL, NULL, NULL, error,
7460                 key, value, ebsql->priv->folderid);
7461         EBSQL_UNLOCK_MUTEX (&ebsql->priv->lock);
7462
7463         return success;
7464 }
7465
7466 /**
7467  * e_book_sqlite_get_key_value_int:
7468  * @ebsql: An #EBookSqlite
7469  * @key: The key to fetch a value for
7470  * @value: (out): A return location to store the value for @key
7471  * @error: (allow-none): A location to store any error that may have occurred.
7472  *
7473  * A convenience function to fetch the value of @key as an integer.
7474  *
7475  * Returns: %TRUE on success, otherwise %FALSE is returned and @error is set appropriately.
7476  *
7477  * Since: 3.12
7478  **/
7479 gboolean
7480 e_book_sqlite_get_key_value_int (EBookSqlite *ebsql,
7481                                  const gchar *key,
7482                                  gint *value,
7483                                  GError **error)
7484 {
7485         gboolean success;
7486         gchar *str_value = NULL;
7487
7488         g_return_val_if_fail (E_IS_BOOK_SQLITE (ebsql), FALSE);
7489         g_return_val_if_fail (key != NULL, FALSE);
7490         g_return_val_if_fail (value != NULL, FALSE);
7491
7492         success = e_book_sqlite_get_key_value (ebsql, key, &str_value, error);
7493
7494         if (success) {
7495
7496                 if (str_value)
7497                         *value = g_ascii_strtoll (str_value, NULL, 10);
7498                 else
7499                         *value = 0;
7500
7501                 g_free (str_value);
7502         }
7503
7504         return success;
7505 }
7506
7507 /**
7508  * e_book_sqlite_set_key_value_int:
7509  * @ebsql: An #EBookSqlite
7510  * @key: The key to fetch a value for
7511  * @value: The new value for @key
7512  * @error: (allow-none): A location to store any error that may have occurred.
7513  *
7514  * A convenience function to set the value of @key as an integer.
7515  *
7516  * Returns: %TRUE on success, otherwise %FALSE is returned and @error is set appropriately.
7517  *
7518  * Since: 3.12
7519  **/
7520 gboolean
7521 e_book_sqlite_set_key_value_int (EBookSqlite *ebsql,
7522                                  const gchar *key,
7523                                  gint value,
7524                                  GError **error)
7525 {
7526         gboolean success;
7527         gchar *str_value = NULL;
7528
7529         g_return_val_if_fail (E_IS_BOOK_SQLITE (ebsql), FALSE);
7530         g_return_val_if_fail (key != NULL, FALSE);
7531
7532         str_value = g_strdup_printf ("%d", value);
7533         success = e_book_sqlite_set_key_value (
7534                 ebsql, key, str_value, error);
7535         g_free (str_value);
7536
7537         return success;
7538 }
7539
7540 /**
7541  * e_book_sqlite_search_data_free:
7542  * @data: An #EbSqlSearchData
7543  *
7544  * Frees an #EbSqlSearchData
7545  *
7546  * Since: 3.12
7547  **/
7548 void
7549 e_book_sqlite_search_data_free (EbSqlSearchData *data)
7550 {
7551         if (data) {
7552                 g_free (data->uid);
7553                 g_free (data->vcard);
7554                 g_free (data->extra);
7555                 g_slice_free (EbSqlSearchData, data);
7556         }
7557 }
7558
7559 /**
7560  * e_book_sqlite_set_locale:
7561  * @ebsql: An #EBookSqlite
7562  * @lc_collate: The new locale for the addressbook
7563  * @cancellable: (allow-none): A #GCancellable
7564  * @error: A location to store any error that may have occurred
7565  *
7566  * Relocalizes any locale specific data in the specified
7567  * new @lc_collate locale.
7568  *
7569  * The @lc_collate locale setting is stored and remembered on
7570  * subsequent accesses of the addressbook, changing the locale
7571  * will store the new locale and will modify sort keys and any
7572  * locale specific data in the addressbook.
7573  *
7574  * As a side effect, it's possible that changing the locale
7575  * will cause stored vcards to change. Notifications for
7576  * these changes can be caught with the #EbSqlVCardCallback
7577  * provided to e_book_sqlite_new_full().
7578  *
7579  * Returns: Whether the new locale was successfully set.
7580  *
7581  * Since: 3.12
7582  */
7583 gboolean
7584 e_book_sqlite_set_locale (EBookSqlite *ebsql,
7585                           const gchar *lc_collate,
7586                           GCancellable *cancellable,
7587                           GError **error)
7588 {
7589         gboolean success;
7590         gchar *stored_lc_collate = NULL;
7591
7592         g_return_val_if_fail (E_IS_BOOK_SQLITE (ebsql), FALSE);
7593
7594         EBSQL_LOCK_OR_RETURN (ebsql, cancellable, FALSE);
7595
7596         if (!ebsql_start_transaction (ebsql, EBSQL_LOCK_WRITE, cancellable, error)) {
7597                 EBSQL_UNLOCK_MUTEX (&ebsql->priv->lock);
7598                 return FALSE;
7599         }
7600
7601         success = ebsql_set_locale_internal (ebsql, lc_collate, error);
7602
7603         if (success)
7604                 success = ebsql_exec_printf (
7605                         ebsql, "SELECT lc_collate FROM folders WHERE folder_id = %Q",
7606                         get_string_cb, &stored_lc_collate, NULL, error,
7607                         ebsql->priv->folderid);
7608
7609         if (success && g_strcmp0 (stored_lc_collate, lc_collate) != 0)
7610                 success = ebsql_upgrade (ebsql, EBSQL_CHANGE_LOCALE_CHANGED, error);
7611
7612         /* If for some reason we failed, then reset the collator to use the old locale */
7613         if (!success && stored_lc_collate && stored_lc_collate[0])
7614                 ebsql_set_locale_internal (ebsql, stored_lc_collate, NULL);
7615
7616         if (success)
7617                 success = ebsql_commit_transaction (ebsql, error);
7618         else
7619                 /* The GError is already set. */
7620                 ebsql_rollback_transaction (ebsql, NULL);
7621
7622         EBSQL_UNLOCK_MUTEX (&ebsql->priv->lock);
7623
7624         g_free (stored_lc_collate);
7625
7626         return success;
7627 }
7628
7629 /**
7630  * e_book_sqlite_get_locale:
7631  * @ebsql: An #EBookSqlite
7632  * @locale_out: (out) (transfer full): The location to return the current locale
7633  * @error: A location to store any error that may have occurred
7634  *
7635  * Fetches the current locale setting for the address-book.
7636  *
7637  * Upon success, @lc_collate_out will hold the returned locale setting,
7638  * otherwise %FALSE will be returned and @error will be updated accordingly.
7639  *
7640  * Returns: Whether the locale was successfully fetched.
7641  *
7642  * Since: 3.12
7643  */
7644 gboolean
7645 e_book_sqlite_get_locale (EBookSqlite *ebsql,
7646                           gchar **locale_out,
7647                           GError **error)
7648 {
7649         gboolean success;
7650         GError *local_error = NULL;
7651
7652         g_return_val_if_fail (E_IS_BOOK_SQLITE (ebsql), FALSE);
7653         g_return_val_if_fail (locale_out != NULL && *locale_out == NULL, FALSE);
7654
7655         EBSQL_LOCK_MUTEX (&ebsql->priv->lock);
7656
7657         success = ebsql_exec_printf (
7658                 ebsql, "SELECT lc_collate FROM folders WHERE folder_id = %Q",
7659                 get_string_cb, locale_out, NULL, error,
7660                 ebsql->priv->folderid);
7661
7662         if (*locale_out == NULL) {
7663
7664                 /* This can't realistically happen, if it does we
7665                  * should warn about it in stdout */
7666                 g_warning ("EBookSqlite has no active locale in the database");
7667
7668                 *locale_out = g_strdup (ebsql->priv->locale);
7669         }
7670
7671         if (success && !ebsql_set_locale_internal (ebsql, *locale_out, &local_error)) {
7672                 g_warning ("Error loading new locale: %s", local_error->message);
7673                 g_clear_error (&local_error);
7674         }
7675
7676         EBSQL_UNLOCK_MUTEX (&ebsql->priv->lock);
7677
7678         return success;
7679 }
7680
7681 /**
7682  * e_book_sqlite_cursor_new:
7683  * @ebsql: An #EBookSqlite
7684  * @sexp: search expression; use NULL or an empty string to get all stored contacts.
7685  * @sort_fields: (array length=n_sort_fields): An array of #EContactFields as sort keys in order of priority
7686  * @sort_types: (array length=n_sort_fields): An array of #EBookCursorSortTypes, one for each field in @sort_fields
7687  * @n_sort_fields: The number of fields to sort results by.
7688  * @error: A return location to store any error that might be reported.
7689  *
7690  * Creates a new #EbSqlCursor.
7691  *
7692  * The cursor should be freed with e_book_sqlite_cursor_free().
7693  *
7694  * Returns: (transfer full): A newly created #EbSqlCursor
7695  *
7696  * Since: 3.12
7697  */
7698 EbSqlCursor *
7699 e_book_sqlite_cursor_new (EBookSqlite *ebsql,
7700                           const gchar *sexp,
7701                           const EContactField *sort_fields,
7702                           const EBookCursorSortType *sort_types,
7703                           guint n_sort_fields,
7704                           GError **error)
7705 {
7706         EbSqlCursor *cursor;
7707         gint i;
7708
7709         g_return_val_if_fail (E_IS_BOOK_SQLITE (ebsql), NULL);
7710
7711         /* We don't like '\0' sexps, prefer NULL */
7712         if (sexp && !sexp[0])
7713                 sexp = NULL;
7714
7715         EBSQL_LOCK_MUTEX (&ebsql->priv->lock);
7716
7717         /* Need one sort key ... */
7718         if (n_sort_fields == 0) {
7719                 EBSQL_SET_ERROR_LITERAL (
7720                         error, E_BOOK_SQLITE_ERROR_INVALID_QUERY,
7721                         _("At least one sort field must be specified to use an EbSqlCursor"));
7722                 EBSQL_UNLOCK_MUTEX (&ebsql->priv->lock);
7723                 return NULL;
7724         }
7725
7726         /* We only support string fields to sort the cursor */
7727         for (i = 0; i < n_sort_fields; i++) {
7728                 EBSQL_NOTE (
7729                         CURSOR,
7730                         g_printerr (
7731                                 "Building cursor to sort '%s' in '%s' order\n",
7732                                 e_contact_field_name (sort_fields[i]),
7733                                 sort_types[i] == E_BOOK_CURSOR_SORT_ASCENDING ?
7734                                 "ascending" : "descending"));
7735
7736                 if (e_contact_field_type (sort_fields[i]) != G_TYPE_STRING) {
7737                         EBSQL_SET_ERROR_LITERAL (
7738                                 error, E_BOOK_SQLITE_ERROR_INVALID_QUERY,
7739                                 _("Cannot sort by a field that is not a string type"));
7740                         EBSQL_UNLOCK_MUTEX (&ebsql->priv->lock);
7741                         return NULL;
7742                 }
7743         }
7744
7745         /* Now we need to create the cursor instance before setting up the query
7746          * (not really true, but more convenient that way).
7747          */
7748         cursor = ebsql_cursor_new (ebsql, sexp, sort_fields, sort_types, n_sort_fields);
7749
7750         /* Setup the cursor's query expression which might fail */
7751         if (!ebsql_cursor_setup_query (ebsql, cursor, sexp, error)) {
7752                 ebsql_cursor_free (cursor);
7753                 cursor = NULL;
7754         }
7755
7756         EBSQL_UNLOCK_MUTEX (&ebsql->priv->lock);
7757
7758         EBSQL_NOTE (
7759                 CURSOR,
7760                 g_printerr (
7761                         "%s cursor with search expression '%s'\n",
7762                         cursor ? "Successfully created" : "Failed to create",
7763                         sexp));
7764
7765         return cursor;
7766 }
7767
7768 /**
7769  * e_book_sqlite_cursor_free:
7770  * @ebsql: An #EBookSqlite
7771  * @cursor: The #EbSqlCursor to free
7772  *
7773  * Frees @cursor.
7774  *
7775  * Since: 3.12
7776  */
7777 void
7778 e_book_sqlite_cursor_free (EBookSqlite *ebsql,
7779                            EbSqlCursor *cursor)
7780 {
7781         g_return_if_fail (E_IS_BOOK_SQLITE (ebsql));
7782
7783         ebsql_cursor_free (cursor);
7784 }
7785
7786 typedef struct {
7787         GSList *results;
7788         gchar *alloc_vcard;
7789         const gchar *last_vcard;
7790
7791         gboolean collect_results;
7792         gint n_results;
7793 } CursorCollectData;
7794
7795 static gint
7796 collect_results_for_cursor_cb (gpointer ref,
7797                                gint ncol,
7798                                gchar **cols,
7799                                gchar **names)
7800 {
7801         CursorCollectData *data = ref;
7802
7803         if (data->collect_results) {
7804                 EbSqlSearchData *search_data;
7805
7806                 search_data = search_data_from_results (ncol, cols, names);
7807
7808                 data->results = g_slist_prepend (data->results, search_data);
7809
7810                 data->last_vcard = search_data->vcard;
7811         } else {
7812                 g_free (data->alloc_vcard);
7813                 data->alloc_vcard = g_strdup (cols[1]);
7814
7815                 data->last_vcard = data->alloc_vcard;
7816         }
7817
7818         data->n_results++;
7819
7820         return 0;
7821 }
7822
7823 /**
7824  * e_book_sqlite_cursor_step:
7825  * @ebsql: An #EBookSqlite
7826  * @cursor: The #EbSqlCursor to use
7827  * @flags: The #EbSqlCursorStepFlags for this step
7828  * @origin: The #EbSqlCursorOrigin from whence to step
7829  * @count: A positive or negative amount of contacts to try and fetch
7830  * @results: (out) (allow-none) (element-type EbSqlSearchData) (transfer full):
7831  *   A return location to store the results, or %NULL if %EBSQL_CURSOR_STEP_FETCH is not specified in %flags.
7832  * @cancellable: (allow-none): A #GCancellable
7833  * @error: A return location to store any error that might be reported.
7834  *
7835  * Steps @cursor through it's sorted query by a maximum of @count contacts
7836  * starting from @origin.
7837  *
7838  * If @count is negative, then the cursor will move through the list in reverse.
7839  *
7840  * If @cursor reaches the beginning or end of the query results, then the
7841  * returned list might not contain the amount of desired contacts, or might
7842  * return no results if the cursor currently points to the last contact.
7843  * Reaching the end of the list is not considered an error condition. Attempts
7844  * to step beyond the end of the list after having reached the end of the list
7845  * will however trigger an %E_BOOK_SQLITE_ERROR_END_OF_LIST error.
7846  *
7847  * If %EBSQL_CURSOR_STEP_FETCH is specified in %flags, a pointer to
7848  * a %NULL #GSList pointer should be provided for the @results parameter.
7849  *
7850  * The result list will be stored to @results and should be freed with g_slist_free()
7851  * and all elements freed with e_book_sqlite_search_data_free().
7852  *
7853  * Returns: The number of contacts traversed if successful, otherwise -1 is
7854  * returned and @error is set.
7855  *
7856  * Since: 3.12
7857  */
7858 gint
7859 e_book_sqlite_cursor_step (EBookSqlite *ebsql,
7860                            EbSqlCursor *cursor,
7861                            EbSqlCursorStepFlags flags,
7862                            EbSqlCursorOrigin origin,
7863                            gint count,
7864                            GSList **results,
7865                            GCancellable *cancellable,
7866                            GError **error)
7867 {
7868         CursorCollectData data = { NULL, NULL, NULL, FALSE, 0 };
7869         CursorState *state;
7870         GString *query;
7871         gboolean success;
7872         EbSqlCursorOrigin try_position;
7873
7874         g_return_val_if_fail (E_IS_BOOK_SQLITE (ebsql), -1);
7875         g_return_val_if_fail (cursor != NULL, -1);
7876         g_return_val_if_fail ((flags & EBSQL_CURSOR_STEP_FETCH) == 0 ||
7877                               (results != NULL && *results == NULL), -1);
7878
7879         /* Lock and check cancellable early */
7880         EBSQL_LOCK_OR_RETURN (ebsql, cancellable, -1);
7881
7882         EBSQL_NOTE (
7883                 CURSOR,
7884                 g_printerr (
7885                         "Cursor requested to step by %d with origin %s will move: %s will fetch: %s\n",
7886                         count, EBSQL_ORIGIN_STR (origin),
7887                         (flags & EBSQL_CURSOR_STEP_MOVE) ? "yes" : "no",
7888                         (flags & EBSQL_CURSOR_STEP_FETCH) ? "yes" : "no"));
7889
7890         /* Check if this step should result in an end of list error first */
7891         try_position = cursor->state.position;
7892         if (origin != EBSQL_CURSOR_ORIGIN_CURRENT)
7893                 try_position = origin;
7894
7895         /* Report errors for requests to run off the end of the list */
7896         if (try_position == EBSQL_CURSOR_ORIGIN_BEGIN && count < 0) {
7897                 EBSQL_SET_ERROR_LITERAL (
7898                         error, E_BOOK_SQLITE_ERROR_END_OF_LIST,
7899                         _("Tried to step a cursor in reverse, "
7900                         "but cursor is already at the beginning of the contact list"));
7901
7902                 EBSQL_UNLOCK_MUTEX (&ebsql->priv->lock);
7903                 return -1;
7904         } else if (try_position == EBSQL_CURSOR_ORIGIN_END && count > 0) {
7905                 EBSQL_SET_ERROR_LITERAL (
7906                         error, E_BOOK_SQLITE_ERROR_END_OF_LIST,
7907                         _("Tried to step a cursor forwards, "
7908                         "but cursor is already at the end of the contact list"));
7909
7910                 EBSQL_UNLOCK_MUTEX (&ebsql->priv->lock);
7911                 return -1;
7912         }
7913
7914         /* Nothing to do, silently return */
7915         if (count == 0 && try_position == EBSQL_CURSOR_ORIGIN_CURRENT) {
7916                 EBSQL_UNLOCK_MUTEX (&ebsql->priv->lock);
7917                 return 0;
7918         }
7919
7920         /* If we're not going to modify the position, just use
7921          * a copy of the current cursor state.
7922          */
7923         if ((flags & EBSQL_CURSOR_STEP_MOVE) != 0)
7924                 state = &(cursor->state);
7925         else
7926                 state = cursor_state_copy (cursor, &(cursor->state));
7927
7928         /* Every query starts with the STATE_CURRENT position, first
7929          * fix up the cursor state according to 'origin'
7930          */
7931         switch (origin) {
7932         case EBSQL_CURSOR_ORIGIN_CURRENT:
7933                 /* Do nothing, normal operation */
7934                 break;
7935
7936         case EBSQL_CURSOR_ORIGIN_BEGIN:
7937         case EBSQL_CURSOR_ORIGIN_END:
7938
7939                 /* Prepare the state before executing the query */
7940                 cursor_state_clear (cursor, state, origin);
7941                 break;
7942         }
7943
7944         /* If count is 0 then there is no need to run any
7945          * query, however it can be useful if you just want
7946          * to move the cursor to the beginning or ending of
7947          * the list.
7948          */
7949         if (count == 0) {
7950
7951                 /* Free the state copy if need be */
7952                 if ((flags & EBSQL_CURSOR_STEP_MOVE) == 0)
7953                         cursor_state_free (cursor, state);
7954
7955                 EBSQL_UNLOCK_MUTEX (&ebsql->priv->lock);
7956                 return 0;
7957         }
7958
7959         query = g_string_new (cursor->select_vcards);
7960
7961         /* Add the filter constraints (if any) */
7962         if (cursor->query) {
7963                 g_string_append (query, " WHERE ");
7964
7965                 g_string_append_c (query, '(');
7966                 g_string_append (query, cursor->query);
7967                 g_string_append_c (query, ')');
7968         }
7969
7970         /* Add the cursor constraints (if any) */
7971         if (state->values[0] != NULL) {
7972                 gchar *constraints = NULL;
7973
7974                 if (!cursor->query)
7975                         g_string_append (query, " WHERE ");
7976                 else
7977                         g_string_append (query, " AND ");
7978
7979                 constraints = ebsql_cursor_constraints (
7980                         ebsql, cursor, state, count < 0, FALSE);
7981
7982                 g_string_append_c (query, '(');
7983                 g_string_append (query, constraints);
7984                 g_string_append_c (query, ')');
7985
7986                 g_free (constraints);
7987         }
7988
7989         /* Add the sort order */
7990         g_string_append_c (query, ' ');
7991         if (count > 0)
7992                 g_string_append (query, cursor->order);
7993         else
7994                 g_string_append (query, cursor->reverse_order);
7995
7996         /* Add the limit */
7997         g_string_append_printf (query, " LIMIT %d", ABS (count));
7998
7999         /* Specify whether we really want results or not */
8000         data.collect_results = (flags & EBSQL_CURSOR_STEP_FETCH) != 0;
8001
8002         /* Execute the query */
8003         success = ebsql_exec (
8004                 ebsql, query->str,
8005                 collect_results_for_cursor_cb, &data,
8006                 cancellable, error);
8007
8008         /* Lock was obtained above */
8009         EBSQL_UNLOCK_MUTEX (&ebsql->priv->lock);
8010
8011         g_string_free (query, TRUE);
8012
8013         /* If there was no error, update the internal cursor state */
8014         if (success) {
8015
8016                 if (data.n_results < ABS (count)) {
8017
8018                         /* We've reached the end, clear the current state */
8019                         if (count < 0)
8020                                 cursor_state_clear (cursor, state, EBSQL_CURSOR_ORIGIN_BEGIN);
8021                         else
8022                                 cursor_state_clear (cursor, state, EBSQL_CURSOR_ORIGIN_END);
8023
8024                 } else if (data.last_vcard) {
8025
8026                         /* Set the cursor state to the last result */
8027                         cursor_state_set_from_vcard (ebsql, cursor, state, data.last_vcard);
8028                 } else
8029                         /* Should never get here */
8030                         g_warn_if_reached ();
8031
8032                 /* Assign the results to return (if any) */
8033                 if (results) {
8034                         /* Correct the order of results at the last minute */
8035                         *results = g_slist_reverse (data.results);
8036                         data.results = NULL;
8037                 }
8038         }
8039
8040         /* Cleanup what was allocated by collect_results_for_cursor_cb() */
8041         if (data.results)
8042                 g_slist_free_full (
8043                         data.results,
8044                         (GDestroyNotify) e_book_sqlite_search_data_free);
8045         g_free (data.alloc_vcard);
8046
8047         /* Free the copy state if we were working with a copy */
8048         if ((flags & EBSQL_CURSOR_STEP_MOVE) == 0)
8049                 cursor_state_free (cursor, state);
8050
8051         if (success)
8052                 return data.n_results;
8053
8054         return -1;
8055 }
8056
8057 /**
8058  * e_book_sqlite_cursor_set_target_alphabetic_index:
8059  * @ebsql: An #EBookSqlite
8060  * @cursor: The #EbSqlCursor to modify
8061  * @idx: The alphabetic index
8062  *
8063  * Sets the @cursor position to an
8064  * <link linkend="cursor-alphabet">Alphabetic Index</link>
8065  * into the alphabet active in @ebsql's locale.
8066  *
8067  * After setting the target to an alphabetic index, for example the
8068  * index for letter 'E', then further calls to e_book_sqlite_cursor_step()
8069  * will return results starting with the letter 'E' (or results starting
8070  * with the last result in 'D', if moving in a negative direction).
8071  *
8072  * The passed index must be a valid index in the active locale, knowledge
8073  * on the currently active alphabet index must be obtained using #ECollator
8074  * APIs.
8075  *
8076  * Use e_book_sqlite_ref_collator() to obtain the active collator for @ebsql.
8077  *
8078  * Since: 3.12
8079  */
8080 void
8081 e_book_sqlite_cursor_set_target_alphabetic_index (EBookSqlite *ebsql,
8082                                                   EbSqlCursor *cursor,
8083                                                   gint idx)
8084 {
8085         gint n_labels = 0;
8086
8087         g_return_if_fail (E_IS_BOOK_SQLITE (ebsql));
8088         g_return_if_fail (cursor != NULL);
8089         g_return_if_fail (idx >= 0);
8090
8091         e_collator_get_index_labels (
8092                 ebsql->priv->collator, &n_labels,
8093                 NULL, NULL, NULL);
8094         g_return_if_fail (idx < n_labels);
8095
8096         cursor_state_clear (cursor, &(cursor->state), EBSQL_CURSOR_ORIGIN_CURRENT);
8097         if (cursor->n_sort_fields > 0) {
8098                 SummaryField *field;
8099                 gchar *index_key;
8100
8101                 index_key = e_collator_generate_key_for_index (ebsql->priv->collator, idx);
8102                 field = summary_field_get (ebsql, cursor->sort_fields[0]);
8103
8104                 if (field && (field->index & INDEX_FLAG (SORT_KEY)) != 0) {
8105                         cursor->state.values[0] = index_key;
8106                 } else {
8107                         cursor->state.values[0] =
8108                                 ebsql_encode_vcard_sort_key (index_key);
8109                         g_free (index_key);
8110                 }
8111         }
8112 }
8113
8114 /**
8115  * e_book_sqlite_cursor_set_sexp:
8116  * @ebsql: An #EBookSqlite
8117  * @cursor: The #EbSqlCursor
8118  * @sexp: The new query expression for @cursor
8119  * @error: A return location to store any error that might be reported.
8120  *
8121  * Modifies the current query expression for @cursor. This will not
8122  * modify @cursor's state, but will change the outcome of any further
8123  * calls to e_book_sqlite_cursor_calculate() or
8124  * e_book_sqlite_cursor_step().
8125  *
8126  * Returns: %TRUE if the expression was valid and accepted by @ebsql
8127  *
8128  * Since: 3.12
8129  */
8130 gboolean
8131 e_book_sqlite_cursor_set_sexp (EBookSqlite *ebsql,
8132                                EbSqlCursor *cursor,
8133                                const gchar *sexp,
8134                                GError **error)
8135 {
8136         gboolean success;
8137
8138         g_return_val_if_fail (E_IS_BOOK_SQLITE (ebsql), FALSE);
8139         g_return_val_if_fail (cursor != NULL, FALSE);
8140
8141         /* We don't like '\0' sexps, prefer NULL */
8142         if (sexp && !sexp[0])
8143                 sexp = NULL;
8144
8145         EBSQL_LOCK_MUTEX (&ebsql->priv->lock);
8146         success = ebsql_cursor_setup_query (ebsql, cursor, sexp, error);
8147         EBSQL_UNLOCK_MUTEX (&ebsql->priv->lock);
8148
8149         return success;
8150 }
8151
8152 /**
8153  * e_book_sqlite_cursor_calculate:
8154  * @ebsql: An #EBookSqlite
8155  * @cursor: The #EbSqlCursor
8156  * @total: (out) (allow-none): A return location to store the total result set for this cursor
8157  * @position: (out) (allow-none): A return location to store the total results before the cursor value
8158  * @cancellable: (allow-none): A #GCancellable
8159  * @error: (allow-none): A return location to store any error that might be reported.
8160  *
8161  * Calculates the @total amount of results for the @cursor's query expression,
8162  * as well as the current @position of @cursor in the results. @position is
8163  * represented as the amount of results which lead up to the current value
8164  * of @cursor, if @cursor currently points to an exact contact, the position
8165  * also includes the cursor contact.
8166  *
8167  * Returns: Whether @total and @position were successfully calculated.
8168  *
8169  * Since: 3.12
8170  */
8171 gboolean
8172 e_book_sqlite_cursor_calculate (EBookSqlite *ebsql,
8173                                 EbSqlCursor *cursor,
8174                                 gint *total,
8175                                 gint *position,
8176                                 GCancellable *cancellable,
8177                                 GError **error)
8178 {
8179         gboolean success = TRUE;
8180         gint local_total = 0;
8181
8182         g_return_val_if_fail (E_IS_BOOK_SQLITE (ebsql), FALSE);
8183         g_return_val_if_fail (cursor != NULL, FALSE);
8184
8185         /* If we're in a clear cursor state, then the position is 0 */
8186         if (position && cursor->state.values[0] == NULL) {
8187
8188                 if (cursor->state.position == EBSQL_CURSOR_ORIGIN_BEGIN) {
8189                         /* Mark the local pointer NULL, no need to calculate this anymore */
8190                         *position = 0;
8191                         position = NULL;
8192                 } else if (cursor->state.position == EBSQL_CURSOR_ORIGIN_END) {
8193
8194                         /* Make sure that we look up the total so we can
8195                          * set the position to 'total + 1'
8196                          */
8197                         if (!total)
8198                                 total = &local_total;
8199                 }
8200         }
8201
8202         /* Early return if there is nothing to do */
8203         if (!total && !position)
8204                 return TRUE;
8205
8206         EBSQL_LOCK_OR_RETURN (ebsql, cancellable, -1);
8207
8208         /* Start a read transaction, it's important our two queries are atomic */
8209         if (!ebsql_start_transaction (ebsql, EBSQL_LOCK_READ, cancellable, error)) {
8210                 EBSQL_UNLOCK_MUTEX (&ebsql->priv->lock);
8211                 return FALSE;
8212         }
8213
8214         if (total)
8215                 success = cursor_count_total_locked (ebsql, cursor, total, error);
8216
8217         if (success && position)
8218                 success = cursor_count_position_locked (ebsql, cursor, position, error);
8219
8220         if (success)
8221                 success = ebsql_commit_transaction (ebsql, error);
8222         else
8223                 /* The GError is already set. */
8224                 ebsql_rollback_transaction (ebsql, NULL);
8225
8226         EBSQL_UNLOCK_MUTEX (&ebsql->priv->lock);
8227
8228         /* In the case we're at the end, we just set the position
8229          * to be the total + 1
8230          */
8231         if (success && position && total &&
8232             cursor->state.position == EBSQL_CURSOR_ORIGIN_END)
8233                 *position = *total + 1;
8234
8235         return success;
8236 }
8237
8238 /**
8239  * e_book_sqlite_cursor_compare_contact:
8240  * @ebsql: An #EBookSqlite
8241  * @cursor: The #EbSqlCursor
8242  * @contact: The #EContact to compare
8243  * @matches_sexp: (out) (allow-none): Whether the contact matches the cursor's search expression
8244  *
8245  * Compares @contact with @cursor and returns whether @contact is less than, equal to, or greater
8246  * than @cursor.
8247  *
8248  * Returns: A value that is less than, equal to, or greater than zero if @contact is found,
8249  * respectively, to be less than, to match, or be greater than the current value of @cursor.
8250  *
8251  * Since: 3.12
8252  */
8253 gint
8254 e_book_sqlite_cursor_compare_contact (EBookSqlite *ebsql,
8255                                       EbSqlCursor *cursor,
8256                                       EContact *contact,
8257                                       gboolean *matches_sexp)
8258 {
8259         EBookSqlitePrivate *priv;
8260         gint i;
8261         gint comparison = 0;
8262
8263         g_return_val_if_fail (E_IS_BOOK_SQLITE (ebsql), -1);
8264         g_return_val_if_fail (E_IS_CONTACT (contact), -1);
8265         g_return_val_if_fail (cursor != NULL, -1);
8266
8267         priv = ebsql->priv;
8268
8269         if (matches_sexp) {
8270                 if (cursor->sexp == NULL)
8271                         *matches_sexp = TRUE;
8272                 else
8273                         *matches_sexp =
8274                                 e_book_backend_sexp_match_contact (cursor->sexp, contact);
8275         }
8276
8277         for (i = 0; i < cursor->n_sort_fields && comparison == 0; i++) {
8278                 SummaryField *field;
8279                 gchar *contact_key = NULL;
8280                 const gchar *cursor_key = NULL;
8281                 const gchar *field_value;
8282                 gchar *freeme = NULL;
8283
8284                 field_value = (const gchar *) e_contact_get_const (contact, cursor->sort_fields[i]);
8285                 if (field_value)
8286                         contact_key = e_collator_generate_key (priv->collator, field_value, NULL);
8287
8288                 field = summary_field_get (ebsql, cursor->sort_fields[i]);
8289
8290                 if (field && (field->index & INDEX_FLAG (SORT_KEY)) != 0) {
8291                         cursor_key = cursor->state.values[i];
8292                 } else {
8293
8294                         if (cursor->state.values[i])
8295                                 freeme = ebsql_decode_vcard_sort_key (cursor->state.values[i]);
8296
8297                         cursor_key = freeme;
8298                 }
8299
8300                 /* Empty state sorts below any contact value, which means the contact sorts above cursor */
8301                 if (cursor_key == NULL)
8302                         comparison = 1;
8303                 else
8304                         /* Check if contact sorts below, equal to, or above the cursor */
8305                         comparison = g_strcmp0 (contact_key, cursor_key);
8306
8307                 g_free (contact_key);
8308                 g_free (freeme);
8309         }
8310
8311         /* UID tie-breaker */
8312         if (comparison == 0) {
8313                 const gchar *uid;
8314
8315                 uid = (const gchar *) e_contact_get_const (contact, E_CONTACT_UID);
8316
8317                 if (cursor->state.last_uid == NULL)
8318                         comparison = 1;
8319                 else if (uid == NULL)
8320                         comparison = -1;
8321                 else
8322                         comparison = strcmp (uid, cursor->state.last_uid);
8323         }
8324
8325         return comparison;
8326 }