camel/providers/imap/camel-imap-folder.c

   1 /* -*- Mode: C; tab-width: 8; indent-tabs-mode: t; c-basic-offset: 8 -*- */
   2 /* camel-imap-folder.c: class for an imap folder */
   3
   4 /*
   5  * Authors:
   6  *   Dan Winship <danw@ximian.com>
   7  *   Jeffrey Stedfast <fejj@ximian.com>
   8  *
   9  * Copyright (C) 1999-2008 Novell, Inc. (www.novell.com)
  10  *
  11  * This program is free software; you can redistribute it and/or
  12  * modify it under the terms of version 2 of the GNU Lesser General Public
  13  * License as published by the Free Software Foundation.
  14  *
  15  * This program is distributed in the hope that it will be useful,
  16  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  17  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  18  * GNU Lesser General Public License for more details.
  19  *
  20  * You should have received a copy of the GNU Lesser General Public License
  21  * along with this program; if not, write to the Free Software
  22  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301
  23  * USA
  24  */
  25
  26 #include <config.h>
  27
  28 #include <ctype.h>
  29 #include <errno.h>
  30 #include <fcntl.h>
  31 #include <stdlib.h>
  32 #include <string.h>
  33 #include <unistd.h>
  34 #include <sys/stat.h>
  35 #include <sys/types.h>
  36
  37 #include <glib/gi18n-lib.h>
  38
  39 #include <camel/camel.h>
  40
  41 #include "camel-imap-command.h"
  42 #include "camel-imap-folder.h"
  43 #include "camel-imap-journal.h"
  44 #include "camel-imap-message-cache.h"
  45 #include "camel-imap-private.h"
  46 #include "camel-imap-search.h"
  47 #include "camel-imap-settings.h"
  48 #include "camel-imap-store.h"
  49 #include "camel-imap-store-summary.h"
  50 #include "camel-imap-summary.h"
  51 #include "camel-imap-utils.h"
  52 #include "camel-imap-wrapper.h"
  53
  54 #define d(x)
  55
  56 /* set to -1 for infinite size (suggested max command-line length is
  57  * 1000 octets (see rfc2683), so we should keep the uid-set length to
  58  * something under that so that our command-lines don't exceed 1000
  59  * octets) */
  60 #define UID_SET_LIMIT  (768)
  61
  62 #define CAMEL_IMAP_FOLDER_GET_PRIVATE(obj) \
  63         (G_TYPE_INSTANCE_GET_PRIVATE \
  64         ((obj), CAMEL_TYPE_IMAP_FOLDER, CamelImapFolderPrivate))
  65
  66 /* The custom property ID is a CamelArg artifact.
  67  * It still identifies the property in state files. */
  68 enum {
  69         PROP_0,
  70         PROP_CHECK_FOLDER = 0x2500,
  71         PROP_APPLY_FILTERS
  72 };
  73
  74 extern gint camel_application_is_exiting;
  75
  76 static gboolean imap_rescan (CamelFolder *folder, gint exists, GCancellable *cancellable, GError **error);
  77 static gboolean imap_refresh_info_sync (CamelFolder *folder, GCancellable *cancellable, GError **error);
  78 static gboolean imap_sync_offline (CamelFolder *folder, GError **error);
  79 static gboolean imap_synchronize_sync (CamelFolder *folder, gboolean expunge, GCancellable *cancellable, GError **error);
  80 static gboolean imap_expunge_uids_online (CamelFolder *folder, GPtrArray *uids, GCancellable *cancellable, GError **error);
  81 static gboolean imap_expunge_uids_offline (CamelFolder *folder, GPtrArray *uids, GCancellable *cancellable, GError **error);
  82 static gboolean imap_expunge_sync (CamelFolder *folder, GCancellable *cancellable, GError **error);
  83 /*static void imap_cache_message (CamelDiscoFolder *disco_folder, const gchar *uid, GError **error);*/
  84 static void imap_rename (CamelFolder *folder, const gchar *new);
  85 static GPtrArray * imap_get_uncached_uids (CamelFolder *folder, GPtrArray * uids, GError **error);
  86 static gchar * imap_get_filename (CamelFolder *folder, const gchar *uid, GError **error);
  87
  88 /* message manipulation */
  89 static CamelMimeMessage *imap_get_message_sync (CamelFolder *folder, const gchar *uid, GCancellable *cancellable,
  90                                            GError **error);
  91 static gboolean imap_synchronize_message_sync (CamelFolder *folder, const gchar *uid, GCancellable *cancellable,
  92                                GError **error);
  93 static gboolean imap_append_online (CamelFolder *folder, CamelMimeMessage *message,
  94                                 CamelMessageInfo *info, gchar **appended_uid,
  95                                 GCancellable *cancellable,
  96                                 GError **error);
  97 static gboolean imap_append_offline (CamelFolder *folder, CamelMimeMessage *message,
  98                                  CamelMessageInfo *info, gchar **appended_uid,
  99                                  GError **error);
 100
 101 static gboolean imap_transfer_online            (CamelFolder *source,
 102                                                  GPtrArray *uids,
 103                                                  CamelFolder *dest,
 104                                                  gboolean delete_originals,
 105                                                  GPtrArray **transferred_uids,
 106                                                  GCancellable *cancellable,
 107                                                  GError **error);
 108 static gboolean imap_transfer_offline           (CamelFolder *source,
 109                                                  GPtrArray *uids,
 110                                                  CamelFolder *dest,
 111                                                  gboolean delete_originals,
 112                                                  GPtrArray **transferred_uids,
 113                                                  GCancellable *cancellable,
 114                                                  GError **error);
 115
 116 /* searching */
 117 static GPtrArray *imap_search_by_expression (CamelFolder *folder, const gchar *expression, GCancellable *cancellable, GError **error);
 118 static guint32 imap_count_by_expression (CamelFolder *folder, const gchar *expression, GCancellable *cancellable, GError **error);
 119 static GPtrArray *imap_search_by_uids       (CamelFolder *folder, const gchar *expression, GPtrArray *uids, GCancellable *cancellable, GError **error);
 120 static void       imap_search_free          (CamelFolder *folder, GPtrArray *uids);
 121
 122 static void imap_thaw (CamelFolder *folder);
 123 static CamelFolderQuotaInfo *
 124                 imap_get_quota_info_sync        (CamelFolder *folder,
 125                                                  GCancellable *cancellable,
 126                                                  GError **error);
 127
 128 static GData *parse_fetch_response (CamelImapFolder *imap_folder, gchar *msg_att);
 129
 130 /* internal helpers */
 131 static CamelImapMessageInfo * imap_folder_summary_uid_or_error (
 132         CamelFolderSummary *summary,
 133         const gchar * uid,
 134         GError **error);
 135
 136 static gboolean imap_transfer_messages          (CamelFolder *source,
 137                                                  GPtrArray *uids,
 138                                                  CamelFolder *dest,
 139                                                  gboolean delete_originals,
 140                                                  GPtrArray **transferred_uids,
 141                                                  gboolean can_call_sync,
 142                                                  GCancellable *cancellable,
 143                                                  GError **error);
 144
 145 static gboolean
 146 imap_folder_get_apply_filters (CamelImapFolder *folder)
 147 {
 148         g_return_val_if_fail (folder != NULL, FALSE);
 149         g_return_val_if_fail (CAMEL_IS_IMAP_FOLDER (folder), FALSE);
 150
 151         return folder->priv->apply_filters;
 152 }
 153
 154 static void
 155 imap_folder_set_apply_filters (CamelImapFolder *folder,
 156                                gboolean apply_filters)
 157 {
 158         g_return_if_fail (folder != NULL);
 159         g_return_if_fail (CAMEL_IS_IMAP_FOLDER (folder));
 160
 161         if ((folder->priv->apply_filters ? 1 : 0) == (apply_filters ? 1 : 0))
 162                 return;
 163
 164         folder->priv->apply_filters = apply_filters;
 165
 166         g_object_notify (G_OBJECT (folder), "apply-filters");
 167 }
 168
 169 #ifdef G_OS_WIN32
 170 /* The strtok() in Microsoft's C library is MT-safe (but still uses
 171  * only one buffer pointer per thread, but for the use of strtok_r()
 172  * here that's enough).
 173  */
 174 #define strtok_r(s,sep,lasts) (*(lasts)=strtok((s),(sep)))
 175 #endif
 176
 177 G_DEFINE_TYPE (CamelImapFolder, camel_imap_folder, CAMEL_TYPE_OFFLINE_FOLDER)
 178
 179 static void
 180 imap_folder_set_property (GObject *object,
 181                           guint property_id,
 182                           const GValue *value,
 183                           GParamSpec *pspec)
 184 {
 185         switch (property_id) {
 186                 case PROP_CHECK_FOLDER:
 187                         camel_imap_folder_set_check_folder (
 188                                 CAMEL_IMAP_FOLDER (object),
 189                                 g_value_get_boolean (value));
 190                         return;
 191
 192                 case PROP_APPLY_FILTERS:
 193                         imap_folder_set_apply_filters (
 194                                 CAMEL_IMAP_FOLDER (object),
 195                                 g_value_get_boolean (value));
 196                         return;
 197         }
 198
 199         G_OBJECT_WARN_INVALID_PROPERTY_ID (object, property_id, pspec);
 200 }
 201
 202 static void
 203 imap_folder_get_property (GObject *object,
 204                           guint property_id,
 205                           GValue *value,
 206                           GParamSpec *pspec)
 207 {
 208         switch (property_id) {
 209                 case PROP_CHECK_FOLDER:
 210                         g_value_set_boolean (
 211                                 value, camel_imap_folder_get_check_folder (
 212                                 CAMEL_IMAP_FOLDER (object)));
 213                         return;
 214
 215                 case PROP_APPLY_FILTERS:
 216                         g_value_set_boolean (
 217                                 value, imap_folder_get_apply_filters (
 218                                 CAMEL_IMAP_FOLDER (object)));
 219                         return;
 220         }
 221
 222         G_OBJECT_WARN_INVALID_PROPERTY_ID (object, property_id, pspec);
 223 }
 224
 225 static void
 226 imap_folder_dispose (GObject *object)
 227 {
 228         CamelImapFolder *imap_folder;
 229         CamelStore *parent_store;
 230
 231         imap_folder = CAMEL_IMAP_FOLDER (object);
 232
 233         parent_store = camel_folder_get_parent_store (CAMEL_FOLDER (imap_folder));
 234         if (parent_store) {
 235                 camel_store_summary_disconnect_folder_summary (
 236                         (CamelStoreSummary *) ((CamelImapStore *) parent_store)->summary,
 237                         CAMEL_FOLDER (imap_folder)->summary);
 238         }
 239
 240         if (imap_folder->search != NULL) {
 241                 g_object_unref (imap_folder->search);
 242                 imap_folder->search = NULL;
 243         }
 244
 245         if (imap_folder->cache != NULL) {
 246                 g_object_unref (imap_folder->cache);
 247                 imap_folder->cache = NULL;
 248         }
 249
 250         if (imap_folder->priv->ignore_recent != NULL) {
 251                 g_hash_table_unref (imap_folder->priv->ignore_recent);
 252                 imap_folder->priv->ignore_recent = NULL;
 253         }
 254
 255         if (imap_folder->journal != NULL) {
 256                 camel_offline_journal_write (imap_folder->journal, NULL);
 257                 g_object_unref (imap_folder->journal);
 258                 imap_folder->journal = NULL;
 259         }
 260
 261         /* Chain up to parent's dispose() method. */
 262         G_OBJECT_CLASS (camel_imap_folder_parent_class)->dispose (object);
 263 }
 264
 265 static void
 266 imap_folder_finalize (GObject *object)
 267 {
 268         CamelImapFolder *imap_folder;
 269
 270         imap_folder = CAMEL_IMAP_FOLDER (object);
 271
 272         g_static_mutex_free (&imap_folder->priv->search_lock);
 273         g_static_rec_mutex_free (&imap_folder->priv->cache_lock);
 274
 275         /* Chain up to parent's finalize() method. */
 276         G_OBJECT_CLASS (camel_imap_folder_parent_class)->finalize (object);
 277 }
 278
 279 static void
 280 imap_folder_constructed (GObject *object)
 281 {
 282         CamelNetworkSettings *network_settings;
 283         CamelSettings *settings;
 284         CamelService *service;
 285         CamelFolder *folder;
 286         CamelStore *parent_store;
 287         const gchar *full_name;
 288         const gchar *host;
 289         const gchar *user;
 290         gchar *description;
 291
 292         folder = CAMEL_FOLDER (object);
 293         full_name = camel_folder_get_full_name (folder);
 294         parent_store = camel_folder_get_parent_store (folder);
 295
 296         service = CAMEL_SERVICE (parent_store);
 297         settings = camel_service_get_settings (service);
 298
 299         network_settings = CAMEL_NETWORK_SETTINGS (settings);
 300         host = camel_network_settings_get_host (network_settings);
 301         user = camel_network_settings_get_user (network_settings);
 302
 303         description = g_strdup_printf (
 304                 "%s@%s:%s", user, host, full_name);
 305         camel_folder_set_description (folder, description);
 306         g_free (description);
 307 }
 308
 309 static void
 310 camel_imap_folder_class_init (CamelImapFolderClass *class)
 311 {
 312         GObjectClass *object_class;
 313         CamelFolderClass *folder_class;
 314
 315         g_type_class_add_private (class, sizeof (CamelImapFolderPrivate));
 316
 317         object_class = G_OBJECT_CLASS (class);
 318         object_class->set_property = imap_folder_set_property;
 319         object_class->get_property = imap_folder_get_property;
 320         object_class->dispose = imap_folder_dispose;
 321         object_class->finalize = imap_folder_finalize;
 322         object_class->constructed = imap_folder_constructed;
 323
 324         folder_class = CAMEL_FOLDER_CLASS (class);
 325         folder_class->rename = imap_rename;
 326         folder_class->search_by_expression = imap_search_by_expression;
 327         folder_class->count_by_expression = imap_count_by_expression;
 328         folder_class->search_by_uids = imap_search_by_uids;
 329         folder_class->search_free = imap_search_free;
 330         folder_class->thaw = imap_thaw;
 331         folder_class->get_uncached_uids = imap_get_uncached_uids;
 332         folder_class->get_filename = imap_get_filename;
 333         folder_class->append_message_sync = imap_append_online;
 334         folder_class->expunge_sync = imap_expunge_sync;
 335         folder_class->get_message_sync = imap_get_message_sync;
 336         folder_class->get_quota_info_sync = imap_get_quota_info_sync;
 337         folder_class->refresh_info_sync = imap_refresh_info_sync;
 338         folder_class->synchronize_sync = imap_synchronize_sync;
 339         folder_class->synchronize_message_sync = imap_synchronize_message_sync;
 340         folder_class->transfer_messages_to_sync = imap_transfer_online;
 341
 342         g_object_class_install_property (
 343                 object_class,
 344                 PROP_CHECK_FOLDER,
 345                 g_param_spec_boolean (
 346                         "check-folder",
 347                         "Check Folder",
 348                         _("Always check for _new mail in this folder"),
 349                         FALSE,
 350                         G_PARAM_READWRITE |
 351                         CAMEL_PARAM_PERSISTENT));
 352
 353         g_object_class_install_property (
 354                 object_class,
 355                 PROP_APPLY_FILTERS,
 356                 g_param_spec_boolean (
 357                         "apply-filters",
 358                         "Apply Filters",
 359                         _("Apply message _filters to this folder"),
 360                         FALSE,
 361                         G_PARAM_READWRITE |
 362                         CAMEL_PARAM_PERSISTENT));
 363 }
 364
 365 static void
 366 camel_imap_folder_init (CamelImapFolder *imap_folder)
 367 {
 368         CamelFolder *folder = CAMEL_FOLDER (imap_folder);
 369
 370         imap_folder->priv = CAMEL_IMAP_FOLDER_GET_PRIVATE (imap_folder);
 371
 372         folder->permanent_flags = CAMEL_MESSAGE_ANSWERED | CAMEL_MESSAGE_DELETED |
 373                 CAMEL_MESSAGE_DRAFT | CAMEL_MESSAGE_FLAGGED | CAMEL_MESSAGE_SEEN;
 374
 375         folder->folder_flags |= (CAMEL_FOLDER_HAS_SUMMARY_CAPABILITY |
 376                                  CAMEL_FOLDER_HAS_SEARCH_CAPABILITY);
 377
 378         g_static_mutex_init (&imap_folder->priv->search_lock);
 379         g_static_rec_mutex_init (&imap_folder->priv->cache_lock);
 380         imap_folder->priv->ignore_recent = NULL;
 381
 382         imap_folder->journal = NULL;
 383         imap_folder->need_rescan = TRUE;
 384 }
 385
 386 static void
 387 replay_offline_journal (CamelImapStore *imap_store,
 388                         CamelImapFolder *imap_folder,
 389                         GCancellable *cancellable,
 390                         GError **error)
 391 {
 392         CamelIMAPJournal *imap_journal;
 393
 394         g_return_if_fail (imap_store != NULL);
 395         g_return_if_fail (imap_folder != NULL);
 396         g_return_if_fail (imap_folder->journal != NULL);
 397
 398         imap_journal = CAMEL_IMAP_JOURNAL (imap_folder->journal);
 399         g_return_if_fail (imap_journal != NULL);
 400
 401         /* do not replay when still in offline */
 402         if (!camel_offline_store_get_online (CAMEL_OFFLINE_STORE (imap_store)) || !camel_imap_store_connected (imap_store, error))
 403                 return;
 404
 405         /* Check if the replay is already in progress as imap_sync would be called while expunge resync */
 406         if (!imap_journal->rp_in_progress) {
 407                 imap_journal->rp_in_progress++;
 408
 409                 camel_offline_journal_replay (
 410                         imap_folder->journal, cancellable, error);
 411                 camel_imap_journal_close_folders (imap_journal);
 412                 camel_offline_journal_write (imap_folder->journal, error);
 413
 414                 imap_journal->rp_in_progress--;
 415                 g_return_if_fail (imap_journal->rp_in_progress >= 0);
 416         }
 417 }
 418
 419 CamelFolder *
 420 camel_imap_folder_new (CamelStore *parent,
 421                        const gchar *folder_name,
 422                        const gchar *folder_dir,
 423                        GError **error)
 424 {
 425         CamelFolder *folder;
 426         CamelImapFolder *imap_folder;
 427         const gchar *short_name;
 428         gchar *state_file, *path;
 429         CamelService *service;
 430         CamelSettings *settings;
 431         gboolean filter_all;
 432         gboolean filter_inbox;
 433         gboolean filter_junk;
 434         gboolean filter_junk_inbox;
 435
 436         if (g_mkdir_with_parents (folder_dir, S_IRWXU) != 0) {
 437                 g_set_error (
 438                         error, G_IO_ERROR,
 439                         g_io_error_from_errno (errno),
 440                         _("Could not create directory %s: %s"),
 441                         folder_dir, g_strerror (errno));
 442                 return NULL;
 443         }
 444
 445         short_name = strrchr (folder_name, '/');
 446         if (short_name)
 447                 short_name++;
 448         else
 449                 short_name = folder_name;
 450         folder = g_object_new (
 451                 CAMEL_TYPE_IMAP_FOLDER,
 452                 "full-name", folder_name,
 453                 "display-name", short_name,
 454                 "parent-store", parent, NULL);
 455
 456         folder->summary = camel_imap_summary_new (folder);
 457         if (!folder->summary) {
 458                 g_object_unref (folder);
 459                 g_set_error (
 460                         error, CAMEL_ERROR, CAMEL_ERROR_GENERIC,
 461                         _("Could not load summary for %s"), folder_name);
 462                 return NULL;
 463         }
 464
 465         imap_folder = CAMEL_IMAP_FOLDER (folder);
 466         path = g_build_filename (folder_dir, "journal", NULL);
 467         imap_folder->journal = camel_imap_journal_new (imap_folder, path);
 468         g_free (path);
 469
 470         /* set/load persistent state */
 471         state_file = g_build_filename (folder_dir, "cmeta", NULL);
 472         camel_object_set_state_filename (CAMEL_OBJECT (folder), state_file);
 473         g_free (state_file);
 474         camel_object_state_read (CAMEL_OBJECT (folder));
 475
 476         imap_folder->cache = camel_imap_message_cache_new (folder_dir, folder->summary, error);
 477         if (!imap_folder->cache) {
 478                 g_object_unref (folder);
 479                 return NULL;
 480         }
 481
 482         service = CAMEL_SERVICE (parent);
 483         settings = camel_service_get_settings (service);
 484
 485         g_object_get (
 486                 settings,
 487                 "filter-all", &filter_all,
 488                 "filter-inbox", &filter_inbox,
 489                 "filter-junk", &filter_junk,
 490                 "filter-junk-inbox", &filter_junk_inbox,
 491                 NULL);
 492
 493         if (g_ascii_strcasecmp (folder_name, "INBOX") == 0) {
 494                 if (filter_inbox || filter_all)
 495                         folder->folder_flags |= CAMEL_FOLDER_FILTER_RECENT;
 496                 if (filter_junk)
 497                         folder->folder_flags |= CAMEL_FOLDER_FILTER_JUNK;
 498                 if (filter_junk_inbox)
 499                         folder->folder_flags |= CAMEL_FOLDER_FILTER_JUNK;
 500         } else {
 501                 gboolean folder_is_trash, folder_is_junk;
 502                 gchar *junk_path;
 503                 gchar *trash_path;
 504
 505                 junk_path = camel_imap_settings_dup_real_junk_path (
 506                         CAMEL_IMAP_SETTINGS (settings));
 507
 508                 /* So we can safely compare strings. */
 509                 if (junk_path == NULL)
 510                         junk_path = g_strdup ("");
 511
 512                 trash_path = camel_imap_settings_dup_real_trash_path (
 513                         CAMEL_IMAP_SETTINGS (settings));
 514
 515                 /* So we can safely compare strings. */
 516                 if (trash_path == NULL)
 517                         trash_path = g_strdup ("");
 518
 519                 if (filter_junk && !filter_junk_inbox)
 520                         folder->folder_flags |= CAMEL_FOLDER_FILTER_JUNK;
 521
 522                 folder_is_trash =
 523                         (parent->flags & CAMEL_STORE_VTRASH) == 0 &&
 524                         g_ascii_strcasecmp (trash_path, folder_name) == 0;
 525
 526                 if (folder_is_trash)
 527                         folder->folder_flags |= CAMEL_FOLDER_IS_TRASH;
 528
 529                 folder_is_junk =
 530                         (parent->flags & CAMEL_STORE_VJUNK) == 0 &&
 531                         g_ascii_strcasecmp (junk_path, folder_name) == 0;
 532
 533                 if (folder_is_junk)
 534                         folder->folder_flags |= CAMEL_FOLDER_IS_JUNK;
 535
 536                 if (filter_all || imap_folder_get_apply_filters (imap_folder))
 537                         folder->folder_flags |= CAMEL_FOLDER_FILTER_RECENT;
 538
 539                 g_free (junk_path);
 540                 g_free (trash_path);
 541         }
 542
 543         imap_folder->search = camel_imap_search_new (folder_dir);
 544
 545         camel_store_summary_connect_folder_summary (
 546                 (CamelStoreSummary *) ((CamelImapStore *) parent)->summary,
 547                 folder_name, folder->summary);
 548
 549         return folder;
 550 }
 551
 552 gboolean
 553 camel_imap_folder_get_check_folder (CamelImapFolder *imap_folder)
 554 {
 555         g_return_val_if_fail (CAMEL_IS_IMAP_FOLDER (imap_folder), FALSE);
 556
 557         return imap_folder->priv->check_folder;
 558 }
 559
 560 void
 561 camel_imap_folder_set_check_folder (CamelImapFolder *imap_folder,
 562                                     gboolean check_folder)
 563 {
 564         CamelFolder *folder;
 565         CamelStore *parent_store;
 566         const gchar *full_name;
 567
 568         g_return_if_fail (CAMEL_IS_IMAP_FOLDER (imap_folder));
 569
 570         imap_folder->priv->check_folder = check_folder;
 571
 572         folder = CAMEL_FOLDER (imap_folder);
 573         full_name = camel_folder_get_full_name (folder);
 574         parent_store = camel_folder_get_parent_store (folder);
 575
 576         /* Update the summary so the value is restored
 577          * correctly the next time the folder is loaded. */
 578         if (CAMEL_IS_IMAP_STORE (parent_store)) {
 579                 CamelImapStore *imap_store;
 580                 CamelStoreSummary *summary;
 581                 CamelStoreInfo *si;
 582
 583                 imap_store = CAMEL_IMAP_STORE (parent_store);
 584                 summary = CAMEL_STORE_SUMMARY (imap_store->summary);
 585
 586                 si = camel_store_summary_path (summary, full_name);
 587                 if (si != NULL) {
 588                         guint32 old_flags = si->flags;
 589
 590                         si->flags &= ~CAMEL_STORE_INFO_FOLDER_CHECK_FOR_NEW;
 591                         si->flags |= check_folder ? CAMEL_STORE_INFO_FOLDER_CHECK_FOR_NEW : 0;
 592
 593                         if (si->flags != old_flags) {
 594                                 camel_store_summary_touch (summary);
 595                                 camel_store_summary_save (summary);
 596                         }
 597
 598                         camel_store_summary_info_free (summary, si);
 599                 }
 600         }
 601
 602         g_object_notify (G_OBJECT (imap_folder), "check-folder");
 603 }
 604
 605 /* Called with the store's connect_lock locked */
 606 gboolean
 607 camel_imap_folder_selected (CamelFolder *folder,
 608                             CamelImapResponse *response,
 609                             GCancellable *cancellable,
 610                             GError **error)
 611 {
 612         CamelImapFolder *imap_folder = CAMEL_IMAP_FOLDER (folder);
 613         CamelImapSummary *imap_summary = CAMEL_IMAP_SUMMARY (folder->summary);
 614         gulong exists = 0, validity = 0, val, uid;
 615         CamelMessageFlags perm_flags = 0;
 616         GData *fetch_data;
 617         gint i, count;
 618         gchar *resp;
 619
 620         count = camel_folder_summary_count (folder->summary);
 621
 622         for (i = 0; i < response->untagged->len; i++) {
 623                 resp = (gchar *) response->untagged->pdata[i] + 2;
 624
 625                 if (!g_ascii_strncasecmp (resp, "FLAGS ", 6) && !perm_flags) {
 626                         resp += 6;
 627                         imap_parse_flag_list (&resp, &folder->permanent_flags, NULL);
 628                 } else if (!g_ascii_strncasecmp (resp, "OK [PERMANENTFLAGS ", 19)) {
 629                         resp += 19;
 630
 631                         /* workaround for broken IMAP servers that send
 632                          * "* OK [PERMANENTFLAGS ()] Permanent flags"
 633                          * even tho they do allow storing flags. */
 634                         imap_parse_flag_list (&resp, &perm_flags, NULL);
 635                         if (perm_flags != 0)
 636                                 folder->permanent_flags = perm_flags;
 637                 } else if (!g_ascii_strncasecmp (resp, "OK [UIDVALIDITY ", 16)) {
 638                         validity = strtoul (resp + 16, NULL, 10);
 639                 } else if (isdigit ((guchar) * resp)) {
 640                         gulong num = strtoul (resp, &resp, 10);
 641
 642                         if (!g_ascii_strncasecmp (resp, " EXISTS", 7)) {
 643                                 exists = num;
 644                                 /* Remove from the response so nothing
 645                                  * else tries to interpret it.
 646                                  */
 647                                 g_free (response->untagged->pdata[i]);
 648                                 g_ptr_array_remove_index (response->untagged, i--);
 649                         }
 650                 }
 651         }
 652
 653         if (camel_strstrcase (response->status, "OK [READ-ONLY]"))
 654                 imap_folder->read_only = TRUE;
 655
 656         if (!imap_summary->validity)
 657                 imap_summary->validity = validity;
 658         else if (validity != imap_summary->validity) {
 659                 imap_summary->validity = validity;
 660                 camel_folder_summary_clear (folder->summary, NULL);
 661                 CAMEL_IMAP_FOLDER_REC_LOCK (imap_folder, cache_lock);
 662                 camel_imap_message_cache_clear (imap_folder->cache);
 663                 CAMEL_IMAP_FOLDER_REC_UNLOCK (imap_folder, cache_lock);
 664                 imap_folder->need_rescan = FALSE;
 665                 return camel_imap_folder_changed (
 666                         folder, exists, NULL, cancellable, error);
 667         }
 668
 669         /* If we've lost messages, we have to rescan everything */
 670         if (exists < count)
 671                 imap_folder->need_rescan = TRUE;
 672         else if (count != 0 && !imap_folder->need_rescan) {
 673                 CamelStore *parent_store;
 674                 CamelImapStore *store;
 675                 GPtrArray *known_uids;
 676                 const gchar *old_uid;
 677
 678                 parent_store = camel_folder_get_parent_store (folder);
 679                 store = CAMEL_IMAP_STORE (parent_store);
 680
 681                 /* Similarly, if the UID of the highest message we
 682                  * know about has changed, then that indicates that
 683                  * messages have been both added and removed, so we
 684                  * have to rescan to find the removed ones. (We pass
 685                  * NULL for the folder since we know that this folder
 686                  * is selected, and we don't want camel_imap_command
 687                  * to worry about it.)
 688                  */
 689                 response = camel_imap_command (store, NULL, cancellable, error, "FETCH %d UID", count);
 690                 if (!response)
 691                         return FALSE;
 692                 uid = 0;
 693                 for (i = 0; i < response->untagged->len; i++) {
 694                         resp = response->untagged->pdata[i];
 695                         val = strtoul (resp + 2, &resp, 10);
 696                         if (val == 0)
 697                                 continue;
 698                         if (!g_ascii_strcasecmp (resp, " EXISTS")) {
 699                                 /* Another one?? */
 700                                 exists = val;
 701                                 continue;
 702                         }
 703                         if (uid != 0 || val != count || g_ascii_strncasecmp (resp, " FETCH (", 8) != 0)
 704                                 continue;
 705
 706                         fetch_data = parse_fetch_response (imap_folder, resp + 7);
 707                         uid = strtoul (g_datalist_get_data (&fetch_data, "UID"), NULL, 10);
 708                         g_datalist_clear (&fetch_data);
 709                 }
 710                 camel_imap_response_free_without_processing (store, response);
 711
 712                 known_uids = camel_folder_summary_get_array (folder->summary);
 713                 camel_folder_sort_uids (folder, known_uids);
 714                 old_uid = NULL;
 715                 if (known_uids && count - 1 >= 0 && count - 1 < known_uids->len)
 716                         old_uid = g_ptr_array_index (known_uids, count - 1);
 717                 if (old_uid) {
 718                         val = strtoul (old_uid, NULL, 10);
 719                         if (uid == 0 || uid != val)
 720                                 imap_folder->need_rescan = TRUE;
 721                 }
 722                 camel_folder_summary_free_array (known_uids);
 723         }
 724
 725         /* Now rescan if we need to */
 726         if (imap_folder->need_rescan)
 727                 return imap_rescan (folder, exists, cancellable, error);
 728
 729         /* If we don't need to rescan completely, but new messages
 730          * have been added, find out about them.
 731          */
 732         if (exists > count)
 733                 camel_imap_folder_changed (
 734                         folder, exists, NULL, cancellable, error);
 735
 736         /* And we're done. */
 737
 738         return TRUE;
 739 }
 740
 741 static gchar *
 742 imap_get_filename (CamelFolder *folder,
 743                    const gchar *uid,
 744                    GError **error)
 745 {
 746         CamelImapFolder *imap_folder = (CamelImapFolder *) folder;
 747
 748         return camel_imap_message_cache_get_filename (imap_folder->cache, uid, "", error);
 749 }
 750
 751 static void
 752 imap_rename (CamelFolder *folder,
 753              const gchar *new)
 754 {
 755         CamelService *service;
 756         CamelStore *parent_store;
 757         CamelImapFolder *imap_folder = (CamelImapFolder *) folder;
 758         const gchar *user_cache_dir;
 759         gchar *folder_dir, *state_file;
 760         gchar *folders;
 761
 762         parent_store = camel_folder_get_parent_store (folder);
 763
 764         service = CAMEL_SERVICE (parent_store);
 765         user_cache_dir = camel_service_get_user_cache_dir (service);
 766
 767         folders = g_build_filename (user_cache_dir, "folders", NULL);
 768         folder_dir = imap_path_to_physical (folders, new);
 769         g_free (folders);
 770
 771         CAMEL_IMAP_FOLDER_REC_LOCK (folder, cache_lock);
 772         camel_imap_message_cache_set_path (imap_folder->cache, folder_dir);
 773         CAMEL_IMAP_FOLDER_REC_UNLOCK (folder, cache_lock);
 774
 775         state_file = g_build_filename (folder_dir, "cmeta", NULL);
 776         camel_object_set_state_filename (CAMEL_OBJECT (folder), state_file);
 777         g_free (state_file);
 778
 779         g_free (folder_dir);
 780
 781         camel_store_summary_disconnect_folder_summary (
 782                 (CamelStoreSummary *) ((CamelImapStore *) parent_store)->summary,
 783                 folder->summary);
 784
 785         CAMEL_FOLDER_CLASS (camel_imap_folder_parent_class)->rename (folder, new);
 786
 787         camel_store_summary_connect_folder_summary (
 788                 (CamelStoreSummary *) ((CamelImapStore *) parent_store)->summary,
 789                 camel_folder_get_full_name (folder), folder->summary);
 790 }
 791
 792 /* called with connect_lock locked */
 793 static gboolean
 794 get_folder_status (CamelFolder *folder,
 795                    guint32 *total,
 796                    guint32 *unread,
 797                    GCancellable *cancellable,
 798                    GError **error)
 799 {
 800         CamelStore *parent_store;
 801         CamelImapStore *imap_store;
 802         CamelImapResponse *response;
 803         const gchar *full_name;
 804         gboolean res = FALSE;
 805
 806         g_return_val_if_fail (folder != NULL, FALSE);
 807
 808         full_name = camel_folder_get_full_name (folder);
 809         parent_store = camel_folder_get_parent_store (folder);
 810
 811         imap_store = CAMEL_IMAP_STORE (parent_store);
 812
 813         response = camel_imap_command (imap_store, folder, cancellable, error, "STATUS %F (MESSAGES UNSEEN)", full_name);
 814
 815         if (response) {
 816                 gint i;
 817
 818                 for (i = 0; i < response->untagged->len; i++) {
 819                         const gchar *resp = response->untagged->pdata[i];
 820
 821                         if (resp && g_str_has_prefix (resp, "* STATUS ")) {
 822                                 const gchar *p = NULL;
 823
 824                                 while (*resp) {
 825                                         if (*resp == '(')
 826                                                 p = resp;
 827                                         resp++;
 828                                 }
 829
 830                                 if (p && *(resp - 1) == ')') {
 831                                         const gchar *msgs = NULL, *unseen = NULL;
 832
 833                                         p++;
 834
 835                                         while (p && (!msgs || !unseen)) {
 836                                                 const gchar **dest = NULL;
 837
 838                                                 if (g_str_has_prefix (p, "MESSAGES "))
 839                                                         dest = &msgs;
 840                                                 else if (g_str_has_prefix (p, "UNSEEN "))
 841                                                         dest = &unseen;
 842
 843                                                 if (dest) {
 844                                                         *dest = imap_next_word (p);
 845
 846                                                         if (!*dest)
 847                                                                 break;
 848
 849                                                         p = imap_next_word (*dest);
 850                                                 } else {
 851                                                         p = imap_next_word (p);
 852                                                         if (p)
 853                                                                 p = imap_next_word (p);
 854                                                 }
 855                                         }
 856
 857                                         if (msgs && unseen) {
 858                                                 res = TRUE;
 859
 860                                                 if (total)
 861                                                         *total = strtoul (msgs, NULL, 10);
 862
 863                                                 if (unread)
 864                                                         *unread = strtoul (unseen, NULL, 10);
 865                                         }
 866                                 }
 867                         }
 868                 }
 869                 camel_imap_response_free (imap_store, response);
 870         }
 871
 872         return res;
 873 }
 874
 875 static gboolean
 876 imap_refresh_info_sync (CamelFolder *folder,
 877                         GCancellable *cancellable,
 878                         GError **error)
 879 {
 880         CamelStore *parent_store;
 881         CamelImapStore *imap_store;
 882         CamelImapFolder *imap_folder = CAMEL_IMAP_FOLDER (folder);
 883         CamelImapResponse *response;
 884         CamelStoreInfo *si;
 885         const gchar *full_name;
 886         gint check_rescan = -1;
 887         GError *local_error = NULL;
 888
 889         parent_store = camel_folder_get_parent_store (folder);
 890         imap_store = CAMEL_IMAP_STORE (parent_store);
 891
 892         if (!camel_offline_store_get_online (CAMEL_OFFLINE_STORE (imap_store)))
 893                 return TRUE;
 894
 895         if (camel_folder_is_frozen (folder)) {
 896                 imap_folder->need_refresh = TRUE;
 897                 return TRUE;
 898         }
 899
 900         /* If the folder isn't selected, select it (which will force
 901          * a rescan if one is needed).
 902          * Also, if this is the INBOX, some servers (cryus) wont tell
 903          * us with a NOOP of new messages, so force a reselect which
 904          * should do it.  */
 905         camel_service_lock (CAMEL_SERVICE (imap_store), CAMEL_SERVICE_REC_CONNECT_LOCK);
 906
 907         if (camel_application_is_exiting  || !camel_imap_store_connected (imap_store, &local_error))
 908                 goto done;
 909
 910         /* try to store local changes first, as the summary contains new local messages */
 911         replay_offline_journal (
 912                 imap_store, imap_folder, cancellable, &local_error);
 913
 914         full_name = camel_folder_get_full_name (folder);
 915
 916         if (imap_store->current_folder != folder
 917             || g_ascii_strcasecmp (full_name, "INBOX") == 0) {
 918                 response = camel_imap_command (imap_store, folder, cancellable, &local_error, NULL);
 919                 if (response) {
 920                         camel_imap_folder_selected (
 921                                 folder, response,
 922                                 cancellable, &local_error);
 923                         camel_imap_response_free (imap_store, response);
 924                 }
 925         } else if (imap_folder->need_rescan) {
 926                 /* Otherwise, if we need a rescan, do it, and if not, just do
 927                  * a NOOP to give the server a chance to tell us about new
 928                  * messages.
 929                  */
 930                 imap_rescan (
 931                         folder, camel_folder_summary_count (
 932                         folder->summary), cancellable, &local_error);
 933                 check_rescan = 0;
 934         } else {
 935 #if 0
 936                 /* on some servers need to CHECKpoint INBOX to recieve new messages?? */
 937                 /* rfc2060 suggests this, but havent seen a server that requires it */
 938                 if (g_ascii_strcasecmp (full_name, "INBOX") == 0) {
 939                         response = camel_imap_command (imap_store, folder, &local_error, "CHECK");
 940                         camel_imap_response_free (imap_store, response);
 941                 }
 942 #endif
 943                 response = camel_imap_command (imap_store, folder, cancellable, &local_error, "NOOP");
 944                 camel_imap_response_free (imap_store, response);
 945         }
 946
 947         si = camel_store_summary_path ((CamelStoreSummary *)((CamelImapStore *) parent_store)->summary, full_name);
 948         if (si) {
 949                 guint32 unread, total;
 950
 951                 total = camel_folder_summary_count (folder->summary);
 952                 unread = camel_folder_summary_get_unread_count (folder->summary);
 953
 954                 if (si->total != total
 955                     || si->unread != unread) {
 956                         si->total = total;
 957                         si->unread = unread;
 958                         camel_store_summary_touch ((CamelStoreSummary *)((CamelImapStore *) parent_store)->summary);
 959                         check_rescan = 0;
 960                 }
 961                 camel_store_summary_info_free ((CamelStoreSummary *)((CamelImapStore *) parent_store)->summary, si);
 962         }
 963
 964         if (check_rescan && !camel_application_is_exiting && local_error == NULL) {
 965                 if (check_rescan == -1) {
 966                         guint32 total, unread = 0, server_total = 0, server_unread = 0;
 967
 968                         check_rescan = 0;
 969
 970                         /* Check whether there are changes in total/unread messages in the folders
 971                          * and if so, then rescan whole summary */
 972                         if (get_folder_status (folder, &server_total, &server_unread, cancellable, &local_error)) {
 973
 974                                 total = camel_folder_summary_count (folder->summary);
 975                                 unread = camel_folder_summary_get_unread_count (folder->summary);
 976
 977                                 if (total != server_total || unread != server_unread)
 978                                         check_rescan = 1;
 979                         }
 980                 }
 981
 982                 if (check_rescan)
 983                         imap_rescan (
 984                                 folder, camel_folder_summary_count (
 985                                 folder->summary), cancellable, &local_error);
 986         }
 987 done:
 988         camel_service_unlock (CAMEL_SERVICE (imap_store), CAMEL_SERVICE_REC_CONNECT_LOCK);
 989
 990         camel_folder_summary_save_to_db (folder->summary, NULL);
 991         camel_store_summary_save ((CamelStoreSummary *)((CamelImapStore *) parent_store)->summary);
 992
 993         if (local_error != NULL) {
 994                 g_propagate_error (error, local_error);
 995                 return FALSE;
 996         }
 997
 998         return TRUE;
 999 }
1000
1001 static void
1002 fillup_custom_flags (CamelMessageInfo *mi,
1003                      gchar *custom_flags)
1004 {
1005         gchar **array_str;
1006         gint index = 0;
1007
1008         array_str = g_strsplit (custom_flags, " ", -1);
1009
1010         while (array_str[index] != NULL) {
1011                 camel_flag_set (&((CamelMessageInfoBase *) mi)->user_flags, array_str[index], TRUE);
1012                 ++ index;
1013         }
1014
1015         g_strfreev (array_str);
1016 }
1017
1018 /* This will merge custom flags with those in message info. Returns whether was some change. */
1019 static gboolean
1020 merge_custom_flags (CamelMessageInfo *mi,
1021                     const gchar *custom_flags)
1022 {
1023         GList *list, *p;
1024         GHashTable *server;
1025         gchar **cflags;
1026         gint i;
1027         const CamelFlag *flag;
1028         gboolean changed = FALSE;
1029
1030         g_return_val_if_fail (mi != NULL, FALSE);
1031
1032         if (!custom_flags)
1033                 custom_flags = "";
1034
1035         list = NULL;
1036         server = g_hash_table_new (g_str_hash, g_str_equal);
1037
1038         cflags = g_strsplit (custom_flags, " ", -1);
1039         for (i = 0; cflags[i]; i++) {
1040                 gchar *name = cflags[i];
1041
1042                 if (name && *name) {
1043                         g_hash_table_insert (server, name, name);
1044                         list = g_list_prepend (list, name);
1045                 }
1046         }
1047
1048         for (flag = camel_message_info_user_flags (mi); flag; flag = flag->next) {
1049                 gchar *name = (gchar *) flag->name;
1050
1051                 if (name && *name)
1052                         list = g_list_prepend (list, name);
1053         }
1054
1055         list = g_list_sort (list, (GCompareFunc) strcmp);
1056         for (p = list; p; p = p->next) {
1057                 if (p->next && strcmp (p->data, p->next->data) == 0) {
1058                         /* This flag is there twice, which means it was on the server and
1059                          * in our local summary too; thus skip these two elements. */
1060                         p = p->next;
1061                 } else {
1062                         /* If this value came from the server, then add it to our local summary,
1063                          * otherwise it was in local summary, but isn't on the server, thus remove it. */
1064                         changed = TRUE;
1065                         mi->dirty = TRUE;
1066                         if (mi->summary)
1067                                 camel_folder_summary_touch (mi->summary);
1068                         camel_flag_set (&((CamelMessageInfoBase *) mi)->user_flags, p->data, g_hash_table_lookup (server, p->data) != NULL);
1069                         ((CamelMessageInfoBase *) mi)->flags |= CAMEL_MESSAGE_FOLDER_FLAGGED;
1070                 }
1071         }
1072
1073         g_list_free (list);
1074         g_hash_table_destroy (server);
1075         g_strfreev (cflags);
1076
1077         return changed;
1078 }
1079
1080 /* Called with the store's connect_lock locked */
1081 static gboolean
1082 imap_rescan (CamelFolder *folder,
1083              gint exists,
1084              GCancellable *cancellable,
1085              GError **error)
1086 {
1087         CamelStore *parent_store;
1088         CamelImapFolder *imap_folder = CAMEL_IMAP_FOLDER (folder);
1089         CamelImapStore *store;
1090         struct {
1091                 gchar *uid;
1092                 guint32 flags;
1093                 gchar *custom_flags;
1094         } *new;
1095         gchar *resp, *uid;
1096         CamelImapResponseType type;
1097         gint i, j, seq, summary_got, del = 0;
1098         guint summary_len;
1099         CamelMessageInfo *info;
1100         CamelImapMessageInfo *iinfo;
1101         GArray *removed;
1102         gboolean ok;
1103         CamelFolderChangeInfo *changes = NULL;
1104         gboolean success;
1105         GPtrArray *known_uids;
1106
1107         parent_store = camel_folder_get_parent_store (folder);
1108         store = CAMEL_IMAP_STORE (parent_store);
1109
1110         if (camel_application_is_exiting)
1111                 return TRUE;
1112
1113         imap_folder->need_rescan = FALSE;
1114
1115         known_uids = camel_folder_summary_get_array (folder->summary);
1116         summary_len = known_uids ? known_uids->len : 0;
1117         if (summary_len == 0) {
1118                 camel_folder_summary_free_array (known_uids);
1119                 if (exists)
1120                         return camel_imap_folder_changed (
1121                                 folder, exists, NULL, cancellable, error);
1122                 return TRUE;
1123         }
1124
1125         /* Check UIDs and flags of all messages we already know of. */
1126         camel_operation_push_message (
1127                 cancellable, _("Scanning for changed messages in %s"),
1128                 camel_folder_get_display_name (folder));
1129
1130         camel_folder_sort_uids (folder, known_uids);
1131
1132         uid = g_ptr_array_index (known_uids, summary_len - 1);
1133         if (!uid) {
1134                 camel_operation_pop_message (cancellable);
1135                 camel_folder_summary_free_array (known_uids);
1136                 return TRUE;
1137         }
1138
1139         ok = camel_imap_command_start (
1140                 store, folder, cancellable, error,
1141                 "UID FETCH 1:%s (FLAGS)", uid);
1142         if (!ok) {
1143                 camel_operation_pop_message (cancellable);
1144                 camel_folder_summary_free_array (known_uids);
1145                 return FALSE;
1146         }
1147
1148         resp = NULL;
1149         new = g_malloc0 (summary_len * sizeof (*new));
1150         summary_got = 0;
1151         while ((type = camel_imap_command_response (store, folder, &resp, cancellable, error)) == CAMEL_IMAP_RESPONSE_UNTAGGED && !camel_application_is_exiting) {
1152                 GData *data;
1153                 gchar *uid;
1154                 guint32 flags;
1155
1156                 data = parse_fetch_response (imap_folder, resp);
1157                 g_free (resp);
1158                 resp = NULL;
1159                 if (!data)
1160                         continue;
1161
1162                 seq = GPOINTER_TO_INT (g_datalist_get_data (&data, "SEQUENCE"));
1163                 uid = g_datalist_get_data (&data, "UID");
1164                 flags = GPOINTER_TO_UINT (g_datalist_get_data (&data, "FLAGS"));
1165
1166                 if (!uid || !seq || seq > summary_len || seq < 0) {
1167                         g_datalist_clear (&data);
1168                         continue;
1169                 }
1170
1171                 camel_operation_progress (
1172                         cancellable, ++summary_got * 100 / summary_len);
1173
1174                 new[seq - 1].uid = g_strdup (uid);
1175                 new[seq - 1].flags = flags;
1176                 new[seq - 1].custom_flags = g_strdup (g_datalist_get_data (&data, "CUSTOM.FLAGS"));
1177                 g_datalist_clear (&data);
1178         }
1179
1180         if (summary_got == 0 && summary_len == 0) {
1181                 camel_operation_pop_message (cancellable);
1182                 camel_service_unlock (CAMEL_SERVICE (store), CAMEL_SERVICE_REC_CONNECT_LOCK);
1183                 g_free (new);
1184                 g_free (resp);
1185
1186                 camel_folder_summary_free_array (known_uids);
1187                 return TRUE;
1188         }
1189
1190         camel_operation_pop_message (cancellable);
1191
1192         if (type == CAMEL_IMAP_RESPONSE_ERROR || camel_application_is_exiting) {
1193                 for (i = 0; i < summary_len && new[i].uid; i++) {
1194                         g_free (new[i].uid);
1195                         g_free (new[i].custom_flags);
1196                 }
1197                 g_free (new);
1198                 g_free (resp);
1199
1200                 if (type != CAMEL_IMAP_RESPONSE_ERROR && type != CAMEL_IMAP_RESPONSE_TAGGED)
1201                         camel_service_unlock (CAMEL_SERVICE (store), CAMEL_SERVICE_REC_CONNECT_LOCK);
1202
1203                 camel_folder_summary_free_array (known_uids);
1204                 return TRUE;
1205         }
1206
1207         /* Free the final tagged response */
1208         g_free (resp);
1209
1210         /* If we find a UID in the summary that doesn't correspond to
1211          * the UID in the folder, then either: (a) it's a real UID,
1212          * but the message was deleted on the server, or (b) it's a
1213          * fake UID, and needs to be removed from the summary in order
1214          * to sync up with the server. So either way, we remove it
1215          * from the summary.
1216          */
1217         removed = g_array_new (FALSE, FALSE, sizeof (gint));
1218
1219         camel_folder_summary_prepare_fetch_all (folder->summary, NULL);
1220
1221         for (i = 0, j = 0; i < summary_len && new[j].uid; i++) {
1222                 gboolean changed = FALSE;
1223
1224                 uid = g_ptr_array_index (known_uids, i);
1225                 if (!uid)
1226                         continue;
1227
1228                 info = camel_folder_summary_get (folder->summary, uid);
1229                 if (!info) {
1230                         if (g_getenv("CRASH_IMAP")) { /* Debug logs to tackle on hard to get imap crasher */
1231                                 printf ("CRASH: %s: %s",
1232                                         camel_folder_get_full_name (folder), uid);
1233                                 g_assert (0);
1234                         } else
1235                                 continue;
1236                 }
1237
1238                 iinfo = (CamelImapMessageInfo *) info;
1239
1240                 if (strcmp (uid, new[j].uid) != 0) {
1241                         seq = i + 1 - del;
1242                         del++;
1243                         g_array_append_val (removed, seq);
1244                         camel_message_info_free (info);
1245                         continue;
1246                 }
1247
1248                 /* Update summary flags */
1249
1250                 if (new[j].flags != iinfo->server_flags) {
1251                         guint32 server_set, server_cleared;
1252
1253                         server_set = new[j].flags & ~iinfo->server_flags;
1254                         server_cleared = iinfo->server_flags & ~new[j].flags;
1255
1256                         camel_message_info_set_flags ((CamelMessageInfo *) iinfo, server_set | server_cleared, (iinfo->info.flags | server_set) & ~server_cleared);
1257                         iinfo->server_flags = new[j].flags;
1258                         /* unset folder_flagged, because these are flags received froma server */
1259                         iinfo->info.flags = iinfo->info.flags & (~CAMEL_MESSAGE_FOLDER_FLAGGED);
1260                         iinfo->info.dirty = TRUE;
1261                         if (info->summary)
1262                                 camel_folder_summary_touch (info->summary);
1263                         changed = TRUE;
1264                 }
1265
1266                 /* Do not merge custom flags when server doesn't support it.
1267                  * Because server always reports NULL, which means none, which
1268                  * will remove user's flags from local machine, which is bad.
1269                 */
1270                 if ((folder->permanent_flags & CAMEL_MESSAGE_USER) != 0 &&
1271                     (iinfo->info.flags & CAMEL_MESSAGE_FOLDER_FLAGGED) == 0 &&
1272                     merge_custom_flags (info, new[j].custom_flags))
1273                         changed = TRUE;
1274
1275                 if (changed) {
1276                         if (changes == NULL)
1277                                 changes = camel_folder_change_info_new ();
1278                         camel_folder_change_info_change_uid (changes, new[j].uid);
1279                 }
1280
1281                 camel_message_info_free (info);
1282                 g_free (new[j].uid);
1283                 g_free (new[j].custom_flags);
1284                 j++;
1285         }
1286
1287         if (changes) {
1288                 camel_folder_changed (folder, changes);
1289                 camel_folder_change_info_free (changes);
1290         }
1291
1292         seq = i + 1;
1293
1294 #if 0
1295         /* FIXME: Srini: I don't think this will be called any longer. */
1296         /* Free remaining memory. */
1297         while (i < summary_len && new[i].uid) {
1298                 g_free (new[i].uid);
1299                 g_free (new[i].custom_flags);
1300                 i++;
1301         }
1302 #endif
1303         g_free (new);
1304
1305         /* Remove any leftover cached summary messages. (Yes, we
1306          * repeatedly add the same number to the removed array.
1307          * See RFC2060 7.4.1)
1308          */
1309
1310         for (i = seq; i <= summary_len; i++) {
1311                 gint j;
1312
1313                 j = seq - del;
1314                 g_array_append_val (removed, j);
1315         }
1316
1317         /* And finally update the summary. */
1318         success = camel_imap_folder_changed (
1319                 folder, exists, removed, cancellable, error);
1320         g_array_free (removed, TRUE);
1321
1322         camel_folder_summary_free_array (known_uids);
1323         return success;
1324 }
1325
1326 static const gchar *
1327 get_message_uid (CamelFolder *folder,
1328                  CamelImapMessageInfo *info)
1329 {
1330         const gchar *uid;
1331
1332         g_return_val_if_fail (folder != NULL, NULL);
1333         g_return_val_if_fail (info != NULL, NULL);
1334
1335         uid = camel_message_info_uid (info);
1336         g_return_val_if_fail (uid != NULL, NULL);
1337
1338         if (!isdigit ((guchar) * uid)) {
1339                 uid = camel_imap_journal_uidmap_lookup ((CamelIMAPJournal *) CAMEL_IMAP_FOLDER (folder)->journal, uid);
1340                 g_return_val_if_fail (uid != NULL, NULL);
1341         }
1342
1343         return uid;
1344 }
1345
1346 /* the max number of chars that an unsigned 32-bit gint can be is 10 chars plus 1 for a possible : */
1347 #define UID_SET_FULL(setlen, maxlen) (maxlen > 0 ? setlen + 11 >= maxlen : FALSE)
1348
1349 /* Find all messages in @folder with flags matching @flags and @mask.
1350  * If no messages match, returns %NULL. Otherwise, returns an array of
1351  * CamelMessageInfo and sets *@set to a message set corresponding the
1352  * UIDs of the matched messages (up to @UID_SET_LIMIT bytes). The
1353  * caller must free the infos, the array, and the set string.
1354  */
1355 static GPtrArray *
1356 get_matching (CamelFolder *folder,
1357               guint32 flags,
1358               guint32 mask,
1359               CamelMessageInfo *master_info,
1360               gchar **set,
1361               GPtrArray *summary,
1362               GPtrArray *deleted_uids,
1363               GPtrArray *junked_uids)
1364 {
1365         GPtrArray *matches;
1366         CamelImapMessageInfo *info;
1367         gint i, max, range, last_range_uid;
1368         GString *gset;
1369         GList *list1 = NULL;
1370         gint count1 = 0;
1371         gchar *uid;
1372
1373         /* use the local rinfo in the close_range, because we want to keep our info untouched */
1374         #define close_range()                                                                           \
1375                 if (range != -1) {                                                                      \
1376                         if (range != i - 1) {                                                           \
1377                                 CamelImapMessageInfo *rinfo = matches->pdata[matches->len - 1];         \
1378                                                                                                         \
1379                                 g_string_append_printf (gset, ":%s", get_message_uid (folder, rinfo));  \
1380                         }                                                                               \
1381                         range = -1;                                                                     \
1382                         last_range_uid = -1;                                                            \
1383                 }
1384
1385         matches = g_ptr_array_new ();
1386         gset = g_string_new ("");
1387         max = summary->len;
1388         range = -1;
1389         last_range_uid = -1;
1390         for (i = 0; i < max && !UID_SET_FULL (gset->len, UID_SET_LIMIT); i++) {
1391                 gint uid_num;
1392                 uid = summary->pdata[i];
1393
1394                 if (uid) {
1395                         info = (CamelImapMessageInfo *) camel_folder_summary_get (folder->summary, uid);
1396                 } else
1397                         continue;
1398
1399                 if (!info)
1400                         continue;
1401
1402                 /* if the resulting flag list is empty, then "concat" other message
1403                  * only when server_flags are same, because there will be a flag removal
1404                  * command for this type of situation */
1405                 if ((info->info.flags & mask) != flags || (flags == 0 && info->server_flags != ((CamelImapMessageInfo *) master_info)->server_flags)) {
1406                         camel_message_info_free ((CamelMessageInfo *) info);
1407                         close_range ();
1408                         continue;
1409                 }
1410
1411                 uid_num = atoi (uid);
1412
1413                 /* we got only changes, thus the uid's can be mixed up, not the consecutive list,
1414                  * thus close range if we are not in it */
1415                 if (last_range_uid != -1 && uid_num != last_range_uid + 1) {
1416                         close_range ();
1417                 }
1418
1419                 /* only check user flags when we see other message than our 'master' */
1420                 if (strcmp (master_info->uid, ((CamelMessageInfo *) info)->uid)) {
1421                         const CamelFlag *flag;
1422                         GList *list2 = NULL, *l1, *l2;
1423                         gint count2 = 0, cmp = 0;
1424
1425                         if (!list1) {
1426                                 for (flag = camel_message_info_user_flags (master_info); flag; flag = flag->next) {
1427                                         if (*flag->name) {
1428                                                 count1++;
1429                                                 list1 = g_list_prepend (list1, (gchar *) flag->name);
1430                                         }
1431                                 }
1432
1433                                 list1 = g_list_sort (list1, (GCompareFunc) strcmp);
1434                         }
1435
1436                         for (flag = camel_message_info_user_flags (info); flag; flag = flag->next) {
1437                                 if (*flag->name) {
1438                                         count2++;
1439                                         list2 = g_list_prepend (list2, (gchar *) flag->name);
1440                                 }
1441                         }
1442
1443                         if (count1 != count2) {
1444                                 g_list_free (list2);
1445                                 close_range ();
1446                                 continue;
1447                         }
1448
1449                         list2 = g_list_sort (list2, (GCompareFunc) strcmp);
1450                         for (l1 = list1, l2 = list2; l1 && l2 && !cmp; l1 = l1->next, l2 = l2->next) {
1451                                 cmp = strcmp (l1->data, l2->data);
1452                         }
1453
1454                         if (cmp) {
1455                                 g_list_free (list2);
1456                                 close_range ();
1457                                 continue;
1458                         }
1459                 }
1460
1461                 if (deleted_uids && (info->info.flags & (CAMEL_MESSAGE_DELETED | CAMEL_IMAP_MESSAGE_MOVED)) == CAMEL_MESSAGE_DELETED) {
1462                         g_ptr_array_add (deleted_uids, (gpointer) camel_pstring_strdup (camel_message_info_uid (info)));
1463                         info->info.flags &= ~CAMEL_MESSAGE_DELETED;
1464                 } else if (junked_uids && (info->info.flags & CAMEL_MESSAGE_JUNK) != 0) {
1465                         g_ptr_array_add (junked_uids, (gpointer) camel_pstring_strdup (camel_message_info_uid (info)));
1466                 }
1467
1468                 g_ptr_array_add (matches, info);
1469                 /* Remove the uid from the list, to optimize*/
1470                 camel_pstring_free (summary->pdata[i]);
1471                 summary->pdata[i] = NULL;
1472
1473                 if (range != -1) {
1474                         last_range_uid = uid_num;
1475                         continue;
1476                 }
1477
1478                 range = i;
1479                 last_range_uid = uid_num;
1480                 if (gset->len)
1481                         g_string_append_c (gset, ',');
1482                 g_string_append_printf (gset, "%s", get_message_uid (folder, info));
1483         }
1484
1485         if (range != -1 && range != max - 1) {
1486                 info = matches->pdata[matches->len - 1];
1487                 g_string_append_printf (gset, ":%s", get_message_uid (folder, info));
1488         }
1489
1490         if (list1)
1491                 g_list_free (list1);
1492
1493         if (matches->len) {
1494                 *set = gset->str;
1495                 g_string_free (gset, FALSE);
1496                 return matches;
1497         } else {
1498                 *set = NULL;
1499                 g_string_free (gset, TRUE);
1500                 g_ptr_array_free (matches, TRUE);
1501                 return NULL;
1502         }
1503
1504         #undef close_range
1505 }
1506
1507 static gboolean
1508 imap_sync_offline (CamelFolder *folder,
1509                    GError **error)
1510 {
1511         CamelStore *parent_store;
1512
1513         parent_store = camel_folder_get_parent_store (folder);
1514
1515         if (folder->summary && (folder->summary->flags & CAMEL_SUMMARY_DIRTY) != 0) {
1516                 CamelStoreInfo *si;
1517                 const gchar *full_name;
1518
1519                 /* ... and store's summary when folder's summary is dirty */
1520                 full_name = camel_folder_get_full_name (folder);
1521                 si = camel_store_summary_path ((CamelStoreSummary *)((CamelImapStore *) parent_store)->summary, full_name);
1522                 if (si) {
1523                         if (si->total != camel_folder_summary_get_saved_count (folder->summary) || si->unread != camel_folder_summary_get_unread_count (folder->summary)) {
1524                                 si->total = camel_folder_summary_get_saved_count (folder->summary);
1525                                 si->unread = camel_folder_summary_get_unread_count (folder->summary);
1526                                 camel_store_summary_touch ((CamelStoreSummary *)((CamelImapStore *) parent_store)->summary);
1527                         }
1528
1529                         camel_store_summary_info_free ((CamelStoreSummary *)((CamelImapStore *) parent_store)->summary, si);
1530                 }
1531         }
1532
1533         camel_folder_summary_save_to_db (folder->summary, NULL);
1534         camel_store_summary_save ((CamelStoreSummary *)((CamelImapStore *) parent_store)->summary);
1535
1536         return TRUE;
1537 }
1538
1539 static gboolean
1540 host_ends_with (const gchar *host,
1541                 const gchar *ends)
1542 {
1543         gint host_len, ends_len;
1544
1545         g_return_val_if_fail (host != NULL, FALSE);
1546         g_return_val_if_fail (ends != NULL, FALSE);
1547
1548         host_len = strlen (host);
1549         ends_len = strlen (ends);
1550
1551         return ends_len <= host_len && g_ascii_strcasecmp (host + host_len - ends_len, ends) == 0;
1552 }
1553
1554 static gboolean
1555 is_google_account (CamelStore *store)
1556 {
1557         CamelNetworkSettings *network_settings;
1558         CamelSettings *settings;
1559         CamelService *service;
1560         gboolean is_google;
1561         gchar *host;
1562
1563         g_return_val_if_fail (store != NULL, FALSE);
1564         g_return_val_if_fail (CAMEL_IS_STORE (store), FALSE);
1565
1566         service = CAMEL_SERVICE (store);
1567         settings = camel_service_get_settings (service);
1568
1569         network_settings = CAMEL_NETWORK_SETTINGS (settings);
1570         host = camel_network_settings_dup_host (network_settings);
1571
1572         is_google =
1573                 (host != NULL) && (
1574                 host_ends_with (host, "gmail.com") ||
1575                 host_ends_with (host, "googlemail.com"));
1576
1577         g_free (host);
1578
1579         return is_google;
1580 }
1581
1582 static void
1583 move_messages (CamelFolder *src_folder,
1584                GPtrArray *uids,
1585                CamelFolder *des_folder,
1586                GCancellable *cancellable,
1587                GError **error)
1588 {
1589         g_return_if_fail (src_folder != NULL);
1590
1591         /* it's OK to have these NULL */
1592         if (!uids || uids->len == 0 || des_folder == NULL)
1593                 return;
1594
1595         /* moving to the same folder means expunge only */
1596         if (src_folder != des_folder) {
1597                 /* do 'copy' to not be bothered with CAMEL_MESSAGE_DELETED again */
1598                 if (!imap_transfer_messages (
1599                         src_folder, uids, des_folder, FALSE,
1600                         NULL, FALSE, cancellable, error))
1601                         return;
1602         }
1603
1604         camel_imap_expunge_uids_only (src_folder, uids, cancellable, error);
1605 }
1606
1607 static gboolean
1608 imap_synchronize_sync (CamelFolder *folder,
1609                        gboolean expunge,
1610                        GCancellable *cancellable,
1611                        GError **error)
1612 {
1613         CamelService *service;
1614         CamelSettings *settings;
1615         CamelStore *parent_store;
1616         CamelImapStore *store;
1617         CamelImapMessageInfo *info;
1618         CamelImapFolder *imap_folder = CAMEL_IMAP_FOLDER (folder);
1619         gboolean success, is_gmail;
1620         CamelFolder *real_junk = NULL;
1621         CamelFolder *real_trash = NULL;
1622         gchar *folder_path;
1623         GError *local_error = NULL;
1624
1625         GPtrArray *matches, *summary, *deleted_uids = NULL, *junked_uids = NULL;
1626         gchar *set, *flaglist, *uid;
1627         gint i, j, max;
1628
1629         parent_store = camel_folder_get_parent_store (folder);
1630         store = CAMEL_IMAP_STORE (parent_store);
1631         is_gmail = is_google_account (parent_store);
1632
1633         service = CAMEL_SERVICE (parent_store);
1634         settings = camel_service_get_settings (service);
1635
1636         if (folder->permanent_flags == 0 || !camel_offline_store_get_online (CAMEL_OFFLINE_STORE (store))) {
1637                 if (expunge) {
1638                         if (!imap_expunge_sync (folder, cancellable, error))
1639                                 return FALSE;
1640                 }
1641                 return imap_sync_offline (folder, error);
1642         }
1643
1644         camel_service_lock (service, CAMEL_SERVICE_REC_CONNECT_LOCK);
1645
1646         /* write local changes first */
1647         replay_offline_journal (store, imap_folder, cancellable, NULL);
1648
1649         /* Find a message with changed flags, find all of the other
1650          * messages like it, sync them as a group, mark them as
1651          * updated, and continue.
1652          */
1653         summary = camel_folder_summary_get_changed (folder->summary); /* These should be in memory anyways */
1654         camel_folder_sort_uids (folder, summary);
1655         max = summary->len;
1656
1657         /* deleted_uids is NULL when not using real trash */
1658         folder_path = camel_imap_settings_dup_real_trash_path (
1659                 CAMEL_IMAP_SETTINGS (settings));
1660         if (folder_path != NULL) {
1661                 if ((folder->folder_flags & CAMEL_FOLDER_IS_TRASH) != 0) {
1662                         /* syncing the trash, expunge deleted when found any */
1663                         real_trash = g_object_ref (folder);
1664                 } else {
1665                         real_trash = camel_store_get_trash_folder_sync (
1666                                 parent_store, cancellable, NULL);
1667
1668                         if (folder_path == NULL && real_trash) {
1669                                 /* failed to open real trash */
1670                                 g_object_unref (real_trash);
1671                                 real_trash = NULL;
1672                         }
1673                 }
1674         }
1675         g_free (folder_path);
1676
1677         if (real_trash)
1678                 deleted_uids = g_ptr_array_new ();
1679
1680         /* junked_uids is NULL when not using real junk */
1681         folder_path = camel_imap_settings_dup_real_junk_path (
1682                 CAMEL_IMAP_SETTINGS (settings));
1683         if (folder_path != NULL) {
1684                 if ((folder->folder_flags & CAMEL_FOLDER_IS_JUNK) != 0) {
1685                         /* syncing the junk, but cannot move
1686                          * messages to itself, thus do nothing */
1687                         real_junk = NULL;
1688                 } else {
1689                         real_junk = camel_store_get_junk_folder_sync (
1690                                 parent_store, cancellable, NULL);
1691
1692                         if (folder_path == NULL && real_junk) {
1693                                 /* failed to open real junk */
1694                                 g_object_unref (real_junk);
1695                                 real_junk = NULL;
1696                         }
1697                 }
1698         }
1699         g_free (folder_path);
1700
1701         if (real_junk)
1702                 junked_uids = g_ptr_array_new ();
1703
1704         for (i = 0; i < max; i++) {
1705                 gboolean unset = FALSE;
1706                 CamelImapResponse *response = NULL;
1707
1708                 uid = summary->pdata[i];
1709
1710                 if (!uid) /* Possibly it was sync by matching flags, which we NULLify */
1711                         continue;
1712
1713                 if (!(info = (CamelImapMessageInfo *) camel_folder_summary_get (folder->summary, uid))) {
1714                         continue;
1715                 }
1716
1717                 if (!(info->info.flags & CAMEL_MESSAGE_FOLDER_FLAGGED)) {
1718                         camel_message_info_free ((CamelMessageInfo *) info);
1719                         continue;
1720                 }
1721
1722                 /* Note: get_matching() uses UID_SET_LIMIT to limit
1723                  * the size of the uid-set string. We don't have to
1724                  * loop here to flush all the matching uids because
1725                  * they will be scooped up later by our parent loop (I
1726                  * think?). -- Jeff */
1727                 matches = get_matching (folder, info->info.flags & (folder->permanent_flags | CAMEL_MESSAGE_FOLDER_FLAGGED),
1728                                         folder->permanent_flags | CAMEL_MESSAGE_FOLDER_FLAGGED, (CamelMessageInfo *) info, &set, summary,
1729                                         deleted_uids, junked_uids);
1730                 if (matches == NULL) {
1731                         camel_message_info_free (info);
1732                         continue;
1733                 }
1734
1735                 /* Make sure we're connected before issuing commands */
1736                 if (!camel_imap_store_connected (store, NULL)) {
1737                         g_free (set);
1738                         camel_message_info_free (info);
1739                         break;
1740                 }
1741
1742                 if (deleted_uids && !is_gmail && (info->info.flags & CAMEL_MESSAGE_DELETED) != 0) {
1743                         /* there is a real trash, do not set it on the server */
1744                         info->info.flags &= ~CAMEL_MESSAGE_DELETED;
1745                 }
1746
1747                 flaglist = imap_create_flag_list (info->info.flags & folder->permanent_flags, (CamelMessageInfo *) info, folder->permanent_flags);
1748
1749                 if (strcmp (flaglist, "()") == 0) {
1750                         /* Note: Cyrus is broken and will not accept an
1751                          * empty-set of flags so... if this is true then we
1752                          * set and unset \Seen flag. It's necessary because
1753                          * we do not know the previously set user flags. */
1754                         unset = TRUE;
1755                         g_free (flaglist);
1756
1757                         /* unset all known server flags, because there left none in the actual flags */
1758                         flaglist =  imap_create_flag_list (info->server_flags & folder->permanent_flags, (CamelMessageInfo *) info, folder->permanent_flags);
1759
1760                         if (strcmp (flaglist, "()") == 0) {
1761                                 /* this should not happen, really */
1762                                 g_free (flaglist);
1763                                 flaglist = strdup ("(\\Seen)");
1764
1765                                 response = camel_imap_command (store, folder, cancellable, &local_error,
1766                                                 "UID STORE %s +FLAGS.SILENT %s",
1767                                                 set, flaglist);
1768                                 if (response)
1769                                         camel_imap_response_free (store, response);
1770
1771                                 response = NULL;
1772                         }
1773                 }
1774
1775                 /* We don't use the info any more */
1776                 camel_message_info_free (info);
1777
1778                 /* Note: to 'unset' flags, use -FLAGS.SILENT (<flag list>) */
1779                 if (local_error == NULL) {
1780                         response = camel_imap_command (store, folder, cancellable, &local_error,
1781                                                "UID STORE %s %sFLAGS.SILENT %s",
1782                                                set, unset ? "-" : "", flaglist);
1783                 }
1784
1785                 g_free (set);
1786                 g_free (flaglist);
1787
1788                 if (response)
1789                         camel_imap_response_free (store, response);
1790
1791                 if (local_error == NULL) {
1792                         for (j = 0; j < matches->len; j++) {
1793                                 info = matches->pdata[j];
1794                                 if (deleted_uids && !is_gmail) {
1795                                         /* there is a real trash, do not keep this set */
1796                                         info->info.flags &= ~CAMEL_MESSAGE_DELETED;
1797                                 }
1798
1799                                 info->info.flags &= ~(CAMEL_MESSAGE_FOLDER_FLAGGED | CAMEL_IMAP_MESSAGE_MOVED);
1800                                 ((CamelImapMessageInfo *) info)->server_flags = info->info.flags & CAMEL_IMAP_SERVER_FLAGS;
1801                                 info->info.dirty = TRUE; /* Sync it back to the DB */
1802                                 if (((CamelMessageInfo *) info)->summary)
1803                                         camel_folder_summary_touch (((CamelMessageInfo *) info)->summary);
1804                         }
1805                         camel_folder_summary_touch (folder->summary);
1806                 }
1807
1808                 for (j = 0; j < matches->len; j++) {
1809                         info = matches->pdata[j];
1810                         camel_message_info_free (&info->info);
1811                 }
1812                 g_ptr_array_free (matches, TRUE);
1813
1814                 /* We unlock here so that other threads can have a chance to grab the connect_lock */
1815                 camel_service_unlock (service, CAMEL_SERVICE_REC_CONNECT_LOCK);
1816
1817                 /* check for an exception */
1818                 if (local_error != NULL) {
1819                         g_propagate_error (error, local_error);
1820                         if (deleted_uids) {
1821                                 g_ptr_array_foreach (deleted_uids, (GFunc) camel_pstring_free, NULL);
1822                                 g_ptr_array_free (deleted_uids, TRUE);
1823                         }
1824                         if (junked_uids) {
1825                                 g_ptr_array_foreach (junked_uids, (GFunc) camel_pstring_free, NULL);
1826                                 g_ptr_array_free (junked_uids, TRUE);
1827                         }
1828                         if (real_trash)
1829                                 g_object_unref (real_trash);
1830                         if (real_junk)
1831                                 g_object_unref (real_junk);
1832                         return FALSE;
1833                 }
1834
1835                 /* Re-lock the connect_lock */
1836                 camel_service_lock (service, CAMEL_SERVICE_REC_CONNECT_LOCK);
1837         }
1838
1839         if (local_error == NULL)
1840                 move_messages (
1841                         folder, deleted_uids, real_trash,
1842                         cancellable, &local_error);
1843         if (local_error == NULL)
1844                 move_messages (
1845                         folder, junked_uids, real_junk,
1846                         cancellable, &local_error);
1847
1848         if (deleted_uids) {
1849                 g_ptr_array_foreach (deleted_uids, (GFunc) camel_pstring_free, NULL);
1850                 g_ptr_array_free (deleted_uids, TRUE);
1851         }
1852         if (junked_uids) {
1853                 g_ptr_array_foreach (junked_uids, (GFunc) camel_pstring_free, NULL);
1854                 g_ptr_array_free (junked_uids, TRUE);
1855         }
1856         if (real_trash)
1857                 g_object_unref (real_trash);
1858         if (real_junk)
1859                 g_object_unref (real_junk);
1860
1861         if (expunge && local_error == NULL)
1862                 imap_expunge_sync (folder, cancellable, &local_error);
1863
1864         if (local_error != NULL)
1865                 g_propagate_error (error, local_error);
1866
1867         g_ptr_array_foreach (summary, (GFunc) camel_pstring_free, NULL);
1868         g_ptr_array_free (summary, TRUE);
1869
1870         /* Save the summary */
1871         success = imap_sync_offline (folder, error);
1872
1873         camel_service_unlock (service, CAMEL_SERVICE_REC_CONNECT_LOCK);
1874
1875         return success;
1876 }
1877
1878 static gint
1879 uid_compar (gconstpointer va,
1880             gconstpointer vb)
1881 {
1882         const gchar **sa = (const gchar **) va, **sb = (const gchar **) vb;
1883         gulong a, b;
1884
1885         a = strtoul (*sa, NULL, 10);
1886         b = strtoul (*sb, NULL, 10);
1887         if (a < b)
1888                 return -1;
1889         else if (a == b)
1890                 return 0;
1891         else
1892                 return 1;
1893 }
1894
1895 static gboolean
1896 imap_expunge_uids_offline (CamelFolder *folder,
1897                            GPtrArray *uids,
1898                            GCancellable *cancellable,
1899                            GError **error)
1900 {
1901         CamelFolderChangeInfo *changes;
1902         CamelStore *parent_store;
1903         GList *list = NULL;
1904         const gchar *full_name;
1905         gint i;
1906
1907         full_name = camel_folder_get_full_name (folder);
1908         parent_store = camel_folder_get_parent_store (folder);
1909
1910         qsort (uids->pdata, uids->len, sizeof (gpointer), uid_compar);
1911
1912         changes = camel_folder_change_info_new ();
1913
1914         for (i = 0; i < uids->len; i++) {
1915                 camel_folder_summary_remove_uid (folder->summary, uids->pdata[i]);
1916                 camel_folder_change_info_remove_uid (changes, uids->pdata[i]);
1917                 list = g_list_prepend (list, (gpointer) uids->pdata[i]);
1918                 /* We intentionally don't remove it from the cache because
1919                  * the cached data may be useful in replaying a COPY later.
1920                  */
1921         }
1922
1923         camel_db_delete_uids (parent_store->cdb_w, full_name, list, NULL);
1924         g_list_free (list);
1925         camel_folder_summary_save_to_db (folder->summary, NULL);
1926
1927         camel_imap_journal_log (CAMEL_IMAP_FOLDER (folder)->journal,
1928                                CAMEL_IMAP_JOURNAL_ENTRY_EXPUNGE, uids);
1929
1930         camel_folder_changed (folder, changes);
1931         camel_folder_change_info_free (changes);
1932
1933         return TRUE;
1934 }
1935
1936 static gboolean
1937 imap_expunge_uids_online (CamelFolder *folder,
1938                           GPtrArray *uids,
1939                           GCancellable *cancellable,
1940                           GError **error)
1941 {
1942         CamelImapStore *store;
1943         CamelImapResponse *response;
1944         gint uid = 0;
1945         gchar *set;
1946         gboolean full_expunge;
1947         CamelFolderChangeInfo *changes;
1948         CamelStore *parent_store;
1949         const gchar *full_name;
1950         gint i;
1951         GList *list = NULL;
1952
1953         full_name = camel_folder_get_full_name (folder);
1954         parent_store = camel_folder_get_parent_store (folder);
1955
1956         store = CAMEL_IMAP_STORE (parent_store);
1957         full_expunge = (store->capabilities & IMAP_CAPABILITY_UIDPLUS) == 0;
1958
1959         camel_service_lock (CAMEL_SERVICE (store), CAMEL_SERVICE_REC_CONNECT_LOCK);
1960
1961         if ((store->capabilities & IMAP_CAPABILITY_UIDPLUS) == 0) {
1962                 if (!CAMEL_FOLDER_GET_CLASS (folder)->synchronize_sync (
1963                         folder, 0, cancellable, error)) {
1964                         camel_service_unlock (
1965                                 CAMEL_SERVICE (store),
1966                                 CAMEL_SERVICE_REC_CONNECT_LOCK);
1967                         return FALSE;
1968                 }
1969         }
1970
1971         qsort (uids->pdata, uids->len, sizeof (gpointer), uid_compar);
1972
1973         while (uid < uids->len) {
1974                 set = imap_uid_array_to_set (folder->summary, uids, uid, UID_SET_LIMIT, &uid);
1975                 response = camel_imap_command (store, folder, cancellable, error,
1976                                                "UID STORE %s +FLAGS.SILENT (\\Deleted)",
1977                                                set);
1978                 if (response)
1979                         camel_imap_response_free (store, response);
1980                 else {
1981                         camel_service_unlock (CAMEL_SERVICE (store), CAMEL_SERVICE_REC_CONNECT_LOCK);
1982                         g_free (set);
1983                         return FALSE;
1984                 }
1985
1986                 if (!full_expunge) {
1987                         GError *local_error = NULL;
1988
1989                         response = camel_imap_command (
1990                                 store, folder, cancellable, &local_error,
1991                                 "UID EXPUNGE %s", set);
1992
1993                         if (local_error != NULL) {
1994                                 g_clear_error (&local_error);
1995
1996                                 /* UID EXPUNGE failed, something is broken on the server probably,
1997                                  * thus fall back to the full expunge. It's not so good, especially
1998                                  * when resyncing, it will remove already marked messages on the
1999                                  * server too. I guess that's fine anyway, isn't it?
2000                                  * For failed command see Gnome's bug #536486 */
2001                                 full_expunge = TRUE;
2002                         }
2003                 }
2004
2005                 if (full_expunge)
2006                         response = camel_imap_command (store, folder, cancellable, NULL, "EXPUNGE");
2007
2008                 if (response)
2009                         camel_imap_response_free (store, response);
2010
2011                 g_free (set);
2012         }
2013
2014         camel_service_unlock (CAMEL_SERVICE (store), CAMEL_SERVICE_REC_CONNECT_LOCK);
2015
2016         changes = camel_folder_change_info_new ();
2017         for (i = 0; i < uids->len; i++) {
2018                 camel_folder_summary_remove_uid (folder->summary, uids->pdata[i]);
2019                 camel_folder_change_info_remove_uid (changes, uids->pdata[i]);
2020                 list = g_list_prepend (list, (gpointer) uids->pdata[i]);
2021                 /* We intentionally don't remove it from the cache because
2022                  * the cached data may be useful in replaying a COPY later.
2023                  */
2024         }
2025
2026         camel_db_delete_uids (parent_store->cdb_w, full_name, list, NULL);
2027         g_list_free (list);
2028         camel_folder_summary_save_to_db (folder->summary, NULL);
2029         camel_folder_changed (folder, changes);
2030         camel_folder_change_info_free (changes);
2031
2032         return TRUE;
2033 }
2034
2035 static gboolean
2036 imap_expunge_sync (CamelFolder *folder,
2037                    GCancellable *cancellable,
2038                    GError **error)
2039 {
2040         CamelStore *parent_store;
2041         GPtrArray *uids = NULL;
2042         const gchar *full_name;
2043         gboolean success, real_trash = FALSE;
2044
2045         full_name = camel_folder_get_full_name (folder);
2046         parent_store = camel_folder_get_parent_store (folder);
2047
2048         camel_folder_summary_save_to_db (folder->summary, NULL);
2049
2050         if ((parent_store->flags & CAMEL_STORE_VTRASH) == 0) {
2051                 CamelFolder *trash;
2052                 GError *local_error = NULL;
2053
2054                 trash = camel_store_get_trash_folder_sync (
2055                         parent_store, cancellable, &local_error);
2056
2057                 if (local_error == NULL && trash && (folder == trash || g_ascii_strcasecmp (full_name, camel_folder_get_full_name (trash)) == 0)) {
2058                         /* it's a real trash folder, thus get all mails from there */
2059                         real_trash = TRUE;
2060                         uids = camel_folder_summary_get_array (folder->summary);
2061                 }
2062
2063                 if (local_error != NULL)
2064                         g_clear_error (&local_error);
2065         }
2066
2067         if (!uids)
2068                 uids = camel_db_get_folder_deleted_uids (parent_store->cdb_r, full_name, NULL);
2069
2070         if (!uids)
2071                 return TRUE;
2072
2073         if (camel_offline_store_get_online (CAMEL_OFFLINE_STORE (parent_store)))
2074                 success = imap_expunge_uids_online (
2075                         folder, uids, cancellable, error);
2076         else
2077                 success = imap_expunge_uids_offline (
2078                         folder, uids, cancellable, error);
2079
2080         if (real_trash) {
2081                 camel_folder_summary_free_array (uids);
2082         } else {
2083                 g_ptr_array_foreach (uids, (GFunc) camel_pstring_free, NULL);
2084                 g_ptr_array_free (uids, TRUE);
2085         }
2086
2087         return success;
2088 }
2089
2090 gboolean
2091 camel_imap_expunge_uids_resyncing (CamelFolder *folder,
2092                                    GPtrArray *uids,
2093                                    GCancellable *cancellable,
2094                                    GError **error)
2095 {
2096         CamelStore *parent_store;
2097         CamelImapFolder *imap_folder = CAMEL_IMAP_FOLDER (folder);
2098         CamelImapStore *store;
2099         GPtrArray *keep_uids, *mark_uids;
2100         CamelImapResponse *response;
2101         gchar *result;
2102
2103         parent_store = camel_folder_get_parent_store (folder);
2104         store = CAMEL_IMAP_STORE (parent_store);
2105
2106         if (imap_folder->read_only)
2107                 return TRUE;
2108
2109         if (store->capabilities & IMAP_CAPABILITY_UIDPLUS)
2110                 return imap_expunge_uids_online (
2111                         folder, uids, cancellable, error);
2112
2113         /* If we don't have UID EXPUNGE we need to avoid expunging any
2114          * of the wrong messages. So we search for deleted messages,
2115          * and any that aren't in our to-expunge list get temporarily
2116          * marked un-deleted.
2117          */
2118
2119         camel_service_lock (CAMEL_SERVICE (store), CAMEL_SERVICE_REC_CONNECT_LOCK);
2120
2121         if (!CAMEL_FOLDER_GET_CLASS (folder)->synchronize_sync (
2122                 folder, 0, cancellable, error)) {
2123                 camel_service_unlock (
2124                         CAMEL_SERVICE (store),
2125                         CAMEL_SERVICE_REC_CONNECT_LOCK);
2126                 return FALSE;
2127         }
2128
2129         response = camel_imap_command (store, folder, cancellable, error, "UID SEARCH DELETED");
2130         if (!response) {
2131                 camel_service_unlock (CAMEL_SERVICE (store), CAMEL_SERVICE_REC_CONNECT_LOCK);
2132                 return FALSE;
2133         }
2134         result = camel_imap_response_extract (store, response, "SEARCH", error);
2135         if (!result) {
2136                 camel_service_unlock (CAMEL_SERVICE (store), CAMEL_SERVICE_REC_CONNECT_LOCK);
2137                 return FALSE;
2138         }
2139
2140         if (result[8] == ' ') {
2141                 gchar *uid, *lasts = NULL;
2142                 gulong euid, kuid;
2143                 gint ei, ki;
2144
2145                 keep_uids = g_ptr_array_new ();
2146                 mark_uids = g_ptr_array_new ();
2147
2148                 /* Parse SEARCH response */
2149                 for (uid = strtok_r (result + 9, " ", &lasts); uid; uid = strtok_r (NULL, " ", &lasts))
2150                         g_ptr_array_add (keep_uids, uid);
2151                 qsort (keep_uids->pdata, keep_uids->len,
2152                        sizeof (gpointer), uid_compar);
2153
2154                 /* Fill in "mark_uids", empty out "keep_uids" as needed */
2155                 for (ei = ki = 0; ei < uids->len; ei++) {
2156                         euid = strtoul (uids->pdata[ei], NULL, 10);
2157
2158                         for (kuid = 0; ki < keep_uids->len; ki++) {
2159                                 kuid = strtoul (keep_uids->pdata[ki], NULL, 10);
2160
2161                                 if (kuid >= euid)
2162                                         break;
2163                         }
2164
2165                         if (euid == kuid)
2166                                 g_ptr_array_remove_index (keep_uids, ki);
2167                         else
2168                                 g_ptr_array_add (mark_uids, uids->pdata[ei]);
2169                 }
2170         } else {
2171                 /* Empty SEARCH result, meaning nothing is marked deleted
2172                  * on server.
2173                  */
2174
2175                 keep_uids = NULL;
2176                 mark_uids = uids;
2177         }
2178
2179         /* Unmark messages to be kept */
2180
2181         if (keep_uids) {
2182                 gchar *uidset;
2183                 gint uid = 0;
2184
2185                 while (uid < keep_uids->len) {
2186                         uidset = imap_uid_array_to_set (folder->summary, keep_uids, uid, UID_SET_LIMIT, &uid);
2187
2188                         response = camel_imap_command (store, folder, cancellable, error,
2189                                                        "UID STORE %s -FLAGS.SILENT (\\Deleted)",
2190                                                        uidset);
2191
2192                         g_free (uidset);
2193
2194                         if (!response) {
2195                                 g_ptr_array_free (keep_uids, TRUE);
2196                                 g_ptr_array_free (mark_uids, TRUE);
2197                                 camel_service_unlock (CAMEL_SERVICE (store), CAMEL_SERVICE_REC_CONNECT_LOCK);
2198                                 return FALSE;
2199                         }
2200                         camel_imap_response_free (store, response);
2201                 }
2202         }
2203
2204         /* Mark any messages that still need to be marked */
2205         if (mark_uids) {
2206                 gchar *uidset;
2207                 gint uid = 0;
2208
2209                 while (uid < mark_uids->len) {
2210                         uidset = imap_uid_array_to_set (folder->summary, mark_uids, uid, UID_SET_LIMIT, &uid);
2211
2212                         response = camel_imap_command (store, folder, cancellable, error,
2213                                                        "UID STORE %s +FLAGS.SILENT (\\Deleted)",
2214                                                        uidset);
2215
2216                         g_free (uidset);
2217
2218                         if (!response) {
2219                                 g_ptr_array_free (keep_uids, TRUE);
2220                                 g_ptr_array_free (mark_uids, TRUE);
2221                                 camel_service_unlock (CAMEL_SERVICE (store), CAMEL_SERVICE_REC_CONNECT_LOCK);
2222                                 return FALSE;
2223                         }
2224                         camel_imap_response_free (store, response);
2225                 }
2226
2227                 if (mark_uids != uids)
2228                         g_ptr_array_free (mark_uids, TRUE);
2229         }
2230
2231         /* Do the actual expunging */
2232         response = camel_imap_command (store, folder, cancellable, NULL, "EXPUNGE");
2233         if (response)
2234                 camel_imap_response_free (store, response);
2235
2236         /* And fix the remaining messages if we mangled them */
2237         if (keep_uids) {
2238                 gchar *uidset;
2239                 gint uid = 0;
2240
2241                 while (uid < keep_uids->len) {
2242                         uidset = imap_uid_array_to_set (folder->summary, keep_uids, uid, UID_SET_LIMIT, &uid);
2243
2244                         response = camel_imap_command (store, folder, cancellable, NULL,
2245                                                        "UID STORE %s +FLAGS.SILENT (\\Deleted)",
2246                                                        uidset);
2247
2248                         g_free (uidset);
2249                         if (response)
2250                                 camel_imap_response_free (store, response);
2251                 }
2252
2253                 g_ptr_array_free (keep_uids, TRUE);
2254         }
2255
2256         /* now we can free this, now that we're done with keep_uids */
2257         g_free (result);
2258
2259         camel_service_unlock (CAMEL_SERVICE (store), CAMEL_SERVICE_REC_CONNECT_LOCK);
2260
2261         return TRUE;
2262 }
2263
2264 gboolean
2265 camel_imap_expunge_uids_only (CamelFolder *folder,
2266                               GPtrArray *uids,
2267                               GCancellable *cancellable,
2268                               GError **error)
2269 {
2270         CamelStore *parent_store;
2271
2272         g_return_val_if_fail (folder != NULL, FALSE);
2273
2274         parent_store = camel_folder_get_parent_store (folder);
2275         g_return_val_if_fail (parent_store != NULL, FALSE);
2276
2277         g_return_val_if_fail (uids != NULL, FALSE);
2278
2279         if (camel_offline_store_get_online (CAMEL_OFFLINE_STORE (parent_store)))
2280                 return camel_imap_expunge_uids_resyncing (
2281                         folder, uids, cancellable, error);
2282         else
2283                 return imap_expunge_uids_offline (
2284                         folder, uids, cancellable, error);
2285 }
2286
2287 static gchar *
2288 get_temp_uid (void)
2289 {
2290         gchar *res;
2291
2292         static gint counter = 0;
2293         G_LOCK_DEFINE_STATIC (lock);
2294
2295         G_LOCK (lock);
2296         res = g_strdup_printf ("tempuid-%lx-%d",
2297                                (gulong) time (NULL),
2298                                counter++);
2299         G_UNLOCK (lock);
2300
2301         return res;
2302 }
2303
2304 static gboolean
2305 imap_append_offline (CamelFolder *folder,
2306                      CamelMimeMessage *message,
2307                      CamelMessageInfo *info,
2308                      gchar **appended_uid,
2309                      GError **error)
2310 {
2311         CamelImapMessageCache *cache = CAMEL_IMAP_FOLDER (folder)->cache;
2312         CamelFolderChangeInfo *changes;
2313         gchar *uid;
2314
2315         uid = get_temp_uid ();
2316
2317         camel_imap_summary_add_offline (
2318                 folder->summary, uid, message, info);
2319         CAMEL_IMAP_FOLDER_REC_LOCK (folder, cache_lock);
2320         camel_imap_message_cache_insert_wrapper (
2321                 cache, uid, "", CAMEL_DATA_WRAPPER (message));
2322         CAMEL_IMAP_FOLDER_REC_UNLOCK (folder, cache_lock);
2323
2324         changes = camel_folder_change_info_new ();
2325         camel_folder_change_info_add_uid (changes, uid);
2326         camel_folder_changed (folder, changes);
2327         camel_folder_change_info_free (changes);
2328
2329         camel_imap_journal_log (CAMEL_IMAP_FOLDER (folder)->journal,
2330                                CAMEL_IMAP_JOURNAL_ENTRY_APPEND, uid);
2331         if (appended_uid)
2332                 *appended_uid = uid;
2333         else
2334                 g_free (uid);
2335
2336         return TRUE;
2337 }
2338
2339 static void
2340 imap_folder_add_ignore_recent (CamelImapFolder *imap_folder,
2341                                const gchar *uid)
2342 {
2343         g_return_if_fail (imap_folder != NULL);
2344         g_return_if_fail (uid != NULL);
2345
2346         if (!imap_folder->priv->ignore_recent)
2347                 imap_folder->priv->ignore_recent = g_hash_table_new_full (g_str_hash, g_str_equal, (GDestroyNotify) g_free, NULL);
2348
2349         g_hash_table_insert (imap_folder->priv->ignore_recent, g_strdup (uid), GINT_TO_POINTER (1));
2350 }
2351
2352 static gboolean
2353 imap_folder_uid_in_ignore_recent (CamelImapFolder *imap_folder,
2354                                   const gchar *uid)
2355 {
2356         g_return_val_if_fail (imap_folder != NULL, FALSE);
2357         g_return_val_if_fail (uid != NULL, FALSE);
2358
2359         return imap_folder->priv->ignore_recent && g_hash_table_lookup (imap_folder->priv->ignore_recent, uid);
2360 }
2361
2362 static CamelImapResponse *
2363 do_append (CamelFolder *folder,
2364            CamelMimeMessage *message,
2365            CamelMessageInfo *info,
2366            gchar **uid,
2367            GCancellable *cancellable,
2368            GError **error)
2369 {
2370         CamelStore *parent_store;
2371         CamelImapStore *store;
2372         CamelImapResponse *response, *response2;
2373         CamelStream *memstream;
2374         CamelMimeFilter *crlf_filter;
2375         CamelStream *streamfilter;
2376         GByteArray *ba;
2377         const gchar *full_name;
2378         gchar *flagstr, *end;
2379         guint32 flags = 0;
2380         GError *local_error = NULL;
2381
2382         parent_store = camel_folder_get_parent_store (folder);
2383         store = CAMEL_IMAP_STORE (parent_store);
2384
2385         /* encode any 8bit parts so we avoid sending embedded nul-chars and such  */
2386         camel_mime_message_encode_8bit_parts (message);
2387
2388         /* FIXME: We could avoid this if we knew how big the message was. */
2389         memstream = camel_stream_mem_new ();
2390         ba = g_byte_array_new ();
2391         camel_stream_mem_set_byte_array (CAMEL_STREAM_MEM (memstream), ba);
2392
2393         streamfilter = camel_stream_filter_new (memstream);
2394         crlf_filter = camel_mime_filter_crlf_new (
2395                 CAMEL_MIME_FILTER_CRLF_ENCODE,
2396                 CAMEL_MIME_FILTER_CRLF_MODE_CRLF_ONLY);
2397         camel_stream_filter_add (
2398                 CAMEL_STREAM_FILTER (streamfilter), crlf_filter);
2399         camel_data_wrapper_write_to_stream_sync (
2400                 CAMEL_DATA_WRAPPER (message), streamfilter, cancellable, NULL);
2401         g_object_unref (streamfilter);
2402         g_object_unref (crlf_filter);
2403         g_object_unref (memstream);
2404
2405         /* Some servers don't let us append with (CamelMessageInfo *)custom flags.  If the command fails for
2406          * whatever reason, assume this is the case and save the state and try again */
2407 retry:
2408         if (info) {
2409                 flags = camel_message_info_flags (info);
2410         }
2411
2412         flags &= folder->permanent_flags;
2413         if (flags)
2414                 flagstr = imap_create_flag_list (flags, (CamelMessageInfo *) info, folder->permanent_flags);
2415         else
2416                 flagstr = NULL;
2417
2418         full_name = camel_folder_get_full_name (folder);
2419         response = camel_imap_command (
2420                 store, NULL, cancellable, &local_error, "APPEND %F%s%s {%d}",
2421                 full_name, flagstr ? " " : "",
2422                 flagstr ? flagstr : "", ba->len);
2423         g_free (flagstr);
2424
2425         if (!response) {
2426                 if (g_error_matches (local_error, CAMEL_SERVICE_ERROR, CAMEL_SERVICE_ERROR_INVALID) && !store->nocustomappend) {
2427                         g_clear_error (&local_error);
2428                         store->nocustomappend = 1;
2429                         goto retry;
2430                 }
2431                 g_propagate_error (error, local_error);
2432                 g_byte_array_free (ba, TRUE);
2433                 return NULL;
2434         }
2435
2436         if (*response->status != '+') {
2437                 camel_imap_response_free (store, response);
2438                 g_byte_array_free (ba, TRUE);
2439                 return NULL;
2440         }
2441
2442         /* send the rest of our data - the mime message */
2443         response2 = camel_imap_command_continuation (store, folder, (const gchar *) ba->data, ba->len, cancellable, error);
2444         g_byte_array_free (ba, TRUE);
2445
2446         /* free it only after message is sent. This may cause more FETCHes. */
2447         camel_imap_response_free (store, response);
2448         if (!response2)
2449                 return response2;
2450
2451         if ((store->capabilities & IMAP_CAPABILITY_UIDPLUS) != 0 ||
2452             is_google_account (parent_store)) {
2453                 *uid = camel_strstrcase (response2->status, "[APPENDUID ");
2454                 if (*uid)
2455                         *uid = strchr (*uid + 11, ' ');
2456                 if (*uid) {
2457                         *uid = g_strndup (*uid + 1, strcspn (*uid + 1, "]"));
2458                         /* Make sure it's a number */
2459                         if (strtoul (*uid, &end, 10) == 0 || *end) {
2460                                 g_free (*uid);
2461                                 *uid = NULL;
2462                         }
2463                 }
2464         } else
2465                 *uid = NULL;
2466
2467         if (*uid)
2468                 imap_folder_add_ignore_recent (CAMEL_IMAP_FOLDER (folder), *uid);
2469
2470         return response2;
2471 }
2472
2473 static gboolean
2474 imap_append_online (CamelFolder *folder,
2475                     CamelMimeMessage *message,
2476                     CamelMessageInfo *info,
2477                     gchar **appended_uid,
2478                     GCancellable *cancellable,
2479                     GError **error)
2480 {
2481         CamelStore *parent_store;
2482         CamelImapStore *store;
2483         CamelImapResponse *response;
2484         gboolean success = TRUE;
2485         gchar *uid;
2486         gint count;
2487
2488         parent_store = camel_folder_get_parent_store (folder);
2489         store = CAMEL_IMAP_STORE (parent_store);
2490
2491         if (!camel_offline_store_get_online (CAMEL_OFFLINE_STORE (store))) {
2492                 return imap_append_offline (
2493                         folder, message, info, appended_uid, error);
2494         }
2495
2496         count = camel_folder_summary_count (folder->summary);
2497         response = do_append (folder, message, info, &uid, cancellable, error);
2498         if (!response)
2499                 return FALSE;
2500
2501         if (uid) {
2502                 /* Cache first, since freeing response may trigger a
2503                  * summary update that will want this information.
2504                  */
2505                 CAMEL_IMAP_FOLDER_REC_LOCK (folder, cache_lock);
2506                 camel_imap_message_cache_insert_wrapper (
2507                         CAMEL_IMAP_FOLDER (folder)->cache, uid,
2508                         "", CAMEL_DATA_WRAPPER (message));
2509                 CAMEL_IMAP_FOLDER_REC_UNLOCK (folder, cache_lock);
2510                 if (appended_uid)
2511                         *appended_uid = uid;
2512                 else
2513                         g_free (uid);
2514         } else if (appended_uid)
2515                 *appended_uid = NULL;
2516
2517         camel_imap_response_free (store, response);
2518
2519         /* Make sure a "folder_changed" is emitted. */
2520         camel_service_lock (CAMEL_SERVICE (store), CAMEL_SERVICE_REC_CONNECT_LOCK);
2521         if (store->current_folder != folder ||
2522             camel_folder_summary_count (folder->summary) == count)
2523                 success = imap_refresh_info_sync (folder, cancellable, error);
2524         camel_service_unlock (CAMEL_SERVICE (store), CAMEL_SERVICE_REC_CONNECT_LOCK);
2525
2526         return success;
2527 }
2528
2529 gboolean
2530 camel_imap_append_resyncing (CamelFolder *folder,
2531                              CamelMimeMessage *message,
2532                              CamelMessageInfo *info,
2533                              gchar **appended_uid,
2534                              GCancellable *cancellable,
2535                              GError **error)
2536 {
2537         CamelStore *parent_store;
2538         CamelImapStore *store;
2539         CamelImapResponse *response;
2540         gchar *uid;
2541
2542         parent_store = camel_folder_get_parent_store (folder);
2543         store = CAMEL_IMAP_STORE (parent_store);
2544
2545         response = do_append (folder, message, info, &uid, cancellable, error);
2546         if (!response)
2547                 return FALSE;
2548
2549         if (uid) {
2550                 CamelImapFolder *imap_folder = CAMEL_IMAP_FOLDER (folder);
2551                 const gchar *olduid = camel_message_info_uid (info);
2552
2553                 CAMEL_IMAP_FOLDER_REC_LOCK (imap_folder, cache_lock);
2554                 camel_imap_message_cache_copy (imap_folder->cache, olduid,
2555                                                imap_folder->cache, uid);
2556                 CAMEL_IMAP_FOLDER_REC_UNLOCK (imap_folder, cache_lock);
2557
2558                 if (appended_uid)
2559                         *appended_uid = uid;
2560                 else
2561                         g_free (uid);
2562         } else if (appended_uid)
2563                 *appended_uid = NULL;
2564
2565         camel_imap_response_free (store, response);
2566
2567         return TRUE;
2568 }
2569
2570 static gboolean
2571 imap_transfer_offline (CamelFolder *source,
2572                        GPtrArray *uids,
2573                        CamelFolder *dest,
2574                        gboolean delete_originals,
2575                        GPtrArray **transferred_uids,
2576                        GCancellable *cancellable,
2577                        GError **error)
2578 {
2579         CamelStore *parent_store;
2580         CamelImapStore *store;
2581         CamelImapMessageCache *sc = CAMEL_IMAP_FOLDER (source)->cache;
2582         CamelImapMessageCache *dc = CAMEL_IMAP_FOLDER (dest)->cache;
2583         CamelFolderChangeInfo *changes;
2584         CamelMimeMessage *message;
2585         CamelMessageInfo *mi;
2586         gchar *uid, *destuid;
2587         gint i;
2588         GError *local_error = NULL;
2589
2590         parent_store = camel_folder_get_parent_store (source);
2591         store = CAMEL_IMAP_STORE (parent_store);
2592
2593         /* We grab the store's command lock first, and then grab the
2594          * source and destination cache_locks. This way we can't
2595          * deadlock in the case where we're simultaneously also trying
2596          * to copy messages in the other direction from another thread.
2597          */
2598         camel_service_lock (CAMEL_SERVICE (store), CAMEL_SERVICE_REC_CONNECT_LOCK);
2599         CAMEL_IMAP_FOLDER_REC_LOCK (source, cache_lock);
2600         CAMEL_IMAP_FOLDER_REC_LOCK (dest, cache_lock);
2601         camel_service_unlock (CAMEL_SERVICE (store), CAMEL_SERVICE_REC_CONNECT_LOCK);
2602
2603         if (transferred_uids) {
2604                 *transferred_uids = g_ptr_array_new ();
2605                 g_ptr_array_set_size (*transferred_uids, uids->len);
2606         }
2607
2608         changes = camel_folder_change_info_new ();
2609
2610         for (i = 0; i < uids->len && local_error == NULL; i++) {
2611                 uid = uids->pdata[i];
2612
2613                 destuid = get_temp_uid ();
2614
2615                 mi = camel_folder_summary_get (source->summary, uid);
2616                 g_return_val_if_fail (mi != NULL, FALSE);
2617
2618                 message = camel_folder_get_message_sync (
2619                         source, uid, cancellable, &local_error);
2620
2621                 if (message) {
2622                         camel_imap_summary_add_offline (
2623                                 dest->summary, destuid, message, mi);
2624                         g_object_unref (message);
2625                 } else
2626                         camel_imap_summary_add_offline_uncached (
2627                                 dest->summary, destuid, mi);
2628
2629                 camel_imap_message_cache_copy (sc, uid, dc, destuid);
2630                 camel_message_info_free (mi);
2631
2632                 camel_folder_change_info_add_uid (changes, destuid);
2633                 if (transferred_uids)
2634                         (*transferred_uids)->pdata[i] = destuid;
2635                 else
2636                         g_free (destuid);
2637
2638                 if (delete_originals && local_error == NULL)
2639                         camel_folder_delete_message (source, uid);
2640         }
2641
2642         CAMEL_IMAP_FOLDER_REC_UNLOCK (dest, cache_lock);
2643         CAMEL_IMAP_FOLDER_REC_UNLOCK (source, cache_lock);
2644
2645         camel_folder_changed (dest, changes);
2646         camel_folder_change_info_free (changes);
2647
2648         camel_imap_journal_log (
2649                 CAMEL_IMAP_FOLDER (source)->journal,
2650                 CAMEL_IMAP_JOURNAL_ENTRY_TRANSFER,
2651                 dest, uids, delete_originals, NULL);
2652
2653         if (local_error != NULL) {
2654                 g_propagate_error (error, local_error);
2655                 return FALSE;
2656         }
2657
2658         return TRUE;
2659 }
2660
2661 static void
2662 handle_copyuid (CamelImapResponse *response,
2663                 CamelFolder *source,
2664                 CamelFolder *destination)
2665 {
2666         CamelImapMessageCache *scache = CAMEL_IMAP_FOLDER (source)->cache;
2667         CamelImapMessageCache *dcache = CAMEL_IMAP_FOLDER (destination)->cache;
2668         gchar *validity, *srcset, *destset;
2669         GPtrArray *src, *dest;
2670         gint i;
2671
2672         validity = camel_strstrcase (response->status, "[COPYUID ");
2673         if (!validity)
2674                 return;
2675         validity += 9;
2676         if (strtoul (validity, NULL, 10) !=
2677             CAMEL_IMAP_SUMMARY (destination->summary)->validity)
2678                 return;
2679
2680         srcset = strchr (validity, ' ');
2681         if (!srcset++)
2682                 goto lose;
2683         destset = strchr (srcset, ' ');
2684         if (!destset++)
2685                 goto lose;
2686
2687         src = imap_uid_set_to_array (source->summary, srcset);
2688         dest = imap_uid_set_to_array (destination->summary, destset);
2689
2690         if (src && dest && src->len == dest->len) {
2691                 /* We don't have to worry about deadlocking on the
2692                  * cache locks here, because we've got the store's
2693                  * command lock too, so no one else could be here.
2694                  */
2695                 CAMEL_IMAP_FOLDER_REC_LOCK (source, cache_lock);
2696                 CAMEL_IMAP_FOLDER_REC_LOCK (destination, cache_lock);
2697                 for (i = 0; i < src->len; i++) {
2698                         camel_imap_message_cache_copy (scache, src->pdata[i],
2699                                                        dcache, dest->pdata[i]);
2700
2701                         imap_folder_add_ignore_recent (CAMEL_IMAP_FOLDER (destination), dest->pdata[i]);
2702                 }
2703                 CAMEL_IMAP_FOLDER_REC_UNLOCK (source, cache_lock);
2704                 CAMEL_IMAP_FOLDER_REC_UNLOCK (destination, cache_lock);
2705
2706                 imap_uid_array_free (src);
2707                 imap_uid_array_free (dest);
2708                 return;
2709         }
2710
2711         if (src)
2712                 imap_uid_array_free (src);
2713         if (dest)
2714                 imap_uid_array_free (dest);
2715  lose:
2716         g_warning ("Bad COPYUID response from server");
2717 }
2718
2719 static void
2720 handle_copyuid_copy_user_tags (CamelImapResponse *response,
2721                                CamelFolder *source,
2722                                CamelFolder *destination,
2723                                GCancellable *cancellable)
2724 {
2725         CamelStore *parent_store;
2726         gchar *validity, *srcset, *destset;
2727         GPtrArray *src, *dest;
2728         gint i;
2729
2730         validity = camel_strstrcase (response->status, "[COPYUID ");
2731         if (!validity)
2732                 return;
2733         validity += 9;
2734         if (strtoul (validity, NULL, 10) !=
2735             CAMEL_IMAP_SUMMARY (destination->summary)->validity)
2736                 return;
2737
2738         srcset = strchr (validity, ' ');
2739         if (!srcset++)
2740                 goto lose;
2741         destset = strchr (srcset, ' ');
2742         if (!destset++)
2743                 goto lose;
2744
2745         /* first do NOOP on the destination folder, so server has enough time to propagate our copy command there */
2746         parent_store = camel_folder_get_parent_store (destination);
2747         camel_imap_response_free (
2748                 CAMEL_IMAP_STORE (parent_store), camel_imap_command (
2749                 CAMEL_IMAP_STORE (parent_store), destination, cancellable, NULL, "NOOP"));
2750
2751         /* refresh folder's summary first, we copied messages there on the server,
2752          * but do not know about it in a local summary */
2753         if (!imap_refresh_info_sync (destination, cancellable, NULL))
2754                 goto lose;
2755
2756         src = imap_uid_set_to_array (source->summary, srcset);
2757         dest = imap_uid_set_to_array (destination->summary, destset);
2758
2759         if (src && dest && src->len == dest->len) {
2760                 /* We don't have to worry about deadlocking on the
2761                  * cache locks here, because we've got the store's
2762                  * command lock too, so no one else could be here.
2763                  */
2764                 CAMEL_IMAP_FOLDER_REC_LOCK (source, cache_lock);
2765                 CAMEL_IMAP_FOLDER_REC_LOCK (destination, cache_lock);
2766                 for (i = 0; i < src->len; i++) {
2767                         CamelMessageInfo *mi = camel_folder_get_message_info (source, src->pdata[i]);
2768
2769                         if (mi) {
2770                                 const CamelTag *tag = camel_message_info_user_tags (mi);
2771
2772                                 while (tag) {
2773                                         camel_folder_set_message_user_tag (destination, dest->pdata[i], tag->name, tag->value);
2774                                         tag = tag->next;
2775                                 }
2776
2777                                 camel_folder_free_message_info (source, mi);
2778                         }
2779                 }
2780                 CAMEL_IMAP_FOLDER_REC_UNLOCK (source, cache_lock);
2781                 CAMEL_IMAP_FOLDER_REC_UNLOCK (destination, cache_lock);
2782
2783                 imap_uid_array_free (src);
2784                 imap_uid_array_free (dest);
2785                 return;
2786         }
2787
2788         if (src)
2789                 imap_uid_array_free (src);
2790         if (dest)
2791                 imap_uid_array_free (dest);
2792  lose:
2793         g_warning ("Bad COPYUID response from server");
2794 }
2795
2796 /* returns whether any of messages from uidset has set any user tag or not */
2797 static gboolean
2798 any_has_user_tag (CamelFolder *source,
2799                   gchar *uidset)
2800 {
2801         GPtrArray *src;
2802
2803         g_return_val_if_fail (source != NULL && uidset != NULL, FALSE);
2804
2805         src = imap_uid_set_to_array (source->summary, uidset);
2806         if (src) {
2807                 gboolean have = FALSE;
2808                 gint i;
2809
2810                 CAMEL_IMAP_FOLDER_REC_LOCK (source, cache_lock);
2811                 for (i = 0; i < src->len && !have; i++) {
2812                         CamelMessageInfo *mi = camel_folder_get_message_info (source, src->pdata[i]);
2813
2814                         if (mi) {
2815                                 have = camel_message_info_user_tags (mi) != NULL;
2816
2817                                 camel_folder_free_message_info (source, mi);
2818                         }
2819                 }
2820                 CAMEL_IMAP_FOLDER_REC_UNLOCK (source, cache_lock);
2821
2822                 imap_uid_array_free (src);
2823
2824                 return have;
2825         }
2826
2827         return FALSE;
2828 }
2829
2830 static gboolean
2831 do_copy (CamelFolder *source,
2832          GPtrArray *uids,
2833          CamelFolder *destination,
2834          gint delete_originals,
2835          GCancellable *cancellable,
2836          GError **error)
2837 {
2838         CamelService *service;
2839         CamelSettings *settings;
2840         CamelStore *parent_store;
2841         CamelImapStore *store;
2842         CamelImapResponse *response;
2843         const gchar *full_name;
2844         gchar *trash_path;
2845         gchar *uidset;
2846         gint uid = 0, last = 0, i;
2847         GError *local_error = NULL;
2848         gboolean mark_moved;
2849         gboolean success = TRUE;
2850
2851         parent_store = camel_folder_get_parent_store (source);
2852         store = CAMEL_IMAP_STORE (parent_store);
2853
2854         service = CAMEL_SERVICE (parent_store);
2855         settings = camel_service_get_settings (service);
2856
2857         trash_path = camel_imap_settings_dup_real_trash_path (
2858                 CAMEL_IMAP_SETTINGS (settings));
2859
2860         mark_moved = is_google_account (parent_store) && trash_path != NULL;
2861
2862         full_name = camel_folder_get_full_name (destination);
2863
2864         while (uid < uids->len && local_error == NULL) {
2865                 uidset = imap_uid_array_to_set (source->summary, uids, uid, UID_SET_LIMIT, &uid);
2866
2867                 /* use XGWMOVE only when none of the moving messages has set any user tag */
2868                 if ((store->capabilities & IMAP_CAPABILITY_XGWMOVE) != 0 && delete_originals && !any_has_user_tag (source, uidset)) {
2869                         camel_service_lock (CAMEL_SERVICE (store), CAMEL_SERVICE_REC_CONNECT_LOCK);
2870                         response = camel_imap_command (
2871                                 store, source, cancellable, &local_error,
2872                                 "UID XGWMOVE %s %F", uidset, full_name);
2873                         /* returns only 'A00012 OK UID XGWMOVE completed' '* 2 XGWMOVE' so nothing useful */
2874                         camel_imap_response_free (store, response);
2875                         camel_service_unlock (CAMEL_SERVICE (store), CAMEL_SERVICE_REC_CONNECT_LOCK);
2876                 } else {
2877                         camel_service_lock (CAMEL_SERVICE (store), CAMEL_SERVICE_REC_CONNECT_LOCK);
2878                         response = camel_imap_command (
2879                                 store, source, cancellable, &local_error,
2880                                 "UID COPY %s %F", uidset, full_name);
2881                         if (response && (store->capabilities & IMAP_CAPABILITY_UIDPLUS))
2882                                 handle_copyuid (response, source, destination);
2883                         if (response)
2884                                 handle_copyuid_copy_user_tags (
2885                                         response, source, destination,
2886                                         cancellable);
2887                         camel_imap_response_free (store, response);
2888                         camel_service_unlock (CAMEL_SERVICE (store), CAMEL_SERVICE_REC_CONNECT_LOCK);
2889                 }
2890
2891                 if (local_error == NULL && delete_originals && (mark_moved || !trash_path)) {
2892                         for (i = last; i < uid; i++) {
2893                                 camel_folder_delete_message (
2894                                         source, uids->pdata[i]);
2895                                 if (mark_moved) {
2896                                         CamelMessageInfoBase *info = (CamelMessageInfoBase *) camel_folder_summary_get (source->summary, uids->pdata[i]);
2897
2898                                         if (info)
2899                                                 info->flags |= CAMEL_IMAP_MESSAGE_MOVED;
2900                                 }
2901                         }
2902                         last = uid;
2903                 }
2904                 g_free (uidset);
2905         }
2906
2907         if (local_error != NULL) {
2908                 g_propagate_error (error, local_error);
2909                 success = FALSE;
2910
2911         /* There is a real trash folder set, which is not on a google account
2912          * and copied messages should be deleted, thus do not move them into
2913          * a trash folder, but just expunge them, because the copy part of
2914          * the operation was successful. */
2915         } else if (trash_path && !mark_moved && delete_originals)
2916                 camel_imap_expunge_uids_only (source, uids, cancellable, NULL);
2917
2918         g_free (trash_path);
2919
2920         return success;
2921 }
2922
2923 static gboolean
2924 imap_transfer_messages (CamelFolder *source,
2925                         GPtrArray *uids,
2926                         CamelFolder *dest,
2927                         gboolean delete_originals,
2928                         GPtrArray **transferred_uids,
2929                         gboolean can_call_sync,
2930                         GCancellable *cancellable,
2931                         GError **error)
2932 {
2933         CamelStore *parent_store;
2934         CamelImapStore *store;
2935         gboolean success = TRUE;
2936         gint count;
2937
2938         parent_store = camel_folder_get_parent_store (source);
2939         store = CAMEL_IMAP_STORE (parent_store);
2940
2941         if (!camel_offline_store_get_online (CAMEL_OFFLINE_STORE (store)))
2942                 return imap_transfer_offline (
2943                         source, uids, dest, delete_originals,
2944                         transferred_uids, cancellable, error);
2945
2946         /* Sync message flags if needed. */
2947         if (can_call_sync && !imap_synchronize_sync (
2948                 source, FALSE, cancellable, error))
2949                 return FALSE;
2950
2951         count = camel_folder_summary_count (dest->summary);
2952
2953         qsort (uids->pdata, uids->len, sizeof (gpointer), uid_compar);
2954
2955         /* Now copy the messages */
2956         if (!do_copy (source, uids, dest, delete_originals, cancellable, error))
2957                 return FALSE;
2958
2959         /* Make the destination notice its new messages */
2960         if (store->current_folder != dest ||
2961             camel_folder_summary_count (dest->summary) == count)
2962                 success = imap_refresh_info_sync (dest, cancellable, error);
2963
2964         /* FIXME */
2965         if (transferred_uids)
2966                 *transferred_uids = NULL;
2967
2968         return success;
2969 }
2970
2971 static gboolean
2972 imap_transfer_online (CamelFolder *source,
2973                       GPtrArray *uids,
2974                       CamelFolder *dest,
2975                       gboolean delete_originals,
2976                       GPtrArray **transferred_uids,
2977                       GCancellable *cancellable,
2978                       GError **error)
2979 {
2980         return imap_transfer_messages (
2981                 source, uids, dest, delete_originals,
2982                 transferred_uids, TRUE, cancellable, error);
2983 }
2984
2985 gboolean
2986 camel_imap_transfer_resyncing (CamelFolder *source,
2987                                GPtrArray *uids,
2988                                CamelFolder *dest,
2989                                gboolean delete_originals,
2990                                GPtrArray **transferred_uids,
2991                                GCancellable *cancellable,
2992                                GError **error)
2993 {
2994         GPtrArray *realuids;
2995         gint first, i;
2996         const gchar *uid;
2997         CamelMimeMessage *message;
2998         CamelMessageInfo *info;
2999         GError *local_error = NULL;
3000
3001         qsort (uids->pdata, uids->len, sizeof (gpointer), uid_compar);
3002
3003         /*This is trickier than append_resyncing, because some of
3004          * the messages we are copying may have been copied or
3005          * appended into @source while we were offline, in which case
3006          * if we don't have UIDPLUS, we won't know their real UIDs,
3007          * so we'll have to append them rather than copying. */
3008
3009         realuids = g_ptr_array_new ();
3010
3011         i = 0;
3012         while (i < uids->len && local_error == NULL) {
3013                  /* Skip past real UIDs  */
3014                 for (first = i; i < uids->len; i++) {
3015                         uid = uids->pdata[i];
3016
3017                         if (!isdigit ((guchar) * uid)) {
3018                                 uid = camel_imap_journal_uidmap_lookup ((CamelIMAPJournal *) CAMEL_IMAP_FOLDER (source)->journal, uid);
3019                                 if (!uid)
3020                                         break;
3021                         }
3022                         g_ptr_array_add (realuids, (gchar *) uid);
3023                 }
3024
3025                 /* If we saw any real UIDs, do a COPY */
3026                 if (i != first) {
3027                         do_copy (
3028                                 source, realuids, dest, delete_originals,
3029                                 cancellable, &local_error);
3030                         g_ptr_array_set_size (realuids, 0);
3031                         if (i == uids->len || local_error != NULL)
3032                                 break;
3033                 }
3034
3035                 /* Deal with fake UIDs */
3036                 while (i < uids->len &&
3037                        !isdigit (*(guchar *)(uids->pdata[i])) &&
3038                        local_error == NULL) {
3039                         uid = uids->pdata[i];
3040                         message = camel_folder_get_message_sync (
3041                                 source, uid, cancellable, NULL);
3042                         if (!message) {
3043                                 /* Message must have been expunged */
3044                                 i++;
3045                                 continue;
3046                         }
3047                         info = camel_folder_get_message_info (source, uid);
3048                         g_return_val_if_fail (info != NULL, FALSE);
3049
3050                         imap_append_online (
3051                                 dest, message, info,
3052                                 NULL, cancellable, &local_error);
3053                         camel_folder_free_message_info (source, info);
3054                         g_object_unref (message);
3055                         if (delete_originals && local_error == NULL)
3056                                 camel_folder_delete_message (source, uid);
3057                         i++;
3058                 }
3059         }
3060
3061         g_ptr_array_free (realuids, FALSE);
3062
3063         /* FIXME */
3064         if (transferred_uids)
3065                 *transferred_uids = NULL;
3066
3067         if (local_error != NULL) {
3068                 g_propagate_error (error, local_error);
3069                 return FALSE;
3070         }
3071
3072         return TRUE;
3073 }
3074
3075 static GPtrArray *
3076 imap_search_by_expression (CamelFolder *folder,
3077                            const gchar *expression,
3078                            GCancellable *cancellable,
3079                            GError **error)
3080 {
3081         CamelImapFolder *imap_folder = CAMEL_IMAP_FOLDER (folder);
3082         GPtrArray *matches;
3083
3084         /* we could get around this by creating a new search object each time,
3085          * but i doubt its worth it since any long operation would lock the
3086          * command channel too */
3087         CAMEL_IMAP_FOLDER_LOCK (folder, search_lock);
3088
3089         camel_folder_search_set_folder (imap_folder->search, folder);
3090         matches = camel_folder_search_search (imap_folder->search, expression, NULL, cancellable, error);
3091
3092         CAMEL_IMAP_FOLDER_UNLOCK (folder, search_lock);
3093
3094         return matches;
3095 }
3096
3097 static guint32
3098 imap_count_by_expression (CamelFolder *folder,
3099                           const gchar *expression,
3100                           GCancellable *cancellable,
3101                           GError **error)
3102 {
3103         CamelImapFolder *imap_folder = CAMEL_IMAP_FOLDER (folder);
3104         guint32 matches;
3105
3106         /* we could get around this by creating a new search object each time,
3107          * but i doubt its worth it since any long operation would lock the
3108          * command channel too */
3109         CAMEL_IMAP_FOLDER_LOCK (folder, search_lock);
3110
3111         camel_folder_search_set_folder (imap_folder->search, folder);
3112         matches = camel_folder_search_count (imap_folder->search, expression, cancellable, error);
3113
3114         CAMEL_IMAP_FOLDER_UNLOCK (folder, search_lock);
3115
3116         return matches;
3117 }
3118
3119 static GPtrArray *
3120 imap_search_by_uids (CamelFolder *folder,
3121                      const gchar *expression,
3122                      GPtrArray *uids,
3123                      GCancellable *cancellable,
3124                      GError **error)
3125 {
3126         CamelImapFolder *imap_folder = CAMEL_IMAP_FOLDER (folder);
3127         GPtrArray *matches;
3128
3129         if (uids->len == 0)
3130                 return g_ptr_array_new ();
3131
3132         CAMEL_IMAP_FOLDER_LOCK (folder, search_lock);
3133
3134         camel_folder_search_set_folder (imap_folder->search, folder);
3135         matches = camel_folder_search_search (imap_folder->search, expression, uids, cancellable, error);
3136
3137         CAMEL_IMAP_FOLDER_UNLOCK (folder, search_lock);
3138
3139         return matches;
3140 }
3141
3142 static void
3143 imap_search_free (CamelFolder *folder,
3144                   GPtrArray *uids)
3145 {
3146         CamelImapFolder *imap_folder = CAMEL_IMAP_FOLDER (folder);
3147
3148         g_return_if_fail (imap_folder->search);
3149
3150         CAMEL_IMAP_FOLDER_LOCK (folder, search_lock);
3151
3152         camel_folder_search_free_result (imap_folder->search, uids);
3153
3154         CAMEL_IMAP_FOLDER_UNLOCK (folder, search_lock);
3155 }
3156
3157 static CamelMimeMessage *get_message (CamelImapFolder *imap_folder,
3158                                       const gchar *uid,
3159                                       CamelMessageContentInfo *ci,
3160                                       GCancellable *cancellable,
3161                                       GError **error);
3162
3163 struct _part_spec_stack {
3164         struct _part_spec_stack *parent;
3165         gint part;
3166 };
3167
3168 static void
3169 part_spec_push (struct _part_spec_stack **stack,
3170                 gint part)
3171 {
3172         struct _part_spec_stack *node;
3173
3174         node = g_new (struct _part_spec_stack, 1);
3175         node->parent = *stack;
3176         node->part = part;
3177
3178         *stack = node;
3179 }
3180
3181 static gint
3182 part_spec_pop (struct _part_spec_stack **stack)
3183 {
3184         struct _part_spec_stack *node;
3185         gint part;
3186
3187         g_return_val_if_fail (*stack != NULL, 0);
3188
3189         node = *stack;
3190         *stack = node->parent;
3191
3192         part = node->part;
3193         g_free (node);
3194
3195         return part;
3196 }
3197
3198 static gchar *
3199 content_info_get_part_spec (CamelMessageContentInfo *ci)
3200 {
3201         struct _part_spec_stack *stack = NULL;
3202         CamelMessageContentInfo *node;
3203         gchar *part_spec, *buf;
3204         gsize len = 1;
3205         gint part;
3206
3207         node = ci;
3208         while (node->parent) {
3209                 CamelMessageContentInfo *child;
3210
3211                 /* FIXME: is this only supposed to apply if 'node' is a multipart? */
3212                 if (node->parent->parent &&
3213                                 camel_content_type_is (node->parent->type, "message", "*") &&
3214                                 !camel_content_type_is (node->parent->parent->type, "message", "*")) {
3215                         node = node->parent;
3216                         continue;
3217                 }
3218
3219                 child = node->parent->childs;
3220                 for (part = 1; child; part++) {
3221                         if (child == node)
3222                                 break;
3223
3224                         child = child->next;
3225                 }
3226
3227                 part_spec_push (&stack, part);
3228
3229                 len += 2;
3230                 while ((part = part / 10))
3231                         len++;
3232
3233                 node = node->parent;
3234         }
3235
3236         buf = part_spec = g_malloc (len);
3237         part_spec[0] = '\0';
3238
3239         while (stack) {
3240                 part = part_spec_pop (&stack);
3241                 buf += sprintf (buf, "%d%s", part, stack ? "." : "");
3242         }
3243
3244         return part_spec;
3245 }
3246
3247 /* Fetch the contents of the MIME part indicated by @ci, which is part
3248  * of message @uid in @folder.
3249  */
3250 static CamelDataWrapper *
3251 get_content (CamelImapFolder *imap_folder,
3252              const gchar *uid,
3253              CamelMimePart *part,
3254              CamelMessageContentInfo *ci,
3255              gint frommsg,
3256              GCancellable *cancellable,
3257              GError **error)
3258 {
3259         CamelDataWrapper *content = NULL;
3260         CamelStream *stream;
3261         gchar *part_spec;
3262
3263         part_spec = content_info_get_part_spec (ci);
3264
3265         d(printf("get content '%s' '%s' (frommsg = %d)\n", part_spec, camel_content_type_format(ci->type), frommsg));
3266
3267         /* There are three cases: multipart/signed, multipart, message/rfc822, and "other" */
3268         if (camel_content_type_is (ci->type, "multipart", "signed")) {
3269                 CamelMultipartSigned *body_mp;
3270                 gchar *spec;
3271                 gboolean success;
3272
3273                 /* Note: because we get the content parts uninterpreted anyway, we could potentially
3274                  * just use the normalmultipart code, except that multipart/signed wont let you yet! */
3275
3276                 body_mp = camel_multipart_signed_new ();
3277                 /* need to set this so it grabs the boundary and other info about the signed type */
3278                 /* we assume that part->content_type is more accurate/full than ci->type */
3279                 camel_data_wrapper_set_mime_type_field (CAMEL_DATA_WRAPPER (body_mp), CAMEL_DATA_WRAPPER (part)->mime_type);
3280
3281                 spec = g_alloca (strlen (part_spec) + 6);
3282                 if (frommsg)
3283                         sprintf(spec, part_spec[0] ? "%s.TEXT" : "TEXT", part_spec);
3284                 else
3285                         strcpy (spec, part_spec);
3286                 g_free (part_spec);
3287
3288                 stream = camel_imap_folder_fetch_data (imap_folder, uid, spec, FALSE, cancellable, error);
3289                 if (stream) {
3290                         success = camel_data_wrapper_construct_from_stream_sync (
3291                                 CAMEL_DATA_WRAPPER (body_mp), stream, cancellable, error);
3292                         g_object_unref (stream);
3293                         if (!success) {
3294                                 g_object_unref ( body_mp);
3295                                 return NULL;
3296                         }
3297                 }
3298
3299                 return (CamelDataWrapper *) body_mp;
3300         } else if (camel_content_type_is (ci->type, "multipart", "*")) {
3301                 CamelMultipart *body_mp;
3302                 gchar *child_spec;
3303                 gint speclen, num, isdigest;
3304
3305                 if (camel_content_type_is (ci->type, "multipart", "encrypted"))
3306                         body_mp = (CamelMultipart *) camel_multipart_encrypted_new ();
3307                 else
3308                         body_mp = camel_multipart_new ();
3309
3310                 /* need to set this so it grabs the boundary and other info about the multipart */
3311                 /* we assume that part->content_type is more accurate/full than ci->type */
3312                 camel_data_wrapper_set_mime_type_field (CAMEL_DATA_WRAPPER (body_mp), CAMEL_DATA_WRAPPER (part)->mime_type);
3313                 isdigest = camel_content_type_is(((CamelDataWrapper *)part)->mime_type, "multipart", "digest");
3314
3315                 speclen = strlen (part_spec);
3316                 child_spec = g_malloc (speclen + 17); /* dot + 10 + dot + MIME + nul */
3317                 memcpy (child_spec, part_spec, speclen);
3318                 if (speclen > 0)
3319                         child_spec[speclen++] = '.';
3320                 g_free (part_spec);
3321
3322                 ci = ci->childs;
3323                 num = 1;
3324                 while (ci) {
3325                         sprintf (child_spec + speclen, "%d.MIME", num++);
3326                         stream = camel_imap_folder_fetch_data (imap_folder, uid, child_spec, FALSE, cancellable, error);
3327                         if (stream) {
3328                                 gboolean success;
3329
3330                                 part = camel_mime_part_new ();
3331                                 success = camel_data_wrapper_construct_from_stream_sync (
3332                                         CAMEL_DATA_WRAPPER (part), stream, cancellable, error);
3333                                 g_object_unref (stream);
3334                                 if (!success) {
3335                                         g_object_unref (part);
3336                                         g_object_unref (body_mp);
3337                                         g_free (child_spec);
3338                                         return NULL;
3339                                 }
3340
3341                                 content = get_content (imap_folder, uid, part, ci, FALSE, cancellable, error);
3342                         }
3343
3344                         if (!stream || !content) {
3345                                 g_object_unref (body_mp);
3346                                 g_free (child_spec);
3347                                 return NULL;
3348                         }
3349
3350                         if (camel_debug("imap:folder")) {
3351                                 gchar *ct = camel_content_type_format (camel_mime_part_get_content_type ((CamelMimePart *) part));
3352                                 gchar *ct2 = camel_content_type_format (ci->type);
3353
3354                                 printf("Setting part content type to '%s' contentinfo type is '%s'\n", ct, ct2);
3355                                 g_free (ct);
3356                                 g_free (ct2);
3357                         }
3358
3359                         /* if we had no content-type header on a multipart/digest sub-part, then we need to
3360                          * treat it as message/rfc822 instead */
3361                         if (isdigest && camel_medium_get_header((CamelMedium *)part, "content-type") == NULL) {
3362                                 CamelContentType *ct = camel_content_type_new("message", "rfc822");
3363
3364                                 camel_data_wrapper_set_mime_type_field (content, ct);
3365                                 camel_content_type_unref (ct);
3366                         } else {
3367                                 camel_data_wrapper_set_mime_type_field (content, camel_mime_part_get_content_type (part));
3368                         }
3369
3370                         camel_medium_set_content (CAMEL_MEDIUM (part), content);
3371                         g_object_unref (content);
3372
3373                         camel_multipart_add_part (body_mp, part);
3374                         g_object_unref (part);
3375
3376                         ci = ci->next;
3377                 }
3378
3379                 g_free (child_spec);
3380
3381                 return (CamelDataWrapper *) body_mp;
3382         } else if (camel_content_type_is (ci->type, "message", "rfc822")) {
3383                 content = (CamelDataWrapper *) get_message (imap_folder, uid, ci->childs, cancellable, error);
3384                 g_free (part_spec);
3385                 return content;
3386         } else {
3387                 CamelTransferEncoding enc;
3388                 gchar *spec;
3389
3390                 /* NB: we need this differently to multipart/signed case above on purpose */
3391                 spec = g_alloca (strlen (part_spec) + 6);
3392                 if (frommsg)
3393                         sprintf(spec, part_spec[0] ? "%s.1" : "1", part_spec);
3394                 else
3395                         strcpy(spec, part_spec[0]?part_spec:"1");
3396
3397                 enc = ci->encoding ? camel_transfer_encoding_from_string (ci->encoding) : CAMEL_TRANSFER_ENCODING_DEFAULT;
3398                 content = camel_imap_wrapper_new (imap_folder, ci->type, enc, uid, spec, part);
3399                 g_free (part_spec);
3400                 return content;
3401         }
3402 }
3403
3404 static CamelMimeMessage *
3405 get_message (CamelImapFolder *imap_folder,
3406              const gchar *uid,
3407              CamelMessageContentInfo *ci,
3408              GCancellable *cancellable,
3409              GError **error)
3410 {
3411         CamelFolder *folder;
3412         CamelStore *parent_store;
3413         CamelImapStore *store;
3414         CamelDataWrapper *content;
3415         CamelMimeMessage *msg;
3416         CamelStream *stream;
3417         gchar *section_text, *part_spec;
3418         gboolean success;
3419
3420         folder = CAMEL_FOLDER (imap_folder);
3421         parent_store = camel_folder_get_parent_store (folder);
3422         store = CAMEL_IMAP_STORE (parent_store);
3423
3424         part_spec = content_info_get_part_spec (ci);
3425         d(printf("get message '%s'\n", part_spec));
3426         section_text = g_strdup_printf ("%s%s%s", part_spec, *part_spec ? "." : "",
3427                                         store->server_level >= IMAP_LEVEL_IMAP4REV1 ? "HEADER" : "0");
3428
3429         stream = camel_imap_folder_fetch_data (imap_folder, uid, section_text, FALSE, cancellable, error);
3430         g_free (section_text);
3431         g_free (part_spec);
3432         if (!stream)
3433                 return NULL;
3434
3435         msg = camel_mime_message_new ();
3436         success = camel_data_wrapper_construct_from_stream_sync (
3437                 CAMEL_DATA_WRAPPER (msg), stream, cancellable, error);
3438         g_object_unref (stream);
3439         if (!success) {
3440                 g_object_unref (msg);
3441                 return NULL;
3442         }
3443
3444         content = get_content (imap_folder, uid, CAMEL_MIME_PART (msg), ci, TRUE, cancellable, error);
3445         if (!content) {
3446                 g_object_unref (msg);
3447                 return NULL;
3448         }
3449
3450         if (camel_debug("imap:folder")) {
3451                 gchar *ct = camel_content_type_format (camel_mime_part_get_content_type ((CamelMimePart *) msg));
3452                 gchar *ct2 = camel_content_type_format (ci->type);
3453
3454                 printf("Setting message content type to '%s' contentinfo type is '%s'\n", ct, ct2);
3455                 g_free (ct);
3456                 g_free (ct2);
3457         }
3458
3459         camel_data_wrapper_set_mime_type_field (content, camel_mime_part_get_content_type ((CamelMimePart *) msg));
3460         camel_medium_set_content (CAMEL_MEDIUM (msg), content);
3461         g_object_unref (content);
3462
3463         return msg;
3464 }
3465
3466 #define IMAP_SMALL_BODY_SIZE 5120
3467
3468 static CamelMimeMessage *
3469 get_message_simple (CamelImapFolder *imap_folder,
3470                     const gchar *uid,
3471                     CamelStream *stream,
3472                     GCancellable *cancellable,
3473                     GError **error)
3474 {
3475         CamelMimeMessage *msg;
3476         gboolean success;
3477
3478         if (!stream) {
3479                 stream = camel_imap_folder_fetch_data (imap_folder, uid, "",
3480                                                        FALSE, cancellable, error);
3481                 if (!stream)
3482                         return NULL;
3483         }
3484
3485         msg = camel_mime_message_new ();
3486         success = camel_data_wrapper_construct_from_stream_sync (
3487                 CAMEL_DATA_WRAPPER (msg), stream, cancellable, error);
3488         g_object_unref (stream);
3489         if (!success) {
3490                 g_prefix_error (error, _("Unable to retrieve message: "));
3491                 g_object_unref (msg);
3492                 return NULL;
3493         }
3494
3495         return msg;
3496 }
3497
3498 static gboolean
3499 content_info_incomplete (CamelMessageContentInfo *ci)
3500 {
3501         if (!ci->type)
3502                 return TRUE;
3503
3504         if (camel_content_type_is (ci->type, "multipart", "*")
3505             || camel_content_type_is (ci->type, "message", "rfc822")) {
3506                 if (!ci->childs)
3507                         return TRUE;
3508                 for (ci = ci->childs; ci; ci = ci->next)
3509                         if (content_info_incomplete (ci))
3510                                 return TRUE;
3511         }
3512
3513         return FALSE;
3514 }
3515
3516 static CamelImapMessageInfo *
3517 imap_folder_summary_uid_or_error (CamelFolderSummary *summary,
3518                                   const gchar *uid,
3519                                   GError **error)
3520 {
3521         CamelImapMessageInfo *mi;
3522         mi = (CamelImapMessageInfo *) camel_folder_summary_get (summary, uid);
3523         if (mi == NULL) {
3524                 g_set_error (
3525                         error, CAMEL_FOLDER_ERROR,
3526                         CAMEL_FOLDER_ERROR_INVALID_UID,
3527                         _("Cannot get message with message ID %s: %s"),
3528                         uid, _("No such message available."));
3529         }
3530         return mi;
3531 }
3532
3533 static CamelMimeMessage *
3534 imap_get_message_sync (CamelFolder *folder,
3535                        const gchar *uid,
3536                        GCancellable *cancellable,
3537                        GError **error)
3538 {
3539         CamelStore *parent_store;
3540         CamelImapFolder *imap_folder = CAMEL_IMAP_FOLDER (folder);
3541         CamelImapStore *store;
3542         CamelImapMessageInfo *mi;
3543         CamelMimeMessage *msg = NULL;
3544         CamelStream *stream = NULL;
3545         gint retry;
3546         GError *local_error = NULL;
3547
3548         parent_store = camel_folder_get_parent_store (folder);
3549         store = CAMEL_IMAP_STORE (parent_store);
3550
3551         mi = imap_folder_summary_uid_or_error (folder->summary, uid, error);
3552         if (!mi)
3553           return NULL;
3554
3555         /* If its cached in full, just get it as is, this is only a shortcut,
3556          * since we get stuff from the cache anyway.  It affects a busted
3557          * connection though. */
3558         stream = camel_imap_folder_fetch_data (imap_folder, uid, "", TRUE, cancellable, NULL);
3559         if (stream != NULL) {
3560                 msg = get_message_simple (imap_folder, uid, stream, cancellable, NULL);
3561                 if (msg != NULL)
3562                         goto done;
3563         }
3564
3565         /* All this mess is so we silently retry a fetch if we fail with
3566          * service_unavailable, without an (equivalent) mess of gotos */
3567         retry = 0;
3568         do {
3569                 retry++;
3570                 g_clear_error (&local_error);
3571
3572                 /* If the message is small or only 1 part, or server doesn't do 4v1 (properly) fetch it in one piece. */
3573                 if (store->server_level < IMAP_LEVEL_IMAP4REV1
3574                     || store->braindamaged
3575                     || mi->info.size < IMAP_SMALL_BODY_SIZE
3576                     || (!content_info_incomplete (mi->info.content) && !mi->info.content->childs)) {
3577                         CamelMessageInfoBase *info = (CamelMessageInfoBase *) camel_folder_summary_get (folder->summary, uid);
3578                         msg = get_message_simple (imap_folder, uid, NULL, cancellable, &local_error);
3579                         if (info && !info->preview && msg && camel_folder_summary_get_need_preview (folder->summary)) {
3580                                 if (camel_mime_message_build_preview ((CamelMimePart *) msg, (CamelMessageInfo *) info) && info->preview)
3581                                         camel_folder_summary_add_preview (folder->summary, (CamelMessageInfo *) info);
3582                         }
3583
3584                         camel_message_info_free (info);
3585                 } else {
3586                         if (content_info_incomplete (mi->info.content)) {
3587                                 /* For larger messages, fetch the structure and build a message
3588                                  * with offline parts. (We check mi->content->type rather than
3589                                  * mi->content because camel_folder_summary_info_new always creates
3590                                  * an empty content struct.)
3591                                  */
3592                                 CamelImapResponse *response;
3593                                 GData *fetch_data = NULL;
3594                                 gchar *body, *found_uid;
3595                                 gint i;
3596
3597                                 camel_service_lock (CAMEL_SERVICE (store), CAMEL_SERVICE_REC_CONNECT_LOCK);
3598                                 if (!camel_imap_store_connected (store, NULL)) {
3599                                         camel_service_unlock (CAMEL_SERVICE (store), CAMEL_SERVICE_REC_CONNECT_LOCK);
3600                                         g_set_error (
3601                                                 error, CAMEL_SERVICE_ERROR,
3602                                                 CAMEL_SERVICE_ERROR_UNAVAILABLE,
3603                                                 _("This message is not currently available"));
3604                                         goto fail;
3605                                 }
3606
3607                                 response = camel_imap_command (store, folder, cancellable, &local_error, "UID FETCH %s BODY", uid);
3608
3609                                 if (response) {
3610                                         for (i = 0, body = NULL; i < response->untagged->len; i++) {
3611                                                 fetch_data = parse_fetch_response (imap_folder, response->untagged->pdata[i]);
3612                                                 if (fetch_data) {
3613                                                         found_uid = g_datalist_get_data (&fetch_data, "UID");
3614                                                         body = g_datalist_get_data (&fetch_data, "BODY");
3615                                                         if (found_uid && body && !strcmp (found_uid, uid))
3616                                                                 break;
3617                                                         g_datalist_clear (&fetch_data);
3618                                                         fetch_data = NULL;
3619                                                         body = NULL;
3620                                                 }
3621                                         }
3622
3623                                         if (body) {
3624                                                 /* NB: small race here, setting the info.content */
3625                                                 imap_parse_body ((const gchar **) &body, folder, mi->info.content);
3626                                                 mi->info.dirty = TRUE;
3627                                                 camel_folder_summary_touch (folder->summary);
3628                                         }
3629
3630                                         if (fetch_data)
3631                                                 g_datalist_clear (&fetch_data);
3632
3633                                         camel_imap_response_free (store, response);
3634                                 } else {
3635                                         g_clear_error (&local_error);
3636                                 }
3637                                 camel_service_unlock (CAMEL_SERVICE (store), CAMEL_SERVICE_REC_CONNECT_LOCK);
3638                         }
3639
3640                         if (camel_debug_start("imap:folder")) {
3641                                 printf("Folder get message '%s' folder info ->\n", uid);
3642                                 camel_message_info_dump ((CamelMessageInfo *) mi);
3643                                 camel_debug_end ();
3644                         }
3645
3646                         /* FETCH returned OK, but we didn't parse a BODY
3647                          * response. Courier will return invalid BODY
3648                          * responses for invalidly MIMEd messages, so
3649                          * fall back to fetching the entire thing and
3650                          * let the mailer's "bad MIME" code handle it.
3651                          */
3652                         if (content_info_incomplete (mi->info.content))
3653                                 msg = get_message_simple (imap_folder, uid, NULL, cancellable, &local_error);
3654                         else
3655                                 msg = get_message (imap_folder, uid, mi->info.content, cancellable, &local_error);
3656                         if (msg && camel_folder_summary_get_need_preview (folder->summary)) {
3657                                 CamelMessageInfoBase *info = (CamelMessageInfoBase *) camel_folder_summary_get (folder->summary, uid);
3658                                 if (info && !info->preview) {
3659                                         if (camel_mime_message_build_preview ((CamelMimePart *) msg, (CamelMessageInfo *) info) && info->preview)
3660                                                 camel_folder_summary_add_preview (folder->summary, (CamelMessageInfo *) info);
3661                                 }
3662                                 camel_message_info_free (info);
3663                         }
3664
3665                 }
3666         } while (msg == NULL
3667                  && retry < 2
3668                  && g_error_matches (local_error, CAMEL_SERVICE_ERROR, CAMEL_SERVICE_ERROR_UNAVAILABLE));
3669
3670 done:
3671         if (msg) {
3672                 gboolean has_attachment;
3673
3674                 if (!mi->info.mlist || !*mi->info.mlist) {
3675                         /* update mailing list information, if necessary */
3676                         gchar *mlist = camel_header_raw_check_mailing_list (&(CAMEL_MIME_PART (msg)->headers));
3677
3678                         if (mlist) {
3679                                 if (mi->info.mlist)
3680                                         camel_pstring_free (mi->info.mlist);
3681                                 mi->info.mlist = camel_pstring_add (mlist, TRUE);
3682                                 mi->info.dirty = TRUE;
3683
3684                                 if (mi->info.summary)
3685                                         camel_folder_summary_touch (mi->info.summary);
3686                         }
3687                 }
3688
3689                 has_attachment = camel_mime_message_has_attachment (msg);
3690                 if (((camel_message_info_flags ((CamelMessageInfo *) mi) & CAMEL_MESSAGE_ATTACHMENTS) && !has_attachment) ||
3691                     ((camel_message_info_flags ((CamelMessageInfo *) mi) & CAMEL_MESSAGE_ATTACHMENTS) == 0 && has_attachment)) {
3692                         camel_message_info_set_flags ((CamelMessageInfo *) mi, CAMEL_MESSAGE_ATTACHMENTS, has_attachment ? CAMEL_MESSAGE_ATTACHMENTS : 0);
3693                 }
3694         }
3695
3696         if (local_error != NULL)
3697                 g_propagate_error (error, local_error);
3698
3699 fail:
3700         camel_message_info_free (&mi->info);
3701
3702         return msg;
3703 }
3704
3705 /**
3706  * imap_synchronize_message_sync
3707  *
3708  * Ensure that a message is cached locally, but don't retrieve the content if
3709  * it is already local.
3710  */
3711 static gboolean
3712 imap_synchronize_message_sync (CamelFolder *folder,
3713                                const gchar *uid,
3714                                GCancellable *cancellable,
3715                                GError **error)
3716 {
3717         CamelImapFolder *imap_folder = CAMEL_IMAP_FOLDER (folder);
3718         CamelImapMessageInfo *mi;
3719         CamelMimeMessage *msg = NULL;
3720         CamelStream *stream = NULL;
3721         gboolean success = FALSE;
3722
3723         mi = imap_folder_summary_uid_or_error (folder->summary, uid, error);
3724         if (!mi)
3725           /* No such UID - is this duplicate work? The sync process selects
3726            * UIDs to start with.
3727            */
3728           return FALSE;
3729         camel_message_info_free (&mi->info);
3730
3731         /* If we can get a stream, assume its fully cached. This may be false
3732          * if partial streams are saved elsewhere in the code - but that seems
3733          * best solved by knowning more about whether a given message is fully
3734          * available locally or not,
3735          */
3736         /* If its cached in full, just get it as is, this is only a shortcut,
3737          * since we get stuff from the cache anyway.  It affects a busted connection though. */
3738         if ((stream = camel_imap_folder_fetch_data(imap_folder, uid, "", TRUE, cancellable, NULL))) {
3739                 g_object_unref (stream);
3740                 return TRUE;
3741         }
3742         msg = imap_get_message_sync (folder, uid, cancellable, error);
3743         if (msg != NULL) {
3744                 g_object_unref (msg);
3745                 success = TRUE;
3746         }
3747
3748         return success;
3749 }
3750
3751 /* We pretend that a FLAGS or RFC822.SIZE response is always exactly
3752  * 20 bytes long, and a BODY[HEADERS] response is always 2000 bytes
3753  * long. Since we know how many of each kind of response we're
3754  * expecting, we can find the total (pretend) amount of server traffic
3755  * to expect and then count off the responses as we read them to update
3756  * the progress bar.
3757  */
3758 #define IMAP_PRETEND_SIZEOF_FLAGS         20
3759 #define IMAP_PRETEND_SIZEOF_SIZE          20
3760 #define IMAP_PRETEND_SIZEOF_HEADERS     2000
3761
3762 static const gchar *tm_months[] = {
3763         "Jan", "Feb", "Mar", "Apr", "May", "Jun",
3764         "Jul", "Aug", "Sep", "Oct", "Nov", "Dec"
3765 };
3766
3767 static gboolean
3768 decode_time (const guchar **in,
3769              gint *hour,
3770              gint *min,
3771              gint *sec)
3772 {
3773         register const guchar *inptr;
3774         gint *val, colons = 0;
3775
3776         *hour = *min = *sec = 0;
3777
3778         val = hour;
3779         for (inptr = *in; *inptr && !isspace ((gint) *inptr); inptr++) {
3780                 if (*inptr == ':') {
3781                         colons++;
3782                         switch (colons) {
3783                         case 1:
3784                                 val = min;
3785                                 break;
3786                         case 2:
3787                                 val = sec;
3788                                 break;
3789                         default:
3790                                 return FALSE;
3791                         }
3792                 } else if (!isdigit ((gint) *inptr))
3793                         return FALSE;
3794                 else
3795                         *val = (*val * 10) + (*inptr - '0');
3796         }
3797
3798         *in = inptr;
3799
3800         return TRUE;
3801 }
3802
3803 static time_t
3804 decode_internaldate (const guchar *in)
3805 {
3806         const guchar *inptr = in;
3807         gint hour, min, sec, n;
3808         guchar *buf;
3809         struct tm tm;
3810         time_t date;
3811
3812         memset ((gpointer) &tm, 0, sizeof (struct tm));
3813
3814         tm.tm_mday = strtoul ((gchar *) inptr, (gchar **) &buf, 10);
3815         if (buf == inptr || *buf != '-')
3816                 return (time_t) -1;
3817
3818         inptr = buf + 1;
3819         if (inptr[3] != '-')
3820                 return (time_t) -1;
3821
3822         for (n = 0; n < 12; n++) {
3823                 if (!g_ascii_strncasecmp ((gchar *) inptr, tm_months[n], 3))
3824                         break;
3825         }
3826
3827         if (n >= 12)
3828                 return (time_t) -1;
3829
3830         tm.tm_mon = n;
3831
3832         inptr += 4;
3833
3834         n = strtoul ((gchar *) inptr, (gchar **) &buf, 10);
3835         if (buf == inptr || *buf != ' ')
3836                 return (time_t) -1;
3837
3838         tm.tm_year = n - 1900;
3839
3840         inptr = buf + 1;
3841         if (!decode_time (&inptr, &hour, &min, &sec))
3842                 return (time_t) -1;
3843
3844         tm.tm_hour = hour;
3845         tm.tm_min = min;
3846         tm.tm_sec = sec;
3847
3848         n = strtol ((gchar *) inptr, NULL, 10);
3849
3850         date = camel_mktime_utc (&tm);
3851
3852         /* date is now GMT of the time we want, but not offset by the timezone ... */
3853
3854         /* this should convert the time to the GMT equiv time */
3855         date -= ((n / 100) * 60 * 60) + (n % 100) * 60;
3856
3857         return date;
3858 }
3859
3860 static void
3861 add_message_from_data (CamelFolder *folder,
3862                        GPtrArray *messages,
3863                        gint first,
3864                        GData *data,
3865                        GCancellable *cancellable)
3866 {
3867         CamelMimeMessage *msg;
3868         CamelStream *stream;
3869         CamelImapMessageInfo *mi;
3870         const gchar *idate;
3871         const gchar *bodystructure;
3872         gint seq;
3873
3874         seq = GPOINTER_TO_INT (g_datalist_get_data (&data, "SEQUENCE"));
3875         if (seq < first)
3876                 return;
3877         stream = g_datalist_get_data (&data, "BODY_PART_STREAM");
3878         if (!stream)
3879                 return;
3880
3881         if (seq - first >= messages->len)
3882                 g_ptr_array_set_size (messages, seq - first + 1);
3883
3884         msg = camel_mime_message_new ();
3885         if (!camel_data_wrapper_construct_from_stream_sync (
3886                 CAMEL_DATA_WRAPPER (msg), stream, cancellable, NULL)) {
3887                 g_object_unref (msg);
3888                 return;
3889         }
3890
3891         bodystructure = g_datalist_get_data (&data, "BODY");
3892
3893         mi = (CamelImapMessageInfo *)
3894                 camel_folder_summary_info_new_from_message (
3895                 folder->summary, msg, bodystructure);
3896         g_object_unref (msg);
3897
3898         if ((idate = g_datalist_get_data (&data, "INTERNALDATE")))
3899                 mi->info.date_received = decode_internaldate ((const guchar *) idate);
3900
3901         if (mi->info.date_received == -1)
3902                 mi->info.date_received = mi->info.date_sent;
3903
3904         messages->pdata[seq - first] = mi;
3905 }
3906
3907 struct _junk_data {
3908         GData *data;
3909         CamelMessageInfoBase *mi;
3910 };
3911
3912 static void
3913 construct_junk_headers (gchar *header,
3914                         gchar *value,
3915                         struct _junk_data *jdata)
3916 {
3917         gchar *bs, *es, *flag = NULL;
3918         gchar *bdata = g_datalist_get_data (&(jdata->data), "BODY_PART_DATA");
3919         struct _camel_header_param *node;
3920
3921         /* FIXME: This can be written in a much clever way.
3922          * We can create HEADERS file or carry all headers till filtering so
3923          * that header based filtering can be much faster. But all that later. */
3924         bs = camel_strstrcase (bdata ? bdata:"", header);
3925         if (bs) {
3926                 bs += strlen (header);
3927                 bs = strchr (bs, ':');
3928                 if (bs) {
3929                         bs++;
3930                         while (*bs == ' ')
3931                                 bs++;
3932                         es = strchr (bs, '\n');
3933                         if (es)
3934                                 flag = g_strndup (bs, es - bs);
3935                         else
3936                                 bs = NULL;
3937                 }
3938
3939         }
3940
3941         if (bs) {
3942                 node = g_new (struct _camel_header_param, 1);
3943                 node->name = g_strdup (header);
3944                 node->value = flag;
3945                 node->next = jdata->mi->headers;
3946                 jdata->mi->headers = node;
3947         }
3948 }
3949
3950 #define CAMEL_MESSAGE_INFO_HEADERS "DATE FROM TO CC SUBJECT REFERENCES IN-REPLY-TO MESSAGE-ID MIME-VERSION CONTENT-TYPE CONTENT-CLASS X-CALENDAR-ATTACHMENT "
3951
3952 /* FIXME: this needs to be kept in sync with camel-mime-utils.c's list
3953  * of mailing-list headers and so might be best if this were
3954  * auto-generated? */
3955 #define MAILING_LIST_HEADERS "X-MAILING-LIST X-LOOP LIST-ID LIST-POST MAILING-LIST ORIGINATOR X-LIST SENDER RETURN-PATH X-BEENTHERE "
3956
3957 static gboolean
3958 imap_update_summary (CamelFolder *folder,
3959                      gint exists,
3960                      CamelFolderChangeInfo *changes,
3961                      GCancellable *cancellable,
3962                      GError **error)
3963 {
3964         CamelStore *parent_store;
3965         CamelService *service;
3966         CamelSettings *settings;
3967         CamelImapStore *store;
3968         CamelImapFolder *imap_folder = CAMEL_IMAP_FOLDER (folder);
3969         GPtrArray *fetch_data = NULL, *messages = NULL, *needheaders;
3970         CamelFetchHeadersType fetch_headers;
3971         gchar **extra_headers;
3972         guint32 flags, uidval;
3973         gint i, seq, first, size, got;
3974         CamelImapResponseType type;
3975         GString *header_spec = NULL;
3976         CamelImapMessageInfo *mi;
3977         CamelStream *stream;
3978         gchar *uid, *resp, *tempuid;
3979         GData *data;
3980         gint k = 0, ct;
3981
3982         parent_store = camel_folder_get_parent_store (folder);
3983         store = CAMEL_IMAP_STORE (parent_store);
3984         service = CAMEL_SERVICE (parent_store);
3985         settings = camel_service_get_settings (service);
3986
3987         fetch_headers = camel_imap_settings_get_fetch_headers (
3988                 CAMEL_IMAP_SETTINGS (settings));
3989
3990         extra_headers = camel_imap_settings_dup_fetch_headers_extra (
3991                 CAMEL_IMAP_SETTINGS (settings));
3992
3993         if (store->server_level >= IMAP_LEVEL_IMAP4REV1) {
3994                 if (fetch_headers == CAMEL_FETCH_HEADERS_ALL)
3995                         header_spec = g_string_new ("HEADER");
3996                 else {
3997                         gchar *temp;
3998                         header_spec = g_string_new ("HEADER.FIELDS (");
3999                         g_string_append (header_spec, CAMEL_MESSAGE_INFO_HEADERS);
4000                         if (fetch_headers == CAMEL_FETCH_HEADERS_BASIC_AND_MAILING_LIST)
4001                                 g_string_append (header_spec, MAILING_LIST_HEADERS);
4002                         if (extra_headers != NULL) {
4003                                 guint length, ii;
4004
4005                                 length = g_strv_length ((gchar **) extra_headers);
4006                                 for (ii = 0; ii < length; ii++) {
4007                                         g_string_append (header_spec, extra_headers[ii]);
4008                                         if (ii + 1 < length)
4009                                                 g_string_append_c (header_spec, ' ');
4010                                 }
4011                         }
4012
4013                         temp = g_string_free (header_spec, FALSE);
4014                         temp = g_strstrip (temp);
4015                         header_spec = g_string_new (temp);
4016                         g_free (temp);
4017                         g_string_append (header_spec, ")");
4018                 }
4019         } else
4020                 header_spec = g_string_new ("0");
4021
4022         g_strfreev (extra_headers);
4023
4024         d(printf("Header is : %s", header_spec->str));
4025
4026         /* Figure out if any of the new messages are already cached (which
4027          * may be the case if we're re-syncing after disconnected operation).
4028          * If so, get their UIDs, FLAGS, and SIZEs. If not, get all that
4029          * and ask for the headers too at the same time.
4030          */
4031         seq = camel_folder_summary_count (folder->summary);
4032         first = seq + 1;
4033         if (seq > 0) {
4034                 GPtrArray *known_uids;
4035
4036                 known_uids = camel_folder_summary_get_array (folder->summary);
4037                 if (known_uids) {
4038                         camel_folder_sort_uids (folder, known_uids);
4039
4040                         tempuid = g_ptr_array_index (known_uids, seq - 1);
4041                         if (tempuid)
4042                                 uidval = strtoul (tempuid, NULL, 10);
4043                         else
4044                                 uidval = 0;
4045
4046                         camel_folder_summary_free_array (known_uids);
4047                 } else
4048                         uidval = 0;
4049         } else
4050                 uidval = 0;
4051
4052         got = 0;
4053         if (!camel_imap_command_start (store, folder, cancellable, error,
4054                                        "UID FETCH %d:* (FLAGS RFC822.SIZE INTERNALDATE BODYSTRUCTURE BODY.PEEK[%s])",
4055                                        uidval + 1, header_spec->str)) {
4056                 g_string_free (header_spec, TRUE);
4057                 return FALSE;
4058         }
4059
4060         camel_operation_push_message (
4061                 cancellable,
4062                 _("Fetching summary information for new messages in %s"),
4063                 camel_folder_get_display_name (folder));
4064
4065         /* Parse the responses. We can't add a message to the summary
4066          * until we've gotten its headers, and there's no guarantee
4067          * the server will send the responses in a useful order...
4068          */
4069         fetch_data = g_ptr_array_new ();
4070         messages = g_ptr_array_new ();
4071         ct = exists - seq;
4072         while ((type = camel_imap_command_response (store, folder, &resp, cancellable, error)) ==
4073                CAMEL_IMAP_RESPONSE_UNTAGGED && !camel_application_is_exiting) {
4074                 data = parse_fetch_response (imap_folder, resp);
4075                 g_free (resp);
4076                 k++;
4077                 if (!data)
4078                         continue;
4079
4080                 seq = GPOINTER_TO_INT (g_datalist_get_data (&data, "SEQUENCE"));
4081                 if (seq < first) {
4082                         g_datalist_clear (&data);
4083                         continue;
4084                 }
4085
4086                 if (g_datalist_get_data (&data, "FLAGS"))
4087                         got += IMAP_PRETEND_SIZEOF_FLAGS;
4088                 if (g_datalist_get_data (&data, "RFC822.SIZE"))
4089                         got += IMAP_PRETEND_SIZEOF_SIZE;
4090                 stream = g_datalist_get_data (&data, "BODY_PART_STREAM");
4091                 if (stream) {
4092                         got += IMAP_PRETEND_SIZEOF_HEADERS;
4093
4094                         /* Use the stream now so we don't tie up many
4095                          * many fds if we're fetching many many messages.
4096                          */
4097                         add_message_from_data (
4098                                 folder, messages, first, data, cancellable);
4099                         g_datalist_set_data (&data, "BODY_PART_STREAM", NULL);
4100                 }
4101
4102                 camel_operation_progress (cancellable, k * 100 / ct);
4103
4104                 g_ptr_array_add (fetch_data, data);
4105         }
4106
4107         camel_operation_pop_message (cancellable);
4108
4109         if (type == CAMEL_IMAP_RESPONSE_ERROR || camel_application_is_exiting) {
4110                 if (type != CAMEL_IMAP_RESPONSE_ERROR && type != CAMEL_IMAP_RESPONSE_TAGGED)
4111                         camel_service_unlock (service, CAMEL_SERVICE_REC_CONNECT_LOCK);
4112
4113                 g_string_free (header_spec, TRUE);
4114                 goto lose;
4115         }
4116
4117         /* Free the final tagged response */
4118         g_free (resp);
4119
4120         /* Figure out which headers we still need to fetch. */
4121         needheaders = g_ptr_array_new ();
4122         size = got = 0;
4123         for (i = 0; i < fetch_data->len; i++) {
4124                 data = fetch_data->pdata[i];
4125                 if (g_datalist_get_data (&data, "BODY_PART_LEN"))
4126                         continue;
4127
4128                 uid = g_datalist_get_data (&data, "UID");
4129                 if (uid) {
4130                         g_ptr_array_add (needheaders, uid);
4131                         size += IMAP_PRETEND_SIZEOF_HEADERS;
4132                 }
4133         }
4134
4135         /* And fetch them */
4136         if (needheaders->len) {
4137                 gchar *uidset;
4138                 gint uid = 0;
4139
4140                 qsort (needheaders->pdata, needheaders->len,
4141                        sizeof (gpointer), uid_compar);
4142
4143                 camel_operation_push_message (
4144                         cancellable,
4145                         _("Fetching summary information for new messages in %s"),
4146                         camel_folder_get_display_name (folder));
4147
4148                 while (uid < needheaders->len && !camel_application_is_exiting) {
4149                         uidset = imap_uid_array_to_set (folder->summary, needheaders, uid, UID_SET_LIMIT, &uid);
4150                         if (!camel_imap_command_start (store, folder, cancellable, error,
4151                                                        "UID FETCH %s BODYSTRUCTURE BODY.PEEK[%s]",
4152                                                        uidset, header_spec->str)) {
4153                                 g_ptr_array_free (needheaders, TRUE);
4154                                 camel_operation_pop_message (cancellable);
4155                                 g_free (uidset);
4156                                 g_string_free (header_spec, TRUE);
4157                                 goto lose;
4158                         }
4159                         g_free (uidset);
4160
4161                         while ((type = camel_imap_command_response (store, folder, &resp, cancellable, error))
4162                                == CAMEL_IMAP_RESPONSE_UNTAGGED && !camel_application_is_exiting) {
4163                                 data = parse_fetch_response (imap_folder, resp);
4164                                 g_free (resp);
4165                                 if (!data)
4166                                         continue;
4167
4168                                 stream = g_datalist_get_data (&data, "BODY_PART_STREAM");
4169                                 if (stream) {
4170                                         add_message_from_data (
4171                                                 folder, messages, first,
4172                                                 data, cancellable);
4173                                         got += IMAP_PRETEND_SIZEOF_HEADERS;
4174                                         camel_operation_progress (
4175                                                 cancellable, got * 100 / size);
4176                                 }
4177                                 g_datalist_clear (&data);
4178                         }
4179
4180                         if (type == CAMEL_IMAP_RESPONSE_ERROR || camel_application_is_exiting) {
4181                                 g_ptr_array_free (needheaders, TRUE);
4182                                 g_string_free (header_spec, TRUE);
4183                                 camel_operation_pop_message (cancellable);
4184
4185                                 if (type != CAMEL_IMAP_RESPONSE_ERROR && type != CAMEL_IMAP_RESPONSE_TAGGED)
4186                                         camel_service_unlock (service, CAMEL_SERVICE_REC_CONNECT_LOCK);
4187
4188                                 goto lose;
4189                         }
4190                 }
4191                 camel_operation_pop_message (cancellable);
4192         }
4193
4194         g_ptr_array_free (needheaders, TRUE);
4195         g_string_free (header_spec, TRUE);
4196
4197         /* Now finish up summary entries (fix UIDs, set flags and size) */
4198         for (i = 0; i < fetch_data->len; i++) {
4199                 struct _junk_data jdata;
4200                 data = fetch_data->pdata[i];
4201
4202                 seq = GPOINTER_TO_INT (g_datalist_get_data (&data, "SEQUENCE"));
4203                 if (seq >= first + messages->len) {
4204                         g_datalist_clear (&data);
4205                         continue;
4206                 }
4207
4208                 mi = messages->pdata[seq - first];
4209                 if (mi == NULL) {
4210                         CamelMessageInfo *pmi = NULL;
4211                         gint j;
4212
4213                         /* This is a kludge around a bug in Exchange
4214                          * 5.5 that sometimes claims multiple messages
4215                          * have the same UID. See bug #17694 for
4216                          * details. The "solution" is to create a fake
4217                          * message-info with the same details as the
4218                          * previously valid message. Yes, the user
4219                          * will have a clone in his/her message-list,
4220                          * but at least we don't crash.
4221                          */
4222
4223                         /* find the previous valid message info */
4224                         for (j = seq - first - 1; j >= 0; j--) {
4225                                 pmi = messages->pdata[j];
4226                                 if (pmi != NULL)
4227                                         break;
4228                         }
4229
4230                         if (pmi == NULL) {
4231                                 continue;
4232                         }
4233
4234                         mi = (CamelImapMessageInfo *) camel_message_info_clone (pmi);
4235                 }
4236
4237                 uid = g_datalist_get_data (&data, "UID");
4238                 if (uid)
4239                         mi->info.uid = camel_pstring_strdup (uid);
4240                 flags = GPOINTER_TO_INT (g_datalist_get_data (&data, "FLAGS"));
4241                 if (flags) {
4242                         gchar *custom_flags = NULL;
4243
4244                         ((CamelImapMessageInfo *) mi)->server_flags = flags;
4245                         /* "or" them in with the existing flags that may
4246                          * have been set by summary_info_new_from_message.
4247                          */
4248                         mi->info.flags |= flags;
4249
4250                         custom_flags = g_datalist_get_data (&data, "CUSTOM.FLAGS");
4251                         if (custom_flags)
4252                                 fillup_custom_flags ((CamelMessageInfo *) mi, custom_flags);
4253                 }
4254                 size = GPOINTER_TO_INT (g_datalist_get_data (&data, "RFC822.SIZE"));
4255                 if (size)
4256                         mi->info.size = size;
4257
4258                 /* Just do this to build the junk required headers to be built*/
4259                 jdata.data = data;
4260                 jdata.mi = (CamelMessageInfoBase *) mi;
4261                 g_hash_table_foreach ((GHashTable *) camel_session_get_junk_headers (camel_service_get_session (service)), (GHFunc) construct_junk_headers, &jdata);
4262                 g_datalist_clear (&data);
4263         }
4264         g_ptr_array_free (fetch_data, TRUE);
4265
4266         if (camel_application_is_exiting) {
4267                 /* it will hopefully update summary next time */
4268                 fetch_data = NULL;
4269                 goto lose;
4270         }
4271
4272         /* And add the entries to the summary, etc. */
4273         for (i = 0; i < messages->len; i++) {
4274                 mi = messages->pdata[i];
4275                 if (!mi) {
4276                         g_warning ("No information for message %d", i + first);
4277                         g_set_error (
4278                                 error, CAMEL_ERROR, CAMEL_ERROR_GENERIC,
4279                                 _("Incomplete server response: "
4280                                   "no information provided for message %d"),
4281                                 i + first);
4282                         break;
4283                 }
4284                 uid = (gchar *) camel_message_info_uid (mi);
4285                 if (uid[0] == 0) {
4286                         g_warning("Server provided no uid: message %d", i + first);
4287                         g_set_error (
4288                                 error, CAMEL_ERROR, CAMEL_ERROR_GENERIC,
4289                                 _("Incomplete server response: "
4290                                   "no UID provided for message %d"),
4291                                 i + first);
4292                         break;
4293                 }
4294
4295                 /* FIXME: If it enters if (info) it will always match the exception. So stupid */
4296                 /* FIXME[disk-summary] Use a db query to see if the DB exists */
4297 /*              info = (CamelImapMessageInfo *)camel_folder_summary_get (folder->summary, uid); */
4298 /*              if (info) { */
4299 /*                      for (seq = 0; seq < camel_folder_summary_count (folder->summary); seq++) { */
4300 /*                              if (folder->summary->messages->pdata[seq] == info) */
4301 /*                                      break; */
4302 /*                      } */
4303
4304                 ((CamelMessageInfoBase *) mi)->dirty = TRUE;
4305                 if (((CamelMessageInfoBase *) mi)->summary)
4306                         camel_folder_summary_touch (((CamelMessageInfoBase *) mi)->summary);
4307                 camel_folder_summary_add (folder->summary, (CamelMessageInfo *) mi);
4308                 camel_folder_change_info_add_uid (changes, camel_message_info_uid (mi));
4309
4310                 /* Report all new messages as recent, even without that flag, thus new
4311                  * messages will be filtered even after saw by other software earlier.
4312                  * Only skip those which we added ourself, like after drag&drop to this folder. */
4313                 if (!imap_folder_uid_in_ignore_recent (imap_folder, camel_message_info_uid (mi))
4314                     && ((mi->info.flags & CAMEL_IMAP_MESSAGE_RECENT) != 0 || getenv ("FILTER_RECENT") == NULL))
4315                         camel_folder_change_info_recent_uid (changes, camel_message_info_uid (mi));
4316
4317         }
4318
4319         g_ptr_array_free (messages, TRUE);
4320
4321         if (imap_folder->priv->ignore_recent) {
4322                 g_hash_table_unref (imap_folder->priv->ignore_recent);
4323                 imap_folder->priv->ignore_recent = NULL;
4324         }
4325
4326         return TRUE;
4327
4328  lose:
4329         if (fetch_data) {
4330                 for (i = 0; i < fetch_data->len; i++) {
4331                         data = fetch_data->pdata[i];
4332                         g_datalist_clear (&data);
4333                 }
4334                 g_ptr_array_free (fetch_data, TRUE);
4335         }
4336         if (messages) {
4337                 for (i = 0; i < messages->len; i++) {
4338                         if (messages->pdata[i])
4339                                 camel_message_info_free (messages->pdata[i]);
4340                 }
4341                 g_ptr_array_free (messages, TRUE);
4342         }
4343
4344         if (imap_folder->priv->ignore_recent) {
4345                 g_hash_table_unref (imap_folder->priv->ignore_recent);
4346                 imap_folder->priv->ignore_recent = NULL;
4347         }
4348
4349         return FALSE;
4350 }
4351
4352 /* Called with the store's connect_lock locked */
4353 gboolean
4354 camel_imap_folder_changed (CamelFolder *folder,
4355                            gint exists,
4356                            GArray *expunged,
4357                            GCancellable *cancellable,
4358                            GError **error)
4359 {
4360         CamelImapFolder *imap_folder = CAMEL_IMAP_FOLDER (folder);
4361         CamelFolderChangeInfo *changes;
4362         gint len;
4363         gboolean success = TRUE;
4364
4365         changes = camel_folder_change_info_new ();
4366         if (expunged) {
4367                 CamelStore *parent_store;
4368                 gint i, id;
4369                 GList *deleted = NULL;
4370                 const gchar *full_name;
4371                 const gchar *uid;
4372                 GPtrArray *known_uids;
4373
4374                 known_uids = camel_folder_summary_get_array (folder->summary);
4375                 camel_folder_sort_uids (folder, known_uids);
4376                 for (i = 0; i < expunged->len; i++) {
4377                         id = g_array_index (expunged, int, i);
4378                         uid = id - 1 + i >= 0 && id - 1 + i < known_uids->len ? g_ptr_array_index (known_uids, id - 1 + i) : NULL;
4379                         if (uid == NULL) {
4380                                 /* FIXME: danw: does this mean that the summary is corrupt? */
4381                                 /* I guess a message that we never retrieved got expunged? */
4382                                 continue;
4383                         }
4384
4385                         deleted = g_list_prepend (deleted, (gpointer) uid);
4386                         camel_folder_change_info_remove_uid (changes, uid);
4387                         CAMEL_IMAP_FOLDER_REC_LOCK (imap_folder, cache_lock);
4388                         camel_imap_message_cache_remove (imap_folder->cache, uid);
4389                         CAMEL_IMAP_FOLDER_REC_UNLOCK (imap_folder, cache_lock);
4390                         camel_folder_summary_remove_uid (folder->summary, uid);
4391                 }
4392
4393                 /* Delete all in one transaction */
4394                 full_name = camel_folder_get_full_name (folder);
4395                 parent_store = camel_folder_get_parent_store (folder);
4396                 camel_db_delete_uids (parent_store->cdb_w, full_name, deleted, NULL);
4397                 g_list_free (deleted);
4398
4399                 camel_folder_summary_free_array (known_uids);
4400         }
4401
4402         len = camel_folder_summary_count (folder->summary);
4403         if (exists > len && !camel_application_is_exiting)
4404                 success = imap_update_summary (
4405                         folder, exists, changes, cancellable, error);
4406
4407         camel_folder_summary_save_to_db (folder->summary, NULL);
4408         if (camel_folder_change_info_changed (changes))
4409                 camel_folder_changed (folder, changes);
4410
4411         camel_folder_change_info_free (changes);
4412
4413         return success;
4414 }
4415
4416 static void
4417 imap_thaw (CamelFolder *folder)
4418 {
4419         CamelImapFolder *imap_folder;
4420
4421         CAMEL_FOLDER_CLASS (camel_imap_folder_parent_class)->thaw (folder);
4422         if (camel_folder_is_frozen (folder))
4423                 return;
4424
4425         /* FIXME imap_refresh_info_sync() may block, but camel_folder_thaw()
4426          *       is not supposed to block.  Potential hang here. */
4427         imap_folder = CAMEL_IMAP_FOLDER (folder);
4428         if (imap_folder->need_refresh) {
4429                 imap_folder->need_refresh = FALSE;
4430                 imap_refresh_info_sync (folder, NULL, NULL);
4431         }
4432 }
4433
4434 CamelStream *
4435 camel_imap_folder_fetch_data (CamelImapFolder *imap_folder,
4436                               const gchar *uid,
4437                               const gchar *section_text,
4438                               gboolean cache_only,
4439                               GCancellable *cancellable,
4440                               GError **error)
4441 {
4442         CamelFolder *folder = CAMEL_FOLDER (imap_folder);
4443         CamelStore *parent_store;
4444         CamelImapStore *store;
4445         CamelImapResponse *response;
4446         CamelStream *stream;
4447         GData *fetch_data;
4448         gchar *found_uid;
4449         gint i;
4450
4451         parent_store = camel_folder_get_parent_store (folder);
4452         store = CAMEL_IMAP_STORE (parent_store);
4453
4454         /* EXPUNGE responses have to modify the cache, which means
4455          * they have to grab the cache_lock while holding the
4456          * connect_lock.
4457          *
4458          * Because getting the service lock may cause MUCH unecessary
4459          * delay when we already have the data locally, we do the
4460          * locking separately.  This could cause a race
4461          * getting the same data from the cache, but that is only
4462          * an inefficiency, and bad luck.
4463          */
4464         CAMEL_IMAP_FOLDER_REC_LOCK (imap_folder, cache_lock);
4465         stream = camel_imap_message_cache_get (imap_folder->cache, uid, section_text, NULL);
4466         if (!stream && (!strcmp (section_text, "HEADER") || !strcmp (section_text, "0"))) {
4467                 stream = camel_imap_message_cache_get (imap_folder->cache, uid, "", NULL);
4468         }
4469         CAMEL_IMAP_FOLDER_REC_UNLOCK (imap_folder, cache_lock);
4470
4471         if (stream || cache_only)
4472                 return stream;
4473
4474         camel_service_lock (CAMEL_SERVICE (store), CAMEL_SERVICE_REC_CONNECT_LOCK);
4475         CAMEL_IMAP_FOLDER_REC_LOCK (imap_folder, cache_lock);
4476
4477         if (!camel_imap_store_connected (store, NULL)) {
4478                 g_set_error (
4479                         error, CAMEL_ERROR,
4480                         CAMEL_SERVICE_ERROR_UNAVAILABLE,
4481                         _("This message is not currently available"));
4482                 CAMEL_IMAP_FOLDER_REC_UNLOCK (imap_folder, cache_lock);
4483                 camel_service_unlock (CAMEL_SERVICE (store), CAMEL_SERVICE_REC_CONNECT_LOCK);
4484                 return NULL;
4485         }
4486
4487         if (store->server_level < IMAP_LEVEL_IMAP4REV1 && !*section_text) {
4488                 response = camel_imap_command (
4489                         store, folder, cancellable, error,
4490                         "UID FETCH %s RFC822.PEEK", uid);
4491         } else {
4492                 response = camel_imap_command (
4493                         store, folder, cancellable, error,
4494                         "UID FETCH %s BODY.PEEK[%s]", uid,
4495                         section_text);
4496         }
4497
4498         if (!response) {
4499                 CAMEL_IMAP_FOLDER_REC_UNLOCK (imap_folder, cache_lock);
4500                 camel_service_unlock (CAMEL_SERVICE (store), CAMEL_SERVICE_REC_CONNECT_LOCK);
4501                 return NULL;
4502         }
4503
4504         for (i = 0; i < response->untagged->len; i++) {
4505                 fetch_data = parse_fetch_response (imap_folder, response->untagged->pdata[i]);
4506                 found_uid = g_datalist_get_data (&fetch_data, "UID");
4507                 stream = g_datalist_get_data (&fetch_data, "BODY_PART_STREAM");
4508                 if (found_uid && stream && !strcmp (uid, found_uid))
4509                         break;
4510
4511                 g_datalist_clear (&fetch_data);
4512                 stream = NULL;
4513         }
4514         camel_imap_response_free (store, response);
4515         CAMEL_IMAP_FOLDER_REC_UNLOCK (imap_folder, cache_lock);
4516         camel_service_unlock (CAMEL_SERVICE (store), CAMEL_SERVICE_REC_CONNECT_LOCK);
4517         if (!stream) {
4518                 g_set_error (
4519                         error, CAMEL_ERROR,
4520                         CAMEL_SERVICE_ERROR_UNAVAILABLE,
4521                         _("Could not find message body in FETCH response."));
4522         } else {
4523                 g_object_ref (stream);
4524                 g_datalist_clear (&fetch_data);
4525         }
4526
4527         return stream;
4528 }
4529
4530 static GData *
4531 parse_fetch_response (CamelImapFolder *imap_folder,
4532                       gchar *response)
4533 {
4534         GData *data = NULL;
4535         gchar *start, *part_spec = NULL, *body = NULL, *uid = NULL, *idate = NULL;
4536         gboolean cache_header = TRUE, header = FALSE;
4537         gsize body_len = 0;
4538
4539         if (*response != '(') {
4540                 glong seq;
4541
4542                 if (*response != '*' || *(response + 1) != ' ')
4543                         return NULL;
4544                 seq = strtoul (response + 2, &response, 10);
4545                 if (seq == 0)
4546                         return NULL;
4547                 if (g_ascii_strncasecmp (response, " FETCH (", 8) != 0)
4548                         return NULL;
4549                 response += 7;
4550
4551                 g_datalist_set_data (&data, "SEQUENCE", GINT_TO_POINTER (seq));
4552         }
4553
4554         do {
4555                 /* Skip the initial '(' or the ' ' between elements */
4556                 response++;
4557
4558                 if (!g_ascii_strncasecmp (response, "FLAGS ", 6)) {
4559                         CamelMessageFlags flags;
4560                         gchar *custom_flags = NULL;
4561
4562                         response += 6;
4563
4564                         if (imap_parse_flag_list (&response, &flags, &custom_flags)) {
4565                                 g_datalist_set_data (&data, "FLAGS", GUINT_TO_POINTER (flags));
4566
4567                                 if (custom_flags)
4568                                         g_datalist_set_data_full (&data, "CUSTOM.FLAGS", custom_flags, g_free);
4569                         }
4570                 } else if (!g_ascii_strncasecmp (response, "RFC822.SIZE ", 12)) {
4571                         gulong size;
4572
4573                         response += 12;
4574                         size = strtoul (response, &response, 10);
4575                         g_datalist_set_data (&data, "RFC822.SIZE", GUINT_TO_POINTER (size));
4576                 } else if (!g_ascii_strncasecmp (response, "BODY[", 5) ||
4577                            !g_ascii_strncasecmp (response, "RFC822 ", 7)) {
4578                         gchar *p;
4579
4580                         if (*response == 'B') {
4581                                 response += 5;
4582
4583                                 /* HEADER], HEADER.FIELDS (...)], or 0] */
4584                                 if (!g_ascii_strncasecmp (response, "HEADER", 6)) {
4585                                         header = TRUE;
4586                                         if (!g_ascii_strncasecmp (response + 6, ".FIELDS", 7))
4587                                                 cache_header = FALSE;
4588                                 } else if (!g_ascii_strncasecmp (response, "0]", 2))
4589                                         header = TRUE;
4590
4591                                 p = strchr (response, ']');
4592                                 if (!p || *(p + 1) != ' ')
4593                                         break;
4594
4595                                 if (cache_header)
4596                                         part_spec = g_strndup (response, p - response);
4597                                 else
4598                                         part_spec = g_strdup ("HEADER.FIELDS");
4599
4600                                 response = p + 2;
4601                         } else {
4602                                 part_spec = g_strdup ("");
4603                                 response += 7;
4604
4605                                 if (!g_ascii_strncasecmp (response, "HEADER", 6))
4606                                         header = TRUE;
4607                         }
4608
4609                         body = imap_parse_nstring ((const gchar **) &response, &body_len);
4610                         if (!response) {
4611                                 g_free (part_spec);
4612                                 break;
4613                         }
4614
4615                         if (!body)
4616                                 body = g_strdup ("");
4617                         g_datalist_set_data_full (&data, "BODY_PART_SPEC", part_spec, g_free);
4618                         g_datalist_set_data_full (&data, "BODY_PART_DATA", body, g_free);
4619                         g_datalist_set_data (&data, "BODY_PART_LEN", GINT_TO_POINTER (body_len));
4620                 } else if (!g_ascii_strncasecmp (response, "BODY ", 5) ||
4621                            !g_ascii_strncasecmp (response, "BODYSTRUCTURE ", 14)) {
4622                         response = strchr (response, ' ') + 1;
4623                         start = response;
4624                         imap_skip_list ((const gchar **) &response);
4625                         if (response && (response != start)) {
4626                                 /* To handle IMAP Server brokenness, Returning empty body, etc. See #355640 */
4627                                 g_datalist_set_data_full (&data, "BODY", g_strndup (start, response - start), g_free);
4628                         }
4629                 } else if (!g_ascii_strncasecmp (response, "UID ", 4)) {
4630                         gint len;
4631
4632                         len = strcspn (response + 4, " )");
4633                         uid = g_strndup (response + 4, len);
4634                         g_datalist_set_data_full (&data, "UID", uid, g_free);
4635                         response += 4 + len;
4636                 } else if (!g_ascii_strncasecmp (response, "INTERNALDATE ", 13)) {
4637                         gint len;
4638
4639                         response += 13;
4640                         if (*response == '"') {
4641                                 response++;
4642                                 len = strcspn (response, "\"");
4643                                 idate = g_strndup (response, len);
4644                                 g_datalist_set_data_full (&data, "INTERNALDATE", idate, g_free);
4645                                 response += len + 1;
4646                         }
4647                 } else {
4648                         g_warning ("Unexpected FETCH response from server: (%s", response);
4649                         break;
4650                 }
4651         } while (response && *response != ')');
4652
4653         if (!response || *response != ')') {
4654                 g_datalist_clear (&data);
4655                 return NULL;
4656         }
4657
4658         if (uid && body) {
4659                 CamelStream *stream;
4660
4661                 if (header && !cache_header) {
4662                         stream = camel_stream_mem_new_with_buffer (body, body_len);
4663                 } else {
4664                         CAMEL_IMAP_FOLDER_REC_LOCK (imap_folder, cache_lock);
4665                         stream = camel_imap_message_cache_insert (imap_folder->cache,
4666                                                                   uid, part_spec,
4667                                                                   body, body_len, NULL, NULL);
4668                         CAMEL_IMAP_FOLDER_REC_UNLOCK (imap_folder, cache_lock);
4669                         if (stream == NULL)
4670                                 stream = camel_stream_mem_new_with_buffer (body, body_len);
4671                 }
4672
4673                 if (stream)
4674                         g_datalist_set_data_full (&data, "BODY_PART_STREAM", stream,
4675                                                   (GDestroyNotify) g_object_unref);
4676         }
4677
4678         return data;
4679 }
4680
4681 /* it uses connect_lock, thus be sure it doesn't run in main thread */
4682 static CamelFolderQuotaInfo *
4683 imap_get_quota_info_sync (CamelFolder *folder,
4684                           GCancellable *cancellable,
4685                           GError **error)
4686 {
4687         CamelStore *parent_store;
4688         CamelImapStore *imap_store;
4689         CamelImapResponse *response;
4690         CamelFolderQuotaInfo *res = NULL, *last = NULL;
4691
4692         parent_store = camel_folder_get_parent_store (folder);
4693         imap_store = CAMEL_IMAP_STORE (parent_store);
4694
4695         if (!camel_offline_store_get_online (CAMEL_OFFLINE_STORE (imap_store)))
4696                 return NULL;
4697
4698         camel_service_lock (CAMEL_SERVICE (imap_store), CAMEL_SERVICE_REC_CONNECT_LOCK);
4699
4700         if (!camel_imap_store_connected (imap_store, NULL))
4701                 goto done;
4702
4703         if (imap_store->capabilities & IMAP_CAPABILITY_QUOTA) {
4704                 const gchar *full_name = camel_folder_get_full_name (folder);
4705                 CamelImapStoreNamespace *ns = camel_imap_store_summary_namespace_find_full (imap_store->summary, full_name);
4706                 gchar *folder_name = camel_imap_store_summary_path_to_full (imap_store->summary, full_name, ns ? ns->sep : '/');
4707
4708                 response = camel_imap_command (imap_store, NULL, cancellable, error, "GETQUOTAROOT \"%s\"", folder_name);
4709
4710                 if (response) {
4711                         gint i;
4712
4713                         for (i = 0; i < response->untagged->len; i++) {
4714                                 const gchar *resp = response->untagged->pdata[i];
4715
4716                                 if (resp && g_str_has_prefix (resp, "* QUOTA ")) {
4717                                         gboolean skipped = TRUE;
4718                                         gsize sz;
4719                                         gchar *astr;
4720
4721                                         resp = resp + 8;
4722                                         astr = imap_parse_astring (&resp, &sz);
4723                                         g_free (astr);
4724
4725                                         while (resp && *resp && *resp != '(')
4726                                                 resp++;
4727
4728                                         if (resp && *resp == '(') {
4729                                                 gchar *name;
4730                                                 const gchar *used = NULL, *total = NULL;
4731
4732                                                 resp++;
4733                                                 name = imap_parse_astring (&resp, &sz);
4734
4735                                                 if (resp)
4736                                                         used = imap_next_word (resp);
4737                                                 if (used)
4738                                                         total = imap_next_word (used);
4739
4740                                                 while (resp && *resp && *resp != ')')
4741                                                         resp++;
4742
4743                                                 if (resp && *resp == ')' && used && total) {
4744                                                         guint64 u, t;
4745
4746                                                         u = strtoull (used, NULL, 10);
4747                                                         t = strtoull (total, NULL, 10);
4748
4749                                                         if (t > 0) {
4750                                                                 CamelFolderQuotaInfo *info = camel_folder_quota_info_new (name, u, t);
4751
4752                                                                 if (last)
4753                                                                         last->next = info;
4754                                                                 else
4755                                                                         res = info;
4756
4757                                                                 last = info;
4758                                                                 skipped = FALSE;
4759                                                         }
4760                                                 }
4761
4762                                                 g_free (name);
4763                                         }
4764
4765                                         if (skipped)
4766                                                 g_debug ("Unexpected quota response '%s'; skipping it...", (const gchar *)response->untagged->pdata[i]);
4767                                 }
4768                         }
4769                         camel_imap_response_free (imap_store, response);
4770                 }
4771
4772                 g_free (folder_name);
4773         }
4774 done:
4775         camel_service_unlock (CAMEL_SERVICE (imap_store), CAMEL_SERVICE_REC_CONNECT_LOCK);
4776         return res;
4777 }
4778
4779 /**
4780  * Scan for messages that are local and return the rest.
4781  */
4782 static GPtrArray *
4783 imap_get_uncached_uids (CamelFolder *folder,
4784                         GPtrArray *uids,
4785                         GError **error)
4786 {
4787         GPtrArray *result;
4788         CamelImapFolder *imap_folder = CAMEL_IMAP_FOLDER (folder);
4789
4790         CAMEL_IMAP_FOLDER_REC_LOCK (imap_folder, cache_lock);
4791
4792         result = camel_imap_message_cache_filter_cached (
4793                 imap_folder->cache, uids, error);
4794
4795         CAMEL_IMAP_FOLDER_REC_UNLOCK (imap_folder, cache_lock);
4796
4797         return result;
4798 }
4799