camel/providers/imap/camel-imap-folder.c

   1 /* -*- Mode: C; tab-width: 8; indent-tabs-mode: t; c-basic-offset: 8 -*- */
   2 /* camel-imap-folder.c: class for an imap folder */
   3
   4 /*
   5  * Authors:
   6  *   Dan Winship <danw@ximian.com>
   7  *   Jeffrey Stedfast <fejj@ximian.com>
   8  *
   9  * Copyright (C) 1999-2008 Novell, Inc. (www.novell.com)
  10  *
  11  * This program is free software; you can redistribute it and/or
  12  * modify it under the terms of version 2 of the GNU Lesser General Public
  13  * License as published by the Free Software Foundation.
  14  *
  15  * This program is distributed in the hope that it will be useful,
  16  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  17  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  18  * GNU Lesser General Public License for more details.
  19  *
  20  * You should have received a copy of the GNU Lesser General Public License
  21  * along with this program; if not, write to the Free Software
  22  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301
  23  * USA
  24  */
  25
  26 #include <config.h>
  27
  28 #include <ctype.h>
  29 #include <errno.h>
  30 #include <fcntl.h>
  31 #include <stdlib.h>
  32 #include <string.h>
  33 #include <unistd.h>
  34 #include <sys/stat.h>
  35 #include <sys/types.h>
  36
  37 #include <glib/gi18n-lib.h>
  38
  39 #include <camel/camel.h>
  40
  41 #include "camel-imap-command.h"
  42 #include "camel-imap-folder.h"
  43 #include "camel-imap-journal.h"
  44 #include "camel-imap-message-cache.h"
  45 #include "camel-imap-private.h"
  46 #include "camel-imap-search.h"
  47 #include "camel-imap-settings.h"
  48 #include "camel-imap-store.h"
  49 #include "camel-imap-store-summary.h"
  50 #include "camel-imap-summary.h"
  51 #include "camel-imap-utils.h"
  52 #include "camel-imap-wrapper.h"
  53
  54 #define d(x)
  55
  56 /* set to -1 for infinite size (suggested max command-line length is
  57  * 1000 octets (see rfc2683), so we should keep the uid-set length to
  58  * something under that so that our command-lines don't exceed 1000
  59  * octets) */
  60 #define UID_SET_LIMIT  (768)
  61
  62 #define CAMEL_IMAP_FOLDER_GET_PRIVATE(obj) \
  63         (G_TYPE_INSTANCE_GET_PRIVATE \
  64         ((obj), CAMEL_TYPE_IMAP_FOLDER, CamelImapFolderPrivate))
  65
  66 /* The custom property ID is a CamelArg artifact.
  67  * It still identifies the property in state files. */
  68 enum {
  69         PROP_0,
  70         PROP_CHECK_FOLDER = 0x2500,
  71         PROP_APPLY_FILTERS
  72 };
  73
  74 extern gint camel_application_is_exiting;
  75
  76 static gboolean imap_rescan (CamelFolder *folder, gint exists, GCancellable *cancellable, GError **error);
  77 static gboolean imap_refresh_info_sync (CamelFolder *folder, GCancellable *cancellable, GError **error);
  78 static gboolean imap_sync_offline (CamelFolder *folder, GError **error);
  79 static gboolean imap_synchronize_sync (CamelFolder *folder, gboolean expunge, GCancellable *cancellable, GError **error);
  80 static gboolean imap_expunge_uids_online (CamelFolder *folder, GPtrArray *uids, GCancellable *cancellable, GError **error);
  81 static gboolean imap_expunge_uids_offline (CamelFolder *folder, GPtrArray *uids, GCancellable *cancellable, GError **error);
  82 static gboolean imap_expunge_sync (CamelFolder *folder, GCancellable *cancellable, GError **error);
  83 /*static void imap_cache_message (CamelDiscoFolder *disco_folder, const gchar *uid, GError **error);*/
  84 static void imap_rename (CamelFolder *folder, const gchar *new);
  85 static GPtrArray * imap_get_uncached_uids (CamelFolder *folder, GPtrArray * uids, GError **error);
  86 static gchar * imap_get_filename (CamelFolder *folder, const gchar *uid, GError **error);
  87
  88 /* message manipulation */
  89 static CamelMimeMessage *imap_get_message_sync (CamelFolder *folder, const gchar *uid, GCancellable *cancellable,
  90                                            GError **error);
  91 static gboolean imap_synchronize_message_sync (CamelFolder *folder, const gchar *uid, GCancellable *cancellable,
  92                                GError **error);
  93 static gboolean imap_append_online (CamelFolder *folder, CamelMimeMessage *message,
  94                                 CamelMessageInfo *info, gchar **appended_uid,
  95                                 GCancellable *cancellable,
  96                                 GError **error);
  97 static gboolean imap_append_offline (CamelFolder *folder, CamelMimeMessage *message,
  98                                  CamelMessageInfo *info, gchar **appended_uid,
  99                                  GError **error);
 100
 101 static gboolean imap_transfer_online            (CamelFolder *source,
 102                                                  GPtrArray *uids,
 103                                                  CamelFolder *dest,
 104                                                  gboolean delete_originals,
 105                                                  GPtrArray **transferred_uids,
 106                                                  GCancellable *cancellable,
 107                                                  GError **error);
 108 static gboolean imap_transfer_offline           (CamelFolder *source,
 109                                                  GPtrArray *uids,
 110                                                  CamelFolder *dest,
 111                                                  gboolean delete_originals,
 112                                                  GPtrArray **transferred_uids,
 113                                                  GCancellable *cancellable,
 114                                                  GError **error);
 115
 116 /* searching */
 117 static GPtrArray *imap_search_by_expression (CamelFolder *folder, const gchar *expression, GCancellable *cancellable, GError **error);
 118 static guint32 imap_count_by_expression (CamelFolder *folder, const gchar *expression, GCancellable *cancellable, GError **error);
 119 static GPtrArray *imap_search_by_uids       (CamelFolder *folder, const gchar *expression, GPtrArray *uids, GCancellable *cancellable, GError **error);
 120 static void       imap_search_free          (CamelFolder *folder, GPtrArray *uids);
 121
 122 static void imap_thaw (CamelFolder *folder);
 123 static CamelFolderQuotaInfo *
 124                 imap_get_quota_info_sync        (CamelFolder *folder,
 125                                                  GCancellable *cancellable,
 126                                                  GError **error);
 127
 128 static GData *parse_fetch_response (CamelImapFolder *imap_folder, gchar *msg_att);
 129
 130 /* internal helpers */
 131 static CamelImapMessageInfo * imap_folder_summary_uid_or_error (
 132         CamelFolderSummary *summary,
 133         const gchar * uid,
 134         GError **error);
 135
 136 static gboolean imap_transfer_messages          (CamelFolder *source,
 137                                                  GPtrArray *uids,
 138                                                  CamelFolder *dest,
 139                                                  gboolean delete_originals,
 140                                                  GPtrArray **transferred_uids,
 141                                                  gboolean can_call_sync,
 142                                                  GCancellable *cancellable,
 143                                                  GError **error);
 144
 145 static gboolean
 146 imap_folder_get_apply_filters (CamelImapFolder *folder)
 147 {
 148         g_return_val_if_fail (folder != NULL, FALSE);
 149         g_return_val_if_fail (CAMEL_IS_IMAP_FOLDER (folder), FALSE);
 150
 151         return folder->priv->apply_filters;
 152 }
 153
 154 static void
 155 imap_folder_set_apply_filters (CamelImapFolder *folder,
 156                                gboolean apply_filters)
 157 {
 158         g_return_if_fail (folder != NULL);
 159         g_return_if_fail (CAMEL_IS_IMAP_FOLDER (folder));
 160
 161         if ((folder->priv->apply_filters ? 1 : 0) == (apply_filters ? 1 : 0))
 162                 return;
 163
 164         folder->priv->apply_filters = apply_filters;
 165
 166         g_object_notify (G_OBJECT (folder), "apply-filters");
 167 }
 168
 169 #ifdef G_OS_WIN32
 170 /* The strtok() in Microsoft's C library is MT-safe (but still uses
 171  * only one buffer pointer per thread, but for the use of strtok_r()
 172  * here that's enough).
 173  */
 174 #define strtok_r(s,sep,lasts) (*(lasts)=strtok((s),(sep)))
 175 #endif
 176
 177 G_DEFINE_TYPE (CamelImapFolder, camel_imap_folder, CAMEL_TYPE_OFFLINE_FOLDER)
 178
 179 static void
 180 imap_folder_set_property (GObject *object,
 181                           guint property_id,
 182                           const GValue *value,
 183                           GParamSpec *pspec)
 184 {
 185         switch (property_id) {
 186                 case PROP_CHECK_FOLDER:
 187                         camel_imap_folder_set_check_folder (
 188                                 CAMEL_IMAP_FOLDER (object),
 189                                 g_value_get_boolean (value));
 190                         return;
 191
 192                 case PROP_APPLY_FILTERS:
 193                         imap_folder_set_apply_filters (
 194                                 CAMEL_IMAP_FOLDER (object),
 195                                 g_value_get_boolean (value));
 196                         return;
 197         }
 198
 199         G_OBJECT_WARN_INVALID_PROPERTY_ID (object, property_id, pspec);
 200 }
 201
 202 static void
 203 imap_folder_get_property (GObject *object,
 204                           guint property_id,
 205                           GValue *value,
 206                           GParamSpec *pspec)
 207 {
 208         switch (property_id) {
 209                 case PROP_CHECK_FOLDER:
 210                         g_value_set_boolean (
 211                                 value, camel_imap_folder_get_check_folder (
 212                                 CAMEL_IMAP_FOLDER (object)));
 213                         return;
 214
 215                 case PROP_APPLY_FILTERS:
 216                         g_value_set_boolean (
 217                                 value, imap_folder_get_apply_filters (
 218                                 CAMEL_IMAP_FOLDER (object)));
 219                         return;
 220         }
 221
 222         G_OBJECT_WARN_INVALID_PROPERTY_ID (object, property_id, pspec);
 223 }
 224
 225 static void
 226 imap_folder_dispose (GObject *object)
 227 {
 228         CamelImapFolder *imap_folder;
 229         CamelStore *parent_store;
 230
 231         imap_folder = CAMEL_IMAP_FOLDER (object);
 232
 233         parent_store = camel_folder_get_parent_store (CAMEL_FOLDER (imap_folder));
 234         if (parent_store) {
 235                 camel_store_summary_disconnect_folder_summary (
 236                         (CamelStoreSummary *) ((CamelImapStore *) parent_store)->summary,
 237                         CAMEL_FOLDER (imap_folder)->summary);
 238         }
 239
 240         if (imap_folder->search != NULL) {
 241                 g_object_unref (imap_folder->search);
 242                 imap_folder->search = NULL;
 243         }
 244
 245         if (imap_folder->cache != NULL) {
 246                 g_object_unref (imap_folder->cache);
 247                 imap_folder->cache = NULL;
 248         }
 249
 250         if (imap_folder->priv->ignore_recent != NULL) {
 251                 g_hash_table_unref (imap_folder->priv->ignore_recent);
 252                 imap_folder->priv->ignore_recent = NULL;
 253         }
 254
 255         if (imap_folder->journal != NULL) {
 256                 camel_offline_journal_write (imap_folder->journal, NULL);
 257                 g_object_unref (imap_folder->journal);
 258                 imap_folder->journal = NULL;
 259         }
 260
 261         /* Chain up to parent's dispose() method. */
 262         G_OBJECT_CLASS (camel_imap_folder_parent_class)->dispose (object);
 263 }
 264
 265 static void
 266 imap_folder_finalize (GObject *object)
 267 {
 268         CamelImapFolder *imap_folder;
 269
 270         imap_folder = CAMEL_IMAP_FOLDER (object);
 271
 272         g_static_mutex_free (&imap_folder->priv->search_lock);
 273         g_static_rec_mutex_free (&imap_folder->priv->cache_lock);
 274
 275         /* Chain up to parent's finalize() method. */
 276         G_OBJECT_CLASS (camel_imap_folder_parent_class)->finalize (object);
 277 }
 278
 279 static void
 280 imap_folder_constructed (GObject *object)
 281 {
 282         CamelNetworkSettings *network_settings;
 283         CamelSettings *settings;
 284         CamelService *service;
 285         CamelFolder *folder;
 286         CamelStore *parent_store;
 287         const gchar *full_name;
 288         gchar *description;
 289         gchar *host;
 290         gchar *user;
 291
 292         folder = CAMEL_FOLDER (object);
 293         full_name = camel_folder_get_full_name (folder);
 294         parent_store = camel_folder_get_parent_store (folder);
 295
 296         service = CAMEL_SERVICE (parent_store);
 297
 298         settings = camel_service_ref_settings (service);
 299
 300         network_settings = CAMEL_NETWORK_SETTINGS (settings);
 301         host = camel_network_settings_dup_host (network_settings);
 302         user = camel_network_settings_dup_user (network_settings);
 303
 304         g_object_unref (settings);
 305
 306         description = g_strdup_printf (
 307                 "%s@%s:%s", user, host, full_name);
 308         camel_folder_set_description (folder, description);
 309         g_free (description);
 310
 311         g_free (host);
 312         g_free (user);
 313 }
 314
 315 static void
 316 camel_imap_folder_class_init (CamelImapFolderClass *class)
 317 {
 318         GObjectClass *object_class;
 319         CamelFolderClass *folder_class;
 320
 321         g_type_class_add_private (class, sizeof (CamelImapFolderPrivate));
 322
 323         object_class = G_OBJECT_CLASS (class);
 324         object_class->set_property = imap_folder_set_property;
 325         object_class->get_property = imap_folder_get_property;
 326         object_class->dispose = imap_folder_dispose;
 327         object_class->finalize = imap_folder_finalize;
 328         object_class->constructed = imap_folder_constructed;
 329
 330         folder_class = CAMEL_FOLDER_CLASS (class);
 331         folder_class->rename = imap_rename;
 332         folder_class->search_by_expression = imap_search_by_expression;
 333         folder_class->count_by_expression = imap_count_by_expression;
 334         folder_class->search_by_uids = imap_search_by_uids;
 335         folder_class->search_free = imap_search_free;
 336         folder_class->thaw = imap_thaw;
 337         folder_class->get_uncached_uids = imap_get_uncached_uids;
 338         folder_class->get_filename = imap_get_filename;
 339         folder_class->append_message_sync = imap_append_online;
 340         folder_class->expunge_sync = imap_expunge_sync;
 341         folder_class->get_message_sync = imap_get_message_sync;
 342         folder_class->get_quota_info_sync = imap_get_quota_info_sync;
 343         folder_class->refresh_info_sync = imap_refresh_info_sync;
 344         folder_class->synchronize_sync = imap_synchronize_sync;
 345         folder_class->synchronize_message_sync = imap_synchronize_message_sync;
 346         folder_class->transfer_messages_to_sync = imap_transfer_online;
 347
 348         g_object_class_install_property (
 349                 object_class,
 350                 PROP_CHECK_FOLDER,
 351                 g_param_spec_boolean (
 352                         "check-folder",
 353                         "Check Folder",
 354                         _("Always check for _new mail in this folder"),
 355                         FALSE,
 356                         G_PARAM_READWRITE |
 357                         CAMEL_PARAM_PERSISTENT));
 358
 359         g_object_class_install_property (
 360                 object_class,
 361                 PROP_APPLY_FILTERS,
 362                 g_param_spec_boolean (
 363                         "apply-filters",
 364                         "Apply Filters",
 365                         _("Apply message _filters to this folder"),
 366                         FALSE,
 367                         G_PARAM_READWRITE |
 368                         CAMEL_PARAM_PERSISTENT));
 369 }
 370
 371 static void
 372 camel_imap_folder_init (CamelImapFolder *imap_folder)
 373 {
 374         CamelFolder *folder = CAMEL_FOLDER (imap_folder);
 375
 376         imap_folder->priv = CAMEL_IMAP_FOLDER_GET_PRIVATE (imap_folder);
 377
 378         folder->permanent_flags = CAMEL_MESSAGE_ANSWERED | CAMEL_MESSAGE_DELETED |
 379                 CAMEL_MESSAGE_DRAFT | CAMEL_MESSAGE_FLAGGED | CAMEL_MESSAGE_SEEN;
 380
 381         folder->folder_flags |= CAMEL_FOLDER_HAS_SUMMARY_CAPABILITY;
 382
 383         g_static_mutex_init (&imap_folder->priv->search_lock);
 384         g_static_rec_mutex_init (&imap_folder->priv->cache_lock);
 385         imap_folder->priv->ignore_recent = NULL;
 386
 387         imap_folder->journal = NULL;
 388         imap_folder->need_rescan = TRUE;
 389 }
 390
 391 static void
 392 replay_offline_journal (CamelImapStore *imap_store,
 393                         CamelImapFolder *imap_folder,
 394                         GCancellable *cancellable,
 395                         GError **error)
 396 {
 397         CamelIMAPJournal *imap_journal;
 398
 399         g_return_if_fail (imap_store != NULL);
 400         g_return_if_fail (imap_folder != NULL);
 401         g_return_if_fail (imap_folder->journal != NULL);
 402
 403         imap_journal = CAMEL_IMAP_JOURNAL (imap_folder->journal);
 404         g_return_if_fail (imap_journal != NULL);
 405
 406         /* do not replay when still in offline */
 407         if (!camel_offline_store_get_online (CAMEL_OFFLINE_STORE (imap_store)) || !camel_imap_store_connected (imap_store, error))
 408                 return;
 409
 410         /* Check if the replay is already in progress as imap_sync would be called while expunge resync */
 411         if (!imap_journal->rp_in_progress) {
 412                 imap_journal->rp_in_progress++;
 413
 414                 camel_offline_journal_replay (
 415                         imap_folder->journal, cancellable, error);
 416                 camel_imap_journal_close_folders (imap_journal);
 417                 camel_offline_journal_write (imap_folder->journal, error);
 418
 419                 imap_journal->rp_in_progress--;
 420                 g_return_if_fail (imap_journal->rp_in_progress >= 0);
 421         }
 422 }
 423
 424 CamelFolder *
 425 camel_imap_folder_new (CamelStore *parent,
 426                        const gchar *folder_name,
 427                        const gchar *folder_dir,
 428                        GError **error)
 429 {
 430         CamelFolder *folder;
 431         CamelImapFolder *imap_folder;
 432         const gchar *short_name;
 433         gchar *state_file, *path;
 434         CamelService *service;
 435         CamelSettings *settings;
 436         gboolean filter_all;
 437         gboolean filter_inbox;
 438         gboolean filter_junk;
 439         gboolean filter_junk_inbox;
 440
 441         if (g_mkdir_with_parents (folder_dir, S_IRWXU) != 0) {
 442                 g_set_error (
 443                         error, G_IO_ERROR,
 444                         g_io_error_from_errno (errno),
 445                         _("Could not create directory %s: %s"),
 446                         folder_dir, g_strerror (errno));
 447                 return NULL;
 448         }
 449
 450         short_name = strrchr (folder_name, '/');
 451         if (short_name)
 452                 short_name++;
 453         else
 454                 short_name = folder_name;
 455         folder = g_object_new (
 456                 CAMEL_TYPE_IMAP_FOLDER,
 457                 "full-name", folder_name,
 458                 "display-name", short_name,
 459                 "parent-store", parent, NULL);
 460
 461         folder->summary = camel_imap_summary_new (folder);
 462         if (!folder->summary) {
 463                 g_object_unref (folder);
 464                 g_set_error (
 465                         error, CAMEL_ERROR, CAMEL_ERROR_GENERIC,
 466                         _("Could not load summary for %s"), folder_name);
 467                 return NULL;
 468         }
 469
 470         imap_folder = CAMEL_IMAP_FOLDER (folder);
 471         path = g_build_filename (folder_dir, "journal", NULL);
 472         imap_folder->journal = camel_imap_journal_new (imap_folder, path);
 473         g_free (path);
 474
 475         /* set/load persistent state */
 476         state_file = g_build_filename (folder_dir, "cmeta", NULL);
 477         camel_object_set_state_filename (CAMEL_OBJECT (folder), state_file);
 478         g_free (state_file);
 479         camel_object_state_read (CAMEL_OBJECT (folder));
 480
 481         imap_folder->cache = camel_imap_message_cache_new (folder_dir, folder->summary, error);
 482         if (!imap_folder->cache) {
 483                 g_object_unref (folder);
 484                 return NULL;
 485         }
 486
 487         service = CAMEL_SERVICE (parent);
 488         settings = camel_service_ref_settings (service);
 489
 490         g_object_get (
 491                 settings,
 492                 "filter-all", &filter_all,
 493                 "filter-inbox", &filter_inbox,
 494                 "filter-junk", &filter_junk,
 495                 "filter-junk-inbox", &filter_junk_inbox,
 496                 NULL);
 497
 498         if (g_ascii_strcasecmp (folder_name, "INBOX") == 0) {
 499                 if (filter_inbox || filter_all)
 500                         folder->folder_flags |= CAMEL_FOLDER_FILTER_RECENT;
 501                 if (filter_junk)
 502                         folder->folder_flags |= CAMEL_FOLDER_FILTER_JUNK;
 503                 if (filter_junk_inbox)
 504                         folder->folder_flags |= CAMEL_FOLDER_FILTER_JUNK;
 505         } else {
 506                 gboolean folder_is_trash, folder_is_junk;
 507                 gchar *junk_path;
 508                 gchar *trash_path;
 509
 510                 junk_path = camel_imap_settings_dup_real_junk_path (
 511                         CAMEL_IMAP_SETTINGS (settings));
 512
 513                 /* So we can safely compare strings. */
 514                 if (junk_path == NULL)
 515                         junk_path = g_strdup ("");
 516
 517                 trash_path = camel_imap_settings_dup_real_trash_path (
 518                         CAMEL_IMAP_SETTINGS (settings));
 519
 520                 /* So we can safely compare strings. */
 521                 if (trash_path == NULL)
 522                         trash_path = g_strdup ("");
 523
 524                 if (filter_junk && !filter_junk_inbox)
 525                         folder->folder_flags |= CAMEL_FOLDER_FILTER_JUNK;
 526
 527                 folder_is_trash =
 528                         (parent->flags & CAMEL_STORE_VTRASH) == 0 &&
 529                         g_ascii_strcasecmp (trash_path, folder_name) == 0;
 530
 531                 if (folder_is_trash)
 532                         folder->folder_flags |= CAMEL_FOLDER_IS_TRASH;
 533
 534                 folder_is_junk =
 535                         (parent->flags & CAMEL_STORE_VJUNK) == 0 &&
 536                         g_ascii_strcasecmp (junk_path, folder_name) == 0;
 537
 538                 if (folder_is_junk)
 539                         folder->folder_flags |= CAMEL_FOLDER_IS_JUNK;
 540
 541                 if (filter_all || imap_folder_get_apply_filters (imap_folder))
 542                         folder->folder_flags |= CAMEL_FOLDER_FILTER_RECENT;
 543
 544                 g_free (junk_path);
 545                 g_free (trash_path);
 546         }
 547
 548         g_object_unref (settings);
 549
 550         imap_folder->search = camel_imap_search_new (folder_dir);
 551
 552         camel_store_summary_connect_folder_summary (
 553                 (CamelStoreSummary *) ((CamelImapStore *) parent)->summary,
 554                 folder_name, folder->summary);
 555
 556         return folder;
 557 }
 558
 559 gboolean
 560 camel_imap_folder_get_check_folder (CamelImapFolder *imap_folder)
 561 {
 562         g_return_val_if_fail (CAMEL_IS_IMAP_FOLDER (imap_folder), FALSE);
 563
 564         return imap_folder->priv->check_folder;
 565 }
 566
 567 void
 568 camel_imap_folder_set_check_folder (CamelImapFolder *imap_folder,
 569                                     gboolean check_folder)
 570 {
 571         CamelFolder *folder;
 572         CamelStore *parent_store;
 573         const gchar *full_name;
 574
 575         g_return_if_fail (CAMEL_IS_IMAP_FOLDER (imap_folder));
 576
 577         if ((imap_folder->priv->check_folder ? 1 : 0) == (check_folder ? 1 : 0))
 578                 return;
 579
 580         imap_folder->priv->check_folder = check_folder;
 581
 582         folder = CAMEL_FOLDER (imap_folder);
 583         full_name = camel_folder_get_full_name (folder);
 584         parent_store = camel_folder_get_parent_store (folder);
 585
 586         /* Update the summary so the value is restored
 587          * correctly the next time the folder is loaded. */
 588         if (CAMEL_IS_IMAP_STORE (parent_store)) {
 589                 CamelImapStore *imap_store;
 590                 CamelStoreSummary *summary;
 591                 CamelStoreInfo *si;
 592
 593                 imap_store = CAMEL_IMAP_STORE (parent_store);
 594                 summary = CAMEL_STORE_SUMMARY (imap_store->summary);
 595
 596                 si = camel_store_summary_path (summary, full_name);
 597                 if (si != NULL) {
 598                         guint32 old_flags = si->flags;
 599
 600                         si->flags &= ~CAMEL_STORE_INFO_FOLDER_CHECK_FOR_NEW;
 601                         si->flags |= check_folder ? CAMEL_STORE_INFO_FOLDER_CHECK_FOR_NEW : 0;
 602
 603                         if (si->flags != old_flags) {
 604                                 camel_store_summary_touch (summary);
 605                                 camel_store_summary_save (summary);
 606                         }
 607
 608                         camel_store_summary_info_free (summary, si);
 609                 }
 610         }
 611
 612         g_object_notify (G_OBJECT (imap_folder), "check-folder");
 613 }
 614
 615 /* Called with the store's connect_lock locked */
 616 gboolean
 617 camel_imap_folder_selected (CamelFolder *folder,
 618                             CamelImapResponse *response,
 619                             GCancellable *cancellable,
 620                             GError **error)
 621 {
 622         CamelImapFolder *imap_folder = CAMEL_IMAP_FOLDER (folder);
 623         CamelImapSummary *imap_summary = CAMEL_IMAP_SUMMARY (folder->summary);
 624         gulong exists = 0, validity = 0, val, uid;
 625         CamelMessageFlags perm_flags = 0;
 626         GData *fetch_data;
 627         gint i, count;
 628         gchar *resp;
 629
 630         count = camel_folder_summary_count (folder->summary);
 631
 632         for (i = 0; i < response->untagged->len; i++) {
 633                 resp = (gchar *) response->untagged->pdata[i] + 2;
 634
 635                 if (!g_ascii_strncasecmp (resp, "FLAGS ", 6) && !perm_flags) {
 636                         resp += 6;
 637                         imap_parse_flag_list (&resp, &folder->permanent_flags, NULL);
 638                 } else if (!g_ascii_strncasecmp (resp, "OK [PERMANENTFLAGS ", 19)) {
 639                         resp += 19;
 640
 641                         /* workaround for broken IMAP servers that send
 642                          * "* OK [PERMANENTFLAGS ()] Permanent flags"
 643                          * even tho they do allow storing flags. */
 644                         imap_parse_flag_list (&resp, &perm_flags, NULL);
 645                         if (perm_flags != 0)
 646                                 folder->permanent_flags = perm_flags;
 647                 } else if (!g_ascii_strncasecmp (resp, "OK [UIDVALIDITY ", 16)) {
 648                         validity = strtoul (resp + 16, NULL, 10);
 649                 } else if (isdigit ((guchar) * resp)) {
 650                         gulong num = strtoul (resp, &resp, 10);
 651
 652                         if (!g_ascii_strncasecmp (resp, " EXISTS", 7)) {
 653                                 exists = num;
 654                                 /* Remove from the response so nothing
 655                                  * else tries to interpret it.
 656                                  */
 657                                 g_free (response->untagged->pdata[i]);
 658                                 g_ptr_array_remove_index (response->untagged, i--);
 659                         }
 660                 }
 661         }
 662
 663         if (camel_strstrcase (response->status, "OK [READ-ONLY]"))
 664                 imap_folder->read_only = TRUE;
 665
 666         if (!imap_summary->validity)
 667                 imap_summary->validity = validity;
 668         else if (validity != imap_summary->validity) {
 669                 imap_summary->validity = validity;
 670                 camel_folder_summary_clear (folder->summary, NULL);
 671                 CAMEL_IMAP_FOLDER_REC_LOCK (imap_folder, cache_lock);
 672                 camel_imap_message_cache_clear (imap_folder->cache);
 673                 CAMEL_IMAP_FOLDER_REC_UNLOCK (imap_folder, cache_lock);
 674                 imap_folder->need_rescan = FALSE;
 675                 return camel_imap_folder_changed (
 676                         folder, exists, NULL, cancellable, error);
 677         }
 678
 679         /* If we've lost messages, we have to rescan everything */
 680         if (exists < count)
 681                 imap_folder->need_rescan = TRUE;
 682         else if (count != 0 && !imap_folder->need_rescan) {
 683                 CamelStore *parent_store;
 684                 CamelImapStore *store;
 685                 GPtrArray *known_uids;
 686                 const gchar *old_uid;
 687
 688                 parent_store = camel_folder_get_parent_store (folder);
 689                 store = CAMEL_IMAP_STORE (parent_store);
 690
 691                 /* Similarly, if the UID of the highest message we
 692                  * know about has changed, then that indicates that
 693                  * messages have been both added and removed, so we
 694                  * have to rescan to find the removed ones. (We pass
 695                  * NULL for the folder since we know that this folder
 696                  * is selected, and we don't want camel_imap_command
 697                  * to worry about it.)
 698                  */
 699                 response = camel_imap_command (store, NULL, cancellable, error, "FETCH %d UID", count);
 700                 if (!response)
 701                         return FALSE;
 702                 uid = 0;
 703                 for (i = 0; i < response->untagged->len; i++) {
 704                         resp = response->untagged->pdata[i];
 705                         val = strtoul (resp + 2, &resp, 10);
 706                         if (val == 0)
 707                                 continue;
 708                         if (!g_ascii_strcasecmp (resp, " EXISTS")) {
 709                                 /* Another one?? */
 710                                 exists = val;
 711                                 continue;
 712                         }
 713                         if (uid != 0 || val != count || g_ascii_strncasecmp (resp, " FETCH (", 8) != 0)
 714                                 continue;
 715
 716                         fetch_data = parse_fetch_response (imap_folder, resp + 7);
 717                         uid = strtoul (g_datalist_get_data (&fetch_data, "UID"), NULL, 10);
 718                         g_datalist_clear (&fetch_data);
 719                 }
 720                 camel_imap_response_free_without_processing (store, response);
 721
 722                 known_uids = camel_folder_summary_get_array (folder->summary);
 723                 camel_folder_sort_uids (folder, known_uids);
 724                 old_uid = NULL;
 725                 if (known_uids && count - 1 >= 0 && count - 1 < known_uids->len)
 726                         old_uid = g_ptr_array_index (known_uids, count - 1);
 727                 if (old_uid) {
 728                         val = strtoul (old_uid, NULL, 10);
 729                         if (uid == 0 || uid != val)
 730                                 imap_folder->need_rescan = TRUE;
 731                 }
 732                 camel_folder_summary_free_array (known_uids);
 733         }
 734
 735         /* Now rescan if we need to */
 736         if (imap_folder->need_rescan)
 737                 return imap_rescan (folder, exists, cancellable, error);
 738
 739         /* If we don't need to rescan completely, but new messages
 740          * have been added, find out about them.
 741          */
 742         if (exists > count)
 743                 camel_imap_folder_changed (
 744                         folder, exists, NULL, cancellable, error);
 745
 746         /* And we're done. */
 747
 748         return TRUE;
 749 }
 750
 751 static gchar *
 752 imap_get_filename (CamelFolder *folder,
 753                    const gchar *uid,
 754                    GError **error)
 755 {
 756         CamelImapFolder *imap_folder = (CamelImapFolder *) folder;
 757
 758         return camel_imap_message_cache_get_filename (imap_folder->cache, uid, "", error);
 759 }
 760
 761 static void
 762 imap_rename (CamelFolder *folder,
 763              const gchar *new)
 764 {
 765         CamelService *service;
 766         CamelStore *parent_store;
 767         CamelImapFolder *imap_folder = (CamelImapFolder *) folder;
 768         const gchar *user_cache_dir;
 769         gchar *folder_dir, *state_file;
 770         gchar *folders;
 771
 772         parent_store = camel_folder_get_parent_store (folder);
 773
 774         service = CAMEL_SERVICE (parent_store);
 775         user_cache_dir = camel_service_get_user_cache_dir (service);
 776
 777         folders = g_build_filename (user_cache_dir, "folders", NULL);
 778         folder_dir = imap_path_to_physical (folders, new);
 779         g_free (folders);
 780
 781         CAMEL_IMAP_FOLDER_REC_LOCK (folder, cache_lock);
 782         camel_imap_message_cache_set_path (imap_folder->cache, folder_dir);
 783         CAMEL_IMAP_FOLDER_REC_UNLOCK (folder, cache_lock);
 784
 785         state_file = g_build_filename (folder_dir, "cmeta", NULL);
 786         camel_object_set_state_filename (CAMEL_OBJECT (folder), state_file);
 787         g_free (state_file);
 788
 789         g_free (folder_dir);
 790
 791         camel_store_summary_disconnect_folder_summary (
 792                 (CamelStoreSummary *) ((CamelImapStore *) parent_store)->summary,
 793                 folder->summary);
 794
 795         CAMEL_FOLDER_CLASS (camel_imap_folder_parent_class)->rename (folder, new);
 796
 797         camel_store_summary_connect_folder_summary (
 798                 (CamelStoreSummary *) ((CamelImapStore *) parent_store)->summary,
 799                 camel_folder_get_full_name (folder), folder->summary);
 800 }
 801
 802 /* called with connect_lock locked */
 803 static gboolean
 804 get_folder_status (CamelFolder *folder,
 805                    guint32 *total,
 806                    guint32 *unread,
 807                    GCancellable *cancellable,
 808                    GError **error)
 809 {
 810         CamelStore *parent_store;
 811         CamelImapStore *imap_store;
 812         CamelImapResponse *response;
 813         const gchar *full_name;
 814         gboolean res = FALSE;
 815
 816         g_return_val_if_fail (folder != NULL, FALSE);
 817
 818         full_name = camel_folder_get_full_name (folder);
 819         parent_store = camel_folder_get_parent_store (folder);
 820
 821         imap_store = CAMEL_IMAP_STORE (parent_store);
 822
 823         response = camel_imap_command (imap_store, folder, cancellable, error, "STATUS %F (MESSAGES UNSEEN)", full_name);
 824
 825         if (response) {
 826                 gint i;
 827
 828                 for (i = 0; i < response->untagged->len; i++) {
 829                         const gchar *resp = response->untagged->pdata[i];
 830
 831                         if (resp && g_str_has_prefix (resp, "* STATUS ")) {
 832                                 const gchar *p = NULL;
 833
 834                                 while (*resp) {
 835                                         if (*resp == '(')
 836                                                 p = resp;
 837                                         resp++;
 838                                 }
 839
 840                                 if (p && *(resp - 1) == ')') {
 841                                         const gchar *msgs = NULL, *unseen = NULL;
 842
 843                                         p++;
 844
 845                                         while (p && (!msgs || !unseen)) {
 846                                                 const gchar **dest = NULL;
 847
 848                                                 if (g_str_has_prefix (p, "MESSAGES "))
 849                                                         dest = &msgs;
 850                                                 else if (g_str_has_prefix (p, "UNSEEN "))
 851                                                         dest = &unseen;
 852
 853                                                 if (dest) {
 854                                                         *dest = imap_next_word (p);
 855
 856                                                         if (!*dest)
 857                                                                 break;
 858
 859                                                         p = imap_next_word (*dest);
 860                                                 } else {
 861                                                         p = imap_next_word (p);
 862                                                         if (p)
 863                                                                 p = imap_next_word (p);
 864                                                 }
 865                                         }
 866
 867                                         if (msgs && unseen) {
 868                                                 res = TRUE;
 869
 870                                                 if (total)
 871                                                         *total = strtoul (msgs, NULL, 10);
 872
 873                                                 if (unread)
 874                                                         *unread = strtoul (unseen, NULL, 10);
 875                                         }
 876                                 }
 877                         }
 878                 }
 879                 camel_imap_response_free (imap_store, response);
 880         }
 881
 882         return res;
 883 }
 884
 885 static gboolean
 886 imap_refresh_info_sync (CamelFolder *folder,
 887                         GCancellable *cancellable,
 888                         GError **error)
 889 {
 890         CamelStore *parent_store;
 891         CamelImapStore *imap_store;
 892         CamelImapFolder *imap_folder = CAMEL_IMAP_FOLDER (folder);
 893         CamelImapResponse *response;
 894         CamelStoreInfo *si;
 895         const gchar *full_name;
 896         gint check_rescan = -1;
 897         GError *local_error = NULL;
 898
 899         parent_store = camel_folder_get_parent_store (folder);
 900         imap_store = CAMEL_IMAP_STORE (parent_store);
 901
 902         if (!camel_offline_store_get_online (CAMEL_OFFLINE_STORE (imap_store)))
 903                 return TRUE;
 904
 905         if (camel_folder_is_frozen (folder)) {
 906                 imap_folder->need_refresh = TRUE;
 907                 return TRUE;
 908         }
 909
 910         /* If the folder isn't selected, select it (which will force
 911          * a rescan if one is needed).
 912          * Also, if this is the INBOX, some servers (cryus) wont tell
 913          * us with a NOOP of new messages, so force a reselect which
 914          * should do it.  */
 915
 916         if (camel_application_is_exiting  || !camel_imap_store_connected (imap_store, &local_error))
 917                 goto done;
 918
 919         /* try to store local changes first, as the summary contains new local messages */
 920         replay_offline_journal (
 921                 imap_store, imap_folder, cancellable, &local_error);
 922
 923         full_name = camel_folder_get_full_name (folder);
 924
 925         if (imap_store->current_folder != folder
 926             || g_ascii_strcasecmp (full_name, "INBOX") == 0) {
 927                 response = camel_imap_command (imap_store, folder, cancellable, &local_error, NULL);
 928                 if (response) {
 929                         camel_imap_folder_selected (
 930                                 folder, response,
 931                                 cancellable, &local_error);
 932                         camel_imap_response_free (imap_store, response);
 933                 }
 934         } else if (imap_folder->need_rescan) {
 935                 /* Otherwise, if we need a rescan, do it, and if not, just do
 936                  * a NOOP to give the server a chance to tell us about new
 937                  * messages.
 938                  */
 939                 imap_rescan (
 940                         folder, camel_folder_summary_count (
 941                         folder->summary), cancellable, &local_error);
 942                 check_rescan = 0;
 943         } else {
 944 #if 0
 945                 /* on some servers need to CHECKpoint INBOX to recieve new messages?? */
 946                 /* rfc2060 suggests this, but havent seen a server that requires it */
 947                 if (g_ascii_strcasecmp (full_name, "INBOX") == 0) {
 948                         response = camel_imap_command (imap_store, folder, &local_error, "CHECK");
 949                         camel_imap_response_free (imap_store, response);
 950                 }
 951 #endif
 952                 response = camel_imap_command (imap_store, folder, cancellable, &local_error, "NOOP");
 953                 camel_imap_response_free (imap_store, response);
 954         }
 955
 956         si = camel_store_summary_path ((CamelStoreSummary *)((CamelImapStore *) parent_store)->summary, full_name);
 957         if (si) {
 958                 guint32 unread, total;
 959
 960                 total = camel_folder_summary_count (folder->summary);
 961                 unread = camel_folder_summary_get_unread_count (folder->summary);
 962
 963                 if (si->total != total
 964                     || si->unread != unread) {
 965                         si->total = total;
 966                         si->unread = unread;
 967                         camel_store_summary_touch ((CamelStoreSummary *)((CamelImapStore *) parent_store)->summary);
 968                         check_rescan = 0;
 969                 }
 970                 camel_store_summary_info_free ((CamelStoreSummary *)((CamelImapStore *) parent_store)->summary, si);
 971         }
 972
 973         if (check_rescan && !camel_application_is_exiting && local_error == NULL) {
 974                 if (check_rescan == -1) {
 975                         guint32 total, unread = 0, server_total = 0, server_unread = 0;
 976
 977                         check_rescan = 0;
 978
 979                         /* Check whether there are changes in total/unread messages in the folders
 980                          * and if so, then rescan whole summary */
 981                         if (get_folder_status (folder, &server_total, &server_unread, cancellable, &local_error)) {
 982
 983                                 total = camel_folder_summary_count (folder->summary);
 984                                 unread = camel_folder_summary_get_unread_count (folder->summary);
 985
 986                                 if (total != server_total || unread != server_unread)
 987                                         check_rescan = 1;
 988                         }
 989                 }
 990
 991                 if (check_rescan)
 992                         imap_rescan (
 993                                 folder, camel_folder_summary_count (
 994                                 folder->summary), cancellable, &local_error);
 995         }
 996
 997 done:
 998         camel_folder_summary_save_to_db (folder->summary, NULL);
 999         camel_store_summary_save ((CamelStoreSummary *)((CamelImapStore *) parent_store)->summary);
1000
1001         if (local_error != NULL) {
1002                 g_propagate_error (error, local_error);
1003                 return FALSE;
1004         }
1005
1006         return TRUE;
1007 }
1008
1009 static void
1010 fillup_custom_flags (CamelMessageInfo *mi,
1011                      gchar *custom_flags)
1012 {
1013         gchar **array_str;
1014         gint index = 0;
1015
1016         array_str = g_strsplit (custom_flags, " ", -1);
1017
1018         while (array_str[index] != NULL) {
1019                 camel_flag_set (&((CamelMessageInfoBase *) mi)->user_flags, array_str[index], TRUE);
1020                 ++ index;
1021         }
1022
1023         g_strfreev (array_str);
1024 }
1025
1026 /* This will merge custom flags with those in message info. Returns whether was some change. */
1027 static gboolean
1028 merge_custom_flags (CamelMessageInfo *mi,
1029                     const gchar *custom_flags)
1030 {
1031         GList *list, *p;
1032         GHashTable *server;
1033         gchar **cflags;
1034         gint i;
1035         const CamelFlag *flag;
1036         gboolean changed = FALSE;
1037
1038         g_return_val_if_fail (mi != NULL, FALSE);
1039
1040         if (!custom_flags)
1041                 custom_flags = "";
1042
1043         list = NULL;
1044         server = g_hash_table_new (g_str_hash, g_str_equal);
1045
1046         cflags = g_strsplit (custom_flags, " ", -1);
1047         for (i = 0; cflags[i]; i++) {
1048                 gchar *name = cflags[i];
1049
1050                 if (name && *name) {
1051                         g_hash_table_insert (server, name, name);
1052                         list = g_list_prepend (list, name);
1053                 }
1054         }
1055
1056         for (flag = camel_message_info_user_flags (mi); flag; flag = flag->next) {
1057                 gchar *name = (gchar *) flag->name;
1058
1059                 if (name && *name)
1060                         list = g_list_prepend (list, name);
1061         }
1062
1063         list = g_list_sort (list, (GCompareFunc) strcmp);
1064         for (p = list; p; p = p->next) {
1065                 if (p->next && strcmp (p->data, p->next->data) == 0) {
1066                         /* This flag is there twice, which means it was on the server and
1067                          * in our local summary too; thus skip these two elements. */
1068                         p = p->next;
1069                 } else {
1070                         /* If this value came from the server, then add it to our local summary,
1071                          * otherwise it was in local summary, but isn't on the server, thus remove it. */
1072                         changed = TRUE;
1073                         mi->dirty = TRUE;
1074                         if (mi->summary)
1075                                 camel_folder_summary_touch (mi->summary);
1076                         camel_flag_set (&((CamelMessageInfoBase *) mi)->user_flags, p->data, g_hash_table_lookup (server, p->data) != NULL);
1077                         ((CamelMessageInfoBase *) mi)->flags |= CAMEL_MESSAGE_FOLDER_FLAGGED;
1078                 }
1079         }
1080
1081         g_list_free (list);
1082         g_hash_table_destroy (server);
1083         g_strfreev (cflags);
1084
1085         return changed;
1086 }
1087
1088 /* Called with the store's connect_lock locked */
1089 static gboolean
1090 imap_rescan (CamelFolder *folder,
1091              gint exists,
1092              GCancellable *cancellable,
1093              GError **error)
1094 {
1095         CamelStore *parent_store;
1096         CamelImapFolder *imap_folder = CAMEL_IMAP_FOLDER (folder);
1097         CamelImapStore *store;
1098         struct {
1099                 gchar *uid;
1100                 guint32 flags;
1101                 gchar *custom_flags;
1102         } *new;
1103         gchar *resp, *uid;
1104         CamelImapResponseType type;
1105         gint i, j, seq, summary_got, del = 0;
1106         guint summary_len;
1107         CamelMessageInfo *info;
1108         CamelImapMessageInfo *iinfo;
1109         GArray *removed;
1110         gboolean ok;
1111         CamelFolderChangeInfo *changes = NULL;
1112         gboolean success;
1113         GPtrArray *known_uids;
1114
1115         parent_store = camel_folder_get_parent_store (folder);
1116         store = CAMEL_IMAP_STORE (parent_store);
1117
1118         if (camel_application_is_exiting)
1119                 return TRUE;
1120
1121         imap_folder->need_rescan = FALSE;
1122
1123         known_uids = camel_folder_summary_get_array (folder->summary);
1124         summary_len = known_uids ? known_uids->len : 0;
1125         if (summary_len == 0) {
1126                 camel_folder_summary_free_array (known_uids);
1127                 if (exists)
1128                         return camel_imap_folder_changed (
1129                                 folder, exists, NULL, cancellable, error);
1130                 return TRUE;
1131         }
1132
1133         /* Check UIDs and flags of all messages we already know of. */
1134         camel_operation_push_message (
1135                 cancellable, _("Scanning for changed messages in %s"),
1136                 camel_folder_get_display_name (folder));
1137
1138         camel_folder_sort_uids (folder, known_uids);
1139
1140         uid = g_ptr_array_index (known_uids, summary_len - 1);
1141         if (!uid) {
1142                 camel_operation_pop_message (cancellable);
1143                 camel_folder_summary_free_array (known_uids);
1144                 return TRUE;
1145         }
1146
1147         if (!camel_imap_store_connected (CAMEL_IMAP_STORE (parent_store), error)) {
1148                 camel_operation_pop_message (cancellable);
1149                 camel_folder_summary_free_array (known_uids);
1150                 return FALSE;
1151         }
1152
1153         ok = camel_imap_command_start (
1154                 store, folder, cancellable, error,
1155                 "UID FETCH 1:%s (FLAGS)", uid);
1156         if (!ok) {
1157                 camel_operation_pop_message (cancellable);
1158                 camel_folder_summary_free_array (known_uids);
1159                 return FALSE;
1160         }
1161
1162         resp = NULL;
1163         new = g_malloc0 (summary_len * sizeof (*new));
1164         summary_got = 0;
1165         while ((type = camel_imap_command_response (store, folder, &resp, cancellable, error)) == CAMEL_IMAP_RESPONSE_UNTAGGED && !camel_application_is_exiting) {
1166                 GData *data;
1167                 gchar *uid;
1168                 guint32 flags;
1169
1170                 data = parse_fetch_response (imap_folder, resp);
1171                 g_free (resp);
1172                 resp = NULL;
1173                 if (!data)
1174                         continue;
1175
1176                 seq = GPOINTER_TO_INT (g_datalist_get_data (&data, "SEQUENCE"));
1177                 uid = g_datalist_get_data (&data, "UID");
1178                 flags = GPOINTER_TO_UINT (g_datalist_get_data (&data, "FLAGS"));
1179
1180                 if (!uid || !seq || seq > summary_len || seq < 0) {
1181                         g_datalist_clear (&data);
1182                         continue;
1183                 }
1184
1185                 camel_operation_progress (
1186                         cancellable, ++summary_got * 100 / summary_len);
1187
1188                 new[seq - 1].uid = g_strdup (uid);
1189                 new[seq - 1].flags = flags;
1190                 new[seq - 1].custom_flags = g_strdup (g_datalist_get_data (&data, "CUSTOM.FLAGS"));
1191                 g_datalist_clear (&data);
1192         }
1193
1194         if (summary_got == 0 && summary_len == 0) {
1195                 camel_operation_pop_message (cancellable);
1196                 g_free (new);
1197                 g_free (resp);
1198
1199                 camel_folder_summary_free_array (known_uids);
1200                 return TRUE;
1201         }
1202
1203         camel_operation_pop_message (cancellable);
1204
1205         if (type == CAMEL_IMAP_RESPONSE_ERROR || camel_application_is_exiting) {
1206                 for (i = 0; i < summary_len && new[i].uid; i++) {
1207                         g_free (new[i].uid);
1208                         g_free (new[i].custom_flags);
1209                 }
1210                 g_free (new);
1211                 g_free (resp);
1212
1213                 camel_folder_summary_free_array (known_uids);
1214                 return TRUE;
1215         }
1216
1217         /* Free the final tagged response */
1218         g_free (resp);
1219
1220         /* If we find a UID in the summary that doesn't correspond to
1221          * the UID in the folder, then either: (a) it's a real UID,
1222          * but the message was deleted on the server, or (b) it's a
1223          * fake UID, and needs to be removed from the summary in order
1224          * to sync up with the server. So either way, we remove it
1225          * from the summary.
1226          */
1227         removed = g_array_new (FALSE, FALSE, sizeof (gint));
1228
1229         camel_folder_summary_prepare_fetch_all (folder->summary, NULL);
1230
1231         for (i = 0, j = 0; i < summary_len && new[j].uid; i++) {
1232                 gboolean changed = FALSE;
1233
1234                 uid = g_ptr_array_index (known_uids, i);
1235                 if (!uid)
1236                         continue;
1237
1238                 info = camel_folder_summary_get (folder->summary, uid);
1239                 if (!info) {
1240                         if (g_getenv("CRASH_IMAP")) { /* Debug logs to tackle on hard to get imap crasher */
1241                                 printf ("CRASH: %s: %s",
1242                                         camel_folder_get_full_name (folder), uid);
1243                                 g_assert (0);
1244                         } else
1245                                 continue;
1246                 }
1247
1248                 iinfo = (CamelImapMessageInfo *) info;
1249
1250                 if (strcmp (uid, new[j].uid) != 0) {
1251                         seq = i + 1 - del;
1252                         del++;
1253                         g_array_append_val (removed, seq);
1254                         camel_message_info_free (info);
1255                         continue;
1256                 }
1257
1258                 /* Update summary flags */
1259
1260                 if (new[j].flags != iinfo->server_flags) {
1261                         guint32 server_set, server_cleared;
1262
1263                         server_set = new[j].flags & ~iinfo->server_flags;
1264                         server_cleared = iinfo->server_flags & ~new[j].flags;
1265
1266                         camel_message_info_set_flags ((CamelMessageInfo *) iinfo, server_set | server_cleared, (iinfo->info.flags | server_set) & ~server_cleared);
1267                         iinfo->server_flags = new[j].flags;
1268                         /* unset folder_flagged, because these are flags received froma server */
1269                         iinfo->info.flags = iinfo->info.flags & (~CAMEL_MESSAGE_FOLDER_FLAGGED);
1270                         iinfo->info.dirty = TRUE;
1271                         if (info->summary)
1272                                 camel_folder_summary_touch (info->summary);
1273                         changed = TRUE;
1274                 }
1275
1276                 /* Do not merge custom flags when server doesn't support it.
1277                  * Because server always reports NULL, which means none, which
1278                  * will remove user's flags from local machine, which is bad.
1279                 */
1280                 if ((folder->permanent_flags & CAMEL_MESSAGE_USER) != 0 &&
1281                     (iinfo->info.flags & CAMEL_MESSAGE_FOLDER_FLAGGED) == 0 &&
1282                     merge_custom_flags (info, new[j].custom_flags))
1283                         changed = TRUE;
1284
1285                 if (changed) {
1286                         if (changes == NULL)
1287                                 changes = camel_folder_change_info_new ();
1288                         camel_folder_change_info_change_uid (changes, new[j].uid);
1289                 }
1290
1291                 camel_message_info_free (info);
1292                 g_free (new[j].uid);
1293                 g_free (new[j].custom_flags);
1294                 j++;
1295         }
1296
1297         if (changes) {
1298                 camel_folder_changed (folder, changes);
1299                 camel_folder_change_info_free (changes);
1300         }
1301
1302         seq = i + 1;
1303
1304 #if 0
1305         /* FIXME: Srini: I don't think this will be called any longer. */
1306         /* Free remaining memory. */
1307         while (i < summary_len && new[i].uid) {
1308                 g_free (new[i].uid);
1309                 g_free (new[i].custom_flags);
1310                 i++;
1311         }
1312 #endif
1313         g_free (new);
1314
1315         /* Remove any leftover cached summary messages. (Yes, we
1316          * repeatedly add the same number to the removed array.
1317          * See RFC2060 7.4.1)
1318          */
1319
1320         for (i = seq; i <= summary_len; i++) {
1321                 gint j;
1322
1323                 j = seq - del;
1324                 g_array_append_val (removed, j);
1325         }
1326
1327         /* And finally update the summary. */
1328         success = camel_imap_folder_changed (
1329                 folder, exists, removed, cancellable, error);
1330         g_array_free (removed, TRUE);
1331
1332         camel_folder_summary_free_array (known_uids);
1333         return success;
1334 }
1335
1336 static const gchar *
1337 get_message_uid (CamelFolder *folder,
1338                  CamelImapMessageInfo *info)
1339 {
1340         const gchar *uid;
1341
1342         g_return_val_if_fail (folder != NULL, NULL);
1343         g_return_val_if_fail (info != NULL, NULL);
1344
1345         uid = camel_message_info_uid (info);
1346         g_return_val_if_fail (uid != NULL, NULL);
1347
1348         if (!isdigit ((guchar) * uid)) {
1349                 uid = camel_imap_journal_uidmap_lookup ((CamelIMAPJournal *) CAMEL_IMAP_FOLDER (folder)->journal, uid);
1350                 g_return_val_if_fail (uid != NULL, NULL);
1351         }
1352
1353         return uid;
1354 }
1355
1356 /* the max number of chars that an unsigned 32-bit gint can be is 10 chars plus 1 for a possible : */
1357 #define UID_SET_FULL(setlen, maxlen) (maxlen > 0 ? setlen + 11 >= maxlen : FALSE)
1358
1359 /* Find all messages in @folder with flags matching @flags and @mask.
1360  * If no messages match, returns %NULL. Otherwise, returns an array of
1361  * CamelMessageInfo and sets *@set to a message set corresponding the
1362  * UIDs of the matched messages (up to @UID_SET_LIMIT bytes). The
1363  * caller must free the infos, the array, and the set string.
1364  */
1365 static GPtrArray *
1366 get_matching (CamelFolder *folder,
1367               guint32 flags,
1368               guint32 mask,
1369               CamelMessageInfo *master_info,
1370               gchar **set,
1371               GPtrArray *summary,
1372               GPtrArray *deleted_uids,
1373               GPtrArray *junked_uids)
1374 {
1375         GPtrArray *matches;
1376         CamelImapMessageInfo *info;
1377         gint i, max, range, last_range_uid;
1378         GString *gset;
1379         GList *list1 = NULL;
1380         gint count1 = 0;
1381         gchar *uid;
1382
1383         /* use the local rinfo in the close_range, because we want to keep our info untouched */
1384         #define close_range()                                                                           \
1385                 if (range != -1) {                                                                      \
1386                         if (range != i - 1) {                                                           \
1387                                 CamelImapMessageInfo *rinfo = matches->pdata[matches->len - 1];         \
1388                                                                                                         \
1389                                 g_string_append_printf (gset, ":%s", get_message_uid (folder, rinfo));  \
1390                         }                                                                               \
1391                         range = -1;                                                                     \
1392                         last_range_uid = -1;                                                            \
1393                 }
1394
1395         matches = g_ptr_array_new ();
1396         gset = g_string_new ("");
1397         max = summary->len;
1398         range = -1;
1399         last_range_uid = -1;
1400         for (i = 0; i < max && !UID_SET_FULL (gset->len, UID_SET_LIMIT); i++) {
1401                 gint uid_num;
1402                 uid = summary->pdata[i];
1403
1404                 if (uid) {
1405                         info = (CamelImapMessageInfo *) camel_folder_summary_get (folder->summary, uid);
1406                 } else
1407                         continue;
1408
1409                 if (!info)
1410                         continue;
1411
1412                 /* if the resulting flag list is empty, then "concat" other message
1413                  * only when server_flags are same, because there will be a flag removal
1414                  * command for this type of situation */
1415                 if ((info->info.flags & mask) != flags || (flags == 0 && info->server_flags != ((CamelImapMessageInfo *) master_info)->server_flags)) {
1416                         camel_message_info_free ((CamelMessageInfo *) info);
1417                         close_range ();
1418                         continue;
1419                 }
1420
1421                 uid_num = atoi (uid);
1422
1423                 /* we got only changes, thus the uid's can be mixed up, not the consecutive list,
1424                  * thus close range if we are not in it */
1425                 if (last_range_uid != -1 && uid_num != last_range_uid + 1) {
1426                         close_range ();
1427                 }
1428
1429                 /* only check user flags when we see other message than our 'master' */
1430                 if (strcmp (master_info->uid, ((CamelMessageInfo *) info)->uid)) {
1431                         const CamelFlag *flag;
1432                         GList *list2 = NULL, *l1, *l2;
1433                         gint count2 = 0, cmp = 0;
1434
1435                         if (!list1) {
1436                                 for (flag = camel_message_info_user_flags (master_info); flag; flag = flag->next) {
1437                                         if (*flag->name) {
1438                                                 count1++;
1439                                                 list1 = g_list_prepend (list1, (gchar *) flag->name);
1440                                         }
1441                                 }
1442
1443                                 list1 = g_list_sort (list1, (GCompareFunc) strcmp);
1444                         }
1445
1446                         for (flag = camel_message_info_user_flags (info); flag; flag = flag->next) {
1447                                 if (*flag->name) {
1448                                         count2++;
1449                                         list2 = g_list_prepend (list2, (gchar *) flag->name);
1450                                 }
1451                         }
1452
1453                         if (count1 != count2) {
1454                                 g_list_free (list2);
1455                                 camel_message_info_free ((CamelMessageInfo *) info);
1456                                 close_range ();
1457                                 continue;
1458                         }
1459
1460                         list2 = g_list_sort (list2, (GCompareFunc) strcmp);
1461                         for (l1 = list1, l2 = list2; l1 && l2 && !cmp; l1 = l1->next, l2 = l2->next) {
1462                                 cmp = strcmp (l1->data, l2->data);
1463                         }
1464
1465                         if (cmp) {
1466                                 g_list_free (list2);
1467                                 camel_message_info_free ((CamelMessageInfo *) info);
1468                                 close_range ();
1469                                 continue;
1470                         }
1471                 }
1472
1473                 if (deleted_uids && (info->info.flags & (CAMEL_MESSAGE_DELETED | CAMEL_IMAP_MESSAGE_MOVED)) == CAMEL_MESSAGE_DELETED) {
1474                         g_ptr_array_add (deleted_uids, (gpointer) camel_pstring_strdup (camel_message_info_uid (info)));
1475                         info->info.flags &= ~CAMEL_MESSAGE_DELETED;
1476                 } else if (junked_uids && (info->info.flags & CAMEL_MESSAGE_JUNK) != 0) {
1477                         g_ptr_array_add (junked_uids, (gpointer) camel_pstring_strdup (camel_message_info_uid (info)));
1478                 }
1479
1480                 g_ptr_array_add (matches, info);
1481                 /* Remove the uid from the list, to optimize*/
1482                 camel_pstring_free (summary->pdata[i]);
1483                 summary->pdata[i] = NULL;
1484
1485                 if (range != -1) {
1486                         last_range_uid = uid_num;
1487                         continue;
1488                 }
1489
1490                 range = i;
1491                 last_range_uid = uid_num;
1492                 if (gset->len)
1493                         g_string_append_c (gset, ',');
1494                 g_string_append_printf (gset, "%s", get_message_uid (folder, info));
1495         }
1496
1497         if (range != -1 && range != max - 1) {
1498                 info = matches->pdata[matches->len - 1];
1499                 g_string_append_printf (gset, ":%s", get_message_uid (folder, info));
1500         }
1501
1502         if (list1)
1503                 g_list_free (list1);
1504
1505         if (matches->len) {
1506                 *set = gset->str;
1507                 g_string_free (gset, FALSE);
1508                 return matches;
1509         } else {
1510                 *set = NULL;
1511                 g_string_free (gset, TRUE);
1512                 g_ptr_array_free (matches, TRUE);
1513                 return NULL;
1514         }
1515
1516         #undef close_range
1517 }
1518
1519 static gboolean
1520 imap_sync_offline (CamelFolder *folder,
1521                    GError **error)
1522 {
1523         CamelStore *parent_store;
1524
1525         parent_store = camel_folder_get_parent_store (folder);
1526
1527         if (folder->summary && (folder->summary->flags & CAMEL_FOLDER_SUMMARY_DIRTY) != 0) {
1528                 CamelStoreInfo *si;
1529                 const gchar *full_name;
1530
1531                 /* ... and store's summary when folder's summary is dirty */
1532                 full_name = camel_folder_get_full_name (folder);
1533                 si = camel_store_summary_path ((CamelStoreSummary *)((CamelImapStore *) parent_store)->summary, full_name);
1534                 if (si) {
1535                         if (si->total != camel_folder_summary_get_saved_count (folder->summary) || si->unread != camel_folder_summary_get_unread_count (folder->summary)) {
1536                                 si->total = camel_folder_summary_get_saved_count (folder->summary);
1537                                 si->unread = camel_folder_summary_get_unread_count (folder->summary);
1538                                 camel_store_summary_touch ((CamelStoreSummary *)((CamelImapStore *) parent_store)->summary);
1539                         }
1540
1541                         camel_store_summary_info_free ((CamelStoreSummary *)((CamelImapStore *) parent_store)->summary, si);
1542                 }
1543         }
1544
1545         camel_folder_summary_save_to_db (folder->summary, NULL);
1546         camel_store_summary_save ((CamelStoreSummary *)((CamelImapStore *) parent_store)->summary);
1547
1548         return TRUE;
1549 }
1550
1551 static gboolean
1552 host_ends_with (const gchar *host,
1553                 const gchar *ends)
1554 {
1555         gint host_len, ends_len;
1556
1557         g_return_val_if_fail (host != NULL, FALSE);
1558         g_return_val_if_fail (ends != NULL, FALSE);
1559
1560         host_len = strlen (host);
1561         ends_len = strlen (ends);
1562
1563         return ends_len <= host_len && g_ascii_strcasecmp (host + host_len - ends_len, ends) == 0;
1564 }
1565
1566 static gboolean
1567 is_google_account (CamelStore *store)
1568 {
1569         CamelNetworkSettings *network_settings;
1570         CamelSettings *settings;
1571         CamelService *service;
1572         gboolean is_google;
1573         gchar *host;
1574
1575         g_return_val_if_fail (store != NULL, FALSE);
1576         g_return_val_if_fail (CAMEL_IS_STORE (store), FALSE);
1577
1578         service = CAMEL_SERVICE (store);
1579
1580         settings = camel_service_ref_settings (service);
1581
1582         network_settings = CAMEL_NETWORK_SETTINGS (settings);
1583         host = camel_network_settings_dup_host (network_settings);
1584
1585         g_object_unref (settings);
1586
1587         is_google =
1588                 (host != NULL) && (
1589                 host_ends_with (host, "gmail.com") ||
1590                 host_ends_with (host, "googlemail.com"));
1591
1592         g_free (host);
1593
1594         return is_google;
1595 }
1596
1597 static void
1598 move_messages (CamelFolder *src_folder,
1599                GPtrArray *uids,
1600                CamelFolder *des_folder,
1601                GCancellable *cancellable,
1602                GError **error)
1603 {
1604         g_return_if_fail (src_folder != NULL);
1605
1606         /* it's OK to have these NULL */
1607         if (!uids || uids->len == 0 || des_folder == NULL)
1608                 return;
1609
1610         /* moving to the same folder means expunge only */
1611         if (src_folder != des_folder) {
1612                 /* do 'copy' to not be bothered with CAMEL_MESSAGE_DELETED again */
1613                 if (!imap_transfer_messages (
1614                         src_folder, uids, des_folder, FALSE,
1615                         NULL, FALSE, cancellable, error))
1616                         return;
1617         }
1618
1619         camel_imap_expunge_uids_only (src_folder, uids, cancellable, error);
1620 }
1621
1622 static gboolean
1623 imap_synchronize_sync (CamelFolder *folder,
1624                        gboolean expunge,
1625                        GCancellable *cancellable,
1626                        GError **error)
1627 {
1628         CamelService *service;
1629         CamelSettings *settings;
1630         CamelStore *parent_store;
1631         CamelImapStore *store;
1632         CamelImapMessageInfo *info;
1633         CamelImapFolder *imap_folder = CAMEL_IMAP_FOLDER (folder);
1634         gboolean is_gmail;
1635         CamelFolder *real_junk = NULL;
1636         CamelFolder *real_trash = NULL;
1637         gchar *folder_path;
1638         GError *local_error = NULL;
1639
1640         GPtrArray *matches, *summary, *deleted_uids = NULL, *junked_uids = NULL;
1641         gchar *set, *flaglist, *uid;
1642         gint i, j, max;
1643
1644         parent_store = camel_folder_get_parent_store (folder);
1645         store = CAMEL_IMAP_STORE (parent_store);
1646         is_gmail = is_google_account (parent_store);
1647
1648         service = CAMEL_SERVICE (parent_store);
1649
1650         if (folder->permanent_flags == 0 || !camel_offline_store_get_online (CAMEL_OFFLINE_STORE (store))) {
1651                 if (expunge) {
1652                         if (!imap_expunge_sync (folder, cancellable, error))
1653                                 return FALSE;
1654                 }
1655                 return imap_sync_offline (folder, error);
1656         }
1657
1658         /* write local changes first */
1659         replay_offline_journal (store, imap_folder, cancellable, NULL);
1660
1661         /* Find a message with changed flags, find all of the other
1662          * messages like it, sync them as a group, mark them as
1663          * updated, and continue.
1664          */
1665         summary = camel_folder_summary_get_changed (folder->summary); /* These should be in memory anyways */
1666         camel_folder_sort_uids (folder, summary);
1667         max = summary->len;
1668
1669         settings = camel_service_ref_settings (service);
1670
1671         /* deleted_uids is NULL when not using real trash */
1672         folder_path = camel_imap_settings_dup_real_trash_path (
1673                 CAMEL_IMAP_SETTINGS (settings));
1674         if (folder_path != NULL && *folder_path) {
1675                 if ((folder->folder_flags & CAMEL_FOLDER_IS_TRASH) != 0) {
1676                         /* syncing the trash, expunge deleted when found any */
1677                         real_trash = g_object_ref (folder);
1678                 } else {
1679                         real_trash = camel_store_get_trash_folder_sync (
1680                                 parent_store, cancellable, NULL);
1681                 }
1682         }
1683         g_free (folder_path);
1684
1685         /* junked_uids is NULL when not using real junk */
1686         folder_path = camel_imap_settings_dup_real_junk_path (
1687                 CAMEL_IMAP_SETTINGS (settings));
1688         if (folder_path != NULL && *folder_path) {
1689                 if ((folder->folder_flags & CAMEL_FOLDER_IS_JUNK) != 0) {
1690                         /* syncing the junk, but cannot move
1691                          * messages to itself, thus do nothing */
1692                         real_junk = NULL;
1693                 } else {
1694                         real_junk = camel_store_get_junk_folder_sync (
1695                                 parent_store, cancellable, NULL);
1696                 }
1697         }
1698         g_free (folder_path);
1699
1700         g_object_unref (settings);
1701
1702         if (real_trash)
1703                 deleted_uids = g_ptr_array_new ();
1704
1705         if (real_junk)
1706                 junked_uids = g_ptr_array_new ();
1707
1708         for (i = 0; i < max; i++) {
1709                 gboolean unset = FALSE;
1710                 CamelImapResponse *response = NULL;
1711
1712                 uid = summary->pdata[i];
1713
1714                 if (!uid) /* Possibly it was sync by matching flags, which we NULLify */
1715                         continue;
1716
1717                 if (!(info = (CamelImapMessageInfo *) camel_folder_summary_get (folder->summary, uid))) {
1718                         continue;
1719                 }
1720
1721                 if (!(info->info.flags & CAMEL_MESSAGE_FOLDER_FLAGGED)) {
1722                         camel_message_info_free ((CamelMessageInfo *) info);
1723                         continue;
1724                 }
1725
1726                 /* Note: get_matching() uses UID_SET_LIMIT to limit
1727                  * the size of the uid-set string. We don't have to
1728                  * loop here to flush all the matching uids because
1729                  * they will be scooped up later by our parent loop (I
1730                  * think?). -- Jeff */
1731                 matches = get_matching (folder, info->info.flags & (folder->permanent_flags | CAMEL_MESSAGE_FOLDER_FLAGGED),
1732                                         folder->permanent_flags | CAMEL_MESSAGE_FOLDER_FLAGGED, (CamelMessageInfo *) info, &set, summary,
1733                                         deleted_uids, junked_uids);
1734                 if (matches == NULL) {
1735                         camel_message_info_free (info);
1736                         continue;
1737                 }
1738
1739                 /* Make sure we're connected before issuing commands */
1740                 if (!camel_imap_store_connected (store, NULL)) {
1741                         g_free (set);
1742                         camel_message_info_free (info);
1743                         g_ptr_array_foreach (matches, (GFunc) camel_message_info_free, NULL);
1744                         g_ptr_array_free (matches, TRUE);
1745                         break;
1746                 }
1747
1748                 if (deleted_uids && !is_gmail && (info->info.flags & CAMEL_MESSAGE_DELETED) != 0) {
1749                         /* there is a real trash, do not set it on the server */
1750                         info->info.flags &= ~CAMEL_MESSAGE_DELETED;
1751                 }
1752
1753                 flaglist = imap_create_flag_list (info->info.flags & folder->permanent_flags, (CamelMessageInfo *) info, folder->permanent_flags);
1754
1755                 if (strcmp (flaglist, "()") == 0) {
1756                         /* Note: Cyrus is broken and will not accept an
1757                          * empty-set of flags so... if this is true then we
1758                          * set and unset \Seen flag. It's necessary because
1759                          * we do not know the previously set user flags. */
1760                         unset = TRUE;
1761                         g_free (flaglist);
1762
1763                         /* unset all known server flags, because there left none in the actual flags */
1764                         flaglist =  imap_create_flag_list (info->server_flags & folder->permanent_flags, (CamelMessageInfo *) info, folder->permanent_flags);
1765
1766                         if (strcmp (flaglist, "()") == 0) {
1767                                 /* this should not happen, really */
1768                                 g_free (flaglist);
1769                                 flaglist = strdup ("(\\Seen)");
1770
1771                                 response = camel_imap_command (store, folder, cancellable, &local_error,
1772                                                 "UID STORE %s +FLAGS.SILENT %s",
1773                                                 set, flaglist);
1774                                 if (response)
1775                                         camel_imap_response_free (store, response);
1776
1777                                 response = NULL;
1778                         }
1779                 }
1780
1781                 /* We don't use the info any more */
1782                 camel_message_info_free (info);
1783
1784                 /* Note: to 'unset' flags, use -FLAGS.SILENT (<flag list>) */
1785                 if (local_error == NULL) {
1786                         response = camel_imap_command (store, folder, cancellable, &local_error,
1787                                                "UID STORE %s %sFLAGS.SILENT %s",
1788                                                set, unset ? "-" : "", flaglist);
1789                 }
1790
1791                 g_free (set);
1792                 g_free (flaglist);
1793
1794                 if (response)
1795                         camel_imap_response_free (store, response);
1796
1797                 if (local_error == NULL) {
1798                         for (j = 0; j < matches->len; j++) {
1799                                 info = matches->pdata[j];
1800                                 if (deleted_uids && !is_gmail) {
1801                                         /* there is a real trash, do not keep this set */
1802                                         info->info.flags &= ~CAMEL_MESSAGE_DELETED;
1803                                 }
1804
1805                                 info->info.flags &= ~(CAMEL_MESSAGE_FOLDER_FLAGGED | CAMEL_IMAP_MESSAGE_MOVED);
1806                                 ((CamelImapMessageInfo *) info)->server_flags = info->info.flags & CAMEL_IMAP_SERVER_FLAGS;
1807                                 info->info.dirty = TRUE; /* Sync it back to the DB */
1808                                 if (((CamelMessageInfo *) info)->summary)
1809                                         camel_folder_summary_touch (((CamelMessageInfo *) info)->summary);
1810                         }
1811                         camel_folder_summary_touch (folder->summary);
1812                 }
1813
1814                 g_ptr_array_foreach (matches, (GFunc) camel_message_info_free, NULL);
1815                 g_ptr_array_free (matches, TRUE);
1816
1817                 /* check for an exception */
1818                 if (local_error != NULL) {
1819                         g_propagate_error (error, local_error);
1820                         if (deleted_uids) {
1821                                 g_ptr_array_foreach (deleted_uids, (GFunc) camel_pstring_free, NULL);
1822                                 g_ptr_array_free (deleted_uids, TRUE);
1823                         }
1824                         if (junked_uids) {
1825                                 g_ptr_array_foreach (junked_uids, (GFunc) camel_pstring_free, NULL);
1826                                 g_ptr_array_free (junked_uids, TRUE);
1827                         }
1828                         if (real_trash)
1829                                 g_object_unref (real_trash);
1830                         if (real_junk)
1831                                 g_object_unref (real_junk);
1832
1833                         g_ptr_array_foreach (summary, (GFunc) camel_pstring_free, NULL);
1834                         g_ptr_array_free (summary, TRUE);
1835
1836                         return FALSE;
1837                 }
1838         }
1839
1840         if (local_error == NULL)
1841                 move_messages (
1842                         folder, deleted_uids, real_trash,
1843                         cancellable, &local_error);
1844         if (local_error == NULL)
1845                 move_messages (
1846                         folder, junked_uids, real_junk,
1847                         cancellable, &local_error);
1848
1849         if (deleted_uids) {
1850                 g_ptr_array_foreach (deleted_uids, (GFunc) camel_pstring_free, NULL);
1851                 g_ptr_array_free (deleted_uids, TRUE);
1852         }
1853         if (junked_uids) {
1854                 g_ptr_array_foreach (junked_uids, (GFunc) camel_pstring_free, NULL);
1855                 g_ptr_array_free (junked_uids, TRUE);
1856         }
1857         if (real_trash)
1858                 g_object_unref (real_trash);
1859         if (real_junk)
1860                 g_object_unref (real_junk);
1861
1862         if (expunge && local_error == NULL)
1863                 imap_expunge_sync (folder, cancellable, &local_error);
1864
1865         if (local_error != NULL)
1866                 g_propagate_error (error, local_error);
1867
1868         g_ptr_array_foreach (summary, (GFunc) camel_pstring_free, NULL);
1869         g_ptr_array_free (summary, TRUE);
1870
1871         /* Save the summary */
1872         return imap_sync_offline (folder, error);
1873 }
1874
1875 static gint
1876 uid_compar (gconstpointer va,
1877             gconstpointer vb)
1878 {
1879         const gchar **sa = (const gchar **) va, **sb = (const gchar **) vb;
1880         gulong a, b;
1881
1882         a = strtoul (*sa, NULL, 10);
1883         b = strtoul (*sb, NULL, 10);
1884         if (a < b)
1885                 return -1;
1886         else if (a == b)
1887                 return 0;
1888         else
1889                 return 1;
1890 }
1891
1892 static gboolean
1893 imap_expunge_uids_offline (CamelFolder *folder,
1894                            GPtrArray *uids,
1895                            GCancellable *cancellable,
1896                            GError **error)
1897 {
1898         CamelFolderChangeInfo *changes;
1899         CamelStore *parent_store;
1900         GList *list = NULL;
1901         const gchar *full_name;
1902         gint i;
1903
1904         full_name = camel_folder_get_full_name (folder);
1905         parent_store = camel_folder_get_parent_store (folder);
1906
1907         qsort (uids->pdata, uids->len, sizeof (gpointer), uid_compar);
1908
1909         changes = camel_folder_change_info_new ();
1910
1911         for (i = 0; i < uids->len; i++) {
1912                 CamelMessageInfo *mi = camel_folder_summary_peek_loaded (folder->summary, uids->pdata[i]);
1913
1914                 if (mi) {
1915                         camel_folder_summary_remove (folder->summary, mi);
1916                         camel_message_info_free (mi);
1917                 } else {
1918                         camel_folder_summary_remove_uid (folder->summary, uids->pdata[i]);
1919                 }
1920
1921                 camel_folder_change_info_remove_uid (changes, uids->pdata[i]);
1922                 list = g_list_prepend (list, (gpointer) uids->pdata[i]);
1923                 /* We intentionally don't remove it from the cache because
1924                  * the cached data may be useful in replaying a COPY later.
1925                  */
1926         }
1927
1928         camel_db_delete_uids (parent_store->cdb_w, full_name, list, NULL);
1929         g_list_free (list);
1930         camel_folder_summary_save_to_db (folder->summary, NULL);
1931
1932         camel_imap_journal_log (CAMEL_IMAP_FOLDER (folder)->journal,
1933                                CAMEL_IMAP_JOURNAL_ENTRY_EXPUNGE, uids);
1934
1935         camel_folder_changed (folder, changes);
1936         camel_folder_change_info_free (changes);
1937
1938         return TRUE;
1939 }
1940
1941 static gboolean
1942 imap_expunge_uids_online (CamelFolder *folder,
1943                           GPtrArray *uids,
1944                           GCancellable *cancellable,
1945                           GError **error)
1946 {
1947         CamelImapStore *store;
1948         CamelImapResponse *response;
1949         gint uid = 0;
1950         gchar *set;
1951         gboolean full_expunge;
1952         CamelFolderChangeInfo *changes;
1953         CamelStore *parent_store;
1954         const gchar *full_name;
1955         gint i;
1956         GList *list = NULL;
1957
1958         full_name = camel_folder_get_full_name (folder);
1959         parent_store = camel_folder_get_parent_store (folder);
1960
1961         store = CAMEL_IMAP_STORE (parent_store);
1962         full_expunge = (store->capabilities & IMAP_CAPABILITY_UIDPLUS) == 0;
1963
1964         if (!camel_imap_store_connected (store, error))
1965                 return FALSE;
1966
1967         if ((store->capabilities & IMAP_CAPABILITY_UIDPLUS) == 0) {
1968                 if (!CAMEL_FOLDER_GET_CLASS (folder)->synchronize_sync (
1969                         folder, 0, cancellable, error)) {
1970                         return FALSE;
1971                 }
1972         }
1973
1974         qsort (uids->pdata, uids->len, sizeof (gpointer), uid_compar);
1975
1976         while (uid < uids->len) {
1977                 set = imap_uid_array_to_set (folder->summary, uids, uid, UID_SET_LIMIT, &uid);
1978                 response = camel_imap_command (store, folder, cancellable, error,
1979                                                "UID STORE %s +FLAGS.SILENT (\\Deleted)",
1980                                                set);
1981                 if (response)
1982                         camel_imap_response_free (store, response);
1983                 else {
1984                         g_free (set);
1985                         return FALSE;
1986                 }
1987
1988                 if (!full_expunge) {
1989                         GError *local_error = NULL;
1990
1991                         response = camel_imap_command (
1992                                 store, folder, cancellable, &local_error,
1993                                 "UID EXPUNGE %s", set);
1994
1995                         if (local_error != NULL) {
1996                                 g_clear_error (&local_error);
1997
1998                                 /* UID EXPUNGE failed, something is broken on the server probably,
1999                                  * thus fall back to the full expunge. It's not so good, especially
2000                                  * when resyncing, it will remove already marked messages on the
2001                                  * server too. I guess that's fine anyway, isn't it?
2002                                  * For failed command see Gnome's bug #536486 */
2003                                 full_expunge = TRUE;
2004                         }
2005                 }
2006
2007                 if (full_expunge)
2008                         response = camel_imap_command (store, folder, cancellable, NULL, "EXPUNGE");
2009
2010                 if (response)
2011                         camel_imap_response_free (store, response);
2012
2013                 g_free (set);
2014         }
2015
2016         changes = camel_folder_change_info_new ();
2017         for (i = 0; i < uids->len; i++) {
2018                 CamelMessageInfo *mi = camel_folder_summary_peek_loaded (folder->summary, uids->pdata[i]);
2019
2020                 if (mi) {
2021                         camel_folder_summary_remove (folder->summary, mi);
2022                         camel_message_info_free (mi);
2023                 } else {
2024                         camel_folder_summary_remove_uid (folder->summary, uids->pdata[i]);
2025                 }
2026
2027                 camel_folder_change_info_remove_uid (changes, uids->pdata[i]);
2028                 list = g_list_prepend (list, (gpointer) uids->pdata[i]);
2029                 /* We intentionally don't remove it from the cache because
2030                  * the cached data may be useful in replaying a COPY later.
2031                  */
2032         }
2033
2034         camel_db_delete_uids (parent_store->cdb_w, full_name, list, NULL);
2035         g_list_free (list);
2036         camel_folder_summary_save_to_db (folder->summary, NULL);
2037         camel_folder_changed (folder, changes);
2038         camel_folder_change_info_free (changes);
2039
2040         return TRUE;
2041 }
2042
2043 static gboolean
2044 imap_expunge_sync (CamelFolder *folder,
2045                    GCancellable *cancellable,
2046                    GError **error)
2047 {
2048         CamelStore *parent_store;
2049         GPtrArray *uids = NULL;
2050         const gchar *full_name;
2051         gboolean success, real_trash = FALSE;
2052
2053         full_name = camel_folder_get_full_name (folder);
2054         parent_store = camel_folder_get_parent_store (folder);
2055
2056         camel_folder_summary_save_to_db (folder->summary, NULL);
2057
2058         if ((parent_store->flags & CAMEL_STORE_VTRASH) == 0) {
2059                 CamelFolder *trash;
2060                 GError *local_error = NULL;
2061
2062                 trash = camel_store_get_trash_folder_sync (
2063                         parent_store, cancellable, &local_error);
2064
2065                 if (local_error == NULL && trash && (folder == trash || g_ascii_strcasecmp (full_name, camel_folder_get_full_name (trash)) == 0)) {
2066                         /* it's a real trash folder, thus get all mails from there */
2067                         real_trash = TRUE;
2068                         uids = camel_folder_summary_get_array (folder->summary);
2069                 }
2070
2071                 if (local_error != NULL)
2072                         g_clear_error (&local_error);
2073         }
2074
2075         if (!uids)
2076                 uids = camel_db_get_folder_deleted_uids (parent_store->cdb_r, full_name, NULL);
2077
2078         if (!uids)
2079                 return TRUE;
2080
2081         if (camel_offline_store_get_online (CAMEL_OFFLINE_STORE (parent_store)))
2082                 success = imap_expunge_uids_online (
2083                         folder, uids, cancellable, error);
2084         else
2085                 success = imap_expunge_uids_offline (
2086                         folder, uids, cancellable, error);
2087
2088         if (real_trash) {
2089                 camel_folder_summary_free_array (uids);
2090         } else {
2091                 g_ptr_array_foreach (uids, (GFunc) camel_pstring_free, NULL);
2092                 g_ptr_array_free (uids, TRUE);
2093         }
2094
2095         return success;
2096 }
2097
2098 gboolean
2099 camel_imap_expunge_uids_resyncing (CamelFolder *folder,
2100                                    GPtrArray *uids,
2101                                    GCancellable *cancellable,
2102                                    GError **error)
2103 {
2104         CamelStore *parent_store;
2105         CamelImapFolder *imap_folder = CAMEL_IMAP_FOLDER (folder);
2106         CamelImapStore *store;
2107         GPtrArray *keep_uids, *mark_uids;
2108         CamelImapResponse *response;
2109         gchar *result;
2110
2111         parent_store = camel_folder_get_parent_store (folder);
2112         store = CAMEL_IMAP_STORE (parent_store);
2113
2114         if (imap_folder->read_only)
2115                 return TRUE;
2116
2117         if (!camel_imap_store_connected (store, error))
2118                 return FALSE;
2119
2120         if (store->capabilities & IMAP_CAPABILITY_UIDPLUS)
2121                 return imap_expunge_uids_online (
2122                         folder, uids, cancellable, error);
2123
2124         /* If we don't have UID EXPUNGE we need to avoid expunging any
2125          * of the wrong messages. So we search for deleted messages,
2126          * and any that aren't in our to-expunge list get temporarily
2127          * marked un-deleted.
2128          */
2129
2130         if (!CAMEL_FOLDER_GET_CLASS (folder)->synchronize_sync (
2131                 folder, 0, cancellable, error))
2132                 return FALSE;
2133
2134         response = camel_imap_command (store, folder, cancellable, error, "UID SEARCH DELETED");
2135         if (!response)
2136                 return FALSE;
2137
2138         result = camel_imap_response_extract (store, response, "SEARCH", error);
2139         if (!result)
2140                 return FALSE;
2141
2142         if (result[8] == ' ') {
2143                 gchar *uid, *lasts = NULL;
2144                 gulong euid, kuid;
2145                 gint ei, ki;
2146
2147                 keep_uids = g_ptr_array_new ();
2148                 mark_uids = g_ptr_array_new ();
2149
2150                 /* Parse SEARCH response */
2151                 for (uid = strtok_r (result + 9, " ", &lasts); uid; uid = strtok_r (NULL, " ", &lasts))
2152                         g_ptr_array_add (keep_uids, uid);
2153                 qsort (keep_uids->pdata, keep_uids->len,
2154                        sizeof (gpointer), uid_compar);
2155
2156                 /* Fill in "mark_uids", empty out "keep_uids" as needed */
2157                 for (ei = ki = 0; ei < uids->len; ei++) {
2158                         euid = strtoul (uids->pdata[ei], NULL, 10);
2159
2160                         for (kuid = 0; ki < keep_uids->len; ki++) {
2161                                 kuid = strtoul (keep_uids->pdata[ki], NULL, 10);
2162
2163                                 if (kuid >= euid)
2164                                         break;
2165                         }
2166
2167                         if (euid == kuid)
2168                                 g_ptr_array_remove_index (keep_uids, ki);
2169                         else
2170                                 g_ptr_array_add (mark_uids, uids->pdata[ei]);
2171                 }
2172         } else {
2173                 /* Empty SEARCH result, meaning nothing is marked deleted
2174                  * on server.
2175                  */
2176
2177                 keep_uids = NULL;
2178                 mark_uids = uids;
2179         }
2180
2181         /* Unmark messages to be kept */
2182
2183         if (keep_uids) {
2184                 gchar *uidset;
2185                 gint uid = 0;
2186
2187                 while (uid < keep_uids->len) {
2188                         uidset = imap_uid_array_to_set (folder->summary, keep_uids, uid, UID_SET_LIMIT, &uid);
2189
2190                         response = camel_imap_command (store, folder, cancellable, error,
2191                                                        "UID STORE %s -FLAGS.SILENT (\\Deleted)",
2192                                                        uidset);
2193
2194                         g_free (uidset);
2195
2196                         if (!response) {
2197                                 g_ptr_array_free (keep_uids, TRUE);
2198                                 g_ptr_array_free (mark_uids, TRUE);
2199                                 return FALSE;
2200                         }
2201                         camel_imap_response_free (store, response);
2202                 }
2203         }
2204
2205         /* Mark any messages that still need to be marked */
2206         if (mark_uids) {
2207                 gchar *uidset;
2208                 gint uid = 0;
2209
2210                 while (uid < mark_uids->len) {
2211                         uidset = imap_uid_array_to_set (folder->summary, mark_uids, uid, UID_SET_LIMIT, &uid);
2212
2213                         response = camel_imap_command (store, folder, cancellable, error,
2214                                                        "UID STORE %s +FLAGS.SILENT (\\Deleted)",
2215                                                        uidset);
2216
2217                         g_free (uidset);
2218
2219                         if (!response) {
2220                                 g_ptr_array_free (keep_uids, TRUE);
2221                                 g_ptr_array_free (mark_uids, TRUE);
2222                                 return FALSE;
2223                         }
2224                         camel_imap_response_free (store, response);
2225                 }
2226
2227                 if (mark_uids != uids)
2228                         g_ptr_array_free (mark_uids, TRUE);
2229         }
2230
2231         /* Do the actual expunging */
2232         response = camel_imap_command (store, folder, cancellable, NULL, "EXPUNGE");
2233         if (response)
2234                 camel_imap_response_free (store, response);
2235
2236         /* And fix the remaining messages if we mangled them */
2237         if (keep_uids) {
2238                 gchar *uidset;
2239                 gint uid = 0;
2240
2241                 while (uid < keep_uids->len) {
2242                         uidset = imap_uid_array_to_set (folder->summary, keep_uids, uid, UID_SET_LIMIT, &uid);
2243
2244                         response = camel_imap_command (store, folder, cancellable, NULL,
2245                                                        "UID STORE %s +FLAGS.SILENT (\\Deleted)",
2246                                                        uidset);
2247
2248                         g_free (uidset);
2249                         if (response)
2250                                 camel_imap_response_free (store, response);
2251                 }
2252
2253                 g_ptr_array_free (keep_uids, TRUE);
2254         }
2255
2256         /* now we can free this, now that we're done with keep_uids */
2257         g_free (result);
2258
2259         return TRUE;
2260 }
2261
2262 gboolean
2263 camel_imap_expunge_uids_only (CamelFolder *folder,
2264                               GPtrArray *uids,
2265                               GCancellable *cancellable,
2266                               GError **error)
2267 {
2268         CamelStore *parent_store;
2269
2270         g_return_val_if_fail (folder != NULL, FALSE);
2271
2272         parent_store = camel_folder_get_parent_store (folder);
2273         g_return_val_if_fail (parent_store != NULL, FALSE);
2274
2275         g_return_val_if_fail (uids != NULL, FALSE);
2276
2277         if (camel_offline_store_get_online (CAMEL_OFFLINE_STORE (parent_store)))
2278                 return camel_imap_expunge_uids_resyncing (
2279                         folder, uids, cancellable, error);
2280         else
2281                 return imap_expunge_uids_offline (
2282                         folder, uids, cancellable, error);
2283 }
2284
2285 static gchar *
2286 get_temp_uid (void)
2287 {
2288         gchar *res;
2289
2290         static gint counter = 0;
2291         G_LOCK_DEFINE_STATIC (lock);
2292
2293         G_LOCK (lock);
2294         res = g_strdup_printf ("tempuid-%lx-%d",
2295                                (gulong) time (NULL),
2296                                counter++);
2297         G_UNLOCK (lock);
2298
2299         return res;
2300 }
2301
2302 static gboolean
2303 imap_append_offline (CamelFolder *folder,
2304                      CamelMimeMessage *message,
2305                      CamelMessageInfo *info,
2306                      gchar **appended_uid,
2307                      GError **error)
2308 {
2309         CamelImapMessageCache *cache = CAMEL_IMAP_FOLDER (folder)->cache;
2310         CamelFolderChangeInfo *changes;
2311         gchar *uid;
2312
2313         uid = get_temp_uid ();
2314
2315         camel_imap_summary_add_offline (
2316                 folder->summary, uid, message, info);
2317         CAMEL_IMAP_FOLDER_REC_LOCK (folder, cache_lock);
2318         camel_imap_message_cache_insert_wrapper (
2319                 cache, uid, "", CAMEL_DATA_WRAPPER (message));
2320         CAMEL_IMAP_FOLDER_REC_UNLOCK (folder, cache_lock);
2321
2322         changes = camel_folder_change_info_new ();
2323         camel_folder_change_info_add_uid (changes, uid);
2324         camel_folder_changed (folder, changes);
2325         camel_folder_change_info_free (changes);
2326
2327         camel_imap_journal_log (CAMEL_IMAP_FOLDER (folder)->journal,
2328                                CAMEL_IMAP_JOURNAL_ENTRY_APPEND, uid);
2329         if (appended_uid)
2330                 *appended_uid = uid;
2331         else
2332                 g_free (uid);
2333
2334         return TRUE;
2335 }
2336
2337 static void
2338 imap_folder_add_ignore_recent (CamelImapFolder *imap_folder,
2339                                const gchar *uid)
2340 {
2341         g_return_if_fail (imap_folder != NULL);
2342         g_return_if_fail (uid != NULL);
2343
2344         if (!imap_folder->priv->ignore_recent)
2345                 imap_folder->priv->ignore_recent = g_hash_table_new_full (g_str_hash, g_str_equal, (GDestroyNotify) g_free, NULL);
2346
2347         g_hash_table_insert (imap_folder->priv->ignore_recent, g_strdup (uid), GINT_TO_POINTER (1));
2348 }
2349
2350 static gboolean
2351 imap_folder_uid_in_ignore_recent (CamelImapFolder *imap_folder,
2352                                   const gchar *uid)
2353 {
2354         g_return_val_if_fail (imap_folder != NULL, FALSE);
2355         g_return_val_if_fail (uid != NULL, FALSE);
2356
2357         return imap_folder->priv->ignore_recent && g_hash_table_lookup (imap_folder->priv->ignore_recent, uid);
2358 }
2359
2360 static CamelImapResponse *
2361 do_append (CamelFolder *folder,
2362            CamelMimeMessage *message,
2363            CamelMessageInfo *info,
2364            gchar **uid,
2365            GCancellable *cancellable,
2366            GError **error)
2367 {
2368         CamelStore *parent_store;
2369         CamelImapStore *store;
2370         CamelImapResponse *response, *response2;
2371         CamelStream *memstream;
2372         CamelMimeFilter *crlf_filter;
2373         CamelStream *streamfilter;
2374         GByteArray *ba;
2375         const gchar *full_name;
2376         gchar *flagstr, *end;
2377         guint32 flags = 0;
2378         GError *local_error = NULL;
2379
2380         parent_store = camel_folder_get_parent_store (folder);
2381         store = CAMEL_IMAP_STORE (parent_store);
2382
2383         /* encode any 8bit parts so we avoid sending embedded nul-chars and such  */
2384         camel_mime_message_encode_8bit_parts (message);
2385
2386         /* FIXME: We could avoid this if we knew how big the message was. */
2387         memstream = camel_stream_mem_new ();
2388         ba = g_byte_array_new ();
2389         camel_stream_mem_set_byte_array (CAMEL_STREAM_MEM (memstream), ba);
2390
2391         streamfilter = camel_stream_filter_new (memstream);
2392         crlf_filter = camel_mime_filter_crlf_new (
2393                 CAMEL_MIME_FILTER_CRLF_ENCODE,
2394                 CAMEL_MIME_FILTER_CRLF_MODE_CRLF_ONLY);
2395         camel_stream_filter_add (
2396                 CAMEL_STREAM_FILTER (streamfilter), crlf_filter);
2397         camel_data_wrapper_write_to_stream_sync (
2398                 CAMEL_DATA_WRAPPER (message), streamfilter, cancellable, NULL);
2399         g_object_unref (streamfilter);
2400         g_object_unref (crlf_filter);
2401         g_object_unref (memstream);
2402
2403         /* Some servers don't let us append with (CamelMessageInfo *)custom flags.  If the command fails for
2404          * whatever reason, assume this is the case and save the state and try again */
2405 retry:
2406         if (info) {
2407                 flags = camel_message_info_flags (info);
2408         }
2409
2410         flags &= folder->permanent_flags;
2411         if (flags)
2412                 flagstr = imap_create_flag_list (flags, (CamelMessageInfo *) info, folder->permanent_flags);
2413         else
2414                 flagstr = NULL;
2415
2416         full_name = camel_folder_get_full_name (folder);
2417         response = camel_imap_command (
2418                 store, NULL, cancellable, &local_error, "APPEND %F%s%s {%d}",
2419                 full_name, flagstr ? " " : "",
2420                 flagstr ? flagstr : "", ba->len);
2421         g_free (flagstr);
2422
2423         if (!response) {
2424                 if (g_error_matches (local_error, CAMEL_SERVICE_ERROR, CAMEL_SERVICE_ERROR_INVALID) && !store->nocustomappend) {
2425                         g_clear_error (&local_error);
2426                         store->nocustomappend = 1;
2427                         goto retry;
2428                 }
2429                 g_propagate_error (error, local_error);
2430                 g_byte_array_free (ba, TRUE);
2431                 return NULL;
2432         }
2433
2434         if (*response->status != '+') {
2435                 camel_imap_response_free (store, response);
2436                 g_byte_array_free (ba, TRUE);
2437                 return NULL;
2438         }
2439
2440         /* send the rest of our data - the mime message */
2441         response2 = camel_imap_command_continuation (store, folder, (const gchar *) ba->data, ba->len, cancellable, error);
2442         g_byte_array_free (ba, TRUE);
2443
2444         /* free it only after message is sent. This may cause more FETCHes. */
2445         camel_imap_response_free (store, response);
2446         if (!response2)
2447                 return response2;
2448
2449         if ((store->capabilities & IMAP_CAPABILITY_UIDPLUS) != 0 ||
2450             is_google_account (parent_store)) {
2451                 *uid = camel_strstrcase (response2->status, "[APPENDUID ");
2452                 if (*uid)
2453                         *uid = strchr (*uid + 11, ' ');
2454                 if (*uid) {
2455                         *uid = g_strndup (*uid + 1, strcspn (*uid + 1, "]"));
2456                         /* Make sure it's a number */
2457                         if (strtoul (*uid, &end, 10) == 0 || *end) {
2458                                 g_free (*uid);
2459                                 *uid = NULL;
2460                         }
2461                 }
2462         } else
2463                 *uid = NULL;
2464
2465         if (*uid)
2466                 imap_folder_add_ignore_recent (CAMEL_IMAP_FOLDER (folder), *uid);
2467
2468         return response2;
2469 }
2470
2471 static gboolean
2472 imap_append_online (CamelFolder *folder,
2473                     CamelMimeMessage *message,
2474                     CamelMessageInfo *info,
2475                     gchar **appended_uid,
2476                     GCancellable *cancellable,
2477                     GError **error)
2478 {
2479         CamelStore *parent_store;
2480         CamelImapStore *store;
2481         CamelImapResponse *response;
2482         gboolean success = TRUE;
2483         gchar *uid;
2484         gint count;
2485
2486         parent_store = camel_folder_get_parent_store (folder);
2487         store = CAMEL_IMAP_STORE (parent_store);
2488
2489         if (!camel_offline_store_get_online (CAMEL_OFFLINE_STORE (store))) {
2490                 return imap_append_offline (
2491                         folder, message, info, appended_uid, error);
2492         }
2493
2494         if (!camel_imap_store_connected (store, error))
2495                 return FALSE;
2496
2497         count = camel_folder_summary_count (folder->summary);
2498         response = do_append (folder, message, info, &uid, cancellable, error);
2499         if (!response)
2500                 return FALSE;
2501
2502         if (uid) {
2503                 /* Cache first, since freeing response may trigger a
2504                  * summary update that will want this information.
2505                  */
2506                 CAMEL_IMAP_FOLDER_REC_LOCK (folder, cache_lock);
2507                 camel_imap_message_cache_insert_wrapper (
2508                         CAMEL_IMAP_FOLDER (folder)->cache, uid,
2509                         "", CAMEL_DATA_WRAPPER (message));
2510                 CAMEL_IMAP_FOLDER_REC_UNLOCK (folder, cache_lock);
2511                 if (appended_uid)
2512                         *appended_uid = uid;
2513                 else
2514                         g_free (uid);
2515         } else if (appended_uid)
2516                 *appended_uid = NULL;
2517
2518         camel_imap_response_free (store, response);
2519
2520         /* Make sure a "folder_changed" is emitted. */
2521         if (store->current_folder != folder ||
2522             camel_folder_summary_count (folder->summary) == count)
2523                 success = imap_refresh_info_sync (folder, cancellable, error);
2524
2525         return success;
2526 }
2527
2528 gboolean
2529 camel_imap_append_resyncing (CamelFolder *folder,
2530                              CamelMimeMessage *message,
2531                              CamelMessageInfo *info,
2532                              gchar **appended_uid,
2533                              GCancellable *cancellable,
2534                              GError **error)
2535 {
2536         CamelStore *parent_store;
2537         CamelImapStore *store;
2538         CamelImapResponse *response;
2539         gchar *uid;
2540
2541         parent_store = camel_folder_get_parent_store (folder);
2542         store = CAMEL_IMAP_STORE (parent_store);
2543
2544         response = do_append (folder, message, info, &uid, cancellable, error);
2545         if (!response)
2546                 return FALSE;
2547
2548         if (uid) {
2549                 CamelImapFolder *imap_folder = CAMEL_IMAP_FOLDER (folder);
2550                 const gchar *olduid = camel_message_info_uid (info);
2551
2552                 CAMEL_IMAP_FOLDER_REC_LOCK (imap_folder, cache_lock);
2553                 camel_imap_message_cache_copy (imap_folder->cache, olduid,
2554                                                imap_folder->cache, uid);
2555                 CAMEL_IMAP_FOLDER_REC_UNLOCK (imap_folder, cache_lock);
2556
2557                 if (appended_uid)
2558                         *appended_uid = uid;
2559                 else
2560                         g_free (uid);
2561         } else if (appended_uid)
2562                 *appended_uid = NULL;
2563
2564         camel_imap_response_free (store, response);
2565
2566         return TRUE;
2567 }
2568
2569 static gboolean
2570 imap_transfer_offline (CamelFolder *source,
2571                        GPtrArray *uids,
2572                        CamelFolder *dest,
2573                        gboolean delete_originals,
2574                        GPtrArray **transferred_uids,
2575                        GCancellable *cancellable,
2576                        GError **error)
2577 {
2578         CamelImapMessageCache *sc = CAMEL_IMAP_FOLDER (source)->cache;
2579         CamelImapMessageCache *dc = CAMEL_IMAP_FOLDER (dest)->cache;
2580         CamelFolderChangeInfo *changes;
2581         CamelMimeMessage *message;
2582         CamelMessageInfo *mi;
2583         gchar *uid, *destuid;
2584         gint i;
2585         GError *local_error = NULL;
2586
2587         /* We grab the store's command lock first, and then grab the
2588          * source and destination cache_locks. This way we can't
2589          * deadlock in the case where we're simultaneously also trying
2590          * to copy messages in the other direction from another thread.
2591          */
2592         CAMEL_IMAP_FOLDER_REC_LOCK (source, cache_lock);
2593         CAMEL_IMAP_FOLDER_REC_LOCK (dest, cache_lock);
2594
2595         if (transferred_uids) {
2596                 *transferred_uids = g_ptr_array_new ();
2597                 g_ptr_array_set_size (*transferred_uids, uids->len);
2598         }
2599
2600         changes = camel_folder_change_info_new ();
2601
2602         for (i = 0; i < uids->len && local_error == NULL; i++) {
2603                 uid = uids->pdata[i];
2604
2605                 destuid = get_temp_uid ();
2606
2607                 mi = camel_folder_summary_get (source->summary, uid);
2608                 g_return_val_if_fail (mi != NULL, FALSE);
2609
2610                 message = camel_folder_get_message_sync (
2611                         source, uid, cancellable, &local_error);
2612
2613                 if (message) {
2614                         camel_imap_summary_add_offline (
2615                                 dest->summary, destuid, message, mi);
2616                         g_object_unref (message);
2617                 } else
2618                         camel_imap_summary_add_offline_uncached (
2619                                 dest->summary, destuid, mi);
2620
2621                 camel_imap_message_cache_copy (sc, uid, dc, destuid);
2622                 camel_message_info_free (mi);
2623
2624                 camel_folder_change_info_add_uid (changes, destuid);
2625                 if (transferred_uids)
2626                         (*transferred_uids)->pdata[i] = destuid;
2627                 else
2628                         g_free (destuid);
2629
2630                 if (delete_originals && local_error == NULL)
2631                         camel_folder_delete_message (source, uid);
2632         }
2633
2634         CAMEL_IMAP_FOLDER_REC_UNLOCK (dest, cache_lock);
2635         CAMEL_IMAP_FOLDER_REC_UNLOCK (source, cache_lock);
2636
2637         camel_folder_changed (dest, changes);
2638         camel_folder_change_info_free (changes);
2639
2640         camel_imap_journal_log (
2641                 CAMEL_IMAP_FOLDER (source)->journal,
2642                 CAMEL_IMAP_JOURNAL_ENTRY_TRANSFER,
2643                 dest, uids, delete_originals, NULL);
2644
2645         if (local_error != NULL) {
2646                 g_propagate_error (error, local_error);
2647                 return FALSE;
2648         }
2649
2650         return TRUE;
2651 }
2652
2653 /* Call with lock held on destination folder cache */
2654 static void
2655 handle_copyuid (CamelImapResponse *response,
2656                 CamelFolder *source,
2657                 CamelFolder *destination)
2658 {
2659         CamelImapMessageCache *scache = CAMEL_IMAP_FOLDER (source)->cache;
2660         CamelImapMessageCache *dcache = CAMEL_IMAP_FOLDER (destination)->cache;
2661         gchar *validity, *srcset, *destset;
2662         GPtrArray *src, *dest;
2663         gint i;
2664
2665         validity = camel_strstrcase (response->status, "[COPYUID ");
2666         if (!validity)
2667                 return;
2668         validity += 9;
2669         if (strtoul (validity, NULL, 10) !=
2670             CAMEL_IMAP_SUMMARY (destination->summary)->validity)
2671                 return;
2672
2673         srcset = strchr (validity, ' ');
2674         if (!srcset++)
2675                 goto lose;
2676         destset = strchr (srcset, ' ');
2677         if (!destset++)
2678                 goto lose;
2679
2680         src = imap_uid_set_to_array (source->summary, srcset);
2681         dest = imap_uid_set_to_array (destination->summary, destset);
2682
2683         if (src && dest && src->len == dest->len) {
2684                 /* We don't have to worry about deadlocking on the
2685                  * cache locks here, because we've got the store's
2686                  * command lock too, so no one else could be here.
2687                  */
2688                 CAMEL_IMAP_FOLDER_REC_LOCK (source, cache_lock);
2689                 for (i = 0; i < src->len; i++) {
2690                         camel_imap_message_cache_copy (scache, src->pdata[i],
2691                                                        dcache, dest->pdata[i]);
2692
2693                         imap_folder_add_ignore_recent (CAMEL_IMAP_FOLDER (destination), dest->pdata[i]);
2694                 }
2695                 CAMEL_IMAP_FOLDER_REC_UNLOCK (source, cache_lock);
2696
2697                 imap_uid_array_free (src);
2698                 imap_uid_array_free (dest);
2699                 return;
2700         }
2701
2702         if (src)
2703                 imap_uid_array_free (src);
2704         if (dest)
2705                 imap_uid_array_free (dest);
2706  lose:
2707         g_warning ("Bad COPYUID response from server");
2708 }
2709
2710 /* Call with lock held on destination folder cache */
2711 static void
2712 handle_copyuid_copy_user_tags (CamelImapResponse *response,
2713                                CamelFolder *source,
2714                                CamelFolder *destination,
2715                                GCancellable *cancellable)
2716 {
2717         CamelStore *parent_store;
2718         gchar *validity, *srcset, *destset;
2719         GPtrArray *src, *dest;
2720         gint i;
2721
2722         validity = camel_strstrcase (response->status, "[COPYUID ");
2723         if (!validity)
2724                 return;
2725         validity += 9;
2726         if (strtoul (validity, NULL, 10) !=
2727             CAMEL_IMAP_SUMMARY (destination->summary)->validity)
2728                 return;
2729
2730         srcset = strchr (validity, ' ');
2731         if (!srcset++)
2732                 goto lose;
2733         destset = strchr (srcset, ' ');
2734         if (!destset++)
2735                 goto lose;
2736
2737         /* first do NOOP on the destination folder, so server has enough time to propagate our copy command there */
2738         parent_store = camel_folder_get_parent_store (destination);
2739         camel_imap_response_free (
2740                 CAMEL_IMAP_STORE (parent_store), camel_imap_command (
2741                 CAMEL_IMAP_STORE (parent_store), destination, cancellable, NULL, "NOOP"));
2742
2743         /* refresh folder's summary first, we copied messages there on the server,
2744          * but do not know about it in a local summary */
2745         if (!imap_refresh_info_sync (destination, cancellable, NULL))
2746                 goto lose;
2747
2748         src = imap_uid_set_to_array (source->summary, srcset);
2749         dest = imap_uid_set_to_array (destination->summary, destset);
2750
2751         if (src && dest && src->len == dest->len) {
2752                 CAMEL_IMAP_FOLDER_REC_LOCK (source, cache_lock);
2753                 for (i = 0; i < src->len; i++) {
2754                         CamelMessageInfo *mi = camel_folder_get_message_info (source, src->pdata[i]);
2755
2756                         if (mi) {
2757                                 const CamelTag *tag = camel_message_info_user_tags (mi);
2758
2759                                 while (tag) {
2760                                         camel_folder_set_message_user_tag (destination, dest->pdata[i], tag->name, tag->value);
2761                                         tag = tag->next;
2762                                 }
2763
2764                                 camel_folder_free_message_info (source, mi);
2765                         }
2766                 }
2767                 CAMEL_IMAP_FOLDER_REC_UNLOCK (source, cache_lock);
2768
2769                 imap_uid_array_free (src);
2770                 imap_uid_array_free (dest);
2771                 return;
2772         }
2773
2774         if (src)
2775                 imap_uid_array_free (src);
2776         if (dest)
2777                 imap_uid_array_free (dest);
2778  lose:
2779         g_warning ("Bad COPYUID response from server");
2780 }
2781
2782 /* returns whether any of messages from uidset has set any user tag or not */
2783 static gboolean
2784 any_has_user_tag (CamelFolder *source,
2785                   gchar *uidset)
2786 {
2787         GPtrArray *src;
2788
2789         g_return_val_if_fail (source != NULL && uidset != NULL, FALSE);
2790
2791         src = imap_uid_set_to_array (source->summary, uidset);
2792         if (src) {
2793                 gboolean have = FALSE;
2794                 gint i;
2795
2796                 CAMEL_IMAP_FOLDER_REC_LOCK (source, cache_lock);
2797                 for (i = 0; i < src->len && !have; i++) {
2798                         CamelMessageInfo *mi = camel_folder_get_message_info (source, src->pdata[i]);
2799
2800                         if (mi) {
2801                                 have = camel_message_info_user_tags (mi) != NULL;
2802
2803                                 camel_folder_free_message_info (source, mi);
2804                         }
2805                 }
2806                 CAMEL_IMAP_FOLDER_REC_UNLOCK (source, cache_lock);
2807
2808                 imap_uid_array_free (src);
2809
2810                 return have;
2811         }
2812
2813         return FALSE;
2814 }
2815
2816 static gboolean
2817 do_copy (CamelFolder *source,
2818          GPtrArray *uids,
2819          CamelFolder *destination,
2820          gint delete_originals,
2821          GCancellable *cancellable,
2822          GError **error)
2823 {
2824         CamelService *service;
2825         CamelSettings *settings;
2826         CamelStore *parent_store;
2827         CamelImapStore *store;
2828         CamelImapResponse *response;
2829         const gchar *full_name;
2830         gchar *trash_path;
2831         gchar *uidset;
2832         gint uid = 0, last = 0, i;
2833         GError *local_error = NULL;
2834         gboolean mark_moved;
2835         gboolean success = TRUE;
2836
2837         parent_store = camel_folder_get_parent_store (source);
2838         store = CAMEL_IMAP_STORE (parent_store);
2839
2840         if (!camel_imap_store_connected (store, error))
2841                 return FALSE;
2842
2843         service = CAMEL_SERVICE (parent_store);
2844
2845         settings = camel_service_ref_settings (service);
2846
2847         trash_path = camel_imap_settings_dup_real_trash_path (
2848                 CAMEL_IMAP_SETTINGS (settings));
2849
2850         g_object_unref (settings);
2851
2852         mark_moved = is_google_account (parent_store) && trash_path != NULL;
2853
2854         full_name = camel_folder_get_full_name (destination);
2855
2856         while (uid < uids->len && local_error == NULL) {
2857                 uidset = imap_uid_array_to_set (source->summary, uids, uid, UID_SET_LIMIT, &uid);
2858
2859                 /* use XGWMOVE only when none of the moving messages has set any user tag */
2860                 if ((store->capabilities & IMAP_CAPABILITY_XGWMOVE) != 0 && delete_originals && !any_has_user_tag (source, uidset)) {
2861                         response = camel_imap_command (
2862                                 store, source, cancellable, &local_error,
2863                                 "UID XGWMOVE %s %F", uidset, full_name);
2864                         /* returns only 'A00012 OK UID XGWMOVE completed' '* 2 XGWMOVE' so nothing useful */
2865                         camel_imap_response_free (store, response);
2866                 } else {
2867                         CAMEL_IMAP_FOLDER_REC_LOCK (destination, cache_lock);
2868                         response = camel_imap_command (
2869                                 store, source, cancellable, &local_error,
2870                                 "UID COPY %s %F", uidset, full_name);
2871                         if (response && (store->capabilities & IMAP_CAPABILITY_UIDPLUS))
2872                                 handle_copyuid (response, source, destination);
2873                         if (response)
2874                                 handle_copyuid_copy_user_tags (
2875                                         response, source, destination,
2876                                         cancellable);
2877                         camel_imap_response_free (store, response);
2878                         CAMEL_IMAP_FOLDER_REC_UNLOCK (destination, cache_lock);
2879                 }
2880
2881                 if (local_error == NULL && delete_originals && (mark_moved || !trash_path)) {
2882                         for (i = last; i < uid; i++) {
2883                                 camel_folder_delete_message (
2884                                         source, uids->pdata[i]);
2885                                 if (mark_moved) {
2886                                         CamelMessageInfoBase *info = (CamelMessageInfoBase *) camel_folder_summary_get (source->summary, uids->pdata[i]);
2887
2888                                         if (info)
2889                                                 info->flags |= CAMEL_IMAP_MESSAGE_MOVED;
2890                                 }
2891                         }
2892                         last = uid;
2893                 }
2894                 g_free (uidset);
2895         }
2896
2897         if (local_error != NULL) {
2898                 g_propagate_error (error, local_error);
2899                 success = FALSE;
2900
2901         /* There is a real trash folder set, which is not on a google account
2902          * and copied messages should be deleted, thus do not move them into
2903          * a trash folder, but just expunge them, because the copy part of
2904          * the operation was successful. */
2905         } else if (trash_path && !mark_moved && delete_originals)
2906                 camel_imap_expunge_uids_only (source, uids, cancellable, NULL);
2907
2908         g_free (trash_path);
2909
2910         return success;
2911 }
2912
2913 static gboolean
2914 imap_transfer_messages (CamelFolder *source,
2915                         GPtrArray *uids,
2916                         CamelFolder *dest,
2917                         gboolean delete_originals,
2918                         GPtrArray **transferred_uids,
2919                         gboolean can_call_sync,
2920                         GCancellable *cancellable,
2921                         GError **error)
2922 {
2923         CamelStore *parent_store;
2924         CamelImapStore *store;
2925         gboolean success = TRUE;
2926         gint count;
2927
2928         parent_store = camel_folder_get_parent_store (source);
2929         store = CAMEL_IMAP_STORE (parent_store);
2930
2931         if (!camel_offline_store_get_online (CAMEL_OFFLINE_STORE (store)))
2932                 return imap_transfer_offline (
2933                         source, uids, dest, delete_originals,
2934                         transferred_uids, cancellable, error);
2935
2936         /* Sync message flags if needed. */
2937         if (can_call_sync && !imap_synchronize_sync (
2938                 source, FALSE, cancellable, error))
2939                 return FALSE;
2940
2941         count = camel_folder_summary_count (dest->summary);
2942
2943         qsort (uids->pdata, uids->len, sizeof (gpointer), uid_compar);
2944
2945         /* Now copy the messages */
2946         if (!do_copy (source, uids, dest, delete_originals, cancellable, error))
2947                 return FALSE;
2948
2949         /* Make the destination notice its new messages */
2950         if (store->current_folder != dest ||
2951             camel_folder_summary_count (dest->summary) == count)
2952                 success = imap_refresh_info_sync (dest, cancellable, error);
2953
2954         /* FIXME */
2955         if (transferred_uids)
2956                 *transferred_uids = NULL;
2957
2958         return success;
2959 }
2960
2961 static gboolean
2962 imap_transfer_online (CamelFolder *source,
2963                       GPtrArray *uids,
2964                       CamelFolder *dest,
2965                       gboolean delete_originals,
2966                       GPtrArray **transferred_uids,
2967                       GCancellable *cancellable,
2968                       GError **error)
2969 {
2970         return imap_transfer_messages (
2971                 source, uids, dest, delete_originals,
2972                 transferred_uids, TRUE, cancellable, error);
2973 }
2974
2975 gboolean
2976 camel_imap_transfer_resyncing (CamelFolder *source,
2977                                GPtrArray *uids,
2978                                CamelFolder *dest,
2979                                gboolean delete_originals,
2980                                GPtrArray **transferred_uids,
2981                                GCancellable *cancellable,
2982                                GError **error)
2983 {
2984         GPtrArray *realuids;
2985         gint first, i;
2986         const gchar *uid;
2987         CamelMimeMessage *message;
2988         CamelMessageInfo *info;
2989         GError *local_error = NULL;
2990
2991         qsort (uids->pdata, uids->len, sizeof (gpointer), uid_compar);
2992
2993         /*This is trickier than append_resyncing, because some of
2994          * the messages we are copying may have been copied or
2995          * appended into @source while we were offline, in which case
2996          * if we don't have UIDPLUS, we won't know their real UIDs,
2997          * so we'll have to append them rather than copying. */
2998
2999         realuids = g_ptr_array_new ();
3000
3001         i = 0;
3002         while (i < uids->len && local_error == NULL) {
3003                  /* Skip past real UIDs  */
3004                 for (first = i; i < uids->len; i++) {
3005                         uid = uids->pdata[i];
3006
3007                         if (!isdigit ((guchar) * uid)) {
3008                                 uid = camel_imap_journal_uidmap_lookup ((CamelIMAPJournal *) CAMEL_IMAP_FOLDER (source)->journal, uid);
3009                                 if (!uid)
3010                                         break;
3011                         }
3012                         g_ptr_array_add (realuids, (gchar *) uid);
3013                 }
3014
3015                 /* If we saw any real UIDs, do a COPY */
3016                 if (i != first) {
3017                         do_copy (
3018                                 source, realuids, dest, delete_originals,
3019                                 cancellable, &local_error);
3020                         g_ptr_array_set_size (realuids, 0);
3021                         if (i == uids->len || local_error != NULL)
3022                                 break;
3023                 }
3024
3025                 /* Deal with fake UIDs */
3026                 while (i < uids->len &&
3027                        !isdigit (*(guchar *)(uids->pdata[i])) &&
3028                        local_error == NULL) {
3029                         uid = uids->pdata[i];
3030                         message = camel_folder_get_message_sync (
3031                                 source, uid, cancellable, NULL);
3032                         if (!message) {
3033                                 /* Message must have been expunged */
3034                                 i++;
3035                                 continue;
3036                         }
3037                         info = camel_folder_get_message_info (source, uid);
3038                         g_return_val_if_fail (info != NULL, FALSE);
3039
3040                         imap_append_online (
3041                                 dest, message, info,
3042                                 NULL, cancellable, &local_error);
3043                         camel_folder_free_message_info (source, info);
3044                         g_object_unref (message);
3045                         if (delete_originals && local_error == NULL)
3046                                 camel_folder_delete_message (source, uid);
3047                         i++;
3048                 }
3049         }
3050
3051         g_ptr_array_free (realuids, FALSE);
3052
3053         /* FIXME */
3054         if (transferred_uids)
3055                 *transferred_uids = NULL;
3056
3057         if (local_error != NULL) {
3058                 g_propagate_error (error, local_error);
3059                 return FALSE;
3060         }
3061
3062         return TRUE;
3063 }
3064
3065 static GPtrArray *
3066 imap_search_by_expression (CamelFolder *folder,
3067                            const gchar *expression,
3068                            GCancellable *cancellable,
3069                            GError **error)
3070 {
3071         CamelImapFolder *imap_folder = CAMEL_IMAP_FOLDER (folder);
3072         GPtrArray *matches;
3073
3074         /* we could get around this by creating a new search object each time,
3075          * but i doubt its worth it since any long operation would lock the
3076          * command channel too */
3077         CAMEL_IMAP_FOLDER_LOCK (folder, search_lock);
3078
3079         camel_folder_search_set_folder (imap_folder->search, folder);
3080         matches = camel_folder_search_search (imap_folder->search, expression, NULL, cancellable, error);
3081
3082         CAMEL_IMAP_FOLDER_UNLOCK (folder, search_lock);
3083
3084         return matches;
3085 }
3086
3087 static guint32
3088 imap_count_by_expression (CamelFolder *folder,
3089                           const gchar *expression,
3090                           GCancellable *cancellable,
3091                           GError **error)
3092 {
3093         CamelImapFolder *imap_folder = CAMEL_IMAP_FOLDER (folder);
3094         guint32 matches;
3095
3096         /* we could get around this by creating a new search object each time,
3097          * but i doubt its worth it since any long operation would lock the
3098          * command channel too */
3099         CAMEL_IMAP_FOLDER_LOCK (folder, search_lock);
3100
3101         camel_folder_search_set_folder (imap_folder->search, folder);
3102         matches = camel_folder_search_count (imap_folder->search, expression, cancellable, error);
3103
3104         CAMEL_IMAP_FOLDER_UNLOCK (folder, search_lock);
3105
3106         return matches;
3107 }
3108
3109 static GPtrArray *
3110 imap_search_by_uids (CamelFolder *folder,
3111                      const gchar *expression,
3112                      GPtrArray *uids,
3113                      GCancellable *cancellable,
3114                      GError **error)
3115 {
3116         CamelImapFolder *imap_folder = CAMEL_IMAP_FOLDER (folder);
3117         GPtrArray *matches;
3118
3119         if (uids->len == 0)
3120                 return g_ptr_array_new ();
3121
3122         CAMEL_IMAP_FOLDER_LOCK (folder, search_lock);
3123
3124         camel_folder_search_set_folder (imap_folder->search, folder);
3125         matches = camel_folder_search_search (imap_folder->search, expression, uids, cancellable, error);
3126
3127         CAMEL_IMAP_FOLDER_UNLOCK (folder, search_lock);
3128
3129         return matches;
3130 }
3131
3132 static void
3133 imap_search_free (CamelFolder *folder,
3134                   GPtrArray *uids)
3135 {
3136         CamelImapFolder *imap_folder = CAMEL_IMAP_FOLDER (folder);
3137
3138         g_return_if_fail (imap_folder->search);
3139
3140         CAMEL_IMAP_FOLDER_LOCK (folder, search_lock);
3141
3142         camel_folder_search_free_result (imap_folder->search, uids);
3143
3144         CAMEL_IMAP_FOLDER_UNLOCK (folder, search_lock);
3145 }
3146
3147 static CamelMimeMessage *get_message (CamelImapFolder *imap_folder,
3148                                       const gchar *uid,
3149                                       CamelMessageContentInfo *ci,
3150                                       GCancellable *cancellable,
3151                                       GError **error);
3152
3153 struct _part_spec_stack {
3154         struct _part_spec_stack *parent;
3155         gint part;
3156 };
3157
3158 static void
3159 part_spec_push (struct _part_spec_stack **stack,
3160                 gint part)
3161 {
3162         struct _part_spec_stack *node;
3163
3164         node = g_new (struct _part_spec_stack, 1);
3165         node->parent = *stack;
3166         node->part = part;
3167
3168         *stack = node;
3169 }
3170
3171 static gint
3172 part_spec_pop (struct _part_spec_stack **stack)
3173 {
3174         struct _part_spec_stack *node;
3175         gint part;
3176
3177         g_return_val_if_fail (*stack != NULL, 0);
3178
3179         node = *stack;
3180         *stack = node->parent;
3181
3182         part = node->part;
3183         g_free (node);
3184
3185         return part;
3186 }
3187
3188 static gchar *
3189 content_info_get_part_spec (CamelMessageContentInfo *ci)
3190 {
3191         struct _part_spec_stack *stack = NULL;
3192         CamelMessageContentInfo *node;
3193         gchar *part_spec, *buf;
3194         gsize len = 1;
3195         gint part;
3196
3197         node = ci;
3198         while (node->parent) {
3199                 CamelMessageContentInfo *child;
3200
3201                 /* FIXME: is this only supposed to apply if 'node' is a multipart? */
3202                 if (node->parent->parent &&
3203                                 camel_content_type_is (node->parent->type, "message", "*") &&
3204                                 !camel_content_type_is (node->parent->parent->type, "message", "*")) {
3205                         node = node->parent;
3206                         continue;
3207                 }
3208
3209                 child = node->parent->childs;
3210                 for (part = 1; child; part++) {
3211                         if (child == node)
3212                                 break;
3213
3214                         child = child->next;
3215                 }
3216
3217                 part_spec_push (&stack, part);
3218
3219                 len += 2;
3220                 while ((part = part / 10))
3221                         len++;
3222
3223                 node = node->parent;
3224         }
3225
3226         buf = part_spec = g_malloc (len);
3227         part_spec[0] = '\0';
3228
3229         while (stack) {
3230                 part = part_spec_pop (&stack);
3231                 buf += sprintf (buf, "%d%s", part, stack ? "." : "");
3232         }
3233
3234         return part_spec;
3235 }
3236
3237 /* Fetch the contents of the MIME part indicated by @ci, which is part
3238  * of message @uid in @folder.
3239  */
3240 static CamelDataWrapper *
3241 get_content (CamelImapFolder *imap_folder,
3242              const gchar *uid,
3243              CamelMimePart *part,
3244              CamelMessageContentInfo *ci,
3245              gint frommsg,
3246              GCancellable *cancellable,
3247              GError **error)
3248 {
3249         CamelDataWrapper *content = NULL;
3250         CamelStream *stream;
3251         gchar *part_spec;
3252
3253         part_spec = content_info_get_part_spec (ci);
3254
3255         d(printf("get content '%s' '%s' (frommsg = %d)\n", part_spec, camel_content_type_format(ci->type), frommsg));
3256
3257         /* There are three cases: multipart/signed, multipart, message/rfc822, and "other" */
3258         if (camel_content_type_is (ci->type, "multipart", "signed")) {
3259                 CamelMultipartSigned *body_mp;
3260                 gchar *spec;
3261                 gboolean success;
3262
3263                 /* Note: because we get the content parts uninterpreted anyway, we could potentially
3264                  * just use the normalmultipart code, except that multipart/signed wont let you yet! */
3265
3266                 body_mp = camel_multipart_signed_new ();
3267                 /* need to set this so it grabs the boundary and other info about the signed type */
3268                 /* we assume that part->content_type is more accurate/full than ci->type */
3269                 camel_data_wrapper_set_mime_type_field (CAMEL_DATA_WRAPPER (body_mp), CAMEL_DATA_WRAPPER (part)->mime_type);
3270
3271                 spec = g_alloca (strlen (part_spec) + 6);
3272                 if (frommsg)
3273                         sprintf(spec, part_spec[0] ? "%s.TEXT" : "TEXT", part_spec);
3274                 else
3275                         strcpy (spec, part_spec);
3276                 g_free (part_spec);
3277
3278                 stream = camel_imap_folder_fetch_data (imap_folder, uid, spec, FALSE, cancellable, error);
3279                 if (stream) {
3280                         success = camel_data_wrapper_construct_from_stream_sync (
3281                                 CAMEL_DATA_WRAPPER (body_mp), stream, cancellable, error);
3282                         g_object_unref (stream);
3283                         if (!success) {
3284                                 g_object_unref ( body_mp);
3285                                 return NULL;
3286                         }
3287                 }
3288
3289                 return (CamelDataWrapper *) body_mp;
3290         } else if (camel_content_type_is (ci->type, "multipart", "*")) {
3291                 CamelMultipart *body_mp;
3292                 gchar *child_spec;
3293                 gint speclen, num, isdigest;
3294
3295                 if (camel_content_type_is (ci->type, "multipart", "encrypted"))
3296                         body_mp = (CamelMultipart *) camel_multipart_encrypted_new ();
3297                 else
3298                         body_mp = camel_multipart_new ();
3299
3300                 /* need to set this so it grabs the boundary and other info about the multipart */
3301                 /* we assume that part->content_type is more accurate/full than ci->type */
3302                 camel_data_wrapper_set_mime_type_field (CAMEL_DATA_WRAPPER (body_mp), CAMEL_DATA_WRAPPER (part)->mime_type);
3303                 isdigest = camel_content_type_is(((CamelDataWrapper *)part)->mime_type, "multipart", "digest");
3304
3305                 speclen = strlen (part_spec);
3306                 child_spec = g_malloc (speclen + 17); /* dot + 10 + dot + MIME + nul */
3307                 memcpy (child_spec, part_spec, speclen);
3308                 if (speclen > 0)
3309                         child_spec[speclen++] = '.';
3310                 g_free (part_spec);
3311
3312                 ci = ci->childs;
3313                 num = 1;
3314                 while (ci) {
3315                         sprintf (child_spec + speclen, "%d.MIME", num++);
3316                         stream = camel_imap_folder_fetch_data (imap_folder, uid, child_spec, FALSE, cancellable, error);
3317                         if (stream) {
3318                                 gboolean success;
3319
3320                                 part = camel_mime_part_new ();
3321                                 success = camel_data_wrapper_construct_from_stream_sync (
3322                                         CAMEL_DATA_WRAPPER (part), stream, cancellable, error);
3323                                 g_object_unref (stream);
3324                                 if (!success) {
3325                                         g_object_unref (part);
3326                                         g_object_unref (body_mp);
3327                                         g_free (child_spec);
3328                                         return NULL;
3329                                 }
3330
3331                                 content = get_content (imap_folder, uid, part, ci, FALSE, cancellable, error);
3332                         }
3333
3334                         if (!stream || !content) {
3335                                 g_object_unref (body_mp);
3336                                 g_free (child_spec);
3337                                 return NULL;
3338                         }
3339
3340                         if (camel_debug("imap:folder")) {
3341                                 gchar *ct = camel_content_type_format (camel_mime_part_get_content_type ((CamelMimePart *) part));
3342                                 gchar *ct2 = camel_content_type_format (ci->type);
3343
3344                                 printf("Setting part content type to '%s' contentinfo type is '%s'\n", ct, ct2);
3345                                 g_free (ct);
3346                                 g_free (ct2);
3347                         }
3348
3349                         /* if we had no content-type header on a multipart/digest sub-part, then we need to
3350                          * treat it as message/rfc822 instead */
3351                         if (isdigest && camel_medium_get_header((CamelMedium *)part, "content-type") == NULL) {
3352                                 CamelContentType *ct = camel_content_type_new("message", "rfc822");
3353
3354                                 camel_data_wrapper_set_mime_type_field (content, ct);
3355                                 camel_content_type_unref (ct);
3356                         } else {
3357                                 camel_data_wrapper_set_mime_type_field (content, camel_mime_part_get_content_type (part));
3358                         }
3359
3360                         camel_medium_set_content (CAMEL_MEDIUM (part), content);
3361                         g_object_unref (content);
3362
3363                         camel_multipart_add_part (body_mp, part);
3364                         g_object_unref (part);
3365
3366                         ci = ci->next;
3367                 }
3368
3369                 g_free (child_spec);
3370
3371                 return (CamelDataWrapper *) body_mp;
3372         } else if (camel_content_type_is (ci->type, "message", "rfc822")) {
3373                 content = (CamelDataWrapper *) get_message (imap_folder, uid, ci->childs, cancellable, error);
3374                 g_free (part_spec);
3375                 return content;
3376         } else {
3377                 CamelTransferEncoding enc;
3378                 gchar *spec;
3379
3380                 /* NB: we need this differently to multipart/signed case above on purpose */
3381                 spec = g_alloca (strlen (part_spec) + 6);
3382                 if (frommsg)
3383                         sprintf(spec, part_spec[0] ? "%s.1" : "1", part_spec);
3384                 else
3385                         strcpy(spec, part_spec[0]?part_spec:"1");
3386
3387                 enc = ci->encoding ? camel_transfer_encoding_from_string (ci->encoding) : CAMEL_TRANSFER_ENCODING_DEFAULT;
3388                 content = camel_imap_wrapper_new (imap_folder, ci->type, enc, uid, spec, part);
3389                 g_free (part_spec);
3390                 return content;
3391         }
3392 }
3393
3394 static CamelMimeMessage *
3395 get_message (CamelImapFolder *imap_folder,
3396              const gchar *uid,
3397              CamelMessageContentInfo *ci,
3398              GCancellable *cancellable,
3399              GError **error)
3400 {
3401         CamelFolder *folder;
3402         CamelStore *parent_store;
3403         CamelImapStore *store;
3404         CamelDataWrapper *content;
3405         CamelMimeMessage *msg;
3406         CamelStream *stream;
3407         gchar *section_text, *part_spec;
3408         gboolean success;
3409
3410         folder = CAMEL_FOLDER (imap_folder);
3411         parent_store = camel_folder_get_parent_store (folder);
3412         store = CAMEL_IMAP_STORE (parent_store);
3413
3414         part_spec = content_info_get_part_spec (ci);
3415         d(printf("get message '%s'\n", part_spec));
3416         section_text = g_strdup_printf ("%s%s%s", part_spec, *part_spec ? "." : "",
3417                                         store->server_level >= IMAP_LEVEL_IMAP4REV1 ? "HEADER" : "0");
3418
3419         stream = camel_imap_folder_fetch_data (imap_folder, uid, section_text, FALSE, cancellable, error);
3420         g_free (section_text);
3421         g_free (part_spec);
3422         if (!stream)
3423                 return NULL;
3424
3425         msg = camel_mime_message_new ();
3426         success = camel_data_wrapper_construct_from_stream_sync (
3427                 CAMEL_DATA_WRAPPER (msg), stream, cancellable, error);
3428         g_object_unref (stream);
3429         if (!success) {
3430                 g_object_unref (msg);
3431                 return NULL;
3432         }
3433
3434         content = get_content (imap_folder, uid, CAMEL_MIME_PART (msg), ci, TRUE, cancellable, error);
3435         if (!content) {
3436                 g_object_unref (msg);
3437                 return NULL;
3438         }
3439
3440         if (camel_debug("imap:folder")) {
3441                 gchar *ct = camel_content_type_format (camel_mime_part_get_content_type ((CamelMimePart *) msg));
3442                 gchar *ct2 = camel_content_type_format (ci->type);
3443
3444                 printf("Setting message content type to '%s' contentinfo type is '%s'\n", ct, ct2);
3445                 g_free (ct);
3446                 g_free (ct2);
3447         }
3448
3449         camel_data_wrapper_set_mime_type_field (content, camel_mime_part_get_content_type ((CamelMimePart *) msg));
3450         camel_medium_set_content (CAMEL_MEDIUM (msg), content);
3451         g_object_unref (content);
3452
3453         return msg;
3454 }
3455
3456 #define IMAP_SMALL_BODY_SIZE 5120
3457
3458 static CamelMimeMessage *
3459 get_message_simple (CamelImapFolder *imap_folder,
3460                     const gchar *uid,
3461                     CamelStream *stream,
3462                     GCancellable *cancellable,
3463                     GError **error)
3464 {
3465         CamelMimeMessage *msg;
3466         gboolean success;
3467
3468         if (!stream) {
3469                 stream = camel_imap_folder_fetch_data (imap_folder, uid, "",
3470                                                        FALSE, cancellable, error);
3471                 if (!stream)
3472                         return NULL;
3473         }
3474
3475         msg = camel_mime_message_new ();
3476         success = camel_data_wrapper_construct_from_stream_sync (
3477                 CAMEL_DATA_WRAPPER (msg), stream, cancellable, error);
3478         g_object_unref (stream);
3479         if (!success) {
3480                 g_prefix_error (error, _("Unable to retrieve message: "));
3481                 g_object_unref (msg);
3482                 return NULL;
3483         }
3484
3485         return msg;
3486 }
3487
3488 static gboolean
3489 content_info_incomplete (CamelMessageContentInfo *ci)
3490 {
3491         if (!ci->type)
3492                 return TRUE;
3493
3494         if (camel_content_type_is (ci->type, "multipart", "*")
3495             || camel_content_type_is (ci->type, "message", "rfc822")) {
3496                 if (!ci->childs)
3497                         return TRUE;
3498                 for (ci = ci->childs; ci; ci = ci->next)
3499                         if (content_info_incomplete (ci))
3500                                 return TRUE;
3501         }
3502
3503         return FALSE;
3504 }
3505
3506 static CamelImapMessageInfo *
3507 imap_folder_summary_uid_or_error (CamelFolderSummary *summary,
3508                                   const gchar *uid,
3509                                   GError **error)
3510 {
3511         CamelImapMessageInfo *mi;
3512         mi = (CamelImapMessageInfo *) camel_folder_summary_get (summary, uid);
3513         if (mi == NULL) {
3514                 g_set_error (
3515                         error, CAMEL_FOLDER_ERROR,
3516                         CAMEL_FOLDER_ERROR_INVALID_UID,
3517                         _("Cannot get message with message ID %s: %s"),
3518                         uid, _("No such message available."));
3519         }
3520         return mi;
3521 }
3522
3523 static CamelMimeMessage *
3524 imap_get_message_sync (CamelFolder *folder,
3525                        const gchar *uid,
3526                        GCancellable *cancellable,
3527                        GError **error)
3528 {
3529         CamelStore *parent_store;
3530         CamelImapFolder *imap_folder = CAMEL_IMAP_FOLDER (folder);
3531         CamelImapStore *store;
3532         CamelImapMessageInfo *mi;
3533         CamelMimeMessage *msg = NULL;
3534         CamelStream *stream = NULL;
3535         gint retry;
3536         GError *local_error = NULL;
3537
3538         parent_store = camel_folder_get_parent_store (folder);
3539         store = CAMEL_IMAP_STORE (parent_store);
3540
3541         if (!camel_imap_store_connected (store, error))
3542                 return NULL;
3543
3544         mi = imap_folder_summary_uid_or_error (folder->summary, uid, error);
3545         if (!mi)
3546                 return NULL;
3547
3548         /* If its cached in full, just get it as is, this is only a shortcut,
3549          * since we get stuff from the cache anyway.  It affects a busted
3550          * connection though. */
3551         stream = camel_imap_folder_fetch_data (imap_folder, uid, "", TRUE, cancellable, NULL);
3552         if (stream != NULL) {
3553                 msg = get_message_simple (imap_folder, uid, stream, cancellable, NULL);
3554                 if (msg != NULL)
3555                         goto done;
3556         }
3557
3558         /* All this mess is so we silently retry a fetch if we fail with
3559          * service_unavailable, without an (equivalent) mess of gotos */
3560         retry = 0;
3561         do {
3562                 retry++;
3563                 g_clear_error (&local_error);
3564
3565                 /* If the message is small or only 1 part, or server doesn't do 4v1 (properly) fetch it in one piece. */
3566                 if (store->server_level < IMAP_LEVEL_IMAP4REV1
3567                     || store->braindamaged
3568                     || mi->info.size < IMAP_SMALL_BODY_SIZE
3569                     || (!content_info_incomplete (mi->info.content) && !mi->info.content->childs)) {
3570                         CamelMessageInfoBase *info = (CamelMessageInfoBase *) camel_folder_summary_get (folder->summary, uid);
3571                         msg = get_message_simple (imap_folder, uid, NULL, cancellable, &local_error);
3572                         if (info && !info->preview && msg && camel_folder_summary_get_need_preview (folder->summary)) {
3573                                 if (camel_mime_message_build_preview ((CamelMimePart *) msg, (CamelMessageInfo *) info) && info->preview)
3574                                         camel_folder_summary_add_preview (folder->summary, (CamelMessageInfo *) info);
3575                         }
3576
3577                         camel_message_info_free (info);
3578                 } else {
3579                         if (content_info_incomplete (mi->info.content)) {
3580                                 /* For larger messages, fetch the structure and build a message
3581                                  * with offline parts. (We check mi->content->type rather than
3582                                  * mi->content because camel_folder_summary_info_new always creates
3583                                  * an empty content struct.)
3584                                  */
3585                                 CamelImapResponse *response;
3586                                 GData *fetch_data = NULL;
3587                                 gchar *body, *found_uid;
3588                                 gint i;
3589
3590                                 if (!camel_imap_store_connected (store, NULL)) {
3591                                         g_set_error (
3592                                                 error, CAMEL_SERVICE_ERROR,
3593                                                 CAMEL_SERVICE_ERROR_UNAVAILABLE,
3594                                                 _("This message is not currently available"));
3595                                         goto fail;
3596                                 }
3597
3598                                 response = camel_imap_command (store, folder, cancellable, &local_error, "UID FETCH %s BODY", uid);
3599
3600                                 if (response) {
3601                                         for (i = 0, body = NULL; i < response->untagged->len; i++) {
3602                                                 fetch_data = parse_fetch_response (imap_folder, response->untagged->pdata[i]);
3603                                                 if (fetch_data) {
3604                                                         found_uid = g_datalist_get_data (&fetch_data, "UID");
3605                                                         body = g_datalist_get_data (&fetch_data, "BODY");
3606                                                         if (found_uid && body && !strcmp (found_uid, uid))
3607                                                                 break;
3608                                                         g_datalist_clear (&fetch_data);
3609                                                         fetch_data = NULL;
3610                                                         body = NULL;
3611                                                 }
3612                                         }
3613
3614                                         if (body) {
3615                                                 /* NB: small race here, setting the info.content */
3616                                                 imap_parse_body ((const gchar **) &body, folder, mi->info.content);
3617                                                 mi->info.dirty = TRUE;
3618                                                 camel_folder_summary_touch (folder->summary);
3619                                         }
3620
3621                                         if (fetch_data)
3622                                                 g_datalist_clear (&fetch_data);
3623
3624                                         camel_imap_response_free (store, response);
3625                                 } else {
3626                                         g_clear_error (&local_error);
3627                                 }
3628                         }
3629
3630                         if (camel_debug_start("imap:folder")) {
3631                                 printf("Folder get message '%s' folder info ->\n", uid);
3632                                 camel_message_info_dump ((CamelMessageInfo *) mi);
3633                                 camel_debug_end ();
3634                         }
3635
3636                         /* FETCH returned OK, but we didn't parse a BODY
3637                          * response. Courier will return invalid BODY
3638                          * responses for invalidly MIMEd messages, so
3639                          * fall back to fetching the entire thing and
3640                          * let the mailer's "bad MIME" code handle it.
3641                          */
3642                         if (content_info_incomplete (mi->info.content))
3643                                 msg = get_message_simple (imap_folder, uid, NULL, cancellable, &local_error);
3644                         else
3645                                 msg = get_message (imap_folder, uid, mi->info.content, cancellable, &local_error);
3646                         if (msg && camel_folder_summary_get_need_preview (folder->summary)) {
3647                                 CamelMessageInfoBase *info = (CamelMessageInfoBase *) camel_folder_summary_get (folder->summary, uid);
3648                                 if (info && !info->preview) {
3649                                         if (camel_mime_message_build_preview ((CamelMimePart *) msg, (CamelMessageInfo *) info) && info->preview)
3650                                                 camel_folder_summary_add_preview (folder->summary, (CamelMessageInfo *) info);
3651                                 }
3652                                 camel_message_info_free (info);
3653                         }
3654
3655                 }
3656         } while (msg == NULL
3657                  && retry < 2
3658                  && g_error_matches (local_error, CAMEL_SERVICE_ERROR, CAMEL_SERVICE_ERROR_UNAVAILABLE));
3659
3660 done:
3661         if (msg) {
3662                 gboolean has_attachment;
3663
3664                 if (!mi->info.mlist || !*mi->info.mlist) {
3665                         /* update mailing list information, if necessary */
3666                         gchar *mlist = camel_header_raw_check_mailing_list (&(CAMEL_MIME_PART (msg)->headers));
3667
3668                         if (mlist) {
3669                                 if (mi->info.mlist)
3670                                         camel_pstring_free (mi->info.mlist);
3671                                 mi->info.mlist = camel_pstring_add (mlist, TRUE);
3672                                 mi->info.dirty = TRUE;
3673
3674                                 if (mi->info.summary)
3675                                         camel_folder_summary_touch (mi->info.summary);
3676                         }
3677                 }
3678
3679                 has_attachment = camel_mime_message_has_attachment (msg);
3680                 if (((camel_message_info_flags ((CamelMessageInfo *) mi) & CAMEL_MESSAGE_ATTACHMENTS) && !has_attachment) ||
3681                     ((camel_message_info_flags ((CamelMessageInfo *) mi) & CAMEL_MESSAGE_ATTACHMENTS) == 0 && has_attachment)) {
3682                         camel_message_info_set_flags ((CamelMessageInfo *) mi, CAMEL_MESSAGE_ATTACHMENTS, has_attachment ? CAMEL_MESSAGE_ATTACHMENTS : 0);
3683                 }
3684         }
3685
3686         if (local_error != NULL)
3687                 g_propagate_error (error, local_error);
3688
3689 fail:
3690         camel_message_info_free (&mi->info);
3691
3692         return msg;
3693 }
3694
3695 /**
3696  * imap_synchronize_message_sync
3697  *
3698  * Ensure that a message is cached locally, but don't retrieve the content if
3699  * it is already local.
3700  */
3701 static gboolean
3702 imap_synchronize_message_sync (CamelFolder *folder,
3703                                const gchar *uid,
3704                                GCancellable *cancellable,
3705                                GError **error)
3706 {
3707         CamelImapFolder *imap_folder = CAMEL_IMAP_FOLDER (folder);
3708         CamelImapMessageInfo *mi;
3709         CamelMimeMessage *msg = NULL;
3710         CamelStream *stream = NULL;
3711         gboolean success = FALSE;
3712
3713         mi = imap_folder_summary_uid_or_error (folder->summary, uid, error);
3714         if (!mi)
3715           /* No such UID - is this duplicate work? The sync process selects
3716            * UIDs to start with.
3717            */
3718           return FALSE;
3719         camel_message_info_free (&mi->info);
3720
3721         /* If we can get a stream, assume its fully cached. This may be false
3722          * if partial streams are saved elsewhere in the code - but that seems
3723          * best solved by knowning more about whether a given message is fully
3724          * available locally or not,
3725          */
3726         /* If its cached in full, just get it as is, this is only a shortcut,
3727          * since we get stuff from the cache anyway.  It affects a busted connection though. */
3728         if ((stream = camel_imap_folder_fetch_data(imap_folder, uid, "", TRUE, cancellable, NULL))) {
3729                 g_object_unref (stream);
3730                 return TRUE;
3731         }
3732         msg = imap_get_message_sync (folder, uid, cancellable, error);
3733         if (msg != NULL) {
3734                 g_object_unref (msg);
3735                 success = TRUE;
3736         }
3737
3738         return success;
3739 }
3740
3741 /* We pretend that a FLAGS or RFC822.SIZE response is always exactly
3742  * 20 bytes long, and a BODY[HEADERS] response is always 2000 bytes
3743  * long. Since we know how many of each kind of response we're
3744  * expecting, we can find the total (pretend) amount of server traffic
3745  * to expect and then count off the responses as we read them to update
3746  * the progress bar.
3747  */
3748 #define IMAP_PRETEND_SIZEOF_FLAGS         20
3749 #define IMAP_PRETEND_SIZEOF_SIZE          20
3750 #define IMAP_PRETEND_SIZEOF_HEADERS     2000
3751
3752 static const gchar *tm_months[] = {
3753         "Jan", "Feb", "Mar", "Apr", "May", "Jun",
3754         "Jul", "Aug", "Sep", "Oct", "Nov", "Dec"
3755 };
3756
3757 static gboolean
3758 decode_time (const guchar **in,
3759              gint *hour,
3760              gint *min,
3761              gint *sec)
3762 {
3763         register const guchar *inptr;
3764         gint *val, colons = 0;
3765
3766         *hour = *min = *sec = 0;
3767
3768         val = hour;
3769         for (inptr = *in; *inptr && !isspace ((gint) *inptr); inptr++) {
3770                 if (*inptr == ':') {
3771                         colons++;
3772                         switch (colons) {
3773                         case 1:
3774                                 val = min;
3775                                 break;
3776                         case 2:
3777                                 val = sec;
3778                                 break;
3779                         default:
3780                                 return FALSE;
3781                         }
3782                 } else if (!isdigit ((gint) *inptr))
3783                         return FALSE;
3784                 else
3785                         *val = (*val * 10) + (*inptr - '0');
3786         }
3787
3788         *in = inptr;
3789
3790         return TRUE;
3791 }
3792
3793 static time_t
3794 decode_internaldate (const guchar *in)
3795 {
3796         const guchar *inptr = in;
3797         gint hour, min, sec, n;
3798         guchar *buf;
3799         struct tm tm;
3800         time_t date;
3801
3802         memset ((gpointer) &tm, 0, sizeof (struct tm));
3803
3804         tm.tm_mday = strtoul ((gchar *) inptr, (gchar **) &buf, 10);
3805         if (buf == inptr || *buf != '-')
3806                 return (time_t) -1;
3807
3808         inptr = buf + 1;
3809         if (inptr[3] != '-')
3810                 return (time_t) -1;
3811
3812         for (n = 0; n < 12; n++) {
3813                 if (!g_ascii_strncasecmp ((gchar *) inptr, tm_months[n], 3))
3814                         break;
3815         }
3816
3817         if (n >= 12)
3818                 return (time_t) -1;
3819
3820         tm.tm_mon = n;
3821
3822         inptr += 4;
3823
3824         n = strtoul ((gchar *) inptr, (gchar **) &buf, 10);
3825         if (buf == inptr || *buf != ' ')
3826                 return (time_t) -1;
3827
3828         tm.tm_year = n - 1900;
3829
3830         inptr = buf + 1;
3831         if (!decode_time (&inptr, &hour, &min, &sec))
3832                 return (time_t) -1;
3833
3834         tm.tm_hour = hour;
3835         tm.tm_min = min;
3836         tm.tm_sec = sec;
3837
3838         n = strtol ((gchar *) inptr, NULL, 10);
3839
3840         date = camel_mktime_utc (&tm);
3841
3842         /* date is now GMT of the time we want, but not offset by the timezone ... */
3843
3844         /* this should convert the time to the GMT equiv time */
3845         date -= ((n / 100) * 60 * 60) + (n % 100) * 60;
3846
3847         return date;
3848 }
3849
3850 static void
3851 add_message_from_data (CamelFolder *folder,
3852                        GPtrArray *messages,
3853                        gint first,
3854                        GData *data,
3855                        GCancellable *cancellable)
3856 {
3857         CamelMimeMessage *msg;
3858         CamelStream *stream;
3859         CamelImapMessageInfo *mi;
3860         const gchar *idate;
3861         const gchar *bodystructure;
3862         gint seq;
3863
3864         seq = GPOINTER_TO_INT (g_datalist_get_data (&data, "SEQUENCE"));
3865         if (seq < first)
3866                 return;
3867         stream = g_datalist_get_data (&data, "BODY_PART_STREAM");
3868         if (!stream)
3869                 return;
3870
3871         if (seq - first >= messages->len)
3872                 g_ptr_array_set_size (messages, seq - first + 1);
3873
3874         msg = camel_mime_message_new ();
3875         if (!camel_data_wrapper_construct_from_stream_sync (
3876                 CAMEL_DATA_WRAPPER (msg), stream, cancellable, NULL)) {
3877                 g_object_unref (msg);
3878                 return;
3879         }
3880
3881         bodystructure = g_datalist_get_data (&data, "BODY");
3882
3883         mi = (CamelImapMessageInfo *)
3884                 camel_folder_summary_info_new_from_message (
3885                 folder->summary, msg, bodystructure);
3886         g_object_unref (msg);
3887
3888         if ((idate = g_datalist_get_data (&data, "INTERNALDATE")))
3889                 mi->info.date_received = decode_internaldate ((const guchar *) idate);
3890
3891         if (mi->info.date_received == -1)
3892                 mi->info.date_received = mi->info.date_sent;
3893
3894         messages->pdata[seq - first] = mi;
3895 }
3896
3897 struct _junk_data {
3898         GData *data;
3899         CamelMessageInfoBase *mi;
3900 };
3901
3902 static void
3903 construct_junk_headers (gchar *header,
3904                         gchar *value,
3905                         struct _junk_data *jdata)
3906 {
3907         gchar *bs, *es, *flag = NULL;
3908         gchar *bdata = g_datalist_get_data (&(jdata->data), "BODY_PART_DATA");
3909         struct _camel_header_param *node;
3910
3911         /* FIXME: This can be written in a much clever way.
3912          * We can create HEADERS file or carry all headers till filtering so
3913          * that header based filtering can be much faster. But all that later. */
3914         bs = camel_strstrcase (bdata ? bdata:"", header);
3915         if (bs) {
3916                 bs += strlen (header);
3917                 bs = strchr (bs, ':');
3918                 if (bs) {
3919                         bs++;
3920                         while (*bs == ' ')
3921                                 bs++;
3922                         es = strchr (bs, '\n');
3923                         if (es)
3924                                 flag = g_strndup (bs, es - bs);
3925                         else
3926                                 bs = NULL;
3927                 }
3928
3929         }
3930
3931         if (bs) {
3932                 node = g_new (struct _camel_header_param, 1);
3933                 node->name = g_strdup (header);
3934                 node->value = flag;
3935                 node->next = jdata->mi->headers;
3936                 jdata->mi->headers = node;
3937         }
3938 }
3939
3940 #define CAMEL_MESSAGE_INFO_HEADERS "DATE FROM TO CC SUBJECT REFERENCES IN-REPLY-TO MESSAGE-ID MIME-VERSION CONTENT-TYPE CONTENT-CLASS X-CALENDAR-ATTACHMENT "
3941
3942 /* FIXME: this needs to be kept in sync with camel-mime-utils.c's list
3943  * of mailing-list headers and so might be best if this were
3944  * auto-generated? */
3945 #define MAILING_LIST_HEADERS "X-MAILING-LIST X-LOOP LIST-ID LIST-POST MAILING-LIST ORIGINATOR X-LIST SENDER RETURN-PATH X-BEENTHERE "
3946
3947 static gboolean
3948 imap_update_summary (CamelFolder *folder,
3949                      gint exists,
3950                      CamelFolderChangeInfo *changes,
3951                      GCancellable *cancellable,
3952                      GError **error)
3953 {
3954         CamelStore *parent_store;
3955         CamelService *service;
3956         CamelSettings *settings;
3957         CamelImapStore *store;
3958         CamelImapFolder *imap_folder = CAMEL_IMAP_FOLDER (folder);
3959         GPtrArray *fetch_data = NULL, *messages = NULL, *needheaders;
3960         CamelFetchHeadersType fetch_headers;
3961         gchar **extra_headers;
3962         guint32 flags, uidval;
3963         gint i, seq, first, size, got;
3964         CamelImapResponseType type;
3965         GString *header_spec = NULL;
3966         CamelImapMessageInfo *mi;
3967         CamelStream *stream;
3968         gchar *uid, *resp, *tempuid;
3969         GData *data;
3970         gint k = 0, ct;
3971
3972         parent_store = camel_folder_get_parent_store (folder);
3973         store = CAMEL_IMAP_STORE (parent_store);
3974         service = CAMEL_SERVICE (parent_store);
3975
3976         if (!camel_imap_store_connected (store, error))
3977                 return FALSE;
3978
3979         settings = camel_service_ref_settings (service);
3980
3981         fetch_headers = camel_imap_settings_get_fetch_headers (
3982                 CAMEL_IMAP_SETTINGS (settings));
3983
3984         extra_headers = camel_imap_settings_dup_fetch_headers_extra (
3985                 CAMEL_IMAP_SETTINGS (settings));
3986
3987         g_object_unref (settings);
3988
3989         if (store->server_level >= IMAP_LEVEL_IMAP4REV1) {
3990                 if (fetch_headers == CAMEL_FETCH_HEADERS_ALL)
3991                         header_spec = g_string_new ("HEADER");
3992                 else {
3993                         gchar *temp;
3994                         header_spec = g_string_new ("HEADER.FIELDS (");
3995                         g_string_append (header_spec, CAMEL_MESSAGE_INFO_HEADERS);
3996                         if (fetch_headers == CAMEL_FETCH_HEADERS_BASIC_AND_MAILING_LIST)
3997                                 g_string_append (header_spec, MAILING_LIST_HEADERS);
3998                         if (extra_headers != NULL) {
3999                                 guint length, ii;
4000
4001                                 length = g_strv_length ((gchar **) extra_headers);
4002                                 for (ii = 0; ii < length; ii++) {
4003                                         g_string_append (header_spec, extra_headers[ii]);
4004                                         if (ii + 1 < length)
4005                                                 g_string_append_c (header_spec, ' ');
4006                                 }
4007                         }
4008
4009                         temp = g_string_free (header_spec, FALSE);
4010                         temp = g_strstrip (temp);
4011                         header_spec = g_string_new (temp);
4012                         g_free (temp);
4013                         g_string_append (header_spec, ")");
4014                 }
4015         } else
4016                 header_spec = g_string_new ("0");
4017
4018         g_strfreev (extra_headers);
4019
4020         d(printf("Header is : %s", header_spec->str));
4021
4022         /* Figure out if any of the new messages are already cached (which
4023          * may be the case if we're re-syncing after disconnected operation).
4024          * If so, get their UIDs, FLAGS, and SIZEs. If not, get all that
4025          * and ask for the headers too at the same time.
4026          */
4027         seq = camel_folder_summary_count (folder->summary);
4028         first = seq + 1;
4029         if (seq > 0) {
4030                 GPtrArray *known_uids;
4031
4032                 known_uids = camel_folder_summary_get_array (folder->summary);
4033                 if (known_uids) {
4034                         camel_folder_sort_uids (folder, known_uids);
4035
4036                         tempuid = g_ptr_array_index (known_uids, seq - 1);
4037                         if (tempuid)
4038                                 uidval = strtoul (tempuid, NULL, 10);
4039                         else
4040                                 uidval = 0;
4041
4042                         camel_folder_summary_free_array (known_uids);
4043                 } else
4044                         uidval = 0;
4045         } else
4046                 uidval = 0;
4047
4048         got = 0;
4049         if (!camel_imap_command_start (store, folder, cancellable, error,
4050                                        "UID FETCH %d:* (FLAGS RFC822.SIZE INTERNALDATE BODYSTRUCTURE BODY.PEEK[%s])",
4051                                        uidval + 1, header_spec->str)) {
4052                 g_string_free (header_spec, TRUE);
4053                 return FALSE;
4054         }
4055
4056         camel_operation_push_message (
4057                 cancellable,
4058                 _("Fetching summary information for new messages in %s"),
4059                 camel_folder_get_display_name (folder));
4060
4061         /* Parse the responses. We can't add a message to the summary
4062          * until we've gotten its headers, and there's no guarantee
4063          * the server will send the responses in a useful order...
4064          */
4065         fetch_data = g_ptr_array_new ();
4066         messages = g_ptr_array_new ();
4067         ct = exists - seq;
4068         while ((type = camel_imap_command_response (store, folder, &resp, cancellable, error)) ==
4069                CAMEL_IMAP_RESPONSE_UNTAGGED && !camel_application_is_exiting) {
4070                 data = parse_fetch_response (imap_folder, resp);
4071                 g_free (resp);
4072                 k++;
4073                 if (!data)
4074                         continue;
4075
4076                 seq = GPOINTER_TO_INT (g_datalist_get_data (&data, "SEQUENCE"));
4077                 if (seq < first) {
4078                         g_datalist_clear (&data);
4079                         continue;
4080                 }
4081
4082                 if (g_datalist_get_data (&data, "FLAGS"))
4083                         got += IMAP_PRETEND_SIZEOF_FLAGS;
4084                 if (g_datalist_get_data (&data, "RFC822.SIZE"))
4085                         got += IMAP_PRETEND_SIZEOF_SIZE;
4086                 stream = g_datalist_get_data (&data, "BODY_PART_STREAM");
4087                 if (stream) {
4088                         got += IMAP_PRETEND_SIZEOF_HEADERS;
4089
4090                         /* Use the stream now so we don't tie up many
4091                          * many fds if we're fetching many many messages.
4092                          */
4093                         add_message_from_data (
4094                                 folder, messages, first, data, cancellable);
4095                         g_datalist_set_data (&data, "BODY_PART_STREAM", NULL);
4096                 }
4097
4098                 camel_operation_progress (cancellable, k * 100 / ct);
4099
4100                 g_ptr_array_add (fetch_data, data);
4101         }
4102
4103         camel_operation_pop_message (cancellable);
4104
4105         if (type == CAMEL_IMAP_RESPONSE_ERROR || camel_application_is_exiting) {
4106                 g_string_free (header_spec, TRUE);
4107                 goto lose;
4108         }
4109
4110         /* Free the final tagged response */
4111         g_free (resp);
4112
4113         /* Figure out which headers we still need to fetch. */
4114         needheaders = g_ptr_array_new ();
4115         size = got = 0;
4116         for (i = 0; i < fetch_data->len; i++) {
4117                 data = fetch_data->pdata[i];
4118                 if (g_datalist_get_data (&data, "BODY_PART_LEN"))
4119                         continue;
4120
4121                 uid = g_datalist_get_data (&data, "UID");
4122                 if (uid) {
4123                         g_ptr_array_add (needheaders, uid);
4124                         size += IMAP_PRETEND_SIZEOF_HEADERS;
4125                 }
4126         }
4127
4128         /* And fetch them */
4129         if (needheaders->len) {
4130                 gchar *uidset;
4131                 gint uid = 0;
4132
4133                 qsort (needheaders->pdata, needheaders->len,
4134                        sizeof (gpointer), uid_compar);
4135
4136                 camel_operation_push_message (
4137                         cancellable,
4138                         _("Fetching summary information for new messages in %s"),
4139                         camel_folder_get_display_name (folder));
4140
4141                 while (uid < needheaders->len && !camel_application_is_exiting) {
4142                         uidset = imap_uid_array_to_set (folder->summary, needheaders, uid, UID_SET_LIMIT, &uid);
4143                         if (!camel_imap_command_start (store, folder, cancellable, error,
4144                                                        "UID FETCH %s BODYSTRUCTURE BODY.PEEK[%s]",
4145                                                        uidset, header_spec->str)) {
4146                                 g_ptr_array_free (needheaders, TRUE);
4147                                 camel_operation_pop_message (cancellable);
4148                                 g_free (uidset);
4149                                 g_string_free (header_spec, TRUE);
4150                                 goto lose;
4151                         }
4152                         g_free (uidset);
4153
4154                         while ((type = camel_imap_command_response (store, folder, &resp, cancellable, error))
4155                                == CAMEL_IMAP_RESPONSE_UNTAGGED && !camel_application_is_exiting) {
4156                                 data = parse_fetch_response (imap_folder, resp);
4157                                 g_free (resp);
4158                                 if (!data)
4159                                         continue;
4160
4161                                 stream = g_datalist_get_data (&data, "BODY_PART_STREAM");
4162                                 if (stream) {
4163                                         add_message_from_data (
4164                                                 folder, messages, first,
4165                                                 data, cancellable);
4166                                         got += IMAP_PRETEND_SIZEOF_HEADERS;
4167                                         camel_operation_progress (
4168                                                 cancellable, got * 100 / size);
4169                                 }
4170                                 g_datalist_clear (&data);
4171                         }
4172
4173                         if (type == CAMEL_IMAP_RESPONSE_ERROR || camel_application_is_exiting) {
4174                                 g_ptr_array_free (needheaders, TRUE);
4175                                 g_string_free (header_spec, TRUE);
4176                                 camel_operation_pop_message (cancellable);
4177
4178                                 goto lose;
4179                         }
4180                 }
4181                 camel_operation_pop_message (cancellable);
4182         }
4183
4184         g_ptr_array_free (needheaders, TRUE);
4185         g_string_free (header_spec, TRUE);
4186
4187         /* Now finish up summary entries (fix UIDs, set flags and size) */
4188         for (i = 0; i < fetch_data->len; i++) {
4189                 struct _junk_data jdata;
4190                 data = fetch_data->pdata[i];
4191
4192                 seq = GPOINTER_TO_INT (g_datalist_get_data (&data, "SEQUENCE"));
4193                 if (seq >= first + messages->len) {
4194                         g_datalist_clear (&data);
4195                         continue;
4196                 }
4197
4198                 mi = messages->pdata[seq - first];
4199                 if (mi == NULL) {
4200                         CamelMessageInfo *pmi = NULL;
4201                         gint j;
4202
4203                         /* This is a kludge around a bug in Exchange
4204                          * 5.5 that sometimes claims multiple messages
4205                          * have the same UID. See bug #17694 for
4206                          * details. The "solution" is to create a fake
4207                          * message-info with the same details as the
4208                          * previously valid message. Yes, the user
4209                          * will have a clone in his/her message-list,
4210                          * but at least we don't crash.
4211                          */
4212
4213                         /* find the previous valid message info */
4214                         for (j = seq - first - 1; j >= 0; j--) {
4215                                 pmi = messages->pdata[j];
4216                                 if (pmi != NULL)
4217                                         break;
4218                         }
4219
4220                         if (pmi == NULL) {
4221                                 continue;
4222                         }
4223
4224                         mi = (CamelImapMessageInfo *) camel_message_info_clone (pmi);
4225                 }
4226
4227                 uid = g_datalist_get_data (&data, "UID");
4228                 if (uid)
4229                         mi->info.uid = camel_pstring_strdup (uid);
4230                 flags = GPOINTER_TO_INT (g_datalist_get_data (&data, "FLAGS"));
4231                 if (flags) {
4232                         gchar *custom_flags = NULL;
4233
4234                         ((CamelImapMessageInfo *) mi)->server_flags = flags;
4235                         /* "or" them in with the existing flags that may
4236                          * have been set by summary_info_new_from_message.
4237                          */
4238                         mi->info.flags |= flags;
4239
4240                         custom_flags = g_datalist_get_data (&data, "CUSTOM.FLAGS");
4241                         if (custom_flags)
4242                                 fillup_custom_flags ((CamelMessageInfo *) mi, custom_flags);
4243                 }
4244                 size = GPOINTER_TO_INT (g_datalist_get_data (&data, "RFC822.SIZE"));
4245                 if (size)
4246                         mi->info.size = size;
4247
4248                 /* Just do this to build the junk required headers to be built*/
4249                 jdata.data = data;
4250                 jdata.mi = (CamelMessageInfoBase *) mi;
4251                 g_hash_table_foreach ((GHashTable *) camel_session_get_junk_headers (camel_service_get_session (service)), (GHFunc) construct_junk_headers, &jdata);
4252                 g_datalist_clear (&data);
4253         }
4254         g_ptr_array_free (fetch_data, TRUE);
4255
4256         if (camel_application_is_exiting) {
4257                 /* it will hopefully update summary next time */
4258                 fetch_data = NULL;
4259                 goto lose;
4260         }
4261
4262         /* And add the entries to the summary, etc. */
4263         for (i = 0; i < messages->len; i++) {
4264                 mi = messages->pdata[i];
4265                 if (!mi) {
4266                         g_warning ("No information for message %d", i + first);
4267                         g_set_error (
4268                                 error, CAMEL_ERROR, CAMEL_ERROR_GENERIC,
4269                                 _("Incomplete server response: "
4270                                   "no information provided for message %d"),
4271                                 i + first);
4272                         break;
4273                 }
4274                 uid = (gchar *) camel_message_info_uid (mi);
4275                 if (uid[0] == 0) {
4276                         g_warning("Server provided no uid: message %d", i + first);
4277                         g_set_error (
4278                                 error, CAMEL_ERROR, CAMEL_ERROR_GENERIC,
4279                                 _("Incomplete server response: "
4280                                   "no UID provided for message %d"),
4281                                 i + first);
4282                         break;
4283                 }
4284
4285                 /* FIXME: If it enters if (info) it will always match the exception. So stupid */
4286                 /* FIXME[disk-summary] Use a db query to see if the DB exists */
4287 /*              info = (CamelImapMessageInfo *)camel_folder_summary_get (folder->summary, uid); */
4288 /*              if (info) { */
4289 /*                      for (seq = 0; seq < camel_folder_summary_count (folder->summary); seq++) { */
4290 /*                              if (folder->summary->messages->pdata[seq] == info) */
4291 /*                                      break; */
4292 /*                      } */
4293
4294                 ((CamelMessageInfoBase *) mi)->dirty = TRUE;
4295                 if (((CamelMessageInfoBase *) mi)->summary)
4296                         camel_folder_summary_touch (((CamelMessageInfoBase *) mi)->summary);
4297                 camel_folder_summary_add (folder->summary, (CamelMessageInfo *) mi);
4298                 camel_folder_change_info_add_uid (changes, camel_message_info_uid (mi));
4299
4300                 /* Report all new messages as recent, even without that flag, thus new
4301                  * messages will be filtered even after saw by other software earlier.
4302                  * Only skip those which we added ourself, like after drag&drop to this folder. */
4303                 if (!imap_folder_uid_in_ignore_recent (imap_folder, camel_message_info_uid (mi))
4304                     && ((mi->info.flags & CAMEL_IMAP_MESSAGE_RECENT) != 0 || getenv ("FILTER_RECENT") == NULL))
4305                         camel_folder_change_info_recent_uid (changes, camel_message_info_uid (mi));
4306
4307         }
4308
4309         g_ptr_array_free (messages, TRUE);
4310
4311         if (imap_folder->priv->ignore_recent) {
4312                 g_hash_table_unref (imap_folder->priv->ignore_recent);
4313                 imap_folder->priv->ignore_recent = NULL;
4314         }
4315
4316         return TRUE;
4317
4318  lose:
4319         if (fetch_data) {
4320                 for (i = 0; i < fetch_data->len; i++) {
4321                         data = fetch_data->pdata[i];
4322                         g_datalist_clear (&data);
4323                 }
4324                 g_ptr_array_free (fetch_data, TRUE);
4325         }
4326         if (messages) {
4327                 for (i = 0; i < messages->len; i++) {
4328                         if (messages->pdata[i])
4329                                 camel_message_info_free (messages->pdata[i]);
4330                 }
4331                 g_ptr_array_free (messages, TRUE);
4332         }
4333
4334         if (imap_folder->priv->ignore_recent) {
4335                 g_hash_table_unref (imap_folder->priv->ignore_recent);
4336                 imap_folder->priv->ignore_recent = NULL;
4337         }
4338
4339         return FALSE;
4340 }
4341
4342 /* Called with the store's connect_lock locked */
4343 gboolean
4344 camel_imap_folder_changed (CamelFolder *folder,
4345                            gint exists,
4346                            GArray *expunged,
4347                            GCancellable *cancellable,
4348                            GError **error)
4349 {
4350         CamelImapFolder *imap_folder = CAMEL_IMAP_FOLDER (folder);
4351         CamelFolderChangeInfo *changes;
4352         gint len;
4353         gboolean success = TRUE;
4354
4355         changes = camel_folder_change_info_new ();
4356         if (expunged) {
4357                 CamelStore *parent_store;
4358                 gint i, id;
4359                 GList *deleted = NULL;
4360                 const gchar *full_name;
4361                 const gchar *uid;
4362                 GPtrArray *known_uids;
4363
4364                 known_uids = camel_folder_summary_get_array (folder->summary);
4365                 camel_folder_sort_uids (folder, known_uids);
4366                 for (i = 0; i < expunged->len; i++) {
4367                         CamelMessageInfo *mi;
4368
4369                         id = g_array_index (expunged, int, i);
4370                         uid = id - 1 + i >= 0 && id - 1 + i < known_uids->len ? g_ptr_array_index (known_uids, id - 1 + i) : NULL;
4371                         if (uid == NULL) {
4372                                 /* FIXME: danw: does this mean that the summary is corrupt? */
4373                                 /* I guess a message that we never retrieved got expunged? */
4374                                 continue;
4375                         }
4376
4377                         deleted = g_list_prepend (deleted, (gpointer) uid);
4378                         camel_folder_change_info_remove_uid (changes, uid);
4379                         CAMEL_IMAP_FOLDER_REC_LOCK (imap_folder, cache_lock);
4380                         camel_imap_message_cache_remove (imap_folder->cache, uid);
4381                         CAMEL_IMAP_FOLDER_REC_UNLOCK (imap_folder, cache_lock);
4382
4383                         mi = camel_folder_summary_peek_loaded (folder->summary, uid);
4384                         if (mi) {
4385                                 camel_folder_summary_remove (folder->summary, mi);
4386                                 camel_message_info_free (mi);
4387                         } else {
4388                                 camel_folder_summary_remove_uid (folder->summary, uid);
4389                         }
4390                 }
4391
4392                 /* Delete all in one transaction */
4393                 full_name = camel_folder_get_full_name (folder);
4394                 parent_store = camel_folder_get_parent_store (folder);
4395                 camel_db_delete_uids (parent_store->cdb_w, full_name, deleted, NULL);
4396                 g_list_free (deleted);
4397
4398                 camel_folder_summary_free_array (known_uids);
4399         }
4400
4401         len = camel_folder_summary_count (folder->summary);
4402         if (exists > len && !camel_application_is_exiting)
4403                 success = imap_update_summary (
4404                         folder, exists, changes, cancellable, error);
4405
4406         camel_folder_summary_save_to_db (folder->summary, NULL);
4407         if (camel_folder_change_info_changed (changes))
4408                 camel_folder_changed (folder, changes);
4409
4410         camel_folder_change_info_free (changes);
4411
4412         return success;
4413 }
4414
4415 static void
4416 imap_thaw (CamelFolder *folder)
4417 {
4418         CamelImapFolder *imap_folder;
4419
4420         CAMEL_FOLDER_CLASS (camel_imap_folder_parent_class)->thaw (folder);
4421         if (camel_folder_is_frozen (folder))
4422                 return;
4423
4424         /* FIXME imap_refresh_info_sync() may block, but camel_folder_thaw()
4425          *       is not supposed to block.  Potential hang here. */
4426         imap_folder = CAMEL_IMAP_FOLDER (folder);
4427         if (imap_folder->need_refresh) {
4428                 imap_folder->need_refresh = FALSE;
4429                 imap_refresh_info_sync (folder, NULL, NULL);
4430         }
4431 }
4432
4433 CamelStream *
4434 camel_imap_folder_fetch_data (CamelImapFolder *imap_folder,
4435                               const gchar *uid,
4436                               const gchar *section_text,
4437                               gboolean cache_only,
4438                               GCancellable *cancellable,
4439                               GError **error)
4440 {
4441         CamelFolder *folder = CAMEL_FOLDER (imap_folder);
4442         CamelStore *parent_store;
4443         CamelImapStore *store;
4444         CamelImapResponse *response;
4445         CamelStream *stream;
4446         GData *fetch_data;
4447         gchar *found_uid;
4448         gint i;
4449
4450         parent_store = camel_folder_get_parent_store (folder);
4451         store = CAMEL_IMAP_STORE (parent_store);
4452
4453         if (!camel_imap_store_connected (store, error))
4454                 return NULL;
4455
4456         /* EXPUNGE responses have to modify the cache, which means
4457          * they have to grab the cache_lock while holding the
4458          * connect_lock.
4459          *
4460          * Because getting the service lock may cause MUCH unecessary
4461          * delay when we already have the data locally, we do the
4462          * locking separately.  This could cause a race
4463          * getting the same data from the cache, but that is only
4464          * an inefficiency, and bad luck.
4465          */
4466         CAMEL_IMAP_FOLDER_REC_LOCK (imap_folder, cache_lock);
4467         stream = camel_imap_message_cache_get (imap_folder->cache, uid, section_text, NULL);
4468         if (!stream && (!strcmp (section_text, "HEADER") || !strcmp (section_text, "0"))) {
4469                 stream = camel_imap_message_cache_get (imap_folder->cache, uid, "", NULL);
4470         }
4471         CAMEL_IMAP_FOLDER_REC_UNLOCK (imap_folder, cache_lock);
4472
4473         if (stream || cache_only)
4474                 return stream;
4475
4476         CAMEL_IMAP_FOLDER_REC_LOCK (imap_folder, cache_lock);
4477
4478         if (!camel_imap_store_connected (store, NULL)) {
4479                 g_set_error (
4480                         error, CAMEL_ERROR,
4481                         CAMEL_SERVICE_ERROR_UNAVAILABLE,
4482                         _("This message is not currently available"));
4483                 CAMEL_IMAP_FOLDER_REC_UNLOCK (imap_folder, cache_lock);
4484                 return NULL;
4485         }
4486
4487         if (store->server_level < IMAP_LEVEL_IMAP4REV1 && !*section_text) {
4488                 response = camel_imap_command (
4489                         store, folder, cancellable, error,
4490                         "UID FETCH %s RFC822.PEEK", uid);
4491         } else {
4492                 response = camel_imap_command (
4493                         store, folder, cancellable, error,
4494                         "UID FETCH %s BODY.PEEK[%s]", uid,
4495                         section_text);
4496         }
4497
4498         if (!response) {
4499                 CAMEL_IMAP_FOLDER_REC_UNLOCK (imap_folder, cache_lock);
4500                 return NULL;
4501         }
4502
4503         for (i = 0; i < response->untagged->len; i++) {
4504                 fetch_data = parse_fetch_response (imap_folder, response->untagged->pdata[i]);
4505                 found_uid = g_datalist_get_data (&fetch_data, "UID");
4506                 stream = g_datalist_get_data (&fetch_data, "BODY_PART_STREAM");
4507                 if (found_uid && stream && !strcmp (uid, found_uid))
4508                         break;
4509
4510                 g_datalist_clear (&fetch_data);
4511                 stream = NULL;
4512         }
4513         camel_imap_response_free (store, response);
4514         CAMEL_IMAP_FOLDER_REC_UNLOCK (imap_folder, cache_lock);
4515         if (!stream) {
4516                 g_set_error (
4517                         error, CAMEL_ERROR,
4518                         CAMEL_SERVICE_ERROR_UNAVAILABLE,
4519                         _("Could not find message body in FETCH response."));
4520         } else {
4521                 g_object_ref (stream);
4522                 g_datalist_clear (&fetch_data);
4523         }
4524
4525         return stream;
4526 }
4527
4528 static GData *
4529 parse_fetch_response (CamelImapFolder *imap_folder,
4530                       gchar *response)
4531 {
4532         GData *data = NULL;
4533         gchar *start, *part_spec = NULL, *body = NULL, *uid = NULL, *idate = NULL;
4534         gboolean cache_header = TRUE, header = FALSE;
4535         gsize body_len = 0;
4536
4537         if (*response != '(') {
4538                 glong seq;
4539
4540                 if (*response != '*' || *(response + 1) != ' ')
4541                         return NULL;
4542                 seq = strtoul (response + 2, &response, 10);
4543                 if (seq == 0)
4544                         return NULL;
4545                 if (g_ascii_strncasecmp (response, " FETCH (", 8) != 0)
4546                         return NULL;
4547                 response += 7;
4548
4549                 g_datalist_set_data (&data, "SEQUENCE", GINT_TO_POINTER (seq));
4550         }
4551
4552         do {
4553                 /* Skip the initial '(' or the ' ' between elements */
4554                 response++;
4555
4556                 if (!g_ascii_strncasecmp (response, "FLAGS ", 6)) {
4557                         CamelMessageFlags flags;
4558                         gchar *custom_flags = NULL;
4559
4560                         response += 6;
4561
4562                         if (imap_parse_flag_list (&response, &flags, &custom_flags)) {
4563                                 g_datalist_set_data (&data, "FLAGS", GUINT_TO_POINTER (flags));
4564
4565                                 if (custom_flags)
4566                                         g_datalist_set_data_full (&data, "CUSTOM.FLAGS", custom_flags, g_free);
4567                         }
4568                 } else if (!g_ascii_strncasecmp (response, "RFC822.SIZE ", 12)) {
4569                         gulong size;
4570
4571                         response += 12;
4572                         size = strtoul (response, &response, 10);
4573                         g_datalist_set_data (&data, "RFC822.SIZE", GUINT_TO_POINTER (size));
4574                 } else if (!g_ascii_strncasecmp (response, "BODY[", 5) ||
4575                            !g_ascii_strncasecmp (response, "RFC822 ", 7)) {
4576                         gchar *p;
4577
4578                         if (*response == 'B') {
4579                                 response += 5;
4580
4581                                 /* HEADER], HEADER.FIELDS (...)], or 0] */
4582                                 if (!g_ascii_strncasecmp (response, "HEADER", 6)) {
4583                                         header = TRUE;
4584                                         if (!g_ascii_strncasecmp (response + 6, ".FIELDS", 7))
4585                                                 cache_header = FALSE;
4586                                 } else if (!g_ascii_strncasecmp (response, "0]", 2))
4587                                         header = TRUE;
4588
4589                                 p = strchr (response, ']');
4590                                 if (!p || *(p + 1) != ' ')
4591                                         break;
4592
4593                                 if (cache_header)
4594                                         part_spec = g_strndup (response, p - response);
4595                                 else
4596                                         part_spec = g_strdup ("HEADER.FIELDS");
4597
4598                                 response = p + 2;
4599                         } else {
4600                                 part_spec = g_strdup ("");
4601                                 response += 7;
4602
4603                                 if (!g_ascii_strncasecmp (response, "HEADER", 6))
4604                                         header = TRUE;
4605                         }
4606
4607                         body = imap_parse_nstring ((const gchar **) &response, &body_len);
4608                         if (!response) {
4609                                 g_free (part_spec);
4610                                 break;
4611                         }
4612
4613                         if (!body)
4614                                 body = g_strdup ("");
4615                         g_datalist_set_data_full (&data, "BODY_PART_SPEC", part_spec, g_free);
4616                         g_datalist_set_data_full (&data, "BODY_PART_DATA", body, g_free);
4617                         g_datalist_set_data (&data, "BODY_PART_LEN", GINT_TO_POINTER (body_len));
4618                 } else if (!g_ascii_strncasecmp (response, "BODY ", 5) ||
4619                            !g_ascii_strncasecmp (response, "BODYSTRUCTURE ", 14)) {
4620                         response = strchr (response, ' ') + 1;
4621                         start = response;
4622                         imap_skip_list ((const gchar **) &response);
4623                         if (response && (response != start)) {
4624                                 /* To handle IMAP Server brokenness, Returning empty body, etc. See #355640 */
4625                                 g_datalist_set_data_full (&data, "BODY", g_strndup (start, response - start), g_free);
4626                         }
4627                 } else if (!g_ascii_strncasecmp (response, "UID ", 4)) {
4628                         gint len;
4629
4630                         len = strcspn (response + 4, " )");
4631                         uid = g_strndup (response + 4, len);
4632                         g_datalist_set_data_full (&data, "UID", uid, g_free);
4633                         response += 4 + len;
4634                 } else if (!g_ascii_strncasecmp (response, "INTERNALDATE ", 13)) {
4635                         gint len;
4636
4637                         response += 13;
4638                         if (*response == '"') {
4639                                 response++;
4640                                 len = strcspn (response, "\"");
4641                                 idate = g_strndup (response, len);
4642                                 g_datalist_set_data_full (&data, "INTERNALDATE", idate, g_free);
4643                                 response += len + 1;
4644                         }
4645                 } else {
4646                         g_warning ("Unexpected FETCH response from server: (%s", response);
4647                         break;
4648                 }
4649         } while (response && *response != ')');
4650
4651         if (!response || *response != ')') {
4652                 g_datalist_clear (&data);
4653                 return NULL;
4654         }
4655
4656         if (uid && body) {
4657                 CamelStream *stream;
4658
4659                 if (header && !cache_header) {
4660                         stream = camel_stream_mem_new_with_buffer (body, body_len);
4661                 } else {
4662                         CAMEL_IMAP_FOLDER_REC_LOCK (imap_folder, cache_lock);
4663                         stream = camel_imap_message_cache_insert (imap_folder->cache,
4664                                                                   uid, part_spec,
4665                                                                   body, body_len, NULL, NULL);
4666                         CAMEL_IMAP_FOLDER_REC_UNLOCK (imap_folder, cache_lock);
4667                         if (stream == NULL)
4668                                 stream = camel_stream_mem_new_with_buffer (body, body_len);
4669                 }
4670
4671                 if (stream)
4672                         g_datalist_set_data_full (&data, "BODY_PART_STREAM", stream,
4673                                                   (GDestroyNotify) g_object_unref);
4674         }
4675
4676         return data;
4677 }
4678
4679 /* it uses connect_lock, thus be sure it doesn't run in main thread */
4680 static CamelFolderQuotaInfo *
4681 imap_get_quota_info_sync (CamelFolder *folder,
4682                           GCancellable *cancellable,
4683                           GError **error)
4684 {
4685         CamelStore *parent_store;
4686         CamelImapStore *imap_store;
4687         CamelImapResponse *response;
4688         CamelFolderQuotaInfo *res = NULL, *last = NULL;
4689
4690         parent_store = camel_folder_get_parent_store (folder);
4691         imap_store = CAMEL_IMAP_STORE (parent_store);
4692
4693         if (!camel_offline_store_get_online (CAMEL_OFFLINE_STORE (imap_store)))
4694                 return NULL;
4695
4696         if (!camel_imap_store_connected (imap_store, NULL))
4697                 return NULL;
4698
4699         if (imap_store->capabilities & IMAP_CAPABILITY_QUOTA) {
4700                 const gchar *full_name = camel_folder_get_full_name (folder);
4701                 CamelImapStoreNamespace *ns = camel_imap_store_summary_namespace_find_full (imap_store->summary, full_name);
4702                 gchar *folder_name = camel_imap_store_summary_path_to_full (imap_store->summary, full_name, ns ? ns->sep : '/');
4703
4704                 response = camel_imap_command (imap_store, NULL, cancellable, error, "GETQUOTAROOT \"%s\"", folder_name);
4705
4706                 if (response) {
4707                         gint i;
4708
4709                         for (i = 0; i < response->untagged->len; i++) {
4710                                 const gchar *resp = response->untagged->pdata[i];
4711
4712                                 if (resp && g_str_has_prefix (resp, "* QUOTA ")) {
4713                                         gboolean skipped = TRUE;
4714                                         gsize sz;
4715                                         gchar *astr;
4716
4717                                         resp = resp + 8;
4718                                         astr = imap_parse_astring (&resp, &sz);
4719                                         g_free (astr);
4720
4721                                         while (resp && *resp && *resp != '(')
4722                                                 resp++;
4723
4724                                         if (resp && *resp == '(') {
4725                                                 gchar *name;
4726                                                 const gchar *used = NULL, *total = NULL;
4727
4728                                                 resp++;
4729                                                 name = imap_parse_astring (&resp, &sz);
4730
4731                                                 if (resp)
4732                                                         used = imap_next_word (resp);
4733                                                 if (used)
4734                                                         total = imap_next_word (used);
4735
4736                                                 while (resp && *resp && *resp != ')')
4737                                                         resp++;
4738
4739                                                 if (resp && *resp == ')' && used && total) {
4740                                                         guint64 u, t;
4741
4742                                                         u = strtoull (used, NULL, 10);
4743                                                         t = strtoull (total, NULL, 10);
4744
4745                                                         if (t > 0) {
4746                                                                 CamelFolderQuotaInfo *info = camel_folder_quota_info_new (name, u, t);
4747
4748                                                                 if (last)
4749                                                                         last->next = info;
4750                                                                 else
4751                                                                         res = info;
4752
4753                                                                 last = info;
4754                                                                 skipped = FALSE;
4755                                                         }
4756                                                 }
4757
4758                                                 g_free (name);
4759                                         }
4760
4761                                         if (skipped)
4762                                                 g_debug ("Unexpected quota response '%s'; skipping it...", (const gchar *)response->untagged->pdata[i]);
4763                                 }
4764                         }
4765                         camel_imap_response_free (imap_store, response);
4766                 }
4767
4768                 g_free (folder_name);
4769         }
4770
4771         return res;
4772 }
4773
4774 /**
4775  * Scan for messages that are local and return the rest.
4776  */
4777 static GPtrArray *
4778 imap_get_uncached_uids (CamelFolder *folder,
4779                         GPtrArray *uids,
4780                         GError **error)
4781 {
4782         GPtrArray *result;
4783         CamelImapFolder *imap_folder = CAMEL_IMAP_FOLDER (folder);
4784
4785         CAMEL_IMAP_FOLDER_REC_LOCK (imap_folder, cache_lock);
4786
4787         result = camel_imap_message_cache_filter_cached (
4788                 imap_folder->cache, uids, error);
4789
4790         CAMEL_IMAP_FOLDER_REC_UNLOCK (imap_folder, cache_lock);
4791
4792         return result;
4793 }
4794