camel/providers/imap/camel-imap-folder.c

   1 /* -*- Mode: C; tab-width: 8; indent-tabs-mode: t; c-basic-offset: 8 -*- */
   2 /* camel-imap-folder.c: class for an imap folder */
   3
   4 /*
   5  * Authors:
   6  *   Dan Winship <danw@ximian.com>
   7  *   Jeffrey Stedfast <fejj@ximian.com>
   8  *
   9  * Copyright (C) 2000, 2001 Ximian, Inc.
  10  *
  11  * This program is free software; you can redistribute it and/or
  12  * modify it under the terms of version 2 of the GNU General Public
  13  * License as published by the Free Software Foundation.
  14  *
  15  * This program is distributed in the hope that it will be useful,
  16  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  17  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  18  * GNU General Public License for more details.
  19  *
  20  * You should have received a copy of the GNU General Public License
  21  * along with this program; if not, write to the Free Software
  22  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
  23  * USA
  24  */
  25
  26
  27 #ifdef HAVE_CONFIG_H
  28 #include <config.h>
  29 #endif
  30
  31 #include <stdlib.h>
  32 #include <sys/types.h>
  33 #include <dirent.h>
  34 #include <sys/stat.h>
  35 #include <unistd.h>
  36 #include <errno.h>
  37 #include <string.h>
  38 #include <fcntl.h>
  39 #include <ctype.h>
  40
  41 #include "e-util/e-path.h"
  42 #include "e-util/e-time-utils.h"
  43
  44 #include "camel-imap-folder.h"
  45 #include "camel-imap-command.h"
  46 #include "camel-imap-message-cache.h"
  47 #include "camel-imap-private.h"
  48 #include "camel-imap-search.h"
  49 #include "camel-imap-store.h"
  50 #include "camel-imap-summary.h"
  51 #include "camel-imap-utils.h"
  52 #include "camel-imap-wrapper.h"
  53 #include "camel-data-wrapper.h"
  54 #include "camel-disco-diary.h"
  55 #include "camel-exception.h"
  56 #include "camel-mime-filter-crlf.h"
  57 #include "camel-mime-filter-from.h"
  58 #include "camel-mime-message.h"
  59 #include "camel-mime-utils.h"
  60 #include "camel-multipart.h"
  61 #include "camel-multipart-signed.h"
  62 #include "camel-multipart-encrypted.h"
  63 #include "camel-operation.h"
  64 #include "camel-session.h"
  65 #include "camel-stream-buffer.h"
  66 #include "camel-stream-filter.h"
  67 #include "camel-stream-mem.h"
  68 #include "camel-stream.h"
  69 #include "string-utils.h"
  70 #include "camel-private.h"
  71
  72
  73 #define d(x)
  74
  75 /* set to -1 for infinite size */
  76 #define UID_SET_LIMIT  (4096)
  77
  78
  79 #define CF_CLASS(o) (CAMEL_FOLDER_CLASS (CAMEL_OBJECT_GET_CLASS(o)))
  80 static CamelDiscoFolderClass *disco_folder_class = NULL;
  81
  82 static void imap_finalize (CamelObject *object);
  83 static int imap_getv(CamelObject *object, CamelException *ex, CamelArgGetV *args);
  84
  85 static void imap_rescan (CamelFolder *folder, int exists, CamelException *ex);
  86 static void imap_refresh_info (CamelFolder *folder, CamelException *ex);
  87 static void imap_sync_online (CamelFolder *folder, CamelException *ex);
  88 static void imap_sync_offline (CamelFolder *folder, CamelException *ex);
  89 static void imap_expunge_uids_online (CamelFolder *folder, GPtrArray *uids, CamelException *ex);
  90 static void imap_expunge_uids_offline (CamelFolder *folder, GPtrArray *uids, CamelException *ex);
  91 static void imap_expunge_uids_resyncing (CamelFolder *folder, GPtrArray *uids, CamelException *ex);
  92 static void imap_cache_message (CamelDiscoFolder *disco_folder, const char *uid, CamelException *ex);
  93 static void imap_rename (CamelFolder *folder, const char *new);
  94
  95 /* message manipulation */
  96 static CamelMimeMessage *imap_get_message (CamelFolder *folder, const gchar *uid,
  97                                            CamelException *ex);
  98 static void imap_append_online (CamelFolder *folder, CamelMimeMessage *message,
  99                                 const CamelMessageInfo *info, char **appended_uid,
 100                                 CamelException *ex);
 101 static void imap_append_offline (CamelFolder *folder, CamelMimeMessage *message,
 102                                  const CamelMessageInfo *info, char **appended_uid,
 103                                  CamelException *ex);
 104 static void imap_append_resyncing (CamelFolder *folder, CamelMimeMessage *message,
 105                                    const CamelMessageInfo *info, char **appended_uid,
 106                                    CamelException *ex);
 107
 108 static void imap_transfer_online (CamelFolder *source, GPtrArray *uids,
 109                                   CamelFolder *dest, GPtrArray **transferred_uids,
 110                                   gboolean delete_originals,
 111                                   CamelException *ex);
 112 static void imap_transfer_offline (CamelFolder *source, GPtrArray *uids,
 113                                    CamelFolder *dest, GPtrArray **transferred_uids,
 114                                    gboolean delete_originals,
 115                                    CamelException *ex);
 116 static void imap_transfer_resyncing (CamelFolder *source, GPtrArray *uids,
 117                                      CamelFolder *dest, GPtrArray **transferred_uids,
 118                                      gboolean delete_originals,
 119                                      CamelException *ex);
 120
 121 /* searching */
 122 static GPtrArray *imap_search_by_expression (CamelFolder *folder, const char *expression, CamelException *ex);
 123 static GPtrArray *imap_search_by_uids       (CamelFolder *folder, const char *expression, GPtrArray *uids, CamelException *ex);
 124 static void       imap_search_free          (CamelFolder *folder, GPtrArray *uids);
 125
 126 static void imap_thaw (CamelFolder *folder);
 127
 128 static CamelObjectClass *parent_class;
 129
 130 static GData *parse_fetch_response (CamelImapFolder *imap_folder, char *msg_att);
 131
 132 static void
 133 camel_imap_folder_class_init (CamelImapFolderClass *camel_imap_folder_class)
 134 {
 135         CamelFolderClass *camel_folder_class = CAMEL_FOLDER_CLASS (camel_imap_folder_class);
 136         CamelDiscoFolderClass *camel_disco_folder_class = CAMEL_DISCO_FOLDER_CLASS (camel_imap_folder_class);
 137
 138         disco_folder_class = CAMEL_DISCO_FOLDER_CLASS (camel_type_get_global_classfuncs (camel_disco_folder_get_type ()));
 139
 140         /* virtual method overload */
 141         ((CamelObjectClass *)camel_imap_folder_class)->getv = imap_getv;
 142
 143         camel_folder_class->get_message = imap_get_message;
 144         camel_folder_class->rename = imap_rename;
 145         camel_folder_class->search_by_expression = imap_search_by_expression;
 146         camel_folder_class->search_by_uids = imap_search_by_uids;
 147         camel_folder_class->search_free = imap_search_free;
 148         camel_folder_class->thaw = imap_thaw;
 149
 150         camel_disco_folder_class->refresh_info_online = imap_refresh_info;
 151         camel_disco_folder_class->sync_online = imap_sync_online;
 152         camel_disco_folder_class->sync_offline = imap_sync_offline;
 153         /* We don't sync flags at resync time: the online code will
 154          * deal with it eventually.
 155          */
 156         camel_disco_folder_class->sync_resyncing = imap_sync_offline;
 157         camel_disco_folder_class->expunge_uids_online = imap_expunge_uids_online;
 158         camel_disco_folder_class->expunge_uids_offline = imap_expunge_uids_offline;
 159         camel_disco_folder_class->expunge_uids_resyncing = imap_expunge_uids_resyncing;
 160         camel_disco_folder_class->append_online = imap_append_online;
 161         camel_disco_folder_class->append_offline = imap_append_offline;
 162         camel_disco_folder_class->append_resyncing = imap_append_resyncing;
 163         camel_disco_folder_class->transfer_online = imap_transfer_online;
 164         camel_disco_folder_class->transfer_offline = imap_transfer_offline;
 165         camel_disco_folder_class->transfer_resyncing = imap_transfer_resyncing;
 166         camel_disco_folder_class->cache_message = imap_cache_message;
 167 }
 168
 169 static void
 170 camel_imap_folder_init (gpointer object, gpointer klass)
 171 {
 172         CamelImapFolder *imap_folder = CAMEL_IMAP_FOLDER (object);
 173         CamelFolder *folder = CAMEL_FOLDER (object);
 174
 175         folder->permanent_flags = CAMEL_MESSAGE_ANSWERED | CAMEL_MESSAGE_DELETED |
 176                 CAMEL_MESSAGE_DRAFT | CAMEL_MESSAGE_FLAGGED | CAMEL_MESSAGE_SEEN;
 177
 178         folder->folder_flags |= (CAMEL_FOLDER_HAS_SUMMARY_CAPABILITY |
 179                                  CAMEL_FOLDER_HAS_SEARCH_CAPABILITY);
 180
 181         imap_folder->priv = g_malloc0(sizeof(*imap_folder->priv));
 182 #ifdef ENABLE_THREADS
 183         imap_folder->priv->search_lock = e_mutex_new(E_MUTEX_SIMPLE);
 184         imap_folder->priv->cache_lock = e_mutex_new(E_MUTEX_REC);
 185 #endif
 186
 187         imap_folder->need_rescan = TRUE;
 188 }
 189
 190 CamelType
 191 camel_imap_folder_get_type (void)
 192 {
 193         static CamelType camel_imap_folder_type = CAMEL_INVALID_TYPE;
 194
 195         if (camel_imap_folder_type == CAMEL_INVALID_TYPE) {
 196                 parent_class = camel_disco_folder_get_type();
 197                 camel_imap_folder_type =
 198                         camel_type_register (parent_class, "CamelImapFolder",
 199                                              sizeof (CamelImapFolder),
 200                                              sizeof (CamelImapFolderClass),
 201                                              (CamelObjectClassInitFunc) camel_imap_folder_class_init,
 202                                              NULL,
 203                                              (CamelObjectInitFunc) camel_imap_folder_init,
 204                                              (CamelObjectFinalizeFunc) imap_finalize);
 205         }
 206
 207         return camel_imap_folder_type;
 208 }
 209
 210 CamelFolder *
 211 camel_imap_folder_new (CamelStore *parent, const char *folder_name,
 212                        const char *folder_dir, CamelException *ex)
 213 {
 214         CamelImapStore *imap_store = CAMEL_IMAP_STORE (parent);
 215         CamelFolder *folder;
 216         CamelImapFolder *imap_folder;
 217         const char *short_name;
 218         char *summary_file;
 219
 220         if (camel_mkdir_hier (folder_dir, S_IRWXU) != 0) {
 221                 camel_exception_setv (ex, CAMEL_EXCEPTION_SYSTEM,
 222                                       _("Could not create directory %s: %s"),
 223                                       folder_dir, g_strerror (errno));
 224                 return NULL;
 225         }
 226
 227         folder = CAMEL_FOLDER (camel_object_new (camel_imap_folder_get_type ()));
 228         short_name = strrchr (folder_name, imap_store->dir_sep);
 229         if (short_name)
 230                 short_name++;
 231         else
 232                 short_name = folder_name;
 233         camel_folder_construct (folder, parent, folder_name, short_name);
 234
 235         summary_file = g_strdup_printf ("%s/summary", folder_dir);
 236         folder->summary = camel_imap_summary_new (summary_file);
 237         g_free (summary_file);
 238         if (!folder->summary) {
 239                 camel_object_unref (CAMEL_OBJECT (folder));
 240                 camel_exception_setv (ex, CAMEL_EXCEPTION_SYSTEM,
 241                                       _("Could not load summary for %s"),
 242                                       folder_name);
 243                 return NULL;
 244         }
 245
 246         imap_folder = CAMEL_IMAP_FOLDER (folder);
 247         imap_folder->cache = camel_imap_message_cache_new (folder_dir, folder->summary, ex);
 248         if (!imap_folder->cache) {
 249                 camel_object_unref (CAMEL_OBJECT (folder));
 250                 return NULL;
 251         }
 252
 253         if ((imap_store->parameters & IMAP_PARAM_FILTER_INBOX) &&
 254             !strcasecmp (folder_name, "INBOX"))
 255                 folder->folder_flags |= CAMEL_FOLDER_FILTER_RECENT;
 256
 257         imap_folder->search = camel_imap_search_new(folder_dir);
 258
 259         return folder;
 260 }
 261
 262 /* Called with the store's connect_lock locked */
 263 void
 264 camel_imap_folder_selected (CamelFolder *folder, CamelImapResponse *response,
 265                             CamelException *ex)
 266 {
 267         CamelImapFolder *imap_folder = CAMEL_IMAP_FOLDER (folder);
 268         CamelImapSummary *imap_summary = CAMEL_IMAP_SUMMARY (folder->summary);
 269         unsigned long exists = 0, validity = 0, val, uid;
 270         CamelMessageInfo *info;
 271         GData *fetch_data;
 272         int i, count;
 273         char *resp;
 274
 275         CAMEL_SERVICE_ASSERT_LOCKED (folder->parent_store, connect_lock);
 276
 277         count = camel_folder_summary_count (folder->summary);
 278
 279         for (i = 0; i < response->untagged->len; i++) {
 280                 resp = response->untagged->pdata[i] + 2;
 281                 if (!strncasecmp (resp, "FLAGS ", 6) &&
 282                     !folder->permanent_flags) {
 283                         resp += 6;
 284                         folder->permanent_flags = imap_parse_flag_list (&resp);
 285                 } else if (!strncasecmp (resp, "OK [PERMANENTFLAGS ", 19)) {
 286                         resp += 19;
 287                         folder->permanent_flags = imap_parse_flag_list (&resp);
 288                 } else if (!strncasecmp (resp, "OK [UIDVALIDITY ", 16)) {
 289                         validity = strtoul (resp + 16, NULL, 10);
 290                 } else if (isdigit ((unsigned char)*resp)) {
 291                         unsigned long num = strtoul (resp, &resp, 10);
 292
 293                         if (!strncasecmp (resp, " EXISTS", 7)) {
 294                                 exists = num;
 295                                 /* Remove from the response so nothing
 296                                  * else tries to interpret it.
 297                                  */
 298                                 g_free (response->untagged->pdata[i]);
 299                                 g_ptr_array_remove_index (response->untagged, i--);
 300                         }
 301                 }
 302         }
 303
 304         if (camel_disco_store_status (CAMEL_DISCO_STORE (folder->parent_store)) == CAMEL_DISCO_STORE_RESYNCING) {
 305                 if (validity != imap_summary->validity) {
 306                         camel_exception_setv (ex, CAMEL_EXCEPTION_FOLDER_SUMMARY_INVALID,
 307                                               _("Folder was destroyed and recreated on server."));
 308                         return;
 309                 }
 310
 311                 /* FIXME: find missing UIDs ? */
 312                 return;
 313         }
 314
 315         if (!imap_summary->validity)
 316                 imap_summary->validity = validity;
 317         else if (validity != imap_summary->validity) {
 318                 imap_summary->validity = validity;
 319                 camel_folder_summary_clear (folder->summary);
 320                 CAMEL_IMAP_FOLDER_LOCK (imap_folder, cache_lock);
 321                 camel_imap_message_cache_clear (imap_folder->cache);
 322                 CAMEL_IMAP_FOLDER_UNLOCK (imap_folder, cache_lock);
 323                 imap_folder->need_rescan = FALSE;
 324                 camel_imap_folder_changed (folder, exists, NULL, ex);
 325                 return;
 326         }
 327
 328         /* If we've lost messages, we have to rescan everything */
 329         if (exists < count)
 330                 imap_folder->need_rescan = TRUE;
 331         else if (count != 0 && !imap_folder->need_rescan) {
 332                 CamelImapStore *store = CAMEL_IMAP_STORE (folder->parent_store);
 333
 334                 /* Similarly, if the UID of the highest message we
 335                  * know about has changed, then that indicates that
 336                  * messages have been both added and removed, so we
 337                  * have to rescan to find the removed ones. (We pass
 338                  * NULL for the folder since we know that this folder
 339                  * is selected, and we don't want camel_imap_command
 340                  * to worry about it.)
 341                  */
 342                 response = camel_imap_command (store, NULL, ex, "FETCH %d UID", count);
 343                 if (!response)
 344                         return;
 345                 uid = 0;
 346                 for (i = 0; i < response->untagged->len; i++) {
 347                         resp = response->untagged->pdata[i];
 348                         val = strtoul (resp + 2, &resp, 10);
 349                         if (val == 0)
 350                                 continue;
 351                         if (!strcasecmp (resp, " EXISTS")) {
 352                                 /* Another one?? */
 353                                 exists = val;
 354                                 continue;
 355                         }
 356                         if (uid != 0 || val != count || strncasecmp (resp, " FETCH (", 8) != 0)
 357                                 continue;
 358
 359                         fetch_data = parse_fetch_response (imap_folder, resp + 7);
 360                         uid = strtoul (g_datalist_get_data (&fetch_data, "UID"), NULL, 10);
 361                         g_datalist_clear (&fetch_data);
 362                 }
 363                 camel_imap_response_free_without_processing (store, response);
 364
 365                 info = camel_folder_summary_index (folder->summary, count - 1);
 366                 val = strtoul (camel_message_info_uid (info), NULL, 10);
 367                 camel_folder_summary_info_free (folder->summary, info);
 368                 if (uid == 0 || uid != val)
 369                         imap_folder->need_rescan = TRUE;
 370         }
 371
 372         /* Now rescan if we need to */
 373         if (imap_folder->need_rescan) {
 374                 imap_rescan (folder, exists, ex);
 375                 return;
 376         }
 377
 378         /* If we don't need to rescan completely, but new messages
 379          * have been added, find out about them.
 380          */
 381         if (exists > count)
 382                 camel_imap_folder_changed (folder, exists, NULL, ex);
 383
 384         /* And we're done. */
 385 }
 386
 387 static void
 388 imap_finalize (CamelObject *object)
 389 {
 390         CamelImapFolder *imap_folder = CAMEL_IMAP_FOLDER (object);
 391
 392         if (imap_folder->search)
 393                 camel_object_unref (CAMEL_OBJECT (imap_folder->search));
 394         if (imap_folder->cache)
 395                 camel_object_unref (CAMEL_OBJECT (imap_folder->cache));
 396
 397 #ifdef ENABLE_THREADS
 398         e_mutex_destroy(imap_folder->priv->search_lock);
 399         e_mutex_destroy(imap_folder->priv->cache_lock);
 400 #endif
 401         g_free(imap_folder->priv);
 402 }
 403
 404 static int
 405 imap_getv(CamelObject *object, CamelException *ex, CamelArgGetV *args)
 406 {
 407         CamelFolder *folder = (CamelFolder *)object;
 408         int i, count=args->argc;
 409         guint32 tag;
 410
 411         for (i=0;i<args->argc;i++) {
 412                 CamelArgGet *arg = &args->argv[i];
 413
 414                 tag = arg->tag;
 415
 416                 switch (tag & CAMEL_ARG_TAG) {
 417                         /* CamelObject args */
 418                 case CAMEL_OBJECT_ARG_DESCRIPTION:
 419                         if (folder->description == NULL) {
 420                                 CamelURL *uri = ((CamelService *)folder->parent_store)->url;
 421
 422                                 /* what if the full name doesn't incclude /'s?  does it matter? */
 423                                 folder->description = g_strdup_printf("%s@%s:%s", uri->user, uri->host, folder->full_name);
 424                         }
 425                         *arg->ca_str = folder->description;
 426                         break;
 427                 default:
 428                         count--;
 429                         continue;
 430                 }
 431
 432                 arg->tag = (tag & CAMEL_ARG_TYPE) | CAMEL_ARG_IGNORE;
 433         }
 434
 435         if (count)
 436                 return ((CamelObjectClass *)parent_class)->getv(object, ex, args);
 437
 438         return 0;
 439 }
 440
 441 static void
 442 imap_rename (CamelFolder *folder, const char *new)
 443 {
 444         CamelImapFolder *imap_folder = (CamelImapFolder *)folder;
 445         CamelImapStore *imap_store = (CamelImapStore *)folder->parent_store;
 446         char *folder_dir, *summary_path;
 447         char *folders;
 448
 449         folders = g_strconcat (imap_store->storage_path, "/folders", NULL);
 450         folder_dir = e_path_to_physical (folders, new);
 451         g_free (folders);
 452         summary_path = g_strdup_printf("%s/summary", folder_dir);
 453
 454         CAMEL_IMAP_FOLDER_LOCK (folder, cache_lock);
 455         camel_imap_message_cache_set_path(imap_folder->cache, folder_dir);
 456         CAMEL_IMAP_FOLDER_UNLOCK (folder, cache_lock);
 457
 458         camel_folder_summary_set_filename(folder->summary, summary_path);
 459
 460         g_free(summary_path);
 461         g_free(folder_dir);
 462
 463         ((CamelFolderClass *)disco_folder_class)->rename(folder, new);
 464 }
 465
 466 static void
 467 imap_refresh_info (CamelFolder *folder, CamelException *ex)
 468 {
 469         CamelImapStore *imap_store = CAMEL_IMAP_STORE (folder->parent_store);
 470         CamelImapFolder *imap_folder = CAMEL_IMAP_FOLDER (folder);
 471         CamelImapResponse *response;
 472
 473         if (camel_disco_store_status (CAMEL_DISCO_STORE (imap_store)) == CAMEL_DISCO_STORE_OFFLINE)
 474                 return;
 475
 476         if (camel_folder_is_frozen (folder)) {
 477                 imap_folder->need_refresh = TRUE;
 478                 return;
 479         }
 480
 481         /* If the folder isn't selected, select it (which will force
 482          * a rescan if one is needed).
 483          * Also, if this is the INBOX, some servers (cryus) wont tell
 484          * us with a NOOP of new messages, so force a reselect which
 485          * should do it.  */
 486         CAMEL_SERVICE_LOCK (imap_store, connect_lock);
 487         if (imap_store->current_folder != folder
 488             || strcasecmp(folder->full_name, "INBOX") == 0) {
 489                 CAMEL_SERVICE_UNLOCK (imap_store, connect_lock);
 490                 response = camel_imap_command (imap_store, folder, ex, NULL);
 491                 if (response) {
 492                         camel_imap_folder_selected (folder, response, ex);
 493                         camel_imap_response_free (imap_store, response);
 494                 }
 495                 return;
 496         }
 497         CAMEL_SERVICE_UNLOCK (imap_store, connect_lock);
 498
 499         /* Otherwise, if we need a rescan, do it, and if not, just do
 500          * a NOOP to give the server a chance to tell us about new
 501          * messages.
 502          */
 503         if (imap_folder->need_rescan)
 504                 imap_rescan (folder, camel_folder_summary_count (folder->summary), ex);
 505         else {
 506 #if 0
 507                 /* on some servers need to CHECKpoint INBOX to recieve new messages?? */
 508                 /* rfc2060 suggests this, but havent seen a server that requires it */
 509                 if (strcasecmp(folder->full_name, "INBOX") == 0) {
 510                         response = camel_imap_command (imap_store, folder, ex, "CHECK");
 511                         camel_imap_response_free (imap_store, response);
 512                 }
 513 #endif
 514                 response = camel_imap_command (imap_store, folder, ex, "NOOP");
 515                 camel_imap_response_free (imap_store, response);
 516         }
 517 }
 518
 519 /* Called with the store's connect_lock locked */
 520 static void
 521 imap_rescan (CamelFolder *folder, int exists, CamelException *ex)
 522 {
 523         CamelImapFolder *imap_folder = CAMEL_IMAP_FOLDER (folder);
 524         CamelImapStore *store = CAMEL_IMAP_STORE (folder->parent_store);
 525         struct {
 526                 char *uid;
 527                 guint32 flags;
 528         } *new;
 529         char *resp;
 530         CamelImapResponseType type;
 531         int i, seq, summary_len, summary_got;
 532         CamelMessageInfo *info;
 533         CamelImapMessageInfo *iinfo;
 534         GArray *removed;
 535         gboolean ok;
 536         CamelFolderChangeInfo *changes = NULL;
 537
 538         CAMEL_SERVICE_ASSERT_LOCKED (store, connect_lock);
 539         imap_folder->need_rescan = FALSE;
 540
 541         summary_len = camel_folder_summary_count (folder->summary);
 542         if (summary_len == 0) {
 543                 if (exists)
 544                         camel_imap_folder_changed (folder, exists, NULL, ex);
 545                 return;
 546         }
 547
 548         /* Check UIDs and flags of all messages we already know of. */
 549         camel_operation_start (NULL, _("Scanning for changed messages"));
 550         info = camel_folder_summary_index (folder->summary, summary_len - 1);
 551         ok = camel_imap_command_start (store, folder, ex,
 552                                        "UID FETCH 1:%s (FLAGS)",
 553                                        camel_message_info_uid (info));
 554         camel_folder_summary_info_free (folder->summary, info);
 555         if (!ok) {
 556                 camel_operation_end (NULL);
 557                 return;
 558         }
 559
 560         new = g_malloc0 (summary_len * sizeof (*new));
 561         summary_got = 0;
 562         while ((type = camel_imap_command_response (store, &resp, ex)) == CAMEL_IMAP_RESPONSE_UNTAGGED) {
 563                 GData *data;
 564                 char *uid;
 565                 guint32 flags;
 566
 567                 data = parse_fetch_response (imap_folder, resp);
 568                 g_free (resp);
 569                 if (!data)
 570                         continue;
 571
 572                 seq = GPOINTER_TO_INT (g_datalist_get_data (&data, "SEQUENCE"));
 573                 uid = g_datalist_get_data (&data, "UID");
 574                 flags = GPOINTER_TO_UINT (g_datalist_get_data (&data, "FLAGS"));
 575
 576                 if (!uid || !seq || seq > summary_len) {
 577                         g_datalist_clear (&data);
 578                         continue;
 579                 }
 580
 581                 camel_operation_progress (NULL, ++summary_got * 100 / summary_len);
 582                 new[seq - 1].uid = g_strdup (uid);
 583                 new[seq - 1].flags = flags;
 584                 g_datalist_clear (&data);
 585         }
 586
 587         camel_operation_end (NULL);
 588         if (type == CAMEL_IMAP_RESPONSE_ERROR) {
 589                 for (i = 0; i < summary_len && new[i].uid; i++)
 590                         g_free (new[i].uid);
 591                 g_free (new);
 592                 return;
 593         }
 594
 595         /* Free the final tagged response */
 596         g_free (resp);
 597
 598         /* If we find a UID in the summary that doesn't correspond to
 599          * the UID in the folder, then either: (a) it's a real UID,
 600          * but the message was deleted on the server, or (b) it's a
 601          * fake UID, and needs to be removed from the summary in order
 602          * to sync up with the server. So either way, we remove it
 603          * from the summary.
 604          */
 605         removed = g_array_new (FALSE, FALSE, sizeof (int));
 606         for (i = 0; i < summary_len && new[i].uid; i++) {
 607                 info = camel_folder_summary_index (folder->summary, i);
 608                 iinfo = (CamelImapMessageInfo *)info;
 609
 610                 if (strcmp (camel_message_info_uid (info), new[i].uid) != 0) {
 611                         camel_folder_summary_info_free(folder->summary, info);
 612                         seq = i + 1;
 613                         g_array_append_val (removed, seq);
 614                         i--;
 615                         summary_len--;
 616                         continue;
 617                 }
 618
 619                 /* Update summary flags */
 620                 if (new[i].flags != iinfo->server_flags) {
 621                         guint32 server_set, server_cleared;
 622
 623                         server_set = new[i].flags & ~iinfo->server_flags;
 624                         server_cleared = iinfo->server_flags & ~new[i].flags;
 625
 626                         info->flags = (info->flags | server_set) & ~server_cleared;
 627                         iinfo->server_flags = new[i].flags;
 628
 629                         if (changes == NULL)
 630                                 changes = camel_folder_change_info_new();
 631                         camel_folder_change_info_change_uid(changes, new[i].uid);
 632                 }
 633
 634                 camel_folder_summary_info_free (folder->summary, info);
 635                 g_free (new[i].uid);
 636         }
 637
 638         if (changes) {
 639                 camel_object_trigger_event(CAMEL_OBJECT (folder), "folder_changed", changes);
 640                 camel_folder_change_info_free(changes);
 641         }
 642
 643         seq = i + 1;
 644
 645         /* Free remaining memory. */
 646         while (i < summary_len && new[i].uid)
 647                 g_free (new[i++].uid);
 648         g_free (new);
 649
 650         /* Remove any leftover cached summary messages. (Yes, we
 651          * repeatedly add the same number to the removed array.
 652          * See RFC2060 7.4.1)
 653          */
 654
 655         for (i = seq; i <= summary_len; i++)
 656                 g_array_append_val (removed, seq);
 657
 658         /* And finally update the summary. */
 659         camel_imap_folder_changed (folder, exists, removed, ex);
 660         g_array_free (removed, TRUE);
 661 }
 662
 663 /* the max number of chars that an unsigned 32-bit int can be is 10 chars plus 1 for a possible : */
 664 #define UID_SET_FULL(setlen, maxlen) (maxlen > 0 ? setlen + 11 >= maxlen : FALSE)
 665
 666 /* Find all messages in @folder with flags matching @flags and @mask.
 667  * If no messages match, returns %NULL. Otherwise, returns an array of
 668  * CamelMessageInfo and sets *@set to a message set corresponding the
 669  * UIDs of the matched messages (up to @UID_SET_LIMIT bytes). The
 670  * caller must free the infos, the array, and the set string.
 671  */
 672 static GPtrArray *
 673 get_matching (CamelFolder *folder, guint32 flags, guint32 mask, char **set)
 674 {
 675         GPtrArray *matches;
 676         CamelMessageInfo *info;
 677         int i, max, range;
 678         GString *gset;
 679
 680         matches = g_ptr_array_new ();
 681         gset = g_string_new ("");
 682         max = camel_folder_summary_count (folder->summary);
 683         range = -1;
 684         for (i = 0; i < max && !UID_SET_FULL (gset->len, UID_SET_LIMIT); i++) {
 685                 info = camel_folder_summary_index (folder->summary, i);
 686                 if (!info)
 687                         continue;
 688                 if ((info->flags & mask) != flags) {
 689                         camel_folder_summary_info_free (folder->summary, info);
 690                         if (range != -1) {
 691                                 if (range != i - 1) {
 692                                         info = matches->pdata[matches->len - 1];
 693                                         g_string_append_printf (gset, ":%s", camel_message_info_uid (info));
 694                                 }
 695                                 range = -1;
 696                         }
 697                         continue;
 698                 }
 699
 700                 g_ptr_array_add (matches, info);
 701                 if (range != -1)
 702                         continue;
 703                 range = i;
 704                 if (gset->len)
 705                         g_string_append_c (gset, ',');
 706                 g_string_append_printf (gset, "%s", camel_message_info_uid (info));
 707         }
 708
 709         if (range != -1 && range != max - 1) {
 710                 info = matches->pdata[matches->len - 1];
 711                 g_string_append_printf (gset, ":%s", camel_message_info_uid (info));
 712         }
 713
 714         if (matches->len) {
 715                 *set = gset->str;
 716                 g_string_free (gset, FALSE);
 717                 return matches;
 718         } else {
 719                 *set = NULL;
 720                 g_string_free (gset, TRUE);
 721                 g_ptr_array_free (matches, TRUE);
 722                 return NULL;
 723         }
 724 }
 725
 726 static void
 727 imap_sync_offline (CamelFolder *folder, CamelException *ex)
 728 {
 729         camel_folder_summary_save (folder->summary);
 730 }
 731
 732 static void
 733 imap_sync_online (CamelFolder *folder, CamelException *ex)
 734 {
 735         CamelImapStore *store = CAMEL_IMAP_STORE (folder->parent_store);
 736         CamelImapResponse *response = NULL;
 737         CamelMessageInfo *info;
 738         CamelException local_ex;
 739         GPtrArray *matches;
 740         char *set, *flaglist;
 741         gboolean unset;
 742         int i, j, max;
 743
 744         camel_exception_init (&local_ex);
 745         CAMEL_SERVICE_LOCK (store, connect_lock);
 746
 747         /* Find a message with changed flags, find all of the other
 748          * messages like it, sync them as a group, mark them as
 749          * updated, and continue.
 750          */
 751         max = camel_folder_summary_count (folder->summary);
 752         for (i = 0; i < max; i++) {
 753                 info = camel_folder_summary_index (folder->summary, i);
 754                 if (!info)
 755                         continue;
 756                 if (!(info->flags & CAMEL_MESSAGE_FOLDER_FLAGGED)) {
 757                         camel_folder_summary_info_free (folder->summary, info);
 758                         continue;
 759                 }
 760
 761                 /* Note: Cyrus is broken and will not accept an
 762                    empty-set of flags so... if this is true then we
 763                    want to unset the previously set flags.*/
 764                 unset = !(info->flags & CAMEL_IMAP_SERVER_FLAGS);
 765
 766                 /* Note: get_matching() uses UID_SET_LIMIT to limit
 767                    the size of the uid-set string. We don't have to
 768                    loop here to flush all the matching uids because
 769                    they will be scooped up later by our parent loop (I
 770                    think?). -- Jeff */
 771                 matches = get_matching (folder, info->flags & (CAMEL_IMAP_SERVER_FLAGS | CAMEL_MESSAGE_FOLDER_FLAGGED),
 772                                         CAMEL_IMAP_SERVER_FLAGS | CAMEL_MESSAGE_FOLDER_FLAGGED, &set);
 773                 camel_folder_summary_info_free (folder->summary, info);
 774                 if (matches == NULL)
 775                         continue;
 776
 777                 /* FIXME: since we don't know the previously set flags,
 778                    if unset is TRUE then just unset all the flags? */
 779                 flaglist = imap_create_flag_list (unset ? CAMEL_IMAP_SERVER_FLAGS : info->flags);
 780
 781                 /* Note: to `unset' flags, use -FLAGS.SILENT (<flag list>) */
 782                 response = camel_imap_command (store, folder, &local_ex,
 783                                                "UID STORE %s %sFLAGS.SILENT %s",
 784                                                set, unset ? "-" : "", flaglist);
 785                 g_free (set);
 786                 g_free (flaglist);
 787
 788                 if (response)
 789                         camel_imap_response_free (store, response);
 790
 791                 if (!camel_exception_is_set (&local_ex)) {
 792                         for (j = 0; j < matches->len; j++) {
 793                                 info = matches->pdata[j];
 794                                 info->flags &= ~CAMEL_MESSAGE_FOLDER_FLAGGED;
 795                                 ((CamelImapMessageInfo*)info)->server_flags =
 796                                         info->flags & CAMEL_IMAP_SERVER_FLAGS;
 797                         }
 798                         camel_folder_summary_touch (folder->summary);
 799                 }
 800
 801                 for (j = 0; j < matches->len; j++) {
 802                         info = matches->pdata[j];
 803                         camel_folder_summary_info_free (folder->summary, info);
 804                 }
 805                 g_ptr_array_free (matches, TRUE);
 806
 807                 /* We unlock here so that other threads can have a chance to grab the connect_lock */
 808                 CAMEL_SERVICE_UNLOCK (store, connect_lock);
 809
 810                 /* check for an exception */
 811                 if (camel_exception_is_set (&local_ex)) {
 812                         camel_exception_xfer (ex, &local_ex);
 813                         return;
 814                 }
 815
 816                 /* Re-lock the connect_lock */
 817                 CAMEL_SERVICE_LOCK (store, connect_lock);
 818         }
 819
 820         /* Save the summary */
 821         imap_sync_offline (folder, ex);
 822
 823         CAMEL_SERVICE_UNLOCK (store, connect_lock);
 824 }
 825
 826 static int
 827 uid_compar (const void *va, const void *vb)
 828 {
 829         const char **sa = (const char **)va, **sb = (const char **)vb;
 830         unsigned long a, b;
 831
 832         a = strtoul (*sa, NULL, 10);
 833         b = strtoul (*sb, NULL, 10);
 834         if (a < b)
 835                 return -1;
 836         else if (a == b)
 837                 return 0;
 838         else
 839                 return 1;
 840 }
 841
 842 static void
 843 imap_expunge_uids_offline (CamelFolder *folder, GPtrArray *uids, CamelException *ex)
 844 {
 845         CamelFolderChangeInfo *changes;
 846         int i;
 847
 848         qsort (uids->pdata, uids->len, sizeof (void *), uid_compar);
 849
 850         changes = camel_folder_change_info_new ();
 851
 852         for (i = 0; i < uids->len; i++) {
 853                 camel_folder_summary_remove_uid (folder->summary, uids->pdata[i]);
 854                 camel_folder_change_info_remove_uid (changes, uids->pdata[i]);
 855                 /* We intentionally don't remove it from the cache because
 856                  * the cached data may be useful in replaying a COPY later.
 857                  */
 858         }
 859         camel_folder_summary_save (folder->summary);
 860
 861         camel_disco_diary_log (CAMEL_DISCO_STORE (folder->parent_store)->diary,
 862                                CAMEL_DISCO_DIARY_FOLDER_EXPUNGE, folder, uids);
 863
 864         camel_object_trigger_event (CAMEL_OBJECT (folder), "folder_changed", changes);
 865         camel_folder_change_info_free (changes);
 866 }
 867
 868 static void
 869 imap_expunge_uids_online (CamelFolder *folder, GPtrArray *uids, CamelException *ex)
 870 {
 871         CamelImapStore *store = CAMEL_IMAP_STORE (folder->parent_store);
 872         CamelImapResponse *response;
 873         int uid = 0;
 874         char *set;
 875
 876         CAMEL_SERVICE_LOCK (store, connect_lock);
 877
 878         if ((store->capabilities & IMAP_CAPABILITY_UIDPLUS) == 0) {
 879                 ((CamelFolderClass *)CAMEL_OBJECT_GET_CLASS(folder))->sync(folder, 0, ex);
 880                 if (camel_exception_is_set(ex)) {
 881                         CAMEL_SERVICE_UNLOCK (store, connect_lock);
 882                         return;
 883                 }
 884         }
 885
 886         qsort (uids->pdata, uids->len, sizeof (void *), uid_compar);
 887
 888         while (uid < uids->len) {
 889                 set = imap_uid_array_to_set (folder->summary, uids, uid, UID_SET_LIMIT, &uid);
 890                 response = camel_imap_command (store, folder, ex,
 891                                                "UID STORE %s +FLAGS.SILENT \\Deleted",
 892                                                set);
 893                 if (response)
 894                         camel_imap_response_free (store, response);
 895                 if (camel_exception_is_set (ex)) {
 896                         CAMEL_SERVICE_UNLOCK (store, connect_lock);
 897                         g_free (set);
 898                         return;
 899                 }
 900
 901                 if (store->capabilities & IMAP_CAPABILITY_UIDPLUS) {
 902                         response = camel_imap_command (store, folder, ex,
 903                                                        "UID EXPUNGE %s", set);
 904                 } else
 905                         response = camel_imap_command (store, folder, ex, "EXPUNGE");
 906
 907                 if (response)
 908                         camel_imap_response_free (store, response);
 909         }
 910
 911         CAMEL_SERVICE_UNLOCK (store, connect_lock);
 912 }
 913
 914 static void
 915 imap_expunge_uids_resyncing (CamelFolder *folder, GPtrArray *uids, CamelException *ex)
 916 {
 917         CamelImapStore *store = CAMEL_IMAP_STORE (folder->parent_store);
 918         GPtrArray *keep_uids, *mark_uids;
 919         CamelImapResponse *response;
 920         char *result;
 921
 922         if (store->capabilities & IMAP_CAPABILITY_UIDPLUS) {
 923                 imap_expunge_uids_online (folder, uids, ex);
 924                 return;
 925         }
 926
 927         /* If we don't have UID EXPUNGE we need to avoid expunging any
 928          * of the wrong messages. So we search for deleted messages,
 929          * and any that aren't in our to-expunge list get temporarily
 930          * marked un-deleted.
 931          */
 932
 933         CAMEL_SERVICE_LOCK (store, connect_lock);
 934
 935         ((CamelFolderClass *)CAMEL_OBJECT_GET_CLASS(folder))->sync(folder, 0, ex);
 936         if (camel_exception_is_set(ex)) {
 937                 CAMEL_SERVICE_UNLOCK (store, connect_lock);
 938                 return;
 939         }
 940
 941         response = camel_imap_command (store, folder, ex, "UID SEARCH DELETED");
 942         if (!response) {
 943                 CAMEL_SERVICE_UNLOCK (store, connect_lock);
 944                 return;
 945         }
 946         result = camel_imap_response_extract (store, response, "SEARCH", ex);
 947         if (!result) {
 948                 CAMEL_SERVICE_UNLOCK (store, connect_lock);
 949                 return;
 950         }
 951
 952         if (result[8] == ' ') {
 953                 char *uid, *lasts = NULL;
 954                 unsigned long euid, kuid;
 955                 int ei, ki;
 956
 957                 keep_uids = g_ptr_array_new ();
 958                 mark_uids = g_ptr_array_new ();
 959
 960                 /* Parse SEARCH response */
 961                 for (uid = strtok_r (result + 9, " ", &lasts); uid; uid = strtok_r (NULL, " ", &lasts))
 962                         g_ptr_array_add (keep_uids, uid);
 963                 qsort (keep_uids->pdata, keep_uids->len,
 964                        sizeof (void *), uid_compar);
 965
 966                 /* Fill in "mark_uids", empty out "keep_uids" as needed */
 967                 for (ei = ki = 0; ei < uids->len; ei++) {
 968                         euid = strtoul (uids->pdata[ei], NULL, 10);
 969
 970                         for (kuid = 0; ki < keep_uids->len; ki++) {
 971                                 kuid = strtoul (keep_uids->pdata[ki], NULL, 10);
 972
 973                                 if (kuid >= euid)
 974                                         break;
 975                         }
 976
 977                         if (euid == kuid)
 978                                 g_ptr_array_remove_index (keep_uids, ki);
 979                         else
 980                                 g_ptr_array_add (mark_uids, uids->pdata[ei]);
 981                 }
 982         } else {
 983                 /* Empty SEARCH result, meaning nothing is marked deleted
 984                  * on server.
 985                  */
 986
 987                 keep_uids = NULL;
 988                 mark_uids = uids;
 989         }
 990
 991         /* Unmark messages to be kept */
 992
 993         if (keep_uids) {
 994                 char *uidset;
 995                 int uid = 0;
 996
 997                 while (uid < keep_uids->len) {
 998                         uidset = imap_uid_array_to_set (folder->summary, keep_uids, uid, UID_SET_LIMIT, &uid);
 999
1000                         response = camel_imap_command (store, folder, ex,
1001                                                        "UID STORE %s -FLAGS.SILENT \\Deleted",
1002                                                        uidset);
1003
1004                         g_free (uidset);
1005
1006                         if (!response) {
1007                                 g_ptr_array_free (keep_uids, TRUE);
1008                                 g_ptr_array_free (mark_uids, TRUE);
1009                                 CAMEL_SERVICE_UNLOCK (store, connect_lock);
1010                                 return;
1011                         }
1012                         camel_imap_response_free (store, response);
1013                 }
1014         }
1015
1016         /* Mark any messages that still need to be marked */
1017         if (mark_uids) {
1018                 char *uidset;
1019                 int uid = 0;
1020
1021                 while (uid < mark_uids->len) {
1022                         uidset = imap_uid_array_to_set (folder->summary, mark_uids, uid, UID_SET_LIMIT, &uid);
1023
1024                         response = camel_imap_command (store, folder, ex,
1025                                                        "UID STORE %s +FLAGS.SILENT \\Deleted",
1026                                                        uidset);
1027
1028                         g_free (uidset);
1029
1030                         if (!response) {
1031                                 g_ptr_array_free (keep_uids, TRUE);
1032                                 g_ptr_array_free (mark_uids, TRUE);
1033                                 CAMEL_SERVICE_UNLOCK (store, connect_lock);
1034                                 return;
1035                         }
1036                         camel_imap_response_free (store, response);
1037                 }
1038
1039                 if (mark_uids != uids)
1040                         g_ptr_array_free (mark_uids, TRUE);
1041         }
1042
1043         /* Do the actual expunging */
1044         response = camel_imap_command (store, folder, ex, "EXPUNGE");
1045         if (response)
1046                 camel_imap_response_free (store, response);
1047
1048         /* And fix the remaining messages if we mangled them */
1049         if (keep_uids) {
1050                 char *uidset;
1051                 int uid = 0;
1052
1053                 while (uid < keep_uids->len) {
1054                         uidset = imap_uid_array_to_set (folder->summary, keep_uids, uid, UID_SET_LIMIT, &uid);
1055
1056                         /* Don't pass ex if it's already been set */
1057                         response = camel_imap_command (store, folder,
1058                                                        camel_exception_is_set (ex) ? NULL : ex,
1059                                                        "UID STORE %s +FLAGS.SILENT \\Deleted",
1060                                                        uidset);
1061
1062                         g_free (uidset);
1063                         if (response)
1064                                 camel_imap_response_free (store, response);
1065                 }
1066
1067                 g_ptr_array_free (keep_uids, TRUE);
1068         }
1069
1070         /* now we can free this, now that we're done with keep_uids */
1071         g_free (result);
1072
1073         CAMEL_SERVICE_UNLOCK (store, connect_lock);
1074 }
1075
1076 static gchar *
1077 get_temp_uid (void)
1078 {
1079         gchar *res;
1080
1081         static int counter = 0;
1082         G_LOCK_DEFINE_STATIC (lock);
1083
1084         G_LOCK (lock);
1085         res = g_strdup_printf ("tempuid-%lx-%d",
1086                                (unsigned long) time (NULL),
1087                                counter++);
1088         G_UNLOCK (lock);
1089
1090         return res;
1091 }
1092
1093 static void
1094 imap_append_offline (CamelFolder *folder, CamelMimeMessage *message,
1095                      const CamelMessageInfo *info, char **appended_uid,
1096                      CamelException *ex)
1097 {
1098         CamelImapStore *imap_store = CAMEL_IMAP_STORE (folder->parent_store);
1099         CamelImapMessageCache *cache = CAMEL_IMAP_FOLDER (folder)->cache;
1100         CamelFolderChangeInfo *changes;
1101         char *uid;
1102
1103         uid = get_temp_uid ();
1104
1105         camel_imap_summary_add_offline (folder->summary, uid, message, info);
1106         CAMEL_IMAP_FOLDER_LOCK (folder, cache_lock);
1107         camel_imap_message_cache_insert_wrapper (cache, uid, "",
1108                                                  CAMEL_DATA_WRAPPER (message), ex);
1109         CAMEL_IMAP_FOLDER_UNLOCK (folder, cache_lock);
1110
1111         changes = camel_folder_change_info_new ();
1112         camel_folder_change_info_add_uid (changes, uid);
1113         camel_object_trigger_event (CAMEL_OBJECT (folder), "folder_changed",
1114                                     changes);
1115         camel_folder_change_info_free (changes);
1116
1117         camel_disco_diary_log (CAMEL_DISCO_STORE (imap_store)->diary,
1118                                CAMEL_DISCO_DIARY_FOLDER_APPEND, folder, uid);
1119         if (appended_uid)
1120                 *appended_uid = uid;
1121         else
1122                 g_free (uid);
1123 }
1124
1125 static CamelImapResponse *
1126 do_append (CamelFolder *folder, CamelMimeMessage *message,
1127            const CamelMessageInfo *info, char **uid,
1128            CamelException *ex)
1129 {
1130         CamelImapStore *store = CAMEL_IMAP_STORE (folder->parent_store);
1131         CamelImapResponse *response;
1132         CamelStream *memstream;
1133         CamelMimeFilter *crlf_filter;
1134         CamelStreamFilter *streamfilter;
1135         GByteArray *ba;
1136         char *flagstr, *result, *end;
1137
1138         /* create flag string param */
1139         if (info && info->flags)
1140                 flagstr = imap_create_flag_list (info->flags);
1141         else
1142                 flagstr = NULL;
1143
1144         /* encode any 8bit parts so we avoid sending embedded nul-chars and such  */
1145         camel_mime_message_encode_8bit_parts (message);
1146
1147         /* FIXME: We could avoid this if we knew how big the message was. */
1148         memstream = camel_stream_mem_new ();
1149         ba = g_byte_array_new ();
1150         camel_stream_mem_set_byte_array (CAMEL_STREAM_MEM (memstream), ba);
1151
1152         streamfilter = camel_stream_filter_new_with_stream (memstream);
1153         crlf_filter = camel_mime_filter_crlf_new (CAMEL_MIME_FILTER_CRLF_ENCODE,
1154                                                   CAMEL_MIME_FILTER_CRLF_MODE_CRLF_ONLY);
1155         camel_stream_filter_add (streamfilter, crlf_filter);
1156         camel_data_wrapper_write_to_stream (CAMEL_DATA_WRAPPER (message),
1157                                             CAMEL_STREAM (streamfilter));
1158         camel_object_unref (CAMEL_OBJECT (streamfilter));
1159         camel_object_unref (CAMEL_OBJECT (crlf_filter));
1160         camel_object_unref (CAMEL_OBJECT (memstream));
1161
1162         response = camel_imap_command (store, NULL, ex, "APPEND %F%s%s {%d}",
1163                                        folder->full_name, flagstr ? " " : "",
1164                                        flagstr ? flagstr : "", ba->len);
1165         g_free (flagstr);
1166
1167         if (!response) {
1168                 g_byte_array_free (ba, TRUE);
1169                 return NULL;
1170         }
1171
1172         result = camel_imap_response_extract_continuation (store, response, ex);
1173         if (!result) {
1174                 g_byte_array_free (ba, TRUE);
1175                 return NULL;
1176         }
1177         g_free (result);
1178
1179         /* send the rest of our data - the mime message */
1180         response = camel_imap_command_continuation (store, ba->data, ba->len, ex);
1181         g_byte_array_free (ba, TRUE);
1182         if (!response)
1183                 return response;
1184
1185         if (store->capabilities & IMAP_CAPABILITY_UIDPLUS) {
1186                 *uid = camel_strstrcase (response->status, "[APPENDUID ");
1187                 if (*uid)
1188                         *uid = strchr (*uid + 11, ' ');
1189                 if (*uid) {
1190                         *uid = g_strndup (*uid + 1, strcspn (*uid + 1, "]"));
1191                         /* Make sure it's a number */
1192                         if (strtoul (*uid, &end, 10) == 0 || *end) {
1193                                 g_free (*uid);
1194                                 *uid = NULL;
1195                         }
1196                 }
1197         } else
1198                 *uid = NULL;
1199
1200         return response;
1201 }
1202
1203 static void
1204 imap_append_online (CamelFolder *folder, CamelMimeMessage *message,
1205                     const CamelMessageInfo *info, char **appended_uid,
1206                     CamelException *ex)
1207 {
1208         CamelImapStore *store = CAMEL_IMAP_STORE (folder->parent_store);
1209         CamelImapResponse *response;
1210         char *uid;
1211         int count;
1212
1213         count = camel_folder_summary_count (folder->summary);
1214         response = do_append (folder, message, info, &uid, ex);
1215         if (!response)
1216                 return;
1217
1218         if (uid) {
1219                 /* Cache first, since freeing response may trigger a
1220                  * summary update that will want this information.
1221                  */
1222                 CAMEL_IMAP_FOLDER_LOCK (folder, cache_lock);
1223                 camel_imap_message_cache_insert_wrapper (
1224                         CAMEL_IMAP_FOLDER (folder)->cache, uid,
1225                         "", CAMEL_DATA_WRAPPER (message), ex);
1226                 CAMEL_IMAP_FOLDER_UNLOCK (folder, cache_lock);
1227                 if (appended_uid)
1228                         *appended_uid = uid;
1229                 else
1230                         g_free (uid);
1231         } else if (appended_uid)
1232                 *appended_uid = NULL;
1233
1234         camel_imap_response_free (store, response);
1235
1236         /* Make sure a "folder_changed" is emitted. */
1237         CAMEL_SERVICE_LOCK (store, connect_lock);
1238         if (store->current_folder != folder ||
1239             camel_folder_summary_count (folder->summary) == count)
1240                 imap_refresh_info (folder, ex);
1241         CAMEL_SERVICE_UNLOCK (store, connect_lock);
1242 }
1243
1244 static void
1245 imap_append_resyncing (CamelFolder *folder, CamelMimeMessage *message,
1246                        const CamelMessageInfo *info, char **appended_uid,
1247                        CamelException *ex)
1248 {
1249         CamelImapStore *store = CAMEL_IMAP_STORE (folder->parent_store);
1250         CamelImapResponse *response;
1251         char *uid;
1252
1253         response = do_append (folder, message, info, &uid, ex);
1254         if (!response)
1255                 return;
1256
1257         if (uid) {
1258                 CamelImapFolder *imap_folder = CAMEL_IMAP_FOLDER (folder);
1259                 const char *olduid = camel_message_info_uid (info);
1260
1261                 CAMEL_IMAP_FOLDER_LOCK (imap_folder, cache_lock);
1262                 camel_imap_message_cache_copy (imap_folder->cache, olduid,
1263                                                imap_folder->cache, uid, ex);
1264                 CAMEL_IMAP_FOLDER_UNLOCK (imap_folder, cache_lock);
1265
1266                 if (appended_uid)
1267                         *appended_uid = uid;
1268                 else
1269                         g_free (uid);
1270         } else if (appended_uid)
1271                 *appended_uid = NULL;
1272
1273         camel_imap_response_free (store, response);
1274 }
1275
1276
1277 static void
1278 imap_transfer_offline (CamelFolder *source, GPtrArray *uids,
1279                        CamelFolder *dest, GPtrArray **transferred_uids,
1280                        gboolean delete_originals, CamelException *ex)
1281 {
1282         CamelImapStore *store = CAMEL_IMAP_STORE (source->parent_store);
1283         CamelImapMessageCache *sc = CAMEL_IMAP_FOLDER (source)->cache;
1284         CamelImapMessageCache *dc = CAMEL_IMAP_FOLDER (dest)->cache;
1285         CamelFolderChangeInfo *changes;
1286         CamelMimeMessage *message;
1287         CamelMessageInfo *mi;
1288         char *uid, *destuid;
1289         int i;
1290
1291         /* We grab the store's command lock first, and then grab the
1292          * source and destination cache_locks. This way we can't
1293          * deadlock in the case where we're simultaneously also trying
1294          * to copy messages in the other direction from another thread.
1295          */
1296         CAMEL_SERVICE_LOCK (store, connect_lock);
1297         CAMEL_IMAP_FOLDER_LOCK (source, cache_lock);
1298         CAMEL_IMAP_FOLDER_LOCK (dest, cache_lock);
1299         CAMEL_SERVICE_UNLOCK (store, connect_lock);
1300
1301         if (transferred_uids) {
1302                 *transferred_uids = g_ptr_array_new ();
1303                 g_ptr_array_set_size (*transferred_uids, uids->len);
1304         }
1305
1306         changes = camel_folder_change_info_new ();
1307
1308         for (i = 0; i < uids->len; i++) {
1309                 uid = uids->pdata[i];
1310
1311                 destuid = get_temp_uid ();
1312
1313                 mi = camel_folder_summary_uid (source->summary, uid);
1314                 g_return_if_fail (mi != NULL);
1315
1316                 message = camel_folder_get_message (source, uid, NULL);
1317
1318                 if (message) {
1319                         camel_imap_summary_add_offline (dest->summary, destuid, message, mi);
1320                         camel_object_unref (CAMEL_OBJECT (message));
1321                 } else
1322                         camel_imap_summary_add_offline_uncached (dest->summary, destuid, mi);
1323
1324                 camel_imap_message_cache_copy (sc, uid, dc, destuid, ex);
1325                 camel_folder_summary_info_free (source->summary, mi);
1326
1327                 camel_folder_change_info_add_uid (changes, destuid);
1328                 if (transferred_uids)
1329                         (*transferred_uids)->pdata[i] = destuid;
1330                 else
1331                         g_free (destuid);
1332
1333                 if (delete_originals)
1334                         camel_folder_delete_message (source, uid);
1335         }
1336
1337         CAMEL_IMAP_FOLDER_UNLOCK (dest, cache_lock);
1338         CAMEL_IMAP_FOLDER_UNLOCK (source, cache_lock);
1339
1340         camel_object_trigger_event (CAMEL_OBJECT (dest), "folder_changed", changes);
1341         camel_folder_change_info_free (changes);
1342
1343         camel_disco_diary_log (CAMEL_DISCO_STORE (store)->diary,
1344                                CAMEL_DISCO_DIARY_FOLDER_TRANSFER,
1345                                source, dest, uids, delete_originals);
1346 }
1347
1348 static void
1349 handle_copyuid (CamelImapResponse *response, CamelFolder *source,
1350                 CamelFolder *destination)
1351 {
1352         CamelImapMessageCache *scache = CAMEL_IMAP_FOLDER (source)->cache;
1353         CamelImapMessageCache *dcache = CAMEL_IMAP_FOLDER (destination)->cache;
1354         char *validity, *srcset, *destset;
1355         GPtrArray *src, *dest;
1356         int i;
1357
1358         validity = camel_strstrcase (response->status, "[COPYUID ");
1359         if (!validity)
1360                 return;
1361         validity += 9;
1362         if (strtoul (validity, NULL, 10) !=
1363             CAMEL_IMAP_SUMMARY (destination->summary)->validity)
1364                 return;
1365
1366         srcset = strchr (validity, ' ');
1367         if (!srcset++)
1368                 goto lose;
1369         destset = strchr (srcset, ' ');
1370         if (!destset++)
1371                 goto lose;
1372
1373         src = imap_uid_set_to_array (source->summary, srcset);
1374         dest = imap_uid_set_to_array (destination->summary, destset);
1375
1376         if (src && dest && src->len == dest->len) {
1377                 /* We don't have to worry about deadlocking on the
1378                  * cache locks here, because we've got the store's
1379                  * command lock too, so no one else could be here.
1380                  */
1381                 CAMEL_IMAP_FOLDER_LOCK (source, cache_lock);
1382                 CAMEL_IMAP_FOLDER_LOCK (destination, cache_lock);
1383                 for (i = 0; i < src->len; i++) {
1384                         camel_imap_message_cache_copy (scache, src->pdata[i],
1385                                                        dcache, dest->pdata[i],
1386                                                        NULL);
1387                 }
1388                 CAMEL_IMAP_FOLDER_UNLOCK (source, cache_lock);
1389                 CAMEL_IMAP_FOLDER_UNLOCK (destination, cache_lock);
1390
1391                 imap_uid_array_free (src);
1392                 imap_uid_array_free (dest);
1393                 return;
1394         }
1395
1396         imap_uid_array_free (src);
1397         imap_uid_array_free (dest);
1398  lose:
1399         g_warning ("Bad COPYUID response from server");
1400 }
1401
1402 static void
1403 do_copy (CamelFolder *source, GPtrArray *uids,
1404          CamelFolder *destination, CamelException *ex)
1405 {
1406         CamelImapStore *store = CAMEL_IMAP_STORE (source->parent_store);
1407         CamelImapResponse *response;
1408         char *uidset;
1409         int uid = 0;
1410
1411         while (uid < uids->len && !camel_exception_is_set (ex)) {
1412                 uidset = imap_uid_array_to_set (source->summary, uids, uid, UID_SET_LIMIT, &uid);
1413
1414                 response = camel_imap_command (store, source, ex, "UID COPY %s %F",
1415                                                uidset, destination->full_name);
1416
1417                 g_free (uidset);
1418
1419                 if (response && (store->capabilities & IMAP_CAPABILITY_UIDPLUS))
1420                         handle_copyuid (response, source, destination);
1421
1422                 camel_imap_response_free (store, response);
1423         }
1424 }
1425
1426 static void
1427 imap_transfer_online (CamelFolder *source, GPtrArray *uids,
1428                       CamelFolder *dest, GPtrArray **transferred_uids,
1429                       gboolean delete_originals, CamelException *ex)
1430 {
1431         CamelImapStore *store = CAMEL_IMAP_STORE (source->parent_store);
1432         int count, i;
1433
1434         /* Sync message flags if needed. */
1435         imap_sync_online (source, ex);
1436         if (camel_exception_is_set (ex))
1437                 return;
1438
1439         count = camel_folder_summary_count (dest->summary);
1440
1441         qsort (uids->pdata, uids->len, sizeof (void *), uid_compar);
1442
1443         /* Now copy the messages */
1444         do_copy (source, uids, dest, ex);
1445         if (camel_exception_is_set (ex))
1446                 return;
1447
1448         /* Make the destination notice its new messages */
1449         CAMEL_SERVICE_LOCK (store, connect_lock);
1450         if (store->current_folder != dest ||
1451             camel_folder_summary_count (dest->summary) == count)
1452                 camel_folder_refresh_info (dest, ex);
1453         CAMEL_SERVICE_UNLOCK (store, connect_lock);
1454
1455         if (delete_originals) {
1456                 for (i = 0; i < uids->len; i++)
1457                         camel_folder_delete_message (source, uids->pdata[i]);
1458         }
1459
1460         /* FIXME */
1461         if (transferred_uids)
1462                 *transferred_uids = NULL;
1463 }
1464
1465 static void
1466 imap_transfer_resyncing (CamelFolder *source, GPtrArray *uids,
1467                          CamelFolder *dest, GPtrArray **transferred_uids,
1468                          gboolean delete_originals, CamelException *ex)
1469 {
1470         CamelDiscoDiary *diary = CAMEL_DISCO_STORE (source->parent_store)->diary;
1471         GPtrArray *realuids;
1472         int first, i;
1473         const char *uid;
1474         CamelMimeMessage *message;
1475         CamelMessageInfo *info;
1476
1477         qsort (uids->pdata, uids->len, sizeof (void *), uid_compar);
1478
1479         /* This is trickier than append_resyncing, because some of
1480          * the messages we are copying may have been copied or
1481          * appended into @source while we were offline, in which case
1482          * if we don't have UIDPLUS, we won't know their real UIDs,
1483          * so we'll have to append them rather than copying.
1484          */
1485
1486         realuids = g_ptr_array_new ();
1487
1488         i = 0;
1489         while (i < uids->len) {
1490                 /* Skip past real UIDs */
1491                 for (first = i; i < uids->len; i++) {
1492                         uid = uids->pdata[i];
1493
1494                         if (!isdigit ((unsigned char)*uid)) {
1495                                 uid = camel_disco_diary_uidmap_lookup (diary, uid);
1496                                 if (!uid)
1497                                         break;
1498                         }
1499                         g_ptr_array_add (realuids, (char *)uid);
1500
1501                         if (delete_originals)
1502                                 camel_folder_delete_message (source, uid);
1503                 }
1504
1505                 /* If we saw any real UIDs, do a COPY */
1506                 if (i != first) {
1507                         do_copy (source, realuids, dest, ex);
1508                         g_ptr_array_set_size (realuids, 0);
1509                         if (i == uids->len || camel_exception_is_set (ex))
1510                                 break;
1511                 }
1512
1513                 /* Deal with fake UIDs */
1514                 while (i < uids->len &&
1515                        !isdigit (*(unsigned char *)(uids->pdata[i])) &&
1516                        !camel_exception_is_set (ex)) {
1517                         uid = uids->pdata[i];
1518                         message = camel_folder_get_message (source, uid, NULL);
1519                         if (!message) {
1520                                 /* Message must have been expunged */
1521                                 continue;
1522                         }
1523                         info = camel_folder_get_message_info (source, uid);
1524                         g_return_if_fail (info != NULL);
1525
1526                         imap_append_online (dest, message, info, NULL, ex);
1527                         camel_folder_free_message_info (source, info);
1528                         camel_object_unref (CAMEL_OBJECT (message));
1529                         if (delete_originals)
1530                                 camel_folder_delete_message (source, uid);
1531                         i++;
1532                 }
1533         }
1534
1535         g_ptr_array_free (realuids, FALSE);
1536
1537         /* FIXME */
1538         if (transferred_uids)
1539                 *transferred_uids = NULL;
1540 }
1541
1542 static GPtrArray *
1543 imap_search_by_expression (CamelFolder *folder, const char *expression, CamelException *ex)
1544 {
1545         CamelImapFolder *imap_folder = CAMEL_IMAP_FOLDER (folder);
1546         GPtrArray *matches, *summary;
1547
1548         /* we could get around this by creating a new search object each time,
1549            but i doubt its worth it since any long operation would lock the
1550            command channel too */
1551         CAMEL_IMAP_FOLDER_LOCK(folder, search_lock);
1552
1553         camel_folder_search_set_folder (imap_folder->search, folder);
1554         summary = camel_folder_get_summary(folder);
1555         camel_folder_search_set_summary(imap_folder->search, summary);
1556         matches = camel_folder_search_execute_expression (imap_folder->search, expression, ex);
1557
1558         CAMEL_IMAP_FOLDER_UNLOCK(folder, search_lock);
1559
1560         camel_folder_free_summary(folder, summary);
1561
1562         return matches;
1563 }
1564
1565 static GPtrArray *
1566 imap_search_by_uids(CamelFolder *folder, const char *expression, GPtrArray *uids, CamelException *ex)
1567 {
1568         CamelImapFolder *imap_folder = CAMEL_IMAP_FOLDER(folder);
1569         GPtrArray *summary, *matches;
1570         int i;
1571
1572         /* NOTE: could get away without the search lock by creating a new
1573            search object each time */
1574
1575         qsort (uids->pdata, uids->len, sizeof (void *), uid_compar);
1576
1577         summary = g_ptr_array_new();
1578         for (i=0;i<uids->len;i++) {
1579                 CamelMessageInfo *info;
1580
1581                 info = camel_folder_get_message_info(folder, uids->pdata[i]);
1582                 if (info)
1583                         g_ptr_array_add(summary, info);
1584         }
1585
1586         if (summary->len == 0)
1587                 return summary;
1588
1589         CAMEL_IMAP_FOLDER_LOCK(folder, search_lock);
1590
1591         camel_folder_search_set_folder(imap_folder->search, folder);
1592         camel_folder_search_set_summary(imap_folder->search, summary);
1593
1594         matches = camel_folder_search_execute_expression(imap_folder->search, expression, ex);
1595
1596         CAMEL_IMAP_FOLDER_UNLOCK(folder, search_lock);
1597
1598         for (i=0;i<summary->len;i++)
1599                 camel_folder_free_message_info(folder, summary->pdata[i]);
1600         g_ptr_array_free(summary, TRUE);
1601
1602         return matches;
1603 }
1604
1605 static void
1606 imap_search_free (CamelFolder *folder, GPtrArray *uids)
1607 {
1608         CamelImapFolder *imap_folder = CAMEL_IMAP_FOLDER (folder);
1609
1610         g_return_if_fail (imap_folder->search);
1611
1612         CAMEL_IMAP_FOLDER_LOCK(folder, search_lock);
1613
1614         camel_folder_search_free_result (imap_folder->search, uids);
1615
1616         CAMEL_IMAP_FOLDER_UNLOCK(folder, search_lock);
1617 }
1618
1619 static CamelMimeMessage *get_message (CamelImapFolder *imap_folder,
1620                                       const char *uid,
1621                                       const char *part_specifier,
1622                                       CamelMessageContentInfo *ci,
1623                                       CamelException *ex);
1624
1625 struct _part_spec_stack {
1626         struct _part_spec_stack *parent;
1627         int part;
1628 };
1629
1630 static void
1631 part_spec_push (struct _part_spec_stack **stack, int part)
1632 {
1633         struct _part_spec_stack *node;
1634
1635         node = g_new (struct _part_spec_stack, 1);
1636         node->parent = *stack;
1637         node->part = part;
1638
1639         *stack = node;
1640 }
1641
1642 static int
1643 part_spec_pop (struct _part_spec_stack **stack)
1644 {
1645         struct _part_spec_stack *node;
1646         int part;
1647
1648         g_return_val_if_fail (*stack != NULL, 0);
1649
1650         node = *stack;
1651         *stack = node->parent;
1652
1653         part = node->part;
1654         g_free (node);
1655
1656         return part;
1657 }
1658
1659 static char *
1660 content_info_get_part_spec (CamelMessageContentInfo *ci)
1661 {
1662         struct _part_spec_stack *stack = NULL;
1663         CamelMessageContentInfo *node;
1664         char *part_spec, *buf;
1665         size_t len = 1;
1666         int part;
1667
1668         node = ci;
1669         while (node->parent) {
1670                 CamelMessageContentInfo *child;
1671
1672                 /* FIXME: is this only supposed to apply if 'node' is a multipart? */
1673                 if (node->parent->parent && header_content_type_is (node->parent->type, "message", "*")) {
1674                         node = node->parent;
1675                         continue;
1676                 }
1677
1678                 child = node->parent->childs;
1679                 for (part = 1; child; part++) {
1680                         if (child == node)
1681                                 break;
1682
1683                         child = child->next;
1684                 }
1685
1686                 part_spec_push (&stack, part);
1687
1688                 len += 2;
1689                 while ((part = part / 10))
1690                         len++;
1691
1692                 node = node->parent;
1693         }
1694
1695         buf = part_spec = g_malloc (len);
1696         part_spec[0] = '\0';
1697
1698         while (stack) {
1699                 part = part_spec_pop (&stack);
1700                 buf += sprintf (buf, "%d%s", part, stack ? "." : "");
1701         }
1702
1703         return part_spec;
1704 }
1705
1706 /* Fetch the contents of the MIME part indicated by @ci, which is part
1707  * of message @uid in @folder.
1708  */
1709 static CamelDataWrapper *
1710 get_content (CamelImapFolder *imap_folder, const char *uid,
1711              CamelMimePart *part, CamelMessageContentInfo *ci,
1712              CamelException *ex)
1713 {
1714         CamelDataWrapper *content = NULL;
1715         CamelStream *stream;
1716         char *part_spec;
1717
1718         part_spec = content_info_get_part_spec (ci);
1719
1720         /* There are three cases: multipart/signed, multipart, message/rfc822, and "other" */
1721         if (header_content_type_is (ci->type, "multipart", "signed")) {
1722                 CamelMultipartSigned *body_mp;
1723                 char *spec;
1724                 int ret;
1725
1726                 /* Note: because we get the content parts uninterpreted anyway, we could potentially
1727                    just use the normalmultipart code, except that multipart/signed wont let you yet! */
1728
1729                 body_mp = camel_multipart_signed_new ();
1730                 /* need to set this so it grabs the boundary and other info about the signed type */
1731                 /* we assume that part->content_type is more accurate/full than ci->type */
1732                 camel_data_wrapper_set_mime_type_field (CAMEL_DATA_WRAPPER (body_mp), part->content_type);
1733
1734                 spec = g_alloca (strlen (part_spec) + 6);
1735                 sprintf (spec, part_spec[0] ? "%s.TEXT" : "TEXT", part_spec);
1736                 g_free (part_spec);
1737
1738                 stream = camel_imap_folder_fetch_data (imap_folder, uid, spec, FALSE, ex);
1739                 if (stream) {
1740                         ret = camel_data_wrapper_construct_from_stream (CAMEL_DATA_WRAPPER (body_mp), stream);
1741                         camel_object_unref (CAMEL_OBJECT (stream));
1742                         if (ret == -1) {
1743                                 camel_object_unref ((CamelObject *) body_mp);
1744                                 return NULL;
1745                         }
1746                 }
1747
1748                 return (CamelDataWrapper *) body_mp;
1749         } else if (header_content_type_is (ci->type, "multipart", "*")) {
1750                 CamelMultipart *body_mp;
1751                 char *child_spec;
1752                 int speclen, num;
1753
1754                 if (header_content_type_is (ci->type, "multipart", "encrypted"))
1755                         body_mp = (CamelMultipart *) camel_multipart_encrypted_new ();
1756                 else
1757                         body_mp = camel_multipart_new ();
1758
1759                 /* need to set this so it grabs the boundary and other info about the multipart */
1760                 /* we assume that part->content_type is more accurate/full than ci->type */
1761                 camel_data_wrapper_set_mime_type_field (CAMEL_DATA_WRAPPER (body_mp), part->content_type);
1762
1763                 speclen = strlen (part_spec);
1764                 child_spec = g_malloc (speclen + 17); /* dot + 10 + dot + MIME + nul */
1765                 memcpy (child_spec, part_spec, speclen);
1766                 if (speclen > 0)
1767                         child_spec[speclen++] = '.';
1768                 g_free (part_spec);
1769
1770                 ci = ci->childs;
1771                 num = 1;
1772                 while (ci) {
1773                         sprintf (child_spec + speclen, "%d.MIME", num++);
1774                         stream = camel_imap_folder_fetch_data (imap_folder, uid, child_spec, FALSE, ex);
1775                         if (stream) {
1776                                 int ret;
1777
1778                                 part = camel_mime_part_new ();
1779                                 ret = camel_data_wrapper_construct_from_stream (CAMEL_DATA_WRAPPER (part), stream);
1780                                 camel_object_unref (CAMEL_OBJECT (stream));
1781                                 if (ret == -1) {
1782                                         camel_object_unref (CAMEL_OBJECT (part));
1783                                         camel_object_unref (CAMEL_OBJECT (body_mp));
1784                                         g_free (child_spec);
1785                                         return NULL;
1786                                 }
1787
1788                                 content = get_content (imap_folder, uid, part, ci, ex);
1789                         }
1790
1791                         if (!stream || !content) {
1792                                 camel_object_unref (CAMEL_OBJECT (body_mp));
1793                                 g_free (child_spec);
1794                                 return NULL;
1795                         }
1796
1797                         camel_medium_set_content_object (CAMEL_MEDIUM (part), content);
1798                         camel_object_unref (CAMEL_OBJECT (content));
1799                         camel_multipart_add_part (body_mp, part);
1800                         camel_object_unref (CAMEL_OBJECT (part));
1801
1802                         ci = ci->next;
1803                 }
1804
1805                 g_free (child_spec);
1806
1807                 return (CamelDataWrapper *) body_mp;
1808         } else if (header_content_type_is (ci->type, "message", "rfc822")) {
1809                 content = (CamelDataWrapper *) get_message (imap_folder, uid, part_spec, ci->childs, ex);
1810                 g_free (part_spec);
1811                 return content;
1812         } else {
1813                 content = camel_imap_wrapper_new (imap_folder, ci->type, uid, *part_spec ? part_spec : "1", part);
1814                 g_free (part_spec);
1815                 return content;
1816         }
1817 }
1818
1819 static CamelMimeMessage *
1820 get_message (CamelImapFolder *imap_folder, const char *uid,
1821              const char *part_spec, CamelMessageContentInfo *ci,
1822              CamelException *ex)
1823 {
1824         CamelImapStore *store = CAMEL_IMAP_STORE (CAMEL_FOLDER (imap_folder)->parent_store);
1825         CamelDataWrapper *content;
1826         CamelMimeMessage *msg;
1827         CamelStream *stream;
1828         char *section_text;
1829         int ret;
1830
1831         section_text = g_strdup_printf ("%s%s%s", part_spec, *part_spec ? "." : "",
1832                                         store->server_level >= IMAP_LEVEL_IMAP4REV1 ? "HEADER" : "0");
1833         stream = camel_imap_folder_fetch_data (imap_folder, uid, section_text, FALSE, ex);
1834         g_free (section_text);
1835         if (!stream)
1836                 return NULL;
1837
1838         msg = camel_mime_message_new ();
1839         ret = camel_data_wrapper_construct_from_stream (CAMEL_DATA_WRAPPER (msg), stream);
1840         camel_object_unref (CAMEL_OBJECT (stream));
1841         if (ret == -1) {
1842                 camel_object_unref (CAMEL_OBJECT (msg));
1843                 return NULL;
1844         }
1845
1846         content = get_content (imap_folder, uid, CAMEL_MIME_PART (msg), ci, ex);
1847         if (!content) {
1848                 camel_object_unref (CAMEL_OBJECT (msg));
1849                 return NULL;
1850         }
1851
1852         camel_medium_set_content_object (CAMEL_MEDIUM (msg), content);
1853         camel_object_unref (CAMEL_OBJECT (content));
1854
1855         return msg;
1856 }
1857
1858 /* FIXME: I pulled this number out of my butt. */
1859 #define IMAP_SMALL_BODY_SIZE 5120
1860
1861 static CamelMimeMessage *
1862 get_message_simple (CamelImapFolder *imap_folder, const char *uid,
1863                     CamelStream *stream, CamelException *ex)
1864 {
1865         CamelMimeMessage *msg;
1866         CamelImapStore *imap_store =
1867                 CAMEL_IMAP_STORE (CAMEL_FOLDER (imap_folder)->parent_store);
1868         int ret;
1869
1870         if (!stream) {
1871                 stream = camel_imap_folder_fetch_data (imap_folder, uid, "",
1872                                                        FALSE, ex);
1873                 if (!stream)
1874                         return NULL;
1875         }
1876
1877         msg = camel_mime_message_new ();
1878         ret = camel_data_wrapper_construct_from_stream (CAMEL_DATA_WRAPPER (msg),
1879                                                         stream);
1880         camel_object_unref (CAMEL_OBJECT (stream));
1881         if (ret == -1) {
1882                 camel_exception_setv (ex, CAMEL_EXCEPTION_SERVICE_UNAVAILABLE,
1883                                       _("Unable to retrieve message: %s"),
1884                                       g_strerror (errno));
1885                 camel_object_unref (CAMEL_OBJECT (msg));
1886                 return NULL;
1887         }
1888
1889         /* FIXME, this shouldn't be done this way. */
1890         camel_medium_set_header (CAMEL_MEDIUM (msg), "X-Evolution-Source",
1891                                  imap_store->base_url);
1892         return msg;
1893 }
1894
1895 static CamelMimeMessage *
1896 imap_get_message (CamelFolder *folder, const char *uid, CamelException *ex)
1897 {
1898         CamelImapFolder *imap_folder = CAMEL_IMAP_FOLDER (folder);
1899         CamelImapStore *store = CAMEL_IMAP_STORE (folder->parent_store);
1900         CamelMessageInfo *mi;
1901         CamelMimeMessage *msg;
1902         CamelStream *stream = NULL;
1903
1904         /* If the server doesn't support IMAP4rev1, or we already have
1905          * the whole thing cached, fetch it in one piece.
1906          */
1907         if (store->server_level < IMAP_LEVEL_IMAP4REV1 ||
1908             (stream = camel_imap_folder_fetch_data (imap_folder, uid, "", TRUE, NULL)))
1909                 return get_message_simple (imap_folder, uid, stream, ex);
1910
1911         /* If we're not actually connected and it's not in the cache,
1912          * that's as far as we can go.
1913          */
1914         if (camel_disco_store_check_online (CAMEL_DISCO_STORE (store), ex) == FALSE)
1915                 return NULL;
1916
1917         mi = camel_folder_summary_uid (folder->summary, uid);
1918         if (mi == NULL) {
1919                 camel_exception_setv(ex, CAMEL_EXCEPTION_FOLDER_INVALID_UID,
1920                                      _("Cannot get message: %s\n  %s"), uid, _("No such message"));
1921                 return NULL;
1922         }
1923
1924         /* If the message is small, fetch it in one piece. */
1925         if (mi->size < IMAP_SMALL_BODY_SIZE) {
1926                 camel_folder_summary_info_free (folder->summary, mi);
1927                 return get_message_simple (imap_folder, uid, NULL, ex);
1928         }
1929
1930         /* For larger messages, fetch the structure and build a message
1931          * with offline parts. (We check mi->content->type rather than
1932          * mi->content because camel_folder_summary_info_new always creates
1933          * an empty content struct.)
1934          */
1935         if (!mi->content->type) {
1936                 CamelImapResponse *response;
1937                 GData *fetch_data = NULL;
1938                 char *body, *found_uid;
1939                 int i;
1940
1941                 if (camel_disco_store_status (CAMEL_DISCO_STORE (store)) == CAMEL_DISCO_STORE_OFFLINE) {
1942                         camel_exception_set (ex, CAMEL_EXCEPTION_SERVICE_UNAVAILABLE,
1943                                              _("This message is not currently available"));
1944                         return NULL;
1945                 }
1946
1947                 response = camel_imap_command (store, folder, ex,
1948                                                "UID FETCH %s BODY", uid);
1949                 if (!response) {
1950                         camel_folder_summary_info_free (folder->summary, mi);
1951                         return NULL;
1952                 }
1953
1954                 for (i = 0, body = NULL; i < response->untagged->len; i++) {
1955                         fetch_data = parse_fetch_response (imap_folder, response->untagged->pdata[i]);
1956                         if (fetch_data) {
1957                                 found_uid = g_datalist_get_data (&fetch_data, "UID");
1958                                 body = g_datalist_get_data (&fetch_data, "BODY");
1959                                 if (found_uid && body && !strcmp (found_uid, uid))
1960                                         break;
1961                                 g_datalist_clear (&fetch_data);
1962                                 fetch_data = NULL;
1963                                 body = NULL;
1964                         }
1965                 }
1966
1967                 if (body)
1968                         imap_parse_body ((const char **) &body, folder, mi->content);
1969
1970                 if (fetch_data)
1971                         g_datalist_clear (&fetch_data);
1972
1973                 camel_imap_response_free (store, response);
1974
1975                 if (!mi->content->type) {
1976                         /* FETCH returned OK, but we didn't parse a BODY
1977                          * response. Courier will return invalid BODY
1978                          * responses for invalidly MIMEd messages, so
1979                          * fall back to fetching the entire thing and
1980                          * let the mailer's "bad MIME" code handle it.
1981                          */
1982                         camel_folder_summary_info_free (folder->summary, mi);
1983                         return get_message_simple (imap_folder, uid, NULL, ex);
1984                 }
1985         }
1986
1987         msg = get_message (imap_folder, uid, "", mi->content, ex);
1988         /* FIXME, this shouldn't be done this way. */
1989         camel_medium_set_header (CAMEL_MEDIUM (msg), "X-Evolution-Source",
1990                                  store->base_url);
1991         camel_folder_summary_info_free (folder->summary, mi);
1992
1993         return msg;
1994 }
1995
1996 static void
1997 imap_cache_message (CamelDiscoFolder *disco_folder, const char *uid,
1998                     CamelException *ex)
1999 {
2000         CamelImapFolder *imap_folder = CAMEL_IMAP_FOLDER (disco_folder);
2001         CamelStream *stream;
2002
2003         stream = camel_imap_folder_fetch_data (imap_folder, uid, "", FALSE, ex);
2004         if (stream)
2005                 camel_object_unref (CAMEL_OBJECT (stream));
2006 }
2007
2008 /* We pretend that a FLAGS or RFC822.SIZE response is always exactly
2009  * 20 bytes long, and a BODY[HEADERS] response is always 2000 bytes
2010  * long. Since we know how many of each kind of response we're
2011  * expecting, we can find the total (pretend) amount of server traffic
2012  * to expect and then count off the responses as we read them to update
2013  * the progress bar.
2014  */
2015 #define IMAP_PRETEND_SIZEOF_FLAGS         20
2016 #define IMAP_PRETEND_SIZEOF_SIZE          20
2017 #define IMAP_PRETEND_SIZEOF_HEADERS     2000
2018
2019 static char *tm_months[] = {
2020         "Jan", "Feb", "Mar", "Apr", "May", "Jun",
2021         "Jul", "Aug", "Sep", "Oct", "Nov", "Dec"
2022 };
2023
2024 static gboolean
2025 decode_time (const unsigned char **in, int *hour, int *min, int *sec)
2026 {
2027         register const unsigned char *inptr;
2028         int *val, colons = 0;
2029
2030         *hour = *min = *sec = 0;
2031
2032         val = hour;
2033         for (inptr = *in; *inptr && !isspace ((int) *inptr); inptr++) {
2034                 if (*inptr == ':') {
2035                         colons++;
2036                         switch (colons) {
2037                         case 1:
2038                                 val = min;
2039                                 break;
2040                         case 2:
2041                                 val = sec;
2042                                 break;
2043                         default:
2044                                 return FALSE;
2045                         }
2046                 } else if (!isdigit ((int) *inptr))
2047                         return FALSE;
2048                 else
2049                         *val = (*val * 10) + (*inptr - '0');
2050         }
2051
2052         *in = inptr;
2053
2054         return TRUE;
2055 }
2056
2057 static time_t
2058 decode_internaldate (const unsigned char *in)
2059 {
2060         const unsigned char *inptr = in;
2061         int hour, min, sec, n;
2062         unsigned char *buf;
2063         struct tm tm;
2064         time_t date;
2065
2066         memset ((void *) &tm, 0, sizeof (struct tm));
2067
2068         tm.tm_mday = strtoul (inptr, (char **) &buf, 10);
2069         if (buf == inptr || *buf != '-')
2070                 return (time_t) -1;
2071
2072         inptr = buf + 1;
2073         if (inptr[3] != '-')
2074                 return (time_t) -1;
2075
2076         for (n = 0; n < 12; n++) {
2077                 if (!strncasecmp (inptr, tm_months[n], 3))
2078                         break;
2079         }
2080
2081         if (n >= 12)
2082                 return (time_t) -1;
2083
2084         tm.tm_mon = n;
2085
2086         inptr += 4;
2087
2088         n = strtoul (inptr, (char **) &buf, 10);
2089         if (buf == inptr || *buf != ' ')
2090                 return (time_t) -1;
2091
2092         tm.tm_year = n - 1900;
2093
2094         inptr = buf + 1;
2095         if (!decode_time (&inptr, &hour, &min, &sec))
2096                 return (time_t) -1;
2097
2098         tm.tm_hour = hour;
2099         tm.tm_min = min;
2100         tm.tm_sec = sec;
2101
2102         n = strtol (inptr, NULL, 10);
2103
2104         date = e_mktime_utc (&tm);
2105
2106         /* date is now GMT of the time we want, but not offset by the timezone ... */
2107
2108         /* this should convert the time to the GMT equiv time */
2109         date -= ((n / 100) * 60 * 60) + (n % 100) * 60;
2110
2111         return date;
2112 }
2113
2114 static void
2115 add_message_from_data (CamelFolder *folder, GPtrArray *messages,
2116                        int first, GData *data)
2117 {
2118         CamelMimeMessage *msg;
2119         CamelStream *stream;
2120         CamelMessageInfo *mi;
2121         const char *idate;
2122         int seq;
2123
2124         seq = GPOINTER_TO_INT (g_datalist_get_data (&data, "SEQUENCE"));
2125         if (seq < first)
2126                 return;
2127         stream = g_datalist_get_data (&data, "BODY_PART_STREAM");
2128         if (!stream)
2129                 return;
2130
2131         if (seq - first >= messages->len)
2132                 g_ptr_array_set_size (messages, seq - first + 1);
2133
2134         msg = camel_mime_message_new ();
2135         if (camel_data_wrapper_construct_from_stream (CAMEL_DATA_WRAPPER (msg), stream) == -1) {
2136                 camel_object_unref (CAMEL_OBJECT (msg));
2137                 return;
2138         }
2139
2140         mi = camel_folder_summary_info_new_from_message (folder->summary, msg);
2141         camel_object_unref (CAMEL_OBJECT (msg));
2142
2143         if ((idate = g_datalist_get_data (&data, "INTERNALDATE")))
2144                 mi->date_received = decode_internaldate (idate);
2145
2146         if (mi->date_received == -1)
2147                 mi->date_received = mi->date_sent;
2148
2149         messages->pdata[seq - first] = mi;
2150 }
2151
2152
2153 #define CAMEL_MESSAGE_INFO_HEADERS "DATE FROM TO CC SUBJECT REFERENCES IN-REPLY-TO MESSAGE-ID MIME-VERSION CONTENT-TYPE"
2154
2155 /* FIXME: this needs to be kept in sync with camel-mime-utils.c's list
2156    of mailing-list headers and so might be best if this were
2157    auto-generated? */
2158 #define MAILING_LIST_HEADERS "X-MAILING-LIST X-LOOP LIST-ID LIST-POST MAILING-LIST ORIGINATOR X-LIST SENDER RETURN-PATH X-BEENTHERE"
2159
2160 static void
2161 imap_update_summary (CamelFolder *folder, int exists,
2162                      CamelFolderChangeInfo *changes,
2163                      CamelException *ex)
2164 {
2165         CamelImapStore *store = CAMEL_IMAP_STORE (folder->parent_store);
2166         CamelImapFolder *imap_folder = CAMEL_IMAP_FOLDER (folder);
2167         GPtrArray *fetch_data = NULL, *messages = NULL, *needheaders;
2168         guint32 flags, uidval;
2169         int i, seq, first, size, got;
2170         CamelImapResponseType type;
2171         const char *header_spec;
2172         CamelMessageInfo *mi, *info;
2173         CamelStream *stream;
2174         char *uid, *resp;
2175         GData *data;
2176
2177         CAMEL_SERVICE_ASSERT_LOCKED (store, connect_lock);
2178         if (store->server_level >= IMAP_LEVEL_IMAP4REV1)
2179                 header_spec = "HEADER.FIELDS.NOT (RECEIVED)";
2180         else
2181                 header_spec = "0";
2182
2183         /* Figure out if any of the new messages are already cached (which
2184          * may be the case if we're re-syncing after disconnected operation).
2185          * If so, get their UIDs, FLAGS, and SIZEs. If not, get all that
2186          * and ask for the headers too at the same time.
2187          */
2188         seq = camel_folder_summary_count (folder->summary);
2189         first = seq + 1;
2190         if (seq > 0) {
2191                 mi = camel_folder_summary_index (folder->summary, seq - 1);
2192                 uidval = strtoul(camel_message_info_uid (mi), NULL, 10);
2193                 camel_folder_summary_info_free (folder->summary, mi);
2194         } else
2195                 uidval = 0;
2196
2197         size = (exists - seq) * (IMAP_PRETEND_SIZEOF_FLAGS + IMAP_PRETEND_SIZEOF_SIZE + IMAP_PRETEND_SIZEOF_HEADERS);
2198         got = 0;
2199         if (!camel_imap_command_start (store, folder, ex,
2200                                        "UID FETCH %d:* (FLAGS RFC822.SIZE INTERNALDATE BODY.PEEK[%s])",
2201                                        uidval + 1, header_spec))
2202                 return;
2203         camel_operation_start (NULL, _("Fetching summary information for new messages"));
2204
2205         /* Parse the responses. We can't add a message to the summary
2206          * until we've gotten its headers, and there's no guarantee
2207          * the server will send the responses in a useful order...
2208          */
2209         fetch_data = g_ptr_array_new ();
2210         messages = g_ptr_array_new ();
2211         while ((type = camel_imap_command_response (store, &resp, ex)) ==
2212                CAMEL_IMAP_RESPONSE_UNTAGGED) {
2213                 data = parse_fetch_response (imap_folder, resp);
2214                 g_free (resp);
2215                 if (!data)
2216                         continue;
2217
2218                 seq = GPOINTER_TO_INT (g_datalist_get_data (&data, "SEQUENCE"));
2219                 if (seq < first) {
2220                         g_datalist_clear (&data);
2221                         continue;
2222                 }
2223
2224                 if (g_datalist_get_data (&data, "FLAGS"))
2225                         got += IMAP_PRETEND_SIZEOF_FLAGS;
2226                 if (g_datalist_get_data (&data, "RFC822.SIZE"))
2227                         got += IMAP_PRETEND_SIZEOF_SIZE;
2228                 stream = g_datalist_get_data (&data, "BODY_PART_STREAM");
2229                 if (stream) {
2230                         got += IMAP_PRETEND_SIZEOF_HEADERS;
2231
2232                         /* Use the stream now so we don't tie up many
2233                          * many fds if we're fetching many many messages.
2234                          */
2235                         add_message_from_data (folder, messages, first, data);
2236                         g_datalist_set_data (&data, "BODY_PART_STREAM", NULL);
2237                 }
2238
2239                 camel_operation_progress (NULL, got * 100 / size);
2240                 g_ptr_array_add (fetch_data, data);
2241         }
2242         camel_operation_end (NULL);
2243
2244         if (type == CAMEL_IMAP_RESPONSE_ERROR)
2245                 goto lose;
2246
2247         /* Free the final tagged response */
2248         g_free (resp);
2249
2250         /* Figure out which headers we still need to fetch. */
2251         needheaders = g_ptr_array_new ();
2252         size = got = 0;
2253         for (i = 0; i < fetch_data->len; i++) {
2254                 data = fetch_data->pdata[i];
2255                 if (g_datalist_get_data (&data, "BODY_PART_LEN"))
2256                         continue;
2257
2258                 uid = g_datalist_get_data (&data, "UID");
2259                 if (uid) {
2260                         g_ptr_array_add (needheaders, uid);
2261                         size += IMAP_PRETEND_SIZEOF_HEADERS;
2262                 }
2263         }
2264
2265         /* And fetch them */
2266         if (needheaders->len) {
2267                 char *uidset;
2268                 int uid = 0;
2269
2270                 qsort (needheaders->pdata, needheaders->len,
2271                        sizeof (void *), uid_compar);
2272
2273                 camel_operation_start (NULL, _("Fetching summary information for new messages"));
2274
2275                 while (uid < needheaders->len) {
2276                         uidset = imap_uid_array_to_set (folder->summary, needheaders, uid, UID_SET_LIMIT, &uid);
2277                         if (!camel_imap_command_start (store, folder, ex,
2278                                                        "UID FETCH %s BODY.PEEK[%s]",
2279                                                        uidset, header_spec)) {
2280                                 g_ptr_array_free (needheaders, TRUE);
2281                                 camel_operation_end (NULL);
2282                                 g_free (uidset);
2283                                 goto lose;
2284                         }
2285                         g_free (uidset);
2286
2287                         while ((type = camel_imap_command_response (store, &resp, ex))
2288                                == CAMEL_IMAP_RESPONSE_UNTAGGED) {
2289                                 data = parse_fetch_response (imap_folder, resp);
2290                                 g_free (resp);
2291                                 if (!data)
2292                                         continue;
2293
2294                                 stream = g_datalist_get_data (&data, "BODY_PART_STREAM");
2295                                 if (stream) {
2296                                         add_message_from_data (folder, messages, first, data);
2297                                         got += IMAP_PRETEND_SIZEOF_HEADERS;
2298                                         camel_operation_progress (NULL, got * 100 / size);
2299                                 }
2300                                 g_datalist_clear (&data);
2301                         }
2302
2303                         if (type == CAMEL_IMAP_RESPONSE_ERROR) {
2304                                 g_ptr_array_free (needheaders, TRUE);
2305                                 camel_operation_end (NULL);
2306                                 goto lose;
2307                         }
2308                 }
2309
2310                 g_ptr_array_free (needheaders, TRUE);
2311                 camel_operation_end (NULL);
2312         }
2313
2314         /* Now finish up summary entries (fix UIDs, set flags and size) */
2315         for (i = 0; i < fetch_data->len; i++) {
2316                 data = fetch_data->pdata[i];
2317
2318                 seq = GPOINTER_TO_INT (g_datalist_get_data (&data, "SEQUENCE"));
2319                 if (seq >= first + messages->len) {
2320                         g_datalist_clear (&data);
2321                         continue;
2322                 }
2323
2324                 mi = messages->pdata[seq - first];
2325                 if (mi == NULL) {
2326                         CamelMessageInfo *pmi = NULL;
2327                         int j;
2328
2329                         /* This is a kludge around a bug in Exchange
2330                          * 5.5 that sometimes claims multiple messages
2331                          * have the same UID. See bug #17694 for
2332                          * details. The "solution" is to create a fake
2333                          * message-info with the same details as the
2334                          * previously valid message. Yes, the user
2335                          * will have a clone in his/her message-list,
2336                          * but at least we don't crash.
2337                          */
2338
2339                         /* find the previous valid message info */
2340                         for (j = seq - first - 1; j >= 0; j--) {
2341                                 pmi = messages->pdata[j];
2342                                 if (pmi != NULL)
2343                                         break;
2344                         }
2345
2346                         if (pmi == NULL) {
2347                                 /* Server response is *really* fucked up,
2348                                    I guess we just pretend it never happened? */
2349                                 continue;
2350                         }
2351
2352                         mi = camel_message_info_new ();
2353                         camel_message_info_dup_to (pmi, mi);
2354                 }
2355
2356                 uid = g_datalist_get_data (&data, "UID");
2357                 if (uid)
2358                         camel_message_info_set_uid (mi, g_strdup (uid));
2359                 flags = GPOINTER_TO_INT (g_datalist_get_data (&data, "FLAGS"));
2360                 if (flags) {
2361                         ((CamelImapMessageInfo *)mi)->server_flags = flags;
2362                         /* "or" them in with the existing flags that may
2363                          * have been set by summary_info_new_from_message.
2364                          */
2365                         mi->flags |= flags;
2366                 }
2367                 size = GPOINTER_TO_INT (g_datalist_get_data (&data, "RFC822.SIZE"));
2368                 if (size)
2369                         mi->size = size;
2370
2371                 g_datalist_clear (&data);
2372         }
2373         g_ptr_array_free (fetch_data, TRUE);
2374
2375         /* And add the entries to the summary, etc. */
2376         for (i = 0; i < messages->len; i++) {
2377                 mi = messages->pdata[i];
2378                 if (!mi) {
2379                         g_warning ("No information for message %d", i + first);
2380                         continue;
2381                 }
2382                 uid = (char *)camel_message_info_uid(mi);
2383                 if (uid[0] == 0) {
2384                         g_warning("Server provided no uid: message %d", i + first);
2385                         continue;
2386                 }
2387                 info = camel_folder_summary_uid(folder->summary, uid);
2388                 if (info) {
2389                         g_warning("Message already present? %s", camel_message_info_uid(mi));
2390                         camel_folder_summary_info_free(folder->summary, info);
2391                         camel_folder_summary_info_free(folder->summary, mi);
2392                         continue;
2393                 }
2394
2395                 camel_folder_summary_add (folder->summary, mi);
2396                 camel_folder_change_info_add_uid (changes, camel_message_info_uid (mi));
2397
2398                 if ((mi->flags & CAMEL_IMAP_MESSAGE_RECENT))
2399                         camel_folder_change_info_recent_uid(changes, camel_message_info_uid (mi));
2400         }
2401         g_ptr_array_free (messages, TRUE);
2402
2403         /* Kludge around Microsoft Exchange 5.5 IMAP - See bug #5348 for details */
2404         if (camel_folder_summary_count (folder->summary) != exists) {
2405                 CamelImapStore *imap_store = (CamelImapStore *) folder->parent_store;
2406                 CamelImapResponse *response;
2407
2408                 /* forget the currently selected folder */
2409                 if (imap_store->current_folder) {
2410                         camel_object_unref (CAMEL_OBJECT (imap_store->current_folder));
2411                         imap_store->current_folder = NULL;
2412                 }
2413
2414                 /* now re-select it and process the EXISTS response */
2415                 response = camel_imap_command (imap_store, folder, ex, NULL);
2416                 if (response) {
2417                         camel_imap_folder_selected (folder, response, NULL);
2418                         camel_imap_response_free (imap_store, response);
2419                 }
2420         }
2421
2422         return;
2423
2424  lose:
2425         if (fetch_data) {
2426                 for (i = 0; i < fetch_data->len; i++) {
2427                         data = fetch_data->pdata[i];
2428                         g_datalist_clear (&data);
2429                 }
2430                 g_ptr_array_free (fetch_data, TRUE);
2431         }
2432         if (messages) {
2433                 for (i = 0; i < messages->len; i++) {
2434                         if (messages->pdata[i])
2435                                 camel_folder_summary_info_free (folder->summary, messages->pdata[i]);
2436                 }
2437                 g_ptr_array_free (messages, TRUE);
2438         }
2439 }
2440
2441 /* Called with the store's connect_lock locked */
2442 void
2443 camel_imap_folder_changed (CamelFolder *folder, int exists,
2444                            GArray *expunged, CamelException *ex)
2445 {
2446         CamelImapFolder *imap_folder = CAMEL_IMAP_FOLDER (folder);
2447         CamelFolderChangeInfo *changes;
2448         CamelMessageInfo *info;
2449         int len;
2450
2451         CAMEL_SERVICE_ASSERT_LOCKED (folder->parent_store, connect_lock);
2452
2453         changes = camel_folder_change_info_new ();
2454         if (expunged) {
2455                 int i, id;
2456
2457                 for (i = 0; i < expunged->len; i++) {
2458                         id = g_array_index (expunged, int, i);
2459                         info = camel_folder_summary_index (folder->summary, id - 1);
2460                         if (info == NULL) {
2461                                 /* FIXME: danw: does this mean that the summary is corrupt? */
2462                                 /* I guess a message that we never retrieved got expunged? */
2463                                 continue;
2464                         }
2465
2466                         camel_folder_change_info_remove_uid (changes, camel_message_info_uid (info));
2467                         CAMEL_IMAP_FOLDER_LOCK (imap_folder, cache_lock);
2468                         camel_imap_message_cache_remove (imap_folder->cache, camel_message_info_uid (info));
2469                         CAMEL_IMAP_FOLDER_UNLOCK (imap_folder, cache_lock);
2470                         camel_folder_summary_remove (folder->summary, info);
2471                         camel_folder_summary_info_free(folder->summary, info);
2472                 }
2473         }
2474
2475         len = camel_folder_summary_count (folder->summary);
2476         if (exists > len)
2477                 imap_update_summary (folder, exists, changes, ex);
2478
2479         if (camel_folder_change_info_changed (changes))
2480                 camel_object_trigger_event (CAMEL_OBJECT (folder), "folder_changed", changes);
2481
2482         camel_folder_change_info_free (changes);
2483         camel_folder_summary_save (folder->summary);
2484 }
2485
2486 static void
2487 imap_thaw (CamelFolder *folder)
2488 {
2489         CamelImapFolder *imap_folder;
2490
2491         CAMEL_FOLDER_CLASS (disco_folder_class)->thaw (folder);
2492         if (camel_folder_is_frozen (folder))
2493                 return;
2494
2495         imap_folder = CAMEL_IMAP_FOLDER (folder);
2496         if (imap_folder->need_refresh) {
2497                 imap_folder->need_refresh = FALSE;
2498                 imap_refresh_info (folder, NULL);
2499         }
2500 }
2501
2502
2503 CamelStream *
2504 camel_imap_folder_fetch_data (CamelImapFolder *imap_folder, const char *uid,
2505                               const char *section_text, gboolean cache_only,
2506                               CamelException *ex)
2507 {
2508         CamelFolder *folder = CAMEL_FOLDER (imap_folder);
2509         CamelImapStore *store = CAMEL_IMAP_STORE (folder->parent_store);
2510         CamelImapResponse *response;
2511         CamelStream *stream;
2512         GData *fetch_data;
2513         char *found_uid;
2514         int i;
2515
2516         /* EXPUNGE responses have to modify the cache, which means
2517          * they have to grab the cache_lock while holding the
2518          * connect_lock. So we grab the connect_lock now, in case
2519          * we're going to need it below, since we can't grab it
2520          * after the cache_lock.
2521          */
2522         CAMEL_SERVICE_LOCK (store, connect_lock);
2523
2524         CAMEL_IMAP_FOLDER_LOCK (imap_folder, cache_lock);
2525         stream = camel_imap_message_cache_get (imap_folder->cache, uid, section_text, ex);
2526         if (!stream && (!strcmp (section_text, "HEADER") || !strcmp (section_text, "0"))) {
2527                 camel_exception_clear (ex);
2528                 stream = camel_imap_message_cache_get (imap_folder->cache, uid, "", ex);
2529         }
2530
2531         if (stream || cache_only) {
2532                 CAMEL_IMAP_FOLDER_UNLOCK (imap_folder, cache_lock);
2533                 CAMEL_SERVICE_UNLOCK (store, connect_lock);
2534                 return stream;
2535         }
2536
2537         if (camel_disco_store_status (CAMEL_DISCO_STORE (store)) == CAMEL_DISCO_STORE_OFFLINE) {
2538                 camel_exception_set (ex, CAMEL_EXCEPTION_SERVICE_UNAVAILABLE,
2539                                      _("This message is not currently available"));
2540                 CAMEL_IMAP_FOLDER_UNLOCK (imap_folder, cache_lock);
2541                 CAMEL_SERVICE_UNLOCK (store, connect_lock);
2542                 return NULL;
2543         }
2544
2545         camel_exception_clear (ex);
2546         if (store->server_level < IMAP_LEVEL_IMAP4REV1 && !*section_text) {
2547                 response = camel_imap_command (store, folder, ex,
2548                                                "UID FETCH %s RFC822.PEEK",
2549                                                uid);
2550         } else {
2551                 response = camel_imap_command (store, folder, ex,
2552                                                "UID FETCH %s BODY.PEEK[%s]",
2553                                                uid, section_text);
2554         }
2555         /* We won't need the connect_lock again after this. */
2556         CAMEL_SERVICE_UNLOCK (store, connect_lock);
2557
2558         if (!response) {
2559                 CAMEL_IMAP_FOLDER_UNLOCK (imap_folder, cache_lock);
2560                 return NULL;
2561         }
2562
2563         for (i = 0; i < response->untagged->len; i++) {
2564                 fetch_data = parse_fetch_response (imap_folder, response->untagged->pdata[i]);
2565                 found_uid = g_datalist_get_data (&fetch_data, "UID");
2566                 stream = g_datalist_get_data (&fetch_data, "BODY_PART_STREAM");
2567                 if (found_uid && stream && !strcmp (uid, found_uid))
2568                         break;
2569
2570                 g_datalist_clear (&fetch_data);
2571                 stream = NULL;
2572         }
2573         camel_imap_response_free (store, response);
2574         CAMEL_IMAP_FOLDER_UNLOCK (imap_folder, cache_lock);
2575         if (!stream) {
2576                 camel_exception_setv (ex, CAMEL_EXCEPTION_SERVICE_UNAVAILABLE,
2577                                       _("Could not find message body in FETCH response."));
2578         } else {
2579                 camel_object_ref (CAMEL_OBJECT (stream));
2580                 g_datalist_clear (&fetch_data);
2581         }
2582
2583         return stream;
2584 }
2585
2586 static GData *
2587 parse_fetch_response (CamelImapFolder *imap_folder, char *response)
2588 {
2589         GData *data = NULL;
2590         char *start, *part_spec = NULL, *body = NULL, *uid = NULL, *idate = NULL;
2591         gboolean cache_header = TRUE, header = FALSE;
2592         size_t body_len = 0;
2593
2594         if (*response != '(') {
2595                 long seq;
2596
2597                 if (*response != '*' || *(response + 1) != ' ')
2598                         return NULL;
2599                 seq = strtol (response + 2, &response, 10);
2600                 if (seq == 0)
2601                         return NULL;
2602                 if (strncasecmp (response, " FETCH (", 8) != 0)
2603                         return NULL;
2604                 response += 7;
2605
2606                 g_datalist_set_data (&data, "SEQUENCE", GINT_TO_POINTER (seq));
2607         }
2608
2609         do {
2610                 /* Skip the initial '(' or the ' ' between elements */
2611                 response++;
2612
2613                 if (!strncasecmp (response, "FLAGS ", 6)) {
2614                         guint32 flags;
2615
2616                         response += 6;
2617                         /* FIXME user flags */
2618                         flags = imap_parse_flag_list (&response);
2619
2620                         g_datalist_set_data (&data, "FLAGS", GUINT_TO_POINTER (flags));
2621                 } else if (!strncasecmp (response, "RFC822.SIZE ", 12)) {
2622                         unsigned long size;
2623
2624                         response += 12;
2625                         size = strtoul (response, &response, 10);
2626                         g_datalist_set_data (&data, "RFC822.SIZE", GUINT_TO_POINTER (size));
2627                 } else if (!strncasecmp (response, "BODY[", 5) ||
2628                            !strncasecmp (response, "RFC822 ", 7)) {
2629                         char *p;
2630
2631                         if (*response == 'B') {
2632                                 response += 5;
2633
2634                                 /* HEADER], HEADER.FIELDS (...)], or 0] */
2635                                 if (!strncasecmp (response, "HEADER", 6)) {
2636                                         header = TRUE;
2637                                         if (!strncasecmp (response + 6, ".FIELDS", 7))
2638                                                 cache_header = FALSE;
2639                                 } else if (!strncasecmp (response, "0]", 2))
2640                                         header = TRUE;
2641
2642                                 p = strchr (response, ']');
2643                                 if (!p || *(p + 1) != ' ')
2644                                         break;
2645
2646                                 if (cache_header)
2647                                         part_spec = g_strndup (response, p - response);
2648                                 else
2649                                         part_spec = g_strdup ("HEADER.FIELDS");
2650
2651                                 response = p + 2;
2652                         } else {
2653                                 part_spec = g_strdup ("");
2654                                 response += 7;
2655
2656                                 if (!strncasecmp (response, "HEADER", 6))
2657                                         header = TRUE;
2658                         }
2659
2660                         body = imap_parse_nstring ((const char **) &response, &body_len);
2661                         if (!response) {
2662                                 g_free (part_spec);
2663                                 break;
2664                         }
2665
2666                         if (!body)
2667                                 body = g_strdup ("");
2668                         g_datalist_set_data_full (&data, "BODY_PART_SPEC", part_spec, g_free);
2669                         g_datalist_set_data_full (&data, "BODY_PART_DATA", body, g_free);
2670                         g_datalist_set_data (&data, "BODY_PART_LEN", GINT_TO_POINTER (body_len));
2671                 } else if (!strncasecmp (response, "BODY ", 5) ||
2672                            !strncasecmp (response, "BODYSTRUCTURE ", 14)) {
2673                         response = strchr (response, ' ') + 1;
2674                         start = response;
2675                         imap_skip_list ((const char **) &response);
2676                         g_datalist_set_data_full (&data, "BODY", g_strndup (start, response - start), g_free);
2677                 } else if (!strncasecmp (response, "UID ", 4)) {
2678                         int len;
2679
2680                         len = strcspn (response + 4, " )");
2681                         uid = g_strndup (response + 4, len);
2682                         g_datalist_set_data_full (&data, "UID", uid, g_free);
2683                         response += 4 + len;
2684                 } else if (!strncasecmp (response, "INTERNALDATE ", 13)) {
2685                         int len;
2686
2687                         response += 13;
2688                         if (*response == '"') {
2689                                 response++;
2690                                 len = strcspn (response, "\"");
2691                                 idate = g_strndup (response, len);
2692                                 g_datalist_set_data_full (&data, "INTERNALDATE", idate, g_free);
2693                                 response += len + 1;
2694                         }
2695                 } else {
2696                         g_warning ("Unexpected FETCH response from server: (%s", response);
2697                         break;
2698                 }
2699         } while (response && *response != ')');
2700
2701         if (!response || *response != ')') {
2702                 g_datalist_clear (&data);
2703                 return NULL;
2704         }
2705
2706         if (uid && body) {
2707                 CamelStream *stream;
2708
2709                 if (header && !cache_header) {
2710                         stream = camel_stream_mem_new_with_buffer (body, body_len);
2711                 } else {
2712                         CAMEL_IMAP_FOLDER_LOCK (imap_folder, cache_lock);
2713                         stream = camel_imap_message_cache_insert (imap_folder->cache,
2714                                                                   uid, part_spec,
2715                                                                   body, body_len, NULL);
2716                         CAMEL_IMAP_FOLDER_UNLOCK (imap_folder, cache_lock);
2717                 }
2718
2719                 if (stream)
2720                         g_datalist_set_data_full (&data, "BODY_PART_STREAM", stream,
2721                                                   (GDestroyNotify) camel_object_unref);
2722         }
2723
2724         return data;
2725 }
2726