camel/providers/imap/camel-imap-folder.c

   1 /* -*- Mode: C; tab-width: 8; indent-tabs-mode: t; c-basic-offset: 8 -*- */
   2 /* camel-imap-folder.c: class for an imap folder */
   3
   4 /*
   5  * Authors:
   6  *   Dan Winship <danw@ximian.com>
   7  *   Jeffrey Stedfast <fejj@ximian.com>
   8  *
   9  * Copyright (C) 2000, 2001 Ximian, Inc.
  10  *
  11  * This program is free software; you can redistribute it and/or
  12  * modify it under the terms of version 2 of the GNU Lesser General Public
  13  * License as published by the Free Software Foundation.
  14  *
  15  * This program is distributed in the hope that it will be useful,
  16  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  17  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  18  * GNU Lesser General Public License for more details.
  19  *
  20  * You should have received a copy of the GNU Lesser General Public License
  21  * along with this program; if not, write to the Free Software
  22  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301
  23  * USA
  24  */
  25
  26 #include <config.h>
  27
  28 #include <ctype.h>
  29 #include <errno.h>
  30 #include <fcntl.h>
  31 #include <stdlib.h>
  32 #include <string.h>
  33 #include <unistd.h>
  34 #include <sys/stat.h>
  35 #include <sys/types.h>
  36
  37 #include <glib/gi18n-lib.h>
  38
  39 #include <libedataserver/e-data-server-util.h>
  40 #include <libedataserver/e-time-utils.h>
  41
  42 #include "camel-data-wrapper.h"
  43 #include "camel-debug.h"
  44 #include "camel-disco-diary.h"
  45 #include "camel-exception.h"
  46 #include "camel-file-utils.h"
  47 #include "camel-mime-filter-crlf.h"
  48 #include "camel-mime-filter-from.h"
  49 #include "camel-mime-message.h"
  50 #include "camel-mime-utils.h"
  51 #include "camel-multipart-encrypted.h"
  52 #include "camel-multipart-signed.h"
  53 #include "camel-multipart.h"
  54 #include "camel-operation.h"
  55 #include "camel-private.h"
  56 #include "camel-session.h"
  57 #include "camel-store-summary.h"
  58 #include "camel-stream-buffer.h"
  59 #include "camel-stream-filter.h"
  60 #include "camel-stream-mem.h"
  61 #include "camel-stream.h"
  62 #include "camel-string-utils.h"
  63
  64 #include "camel-imap-command.h"
  65 #include "camel-imap-folder.h"
  66 #include "camel-imap-message-cache.h"
  67 #include "camel-imap-private.h"
  68 #include "camel-imap-search.h"
  69 #include "camel-imap-store.h"
  70 #include "camel-imap-summary.h"
  71 #include "camel-imap-utils.h"
  72 #include "camel-imap-wrapper.h"
  73
  74 #define d(x)
  75
  76 /* set to -1 for infinite size (suggested max command-line length is
  77  * 1000 octets (see rfc2683), so we should keep the uid-set length to
  78  * something under that so that our command-lines don't exceed 1000
  79  * octets) */
  80 #define UID_SET_LIMIT  (768)
  81
  82
  83 #define CF_CLASS(o) (CAMEL_FOLDER_CLASS (CAMEL_OBJECT_GET_CLASS(o)))
  84 static CamelDiscoFolderClass *disco_folder_class = NULL;
  85
  86 static void imap_finalize (CamelObject *object);
  87 static int imap_getv(CamelObject *object, CamelException *ex, CamelArgGetV *args);
  88
  89 static void imap_rescan (CamelFolder *folder, int exists, CamelException *ex);
  90 static void imap_refresh_info (CamelFolder *folder, CamelException *ex);
  91 static void imap_sync_online (CamelFolder *folder, CamelException *ex);
  92 static void imap_sync_offline (CamelFolder *folder, CamelException *ex);
  93 static void imap_expunge_uids_online (CamelFolder *folder, GPtrArray *uids, CamelException *ex);
  94 static void imap_expunge_uids_offline (CamelFolder *folder, GPtrArray *uids, CamelException *ex);
  95 static void imap_expunge_uids_resyncing (CamelFolder *folder, GPtrArray *uids, CamelException *ex);
  96 static void imap_cache_message (CamelDiscoFolder *disco_folder, const char *uid, CamelException *ex);
  97 static void imap_rename (CamelFolder *folder, const char *new);
  98
  99 /* message manipulation */
 100 static CamelMimeMessage *imap_get_message (CamelFolder *folder, const gchar *uid,
 101                                            CamelException *ex);
 102 static void imap_append_online (CamelFolder *folder, CamelMimeMessage *message,
 103                                 const CamelMessageInfo *info, char **appended_uid,
 104                                 CamelException *ex);
 105 static void imap_append_offline (CamelFolder *folder, CamelMimeMessage *message,
 106                                  const CamelMessageInfo *info, char **appended_uid,
 107                                  CamelException *ex);
 108 static void imap_append_resyncing (CamelFolder *folder, CamelMimeMessage *message,
 109                                    const CamelMessageInfo *info, char **appended_uid,
 110                                    CamelException *ex);
 111
 112 static void imap_transfer_online (CamelFolder *source, GPtrArray *uids,
 113                                   CamelFolder *dest, GPtrArray **transferred_uids,
 114                                   gboolean delete_originals,
 115                                   CamelException *ex);
 116 static void imap_transfer_offline (CamelFolder *source, GPtrArray *uids,
 117                                    CamelFolder *dest, GPtrArray **transferred_uids,
 118                                    gboolean delete_originals,
 119                                    CamelException *ex);
 120 static void imap_transfer_resyncing (CamelFolder *source, GPtrArray *uids,
 121                                      CamelFolder *dest, GPtrArray **transferred_uids,
 122                                      gboolean delete_originals,
 123                                      CamelException *ex);
 124
 125 /* searching */
 126 static GPtrArray *imap_search_by_expression (CamelFolder *folder, const char *expression, CamelException *ex);
 127 static GPtrArray *imap_search_by_uids       (CamelFolder *folder, const char *expression, GPtrArray *uids, CamelException *ex);
 128 static void       imap_search_free          (CamelFolder *folder, GPtrArray *uids);
 129
 130 static void imap_thaw (CamelFolder *folder);
 131
 132 static CamelObjectClass *parent_class;
 133
 134 static GData *parse_fetch_response (CamelImapFolder *imap_folder, char *msg_att);
 135
 136 #ifdef G_OS_WIN32
 137 /* The strtok() in Microsoft's C library is MT-safe (but still uses
 138  * only one buffer pointer per thread, but for the use of strtok_r()
 139  * here that's enough).
 140  */
 141 #define strtok_r(s,sep,lasts) (*(lasts)=strtok((s),(sep)))
 142 #endif
 143
 144 static void
 145 camel_imap_folder_class_init (CamelImapFolderClass *camel_imap_folder_class)
 146 {
 147         CamelFolderClass *camel_folder_class = CAMEL_FOLDER_CLASS (camel_imap_folder_class);
 148         CamelDiscoFolderClass *camel_disco_folder_class = CAMEL_DISCO_FOLDER_CLASS (camel_imap_folder_class);
 149
 150         disco_folder_class = CAMEL_DISCO_FOLDER_CLASS (camel_type_get_global_classfuncs (camel_disco_folder_get_type ()));
 151
 152         /* virtual method overload */
 153         ((CamelObjectClass *)camel_imap_folder_class)->getv = imap_getv;
 154
 155         camel_folder_class->get_message = imap_get_message;
 156         camel_folder_class->rename = imap_rename;
 157         camel_folder_class->search_by_expression = imap_search_by_expression;
 158         camel_folder_class->search_by_uids = imap_search_by_uids;
 159         camel_folder_class->search_free = imap_search_free;
 160         camel_folder_class->thaw = imap_thaw;
 161
 162         camel_disco_folder_class->refresh_info_online = imap_refresh_info;
 163         camel_disco_folder_class->sync_online = imap_sync_online;
 164         camel_disco_folder_class->sync_offline = imap_sync_offline;
 165         /* We don't sync flags at resync time: the online code will
 166          * deal with it eventually.
 167          */
 168         camel_disco_folder_class->sync_resyncing = imap_sync_offline;
 169         camel_disco_folder_class->expunge_uids_online = imap_expunge_uids_online;
 170         camel_disco_folder_class->expunge_uids_offline = imap_expunge_uids_offline;
 171         camel_disco_folder_class->expunge_uids_resyncing = imap_expunge_uids_resyncing;
 172         camel_disco_folder_class->append_online = imap_append_online;
 173         camel_disco_folder_class->append_offline = imap_append_offline;
 174         camel_disco_folder_class->append_resyncing = imap_append_resyncing;
 175         camel_disco_folder_class->transfer_online = imap_transfer_online;
 176         camel_disco_folder_class->transfer_offline = imap_transfer_offline;
 177         camel_disco_folder_class->transfer_resyncing = imap_transfer_resyncing;
 178         camel_disco_folder_class->cache_message = imap_cache_message;
 179 }
 180
 181 static void
 182 camel_imap_folder_init (gpointer object, gpointer klass)
 183 {
 184         CamelImapFolder *imap_folder = CAMEL_IMAP_FOLDER (object);
 185         CamelFolder *folder = CAMEL_FOLDER (object);
 186
 187         folder->permanent_flags = CAMEL_MESSAGE_ANSWERED | CAMEL_MESSAGE_DELETED |
 188                 CAMEL_MESSAGE_DRAFT | CAMEL_MESSAGE_FLAGGED | CAMEL_MESSAGE_SEEN;
 189
 190         folder->folder_flags |= (CAMEL_FOLDER_HAS_SUMMARY_CAPABILITY |
 191                                  CAMEL_FOLDER_HAS_SEARCH_CAPABILITY);
 192
 193         imap_folder->priv = g_malloc0(sizeof(*imap_folder->priv));
 194 #ifdef ENABLE_THREADS
 195         g_static_mutex_init(&imap_folder->priv->search_lock);
 196         g_static_rec_mutex_init(&imap_folder->priv->cache_lock);
 197 #endif
 198
 199         imap_folder->need_rescan = TRUE;
 200 }
 201
 202 CamelType
 203 camel_imap_folder_get_type (void)
 204 {
 205         static CamelType camel_imap_folder_type = CAMEL_INVALID_TYPE;
 206
 207         if (camel_imap_folder_type == CAMEL_INVALID_TYPE) {
 208                 parent_class = camel_disco_folder_get_type();
 209                 camel_imap_folder_type =
 210                         camel_type_register (parent_class, "CamelImapFolder",
 211                                              sizeof (CamelImapFolder),
 212                                              sizeof (CamelImapFolderClass),
 213                                              (CamelObjectClassInitFunc) camel_imap_folder_class_init,
 214                                              NULL,
 215                                              (CamelObjectInitFunc) camel_imap_folder_init,
 216                                              (CamelObjectFinalizeFunc) imap_finalize);
 217         }
 218
 219         return camel_imap_folder_type;
 220 }
 221
 222 CamelFolder *
 223 camel_imap_folder_new (CamelStore *parent, const char *folder_name,
 224                        const char *folder_dir, CamelException *ex)
 225 {
 226         CamelImapStore *imap_store = CAMEL_IMAP_STORE (parent);
 227         CamelFolder *folder;
 228         CamelImapFolder *imap_folder;
 229         const char *short_name;
 230         char *summary_file, *state_file;
 231
 232         if (g_mkdir_with_parents (folder_dir, S_IRWXU) != 0) {
 233                 camel_exception_setv (ex, CAMEL_EXCEPTION_SYSTEM,
 234                                       _("Could not create directory %s: %s"),
 235                                       folder_dir, g_strerror (errno));
 236                 return NULL;
 237         }
 238
 239         folder = CAMEL_FOLDER (camel_object_new (camel_imap_folder_get_type ()));
 240         short_name = strrchr (folder_name, '/');
 241         if (short_name)
 242                 short_name++;
 243         else
 244                 short_name = folder_name;
 245         camel_folder_construct (folder, parent, folder_name, short_name);
 246
 247         summary_file = g_strdup_printf ("%s/summary", folder_dir);
 248         folder->summary = camel_imap_summary_new (folder, summary_file);
 249         g_free (summary_file);
 250         if (!folder->summary) {
 251                 camel_object_unref (CAMEL_OBJECT (folder));
 252                 camel_exception_setv (ex, CAMEL_EXCEPTION_SYSTEM,
 253                                       _("Could not load summary for %s"),
 254                                       folder_name);
 255                 return NULL;
 256         }
 257
 258         /* set/load persistent state */
 259         state_file = g_strdup_printf ("%s/cmeta", folder_dir);
 260         camel_object_set(folder, NULL, CAMEL_OBJECT_STATE_FILE, state_file, NULL);
 261         g_free(state_file);
 262         camel_object_state_read(folder);
 263
 264         imap_folder = CAMEL_IMAP_FOLDER (folder);
 265         imap_folder->cache = camel_imap_message_cache_new (folder_dir, folder->summary, ex);
 266         if (!imap_folder->cache) {
 267                 camel_object_unref (CAMEL_OBJECT (folder));
 268                 return NULL;
 269         }
 270
 271         if (!g_ascii_strcasecmp (folder_name, "INBOX")) {
 272                 if ((imap_store->parameters & IMAP_PARAM_FILTER_INBOX))
 273                         folder->folder_flags |= CAMEL_FOLDER_FILTER_RECENT;
 274                 if ((imap_store->parameters & IMAP_PARAM_FILTER_JUNK))
 275                         folder->folder_flags |= CAMEL_FOLDER_FILTER_JUNK;
 276         } else {
 277                 if ((imap_store->parameters & (IMAP_PARAM_FILTER_JUNK|IMAP_PARAM_FILTER_JUNK_INBOX)) == (IMAP_PARAM_FILTER_JUNK))
 278                         folder->folder_flags |= CAMEL_FOLDER_FILTER_JUNK;
 279         }
 280
 281         imap_folder->search = camel_imap_search_new(folder_dir);
 282
 283         return folder;
 284 }
 285
 286 /* Called with the store's connect_lock locked */
 287 void
 288 camel_imap_folder_selected (CamelFolder *folder, CamelImapResponse *response,
 289                             CamelException *ex)
 290 {
 291         CamelImapFolder *imap_folder = CAMEL_IMAP_FOLDER (folder);
 292         CamelImapSummary *imap_summary = CAMEL_IMAP_SUMMARY (folder->summary);
 293         unsigned long exists = 0, validity = 0, val, uid;
 294         CamelMessageInfo *info;
 295         guint32 perm_flags = 0;
 296         GData *fetch_data;
 297         int i, count;
 298         char *resp;
 299
 300         count = camel_folder_summary_count (folder->summary);
 301
 302         for (i = 0; i < response->untagged->len; i++) {
 303                 resp = response->untagged->pdata[i] + 2;
 304                 if (!g_ascii_strncasecmp (resp, "FLAGS ", 6) && !perm_flags) {
 305                         resp += 6;
 306                         folder->permanent_flags = imap_parse_flag_list (&resp);
 307                 } else if (!g_ascii_strncasecmp (resp, "OK [PERMANENTFLAGS ", 19)) {
 308                         resp += 19;
 309
 310                         /* workaround for broken IMAP servers that send "* OK [PERMANENTFLAGS ()] Permanent flags"
 311                          * even tho they do allow storing flags. *Sigh* So many fucking broken IMAP servers out there. */
 312                         if ((perm_flags = imap_parse_flag_list (&resp)) != 0)
 313                                 folder->permanent_flags = perm_flags;
 314                 } else if (!g_ascii_strncasecmp (resp, "OK [UIDVALIDITY ", 16)) {
 315                         validity = strtoul (resp + 16, NULL, 10);
 316                 } else if (isdigit ((unsigned char)*resp)) {
 317                         unsigned long num = strtoul (resp, &resp, 10);
 318
 319                         if (!g_ascii_strncasecmp (resp, " EXISTS", 7)) {
 320                                 exists = num;
 321                                 /* Remove from the response so nothing
 322                                  * else tries to interpret it.
 323                                  */
 324                                 g_free (response->untagged->pdata[i]);
 325                                 g_ptr_array_remove_index (response->untagged, i--);
 326                         }
 327                 }
 328         }
 329
 330         if (camel_strstrcase (response->status, "OK [READ-ONLY]"))
 331                 imap_folder->read_only = TRUE;
 332
 333         if (camel_disco_store_status (CAMEL_DISCO_STORE (folder->parent_store)) == CAMEL_DISCO_STORE_RESYNCING) {
 334                 if (validity != imap_summary->validity) {
 335                         camel_exception_setv (ex, CAMEL_EXCEPTION_FOLDER_SUMMARY_INVALID,
 336                                               _("Folder was destroyed and recreated on server."));
 337                         return;
 338                 }
 339
 340                 /* FIXME: find missing UIDs ? */
 341                 return;
 342         }
 343
 344         if (!imap_summary->validity)
 345                 imap_summary->validity = validity;
 346         else if (validity != imap_summary->validity) {
 347                 imap_summary->validity = validity;
 348                 camel_folder_summary_clear (folder->summary);
 349                 CAMEL_IMAP_FOLDER_REC_LOCK (imap_folder, cache_lock);
 350                 camel_imap_message_cache_clear (imap_folder->cache);
 351                 CAMEL_IMAP_FOLDER_REC_UNLOCK (imap_folder, cache_lock);
 352                 imap_folder->need_rescan = FALSE;
 353                 camel_imap_folder_changed (folder, exists, NULL, ex);
 354                 return;
 355         }
 356
 357         /* If we've lost messages, we have to rescan everything */
 358         if (exists < count)
 359                 imap_folder->need_rescan = TRUE;
 360         else if (count != 0 && !imap_folder->need_rescan) {
 361                 CamelImapStore *store = CAMEL_IMAP_STORE (folder->parent_store);
 362
 363                 /* Similarly, if the UID of the highest message we
 364                  * know about has changed, then that indicates that
 365                  * messages have been both added and removed, so we
 366                  * have to rescan to find the removed ones. (We pass
 367                  * NULL for the folder since we know that this folder
 368                  * is selected, and we don't want camel_imap_command
 369                  * to worry about it.)
 370                  */
 371                 response = camel_imap_command (store, NULL, ex, "FETCH %d UID", count);
 372                 if (!response)
 373                         return;
 374                 uid = 0;
 375                 for (i = 0; i < response->untagged->len; i++) {
 376                         resp = response->untagged->pdata[i];
 377                         val = strtoul (resp + 2, &resp, 10);
 378                         if (val == 0)
 379                                 continue;
 380                         if (!g_ascii_strcasecmp (resp, " EXISTS")) {
 381                                 /* Another one?? */
 382                                 exists = val;
 383                                 continue;
 384                         }
 385                         if (uid != 0 || val != count || g_ascii_strncasecmp (resp, " FETCH (", 8) != 0)
 386                                 continue;
 387
 388                         fetch_data = parse_fetch_response (imap_folder, resp + 7);
 389                         uid = strtoul (g_datalist_get_data (&fetch_data, "UID"), NULL, 10);
 390                         g_datalist_clear (&fetch_data);
 391                 }
 392                 camel_imap_response_free_without_processing (store, response);
 393
 394                 info = camel_folder_summary_index (folder->summary, count - 1);
 395                 val = strtoul (camel_message_info_uid (info), NULL, 10);
 396                 camel_message_info_free(info);
 397                 if (uid == 0 || uid != val)
 398                         imap_folder->need_rescan = TRUE;
 399         }
 400
 401         /* Now rescan if we need to */
 402         if (imap_folder->need_rescan) {
 403                 imap_rescan (folder, exists, ex);
 404                 return;
 405         }
 406
 407         /* If we don't need to rescan completely, but new messages
 408          * have been added, find out about them.
 409          */
 410         if (exists > count)
 411                 camel_imap_folder_changed (folder, exists, NULL, ex);
 412
 413         /* And we're done. */
 414 }
 415
 416 static void
 417 imap_finalize (CamelObject *object)
 418 {
 419         CamelImapFolder *imap_folder = CAMEL_IMAP_FOLDER (object);
 420
 421         if (imap_folder->search)
 422                 camel_object_unref (CAMEL_OBJECT (imap_folder->search));
 423         if (imap_folder->cache)
 424                 camel_object_unref (CAMEL_OBJECT (imap_folder->cache));
 425
 426 #ifdef ENABLE_THREADS
 427         g_static_mutex_free(&imap_folder->priv->search_lock);
 428         g_static_rec_mutex_free(&imap_folder->priv->cache_lock);
 429 #endif
 430         g_free(imap_folder->priv);
 431 }
 432
 433 static int
 434 imap_getv(CamelObject *object, CamelException *ex, CamelArgGetV *args)
 435 {
 436         CamelFolder *folder = (CamelFolder *)object;
 437         int i, count=0;
 438         guint32 tag;
 439
 440         for (i=0;i<args->argc;i++) {
 441                 CamelArgGet *arg = &args->argv[i];
 442
 443                 tag = arg->tag;
 444
 445                 switch (tag & CAMEL_ARG_TAG) {
 446                         /* CamelObject args */
 447                 case CAMEL_OBJECT_ARG_DESCRIPTION:
 448                         if (folder->description == NULL) {
 449                                 CamelURL *uri = ((CamelService *)folder->parent_store)->url;
 450
 451                                 /* what if the full name doesn't incclude /'s?  does it matter? */
 452                                 folder->description = g_strdup_printf("%s@%s:%s", uri->user, uri->host, folder->full_name);
 453                         }
 454                         *arg->ca_str = folder->description;
 455                         break;
 456                 default:
 457                         count++;
 458                         continue;
 459                 }
 460
 461                 arg->tag = (tag & CAMEL_ARG_TYPE) | CAMEL_ARG_IGNORE;
 462         }
 463
 464         if (count)
 465                 return ((CamelObjectClass *)parent_class)->getv(object, ex, args);
 466
 467         return 0;
 468 }
 469
 470 static void
 471 imap_rename (CamelFolder *folder, const char *new)
 472 {
 473         CamelImapFolder *imap_folder = (CamelImapFolder *)folder;
 474         CamelImapStore *imap_store = (CamelImapStore *)folder->parent_store;
 475         char *folder_dir, *summary_path, *state_file;
 476         char *folders;
 477
 478         folders = g_strconcat (imap_store->storage_path, "/folders", NULL);
 479         folder_dir = imap_path_to_physical (folders, new);
 480         g_free (folders);
 481         summary_path = g_strdup_printf("%s/summary", folder_dir);
 482
 483         CAMEL_IMAP_FOLDER_REC_LOCK (folder, cache_lock);
 484         camel_imap_message_cache_set_path(imap_folder->cache, folder_dir);
 485         CAMEL_IMAP_FOLDER_REC_UNLOCK (folder, cache_lock);
 486
 487         camel_folder_summary_set_filename(folder->summary, summary_path);
 488
 489         state_file = g_strdup_printf ("%s/cmeta", folder_dir);
 490         camel_object_set(folder, NULL, CAMEL_OBJECT_STATE_FILE, state_file, NULL);
 491         g_free(state_file);
 492
 493         g_free(summary_path);
 494         g_free(folder_dir);
 495
 496         ((CamelFolderClass *)disco_folder_class)->rename(folder, new);
 497 }
 498
 499 static void
 500 imap_refresh_info (CamelFolder *folder, CamelException *ex)
 501 {
 502         CamelImapStore *imap_store = CAMEL_IMAP_STORE (folder->parent_store);
 503         CamelImapFolder *imap_folder = CAMEL_IMAP_FOLDER (folder);
 504         CamelImapResponse *response;
 505         CamelStoreInfo *si;
 506
 507         if (camel_disco_store_status (CAMEL_DISCO_STORE (imap_store)) == CAMEL_DISCO_STORE_OFFLINE)
 508                 return;
 509
 510         if (camel_folder_is_frozen (folder)) {
 511                 imap_folder->need_refresh = TRUE;
 512                 return;
 513         }
 514
 515         /* If the folder isn't selected, select it (which will force
 516          * a rescan if one is needed).
 517          * Also, if this is the INBOX, some servers (cryus) wont tell
 518          * us with a NOOP of new messages, so force a reselect which
 519          * should do it.  */
 520         CAMEL_SERVICE_REC_LOCK (imap_store, connect_lock);
 521
 522         if (!camel_imap_store_connected(imap_store, ex))
 523                 goto done;
 524
 525         if (imap_store->current_folder != folder
 526             || g_ascii_strcasecmp(folder->full_name, "INBOX") == 0) {
 527                 response = camel_imap_command (imap_store, folder, ex, NULL);
 528                 if (response) {
 529                         camel_imap_folder_selected (folder, response, ex);
 530                         camel_imap_response_free (imap_store, response);
 531                 }
 532         } else if (imap_folder->need_rescan) {
 533                 /* Otherwise, if we need a rescan, do it, and if not, just do
 534                  * a NOOP to give the server a chance to tell us about new
 535                  * messages.
 536                  */
 537                 imap_rescan (folder, camel_folder_summary_count (folder->summary), ex);
 538         } else {
 539 #if 0
 540                 /* on some servers need to CHECKpoint INBOX to recieve new messages?? */
 541                 /* rfc2060 suggests this, but havent seen a server that requires it */
 542                 if (g_ascii_strcasecmp(folder->full_name, "INBOX") == 0) {
 543                         response = camel_imap_command (imap_store, folder, ex, "CHECK");
 544                         camel_imap_response_free (imap_store, response);
 545                 }
 546 #endif
 547                 response = camel_imap_command (imap_store, folder, ex, "NOOP");
 548                 camel_imap_response_free (imap_store, response);
 549         }
 550
 551         si = camel_store_summary_path((CamelStoreSummary *)((CamelImapStore *)folder->parent_store)->summary, folder->full_name);
 552         if (si) {
 553                 guint32 unread, total;
 554
 555                 camel_object_get(folder, NULL, CAMEL_FOLDER_TOTAL, &total, CAMEL_FOLDER_UNREAD, &unread, NULL);
 556                 if (si->total != total
 557                     || si->unread != unread){
 558                         si->total = total;
 559                         si->unread = unread;
 560                         camel_store_summary_touch((CamelStoreSummary *)((CamelImapStore *)folder->parent_store)->summary);
 561                 }
 562                 camel_store_summary_info_free((CamelStoreSummary *)((CamelImapStore *)folder->parent_store)->summary, si);
 563         }
 564 done:
 565         CAMEL_SERVICE_REC_UNLOCK (imap_store, connect_lock);
 566
 567         camel_folder_summary_save(folder->summary);
 568         camel_store_summary_save((CamelStoreSummary *)((CamelImapStore *)folder->parent_store)->summary);
 569 }
 570
 571 static void
 572 flags_to_label(CamelFolder *folder, CamelImapMessageInfo *mi)
 573 {
 574         /* We snoop the server flag setting, and map it to
 575            the label 'user tag'.  We also clean it up at this
 576            point, there can only be 1 label set at a time */
 577         if (folder->permanent_flags & CAMEL_IMAP_MESSAGE_LABEL_MASK) {
 578                 const char *label = NULL;
 579                 guint32 mask = 0;
 580
 581                 if (mi->info.flags & CAMEL_IMAP_MESSAGE_LABEL1) {
 582                         mask = CAMEL_IMAP_MESSAGE_LABEL1;
 583                         label = "important";
 584                 } else if (mi->info.flags & CAMEL_IMAP_MESSAGE_LABEL2) {
 585                         mask = CAMEL_IMAP_MESSAGE_LABEL2;
 586                         label = "work";
 587                 } else if (mi->info.flags & CAMEL_IMAP_MESSAGE_LABEL3) {
 588                         mask = CAMEL_IMAP_MESSAGE_LABEL3;
 589                         label = "personal";
 590                 } else if (mi->info.flags & CAMEL_IMAP_MESSAGE_LABEL4) {
 591                         mask = CAMEL_IMAP_MESSAGE_LABEL4;
 592                         label = "todo";
 593                 } else if (mi->info.flags & CAMEL_IMAP_MESSAGE_LABEL5) {
 594                         mask = CAMEL_IMAP_MESSAGE_LABEL5;
 595                         label = "later";
 596                 }
 597
 598                 mi->info.flags = (mi->info.flags & ~CAMEL_IMAP_MESSAGE_LABEL_MASK) | mask;
 599                 camel_tag_set(&mi->info.user_tags, "label", label);
 600         }
 601 }
 602
 603 /* Called with the store's connect_lock locked */
 604 static void
 605 imap_rescan (CamelFolder *folder, int exists, CamelException *ex)
 606 {
 607         CamelImapFolder *imap_folder = CAMEL_IMAP_FOLDER (folder);
 608         CamelImapStore *store = CAMEL_IMAP_STORE (folder->parent_store);
 609         struct {
 610                 char *uid;
 611                 guint32 flags;
 612         } *new;
 613         char *resp;
 614         CamelImapResponseType type;
 615         int i, seq, summary_len, summary_got;
 616         CamelMessageInfo *info;
 617         CamelImapMessageInfo *iinfo;
 618         GArray *removed;
 619         gboolean ok;
 620         CamelFolderChangeInfo *changes = NULL;
 621
 622         imap_folder->need_rescan = FALSE;
 623
 624         summary_len = camel_folder_summary_count (folder->summary);
 625         if (summary_len == 0) {
 626                 if (exists)
 627                         camel_imap_folder_changed (folder, exists, NULL, ex);
 628                 return;
 629         }
 630
 631         /* Check UIDs and flags of all messages we already know of. */
 632         camel_operation_start (NULL, _("Scanning for changed messages in %s"), folder->name);
 633         info = camel_folder_summary_index (folder->summary, summary_len - 1);
 634         ok = camel_imap_command_start (store, folder, ex,
 635                                        "UID FETCH 1:%s (FLAGS)",
 636                                        camel_message_info_uid (info));
 637         camel_message_info_free(info);
 638         if (!ok) {
 639                 camel_operation_end (NULL);
 640                 return;
 641         }
 642
 643         new = g_malloc0 (summary_len * sizeof (*new));
 644         summary_got = 0;
 645         while ((type = camel_imap_command_response (store, &resp, ex)) == CAMEL_IMAP_RESPONSE_UNTAGGED) {
 646                 GData *data;
 647                 char *uid;
 648                 guint32 flags;
 649
 650                 data = parse_fetch_response (imap_folder, resp);
 651                 g_free (resp);
 652                 if (!data)
 653                         continue;
 654
 655                 seq = GPOINTER_TO_INT (g_datalist_get_data (&data, "SEQUENCE"));
 656                 uid = g_datalist_get_data (&data, "UID");
 657                 flags = GPOINTER_TO_UINT (g_datalist_get_data (&data, "FLAGS"));
 658
 659                 if (!uid || !seq || seq > summary_len || seq < 0) {
 660                         g_datalist_clear (&data);
 661                         continue;
 662                 }
 663
 664                 camel_operation_progress (NULL, ++summary_got * 100 / summary_len);
 665                 new[seq - 1].uid = g_strdup (uid);
 666                 new[seq - 1].flags = flags;
 667                 g_datalist_clear (&data);
 668         }
 669
 670         camel_operation_end (NULL);
 671         if (type == CAMEL_IMAP_RESPONSE_ERROR) {
 672                 for (i = 0; i < summary_len && new[i].uid; i++)
 673                         g_free (new[i].uid);
 674                 g_free (new);
 675                 return;
 676         }
 677
 678         /* Free the final tagged response */
 679         g_free (resp);
 680
 681         /* If we find a UID in the summary that doesn't correspond to
 682          * the UID in the folder, then either: (a) it's a real UID,
 683          * but the message was deleted on the server, or (b) it's a
 684          * fake UID, and needs to be removed from the summary in order
 685          * to sync up with the server. So either way, we remove it
 686          * from the summary.
 687          */
 688         removed = g_array_new (FALSE, FALSE, sizeof (int));
 689         for (i = 0; i < summary_len && new[i].uid; i++) {
 690                 info = camel_folder_summary_index (folder->summary, i);
 691                 iinfo = (CamelImapMessageInfo *)info;
 692
 693                 if (strcmp (camel_message_info_uid (info), new[i].uid) != 0) {
 694                         camel_message_info_free(info);
 695                         seq = i + 1;
 696                         g_array_append_val (removed, seq);
 697                         i--;
 698                         summary_len--;
 699                         continue;
 700                 }
 701
 702                 /* Update summary flags */
 703                 if (new[i].flags != iinfo->server_flags) {
 704                         guint32 server_set, server_cleared;
 705
 706                         server_set = new[i].flags & ~iinfo->server_flags;
 707                         server_cleared = iinfo->server_flags & ~new[i].flags;
 708
 709                         iinfo->info.flags = (iinfo->info.flags | server_set) & ~server_cleared;
 710                         iinfo->server_flags = new[i].flags;
 711
 712                         if (changes == NULL)
 713                                 changes = camel_folder_change_info_new();
 714                         camel_folder_change_info_change_uid(changes, new[i].uid);
 715                         flags_to_label(folder, (CamelImapMessageInfo *)info);
 716                 }
 717
 718                 camel_message_info_free(info);
 719                 g_free (new[i].uid);
 720         }
 721
 722         if (changes) {
 723                 camel_object_trigger_event(CAMEL_OBJECT (folder), "folder_changed", changes);
 724                 camel_folder_change_info_free(changes);
 725         }
 726
 727         seq = i + 1;
 728
 729         /* Free remaining memory. */
 730         while (i < summary_len && new[i].uid)
 731                 g_free (new[i++].uid);
 732         g_free (new);
 733
 734         /* Remove any leftover cached summary messages. (Yes, we
 735          * repeatedly add the same number to the removed array.
 736          * See RFC2060 7.4.1)
 737          */
 738
 739         for (i = seq; i <= summary_len; i++)
 740                 g_array_append_val (removed, seq);
 741
 742         /* And finally update the summary. */
 743         camel_imap_folder_changed (folder, exists, removed, ex);
 744         g_array_free (removed, TRUE);
 745 }
 746
 747 /* the max number of chars that an unsigned 32-bit int can be is 10 chars plus 1 for a possible : */
 748 #define UID_SET_FULL(setlen, maxlen) (maxlen > 0 ? setlen + 11 >= maxlen : FALSE)
 749
 750 /* Find all messages in @folder with flags matching @flags and @mask.
 751  * If no messages match, returns %NULL. Otherwise, returns an array of
 752  * CamelMessageInfo and sets *@set to a message set corresponding the
 753  * UIDs of the matched messages (up to @UID_SET_LIMIT bytes). The
 754  * caller must free the infos, the array, and the set string.
 755  */
 756 static GPtrArray *
 757 get_matching (CamelFolder *folder, guint32 flags, guint32 mask, char **set)
 758 {
 759         GPtrArray *matches;
 760         CamelImapMessageInfo *info;
 761         int i, max, range;
 762         GString *gset;
 763
 764         matches = g_ptr_array_new ();
 765         gset = g_string_new ("");
 766         max = camel_folder_summary_count (folder->summary);
 767         range = -1;
 768         for (i = 0; i < max && !UID_SET_FULL (gset->len, UID_SET_LIMIT); i++) {
 769                 info = (CamelImapMessageInfo *)camel_folder_summary_index (folder->summary, i);
 770                 if (!info)
 771                         continue;
 772                 if ((info->info.flags & mask) != flags) {
 773                         camel_message_info_free((CamelMessageInfo *)info);
 774                         if (range != -1) {
 775                                 if (range != i - 1) {
 776                                         info = matches->pdata[matches->len - 1];
 777                                         g_string_append_printf (gset, ":%s", camel_message_info_uid (info));
 778                                 }
 779                                 range = -1;
 780                         }
 781                         continue;
 782                 }
 783
 784                 g_ptr_array_add (matches, info);
 785                 if (range != -1)
 786                         continue;
 787                 range = i;
 788                 if (gset->len)
 789                         g_string_append_c (gset, ',');
 790                 g_string_append_printf (gset, "%s", camel_message_info_uid (info));
 791         }
 792
 793         if (range != -1 && range != max - 1) {
 794                 info = matches->pdata[matches->len - 1];
 795                 g_string_append_printf (gset, ":%s", camel_message_info_uid (info));
 796         }
 797
 798         if (matches->len) {
 799                 *set = gset->str;
 800                 g_string_free (gset, FALSE);
 801                 return matches;
 802         } else {
 803                 *set = NULL;
 804                 g_string_free (gset, TRUE);
 805                 g_ptr_array_free (matches, TRUE);
 806                 return NULL;
 807         }
 808 }
 809
 810 static void
 811 imap_sync_offline (CamelFolder *folder, CamelException *ex)
 812 {
 813         camel_folder_summary_save (folder->summary);
 814         camel_store_summary_save((CamelStoreSummary *)((CamelImapStore *)folder->parent_store)->summary);
 815 }
 816
 817 static void
 818 imap_sync_online (CamelFolder *folder, CamelException *ex)
 819 {
 820         CamelImapStore *store = CAMEL_IMAP_STORE (folder->parent_store);
 821         CamelImapResponse *response = NULL;
 822         CamelImapMessageInfo *info;
 823         CamelException local_ex;
 824         GPtrArray *matches;
 825         char *set, *flaglist;
 826         gboolean unset;
 827         int i, j, max;
 828
 829         if (folder->permanent_flags == 0) {
 830                 imap_sync_offline (folder, ex);
 831                 return;
 832         }
 833
 834         camel_exception_init (&local_ex);
 835         CAMEL_SERVICE_REC_LOCK (store, connect_lock);
 836
 837         /* Find a message with changed flags, find all of the other
 838          * messages like it, sync them as a group, mark them as
 839          * updated, and continue.
 840          */
 841         max = camel_folder_summary_count (folder->summary);
 842         for (i = 0; i < max; i++) {
 843                 if (!(info = (CamelImapMessageInfo *)camel_folder_summary_index (folder->summary, i)))
 844                         continue;
 845
 846                 if (!(info->info.flags & CAMEL_MESSAGE_FOLDER_FLAGGED)) {
 847                         camel_message_info_free((CamelMessageInfo *)info);
 848                         continue;
 849                 }
 850
 851                 /* Note: Cyrus is broken and will not accept an
 852                    empty-set of flags so... if this is true then we
 853                    want to unset the previously set flags.*/
 854                 unset = !(info->info.flags & folder->permanent_flags);
 855
 856                 /* Note: get_matching() uses UID_SET_LIMIT to limit
 857                    the size of the uid-set string. We don't have to
 858                    loop here to flush all the matching uids because
 859                    they will be scooped up later by our parent loop (I
 860                    think?). -- Jeff */
 861                 matches = get_matching (folder, info->info.flags & (folder->permanent_flags | CAMEL_MESSAGE_FOLDER_FLAGGED),
 862                                         folder->permanent_flags | CAMEL_MESSAGE_FOLDER_FLAGGED, &set);
 863                 camel_message_info_free(info);
 864                 if (matches == NULL)
 865                         continue;
 866
 867                 /* Make sure we're connected before issuing commands */
 868                 if (!camel_imap_store_connected(store, ex)) {
 869                         g_free(set);
 870                         break;
 871                 }
 872
 873                 /* FIXME: since we don't know the previously set flags,
 874                    if unset is TRUE then just unset all the flags? */
 875                 flaglist = imap_create_flag_list (unset ? folder->permanent_flags : info->info.flags & folder->permanent_flags);
 876
 877                 /* Note: to `unset' flags, use -FLAGS.SILENT (<flag list>) */
 878                 response = camel_imap_command (store, folder, &local_ex,
 879                                                "UID STORE %s %sFLAGS.SILENT %s",
 880                                                set, unset ? "-" : "", flaglist);
 881                 g_free (set);
 882                 g_free (flaglist);
 883
 884                 if (response)
 885                         camel_imap_response_free (store, response);
 886
 887                 if (!camel_exception_is_set (&local_ex)) {
 888                         for (j = 0; j < matches->len; j++) {
 889                                 info = matches->pdata[j];
 890                                 info->info.flags &= ~CAMEL_MESSAGE_FOLDER_FLAGGED;
 891                                 ((CamelImapMessageInfo *) info)->server_flags = info->info.flags & CAMEL_IMAP_SERVER_FLAGS;
 892                         }
 893                         camel_folder_summary_touch (folder->summary);
 894                 }
 895
 896                 for (j = 0; j < matches->len; j++) {
 897                         info = matches->pdata[j];
 898                         camel_message_info_free(&info->info);
 899                 }
 900                 g_ptr_array_free (matches, TRUE);
 901
 902                 /* We unlock here so that other threads can have a chance to grab the connect_lock */
 903                 CAMEL_SERVICE_REC_UNLOCK (store, connect_lock);
 904
 905                 /* check for an exception */
 906                 if (camel_exception_is_set (&local_ex)) {
 907                         camel_exception_xfer (ex, &local_ex);
 908                         return;
 909                 }
 910
 911                 /* Re-lock the connect_lock */
 912                 CAMEL_SERVICE_REC_LOCK (store, connect_lock);
 913         }
 914
 915         /* Save the summary */
 916         imap_sync_offline (folder, ex);
 917
 918         CAMEL_SERVICE_REC_UNLOCK (store, connect_lock);
 919 }
 920
 921 static int
 922 uid_compar (const void *va, const void *vb)
 923 {
 924         const char **sa = (const char **)va, **sb = (const char **)vb;
 925         unsigned long a, b;
 926
 927         a = strtoul (*sa, NULL, 10);
 928         b = strtoul (*sb, NULL, 10);
 929         if (a < b)
 930                 return -1;
 931         else if (a == b)
 932                 return 0;
 933         else
 934                 return 1;
 935 }
 936
 937 static void
 938 imap_expunge_uids_offline (CamelFolder *folder, GPtrArray *uids, CamelException *ex)
 939 {
 940         CamelFolderChangeInfo *changes;
 941         int i;
 942
 943         qsort (uids->pdata, uids->len, sizeof (void *), uid_compar);
 944
 945         changes = camel_folder_change_info_new ();
 946
 947         for (i = 0; i < uids->len; i++) {
 948                 camel_folder_summary_remove_uid (folder->summary, uids->pdata[i]);
 949                 camel_folder_change_info_remove_uid (changes, uids->pdata[i]);
 950                 /* We intentionally don't remove it from the cache because
 951                  * the cached data may be useful in replaying a COPY later.
 952                  */
 953         }
 954         camel_folder_summary_save (folder->summary);
 955
 956         camel_disco_diary_log (CAMEL_DISCO_STORE (folder->parent_store)->diary,
 957                                CAMEL_DISCO_DIARY_FOLDER_EXPUNGE, folder, uids);
 958
 959         camel_object_trigger_event (CAMEL_OBJECT (folder), "folder_changed", changes);
 960         camel_folder_change_info_free (changes);
 961 }
 962
 963 static void
 964 imap_expunge_uids_online (CamelFolder *folder, GPtrArray *uids, CamelException *ex)
 965 {
 966         CamelImapStore *store = CAMEL_IMAP_STORE (folder->parent_store);
 967         CamelImapResponse *response;
 968         int uid = 0;
 969         char *set;
 970
 971         CAMEL_SERVICE_REC_LOCK (store, connect_lock);
 972
 973         if ((store->capabilities & IMAP_CAPABILITY_UIDPLUS) == 0) {
 974                 ((CamelFolderClass *)CAMEL_OBJECT_GET_CLASS(folder))->sync(folder, 0, ex);
 975                 if (camel_exception_is_set(ex)) {
 976                         CAMEL_SERVICE_REC_UNLOCK (store, connect_lock);
 977                         return;
 978                 }
 979         }
 980
 981         qsort (uids->pdata, uids->len, sizeof (void *), uid_compar);
 982
 983         while (uid < uids->len) {
 984                 set = imap_uid_array_to_set (folder->summary, uids, uid, UID_SET_LIMIT, &uid);
 985                 response = camel_imap_command (store, folder, ex,
 986                                                "UID STORE %s +FLAGS.SILENT (\\Deleted)",
 987                                                set);
 988                 if (response)
 989                         camel_imap_response_free (store, response);
 990                 if (camel_exception_is_set (ex)) {
 991                         CAMEL_SERVICE_REC_UNLOCK (store, connect_lock);
 992                         g_free (set);
 993                         return;
 994                 }
 995
 996                 if (store->capabilities & IMAP_CAPABILITY_UIDPLUS) {
 997                         response = camel_imap_command (store, folder, ex,
 998                                                        "UID EXPUNGE %s", set);
 999                 } else
1000                         response = camel_imap_command (store, folder, ex, "EXPUNGE");
1001
1002                 if (response)
1003                         camel_imap_response_free (store, response);
1004         }
1005
1006         CAMEL_SERVICE_REC_UNLOCK (store, connect_lock);
1007 }
1008
1009 static void
1010 imap_expunge_uids_resyncing (CamelFolder *folder, GPtrArray *uids, CamelException *ex)
1011 {
1012         CamelImapFolder *imap_folder = CAMEL_IMAP_FOLDER (folder);
1013         CamelImapStore *store = CAMEL_IMAP_STORE (folder->parent_store);
1014         GPtrArray *keep_uids, *mark_uids;
1015         CamelImapResponse *response;
1016         char *result;
1017
1018         if (imap_folder->read_only)
1019                 return;
1020
1021         if (store->capabilities & IMAP_CAPABILITY_UIDPLUS) {
1022                 imap_expunge_uids_online (folder, uids, ex);
1023                 return;
1024         }
1025
1026         /* If we don't have UID EXPUNGE we need to avoid expunging any
1027          * of the wrong messages. So we search for deleted messages,
1028          * and any that aren't in our to-expunge list get temporarily
1029          * marked un-deleted.
1030          */
1031
1032         CAMEL_SERVICE_REC_LOCK (store, connect_lock);
1033
1034         ((CamelFolderClass *)CAMEL_OBJECT_GET_CLASS(folder))->sync(folder, 0, ex);
1035         if (camel_exception_is_set(ex)) {
1036                 CAMEL_SERVICE_REC_UNLOCK (store, connect_lock);
1037                 return;
1038         }
1039
1040         response = camel_imap_command (store, folder, ex, "UID SEARCH DELETED");
1041         if (!response) {
1042                 CAMEL_SERVICE_REC_UNLOCK (store, connect_lock);
1043                 return;
1044         }
1045         result = camel_imap_response_extract (store, response, "SEARCH", ex);
1046         if (!result) {
1047                 CAMEL_SERVICE_REC_UNLOCK (store, connect_lock);
1048                 return;
1049         }
1050
1051         if (result[8] == ' ') {
1052                 char *uid, *lasts = NULL;
1053                 unsigned long euid, kuid;
1054                 int ei, ki;
1055
1056                 keep_uids = g_ptr_array_new ();
1057                 mark_uids = g_ptr_array_new ();
1058
1059                 /* Parse SEARCH response */
1060                 for (uid = strtok_r (result + 9, " ", &lasts); uid; uid = strtok_r (NULL, " ", &lasts))
1061                         g_ptr_array_add (keep_uids, uid);
1062                 qsort (keep_uids->pdata, keep_uids->len,
1063                        sizeof (void *), uid_compar);
1064
1065                 /* Fill in "mark_uids", empty out "keep_uids" as needed */
1066                 for (ei = ki = 0; ei < uids->len; ei++) {
1067                         euid = strtoul (uids->pdata[ei], NULL, 10);
1068
1069                         for (kuid = 0; ki < keep_uids->len; ki++) {
1070                                 kuid = strtoul (keep_uids->pdata[ki], NULL, 10);
1071
1072                                 if (kuid >= euid)
1073                                         break;
1074                         }
1075
1076                         if (euid == kuid)
1077                                 g_ptr_array_remove_index (keep_uids, ki);
1078                         else
1079                                 g_ptr_array_add (mark_uids, uids->pdata[ei]);
1080                 }
1081         } else {
1082                 /* Empty SEARCH result, meaning nothing is marked deleted
1083                  * on server.
1084                  */
1085
1086                 keep_uids = NULL;
1087                 mark_uids = uids;
1088         }
1089
1090         /* Unmark messages to be kept */
1091
1092         if (keep_uids) {
1093                 char *uidset;
1094                 int uid = 0;
1095
1096                 while (uid < keep_uids->len) {
1097                         uidset = imap_uid_array_to_set (folder->summary, keep_uids, uid, UID_SET_LIMIT, &uid);
1098
1099                         response = camel_imap_command (store, folder, ex,
1100                                                        "UID STORE %s -FLAGS.SILENT (\\Deleted)",
1101                                                        uidset);
1102
1103                         g_free (uidset);
1104
1105                         if (!response) {
1106                                 g_ptr_array_free (keep_uids, TRUE);
1107                                 g_ptr_array_free (mark_uids, TRUE);
1108                                 CAMEL_SERVICE_REC_UNLOCK (store, connect_lock);
1109                                 return;
1110                         }
1111                         camel_imap_response_free (store, response);
1112                 }
1113         }
1114
1115         /* Mark any messages that still need to be marked */
1116         if (mark_uids) {
1117                 char *uidset;
1118                 int uid = 0;
1119
1120                 while (uid < mark_uids->len) {
1121                         uidset = imap_uid_array_to_set (folder->summary, mark_uids, uid, UID_SET_LIMIT, &uid);
1122
1123                         response = camel_imap_command (store, folder, ex,
1124                                                        "UID STORE %s +FLAGS.SILENT (\\Deleted)",
1125                                                        uidset);
1126
1127                         g_free (uidset);
1128
1129                         if (!response) {
1130                                 g_ptr_array_free (keep_uids, TRUE);
1131                                 g_ptr_array_free (mark_uids, TRUE);
1132                                 CAMEL_SERVICE_REC_UNLOCK (store, connect_lock);
1133                                 return;
1134                         }
1135                         camel_imap_response_free (store, response);
1136                 }
1137
1138                 if (mark_uids != uids)
1139                         g_ptr_array_free (mark_uids, TRUE);
1140         }
1141
1142         /* Do the actual expunging */
1143         response = camel_imap_command (store, folder, ex, "EXPUNGE");
1144         if (response)
1145                 camel_imap_response_free (store, response);
1146
1147         /* And fix the remaining messages if we mangled them */
1148         if (keep_uids) {
1149                 char *uidset;
1150                 int uid = 0;
1151
1152                 while (uid < keep_uids->len) {
1153                         uidset = imap_uid_array_to_set (folder->summary, keep_uids, uid, UID_SET_LIMIT, &uid);
1154
1155                         /* Don't pass ex if it's already been set */
1156                         response = camel_imap_command (store, folder,
1157                                                        camel_exception_is_set (ex) ? NULL : ex,
1158                                                        "UID STORE %s +FLAGS.SILENT (\\Deleted)",
1159                                                        uidset);
1160
1161                         g_free (uidset);
1162                         if (response)
1163                                 camel_imap_response_free (store, response);
1164                 }
1165
1166                 g_ptr_array_free (keep_uids, TRUE);
1167         }
1168
1169         /* now we can free this, now that we're done with keep_uids */
1170         g_free (result);
1171
1172         CAMEL_SERVICE_REC_UNLOCK (store, connect_lock);
1173 }
1174
1175 static gchar *
1176 get_temp_uid (void)
1177 {
1178         gchar *res;
1179
1180         static int counter = 0;
1181         G_LOCK_DEFINE_STATIC (lock);
1182
1183         G_LOCK (lock);
1184         res = g_strdup_printf ("tempuid-%lx-%d",
1185                                (unsigned long) time (NULL),
1186                                counter++);
1187         G_UNLOCK (lock);
1188
1189         return res;
1190 }
1191
1192 static void
1193 imap_append_offline (CamelFolder *folder, CamelMimeMessage *message,
1194                      const CamelMessageInfo *info, char **appended_uid,
1195                      CamelException *ex)
1196 {
1197         CamelImapStore *imap_store = CAMEL_IMAP_STORE (folder->parent_store);
1198         CamelImapMessageCache *cache = CAMEL_IMAP_FOLDER (folder)->cache;
1199         CamelFolderChangeInfo *changes;
1200         char *uid;
1201
1202         uid = get_temp_uid ();
1203
1204         camel_imap_summary_add_offline (folder->summary, uid, message, info);
1205         CAMEL_IMAP_FOLDER_REC_LOCK (folder, cache_lock);
1206         camel_imap_message_cache_insert_wrapper (cache, uid, "",
1207                                                  CAMEL_DATA_WRAPPER (message), ex);
1208         CAMEL_IMAP_FOLDER_REC_UNLOCK (folder, cache_lock);
1209
1210         changes = camel_folder_change_info_new ();
1211         camel_folder_change_info_add_uid (changes, uid);
1212         camel_object_trigger_event (CAMEL_OBJECT (folder), "folder_changed",
1213                                     changes);
1214         camel_folder_change_info_free (changes);
1215
1216         camel_disco_diary_log (CAMEL_DISCO_STORE (imap_store)->diary,
1217                                CAMEL_DISCO_DIARY_FOLDER_APPEND, folder, uid);
1218         if (appended_uid)
1219                 *appended_uid = uid;
1220         else
1221                 g_free (uid);
1222 }
1223
1224 static CamelImapResponse *
1225 do_append (CamelFolder *folder, CamelMimeMessage *message,
1226            const CamelMessageInfo *info, char **uid,
1227            CamelException *ex)
1228 {
1229         CamelImapStore *store = CAMEL_IMAP_STORE (folder->parent_store);
1230         CamelImapResponse *response, *response2;
1231         CamelStream *memstream;
1232         CamelMimeFilter *crlf_filter;
1233         CamelStreamFilter *streamfilter;
1234         GByteArray *ba;
1235         char *flagstr, *end;
1236         guint32 flags = 0;
1237
1238         /* encode any 8bit parts so we avoid sending embedded nul-chars and such  */
1239         camel_mime_message_encode_8bit_parts (message);
1240
1241         /* FIXME: We could avoid this if we knew how big the message was. */
1242         memstream = camel_stream_mem_new ();
1243         ba = g_byte_array_new ();
1244         camel_stream_mem_set_byte_array (CAMEL_STREAM_MEM (memstream), ba);
1245
1246         streamfilter = camel_stream_filter_new_with_stream (memstream);
1247         crlf_filter = camel_mime_filter_crlf_new (CAMEL_MIME_FILTER_CRLF_ENCODE,
1248                                                   CAMEL_MIME_FILTER_CRLF_MODE_CRLF_ONLY);
1249         camel_stream_filter_add (streamfilter, crlf_filter);
1250         camel_data_wrapper_write_to_stream (CAMEL_DATA_WRAPPER (message),
1251                                             CAMEL_STREAM (streamfilter));
1252         camel_object_unref (CAMEL_OBJECT (streamfilter));
1253         camel_object_unref (CAMEL_OBJECT (crlf_filter));
1254         camel_object_unref (CAMEL_OBJECT (memstream));
1255
1256         /* Some servers dont let us append with custom flags.  If the command fails for
1257            whatever reason, assume this is the case and save the state and try again */
1258 retry:
1259         if (info) {
1260                 flags = camel_message_info_flags(info);
1261                 if (!store->nocustomappend)
1262                         flags |= imap_label_to_flags((CamelMessageInfo *)info);
1263         }
1264
1265         flags &= folder->permanent_flags;
1266         if (flags)
1267                 flagstr = imap_create_flag_list (flags);
1268         else
1269                 flagstr = NULL;
1270
1271         response = camel_imap_command (store, NULL, ex, "APPEND %F%s%s {%d}",
1272                                        folder->full_name, flagstr ? " " : "",
1273                                        flagstr ? flagstr : "", ba->len);
1274         g_free (flagstr);
1275
1276         if (!response) {
1277                 if (camel_exception_get_id(ex) == CAMEL_EXCEPTION_SERVICE_INVALID && !store->nocustomappend) {
1278                         camel_exception_clear(ex);
1279                         store->nocustomappend = 1;
1280                         goto retry;
1281                 }
1282                 g_byte_array_free (ba, TRUE);
1283                 return NULL;
1284         }
1285
1286         if (*response->status != '+') {
1287                 camel_imap_response_free (store, response);
1288                 g_byte_array_free (ba, TRUE);
1289                 return NULL;
1290         }
1291
1292         /* send the rest of our data - the mime message */
1293         response2 = camel_imap_command_continuation (store, (const char *) ba->data, ba->len, ex);
1294         g_byte_array_free (ba, TRUE);
1295
1296         /* free it only after message is sent. This may cause more FETCHes. */
1297         camel_imap_response_free (store, response);
1298         if (!response2)
1299                 return response2;
1300
1301         if (store->capabilities & IMAP_CAPABILITY_UIDPLUS) {
1302                 *uid = camel_strstrcase (response2->status, "[APPENDUID ");
1303                 if (*uid)
1304                         *uid = strchr (*uid + 11, ' ');
1305                 if (*uid) {
1306                         *uid = g_strndup (*uid + 1, strcspn (*uid + 1, "]"));
1307                         /* Make sure it's a number */
1308                         if (strtoul (*uid, &end, 10) == 0 || *end) {
1309                                 g_free (*uid);
1310                                 *uid = NULL;
1311                         }
1312                 }
1313         } else
1314                 *uid = NULL;
1315
1316         return response2;
1317 }
1318
1319 static void
1320 imap_append_online (CamelFolder *folder, CamelMimeMessage *message,
1321                     const CamelMessageInfo *info, char **appended_uid,
1322                     CamelException *ex)
1323 {
1324         CamelImapStore *store = CAMEL_IMAP_STORE (folder->parent_store);
1325         CamelImapResponse *response;
1326         char *uid;
1327         int count;
1328
1329         count = camel_folder_summary_count (folder->summary);
1330         response = do_append (folder, message, info, &uid, ex);
1331         if (!response)
1332                 return;
1333
1334         if (uid) {
1335                 /* Cache first, since freeing response may trigger a
1336                  * summary update that will want this information.
1337                  */
1338                 CAMEL_IMAP_FOLDER_REC_LOCK (folder, cache_lock);
1339                 camel_imap_message_cache_insert_wrapper (
1340                         CAMEL_IMAP_FOLDER (folder)->cache, uid,
1341                         "", CAMEL_DATA_WRAPPER (message), ex);
1342                 CAMEL_IMAP_FOLDER_REC_UNLOCK (folder, cache_lock);
1343                 if (appended_uid)
1344                         *appended_uid = uid;
1345                 else
1346                         g_free (uid);
1347         } else if (appended_uid)
1348                 *appended_uid = NULL;
1349
1350         camel_imap_response_free (store, response);
1351
1352         /* Make sure a "folder_changed" is emitted. */
1353         CAMEL_SERVICE_REC_LOCK (store, connect_lock);
1354         if (store->current_folder != folder ||
1355             camel_folder_summary_count (folder->summary) == count)
1356                 imap_refresh_info (folder, ex);
1357         CAMEL_SERVICE_REC_UNLOCK (store, connect_lock);
1358 }
1359
1360 static void
1361 imap_append_resyncing (CamelFolder *folder, CamelMimeMessage *message,
1362                        const CamelMessageInfo *info, char **appended_uid,
1363                        CamelException *ex)
1364 {
1365         CamelImapStore *store = CAMEL_IMAP_STORE (folder->parent_store);
1366         CamelImapResponse *response;
1367         char *uid;
1368
1369         response = do_append (folder, message, info, &uid, ex);
1370         if (!response)
1371                 return;
1372
1373         if (uid) {
1374                 CamelImapFolder *imap_folder = CAMEL_IMAP_FOLDER (folder);
1375                 const char *olduid = camel_message_info_uid (info);
1376
1377                 CAMEL_IMAP_FOLDER_REC_LOCK (imap_folder, cache_lock);
1378                 camel_imap_message_cache_copy (imap_folder->cache, olduid,
1379                                                imap_folder->cache, uid, ex);
1380                 CAMEL_IMAP_FOLDER_REC_UNLOCK (imap_folder, cache_lock);
1381
1382                 if (appended_uid)
1383                         *appended_uid = uid;
1384                 else
1385                         g_free (uid);
1386         } else if (appended_uid)
1387                 *appended_uid = NULL;
1388
1389         camel_imap_response_free (store, response);
1390 }
1391
1392
1393 static void
1394 imap_transfer_offline (CamelFolder *source, GPtrArray *uids,
1395                        CamelFolder *dest, GPtrArray **transferred_uids,
1396                        gboolean delete_originals, CamelException *ex)
1397 {
1398         CamelImapStore *store = CAMEL_IMAP_STORE (source->parent_store);
1399         CamelImapMessageCache *sc = CAMEL_IMAP_FOLDER (source)->cache;
1400         CamelImapMessageCache *dc = CAMEL_IMAP_FOLDER (dest)->cache;
1401         CamelFolderChangeInfo *changes;
1402         CamelMimeMessage *message;
1403         CamelMessageInfo *mi;
1404         char *uid, *destuid;
1405         int i;
1406
1407         /* We grab the store's command lock first, and then grab the
1408          * source and destination cache_locks. This way we can't
1409          * deadlock in the case where we're simultaneously also trying
1410          * to copy messages in the other direction from another thread.
1411          */
1412         CAMEL_SERVICE_REC_LOCK (store, connect_lock);
1413         CAMEL_IMAP_FOLDER_REC_LOCK (source, cache_lock);
1414         CAMEL_IMAP_FOLDER_REC_LOCK (dest, cache_lock);
1415         CAMEL_SERVICE_REC_UNLOCK (store, connect_lock);
1416
1417         if (transferred_uids) {
1418                 *transferred_uids = g_ptr_array_new ();
1419                 g_ptr_array_set_size (*transferred_uids, uids->len);
1420         }
1421
1422         changes = camel_folder_change_info_new ();
1423
1424         for (i = 0; i < uids->len; i++) {
1425                 uid = uids->pdata[i];
1426
1427                 destuid = get_temp_uid ();
1428
1429                 mi = camel_folder_summary_uid (source->summary, uid);
1430                 g_return_if_fail (mi != NULL);
1431
1432                 message = camel_folder_get_message (source, uid, NULL);
1433
1434                 if (message) {
1435                         camel_imap_summary_add_offline (dest->summary, destuid, message, mi);
1436                         camel_object_unref (CAMEL_OBJECT (message));
1437                 } else
1438                         camel_imap_summary_add_offline_uncached (dest->summary, destuid, mi);
1439
1440                 camel_imap_message_cache_copy (sc, uid, dc, destuid, ex);
1441                 camel_message_info_free(mi);
1442
1443                 camel_folder_change_info_add_uid (changes, destuid);
1444                 if (transferred_uids)
1445                         (*transferred_uids)->pdata[i] = destuid;
1446                 else
1447                         g_free (destuid);
1448
1449                 if (delete_originals)
1450                         camel_folder_delete_message (source, uid);
1451         }
1452
1453         CAMEL_IMAP_FOLDER_REC_UNLOCK (dest, cache_lock);
1454         CAMEL_IMAP_FOLDER_REC_UNLOCK (source, cache_lock);
1455
1456         camel_object_trigger_event (CAMEL_OBJECT (dest), "folder_changed", changes);
1457         camel_folder_change_info_free (changes);
1458
1459         camel_disco_diary_log (CAMEL_DISCO_STORE (store)->diary,
1460                                CAMEL_DISCO_DIARY_FOLDER_TRANSFER,
1461                                source, dest, uids, delete_originals);
1462 }
1463
1464 static void
1465 handle_copyuid (CamelImapResponse *response, CamelFolder *source,
1466                 CamelFolder *destination)
1467 {
1468         CamelImapMessageCache *scache = CAMEL_IMAP_FOLDER (source)->cache;
1469         CamelImapMessageCache *dcache = CAMEL_IMAP_FOLDER (destination)->cache;
1470         char *validity, *srcset, *destset;
1471         GPtrArray *src, *dest;
1472         int i;
1473
1474         validity = camel_strstrcase (response->status, "[COPYUID ");
1475         if (!validity)
1476                 return;
1477         validity += 9;
1478         if (strtoul (validity, NULL, 10) !=
1479             CAMEL_IMAP_SUMMARY (destination->summary)->validity)
1480                 return;
1481
1482         srcset = strchr (validity, ' ');
1483         if (!srcset++)
1484                 goto lose;
1485         destset = strchr (srcset, ' ');
1486         if (!destset++)
1487                 goto lose;
1488
1489         src = imap_uid_set_to_array (source->summary, srcset);
1490         dest = imap_uid_set_to_array (destination->summary, destset);
1491
1492         if (src && dest && src->len == dest->len) {
1493                 /* We don't have to worry about deadlocking on the
1494                  * cache locks here, because we've got the store's
1495                  * command lock too, so no one else could be here.
1496                  */
1497                 CAMEL_IMAP_FOLDER_REC_LOCK (source, cache_lock);
1498                 CAMEL_IMAP_FOLDER_REC_LOCK (destination, cache_lock);
1499                 for (i = 0; i < src->len; i++) {
1500                         camel_imap_message_cache_copy (scache, src->pdata[i],
1501                                                        dcache, dest->pdata[i],
1502                                                        NULL);
1503                 }
1504                 CAMEL_IMAP_FOLDER_REC_UNLOCK (source, cache_lock);
1505                 CAMEL_IMAP_FOLDER_REC_UNLOCK (destination, cache_lock);
1506
1507                 imap_uid_array_free (src);
1508                 imap_uid_array_free (dest);
1509                 return;
1510         }
1511
1512         imap_uid_array_free (src);
1513         imap_uid_array_free (dest);
1514  lose:
1515         g_warning ("Bad COPYUID response from server");
1516 }
1517
1518 static void
1519 do_copy (CamelFolder *source, GPtrArray *uids,
1520          CamelFolder *destination, int delete_originals, CamelException *ex)
1521 {
1522         CamelImapStore *store = CAMEL_IMAP_STORE (source->parent_store);
1523         CamelImapResponse *response;
1524         char *uidset;
1525         int uid = 0, last=0, i;
1526
1527         while (uid < uids->len && !camel_exception_is_set (ex)) {
1528                 uidset = imap_uid_array_to_set (source->summary, uids, uid, UID_SET_LIMIT, &uid);
1529
1530                 if ((store->capabilities & IMAP_CAPABILITY_XGWMOVE) && delete_originals) {
1531                         response = camel_imap_command (store, source, ex, "UID XGWMOVE %s %F", uidset, destination->full_name);
1532                         /* TODO: EXPUNGE returns??? */
1533                         camel_imap_response_free (store, response);
1534                 } else {
1535                         response = camel_imap_command (store, source, ex, "UID COPY %s %F", uidset, destination->full_name);
1536                         if (response && (store->capabilities & IMAP_CAPABILITY_UIDPLUS))
1537                                 handle_copyuid (response, source, destination);
1538                         camel_imap_response_free (store, response);
1539
1540                         if (!camel_exception_is_set(ex) && delete_originals) {
1541                                 for (i=last;i<uid;i++)
1542                                         camel_folder_delete_message(source, uids->pdata[i]);
1543                                 last = uid;
1544                         }
1545                 }
1546                 g_free (uidset);
1547         }
1548 }
1549
1550 static void
1551 imap_transfer_online (CamelFolder *source, GPtrArray *uids,
1552                       CamelFolder *dest, GPtrArray **transferred_uids,
1553                       gboolean delete_originals, CamelException *ex)
1554 {
1555         CamelImapStore *store = CAMEL_IMAP_STORE (source->parent_store);
1556         int count;
1557
1558         /* Sync message flags if needed. */
1559         imap_sync_online (source, ex);
1560         if (camel_exception_is_set (ex))
1561                 return;
1562
1563         count = camel_folder_summary_count (dest->summary);
1564
1565         qsort (uids->pdata, uids->len, sizeof (void *), uid_compar);
1566
1567         /* Now copy the messages */
1568         do_copy(source, uids, dest, delete_originals, ex);
1569         if (camel_exception_is_set (ex))
1570                 return;
1571
1572         /* Make the destination notice its new messages */
1573         if (store->current_folder != dest ||
1574             camel_folder_summary_count (dest->summary) == count)
1575                 camel_folder_refresh_info (dest, ex);
1576
1577         /* FIXME */
1578         if (transferred_uids)
1579                 *transferred_uids = NULL;
1580 }
1581
1582 static void
1583 imap_transfer_resyncing (CamelFolder *source, GPtrArray *uids,
1584                          CamelFolder *dest, GPtrArray **transferred_uids,
1585                          gboolean delete_originals, CamelException *ex)
1586 {
1587         CamelDiscoDiary *diary = CAMEL_DISCO_STORE (source->parent_store)->diary;
1588         GPtrArray *realuids;
1589         int first, i;
1590         const char *uid;
1591         CamelMimeMessage *message;
1592         CamelMessageInfo *info;
1593
1594         qsort (uids->pdata, uids->len, sizeof (void *), uid_compar);
1595
1596         /* This is trickier than append_resyncing, because some of
1597          * the messages we are copying may have been copied or
1598          * appended into @source while we were offline, in which case
1599          * if we don't have UIDPLUS, we won't know their real UIDs,
1600          * so we'll have to append them rather than copying.
1601          */
1602
1603         realuids = g_ptr_array_new ();
1604
1605         i = 0;
1606         while (i < uids->len) {
1607                 /* Skip past real UIDs */
1608                 for (first = i; i < uids->len; i++) {
1609                         uid = uids->pdata[i];
1610
1611                         if (!isdigit ((unsigned char)*uid)) {
1612                                 uid = camel_disco_diary_uidmap_lookup (diary, uid);
1613                                 if (!uid)
1614                                         break;
1615                         }
1616                         g_ptr_array_add (realuids, (char *)uid);
1617                 }
1618
1619                 /* If we saw any real UIDs, do a COPY */
1620                 if (i != first) {
1621                         do_copy (source, realuids, dest, delete_originals, ex);
1622                         g_ptr_array_set_size (realuids, 0);
1623                         if (i == uids->len || camel_exception_is_set (ex))
1624                                 break;
1625                 }
1626
1627                 /* Deal with fake UIDs */
1628                 while (i < uids->len &&
1629                        !isdigit (*(unsigned char *)(uids->pdata[i])) &&
1630                        !camel_exception_is_set (ex)) {
1631                         uid = uids->pdata[i];
1632                         message = camel_folder_get_message (source, uid, NULL);
1633                         if (!message) {
1634                                 /* Message must have been expunged */
1635                                 continue;
1636                         }
1637                         info = camel_folder_get_message_info (source, uid);
1638                         g_return_if_fail (info != NULL);
1639
1640                         imap_append_online (dest, message, info, NULL, ex);
1641                         camel_folder_free_message_info (source, info);
1642                         camel_object_unref (CAMEL_OBJECT (message));
1643                         if (delete_originals)
1644                                 camel_folder_delete_message (source, uid);
1645                         i++;
1646                 }
1647         }
1648
1649         g_ptr_array_free (realuids, FALSE);
1650
1651         /* FIXME */
1652         if (transferred_uids)
1653                 *transferred_uids = NULL;
1654 }
1655
1656 static GPtrArray *
1657 imap_search_by_expression (CamelFolder *folder, const char *expression, CamelException *ex)
1658 {
1659         CamelImapFolder *imap_folder = CAMEL_IMAP_FOLDER (folder);
1660         GPtrArray *matches;
1661
1662         /* we could get around this by creating a new search object each time,
1663            but i doubt its worth it since any long operation would lock the
1664            command channel too */
1665         CAMEL_IMAP_FOLDER_LOCK(folder, search_lock);
1666
1667         camel_folder_search_set_folder (imap_folder->search, folder);
1668         matches = camel_folder_search_search(imap_folder->search, expression, NULL, ex);
1669
1670         CAMEL_IMAP_FOLDER_UNLOCK(folder, search_lock);
1671
1672         return matches;
1673 }
1674
1675 static GPtrArray *
1676 imap_search_by_uids(CamelFolder *folder, const char *expression, GPtrArray *uids, CamelException *ex)
1677 {
1678         CamelImapFolder *imap_folder = CAMEL_IMAP_FOLDER(folder);
1679         GPtrArray *matches;
1680
1681         if (uids->len == 0)
1682                 return g_ptr_array_new();
1683
1684         CAMEL_IMAP_FOLDER_LOCK(folder, search_lock);
1685
1686         camel_folder_search_set_folder(imap_folder->search, folder);
1687         matches = camel_folder_search_search(imap_folder->search, expression, uids, ex);
1688
1689         CAMEL_IMAP_FOLDER_UNLOCK(folder, search_lock);
1690
1691         return matches;
1692 }
1693
1694 static void
1695 imap_search_free (CamelFolder *folder, GPtrArray *uids)
1696 {
1697         CamelImapFolder *imap_folder = CAMEL_IMAP_FOLDER (folder);
1698
1699         g_return_if_fail (imap_folder->search);
1700
1701         CAMEL_IMAP_FOLDER_LOCK(folder, search_lock);
1702
1703         camel_folder_search_free_result (imap_folder->search, uids);
1704
1705         CAMEL_IMAP_FOLDER_UNLOCK(folder, search_lock);
1706 }
1707
1708 static CamelMimeMessage *get_message (CamelImapFolder *imap_folder,
1709                                       const char *uid,
1710                                       CamelMessageContentInfo *ci,
1711                                       CamelException *ex);
1712
1713 struct _part_spec_stack {
1714         struct _part_spec_stack *parent;
1715         int part;
1716 };
1717
1718 static void
1719 part_spec_push (struct _part_spec_stack **stack, int part)
1720 {
1721         struct _part_spec_stack *node;
1722
1723         node = g_new (struct _part_spec_stack, 1);
1724         node->parent = *stack;
1725         node->part = part;
1726
1727         *stack = node;
1728 }
1729
1730 static int
1731 part_spec_pop (struct _part_spec_stack **stack)
1732 {
1733         struct _part_spec_stack *node;
1734         int part;
1735
1736         g_return_val_if_fail (*stack != NULL, 0);
1737
1738         node = *stack;
1739         *stack = node->parent;
1740
1741         part = node->part;
1742         g_free (node);
1743
1744         return part;
1745 }
1746
1747 static char *
1748 content_info_get_part_spec (CamelMessageContentInfo *ci)
1749 {
1750         struct _part_spec_stack *stack = NULL;
1751         CamelMessageContentInfo *node;
1752         char *part_spec, *buf;
1753         size_t len = 1;
1754         int part;
1755
1756         node = ci;
1757         while (node->parent) {
1758                 CamelMessageContentInfo *child;
1759
1760                 /* FIXME: is this only supposed to apply if 'node' is a multipart? */
1761                 if (node->parent->parent &&
1762                                 camel_content_type_is (node->parent->type, "message", "*") &&
1763                                 !camel_content_type_is (node->parent->parent->type, "message", "*")) {
1764                         node = node->parent;
1765                         continue;
1766                 }
1767
1768                 child = node->parent->childs;
1769                 for (part = 1; child; part++) {
1770                         if (child == node)
1771                                 break;
1772
1773                         child = child->next;
1774                 }
1775
1776                 part_spec_push (&stack, part);
1777
1778                 len += 2;
1779                 while ((part = part / 10))
1780                         len++;
1781
1782                 node = node->parent;
1783         }
1784
1785         buf = part_spec = g_malloc (len);
1786         part_spec[0] = '\0';
1787
1788         while (stack) {
1789                 part = part_spec_pop (&stack);
1790                 buf += sprintf (buf, "%d%s", part, stack ? "." : "");
1791         }
1792
1793         return part_spec;
1794 }
1795
1796 /* Fetch the contents of the MIME part indicated by @ci, which is part
1797  * of message @uid in @folder.
1798  */
1799 static CamelDataWrapper *
1800 get_content (CamelImapFolder *imap_folder, const char *uid,
1801              CamelMimePart *part, CamelMessageContentInfo *ci,
1802              int frommsg,
1803              CamelException *ex)
1804 {
1805         CamelDataWrapper *content = NULL;
1806         CamelStream *stream;
1807         char *part_spec;
1808
1809         part_spec = content_info_get_part_spec (ci);
1810
1811         d(printf("get content '%s' '%s' (frommsg = %d)\n", part_spec, camel_content_type_format(ci->type), frommsg));
1812
1813         /* There are three cases: multipart/signed, multipart, message/rfc822, and "other" */
1814         if (camel_content_type_is (ci->type, "multipart", "signed")) {
1815                 CamelMultipartSigned *body_mp;
1816                 char *spec;
1817                 int ret;
1818
1819                 /* Note: because we get the content parts uninterpreted anyway, we could potentially
1820                    just use the normalmultipart code, except that multipart/signed wont let you yet! */
1821
1822                 body_mp = camel_multipart_signed_new ();
1823                 /* need to set this so it grabs the boundary and other info about the signed type */
1824                 /* we assume that part->content_type is more accurate/full than ci->type */
1825                 camel_data_wrapper_set_mime_type_field (CAMEL_DATA_WRAPPER (body_mp), CAMEL_DATA_WRAPPER (part)->mime_type);
1826
1827                 spec = g_alloca(strlen(part_spec) + 6);
1828                 if (frommsg)
1829                         sprintf(spec, part_spec[0] ? "%s.TEXT" : "TEXT", part_spec);
1830                 else
1831                         strcpy(spec, part_spec);
1832                 g_free(part_spec);
1833
1834                 stream = camel_imap_folder_fetch_data (imap_folder, uid, spec, FALSE, ex);
1835                 if (stream) {
1836                         ret = camel_data_wrapper_construct_from_stream (CAMEL_DATA_WRAPPER (body_mp), stream);
1837                         camel_object_unref (CAMEL_OBJECT (stream));
1838                         if (ret == -1) {
1839                                 camel_object_unref ((CamelObject *) body_mp);
1840                                 return NULL;
1841                         }
1842                 }
1843
1844                 return (CamelDataWrapper *) body_mp;
1845         } else if (camel_content_type_is (ci->type, "multipart", "*")) {
1846                 CamelMultipart *body_mp;
1847                 char *child_spec;
1848                 int speclen, num, isdigest;
1849
1850                 if (camel_content_type_is (ci->type, "multipart", "encrypted"))
1851                         body_mp = (CamelMultipart *) camel_multipart_encrypted_new ();
1852                 else
1853                         body_mp = camel_multipart_new ();
1854
1855                 /* need to set this so it grabs the boundary and other info about the multipart */
1856                 /* we assume that part->content_type is more accurate/full than ci->type */
1857                 camel_data_wrapper_set_mime_type_field (CAMEL_DATA_WRAPPER (body_mp), CAMEL_DATA_WRAPPER (part)->mime_type);
1858                 isdigest = camel_content_type_is(((CamelDataWrapper *)part)->mime_type, "multipart", "digest");
1859
1860                 speclen = strlen (part_spec);
1861                 child_spec = g_malloc (speclen + 17); /* dot + 10 + dot + MIME + nul */
1862                 memcpy (child_spec, part_spec, speclen);
1863                 if (speclen > 0)
1864                         child_spec[speclen++] = '.';
1865                 g_free (part_spec);
1866
1867                 ci = ci->childs;
1868                 num = 1;
1869                 while (ci) {
1870                         sprintf (child_spec + speclen, "%d.MIME", num++);
1871                         stream = camel_imap_folder_fetch_data (imap_folder, uid, child_spec, FALSE, ex);
1872                         if (stream) {
1873                                 int ret;
1874
1875                                 part = camel_mime_part_new ();
1876                                 ret = camel_data_wrapper_construct_from_stream (CAMEL_DATA_WRAPPER (part), stream);
1877                                 camel_object_unref (CAMEL_OBJECT (stream));
1878                                 if (ret == -1) {
1879                                         camel_object_unref (CAMEL_OBJECT (part));
1880                                         camel_object_unref (CAMEL_OBJECT (body_mp));
1881                                         g_free (child_spec);
1882                                         return NULL;
1883                                 }
1884
1885                                 content = get_content (imap_folder, uid, part, ci, FALSE, ex);
1886                         }
1887
1888                         if (!stream || !content) {
1889                                 camel_object_unref (CAMEL_OBJECT (body_mp));
1890                                 g_free (child_spec);
1891                                 return NULL;
1892                         }
1893
1894                         if (camel_debug("imap:folder")) {
1895                                 char *ct = camel_content_type_format(camel_mime_part_get_content_type((CamelMimePart *)part));
1896                                 char *ct2 = camel_content_type_format(ci->type);
1897
1898                                 printf("Setting part content type to '%s' contentinfo type is '%s'\n", ct, ct2);
1899                                 g_free(ct);
1900                                 g_free(ct2);
1901                         }
1902
1903                         /* if we had no content-type header on a multipart/digest sub-part, then we need to
1904                            treat it as message/rfc822 instead */
1905                         if (isdigest && camel_medium_get_header((CamelMedium *)part, "content-type") == NULL) {
1906                                 CamelContentType *ct = camel_content_type_new("message", "rfc822");
1907
1908                                 camel_data_wrapper_set_mime_type_field(content, ct);
1909                                 camel_content_type_unref(ct);
1910                         } else {
1911                                 camel_data_wrapper_set_mime_type_field(content, camel_mime_part_get_content_type(part));
1912                         }
1913
1914                         camel_medium_set_content_object (CAMEL_MEDIUM (part), content);
1915                         camel_object_unref(content);
1916
1917                         camel_multipart_add_part (body_mp, part);
1918                         camel_object_unref(part);
1919
1920                         ci = ci->next;
1921                 }
1922
1923                 g_free (child_spec);
1924
1925                 return (CamelDataWrapper *) body_mp;
1926         } else if (camel_content_type_is (ci->type, "message", "rfc822")) {
1927                 content = (CamelDataWrapper *) get_message (imap_folder, uid, ci->childs, ex);
1928                 g_free (part_spec);
1929                 return content;
1930         } else {
1931                 CamelTransferEncoding enc;
1932                 char *spec;
1933
1934                 /* NB: we need this differently to multipart/signed case above on purpose */
1935                 spec = g_alloca(strlen(part_spec) + 6);
1936                 if (frommsg)
1937                         sprintf(spec, part_spec[0] ? "%s.1" : "1", part_spec);
1938                 else
1939                         strcpy(spec, part_spec[0]?part_spec:"1");
1940
1941                 enc = ci->encoding?camel_transfer_encoding_from_string(ci->encoding):CAMEL_TRANSFER_ENCODING_DEFAULT;
1942                 content = camel_imap_wrapper_new (imap_folder, ci->type, enc, uid, spec, part);
1943                 g_free (part_spec);
1944                 return content;
1945         }
1946 }
1947
1948 static CamelMimeMessage *
1949 get_message (CamelImapFolder *imap_folder, const char *uid,
1950              CamelMessageContentInfo *ci,
1951              CamelException *ex)
1952 {
1953         CamelImapStore *store = CAMEL_IMAP_STORE (CAMEL_FOLDER (imap_folder)->parent_store);
1954         CamelDataWrapper *content;
1955         CamelMimeMessage *msg;
1956         CamelStream *stream;
1957         char *section_text, *part_spec;
1958         int ret;
1959
1960         part_spec = content_info_get_part_spec(ci);
1961         d(printf("get message '%s'\n", part_spec));
1962         section_text = g_strdup_printf ("%s%s%s", part_spec, *part_spec ? "." : "",
1963                                         store->server_level >= IMAP_LEVEL_IMAP4REV1 ? "HEADER" : "0");
1964
1965         stream = camel_imap_folder_fetch_data (imap_folder, uid, section_text, FALSE, ex);
1966         g_free (section_text);
1967         g_free(part_spec);
1968         if (!stream)
1969                 return NULL;
1970
1971         msg = camel_mime_message_new ();
1972         ret = camel_data_wrapper_construct_from_stream (CAMEL_DATA_WRAPPER (msg), stream);
1973         camel_object_unref (CAMEL_OBJECT (stream));
1974         if (ret == -1) {
1975                 camel_object_unref (CAMEL_OBJECT (msg));
1976                 return NULL;
1977         }
1978
1979         content = get_content (imap_folder, uid, CAMEL_MIME_PART (msg), ci, TRUE, ex);
1980         if (!content) {
1981                 camel_object_unref (CAMEL_OBJECT (msg));
1982                 return NULL;
1983         }
1984
1985         if (camel_debug("imap:folder")) {
1986                 char *ct = camel_content_type_format(camel_mime_part_get_content_type((CamelMimePart *)msg));
1987                 char *ct2 = camel_content_type_format(ci->type);
1988
1989                 printf("Setting message content type to '%s' contentinfo type is '%s'\n", ct, ct2);
1990                 g_free(ct);
1991                 g_free(ct2);
1992         }
1993
1994         camel_data_wrapper_set_mime_type_field(content, camel_mime_part_get_content_type((CamelMimePart *)msg));
1995         camel_medium_set_content_object (CAMEL_MEDIUM (msg), content);
1996         camel_object_unref (CAMEL_OBJECT (content));
1997
1998         return msg;
1999 }
2000
2001 #define IMAP_SMALL_BODY_SIZE 5120
2002
2003 static CamelMimeMessage *
2004 get_message_simple (CamelImapFolder *imap_folder, const char *uid,
2005                     CamelStream *stream, CamelException *ex)
2006 {
2007         CamelMimeMessage *msg;
2008         int ret;
2009
2010         if (!stream) {
2011                 stream = camel_imap_folder_fetch_data (imap_folder, uid, "",
2012                                                        FALSE, ex);
2013                 if (!stream)
2014                         return NULL;
2015         }
2016
2017         msg = camel_mime_message_new ();
2018         ret = camel_data_wrapper_construct_from_stream (CAMEL_DATA_WRAPPER (msg),
2019                                                         stream);
2020         camel_object_unref (CAMEL_OBJECT (stream));
2021         if (ret == -1) {
2022                 camel_exception_setv (ex, CAMEL_EXCEPTION_SERVICE_UNAVAILABLE,
2023                                       _("Unable to retrieve message: %s"),
2024                                       g_strerror (errno));
2025                 camel_object_unref (CAMEL_OBJECT (msg));
2026                 return NULL;
2027         }
2028
2029         return msg;
2030 }
2031
2032 static gboolean
2033 content_info_incomplete (CamelMessageContentInfo *ci)
2034 {
2035         if (!ci->type)
2036                 return TRUE;
2037
2038         if (camel_content_type_is (ci->type, "multipart", "*")
2039             || camel_content_type_is (ci->type, "message", "rfc822")) {
2040                 if (!ci->childs)
2041                         return TRUE;
2042                 for (ci = ci->childs;ci;ci=ci->next)
2043                         if (content_info_incomplete(ci))
2044                                 return TRUE;
2045         }
2046
2047         return FALSE;
2048 }
2049
2050 static CamelMimeMessage *
2051 imap_get_message (CamelFolder *folder, const char *uid, CamelException *ex)
2052 {
2053         CamelImapFolder *imap_folder = CAMEL_IMAP_FOLDER (folder);
2054         CamelImapStore *store = CAMEL_IMAP_STORE (folder->parent_store);
2055         CamelImapMessageInfo *mi;
2056         CamelMimeMessage *msg = NULL;
2057         CamelStream *stream = NULL;
2058         int retry;
2059
2060         mi = (CamelImapMessageInfo *)camel_folder_summary_uid (folder->summary, uid);
2061         if (mi == NULL) {
2062                 camel_exception_setv(ex, CAMEL_EXCEPTION_FOLDER_INVALID_UID,
2063                                      _("Cannot get message: %s\n  %s"), uid, _("No such message"));
2064                 return NULL;
2065         }
2066
2067         /* If its cached in full, just get it as is, this is only a shortcut,
2068            since we get stuff from the cache anyway.  It affects a busted connection though. */
2069         if ( (stream = camel_imap_folder_fetch_data(imap_folder, uid, "", TRUE, NULL))
2070              && (msg = get_message_simple(imap_folder, uid, stream, ex)))
2071                 goto done;
2072
2073         /* All this mess is so we silently retry a fetch if we fail with
2074            service_unavailable, without an (equivalent) mess of gotos */
2075         retry = 0;
2076         do {
2077                 retry++;
2078                 camel_exception_clear(ex);
2079
2080                 /* If the message is small or only 1 part, or server doesn't do 4v1 (properly) fetch it in one piece. */
2081                 if (store->server_level < IMAP_LEVEL_IMAP4REV1
2082                     || store->braindamaged
2083                     || mi->info.size < IMAP_SMALL_BODY_SIZE
2084                     || (!content_info_incomplete(mi->info.content) && !mi->info.content->childs)) {
2085                         msg = get_message_simple (imap_folder, uid, NULL, ex);
2086                 } else {
2087                         if (content_info_incomplete (mi->info.content)) {
2088                                 /* For larger messages, fetch the structure and build a message
2089                                  * with offline parts. (We check mi->content->type rather than
2090                                  * mi->content because camel_folder_summary_info_new always creates
2091                                  * an empty content struct.)
2092                                  */
2093                                 CamelImapResponse *response;
2094                                 GData *fetch_data = NULL;
2095                                 char *body, *found_uid;
2096                                 int i;
2097
2098                                 CAMEL_SERVICE_REC_LOCK(store, connect_lock);
2099                                 if (!camel_imap_store_connected(store, ex)) {
2100                                         CAMEL_SERVICE_REC_UNLOCK(store, connect_lock);
2101                                         camel_exception_set (ex, CAMEL_EXCEPTION_SERVICE_UNAVAILABLE,
2102                                                              _("This message is not currently available"));
2103                                         goto fail;
2104                                 }
2105
2106                                 response = camel_imap_command (store, folder, ex, "UID FETCH %s BODY", uid);
2107                                 CAMEL_SERVICE_REC_UNLOCK(store, connect_lock);
2108
2109                                 if (response) {
2110                                         for (i = 0, body = NULL; i < response->untagged->len; i++) {
2111                                                 fetch_data = parse_fetch_response (imap_folder, response->untagged->pdata[i]);
2112                                                 if (fetch_data) {
2113                                                         found_uid = g_datalist_get_data (&fetch_data, "UID");
2114                                                         body = g_datalist_get_data (&fetch_data, "BODY");
2115                                                         if (found_uid && body && !strcmp (found_uid, uid))
2116                                                                 break;
2117                                                         g_datalist_clear (&fetch_data);
2118                                                         fetch_data = NULL;
2119                                                         body = NULL;
2120                                                 }
2121                                         }
2122
2123                                         if (body) {
2124                                                 /* NB: small race here, setting the info.content */
2125                                                 imap_parse_body ((const char **) &body, folder, mi->info.content);
2126                                                 camel_folder_summary_touch (folder->summary);
2127                                         }
2128
2129                                         if (fetch_data)
2130                                                 g_datalist_clear (&fetch_data);
2131
2132                                         camel_imap_response_free (store, response);
2133                                 } else {
2134                                         camel_exception_clear(ex);
2135                                 }
2136                         }
2137
2138                         if (camel_debug_start("imap:folder")) {
2139                                 printf("Folder get message '%s' folder info ->\n", uid);
2140                                 camel_message_info_dump((CamelMessageInfo *)mi);
2141                                 camel_debug_end();
2142                         }
2143
2144                         /* FETCH returned OK, but we didn't parse a BODY
2145                          * response. Courier will return invalid BODY
2146                          * responses for invalidly MIMEd messages, so
2147                          * fall back to fetching the entire thing and
2148                          * let the mailer's "bad MIME" code handle it.
2149                          */
2150                         if (content_info_incomplete (mi->info.content))
2151                                 msg = get_message_simple (imap_folder, uid, NULL, ex);
2152                         else
2153                                 msg = get_message (imap_folder, uid, mi->info.content, ex);
2154                 }
2155         } while (msg == NULL
2156                  && retry < 2
2157                  && camel_exception_get_id(ex) == CAMEL_EXCEPTION_SERVICE_UNAVAILABLE);
2158
2159 done:   /* FIXME, this shouldn't be done this way. */
2160         if (msg)
2161                 camel_medium_set_header (CAMEL_MEDIUM (msg), "X-Evolution-Source", store->base_url);
2162 fail:
2163         camel_message_info_free(&mi->info);
2164
2165         return msg;
2166 }
2167
2168 static void
2169 imap_cache_message (CamelDiscoFolder *disco_folder, const char *uid,
2170                     CamelException *ex)
2171 {
2172         CamelImapFolder *imap_folder = CAMEL_IMAP_FOLDER (disco_folder);
2173         CamelStream *stream;
2174
2175         stream = camel_imap_folder_fetch_data (imap_folder, uid, "", FALSE, ex);
2176         if (stream)
2177                 camel_object_unref (CAMEL_OBJECT (stream));
2178 }
2179
2180 /* We pretend that a FLAGS or RFC822.SIZE response is always exactly
2181  * 20 bytes long, and a BODY[HEADERS] response is always 2000 bytes
2182  * long. Since we know how many of each kind of response we're
2183  * expecting, we can find the total (pretend) amount of server traffic
2184  * to expect and then count off the responses as we read them to update
2185  * the progress bar.
2186  */
2187 #define IMAP_PRETEND_SIZEOF_FLAGS         20
2188 #define IMAP_PRETEND_SIZEOF_SIZE          20
2189 #define IMAP_PRETEND_SIZEOF_HEADERS     2000
2190
2191 static char *tm_months[] = {
2192         "Jan", "Feb", "Mar", "Apr", "May", "Jun",
2193         "Jul", "Aug", "Sep", "Oct", "Nov", "Dec"
2194 };
2195
2196 static gboolean
2197 decode_time (const unsigned char **in, int *hour, int *min, int *sec)
2198 {
2199         register const unsigned char *inptr;
2200         int *val, colons = 0;
2201
2202         *hour = *min = *sec = 0;
2203
2204         val = hour;
2205         for (inptr = *in; *inptr && !isspace ((int) *inptr); inptr++) {
2206                 if (*inptr == ':') {
2207                         colons++;
2208                         switch (colons) {
2209                         case 1:
2210                                 val = min;
2211                                 break;
2212                         case 2:
2213                                 val = sec;
2214                                 break;
2215                         default:
2216                                 return FALSE;
2217                         }
2218                 } else if (!isdigit ((int) *inptr))
2219                         return FALSE;
2220                 else
2221                         *val = (*val * 10) + (*inptr - '0');
2222         }
2223
2224         *in = inptr;
2225
2226         return TRUE;
2227 }
2228
2229 static time_t
2230 decode_internaldate (const unsigned char *in)
2231 {
2232         const unsigned char *inptr = in;
2233         int hour, min, sec, n;
2234         unsigned char *buf;
2235         struct tm tm;
2236         time_t date;
2237
2238         memset ((void *) &tm, 0, sizeof (struct tm));
2239
2240         tm.tm_mday = strtoul ((char *) inptr, (char **) &buf, 10);
2241         if (buf == inptr || *buf != '-')
2242                 return (time_t) -1;
2243
2244         inptr = buf + 1;
2245         if (inptr[3] != '-')
2246                 return (time_t) -1;
2247
2248         for (n = 0; n < 12; n++) {
2249                 if (!g_ascii_strncasecmp ((gchar *) inptr, tm_months[n], 3))
2250                         break;
2251         }
2252
2253         if (n >= 12)
2254                 return (time_t) -1;
2255
2256         tm.tm_mon = n;
2257
2258         inptr += 4;
2259
2260         n = strtoul ((char *) inptr, (char **) &buf, 10);
2261         if (buf == inptr || *buf != ' ')
2262                 return (time_t) -1;
2263
2264         tm.tm_year = n - 1900;
2265
2266         inptr = buf + 1;
2267         if (!decode_time (&inptr, &hour, &min, &sec))
2268                 return (time_t) -1;
2269
2270         tm.tm_hour = hour;
2271         tm.tm_min = min;
2272         tm.tm_sec = sec;
2273
2274         n = strtol ((char *) inptr, NULL, 10);
2275
2276         date = e_mktime_utc (&tm);
2277
2278         /* date is now GMT of the time we want, but not offset by the timezone ... */
2279
2280         /* this should convert the time to the GMT equiv time */
2281         date -= ((n / 100) * 60 * 60) + (n % 100) * 60;
2282
2283         return date;
2284 }
2285
2286 static void
2287 add_message_from_data (CamelFolder *folder, GPtrArray *messages,
2288                        int first, GData *data)
2289 {
2290         CamelMimeMessage *msg;
2291         CamelStream *stream;
2292         CamelImapMessageInfo *mi;
2293         const char *idate;
2294         int seq;
2295
2296         seq = GPOINTER_TO_INT (g_datalist_get_data (&data, "SEQUENCE"));
2297         if (seq < first)
2298                 return;
2299         stream = g_datalist_get_data (&data, "BODY_PART_STREAM");
2300         if (!stream)
2301                 return;
2302
2303         if (seq - first >= messages->len)
2304                 g_ptr_array_set_size (messages, seq - first + 1);
2305
2306         msg = camel_mime_message_new ();
2307         if (camel_data_wrapper_construct_from_stream (CAMEL_DATA_WRAPPER (msg), stream) == -1) {
2308                 camel_object_unref (CAMEL_OBJECT (msg));
2309                 return;
2310         }
2311
2312         mi = (CamelImapMessageInfo *)camel_folder_summary_info_new_from_message (folder->summary, msg);
2313         camel_object_unref (CAMEL_OBJECT (msg));
2314
2315         if ((idate = g_datalist_get_data (&data, "INTERNALDATE")))
2316                 mi->info.date_received = decode_internaldate ((const unsigned char *) idate);
2317
2318         if (mi->info.date_received == -1)
2319                 mi->info.date_received = mi->info.date_sent;
2320
2321         messages->pdata[seq - first] = mi;
2322 }
2323
2324
2325 #define CAMEL_MESSAGE_INFO_HEADERS "DATE FROM TO CC SUBJECT REFERENCES IN-REPLY-TO MESSAGE-ID MIME-VERSION CONTENT-TYPE "
2326
2327 /* FIXME: this needs to be kept in sync with camel-mime-utils.c's list
2328    of mailing-list headers and so might be best if this were
2329    auto-generated? */
2330 #define MAILING_LIST_HEADERS "X-MAILING-LIST X-LOOP LIST-ID LIST-POST MAILING-LIST ORIGINATOR X-LIST SENDER RETURN-PATH X-BEENTHERE "
2331
2332 static void
2333 imap_update_summary (CamelFolder *folder, int exists,
2334                      CamelFolderChangeInfo *changes,
2335                      CamelException *ex)
2336 {
2337         CamelImapStore *store = CAMEL_IMAP_STORE (folder->parent_store);
2338         CamelImapFolder *imap_folder = CAMEL_IMAP_FOLDER (folder);
2339         GPtrArray *fetch_data = NULL, *messages = NULL, *needheaders;
2340         guint32 flags, uidval;
2341         int i, seq, first, size, got;
2342         CamelImapResponseType type;
2343         GString *header_spec = NULL;
2344         CamelImapMessageInfo *mi, *info;
2345         CamelStream *stream;
2346         char *uid, *resp;
2347         GData *data;
2348
2349         if (store->server_level >= IMAP_LEVEL_IMAP4REV1) {
2350                 if (store->headers == IMAP_FETCH_ALL_HEADERS)
2351                         header_spec = g_string_new ("HEADER");
2352                 else {
2353                         gchar *temp;
2354                         header_spec = g_string_new ("HEADER.FIELDS (");
2355                         header_spec = g_string_append (header_spec, CAMEL_MESSAGE_INFO_HEADERS);
2356                         if (store->headers == IMAP_FETCH_MAILING_LIST_HEADERS)
2357                                 header_spec = g_string_append (header_spec, MAILING_LIST_HEADERS);
2358                         if (store->custom_headers)
2359                                 header_spec = g_string_append (header_spec, store->custom_headers);
2360
2361                         temp = g_strdup(header_spec->str);
2362                         temp = g_strstrip (temp);
2363                         header_spec = g_string_new (temp);
2364                         g_free (temp);
2365                         header_spec = g_string_append (header_spec, ")");
2366                 }
2367         } else
2368                 header_spec = g_string_new ("0");
2369
2370         d(printf("Header is : %s", header_spec->str));
2371
2372         /* Figure out if any of the new messages are already cached (which
2373          * may be the case if we're re-syncing after disconnected operation).
2374          * If so, get their UIDs, FLAGS, and SIZEs. If not, get all that
2375          * and ask for the headers too at the same time.
2376          */
2377         seq = camel_folder_summary_count (folder->summary);
2378         first = seq + 1;
2379         if (seq > 0) {
2380                 mi = (CamelImapMessageInfo *)camel_folder_summary_index (folder->summary, seq - 1);
2381                 uidval = strtoul(camel_message_info_uid (mi), NULL, 10);
2382                 camel_message_info_free(&mi->info);
2383         } else
2384                 uidval = 0;
2385
2386         size = (exists - seq) * (IMAP_PRETEND_SIZEOF_FLAGS + IMAP_PRETEND_SIZEOF_SIZE + IMAP_PRETEND_SIZEOF_HEADERS);
2387         got = 0;
2388         if (!camel_imap_command_start (store, folder, ex,
2389                                        "UID FETCH %d:* (FLAGS RFC822.SIZE INTERNALDATE BODY.PEEK[%s])",
2390                                        uidval + 1, header_spec->str)) {
2391                 g_string_free (header_spec, TRUE);
2392                 return;
2393         }
2394         camel_operation_start (NULL, _("Fetching summary information for new messages in %s"), folder->name);
2395
2396         /* Parse the responses. We can't add a message to the summary
2397          * until we've gotten its headers, and there's no guarantee
2398          * the server will send the responses in a useful order...
2399          */
2400         fetch_data = g_ptr_array_new ();
2401         messages = g_ptr_array_new ();
2402         while ((type = camel_imap_command_response (store, &resp, ex)) ==
2403                CAMEL_IMAP_RESPONSE_UNTAGGED) {
2404                 data = parse_fetch_response (imap_folder, resp);
2405                 g_free (resp);
2406                 if (!data)
2407                         continue;
2408
2409                 seq = GPOINTER_TO_INT (g_datalist_get_data (&data, "SEQUENCE"));
2410                 if (seq < first) {
2411                         g_datalist_clear (&data);
2412                         continue;
2413                 }
2414
2415                 if (g_datalist_get_data (&data, "FLAGS"))
2416                         got += IMAP_PRETEND_SIZEOF_FLAGS;
2417                 if (g_datalist_get_data (&data, "RFC822.SIZE"))
2418                         got += IMAP_PRETEND_SIZEOF_SIZE;
2419                 stream = g_datalist_get_data (&data, "BODY_PART_STREAM");
2420                 if (stream) {
2421                         got += IMAP_PRETEND_SIZEOF_HEADERS;
2422
2423                         /* Use the stream now so we don't tie up many
2424                          * many fds if we're fetching many many messages.
2425                          */
2426                         add_message_from_data (folder, messages, first, data);
2427                         g_datalist_set_data (&data, "BODY_PART_STREAM", NULL);
2428                 }
2429
2430                 camel_operation_progress (NULL, got * 100 / size);
2431                 g_ptr_array_add (fetch_data, data);
2432         }
2433         camel_operation_end (NULL);
2434
2435         if (type == CAMEL_IMAP_RESPONSE_ERROR)
2436                 goto lose;
2437
2438         /* Free the final tagged response */
2439         g_free (resp);
2440
2441         /* Figure out which headers we still need to fetch. */
2442         needheaders = g_ptr_array_new ();
2443         size = got = 0;
2444         for (i = 0; i < fetch_data->len; i++) {
2445                 data = fetch_data->pdata[i];
2446                 if (g_datalist_get_data (&data, "BODY_PART_LEN"))
2447                         continue;
2448
2449                 uid = g_datalist_get_data (&data, "UID");
2450                 if (uid) {
2451                         g_ptr_array_add (needheaders, uid);
2452                         size += IMAP_PRETEND_SIZEOF_HEADERS;
2453                 }
2454         }
2455
2456         /* And fetch them */
2457         if (needheaders->len) {
2458                 char *uidset;
2459                 int uid = 0;
2460
2461                 qsort (needheaders->pdata, needheaders->len,
2462                        sizeof (void *), uid_compar);
2463
2464                 camel_operation_start (NULL, _("Fetching summary information for new messages in %s"), folder->name);
2465
2466                 while (uid < needheaders->len) {
2467                         uidset = imap_uid_array_to_set (folder->summary, needheaders, uid, UID_SET_LIMIT, &uid);
2468                         if (!camel_imap_command_start (store, folder, ex,
2469                                                        "UID FETCH %s BODY.PEEK[%s]",
2470                                                        uidset, header_spec->str)) {
2471                                 g_ptr_array_free (needheaders, TRUE);
2472                                 camel_operation_end (NULL);
2473                                 g_free (uidset);
2474                                 g_string_free (header_spec, TRUE);
2475                                 goto lose;
2476                         }
2477                         g_free (uidset);
2478
2479                         while ((type = camel_imap_command_response (store, &resp, ex))
2480                                == CAMEL_IMAP_RESPONSE_UNTAGGED) {
2481                                 data = parse_fetch_response (imap_folder, resp);
2482                                 g_free (resp);
2483                                 if (!data)
2484                                         continue;
2485
2486                                 stream = g_datalist_get_data (&data, "BODY_PART_STREAM");
2487                                 if (stream) {
2488                                         add_message_from_data (folder, messages, first, data);
2489                                         got += IMAP_PRETEND_SIZEOF_HEADERS;
2490                                         camel_operation_progress (NULL, got * 100 / size);
2491                                 }
2492                                 g_datalist_clear (&data);
2493                         }
2494
2495                         if (type == CAMEL_IMAP_RESPONSE_ERROR) {
2496                                 g_ptr_array_free (needheaders, TRUE);
2497                                 camel_operation_end (NULL);
2498                                 goto lose;
2499                         }
2500                 }
2501                 g_string_free (header_spec, TRUE);
2502                 g_ptr_array_free (needheaders, TRUE);
2503                 camel_operation_end (NULL);
2504         }
2505
2506         /* Now finish up summary entries (fix UIDs, set flags and size) */
2507         for (i = 0; i < fetch_data->len; i++) {
2508                 data = fetch_data->pdata[i];
2509
2510                 seq = GPOINTER_TO_INT (g_datalist_get_data (&data, "SEQUENCE"));
2511                 if (seq >= first + messages->len) {
2512                         g_datalist_clear (&data);
2513                         continue;
2514                 }
2515
2516                 mi = messages->pdata[seq - first];
2517                 if (mi == NULL) {
2518                         CamelMessageInfo *pmi = NULL;
2519                         int j;
2520
2521                         /* This is a kludge around a bug in Exchange
2522                          * 5.5 that sometimes claims multiple messages
2523                          * have the same UID. See bug #17694 for
2524                          * details. The "solution" is to create a fake
2525                          * message-info with the same details as the
2526                          * previously valid message. Yes, the user
2527                          * will have a clone in his/her message-list,
2528                          * but at least we don't crash.
2529                          */
2530
2531                         /* find the previous valid message info */
2532                         for (j = seq - first - 1; j >= 0; j--) {
2533                                 pmi = messages->pdata[j];
2534                                 if (pmi != NULL)
2535                                         break;
2536                         }
2537
2538                         if (pmi == NULL) {
2539                                 /* Server response is *really* fucked up,
2540                                    I guess we just pretend it never happened? */
2541                                 continue;
2542                         }
2543
2544                         mi = (CamelImapMessageInfo *)camel_message_info_clone(pmi);
2545                 }
2546
2547                 uid = g_datalist_get_data (&data, "UID");
2548                 if (uid)
2549                         mi->info.uid = g_strdup (uid);
2550                 flags = GPOINTER_TO_INT (g_datalist_get_data (&data, "FLAGS"));
2551                 if (flags) {
2552                         ((CamelImapMessageInfo *)mi)->server_flags = flags;
2553                         /* "or" them in with the existing flags that may
2554                          * have been set by summary_info_new_from_message.
2555                          */
2556                         mi->info.flags |= flags;
2557                         flags_to_label(folder, mi);
2558                 }
2559                 size = GPOINTER_TO_INT (g_datalist_get_data (&data, "RFC822.SIZE"));
2560                 if (size)
2561                         mi->info.size = size;
2562
2563                 g_datalist_clear (&data);
2564         }
2565         g_ptr_array_free (fetch_data, TRUE);
2566
2567         /* And add the entries to the summary, etc. */
2568         for (i = 0; i < messages->len; i++) {
2569                 mi = messages->pdata[i];
2570                 if (!mi) {
2571                         g_warning ("No information for message %d", i + first);
2572                         camel_exception_setv (ex, CAMEL_EXCEPTION_SYSTEM,
2573                                               _("Incomplete server response: no information provided for message %d"),
2574                                               i + first);
2575                         break;
2576                 }
2577                 uid = (char *)camel_message_info_uid(mi);
2578                 if (uid[0] == 0) {
2579                         g_warning("Server provided no uid: message %d", i + first);
2580                         camel_exception_setv (ex, CAMEL_EXCEPTION_SYSTEM,
2581                                               _("Incomplete server response: no UID provided for message %d"),
2582                                               i + first);
2583                         break;
2584                 }
2585                 info = (CamelImapMessageInfo *)camel_folder_summary_uid(folder->summary, uid);
2586                 if (info) {
2587                         for (seq = 0; seq < camel_folder_summary_count (folder->summary); seq++) {
2588                                 if (folder->summary->messages->pdata[seq] == info)
2589                                         break;
2590                         }
2591
2592                         g_warning("Message already present? %s", camel_message_info_uid(mi));
2593                         camel_exception_setv (ex, CAMEL_EXCEPTION_SYSTEM,
2594                                               _("Unexpected server response: Identical UIDs provided for messages %d and %d"),
2595                                               seq + 1, i + first);
2596
2597                         camel_message_info_free(&info->info);
2598                         break;
2599                 }
2600
2601                 camel_folder_summary_add (folder->summary, (CamelMessageInfo *)mi);
2602                 camel_folder_change_info_add_uid (changes, camel_message_info_uid (mi));
2603
2604                 if ((mi->info.flags & CAMEL_IMAP_MESSAGE_RECENT))
2605                         camel_folder_change_info_recent_uid(changes, camel_message_info_uid (mi));
2606         }
2607
2608         for ( ; i < messages->len; i++) {
2609                 if ((mi = messages->pdata[i]))
2610                         camel_message_info_free(&mi->info);
2611         }
2612
2613         g_ptr_array_free (messages, TRUE);
2614
2615         return;
2616
2617  lose:
2618         if (fetch_data) {
2619                 for (i = 0; i < fetch_data->len; i++) {
2620                         data = fetch_data->pdata[i];
2621                         g_datalist_clear (&data);
2622                 }
2623                 g_ptr_array_free (fetch_data, TRUE);
2624         }
2625         if (messages) {
2626                 for (i = 0; i < messages->len; i++) {
2627                         if (messages->pdata[i])
2628                                 camel_message_info_free(messages->pdata[i]);
2629                 }
2630                 g_ptr_array_free (messages, TRUE);
2631         }
2632 }
2633
2634 /* Called with the store's connect_lock locked */
2635 void
2636 camel_imap_folder_changed (CamelFolder *folder, int exists,
2637                            GArray *expunged, CamelException *ex)
2638 {
2639         CamelImapFolder *imap_folder = CAMEL_IMAP_FOLDER (folder);
2640         CamelFolderChangeInfo *changes;
2641         CamelMessageInfo *info;
2642         int len;
2643
2644         changes = camel_folder_change_info_new ();
2645         if (expunged) {
2646                 int i, id;
2647
2648                 for (i = 0; i < expunged->len; i++) {
2649                         id = g_array_index (expunged, int, i);
2650                         info = camel_folder_summary_index (folder->summary, id - 1);
2651                         if (info == NULL) {
2652                                 /* FIXME: danw: does this mean that the summary is corrupt? */
2653                                 /* I guess a message that we never retrieved got expunged? */
2654                                 continue;
2655                         }
2656
2657                         camel_folder_change_info_remove_uid (changes, camel_message_info_uid (info));
2658                         CAMEL_IMAP_FOLDER_REC_LOCK (imap_folder, cache_lock);
2659                         camel_imap_message_cache_remove (imap_folder->cache, camel_message_info_uid (info));
2660                         CAMEL_IMAP_FOLDER_REC_UNLOCK (imap_folder, cache_lock);
2661                         camel_folder_summary_remove (folder->summary, info);
2662                         camel_message_info_free(info);
2663                 }
2664         }
2665
2666         len = camel_folder_summary_count (folder->summary);
2667         if (exists > len)
2668                 imap_update_summary (folder, exists, changes, ex);
2669
2670         if (camel_folder_change_info_changed (changes))
2671                 camel_object_trigger_event (CAMEL_OBJECT (folder), "folder_changed", changes);
2672
2673         camel_folder_change_info_free (changes);
2674         camel_folder_summary_save (folder->summary);
2675 }
2676
2677 static void
2678 imap_thaw (CamelFolder *folder)
2679 {
2680         CamelImapFolder *imap_folder;
2681
2682         CAMEL_FOLDER_CLASS (disco_folder_class)->thaw (folder);
2683         if (camel_folder_is_frozen (folder))
2684                 return;
2685
2686         imap_folder = CAMEL_IMAP_FOLDER (folder);
2687         if (imap_folder->need_refresh) {
2688                 imap_folder->need_refresh = FALSE;
2689                 imap_refresh_info (folder, NULL);
2690         }
2691 }
2692
2693
2694 CamelStream *
2695 camel_imap_folder_fetch_data (CamelImapFolder *imap_folder, const char *uid,
2696                               const char *section_text, gboolean cache_only,
2697                               CamelException *ex)
2698 {
2699         CamelFolder *folder = CAMEL_FOLDER (imap_folder);
2700         CamelImapStore *store = CAMEL_IMAP_STORE (folder->parent_store);
2701         CamelImapResponse *response;
2702         CamelStream *stream;
2703         GData *fetch_data;
2704         char *found_uid;
2705         int i;
2706
2707         /* EXPUNGE responses have to modify the cache, which means
2708          * they have to grab the cache_lock while holding the
2709          * connect_lock.
2710
2711          * Because getting the service lock may cause MUCH unecessary
2712          * delay when we already have the data locally, we do the
2713          * locking separately.  This could cause a race
2714          * getting the same data from the cache, but that is only
2715          * an inefficiency, and bad luck.
2716          */
2717         CAMEL_IMAP_FOLDER_REC_LOCK (imap_folder, cache_lock);
2718         stream = camel_imap_message_cache_get (imap_folder->cache, uid, section_text, ex);
2719         if (!stream && (!strcmp (section_text, "HEADER") || !strcmp (section_text, "0"))) {
2720                 camel_exception_clear (ex);
2721                 stream = camel_imap_message_cache_get (imap_folder->cache, uid, "", ex);
2722         }
2723         CAMEL_IMAP_FOLDER_REC_UNLOCK (imap_folder, cache_lock);
2724
2725         if (stream || cache_only)
2726                 return stream;
2727
2728         camel_exception_clear(ex);
2729
2730         CAMEL_SERVICE_REC_LOCK (store, connect_lock);
2731         CAMEL_IMAP_FOLDER_REC_LOCK (imap_folder, cache_lock);
2732
2733         if (!camel_imap_store_connected(store, ex)) {
2734                 camel_exception_set (ex, CAMEL_EXCEPTION_SERVICE_UNAVAILABLE,
2735                                      _("This message is not currently available"));
2736                 CAMEL_IMAP_FOLDER_REC_UNLOCK (imap_folder, cache_lock);
2737                 CAMEL_SERVICE_REC_UNLOCK (store, connect_lock);
2738                 return NULL;
2739         }
2740
2741         camel_exception_clear (ex);
2742         if (store->server_level < IMAP_LEVEL_IMAP4REV1 && !*section_text) {
2743                 response = camel_imap_command (store, folder, ex,
2744                                                "UID FETCH %s RFC822.PEEK",
2745                                                uid);
2746         } else {
2747                 response = camel_imap_command (store, folder, ex,
2748                                                "UID FETCH %s BODY.PEEK[%s]",
2749                                                uid, section_text);
2750         }
2751         /* We won't need the connect_lock again after this. */
2752         CAMEL_SERVICE_REC_UNLOCK (store, connect_lock);
2753
2754         if (!response) {
2755                 CAMEL_IMAP_FOLDER_REC_UNLOCK (imap_folder, cache_lock);
2756                 return NULL;
2757         }
2758
2759         for (i = 0; i < response->untagged->len; i++) {
2760                 fetch_data = parse_fetch_response (imap_folder, response->untagged->pdata[i]);
2761                 found_uid = g_datalist_get_data (&fetch_data, "UID");
2762                 stream = g_datalist_get_data (&fetch_data, "BODY_PART_STREAM");
2763                 if (found_uid && stream && !strcmp (uid, found_uid))
2764                         break;
2765
2766                 g_datalist_clear (&fetch_data);
2767                 stream = NULL;
2768         }
2769         camel_imap_response_free (store, response);
2770         CAMEL_IMAP_FOLDER_REC_UNLOCK (imap_folder, cache_lock);
2771         if (!stream) {
2772                 camel_exception_setv (ex, CAMEL_EXCEPTION_SERVICE_UNAVAILABLE,
2773                                       _("Could not find message body in FETCH response."));
2774         } else {
2775                 camel_object_ref (CAMEL_OBJECT (stream));
2776                 g_datalist_clear (&fetch_data);
2777         }
2778
2779         return stream;
2780 }
2781
2782 static GData *
2783 parse_fetch_response (CamelImapFolder *imap_folder, char *response)
2784 {
2785         GData *data = NULL;
2786         char *start, *part_spec = NULL, *body = NULL, *uid = NULL, *idate = NULL;
2787         gboolean cache_header = TRUE, header = FALSE;
2788         size_t body_len = 0;
2789
2790         if (*response != '(') {
2791                 long seq;
2792
2793                 if (*response != '*' || *(response + 1) != ' ')
2794                         return NULL;
2795                 seq = strtoul (response + 2, &response, 10);
2796                 if (seq == 0)
2797                         return NULL;
2798                 if (g_ascii_strncasecmp (response, " FETCH (", 8) != 0)
2799                         return NULL;
2800                 response += 7;
2801
2802                 g_datalist_set_data (&data, "SEQUENCE", GINT_TO_POINTER (seq));
2803         }
2804
2805         do {
2806                 /* Skip the initial '(' or the ' ' between elements */
2807                 response++;
2808
2809                 if (!g_ascii_strncasecmp (response, "FLAGS ", 6)) {
2810                         guint32 flags;
2811
2812                         response += 6;
2813                         /* FIXME user flags */
2814                         flags = imap_parse_flag_list (&response);
2815
2816                         g_datalist_set_data (&data, "FLAGS", GUINT_TO_POINTER (flags));
2817                 } else if (!g_ascii_strncasecmp (response, "RFC822.SIZE ", 12)) {
2818                         unsigned long size;
2819
2820                         response += 12;
2821                         size = strtoul (response, &response, 10);
2822                         g_datalist_set_data (&data, "RFC822.SIZE", GUINT_TO_POINTER (size));
2823                 } else if (!g_ascii_strncasecmp (response, "BODY[", 5) ||
2824                            !g_ascii_strncasecmp (response, "RFC822 ", 7)) {
2825                         char *p;
2826
2827                         if (*response == 'B') {
2828                                 response += 5;
2829
2830                                 /* HEADER], HEADER.FIELDS (...)], or 0] */
2831                                 if (!g_ascii_strncasecmp (response, "HEADER", 6)) {
2832                                         header = TRUE;
2833                                         if (!g_ascii_strncasecmp (response + 6, ".FIELDS", 7))
2834                                                 cache_header = FALSE;
2835                                 } else if (!g_ascii_strncasecmp (response, "0]", 2))
2836                                         header = TRUE;
2837
2838                                 p = strchr (response, ']');
2839                                 if (!p || *(p + 1) != ' ')
2840                                         break;
2841
2842                                 if (cache_header)
2843                                         part_spec = g_strndup (response, p - response);
2844                                 else
2845                                         part_spec = g_strdup ("HEADER.FIELDS");
2846
2847                                 response = p + 2;
2848                         } else {
2849                                 part_spec = g_strdup ("");
2850                                 response += 7;
2851
2852                                 if (!g_ascii_strncasecmp (response, "HEADER", 6))
2853                                         header = TRUE;
2854                         }
2855
2856                         body = imap_parse_nstring ((const char **) &response, &body_len);
2857                         if (!response) {
2858                                 g_free (part_spec);
2859                                 break;
2860                         }
2861
2862                         if (!body)
2863                                 body = g_strdup ("");
2864                         g_datalist_set_data_full (&data, "BODY_PART_SPEC", part_spec, g_free);
2865                         g_datalist_set_data_full (&data, "BODY_PART_DATA", body, g_free);
2866                         g_datalist_set_data (&data, "BODY_PART_LEN", GINT_TO_POINTER (body_len));
2867                 } else if (!g_ascii_strncasecmp (response, "BODY ", 5) ||
2868                            !g_ascii_strncasecmp (response, "BODYSTRUCTURE ", 14)) {
2869                         response = strchr (response, ' ') + 1;
2870                         start = response;
2871                         imap_skip_list ((const char **) &response);
2872                         if (response && (response != start)) {
2873                                 /* To handle IMAP Server brokenness, Returning empty body, etc. See #355640 */
2874                                 g_datalist_set_data_full (&data, "BODY", g_strndup (start, response - start), g_free);
2875                         }
2876                 } else if (!g_ascii_strncasecmp (response, "UID ", 4)) {
2877                         int len;
2878
2879                         len = strcspn (response + 4, " )");
2880                         uid = g_strndup (response + 4, len);
2881                         g_datalist_set_data_full (&data, "UID", uid, g_free);
2882                         response += 4 + len;
2883                 } else if (!g_ascii_strncasecmp (response, "INTERNALDATE ", 13)) {
2884                         int len;
2885
2886                         response += 13;
2887                         if (*response == '"') {
2888                                 response++;
2889                                 len = strcspn (response, "\"");
2890                                 idate = g_strndup (response, len);
2891                                 g_datalist_set_data_full (&data, "INTERNALDATE", idate, g_free);
2892                                 response += len + 1;
2893                         }
2894                 } else {
2895                         g_warning ("Unexpected FETCH response from server: (%s", response);
2896                         break;
2897                 }
2898         } while (response && *response != ')');
2899
2900         if (!response || *response != ')') {
2901                 g_datalist_clear (&data);
2902                 return NULL;
2903         }
2904
2905         if (uid && body) {
2906                 CamelStream *stream;
2907
2908                 if (header && !cache_header) {
2909                         stream = camel_stream_mem_new_with_buffer (body, body_len);
2910                 } else {
2911                         CAMEL_IMAP_FOLDER_REC_LOCK (imap_folder, cache_lock);
2912                         stream = camel_imap_message_cache_insert (imap_folder->cache,
2913                                                                   uid, part_spec,
2914                                                                   body, body_len, NULL);
2915                         CAMEL_IMAP_FOLDER_REC_UNLOCK (imap_folder, cache_lock);
2916                         if (stream == NULL)
2917                                 stream = camel_stream_mem_new_with_buffer (body, body_len);
2918                 }
2919
2920                 if (stream)
2921                         g_datalist_set_data_full (&data, "BODY_PART_STREAM", stream,
2922                                                   (GDestroyNotify) camel_object_unref);
2923         }
2924
2925         return data;
2926 }
2927