camel/camel-imapx-stream.c

   1 /* -*- Mode: C; tab-width: 8; indent-tabs-mode: t; c-basic-offset: 8; fill-column: 160 -*-
   2  *
   3  * Author:
   4  *  Michael Zucchi <notzed@ximian.com>
   5  *
   6  * Copyright (C) 1999-2008 Novell, Inc. (www.novell.com)
   7  *
   8  * This library is free software; you can redistribute it and/or
   9  * modify it under the terms of version 2 of the GNU Lesser General Public
  10  * License as published by the Free Software Foundation.
  11  *
  12  * This program is distributed in the hope that it will be useful,
  13  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  14  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  15  * General Public License for more details.
  16  *
  17  * You should have received a copy of the GNU Lesser General Public
  18  * License along with this library; if not, write to the
  19  * Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
  20  * Boston, MA 02110-1301, USA.
  21  */
  22
  23 #ifdef HAVE_CONFIG_H
  24 #include <config.h>
  25 #endif
  26
  27 #include <string.h>
  28 #include <stdio.h>
  29 #include <ctype.h>
  30 #include <errno.h>
  31
  32 #include <glib/gi18n-lib.h>
  33
  34 #include <camel/camel-stream-mem.h>
  35
  36 #include "camel-imapx-utils.h"
  37 #include "camel-imapx-stream.h"
  38
  39 #define t(...) camel_imapx_debug(token, __VA_ARGS__)
  40 #define io(...) camel_imapx_debug(io, __VA_ARGS__)
  41
  42 G_DEFINE_TYPE (CamelIMAPXStream, camel_imapx_stream, CAMEL_TYPE_STREAM)
  43
  44 static gint
  45 imapx_stream_fill (CamelIMAPXStream *is,
  46                    GCancellable *cancellable,
  47                    GError **error)
  48 {
  49         gint left = 0;
  50
  51         if (is->source) {
  52                 left = is->end - is->ptr;
  53                 memcpy (is->buf, is->ptr, left);
  54                 is->end = is->buf + left;
  55                 is->ptr = is->buf;
  56                 left = camel_stream_read (
  57                         is->source, (gchar *) is->end,
  58                         is->bufsize - (is->end - is->buf),
  59                         cancellable, error);
  60                 if (left > 0) {
  61                         is->end += left;
  62                         io(is->tagprefix, "camel_imapx_read: buffer is '%.*s'\n", (gint)(is->end - is->ptr), is->ptr);
  63                         return is->end - is->ptr;
  64                 } else {
  65                         io(is->tagprefix, "camel_imapx_read: -1\n");
  66                         /* If returning zero, camel_stream_read() doesn't consider
  67                          * that to be an error. But we do -- we should only be here
  68                          * if we *know* there are data to receive. So set the error
  69                          * accordingly */
  70                         if (!left)
  71                                 g_set_error (error, CAMEL_ERROR, CAMEL_ERROR_GENERIC,
  72                                             _("Source stream returned no data"));
  73                         return -1;
  74                 }
  75         }
  76
  77         io(is->tagprefix, "camel_imapx_read: -1\n");
  78
  79         g_set_error (
  80                 error, CAMEL_ERROR, CAMEL_ERROR_GENERIC,
  81                 _("Source stream unavailable"));
  82
  83         return -1;
  84 }
  85
  86 static void
  87 imapx_stream_dispose (GObject *object)
  88 {
  89         CamelIMAPXStream *stream = CAMEL_IMAPX_STREAM (object);
  90
  91         if (stream->source != NULL) {
  92                 g_object_unref (stream->source);
  93                 stream->source = NULL;
  94         }
  95
  96         /* Chain up to parent's dispose() method. */
  97         G_OBJECT_CLASS (camel_imapx_stream_parent_class)->dispose (object);
  98 }
  99
 100 static void
 101 imapx_stream_finalize (GObject *object)
 102 {
 103         CamelIMAPXStream *stream = CAMEL_IMAPX_STREAM (object);
 104
 105         g_free (stream->buf);
 106         g_free (stream->tokenbuf);
 107
 108         /* Chain up to parent's finalize() method. */
 109         G_OBJECT_CLASS (camel_imapx_stream_parent_class)->finalize (object);
 110 }
 111
 112 static gssize
 113 imapx_stream_read (CamelStream *stream,
 114                    gchar *buffer,
 115                    gsize n,
 116                    GCancellable *cancellable,
 117                    GError **error)
 118 {
 119         CamelIMAPXStream *is = (CamelIMAPXStream *) stream;
 120         gssize max;
 121
 122         if (is->literal == 0 || n == 0)
 123                 return 0;
 124
 125         max = is->end - is->ptr;
 126         if (max > 0) {
 127                 max = MIN (max, is->literal);
 128                 max = MIN (max, n);
 129                 memcpy (buffer, is->ptr, max);
 130                 is->ptr += max;
 131         } else {
 132                 max = MIN (is->literal, n);
 133                 max = camel_stream_read (is->source, buffer, max, cancellable, error);
 134                 if (max <= 0)
 135                         return max;
 136         }
 137
 138         io(is->tagprefix, "camel_imapx_read(literal): '%.*s'\n", (gint)max, buffer);
 139
 140         is->literal -= max;
 141
 142         return max;
 143 }
 144
 145 static gssize
 146 imapx_stream_write (CamelStream *stream,
 147                     const gchar *buffer,
 148                     gsize n,
 149                     GCancellable *cancellable,
 150                     GError **error)
 151 {
 152         CamelIMAPXStream *is = (CamelIMAPXStream *) stream;
 153
 154         if (g_strstr_len (buffer, n, "LOGIN")) {
 155                 io(is->tagprefix, "camel_imapx_write: 'LOGIN...'\n");
 156         } else {
 157                 io(is->tagprefix, "camel_imapx_write: '%.*s'\n", (gint)n, buffer);
 158         }
 159
 160         return camel_stream_write (is->source, buffer, n, cancellable, error);
 161 }
 162
 163 static gint
 164 imapx_stream_close (CamelStream *stream,
 165                     GCancellable *cancellable,
 166                     GError **error)
 167 {
 168         /* nop? */
 169         return 0;
 170 }
 171
 172 static gint
 173 imapx_stream_flush (CamelStream *stream,
 174                     GCancellable *cancellable,
 175                     GError **error)
 176 {
 177         /* nop? */
 178         return 0;
 179 }
 180
 181 static gboolean
 182 imapx_stream_eos (CamelStream *stream)
 183 {
 184         CamelIMAPXStream *is = (CamelIMAPXStream *) stream;
 185
 186         return is->literal == 0;
 187 }
 188
 189 static void
 190 camel_imapx_stream_class_init (CamelIMAPXStreamClass *class)
 191 {
 192         GObjectClass *object_class;
 193         CamelStreamClass *stream_class;
 194
 195         object_class = G_OBJECT_CLASS (class);
 196         object_class->dispose = imapx_stream_dispose;
 197         object_class->finalize = imapx_stream_finalize;
 198
 199         stream_class = CAMEL_STREAM_CLASS (class);
 200         stream_class->read = imapx_stream_read;
 201         stream_class->write = imapx_stream_write;
 202         stream_class->close = imapx_stream_close;
 203         stream_class->flush = imapx_stream_flush;
 204         stream_class->eos = imapx_stream_eos;
 205 }
 206
 207 static void
 208 camel_imapx_stream_init (CamelIMAPXStream *is)
 209 {
 210         /* +1 is room for appending a 0 if we need to for a token */
 211         is->bufsize = 4096;
 212         is->ptr = is->end = is->buf = g_malloc (is->bufsize + 1);
 213         is->tokenbuf = g_malloc (is->bufsize + 1);
 214 }
 215
 216 static void camel_imapx_stream_grow (CamelIMAPXStream *is, guint len, guchar **bufptr, guchar **tokptr)
 217 {
 218         guchar *oldtok = is->tokenbuf;
 219         guchar *oldbuf = is->buf;
 220
 221         do {
 222                 is->bufsize <<= 1;
 223         } while (is->bufsize <= len);
 224
 225         io(is->tagprefix, "Grow imapx buffers to %d bytes\n", is->bufsize);
 226
 227         is->tokenbuf = g_realloc (is->tokenbuf, is->bufsize + 1);
 228         if (tokptr)
 229                 *tokptr = is->tokenbuf + (*tokptr - oldtok);
 230         if (is->unget)
 231                 is->unget_token = is->tokenbuf + (is->unget_token - oldtok);
 232
 233         //io(is->tagprefix, "buf was %p, ptr %p end %p\n", is->buf, is->ptr, is->end);
 234         is->buf = g_realloc (is->buf, is->bufsize + 1);
 235         is->ptr = is->buf + (is->ptr - oldbuf);
 236         is->end = is->buf + (is->end - oldbuf);
 237         //io(is->tagprefix, "buf now %p, ptr %p end %p\n", is->buf, is->ptr, is->end);
 238         if (bufptr)
 239                 *bufptr = is->buf + (*bufptr - oldbuf);
 240 }
 241
 242 /**
 243  * camel_imapx_stream_new:
 244  *
 245  * Returns a NULL stream.  A null stream is always at eof, and
 246  * always returns success for all reads and writes.
 247  *
 248  * Returns: the stream
 249  **/
 250 CamelStream *
 251 camel_imapx_stream_new (CamelStream *source)
 252 {
 253         CamelIMAPXStream *is;
 254
 255         is = g_object_new (CAMEL_TYPE_IMAPX_STREAM, NULL);
 256         is->source = g_object_ref (source);
 257
 258         return (CamelStream *) is;
 259 }
 260
 261 GQuark
 262 camel_imapx_error_quark (void)
 263 {
 264         static GQuark quark = 0;
 265
 266         if (G_UNLIKELY (quark == 0)) {
 267                 const gchar *string = "camel-imapx-error-quark";
 268                 quark = g_quark_from_static_string (string);
 269         }
 270
 271         return quark;
 272 }
 273
 274 /* Returns if there is any data buffered that is ready for processing */
 275 gint
 276 camel_imapx_stream_buffered (CamelIMAPXStream *is)
 277 {
 278         return is->end - is->ptr;
 279 }
 280
 281 #if 0
 282
 283 static gint
 284 skip_ws (CamelIMAPXStream *is,
 285          guchar *pp,
 286          guchar *pe)
 287 {
 288         register guchar c, *p;
 289         guchar *e;
 290
 291         p = is->ptr;
 292         e = is->end;
 293
 294         do {
 295                 while (p >= e ) {
 296                         is->ptr = p;
 297                         if (imapx_stream_fill (is, NULL) == IMAPX_TOK_ERROR)
 298                                 return IMAPX_TOK_ERROR;
 299                         p = is->ptr;
 300                         e = is->end;
 301                 }
 302                 c = *p++;
 303         } while (c == ' ' || c == '\r');
 304
 305         is->ptr = p;
 306         is->end = e;
 307
 308         return c;
 309 }
 310 #endif
 311
 312 /* FIXME: these should probably handle it themselves,
 313  * and get rid of the token interface? */
 314 gint
 315 camel_imapx_stream_atom (CamelIMAPXStream *is,
 316                          guchar **data,
 317                          guint *lenp,
 318                          GCancellable *cancellable,
 319                          GError **error)
 320 {
 321         guchar *p, c;
 322         GError *local_error = NULL;
 323
 324         /* this is only 'approximate' atom */
 325         switch (camel_imapx_stream_token (is, data, lenp, cancellable, &local_error)) {
 326         case IMAPX_TOK_TOKEN:
 327                 p = *data;
 328                 while ((c = *p))
 329                         *p++ = toupper(c);
 330         case IMAPX_TOK_INT:
 331                 return 0;
 332         case IMAPX_TOK_ERROR:
 333                 if (local_error != NULL)
 334                         g_propagate_error (error, local_error);
 335                 return IMAPX_TOK_ERROR;
 336         default:
 337                 if (local_error == NULL)
 338                         g_set_error (error, CAMEL_IMAPX_ERROR, 1, "expecting atom");
 339                 else
 340                         g_propagate_error (error, local_error);
 341                 io(is->tagprefix, "expecting atom!\n");
 342                 return IMAPX_TOK_PROTOCOL;
 343         }
 344 }
 345
 346 /* gets an atom, a quoted_string, or a literal */
 347 gint
 348 camel_imapx_stream_astring (CamelIMAPXStream *is,
 349                             guchar **data,
 350                             GCancellable *cancellable,
 351                             GError **error)
 352 {
 353         guchar *p, *start;
 354         guint len, inlen;
 355         gint ret;
 356         GError *local_error = NULL;
 357
 358         switch (camel_imapx_stream_token (is, data, &len, cancellable, &local_error)) {
 359         case IMAPX_TOK_TOKEN:
 360         case IMAPX_TOK_INT:
 361         case IMAPX_TOK_STRING:
 362                 return 0;
 363         case IMAPX_TOK_LITERAL:
 364                 if (len >= is->bufsize)
 365                         camel_imapx_stream_grow (is, len, NULL, NULL);
 366                 p = is->tokenbuf;
 367                 camel_imapx_stream_set_literal (is, len);
 368                 do {
 369                         ret = camel_imapx_stream_getl (is, &start, &inlen, cancellable, error);
 370                         if (ret < 0)
 371                                 return ret;
 372                         memcpy (p, start, inlen);
 373                         p += inlen;
 374                 } while (ret > 0);
 375                 *p = 0;
 376                 *data = is->tokenbuf;
 377                 return 0;
 378         case IMAPX_TOK_ERROR:
 379                 /* wont get unless no exception hanlder*/
 380                 if (local_error != NULL)
 381                         g_propagate_error (error, local_error);
 382                 return IMAPX_TOK_ERROR;
 383         default:
 384                 if (local_error == NULL)
 385                         g_set_error (error, CAMEL_IMAPX_ERROR, 1, "expecting astring");
 386                 else
 387                         g_propagate_error (error, local_error);
 388                 io(is->tagprefix, "expecting astring!\n");
 389                 return IMAPX_TOK_PROTOCOL;
 390         }
 391 }
 392
 393 /* check for NIL or (small) quoted_string or literal */
 394 gint
 395 camel_imapx_stream_nstring (CamelIMAPXStream *is,
 396                             guchar **data,
 397                             GCancellable *cancellable,
 398                             GError **error)
 399 {
 400         guchar *p, *start;
 401         guint len, inlen;
 402         gint ret;
 403         GError *local_error = NULL;
 404
 405         switch (camel_imapx_stream_token (is, data, &len, cancellable, &local_error)) {
 406         case IMAPX_TOK_STRING:
 407                 return 0;
 408         case IMAPX_TOK_LITERAL:
 409                 if (len >= is->bufsize)
 410                         camel_imapx_stream_grow (is, len, NULL, NULL);
 411                 p = is->tokenbuf;
 412                 camel_imapx_stream_set_literal (is, len);
 413                 do {
 414                         ret = camel_imapx_stream_getl (is, &start, &inlen, cancellable, error);
 415                         if (ret < 0)
 416                                 return ret;
 417                         memcpy (p, start, inlen);
 418                         p += inlen;
 419                 } while (ret > 0);
 420                 *p = 0;
 421                 *data = is->tokenbuf;
 422                 return 0;
 423         case IMAPX_TOK_TOKEN:
 424                 p = *data;
 425                 if (toupper (p[0]) == 'N' && toupper (p[1]) == 'I' && toupper (p[2]) == 'L' && p[3] == 0) {
 426                         *data = NULL;
 427                         return 0;
 428                 }
 429         default:
 430                 if (local_error == NULL)
 431                         g_set_error (error, CAMEL_IMAPX_ERROR, 1, "expecting nstring");
 432                 else
 433                         g_propagate_error (error, local_error);
 434                 return IMAPX_TOK_PROTOCOL;
 435         case IMAPX_TOK_ERROR:
 436                 /* we'll never get this unless there are no exception  handlers anyway */
 437                 if (local_error != NULL)
 438                         g_propagate_error (error, local_error);
 439                 return IMAPX_TOK_ERROR;
 440
 441         }
 442 }
 443
 444 /* parse an nstring as a stream */
 445 gint
 446 camel_imapx_stream_nstring_stream (CamelIMAPXStream *is,
 447                                    CamelStream **stream,
 448                                    GCancellable *cancellable,
 449                                    GError **error)
 450 /* throws IO,PARSE exception */
 451 {
 452         guchar *token;
 453         guint len;
 454         gint ret = 0;
 455         CamelStream * mem = NULL;
 456         GError *local_error = NULL;
 457
 458         *stream = NULL;
 459
 460         switch (camel_imapx_stream_token (is, &token, &len, cancellable, &local_error)) {
 461                 case IMAPX_TOK_STRING:
 462                         mem = camel_stream_mem_new_with_buffer ((gchar *) token, len);
 463                         *stream = mem;
 464                         break;
 465                 case IMAPX_TOK_LITERAL:
 466                         /* if len is big, we could automatically use a file backing */
 467                         camel_imapx_stream_set_literal (is, len);
 468                         mem = camel_stream_mem_new ();
 469                         if (camel_stream_write_to_stream ((CamelStream *) is, mem, cancellable, error) == -1) {
 470                                 g_object_unref (mem);
 471                                 ret = -1;
 472                                 break;
 473                         }
 474
 475                         g_seekable_seek (
 476                                 G_SEEKABLE (mem), 0,
 477                                 G_SEEK_SET, NULL, NULL);
 478
 479                         *stream = mem;
 480                         break;
 481                 case IMAPX_TOK_TOKEN:
 482                         if (toupper (token[0]) == 'N' && toupper (token[1]) == 'I' && toupper (token[2]) == 'L' && token[3] == 0) {
 483                                 *stream = NULL;
 484                                 break;
 485                         }
 486                 default:
 487                         ret = -1;
 488                         if (local_error == NULL)
 489                                 g_set_error (error, CAMEL_IMAPX_ERROR, 1, "nstring: token not string");
 490                         else
 491                                 g_propagate_error (error, local_error);
 492         }
 493
 494         return ret;
 495 }
 496
 497 guint64
 498 camel_imapx_stream_number (CamelIMAPXStream *is,
 499                            GCancellable *cancellable,
 500                            GError **error)
 501 {
 502         guchar *token;
 503         guint len;
 504         GError *local_error = NULL;
 505
 506         if (camel_imapx_stream_token (is, &token, &len, cancellable, &local_error) != IMAPX_TOK_INT) {
 507                 if (local_error == NULL)
 508                         g_set_error (error, CAMEL_IMAPX_ERROR, 1, "expecting number");
 509                 else
 510                         g_propagate_error (error, local_error);
 511                 return 0;
 512         }
 513
 514         return strtoull ((gchar *) token, 0, 10);
 515 }
 516
 517 gint
 518 camel_imapx_stream_text (CamelIMAPXStream *is,
 519                          guchar **text,
 520                          GCancellable *cancellable,
 521                          GError **error)
 522 {
 523         GByteArray *build = g_byte_array_new ();
 524         guchar *token;
 525         guint len;
 526         gint tok;
 527
 528         while (is->unget > 0) {
 529                 switch (is->unget_tok) {
 530                         case IMAPX_TOK_TOKEN:
 531                         case IMAPX_TOK_STRING:
 532                         case IMAPX_TOK_INT:
 533                                 g_byte_array_append (build, (guint8 *) is->unget_token, is->unget_len);
 534                                 g_byte_array_append(build, (guint8 *) " ", 1);
 535                         default: /* invalid, but we'll ignore */
 536                                 break;
 537                 }
 538                 is->unget--;
 539         }
 540
 541         do {
 542                 tok = camel_imapx_stream_gets (is, &token, &len, cancellable, error);
 543                 if (tok < 0) {
 544                         *text = NULL;
 545                         g_byte_array_free (build, TRUE);
 546                         return -1;
 547                 }
 548                 if (len)
 549                         g_byte_array_append (build, token, len);
 550         } while (tok > 0);
 551
 552         g_byte_array_append(build, (guint8 *) "", 1);
 553         *text = build->data;
 554         g_byte_array_free (build, FALSE);
 555
 556         return 0;
 557 }
 558
 559 /* Get one token from the imap stream */
 560 camel_imapx_token_t
 561 /* throws IO,PARSE exception */
 562 camel_imapx_stream_token (CamelIMAPXStream *is,
 563                           guchar **data,
 564                           guint *len,
 565                           GCancellable *cancellable,
 566                           GError **error)
 567 {
 568         register guchar c, *oe;
 569         guchar *o, *p, *e;
 570         guint literal;
 571         gint digits;
 572
 573         if (is->unget > 0) {
 574                 is->unget--;
 575                 *data = is->unget_token;
 576                 *len = is->unget_len;
 577                 /*printf("token UNGET '%c' %s\n", is->unget_tok, is->unget_token);*/
 578                 return is->unget_tok;
 579         }
 580
 581         if (is->literal > 0)
 582                 g_warning("stream_token called with literal %d", is->literal);
 583
 584         p = is->ptr;
 585         e = is->end;
 586
 587         /* skip whitespace/prefill buffer */
 588         do {
 589                 while (p >= e ) {
 590                         is->ptr = p;
 591                         if (imapx_stream_fill (is, cancellable, error) == IMAPX_TOK_ERROR)
 592                                 return IMAPX_TOK_ERROR;
 593                         p = is->ptr;
 594                         e = is->end;
 595                 }
 596                 c = *p++;
 597         } while (c == ' ' || c == '\r');
 598
 599         /*strchr("\n*()[]+", c)*/
 600         if (imapx_is_token_char (c)) {
 601                 is->ptr = p;
 602                 t(is->tagprefix, "token '%c'\n", c);
 603                 return c;
 604         } else if (c == '{') {
 605                 literal = 0;
 606                 *data = p;
 607                 while (1) {
 608                         while (p < e) {
 609                                 c = *p++;
 610                                 if (isdigit (c) && literal < (UINT_MAX / 10)) {
 611                                         literal = literal * 10 + (c - '0');
 612                                 } else if (c == '}') {
 613                                         while (1) {
 614                                                 while (p < e) {
 615                                                         c = *p++;
 616                                                         if (c == '\n') {
 617                                                                 *len = literal;
 618                                                                 is->ptr = p;
 619                                                                 is->literal = literal;
 620                                                                 t(is->tagprefix, "token LITERAL %d\n", literal);
 621                                                                 return IMAPX_TOK_LITERAL;
 622                                                         }
 623                                                 }
 624                                                 is->ptr = p;
 625                                                 if (imapx_stream_fill (is, cancellable, error) == IMAPX_TOK_ERROR)
 626                                                         return IMAPX_TOK_ERROR;
 627                                                 p = is->ptr;
 628                                                 e = is->end;
 629                                         }
 630                                 } else {
 631                                         if (isdigit (c)) {
 632                                                 io(is->tagprefix, "Protocol error: literal too big\n");
 633                                         } else {
 634                                                 io(is->tagprefix, "Protocol error: literal contains invalid gchar %02x '%c'\n", c, isprint(c)?c:c);
 635                                         }
 636                                         goto protocol_error;
 637                                 }
 638                         }
 639                         is->ptr = p;
 640                         if (imapx_stream_fill (is, cancellable, error) == IMAPX_TOK_ERROR)
 641                                 return IMAPX_TOK_ERROR;
 642                         p = is->ptr;
 643                         e = is->end;
 644                 }
 645         } else if (c == '"') {
 646                 o = is->tokenbuf;
 647                 oe = is->tokenbuf + is->bufsize - 1;
 648                 while (1) {
 649                         while (p < e) {
 650                                 c = *p++;
 651                                 if (c == '\\') {
 652                                         while (p >= e) {
 653                                                 is->ptr = p;
 654                                                 if (imapx_stream_fill (is, cancellable, error) == IMAPX_TOK_ERROR)
 655                                                         return IMAPX_TOK_ERROR;
 656                                                 p = is->ptr;
 657                                                 e = is->end;
 658                                         }
 659                                         c = *p++;
 660                                 } else if (c == '\"') {
 661                                         is->ptr = p;
 662                                         *o = 0;
 663                                         *data = is->tokenbuf;
 664                                         *len = o - is->tokenbuf;
 665                                         t(is->tagprefix, "token STRING '%s'\n", is->tokenbuf);
 666                                         return IMAPX_TOK_STRING;
 667                                 }
 668                                 if (c == '\n' || c == '\r') {
 669                                         io(is->tagprefix, "Protocol error: truncated string\n");
 670                                         goto protocol_error;
 671                                 }
 672                                 if (o >= oe) {
 673                                         camel_imapx_stream_grow (is, 0, &p, &o);
 674                                         oe = is->tokenbuf + is->bufsize - 1;
 675                                         e = is->end;
 676                                 }
 677                                 *o++ = c;
 678                         }
 679                         is->ptr = p;
 680                         if (imapx_stream_fill (is, cancellable, error) == IMAPX_TOK_ERROR)
 681                                 return IMAPX_TOK_ERROR;
 682                         p = is->ptr;
 683                         e = is->end;
 684                 }
 685         } else {
 686                 o = is->tokenbuf;
 687                 oe = is->tokenbuf + is->bufsize - 1;
 688                 digits = isdigit (c);
 689                 *o++ = c;
 690                 while (1) {
 691                         while (p < e) {
 692                                 c = *p++;
 693                                 /*if (strchr(" \r\n*()[]+", c) != NULL) {*/
 694                                 if (imapx_is_notid_char (c)) {
 695                                         if (c == ' ' || c == '\r')
 696                                                 is->ptr = p;
 697                                         else
 698                                                 is->ptr = p - 1;
 699                                         *o = 0;
 700                                         *data = is->tokenbuf;
 701                                         *len = o - is->tokenbuf;
 702                                         t(is->tagprefix, "token TOKEN '%s'\n", is->tokenbuf);
 703                                         return digits ? IMAPX_TOK_INT : IMAPX_TOK_TOKEN;
 704                                 }
 705
 706                                 if (o >= oe) {
 707                                         camel_imapx_stream_grow (is, 0, &p, &o);
 708                                         oe = is->tokenbuf + is->bufsize - 1;
 709                                         e = is->end;
 710                                 }
 711                                 digits &= isdigit (c);
 712                                 *o++ = c;
 713                         }
 714                         is->ptr = p;
 715                         if (imapx_stream_fill (is, cancellable, error) == IMAPX_TOK_ERROR)
 716                                 return IMAPX_TOK_ERROR;
 717                         p = is->ptr;
 718                         e = is->end;
 719                 }
 720         }
 721
 722         /* Protocol error, skip until next lf? */
 723 protocol_error:
 724         io(is->tagprefix, "Got protocol error\n");
 725
 726         if (c == '\n')
 727                 is->ptr = p - 1;
 728         else
 729                 is->ptr = p;
 730
 731         g_set_error (error, CAMEL_IMAPX_ERROR, 1, "protocol error");
 732         return IMAPX_TOK_PROTOCOL;
 733 }
 734
 735 void
 736 camel_imapx_stream_ungettoken (CamelIMAPXStream *is,
 737                                camel_imapx_token_t tok,
 738                                guchar *token,
 739                                guint len)
 740 {
 741         /*printf("ungettoken: '%c' '%s'\n", tok, token);*/
 742         is->unget_tok = tok;
 743         is->unget_token = token;
 744         is->unget_len = len;
 745         is->unget++;
 746 }
 747
 748 /* returns -1 on error, 0 if last lot of data, >0 if more remaining */
 749 gint
 750 camel_imapx_stream_gets (CamelIMAPXStream *is,
 751                          guchar **start,
 752                          guint *len,
 753                          GCancellable *cancellable,
 754                          GError **error)
 755 {
 756         gint max;
 757         guchar *end;
 758
 759         *len = 0;
 760
 761         max = is->end - is->ptr;
 762         if (max == 0) {
 763                 max = imapx_stream_fill (is, cancellable, error);
 764                 if (max <= 0)
 765                         return max;
 766         }
 767
 768         *start = is->ptr;
 769         end = memchr (is->ptr, '\n', max);
 770         if (end)
 771                 max = (end - is->ptr) + 1;
 772         *start = is->ptr;
 773         *len = max;
 774         is->ptr += max;
 775
 776         return end == NULL ? 1 : 0;
 777 }
 778
 779 void camel_imapx_stream_set_literal (CamelIMAPXStream *is, guint literal)
 780 {
 781         is->literal = literal;
 782 }
 783
 784 /* returns -1 on erorr, 0 if last data, >0 if more data left */
 785 gint
 786 camel_imapx_stream_getl (CamelIMAPXStream *is,
 787                          guchar **start,
 788                          guint *len,
 789                          GCancellable *cancellable,
 790                          GError **error)
 791 {
 792         gint max;
 793
 794         *len = 0;
 795
 796         if (is->literal > 0) {
 797                 max = is->end - is->ptr;
 798                 if (max == 0) {
 799                         max = imapx_stream_fill (is, cancellable, error);
 800                         if (max <= 0)
 801                                 return max;
 802                 }
 803
 804                 max = MIN (max, is->literal);
 805                 *start = is->ptr;
 806                 *len = max;
 807                 is->ptr += max;
 808                 is->literal -= max;
 809         }
 810
 811         if (is->literal > 0)
 812                 return 1;
 813
 814         return 0;
 815 }
 816
 817 /* skip the rest of the line of tokens */
 818 gint
 819 camel_imapx_stream_skip (CamelIMAPXStream *is,
 820                          GCancellable *cancellable,
 821                          GError **error)
 822 {
 823         gint tok;
 824         guchar *token;
 825         guint len;
 826
 827         do {
 828                 tok = camel_imapx_stream_token (is, &token, &len, cancellable, error);
 829                 if (tok == IMAPX_TOK_LITERAL) {
 830                         camel_imapx_stream_set_literal (is, len);
 831                         while ((tok = camel_imapx_stream_getl (is, &token, &len, cancellable, error)) > 0) {
 832                                 io(is->tagprefix, "Skip literal data '%.*s'\n", (gint)len, token);
 833                         }
 834                 }
 835         } while (tok != '\n' && tok >= 0);
 836
 837         if (tok < 0)
 838                 return -1;
 839
 840         return 0;
 841 }