libsoup/soup-cache.c

   1 /* -*- Mode: C; tab-width: 8; indent-tabs-mode: t; c-basic-offset: 8 -*- */
   2 /*
   3  * soup-cache.c
   4  *
   5  * Copyright (C) 2009, 2010 Igalia S.L.
   6  *
   7  * This library is free software; you can redistribute it and/or
   8  * modify it under the terms of the GNU Library General Public
   9  * License as published by the Free Software Foundation; either
  10  * version 2 of the License, or (at your option) any later version.
  11  *
  12  * This library is distributed in the hope that it will be useful,
  13  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  14  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  15  * Library General Public License for more details.
  16  *
  17  * You should have received a copy of the GNU Library General Public License
  18  * along with this library; see the file COPYING.LIB.  If not, write to
  19  * the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
  20  * Boston, MA 02110-1301, USA.
  21  */
  22
  23 /* TODO:
  24  * - Need to hook the feature in the sync SoupSession.
  25  * - Need more tests.
  26  */
  27
  28 #ifdef HAVE_CONFIG_H
  29 #include <config.h>
  30 #endif
  31
  32 #include <stdlib.h>
  33 #include <gio/gio.h>
  34
  35 #define LIBSOUP_USE_UNSTABLE_REQUEST_API
  36
  37 #include "soup-cache.h"
  38 #include "soup-cache-private.h"
  39 #include "soup-date.h"
  40 #include "soup-enum-types.h"
  41 #include "soup-headers.h"
  42 #include "soup-session.h"
  43 #include "soup-session-feature.h"
  44 #include "soup-uri.h"
  45
  46 static SoupSessionFeatureInterface *soup_cache_default_feature_interface;
  47 static void soup_cache_session_feature_init (SoupSessionFeatureInterface *feature_interface, gpointer interface_data);
  48
  49 #define DEFAULT_MAX_SIZE 50 * 1024 * 1024
  50 #define MAX_ENTRY_DATA_PERCENTAGE 10 /* Percentage of the total size
  51                                         of the cache that can be
  52                                         filled by a single entry */
  53
  54 typedef struct _SoupCacheEntry {
  55         char *key;
  56         char *filename;
  57         guint freshness_lifetime;
  58         gboolean must_revalidate;
  59         GString *data;
  60         gsize pos;
  61         gsize length;
  62         time_t corrected_initial_age;
  63         time_t response_time;
  64         gboolean writing;
  65         gboolean dirty;
  66         gboolean got_body;
  67         gboolean being_validated;
  68         SoupMessageHeaders *headers;
  69         GOutputStream *stream;
  70         GError *error;
  71         guint hits;
  72         GCancellable *cancellable;
  73 } SoupCacheEntry;
  74
  75 struct _SoupCachePrivate {
  76         char *cache_dir;
  77         GHashTable *cache;
  78         guint n_pending;
  79         SoupSession *session;
  80         SoupCacheType cache_type;
  81         guint size;
  82         guint max_size;
  83         guint max_entry_data_size; /* Computed value. Here for performance reasons */
  84         GList *lru_start;
  85 };
  86
  87 typedef struct {
  88         SoupCache *cache;
  89         SoupCacheEntry *entry;
  90         SoupMessage *msg;
  91         gulong got_chunk_handler;
  92         gulong got_body_handler;
  93         gulong restarted_handler;
  94 } SoupCacheWritingFixture;
  95
  96 enum {
  97         PROP_0,
  98         PROP_CACHE_DIR,
  99         PROP_CACHE_TYPE
 100 };
 101
 102 #define SOUP_CACHE_GET_PRIVATE(o) (G_TYPE_INSTANCE_GET_PRIVATE ((o), SOUP_TYPE_CACHE, SoupCachePrivate))
 103
 104 G_DEFINE_TYPE_WITH_CODE (SoupCache, soup_cache, G_TYPE_OBJECT,
 105                          G_IMPLEMENT_INTERFACE (SOUP_TYPE_SESSION_FEATURE,
 106                                                 soup_cache_session_feature_init))
 107
 108 static gboolean soup_cache_entry_remove (SoupCache *cache, SoupCacheEntry *entry);
 109 static void make_room_for_new_entry (SoupCache *cache, guint length_to_add);
 110 static gboolean cache_accepts_entries_of_size (SoupCache *cache, guint length_to_add);
 111
 112 static SoupCacheability
 113 get_cacheability (SoupCache *cache, SoupMessage *msg)
 114 {
 115         SoupCacheability cacheability;
 116         const char *cache_control, *content_type;
 117
 118         /* 1. The request method must be cacheable */
 119         if (msg->method == SOUP_METHOD_GET)
 120                 cacheability = SOUP_CACHE_CACHEABLE;
 121         else if (msg->method == SOUP_METHOD_HEAD ||
 122                  msg->method == SOUP_METHOD_TRACE ||
 123                  msg->method == SOUP_METHOD_CONNECT)
 124                 return SOUP_CACHE_UNCACHEABLE;
 125         else
 126                 return (SOUP_CACHE_UNCACHEABLE | SOUP_CACHE_INVALIDATES);
 127
 128         content_type = soup_message_headers_get_content_type (msg->response_headers, NULL);
 129         if (content_type && !g_ascii_strcasecmp (content_type, "multipart/x-mixed-replace"))
 130                 return SOUP_CACHE_UNCACHEABLE;
 131
 132         cache_control = soup_message_headers_get (msg->response_headers, "Cache-Control");
 133         if (cache_control) {
 134                 GHashTable *hash;
 135                 SoupCachePrivate *priv = SOUP_CACHE_GET_PRIVATE (cache);
 136
 137                 hash = soup_header_parse_param_list (cache_control);
 138
 139                 /* Shared caches MUST NOT store private resources */
 140                 if (priv->cache_type == SOUP_CACHE_SHARED) {
 141                         if (g_hash_table_lookup_extended (hash, "private", NULL, NULL)) {
 142                                 soup_header_free_param_list (hash);
 143                                 return SOUP_CACHE_UNCACHEABLE;
 144                         }
 145                 }
 146
 147                 /* 2. The 'no-store' cache directive does not appear in the
 148                  * headers
 149                  */
 150                 if (g_hash_table_lookup_extended (hash, "no-store", NULL, NULL)) {
 151                         soup_header_free_param_list (hash);
 152                         return SOUP_CACHE_UNCACHEABLE;
 153                 }
 154
 155                 /* This does not appear in section 2.1, but I think it makes
 156                  * sense to check it too?
 157                  */
 158                 if (g_hash_table_lookup_extended (hash, "no-cache", NULL, NULL)) {
 159                         soup_header_free_param_list (hash);
 160                         return SOUP_CACHE_UNCACHEABLE;
 161                 }
 162
 163                 soup_header_free_param_list (hash);
 164         }
 165
 166         switch (msg->status_code) {
 167         case SOUP_STATUS_PARTIAL_CONTENT:
 168                 /* We don't cache partial responses, but they only
 169                  * invalidate cached full responses if the headers
 170                  * don't match.
 171                  */
 172                 cacheability = SOUP_CACHE_UNCACHEABLE;
 173                 break;
 174
 175         case SOUP_STATUS_NOT_MODIFIED:
 176                 /* A 304 response validates an existing cache entry */
 177                 cacheability = SOUP_CACHE_VALIDATES;
 178                 break;
 179
 180         case SOUP_STATUS_MULTIPLE_CHOICES:
 181         case SOUP_STATUS_MOVED_PERMANENTLY:
 182         case SOUP_STATUS_GONE:
 183                 /* FIXME: cacheable unless indicated otherwise */
 184                 cacheability = SOUP_CACHE_UNCACHEABLE;
 185                 break;
 186
 187         case SOUP_STATUS_FOUND:
 188         case SOUP_STATUS_TEMPORARY_REDIRECT:
 189                 /* FIXME: cacheable if explicitly indicated */
 190                 cacheability = SOUP_CACHE_UNCACHEABLE;
 191                 break;
 192
 193         case SOUP_STATUS_SEE_OTHER:
 194         case SOUP_STATUS_FORBIDDEN:
 195         case SOUP_STATUS_NOT_FOUND:
 196         case SOUP_STATUS_METHOD_NOT_ALLOWED:
 197                 return (SOUP_CACHE_UNCACHEABLE | SOUP_CACHE_INVALIDATES);
 198
 199         default:
 200                 /* Any 5xx status or any 4xx status not handled above
 201                  * is uncacheable but doesn't break the cache.
 202                  */
 203                 if ((msg->status_code >= SOUP_STATUS_BAD_REQUEST &&
 204                      msg->status_code <= SOUP_STATUS_FAILED_DEPENDENCY) ||
 205                     msg->status_code >= SOUP_STATUS_INTERNAL_SERVER_ERROR)
 206                         return SOUP_CACHE_UNCACHEABLE;
 207
 208                 /* An unrecognized 2xx, 3xx, or 4xx response breaks
 209                  * the cache.
 210                  */
 211                 if ((msg->status_code > SOUP_STATUS_PARTIAL_CONTENT &&
 212                      msg->status_code < SOUP_STATUS_MULTIPLE_CHOICES) ||
 213                     (msg->status_code > SOUP_STATUS_TEMPORARY_REDIRECT &&
 214                      msg->status_code < SOUP_STATUS_INTERNAL_SERVER_ERROR))
 215                         return (SOUP_CACHE_UNCACHEABLE | SOUP_CACHE_INVALIDATES);
 216                 break;
 217         }
 218
 219         return cacheability;
 220 }
 221
 222 static void
 223 soup_cache_entry_free (SoupCacheEntry *entry, gboolean purge)
 224 {
 225         if (purge) {
 226                 GFile *file = g_file_new_for_path (entry->filename);
 227                 g_file_delete (file, NULL, NULL);
 228                 g_object_unref (file);
 229         }
 230
 231         g_free (entry->filename);
 232         entry->filename = NULL;
 233         g_free (entry->key);
 234         entry->key = NULL;
 235
 236         if (entry->headers) {
 237                 soup_message_headers_free (entry->headers);
 238                 entry->headers = NULL;
 239         }
 240
 241         if (entry->data) {
 242                 g_string_free (entry->data, TRUE);
 243                 entry->data = NULL;
 244         }
 245         if (entry->error) {
 246                 g_error_free (entry->error);
 247                 entry->error = NULL;
 248         }
 249         if (entry->cancellable) {
 250                 g_object_unref (entry->cancellable);
 251                 entry->cancellable = NULL;
 252         }
 253
 254         g_slice_free (SoupCacheEntry, entry);
 255 }
 256
 257 static void
 258 copy_headers (const char *name, const char *value, SoupMessageHeaders *headers)
 259 {
 260         soup_message_headers_append (headers, name, value);
 261 }
 262
 263 static void
 264 update_headers (const char *name, const char *value, SoupMessageHeaders *headers)
 265 {
 266         if (soup_message_headers_get (headers, name))
 267                 soup_message_headers_replace (headers, name, value);
 268         else
 269                 soup_message_headers_append (headers, name, value);
 270 }
 271
 272 static guint
 273 soup_cache_entry_get_current_age (SoupCacheEntry *entry)
 274 {
 275         time_t now = time (NULL);
 276         time_t resident_time;
 277
 278         resident_time = now - entry->response_time;
 279         return entry->corrected_initial_age + resident_time;
 280 }
 281
 282 static gboolean
 283 soup_cache_entry_is_fresh_enough (SoupCacheEntry *entry, gint min_fresh)
 284 {
 285         guint limit = (min_fresh == -1) ? soup_cache_entry_get_current_age (entry) : (guint) min_fresh;
 286         return entry->freshness_lifetime > limit;
 287 }
 288
 289 static char *
 290 soup_message_get_cache_key (SoupMessage *msg)
 291 {
 292         SoupURI *uri = soup_message_get_uri (msg);
 293         return soup_uri_to_string (uri, FALSE);
 294 }
 295
 296 static void
 297 soup_cache_entry_set_freshness (SoupCacheEntry *entry, SoupMessage *msg, SoupCache *cache)
 298 {
 299         const char *cache_control;
 300         const char *expires, *date, *last_modified;
 301
 302         cache_control = soup_message_headers_get (entry->headers, "Cache-Control");
 303         if (cache_control) {
 304                 const char *max_age, *s_maxage;
 305                 gint64 freshness_lifetime = 0;
 306                 GHashTable *hash;
 307                 SoupCachePrivate *priv = SOUP_CACHE_GET_PRIVATE (cache);
 308
 309                 hash = soup_header_parse_param_list (cache_control);
 310
 311                 /* Should we re-validate the entry when it goes stale */
 312                 entry->must_revalidate = g_hash_table_lookup_extended (hash, "must-revalidate", NULL, NULL);
 313
 314                 /* Section 2.3.1 */
 315                 if (priv->cache_type == SOUP_CACHE_SHARED) {
 316                         s_maxage = g_hash_table_lookup (hash, "s-maxage");
 317                         if (s_maxage) {
 318                                 freshness_lifetime = g_ascii_strtoll (s_maxage, NULL, 10);
 319                                 if (freshness_lifetime) {
 320                                         /* Implies proxy-revalidate. TODO: is it true? */
 321                                         entry->must_revalidate = TRUE;
 322                                         soup_header_free_param_list (hash);
 323                                         return;
 324                                 }
 325                         }
 326                 }
 327
 328                 /* If 'max-age' cache directive is present, use that */
 329                 max_age = g_hash_table_lookup (hash, "max-age");
 330                 if (max_age)
 331                         freshness_lifetime = g_ascii_strtoll (max_age, NULL, 10);
 332
 333                 if (freshness_lifetime) {
 334                         entry->freshness_lifetime = (guint)MIN (freshness_lifetime, G_MAXUINT32);
 335                         soup_header_free_param_list (hash);
 336                         return;
 337                 }
 338
 339                 soup_header_free_param_list (hash);
 340         }
 341
 342         /* If the 'Expires' response header is present, use its value
 343          * minus the value of the 'Date' response header
 344          */
 345         expires = soup_message_headers_get (entry->headers, "Expires");
 346         date = soup_message_headers_get (entry->headers, "Date");
 347         if (expires && date) {
 348                 SoupDate *expires_d, *date_d;
 349                 time_t expires_t, date_t;
 350
 351                 expires_d = soup_date_new_from_string (expires);
 352                 if (expires_d) {
 353                         date_d = soup_date_new_from_string (date);
 354
 355                         expires_t = soup_date_to_time_t (expires_d);
 356                         date_t = soup_date_to_time_t (date_d);
 357
 358                         soup_date_free (expires_d);
 359                         soup_date_free (date_d);
 360
 361                         if (expires_t && date_t) {
 362                                 entry->freshness_lifetime = (guint)MAX (expires_t - date_t, 0);
 363                                 return;
 364                         }
 365                 } else {
 366                         /* If Expires is not a valid date we should
 367                            treat it as already expired, see section
 368                            3.3 */
 369                         entry->freshness_lifetime = 0;
 370                         return;
 371                 }
 372         }
 373
 374         /* Otherwise an heuristic may be used */
 375
 376         /* Heuristics MUST NOT be used with these status codes
 377            (section 2.3.1.1) */
 378         if (msg->status_code != SOUP_STATUS_OK &&
 379             msg->status_code != SOUP_STATUS_NON_AUTHORITATIVE &&
 380             msg->status_code != SOUP_STATUS_PARTIAL_CONTENT &&
 381             msg->status_code != SOUP_STATUS_MULTIPLE_CHOICES &&
 382             msg->status_code != SOUP_STATUS_MOVED_PERMANENTLY &&
 383             msg->status_code != SOUP_STATUS_GONE)
 384                 goto expire;
 385
 386         /* TODO: attach warning 113 if response's current_age is more
 387            than 24h (section 2.3.1.1) when using heuristics */
 388
 389         /* Last-Modified based heuristic */
 390         last_modified = soup_message_headers_get (entry->headers, "Last-Modified");
 391         if (last_modified) {
 392                 SoupDate *soup_date;
 393                 time_t now, last_modified_t;
 394
 395                 soup_date = soup_date_new_from_string (last_modified);
 396                 last_modified_t = soup_date_to_time_t (soup_date);
 397                 now = time (NULL);
 398
 399 #define HEURISTIC_FACTOR 0.1 /* From Section 2.3.1.1 */
 400
 401                 entry->freshness_lifetime = MAX (0, (now - last_modified_t) * HEURISTIC_FACTOR);
 402                 soup_date_free (soup_date);
 403         }
 404
 405         return;
 406
 407  expire:
 408         /* If all else fails, make the entry expire immediately */
 409         entry->freshness_lifetime = 0;
 410 }
 411
 412 static SoupCacheEntry *
 413 soup_cache_entry_new (SoupCache *cache, SoupMessage *msg, time_t request_time, time_t response_time)
 414 {
 415         SoupCacheEntry *entry;
 416         SoupMessageHeaders *headers;
 417         const char *date;
 418         char *md5;
 419
 420         entry = g_slice_new0 (SoupCacheEntry);
 421         entry->dirty = FALSE;
 422         entry->writing = FALSE;
 423         entry->got_body = FALSE;
 424         entry->being_validated = FALSE;
 425         entry->data = g_string_new (NULL);
 426         entry->pos = 0;
 427         entry->error = NULL;
 428
 429         /* key & filename */
 430         entry->key = soup_message_get_cache_key (msg);
 431         md5 = g_compute_checksum_for_string (G_CHECKSUM_MD5, entry->key, -1);
 432         entry->filename = g_build_filename (cache->priv->cache_dir, md5, NULL);
 433         g_free (md5);
 434
 435         /* Headers */
 436         headers = soup_message_headers_new (SOUP_MESSAGE_HEADERS_RESPONSE);
 437         soup_message_headers_foreach (msg->response_headers,
 438                                       (SoupMessageHeadersForeachFunc)copy_headers,
 439                                       headers);
 440         entry->headers = headers;
 441
 442         /* LRU list */
 443         entry->hits = 0;
 444
 445         /* Section 2.3.1, Freshness Lifetime */
 446         soup_cache_entry_set_freshness (entry, msg, cache);
 447
 448         /* Section 2.3.2, Calculating Age */
 449         date = soup_message_headers_get (entry->headers, "Date");
 450
 451         if (date) {
 452                 SoupDate *soup_date;
 453                 const char *age;
 454                 time_t date_value, apparent_age, corrected_received_age, response_delay, age_value = 0;
 455
 456                 soup_date = soup_date_new_from_string (date);
 457                 date_value = soup_date_to_time_t (soup_date);
 458                 soup_date_free (soup_date);
 459
 460                 age = soup_message_headers_get (entry->headers, "Age");
 461                 if (age)
 462                         age_value = g_ascii_strtoll (age, NULL, 10);
 463
 464                 entry->response_time = response_time;
 465                 apparent_age = MAX (0, entry->response_time - date_value);
 466                 corrected_received_age = MAX (apparent_age, age_value);
 467                 response_delay = entry->response_time - request_time;
 468                 entry->corrected_initial_age = corrected_received_age + response_delay;
 469         } else {
 470                 /* Is this correct ? */
 471                 entry->corrected_initial_age = time (NULL);
 472         }
 473
 474         return entry;
 475 }
 476
 477 static void
 478 soup_cache_writing_fixture_free (SoupCacheWritingFixture *fixture)
 479 {
 480         /* Free fixture. And disconnect signals, we don't want to
 481            listen to more SoupMessage events as we're finished with
 482            this resource */
 483         if (g_signal_handler_is_connected (fixture->msg, fixture->got_chunk_handler))
 484                 g_signal_handler_disconnect (fixture->msg, fixture->got_chunk_handler);
 485         if (g_signal_handler_is_connected (fixture->msg, fixture->got_body_handler))
 486                 g_signal_handler_disconnect (fixture->msg, fixture->got_body_handler);
 487         if (g_signal_handler_is_connected (fixture->msg, fixture->restarted_handler))
 488                 g_signal_handler_disconnect (fixture->msg, fixture->restarted_handler);
 489         g_object_unref (fixture->msg);
 490         g_object_unref (fixture->cache);
 491         g_slice_free (SoupCacheWritingFixture, fixture);
 492 }
 493
 494 static void
 495 close_ready_cb (GObject *source, GAsyncResult *result, SoupCacheWritingFixture *fixture)
 496 {
 497         SoupCacheEntry *entry = fixture->entry;
 498         SoupCache *cache = fixture->cache;
 499         GOutputStream *stream = G_OUTPUT_STREAM (source);
 500         goffset content_length;
 501
 502         g_warn_if_fail (entry->error == NULL);
 503
 504         /* FIXME: what do we do on error ? */
 505
 506         if (stream) {
 507                 g_output_stream_close_finish (stream, result, NULL);
 508                 g_object_unref (stream);
 509         }
 510         entry->stream = NULL;
 511
 512         content_length = soup_message_headers_get_content_length (entry->headers);
 513
 514         /* If the process was cancelled, then delete the entry from
 515            the cache. Do it also if the size of a chunked resource is
 516            too much for the cache */
 517         if (g_cancellable_is_cancelled (entry->cancellable)) {
 518                 entry->dirty = FALSE;
 519                 soup_cache_entry_remove (cache, entry);
 520                 soup_cache_entry_free (entry, TRUE);
 521                 entry = NULL;
 522         } else if ((soup_message_headers_get_encoding (entry->headers) == SOUP_ENCODING_CHUNKED) ||
 523                    entry->length != (gsize) content_length) {
 524                 /** Two options here:
 525                  *
 526                  * 1. "chunked" data, entry was temporarily added to
 527                  * cache (as content-length is 0) and now that we have
 528                  * the actual size we have to evaluate if we want it
 529                  * in the cache or not
 530                  *
 531                  * 2. Content-Length has a different value than actual
 532                  * length, means that the content was encoded for
 533                  * transmission (typically compressed) and thus we
 534                  * have to substract the content-length value that was
 535                  * added to the cache and add the unencoded length
 536                  **/
 537                 gint length_to_add = entry->length - content_length;
 538
 539                 /* Make room in cache if needed */
 540                 if (cache_accepts_entries_of_size (cache, length_to_add)) {
 541                         make_room_for_new_entry (cache, length_to_add);
 542
 543                         cache->priv->size += length_to_add;
 544                 } else {
 545                         entry->dirty = FALSE;
 546                         soup_cache_entry_remove (cache, entry);
 547                         soup_cache_entry_free (entry, TRUE);
 548                         entry = NULL;
 549                 }
 550         }
 551
 552         if (entry) {
 553                 /* Get rid of the GString in memory for the resource now */
 554                 if (entry->data) {
 555                         g_string_free (entry->data, TRUE);
 556                         entry->data = NULL;
 557                 }
 558
 559                 entry->dirty = FALSE;
 560                 entry->writing = FALSE;
 561                 entry->got_body = FALSE;
 562                 entry->pos = 0;
 563
 564                 g_object_unref (entry->cancellable);
 565                 entry->cancellable = NULL;
 566         }
 567
 568         cache->priv->n_pending--;
 569
 570         /* Frees */
 571         soup_cache_writing_fixture_free (fixture);
 572 }
 573
 574 static void
 575 write_ready_cb (GObject *source, GAsyncResult *result, SoupCacheWritingFixture *fixture)
 576 {
 577         GOutputStream *stream = G_OUTPUT_STREAM (source);
 578         GError *error = NULL;
 579         gssize write_size;
 580         SoupCacheEntry *entry = fixture->entry;
 581
 582         if (g_cancellable_is_cancelled (entry->cancellable)) {
 583                 g_output_stream_close_async (stream,
 584                                              G_PRIORITY_LOW,
 585                                              entry->cancellable,
 586                                              (GAsyncReadyCallback)close_ready_cb,
 587                                              fixture);
 588                 return;
 589         }
 590
 591         write_size = g_output_stream_write_finish (stream, result, &error);
 592         if (write_size <= 0 || error) {
 593                 if (error)
 594                         entry->error = error;
 595                 g_output_stream_close_async (stream,
 596                                              G_PRIORITY_LOW,
 597                                              entry->cancellable,
 598                                              (GAsyncReadyCallback)close_ready_cb,
 599                                              fixture);
 600                 /* FIXME: We should completely stop caching the
 601                    resource at this point */
 602         } else {
 603                 entry->pos += write_size;
 604
 605                 /* Are we still writing and is there new data to write
 606                    already ? */
 607                 if (entry->data && entry->pos < entry->data->len) {
 608                         g_output_stream_write_async (entry->stream,
 609                                                      entry->data->str + entry->pos,
 610                                                      entry->data->len - entry->pos,
 611                                                      G_PRIORITY_LOW,
 612                                                      entry->cancellable,
 613                                                      (GAsyncReadyCallback)write_ready_cb,
 614                                                      fixture);
 615                 } else {
 616                         entry->writing = FALSE;
 617
 618                         if (entry->got_body) {
 619                                 /* If we already received 'got-body'
 620                                    and we have written all the data,
 621                                    we can close the stream */
 622                                 g_output_stream_close_async (entry->stream,
 623                                                              G_PRIORITY_LOW,
 624                                                              entry->cancellable,
 625                                                              (GAsyncReadyCallback)close_ready_cb,
 626                                                              fixture);
 627                         }
 628                 }
 629         }
 630 }
 631
 632 static void
 633 msg_got_chunk_cb (SoupMessage *msg, SoupBuffer *chunk, SoupCacheWritingFixture *fixture)
 634 {
 635         SoupCacheEntry *entry = fixture->entry;
 636
 637         g_return_if_fail (chunk->data && chunk->length);
 638         g_return_if_fail (entry);
 639
 640         /* Ignore this if the writing or appending was cancelled */
 641         if (!g_cancellable_is_cancelled (entry->cancellable)) {
 642                 g_string_append_len (entry->data, chunk->data, chunk->length);
 643                 entry->length = entry->data->len;
 644
 645                 if (!cache_accepts_entries_of_size (fixture->cache, entry->length)) {
 646                         /* Quickly cancel the caching of the resource */
 647                         g_cancellable_cancel (entry->cancellable);
 648                 }
 649         }
 650
 651         /* FIXME: remove the error check when we cancel the caching at
 652            the first write error */
 653         /* Only write if the entry stream is ready */
 654         if (entry->writing == FALSE && entry->error == NULL && entry->stream) {
 655                 GString *data = entry->data;
 656                 entry->writing = TRUE;
 657                 g_output_stream_write_async (entry->stream,
 658                                              data->str + entry->pos,
 659                                              data->len - entry->pos,
 660                                              G_PRIORITY_LOW,
 661                                              entry->cancellable,
 662                                              (GAsyncReadyCallback)write_ready_cb,
 663                                              fixture);
 664         }
 665 }
 666
 667 static void
 668 msg_got_body_cb (SoupMessage *msg, SoupCacheWritingFixture *fixture)
 669 {
 670         SoupCacheEntry *entry = fixture->entry;
 671         g_return_if_fail (entry);
 672
 673         entry->got_body = TRUE;
 674
 675         if (!entry->stream && entry->pos != entry->length)
 676                 /* The stream is not ready to be written but we still
 677                    have data to write, we'll write it when the stream
 678                    is opened for writing */
 679                 return;
 680
 681
 682         if (entry->pos != entry->length) {
 683                 /* If we still have data to write, write it,
 684                    write_ready_cb will close the stream */
 685                 if (entry->writing == FALSE && entry->error == NULL && entry->stream) {
 686                         g_output_stream_write_async (entry->stream,
 687                                                      entry->data->str + entry->pos,
 688                                                      entry->data->len - entry->pos,
 689                                                      G_PRIORITY_LOW,
 690                                                      entry->cancellable,
 691                                                      (GAsyncReadyCallback)write_ready_cb,
 692                                                      fixture);
 693                 }
 694                 return;
 695         }
 696
 697         if (entry->stream && !entry->writing)
 698                 g_output_stream_close_async (entry->stream,
 699                                              G_PRIORITY_LOW,
 700                                              entry->cancellable,
 701                                              (GAsyncReadyCallback)close_ready_cb,
 702                                              fixture);
 703 }
 704
 705 static gboolean
 706 soup_cache_entry_remove (SoupCache *cache, SoupCacheEntry *entry)
 707 {
 708         GList *lru_item;
 709
 710         /* if (entry->dirty && !g_cancellable_is_cancelled (entry->cancellable)) { */
 711         if (entry->dirty) {
 712                 g_cancellable_cancel (entry->cancellable);
 713                 return FALSE;
 714         }
 715
 716         g_assert (!entry->dirty);
 717         g_assert (g_list_length (cache->priv->lru_start) == g_hash_table_size (cache->priv->cache));
 718
 719         /* Remove from cache */
 720         if (!g_hash_table_remove (cache->priv->cache, entry->key))
 721                 return FALSE;
 722
 723         /* Remove from LRU */
 724         lru_item = g_list_find (cache->priv->lru_start, entry);
 725         cache->priv->lru_start = g_list_delete_link (cache->priv->lru_start, lru_item);
 726
 727         /* Adjust cache size */
 728         cache->priv->size -= entry->length;
 729
 730         g_assert (g_list_length (cache->priv->lru_start) == g_hash_table_size (cache->priv->cache));
 731
 732         return TRUE;
 733 }
 734
 735 static gint
 736 lru_compare_func (gconstpointer a, gconstpointer b)
 737 {
 738         SoupCacheEntry *entry_a = (SoupCacheEntry *)a;
 739         SoupCacheEntry *entry_b = (SoupCacheEntry *)b;
 740
 741         /** The rationale of this sorting func is
 742          *
 743          * 1. sort by hits -> LRU algorithm, then
 744          *
 745          * 2. sort by freshness lifetime, we better discard first
 746          * entries that are close to expire
 747          *
 748          * 3. sort by size, replace first small size resources as they
 749          * are cheaper to download
 750          **/
 751
 752         /* Sort by hits */
 753         if (entry_a->hits != entry_b->hits)
 754                 return entry_a->hits - entry_b->hits;
 755
 756         /* Sort by freshness_lifetime */
 757         if (entry_a->freshness_lifetime != entry_b->freshness_lifetime)
 758                 return entry_a->freshness_lifetime - entry_b->freshness_lifetime;
 759
 760         /* Sort by size */
 761         return entry_a->length - entry_b->length;
 762 }
 763
 764 static gboolean
 765 cache_accepts_entries_of_size (SoupCache *cache, guint length_to_add)
 766 {
 767         /* We could add here some more heuristics. TODO: review how
 768            this is done by other HTTP caches */
 769
 770         return length_to_add <= cache->priv->max_entry_data_size;
 771 }
 772
 773 static void
 774 make_room_for_new_entry (SoupCache *cache, guint length_to_add)
 775 {
 776         GList *lru_entry = cache->priv->lru_start;
 777
 778         /* Check that there is enough room for the new entry. This is
 779            an approximation as we're not working out the size of the
 780            cache file or the size of the headers for performance
 781            reasons. TODO: check if that would be really that expensive */
 782
 783         while (lru_entry &&
 784                (length_to_add + cache->priv->size > cache->priv->max_size)) {
 785                 SoupCacheEntry *old_entry = (SoupCacheEntry *)lru_entry->data;
 786
 787                 /* Discard entries. Once cancelled resources will be
 788                  * freed in close_ready_cb
 789                  */
 790                 if (soup_cache_entry_remove (cache, old_entry)) {
 791                         soup_cache_entry_free (old_entry, TRUE);
 792                         lru_entry = cache->priv->lru_start;
 793                 } else
 794                         lru_entry = g_list_next (lru_entry);
 795         }
 796 }
 797
 798 static gboolean
 799 soup_cache_entry_insert_by_key (SoupCache *cache,
 800                                 const char *key,
 801                                 SoupCacheEntry *entry,
 802                                 gboolean sort)
 803 {
 804         guint length_to_add = 0;
 805
 806         if (soup_message_headers_get_encoding (entry->headers) != SOUP_ENCODING_CHUNKED)
 807                 length_to_add = soup_message_headers_get_content_length (entry->headers);
 808
 809         /* Check if we are going to store the resource depending on its size */
 810         if (length_to_add) {
 811                 if (!cache_accepts_entries_of_size (cache, length_to_add))
 812                         return FALSE;
 813
 814                 /* Make room for new entry if needed */
 815                 make_room_for_new_entry (cache, length_to_add);
 816         }
 817
 818         g_hash_table_insert (cache->priv->cache, g_strdup (key), entry);
 819
 820         /* Compute new cache size */
 821         cache->priv->size += length_to_add;
 822
 823         /* Update LRU */
 824         if (sort)
 825                 cache->priv->lru_start = g_list_insert_sorted (cache->priv->lru_start, entry, lru_compare_func);
 826         else
 827                 cache->priv->lru_start = g_list_prepend (cache->priv->lru_start, entry);
 828
 829         g_assert (g_list_length (cache->priv->lru_start) == g_hash_table_size (cache->priv->cache));
 830
 831         return TRUE;
 832 }
 833
 834 static void
 835 msg_restarted_cb (SoupMessage *msg, SoupCacheEntry *entry)
 836 {
 837         /* FIXME: What should we do here exactly? */
 838 }
 839
 840 static void
 841 replace_cb (GObject *source, GAsyncResult *result, SoupCacheWritingFixture *fixture)
 842 {
 843         SoupCacheEntry *entry = fixture->entry;
 844         GOutputStream *stream = (GOutputStream *) g_file_replace_finish (G_FILE (source),
 845                                                                          result, &entry->error);
 846
 847         if (g_cancellable_is_cancelled (entry->cancellable) || entry->error) {
 848                 if (stream)
 849                         g_object_unref (stream);
 850                 fixture->cache->priv->n_pending--;
 851                 entry->dirty = FALSE;
 852                 soup_cache_entry_remove (fixture->cache, entry);
 853                 soup_cache_entry_free (entry, TRUE);
 854                 soup_cache_writing_fixture_free (fixture);
 855                 return;
 856         }
 857
 858         entry->stream = stream;
 859
 860         /* If we already got all the data we have to initiate the
 861          * writing here, since we won't get more 'got-chunk'
 862          * signals
 863          */
 864         if (!entry->got_body)
 865                 return;
 866
 867         /* It could happen that reading the data from server
 868          * was completed before this happens. In that case
 869          * there is no data
 870          */
 871         if (entry->data) {
 872                 entry->writing = TRUE;
 873                 g_output_stream_write_async (entry->stream,
 874                                              entry->data->str + entry->pos,
 875                                              entry->data->len - entry->pos,
 876                                              G_PRIORITY_LOW,
 877                                              entry->cancellable,
 878                                              (GAsyncReadyCallback)write_ready_cb,
 879                                              fixture);
 880         }
 881 }
 882
 883 typedef struct {
 884         time_t request_time;
 885         SoupSessionFeature *feature;
 886         gulong got_headers_handler;
 887 } RequestHelper;
 888
 889 static void
 890 msg_got_headers_cb (SoupMessage *msg, gpointer user_data)
 891 {
 892         SoupCache *cache;
 893         SoupCacheability cacheable;
 894         RequestHelper *helper;
 895         time_t request_time, response_time;
 896
 897         response_time = time (NULL);
 898
 899         helper = (RequestHelper *)user_data;
 900         cache = SOUP_CACHE (helper->feature);
 901         request_time = helper->request_time;
 902         g_signal_handlers_disconnect_by_func (msg, msg_got_headers_cb, user_data);
 903         g_slice_free (RequestHelper, helper);
 904
 905         cacheable = soup_cache_get_cacheability (cache, msg);
 906
 907         if (cacheable & SOUP_CACHE_CACHEABLE) {
 908                 SoupCacheEntry *entry;
 909                 char *key;
 910                 GFile *file;
 911                 SoupCacheWritingFixture *fixture;
 912
 913                 /* Check if we are already caching this resource */
 914                 key = soup_message_get_cache_key (msg);
 915                 entry = g_hash_table_lookup (cache->priv->cache, key);
 916                 g_free (key);
 917
 918                 if (entry && entry->dirty)
 919                         return;
 920
 921                 /* Create a new entry, deleting any old one if present */
 922                 if (entry) {
 923                         soup_cache_entry_remove (cache, entry);
 924                         soup_cache_entry_free (entry, TRUE);
 925                 }
 926
 927                 entry = soup_cache_entry_new (cache, msg, request_time, response_time);
 928                 entry->hits = 1;
 929
 930                 /* Do not continue if it can not be stored */
 931                 if (!soup_cache_entry_insert_by_key (cache, (const gchar *)entry->key, entry, TRUE)) {
 932                         soup_cache_entry_free (entry, TRUE);
 933                         return;
 934                 }
 935
 936                 fixture = g_slice_new0 (SoupCacheWritingFixture);
 937                 fixture->cache = g_object_ref (cache);
 938                 fixture->entry = entry;
 939                 fixture->msg = g_object_ref (msg);
 940
 941                 /* We connect now to these signals and buffer the data
 942                    if it comes before the file is ready for writing */
 943                 fixture->got_chunk_handler =
 944                         g_signal_connect (msg, "got-chunk", G_CALLBACK (msg_got_chunk_cb), fixture);
 945                 fixture->got_body_handler =
 946                         g_signal_connect (msg, "got-body", G_CALLBACK (msg_got_body_cb), fixture);
 947                 fixture->restarted_handler =
 948                         g_signal_connect (msg, "restarted", G_CALLBACK (msg_restarted_cb), entry);
 949
 950                 /* Prepare entry */
 951                 file = g_file_new_for_path (entry->filename);
 952                 cache->priv->n_pending++;
 953
 954                 entry->dirty = TRUE;
 955                 entry->cancellable = g_cancellable_new ();
 956                 g_file_replace_async (file, NULL, FALSE,
 957                                       G_FILE_CREATE_PRIVATE | G_FILE_CREATE_REPLACE_DESTINATION,
 958                                       G_PRIORITY_LOW, entry->cancellable,
 959                                       (GAsyncReadyCallback) replace_cb, fixture);
 960                 g_object_unref (file);
 961         } else if (cacheable & SOUP_CACHE_INVALIDATES) {
 962                 char *key;
 963                 SoupCacheEntry *entry;
 964
 965                 key = soup_message_get_cache_key (msg);
 966                 entry = g_hash_table_lookup (cache->priv->cache, key);
 967                 g_free (key);
 968
 969                 if (entry) {
 970                         if (soup_cache_entry_remove (cache, entry))
 971                                 soup_cache_entry_free (entry, TRUE);
 972                 }
 973         } else if (cacheable & SOUP_CACHE_VALIDATES) {
 974                 char *key;
 975                 SoupCacheEntry *entry;
 976
 977                 key = soup_message_get_cache_key (msg);
 978                 entry = g_hash_table_lookup (cache->priv->cache, key);
 979                 g_free (key);
 980
 981                 /* It's possible to get a CACHE_VALIDATES with no
 982                  * entry in the hash table. This could happen if for
 983                  * example the soup client is the one creating the
 984                  * conditional request.
 985                  */
 986                 if (entry) {
 987                         entry->being_validated = FALSE;
 988
 989                         /* We update the headers of the existing cache item,
 990                            plus its age */
 991                         soup_message_headers_foreach (msg->response_headers,
 992                                                       (SoupMessageHeadersForeachFunc)update_headers,
 993                                                       entry->headers);
 994                         soup_cache_entry_set_freshness (entry, msg, cache);
 995                 }
 996         }
 997 }
 998
 999 GInputStream *
1000 soup_cache_send_response (SoupCache *cache, SoupMessage *msg)
1001 {
1002         char *key;
1003         SoupCacheEntry *entry;
1004         char *current_age;
1005         GInputStream *stream = NULL;
1006         GFile *file;
1007
1008         g_return_val_if_fail (SOUP_IS_CACHE (cache), NULL);
1009         g_return_val_if_fail (SOUP_IS_MESSAGE (msg), NULL);
1010
1011         key = soup_message_get_cache_key (msg);
1012         entry = g_hash_table_lookup (cache->priv->cache, key);
1013         g_free (key);
1014         g_return_val_if_fail (entry, NULL);
1015
1016         /* If we are told to send a response from cache any validation
1017            in course is over by now */
1018         entry->being_validated = FALSE;
1019
1020         /* Headers */
1021         soup_message_headers_foreach (entry->headers,
1022                                       (SoupMessageHeadersForeachFunc)update_headers,
1023                                       msg->response_headers);
1024
1025         /* Add 'Age' header with the current age */
1026         current_age = g_strdup_printf ("%d", soup_cache_entry_get_current_age (entry));
1027         soup_message_headers_replace (msg->response_headers,
1028                                       "Age",
1029                                       current_age);
1030         g_free (current_age);
1031
1032         /* TODO: the original idea was to save reads, but current code
1033            assumes that a stream is always returned. Need to reach
1034            some agreement here. Also we have to handle the situation
1035            were the file was no longer there (for example files
1036            removed without notifying the cache */
1037         file = g_file_new_for_path (entry->filename);
1038         stream = G_INPUT_STREAM (g_file_read (file, NULL, NULL));
1039         g_object_unref (file);
1040
1041         return stream;
1042 }
1043
1044 static void
1045 request_started (SoupSessionFeature *feature, SoupSession *session,
1046                  SoupMessage *msg, SoupSocket *socket)
1047 {
1048         RequestHelper *helper = g_slice_new0 (RequestHelper);
1049         helper->request_time = time (NULL);
1050         helper->feature = feature;
1051         helper->got_headers_handler = g_signal_connect (msg, "got-headers",
1052                                                         G_CALLBACK (msg_got_headers_cb),
1053                                                         helper);
1054 }
1055
1056 static void
1057 attach (SoupSessionFeature *feature, SoupSession *session)
1058 {
1059         SoupCache *cache = SOUP_CACHE (feature);
1060         cache->priv->session = session;
1061
1062         soup_cache_default_feature_interface->attach (feature, session);
1063 }
1064
1065 static void
1066 soup_cache_session_feature_init (SoupSessionFeatureInterface *feature_interface,
1067                                         gpointer interface_data)
1068 {
1069         soup_cache_default_feature_interface =
1070                 g_type_default_interface_peek (SOUP_TYPE_SESSION_FEATURE);
1071
1072         feature_interface->attach = attach;
1073         feature_interface->request_started = request_started;
1074 }
1075
1076 static void
1077 soup_cache_init (SoupCache *cache)
1078 {
1079         SoupCachePrivate *priv;
1080
1081         priv = cache->priv = SOUP_CACHE_GET_PRIVATE (cache);
1082
1083         priv->cache = g_hash_table_new_full (g_str_hash,
1084                                              g_str_equal,
1085                                              (GDestroyNotify)g_free,
1086                                              NULL);
1087
1088         /* LRU */
1089         priv->lru_start = NULL;
1090
1091         /* */
1092         priv->n_pending = 0;
1093
1094         /* Cache size */
1095         priv->max_size = DEFAULT_MAX_SIZE;
1096         priv->max_entry_data_size = priv->max_size / MAX_ENTRY_DATA_PERCENTAGE;
1097         priv->size = 0;
1098 }
1099
1100 static void
1101 remove_cache_item (gpointer data,
1102                    gpointer user_data)
1103 {
1104         SoupCache *cache = (SoupCache *) user_data;
1105         SoupCacheEntry *entry = (SoupCacheEntry *) data;
1106
1107         if (soup_cache_entry_remove (cache, entry))
1108                 soup_cache_entry_free (entry, FALSE);
1109 }
1110
1111 static void
1112 soup_cache_finalize (GObject *object)
1113 {
1114         SoupCachePrivate *priv;
1115         GList *entries;
1116
1117         priv = SOUP_CACHE (object)->priv;
1118
1119         // Cannot use g_hash_table_foreach as callbacks must not modify the hash table
1120         entries = g_hash_table_get_values (priv->cache);
1121         g_list_foreach (entries, remove_cache_item, object);
1122         g_list_free (entries);
1123
1124         g_hash_table_destroy (priv->cache);
1125         g_free (priv->cache_dir);
1126
1127         g_list_free (priv->lru_start);
1128         priv->lru_start = NULL;
1129
1130         G_OBJECT_CLASS (soup_cache_parent_class)->finalize (object);
1131 }
1132
1133 static void
1134 soup_cache_set_property (GObject *object, guint prop_id,
1135                                 const GValue *value, GParamSpec *pspec)
1136 {
1137         SoupCachePrivate *priv = SOUP_CACHE (object)->priv;
1138
1139         switch (prop_id) {
1140         case PROP_CACHE_DIR:
1141                 priv->cache_dir = g_value_dup_string (value);
1142                 /* Create directory if it does not exist (FIXME: should we?) */
1143                 if (!g_file_test (priv->cache_dir, G_FILE_TEST_EXISTS | G_FILE_TEST_IS_DIR))
1144                         g_mkdir_with_parents (priv->cache_dir, 0700);
1145                 break;
1146         case PROP_CACHE_TYPE:
1147                 priv->cache_type = g_value_get_enum (value);
1148                 /* TODO: clear private entries and issue a warning if moving to shared? */
1149                 break;
1150         default:
1151                 G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
1152                 break;
1153         }
1154 }
1155
1156 static void
1157 soup_cache_get_property (GObject *object, guint prop_id,
1158                          GValue *value, GParamSpec *pspec)
1159 {
1160         SoupCachePrivate *priv = SOUP_CACHE (object)->priv;
1161
1162         switch (prop_id) {
1163         case PROP_CACHE_DIR:
1164                 g_value_set_string (value, priv->cache_dir);
1165                 break;
1166         case PROP_CACHE_TYPE:
1167                 g_value_set_enum (value, priv->cache_type);
1168                 break;
1169         default:
1170                 G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
1171                 break;
1172         }
1173 }
1174
1175 static void
1176 soup_cache_constructed (GObject *object)
1177 {
1178         SoupCachePrivate *priv;
1179
1180         priv = SOUP_CACHE (object)->priv;
1181
1182         if (!priv->cache_dir) {
1183                 /* Set a default cache dir, different for each user */
1184                 priv->cache_dir = g_build_filename (g_get_user_cache_dir (),
1185                                                     "httpcache",
1186                                                     NULL);
1187                 if (!g_file_test (priv->cache_dir, G_FILE_TEST_EXISTS | G_FILE_TEST_IS_DIR))
1188                         g_mkdir_with_parents (priv->cache_dir, 0700);
1189         }
1190
1191         if (G_OBJECT_CLASS (soup_cache_parent_class)->constructed)
1192                 G_OBJECT_CLASS (soup_cache_parent_class)->constructed (object);
1193 }
1194
1195 static void
1196 soup_cache_class_init (SoupCacheClass *cache_class)
1197 {
1198         GObjectClass *gobject_class = (GObjectClass *)cache_class;
1199
1200         gobject_class->finalize = soup_cache_finalize;
1201         gobject_class->constructed = soup_cache_constructed;
1202         gobject_class->set_property = soup_cache_set_property;
1203         gobject_class->get_property = soup_cache_get_property;
1204
1205         cache_class->get_cacheability = get_cacheability;
1206
1207         g_object_class_install_property (gobject_class, PROP_CACHE_DIR,
1208                                          g_param_spec_string ("cache-dir",
1209                                                               "Cache directory",
1210                                                               "The directory to store the cache files",
1211                                                               NULL,
1212                                                               G_PARAM_READWRITE | G_PARAM_CONSTRUCT_ONLY));
1213
1214         g_object_class_install_property (gobject_class, PROP_CACHE_TYPE,
1215                                          g_param_spec_enum ("cache-type",
1216                                                             "Cache type",
1217                                                             "Whether the cache is private or shared",
1218                                                             SOUP_TYPE_CACHE_TYPE,
1219                                                             SOUP_CACHE_SINGLE_USER,
1220                                                             G_PARAM_READWRITE | G_PARAM_CONSTRUCT_ONLY));
1221
1222         g_type_class_add_private (cache_class, sizeof (SoupCachePrivate));
1223 }
1224
1225 /**
1226  * soup_cache_new:
1227  * @cache_dir: the directory to store the cached data, or %NULL to use the default one
1228  * @cache_type: the #SoupCacheType of the cache
1229  *
1230  * Creates a new #SoupCache.
1231  *
1232  * Returns: a new #SoupCache
1233  *
1234  * Since: 2.28
1235  **/
1236 SoupCache *
1237 soup_cache_new (const char *cache_dir, SoupCacheType cache_type)
1238 {
1239         return g_object_new (SOUP_TYPE_CACHE,
1240                              "cache-dir", cache_dir,
1241                              "cache-type", cache_type,
1242                              NULL);
1243 }
1244
1245 /**
1246  * soup_cache_has_response:
1247  * @cache: a #SoupCache
1248  * @msg: a #SoupMessage
1249  *
1250  * This function calculates whether the @cache object has a proper
1251  * response for the request @msg given the flags both in the request
1252  * and the cached reply and the time ellapsed since it was cached.
1253  *
1254  * Returns: whether or not the @cache has a valid response for @msg
1255  **/
1256 SoupCacheResponse
1257 soup_cache_has_response (SoupCache *cache, SoupMessage *msg)
1258 {
1259         char *key;
1260         SoupCacheEntry *entry;
1261         const char *cache_control, *pragma;
1262         gpointer value;
1263         int max_age, max_stale, min_fresh;
1264         GList *lru_item, *item;
1265
1266         key = soup_message_get_cache_key (msg);
1267         entry = g_hash_table_lookup (cache->priv->cache, key);
1268         g_free (key);
1269
1270         /* 1. The presented Request-URI and that of stored response
1271          * match
1272          */
1273         if (!entry)
1274                 return SOUP_CACHE_RESPONSE_STALE;
1275
1276         /* Increase hit count. Take sorting into account */
1277         entry->hits++;
1278         lru_item = g_list_find (cache->priv->lru_start, entry);
1279         item = lru_item;
1280         while (item->next && lru_compare_func (item->data, item->next->data) > 0)
1281                 item = g_list_next (item);
1282
1283         if (item != lru_item) {
1284                 cache->priv->lru_start = g_list_remove_link (cache->priv->lru_start, lru_item);
1285                 item = g_list_insert_sorted (item, lru_item->data, lru_compare_func);
1286                 g_list_free (lru_item);
1287         }
1288
1289         if (entry->dirty || entry->being_validated)
1290                 return SOUP_CACHE_RESPONSE_STALE;
1291
1292         /* 2. The request method associated with the stored response
1293          *  allows it to be used for the presented request
1294          */
1295
1296         /* In practice this means we only return our resource for GET,
1297          * cacheability for other methods is a TODO in the RFC
1298          * (TODO: although we could return the headers for HEAD
1299          * probably).
1300          */
1301         if (msg->method != SOUP_METHOD_GET)
1302                 return SOUP_CACHE_RESPONSE_STALE;
1303
1304         /* 3. Selecting request-headers nominated by the stored
1305          * response (if any) match those presented.
1306          */
1307
1308         /* TODO */
1309
1310         /* 4. The request is a conditional request issued by the client.
1311          */
1312         if (soup_message_headers_get (msg->request_headers, "If-Modified-Since") ||
1313             soup_message_headers_get (msg->request_headers, "If-None-Match"))
1314                 return SOUP_CACHE_RESPONSE_STALE;
1315
1316         /* 5. The presented request and stored response are free from
1317          * directives that would prevent its use.
1318          */
1319
1320         max_age = max_stale = min_fresh = -1;
1321
1322         /* For HTTP 1.0 compatibility. RFC2616 section 14.9.4
1323          */
1324         pragma = soup_message_headers_get (msg->request_headers, "Pragma");
1325         if (pragma && soup_header_contains (pragma, "no-cache"))
1326                 return SOUP_CACHE_RESPONSE_STALE;
1327
1328         cache_control = soup_message_headers_get (msg->request_headers, "Cache-Control");
1329         if (cache_control) {
1330                 GHashTable *hash = soup_header_parse_param_list (cache_control);
1331
1332                 if (g_hash_table_lookup_extended (hash, "no-store", NULL, NULL)) {
1333                         soup_header_free_param_list (hash);
1334                         return SOUP_CACHE_RESPONSE_STALE;
1335                 }
1336
1337                 if (g_hash_table_lookup_extended (hash, "no-cache", NULL, NULL)) {
1338                         soup_header_free_param_list (hash);
1339                         return SOUP_CACHE_RESPONSE_STALE;
1340                 }
1341
1342                 if (g_hash_table_lookup_extended (hash, "max-age", NULL, &value)) {
1343                         max_age = (int)MIN (g_ascii_strtoll (value, NULL, 10), G_MAXINT32);
1344                         /* Forcing cache revalidaton
1345                          */
1346                         if (!max_age)
1347                                 return SOUP_CACHE_RESPONSE_NEEDS_VALIDATION;
1348                 }
1349
1350                 /* max-stale can have no value set, we need to use _extended */
1351                 if (g_hash_table_lookup_extended (hash, "max-stale", NULL, &value)) {
1352                         if (value)
1353                                 max_stale = (int)MIN (g_ascii_strtoll (value, NULL, 10), G_MAXINT32);
1354                         else
1355                                 max_stale = G_MAXINT32;
1356                 }
1357
1358                 value = g_hash_table_lookup (hash, "min-fresh");
1359                 if (value)
1360                         min_fresh = (int)MIN (g_ascii_strtoll (value, NULL, 10), G_MAXINT32);
1361
1362                 soup_header_free_param_list (hash);
1363
1364                 if (max_age > 0) {
1365                         guint current_age = soup_cache_entry_get_current_age (entry);
1366
1367                         /* If we are over max-age and max-stale is not
1368                            set, do not use the value from the cache
1369                            without validation */
1370                         if ((guint) max_age <= current_age && max_stale == -1)
1371                                 return SOUP_CACHE_RESPONSE_NEEDS_VALIDATION;
1372                 }
1373         }
1374
1375         /* 6. The stored response is either: fresh, allowed to be
1376          * served stale or succesfully validated
1377          */
1378         /* TODO consider also proxy-revalidate & s-maxage */
1379         if (entry->must_revalidate)
1380                 return SOUP_CACHE_RESPONSE_NEEDS_VALIDATION;
1381
1382         if (!soup_cache_entry_is_fresh_enough (entry, min_fresh)) {
1383                 /* Not fresh, can it be served stale? */
1384                 if (max_stale != -1) {
1385                         /* G_MAXINT32 means we accept any staleness */
1386                         if (max_stale == G_MAXINT32)
1387                                 return SOUP_CACHE_RESPONSE_FRESH;
1388
1389                         if ((soup_cache_entry_get_current_age (entry) - entry->freshness_lifetime) <= (guint) max_stale)
1390                                 return SOUP_CACHE_RESPONSE_FRESH;
1391                 }
1392
1393                 return SOUP_CACHE_RESPONSE_NEEDS_VALIDATION;
1394         }
1395
1396         return SOUP_CACHE_RESPONSE_FRESH;
1397 }
1398
1399 /**
1400  * soup_cache_get_cacheability:
1401  * @cache: a #SoupCache
1402  * @msg: a #SoupMessage
1403  *
1404  * Calculates whether the @msg can be cached or not.
1405  *
1406  * Returns: a #SoupCacheability value indicating whether the @msg can be cached or not.
1407  **/
1408 SoupCacheability
1409 soup_cache_get_cacheability (SoupCache *cache, SoupMessage *msg)
1410 {
1411         g_return_val_if_fail (SOUP_IS_CACHE (cache), SOUP_CACHE_UNCACHEABLE);
1412         g_return_val_if_fail (SOUP_IS_MESSAGE (msg), SOUP_CACHE_UNCACHEABLE);
1413
1414         return SOUP_CACHE_GET_CLASS (cache)->get_cacheability (cache, msg);
1415 }
1416
1417 static gboolean
1418 force_flush_timeout (gpointer data)
1419 {
1420         gboolean *forced = (gboolean *)data;
1421         *forced = TRUE;
1422
1423         return FALSE;
1424 }
1425
1426 /**
1427  * soup_cache_flush:
1428  * @cache: a #SoupCache
1429  * @session: the #SoupSession associated with the @cache
1430  *
1431  * This function will force all pending writes in the @cache to be
1432  * committed to disk. For doing so it will iterate the #GMainContext
1433  * associated with the @session (which can be the default one) as long
1434  * as needed.
1435  **/
1436 void
1437 soup_cache_flush (SoupCache *cache)
1438 {
1439         GMainContext *async_context;
1440         SoupSession *session;
1441         guint timeout_id;
1442         gboolean forced = FALSE;
1443
1444         g_return_if_fail (SOUP_IS_CACHE (cache));
1445
1446         session = cache->priv->session;
1447         g_return_if_fail (SOUP_IS_SESSION (session));
1448         async_context = soup_session_get_async_context (session);
1449
1450         /* We give cache 10 secs to finish */
1451         timeout_id = g_timeout_add (10000, force_flush_timeout, &forced);
1452
1453         while (!forced && cache->priv->n_pending > 0)
1454                 g_main_context_iteration (async_context, FALSE);
1455
1456         if (!forced)
1457                 g_source_remove (timeout_id);
1458         else
1459                 g_warning ("Cache flush finished despite %d pending requests", cache->priv->n_pending);
1460 }
1461
1462 static void
1463 clear_cache_item (gpointer data,
1464                   gpointer user_data)
1465 {
1466         SoupCache *cache = (SoupCache *) user_data;
1467         SoupCacheEntry *entry = (SoupCacheEntry *) data;
1468
1469         if (soup_cache_entry_remove (cache, entry))
1470                 soup_cache_entry_free (entry, TRUE);
1471 }
1472
1473 /**
1474  * soup_cache_clear:
1475  * @cache: a #SoupCache
1476  *
1477  * Will remove all entries in the @cache plus all the cache files
1478  * associated with them.
1479  **/
1480 void
1481 soup_cache_clear (SoupCache *cache)
1482 {
1483         GHashTable *hash;
1484         GList *entries;
1485
1486         g_return_if_fail (SOUP_IS_CACHE (cache));
1487
1488         hash = cache->priv->cache;
1489         g_return_if_fail (hash);
1490
1491         // Cannot use g_hash_table_foreach as callbacks must not modify the hash table
1492         entries = g_hash_table_get_values (hash);
1493         g_list_foreach (entries, clear_cache_item, cache);
1494         g_list_free (entries);
1495 }
1496
1497 SoupMessage *
1498 soup_cache_generate_conditional_request (SoupCache *cache, SoupMessage *original)
1499 {
1500         SoupMessage *msg;
1501         SoupURI *uri;
1502         SoupCacheEntry *entry;
1503         char *key;
1504         const char *value;
1505
1506         g_return_val_if_fail (SOUP_IS_CACHE (cache), NULL);
1507         g_return_val_if_fail (SOUP_IS_MESSAGE (original), NULL);
1508
1509         /* First copy the data we need from the original message */
1510         uri = soup_message_get_uri (original);
1511         msg = soup_message_new_from_uri (original->method, uri);
1512
1513         soup_message_headers_foreach (original->request_headers,
1514                                       (SoupMessageHeadersForeachFunc)copy_headers,
1515                                       msg->request_headers);
1516
1517         /* Now add the validator entries in the header from the cached
1518            data */
1519         key = soup_message_get_cache_key (original);
1520         entry = g_hash_table_lookup (cache->priv->cache, key);
1521         g_free (key);
1522
1523         g_return_val_if_fail (entry, NULL);
1524
1525         entry->being_validated = TRUE;
1526
1527         value = soup_message_headers_get (entry->headers, "Last-Modified");
1528         if (value)
1529                 soup_message_headers_append (msg->request_headers,
1530                                              "If-Modified-Since",
1531                                              value);
1532         value = soup_message_headers_get (entry->headers, "ETag");
1533         if (value)
1534                 soup_message_headers_append (msg->request_headers,
1535                                              "If-None-Match",
1536                                              value);
1537         return msg;
1538 }
1539
1540 #define SOUP_CACHE_FILE "soup.cache"
1541
1542 #define SOUP_CACHE_HEADERS_FORMAT "{ss}"
1543 #define SOUP_CACHE_PHEADERS_FORMAT "(ssbuuuuua" SOUP_CACHE_HEADERS_FORMAT ")"
1544 #define SOUP_CACHE_ENTRIES_FORMAT "a" SOUP_CACHE_PHEADERS_FORMAT
1545
1546 /* Basically the same format than above except that some strings are
1547    prepended with &. This way the GVariant returns a pointer to the
1548    data instead of duplicating the string */
1549 #define SOUP_CACHE_DECODE_HEADERS_FORMAT "{&s&s}"
1550
1551 static void
1552 pack_entry (gpointer data,
1553             gpointer user_data)
1554 {
1555         SoupCacheEntry *entry = (SoupCacheEntry *) data;
1556         SoupMessageHeadersIter iter;
1557         const gchar *header_key, *header_value;
1558         GVariantBuilder *headers_builder;
1559         GVariantBuilder *entries_builder = (GVariantBuilder *)user_data;
1560
1561         /* Do not store non-consolidated entries */
1562         if (entry->dirty || entry->writing || !entry->key)
1563                 return;
1564
1565         /* Pack headers */
1566         headers_builder = g_variant_builder_new (G_VARIANT_TYPE_ARRAY);
1567         soup_message_headers_iter_init (&iter, entry->headers);
1568         while (soup_message_headers_iter_next (&iter, &header_key, &header_value)) {
1569                 if (g_utf8_validate (header_value, -1, NULL))
1570                         g_variant_builder_add (headers_builder, SOUP_CACHE_HEADERS_FORMAT,
1571                                                header_key, header_value);
1572         }
1573
1574         /* Entry data */
1575         g_variant_builder_add (entries_builder, SOUP_CACHE_PHEADERS_FORMAT,
1576                                entry->key, entry->filename, entry->must_revalidate,
1577                                entry->freshness_lifetime, entry->corrected_initial_age,
1578                                entry->response_time, entry->hits, entry->length, headers_builder);
1579
1580         g_variant_builder_unref (headers_builder);
1581 }
1582
1583 void
1584 soup_cache_dump (SoupCache *cache)
1585 {
1586         SoupCachePrivate *priv = SOUP_CACHE_GET_PRIVATE (cache);
1587         gchar *filename;
1588         GVariantBuilder *entries_builder;
1589         GVariant *cache_variant;
1590
1591         if (!g_list_length (cache->priv->lru_start))
1592                 return;
1593
1594         /* Create the builder and iterate over all entries */
1595         entries_builder = g_variant_builder_new (G_VARIANT_TYPE_ARRAY);
1596         g_list_foreach (cache->priv->lru_start, pack_entry, entries_builder);
1597
1598         /* Serialize and dump */
1599         cache_variant = g_variant_new (SOUP_CACHE_ENTRIES_FORMAT, entries_builder);
1600         g_variant_builder_unref (entries_builder);
1601
1602         filename = g_build_filename (priv->cache_dir, SOUP_CACHE_FILE, NULL);
1603         g_file_set_contents (filename, (const gchar *)g_variant_get_data (cache_variant),
1604                              g_variant_get_size (cache_variant), NULL);
1605         g_free (filename);
1606         g_variant_unref (cache_variant);
1607 }
1608
1609 void
1610 soup_cache_load (SoupCache *cache)
1611 {
1612         gchar *filename = NULL, *contents = NULL;
1613         GVariant *cache_variant;
1614         GVariantIter *entries_iter, *headers_iter;
1615         GVariantType *variant_format;
1616         gsize length;
1617         SoupCacheEntry *entry;
1618         SoupCachePrivate *priv = cache->priv;
1619
1620         filename = g_build_filename (priv->cache_dir, SOUP_CACHE_FILE, NULL);
1621         if (!g_file_get_contents (filename, &contents, &length, NULL)) {
1622                 g_free (filename);
1623                 g_free (contents);
1624                 return;
1625         }
1626         g_free (filename);
1627
1628         variant_format = g_variant_type_new (SOUP_CACHE_ENTRIES_FORMAT);
1629         cache_variant = g_variant_new_from_data (variant_format, (const gchar *)contents, length, FALSE, g_free, contents);
1630         g_variant_type_free (variant_format);
1631
1632         g_variant_get (cache_variant, SOUP_CACHE_ENTRIES_FORMAT, &entries_iter);
1633         entry = g_slice_new0 (SoupCacheEntry);
1634
1635         while (g_variant_iter_loop (entries_iter, SOUP_CACHE_PHEADERS_FORMAT,
1636                                     &entry->key, &entry->filename, &entry->must_revalidate,
1637                                     &entry->freshness_lifetime, &entry->corrected_initial_age,
1638                                     &entry->response_time, &entry->hits, &entry->length,
1639                                     &headers_iter)) {
1640                 const gchar *header_key, *header_value;
1641
1642                 /* SoupMessage Headers */
1643                 entry->headers = soup_message_headers_new (SOUP_MESSAGE_HEADERS_RESPONSE);
1644                 while (g_variant_iter_loop (headers_iter, SOUP_CACHE_DECODE_HEADERS_FORMAT, &header_key, &header_value))
1645                         soup_message_headers_append (entry->headers, header_key, header_value);
1646
1647                 /* Insert in cache */
1648                 if (!soup_cache_entry_insert_by_key (cache, (const gchar *)entry->key, entry, FALSE))
1649                         soup_cache_entry_free (entry, TRUE);
1650
1651                 /* New entry for the next iteration. This creates an
1652                    extra object the last iteration but it's worth it
1653                    as we save several if's */
1654                 entry = g_slice_new0 (SoupCacheEntry);
1655         }
1656         /* Remove last created entry */
1657         g_slice_free (SoupCacheEntry, entry);
1658
1659         /* Sort LRU (shouldn't be needed). First reverse as elements
1660          * are always prepended when inserting
1661          */
1662         cache->priv->lru_start = g_list_reverse (cache->priv->lru_start);
1663         cache->priv->lru_start = g_list_sort (cache->priv->lru_start, lru_compare_func);
1664
1665         /* frees */
1666         g_variant_iter_free (entries_iter);
1667         g_variant_unref (cache_variant);
1668 }
1669
1670 void
1671 soup_cache_set_max_size (SoupCache *cache,
1672                          guint      max_size)
1673 {
1674         cache->priv->max_size = max_size;
1675         cache->priv->max_entry_data_size = cache->priv->max_size / MAX_ENTRY_DATA_PERCENTAGE;
1676 }
1677
1678 guint
1679 soup_cache_get_max_size (SoupCache *cache)
1680 {
1681         return cache->priv->max_size;
1682 }