libsoup/soup-cache.c

   1 /* -*- Mode: C; tab-width: 8; indent-tabs-mode: t; c-basic-offset: 8 -*- */
   2 /*
   3  * soup-cache.c
   4  *
   5  * Copyright (C) 2009, 2010 Igalia S.L.
   6  *
   7  * This library is free software; you can redistribute it and/or
   8  * modify it under the terms of the GNU Library General Public
   9  * License as published by the Free Software Foundation; either
  10  * version 2 of the License, or (at your option) any later version.
  11  *
  12  * This library is distributed in the hope that it will be useful,
  13  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  14  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  15  * Library General Public License for more details.
  16  *
  17  * You should have received a copy of the GNU Library General Public License
  18  * along with this library; see the file COPYING.LIB.  If not, write to
  19  * the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
  20  * Boston, MA 02110-1301, USA.
  21  */
  22
  23 /* TODO:
  24  * - Need to hook the feature in the sync SoupSession.
  25  * - Need more tests.
  26  */
  27
  28 #ifdef HAVE_CONFIG_H
  29 #include <config.h>
  30 #endif
  31
  32 #include <string.h>
  33
  34 #include "soup-cache.h"
  35 #include "soup-body-input-stream.h"
  36 #include "soup-cache-input-stream.h"
  37 #include "soup-cache-private.h"
  38 #include "soup-content-processor.h"
  39 #include "soup-message-private.h"
  40 #include "soup.h"
  41
  42 /**
  43  * SECTION:soup-cache
  44  * @short_description: Caching support
  45  *
  46  * #SoupCache implements a file-based cache for HTTP resources.
  47  */
  48
  49 static SoupSessionFeatureInterface *soup_cache_default_feature_interface;
  50 static void soup_cache_session_feature_init (SoupSessionFeatureInterface *feature_interface, gpointer interface_data);
  51
  52 static SoupContentProcessorInterface *soup_cache_default_content_processor_interface;
  53 static void soup_cache_content_processor_init (SoupContentProcessorInterface *interface, gpointer interface_data);
  54
  55 #define DEFAULT_MAX_SIZE 50 * 1024 * 1024
  56 #define MAX_ENTRY_DATA_PERCENTAGE 10 /* Percentage of the total size
  57                                         of the cache that can be
  58                                         filled by a single entry */
  59
  60 /*
  61  * Version 2: cache is now saved in soup.cache2. Added the version
  62  * number to the beginning of the file.
  63  *
  64  * Version 3: added HTTP status code to the cache entries.
  65  *
  66  * Version 4: replaced several types.
  67  *   - freshness_lifetime,corrected_initial_age,response_time: time_t -> guint32
  68  *   - status_code: guint -> guint16
  69  *   - hits: guint -> guint32
  70  *
  71  * Version 5: key is no longer stored on disk as it can be easily
  72  * built from the URI. Apart from that some fields in the
  73  * SoupCacheEntry have changed:
  74  *   - entry key is now a uint32 instead of a (char *).
  75  *   - added uri, used to check for collisions
  76  *   - removed filename, it's built from the entry key.
  77  */
  78 #define SOUP_CACHE_CURRENT_VERSION 5
  79
  80 #define OLD_SOUP_CACHE_FILE "soup.cache"
  81 #define SOUP_CACHE_FILE "soup.cache2"
  82
  83 #define SOUP_CACHE_HEADERS_FORMAT "{ss}"
  84 #define SOUP_CACHE_PHEADERS_FORMAT "(sbuuuuuqa" SOUP_CACHE_HEADERS_FORMAT ")"
  85 #define SOUP_CACHE_ENTRIES_FORMAT "(qa" SOUP_CACHE_PHEADERS_FORMAT ")"
  86
  87 /* Basically the same format than above except that some strings are
  88    prepended with &. This way the GVariant returns a pointer to the
  89    data instead of duplicating the string */
  90 #define SOUP_CACHE_DECODE_HEADERS_FORMAT "{&s&s}"
  91
  92
  93 typedef struct _SoupCacheEntry {
  94         guint32 key;
  95         char *uri;
  96         guint32 freshness_lifetime;
  97         gboolean must_revalidate;
  98         gsize length;
  99         guint32 corrected_initial_age;
 100         guint32 response_time;
 101         gboolean dirty;
 102         gboolean being_validated;
 103         SoupMessageHeaders *headers;
 104         guint32 hits;
 105         GCancellable *cancellable;
 106         guint16 status_code;
 107 } SoupCacheEntry;
 108
 109 struct _SoupCachePrivate {
 110         char *cache_dir;
 111         GHashTable *cache;
 112         guint n_pending;
 113         SoupSession *session;
 114         SoupCacheType cache_type;
 115         guint size;
 116         guint max_size;
 117         guint max_entry_data_size; /* Computed value. Here for performance reasons */
 118         GList *lru_start;
 119 };
 120
 121 enum {
 122         PROP_0,
 123         PROP_CACHE_DIR,
 124         PROP_CACHE_TYPE
 125 };
 126
 127 #define SOUP_CACHE_GET_PRIVATE(o) (G_TYPE_INSTANCE_GET_PRIVATE ((o), SOUP_TYPE_CACHE, SoupCachePrivate))
 128
 129 G_DEFINE_TYPE_WITH_CODE (SoupCache, soup_cache, G_TYPE_OBJECT,
 130                          G_IMPLEMENT_INTERFACE (SOUP_TYPE_SESSION_FEATURE,
 131                                                 soup_cache_session_feature_init)
 132                          G_IMPLEMENT_INTERFACE (SOUP_TYPE_CONTENT_PROCESSOR,
 133                                                 soup_cache_content_processor_init))
 134
 135 static gboolean soup_cache_entry_remove (SoupCache *cache, SoupCacheEntry *entry, gboolean purge);
 136 static void make_room_for_new_entry (SoupCache *cache, guint length_to_add);
 137 static gboolean cache_accepts_entries_of_size (SoupCache *cache, guint length_to_add);
 138
 139 static GFile *
 140 get_file_from_entry (SoupCache *cache, SoupCacheEntry *entry)
 141 {
 142         char *filename = g_strdup_printf ("%s%s%u", cache->priv->cache_dir,
 143                                           G_DIR_SEPARATOR_S, (guint) entry->key);
 144         GFile *file = g_file_new_for_path (filename);
 145         g_free (filename);
 146
 147         return file;
 148 }
 149
 150 static SoupCacheability
 151 get_cacheability (SoupCache *cache, SoupMessage *msg)
 152 {
 153         SoupCacheability cacheability;
 154         const char *cache_control, *content_type;
 155         gboolean has_max_age = FALSE;
 156
 157         /* 1. The request method must be cacheable */
 158         if (msg->method == SOUP_METHOD_GET)
 159                 cacheability = SOUP_CACHE_CACHEABLE;
 160         else if (msg->method == SOUP_METHOD_HEAD ||
 161                  msg->method == SOUP_METHOD_TRACE ||
 162                  msg->method == SOUP_METHOD_CONNECT)
 163                 return SOUP_CACHE_UNCACHEABLE;
 164         else
 165                 return (SOUP_CACHE_UNCACHEABLE | SOUP_CACHE_INVALIDATES);
 166
 167         content_type = soup_message_headers_get_content_type (msg->response_headers, NULL);
 168         if (content_type && !g_ascii_strcasecmp (content_type, "multipart/x-mixed-replace"))
 169                 return SOUP_CACHE_UNCACHEABLE;
 170
 171         cache_control = soup_message_headers_get_list (msg->response_headers, "Cache-Control");
 172         if (cache_control && *cache_control) {
 173                 GHashTable *hash;
 174                 SoupCachePrivate *priv = SOUP_CACHE_GET_PRIVATE (cache);
 175
 176                 hash = soup_header_parse_param_list (cache_control);
 177
 178                 /* Shared caches MUST NOT store private resources */
 179                 if (priv->cache_type == SOUP_CACHE_SHARED) {
 180                         if (g_hash_table_lookup_extended (hash, "private", NULL, NULL)) {
 181                                 soup_header_free_param_list (hash);
 182                                 return SOUP_CACHE_UNCACHEABLE;
 183                         }
 184                 }
 185
 186                 /* 2. The 'no-store' cache directive does not appear in the
 187                  * headers
 188                  */
 189                 if (g_hash_table_lookup_extended (hash, "no-store", NULL, NULL)) {
 190                         soup_header_free_param_list (hash);
 191                         return SOUP_CACHE_UNCACHEABLE;
 192                 }
 193
 194                 if (g_hash_table_lookup_extended (hash, "max-age", NULL, NULL))
 195                         has_max_age = TRUE;
 196
 197                 /* This does not appear in section 2.1, but I think it makes
 198                  * sense to check it too?
 199                  */
 200                 if (g_hash_table_lookup_extended (hash, "no-cache", NULL, NULL)) {
 201                         soup_header_free_param_list (hash);
 202                         return SOUP_CACHE_UNCACHEABLE;
 203                 }
 204
 205                 soup_header_free_param_list (hash);
 206         }
 207
 208         /* Section 13.9 */
 209         if ((soup_message_get_uri (msg))->query &&
 210             !soup_message_headers_get_one (msg->response_headers, "Expires") &&
 211             !has_max_age)
 212                 return SOUP_CACHE_UNCACHEABLE;
 213
 214         switch (msg->status_code) {
 215         case SOUP_STATUS_PARTIAL_CONTENT:
 216                 /* We don't cache partial responses, but they only
 217                  * invalidate cached full responses if the headers
 218                  * don't match.
 219                  */
 220                 cacheability = SOUP_CACHE_UNCACHEABLE;
 221                 break;
 222
 223         case SOUP_STATUS_NOT_MODIFIED:
 224                 /* A 304 response validates an existing cache entry */
 225                 cacheability = SOUP_CACHE_VALIDATES;
 226                 break;
 227
 228         case SOUP_STATUS_MULTIPLE_CHOICES:
 229         case SOUP_STATUS_MOVED_PERMANENTLY:
 230         case SOUP_STATUS_GONE:
 231                 /* FIXME: cacheable unless indicated otherwise */
 232                 cacheability = SOUP_CACHE_UNCACHEABLE;
 233                 break;
 234
 235         case SOUP_STATUS_FOUND:
 236         case SOUP_STATUS_TEMPORARY_REDIRECT:
 237                 /* FIXME: cacheable if explicitly indicated */
 238                 cacheability = SOUP_CACHE_UNCACHEABLE;
 239                 break;
 240
 241         case SOUP_STATUS_SEE_OTHER:
 242         case SOUP_STATUS_FORBIDDEN:
 243         case SOUP_STATUS_NOT_FOUND:
 244         case SOUP_STATUS_METHOD_NOT_ALLOWED:
 245                 return (SOUP_CACHE_UNCACHEABLE | SOUP_CACHE_INVALIDATES);
 246
 247         default:
 248                 /* Any 5xx status or any 4xx status not handled above
 249                  * is uncacheable but doesn't break the cache.
 250                  */
 251                 if ((msg->status_code >= SOUP_STATUS_BAD_REQUEST &&
 252                      msg->status_code <= SOUP_STATUS_FAILED_DEPENDENCY) ||
 253                     msg->status_code >= SOUP_STATUS_INTERNAL_SERVER_ERROR)
 254                         return SOUP_CACHE_UNCACHEABLE;
 255
 256                 /* An unrecognized 2xx, 3xx, or 4xx response breaks
 257                  * the cache.
 258                  */
 259                 if ((msg->status_code > SOUP_STATUS_PARTIAL_CONTENT &&
 260                      msg->status_code < SOUP_STATUS_MULTIPLE_CHOICES) ||
 261                     (msg->status_code > SOUP_STATUS_TEMPORARY_REDIRECT &&
 262                      msg->status_code < SOUP_STATUS_INTERNAL_SERVER_ERROR))
 263                         return (SOUP_CACHE_UNCACHEABLE | SOUP_CACHE_INVALIDATES);
 264                 break;
 265         }
 266
 267         return cacheability;
 268 }
 269
 270 /* NOTE: this function deletes the file pointed by the file argument
 271  * and also unref's the GFile object representing it.
 272  */
 273 static void
 274 soup_cache_entry_free (SoupCacheEntry *entry)
 275 {
 276         g_free (entry->uri);
 277         g_clear_pointer (&entry->headers, soup_message_headers_free);
 278         g_clear_object (&entry->cancellable);
 279
 280         g_slice_free (SoupCacheEntry, entry);
 281 }
 282
 283 static void
 284 copy_headers (const char *name, const char *value, SoupMessageHeaders *headers)
 285 {
 286         soup_message_headers_append (headers, name, value);
 287 }
 288
 289 static char *hop_by_hop_headers[] = {"Connection", "Keep-Alive", "Proxy-Authenticate", "Proxy-Authorization", "TE", "Trailer", "Transfer-Encoding", "Upgrade"};
 290
 291 static void
 292 copy_end_to_end_headers (SoupMessageHeaders *source, SoupMessageHeaders *destination)
 293 {
 294         int i;
 295
 296         soup_message_headers_foreach (source, (SoupMessageHeadersForeachFunc) copy_headers, destination);
 297         for (i = 0; i < G_N_ELEMENTS (hop_by_hop_headers); i++)
 298                 soup_message_headers_remove (destination, hop_by_hop_headers[i]);
 299         soup_message_headers_clean_connection_headers (destination);
 300 }
 301
 302 static guint
 303 soup_cache_entry_get_current_age (SoupCacheEntry *entry)
 304 {
 305         time_t now = time (NULL);
 306         time_t resident_time;
 307
 308         resident_time = now - entry->response_time;
 309         return entry->corrected_initial_age + resident_time;
 310 }
 311
 312 static gboolean
 313 soup_cache_entry_is_fresh_enough (SoupCacheEntry *entry, gint min_fresh)
 314 {
 315         guint limit = (min_fresh == -1) ? soup_cache_entry_get_current_age (entry) : (guint) min_fresh;
 316         return entry->freshness_lifetime > limit;
 317 }
 318
 319 static inline guint32
 320 get_cache_key_from_uri (const char *uri)
 321 {
 322         return (guint32) g_str_hash (uri);
 323 }
 324
 325 static void
 326 soup_cache_entry_set_freshness (SoupCacheEntry *entry, SoupMessage *msg, SoupCache *cache)
 327 {
 328         const char *cache_control;
 329         const char *expires, *date, *last_modified;
 330
 331         cache_control = soup_message_headers_get_list (entry->headers, "Cache-Control");
 332         if (cache_control && *cache_control) {
 333                 const char *max_age, *s_maxage;
 334                 gint64 freshness_lifetime = 0;
 335                 GHashTable *hash;
 336                 SoupCachePrivate *priv = SOUP_CACHE_GET_PRIVATE (cache);
 337
 338                 hash = soup_header_parse_param_list (cache_control);
 339
 340                 /* Should we re-validate the entry when it goes stale */
 341                 entry->must_revalidate = g_hash_table_lookup_extended (hash, "must-revalidate", NULL, NULL);
 342
 343                 /* Section 2.3.1 */
 344                 if (priv->cache_type == SOUP_CACHE_SHARED) {
 345                         s_maxage = g_hash_table_lookup (hash, "s-maxage");
 346                         if (s_maxage) {
 347                                 freshness_lifetime = g_ascii_strtoll (s_maxage, NULL, 10);
 348                                 if (freshness_lifetime) {
 349                                         /* Implies proxy-revalidate. TODO: is it true? */
 350                                         entry->must_revalidate = TRUE;
 351                                         soup_header_free_param_list (hash);
 352                                         return;
 353                                 }
 354                         }
 355                 }
 356
 357                 /* If 'max-age' cache directive is present, use that */
 358                 max_age = g_hash_table_lookup (hash, "max-age");
 359                 if (max_age)
 360                         freshness_lifetime = g_ascii_strtoll (max_age, NULL, 10);
 361
 362                 if (freshness_lifetime) {
 363                         entry->freshness_lifetime = (guint32) MIN (freshness_lifetime, G_MAXUINT32);
 364                         soup_header_free_param_list (hash);
 365                         return;
 366                 }
 367
 368                 soup_header_free_param_list (hash);
 369         }
 370
 371         /* If the 'Expires' response header is present, use its value
 372          * minus the value of the 'Date' response header
 373          */
 374         expires = soup_message_headers_get_one (entry->headers, "Expires");
 375         date = soup_message_headers_get_one (entry->headers, "Date");
 376         if (expires && date) {
 377                 SoupDate *expires_d, *date_d;
 378                 time_t expires_t, date_t;
 379
 380                 expires_d = soup_date_new_from_string (expires);
 381                 if (expires_d) {
 382                         date_d = soup_date_new_from_string (date);
 383
 384                         expires_t = soup_date_to_time_t (expires_d);
 385                         date_t = soup_date_to_time_t (date_d);
 386
 387                         soup_date_free (expires_d);
 388                         soup_date_free (date_d);
 389
 390                         if (expires_t && date_t) {
 391                                 entry->freshness_lifetime = (guint32) MAX (expires_t - date_t, 0);
 392                                 return;
 393                         }
 394                 } else {
 395                         /* If Expires is not a valid date we should
 396                            treat it as already expired, see section
 397                            3.3 */
 398                         entry->freshness_lifetime = 0;
 399                         return;
 400                 }
 401         }
 402
 403         /* Otherwise an heuristic may be used */
 404
 405         /* Heuristics MUST NOT be used with stored responses with
 406            these status codes (section 2.3.1.1) */
 407         if (entry->status_code != SOUP_STATUS_OK &&
 408             entry->status_code != SOUP_STATUS_NON_AUTHORITATIVE &&
 409             entry->status_code != SOUP_STATUS_PARTIAL_CONTENT &&
 410             entry->status_code != SOUP_STATUS_MULTIPLE_CHOICES &&
 411             entry->status_code != SOUP_STATUS_MOVED_PERMANENTLY &&
 412             entry->status_code != SOUP_STATUS_GONE)
 413                 goto expire;
 414
 415         /* TODO: attach warning 113 if response's current_age is more
 416            than 24h (section 2.3.1.1) when using heuristics */
 417
 418         /* Last-Modified based heuristic */
 419         last_modified = soup_message_headers_get_one (entry->headers, "Last-Modified");
 420         if (last_modified) {
 421                 SoupDate *soup_date;
 422                 time_t now, last_modified_t;
 423
 424                 soup_date = soup_date_new_from_string (last_modified);
 425                 last_modified_t = soup_date_to_time_t (soup_date);
 426                 now = time (NULL);
 427
 428 #define HEURISTIC_FACTOR 0.1 /* From Section 2.3.1.1 */
 429
 430                 entry->freshness_lifetime = MAX (0, (now - last_modified_t) * HEURISTIC_FACTOR);
 431                 soup_date_free (soup_date);
 432         }
 433
 434         return;
 435
 436  expire:
 437         /* If all else fails, make the entry expire immediately */
 438         entry->freshness_lifetime = 0;
 439 }
 440
 441 static SoupCacheEntry *
 442 soup_cache_entry_new (SoupCache *cache, SoupMessage *msg, time_t request_time, time_t response_time)
 443 {
 444         SoupCacheEntry *entry;
 445         const char *date;
 446
 447         entry = g_slice_new0 (SoupCacheEntry);
 448         entry->dirty = FALSE;
 449         entry->being_validated = FALSE;
 450         entry->status_code = msg->status_code;
 451         entry->response_time = response_time;
 452         entry->uri = soup_uri_to_string (soup_message_get_uri (msg), FALSE);
 453
 454         /* Headers */
 455         entry->headers = soup_message_headers_new (SOUP_MESSAGE_HEADERS_RESPONSE);
 456         copy_end_to_end_headers (msg->response_headers, entry->headers);
 457
 458         /* LRU list */
 459         entry->hits = 0;
 460
 461         /* Section 2.3.1, Freshness Lifetime */
 462         soup_cache_entry_set_freshness (entry, msg, cache);
 463
 464         /* Section 2.3.2, Calculating Age */
 465         date = soup_message_headers_get_one (entry->headers, "Date");
 466
 467         if (date) {
 468                 SoupDate *soup_date;
 469                 const char *age;
 470                 time_t date_value, apparent_age, corrected_received_age, response_delay, age_value = 0;
 471
 472                 soup_date = soup_date_new_from_string (date);
 473                 date_value = soup_date_to_time_t (soup_date);
 474                 soup_date_free (soup_date);
 475
 476                 age = soup_message_headers_get_one (entry->headers, "Age");
 477                 if (age)
 478                         age_value = g_ascii_strtoll (age, NULL, 10);
 479
 480                 apparent_age = MAX (0, entry->response_time - date_value);
 481                 corrected_received_age = MAX (apparent_age, age_value);
 482                 response_delay = entry->response_time - request_time;
 483                 entry->corrected_initial_age = corrected_received_age + response_delay;
 484         } else {
 485                 /* Is this correct ? */
 486                 entry->corrected_initial_age = time (NULL);
 487         }
 488
 489         return entry;
 490 }
 491
 492 static gboolean
 493 soup_cache_entry_remove (SoupCache *cache, SoupCacheEntry *entry, gboolean purge)
 494 {
 495         GList *lru_item;
 496
 497         if (entry->dirty) {
 498                 g_cancellable_cancel (entry->cancellable);
 499                 return FALSE;
 500         }
 501
 502         g_assert (!entry->dirty);
 503         g_assert (g_list_length (cache->priv->lru_start) == g_hash_table_size (cache->priv->cache));
 504
 505         if (!g_hash_table_remove (cache->priv->cache, GUINT_TO_POINTER (entry->key)))
 506                 return FALSE;
 507
 508         /* Remove from LRU */
 509         lru_item = g_list_find (cache->priv->lru_start, entry);
 510         cache->priv->lru_start = g_list_delete_link (cache->priv->lru_start, lru_item);
 511
 512         /* Adjust cache size */
 513         cache->priv->size -= entry->length;
 514
 515         g_assert (g_list_length (cache->priv->lru_start) == g_hash_table_size (cache->priv->cache));
 516
 517         /* Free resources */
 518         if (purge) {
 519                 GFile *file = get_file_from_entry (cache, entry);
 520                 g_file_delete (file, NULL, NULL);
 521                 g_object_unref (file);
 522         }
 523         soup_cache_entry_free (entry);
 524
 525         return TRUE;
 526 }
 527
 528 static gint
 529 lru_compare_func (gconstpointer a, gconstpointer b)
 530 {
 531         SoupCacheEntry *entry_a = (SoupCacheEntry *)a;
 532         SoupCacheEntry *entry_b = (SoupCacheEntry *)b;
 533
 534         /* The rationale of this sorting func is
 535          *
 536          * 1. sort by hits -> LRU algorithm, then
 537          *
 538          * 2. sort by freshness lifetime, we better discard first
 539          * entries that are close to expire
 540          *
 541          * 3. sort by size, replace first small size resources as they
 542          * are cheaper to download
 543          */
 544
 545         /* Sort by hits */
 546         if (entry_a->hits != entry_b->hits)
 547                 return entry_a->hits - entry_b->hits;
 548
 549         /* Sort by freshness_lifetime */
 550         if (entry_a->freshness_lifetime != entry_b->freshness_lifetime)
 551                 return entry_a->freshness_lifetime - entry_b->freshness_lifetime;
 552
 553         /* Sort by size */
 554         return entry_a->length - entry_b->length;
 555 }
 556
 557 static gboolean
 558 cache_accepts_entries_of_size (SoupCache *cache, guint length_to_add)
 559 {
 560         /* We could add here some more heuristics. TODO: review how
 561            this is done by other HTTP caches */
 562
 563         return length_to_add <= cache->priv->max_entry_data_size;
 564 }
 565
 566 static void
 567 make_room_for_new_entry (SoupCache *cache, guint length_to_add)
 568 {
 569         GList *lru_entry = cache->priv->lru_start;
 570
 571         /* Check that there is enough room for the new entry. This is
 572            an approximation as we're not working out the size of the
 573            cache file or the size of the headers for performance
 574            reasons. TODO: check if that would be really that expensive */
 575
 576         while (lru_entry &&
 577                (length_to_add + cache->priv->size > cache->priv->max_size)) {
 578                 SoupCacheEntry *old_entry = (SoupCacheEntry *)lru_entry->data;
 579
 580                 /* Discard entries. Once cancelled resources will be
 581                  * freed in close_ready_cb
 582                  */
 583                 if (soup_cache_entry_remove (cache, old_entry, TRUE))
 584                         lru_entry = cache->priv->lru_start;
 585                 else
 586                         lru_entry = g_list_next (lru_entry);
 587         }
 588 }
 589
 590 static gboolean
 591 soup_cache_entry_insert (SoupCache *cache,
 592                          SoupCacheEntry *entry,
 593                          gboolean sort)
 594 {
 595         guint length_to_add = 0;
 596         SoupCacheEntry *old_entry;
 597
 598         /* Fill the key */
 599         entry->key = get_cache_key_from_uri ((const char *) entry->uri);
 600
 601         if (soup_message_headers_get_encoding (entry->headers) == SOUP_ENCODING_CONTENT_LENGTH)
 602                 length_to_add = soup_message_headers_get_content_length (entry->headers);
 603
 604         /* Check if we are going to store the resource depending on its size */
 605         if (length_to_add) {
 606                 if (!cache_accepts_entries_of_size (cache, length_to_add))
 607                         return FALSE;
 608
 609                 /* Make room for new entry if needed */
 610                 make_room_for_new_entry (cache, length_to_add);
 611         }
 612
 613         /* Remove any previous entry */
 614         if ((old_entry = g_hash_table_lookup (cache->priv->cache, GUINT_TO_POINTER (entry->key))) != NULL) {
 615                 if (!soup_cache_entry_remove (cache, old_entry, TRUE))
 616                         return FALSE;
 617         }
 618
 619         /* Add to hash table */
 620         g_hash_table_insert (cache->priv->cache, GUINT_TO_POINTER (entry->key), entry);
 621
 622         /* Compute new cache size */
 623         cache->priv->size += length_to_add;
 624
 625         /* Update LRU */
 626         if (sort)
 627                 cache->priv->lru_start = g_list_insert_sorted (cache->priv->lru_start, entry, lru_compare_func);
 628         else
 629                 cache->priv->lru_start = g_list_prepend (cache->priv->lru_start, entry);
 630
 631         g_assert (g_list_length (cache->priv->lru_start) == g_hash_table_size (cache->priv->cache));
 632
 633         return TRUE;
 634 }
 635
 636 static SoupCacheEntry*
 637 soup_cache_entry_lookup (SoupCache *cache,
 638                          SoupMessage *msg)
 639 {
 640         SoupCacheEntry *entry;
 641         guint32 key;
 642         char *uri = NULL;
 643
 644         uri = soup_uri_to_string (soup_message_get_uri (msg), FALSE);
 645         key = get_cache_key_from_uri ((const char *) uri);
 646
 647         entry = g_hash_table_lookup (cache->priv->cache, GUINT_TO_POINTER (key));
 648
 649         if (entry != NULL && (strcmp (entry->uri, uri) != 0))
 650                 entry = NULL;
 651
 652         g_free (uri);
 653         return entry;
 654 }
 655
 656 GInputStream *
 657 soup_cache_send_response (SoupCache *cache, SoupMessage *msg)
 658 {
 659         SoupCacheEntry *entry;
 660         char *current_age;
 661         GInputStream *file_stream, *body_stream, *cache_stream;
 662         GFile *file;
 663
 664         g_return_val_if_fail (SOUP_IS_CACHE (cache), NULL);
 665         g_return_val_if_fail (SOUP_IS_MESSAGE (msg), NULL);
 666
 667         entry = soup_cache_entry_lookup (cache, msg);
 668         g_return_val_if_fail (entry, NULL);
 669
 670         file = get_file_from_entry (cache, entry);
 671         file_stream = G_INPUT_STREAM (g_file_read (file, NULL, NULL));
 672         g_object_unref (file);
 673
 674         /* Do not change the original message if there is no resource */
 675         if (!file_stream)
 676                 return NULL;
 677
 678         body_stream = soup_body_input_stream_new (file_stream, SOUP_ENCODING_CONTENT_LENGTH, entry->length);
 679         g_object_unref (file_stream);
 680
 681         if (!body_stream)
 682                 return NULL;
 683
 684         /* If we are told to send a response from cache any validation
 685            in course is over by now */
 686         entry->being_validated = FALSE;
 687
 688         /* Status */
 689         soup_message_set_status (msg, entry->status_code);
 690
 691         /* Headers */
 692         copy_end_to_end_headers (entry->headers, msg->response_headers);
 693
 694         /* Add 'Age' header with the current age */
 695         current_age = g_strdup_printf ("%d", soup_cache_entry_get_current_age (entry));
 696         soup_message_headers_replace (msg->response_headers,
 697                                       "Age",
 698                                       current_age);
 699         g_free (current_age);
 700
 701         /* Create the cache stream. */
 702         soup_message_disable_feature (msg, SOUP_TYPE_CACHE);
 703         cache_stream = soup_message_setup_body_istream (body_stream, msg,
 704                                                         cache->priv->session,
 705                                                         SOUP_STAGE_ENTITY_BODY);
 706         g_object_unref (body_stream);
 707
 708         return cache_stream;
 709 }
 710
 711 static void
 712 msg_got_headers_cb (SoupMessage *msg, gpointer user_data)
 713 {
 714         g_object_set_data (G_OBJECT (msg), "response-time", GINT_TO_POINTER (time (NULL)));
 715         g_signal_handlers_disconnect_by_func (msg, msg_got_headers_cb, user_data);
 716 }
 717
 718 static void
 719 request_started (SoupSessionFeature *feature, SoupSession *session,
 720                  SoupMessage *msg, SoupSocket *socket)
 721 {
 722         g_object_set_data (G_OBJECT (msg), "request-time", GINT_TO_POINTER (time (NULL)));
 723         g_signal_connect (msg, "got-headers", G_CALLBACK (msg_got_headers_cb), NULL);
 724 }
 725
 726 static void
 727 attach (SoupSessionFeature *feature, SoupSession *session)
 728 {
 729         SoupCache *cache = SOUP_CACHE (feature);
 730         cache->priv->session = session;
 731
 732         soup_cache_default_feature_interface->attach (feature, session);
 733 }
 734
 735 static void
 736 soup_cache_session_feature_init (SoupSessionFeatureInterface *feature_interface,
 737                                         gpointer interface_data)
 738 {
 739         soup_cache_default_feature_interface =
 740                 g_type_default_interface_peek (SOUP_TYPE_SESSION_FEATURE);
 741
 742         feature_interface->attach = attach;
 743         feature_interface->request_started = request_started;
 744 }
 745
 746 typedef struct {
 747         SoupCache *cache;
 748         SoupCacheEntry *entry;
 749 } StreamHelper;
 750
 751 static void
 752 istream_cache_cb (GObject *source,
 753                   GAsyncResult *res,
 754                   gpointer user_data)
 755 {
 756         SoupCacheInputStream *istream = SOUP_CACHE_INPUT_STREAM (source);
 757         StreamHelper *helper = (StreamHelper *) user_data;
 758         SoupCache *cache = helper->cache;
 759         SoupCacheEntry *entry = helper->entry;
 760         GError *error = NULL;
 761
 762         entry->dirty = FALSE;
 763         g_clear_object (&entry->cancellable);
 764         --cache->priv->n_pending;
 765
 766         entry->length = soup_cache_input_stream_cache_finish (istream, res, &error);
 767
 768         if (error) {
 769                 /* Update cache size */
 770                 if (soup_message_headers_get_encoding (entry->headers) == SOUP_ENCODING_CONTENT_LENGTH)
 771                         cache->priv->size -= soup_message_headers_get_content_length (entry->headers);
 772
 773                 soup_cache_entry_remove (cache, entry, TRUE);
 774                 helper->entry = entry = NULL;
 775                 goto cleanup;
 776         }
 777
 778         if (soup_message_headers_get_encoding (entry->headers) != SOUP_ENCODING_CONTENT_LENGTH) {
 779
 780                 if (cache_accepts_entries_of_size (cache, entry->length)) {
 781                         make_room_for_new_entry (cache, entry->length);
 782                         cache->priv->size += entry->length;
 783                 } else {
 784                         soup_cache_entry_remove (cache, entry, TRUE);
 785                         helper->entry = entry = NULL;
 786                 }
 787         }
 788
 789  cleanup:
 790         g_object_unref (helper->cache);
 791         g_slice_free (StreamHelper, helper);
 792 }
 793
 794
 795 static GInputStream*
 796 soup_cache_content_processor_wrap_input (SoupContentProcessor *processor,
 797                                          GInputStream *base_stream,
 798                                          SoupMessage *msg,
 799                                          GError **error)
 800 {
 801         SoupCache *cache = (SoupCache*) processor;
 802         SoupCacheEntry *entry;
 803         SoupCacheability cacheability;
 804         GInputStream *istream;
 805         GFile *file;
 806         StreamHelper *helper;
 807         time_t request_time, response_time;
 808
 809         /* First of all, check if we should cache the resource. */
 810         cacheability = soup_cache_get_cacheability (cache, msg);
 811         entry = soup_cache_entry_lookup (cache, msg);
 812
 813         if (cacheability & SOUP_CACHE_INVALIDATES) {
 814                 if (entry)
 815                         soup_cache_entry_remove (cache, entry, TRUE);
 816                 return NULL;
 817         }
 818
 819         if (cacheability & SOUP_CACHE_VALIDATES) {
 820                 /* It's possible to get a CACHE_VALIDATES with no
 821                  * entry in the hash table. This could happen if for
 822                  * example the soup client is the one creating the
 823                  * conditional request.
 824                  */
 825                 if (entry) {
 826                         entry->being_validated = FALSE;
 827                         copy_end_to_end_headers (msg->response_headers, entry->headers);
 828                         soup_cache_entry_set_freshness (entry, msg, cache);
 829                 }
 830                 return NULL;
 831         }
 832
 833         if (!(cacheability & SOUP_CACHE_CACHEABLE))
 834                 return NULL;
 835
 836         /* Check if we are already caching this resource */
 837         if (entry && (entry->dirty || entry->being_validated))
 838                 return NULL;
 839
 840         /* Create a new entry, deleting any old one if present */
 841         if (entry)
 842                 soup_cache_entry_remove (cache, entry, TRUE);
 843
 844         request_time = GPOINTER_TO_INT (g_object_get_data (G_OBJECT (msg), "request-time"));
 845         response_time = GPOINTER_TO_INT (g_object_get_data (G_OBJECT (msg), "request-time"));
 846         entry = soup_cache_entry_new (cache, msg, request_time, response_time);
 847         entry->hits = 1;
 848         entry->dirty = TRUE;
 849
 850         /* Do not continue if it can not be stored */
 851         if (!soup_cache_entry_insert (cache, entry, TRUE)) {
 852                 soup_cache_entry_free (entry);
 853                 return NULL;
 854         }
 855
 856         ++cache->priv->n_pending;
 857
 858         istream = soup_cache_input_stream_new (base_stream);
 859
 860         file = get_file_from_entry (cache, entry);
 861         entry->cancellable = g_cancellable_new ();
 862
 863         helper = g_slice_new (StreamHelper);
 864         helper->cache = g_object_ref (cache);
 865         helper->entry = entry;
 866
 867         soup_cache_input_stream_cache (SOUP_CACHE_INPUT_STREAM (istream), file, entry->cancellable,
 868                                        (GAsyncReadyCallback) istream_cache_cb, helper);
 869         g_object_unref (file);
 870
 871         return istream;
 872 }
 873
 874 static void
 875 soup_cache_content_processor_init (SoupContentProcessorInterface *processor_interface,
 876                                    gpointer interface_data)
 877 {
 878         soup_cache_default_content_processor_interface =
 879                 g_type_default_interface_peek (SOUP_TYPE_CONTENT_PROCESSOR);
 880
 881         processor_interface->processing_stage = SOUP_STAGE_ENTITY_BODY;
 882         processor_interface->wrap_input = soup_cache_content_processor_wrap_input;
 883 }
 884
 885 static void
 886 soup_cache_init (SoupCache *cache)
 887 {
 888         SoupCachePrivate *priv;
 889
 890         priv = cache->priv = SOUP_CACHE_GET_PRIVATE (cache);
 891
 892         priv->cache = g_hash_table_new (g_direct_hash, g_direct_equal);
 893         /* LRU */
 894         priv->lru_start = NULL;
 895
 896         /* */
 897         priv->n_pending = 0;
 898
 899         /* Cache size */
 900         priv->max_size = DEFAULT_MAX_SIZE;
 901         priv->max_entry_data_size = priv->max_size / MAX_ENTRY_DATA_PERCENTAGE;
 902         priv->size = 0;
 903 }
 904
 905 static void
 906 remove_cache_item (gpointer data,
 907                    gpointer user_data)
 908 {
 909         soup_cache_entry_remove ((SoupCache *) user_data, (SoupCacheEntry *) data, FALSE);
 910 }
 911
 912 static void
 913 soup_cache_finalize (GObject *object)
 914 {
 915         SoupCachePrivate *priv;
 916         GList *entries;
 917
 918         priv = SOUP_CACHE (object)->priv;
 919
 920         // Cannot use g_hash_table_foreach as callbacks must not modify the hash table
 921         entries = g_hash_table_get_values (priv->cache);
 922         g_list_foreach (entries, remove_cache_item, object);
 923         g_list_free (entries);
 924
 925         g_hash_table_destroy (priv->cache);
 926         g_free (priv->cache_dir);
 927
 928         g_list_free (priv->lru_start);
 929
 930         G_OBJECT_CLASS (soup_cache_parent_class)->finalize (object);
 931 }
 932
 933 static void
 934 soup_cache_set_property (GObject *object, guint prop_id,
 935                                 const GValue *value, GParamSpec *pspec)
 936 {
 937         SoupCachePrivate *priv = SOUP_CACHE (object)->priv;
 938
 939         switch (prop_id) {
 940         case PROP_CACHE_DIR:
 941                 priv->cache_dir = g_value_dup_string (value);
 942                 /* Create directory if it does not exist (FIXME: should we?) */
 943                 if (!g_file_test (priv->cache_dir, G_FILE_TEST_EXISTS | G_FILE_TEST_IS_DIR))
 944                         g_mkdir_with_parents (priv->cache_dir, 0700);
 945                 break;
 946         case PROP_CACHE_TYPE:
 947                 priv->cache_type = g_value_get_enum (value);
 948                 /* TODO: clear private entries and issue a warning if moving to shared? */
 949                 break;
 950         default:
 951                 G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
 952                 break;
 953         }
 954 }
 955
 956 static void
 957 soup_cache_get_property (GObject *object, guint prop_id,
 958                          GValue *value, GParamSpec *pspec)
 959 {
 960         SoupCachePrivate *priv = SOUP_CACHE (object)->priv;
 961
 962         switch (prop_id) {
 963         case PROP_CACHE_DIR:
 964                 g_value_set_string (value, priv->cache_dir);
 965                 break;
 966         case PROP_CACHE_TYPE:
 967                 g_value_set_enum (value, priv->cache_type);
 968                 break;
 969         default:
 970                 G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
 971                 break;
 972         }
 973 }
 974
 975 static void
 976 soup_cache_constructed (GObject *object)
 977 {
 978         SoupCachePrivate *priv;
 979
 980         priv = SOUP_CACHE (object)->priv;
 981
 982         if (!priv->cache_dir) {
 983                 /* Set a default cache dir, different for each user */
 984                 priv->cache_dir = g_build_filename (g_get_user_cache_dir (),
 985                                                     "httpcache",
 986                                                     NULL);
 987                 if (!g_file_test (priv->cache_dir, G_FILE_TEST_EXISTS | G_FILE_TEST_IS_DIR))
 988                         g_mkdir_with_parents (priv->cache_dir, 0700);
 989         }
 990
 991         if (G_OBJECT_CLASS (soup_cache_parent_class)->constructed)
 992                 G_OBJECT_CLASS (soup_cache_parent_class)->constructed (object);
 993 }
 994
 995 static void
 996 soup_cache_class_init (SoupCacheClass *cache_class)
 997 {
 998         GObjectClass *gobject_class = (GObjectClass *)cache_class;
 999
1000         gobject_class->finalize = soup_cache_finalize;
1001         gobject_class->constructed = soup_cache_constructed;
1002         gobject_class->set_property = soup_cache_set_property;
1003         gobject_class->get_property = soup_cache_get_property;
1004
1005         cache_class->get_cacheability = get_cacheability;
1006
1007         g_object_class_install_property (gobject_class, PROP_CACHE_DIR,
1008                                          g_param_spec_string ("cache-dir",
1009                                                               "Cache directory",
1010                                                               "The directory to store the cache files",
1011                                                               NULL,
1012                                                               G_PARAM_READWRITE | G_PARAM_CONSTRUCT_ONLY));
1013
1014         g_object_class_install_property (gobject_class, PROP_CACHE_TYPE,
1015                                          g_param_spec_enum ("cache-type",
1016                                                             "Cache type",
1017                                                             "Whether the cache is private or shared",
1018                                                             SOUP_TYPE_CACHE_TYPE,
1019                                                             SOUP_CACHE_SINGLE_USER,
1020                                                             G_PARAM_READWRITE | G_PARAM_CONSTRUCT_ONLY));
1021
1022         g_type_class_add_private (cache_class, sizeof (SoupCachePrivate));
1023 }
1024
1025 /**
1026  * SoupCacheType:
1027  * @SOUP_CACHE_SINGLE_USER: a single-user cache
1028  * @SOUP_CACHE_SHARED: a shared cache
1029  *
1030  * The type of cache; this affects what kinds of responses will be
1031  * saved.
1032  *
1033  * Since: 2.34
1034  */
1035
1036 /**
1037  * soup_cache_new:
1038  * @cache_dir: the directory to store the cached data, or %NULL to use the default one
1039  * @cache_type: the #SoupCacheType of the cache
1040  *
1041  * Creates a new #SoupCache.
1042  *
1043  * Returns: a new #SoupCache
1044  *
1045  * Since: 2.34
1046  */
1047 SoupCache *
1048 soup_cache_new (const char *cache_dir, SoupCacheType cache_type)
1049 {
1050         return g_object_new (SOUP_TYPE_CACHE,
1051                              "cache-dir", cache_dir,
1052                              "cache-type", cache_type,
1053                              NULL);
1054 }
1055
1056 /**
1057  * soup_cache_has_response:
1058  * @cache: a #SoupCache
1059  * @msg: a #SoupMessage
1060  *
1061  * This function calculates whether the @cache object has a proper
1062  * response for the request @msg given the flags both in the request
1063  * and the cached reply and the time ellapsed since it was cached.
1064  *
1065  * Returns: whether or not the @cache has a valid response for @msg
1066  *
1067  * Since: 2.34
1068  */
1069 SoupCacheResponse
1070 soup_cache_has_response (SoupCache *cache, SoupMessage *msg)
1071 {
1072         SoupCacheEntry *entry;
1073         const char *cache_control, *pragma;
1074         gpointer value;
1075         int max_age, max_stale, min_fresh;
1076         GList *lru_item, *item;
1077
1078         entry = soup_cache_entry_lookup (cache, msg);
1079
1080         /* 1. The presented Request-URI and that of stored response
1081          * match
1082          */
1083         if (!entry)
1084                 return SOUP_CACHE_RESPONSE_STALE;
1085
1086         /* Increase hit count. Take sorting into account */
1087         entry->hits++;
1088         lru_item = g_list_find (cache->priv->lru_start, entry);
1089         item = lru_item;
1090         while (item->next && lru_compare_func (item->data, item->next->data) > 0)
1091                 item = g_list_next (item);
1092
1093         if (item != lru_item) {
1094                 cache->priv->lru_start = g_list_remove_link (cache->priv->lru_start, lru_item);
1095                 item = g_list_insert_sorted (item, lru_item->data, lru_compare_func);
1096                 g_list_free (lru_item);
1097         }
1098
1099         if (entry->dirty || entry->being_validated)
1100                 return SOUP_CACHE_RESPONSE_STALE;
1101
1102         /* 2. The request method associated with the stored response
1103          *  allows it to be used for the presented request
1104          */
1105
1106         /* In practice this means we only return our resource for GET,
1107          * cacheability for other methods is a TODO in the RFC
1108          * (TODO: although we could return the headers for HEAD
1109          * probably).
1110          */
1111         if (msg->method != SOUP_METHOD_GET)
1112                 return SOUP_CACHE_RESPONSE_STALE;
1113
1114         /* 3. Selecting request-headers nominated by the stored
1115          * response (if any) match those presented.
1116          */
1117
1118         /* TODO */
1119
1120         /* 4. The request is a conditional request issued by the client.
1121          */
1122         if (soup_message_headers_get_one (msg->request_headers, "If-Modified-Since") ||
1123             soup_message_headers_get_list (msg->request_headers, "If-None-Match"))
1124                 return SOUP_CACHE_RESPONSE_STALE;
1125
1126         /* 5. The presented request and stored response are free from
1127          * directives that would prevent its use.
1128          */
1129
1130         max_age = max_stale = min_fresh = -1;
1131
1132         /* For HTTP 1.0 compatibility. RFC2616 section 14.9.4
1133          */
1134         pragma = soup_message_headers_get_list (msg->request_headers, "Pragma");
1135         if (pragma && soup_header_contains (pragma, "no-cache"))
1136                 return SOUP_CACHE_RESPONSE_STALE;
1137
1138         cache_control = soup_message_headers_get_list (msg->request_headers, "Cache-Control");
1139         if (cache_control && *cache_control) {
1140                 GHashTable *hash = soup_header_parse_param_list (cache_control);
1141
1142                 if (g_hash_table_lookup_extended (hash, "no-store", NULL, NULL)) {
1143                         soup_header_free_param_list (hash);
1144                         return SOUP_CACHE_RESPONSE_STALE;
1145                 }
1146
1147                 if (g_hash_table_lookup_extended (hash, "no-cache", NULL, NULL)) {
1148                         soup_header_free_param_list (hash);
1149                         return SOUP_CACHE_RESPONSE_STALE;
1150                 }
1151
1152                 if (g_hash_table_lookup_extended (hash, "max-age", NULL, &value)) {
1153                         max_age = (int)MIN (g_ascii_strtoll (value, NULL, 10), G_MAXINT32);
1154                         /* Forcing cache revalidaton
1155                          */
1156                         if (!max_age) {
1157                                 soup_header_free_param_list (hash);
1158                                 return SOUP_CACHE_RESPONSE_NEEDS_VALIDATION;
1159                         }
1160                 }
1161
1162                 /* max-stale can have no value set, we need to use _extended */
1163                 if (g_hash_table_lookup_extended (hash, "max-stale", NULL, &value)) {
1164                         if (value)
1165                                 max_stale = (int)MIN (g_ascii_strtoll (value, NULL, 10), G_MAXINT32);
1166                         else
1167                                 max_stale = G_MAXINT32;
1168                 }
1169
1170                 value = g_hash_table_lookup (hash, "min-fresh");
1171                 if (value)
1172                         min_fresh = (int)MIN (g_ascii_strtoll (value, NULL, 10), G_MAXINT32);
1173
1174                 soup_header_free_param_list (hash);
1175
1176                 if (max_age > 0) {
1177                         guint current_age = soup_cache_entry_get_current_age (entry);
1178
1179                         /* If we are over max-age and max-stale is not
1180                            set, do not use the value from the cache
1181                            without validation */
1182                         if ((guint) max_age <= current_age && max_stale == -1)
1183                                 return SOUP_CACHE_RESPONSE_NEEDS_VALIDATION;
1184                 }
1185         }
1186
1187         /* 6. The stored response is either: fresh, allowed to be
1188          * served stale or succesfully validated
1189          */
1190         /* TODO consider also proxy-revalidate & s-maxage */
1191         if (entry->must_revalidate)
1192                 return SOUP_CACHE_RESPONSE_NEEDS_VALIDATION;
1193
1194         if (!soup_cache_entry_is_fresh_enough (entry, min_fresh)) {
1195                 /* Not fresh, can it be served stale? */
1196                 if (max_stale != -1) {
1197                         /* G_MAXINT32 means we accept any staleness */
1198                         if (max_stale == G_MAXINT32)
1199                                 return SOUP_CACHE_RESPONSE_FRESH;
1200
1201                         if ((soup_cache_entry_get_current_age (entry) - entry->freshness_lifetime) <= (guint) max_stale)
1202                                 return SOUP_CACHE_RESPONSE_FRESH;
1203                 }
1204
1205                 return SOUP_CACHE_RESPONSE_NEEDS_VALIDATION;
1206         }
1207
1208         return SOUP_CACHE_RESPONSE_FRESH;
1209 }
1210
1211 /**
1212  * soup_cache_get_cacheability:
1213  * @cache: a #SoupCache
1214  * @msg: a #SoupMessage
1215  *
1216  * Calculates whether the @msg can be cached or not.
1217  *
1218  * Returns: a #SoupCacheability value indicating whether the @msg can be cached or not.
1219  *
1220  * Since: 2.34
1221  */
1222 SoupCacheability
1223 soup_cache_get_cacheability (SoupCache *cache, SoupMessage *msg)
1224 {
1225         g_return_val_if_fail (SOUP_IS_CACHE (cache), SOUP_CACHE_UNCACHEABLE);
1226         g_return_val_if_fail (SOUP_IS_MESSAGE (msg), SOUP_CACHE_UNCACHEABLE);
1227
1228         return SOUP_CACHE_GET_CLASS (cache)->get_cacheability (cache, msg);
1229 }
1230
1231 static gboolean
1232 force_flush_timeout (gpointer data)
1233 {
1234         gboolean *forced = (gboolean *)data;
1235         *forced = TRUE;
1236
1237         return FALSE;
1238 }
1239
1240 /**
1241  * soup_cache_flush:
1242  * @cache: a #SoupCache
1243  *
1244  * This function will force all pending writes in the @cache to be
1245  * committed to disk. For doing so it will iterate the #GMainContext
1246  * associated with @cache's session as long as needed.
1247  *
1248  * Contrast with soup_cache_dump(), which writes out the cache index
1249  * file.
1250  *
1251  * Since: 2.34
1252  */
1253 void
1254 soup_cache_flush (SoupCache *cache)
1255 {
1256         GMainContext *async_context;
1257         SoupSession *session;
1258         GSource *timeout;
1259         gboolean forced = FALSE;
1260
1261         g_return_if_fail (SOUP_IS_CACHE (cache));
1262
1263         session = cache->priv->session;
1264         g_return_if_fail (SOUP_IS_SESSION (session));
1265         async_context = soup_session_get_async_context (session);
1266
1267         /* We give cache 10 secs to finish */
1268         timeout = soup_add_timeout (async_context, 10000, force_flush_timeout, &forced);
1269
1270         while (!forced && cache->priv->n_pending > 0)
1271                 g_main_context_iteration (async_context, FALSE);
1272
1273         if (!forced)
1274                 g_source_destroy (timeout);
1275         else
1276                 g_warning ("Cache flush finished despite %d pending requests", cache->priv->n_pending);
1277 }
1278
1279 static void
1280 clear_cache_item (gpointer data,
1281                   gpointer user_data)
1282 {
1283         soup_cache_entry_remove ((SoupCache *) user_data, (SoupCacheEntry *) data, TRUE);
1284 }
1285
1286 static void
1287 clear_cache_files (SoupCache *cache)
1288 {
1289         GFileInfo *file_info;
1290         GFileEnumerator *file_enumerator;
1291         GFile *cache_dir_file = g_file_new_for_path (cache->priv->cache_dir);
1292
1293         file_enumerator = g_file_enumerate_children (cache_dir_file, G_FILE_ATTRIBUTE_STANDARD_NAME,
1294                                                      G_FILE_QUERY_INFO_NONE, NULL, NULL);
1295         if (file_enumerator) {
1296                 while ((file_info = g_file_enumerator_next_file (file_enumerator, NULL, NULL)) != NULL) {
1297                         const char *filename = g_file_info_get_name (file_info);
1298
1299                         if (strcmp (filename, SOUP_CACHE_FILE) != 0) {
1300                                 GFile *cache_file = g_file_get_child (cache_dir_file, filename);
1301                                 g_file_delete (cache_file, NULL, NULL);
1302                                 g_object_unref (cache_file);
1303                         }
1304                         g_object_unref (file_info);
1305                 }
1306                 g_object_unref (file_enumerator);
1307         }
1308         g_object_unref (cache_dir_file);
1309 }
1310
1311 /**
1312  * soup_cache_clear:
1313  * @cache: a #SoupCache
1314  *
1315  * Will remove all entries in the @cache plus all the cache files.
1316  *
1317  * Since: 2.34
1318  */
1319 void
1320 soup_cache_clear (SoupCache *cache)
1321 {
1322         GList *entries;
1323
1324         g_return_if_fail (SOUP_IS_CACHE (cache));
1325         g_return_if_fail (cache->priv->cache);
1326
1327         // Cannot use g_hash_table_foreach as callbacks must not modify the hash table
1328         entries = g_hash_table_get_values (cache->priv->cache);
1329         g_list_foreach (entries, clear_cache_item, cache);
1330         g_list_free (entries);
1331
1332         /* Remove also any file not associated with a cache entry. */
1333         clear_cache_files (cache);
1334 }
1335
1336 SoupMessage *
1337 soup_cache_generate_conditional_request (SoupCache *cache, SoupMessage *original)
1338 {
1339         SoupMessage *msg;
1340         SoupURI *uri;
1341         SoupCacheEntry *entry;
1342         const char *last_modified, *etag;
1343
1344         g_return_val_if_fail (SOUP_IS_CACHE (cache), NULL);
1345         g_return_val_if_fail (SOUP_IS_MESSAGE (original), NULL);
1346
1347         /* Add the validator entries in the header from the cached data */
1348         entry = soup_cache_entry_lookup (cache, original);
1349         g_return_val_if_fail (entry, NULL);
1350
1351         last_modified = soup_message_headers_get_one (entry->headers, "Last-Modified");
1352         etag = soup_message_headers_get_one (entry->headers, "ETag");
1353
1354         if (!last_modified && !etag)
1355                 return NULL;
1356
1357         entry->being_validated = TRUE;
1358
1359         /* Copy the data we need from the original message */
1360         uri = soup_message_get_uri (original);
1361         msg = soup_message_new_from_uri (original->method, uri);
1362         soup_message_disable_feature (msg, SOUP_TYPE_CACHE);
1363
1364         soup_message_headers_foreach (original->request_headers,
1365                                       (SoupMessageHeadersForeachFunc)copy_headers,
1366                                       msg->request_headers);
1367
1368         if (last_modified)
1369                 soup_message_headers_append (msg->request_headers,
1370                                              "If-Modified-Since",
1371                                              last_modified);
1372         if (etag)
1373                 soup_message_headers_append (msg->request_headers,
1374                                              "If-None-Match",
1375                                              etag);
1376
1377         return msg;
1378 }
1379
1380 static void
1381 pack_entry (gpointer data,
1382             gpointer user_data)
1383 {
1384         SoupCacheEntry *entry = (SoupCacheEntry *) data;
1385         SoupMessageHeadersIter iter;
1386         const char *header_key, *header_value;
1387         GVariantBuilder *entries_builder = (GVariantBuilder *)user_data;
1388
1389         /* Do not store non-consolidated entries */
1390         if (entry->dirty || !entry->key)
1391                 return;
1392
1393         g_variant_builder_open (entries_builder, G_VARIANT_TYPE (SOUP_CACHE_PHEADERS_FORMAT));
1394         g_variant_builder_add (entries_builder, "s", entry->uri);
1395         g_variant_builder_add (entries_builder, "b", entry->must_revalidate);
1396         g_variant_builder_add (entries_builder, "u", entry->freshness_lifetime);
1397         g_variant_builder_add (entries_builder, "u", entry->corrected_initial_age);
1398         g_variant_builder_add (entries_builder, "u", entry->response_time);
1399         g_variant_builder_add (entries_builder, "u", entry->hits);
1400         g_variant_builder_add (entries_builder, "u", entry->length);
1401         g_variant_builder_add (entries_builder, "q", entry->status_code);
1402
1403         /* Pack headers */
1404         g_variant_builder_open (entries_builder, G_VARIANT_TYPE ("a" SOUP_CACHE_HEADERS_FORMAT));
1405         soup_message_headers_iter_init (&iter, entry->headers);
1406         while (soup_message_headers_iter_next (&iter, &header_key, &header_value)) {
1407                 if (g_utf8_validate (header_value, -1, NULL))
1408                         g_variant_builder_add (entries_builder, SOUP_CACHE_HEADERS_FORMAT,
1409                                                header_key, header_value);
1410         }
1411         g_variant_builder_close (entries_builder); /* "a" SOUP_CACHE_HEADERS_FORMAT */
1412         g_variant_builder_close (entries_builder); /* SOUP_CACHE_PHEADERS_FORMAT */
1413 }
1414
1415 /**
1416  * soup_cache_dump:
1417  * @cache: a #SoupCache
1418  *
1419  * Synchronously writes the cache index out to disk. Contrast with
1420  * soup_cache_flush(), which writes pending cache
1421  * <emphasis>entries</emphasis> to disk.
1422  *
1423  * You must call this before exiting if you want your cache data to
1424  * persist between sessions.
1425  *
1426  * Since: 2.34.
1427  */
1428 void
1429 soup_cache_dump (SoupCache *cache)
1430 {
1431         SoupCachePrivate *priv = SOUP_CACHE_GET_PRIVATE (cache);
1432         char *filename;
1433         GVariantBuilder entries_builder;
1434         GVariant *cache_variant;
1435
1436         if (!g_list_length (cache->priv->lru_start))
1437                 return;
1438
1439         /* Create the builder and iterate over all entries */
1440         g_variant_builder_init (&entries_builder, G_VARIANT_TYPE (SOUP_CACHE_ENTRIES_FORMAT));
1441         g_variant_builder_add (&entries_builder, "q", SOUP_CACHE_CURRENT_VERSION);
1442         g_variant_builder_open (&entries_builder, G_VARIANT_TYPE ("a" SOUP_CACHE_PHEADERS_FORMAT));
1443         g_list_foreach (cache->priv->lru_start, pack_entry, &entries_builder);
1444         g_variant_builder_close (&entries_builder);
1445
1446         /* Serialize and dump */
1447         cache_variant = g_variant_builder_end (&entries_builder);
1448         g_variant_ref_sink (cache_variant);
1449         filename = g_build_filename (priv->cache_dir, SOUP_CACHE_FILE, NULL);
1450         g_file_set_contents (filename, (const char *) g_variant_get_data (cache_variant),
1451                              g_variant_get_size (cache_variant), NULL);
1452         g_free (filename);
1453         g_variant_unref (cache_variant);
1454 }
1455
1456 /**
1457  * soup_cache_load:
1458  * @cache: a #SoupCache
1459  *
1460  * Loads the contents of @cache's index into memory.
1461  *
1462  * Since: 2.34
1463  */
1464 void
1465 soup_cache_load (SoupCache *cache)
1466 {
1467         gboolean must_revalidate;
1468         guint32 freshness_lifetime, hits;
1469         guint32 corrected_initial_age, response_time;
1470         char *url, *filename = NULL, *contents = NULL;
1471         GVariant *cache_variant;
1472         GVariantIter *entries_iter = NULL, *headers_iter = NULL;
1473         gsize length;
1474         SoupCacheEntry *entry;
1475         SoupCachePrivate *priv = cache->priv;
1476         guint16 version, status_code;
1477
1478         filename = g_build_filename (priv->cache_dir, SOUP_CACHE_FILE, NULL);
1479         if (!g_file_get_contents (filename, &contents, &length, NULL)) {
1480                 g_free (filename);
1481                 g_free (contents);
1482                 clear_cache_files (cache);
1483                 return;
1484         }
1485         g_free (filename);
1486
1487         cache_variant = g_variant_new_from_data (G_VARIANT_TYPE (SOUP_CACHE_ENTRIES_FORMAT),
1488                                                  (const char *) contents, length, FALSE, g_free, contents);
1489         g_variant_get (cache_variant, SOUP_CACHE_ENTRIES_FORMAT, &version, &entries_iter);
1490         if (version != SOUP_CACHE_CURRENT_VERSION) {
1491                 g_variant_iter_free (entries_iter);
1492                 g_variant_unref (cache_variant);
1493                 clear_cache_files (cache);
1494                 return;
1495         }
1496
1497         while (g_variant_iter_loop (entries_iter, SOUP_CACHE_PHEADERS_FORMAT,
1498                                     &url, &must_revalidate, &freshness_lifetime, &corrected_initial_age,
1499                                     &response_time, &hits, &length, &status_code,
1500                                     &headers_iter)) {
1501                 const char *header_key, *header_value;
1502                 SoupMessageHeaders *headers;
1503                 SoupMessageHeadersIter soup_headers_iter;
1504
1505                 /* SoupMessage Headers */
1506                 headers = soup_message_headers_new (SOUP_MESSAGE_HEADERS_RESPONSE);
1507                 while (g_variant_iter_loop (headers_iter, SOUP_CACHE_HEADERS_FORMAT, &header_key, &header_value))
1508                         if (*header_key && *header_value)
1509                                 soup_message_headers_append (headers, header_key, header_value);
1510
1511                 /* Check that we have headers */
1512                 soup_message_headers_iter_init (&soup_headers_iter, headers);
1513                 if (!soup_message_headers_iter_next (&soup_headers_iter, &header_key, &header_value)) {
1514                         soup_message_headers_free (headers);
1515                         continue;
1516                 }
1517
1518                 /* Insert in cache */
1519                 entry = g_slice_new0 (SoupCacheEntry);
1520                 entry->uri = g_strdup (url);
1521                 entry->must_revalidate = must_revalidate;
1522                 entry->freshness_lifetime = freshness_lifetime;
1523                 entry->corrected_initial_age = corrected_initial_age;
1524                 entry->response_time = response_time;
1525                 entry->hits = hits;
1526                 entry->length = length;
1527                 entry->headers = headers;
1528                 entry->status_code = status_code;
1529
1530                 if (!soup_cache_entry_insert (cache, entry, FALSE))
1531                         soup_cache_entry_free (entry);
1532         }
1533
1534         cache->priv->lru_start = g_list_reverse (cache->priv->lru_start);
1535
1536         /* frees */
1537         g_variant_iter_free (entries_iter);
1538         g_variant_unref (cache_variant);
1539 }
1540
1541 /**
1542  * soup_cache_set_max_size:
1543  * @cache: a #SoupCache
1544  * @max_size: the maximum size of the cache, in bytes
1545  *
1546  * Sets the maximum size of the cache.
1547  *
1548  * Since: 2.34
1549  */
1550 void
1551 soup_cache_set_max_size (SoupCache *cache,
1552                          guint      max_size)
1553 {
1554         cache->priv->max_size = max_size;
1555         cache->priv->max_entry_data_size = cache->priv->max_size / MAX_ENTRY_DATA_PERCENTAGE;
1556 }
1557
1558 /**
1559  * soup_cache_get_max_size:
1560  * @cache: a #SoupCache
1561  *
1562  * Gets the maximum size of the cache.
1563  *
1564  * Return value: the maximum size of the cache, in bytes.
1565  *
1566  * Since: 2.34
1567  */
1568 guint
1569 soup_cache_get_max_size (SoupCache *cache)
1570 {
1571         return cache->priv->max_size;
1572 }