libsoup/soup-cache.c

   1 /* -*- Mode: C; tab-width: 8; indent-tabs-mode: t; c-basic-offset: 8 -*- */
   2 /*
   3  * soup-cache.c
   4  *
   5  * Copyright (C) 2009, 2010 Igalia S.L.
   6  *
   7  * This library is free software; you can redistribute it and/or
   8  * modify it under the terms of the GNU Library General Public
   9  * License as published by the Free Software Foundation; either
  10  * version 2 of the License, or (at your option) any later version.
  11  *
  12  * This library is distributed in the hope that it will be useful,
  13  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  14  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  15  * Library General Public License for more details.
  16  *
  17  * You should have received a copy of the GNU Library General Public License
  18  * along with this library; see the file COPYING.LIB.  If not, write to
  19  * the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
  20  * Boston, MA 02110-1301, USA.
  21  */
  22
  23 /* TODO:
  24  * - Need to hook the feature in the sync SoupSession.
  25  * - Need more tests.
  26  */
  27
  28 #ifdef HAVE_CONFIG_H
  29 #include <config.h>
  30 #endif
  31
  32 #include <string.h>
  33
  34 #include "soup-cache.h"
  35 #include "soup-body-input-stream.h"
  36 #include "soup-cache-input-stream.h"
  37 #include "soup-cache-private.h"
  38 #include "soup-content-processor.h"
  39 #include "soup-message-private.h"
  40 #include "soup.h"
  41 #include "soup-message-private.h"
  42
  43 /**
  44  * SECTION:soup-cache
  45  * @short_description: Caching support
  46  *
  47  * #SoupCache implements a file-based cache for HTTP resources.
  48  */
  49
  50 static SoupSessionFeatureInterface *soup_cache_default_feature_interface;
  51 static void soup_cache_session_feature_init (SoupSessionFeatureInterface *feature_interface, gpointer interface_data);
  52
  53 static SoupContentProcessorInterface *soup_cache_default_content_processor_interface;
  54 static void soup_cache_content_processor_init (SoupContentProcessorInterface *interface, gpointer interface_data);
  55
  56 #define DEFAULT_MAX_SIZE 50 * 1024 * 1024
  57 #define MAX_ENTRY_DATA_PERCENTAGE 10 /* Percentage of the total size
  58                                         of the cache that can be
  59                                         filled by a single entry */
  60
  61 /*
  62  * Version 2: cache is now saved in soup.cache2. Added the version
  63  * number to the beginning of the file.
  64  *
  65  * Version 3: added HTTP status code to the cache entries.
  66  *
  67  * Version 4: replaced several types.
  68  *   - freshness_lifetime,corrected_initial_age,response_time: time_t -> guint32
  69  *   - status_code: guint -> guint16
  70  *   - hits: guint -> guint32
  71  *
  72  * Version 5: key is no longer stored on disk as it can be easily
  73  * built from the URI. Apart from that some fields in the
  74  * SoupCacheEntry have changed:
  75  *   - entry key is now a uint32 instead of a (char *).
  76  *   - added uri, used to check for collisions
  77  *   - removed filename, it's built from the entry key.
  78  */
  79 #define SOUP_CACHE_CURRENT_VERSION 5
  80
  81 #define OLD_SOUP_CACHE_FILE "soup.cache"
  82 #define SOUP_CACHE_FILE "soup.cache2"
  83
  84 #define SOUP_CACHE_HEADERS_FORMAT "{ss}"
  85 #define SOUP_CACHE_PHEADERS_FORMAT "(sbuuuuuqa" SOUP_CACHE_HEADERS_FORMAT ")"
  86 #define SOUP_CACHE_ENTRIES_FORMAT "(qa" SOUP_CACHE_PHEADERS_FORMAT ")"
  87
  88 /* Basically the same format than above except that some strings are
  89    prepended with &. This way the GVariant returns a pointer to the
  90    data instead of duplicating the string */
  91 #define SOUP_CACHE_DECODE_HEADERS_FORMAT "{&s&s}"
  92
  93
  94 typedef struct _SoupCacheEntry {
  95         guint32 key;
  96         char *uri;
  97         guint32 freshness_lifetime;
  98         gboolean must_revalidate;
  99         gsize length;
 100         guint32 corrected_initial_age;
 101         guint32 response_time;
 102         gboolean dirty;
 103         gboolean being_validated;
 104         SoupMessageHeaders *headers;
 105         guint32 hits;
 106         GCancellable *cancellable;
 107         guint16 status_code;
 108 } SoupCacheEntry;
 109
 110 struct _SoupCachePrivate {
 111         char *cache_dir;
 112         GHashTable *cache;
 113         guint n_pending;
 114         SoupSession *session;
 115         SoupCacheType cache_type;
 116         guint size;
 117         guint max_size;
 118         guint max_entry_data_size; /* Computed value. Here for performance reasons */
 119         GList *lru_start;
 120 };
 121
 122 enum {
 123         PROP_0,
 124         PROP_CACHE_DIR,
 125         PROP_CACHE_TYPE
 126 };
 127
 128 #define SOUP_CACHE_GET_PRIVATE(o) (G_TYPE_INSTANCE_GET_PRIVATE ((o), SOUP_TYPE_CACHE, SoupCachePrivate))
 129
 130 G_DEFINE_TYPE_WITH_CODE (SoupCache, soup_cache, G_TYPE_OBJECT,
 131                          G_IMPLEMENT_INTERFACE (SOUP_TYPE_SESSION_FEATURE,
 132                                                 soup_cache_session_feature_init)
 133                          G_IMPLEMENT_INTERFACE (SOUP_TYPE_CONTENT_PROCESSOR,
 134                                                 soup_cache_content_processor_init))
 135
 136 static gboolean soup_cache_entry_remove (SoupCache *cache, SoupCacheEntry *entry, gboolean purge);
 137 static void make_room_for_new_entry (SoupCache *cache, guint length_to_add);
 138 static gboolean cache_accepts_entries_of_size (SoupCache *cache, guint length_to_add);
 139
 140 static GFile *
 141 get_file_from_entry (SoupCache *cache, SoupCacheEntry *entry)
 142 {
 143         char *filename = g_strdup_printf ("%s%s%u", cache->priv->cache_dir,
 144                                           G_DIR_SEPARATOR_S, (guint) entry->key);
 145         GFile *file = g_file_new_for_path (filename);
 146         g_free (filename);
 147
 148         return file;
 149 }
 150
 151 static SoupCacheability
 152 get_cacheability (SoupCache *cache, SoupMessage *msg)
 153 {
 154         SoupCacheability cacheability;
 155         const char *cache_control, *content_type;
 156         gboolean has_max_age = FALSE;
 157
 158         /* 1. The request method must be cacheable */
 159         if (msg->method == SOUP_METHOD_GET)
 160                 cacheability = SOUP_CACHE_CACHEABLE;
 161         else if (msg->method == SOUP_METHOD_HEAD ||
 162                  msg->method == SOUP_METHOD_TRACE ||
 163                  msg->method == SOUP_METHOD_CONNECT)
 164                 return SOUP_CACHE_UNCACHEABLE;
 165         else
 166                 return (SOUP_CACHE_UNCACHEABLE | SOUP_CACHE_INVALIDATES);
 167
 168         content_type = soup_message_headers_get_content_type (msg->response_headers, NULL);
 169         if (content_type && !g_ascii_strcasecmp (content_type, "multipart/x-mixed-replace"))
 170                 return SOUP_CACHE_UNCACHEABLE;
 171
 172         cache_control = soup_message_headers_get_list (msg->response_headers, "Cache-Control");
 173         if (cache_control && *cache_control) {
 174                 GHashTable *hash;
 175                 SoupCachePrivate *priv = SOUP_CACHE_GET_PRIVATE (cache);
 176
 177                 hash = soup_header_parse_param_list (cache_control);
 178
 179                 /* Shared caches MUST NOT store private resources */
 180                 if (priv->cache_type == SOUP_CACHE_SHARED) {
 181                         if (g_hash_table_lookup_extended (hash, "private", NULL, NULL)) {
 182                                 soup_header_free_param_list (hash);
 183                                 return SOUP_CACHE_UNCACHEABLE;
 184                         }
 185                 }
 186
 187                 /* 2. The 'no-store' cache directive does not appear in the
 188                  * headers
 189                  */
 190                 if (g_hash_table_lookup_extended (hash, "no-store", NULL, NULL)) {
 191                         soup_header_free_param_list (hash);
 192                         return SOUP_CACHE_UNCACHEABLE;
 193                 }
 194
 195                 if (g_hash_table_lookup_extended (hash, "max-age", NULL, NULL))
 196                         has_max_age = TRUE;
 197
 198                 /* This does not appear in section 2.1, but I think it makes
 199                  * sense to check it too?
 200                  */
 201                 if (g_hash_table_lookup_extended (hash, "no-cache", NULL, NULL)) {
 202                         soup_header_free_param_list (hash);
 203                         return SOUP_CACHE_UNCACHEABLE;
 204                 }
 205
 206                 soup_header_free_param_list (hash);
 207         }
 208
 209         /* Section 13.9 */
 210         if ((soup_message_get_uri (msg))->query &&
 211             !soup_message_headers_get_one (msg->response_headers, "Expires") &&
 212             !has_max_age)
 213                 return SOUP_CACHE_UNCACHEABLE;
 214
 215         switch (msg->status_code) {
 216         case SOUP_STATUS_PARTIAL_CONTENT:
 217                 /* We don't cache partial responses, but they only
 218                  * invalidate cached full responses if the headers
 219                  * don't match.
 220                  */
 221                 cacheability = SOUP_CACHE_UNCACHEABLE;
 222                 break;
 223
 224         case SOUP_STATUS_NOT_MODIFIED:
 225                 /* A 304 response validates an existing cache entry */
 226                 cacheability = SOUP_CACHE_VALIDATES;
 227                 break;
 228
 229         case SOUP_STATUS_MULTIPLE_CHOICES:
 230         case SOUP_STATUS_MOVED_PERMANENTLY:
 231         case SOUP_STATUS_GONE:
 232                 /* FIXME: cacheable unless indicated otherwise */
 233                 cacheability = SOUP_CACHE_UNCACHEABLE;
 234                 break;
 235
 236         case SOUP_STATUS_FOUND:
 237         case SOUP_STATUS_TEMPORARY_REDIRECT:
 238                 /* FIXME: cacheable if explicitly indicated */
 239                 cacheability = SOUP_CACHE_UNCACHEABLE;
 240                 break;
 241
 242         case SOUP_STATUS_SEE_OTHER:
 243         case SOUP_STATUS_FORBIDDEN:
 244         case SOUP_STATUS_NOT_FOUND:
 245         case SOUP_STATUS_METHOD_NOT_ALLOWED:
 246                 return (SOUP_CACHE_UNCACHEABLE | SOUP_CACHE_INVALIDATES);
 247
 248         default:
 249                 /* Any 5xx status or any 4xx status not handled above
 250                  * is uncacheable but doesn't break the cache.
 251                  */
 252                 if ((msg->status_code >= SOUP_STATUS_BAD_REQUEST &&
 253                      msg->status_code <= SOUP_STATUS_FAILED_DEPENDENCY) ||
 254                     msg->status_code >= SOUP_STATUS_INTERNAL_SERVER_ERROR)
 255                         return SOUP_CACHE_UNCACHEABLE;
 256
 257                 /* An unrecognized 2xx, 3xx, or 4xx response breaks
 258                  * the cache.
 259                  */
 260                 if ((msg->status_code > SOUP_STATUS_PARTIAL_CONTENT &&
 261                      msg->status_code < SOUP_STATUS_MULTIPLE_CHOICES) ||
 262                     (msg->status_code > SOUP_STATUS_TEMPORARY_REDIRECT &&
 263                      msg->status_code < SOUP_STATUS_INTERNAL_SERVER_ERROR))
 264                         return (SOUP_CACHE_UNCACHEABLE | SOUP_CACHE_INVALIDATES);
 265                 break;
 266         }
 267
 268         return cacheability;
 269 }
 270
 271 /* NOTE: this function deletes the file pointed by the file argument
 272  * and also unref's the GFile object representing it.
 273  */
 274 static void
 275 soup_cache_entry_free (SoupCacheEntry *entry)
 276 {
 277         g_free (entry->uri);
 278         g_clear_pointer (&entry->headers, soup_message_headers_free);
 279         g_clear_object (&entry->cancellable);
 280
 281         g_slice_free (SoupCacheEntry, entry);
 282 }
 283
 284 static void
 285 copy_headers (const char *name, const char *value, SoupMessageHeaders *headers)
 286 {
 287         soup_message_headers_append (headers, name, value);
 288 }
 289
 290 static void
 291 remove_headers (const char *name, const char *value, SoupMessageHeaders *headers)
 292 {
 293         soup_message_headers_remove (headers, name);
 294 }
 295
 296 static char *hop_by_hop_headers[] = {"Connection", "Keep-Alive", "Proxy-Authenticate", "Proxy-Authorization", "TE", "Trailer", "Transfer-Encoding", "Upgrade"};
 297
 298 static void
 299 copy_end_to_end_headers (SoupMessageHeaders *source, SoupMessageHeaders *destination)
 300 {
 301         int i;
 302
 303         soup_message_headers_foreach (source, (SoupMessageHeadersForeachFunc) copy_headers, destination);
 304         for (i = 0; i < G_N_ELEMENTS (hop_by_hop_headers); i++)
 305                 soup_message_headers_remove (destination, hop_by_hop_headers[i]);
 306         soup_message_headers_clean_connection_headers (destination);
 307 }
 308
 309 static guint
 310 soup_cache_entry_get_current_age (SoupCacheEntry *entry)
 311 {
 312         time_t now = time (NULL);
 313         time_t resident_time;
 314
 315         resident_time = now - entry->response_time;
 316         return entry->corrected_initial_age + resident_time;
 317 }
 318
 319 static gboolean
 320 soup_cache_entry_is_fresh_enough (SoupCacheEntry *entry, gint min_fresh)
 321 {
 322         guint limit = (min_fresh == -1) ? soup_cache_entry_get_current_age (entry) : (guint) min_fresh;
 323         return entry->freshness_lifetime > limit;
 324 }
 325
 326 static inline guint32
 327 get_cache_key_from_uri (const char *uri)
 328 {
 329         return (guint32) g_str_hash (uri);
 330 }
 331
 332 static void
 333 soup_cache_entry_set_freshness (SoupCacheEntry *entry, SoupMessage *msg, SoupCache *cache)
 334 {
 335         const char *cache_control;
 336         const char *expires, *date, *last_modified;
 337
 338         /* Reset these values. We have to do this to ensure that
 339          * revalidations overwrite previous values for the headers.
 340          */
 341         entry->must_revalidate = FALSE;
 342         entry->freshness_lifetime = 0;
 343
 344         cache_control = soup_message_headers_get_list (entry->headers, "Cache-Control");
 345         if (cache_control && *cache_control) {
 346                 const char *max_age, *s_maxage;
 347                 gint64 freshness_lifetime = 0;
 348                 GHashTable *hash;
 349                 SoupCachePrivate *priv = SOUP_CACHE_GET_PRIVATE (cache);
 350
 351                 hash = soup_header_parse_param_list (cache_control);
 352
 353                 /* Should we re-validate the entry when it goes stale */
 354                 entry->must_revalidate = g_hash_table_lookup_extended (hash, "must-revalidate", NULL, NULL);
 355
 356                 /* Section 2.3.1 */
 357                 if (priv->cache_type == SOUP_CACHE_SHARED) {
 358                         s_maxage = g_hash_table_lookup (hash, "s-maxage");
 359                         if (s_maxage) {
 360                                 freshness_lifetime = g_ascii_strtoll (s_maxage, NULL, 10);
 361                                 if (freshness_lifetime) {
 362                                         /* Implies proxy-revalidate. TODO: is it true? */
 363                                         entry->must_revalidate = TRUE;
 364                                         soup_header_free_param_list (hash);
 365                                         return;
 366                                 }
 367                         }
 368                 }
 369
 370                 /* If 'max-age' cache directive is present, use that */
 371                 max_age = g_hash_table_lookup (hash, "max-age");
 372                 if (max_age)
 373                         freshness_lifetime = g_ascii_strtoll (max_age, NULL, 10);
 374
 375                 if (freshness_lifetime) {
 376                         entry->freshness_lifetime = (guint32) MIN (freshness_lifetime, G_MAXUINT32);
 377                         soup_header_free_param_list (hash);
 378                         return;
 379                 }
 380
 381                 soup_header_free_param_list (hash);
 382         }
 383
 384         /* If the 'Expires' response header is present, use its value
 385          * minus the value of the 'Date' response header
 386          */
 387         expires = soup_message_headers_get_one (entry->headers, "Expires");
 388         date = soup_message_headers_get_one (entry->headers, "Date");
 389         if (expires && date) {
 390                 SoupDate *expires_d, *date_d;
 391                 time_t expires_t, date_t;
 392
 393                 expires_d = soup_date_new_from_string (expires);
 394                 if (expires_d) {
 395                         date_d = soup_date_new_from_string (date);
 396
 397                         expires_t = soup_date_to_time_t (expires_d);
 398                         date_t = soup_date_to_time_t (date_d);
 399
 400                         soup_date_free (expires_d);
 401                         soup_date_free (date_d);
 402
 403                         if (expires_t && date_t) {
 404                                 entry->freshness_lifetime = (guint32) MAX (expires_t - date_t, 0);
 405                                 return;
 406                         }
 407                 } else {
 408                         /* If Expires is not a valid date we should
 409                            treat it as already expired, see section
 410                            3.3 */
 411                         entry->freshness_lifetime = 0;
 412                         return;
 413                 }
 414         }
 415
 416         /* Otherwise an heuristic may be used */
 417
 418         /* Heuristics MUST NOT be used with stored responses with
 419            these status codes (section 2.3.1.1) */
 420         if (entry->status_code != SOUP_STATUS_OK &&
 421             entry->status_code != SOUP_STATUS_NON_AUTHORITATIVE &&
 422             entry->status_code != SOUP_STATUS_PARTIAL_CONTENT &&
 423             entry->status_code != SOUP_STATUS_MULTIPLE_CHOICES &&
 424             entry->status_code != SOUP_STATUS_MOVED_PERMANENTLY &&
 425             entry->status_code != SOUP_STATUS_GONE)
 426                 goto expire;
 427
 428         /* TODO: attach warning 113 if response's current_age is more
 429            than 24h (section 2.3.1.1) when using heuristics */
 430
 431         /* Last-Modified based heuristic */
 432         last_modified = soup_message_headers_get_one (entry->headers, "Last-Modified");
 433         if (last_modified) {
 434                 SoupDate *soup_date;
 435                 time_t now, last_modified_t;
 436
 437                 soup_date = soup_date_new_from_string (last_modified);
 438                 last_modified_t = soup_date_to_time_t (soup_date);
 439                 now = time (NULL);
 440
 441 #define HEURISTIC_FACTOR 0.1 /* From Section 2.3.1.1 */
 442
 443                 entry->freshness_lifetime = MAX (0, (now - last_modified_t) * HEURISTIC_FACTOR);
 444                 soup_date_free (soup_date);
 445         }
 446
 447         return;
 448
 449  expire:
 450         /* If all else fails, make the entry expire immediately */
 451         entry->freshness_lifetime = 0;
 452 }
 453
 454 static SoupCacheEntry *
 455 soup_cache_entry_new (SoupCache *cache, SoupMessage *msg, time_t request_time, time_t response_time)
 456 {
 457         SoupCacheEntry *entry;
 458         const char *date;
 459
 460         entry = g_slice_new0 (SoupCacheEntry);
 461         entry->dirty = FALSE;
 462         entry->being_validated = FALSE;
 463         entry->status_code = msg->status_code;
 464         entry->response_time = response_time;
 465         entry->uri = soup_uri_to_string (soup_message_get_uri (msg), FALSE);
 466
 467         /* Headers */
 468         entry->headers = soup_message_headers_new (SOUP_MESSAGE_HEADERS_RESPONSE);
 469         copy_end_to_end_headers (msg->response_headers, entry->headers);
 470
 471         /* LRU list */
 472         entry->hits = 0;
 473
 474         /* Section 2.3.1, Freshness Lifetime */
 475         soup_cache_entry_set_freshness (entry, msg, cache);
 476
 477         /* Section 2.3.2, Calculating Age */
 478         date = soup_message_headers_get_one (entry->headers, "Date");
 479
 480         if (date) {
 481                 SoupDate *soup_date;
 482                 const char *age;
 483                 time_t date_value, apparent_age, corrected_received_age, response_delay, age_value = 0;
 484
 485                 soup_date = soup_date_new_from_string (date);
 486                 date_value = soup_date_to_time_t (soup_date);
 487                 soup_date_free (soup_date);
 488
 489                 age = soup_message_headers_get_one (entry->headers, "Age");
 490                 if (age)
 491                         age_value = g_ascii_strtoll (age, NULL, 10);
 492
 493                 apparent_age = MAX (0, entry->response_time - date_value);
 494                 corrected_received_age = MAX (apparent_age, age_value);
 495                 response_delay = entry->response_time - request_time;
 496                 entry->corrected_initial_age = corrected_received_age + response_delay;
 497         } else {
 498                 /* Is this correct ? */
 499                 entry->corrected_initial_age = time (NULL);
 500         }
 501
 502         return entry;
 503 }
 504
 505 static gboolean
 506 soup_cache_entry_remove (SoupCache *cache, SoupCacheEntry *entry, gboolean purge)
 507 {
 508         GList *lru_item;
 509
 510         if (entry->dirty) {
 511                 g_cancellable_cancel (entry->cancellable);
 512                 return FALSE;
 513         }
 514
 515         g_assert (!entry->dirty);
 516         g_assert (g_list_length (cache->priv->lru_start) == g_hash_table_size (cache->priv->cache));
 517
 518         if (!g_hash_table_remove (cache->priv->cache, GUINT_TO_POINTER (entry->key)))
 519                 return FALSE;
 520
 521         /* Remove from LRU */
 522         lru_item = g_list_find (cache->priv->lru_start, entry);
 523         cache->priv->lru_start = g_list_delete_link (cache->priv->lru_start, lru_item);
 524
 525         /* Adjust cache size */
 526         cache->priv->size -= entry->length;
 527
 528         g_assert (g_list_length (cache->priv->lru_start) == g_hash_table_size (cache->priv->cache));
 529
 530         /* Free resources */
 531         if (purge) {
 532                 GFile *file = get_file_from_entry (cache, entry);
 533                 g_file_delete (file, NULL, NULL);
 534                 g_object_unref (file);
 535         }
 536         soup_cache_entry_free (entry);
 537
 538         return TRUE;
 539 }
 540
 541 static gint
 542 lru_compare_func (gconstpointer a, gconstpointer b)
 543 {
 544         SoupCacheEntry *entry_a = (SoupCacheEntry *)a;
 545         SoupCacheEntry *entry_b = (SoupCacheEntry *)b;
 546
 547         /* The rationale of this sorting func is
 548          *
 549          * 1. sort by hits -> LRU algorithm, then
 550          *
 551          * 2. sort by freshness lifetime, we better discard first
 552          * entries that are close to expire
 553          *
 554          * 3. sort by size, replace first small size resources as they
 555          * are cheaper to download
 556          */
 557
 558         /* Sort by hits */
 559         if (entry_a->hits != entry_b->hits)
 560                 return entry_a->hits - entry_b->hits;
 561
 562         /* Sort by freshness_lifetime */
 563         if (entry_a->freshness_lifetime != entry_b->freshness_lifetime)
 564                 return entry_a->freshness_lifetime - entry_b->freshness_lifetime;
 565
 566         /* Sort by size */
 567         return entry_a->length - entry_b->length;
 568 }
 569
 570 static gboolean
 571 cache_accepts_entries_of_size (SoupCache *cache, guint length_to_add)
 572 {
 573         /* We could add here some more heuristics. TODO: review how
 574            this is done by other HTTP caches */
 575
 576         return length_to_add <= cache->priv->max_entry_data_size;
 577 }
 578
 579 static void
 580 make_room_for_new_entry (SoupCache *cache, guint length_to_add)
 581 {
 582         GList *lru_entry = cache->priv->lru_start;
 583
 584         /* Check that there is enough room for the new entry. This is
 585            an approximation as we're not working out the size of the
 586            cache file or the size of the headers for performance
 587            reasons. TODO: check if that would be really that expensive */
 588
 589         while (lru_entry &&
 590                (length_to_add + cache->priv->size > cache->priv->max_size)) {
 591                 SoupCacheEntry *old_entry = (SoupCacheEntry *)lru_entry->data;
 592
 593                 /* Discard entries. Once cancelled resources will be
 594                  * freed in close_ready_cb
 595                  */
 596                 if (soup_cache_entry_remove (cache, old_entry, TRUE))
 597                         lru_entry = cache->priv->lru_start;
 598                 else
 599                         lru_entry = g_list_next (lru_entry);
 600         }
 601 }
 602
 603 static gboolean
 604 soup_cache_entry_insert (SoupCache *cache,
 605                          SoupCacheEntry *entry,
 606                          gboolean sort)
 607 {
 608         guint length_to_add = 0;
 609         SoupCacheEntry *old_entry;
 610
 611         /* Fill the key */
 612         entry->key = get_cache_key_from_uri ((const char *) entry->uri);
 613
 614         if (soup_message_headers_get_encoding (entry->headers) == SOUP_ENCODING_CONTENT_LENGTH)
 615                 length_to_add = soup_message_headers_get_content_length (entry->headers);
 616
 617         /* Check if we are going to store the resource depending on its size */
 618         if (length_to_add) {
 619                 if (!cache_accepts_entries_of_size (cache, length_to_add))
 620                         return FALSE;
 621
 622                 /* Make room for new entry if needed */
 623                 make_room_for_new_entry (cache, length_to_add);
 624         }
 625
 626         /* Remove any previous entry */
 627         if ((old_entry = g_hash_table_lookup (cache->priv->cache, GUINT_TO_POINTER (entry->key))) != NULL) {
 628                 if (!soup_cache_entry_remove (cache, old_entry, TRUE))
 629                         return FALSE;
 630         }
 631
 632         /* Add to hash table */
 633         g_hash_table_insert (cache->priv->cache, GUINT_TO_POINTER (entry->key), entry);
 634
 635         /* Compute new cache size */
 636         cache->priv->size += length_to_add;
 637
 638         /* Update LRU */
 639         if (sort)
 640                 cache->priv->lru_start = g_list_insert_sorted (cache->priv->lru_start, entry, lru_compare_func);
 641         else
 642                 cache->priv->lru_start = g_list_prepend (cache->priv->lru_start, entry);
 643
 644         g_assert (g_list_length (cache->priv->lru_start) == g_hash_table_size (cache->priv->cache));
 645
 646         return TRUE;
 647 }
 648
 649 static SoupCacheEntry*
 650 soup_cache_entry_lookup (SoupCache *cache,
 651                          SoupMessage *msg)
 652 {
 653         SoupCacheEntry *entry;
 654         guint32 key;
 655         char *uri = NULL;
 656
 657         uri = soup_uri_to_string (soup_message_get_uri (msg), FALSE);
 658         key = get_cache_key_from_uri ((const char *) uri);
 659
 660         entry = g_hash_table_lookup (cache->priv->cache, GUINT_TO_POINTER (key));
 661
 662         if (entry != NULL && (strcmp (entry->uri, uri) != 0))
 663                 entry = NULL;
 664
 665         g_free (uri);
 666         return entry;
 667 }
 668
 669 GInputStream *
 670 soup_cache_send_response (SoupCache *cache, SoupMessage *msg)
 671 {
 672         SoupCacheEntry *entry;
 673         char *current_age;
 674         GInputStream *file_stream, *body_stream, *cache_stream;
 675         GFile *file;
 676
 677         g_return_val_if_fail (SOUP_IS_CACHE (cache), NULL);
 678         g_return_val_if_fail (SOUP_IS_MESSAGE (msg), NULL);
 679
 680         entry = soup_cache_entry_lookup (cache, msg);
 681         g_return_val_if_fail (entry, NULL);
 682
 683         file = get_file_from_entry (cache, entry);
 684         file_stream = G_INPUT_STREAM (g_file_read (file, NULL, NULL));
 685         g_object_unref (file);
 686
 687         /* Do not change the original message if there is no resource */
 688         if (!file_stream)
 689                 return NULL;
 690
 691         body_stream = soup_body_input_stream_new (file_stream, SOUP_ENCODING_CONTENT_LENGTH, entry->length);
 692         g_object_unref (file_stream);
 693
 694         if (!body_stream)
 695                 return NULL;
 696
 697         /* If we are told to send a response from cache any validation
 698            in course is over by now */
 699         entry->being_validated = FALSE;
 700
 701         /* Status */
 702         soup_message_set_status (msg, entry->status_code);
 703
 704         /* Headers */
 705         copy_end_to_end_headers (entry->headers, msg->response_headers);
 706
 707         /* Add 'Age' header with the current age */
 708         current_age = g_strdup_printf ("%d", soup_cache_entry_get_current_age (entry));
 709         soup_message_headers_replace (msg->response_headers,
 710                                       "Age",
 711                                       current_age);
 712         g_free (current_age);
 713
 714         /* Create the cache stream. */
 715         soup_message_disable_feature (msg, SOUP_TYPE_CACHE);
 716         cache_stream = soup_message_setup_body_istream (body_stream, msg,
 717                                                         cache->priv->session,
 718                                                         SOUP_STAGE_ENTITY_BODY);
 719         g_object_unref (body_stream);
 720
 721         return cache_stream;
 722 }
 723
 724 static void
 725 msg_got_headers_cb (SoupMessage *msg, gpointer user_data)
 726 {
 727         g_object_set_data (G_OBJECT (msg), "response-time", GINT_TO_POINTER (time (NULL)));
 728         g_signal_handlers_disconnect_by_func (msg, msg_got_headers_cb, user_data);
 729 }
 730
 731 static void
 732 request_started (SoupSessionFeature *feature, SoupSession *session,
 733                  SoupMessage *msg, SoupSocket *socket)
 734 {
 735         g_object_set_data (G_OBJECT (msg), "request-time", GINT_TO_POINTER (time (NULL)));
 736         g_signal_connect (msg, "got-headers", G_CALLBACK (msg_got_headers_cb), NULL);
 737 }
 738
 739 static void
 740 attach (SoupSessionFeature *feature, SoupSession *session)
 741 {
 742         SoupCache *cache = SOUP_CACHE (feature);
 743         cache->priv->session = session;
 744
 745         soup_cache_default_feature_interface->attach (feature, session);
 746 }
 747
 748 static void
 749 soup_cache_session_feature_init (SoupSessionFeatureInterface *feature_interface,
 750                                         gpointer interface_data)
 751 {
 752         soup_cache_default_feature_interface =
 753                 g_type_default_interface_peek (SOUP_TYPE_SESSION_FEATURE);
 754
 755         feature_interface->attach = attach;
 756         feature_interface->request_started = request_started;
 757 }
 758
 759 typedef struct {
 760         SoupCache *cache;
 761         SoupCacheEntry *entry;
 762 } StreamHelper;
 763
 764 static void
 765 istream_caching_finished (SoupCacheInputStream *istream,
 766                           gsize                 bytes_written,
 767                           GError               *error,
 768                           gpointer              user_data)
 769 {
 770         StreamHelper *helper = (StreamHelper *) user_data;
 771         SoupCache *cache = helper->cache;
 772         SoupCacheEntry *entry = helper->entry;
 773
 774         --cache->priv->n_pending;
 775
 776         entry->dirty = FALSE;
 777         entry->length = bytes_written;
 778         g_clear_object (&entry->cancellable);
 779
 780         if (error) {
 781                 /* Update cache size */
 782                 if (soup_message_headers_get_encoding (entry->headers) == SOUP_ENCODING_CONTENT_LENGTH)
 783                         cache->priv->size -= soup_message_headers_get_content_length (entry->headers);
 784
 785                 soup_cache_entry_remove (cache, entry, TRUE);
 786                 helper->entry = entry = NULL;
 787                 goto cleanup;
 788         }
 789
 790         if (soup_message_headers_get_encoding (entry->headers) != SOUP_ENCODING_CONTENT_LENGTH) {
 791
 792                 if (cache_accepts_entries_of_size (cache, entry->length)) {
 793                         make_room_for_new_entry (cache, entry->length);
 794                         cache->priv->size += entry->length;
 795                 } else {
 796                         soup_cache_entry_remove (cache, entry, TRUE);
 797                         helper->entry = entry = NULL;
 798                 }
 799         }
 800
 801  cleanup:
 802         g_object_unref (helper->cache);
 803         g_slice_free (StreamHelper, helper);
 804 }
 805
 806 static GInputStream*
 807 soup_cache_content_processor_wrap_input (SoupContentProcessor *processor,
 808                                          GInputStream *base_stream,
 809                                          SoupMessage *msg,
 810                                          GError **error)
 811 {
 812         SoupCache *cache = (SoupCache*) processor;
 813         SoupCacheEntry *entry;
 814         SoupCacheability cacheability;
 815         GInputStream *istream;
 816         GFile *file;
 817         StreamHelper *helper;
 818         time_t request_time, response_time;
 819
 820         /* First of all, check if we should cache the resource. */
 821         cacheability = soup_cache_get_cacheability (cache, msg);
 822         entry = soup_cache_entry_lookup (cache, msg);
 823
 824         if (cacheability & SOUP_CACHE_INVALIDATES) {
 825                 if (entry)
 826                         soup_cache_entry_remove (cache, entry, TRUE);
 827                 return NULL;
 828         }
 829
 830         if (cacheability & SOUP_CACHE_VALIDATES) {
 831                 /* It's possible to get a CACHE_VALIDATES with no
 832                  * entry in the hash table. This could happen if for
 833                  * example the soup client is the one creating the
 834                  * conditional request.
 835                  */
 836                 if (entry)
 837                         soup_cache_update_from_conditional_request (cache, msg);
 838                 return NULL;
 839         }
 840
 841         if (!(cacheability & SOUP_CACHE_CACHEABLE))
 842                 return NULL;
 843
 844         /* Check if we are already caching this resource */
 845         if (entry && (entry->dirty || entry->being_validated))
 846                 return NULL;
 847
 848         /* Create a new entry, deleting any old one if present */
 849         if (entry)
 850                 soup_cache_entry_remove (cache, entry, TRUE);
 851
 852         request_time = GPOINTER_TO_INT (g_object_get_data (G_OBJECT (msg), "request-time"));
 853         response_time = GPOINTER_TO_INT (g_object_get_data (G_OBJECT (msg), "request-time"));
 854         entry = soup_cache_entry_new (cache, msg, request_time, response_time);
 855         entry->hits = 1;
 856         entry->dirty = TRUE;
 857
 858         /* Do not continue if it can not be stored */
 859         if (!soup_cache_entry_insert (cache, entry, TRUE)) {
 860                 soup_cache_entry_free (entry);
 861                 return NULL;
 862         }
 863
 864         entry->cancellable = g_cancellable_new ();
 865         ++cache->priv->n_pending;
 866
 867         helper = g_slice_new (StreamHelper);
 868         helper->cache = g_object_ref (cache);
 869         helper->entry = entry;
 870
 871         file = get_file_from_entry (cache, entry);
 872         istream = soup_cache_input_stream_new (base_stream, file);
 873         g_object_unref (file);
 874
 875         g_signal_connect (istream, "caching-finished", G_CALLBACK (istream_caching_finished), helper);
 876
 877         return istream;
 878 }
 879
 880 static void
 881 soup_cache_content_processor_init (SoupContentProcessorInterface *processor_interface,
 882                                    gpointer interface_data)
 883 {
 884         soup_cache_default_content_processor_interface =
 885                 g_type_default_interface_peek (SOUP_TYPE_CONTENT_PROCESSOR);
 886
 887         processor_interface->processing_stage = SOUP_STAGE_ENTITY_BODY;
 888         processor_interface->wrap_input = soup_cache_content_processor_wrap_input;
 889 }
 890
 891 static void
 892 soup_cache_init (SoupCache *cache)
 893 {
 894         SoupCachePrivate *priv;
 895
 896         priv = cache->priv = SOUP_CACHE_GET_PRIVATE (cache);
 897
 898         priv->cache = g_hash_table_new (g_direct_hash, g_direct_equal);
 899         /* LRU */
 900         priv->lru_start = NULL;
 901
 902         /* */
 903         priv->n_pending = 0;
 904
 905         /* Cache size */
 906         priv->max_size = DEFAULT_MAX_SIZE;
 907         priv->max_entry_data_size = priv->max_size / MAX_ENTRY_DATA_PERCENTAGE;
 908         priv->size = 0;
 909 }
 910
 911 static void
 912 remove_cache_item (gpointer data,
 913                    gpointer user_data)
 914 {
 915         soup_cache_entry_remove ((SoupCache *) user_data, (SoupCacheEntry *) data, FALSE);
 916 }
 917
 918 static void
 919 soup_cache_finalize (GObject *object)
 920 {
 921         SoupCachePrivate *priv;
 922         GList *entries;
 923
 924         priv = SOUP_CACHE (object)->priv;
 925
 926         // Cannot use g_hash_table_foreach as callbacks must not modify the hash table
 927         entries = g_hash_table_get_values (priv->cache);
 928         g_list_foreach (entries, remove_cache_item, object);
 929         g_list_free (entries);
 930
 931         g_hash_table_destroy (priv->cache);
 932         g_free (priv->cache_dir);
 933
 934         g_list_free (priv->lru_start);
 935
 936         G_OBJECT_CLASS (soup_cache_parent_class)->finalize (object);
 937 }
 938
 939 static void
 940 soup_cache_set_property (GObject *object, guint prop_id,
 941                                 const GValue *value, GParamSpec *pspec)
 942 {
 943         SoupCachePrivate *priv = SOUP_CACHE (object)->priv;
 944
 945         switch (prop_id) {
 946         case PROP_CACHE_DIR:
 947                 priv->cache_dir = g_value_dup_string (value);
 948                 /* Create directory if it does not exist (FIXME: should we?) */
 949                 if (!g_file_test (priv->cache_dir, G_FILE_TEST_EXISTS | G_FILE_TEST_IS_DIR))
 950                         g_mkdir_with_parents (priv->cache_dir, 0700);
 951                 break;
 952         case PROP_CACHE_TYPE:
 953                 priv->cache_type = g_value_get_enum (value);
 954                 /* TODO: clear private entries and issue a warning if moving to shared? */
 955                 break;
 956         default:
 957                 G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
 958                 break;
 959         }
 960 }
 961
 962 static void
 963 soup_cache_get_property (GObject *object, guint prop_id,
 964                          GValue *value, GParamSpec *pspec)
 965 {
 966         SoupCachePrivate *priv = SOUP_CACHE (object)->priv;
 967
 968         switch (prop_id) {
 969         case PROP_CACHE_DIR:
 970                 g_value_set_string (value, priv->cache_dir);
 971                 break;
 972         case PROP_CACHE_TYPE:
 973                 g_value_set_enum (value, priv->cache_type);
 974                 break;
 975         default:
 976                 G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
 977                 break;
 978         }
 979 }
 980
 981 static void
 982 soup_cache_constructed (GObject *object)
 983 {
 984         SoupCachePrivate *priv;
 985
 986         priv = SOUP_CACHE (object)->priv;
 987
 988         if (!priv->cache_dir) {
 989                 /* Set a default cache dir, different for each user */
 990                 priv->cache_dir = g_build_filename (g_get_user_cache_dir (),
 991                                                     "httpcache",
 992                                                     NULL);
 993                 if (!g_file_test (priv->cache_dir, G_FILE_TEST_EXISTS | G_FILE_TEST_IS_DIR))
 994                         g_mkdir_with_parents (priv->cache_dir, 0700);
 995         }
 996
 997         if (G_OBJECT_CLASS (soup_cache_parent_class)->constructed)
 998                 G_OBJECT_CLASS (soup_cache_parent_class)->constructed (object);
 999 }
1000
1001 static void
1002 soup_cache_class_init (SoupCacheClass *cache_class)
1003 {
1004         GObjectClass *gobject_class = (GObjectClass *)cache_class;
1005
1006         gobject_class->finalize = soup_cache_finalize;
1007         gobject_class->constructed = soup_cache_constructed;
1008         gobject_class->set_property = soup_cache_set_property;
1009         gobject_class->get_property = soup_cache_get_property;
1010
1011         cache_class->get_cacheability = get_cacheability;
1012
1013         g_object_class_install_property (gobject_class, PROP_CACHE_DIR,
1014                                          g_param_spec_string ("cache-dir",
1015                                                               "Cache directory",
1016                                                               "The directory to store the cache files",
1017                                                               NULL,
1018                                                               G_PARAM_READWRITE | G_PARAM_CONSTRUCT_ONLY));
1019
1020         g_object_class_install_property (gobject_class, PROP_CACHE_TYPE,
1021                                          g_param_spec_enum ("cache-type",
1022                                                             "Cache type",
1023                                                             "Whether the cache is private or shared",
1024                                                             SOUP_TYPE_CACHE_TYPE,
1025                                                             SOUP_CACHE_SINGLE_USER,
1026                                                             G_PARAM_READWRITE | G_PARAM_CONSTRUCT_ONLY));
1027
1028         g_type_class_add_private (cache_class, sizeof (SoupCachePrivate));
1029 }
1030
1031 /**
1032  * SoupCacheType:
1033  * @SOUP_CACHE_SINGLE_USER: a single-user cache
1034  * @SOUP_CACHE_SHARED: a shared cache
1035  *
1036  * The type of cache; this affects what kinds of responses will be
1037  * saved.
1038  *
1039  * Since: 2.34
1040  */
1041
1042 /**
1043  * soup_cache_new:
1044  * @cache_dir: the directory to store the cached data, or %NULL to use the default one
1045  * @cache_type: the #SoupCacheType of the cache
1046  *
1047  * Creates a new #SoupCache.
1048  *
1049  * Returns: a new #SoupCache
1050  *
1051  * Since: 2.34
1052  */
1053 SoupCache *
1054 soup_cache_new (const char *cache_dir, SoupCacheType cache_type)
1055 {
1056         return g_object_new (SOUP_TYPE_CACHE,
1057                              "cache-dir", cache_dir,
1058                              "cache-type", cache_type,
1059                              NULL);
1060 }
1061
1062 /**
1063  * soup_cache_has_response:
1064  * @cache: a #SoupCache
1065  * @msg: a #SoupMessage
1066  *
1067  * This function calculates whether the @cache object has a proper
1068  * response for the request @msg given the flags both in the request
1069  * and the cached reply and the time ellapsed since it was cached.
1070  *
1071  * Returns: whether or not the @cache has a valid response for @msg
1072  *
1073  * Since: 2.34
1074  */
1075 SoupCacheResponse
1076 soup_cache_has_response (SoupCache *cache, SoupMessage *msg)
1077 {
1078         SoupCacheEntry *entry;
1079         const char *cache_control, *pragma;
1080         gpointer value;
1081         int max_age, max_stale, min_fresh;
1082         GList *lru_item, *item;
1083
1084         entry = soup_cache_entry_lookup (cache, msg);
1085
1086         /* 1. The presented Request-URI and that of stored response
1087          * match
1088          */
1089         if (!entry)
1090                 return SOUP_CACHE_RESPONSE_STALE;
1091
1092         /* Increase hit count. Take sorting into account */
1093         entry->hits++;
1094         lru_item = g_list_find (cache->priv->lru_start, entry);
1095         item = lru_item;
1096         while (item->next && lru_compare_func (item->data, item->next->data) > 0)
1097                 item = g_list_next (item);
1098
1099         if (item != lru_item) {
1100                 cache->priv->lru_start = g_list_remove_link (cache->priv->lru_start, lru_item);
1101                 item = g_list_insert_sorted (item, lru_item->data, lru_compare_func);
1102                 g_list_free (lru_item);
1103         }
1104
1105         if (entry->dirty || entry->being_validated)
1106                 return SOUP_CACHE_RESPONSE_STALE;
1107
1108         /* 2. The request method associated with the stored response
1109          *  allows it to be used for the presented request
1110          */
1111
1112         /* In practice this means we only return our resource for GET,
1113          * cacheability for other methods is a TODO in the RFC
1114          * (TODO: although we could return the headers for HEAD
1115          * probably).
1116          */
1117         if (msg->method != SOUP_METHOD_GET)
1118                 return SOUP_CACHE_RESPONSE_STALE;
1119
1120         /* 3. Selecting request-headers nominated by the stored
1121          * response (if any) match those presented.
1122          */
1123
1124         /* TODO */
1125
1126         /* 4. The request is a conditional request issued by the client.
1127          */
1128         if (soup_message_headers_get_one (msg->request_headers, "If-Modified-Since") ||
1129             soup_message_headers_get_list (msg->request_headers, "If-None-Match"))
1130                 return SOUP_CACHE_RESPONSE_STALE;
1131
1132         /* 5. The presented request and stored response are free from
1133          * directives that would prevent its use.
1134          */
1135
1136         max_age = max_stale = min_fresh = -1;
1137
1138         /* For HTTP 1.0 compatibility. RFC2616 section 14.9.4
1139          */
1140         pragma = soup_message_headers_get_list (msg->request_headers, "Pragma");
1141         if (pragma && soup_header_contains (pragma, "no-cache"))
1142                 return SOUP_CACHE_RESPONSE_STALE;
1143
1144         cache_control = soup_message_headers_get_list (msg->request_headers, "Cache-Control");
1145         if (cache_control && *cache_control) {
1146                 GHashTable *hash = soup_header_parse_param_list (cache_control);
1147
1148                 if (g_hash_table_lookup_extended (hash, "no-store", NULL, NULL)) {
1149                         soup_header_free_param_list (hash);
1150                         return SOUP_CACHE_RESPONSE_STALE;
1151                 }
1152
1153                 if (g_hash_table_lookup_extended (hash, "no-cache", NULL, NULL)) {
1154                         soup_header_free_param_list (hash);
1155                         return SOUP_CACHE_RESPONSE_STALE;
1156                 }
1157
1158                 if (g_hash_table_lookup_extended (hash, "max-age", NULL, &value)) {
1159                         max_age = (int)MIN (g_ascii_strtoll (value, NULL, 10), G_MAXINT32);
1160                         /* Forcing cache revalidaton
1161                          */
1162                         if (!max_age) {
1163                                 soup_header_free_param_list (hash);
1164                                 return SOUP_CACHE_RESPONSE_NEEDS_VALIDATION;
1165                         }
1166                 }
1167
1168                 /* max-stale can have no value set, we need to use _extended */
1169                 if (g_hash_table_lookup_extended (hash, "max-stale", NULL, &value)) {
1170                         if (value)
1171                                 max_stale = (int)MIN (g_ascii_strtoll (value, NULL, 10), G_MAXINT32);
1172                         else
1173                                 max_stale = G_MAXINT32;
1174                 }
1175
1176                 value = g_hash_table_lookup (hash, "min-fresh");
1177                 if (value)
1178                         min_fresh = (int)MIN (g_ascii_strtoll (value, NULL, 10), G_MAXINT32);
1179
1180                 soup_header_free_param_list (hash);
1181
1182                 if (max_age > 0) {
1183                         guint current_age = soup_cache_entry_get_current_age (entry);
1184
1185                         /* If we are over max-age and max-stale is not
1186                            set, do not use the value from the cache
1187                            without validation */
1188                         if ((guint) max_age <= current_age && max_stale == -1)
1189                                 return SOUP_CACHE_RESPONSE_NEEDS_VALIDATION;
1190                 }
1191         }
1192
1193         /* 6. The stored response is either: fresh, allowed to be
1194          * served stale or succesfully validated
1195          */
1196         /* TODO consider also proxy-revalidate & s-maxage */
1197         if (entry->must_revalidate)
1198                 return SOUP_CACHE_RESPONSE_NEEDS_VALIDATION;
1199
1200         if (!soup_cache_entry_is_fresh_enough (entry, min_fresh)) {
1201                 /* Not fresh, can it be served stale? */
1202                 if (max_stale != -1) {
1203                         /* G_MAXINT32 means we accept any staleness */
1204                         if (max_stale == G_MAXINT32)
1205                                 return SOUP_CACHE_RESPONSE_FRESH;
1206
1207                         if ((soup_cache_entry_get_current_age (entry) - entry->freshness_lifetime) <= (guint) max_stale)
1208                                 return SOUP_CACHE_RESPONSE_FRESH;
1209                 }
1210
1211                 return SOUP_CACHE_RESPONSE_NEEDS_VALIDATION;
1212         }
1213
1214         return SOUP_CACHE_RESPONSE_FRESH;
1215 }
1216
1217 /**
1218  * soup_cache_get_cacheability:
1219  * @cache: a #SoupCache
1220  * @msg: a #SoupMessage
1221  *
1222  * Calculates whether the @msg can be cached or not.
1223  *
1224  * Returns: a #SoupCacheability value indicating whether the @msg can be cached or not.
1225  *
1226  * Since: 2.34
1227  */
1228 SoupCacheability
1229 soup_cache_get_cacheability (SoupCache *cache, SoupMessage *msg)
1230 {
1231         g_return_val_if_fail (SOUP_IS_CACHE (cache), SOUP_CACHE_UNCACHEABLE);
1232         g_return_val_if_fail (SOUP_IS_MESSAGE (msg), SOUP_CACHE_UNCACHEABLE);
1233
1234         return SOUP_CACHE_GET_CLASS (cache)->get_cacheability (cache, msg);
1235 }
1236
1237 static gboolean
1238 force_flush_timeout (gpointer data)
1239 {
1240         gboolean *forced = (gboolean *)data;
1241         *forced = TRUE;
1242
1243         return FALSE;
1244 }
1245
1246 /**
1247  * soup_cache_flush:
1248  * @cache: a #SoupCache
1249  *
1250  * This function will force all pending writes in the @cache to be
1251  * committed to disk. For doing so it will iterate the #GMainContext
1252  * associated with @cache's session as long as needed.
1253  *
1254  * Contrast with soup_cache_dump(), which writes out the cache index
1255  * file.
1256  *
1257  * Since: 2.34
1258  */
1259 void
1260 soup_cache_flush (SoupCache *cache)
1261 {
1262         GMainContext *async_context;
1263         SoupSession *session;
1264         GSource *timeout;
1265         gboolean forced = FALSE;
1266
1267         g_return_if_fail (SOUP_IS_CACHE (cache));
1268
1269         session = cache->priv->session;
1270         g_return_if_fail (SOUP_IS_SESSION (session));
1271         async_context = soup_session_get_async_context (session);
1272
1273         /* We give cache 10 secs to finish */
1274         timeout = soup_add_timeout (async_context, 10000, force_flush_timeout, &forced);
1275
1276         while (!forced && cache->priv->n_pending > 0)
1277                 g_main_context_iteration (async_context, FALSE);
1278
1279         if (!forced)
1280                 g_source_destroy (timeout);
1281         else
1282                 g_warning ("Cache flush finished despite %d pending requests", cache->priv->n_pending);
1283 }
1284
1285 static void
1286 clear_cache_item (gpointer data,
1287                   gpointer user_data)
1288 {
1289         soup_cache_entry_remove ((SoupCache *) user_data, (SoupCacheEntry *) data, TRUE);
1290 }
1291
1292 static void
1293 clear_cache_files (SoupCache *cache)
1294 {
1295         GFileInfo *file_info;
1296         GFileEnumerator *file_enumerator;
1297         GFile *cache_dir_file = g_file_new_for_path (cache->priv->cache_dir);
1298
1299         file_enumerator = g_file_enumerate_children (cache_dir_file, G_FILE_ATTRIBUTE_STANDARD_NAME,
1300                                                      G_FILE_QUERY_INFO_NONE, NULL, NULL);
1301         if (file_enumerator) {
1302                 while ((file_info = g_file_enumerator_next_file (file_enumerator, NULL, NULL)) != NULL) {
1303                         const char *filename = g_file_info_get_name (file_info);
1304
1305                         if (strcmp (filename, SOUP_CACHE_FILE) != 0) {
1306                                 GFile *cache_file = g_file_get_child (cache_dir_file, filename);
1307                                 g_file_delete (cache_file, NULL, NULL);
1308                                 g_object_unref (cache_file);
1309                         }
1310                         g_object_unref (file_info);
1311                 }
1312                 g_object_unref (file_enumerator);
1313         }
1314         g_object_unref (cache_dir_file);
1315 }
1316
1317 /**
1318  * soup_cache_clear:
1319  * @cache: a #SoupCache
1320  *
1321  * Will remove all entries in the @cache plus all the cache files.
1322  *
1323  * Since: 2.34
1324  */
1325 void
1326 soup_cache_clear (SoupCache *cache)
1327 {
1328         GList *entries;
1329
1330         g_return_if_fail (SOUP_IS_CACHE (cache));
1331         g_return_if_fail (cache->priv->cache);
1332
1333         // Cannot use g_hash_table_foreach as callbacks must not modify the hash table
1334         entries = g_hash_table_get_values (cache->priv->cache);
1335         g_list_foreach (entries, clear_cache_item, cache);
1336         g_list_free (entries);
1337
1338         /* Remove also any file not associated with a cache entry. */
1339         clear_cache_files (cache);
1340 }
1341
1342 SoupMessage *
1343 soup_cache_generate_conditional_request (SoupCache *cache, SoupMessage *original)
1344 {
1345         SoupMessage *msg;
1346         SoupURI *uri;
1347         SoupCacheEntry *entry;
1348         const char *last_modified, *etag;
1349         SoupMessagePrivate *origpriv;
1350         GSList *f;
1351
1352         g_return_val_if_fail (SOUP_IS_CACHE (cache), NULL);
1353         g_return_val_if_fail (SOUP_IS_MESSAGE (original), NULL);
1354
1355         /* Add the validator entries in the header from the cached data */
1356         entry = soup_cache_entry_lookup (cache, original);
1357         g_return_val_if_fail (entry, NULL);
1358
1359         last_modified = soup_message_headers_get_one (entry->headers, "Last-Modified");
1360         etag = soup_message_headers_get_one (entry->headers, "ETag");
1361
1362         if (!last_modified && !etag)
1363                 return NULL;
1364
1365         entry->being_validated = TRUE;
1366
1367         /* Copy the data we need from the original message */
1368         uri = soup_message_get_uri (original);
1369         msg = soup_message_new_from_uri (original->method, uri);
1370         soup_message_disable_feature (msg, SOUP_TYPE_CACHE);
1371
1372         soup_message_headers_foreach (original->request_headers,
1373                                       (SoupMessageHeadersForeachFunc)copy_headers,
1374                                       msg->request_headers);
1375
1376         origpriv = SOUP_MESSAGE_GET_PRIVATE (original);
1377         for (f = origpriv->disabled_features; f; f = f->next)
1378                 soup_message_disable_feature (msg, (GType) GPOINTER_TO_SIZE (f->data));
1379
1380         if (last_modified)
1381                 soup_message_headers_append (msg->request_headers,
1382                                              "If-Modified-Since",
1383                                              last_modified);
1384         if (etag)
1385                 soup_message_headers_append (msg->request_headers,
1386                                              "If-None-Match",
1387                                              etag);
1388
1389         return msg;
1390 }
1391
1392 void
1393 soup_cache_cancel_conditional_request (SoupCache   *cache,
1394                                        SoupMessage *msg)
1395 {
1396         SoupCacheEntry *entry;
1397
1398         entry = soup_cache_entry_lookup (cache, msg);
1399         if (entry)
1400                 entry->being_validated = FALSE;
1401
1402         soup_session_cancel_message (cache->priv->session, msg, SOUP_STATUS_CANCELLED);
1403 }
1404
1405 void
1406 soup_cache_update_from_conditional_request (SoupCache   *cache,
1407                                             SoupMessage *msg)
1408 {
1409         SoupCacheEntry *entry = soup_cache_entry_lookup (cache, msg);
1410         if (!entry)
1411                 return;
1412
1413         entry->being_validated = FALSE;
1414
1415         if (msg->status_code == SOUP_STATUS_NOT_MODIFIED) {
1416                 soup_message_headers_foreach (msg->response_headers,
1417                                               (SoupMessageHeadersForeachFunc) remove_headers,
1418                                               entry->headers);
1419                 copy_end_to_end_headers (msg->response_headers, entry->headers);
1420
1421                 soup_cache_entry_set_freshness (entry, msg, cache);
1422         }
1423 }
1424
1425 static void
1426 pack_entry (gpointer data,
1427             gpointer user_data)
1428 {
1429         SoupCacheEntry *entry = (SoupCacheEntry *) data;
1430         SoupMessageHeadersIter iter;
1431         const char *header_key, *header_value;
1432         GVariantBuilder *entries_builder = (GVariantBuilder *)user_data;
1433
1434         /* Do not store non-consolidated entries */
1435         if (entry->dirty || !entry->key)
1436                 return;
1437
1438         g_variant_builder_open (entries_builder, G_VARIANT_TYPE (SOUP_CACHE_PHEADERS_FORMAT));
1439         g_variant_builder_add (entries_builder, "s", entry->uri);
1440         g_variant_builder_add (entries_builder, "b", entry->must_revalidate);
1441         g_variant_builder_add (entries_builder, "u", entry->freshness_lifetime);
1442         g_variant_builder_add (entries_builder, "u", entry->corrected_initial_age);
1443         g_variant_builder_add (entries_builder, "u", entry->response_time);
1444         g_variant_builder_add (entries_builder, "u", entry->hits);
1445         g_variant_builder_add (entries_builder, "u", entry->length);
1446         g_variant_builder_add (entries_builder, "q", entry->status_code);
1447
1448         /* Pack headers */
1449         g_variant_builder_open (entries_builder, G_VARIANT_TYPE ("a" SOUP_CACHE_HEADERS_FORMAT));
1450         soup_message_headers_iter_init (&iter, entry->headers);
1451         while (soup_message_headers_iter_next (&iter, &header_key, &header_value)) {
1452                 if (g_utf8_validate (header_value, -1, NULL))
1453                         g_variant_builder_add (entries_builder, SOUP_CACHE_HEADERS_FORMAT,
1454                                                header_key, header_value);
1455         }
1456         g_variant_builder_close (entries_builder); /* "a" SOUP_CACHE_HEADERS_FORMAT */
1457         g_variant_builder_close (entries_builder); /* SOUP_CACHE_PHEADERS_FORMAT */
1458 }
1459
1460 /**
1461  * soup_cache_dump:
1462  * @cache: a #SoupCache
1463  *
1464  * Synchronously writes the cache index out to disk. Contrast with
1465  * soup_cache_flush(), which writes pending cache
1466  * <emphasis>entries</emphasis> to disk.
1467  *
1468  * You must call this before exiting if you want your cache data to
1469  * persist between sessions.
1470  *
1471  * Since: 2.34.
1472  */
1473 void
1474 soup_cache_dump (SoupCache *cache)
1475 {
1476         SoupCachePrivate *priv = SOUP_CACHE_GET_PRIVATE (cache);
1477         char *filename;
1478         GVariantBuilder entries_builder;
1479         GVariant *cache_variant;
1480
1481         if (!g_list_length (cache->priv->lru_start))
1482                 return;
1483
1484         /* Create the builder and iterate over all entries */
1485         g_variant_builder_init (&entries_builder, G_VARIANT_TYPE (SOUP_CACHE_ENTRIES_FORMAT));
1486         g_variant_builder_add (&entries_builder, "q", SOUP_CACHE_CURRENT_VERSION);
1487         g_variant_builder_open (&entries_builder, G_VARIANT_TYPE ("a" SOUP_CACHE_PHEADERS_FORMAT));
1488         g_list_foreach (cache->priv->lru_start, pack_entry, &entries_builder);
1489         g_variant_builder_close (&entries_builder);
1490
1491         /* Serialize and dump */
1492         cache_variant = g_variant_builder_end (&entries_builder);
1493         g_variant_ref_sink (cache_variant);
1494         filename = g_build_filename (priv->cache_dir, SOUP_CACHE_FILE, NULL);
1495         g_file_set_contents (filename, (const char *) g_variant_get_data (cache_variant),
1496                              g_variant_get_size (cache_variant), NULL);
1497         g_free (filename);
1498         g_variant_unref (cache_variant);
1499 }
1500
1501 /**
1502  * soup_cache_load:
1503  * @cache: a #SoupCache
1504  *
1505  * Loads the contents of @cache's index into memory.
1506  *
1507  * Since: 2.34
1508  */
1509 void
1510 soup_cache_load (SoupCache *cache)
1511 {
1512         gboolean must_revalidate;
1513         guint32 freshness_lifetime, hits;
1514         guint32 corrected_initial_age, response_time;
1515         char *url, *filename = NULL, *contents = NULL;
1516         GVariant *cache_variant;
1517         GVariantIter *entries_iter = NULL, *headers_iter = NULL;
1518         gsize length;
1519         SoupCacheEntry *entry;
1520         SoupCachePrivate *priv = cache->priv;
1521         guint16 version, status_code;
1522
1523         filename = g_build_filename (priv->cache_dir, SOUP_CACHE_FILE, NULL);
1524         if (!g_file_get_contents (filename, &contents, &length, NULL)) {
1525                 g_free (filename);
1526                 g_free (contents);
1527                 clear_cache_files (cache);
1528                 return;
1529         }
1530         g_free (filename);
1531
1532         cache_variant = g_variant_new_from_data (G_VARIANT_TYPE (SOUP_CACHE_ENTRIES_FORMAT),
1533                                                  (const char *) contents, length, FALSE, g_free, contents);
1534         g_variant_get (cache_variant, SOUP_CACHE_ENTRIES_FORMAT, &version, &entries_iter);
1535         if (version != SOUP_CACHE_CURRENT_VERSION) {
1536                 g_variant_iter_free (entries_iter);
1537                 g_variant_unref (cache_variant);
1538                 clear_cache_files (cache);
1539                 return;
1540         }
1541
1542         while (g_variant_iter_loop (entries_iter, SOUP_CACHE_PHEADERS_FORMAT,
1543                                     &url, &must_revalidate, &freshness_lifetime, &corrected_initial_age,
1544                                     &response_time, &hits, &length, &status_code,
1545                                     &headers_iter)) {
1546                 const char *header_key, *header_value;
1547                 SoupMessageHeaders *headers;
1548                 SoupMessageHeadersIter soup_headers_iter;
1549
1550                 /* SoupMessage Headers */
1551                 headers = soup_message_headers_new (SOUP_MESSAGE_HEADERS_RESPONSE);
1552                 while (g_variant_iter_loop (headers_iter, SOUP_CACHE_HEADERS_FORMAT, &header_key, &header_value))
1553                         if (*header_key && *header_value)
1554                                 soup_message_headers_append (headers, header_key, header_value);
1555
1556                 /* Check that we have headers */
1557                 soup_message_headers_iter_init (&soup_headers_iter, headers);
1558                 if (!soup_message_headers_iter_next (&soup_headers_iter, &header_key, &header_value)) {
1559                         soup_message_headers_free (headers);
1560                         continue;
1561                 }
1562
1563                 /* Insert in cache */
1564                 entry = g_slice_new0 (SoupCacheEntry);
1565                 entry->uri = g_strdup (url);
1566                 entry->must_revalidate = must_revalidate;
1567                 entry->freshness_lifetime = freshness_lifetime;
1568                 entry->corrected_initial_age = corrected_initial_age;
1569                 entry->response_time = response_time;
1570                 entry->hits = hits;
1571                 entry->length = length;
1572                 entry->headers = headers;
1573                 entry->status_code = status_code;
1574
1575                 if (!soup_cache_entry_insert (cache, entry, FALSE))
1576                         soup_cache_entry_free (entry);
1577         }
1578
1579         cache->priv->lru_start = g_list_reverse (cache->priv->lru_start);
1580
1581         /* frees */
1582         g_variant_iter_free (entries_iter);
1583         g_variant_unref (cache_variant);
1584 }
1585
1586 /**
1587  * soup_cache_set_max_size:
1588  * @cache: a #SoupCache
1589  * @max_size: the maximum size of the cache, in bytes
1590  *
1591  * Sets the maximum size of the cache.
1592  *
1593  * Since: 2.34
1594  */
1595 void
1596 soup_cache_set_max_size (SoupCache *cache,
1597                          guint      max_size)
1598 {
1599         cache->priv->max_size = max_size;
1600         cache->priv->max_entry_data_size = cache->priv->max_size / MAX_ENTRY_DATA_PERCENTAGE;
1601 }
1602
1603 /**
1604  * soup_cache_get_max_size:
1605  * @cache: a #SoupCache
1606  *
1607  * Gets the maximum size of the cache.
1608  *
1609  * Return value: the maximum size of the cache, in bytes.
1610  *
1611  * Since: 2.34
1612  */
1613 guint
1614 soup_cache_get_max_size (SoupCache *cache)
1615 {
1616         return cache->priv->max_size;
1617 }