libsoup/soup-cache.c

   1 /* -*- Mode: C; tab-width: 8; indent-tabs-mode: t; c-basic-offset: 8 -*- */
   2 /*
   3  * soup-cache.c
   4  *
   5  * Copyright (C) 2009, 2010 Igalia S.L.
   6  *
   7  * This library is free software; you can redistribute it and/or
   8  * modify it under the terms of the GNU Library General Public
   9  * License as published by the Free Software Foundation; either
  10  * version 2 of the License, or (at your option) any later version.
  11  *
  12  * This library is distributed in the hope that it will be useful,
  13  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  14  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  15  * Library General Public License for more details.
  16  *
  17  * You should have received a copy of the GNU Library General Public License
  18  * along with this library; see the file COPYING.LIB.  If not, write to
  19  * the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
  20  * Boston, MA 02110-1301, USA.
  21  */
  22
  23 /* TODO:
  24  * - Need to hook the feature in the sync SoupSession.
  25  * - Need more tests.
  26  */
  27
  28 #ifdef HAVE_CONFIG_H
  29 #include <config.h>
  30 #endif
  31
  32 #include <stdlib.h>
  33 #include <string.h>
  34 #include <gio/gio.h>
  35
  36 #define LIBSOUP_USE_UNSTABLE_REQUEST_API
  37
  38 #include "soup-cache.h"
  39 #include "soup-cache-private.h"
  40 #include "soup-date.h"
  41 #include "soup-enum-types.h"
  42 #include "soup-headers.h"
  43 #include "soup-session.h"
  44 #include "soup-session-feature.h"
  45 #include "soup-uri.h"
  46
  47 static SoupSessionFeatureInterface *soup_cache_default_feature_interface;
  48 static void soup_cache_session_feature_init (SoupSessionFeatureInterface *feature_interface, gpointer interface_data);
  49
  50 #define DEFAULT_MAX_SIZE 50 * 1024 * 1024
  51 #define MAX_ENTRY_DATA_PERCENTAGE 10 /* Percentage of the total size
  52                                         of the cache that can be
  53                                         filled by a single entry */
  54
  55 /*
  56  * Version 2: cache is now saved in soup.cache2. Added the version
  57  * number to the beginning of the file.
  58  *
  59  * Version 3: added HTTP status code to the cache entries.
  60  *
  61  * Version 4: replaced several types.
  62  *   - freshness_lifetime,corrected_initial_age,response_time: time_t -> guint32
  63  *   - status_code: guint -> guint16
  64  *   - hits: guint -> guint32
  65  *
  66  * Version 5: key is no longer stored on disk as it can be easily
  67  * built from the URI. Apart from that some fields in the
  68  * SoupCacheEntry have changed:
  69  *   - entry key is now a uint32 instead of a (char *).
  70  *   - added uri, used to check for collisions
  71  *   - removed filename, it's built from the entry key.
  72  */
  73 #define SOUP_CACHE_CURRENT_VERSION 5
  74
  75 typedef struct _SoupCacheEntry {
  76         guint32 key;
  77         char *uri;
  78         guint32 freshness_lifetime;
  79         gboolean must_revalidate;
  80         gsize length;
  81         guint32 corrected_initial_age;
  82         guint32 response_time;
  83         SoupBuffer *current_writing_buffer;
  84         gboolean dirty;
  85         gboolean got_body;
  86         gboolean being_validated;
  87         SoupMessageHeaders *headers;
  88         GOutputStream *stream;
  89         GError *error;
  90         guint32 hits;
  91         GCancellable *cancellable;
  92         guint16 status_code;
  93 } SoupCacheEntry;
  94
  95 struct _SoupCachePrivate {
  96         char *cache_dir;
  97         GHashTable *cache;
  98         guint n_pending;
  99         SoupSession *session;
 100         SoupCacheType cache_type;
 101         guint size;
 102         guint max_size;
 103         guint max_entry_data_size; /* Computed value. Here for performance reasons */
 104         GList *lru_start;
 105 };
 106
 107 typedef struct {
 108         SoupCache *cache;
 109         SoupCacheEntry *entry;
 110         SoupMessage *msg;
 111         gulong got_chunk_handler;
 112         gulong got_body_handler;
 113         gulong restarted_handler;
 114         GQueue *buffer_queue;
 115 } SoupCacheWritingFixture;
 116
 117 enum {
 118         PROP_0,
 119         PROP_CACHE_DIR,
 120         PROP_CACHE_TYPE
 121 };
 122
 123 #define SOUP_CACHE_GET_PRIVATE(o) (G_TYPE_INSTANCE_GET_PRIVATE ((o), SOUP_TYPE_CACHE, SoupCachePrivate))
 124
 125 G_DEFINE_TYPE_WITH_CODE (SoupCache, soup_cache, G_TYPE_OBJECT,
 126                          G_IMPLEMENT_INTERFACE (SOUP_TYPE_SESSION_FEATURE,
 127                                                 soup_cache_session_feature_init))
 128
 129 static gboolean soup_cache_entry_remove (SoupCache *cache, SoupCacheEntry *entry);
 130 static void make_room_for_new_entry (SoupCache *cache, guint length_to_add);
 131 static gboolean cache_accepts_entries_of_size (SoupCache *cache, guint length_to_add);
 132 static gboolean write_next_buffer (SoupCacheEntry *entry, SoupCacheWritingFixture *fixture);
 133
 134 static GFile *
 135 get_file_from_entry (SoupCache *cache, SoupCacheEntry *entry)
 136 {
 137         char *filename = g_strdup_printf ("%s%s%u", cache->priv->cache_dir,
 138                                           G_DIR_SEPARATOR_S, (guint) entry->key);
 139         GFile *file = g_file_new_for_path (filename);
 140         g_free (filename);
 141
 142         return file;
 143 }
 144
 145 static SoupCacheability
 146 get_cacheability (SoupCache *cache, SoupMessage *msg)
 147 {
 148         SoupCacheability cacheability;
 149         const char *cache_control, *content_type;
 150         gboolean has_max_age = FALSE;
 151
 152         /* 1. The request method must be cacheable */
 153         if (msg->method == SOUP_METHOD_GET)
 154                 cacheability = SOUP_CACHE_CACHEABLE;
 155         else if (msg->method == SOUP_METHOD_HEAD ||
 156                  msg->method == SOUP_METHOD_TRACE ||
 157                  msg->method == SOUP_METHOD_CONNECT)
 158                 return SOUP_CACHE_UNCACHEABLE;
 159         else
 160                 return (SOUP_CACHE_UNCACHEABLE | SOUP_CACHE_INVALIDATES);
 161
 162         content_type = soup_message_headers_get_content_type (msg->response_headers, NULL);
 163         if (content_type && !g_ascii_strcasecmp (content_type, "multipart/x-mixed-replace"))
 164                 return SOUP_CACHE_UNCACHEABLE;
 165
 166         cache_control = soup_message_headers_get (msg->response_headers, "Cache-Control");
 167         if (cache_control && *cache_control) {
 168                 GHashTable *hash;
 169                 SoupCachePrivate *priv = SOUP_CACHE_GET_PRIVATE (cache);
 170
 171                 hash = soup_header_parse_param_list (cache_control);
 172
 173                 /* Shared caches MUST NOT store private resources */
 174                 if (priv->cache_type == SOUP_CACHE_SHARED) {
 175                         if (g_hash_table_lookup_extended (hash, "private", NULL, NULL)) {
 176                                 soup_header_free_param_list (hash);
 177                                 return SOUP_CACHE_UNCACHEABLE;
 178                         }
 179                 }
 180
 181                 /* 2. The 'no-store' cache directive does not appear in the
 182                  * headers
 183                  */
 184                 if (g_hash_table_lookup_extended (hash, "no-store", NULL, NULL)) {
 185                         soup_header_free_param_list (hash);
 186                         return SOUP_CACHE_UNCACHEABLE;
 187                 }
 188
 189                 if (g_hash_table_lookup_extended (hash, "max-age", NULL, NULL))
 190                         has_max_age = TRUE;
 191
 192                 /* This does not appear in section 2.1, but I think it makes
 193                  * sense to check it too?
 194                  */
 195                 if (g_hash_table_lookup_extended (hash, "no-cache", NULL, NULL)) {
 196                         soup_header_free_param_list (hash);
 197                         return SOUP_CACHE_UNCACHEABLE;
 198                 }
 199
 200                 soup_header_free_param_list (hash);
 201         }
 202
 203         /* Section 13.9 */
 204         if ((soup_message_get_uri (msg))->query &&
 205             !soup_message_headers_get_one (msg->response_headers, "Expires") &&
 206             !has_max_age)
 207                 return SOUP_CACHE_UNCACHEABLE;
 208
 209         switch (msg->status_code) {
 210         case SOUP_STATUS_PARTIAL_CONTENT:
 211                 /* We don't cache partial responses, but they only
 212                  * invalidate cached full responses if the headers
 213                  * don't match.
 214                  */
 215                 cacheability = SOUP_CACHE_UNCACHEABLE;
 216                 break;
 217
 218         case SOUP_STATUS_NOT_MODIFIED:
 219                 /* A 304 response validates an existing cache entry */
 220                 cacheability = SOUP_CACHE_VALIDATES;
 221                 break;
 222
 223         case SOUP_STATUS_MULTIPLE_CHOICES:
 224         case SOUP_STATUS_MOVED_PERMANENTLY:
 225         case SOUP_STATUS_GONE:
 226                 /* FIXME: cacheable unless indicated otherwise */
 227                 cacheability = SOUP_CACHE_UNCACHEABLE;
 228                 break;
 229
 230         case SOUP_STATUS_FOUND:
 231         case SOUP_STATUS_TEMPORARY_REDIRECT:
 232                 /* FIXME: cacheable if explicitly indicated */
 233                 cacheability = SOUP_CACHE_UNCACHEABLE;
 234                 break;
 235
 236         case SOUP_STATUS_SEE_OTHER:
 237         case SOUP_STATUS_FORBIDDEN:
 238         case SOUP_STATUS_NOT_FOUND:
 239         case SOUP_STATUS_METHOD_NOT_ALLOWED:
 240                 return (SOUP_CACHE_UNCACHEABLE | SOUP_CACHE_INVALIDATES);
 241
 242         default:
 243                 /* Any 5xx status or any 4xx status not handled above
 244                  * is uncacheable but doesn't break the cache.
 245                  */
 246                 if ((msg->status_code >= SOUP_STATUS_BAD_REQUEST &&
 247                      msg->status_code <= SOUP_STATUS_FAILED_DEPENDENCY) ||
 248                     msg->status_code >= SOUP_STATUS_INTERNAL_SERVER_ERROR)
 249                         return SOUP_CACHE_UNCACHEABLE;
 250
 251                 /* An unrecognized 2xx, 3xx, or 4xx response breaks
 252                  * the cache.
 253                  */
 254                 if ((msg->status_code > SOUP_STATUS_PARTIAL_CONTENT &&
 255                      msg->status_code < SOUP_STATUS_MULTIPLE_CHOICES) ||
 256                     (msg->status_code > SOUP_STATUS_TEMPORARY_REDIRECT &&
 257                      msg->status_code < SOUP_STATUS_INTERNAL_SERVER_ERROR))
 258                         return (SOUP_CACHE_UNCACHEABLE | SOUP_CACHE_INVALIDATES);
 259                 break;
 260         }
 261
 262         return cacheability;
 263 }
 264
 265 /* NOTE: this function deletes the file pointed by the file argument
 266  * and also unref's the GFile object representing it.
 267  */
 268 static void
 269 soup_cache_entry_free (SoupCacheEntry *entry, GFile *file)
 270 {
 271         if (file) {
 272                 g_file_delete (file, NULL, NULL);
 273                 g_object_unref (file);
 274         }
 275
 276         g_free (entry->uri);
 277         entry->uri = NULL;
 278
 279         if (entry->current_writing_buffer) {
 280                 soup_buffer_free (entry->current_writing_buffer);
 281                 entry->current_writing_buffer = NULL;
 282         }
 283
 284         if (entry->headers) {
 285                 soup_message_headers_free (entry->headers);
 286                 entry->headers = NULL;
 287         }
 288         if (entry->error) {
 289                 g_error_free (entry->error);
 290                 entry->error = NULL;
 291         }
 292         if (entry->cancellable) {
 293                 g_object_unref (entry->cancellable);
 294                 entry->cancellable = NULL;
 295         }
 296
 297         g_slice_free (SoupCacheEntry, entry);
 298 }
 299
 300 static void
 301 copy_headers (const char *name, const char *value, SoupMessageHeaders *headers)
 302 {
 303         soup_message_headers_append (headers, name, value);
 304 }
 305
 306 static char *hop_by_hop_headers[] = {"Connection", "Keep-Alive", "Proxy-Authenticate", "Proxy-Authorization", "TE", "Trailer", "Transfer-Encoding", "Upgrade"};
 307
 308 static void
 309 copy_end_to_end_headers (SoupMessageHeaders *source, SoupMessageHeaders *destination)
 310 {
 311         int i;
 312
 313         soup_message_headers_foreach (source, (SoupMessageHeadersForeachFunc) copy_headers, destination);
 314         for (i = 0; i < G_N_ELEMENTS (hop_by_hop_headers); i++)
 315                 soup_message_headers_remove (destination, hop_by_hop_headers[i]);
 316         soup_message_headers_clean_connection_headers (destination);
 317 }
 318
 319 static guint
 320 soup_cache_entry_get_current_age (SoupCacheEntry *entry)
 321 {
 322         time_t now = time (NULL);
 323         time_t resident_time;
 324
 325         resident_time = now - entry->response_time;
 326         return entry->corrected_initial_age + resident_time;
 327 }
 328
 329 static gboolean
 330 soup_cache_entry_is_fresh_enough (SoupCacheEntry *entry, gint min_fresh)
 331 {
 332         guint limit = (min_fresh == -1) ? soup_cache_entry_get_current_age (entry) : (guint) min_fresh;
 333         return entry->freshness_lifetime > limit;
 334 }
 335
 336 static inline guint32
 337 get_cache_key_from_uri (const char *uri)
 338 {
 339         return (guint32) g_str_hash (uri);
 340 }
 341
 342 static void
 343 soup_cache_entry_set_freshness (SoupCacheEntry *entry, SoupMessage *msg, SoupCache *cache)
 344 {
 345         const char *cache_control;
 346         const char *expires, *date, *last_modified;
 347
 348         cache_control = soup_message_headers_get (entry->headers, "Cache-Control");
 349         if (cache_control) {
 350                 const char *max_age, *s_maxage;
 351                 gint64 freshness_lifetime = 0;
 352                 GHashTable *hash;
 353                 SoupCachePrivate *priv = SOUP_CACHE_GET_PRIVATE (cache);
 354
 355                 hash = soup_header_parse_param_list (cache_control);
 356
 357                 /* Should we re-validate the entry when it goes stale */
 358                 entry->must_revalidate = g_hash_table_lookup_extended (hash, "must-revalidate", NULL, NULL);
 359
 360                 /* Section 2.3.1 */
 361                 if (priv->cache_type == SOUP_CACHE_SHARED) {
 362                         s_maxage = g_hash_table_lookup (hash, "s-maxage");
 363                         if (s_maxage) {
 364                                 freshness_lifetime = g_ascii_strtoll (s_maxage, NULL, 10);
 365                                 if (freshness_lifetime) {
 366                                         /* Implies proxy-revalidate. TODO: is it true? */
 367                                         entry->must_revalidate = TRUE;
 368                                         soup_header_free_param_list (hash);
 369                                         return;
 370                                 }
 371                         }
 372                 }
 373
 374                 /* If 'max-age' cache directive is present, use that */
 375                 max_age = g_hash_table_lookup (hash, "max-age");
 376                 if (max_age)
 377                         freshness_lifetime = g_ascii_strtoll (max_age, NULL, 10);
 378
 379                 if (freshness_lifetime) {
 380                         entry->freshness_lifetime = (guint32) MIN (freshness_lifetime, G_MAXUINT32);
 381                         soup_header_free_param_list (hash);
 382                         return;
 383                 }
 384
 385                 soup_header_free_param_list (hash);
 386         }
 387
 388         /* If the 'Expires' response header is present, use its value
 389          * minus the value of the 'Date' response header
 390          */
 391         expires = soup_message_headers_get (entry->headers, "Expires");
 392         date = soup_message_headers_get (entry->headers, "Date");
 393         if (expires && date) {
 394                 SoupDate *expires_d, *date_d;
 395                 time_t expires_t, date_t;
 396
 397                 expires_d = soup_date_new_from_string (expires);
 398                 if (expires_d) {
 399                         date_d = soup_date_new_from_string (date);
 400
 401                         expires_t = soup_date_to_time_t (expires_d);
 402                         date_t = soup_date_to_time_t (date_d);
 403
 404                         soup_date_free (expires_d);
 405                         soup_date_free (date_d);
 406
 407                         if (expires_t && date_t) {
 408                                 entry->freshness_lifetime = (guint32) MAX (expires_t - date_t, 0);
 409                                 return;
 410                         }
 411                 } else {
 412                         /* If Expires is not a valid date we should
 413                            treat it as already expired, see section
 414                            3.3 */
 415                         entry->freshness_lifetime = 0;
 416                         return;
 417                 }
 418         }
 419
 420         /* Otherwise an heuristic may be used */
 421
 422         /* Heuristics MUST NOT be used with stored responses with
 423            these status codes (section 2.3.1.1) */
 424         if (entry->status_code != SOUP_STATUS_OK &&
 425             entry->status_code != SOUP_STATUS_NON_AUTHORITATIVE &&
 426             entry->status_code != SOUP_STATUS_PARTIAL_CONTENT &&
 427             entry->status_code != SOUP_STATUS_MULTIPLE_CHOICES &&
 428             entry->status_code != SOUP_STATUS_MOVED_PERMANENTLY &&
 429             entry->status_code != SOUP_STATUS_GONE)
 430                 goto expire;
 431
 432         /* TODO: attach warning 113 if response's current_age is more
 433            than 24h (section 2.3.1.1) when using heuristics */
 434
 435         /* Last-Modified based heuristic */
 436         last_modified = soup_message_headers_get (entry->headers, "Last-Modified");
 437         if (last_modified) {
 438                 SoupDate *soup_date;
 439                 time_t now, last_modified_t;
 440
 441                 soup_date = soup_date_new_from_string (last_modified);
 442                 last_modified_t = soup_date_to_time_t (soup_date);
 443                 now = time (NULL);
 444
 445 #define HEURISTIC_FACTOR 0.1 /* From Section 2.3.1.1 */
 446
 447                 entry->freshness_lifetime = MAX (0, (now - last_modified_t) * HEURISTIC_FACTOR);
 448                 soup_date_free (soup_date);
 449         }
 450
 451         return;
 452
 453  expire:
 454         /* If all else fails, make the entry expire immediately */
 455         entry->freshness_lifetime = 0;
 456 }
 457
 458 static SoupCacheEntry *
 459 soup_cache_entry_new (SoupCache *cache, SoupMessage *msg, time_t request_time, time_t response_time)
 460 {
 461         SoupCacheEntry *entry;
 462         const char *date;
 463
 464         entry = g_slice_new0 (SoupCacheEntry);
 465         entry->dirty = FALSE;
 466         entry->current_writing_buffer = NULL;
 467         entry->got_body = FALSE;
 468         entry->being_validated = FALSE;
 469         entry->error = NULL;
 470         entry->status_code = msg->status_code;
 471         entry->response_time = response_time;
 472         entry->uri = soup_uri_to_string (soup_message_get_uri (msg), FALSE);
 473
 474         /* Headers */
 475         entry->headers = soup_message_headers_new (SOUP_MESSAGE_HEADERS_RESPONSE);
 476         copy_end_to_end_headers (msg->response_headers, entry->headers);
 477
 478         /* LRU list */
 479         entry->hits = 0;
 480
 481         /* Section 2.3.1, Freshness Lifetime */
 482         soup_cache_entry_set_freshness (entry, msg, cache);
 483
 484         /* Section 2.3.2, Calculating Age */
 485         date = soup_message_headers_get (entry->headers, "Date");
 486
 487         if (date) {
 488                 SoupDate *soup_date;
 489                 const char *age;
 490                 time_t date_value, apparent_age, corrected_received_age, response_delay, age_value = 0;
 491
 492                 soup_date = soup_date_new_from_string (date);
 493                 date_value = soup_date_to_time_t (soup_date);
 494                 soup_date_free (soup_date);
 495
 496                 age = soup_message_headers_get (entry->headers, "Age");
 497                 if (age)
 498                         age_value = g_ascii_strtoll (age, NULL, 10);
 499
 500                 apparent_age = MAX (0, entry->response_time - date_value);
 501                 corrected_received_age = MAX (apparent_age, age_value);
 502                 response_delay = entry->response_time - request_time;
 503                 entry->corrected_initial_age = corrected_received_age + response_delay;
 504         } else {
 505                 /* Is this correct ? */
 506                 entry->corrected_initial_age = time (NULL);
 507         }
 508
 509         return entry;
 510 }
 511
 512 static void
 513 soup_cache_writing_fixture_free (SoupCacheWritingFixture *fixture)
 514 {
 515         /* Free fixture. And disconnect signals, we don't want to
 516            listen to more SoupMessage events as we're finished with
 517            this resource */
 518         if (g_signal_handler_is_connected (fixture->msg, fixture->got_chunk_handler))
 519                 g_signal_handler_disconnect (fixture->msg, fixture->got_chunk_handler);
 520         if (g_signal_handler_is_connected (fixture->msg, fixture->got_body_handler))
 521                 g_signal_handler_disconnect (fixture->msg, fixture->got_body_handler);
 522         if (g_signal_handler_is_connected (fixture->msg, fixture->restarted_handler))
 523                 g_signal_handler_disconnect (fixture->msg, fixture->restarted_handler);
 524         g_queue_foreach (fixture->buffer_queue, (GFunc) soup_buffer_free, NULL);
 525         g_queue_free (fixture->buffer_queue);
 526         g_object_unref (fixture->msg);
 527         g_object_unref (fixture->cache);
 528         g_slice_free (SoupCacheWritingFixture, fixture);
 529 }
 530
 531 static void
 532 close_ready_cb (GObject *source, GAsyncResult *result, SoupCacheWritingFixture *fixture)
 533 {
 534         SoupCacheEntry *entry = fixture->entry;
 535         SoupCache *cache = fixture->cache;
 536         GOutputStream *stream = G_OUTPUT_STREAM (source);
 537         goffset content_length;
 538
 539         g_warn_if_fail (entry->error == NULL);
 540
 541         /* FIXME: what do we do on error ? */
 542
 543         if (stream) {
 544                 g_output_stream_close_finish (stream, result, NULL);
 545                 g_object_unref (stream);
 546         }
 547         entry->stream = NULL;
 548
 549         content_length = soup_message_headers_get_content_length (entry->headers);
 550
 551         /* If the process was cancelled, then delete the entry from
 552            the cache. Do it also if the size of a chunked resource is
 553            too much for the cache */
 554         if (g_cancellable_is_cancelled (entry->cancellable)) {
 555                 entry->dirty = FALSE;
 556                 soup_cache_entry_remove (cache, entry);
 557                 soup_cache_entry_free (entry, get_file_from_entry (cache, entry));
 558                 entry = NULL;
 559         } else if ((soup_message_headers_get_encoding (entry->headers) == SOUP_ENCODING_CHUNKED) ||
 560                    entry->length != (gsize) content_length) {
 561                 /* Two options here:
 562                  *
 563                  * 1. "chunked" data, entry was temporarily added to
 564                  * cache (as content-length is 0) and now that we have
 565                  * the actual size we have to evaluate if we want it
 566                  * in the cache or not
 567                  *
 568                  * 2. Content-Length has a different value than actual
 569                  * length, means that the content was encoded for
 570                  * transmission (typically compressed) and thus we
 571                  * have to substract the content-length value that was
 572                  * added to the cache and add the unencoded length
 573                  */
 574                 gint length_to_add = entry->length - content_length;
 575
 576                 /* Make room in cache if needed */
 577                 if (cache_accepts_entries_of_size (cache, length_to_add)) {
 578                         make_room_for_new_entry (cache, length_to_add);
 579
 580                         cache->priv->size += length_to_add;
 581                 } else {
 582                         entry->dirty = FALSE;
 583                         soup_cache_entry_remove (cache, entry);
 584                         soup_cache_entry_free (entry, get_file_from_entry (cache, entry));
 585                         entry = NULL;
 586                 }
 587         }
 588
 589         if (entry) {
 590                 entry->dirty = FALSE;
 591                 entry->got_body = FALSE;
 592
 593                 if (entry->current_writing_buffer) {
 594                         soup_buffer_free (entry->current_writing_buffer);
 595                         entry->current_writing_buffer = NULL;
 596                 }
 597
 598                 g_object_unref (entry->cancellable);
 599                 entry->cancellable = NULL;
 600         }
 601
 602         cache->priv->n_pending--;
 603
 604         /* Frees */
 605         soup_cache_writing_fixture_free (fixture);
 606 }
 607
 608 static void
 609 write_ready_cb (GObject *source, GAsyncResult *result, SoupCacheWritingFixture *fixture)
 610 {
 611         GOutputStream *stream = G_OUTPUT_STREAM (source);
 612         GError *error = NULL;
 613         gssize write_size;
 614         SoupCacheEntry *entry = fixture->entry;
 615
 616         if (g_cancellable_is_cancelled (entry->cancellable)) {
 617                 g_output_stream_close_async (stream,
 618                                              G_PRIORITY_LOW,
 619                                              entry->cancellable,
 620                                              (GAsyncReadyCallback)close_ready_cb,
 621                                              fixture);
 622                 return;
 623         }
 624
 625         write_size = g_output_stream_write_finish (stream, result, &error);
 626         if (write_size <= 0 || error) {
 627                 if (error)
 628                         entry->error = error;
 629                 g_output_stream_close_async (stream,
 630                                              G_PRIORITY_LOW,
 631                                              entry->cancellable,
 632                                              (GAsyncReadyCallback)close_ready_cb,
 633                                              fixture);
 634                 /* FIXME: We should completely stop caching the
 635                    resource at this point */
 636         } else {
 637                 /* Are we still writing and is there new data to write
 638                    already ? */
 639                 if (fixture->buffer_queue->length > 0)
 640                         write_next_buffer (entry, fixture);
 641                 else {
 642                         soup_buffer_free (entry->current_writing_buffer);
 643                         entry->current_writing_buffer = NULL;
 644
 645                         if (entry->got_body) {
 646                                 /* If we already received 'got-body'
 647                                    and we have written all the data,
 648                                    we can close the stream */
 649                                 g_output_stream_close_async (entry->stream,
 650                                                              G_PRIORITY_LOW,
 651                                                              entry->cancellable,
 652                                                              (GAsyncReadyCallback)close_ready_cb,
 653                                                              fixture);
 654                         }
 655                 }
 656         }
 657 }
 658
 659 static gboolean
 660 write_next_buffer (SoupCacheEntry *entry, SoupCacheWritingFixture *fixture)
 661 {
 662         SoupBuffer *buffer = g_queue_pop_head (fixture->buffer_queue);
 663
 664         if (buffer == NULL)
 665                 return FALSE;
 666
 667         /* Free the old buffer */
 668         if (entry->current_writing_buffer) {
 669                 soup_buffer_free (entry->current_writing_buffer);
 670                 entry->current_writing_buffer = NULL;
 671         }
 672         entry->current_writing_buffer = buffer;
 673
 674         g_output_stream_write_async (entry->stream, buffer->data, buffer->length,
 675                                      G_PRIORITY_LOW, entry->cancellable,
 676                                      (GAsyncReadyCallback) write_ready_cb,
 677                                      fixture);
 678         return TRUE;
 679 }
 680
 681 static void
 682 msg_got_chunk_cb (SoupMessage *msg, SoupBuffer *chunk, SoupCacheWritingFixture *fixture)
 683 {
 684         SoupCacheEntry *entry = fixture->entry;
 685
 686         /* Ignore this if the writing or appending was cancelled */
 687         if (!g_cancellable_is_cancelled (entry->cancellable)) {
 688                 g_queue_push_tail (fixture->buffer_queue, soup_buffer_copy (chunk));
 689                 entry->length += chunk->length;
 690
 691                 if (!cache_accepts_entries_of_size (fixture->cache, entry->length)) {
 692                         /* Quickly cancel the caching of the resource */
 693                         g_cancellable_cancel (entry->cancellable);
 694                 }
 695         }
 696
 697         /* FIXME: remove the error check when we cancel the caching at
 698            the first write error */
 699         /* Only write if the entry stream is ready */
 700         if (entry->current_writing_buffer == NULL && entry->error == NULL && entry->stream)
 701                 write_next_buffer (entry, fixture);
 702 }
 703
 704 static void
 705 msg_got_body_cb (SoupMessage *msg, SoupCacheWritingFixture *fixture)
 706 {
 707         SoupCacheEntry *entry = fixture->entry;
 708         g_return_if_fail (entry);
 709
 710         entry->got_body = TRUE;
 711
 712         if (!entry->stream && fixture->buffer_queue->length > 0)
 713                 /* The stream is not ready to be written but we still
 714                    have data to write, we'll write it when the stream
 715                    is opened for writing */
 716                 return;
 717
 718
 719         if (fixture->buffer_queue->length > 0) {
 720                 /* If we still have data to write, write it,
 721                    write_ready_cb will close the stream */
 722                 if (entry->current_writing_buffer == NULL && entry->error == NULL && entry->stream)
 723                         write_next_buffer (entry, fixture);
 724                 return;
 725         }
 726
 727         if (entry->stream && entry->current_writing_buffer == NULL)
 728                 g_output_stream_close_async (entry->stream,
 729                                              G_PRIORITY_LOW,
 730                                              entry->cancellable,
 731                                              (GAsyncReadyCallback)close_ready_cb,
 732                                              fixture);
 733 }
 734
 735 static gboolean
 736 soup_cache_entry_remove (SoupCache *cache, SoupCacheEntry *entry)
 737 {
 738         GList *lru_item;
 739
 740         /* if (entry->dirty && !g_cancellable_is_cancelled (entry->cancellable)) { */
 741         if (entry->dirty) {
 742                 g_cancellable_cancel (entry->cancellable);
 743                 return FALSE;
 744         }
 745
 746         g_assert (!entry->dirty);
 747         g_assert (g_list_length (cache->priv->lru_start) == g_hash_table_size (cache->priv->cache));
 748
 749         if (!g_hash_table_remove (cache->priv->cache, GUINT_TO_POINTER (entry->key)))
 750                 return FALSE;
 751
 752         /* Remove from LRU */
 753         lru_item = g_list_find (cache->priv->lru_start, entry);
 754         cache->priv->lru_start = g_list_delete_link (cache->priv->lru_start, lru_item);
 755
 756         /* Adjust cache size */
 757         cache->priv->size -= entry->length;
 758
 759         g_assert (g_list_length (cache->priv->lru_start) == g_hash_table_size (cache->priv->cache));
 760
 761         return TRUE;
 762 }
 763
 764 static gint
 765 lru_compare_func (gconstpointer a, gconstpointer b)
 766 {
 767         SoupCacheEntry *entry_a = (SoupCacheEntry *)a;
 768         SoupCacheEntry *entry_b = (SoupCacheEntry *)b;
 769
 770         /* The rationale of this sorting func is
 771          *
 772          * 1. sort by hits -> LRU algorithm, then
 773          *
 774          * 2. sort by freshness lifetime, we better discard first
 775          * entries that are close to expire
 776          *
 777          * 3. sort by size, replace first small size resources as they
 778          * are cheaper to download
 779          */
 780
 781         /* Sort by hits */
 782         if (entry_a->hits != entry_b->hits)
 783                 return entry_a->hits - entry_b->hits;
 784
 785         /* Sort by freshness_lifetime */
 786         if (entry_a->freshness_lifetime != entry_b->freshness_lifetime)
 787                 return entry_a->freshness_lifetime - entry_b->freshness_lifetime;
 788
 789         /* Sort by size */
 790         return entry_a->length - entry_b->length;
 791 }
 792
 793 static gboolean
 794 cache_accepts_entries_of_size (SoupCache *cache, guint length_to_add)
 795 {
 796         /* We could add here some more heuristics. TODO: review how
 797            this is done by other HTTP caches */
 798
 799         return length_to_add <= cache->priv->max_entry_data_size;
 800 }
 801
 802 static void
 803 make_room_for_new_entry (SoupCache *cache, guint length_to_add)
 804 {
 805         GList *lru_entry = cache->priv->lru_start;
 806
 807         /* Check that there is enough room for the new entry. This is
 808            an approximation as we're not working out the size of the
 809            cache file or the size of the headers for performance
 810            reasons. TODO: check if that would be really that expensive */
 811
 812         while (lru_entry &&
 813                (length_to_add + cache->priv->size > cache->priv->max_size)) {
 814                 SoupCacheEntry *old_entry = (SoupCacheEntry *)lru_entry->data;
 815
 816                 /* Discard entries. Once cancelled resources will be
 817                  * freed in close_ready_cb
 818                  */
 819                 if (soup_cache_entry_remove (cache, old_entry)) {
 820                         soup_cache_entry_free (old_entry, get_file_from_entry (cache, old_entry));
 821                         lru_entry = cache->priv->lru_start;
 822                 } else
 823                         lru_entry = g_list_next (lru_entry);
 824         }
 825 }
 826
 827 static gboolean
 828 soup_cache_entry_insert (SoupCache *cache,
 829                          SoupCacheEntry *entry,
 830                          gboolean sort)
 831 {
 832         guint length_to_add = 0;
 833         SoupCacheEntry *old_entry;
 834
 835         /* Fill the key */
 836         entry->key = get_cache_key_from_uri ((const char *) entry->uri);
 837
 838         if (soup_message_headers_get_encoding (entry->headers) != SOUP_ENCODING_CHUNKED)
 839                 length_to_add = soup_message_headers_get_content_length (entry->headers);
 840
 841         /* Check if we are going to store the resource depending on its size */
 842         if (length_to_add) {
 843                 if (!cache_accepts_entries_of_size (cache, length_to_add))
 844                         return FALSE;
 845
 846                 /* Make room for new entry if needed */
 847                 make_room_for_new_entry (cache, length_to_add);
 848         }
 849
 850         /* Remove any previous entry */
 851         if ((old_entry = g_hash_table_lookup (cache->priv->cache, GUINT_TO_POINTER (entry->key))) != NULL) {
 852                 if (soup_cache_entry_remove (cache, old_entry))
 853                         soup_cache_entry_free (old_entry, get_file_from_entry (cache, old_entry));
 854                 else
 855                         return FALSE;
 856         }
 857
 858         /* Add to hash table */
 859         g_hash_table_insert (cache->priv->cache, GUINT_TO_POINTER (entry->key), entry);
 860
 861         /* Compute new cache size */
 862         cache->priv->size += length_to_add;
 863
 864         /* Update LRU */
 865         if (sort)
 866                 cache->priv->lru_start = g_list_insert_sorted (cache->priv->lru_start, entry, lru_compare_func);
 867         else
 868                 cache->priv->lru_start = g_list_prepend (cache->priv->lru_start, entry);
 869
 870         g_assert (g_list_length (cache->priv->lru_start) == g_hash_table_size (cache->priv->cache));
 871
 872         return TRUE;
 873 }
 874
 875 static SoupCacheEntry*
 876 soup_cache_entry_lookup (SoupCache *cache,
 877                          SoupMessage *msg)
 878 {
 879         SoupCacheEntry *entry;
 880         guint32 key;
 881         char *uri = NULL;
 882
 883         uri = soup_uri_to_string (soup_message_get_uri (msg), FALSE);
 884         key = get_cache_key_from_uri ((const char *) uri);
 885
 886         entry = g_hash_table_lookup (cache->priv->cache, GUINT_TO_POINTER (key));
 887
 888         if (entry != NULL && (strcmp (entry->uri, uri) != 0))
 889                 entry = NULL;
 890
 891         g_free (uri);
 892         return entry;
 893 }
 894
 895 static void
 896 msg_restarted_cb (SoupMessage *msg, SoupCacheEntry *entry)
 897 {
 898         /* FIXME: What should we do here exactly? */
 899 }
 900
 901 static void
 902 replace_cb (GObject *source, GAsyncResult *result, SoupCacheWritingFixture *fixture)
 903 {
 904         SoupCacheEntry *entry = fixture->entry;
 905         GOutputStream *stream = (GOutputStream *) g_file_replace_finish (G_FILE (source),
 906                                                                          result, &entry->error);
 907
 908         if (g_cancellable_is_cancelled (entry->cancellable) || entry->error) {
 909                 if (stream)
 910                         g_object_unref (stream);
 911                 fixture->cache->priv->n_pending--;
 912                 entry->dirty = FALSE;
 913                 soup_cache_entry_remove (fixture->cache, entry);
 914                 soup_cache_entry_free (entry, get_file_from_entry (fixture->cache, entry));
 915                 soup_cache_writing_fixture_free (fixture);
 916                 return;
 917         }
 918
 919         entry->stream = stream;
 920
 921         /* If we already got all the data we have to initiate the
 922          * writing here, since we won't get more 'got-chunk'
 923          * signals
 924          */
 925         if (!entry->got_body)
 926                 return;
 927
 928         /* It could happen that reading the data from server
 929          * was completed before this happens. In that case
 930          * there is no data
 931          */
 932         if (!write_next_buffer (entry, fixture))
 933                 /* Could happen if the resource is empty */
 934                 g_output_stream_close_async (stream, G_PRIORITY_LOW, entry->cancellable,
 935                                              (GAsyncReadyCallback) close_ready_cb,
 936                                              fixture);
 937 }
 938
 939 typedef struct {
 940         time_t request_time;
 941         SoupSessionFeature *feature;
 942         gulong got_headers_handler;
 943 } RequestHelper;
 944
 945 static void
 946 msg_got_headers_cb (SoupMessage *msg, gpointer user_data)
 947 {
 948         SoupCache *cache;
 949         SoupCacheability cacheable;
 950         RequestHelper *helper;
 951         time_t request_time, response_time;
 952         SoupCacheEntry *entry;
 953
 954         response_time = time (NULL);
 955
 956         helper = (RequestHelper *)user_data;
 957         cache = SOUP_CACHE (helper->feature);
 958         request_time = helper->request_time;
 959         g_signal_handlers_disconnect_by_func (msg, msg_got_headers_cb, user_data);
 960         g_slice_free (RequestHelper, helper);
 961
 962         cacheable = soup_cache_get_cacheability (cache, msg);
 963
 964         if (cacheable & SOUP_CACHE_CACHEABLE) {
 965                 GFile *file;
 966                 SoupCacheWritingFixture *fixture;
 967
 968                 /* Check if we are already caching this resource */
 969                 entry = soup_cache_entry_lookup (cache, msg);
 970
 971                 if (entry && (entry->dirty || entry->being_validated))
 972                         return;
 973
 974                 /* Create a new entry, deleting any old one if present */
 975                 if (entry) {
 976                         soup_cache_entry_remove (cache, entry);
 977                         soup_cache_entry_free (entry, get_file_from_entry (cache, entry));
 978                 }
 979
 980                 entry = soup_cache_entry_new (cache, msg, request_time, response_time);
 981                 entry->hits = 1;
 982
 983                 /* Do not continue if it can not be stored */
 984                 if (!soup_cache_entry_insert (cache, entry, TRUE)) {
 985                         soup_cache_entry_free (entry, get_file_from_entry (cache, entry));
 986                         return;
 987                 }
 988
 989                 fixture = g_slice_new0 (SoupCacheWritingFixture);
 990                 fixture->cache = g_object_ref (cache);
 991                 fixture->entry = entry;
 992                 fixture->msg = g_object_ref (msg);
 993                 fixture->buffer_queue = g_queue_new ();
 994
 995                 /* We connect now to these signals and buffer the data
 996                    if it comes before the file is ready for writing */
 997                 fixture->got_chunk_handler =
 998                         g_signal_connect (msg, "got-chunk", G_CALLBACK (msg_got_chunk_cb), fixture);
 999                 fixture->got_body_handler =
1000                         g_signal_connect (msg, "got-body", G_CALLBACK (msg_got_body_cb), fixture);
1001                 fixture->restarted_handler =
1002                         g_signal_connect (msg, "restarted", G_CALLBACK (msg_restarted_cb), entry);
1003
1004                 /* Prepare entry */
1005                 cache->priv->n_pending++;
1006
1007                 entry->dirty = TRUE;
1008                 entry->cancellable = g_cancellable_new ();
1009                 file = get_file_from_entry (cache, entry);
1010                 g_file_replace_async (file, NULL, FALSE,
1011                                       G_FILE_CREATE_PRIVATE | G_FILE_CREATE_REPLACE_DESTINATION,
1012                                       G_PRIORITY_LOW, entry->cancellable,
1013                                       (GAsyncReadyCallback) replace_cb, fixture);
1014                 g_object_unref (file);
1015         } else if (cacheable & SOUP_CACHE_INVALIDATES) {
1016                 entry = soup_cache_entry_lookup (cache, msg);
1017
1018                 if (entry) {
1019                         if (soup_cache_entry_remove (cache, entry))
1020                                 soup_cache_entry_free (entry, get_file_from_entry (cache, entry));
1021                 }
1022         } else if (cacheable & SOUP_CACHE_VALIDATES) {
1023                 entry = soup_cache_entry_lookup (cache, msg);
1024
1025                 /* It's possible to get a CACHE_VALIDATES with no
1026                  * entry in the hash table. This could happen if for
1027                  * example the soup client is the one creating the
1028                  * conditional request.
1029                  */
1030                 if (entry) {
1031                         entry->being_validated = FALSE;
1032                         copy_end_to_end_headers (msg->response_headers, entry->headers);
1033                         soup_cache_entry_set_freshness (entry, msg, cache);
1034                 }
1035         }
1036 }
1037
1038 GInputStream *
1039 soup_cache_send_response (SoupCache *cache, SoupMessage *msg)
1040 {
1041         SoupCacheEntry *entry;
1042         char *current_age;
1043         GInputStream *stream = NULL;
1044         GFile *file;
1045
1046         g_return_val_if_fail (SOUP_IS_CACHE (cache), NULL);
1047         g_return_val_if_fail (SOUP_IS_MESSAGE (msg), NULL);
1048
1049         entry = soup_cache_entry_lookup (cache, msg);
1050         g_return_val_if_fail (entry, NULL);
1051
1052         /* TODO: the original idea was to save reads, but current code
1053            assumes that a stream is always returned. Need to reach
1054            some agreement here. Also we have to handle the situation
1055            were the file was no longer there (for example files
1056            removed without notifying the cache */
1057         file = get_file_from_entry (cache, entry);
1058         stream = G_INPUT_STREAM (g_file_read (file, NULL, NULL));
1059         g_object_unref (file);
1060
1061         /* Do not change the original message if there is no resource */
1062         if (stream == NULL)
1063                 return stream;
1064
1065         /* If we are told to send a response from cache any validation
1066            in course is over by now */
1067         entry->being_validated = FALSE;
1068
1069         /* Status */
1070         soup_message_set_status (msg, entry->status_code);
1071
1072         /* Headers */
1073         copy_end_to_end_headers (entry->headers, msg->response_headers);
1074
1075         /* Add 'Age' header with the current age */
1076         current_age = g_strdup_printf ("%d", soup_cache_entry_get_current_age (entry));
1077         soup_message_headers_replace (msg->response_headers,
1078                                       "Age",
1079                                       current_age);
1080         g_free (current_age);
1081
1082         return stream;
1083 }
1084
1085 static void
1086 request_started (SoupSessionFeature *feature, SoupSession *session,
1087                  SoupMessage *msg, SoupSocket *socket)
1088 {
1089         RequestHelper *helper = g_slice_new0 (RequestHelper);
1090         helper->request_time = time (NULL);
1091         helper->feature = feature;
1092         helper->got_headers_handler = g_signal_connect (msg, "got-headers",
1093                                                         G_CALLBACK (msg_got_headers_cb),
1094                                                         helper);
1095 }
1096
1097 static void
1098 attach (SoupSessionFeature *feature, SoupSession *session)
1099 {
1100         SoupCache *cache = SOUP_CACHE (feature);
1101         cache->priv->session = session;
1102
1103         soup_cache_default_feature_interface->attach (feature, session);
1104 }
1105
1106 static void
1107 soup_cache_session_feature_init (SoupSessionFeatureInterface *feature_interface,
1108                                         gpointer interface_data)
1109 {
1110         soup_cache_default_feature_interface =
1111                 g_type_default_interface_peek (SOUP_TYPE_SESSION_FEATURE);
1112
1113         feature_interface->attach = attach;
1114         feature_interface->request_started = request_started;
1115 }
1116
1117 static void
1118 soup_cache_init (SoupCache *cache)
1119 {
1120         SoupCachePrivate *priv;
1121
1122         priv = cache->priv = SOUP_CACHE_GET_PRIVATE (cache);
1123
1124         priv->cache = g_hash_table_new (g_direct_hash, g_direct_equal);
1125         /* LRU */
1126         priv->lru_start = NULL;
1127
1128         /* */
1129         priv->n_pending = 0;
1130
1131         /* Cache size */
1132         priv->max_size = DEFAULT_MAX_SIZE;
1133         priv->max_entry_data_size = priv->max_size / MAX_ENTRY_DATA_PERCENTAGE;
1134         priv->size = 0;
1135 }
1136
1137 static void
1138 remove_cache_item (gpointer data,
1139                    gpointer user_data)
1140 {
1141         SoupCache *cache = (SoupCache *) user_data;
1142         SoupCacheEntry *entry = (SoupCacheEntry *) data;
1143
1144         if (soup_cache_entry_remove (cache, entry))
1145                 soup_cache_entry_free (entry, NULL);
1146 }
1147
1148 static void
1149 soup_cache_finalize (GObject *object)
1150 {
1151         SoupCachePrivate *priv;
1152         GList *entries;
1153
1154         priv = SOUP_CACHE (object)->priv;
1155
1156         // Cannot use g_hash_table_foreach as callbacks must not modify the hash table
1157         entries = g_hash_table_get_values (priv->cache);
1158         g_list_foreach (entries, remove_cache_item, object);
1159         g_list_free (entries);
1160
1161         g_hash_table_destroy (priv->cache);
1162         g_free (priv->cache_dir);
1163
1164         g_list_free (priv->lru_start);
1165         priv->lru_start = NULL;
1166
1167         G_OBJECT_CLASS (soup_cache_parent_class)->finalize (object);
1168 }
1169
1170 static void
1171 soup_cache_set_property (GObject *object, guint prop_id,
1172                                 const GValue *value, GParamSpec *pspec)
1173 {
1174         SoupCachePrivate *priv = SOUP_CACHE (object)->priv;
1175
1176         switch (prop_id) {
1177         case PROP_CACHE_DIR:
1178                 priv->cache_dir = g_value_dup_string (value);
1179                 /* Create directory if it does not exist (FIXME: should we?) */
1180                 if (!g_file_test (priv->cache_dir, G_FILE_TEST_EXISTS | G_FILE_TEST_IS_DIR))
1181                         g_mkdir_with_parents (priv->cache_dir, 0700);
1182                 break;
1183         case PROP_CACHE_TYPE:
1184                 priv->cache_type = g_value_get_enum (value);
1185                 /* TODO: clear private entries and issue a warning if moving to shared? */
1186                 break;
1187         default:
1188                 G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
1189                 break;
1190         }
1191 }
1192
1193 static void
1194 soup_cache_get_property (GObject *object, guint prop_id,
1195                          GValue *value, GParamSpec *pspec)
1196 {
1197         SoupCachePrivate *priv = SOUP_CACHE (object)->priv;
1198
1199         switch (prop_id) {
1200         case PROP_CACHE_DIR:
1201                 g_value_set_string (value, priv->cache_dir);
1202                 break;
1203         case PROP_CACHE_TYPE:
1204                 g_value_set_enum (value, priv->cache_type);
1205                 break;
1206         default:
1207                 G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
1208                 break;
1209         }
1210 }
1211
1212 static void
1213 soup_cache_constructed (GObject *object)
1214 {
1215         SoupCachePrivate *priv;
1216
1217         priv = SOUP_CACHE (object)->priv;
1218
1219         if (!priv->cache_dir) {
1220                 /* Set a default cache dir, different for each user */
1221                 priv->cache_dir = g_build_filename (g_get_user_cache_dir (),
1222                                                     "httpcache",
1223                                                     NULL);
1224                 if (!g_file_test (priv->cache_dir, G_FILE_TEST_EXISTS | G_FILE_TEST_IS_DIR))
1225                         g_mkdir_with_parents (priv->cache_dir, 0700);
1226         }
1227
1228         if (G_OBJECT_CLASS (soup_cache_parent_class)->constructed)
1229                 G_OBJECT_CLASS (soup_cache_parent_class)->constructed (object);
1230 }
1231
1232 static void
1233 soup_cache_class_init (SoupCacheClass *cache_class)
1234 {
1235         GObjectClass *gobject_class = (GObjectClass *)cache_class;
1236
1237         gobject_class->finalize = soup_cache_finalize;
1238         gobject_class->constructed = soup_cache_constructed;
1239         gobject_class->set_property = soup_cache_set_property;
1240         gobject_class->get_property = soup_cache_get_property;
1241
1242         cache_class->get_cacheability = get_cacheability;
1243
1244         g_object_class_install_property (gobject_class, PROP_CACHE_DIR,
1245                                          g_param_spec_string ("cache-dir",
1246                                                               "Cache directory",
1247                                                               "The directory to store the cache files",
1248                                                               NULL,
1249                                                               G_PARAM_READWRITE | G_PARAM_CONSTRUCT_ONLY));
1250
1251         g_object_class_install_property (gobject_class, PROP_CACHE_TYPE,
1252                                          g_param_spec_enum ("cache-type",
1253                                                             "Cache type",
1254                                                             "Whether the cache is private or shared",
1255                                                             SOUP_TYPE_CACHE_TYPE,
1256                                                             SOUP_CACHE_SINGLE_USER,
1257                                                             G_PARAM_READWRITE | G_PARAM_CONSTRUCT_ONLY));
1258
1259         g_type_class_add_private (cache_class, sizeof (SoupCachePrivate));
1260 }
1261
1262 /**
1263  * SoupCacheType:
1264  * @SOUP_CACHE_SINGLE_USER: a single-user cache
1265  * @SOUP_CACHE_SHARED: a shared cache
1266  *
1267  * The type of cache; this affects what kinds of responses will be
1268  * saved.
1269  *
1270  * Since: 2.34
1271  */
1272
1273 /**
1274  * soup_cache_new:
1275  * @cache_dir: the directory to store the cached data, or %NULL to use the default one
1276  * @cache_type: the #SoupCacheType of the cache
1277  *
1278  * Creates a new #SoupCache.
1279  *
1280  * Returns: a new #SoupCache
1281  *
1282  * Since: 2.34
1283  */
1284 SoupCache *
1285 soup_cache_new (const char *cache_dir, SoupCacheType cache_type)
1286 {
1287         return g_object_new (SOUP_TYPE_CACHE,
1288                              "cache-dir", cache_dir,
1289                              "cache-type", cache_type,
1290                              NULL);
1291 }
1292
1293 /**
1294  * soup_cache_has_response:
1295  * @cache: a #SoupCache
1296  * @msg: a #SoupMessage
1297  *
1298  * This function calculates whether the @cache object has a proper
1299  * response for the request @msg given the flags both in the request
1300  * and the cached reply and the time ellapsed since it was cached.
1301  *
1302  * Returns: whether or not the @cache has a valid response for @msg
1303  *
1304  * Since: 2.34
1305  */
1306 SoupCacheResponse
1307 soup_cache_has_response (SoupCache *cache, SoupMessage *msg)
1308 {
1309         SoupCacheEntry *entry;
1310         const char *cache_control, *pragma;
1311         gpointer value;
1312         int max_age, max_stale, min_fresh;
1313         GList *lru_item, *item;
1314
1315         entry = soup_cache_entry_lookup (cache, msg);
1316
1317         /* 1. The presented Request-URI and that of stored response
1318          * match
1319          */
1320         if (!entry)
1321                 return SOUP_CACHE_RESPONSE_STALE;
1322
1323         /* Increase hit count. Take sorting into account */
1324         entry->hits++;
1325         lru_item = g_list_find (cache->priv->lru_start, entry);
1326         item = lru_item;
1327         while (item->next && lru_compare_func (item->data, item->next->data) > 0)
1328                 item = g_list_next (item);
1329
1330         if (item != lru_item) {
1331                 cache->priv->lru_start = g_list_remove_link (cache->priv->lru_start, lru_item);
1332                 item = g_list_insert_sorted (item, lru_item->data, lru_compare_func);
1333                 g_list_free (lru_item);
1334         }
1335
1336         if (entry->dirty || entry->being_validated)
1337                 return SOUP_CACHE_RESPONSE_STALE;
1338
1339         /* 2. The request method associated with the stored response
1340          *  allows it to be used for the presented request
1341          */
1342
1343         /* In practice this means we only return our resource for GET,
1344          * cacheability for other methods is a TODO in the RFC
1345          * (TODO: although we could return the headers for HEAD
1346          * probably).
1347          */
1348         if (msg->method != SOUP_METHOD_GET)
1349                 return SOUP_CACHE_RESPONSE_STALE;
1350
1351         /* 3. Selecting request-headers nominated by the stored
1352          * response (if any) match those presented.
1353          */
1354
1355         /* TODO */
1356
1357         /* 4. The request is a conditional request issued by the client.
1358          */
1359         if (soup_message_headers_get (msg->request_headers, "If-Modified-Since") ||
1360             soup_message_headers_get (msg->request_headers, "If-None-Match"))
1361                 return SOUP_CACHE_RESPONSE_STALE;
1362
1363         /* 5. The presented request and stored response are free from
1364          * directives that would prevent its use.
1365          */
1366
1367         max_age = max_stale = min_fresh = -1;
1368
1369         /* For HTTP 1.0 compatibility. RFC2616 section 14.9.4
1370          */
1371         pragma = soup_message_headers_get (msg->request_headers, "Pragma");
1372         if (pragma && soup_header_contains (pragma, "no-cache"))
1373                 return SOUP_CACHE_RESPONSE_STALE;
1374
1375         cache_control = soup_message_headers_get (msg->request_headers, "Cache-Control");
1376         if (cache_control) {
1377                 GHashTable *hash = soup_header_parse_param_list (cache_control);
1378
1379                 if (g_hash_table_lookup_extended (hash, "no-store", NULL, NULL)) {
1380                         soup_header_free_param_list (hash);
1381                         return SOUP_CACHE_RESPONSE_STALE;
1382                 }
1383
1384                 if (g_hash_table_lookup_extended (hash, "no-cache", NULL, NULL)) {
1385                         soup_header_free_param_list (hash);
1386                         return SOUP_CACHE_RESPONSE_STALE;
1387                 }
1388
1389                 if (g_hash_table_lookup_extended (hash, "max-age", NULL, &value)) {
1390                         max_age = (int)MIN (g_ascii_strtoll (value, NULL, 10), G_MAXINT32);
1391                         /* Forcing cache revalidaton
1392                          */
1393                         if (!max_age) {
1394                                 soup_header_free_param_list (hash);
1395                                 return SOUP_CACHE_RESPONSE_NEEDS_VALIDATION;
1396                         }
1397                 }
1398
1399                 /* max-stale can have no value set, we need to use _extended */
1400                 if (g_hash_table_lookup_extended (hash, "max-stale", NULL, &value)) {
1401                         if (value)
1402                                 max_stale = (int)MIN (g_ascii_strtoll (value, NULL, 10), G_MAXINT32);
1403                         else
1404                                 max_stale = G_MAXINT32;
1405                 }
1406
1407                 value = g_hash_table_lookup (hash, "min-fresh");
1408                 if (value)
1409                         min_fresh = (int)MIN (g_ascii_strtoll (value, NULL, 10), G_MAXINT32);
1410
1411                 soup_header_free_param_list (hash);
1412
1413                 if (max_age > 0) {
1414                         guint current_age = soup_cache_entry_get_current_age (entry);
1415
1416                         /* If we are over max-age and max-stale is not
1417                            set, do not use the value from the cache
1418                            without validation */
1419                         if ((guint) max_age <= current_age && max_stale == -1)
1420                                 return SOUP_CACHE_RESPONSE_NEEDS_VALIDATION;
1421                 }
1422         }
1423
1424         /* 6. The stored response is either: fresh, allowed to be
1425          * served stale or succesfully validated
1426          */
1427         /* TODO consider also proxy-revalidate & s-maxage */
1428         if (entry->must_revalidate)
1429                 return SOUP_CACHE_RESPONSE_NEEDS_VALIDATION;
1430
1431         if (!soup_cache_entry_is_fresh_enough (entry, min_fresh)) {
1432                 /* Not fresh, can it be served stale? */
1433                 if (max_stale != -1) {
1434                         /* G_MAXINT32 means we accept any staleness */
1435                         if (max_stale == G_MAXINT32)
1436                                 return SOUP_CACHE_RESPONSE_FRESH;
1437
1438                         if ((soup_cache_entry_get_current_age (entry) - entry->freshness_lifetime) <= (guint) max_stale)
1439                                 return SOUP_CACHE_RESPONSE_FRESH;
1440                 }
1441
1442                 return SOUP_CACHE_RESPONSE_NEEDS_VALIDATION;
1443         }
1444
1445         return SOUP_CACHE_RESPONSE_FRESH;
1446 }
1447
1448 /**
1449  * soup_cache_get_cacheability:
1450  * @cache: a #SoupCache
1451  * @msg: a #SoupMessage
1452  *
1453  * Calculates whether the @msg can be cached or not.
1454  *
1455  * Returns: a #SoupCacheability value indicating whether the @msg can be cached or not.
1456  *
1457  * Since: 2.34
1458  */
1459 SoupCacheability
1460 soup_cache_get_cacheability (SoupCache *cache, SoupMessage *msg)
1461 {
1462         g_return_val_if_fail (SOUP_IS_CACHE (cache), SOUP_CACHE_UNCACHEABLE);
1463         g_return_val_if_fail (SOUP_IS_MESSAGE (msg), SOUP_CACHE_UNCACHEABLE);
1464
1465         return SOUP_CACHE_GET_CLASS (cache)->get_cacheability (cache, msg);
1466 }
1467
1468 static gboolean
1469 force_flush_timeout (gpointer data)
1470 {
1471         gboolean *forced = (gboolean *)data;
1472         *forced = TRUE;
1473
1474         return FALSE;
1475 }
1476
1477 /**
1478  * soup_cache_flush:
1479  * @cache: a #SoupCache
1480  *
1481  * This function will force all pending writes in the @cache to be
1482  * committed to disk. For doing so it will iterate the #GMainContext
1483  * associated with @cache's session as long as needed.
1484  *
1485  * Since: 2.34
1486  */
1487 void
1488 soup_cache_flush (SoupCache *cache)
1489 {
1490         GMainContext *async_context;
1491         SoupSession *session;
1492         GSource *timeout;
1493         gboolean forced = FALSE;
1494
1495         g_return_if_fail (SOUP_IS_CACHE (cache));
1496
1497         session = cache->priv->session;
1498         g_return_if_fail (SOUP_IS_SESSION (session));
1499         async_context = soup_session_get_async_context (session);
1500
1501         /* We give cache 10 secs to finish */
1502         timeout = soup_add_timeout (async_context, 10000, force_flush_timeout, &forced);
1503
1504         while (!forced && cache->priv->n_pending > 0)
1505                 g_main_context_iteration (async_context, FALSE);
1506
1507         if (!forced)
1508                 g_source_destroy (timeout);
1509         else
1510                 g_warning ("Cache flush finished despite %d pending requests", cache->priv->n_pending);
1511 }
1512
1513 static void
1514 clear_cache_item (gpointer data,
1515                   gpointer user_data)
1516 {
1517         SoupCache *cache = (SoupCache *) user_data;
1518         SoupCacheEntry *entry = (SoupCacheEntry *) data;
1519
1520         if (soup_cache_entry_remove (cache, entry))
1521                 soup_cache_entry_free (entry, get_file_from_entry (cache, entry));
1522 }
1523
1524 /**
1525  * soup_cache_clear:
1526  * @cache: a #SoupCache
1527  *
1528  * Will remove all entries in the @cache plus all the cache files
1529  * associated with them.
1530  *
1531  * Since: 2.34
1532  */
1533 void
1534 soup_cache_clear (SoupCache *cache)
1535 {
1536         GList *entries;
1537
1538         g_return_if_fail (SOUP_IS_CACHE (cache));
1539         g_return_if_fail (cache->priv->cache);
1540
1541         // Cannot use g_hash_table_foreach as callbacks must not modify the hash table
1542         entries = g_hash_table_get_values (cache->priv->cache);
1543         g_list_foreach (entries, clear_cache_item, cache);
1544         g_list_free (entries);
1545 }
1546
1547 SoupMessage *
1548 soup_cache_generate_conditional_request (SoupCache *cache, SoupMessage *original)
1549 {
1550         SoupMessage *msg;
1551         SoupURI *uri;
1552         SoupCacheEntry *entry;
1553         const char *last_modified, *etag;
1554
1555         g_return_val_if_fail (SOUP_IS_CACHE (cache), NULL);
1556         g_return_val_if_fail (SOUP_IS_MESSAGE (original), NULL);
1557
1558         /* Add the validator entries in the header from the cached data */
1559         entry = soup_cache_entry_lookup (cache, original);
1560         g_return_val_if_fail (entry, NULL);
1561
1562         last_modified = soup_message_headers_get_one (entry->headers, "Last-Modified");
1563         etag = soup_message_headers_get_one (entry->headers, "ETag");
1564
1565         if (!last_modified && !etag)
1566                 return NULL;
1567
1568         entry->being_validated = TRUE;
1569
1570         /* Copy the data we need from the original message */
1571         uri = soup_message_get_uri (original);
1572         msg = soup_message_new_from_uri (original->method, uri);
1573
1574         soup_message_headers_foreach (original->request_headers,
1575                                       (SoupMessageHeadersForeachFunc)copy_headers,
1576                                       msg->request_headers);
1577
1578         if (last_modified)
1579                 soup_message_headers_append (msg->request_headers,
1580                                              "If-Modified-Since",
1581                                              last_modified);
1582         if (etag)
1583                 soup_message_headers_append (msg->request_headers,
1584                                              "If-None-Match",
1585                                              etag);
1586
1587         return msg;
1588 }
1589
1590 #define OLD_SOUP_CACHE_FILE "soup.cache"
1591 #define SOUP_CACHE_FILE "soup.cache2"
1592
1593 #define SOUP_CACHE_HEADERS_FORMAT "{ss}"
1594 #define SOUP_CACHE_PHEADERS_FORMAT "(sbuuuuuqa" SOUP_CACHE_HEADERS_FORMAT ")"
1595 #define SOUP_CACHE_ENTRIES_FORMAT "(qa" SOUP_CACHE_PHEADERS_FORMAT ")"
1596
1597 /* Basically the same format than above except that some strings are
1598    prepended with &. This way the GVariant returns a pointer to the
1599    data instead of duplicating the string */
1600 #define SOUP_CACHE_DECODE_HEADERS_FORMAT "{&s&s}"
1601
1602 static void
1603 pack_entry (gpointer data,
1604             gpointer user_data)
1605 {
1606         SoupCacheEntry *entry = (SoupCacheEntry *) data;
1607         SoupMessageHeadersIter iter;
1608         const char *header_key, *header_value;
1609         GVariantBuilder *entries_builder = (GVariantBuilder *)user_data;
1610
1611         /* Do not store non-consolidated entries */
1612         if (entry->dirty || entry->current_writing_buffer != NULL || !entry->key)
1613                 return;
1614
1615         g_variant_builder_open (entries_builder, G_VARIANT_TYPE (SOUP_CACHE_PHEADERS_FORMAT));
1616         g_variant_builder_add (entries_builder, "s", entry->uri);
1617         g_variant_builder_add (entries_builder, "b", entry->must_revalidate);
1618         g_variant_builder_add (entries_builder, "u", entry->freshness_lifetime);
1619         g_variant_builder_add (entries_builder, "u", entry->corrected_initial_age);
1620         g_variant_builder_add (entries_builder, "u", entry->response_time);
1621         g_variant_builder_add (entries_builder, "u", entry->hits);
1622         g_variant_builder_add (entries_builder, "u", entry->length);
1623         g_variant_builder_add (entries_builder, "q", entry->status_code);
1624
1625         /* Pack headers */
1626         g_variant_builder_open (entries_builder, G_VARIANT_TYPE ("a" SOUP_CACHE_HEADERS_FORMAT));
1627         soup_message_headers_iter_init (&iter, entry->headers);
1628         while (soup_message_headers_iter_next (&iter, &header_key, &header_value)) {
1629                 if (g_utf8_validate (header_value, -1, NULL))
1630                         g_variant_builder_add (entries_builder, SOUP_CACHE_HEADERS_FORMAT,
1631                                                header_key, header_value);
1632         }
1633         g_variant_builder_close (entries_builder); /* "a" SOUP_CACHE_HEADERS_FORMAT */
1634         g_variant_builder_close (entries_builder); /* SOUP_CACHE_PHEADERS_FORMAT */
1635 }
1636
1637 void
1638 soup_cache_dump (SoupCache *cache)
1639 {
1640         SoupCachePrivate *priv = SOUP_CACHE_GET_PRIVATE (cache);
1641         char *filename;
1642         GVariantBuilder entries_builder;
1643         GVariant *cache_variant;
1644
1645         if (!g_list_length (cache->priv->lru_start))
1646                 return;
1647
1648         /* Create the builder and iterate over all entries */
1649         g_variant_builder_init (&entries_builder, G_VARIANT_TYPE (SOUP_CACHE_ENTRIES_FORMAT));
1650         g_variant_builder_add (&entries_builder, "q", SOUP_CACHE_CURRENT_VERSION);
1651         g_variant_builder_open (&entries_builder, G_VARIANT_TYPE ("a" SOUP_CACHE_PHEADERS_FORMAT));
1652         g_list_foreach (cache->priv->lru_start, pack_entry, &entries_builder);
1653         g_variant_builder_close (&entries_builder);
1654
1655         /* Serialize and dump */
1656         cache_variant = g_variant_builder_end (&entries_builder);
1657         g_variant_ref_sink (cache_variant);
1658         filename = g_build_filename (priv->cache_dir, SOUP_CACHE_FILE, NULL);
1659         g_file_set_contents (filename, (const char *) g_variant_get_data (cache_variant),
1660                              g_variant_get_size (cache_variant), NULL);
1661         g_free (filename);
1662         g_variant_unref (cache_variant);
1663 }
1664
1665 static void
1666 clear_cache_files (SoupCache *cache)
1667 {
1668         GFileInfo *file_info;
1669         GFileEnumerator *file_enumerator;
1670         GFile *cache_dir_file = g_file_new_for_path (cache->priv->cache_dir);
1671
1672         file_enumerator = g_file_enumerate_children (cache_dir_file, G_FILE_ATTRIBUTE_STANDARD_NAME,
1673                                                      G_FILE_QUERY_INFO_NONE, NULL, NULL);
1674         if (file_enumerator) {
1675                 while ((file_info = g_file_enumerator_next_file (file_enumerator, NULL, NULL)) != NULL) {
1676                         const char *filename = g_file_info_get_name (file_info);
1677
1678                         if (strcmp (filename, SOUP_CACHE_FILE) != 0) {
1679                                 GFile *cache_file = g_file_get_child (cache_dir_file, filename);
1680                                 g_file_delete (cache_file, NULL, NULL);
1681                                 g_object_unref (cache_file);
1682                         }
1683                 }
1684                 g_object_unref (file_enumerator);
1685         }
1686         g_object_unref (cache_dir_file);
1687 }
1688
1689 void
1690 soup_cache_load (SoupCache *cache)
1691 {
1692         gboolean must_revalidate;
1693         guint32 freshness_lifetime, hits;
1694         guint32 corrected_initial_age, response_time;
1695         char *url, *filename = NULL, *contents = NULL;
1696         GVariant *cache_variant;
1697         GVariantIter *entries_iter = NULL, *headers_iter = NULL;
1698         gsize length;
1699         SoupCacheEntry *entry;
1700         SoupCachePrivate *priv = cache->priv;
1701         guint16 version, status_code;
1702
1703         filename = g_build_filename (priv->cache_dir, SOUP_CACHE_FILE, NULL);
1704         if (!g_file_get_contents (filename, &contents, &length, NULL)) {
1705                 g_free (filename);
1706                 g_free (contents);
1707                 clear_cache_files (cache);
1708                 return;
1709         }
1710         g_free (filename);
1711
1712         cache_variant = g_variant_new_from_data (G_VARIANT_TYPE (SOUP_CACHE_ENTRIES_FORMAT),
1713                                                  (const char *) contents, length, FALSE, g_free, contents);
1714         g_variant_get (cache_variant, SOUP_CACHE_ENTRIES_FORMAT, &version, &entries_iter);
1715         if (version != SOUP_CACHE_CURRENT_VERSION) {
1716                 g_variant_iter_free (entries_iter);
1717                 g_variant_unref (cache_variant);
1718                 clear_cache_files (cache);
1719                 return;
1720         }
1721
1722         while (g_variant_iter_loop (entries_iter, SOUP_CACHE_PHEADERS_FORMAT,
1723                                     &url, &must_revalidate, &freshness_lifetime, &corrected_initial_age,
1724                                     &response_time, &hits, &length, &status_code,
1725                                     &headers_iter)) {
1726                 const char *header_key, *header_value;
1727                 SoupMessageHeaders *headers;
1728                 SoupMessageHeadersIter soup_headers_iter;
1729
1730                 /* SoupMessage Headers */
1731                 headers = soup_message_headers_new (SOUP_MESSAGE_HEADERS_RESPONSE);
1732                 while (g_variant_iter_loop (headers_iter, SOUP_CACHE_HEADERS_FORMAT, &header_key, &header_value))
1733                         if (*header_key && *header_value)
1734                                 soup_message_headers_append (headers, header_key, header_value);
1735
1736                 /* Check that we have headers */
1737                 soup_message_headers_iter_init (&soup_headers_iter, headers);
1738                 if (!soup_message_headers_iter_next (&soup_headers_iter, &header_key, &header_value)) {
1739                         soup_message_headers_free (headers);
1740                         continue;
1741                 }
1742
1743                 /* Insert in cache */
1744                 entry = g_slice_new0 (SoupCacheEntry);
1745                 entry->uri = g_strdup (url);
1746                 entry->must_revalidate = must_revalidate;
1747                 entry->freshness_lifetime = freshness_lifetime;
1748                 entry->corrected_initial_age = corrected_initial_age;
1749                 entry->response_time = response_time;
1750                 entry->hits = hits;
1751                 entry->length = length;
1752                 entry->headers = headers;
1753                 entry->status_code = status_code;
1754
1755                 if (!soup_cache_entry_insert (cache, entry, FALSE))
1756                         soup_cache_entry_free (entry, get_file_from_entry (cache, entry));
1757         }
1758
1759         cache->priv->lru_start = g_list_reverse (cache->priv->lru_start);
1760
1761         /* frees */
1762         g_variant_iter_free (entries_iter);
1763         g_variant_unref (cache_variant);
1764 }
1765
1766 void
1767 soup_cache_set_max_size (SoupCache *cache,
1768                          guint      max_size)
1769 {
1770         cache->priv->max_size = max_size;
1771         cache->priv->max_entry_data_size = cache->priv->max_size / MAX_ENTRY_DATA_PERCENTAGE;
1772 }
1773
1774 guint
1775 soup_cache_get_max_size (SoupCache *cache)
1776 {
1777         return cache->priv->max_size;
1778 }