src/webkit/browser/appcache/appcache_update_job.cc

   1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "webkit/browser/appcache/appcache_update_job.h"
   6
   7 #include "base/bind.h"
   8 #include "base/bind_helpers.h"
   9 #include "base/compiler_specific.h"
  10 #include "base/message_loop/message_loop.h"
  11 #include "base/strings/string_util.h"
  12 #include "base/strings/stringprintf.h"
  13 #include "net/base/io_buffer.h"
  14 #include "net/base/load_flags.h"
  15 #include "net/base/net_errors.h"
  16 #include "net/base/request_priority.h"
  17 #include "net/http/http_request_headers.h"
  18 #include "net/http/http_response_headers.h"
  19 #include "net/url_request/url_request_context.h"
  20 #include "webkit/browser/appcache/appcache_group.h"
  21 #include "webkit/browser/appcache/appcache_histograms.h"
  22
  23 namespace appcache {
  24
  25 static const int kBufferSize = 32768;
  26 static const size_t kMaxConcurrentUrlFetches = 2;
  27 static const int kMax503Retries = 3;
  28
  29 // Helper class for collecting hosts per frontend when sending notifications
  30 // so that only one notification is sent for all hosts using the same frontend.
  31 class HostNotifier {
  32  public:
  33   typedef std::vector<int> HostIds;
  34   typedef std::map<AppCacheFrontend*, HostIds> NotifyHostMap;
  35
  36   // Caller is responsible for ensuring there will be no duplicate hosts.
  37   void AddHost(AppCacheHost* host) {
  38     std::pair<NotifyHostMap::iterator , bool> ret = hosts_to_notify.insert(
  39         NotifyHostMap::value_type(host->frontend(), HostIds()));
  40     ret.first->second.push_back(host->host_id());
  41   }
  42
  43   void AddHosts(const std::set<AppCacheHost*>& hosts) {
  44     for (std::set<AppCacheHost*>::const_iterator it = hosts.begin();
  45          it != hosts.end(); ++it) {
  46       AddHost(*it);
  47     }
  48   }
  49
  50   void SendNotifications(EventID event_id) {
  51     for (NotifyHostMap::iterator it = hosts_to_notify.begin();
  52          it != hosts_to_notify.end(); ++it) {
  53       AppCacheFrontend* frontend = it->first;
  54       frontend->OnEventRaised(it->second, event_id);
  55     }
  56   }
  57
  58   void SendProgressNotifications(
  59       const GURL& url, int num_total, int num_complete) {
  60     for (NotifyHostMap::iterator it = hosts_to_notify.begin();
  61          it != hosts_to_notify.end(); ++it) {
  62       AppCacheFrontend* frontend = it->first;
  63       frontend->OnProgressEventRaised(it->second, url,
  64                                       num_total, num_complete);
  65     }
  66   }
  67
  68   void SendErrorNotifications(const std::string& error_message) {
  69     DCHECK(!error_message.empty());
  70     for (NotifyHostMap::iterator it = hosts_to_notify.begin();
  71          it != hosts_to_notify.end(); ++it) {
  72       AppCacheFrontend* frontend = it->first;
  73       frontend->OnErrorEventRaised(it->second, error_message);
  74     }
  75   }
  76
  77  private:
  78   NotifyHostMap hosts_to_notify;
  79 };
  80
  81 AppCacheUpdateJob::UrlToFetch::UrlToFetch(const GURL& url,
  82                                           bool checked,
  83                                           AppCacheResponseInfo* info)
  84     : url(url),
  85       storage_checked(checked),
  86       existing_response_info(info) {
  87 }
  88
  89 AppCacheUpdateJob::UrlToFetch::~UrlToFetch() {
  90 }
  91
  92 // Helper class to fetch resources. Depending on the fetch type,
  93 // can either fetch to an in-memory string or write the response
  94 // data out to the disk cache.
  95 AppCacheUpdateJob::URLFetcher::URLFetcher(const GURL& url,
  96                                           FetchType fetch_type,
  97                                           AppCacheUpdateJob* job)
  98     : url_(url),
  99       job_(job),
 100       fetch_type_(fetch_type),
 101       retry_503_attempts_(0),
 102       buffer_(new net::IOBuffer(kBufferSize)),
 103       request_(job->service_->request_context()
 104                    ->CreateRequest(url, net::DEFAULT_PRIORITY, this)) {}
 105
 106 AppCacheUpdateJob::URLFetcher::~URLFetcher() {
 107 }
 108
 109 void AppCacheUpdateJob::URLFetcher::Start() {
 110   request_->set_first_party_for_cookies(job_->manifest_url_);
 111   request_->set_load_flags(request_->load_flags() |
 112                            net::LOAD_DISABLE_INTERCEPT);
 113   if (existing_response_headers_.get())
 114     AddConditionalHeaders(existing_response_headers_.get());
 115   request_->Start();
 116 }
 117
 118 void AppCacheUpdateJob::URLFetcher::OnReceivedRedirect(
 119     net::URLRequest* request, const GURL& new_url,  bool* defer_redirect) {
 120   DCHECK(request_ == request);
 121   // Redirect is not allowed by the update process.
 122   request->Cancel();
 123   OnResponseCompleted();
 124 }
 125
 126 void AppCacheUpdateJob::URLFetcher::OnResponseStarted(
 127     net::URLRequest *request) {
 128   DCHECK(request == request_);
 129   if (request->status().is_success() &&
 130       (request->GetResponseCode() / 100) == 2) {
 131
 132     // See http://code.google.com/p/chromium/issues/detail?id=69594
 133     // We willfully violate the HTML5 spec at this point in order
 134     // to support the appcaching of cross-origin HTTPS resources.
 135     // We've opted for a milder constraint and allow caching unless
 136     // the resource has a "no-store" header. A spec change has been
 137     // requested on the whatwg list.
 138     // TODO(michaeln): Consider doing this for cross-origin HTTP resources too.
 139     if (url_.SchemeIsSecure() &&
 140         url_.GetOrigin() != job_->manifest_url_.GetOrigin()) {
 141       if (request->response_headers()->
 142               HasHeaderValue("cache-control", "no-store")) {
 143         request->Cancel();
 144         OnResponseCompleted();
 145         return;
 146       }
 147     }
 148
 149     // Write response info to storage for URL fetches. Wait for async write
 150     // completion before reading any response data.
 151     if (fetch_type_ == URL_FETCH || fetch_type_ == MASTER_ENTRY_FETCH) {
 152       response_writer_.reset(job_->CreateResponseWriter());
 153       scoped_refptr<HttpResponseInfoIOBuffer> io_buffer(
 154           new HttpResponseInfoIOBuffer(
 155               new net::HttpResponseInfo(request->response_info())));
 156       response_writer_->WriteInfo(
 157           io_buffer.get(),
 158           base::Bind(&URLFetcher::OnWriteComplete, base::Unretained(this)));
 159     } else {
 160       ReadResponseData();
 161     }
 162   } else {
 163     OnResponseCompleted();
 164   }
 165 }
 166
 167 void AppCacheUpdateJob::URLFetcher::OnReadCompleted(
 168     net::URLRequest* request, int bytes_read) {
 169   DCHECK(request_ == request);
 170   bool data_consumed = true;
 171   if (request->status().is_success() && bytes_read > 0) {
 172     data_consumed = ConsumeResponseData(bytes_read);
 173     if (data_consumed) {
 174       bytes_read = 0;
 175       while (request->Read(buffer_.get(), kBufferSize, &bytes_read)) {
 176         if (bytes_read > 0) {
 177           data_consumed = ConsumeResponseData(bytes_read);
 178           if (!data_consumed)
 179             break;  // wait for async data processing, then read more
 180         } else {
 181           break;
 182         }
 183       }
 184     }
 185   }
 186   if (data_consumed && !request->status().is_io_pending())
 187     OnResponseCompleted();
 188 }
 189
 190 void AppCacheUpdateJob::URLFetcher::AddConditionalHeaders(
 191     const net::HttpResponseHeaders* headers) {
 192   DCHECK(request_.get() && headers);
 193   net::HttpRequestHeaders extra_headers;
 194
 195   // Add If-Modified-Since header if response info has Last-Modified header.
 196   const std::string last_modified = "Last-Modified";
 197   std::string last_modified_value;
 198   headers->EnumerateHeader(NULL, last_modified, &last_modified_value);
 199   if (!last_modified_value.empty()) {
 200     extra_headers.SetHeader(net::HttpRequestHeaders::kIfModifiedSince,
 201                             last_modified_value);
 202   }
 203
 204   // Add If-None-Match header if response info has ETag header.
 205   const std::string etag = "ETag";
 206   std::string etag_value;
 207   headers->EnumerateHeader(NULL, etag, &etag_value);
 208   if (!etag_value.empty()) {
 209     extra_headers.SetHeader(net::HttpRequestHeaders::kIfNoneMatch,
 210                             etag_value);
 211   }
 212   if (!extra_headers.IsEmpty())
 213     request_->SetExtraRequestHeaders(extra_headers);
 214 }
 215
 216 void  AppCacheUpdateJob::URLFetcher::OnWriteComplete(int result) {
 217   if (result < 0) {
 218     request_->Cancel();
 219     OnResponseCompleted();
 220     return;
 221   }
 222   ReadResponseData();
 223 }
 224
 225 void AppCacheUpdateJob::URLFetcher::ReadResponseData() {
 226   InternalUpdateState state = job_->internal_state_;
 227   if (state == CACHE_FAILURE || state == CANCELLED || state == COMPLETED)
 228     return;
 229   int bytes_read = 0;
 230   request_->Read(buffer_.get(), kBufferSize, &bytes_read);
 231   OnReadCompleted(request_.get(), bytes_read);
 232 }
 233
 234 // Returns false if response data is processed asynchronously, in which
 235 // case ReadResponseData will be invoked when it is safe to continue
 236 // reading more response data from the request.
 237 bool AppCacheUpdateJob::URLFetcher::ConsumeResponseData(int bytes_read) {
 238   DCHECK_GT(bytes_read, 0);
 239   switch (fetch_type_) {
 240     case MANIFEST_FETCH:
 241     case MANIFEST_REFETCH:
 242       manifest_data_.append(buffer_->data(), bytes_read);
 243       break;
 244     case URL_FETCH:
 245     case MASTER_ENTRY_FETCH:
 246       DCHECK(response_writer_.get());
 247       response_writer_->WriteData(
 248           buffer_.get(),
 249           bytes_read,
 250           base::Bind(&URLFetcher::OnWriteComplete, base::Unretained(this)));
 251       return false;  // wait for async write completion to continue reading
 252     default:
 253       NOTREACHED();
 254   }
 255   return true;
 256 }
 257
 258 void AppCacheUpdateJob::URLFetcher::OnResponseCompleted() {
 259   // Retry for 503s where retry-after is 0.
 260   if (request_->status().is_success() &&
 261       request_->GetResponseCode() == 503 &&
 262       MaybeRetryRequest()) {
 263     return;
 264   }
 265
 266   switch (fetch_type_) {
 267     case MANIFEST_FETCH:
 268       job_->HandleManifestFetchCompleted(this);
 269       break;
 270     case URL_FETCH:
 271       job_->HandleUrlFetchCompleted(this);
 272       break;
 273     case MASTER_ENTRY_FETCH:
 274       job_->HandleMasterEntryFetchCompleted(this);
 275       break;
 276     case MANIFEST_REFETCH:
 277       job_->HandleManifestRefetchCompleted(this);
 278       break;
 279     default:
 280       NOTREACHED();
 281   }
 282
 283   delete this;
 284 }
 285
 286 bool AppCacheUpdateJob::URLFetcher::MaybeRetryRequest() {
 287   if (retry_503_attempts_ >= kMax503Retries ||
 288       !request_->response_headers()->HasHeaderValue("retry-after", "0")) {
 289     return false;
 290   }
 291   ++retry_503_attempts_;
 292   request_ = job_->service_->request_context()->CreateRequest(
 293       url_, net::DEFAULT_PRIORITY, this);
 294   Start();
 295   return true;
 296 }
 297
 298 AppCacheUpdateJob::AppCacheUpdateJob(AppCacheService* service,
 299                                      AppCacheGroup* group)
 300     : service_(service),
 301       manifest_url_(group->manifest_url()),
 302       group_(group),
 303       update_type_(UNKNOWN_TYPE),
 304       internal_state_(FETCH_MANIFEST),
 305       master_entries_completed_(0),
 306       url_fetches_completed_(0),
 307       manifest_fetcher_(NULL),
 308       stored_state_(UNSTORED),
 309       storage_(service->storage()) {
 310     service_->AddObserver(this);
 311 }
 312
 313 AppCacheUpdateJob::~AppCacheUpdateJob() {
 314   if (service_)
 315     service_->RemoveObserver(this);
 316   if (internal_state_ != COMPLETED)
 317     Cancel();
 318
 319   DCHECK(!manifest_fetcher_);
 320   DCHECK(pending_url_fetches_.empty());
 321   DCHECK(!inprogress_cache_.get());
 322   DCHECK(pending_master_entries_.empty());
 323   DCHECK(master_entry_fetches_.empty());
 324
 325   if (group_)
 326     group_->SetUpdateStatus(AppCacheGroup::IDLE);
 327 }
 328
 329 void AppCacheUpdateJob::StartUpdate(AppCacheHost* host,
 330                                     const GURL& new_master_resource) {
 331   DCHECK(group_->update_job() == this);
 332   DCHECK(!group_->is_obsolete());
 333
 334   bool is_new_pending_master_entry = false;
 335   if (!new_master_resource.is_empty()) {
 336     DCHECK(new_master_resource == host->pending_master_entry_url());
 337     DCHECK(!new_master_resource.has_ref());
 338     DCHECK(new_master_resource.GetOrigin() == manifest_url_.GetOrigin());
 339
 340     // Cannot add more to this update if already terminating.
 341     if (IsTerminating()) {
 342       group_->QueueUpdate(host, new_master_resource);
 343       return;
 344     }
 345
 346     std::pair<PendingMasters::iterator, bool> ret =
 347         pending_master_entries_.insert(
 348             PendingMasters::value_type(new_master_resource, PendingHosts()));
 349     is_new_pending_master_entry = ret.second;
 350     ret.first->second.push_back(host);
 351     host->AddObserver(this);
 352   }
 353
 354   // Notify host (if any) if already checking or downloading.
 355   AppCacheGroup::UpdateStatus update_status = group_->update_status();
 356   if (update_status == AppCacheGroup::CHECKING ||
 357       update_status == AppCacheGroup::DOWNLOADING) {
 358     if (host) {
 359       NotifySingleHost(host, CHECKING_EVENT);
 360       if (update_status == AppCacheGroup::DOWNLOADING)
 361         NotifySingleHost(host, DOWNLOADING_EVENT);
 362
 363       // Add to fetch list or an existing entry if already fetched.
 364       if (!new_master_resource.is_empty()) {
 365         AddMasterEntryToFetchList(host, new_master_resource,
 366                                   is_new_pending_master_entry);
 367       }
 368     }
 369     return;
 370   }
 371
 372   // Begin update process for the group.
 373   group_->SetUpdateStatus(AppCacheGroup::CHECKING);
 374   if (group_->HasCache()) {
 375     update_type_ = UPGRADE_ATTEMPT;
 376     NotifyAllAssociatedHosts(CHECKING_EVENT);
 377   } else {
 378     update_type_ = CACHE_ATTEMPT;
 379     DCHECK(host);
 380     NotifySingleHost(host, CHECKING_EVENT);
 381   }
 382
 383   if (!new_master_resource.is_empty()) {
 384     AddMasterEntryToFetchList(host, new_master_resource,
 385                               is_new_pending_master_entry);
 386   }
 387
 388   FetchManifest(true);
 389 }
 390
 391 AppCacheResponseWriter* AppCacheUpdateJob::CreateResponseWriter() {
 392   AppCacheResponseWriter* writer =
 393       storage_->CreateResponseWriter(manifest_url_,
 394                                                 group_->group_id());
 395   stored_response_ids_.push_back(writer->response_id());
 396   return writer;
 397 }
 398
 399 void AppCacheUpdateJob::HandleCacheFailure(const std::string& error_message) {
 400   // 6.9.4 cache failure steps 2-8.
 401   DCHECK(internal_state_ != CACHE_FAILURE);
 402   DCHECK(!error_message.empty());
 403   internal_state_ = CACHE_FAILURE;
 404   CancelAllUrlFetches();
 405   CancelAllMasterEntryFetches(error_message);
 406   NotifyAllError(error_message);
 407   DiscardInprogressCache();
 408   internal_state_ = COMPLETED;
 409   DeleteSoon();  // To unwind the stack prior to deletion.
 410 }
 411
 412 void AppCacheUpdateJob::FetchManifest(bool is_first_fetch) {
 413   DCHECK(!manifest_fetcher_);
 414   manifest_fetcher_ = new URLFetcher(
 415      manifest_url_,
 416      is_first_fetch ? URLFetcher::MANIFEST_FETCH :
 417                       URLFetcher::MANIFEST_REFETCH,
 418      this);
 419
 420   // Add any necessary Http headers before sending fetch request.
 421   if (is_first_fetch) {
 422     AppCacheEntry* entry = (update_type_ == UPGRADE_ATTEMPT) ?
 423         group_->newest_complete_cache()->GetEntry(manifest_url_) : NULL;
 424     if (entry) {
 425       // Asynchronously load response info for manifest from newest cache.
 426       storage_->LoadResponseInfo(manifest_url_, group_->group_id(),
 427                                  entry->response_id(), this);
 428     } else {
 429       manifest_fetcher_->Start();
 430     }
 431   } else {
 432     DCHECK(internal_state_ == REFETCH_MANIFEST);
 433     DCHECK(manifest_response_info_.get());
 434     manifest_fetcher_->set_existing_response_headers(
 435         manifest_response_info_->headers.get());
 436     manifest_fetcher_->Start();
 437   }
 438 }
 439
 440
 441 void AppCacheUpdateJob::HandleManifestFetchCompleted(
 442     URLFetcher* fetcher) {
 443   DCHECK_EQ(internal_state_, FETCH_MANIFEST);
 444   DCHECK_EQ(manifest_fetcher_, fetcher);
 445   manifest_fetcher_ = NULL;
 446
 447   net::URLRequest* request = fetcher->request();
 448   int response_code = -1;
 449   bool is_valid_response_code = false;
 450   if (request->status().is_success()) {
 451     response_code = request->GetResponseCode();
 452     is_valid_response_code = (response_code / 100 == 2);
 453   }
 454
 455   if (is_valid_response_code) {
 456     manifest_data_ = fetcher->manifest_data();
 457     manifest_response_info_.reset(
 458         new net::HttpResponseInfo(request->response_info()));
 459     if (update_type_ == UPGRADE_ATTEMPT)
 460       CheckIfManifestChanged();  // continues asynchronously
 461     else
 462       ContinueHandleManifestFetchCompleted(true);
 463   } else if (response_code == 304 && update_type_ == UPGRADE_ATTEMPT) {
 464     ContinueHandleManifestFetchCompleted(false);
 465   } else if ((response_code == 404 || response_code == 410) &&
 466              update_type_ == UPGRADE_ATTEMPT) {
 467     storage_->MakeGroupObsolete(group_, this);  // async
 468   } else {
 469     const char* kFormatString = "Manifest fetch failed (%d) %s";
 470     std::string message = base::StringPrintf(kFormatString, response_code,
 471                                              manifest_url_.spec().c_str());
 472     HandleCacheFailure(message);
 473   }
 474 }
 475
 476 void AppCacheUpdateJob::OnGroupMadeObsolete(AppCacheGroup* group,
 477                                             bool success) {
 478   DCHECK(master_entry_fetches_.empty());
 479   CancelAllMasterEntryFetches("The cache has been made obsolete, "
 480                               "the manifest file returned 404 or 410");
 481   if (success) {
 482     DCHECK(group->is_obsolete());
 483     NotifyAllAssociatedHosts(OBSOLETE_EVENT);
 484     internal_state_ = COMPLETED;
 485     MaybeCompleteUpdate();
 486   } else {
 487     // Treat failure to mark group obsolete as a cache failure.
 488     HandleCacheFailure("Failed to mark the cache as obsolete");
 489   }
 490 }
 491
 492 void AppCacheUpdateJob::ContinueHandleManifestFetchCompleted(bool changed) {
 493   DCHECK(internal_state_ == FETCH_MANIFEST);
 494
 495   if (!changed) {
 496     DCHECK(update_type_ == UPGRADE_ATTEMPT);
 497     internal_state_ = NO_UPDATE;
 498
 499     // Wait for pending master entries to download.
 500     FetchMasterEntries();
 501     MaybeCompleteUpdate();  // if not done, run async 6.9.4 step 7 substeps
 502     return;
 503   }
 504
 505   Manifest manifest;
 506   if (!ParseManifest(manifest_url_, manifest_data_.data(),
 507                      manifest_data_.length(), manifest)) {
 508     const char* kFormatString = "Failed to parse manifest %s";
 509     const std::string message = base::StringPrintf(kFormatString,
 510         manifest_url_.spec().c_str());
 511     HandleCacheFailure(message);
 512     VLOG(1) << message;
 513     return;
 514   }
 515
 516   // Proceed with update process. Section 6.9.4 steps 8-20.
 517   internal_state_ = DOWNLOADING;
 518   inprogress_cache_ = new AppCache(storage_, storage_->NewCacheId());
 519   BuildUrlFileList(manifest);
 520   inprogress_cache_->InitializeWithManifest(&manifest);
 521
 522   // Associate all pending master hosts with the newly created cache.
 523   for (PendingMasters::iterator it = pending_master_entries_.begin();
 524        it != pending_master_entries_.end(); ++it) {
 525     PendingHosts& hosts = it->second;
 526     for (PendingHosts::iterator host_it = hosts.begin();
 527          host_it != hosts.end(); ++host_it) {
 528       (*host_it)
 529           ->AssociateIncompleteCache(inprogress_cache_.get(), manifest_url_);
 530     }
 531   }
 532
 533   group_->SetUpdateStatus(AppCacheGroup::DOWNLOADING);
 534   NotifyAllAssociatedHosts(DOWNLOADING_EVENT);
 535   FetchUrls();
 536   FetchMasterEntries();
 537   MaybeCompleteUpdate();  // if not done, continues when async fetches complete
 538 }
 539
 540 void AppCacheUpdateJob::HandleUrlFetchCompleted(URLFetcher* fetcher) {
 541   DCHECK(internal_state_ == DOWNLOADING);
 542
 543   net::URLRequest* request = fetcher->request();
 544   const GURL& url = request->original_url();
 545   pending_url_fetches_.erase(url);
 546   NotifyAllProgress(url);
 547   ++url_fetches_completed_;
 548
 549   int response_code = request->status().is_success()
 550       ? request->GetResponseCode() : -1;
 551   AppCacheEntry& entry = url_file_list_.find(url)->second;
 552
 553   if (response_code / 100 == 2) {
 554     // Associate storage with the new entry.
 555     DCHECK(fetcher->response_writer());
 556     entry.set_response_id(fetcher->response_writer()->response_id());
 557     entry.set_response_size(fetcher->response_writer()->amount_written());
 558     if (!inprogress_cache_->AddOrModifyEntry(url, entry))
 559       duplicate_response_ids_.push_back(entry.response_id());
 560
 561     // TODO(michaeln): Check for <html manifest=xxx>
 562     // See http://code.google.com/p/chromium/issues/detail?id=97930
 563     // if (entry.IsMaster() && !(entry.IsExplicit() || fallback || intercept))
 564     //   if (!manifestAttribute) skip it
 565
 566     // Foreign entries will be detected during cache selection.
 567     // Note: 6.9.4, step 17.9 possible optimization: if resource is HTML or XML
 568     // file whose root element is an html element with a manifest attribute
 569     // whose value doesn't match the manifest url of the application cache
 570     // being processed, mark the entry as being foreign.
 571   } else {
 572     VLOG(1) << "Request status: " << request->status().status()
 573             << " error: " << request->status().error()
 574             << " response code: " << response_code;
 575     if (entry.IsExplicit() || entry.IsFallback() || entry.IsIntercept()) {
 576       if (response_code == 304 && fetcher->existing_entry().has_response_id()) {
 577         // Keep the existing response.
 578         entry.set_response_id(fetcher->existing_entry().response_id());
 579         entry.set_response_size(fetcher->existing_entry().response_size());
 580         inprogress_cache_->AddOrModifyEntry(url, entry);
 581       } else {
 582         const char* kFormatString = "Resource fetch failed (%d) %s";
 583         const std::string message = base::StringPrintf(kFormatString,
 584             response_code, url.spec().c_str());
 585         HandleCacheFailure(message);
 586         return;
 587       }
 588     } else if (response_code == 404 || response_code == 410) {
 589       // Entry is skipped.  They are dropped from the cache.
 590     } else if (update_type_ == UPGRADE_ATTEMPT &&
 591                fetcher->existing_entry().has_response_id()) {
 592       // Keep the existing response.
 593       // TODO(michaeln): Not sure this is a good idea. This is spec compliant
 594       // but the old resource may or may not be compatible with the new contents
 595       // of the cache. Impossible to know one way or the other.
 596       entry.set_response_id(fetcher->existing_entry().response_id());
 597       entry.set_response_size(fetcher->existing_entry().response_size());
 598       inprogress_cache_->AddOrModifyEntry(url, entry);
 599     }
 600   }
 601
 602   // Fetch another URL now that one request has completed.
 603   DCHECK(internal_state_ != CACHE_FAILURE);
 604   FetchUrls();
 605   MaybeCompleteUpdate();
 606 }
 607
 608 void AppCacheUpdateJob::HandleMasterEntryFetchCompleted(
 609     URLFetcher* fetcher) {
 610   DCHECK(internal_state_ == NO_UPDATE || internal_state_ == DOWNLOADING);
 611
 612   // TODO(jennb): Handle downloads completing during cache failure when update
 613   // no longer fetches master entries directly. For now, we cancel all pending
 614   // master entry fetches when entering cache failure state so this will never
 615   // be called in CACHE_FAILURE state.
 616
 617   net::URLRequest* request = fetcher->request();
 618   const GURL& url = request->original_url();
 619   master_entry_fetches_.erase(url);
 620   ++master_entries_completed_;
 621
 622   int response_code = request->status().is_success()
 623       ? request->GetResponseCode() : -1;
 624
 625   PendingMasters::iterator found = pending_master_entries_.find(url);
 626   DCHECK(found != pending_master_entries_.end());
 627   PendingHosts& hosts = found->second;
 628
 629   // Section 6.9.4. No update case: step 7.3, else step 22.
 630   if (response_code / 100 == 2) {
 631     // Add fetched master entry to the appropriate cache.
 632     AppCache* cache = inprogress_cache_.get() ? inprogress_cache_.get()
 633                                               : group_->newest_complete_cache();
 634     DCHECK(fetcher->response_writer());
 635     AppCacheEntry master_entry(AppCacheEntry::MASTER,
 636                                fetcher->response_writer()->response_id(),
 637                                fetcher->response_writer()->amount_written());
 638     if (cache->AddOrModifyEntry(url, master_entry))
 639       added_master_entries_.push_back(url);
 640     else
 641       duplicate_response_ids_.push_back(master_entry.response_id());
 642
 643     // In no-update case, associate host with the newest cache.
 644     if (!inprogress_cache_.get()) {
 645       // TODO(michaeln): defer until the updated cache has been stored
 646       DCHECK(cache == group_->newest_complete_cache());
 647       for (PendingHosts::iterator host_it = hosts.begin();
 648            host_it != hosts.end(); ++host_it) {
 649         (*host_it)->AssociateCompleteCache(cache);
 650       }
 651     }
 652   } else {
 653     HostNotifier host_notifier;
 654     for (PendingHosts::iterator host_it = hosts.begin();
 655          host_it != hosts.end(); ++host_it) {
 656       AppCacheHost* host = *host_it;
 657       host_notifier.AddHost(host);
 658
 659       // In downloading case, disassociate host from inprogress cache.
 660       if (inprogress_cache_.get())
 661         host->AssociateNoCache(GURL());
 662
 663       host->RemoveObserver(this);
 664     }
 665     hosts.clear();
 666
 667     const char* kFormatString = "Master entry fetch failed (%d) %s";
 668     const std::string message = base::StringPrintf(kFormatString,
 669         response_code, request->url().spec().c_str());
 670     host_notifier.SendErrorNotifications(message);
 671
 672     // In downloading case, update result is different if all master entries
 673     // failed vs. only some failing.
 674     if (inprogress_cache_.get()) {
 675       // Only count successful downloads to know if all master entries failed.
 676       pending_master_entries_.erase(found);
 677       --master_entries_completed_;
 678
 679       // Section 6.9.4, step 22.3.
 680       if (update_type_ == CACHE_ATTEMPT && pending_master_entries_.empty()) {
 681         HandleCacheFailure(message);
 682         return;
 683       }
 684     }
 685   }
 686
 687   DCHECK(internal_state_ != CACHE_FAILURE);
 688   FetchMasterEntries();
 689   MaybeCompleteUpdate();
 690 }
 691
 692 void AppCacheUpdateJob::HandleManifestRefetchCompleted(
 693     URLFetcher* fetcher) {
 694   DCHECK(internal_state_ == REFETCH_MANIFEST);
 695   DCHECK(manifest_fetcher_ == fetcher);
 696   manifest_fetcher_ = NULL;
 697
 698   net::URLRequest* request = fetcher->request();
 699   int response_code = request->status().is_success()
 700       ? request->GetResponseCode() : -1;
 701   if (response_code == 304 || manifest_data_ == fetcher->manifest_data()) {
 702     // Only need to store response in storage if manifest is not already
 703     // an entry in the cache.
 704     AppCacheEntry* entry = inprogress_cache_->GetEntry(manifest_url_);
 705     if (entry) {
 706       entry->add_types(AppCacheEntry::MANIFEST);
 707       StoreGroupAndCache();
 708     } else {
 709       manifest_response_writer_.reset(CreateResponseWriter());
 710       scoped_refptr<HttpResponseInfoIOBuffer> io_buffer(
 711           new HttpResponseInfoIOBuffer(manifest_response_info_.release()));
 712       manifest_response_writer_->WriteInfo(
 713           io_buffer.get(),
 714           base::Bind(&AppCacheUpdateJob::OnManifestInfoWriteComplete,
 715                      base::Unretained(this)));
 716     }
 717   } else {
 718     VLOG(1) << "Request status: " << request->status().status()
 719             << " error: " << request->status().error()
 720             << " response code: " << response_code;
 721     ScheduleUpdateRetry(kRerunDelayMs);
 722     HandleCacheFailure("Manifest changed during update, scheduling retry");
 723   }
 724 }
 725
 726 void AppCacheUpdateJob::OnManifestInfoWriteComplete(int result) {
 727   if (result > 0) {
 728     scoped_refptr<net::StringIOBuffer> io_buffer(
 729         new net::StringIOBuffer(manifest_data_));
 730     manifest_response_writer_->WriteData(
 731         io_buffer.get(),
 732         manifest_data_.length(),
 733         base::Bind(&AppCacheUpdateJob::OnManifestDataWriteComplete,
 734                    base::Unretained(this)));
 735   } else {
 736     HandleCacheFailure("Failed to write the manifest headers to storage");
 737   }
 738 }
 739
 740 void AppCacheUpdateJob::OnManifestDataWriteComplete(int result) {
 741   if (result > 0) {
 742     AppCacheEntry entry(AppCacheEntry::MANIFEST,
 743         manifest_response_writer_->response_id(),
 744         manifest_response_writer_->amount_written());
 745     if (!inprogress_cache_->AddOrModifyEntry(manifest_url_, entry))
 746       duplicate_response_ids_.push_back(entry.response_id());
 747     StoreGroupAndCache();
 748   } else {
 749     HandleCacheFailure("Failed to write the manifest data to storage");
 750   }
 751 }
 752
 753 void AppCacheUpdateJob::StoreGroupAndCache() {
 754   DCHECK(stored_state_ == UNSTORED);
 755   stored_state_ = STORING;
 756   scoped_refptr<AppCache> newest_cache;
 757   if (inprogress_cache_.get())
 758     newest_cache.swap(inprogress_cache_);
 759   else
 760     newest_cache = group_->newest_complete_cache();
 761   newest_cache->set_update_time(base::Time::Now());
 762
 763   // TODO(michaeln): dcheck is fishing for clues to crbug/95101
 764   DCHECK_EQ(manifest_url_, group_->manifest_url());
 765   storage_->StoreGroupAndNewestCache(group_, newest_cache.get(), this);
 766 }
 767
 768 void AppCacheUpdateJob::OnGroupAndNewestCacheStored(AppCacheGroup* group,
 769                                                     AppCache* newest_cache,
 770                                                     bool success,
 771                                                     bool would_exceed_quota) {
 772   DCHECK(stored_state_ == STORING);
 773   if (success) {
 774     stored_state_ = STORED;
 775     MaybeCompleteUpdate();  // will definitely complete
 776   } else {
 777     // Restore inprogress_cache_ to get the proper events delivered
 778     // and the proper cleanup to occur.
 779     if (newest_cache != group->newest_complete_cache())
 780       inprogress_cache_ = newest_cache;
 781
 782     std::string message("Failed to commit new cache to storage");
 783     if (would_exceed_quota)
 784       message.append(", would exceed quota");
 785     HandleCacheFailure(message);
 786   }
 787 }
 788
 789 void AppCacheUpdateJob::NotifySingleHost(AppCacheHost* host,
 790                                          EventID event_id) {
 791   std::vector<int> ids(1, host->host_id());
 792   host->frontend()->OnEventRaised(ids, event_id);
 793 }
 794
 795 void AppCacheUpdateJob::NotifyAllAssociatedHosts(EventID event_id) {
 796   HostNotifier host_notifier;
 797   AddAllAssociatedHostsToNotifier(&host_notifier);
 798   host_notifier.SendNotifications(event_id);
 799 }
 800
 801 void AppCacheUpdateJob::NotifyAllProgress(const GURL& url) {
 802   HostNotifier host_notifier;
 803   AddAllAssociatedHostsToNotifier(&host_notifier);
 804   host_notifier.SendProgressNotifications(
 805       url, url_file_list_.size(), url_fetches_completed_);
 806 }
 807
 808 void AppCacheUpdateJob::NotifyAllFinalProgress() {
 809   DCHECK(url_file_list_.size() == url_fetches_completed_);
 810   NotifyAllProgress(GURL());
 811 }
 812
 813 void AppCacheUpdateJob::NotifyAllError(const std::string& error_message) {
 814   HostNotifier host_notifier;
 815   AddAllAssociatedHostsToNotifier(&host_notifier);
 816   host_notifier.SendErrorNotifications(error_message);
 817 }
 818
 819 void AppCacheUpdateJob::AddAllAssociatedHostsToNotifier(
 820     HostNotifier* host_notifier) {
 821   // Collect hosts so we only send one notification per frontend.
 822   // A host can only be associated with a single cache so no need to worry
 823   // about duplicate hosts being added to the notifier.
 824   if (inprogress_cache_.get()) {
 825     DCHECK(internal_state_ == DOWNLOADING || internal_state_ == CACHE_FAILURE);
 826     host_notifier->AddHosts(inprogress_cache_->associated_hosts());
 827   }
 828
 829   AppCacheGroup::Caches old_caches = group_->old_caches();
 830   for (AppCacheGroup::Caches::const_iterator it = old_caches.begin();
 831        it != old_caches.end(); ++it) {
 832     host_notifier->AddHosts((*it)->associated_hosts());
 833   }
 834
 835   AppCache* newest_cache = group_->newest_complete_cache();
 836   if (newest_cache)
 837     host_notifier->AddHosts(newest_cache->associated_hosts());
 838 }
 839
 840 void AppCacheUpdateJob::OnDestructionImminent(AppCacheHost* host) {
 841   // The host is about to be deleted; remove from our collection.
 842   PendingMasters::iterator found =
 843       pending_master_entries_.find(host->pending_master_entry_url());
 844   DCHECK(found != pending_master_entries_.end());
 845   PendingHosts& hosts = found->second;
 846   PendingHosts::iterator it = std::find(hosts.begin(), hosts.end(), host);
 847   DCHECK(it != hosts.end());
 848   hosts.erase(it);
 849 }
 850
 851 void AppCacheUpdateJob::OnServiceReinitialized(
 852     AppCacheStorageReference* old_storage_ref) {
 853   // We continue to use the disabled instance, but arrange for its
 854   // deletion when its no longer needed.
 855   if (old_storage_ref->storage() == storage_)
 856     disabled_storage_reference_ = old_storage_ref;
 857 }
 858
 859 void AppCacheUpdateJob::CheckIfManifestChanged() {
 860   DCHECK(update_type_ == UPGRADE_ATTEMPT);
 861   AppCacheEntry* entry = NULL;
 862   if (group_->newest_complete_cache())
 863     entry = group_->newest_complete_cache()->GetEntry(manifest_url_);
 864   if (!entry) {
 865     // TODO(michaeln): This is just a bandaid to avoid a crash.
 866     // http://code.google.com/p/chromium/issues/detail?id=95101
 867     if (service_->storage() == storage_) {
 868       // Use a local variable because service_ is reset in HandleCacheFailure.
 869       AppCacheService* service = service_;
 870       HandleCacheFailure("Manifest entry not found in existing cache");
 871       AppCacheHistograms::AddMissingManifestEntrySample();
 872       service->DeleteAppCacheGroup(manifest_url_, net::CompletionCallback());
 873     }
 874     return;
 875   }
 876
 877   // Load manifest data from storage to compare against fetched manifest.
 878   manifest_response_reader_.reset(
 879       storage_->CreateResponseReader(manifest_url_,
 880                                      group_->group_id(),
 881                                      entry->response_id()));
 882   read_manifest_buffer_ = new net::IOBuffer(kBufferSize);
 883   manifest_response_reader_->ReadData(
 884       read_manifest_buffer_.get(),
 885       kBufferSize,
 886       base::Bind(&AppCacheUpdateJob::OnManifestDataReadComplete,
 887                  base::Unretained(this)));  // async read
 888 }
 889
 890 void AppCacheUpdateJob::OnManifestDataReadComplete(int result) {
 891   if (result > 0) {
 892     loaded_manifest_data_.append(read_manifest_buffer_->data(), result);
 893     manifest_response_reader_->ReadData(
 894         read_manifest_buffer_.get(),
 895         kBufferSize,
 896         base::Bind(&AppCacheUpdateJob::OnManifestDataReadComplete,
 897                    base::Unretained(this)));  // read more
 898   } else {
 899     read_manifest_buffer_ = NULL;
 900     manifest_response_reader_.reset();
 901     ContinueHandleManifestFetchCompleted(
 902         result < 0 || manifest_data_ != loaded_manifest_data_);
 903   }
 904 }
 905
 906 void AppCacheUpdateJob::BuildUrlFileList(const Manifest& manifest) {
 907   for (base::hash_set<std::string>::const_iterator it =
 908            manifest.explicit_urls.begin();
 909        it != manifest.explicit_urls.end(); ++it) {
 910     AddUrlToFileList(GURL(*it), AppCacheEntry::EXPLICIT);
 911   }
 912
 913   const std::vector<Namespace>& intercepts =
 914       manifest.intercept_namespaces;
 915   for (std::vector<Namespace>::const_iterator it = intercepts.begin();
 916        it != intercepts.end(); ++it) {
 917     int flags = AppCacheEntry::INTERCEPT;
 918     if (it->is_executable)
 919       flags |= AppCacheEntry::EXECUTABLE;
 920     AddUrlToFileList(it->target_url, flags);
 921   }
 922
 923   const std::vector<Namespace>& fallbacks =
 924       manifest.fallback_namespaces;
 925   for (std::vector<Namespace>::const_iterator it = fallbacks.begin();
 926        it != fallbacks.end(); ++it) {
 927      AddUrlToFileList(it->target_url, AppCacheEntry::FALLBACK);
 928   }
 929
 930   // Add all master entries from newest complete cache.
 931   if (update_type_ == UPGRADE_ATTEMPT) {
 932     const AppCache::EntryMap& entries =
 933         group_->newest_complete_cache()->entries();
 934     for (AppCache::EntryMap::const_iterator it = entries.begin();
 935          it != entries.end(); ++it) {
 936       const AppCacheEntry& entry = it->second;
 937       if (entry.IsMaster())
 938         AddUrlToFileList(it->first, AppCacheEntry::MASTER);
 939     }
 940   }
 941 }
 942
 943 void AppCacheUpdateJob::AddUrlToFileList(const GURL& url, int type) {
 944   std::pair<AppCache::EntryMap::iterator, bool> ret = url_file_list_.insert(
 945       AppCache::EntryMap::value_type(url, AppCacheEntry(type)));
 946
 947   if (ret.second)
 948     urls_to_fetch_.push_back(UrlToFetch(url, false, NULL));
 949   else
 950     ret.first->second.add_types(type);  // URL already exists. Merge types.
 951 }
 952
 953 void AppCacheUpdateJob::FetchUrls() {
 954   DCHECK(internal_state_ == DOWNLOADING);
 955
 956   // Fetch each URL in the list according to section 6.9.4 step 17.1-17.3.
 957   // Fetch up to the concurrent limit. Other fetches will be triggered as each
 958   // each fetch completes.
 959   while (pending_url_fetches_.size() < kMaxConcurrentUrlFetches &&
 960          !urls_to_fetch_.empty()) {
 961     UrlToFetch url_to_fetch = urls_to_fetch_.front();
 962     urls_to_fetch_.pop_front();
 963
 964     AppCache::EntryMap::iterator it = url_file_list_.find(url_to_fetch.url);
 965     DCHECK(it != url_file_list_.end());
 966     AppCacheEntry& entry = it->second;
 967     if (ShouldSkipUrlFetch(entry)) {
 968       NotifyAllProgress(url_to_fetch.url);
 969       ++url_fetches_completed_;
 970     } else if (AlreadyFetchedEntry(url_to_fetch.url, entry.types())) {
 971       NotifyAllProgress(url_to_fetch.url);
 972       ++url_fetches_completed_;  // saved a URL request
 973     } else if (!url_to_fetch.storage_checked &&
 974                MaybeLoadFromNewestCache(url_to_fetch.url, entry)) {
 975       // Continues asynchronously after data is loaded from newest cache.
 976     } else {
 977       URLFetcher* fetcher = new URLFetcher(
 978           url_to_fetch.url, URLFetcher::URL_FETCH, this);
 979       if (url_to_fetch.existing_response_info.get()) {
 980         DCHECK(group_->newest_complete_cache());
 981         AppCacheEntry* existing_entry =
 982             group_->newest_complete_cache()->GetEntry(url_to_fetch.url);
 983         DCHECK(existing_entry);
 984         DCHECK(existing_entry->response_id() ==
 985                url_to_fetch.existing_response_info->response_id());
 986         fetcher->set_existing_response_headers(
 987             url_to_fetch.existing_response_info->http_response_info()->headers
 988                 .get());
 989         fetcher->set_existing_entry(*existing_entry);
 990       }
 991       fetcher->Start();
 992       pending_url_fetches_.insert(
 993           PendingUrlFetches::value_type(url_to_fetch.url, fetcher));
 994     }
 995   }
 996 }
 997
 998 void AppCacheUpdateJob::CancelAllUrlFetches() {
 999   // Cancel any pending URL requests.
1000   for (PendingUrlFetches::iterator it = pending_url_fetches_.begin();
1001        it != pending_url_fetches_.end(); ++it) {
1002     delete it->second;
1003   }
1004
1005   url_fetches_completed_ +=
1006       pending_url_fetches_.size() + urls_to_fetch_.size();
1007   pending_url_fetches_.clear();
1008   urls_to_fetch_.clear();
1009 }
1010
1011 bool AppCacheUpdateJob::ShouldSkipUrlFetch(const AppCacheEntry& entry) {
1012   // 6.6.4 Step 17
1013   // If the resource URL being processed was flagged as neither an
1014   // "explicit entry" nor or a "fallback entry", then the user agent
1015   // may skip this URL.
1016   if (entry.IsExplicit() || entry.IsFallback() || entry.IsIntercept())
1017     return false;
1018
1019   // TODO(jennb): decide if entry should be skipped to expire it from cache
1020   return false;
1021 }
1022
1023 bool AppCacheUpdateJob::AlreadyFetchedEntry(const GURL& url,
1024                                             int entry_type) {
1025   DCHECK(internal_state_ == DOWNLOADING || internal_state_ == NO_UPDATE);
1026   AppCacheEntry* existing =
1027       inprogress_cache_.get() ? inprogress_cache_->GetEntry(url)
1028                               : group_->newest_complete_cache()->GetEntry(url);
1029   if (existing) {
1030     existing->add_types(entry_type);
1031     return true;
1032   }
1033   return false;
1034 }
1035
1036 void AppCacheUpdateJob::AddMasterEntryToFetchList(AppCacheHost* host,
1037                                                   const GURL& url,
1038                                                   bool is_new) {
1039   DCHECK(!IsTerminating());
1040
1041   if (internal_state_ == DOWNLOADING || internal_state_ == NO_UPDATE) {
1042     AppCache* cache;
1043     if (inprogress_cache_.get()) {
1044       // always associate
1045       host->AssociateIncompleteCache(inprogress_cache_.get(), manifest_url_);
1046       cache = inprogress_cache_.get();
1047     } else {
1048       cache = group_->newest_complete_cache();
1049     }
1050
1051     // Update existing entry if it has already been fetched.
1052     AppCacheEntry* entry = cache->GetEntry(url);
1053     if (entry) {
1054       entry->add_types(AppCacheEntry::MASTER);
1055       if (internal_state_ == NO_UPDATE && !inprogress_cache_.get()) {
1056         // only associate if have entry
1057         host->AssociateCompleteCache(cache);
1058       }
1059       if (is_new)
1060         ++master_entries_completed_;  // pretend fetching completed
1061       return;
1062     }
1063   }
1064
1065   // Add to fetch list if not already fetching.
1066   if (master_entry_fetches_.find(url) == master_entry_fetches_.end()) {
1067     master_entries_to_fetch_.insert(url);
1068     if (internal_state_ == DOWNLOADING || internal_state_ == NO_UPDATE)
1069       FetchMasterEntries();
1070   }
1071 }
1072
1073 void AppCacheUpdateJob::FetchMasterEntries() {
1074   DCHECK(internal_state_ == NO_UPDATE || internal_state_ == DOWNLOADING);
1075
1076   // Fetch each master entry in the list, up to the concurrent limit.
1077   // Additional fetches will be triggered as each fetch completes.
1078   while (master_entry_fetches_.size() < kMaxConcurrentUrlFetches &&
1079          !master_entries_to_fetch_.empty()) {
1080     const GURL& url = *master_entries_to_fetch_.begin();
1081
1082     if (AlreadyFetchedEntry(url, AppCacheEntry::MASTER)) {
1083       ++master_entries_completed_;  // saved a URL request
1084
1085       // In no update case, associate hosts to newest cache in group
1086       // now that master entry has been "successfully downloaded".
1087       if (internal_state_ == NO_UPDATE) {
1088         // TODO(michaeln): defer until the updated cache has been stored.
1089         DCHECK(!inprogress_cache_.get());
1090         AppCache* cache = group_->newest_complete_cache();
1091         PendingMasters::iterator found = pending_master_entries_.find(url);
1092         DCHECK(found != pending_master_entries_.end());
1093         PendingHosts& hosts = found->second;
1094         for (PendingHosts::iterator host_it = hosts.begin();
1095              host_it != hosts.end(); ++host_it) {
1096           (*host_it)->AssociateCompleteCache(cache);
1097         }
1098       }
1099     } else {
1100       URLFetcher* fetcher = new URLFetcher(
1101           url, URLFetcher::MASTER_ENTRY_FETCH, this);
1102       fetcher->Start();
1103       master_entry_fetches_.insert(PendingUrlFetches::value_type(url, fetcher));
1104     }
1105
1106     master_entries_to_fetch_.erase(master_entries_to_fetch_.begin());
1107   }
1108 }
1109
1110 void AppCacheUpdateJob::CancelAllMasterEntryFetches(
1111     const std::string& error_message) {
1112   // For now, cancel all in-progress fetches for master entries and pretend
1113   // all master entries fetches have completed.
1114   // TODO(jennb): Delete this when update no longer fetches master entries
1115   // directly.
1116
1117   // Cancel all in-progress fetches.
1118   for (PendingUrlFetches::iterator it = master_entry_fetches_.begin();
1119        it != master_entry_fetches_.end(); ++it) {
1120     delete it->second;
1121     master_entries_to_fetch_.insert(it->first);  // back in unfetched list
1122   }
1123   master_entry_fetches_.clear();
1124
1125   master_entries_completed_ += master_entries_to_fetch_.size();
1126
1127   // Cache failure steps, step 2.
1128   // Pretend all master entries that have not yet been fetched have completed
1129   // downloading. Unassociate hosts from any appcache and send ERROR event.
1130   HostNotifier host_notifier;
1131   while (!master_entries_to_fetch_.empty()) {
1132     const GURL& url = *master_entries_to_fetch_.begin();
1133     PendingMasters::iterator found = pending_master_entries_.find(url);
1134     DCHECK(found != pending_master_entries_.end());
1135     PendingHosts& hosts = found->second;
1136     for (PendingHosts::iterator host_it = hosts.begin();
1137          host_it != hosts.end(); ++host_it) {
1138       AppCacheHost* host = *host_it;
1139       host->AssociateNoCache(GURL());
1140       host_notifier.AddHost(host);
1141       host->RemoveObserver(this);
1142     }
1143     hosts.clear();
1144
1145     master_entries_to_fetch_.erase(master_entries_to_fetch_.begin());
1146   }
1147   host_notifier.SendErrorNotifications(error_message);
1148 }
1149
1150 bool AppCacheUpdateJob::MaybeLoadFromNewestCache(const GURL& url,
1151                                                  AppCacheEntry& entry) {
1152   if (update_type_ != UPGRADE_ATTEMPT)
1153     return false;
1154
1155   AppCache* newest = group_->newest_complete_cache();
1156   AppCacheEntry* copy_me = newest->GetEntry(url);
1157   if (!copy_me || !copy_me->has_response_id())
1158     return false;
1159
1160   // Load HTTP headers for entry from newest cache.
1161   loading_responses_.insert(
1162       LoadingResponses::value_type(copy_me->response_id(), url));
1163   storage_->LoadResponseInfo(manifest_url_, group_->group_id(),
1164                              copy_me->response_id(),
1165                              this);
1166   // Async: wait for OnResponseInfoLoaded to complete.
1167   return true;
1168 }
1169
1170 void AppCacheUpdateJob::OnResponseInfoLoaded(
1171     AppCacheResponseInfo* response_info, int64 response_id) {
1172   const net::HttpResponseInfo* http_info = response_info ?
1173       response_info->http_response_info() : NULL;
1174
1175   // Needed response info for a manifest fetch request.
1176   if (internal_state_ == FETCH_MANIFEST) {
1177     if (http_info)
1178       manifest_fetcher_->set_existing_response_headers(
1179           http_info->headers.get());
1180     manifest_fetcher_->Start();
1181     return;
1182   }
1183
1184   LoadingResponses::iterator found = loading_responses_.find(response_id);
1185   DCHECK(found != loading_responses_.end());
1186   const GURL& url = found->second;
1187
1188   if (!http_info) {
1189     LoadFromNewestCacheFailed(url, NULL);  // no response found
1190   } else {
1191     // Check if response can be re-used according to HTTP caching semantics.
1192     // Responses with a "vary" header get treated as expired.
1193     const std::string name = "vary";
1194     std::string value;
1195     void* iter = NULL;
1196     if (!http_info->headers.get() ||
1197         http_info->headers->RequiresValidation(http_info->request_time,
1198                                                http_info->response_time,
1199                                                base::Time::Now()) ||
1200         http_info->headers->EnumerateHeader(&iter, name, &value)) {
1201       LoadFromNewestCacheFailed(url, response_info);
1202     } else {
1203       DCHECK(group_->newest_complete_cache());
1204       AppCacheEntry* copy_me = group_->newest_complete_cache()->GetEntry(url);
1205       DCHECK(copy_me);
1206       DCHECK(copy_me->response_id() == response_id);
1207
1208       AppCache::EntryMap::iterator it = url_file_list_.find(url);
1209       DCHECK(it != url_file_list_.end());
1210       AppCacheEntry& entry = it->second;
1211       entry.set_response_id(response_id);
1212       entry.set_response_size(copy_me->response_size());
1213       inprogress_cache_->AddOrModifyEntry(url, entry);
1214       NotifyAllProgress(url);
1215       ++url_fetches_completed_;
1216     }
1217   }
1218   loading_responses_.erase(found);
1219
1220   MaybeCompleteUpdate();
1221 }
1222
1223 void AppCacheUpdateJob::LoadFromNewestCacheFailed(
1224     const GURL& url, AppCacheResponseInfo* response_info) {
1225   if (internal_state_ == CACHE_FAILURE)
1226     return;
1227
1228   // Re-insert url at front of fetch list. Indicate storage has been checked.
1229   urls_to_fetch_.push_front(UrlToFetch(url, true, response_info));
1230   FetchUrls();
1231 }
1232
1233 void AppCacheUpdateJob::MaybeCompleteUpdate() {
1234   DCHECK(internal_state_ != CACHE_FAILURE);
1235
1236   // Must wait for any pending master entries or url fetches to complete.
1237   if (master_entries_completed_ != pending_master_entries_.size() ||
1238       url_fetches_completed_ != url_file_list_.size()) {
1239     DCHECK(internal_state_ != COMPLETED);
1240     return;
1241   }
1242
1243   switch (internal_state_) {
1244     case NO_UPDATE:
1245       if (master_entries_completed_ > 0) {
1246         switch (stored_state_) {
1247           case UNSTORED:
1248             StoreGroupAndCache();
1249             return;
1250           case STORING:
1251             return;
1252           case STORED:
1253             break;
1254         }
1255       }
1256       // 6.9.4 steps 7.3-7.7.
1257       NotifyAllAssociatedHosts(NO_UPDATE_EVENT);
1258       DiscardDuplicateResponses();
1259       internal_state_ = COMPLETED;
1260       break;
1261     case DOWNLOADING:
1262       internal_state_ = REFETCH_MANIFEST;
1263       FetchManifest(false);
1264       break;
1265     case REFETCH_MANIFEST:
1266       DCHECK(stored_state_ == STORED);
1267       NotifyAllFinalProgress();
1268       if (update_type_ == CACHE_ATTEMPT)
1269         NotifyAllAssociatedHosts(CACHED_EVENT);
1270       else
1271         NotifyAllAssociatedHosts(UPDATE_READY_EVENT);
1272       DiscardDuplicateResponses();
1273       internal_state_ = COMPLETED;
1274       break;
1275     case CACHE_FAILURE:
1276       NOTREACHED();  // See HandleCacheFailure
1277       break;
1278     default:
1279       break;
1280   }
1281
1282   // Let the stack unwind before deletion to make it less risky as this
1283   // method is called from multiple places in this file.
1284   if (internal_state_ == COMPLETED)
1285     DeleteSoon();
1286 }
1287
1288 void AppCacheUpdateJob::ScheduleUpdateRetry(int delay_ms) {
1289   // TODO(jennb): post a delayed task with the "same parameters" as this job
1290   // to retry the update at a later time. Need group, URLs of pending master
1291   // entries and their hosts.
1292 }
1293
1294 void AppCacheUpdateJob::Cancel() {
1295   internal_state_ = CANCELLED;
1296
1297   if (manifest_fetcher_) {
1298     delete manifest_fetcher_;
1299     manifest_fetcher_ = NULL;
1300   }
1301
1302   for (PendingUrlFetches::iterator it = pending_url_fetches_.begin();
1303        it != pending_url_fetches_.end(); ++it) {
1304     delete it->second;
1305   }
1306   pending_url_fetches_.clear();
1307
1308   for (PendingUrlFetches::iterator it = master_entry_fetches_.begin();
1309        it != master_entry_fetches_.end(); ++it) {
1310     delete it->second;
1311   }
1312   master_entry_fetches_.clear();
1313
1314   ClearPendingMasterEntries();
1315   DiscardInprogressCache();
1316
1317   // Delete response writer to avoid any callbacks.
1318   if (manifest_response_writer_)
1319     manifest_response_writer_.reset();
1320
1321   storage_->CancelDelegateCallbacks(this);
1322 }
1323
1324 void AppCacheUpdateJob::ClearPendingMasterEntries() {
1325   for (PendingMasters::iterator it = pending_master_entries_.begin();
1326        it != pending_master_entries_.end(); ++it) {
1327     PendingHosts& hosts = it->second;
1328     for (PendingHosts::iterator host_it = hosts.begin();
1329          host_it != hosts.end(); ++host_it) {
1330       (*host_it)->RemoveObserver(this);
1331     }
1332   }
1333
1334   pending_master_entries_.clear();
1335 }
1336
1337 void AppCacheUpdateJob::DiscardInprogressCache() {
1338   storage_->DoomResponses(manifest_url_, stored_response_ids_);
1339
1340   if (!inprogress_cache_.get()) {
1341     // We have to undo the changes we made, if any, to the existing cache.
1342     for (std::vector<GURL>::iterator iter = added_master_entries_.begin();
1343          iter != added_master_entries_.end(); ++iter) {
1344       DCHECK(group_->newest_complete_cache());
1345       group_->newest_complete_cache()->RemoveEntry(*iter);
1346     }
1347     return;
1348   }
1349
1350   AppCache::AppCacheHosts& hosts = inprogress_cache_->associated_hosts();
1351   while (!hosts.empty())
1352     (*hosts.begin())->AssociateNoCache(GURL());
1353
1354   inprogress_cache_ = NULL;
1355 }
1356
1357 void AppCacheUpdateJob::DiscardDuplicateResponses() {
1358   storage_->DoomResponses(manifest_url_, duplicate_response_ids_);
1359 }
1360
1361 void AppCacheUpdateJob::DeleteSoon() {
1362   ClearPendingMasterEntries();
1363   manifest_response_writer_.reset();
1364   storage_->CancelDelegateCallbacks(this);
1365   service_->RemoveObserver(this);
1366   service_ = NULL;
1367
1368   // Break the connection with the group so the group cannot call delete
1369   // on this object after we've posted a task to delete ourselves.
1370   group_->SetUpdateStatus(AppCacheGroup::IDLE);
1371   group_ = NULL;
1372
1373   base::MessageLoop::current()->DeleteSoon(FROM_HERE, this);
1374 }
1375
1376 }  // namespace appcache