src/webkit/browser/appcache/appcache_update_job.cc

   1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "webkit/browser/appcache/appcache_update_job.h"
   6
   7 #include "base/bind.h"
   8 #include "base/bind_helpers.h"
   9 #include "base/compiler_specific.h"
  10 #include "base/message_loop/message_loop.h"
  11 #include "base/strings/string_util.h"
  12 #include "base/strings/stringprintf.h"
  13 #include "net/base/io_buffer.h"
  14 #include "net/base/load_flags.h"
  15 #include "net/base/net_errors.h"
  16 #include "net/base/request_priority.h"
  17 #include "net/http/http_request_headers.h"
  18 #include "net/http/http_response_headers.h"
  19 #include "net/url_request/url_request_context.h"
  20 #include "webkit/browser/appcache/appcache_group.h"
  21 #include "webkit/browser/appcache/appcache_histograms.h"
  22
  23 namespace appcache {
  24
  25 static const int kBufferSize = 32768;
  26 static const size_t kMaxConcurrentUrlFetches = 2;
  27 static const int kMax503Retries = 3;
  28
  29 static std::string FormatUrlErrorMessage(
  30       const char* format, const GURL& url,
  31       AppCacheUpdateJob::ResultType error,
  32       int response_code) {
  33     // Show the net response code if we have one.
  34     int code = response_code;
  35     if (error != AppCacheUpdateJob::SERVER_ERROR)
  36       code = static_cast<int>(error);
  37     return base::StringPrintf(format, code, url.spec().c_str());
  38 }
  39
  40 // Helper class for collecting hosts per frontend when sending notifications
  41 // so that only one notification is sent for all hosts using the same frontend.
  42 class HostNotifier {
  43  public:
  44   typedef std::vector<int> HostIds;
  45   typedef std::map<AppCacheFrontend*, HostIds> NotifyHostMap;
  46
  47   // Caller is responsible for ensuring there will be no duplicate hosts.
  48   void AddHost(AppCacheHost* host) {
  49     std::pair<NotifyHostMap::iterator , bool> ret = hosts_to_notify.insert(
  50         NotifyHostMap::value_type(host->frontend(), HostIds()));
  51     ret.first->second.push_back(host->host_id());
  52   }
  53
  54   void AddHosts(const std::set<AppCacheHost*>& hosts) {
  55     for (std::set<AppCacheHost*>::const_iterator it = hosts.begin();
  56          it != hosts.end(); ++it) {
  57       AddHost(*it);
  58     }
  59   }
  60
  61   void SendNotifications(EventID event_id) {
  62     for (NotifyHostMap::iterator it = hosts_to_notify.begin();
  63          it != hosts_to_notify.end(); ++it) {
  64       AppCacheFrontend* frontend = it->first;
  65       frontend->OnEventRaised(it->second, event_id);
  66     }
  67   }
  68
  69   void SendProgressNotifications(
  70       const GURL& url, int num_total, int num_complete) {
  71     for (NotifyHostMap::iterator it = hosts_to_notify.begin();
  72          it != hosts_to_notify.end(); ++it) {
  73       AppCacheFrontend* frontend = it->first;
  74       frontend->OnProgressEventRaised(it->second, url,
  75                                       num_total, num_complete);
  76     }
  77   }
  78
  79   void SendErrorNotifications(const std::string& error_message) {
  80     DCHECK(!error_message.empty());
  81     for (NotifyHostMap::iterator it = hosts_to_notify.begin();
  82          it != hosts_to_notify.end(); ++it) {
  83       AppCacheFrontend* frontend = it->first;
  84       frontend->OnErrorEventRaised(it->second, error_message);
  85     }
  86   }
  87
  88  private:
  89   NotifyHostMap hosts_to_notify;
  90 };
  91
  92 AppCacheUpdateJob::UrlToFetch::UrlToFetch(const GURL& url,
  93                                           bool checked,
  94                                           AppCacheResponseInfo* info)
  95     : url(url),
  96       storage_checked(checked),
  97       existing_response_info(info) {
  98 }
  99
 100 AppCacheUpdateJob::UrlToFetch::~UrlToFetch() {
 101 }
 102
 103 // Helper class to fetch resources. Depending on the fetch type,
 104 // can either fetch to an in-memory string or write the response
 105 // data out to the disk cache.
 106 AppCacheUpdateJob::URLFetcher::URLFetcher(const GURL& url,
 107                                           FetchType fetch_type,
 108                                           AppCacheUpdateJob* job)
 109     : url_(url),
 110       job_(job),
 111       fetch_type_(fetch_type),
 112       retry_503_attempts_(0),
 113       buffer_(new net::IOBuffer(kBufferSize)),
 114       request_(job->service_->request_context()
 115                    ->CreateRequest(url, net::DEFAULT_PRIORITY, this)),
 116       result_(UPDATE_OK) {}
 117
 118 AppCacheUpdateJob::URLFetcher::~URLFetcher() {
 119 }
 120
 121 void AppCacheUpdateJob::URLFetcher::Start() {
 122   request_->set_first_party_for_cookies(job_->manifest_url_);
 123   request_->SetLoadFlags(request_->load_flags() |
 124                          net::LOAD_DISABLE_INTERCEPT);
 125   if (existing_response_headers_.get())
 126     AddConditionalHeaders(existing_response_headers_.get());
 127   request_->Start();
 128 }
 129
 130 void AppCacheUpdateJob::URLFetcher::OnReceivedRedirect(
 131     net::URLRequest* request, const GURL& new_url,  bool* defer_redirect) {
 132   DCHECK(request_ == request);
 133   // Redirect is not allowed by the update process.
 134   request->Cancel();
 135   result_ = REDIRECT_ERROR;
 136   OnResponseCompleted();
 137 }
 138
 139 void AppCacheUpdateJob::URLFetcher::OnResponseStarted(
 140     net::URLRequest *request) {
 141   DCHECK(request == request_);
 142   int response_code = -1;
 143   if (request->status().is_success())
 144     response_code = request->GetResponseCode();
 145   if ((response_code / 100) == 2) {
 146
 147     // See http://code.google.com/p/chromium/issues/detail?id=69594
 148     // We willfully violate the HTML5 spec at this point in order
 149     // to support the appcaching of cross-origin HTTPS resources.
 150     // We've opted for a milder constraint and allow caching unless
 151     // the resource has a "no-store" header. A spec change has been
 152     // requested on the whatwg list.
 153     // TODO(michaeln): Consider doing this for cross-origin HTTP resources too.
 154     if (url_.SchemeIsSecure() &&
 155         url_.GetOrigin() != job_->manifest_url_.GetOrigin()) {
 156       if (request->response_headers()->
 157               HasHeaderValue("cache-control", "no-store")) {
 158         request->Cancel();
 159         result_ = SERVER_ERROR;  // Not the best match?
 160         OnResponseCompleted();
 161         return;
 162       }
 163     }
 164
 165     // Write response info to storage for URL fetches. Wait for async write
 166     // completion before reading any response data.
 167     if (fetch_type_ == URL_FETCH || fetch_type_ == MASTER_ENTRY_FETCH) {
 168       response_writer_.reset(job_->CreateResponseWriter());
 169       scoped_refptr<HttpResponseInfoIOBuffer> io_buffer(
 170           new HttpResponseInfoIOBuffer(
 171               new net::HttpResponseInfo(request->response_info())));
 172       response_writer_->WriteInfo(
 173           io_buffer.get(),
 174           base::Bind(&URLFetcher::OnWriteComplete, base::Unretained(this)));
 175     } else {
 176       ReadResponseData();
 177     }
 178   } else {
 179     if (response_code > 0)
 180       result_ = SERVER_ERROR;
 181     else
 182       result_ = NETWORK_ERROR;
 183     OnResponseCompleted();
 184   }
 185 }
 186
 187 void AppCacheUpdateJob::URLFetcher::OnReadCompleted(
 188     net::URLRequest* request, int bytes_read) {
 189   DCHECK(request_ == request);
 190   bool data_consumed = true;
 191   if (request->status().is_success() && bytes_read > 0) {
 192     data_consumed = ConsumeResponseData(bytes_read);
 193     if (data_consumed) {
 194       bytes_read = 0;
 195       while (request->Read(buffer_.get(), kBufferSize, &bytes_read)) {
 196         if (bytes_read > 0) {
 197           data_consumed = ConsumeResponseData(bytes_read);
 198           if (!data_consumed)
 199             break;  // wait for async data processing, then read more
 200         } else {
 201           break;
 202         }
 203       }
 204     }
 205   }
 206   if (data_consumed && !request->status().is_io_pending())
 207     OnResponseCompleted();
 208 }
 209
 210 void AppCacheUpdateJob::URLFetcher::AddConditionalHeaders(
 211     const net::HttpResponseHeaders* headers) {
 212   DCHECK(request_.get() && headers);
 213   net::HttpRequestHeaders extra_headers;
 214
 215   // Add If-Modified-Since header if response info has Last-Modified header.
 216   const std::string last_modified = "Last-Modified";
 217   std::string last_modified_value;
 218   headers->EnumerateHeader(NULL, last_modified, &last_modified_value);
 219   if (!last_modified_value.empty()) {
 220     extra_headers.SetHeader(net::HttpRequestHeaders::kIfModifiedSince,
 221                             last_modified_value);
 222   }
 223
 224   // Add If-None-Match header if response info has ETag header.
 225   const std::string etag = "ETag";
 226   std::string etag_value;
 227   headers->EnumerateHeader(NULL, etag, &etag_value);
 228   if (!etag_value.empty()) {
 229     extra_headers.SetHeader(net::HttpRequestHeaders::kIfNoneMatch,
 230                             etag_value);
 231   }
 232   if (!extra_headers.IsEmpty())
 233     request_->SetExtraRequestHeaders(extra_headers);
 234 }
 235
 236 void  AppCacheUpdateJob::URLFetcher::OnWriteComplete(int result) {
 237   if (result < 0) {
 238     request_->Cancel();
 239     result_ = DISKCACHE_ERROR;
 240     OnResponseCompleted();
 241     return;
 242   }
 243   ReadResponseData();
 244 }
 245
 246 void AppCacheUpdateJob::URLFetcher::ReadResponseData() {
 247   InternalUpdateState state = job_->internal_state_;
 248   if (state == CACHE_FAILURE || state == CANCELLED || state == COMPLETED)
 249     return;
 250   int bytes_read = 0;
 251   request_->Read(buffer_.get(), kBufferSize, &bytes_read);
 252   OnReadCompleted(request_.get(), bytes_read);
 253 }
 254
 255 // Returns false if response data is processed asynchronously, in which
 256 // case ReadResponseData will be invoked when it is safe to continue
 257 // reading more response data from the request.
 258 bool AppCacheUpdateJob::URLFetcher::ConsumeResponseData(int bytes_read) {
 259   DCHECK_GT(bytes_read, 0);
 260   switch (fetch_type_) {
 261     case MANIFEST_FETCH:
 262     case MANIFEST_REFETCH:
 263       manifest_data_.append(buffer_->data(), bytes_read);
 264       break;
 265     case URL_FETCH:
 266     case MASTER_ENTRY_FETCH:
 267       DCHECK(response_writer_.get());
 268       response_writer_->WriteData(
 269           buffer_.get(),
 270           bytes_read,
 271           base::Bind(&URLFetcher::OnWriteComplete, base::Unretained(this)));
 272       return false;  // wait for async write completion to continue reading
 273     default:
 274       NOTREACHED();
 275   }
 276   return true;
 277 }
 278
 279 void AppCacheUpdateJob::URLFetcher::OnResponseCompleted() {
 280   // Retry for 503s where retry-after is 0.
 281   if (request_->status().is_success() &&
 282       request_->GetResponseCode() == 503 &&
 283       MaybeRetryRequest()) {
 284     return;
 285   }
 286
 287   switch (fetch_type_) {
 288     case MANIFEST_FETCH:
 289       job_->HandleManifestFetchCompleted(this);
 290       break;
 291     case URL_FETCH:
 292       job_->HandleUrlFetchCompleted(this);
 293       break;
 294     case MASTER_ENTRY_FETCH:
 295       job_->HandleMasterEntryFetchCompleted(this);
 296       break;
 297     case MANIFEST_REFETCH:
 298       job_->HandleManifestRefetchCompleted(this);
 299       break;
 300     default:
 301       NOTREACHED();
 302   }
 303
 304   delete this;
 305 }
 306
 307 bool AppCacheUpdateJob::URLFetcher::MaybeRetryRequest() {
 308   if (retry_503_attempts_ >= kMax503Retries ||
 309       !request_->response_headers()->HasHeaderValue("retry-after", "0")) {
 310     return false;
 311   }
 312   ++retry_503_attempts_;
 313   request_ = job_->service_->request_context()->CreateRequest(
 314       url_, net::DEFAULT_PRIORITY, this);
 315   Start();
 316   return true;
 317 }
 318
 319 AppCacheUpdateJob::AppCacheUpdateJob(AppCacheService* service,
 320                                      AppCacheGroup* group)
 321     : service_(service),
 322       manifest_url_(group->manifest_url()),
 323       group_(group),
 324       update_type_(UNKNOWN_TYPE),
 325       internal_state_(FETCH_MANIFEST),
 326       master_entries_completed_(0),
 327       url_fetches_completed_(0),
 328       manifest_fetcher_(NULL),
 329       stored_state_(UNSTORED),
 330       storage_(service->storage()) {
 331     service_->AddObserver(this);
 332 }
 333
 334 AppCacheUpdateJob::~AppCacheUpdateJob() {
 335   if (service_)
 336     service_->RemoveObserver(this);
 337   if (internal_state_ != COMPLETED)
 338     Cancel();
 339
 340   DCHECK(!manifest_fetcher_);
 341   DCHECK(pending_url_fetches_.empty());
 342   DCHECK(!inprogress_cache_.get());
 343   DCHECK(pending_master_entries_.empty());
 344   DCHECK(master_entry_fetches_.empty());
 345
 346   if (group_)
 347     group_->SetUpdateStatus(AppCacheGroup::IDLE);
 348 }
 349
 350 void AppCacheUpdateJob::StartUpdate(AppCacheHost* host,
 351                                     const GURL& new_master_resource) {
 352   DCHECK(group_->update_job() == this);
 353   DCHECK(!group_->is_obsolete());
 354
 355   bool is_new_pending_master_entry = false;
 356   if (!new_master_resource.is_empty()) {
 357     DCHECK(new_master_resource == host->pending_master_entry_url());
 358     DCHECK(!new_master_resource.has_ref());
 359     DCHECK(new_master_resource.GetOrigin() == manifest_url_.GetOrigin());
 360
 361     // Cannot add more to this update if already terminating.
 362     if (IsTerminating()) {
 363       group_->QueueUpdate(host, new_master_resource);
 364       return;
 365     }
 366
 367     std::pair<PendingMasters::iterator, bool> ret =
 368         pending_master_entries_.insert(
 369             PendingMasters::value_type(new_master_resource, PendingHosts()));
 370     is_new_pending_master_entry = ret.second;
 371     ret.first->second.push_back(host);
 372     host->AddObserver(this);
 373   }
 374
 375   // Notify host (if any) if already checking or downloading.
 376   AppCacheGroup::UpdateStatus update_status = group_->update_status();
 377   if (update_status == AppCacheGroup::CHECKING ||
 378       update_status == AppCacheGroup::DOWNLOADING) {
 379     if (host) {
 380       NotifySingleHost(host, CHECKING_EVENT);
 381       if (update_status == AppCacheGroup::DOWNLOADING)
 382         NotifySingleHost(host, DOWNLOADING_EVENT);
 383
 384       // Add to fetch list or an existing entry if already fetched.
 385       if (!new_master_resource.is_empty()) {
 386         AddMasterEntryToFetchList(host, new_master_resource,
 387                                   is_new_pending_master_entry);
 388       }
 389     }
 390     return;
 391   }
 392
 393   // Begin update process for the group.
 394   group_->SetUpdateStatus(AppCacheGroup::CHECKING);
 395   if (group_->HasCache()) {
 396     update_type_ = UPGRADE_ATTEMPT;
 397     NotifyAllAssociatedHosts(CHECKING_EVENT);
 398   } else {
 399     update_type_ = CACHE_ATTEMPT;
 400     DCHECK(host);
 401     NotifySingleHost(host, CHECKING_EVENT);
 402   }
 403
 404   if (!new_master_resource.is_empty()) {
 405     AddMasterEntryToFetchList(host, new_master_resource,
 406                               is_new_pending_master_entry);
 407   }
 408
 409   FetchManifest(true);
 410 }
 411
 412 AppCacheResponseWriter* AppCacheUpdateJob::CreateResponseWriter() {
 413   AppCacheResponseWriter* writer =
 414       storage_->CreateResponseWriter(manifest_url_,
 415                                                 group_->group_id());
 416   stored_response_ids_.push_back(writer->response_id());
 417   return writer;
 418 }
 419
 420 void AppCacheUpdateJob::HandleCacheFailure(
 421     const std::string& error_message,
 422     ResultType result) {
 423   // 6.9.4 cache failure steps 2-8.
 424   DCHECK(internal_state_ != CACHE_FAILURE);
 425   DCHECK(!error_message.empty());
 426   DCHECK(result != UPDATE_OK);
 427   internal_state_ = CACHE_FAILURE;
 428   CancelAllUrlFetches();
 429   CancelAllMasterEntryFetches(error_message);
 430   NotifyAllError(error_message);
 431   DiscardInprogressCache();
 432   internal_state_ = COMPLETED;
 433   AppCacheHistograms::CountUpdateJobResult(
 434       result, manifest_url_.GetOrigin());
 435   DeleteSoon();  // To unwind the stack prior to deletion.
 436 }
 437
 438 void AppCacheUpdateJob::FetchManifest(bool is_first_fetch) {
 439   DCHECK(!manifest_fetcher_);
 440   manifest_fetcher_ = new URLFetcher(
 441      manifest_url_,
 442      is_first_fetch ? URLFetcher::MANIFEST_FETCH :
 443                       URLFetcher::MANIFEST_REFETCH,
 444      this);
 445
 446   // Add any necessary Http headers before sending fetch request.
 447   if (is_first_fetch) {
 448     AppCacheEntry* entry = (update_type_ == UPGRADE_ATTEMPT) ?
 449         group_->newest_complete_cache()->GetEntry(manifest_url_) : NULL;
 450     if (entry) {
 451       // Asynchronously load response info for manifest from newest cache.
 452       storage_->LoadResponseInfo(manifest_url_, group_->group_id(),
 453                                  entry->response_id(), this);
 454     } else {
 455       manifest_fetcher_->Start();
 456     }
 457   } else {
 458     DCHECK(internal_state_ == REFETCH_MANIFEST);
 459     DCHECK(manifest_response_info_.get());
 460     manifest_fetcher_->set_existing_response_headers(
 461         manifest_response_info_->headers.get());
 462     manifest_fetcher_->Start();
 463   }
 464 }
 465
 466
 467 void AppCacheUpdateJob::HandleManifestFetchCompleted(
 468     URLFetcher* fetcher) {
 469   DCHECK_EQ(internal_state_, FETCH_MANIFEST);
 470   DCHECK_EQ(manifest_fetcher_, fetcher);
 471   manifest_fetcher_ = NULL;
 472
 473   net::URLRequest* request = fetcher->request();
 474   int response_code = -1;
 475   bool is_valid_response_code = false;
 476   if (request->status().is_success()) {
 477     response_code = request->GetResponseCode();
 478     is_valid_response_code = (response_code / 100 == 2);
 479   }
 480
 481   if (is_valid_response_code) {
 482     manifest_data_ = fetcher->manifest_data();
 483     manifest_response_info_.reset(
 484         new net::HttpResponseInfo(request->response_info()));
 485     if (update_type_ == UPGRADE_ATTEMPT)
 486       CheckIfManifestChanged();  // continues asynchronously
 487     else
 488       ContinueHandleManifestFetchCompleted(true);
 489   } else if (response_code == 304 && update_type_ == UPGRADE_ATTEMPT) {
 490     ContinueHandleManifestFetchCompleted(false);
 491   } else if ((response_code == 404 || response_code == 410) &&
 492              update_type_ == UPGRADE_ATTEMPT) {
 493     storage_->MakeGroupObsolete(group_, this);  // async
 494   } else {
 495     const char* kFormatString = "Manifest fetch failed (%d) %s";
 496     std::string message = FormatUrlErrorMessage(
 497         kFormatString, manifest_url_, fetcher->result(), response_code);
 498     HandleCacheFailure(message, fetcher->result());
 499   }
 500 }
 501
 502 void AppCacheUpdateJob::OnGroupMadeObsolete(AppCacheGroup* group,
 503                                             bool success) {
 504   DCHECK(master_entry_fetches_.empty());
 505   CancelAllMasterEntryFetches("The cache has been made obsolete, "
 506                               "the manifest file returned 404 or 410");
 507   if (success) {
 508     DCHECK(group->is_obsolete());
 509     NotifyAllAssociatedHosts(OBSOLETE_EVENT);
 510     internal_state_ = COMPLETED;
 511     MaybeCompleteUpdate();
 512   } else {
 513     // Treat failure to mark group obsolete as a cache failure.
 514     HandleCacheFailure("Failed to mark the cache as obsolete", DB_ERROR);
 515   }
 516 }
 517
 518 void AppCacheUpdateJob::ContinueHandleManifestFetchCompleted(bool changed) {
 519   DCHECK(internal_state_ == FETCH_MANIFEST);
 520
 521   if (!changed) {
 522     DCHECK(update_type_ == UPGRADE_ATTEMPT);
 523     internal_state_ = NO_UPDATE;
 524
 525     // Wait for pending master entries to download.
 526     FetchMasterEntries();
 527     MaybeCompleteUpdate();  // if not done, run async 6.9.4 step 7 substeps
 528     return;
 529   }
 530
 531   Manifest manifest;
 532   if (!ParseManifest(manifest_url_, manifest_data_.data(),
 533                      manifest_data_.length(), manifest)) {
 534     const char* kFormatString = "Failed to parse manifest %s";
 535     const std::string message = base::StringPrintf(kFormatString,
 536         manifest_url_.spec().c_str());
 537     HandleCacheFailure(message, MANIFEST_ERROR);
 538     VLOG(1) << message;
 539     return;
 540   }
 541
 542   // Proceed with update process. Section 6.9.4 steps 8-20.
 543   internal_state_ = DOWNLOADING;
 544   inprogress_cache_ = new AppCache(storage_, storage_->NewCacheId());
 545   BuildUrlFileList(manifest);
 546   inprogress_cache_->InitializeWithManifest(&manifest);
 547
 548   // Associate all pending master hosts with the newly created cache.
 549   for (PendingMasters::iterator it = pending_master_entries_.begin();
 550        it != pending_master_entries_.end(); ++it) {
 551     PendingHosts& hosts = it->second;
 552     for (PendingHosts::iterator host_it = hosts.begin();
 553          host_it != hosts.end(); ++host_it) {
 554       (*host_it)
 555           ->AssociateIncompleteCache(inprogress_cache_.get(), manifest_url_);
 556     }
 557   }
 558
 559   group_->SetUpdateStatus(AppCacheGroup::DOWNLOADING);
 560   NotifyAllAssociatedHosts(DOWNLOADING_EVENT);
 561   FetchUrls();
 562   FetchMasterEntries();
 563   MaybeCompleteUpdate();  // if not done, continues when async fetches complete
 564 }
 565
 566 void AppCacheUpdateJob::HandleUrlFetchCompleted(URLFetcher* fetcher) {
 567   DCHECK(internal_state_ == DOWNLOADING);
 568
 569   net::URLRequest* request = fetcher->request();
 570   const GURL& url = request->original_url();
 571   pending_url_fetches_.erase(url);
 572   NotifyAllProgress(url);
 573   ++url_fetches_completed_;
 574
 575   int response_code = request->status().is_success()
 576       ? request->GetResponseCode() : -1;
 577   AppCacheEntry& entry = url_file_list_.find(url)->second;
 578
 579   if (response_code / 100 == 2) {
 580     // Associate storage with the new entry.
 581     DCHECK(fetcher->response_writer());
 582     entry.set_response_id(fetcher->response_writer()->response_id());
 583     entry.set_response_size(fetcher->response_writer()->amount_written());
 584     if (!inprogress_cache_->AddOrModifyEntry(url, entry))
 585       duplicate_response_ids_.push_back(entry.response_id());
 586
 587     // TODO(michaeln): Check for <html manifest=xxx>
 588     // See http://code.google.com/p/chromium/issues/detail?id=97930
 589     // if (entry.IsMaster() && !(entry.IsExplicit() || fallback || intercept))
 590     //   if (!manifestAttribute) skip it
 591
 592     // Foreign entries will be detected during cache selection.
 593     // Note: 6.9.4, step 17.9 possible optimization: if resource is HTML or XML
 594     // file whose root element is an html element with a manifest attribute
 595     // whose value doesn't match the manifest url of the application cache
 596     // being processed, mark the entry as being foreign.
 597   } else {
 598     VLOG(1) << "Request status: " << request->status().status()
 599             << " error: " << request->status().error()
 600             << " response code: " << response_code;
 601     if (entry.IsExplicit() || entry.IsFallback() || entry.IsIntercept()) {
 602       if (response_code == 304 && fetcher->existing_entry().has_response_id()) {
 603         // Keep the existing response.
 604         entry.set_response_id(fetcher->existing_entry().response_id());
 605         entry.set_response_size(fetcher->existing_entry().response_size());
 606         inprogress_cache_->AddOrModifyEntry(url, entry);
 607       } else {
 608         const char* kFormatString = "Resource fetch failed (%d) %s";
 609         std::string message = FormatUrlErrorMessage(
 610             kFormatString, url, fetcher->result(), response_code);
 611         HandleCacheFailure(message, fetcher->result());
 612         return;
 613       }
 614     } else if (response_code == 404 || response_code == 410) {
 615       // Entry is skipped.  They are dropped from the cache.
 616     } else if (update_type_ == UPGRADE_ATTEMPT &&
 617                fetcher->existing_entry().has_response_id()) {
 618       // Keep the existing response.
 619       // TODO(michaeln): Not sure this is a good idea. This is spec compliant
 620       // but the old resource may or may not be compatible with the new contents
 621       // of the cache. Impossible to know one way or the other.
 622       entry.set_response_id(fetcher->existing_entry().response_id());
 623       entry.set_response_size(fetcher->existing_entry().response_size());
 624       inprogress_cache_->AddOrModifyEntry(url, entry);
 625     }
 626   }
 627
 628   // Fetch another URL now that one request has completed.
 629   DCHECK(internal_state_ != CACHE_FAILURE);
 630   FetchUrls();
 631   MaybeCompleteUpdate();
 632 }
 633
 634 void AppCacheUpdateJob::HandleMasterEntryFetchCompleted(
 635     URLFetcher* fetcher) {
 636   DCHECK(internal_state_ == NO_UPDATE || internal_state_ == DOWNLOADING);
 637
 638   // TODO(jennb): Handle downloads completing during cache failure when update
 639   // no longer fetches master entries directly. For now, we cancel all pending
 640   // master entry fetches when entering cache failure state so this will never
 641   // be called in CACHE_FAILURE state.
 642
 643   net::URLRequest* request = fetcher->request();
 644   const GURL& url = request->original_url();
 645   master_entry_fetches_.erase(url);
 646   ++master_entries_completed_;
 647
 648   int response_code = request->status().is_success()
 649       ? request->GetResponseCode() : -1;
 650
 651   PendingMasters::iterator found = pending_master_entries_.find(url);
 652   DCHECK(found != pending_master_entries_.end());
 653   PendingHosts& hosts = found->second;
 654
 655   // Section 6.9.4. No update case: step 7.3, else step 22.
 656   if (response_code / 100 == 2) {
 657     // Add fetched master entry to the appropriate cache.
 658     AppCache* cache = inprogress_cache_.get() ? inprogress_cache_.get()
 659                                               : group_->newest_complete_cache();
 660     DCHECK(fetcher->response_writer());
 661     AppCacheEntry master_entry(AppCacheEntry::MASTER,
 662                                fetcher->response_writer()->response_id(),
 663                                fetcher->response_writer()->amount_written());
 664     if (cache->AddOrModifyEntry(url, master_entry))
 665       added_master_entries_.push_back(url);
 666     else
 667       duplicate_response_ids_.push_back(master_entry.response_id());
 668
 669     // In no-update case, associate host with the newest cache.
 670     if (!inprogress_cache_.get()) {
 671       // TODO(michaeln): defer until the updated cache has been stored
 672       DCHECK(cache == group_->newest_complete_cache());
 673       for (PendingHosts::iterator host_it = hosts.begin();
 674            host_it != hosts.end(); ++host_it) {
 675         (*host_it)->AssociateCompleteCache(cache);
 676       }
 677     }
 678   } else {
 679     HostNotifier host_notifier;
 680     for (PendingHosts::iterator host_it = hosts.begin();
 681          host_it != hosts.end(); ++host_it) {
 682       AppCacheHost* host = *host_it;
 683       host_notifier.AddHost(host);
 684
 685       // In downloading case, disassociate host from inprogress cache.
 686       if (inprogress_cache_.get())
 687         host->AssociateNoCache(GURL());
 688
 689       host->RemoveObserver(this);
 690     }
 691     hosts.clear();
 692
 693     const char* kFormatString = "Manifest fetch failed (%d) %s";
 694     std::string message = FormatUrlErrorMessage(
 695         kFormatString, request->url(), fetcher->result(), response_code);
 696     host_notifier.SendErrorNotifications(message);
 697
 698     // In downloading case, update result is different if all master entries
 699     // failed vs. only some failing.
 700     if (inprogress_cache_.get()) {
 701       // Only count successful downloads to know if all master entries failed.
 702       pending_master_entries_.erase(found);
 703       --master_entries_completed_;
 704
 705       // Section 6.9.4, step 22.3.
 706       if (update_type_ == CACHE_ATTEMPT && pending_master_entries_.empty()) {
 707         HandleCacheFailure(message, fetcher->result());
 708         return;
 709       }
 710     }
 711   }
 712
 713   DCHECK(internal_state_ != CACHE_FAILURE);
 714   FetchMasterEntries();
 715   MaybeCompleteUpdate();
 716 }
 717
 718 void AppCacheUpdateJob::HandleManifestRefetchCompleted(
 719     URLFetcher* fetcher) {
 720   DCHECK(internal_state_ == REFETCH_MANIFEST);
 721   DCHECK(manifest_fetcher_ == fetcher);
 722   manifest_fetcher_ = NULL;
 723
 724   net::URLRequest* request = fetcher->request();
 725   int response_code = request->status().is_success()
 726       ? request->GetResponseCode() : -1;
 727   if (response_code == 304 || manifest_data_ == fetcher->manifest_data()) {
 728     // Only need to store response in storage if manifest is not already
 729     // an entry in the cache.
 730     AppCacheEntry* entry = inprogress_cache_->GetEntry(manifest_url_);
 731     if (entry) {
 732       entry->add_types(AppCacheEntry::MANIFEST);
 733       StoreGroupAndCache();
 734     } else {
 735       manifest_response_writer_.reset(CreateResponseWriter());
 736       scoped_refptr<HttpResponseInfoIOBuffer> io_buffer(
 737           new HttpResponseInfoIOBuffer(manifest_response_info_.release()));
 738       manifest_response_writer_->WriteInfo(
 739           io_buffer.get(),
 740           base::Bind(&AppCacheUpdateJob::OnManifestInfoWriteComplete,
 741                      base::Unretained(this)));
 742     }
 743   } else {
 744     VLOG(1) << "Request status: " << request->status().status()
 745             << " error: " << request->status().error()
 746             << " response code: " << response_code;
 747     ScheduleUpdateRetry(kRerunDelayMs);
 748     if (response_code == 200) {
 749       HandleCacheFailure("Manifest changed during update", MANIFEST_ERROR);
 750     } else {
 751       const char* kFormatString = "Manifest re-fetch failed (%d) %s";
 752       std::string message = FormatUrlErrorMessage(
 753           kFormatString, manifest_url_, fetcher->result(), response_code);
 754       HandleCacheFailure(message, fetcher->result());
 755     }
 756   }
 757 }
 758
 759 void AppCacheUpdateJob::OnManifestInfoWriteComplete(int result) {
 760   if (result > 0) {
 761     scoped_refptr<net::StringIOBuffer> io_buffer(
 762         new net::StringIOBuffer(manifest_data_));
 763     manifest_response_writer_->WriteData(
 764         io_buffer.get(),
 765         manifest_data_.length(),
 766         base::Bind(&AppCacheUpdateJob::OnManifestDataWriteComplete,
 767                    base::Unretained(this)));
 768   } else {
 769     HandleCacheFailure("Failed to write the manifest headers to storage",
 770                        DISKCACHE_ERROR);
 771   }
 772 }
 773
 774 void AppCacheUpdateJob::OnManifestDataWriteComplete(int result) {
 775   if (result > 0) {
 776     AppCacheEntry entry(AppCacheEntry::MANIFEST,
 777         manifest_response_writer_->response_id(),
 778         manifest_response_writer_->amount_written());
 779     if (!inprogress_cache_->AddOrModifyEntry(manifest_url_, entry))
 780       duplicate_response_ids_.push_back(entry.response_id());
 781     StoreGroupAndCache();
 782   } else {
 783     HandleCacheFailure("Failed to write the manifest data to storage",
 784                        DISKCACHE_ERROR);
 785   }
 786 }
 787
 788 void AppCacheUpdateJob::StoreGroupAndCache() {
 789   DCHECK(stored_state_ == UNSTORED);
 790   stored_state_ = STORING;
 791   scoped_refptr<AppCache> newest_cache;
 792   if (inprogress_cache_.get())
 793     newest_cache.swap(inprogress_cache_);
 794   else
 795     newest_cache = group_->newest_complete_cache();
 796   newest_cache->set_update_time(base::Time::Now());
 797
 798   // TODO(michaeln): dcheck is fishing for clues to crbug/95101
 799   DCHECK_EQ(manifest_url_, group_->manifest_url());
 800   storage_->StoreGroupAndNewestCache(group_, newest_cache.get(), this);
 801 }
 802
 803 void AppCacheUpdateJob::OnGroupAndNewestCacheStored(AppCacheGroup* group,
 804                                                     AppCache* newest_cache,
 805                                                     bool success,
 806                                                     bool would_exceed_quota) {
 807   DCHECK(stored_state_ == STORING);
 808   if (success) {
 809     stored_state_ = STORED;
 810     MaybeCompleteUpdate();  // will definitely complete
 811   } else {
 812     stored_state_ = UNSTORED;
 813
 814     // Restore inprogress_cache_ to get the proper events delivered
 815     // and the proper cleanup to occur.
 816     if (newest_cache != group->newest_complete_cache())
 817       inprogress_cache_ = newest_cache;
 818
 819     ResultType result = DB_ERROR;
 820     std::string message("Failed to commit new cache to storage");
 821     if (would_exceed_quota) {
 822       message.append(", would exceed quota");
 823       result = QUOTA_ERROR;
 824     }
 825     HandleCacheFailure(message, result);
 826   }
 827 }
 828
 829 void AppCacheUpdateJob::NotifySingleHost(AppCacheHost* host,
 830                                          EventID event_id) {
 831   std::vector<int> ids(1, host->host_id());
 832   host->frontend()->OnEventRaised(ids, event_id);
 833 }
 834
 835 void AppCacheUpdateJob::NotifyAllAssociatedHosts(EventID event_id) {
 836   HostNotifier host_notifier;
 837   AddAllAssociatedHostsToNotifier(&host_notifier);
 838   host_notifier.SendNotifications(event_id);
 839 }
 840
 841 void AppCacheUpdateJob::NotifyAllProgress(const GURL& url) {
 842   HostNotifier host_notifier;
 843   AddAllAssociatedHostsToNotifier(&host_notifier);
 844   host_notifier.SendProgressNotifications(
 845       url, url_file_list_.size(), url_fetches_completed_);
 846 }
 847
 848 void AppCacheUpdateJob::NotifyAllFinalProgress() {
 849   DCHECK(url_file_list_.size() == url_fetches_completed_);
 850   NotifyAllProgress(GURL());
 851 }
 852
 853 void AppCacheUpdateJob::NotifyAllError(const std::string& error_message) {
 854   HostNotifier host_notifier;
 855   AddAllAssociatedHostsToNotifier(&host_notifier);
 856   host_notifier.SendErrorNotifications(error_message);
 857 }
 858
 859 void AppCacheUpdateJob::AddAllAssociatedHostsToNotifier(
 860     HostNotifier* host_notifier) {
 861   // Collect hosts so we only send one notification per frontend.
 862   // A host can only be associated with a single cache so no need to worry
 863   // about duplicate hosts being added to the notifier.
 864   if (inprogress_cache_.get()) {
 865     DCHECK(internal_state_ == DOWNLOADING || internal_state_ == CACHE_FAILURE);
 866     host_notifier->AddHosts(inprogress_cache_->associated_hosts());
 867   }
 868
 869   AppCacheGroup::Caches old_caches = group_->old_caches();
 870   for (AppCacheGroup::Caches::const_iterator it = old_caches.begin();
 871        it != old_caches.end(); ++it) {
 872     host_notifier->AddHosts((*it)->associated_hosts());
 873   }
 874
 875   AppCache* newest_cache = group_->newest_complete_cache();
 876   if (newest_cache)
 877     host_notifier->AddHosts(newest_cache->associated_hosts());
 878 }
 879
 880 void AppCacheUpdateJob::OnDestructionImminent(AppCacheHost* host) {
 881   // The host is about to be deleted; remove from our collection.
 882   PendingMasters::iterator found =
 883       pending_master_entries_.find(host->pending_master_entry_url());
 884   DCHECK(found != pending_master_entries_.end());
 885   PendingHosts& hosts = found->second;
 886   PendingHosts::iterator it = std::find(hosts.begin(), hosts.end(), host);
 887   DCHECK(it != hosts.end());
 888   hosts.erase(it);
 889 }
 890
 891 void AppCacheUpdateJob::OnServiceReinitialized(
 892     AppCacheStorageReference* old_storage_ref) {
 893   // We continue to use the disabled instance, but arrange for its
 894   // deletion when its no longer needed.
 895   if (old_storage_ref->storage() == storage_)
 896     disabled_storage_reference_ = old_storage_ref;
 897 }
 898
 899 void AppCacheUpdateJob::CheckIfManifestChanged() {
 900   DCHECK(update_type_ == UPGRADE_ATTEMPT);
 901   AppCacheEntry* entry = NULL;
 902   if (group_->newest_complete_cache())
 903     entry = group_->newest_complete_cache()->GetEntry(manifest_url_);
 904   if (!entry) {
 905     // TODO(michaeln): This is just a bandaid to avoid a crash.
 906     // http://code.google.com/p/chromium/issues/detail?id=95101
 907     if (service_->storage() == storage_) {
 908       // Use a local variable because service_ is reset in HandleCacheFailure.
 909       AppCacheService* service = service_;
 910       HandleCacheFailure("Manifest entry not found in existing cache",
 911                          DB_ERROR);
 912       AppCacheHistograms::AddMissingManifestEntrySample();
 913       service->DeleteAppCacheGroup(manifest_url_, net::CompletionCallback());
 914     }
 915     return;
 916   }
 917
 918   // Load manifest data from storage to compare against fetched manifest.
 919   manifest_response_reader_.reset(
 920       storage_->CreateResponseReader(manifest_url_,
 921                                      group_->group_id(),
 922                                      entry->response_id()));
 923   read_manifest_buffer_ = new net::IOBuffer(kBufferSize);
 924   manifest_response_reader_->ReadData(
 925       read_manifest_buffer_.get(),
 926       kBufferSize,
 927       base::Bind(&AppCacheUpdateJob::OnManifestDataReadComplete,
 928                  base::Unretained(this)));  // async read
 929 }
 930
 931 void AppCacheUpdateJob::OnManifestDataReadComplete(int result) {
 932   if (result > 0) {
 933     loaded_manifest_data_.append(read_manifest_buffer_->data(), result);
 934     manifest_response_reader_->ReadData(
 935         read_manifest_buffer_.get(),
 936         kBufferSize,
 937         base::Bind(&AppCacheUpdateJob::OnManifestDataReadComplete,
 938                    base::Unretained(this)));  // read more
 939   } else {
 940     read_manifest_buffer_ = NULL;
 941     manifest_response_reader_.reset();
 942     ContinueHandleManifestFetchCompleted(
 943         result < 0 || manifest_data_ != loaded_manifest_data_);
 944   }
 945 }
 946
 947 void AppCacheUpdateJob::BuildUrlFileList(const Manifest& manifest) {
 948   for (base::hash_set<std::string>::const_iterator it =
 949            manifest.explicit_urls.begin();
 950        it != manifest.explicit_urls.end(); ++it) {
 951     AddUrlToFileList(GURL(*it), AppCacheEntry::EXPLICIT);
 952   }
 953
 954   const std::vector<Namespace>& intercepts =
 955       manifest.intercept_namespaces;
 956   for (std::vector<Namespace>::const_iterator it = intercepts.begin();
 957        it != intercepts.end(); ++it) {
 958     int flags = AppCacheEntry::INTERCEPT;
 959     if (it->is_executable)
 960       flags |= AppCacheEntry::EXECUTABLE;
 961     AddUrlToFileList(it->target_url, flags);
 962   }
 963
 964   const std::vector<Namespace>& fallbacks =
 965       manifest.fallback_namespaces;
 966   for (std::vector<Namespace>::const_iterator it = fallbacks.begin();
 967        it != fallbacks.end(); ++it) {
 968      AddUrlToFileList(it->target_url, AppCacheEntry::FALLBACK);
 969   }
 970
 971   // Add all master entries from newest complete cache.
 972   if (update_type_ == UPGRADE_ATTEMPT) {
 973     const AppCache::EntryMap& entries =
 974         group_->newest_complete_cache()->entries();
 975     for (AppCache::EntryMap::const_iterator it = entries.begin();
 976          it != entries.end(); ++it) {
 977       const AppCacheEntry& entry = it->second;
 978       if (entry.IsMaster())
 979         AddUrlToFileList(it->first, AppCacheEntry::MASTER);
 980     }
 981   }
 982 }
 983
 984 void AppCacheUpdateJob::AddUrlToFileList(const GURL& url, int type) {
 985   std::pair<AppCache::EntryMap::iterator, bool> ret = url_file_list_.insert(
 986       AppCache::EntryMap::value_type(url, AppCacheEntry(type)));
 987
 988   if (ret.second)
 989     urls_to_fetch_.push_back(UrlToFetch(url, false, NULL));
 990   else
 991     ret.first->second.add_types(type);  // URL already exists. Merge types.
 992 }
 993
 994 void AppCacheUpdateJob::FetchUrls() {
 995   DCHECK(internal_state_ == DOWNLOADING);
 996
 997   // Fetch each URL in the list according to section 6.9.4 step 17.1-17.3.
 998   // Fetch up to the concurrent limit. Other fetches will be triggered as each
 999   // each fetch completes.
1000   while (pending_url_fetches_.size() < kMaxConcurrentUrlFetches &&
1001          !urls_to_fetch_.empty()) {
1002     UrlToFetch url_to_fetch = urls_to_fetch_.front();
1003     urls_to_fetch_.pop_front();
1004
1005     AppCache::EntryMap::iterator it = url_file_list_.find(url_to_fetch.url);
1006     DCHECK(it != url_file_list_.end());
1007     AppCacheEntry& entry = it->second;
1008     if (ShouldSkipUrlFetch(entry)) {
1009       NotifyAllProgress(url_to_fetch.url);
1010       ++url_fetches_completed_;
1011     } else if (AlreadyFetchedEntry(url_to_fetch.url, entry.types())) {
1012       NotifyAllProgress(url_to_fetch.url);
1013       ++url_fetches_completed_;  // saved a URL request
1014     } else if (!url_to_fetch.storage_checked &&
1015                MaybeLoadFromNewestCache(url_to_fetch.url, entry)) {
1016       // Continues asynchronously after data is loaded from newest cache.
1017     } else {
1018       URLFetcher* fetcher = new URLFetcher(
1019           url_to_fetch.url, URLFetcher::URL_FETCH, this);
1020       if (url_to_fetch.existing_response_info.get()) {
1021         DCHECK(group_->newest_complete_cache());
1022         AppCacheEntry* existing_entry =
1023             group_->newest_complete_cache()->GetEntry(url_to_fetch.url);
1024         DCHECK(existing_entry);
1025         DCHECK(existing_entry->response_id() ==
1026                url_to_fetch.existing_response_info->response_id());
1027         fetcher->set_existing_response_headers(
1028             url_to_fetch.existing_response_info->http_response_info()->headers
1029                 .get());
1030         fetcher->set_existing_entry(*existing_entry);
1031       }
1032       fetcher->Start();
1033       pending_url_fetches_.insert(
1034           PendingUrlFetches::value_type(url_to_fetch.url, fetcher));
1035     }
1036   }
1037 }
1038
1039 void AppCacheUpdateJob::CancelAllUrlFetches() {
1040   // Cancel any pending URL requests.
1041   for (PendingUrlFetches::iterator it = pending_url_fetches_.begin();
1042        it != pending_url_fetches_.end(); ++it) {
1043     delete it->second;
1044   }
1045
1046   url_fetches_completed_ +=
1047       pending_url_fetches_.size() + urls_to_fetch_.size();
1048   pending_url_fetches_.clear();
1049   urls_to_fetch_.clear();
1050 }
1051
1052 bool AppCacheUpdateJob::ShouldSkipUrlFetch(const AppCacheEntry& entry) {
1053   // 6.6.4 Step 17
1054   // If the resource URL being processed was flagged as neither an
1055   // "explicit entry" nor or a "fallback entry", then the user agent
1056   // may skip this URL.
1057   if (entry.IsExplicit() || entry.IsFallback() || entry.IsIntercept())
1058     return false;
1059
1060   // TODO(jennb): decide if entry should be skipped to expire it from cache
1061   return false;
1062 }
1063
1064 bool AppCacheUpdateJob::AlreadyFetchedEntry(const GURL& url,
1065                                             int entry_type) {
1066   DCHECK(internal_state_ == DOWNLOADING || internal_state_ == NO_UPDATE);
1067   AppCacheEntry* existing =
1068       inprogress_cache_.get() ? inprogress_cache_->GetEntry(url)
1069                               : group_->newest_complete_cache()->GetEntry(url);
1070   if (existing) {
1071     existing->add_types(entry_type);
1072     return true;
1073   }
1074   return false;
1075 }
1076
1077 void AppCacheUpdateJob::AddMasterEntryToFetchList(AppCacheHost* host,
1078                                                   const GURL& url,
1079                                                   bool is_new) {
1080   DCHECK(!IsTerminating());
1081
1082   if (internal_state_ == DOWNLOADING || internal_state_ == NO_UPDATE) {
1083     AppCache* cache;
1084     if (inprogress_cache_.get()) {
1085       // always associate
1086       host->AssociateIncompleteCache(inprogress_cache_.get(), manifest_url_);
1087       cache = inprogress_cache_.get();
1088     } else {
1089       cache = group_->newest_complete_cache();
1090     }
1091
1092     // Update existing entry if it has already been fetched.
1093     AppCacheEntry* entry = cache->GetEntry(url);
1094     if (entry) {
1095       entry->add_types(AppCacheEntry::MASTER);
1096       if (internal_state_ == NO_UPDATE && !inprogress_cache_.get()) {
1097         // only associate if have entry
1098         host->AssociateCompleteCache(cache);
1099       }
1100       if (is_new)
1101         ++master_entries_completed_;  // pretend fetching completed
1102       return;
1103     }
1104   }
1105
1106   // Add to fetch list if not already fetching.
1107   if (master_entry_fetches_.find(url) == master_entry_fetches_.end()) {
1108     master_entries_to_fetch_.insert(url);
1109     if (internal_state_ == DOWNLOADING || internal_state_ == NO_UPDATE)
1110       FetchMasterEntries();
1111   }
1112 }
1113
1114 void AppCacheUpdateJob::FetchMasterEntries() {
1115   DCHECK(internal_state_ == NO_UPDATE || internal_state_ == DOWNLOADING);
1116
1117   // Fetch each master entry in the list, up to the concurrent limit.
1118   // Additional fetches will be triggered as each fetch completes.
1119   while (master_entry_fetches_.size() < kMaxConcurrentUrlFetches &&
1120          !master_entries_to_fetch_.empty()) {
1121     const GURL& url = *master_entries_to_fetch_.begin();
1122
1123     if (AlreadyFetchedEntry(url, AppCacheEntry::MASTER)) {
1124       ++master_entries_completed_;  // saved a URL request
1125
1126       // In no update case, associate hosts to newest cache in group
1127       // now that master entry has been "successfully downloaded".
1128       if (internal_state_ == NO_UPDATE) {
1129         // TODO(michaeln): defer until the updated cache has been stored.
1130         DCHECK(!inprogress_cache_.get());
1131         AppCache* cache = group_->newest_complete_cache();
1132         PendingMasters::iterator found = pending_master_entries_.find(url);
1133         DCHECK(found != pending_master_entries_.end());
1134         PendingHosts& hosts = found->second;
1135         for (PendingHosts::iterator host_it = hosts.begin();
1136              host_it != hosts.end(); ++host_it) {
1137           (*host_it)->AssociateCompleteCache(cache);
1138         }
1139       }
1140     } else {
1141       URLFetcher* fetcher = new URLFetcher(
1142           url, URLFetcher::MASTER_ENTRY_FETCH, this);
1143       fetcher->Start();
1144       master_entry_fetches_.insert(PendingUrlFetches::value_type(url, fetcher));
1145     }
1146
1147     master_entries_to_fetch_.erase(master_entries_to_fetch_.begin());
1148   }
1149 }
1150
1151 void AppCacheUpdateJob::CancelAllMasterEntryFetches(
1152     const std::string& error_message) {
1153   // For now, cancel all in-progress fetches for master entries and pretend
1154   // all master entries fetches have completed.
1155   // TODO(jennb): Delete this when update no longer fetches master entries
1156   // directly.
1157
1158   // Cancel all in-progress fetches.
1159   for (PendingUrlFetches::iterator it = master_entry_fetches_.begin();
1160        it != master_entry_fetches_.end(); ++it) {
1161     delete it->second;
1162     master_entries_to_fetch_.insert(it->first);  // back in unfetched list
1163   }
1164   master_entry_fetches_.clear();
1165
1166   master_entries_completed_ += master_entries_to_fetch_.size();
1167
1168   // Cache failure steps, step 2.
1169   // Pretend all master entries that have not yet been fetched have completed
1170   // downloading. Unassociate hosts from any appcache and send ERROR event.
1171   HostNotifier host_notifier;
1172   while (!master_entries_to_fetch_.empty()) {
1173     const GURL& url = *master_entries_to_fetch_.begin();
1174     PendingMasters::iterator found = pending_master_entries_.find(url);
1175     DCHECK(found != pending_master_entries_.end());
1176     PendingHosts& hosts = found->second;
1177     for (PendingHosts::iterator host_it = hosts.begin();
1178          host_it != hosts.end(); ++host_it) {
1179       AppCacheHost* host = *host_it;
1180       host->AssociateNoCache(GURL());
1181       host_notifier.AddHost(host);
1182       host->RemoveObserver(this);
1183     }
1184     hosts.clear();
1185
1186     master_entries_to_fetch_.erase(master_entries_to_fetch_.begin());
1187   }
1188   host_notifier.SendErrorNotifications(error_message);
1189 }
1190
1191 bool AppCacheUpdateJob::MaybeLoadFromNewestCache(const GURL& url,
1192                                                  AppCacheEntry& entry) {
1193   if (update_type_ != UPGRADE_ATTEMPT)
1194     return false;
1195
1196   AppCache* newest = group_->newest_complete_cache();
1197   AppCacheEntry* copy_me = newest->GetEntry(url);
1198   if (!copy_me || !copy_me->has_response_id())
1199     return false;
1200
1201   // Load HTTP headers for entry from newest cache.
1202   loading_responses_.insert(
1203       LoadingResponses::value_type(copy_me->response_id(), url));
1204   storage_->LoadResponseInfo(manifest_url_, group_->group_id(),
1205                              copy_me->response_id(),
1206                              this);
1207   // Async: wait for OnResponseInfoLoaded to complete.
1208   return true;
1209 }
1210
1211 void AppCacheUpdateJob::OnResponseInfoLoaded(
1212     AppCacheResponseInfo* response_info, int64 response_id) {
1213   const net::HttpResponseInfo* http_info = response_info ?
1214       response_info->http_response_info() : NULL;
1215
1216   // Needed response info for a manifest fetch request.
1217   if (internal_state_ == FETCH_MANIFEST) {
1218     if (http_info)
1219       manifest_fetcher_->set_existing_response_headers(
1220           http_info->headers.get());
1221     manifest_fetcher_->Start();
1222     return;
1223   }
1224
1225   LoadingResponses::iterator found = loading_responses_.find(response_id);
1226   DCHECK(found != loading_responses_.end());
1227   const GURL& url = found->second;
1228
1229   if (!http_info) {
1230     LoadFromNewestCacheFailed(url, NULL);  // no response found
1231   } else {
1232     // Check if response can be re-used according to HTTP caching semantics.
1233     // Responses with a "vary" header get treated as expired.
1234     const std::string name = "vary";
1235     std::string value;
1236     void* iter = NULL;
1237     if (!http_info->headers.get() ||
1238         http_info->headers->RequiresValidation(http_info->request_time,
1239                                                http_info->response_time,
1240                                                base::Time::Now()) ||
1241         http_info->headers->EnumerateHeader(&iter, name, &value)) {
1242       LoadFromNewestCacheFailed(url, response_info);
1243     } else {
1244       DCHECK(group_->newest_complete_cache());
1245       AppCacheEntry* copy_me = group_->newest_complete_cache()->GetEntry(url);
1246       DCHECK(copy_me);
1247       DCHECK(copy_me->response_id() == response_id);
1248
1249       AppCache::EntryMap::iterator it = url_file_list_.find(url);
1250       DCHECK(it != url_file_list_.end());
1251       AppCacheEntry& entry = it->second;
1252       entry.set_response_id(response_id);
1253       entry.set_response_size(copy_me->response_size());
1254       inprogress_cache_->AddOrModifyEntry(url, entry);
1255       NotifyAllProgress(url);
1256       ++url_fetches_completed_;
1257     }
1258   }
1259   loading_responses_.erase(found);
1260
1261   MaybeCompleteUpdate();
1262 }
1263
1264 void AppCacheUpdateJob::LoadFromNewestCacheFailed(
1265     const GURL& url, AppCacheResponseInfo* response_info) {
1266   if (internal_state_ == CACHE_FAILURE)
1267     return;
1268
1269   // Re-insert url at front of fetch list. Indicate storage has been checked.
1270   urls_to_fetch_.push_front(UrlToFetch(url, true, response_info));
1271   FetchUrls();
1272 }
1273
1274 void AppCacheUpdateJob::MaybeCompleteUpdate() {
1275   DCHECK(internal_state_ != CACHE_FAILURE);
1276
1277   // Must wait for any pending master entries or url fetches to complete.
1278   if (master_entries_completed_ != pending_master_entries_.size() ||
1279       url_fetches_completed_ != url_file_list_.size()) {
1280     DCHECK(internal_state_ != COMPLETED);
1281     return;
1282   }
1283
1284   switch (internal_state_) {
1285     case NO_UPDATE:
1286       if (master_entries_completed_ > 0) {
1287         switch (stored_state_) {
1288           case UNSTORED:
1289             StoreGroupAndCache();
1290             return;
1291           case STORING:
1292             return;
1293           case STORED:
1294             break;
1295         }
1296       }
1297       // 6.9.4 steps 7.3-7.7.
1298       NotifyAllAssociatedHosts(NO_UPDATE_EVENT);
1299       DiscardDuplicateResponses();
1300       internal_state_ = COMPLETED;
1301       break;
1302     case DOWNLOADING:
1303       internal_state_ = REFETCH_MANIFEST;
1304       FetchManifest(false);
1305       break;
1306     case REFETCH_MANIFEST:
1307       DCHECK(stored_state_ == STORED);
1308       NotifyAllFinalProgress();
1309       if (update_type_ == CACHE_ATTEMPT)
1310         NotifyAllAssociatedHosts(CACHED_EVENT);
1311       else
1312         NotifyAllAssociatedHosts(UPDATE_READY_EVENT);
1313       DiscardDuplicateResponses();
1314       internal_state_ = COMPLETED;
1315       AppCacheHistograms::CountUpdateJobResult(
1316           UPDATE_OK, manifest_url_.GetOrigin());
1317       break;
1318     case CACHE_FAILURE:
1319       NOTREACHED();  // See HandleCacheFailure
1320       break;
1321     default:
1322       break;
1323   }
1324
1325   // Let the stack unwind before deletion to make it less risky as this
1326   // method is called from multiple places in this file.
1327   if (internal_state_ == COMPLETED)
1328     DeleteSoon();
1329 }
1330
1331 void AppCacheUpdateJob::ScheduleUpdateRetry(int delay_ms) {
1332   // TODO(jennb): post a delayed task with the "same parameters" as this job
1333   // to retry the update at a later time. Need group, URLs of pending master
1334   // entries and their hosts.
1335 }
1336
1337 void AppCacheUpdateJob::Cancel() {
1338   internal_state_ = CANCELLED;
1339
1340   if (manifest_fetcher_) {
1341     delete manifest_fetcher_;
1342     manifest_fetcher_ = NULL;
1343   }
1344
1345   for (PendingUrlFetches::iterator it = pending_url_fetches_.begin();
1346        it != pending_url_fetches_.end(); ++it) {
1347     delete it->second;
1348   }
1349   pending_url_fetches_.clear();
1350
1351   for (PendingUrlFetches::iterator it = master_entry_fetches_.begin();
1352        it != master_entry_fetches_.end(); ++it) {
1353     delete it->second;
1354   }
1355   master_entry_fetches_.clear();
1356
1357   ClearPendingMasterEntries();
1358   DiscardInprogressCache();
1359
1360   // Delete response writer to avoid any callbacks.
1361   if (manifest_response_writer_)
1362     manifest_response_writer_.reset();
1363
1364   storage_->CancelDelegateCallbacks(this);
1365 }
1366
1367 void AppCacheUpdateJob::ClearPendingMasterEntries() {
1368   for (PendingMasters::iterator it = pending_master_entries_.begin();
1369        it != pending_master_entries_.end(); ++it) {
1370     PendingHosts& hosts = it->second;
1371     for (PendingHosts::iterator host_it = hosts.begin();
1372          host_it != hosts.end(); ++host_it) {
1373       (*host_it)->RemoveObserver(this);
1374     }
1375   }
1376
1377   pending_master_entries_.clear();
1378 }
1379
1380 void AppCacheUpdateJob::DiscardInprogressCache() {
1381   if (stored_state_ == STORING) {
1382     // We can make no assumptions about whether the StoreGroupAndCacheTask
1383     // actually completed or not. This condition should only be reachable
1384     // during shutdown. Free things up and return to do no harm.
1385     inprogress_cache_ = NULL;
1386     added_master_entries_.clear();
1387     return;
1388   }
1389
1390   storage_->DoomResponses(manifest_url_, stored_response_ids_);
1391
1392   if (!inprogress_cache_.get()) {
1393     // We have to undo the changes we made, if any, to the existing cache.
1394     if (group_ && group_->newest_complete_cache()) {
1395       for (std::vector<GURL>::iterator iter = added_master_entries_.begin();
1396            iter != added_master_entries_.end(); ++iter) {
1397         group_->newest_complete_cache()->RemoveEntry(*iter);
1398       }
1399     }
1400     added_master_entries_.clear();
1401     return;
1402   }
1403
1404   AppCache::AppCacheHosts& hosts = inprogress_cache_->associated_hosts();
1405   while (!hosts.empty())
1406     (*hosts.begin())->AssociateNoCache(GURL());
1407
1408   inprogress_cache_ = NULL;
1409   added_master_entries_.clear();
1410 }
1411
1412 void AppCacheUpdateJob::DiscardDuplicateResponses() {
1413   storage_->DoomResponses(manifest_url_, duplicate_response_ids_);
1414 }
1415
1416 void AppCacheUpdateJob::DeleteSoon() {
1417   ClearPendingMasterEntries();
1418   manifest_response_writer_.reset();
1419   storage_->CancelDelegateCallbacks(this);
1420   service_->RemoveObserver(this);
1421   service_ = NULL;
1422
1423   // Break the connection with the group so the group cannot call delete
1424   // on this object after we've posted a task to delete ourselves.
1425   group_->SetUpdateStatus(AppCacheGroup::IDLE);
1426   group_ = NULL;
1427
1428   base::MessageLoop::current()->DeleteSoon(FROM_HERE, this);
1429 }
1430
1431 }  // namespace appcache