src/webkit/browser/appcache/appcache_update_job.cc

   1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "webkit/browser/appcache/appcache_update_job.h"
   6
   7 #include "base/bind.h"
   8 #include "base/bind_helpers.h"
   9 #include "base/compiler_specific.h"
  10 #include "base/message_loop/message_loop.h"
  11 #include "base/strings/string_util.h"
  12 #include "base/strings/stringprintf.h"
  13 #include "net/base/io_buffer.h"
  14 #include "net/base/load_flags.h"
  15 #include "net/base/net_errors.h"
  16 #include "net/base/request_priority.h"
  17 #include "net/http/http_request_headers.h"
  18 #include "net/http/http_response_headers.h"
  19 #include "net/url_request/url_request_context.h"
  20 #include "webkit/browser/appcache/appcache_group.h"
  21 #include "webkit/browser/appcache/appcache_histograms.h"
  22
  23 namespace appcache {
  24
  25 static const int kBufferSize = 32768;
  26 static const size_t kMaxConcurrentUrlFetches = 2;
  27 static const int kMax503Retries = 3;
  28
  29 static std::string FormatUrlErrorMessage(
  30       const char* format, const GURL& url,
  31       AppCacheUpdateJob::ResultType error,
  32       int response_code) {
  33     // Show the net response code if we have one.
  34     int code = response_code;
  35     if (error != AppCacheUpdateJob::SERVER_ERROR)
  36       code = static_cast<int>(error);
  37     return base::StringPrintf(format, code, url.spec().c_str());
  38 }
  39
  40 // Helper class for collecting hosts per frontend when sending notifications
  41 // so that only one notification is sent for all hosts using the same frontend.
  42 class HostNotifier {
  43  public:
  44   typedef std::vector<int> HostIds;
  45   typedef std::map<AppCacheFrontend*, HostIds> NotifyHostMap;
  46
  47   // Caller is responsible for ensuring there will be no duplicate hosts.
  48   void AddHost(AppCacheHost* host) {
  49     std::pair<NotifyHostMap::iterator , bool> ret = hosts_to_notify.insert(
  50         NotifyHostMap::value_type(host->frontend(), HostIds()));
  51     ret.first->second.push_back(host->host_id());
  52   }
  53
  54   void AddHosts(const std::set<AppCacheHost*>& hosts) {
  55     for (std::set<AppCacheHost*>::const_iterator it = hosts.begin();
  56          it != hosts.end(); ++it) {
  57       AddHost(*it);
  58     }
  59   }
  60
  61   void SendNotifications(AppCacheEventID event_id) {
  62     for (NotifyHostMap::iterator it = hosts_to_notify.begin();
  63          it != hosts_to_notify.end(); ++it) {
  64       AppCacheFrontend* frontend = it->first;
  65       frontend->OnEventRaised(it->second, event_id);
  66     }
  67   }
  68
  69   void SendProgressNotifications(
  70       const GURL& url, int num_total, int num_complete) {
  71     for (NotifyHostMap::iterator it = hosts_to_notify.begin();
  72          it != hosts_to_notify.end(); ++it) {
  73       AppCacheFrontend* frontend = it->first;
  74       frontend->OnProgressEventRaised(it->second, url,
  75                                       num_total, num_complete);
  76     }
  77   }
  78
  79   void SendErrorNotifications(const AppCacheErrorDetails& details) {
  80     DCHECK(!details.message.empty());
  81     for (NotifyHostMap::iterator it = hosts_to_notify.begin();
  82          it != hosts_to_notify.end(); ++it) {
  83       AppCacheFrontend* frontend = it->first;
  84       frontend->OnErrorEventRaised(it->second, details);
  85     }
  86   }
  87
  88   void SendLogMessage(const std::string& message) {
  89     for (NotifyHostMap::iterator it = hosts_to_notify.begin();
  90          it != hosts_to_notify.end(); ++it) {
  91       AppCacheFrontend* frontend = it->first;
  92       for (HostIds::iterator id = it->second.begin();
  93            id != it->second.end(); ++id) {
  94         frontend->OnLogMessage(*id, APPCACHE_LOG_WARNING, message);
  95       }
  96     }
  97   }
  98
  99  private:
 100   NotifyHostMap hosts_to_notify;
 101 };
 102
 103 AppCacheUpdateJob::UrlToFetch::UrlToFetch(const GURL& url,
 104                                           bool checked,
 105                                           AppCacheResponseInfo* info)
 106     : url(url),
 107       storage_checked(checked),
 108       existing_response_info(info) {
 109 }
 110
 111 AppCacheUpdateJob::UrlToFetch::~UrlToFetch() {
 112 }
 113
 114 // Helper class to fetch resources. Depending on the fetch type,
 115 // can either fetch to an in-memory string or write the response
 116 // data out to the disk cache.
 117 AppCacheUpdateJob::URLFetcher::URLFetcher(const GURL& url,
 118                                           FetchType fetch_type,
 119                                           AppCacheUpdateJob* job)
 120     : url_(url),
 121       job_(job),
 122       fetch_type_(fetch_type),
 123       retry_503_attempts_(0),
 124       buffer_(new net::IOBuffer(kBufferSize)),
 125       request_(job->service_->request_context()
 126                    ->CreateRequest(url, net::DEFAULT_PRIORITY, this, NULL)),
 127       result_(UPDATE_OK),
 128       redirect_response_code_(-1) {}
 129
 130 AppCacheUpdateJob::URLFetcher::~URLFetcher() {
 131 }
 132
 133 void AppCacheUpdateJob::URLFetcher::Start() {
 134   request_->set_first_party_for_cookies(job_->manifest_url_);
 135   request_->SetLoadFlags(request_->load_flags() |
 136                          net::LOAD_DISABLE_INTERCEPT);
 137   if (existing_response_headers_.get())
 138     AddConditionalHeaders(existing_response_headers_.get());
 139   request_->Start();
 140 }
 141
 142 void AppCacheUpdateJob::URLFetcher::OnReceivedRedirect(
 143     net::URLRequest* request, const GURL& new_url,  bool* defer_redirect) {
 144   DCHECK(request_ == request);
 145   // Redirect is not allowed by the update process.
 146   job_->MadeProgress();
 147   redirect_response_code_ = request->GetResponseCode();
 148   request->Cancel();
 149   result_ = REDIRECT_ERROR;
 150   OnResponseCompleted();
 151 }
 152
 153 void AppCacheUpdateJob::URLFetcher::OnResponseStarted(
 154     net::URLRequest *request) {
 155   DCHECK(request == request_);
 156   int response_code = -1;
 157   if (request->status().is_success()) {
 158     response_code = request->GetResponseCode();
 159     job_->MadeProgress();
 160   }
 161   if ((response_code / 100) == 2) {
 162
 163     // See http://code.google.com/p/chromium/issues/detail?id=69594
 164     // We willfully violate the HTML5 spec at this point in order
 165     // to support the appcaching of cross-origin HTTPS resources.
 166     // We've opted for a milder constraint and allow caching unless
 167     // the resource has a "no-store" header. A spec change has been
 168     // requested on the whatwg list.
 169     // TODO(michaeln): Consider doing this for cross-origin HTTP resources too.
 170     if (url_.SchemeIsSecure() &&
 171         url_.GetOrigin() != job_->manifest_url_.GetOrigin()) {
 172       if (request->response_headers()->
 173               HasHeaderValue("cache-control", "no-store")) {
 174         DCHECK_EQ(-1, redirect_response_code_);
 175         request->Cancel();
 176         result_ = SERVER_ERROR;  // Not the best match?
 177         OnResponseCompleted();
 178         return;
 179       }
 180     }
 181
 182     // Write response info to storage for URL fetches. Wait for async write
 183     // completion before reading any response data.
 184     if (fetch_type_ == URL_FETCH || fetch_type_ == MASTER_ENTRY_FETCH) {
 185       response_writer_.reset(job_->CreateResponseWriter());
 186       scoped_refptr<HttpResponseInfoIOBuffer> io_buffer(
 187           new HttpResponseInfoIOBuffer(
 188               new net::HttpResponseInfo(request->response_info())));
 189       response_writer_->WriteInfo(
 190           io_buffer.get(),
 191           base::Bind(&URLFetcher::OnWriteComplete, base::Unretained(this)));
 192     } else {
 193       ReadResponseData();
 194     }
 195   } else {
 196     if (response_code > 0)
 197       result_ = SERVER_ERROR;
 198     else
 199       result_ = NETWORK_ERROR;
 200     OnResponseCompleted();
 201   }
 202 }
 203
 204 void AppCacheUpdateJob::URLFetcher::OnReadCompleted(
 205     net::URLRequest* request, int bytes_read) {
 206   DCHECK(request_ == request);
 207   bool data_consumed = true;
 208   if (request->status().is_success() && bytes_read > 0) {
 209     job_->MadeProgress();
 210     data_consumed = ConsumeResponseData(bytes_read);
 211     if (data_consumed) {
 212       bytes_read = 0;
 213       while (request->Read(buffer_.get(), kBufferSize, &bytes_read)) {
 214         if (bytes_read > 0) {
 215           data_consumed = ConsumeResponseData(bytes_read);
 216           if (!data_consumed)
 217             break;  // wait for async data processing, then read more
 218         } else {
 219           break;
 220         }
 221       }
 222     }
 223   }
 224   if (data_consumed && !request->status().is_io_pending()) {
 225     DCHECK_EQ(UPDATE_OK, result_);
 226     OnResponseCompleted();
 227   }
 228 }
 229
 230 void AppCacheUpdateJob::URLFetcher::AddConditionalHeaders(
 231     const net::HttpResponseHeaders* headers) {
 232   DCHECK(request_.get() && headers);
 233   net::HttpRequestHeaders extra_headers;
 234
 235   // Add If-Modified-Since header if response info has Last-Modified header.
 236   const std::string last_modified = "Last-Modified";
 237   std::string last_modified_value;
 238   headers->EnumerateHeader(NULL, last_modified, &last_modified_value);
 239   if (!last_modified_value.empty()) {
 240     extra_headers.SetHeader(net::HttpRequestHeaders::kIfModifiedSince,
 241                             last_modified_value);
 242   }
 243
 244   // Add If-None-Match header if response info has ETag header.
 245   const std::string etag = "ETag";
 246   std::string etag_value;
 247   headers->EnumerateHeader(NULL, etag, &etag_value);
 248   if (!etag_value.empty()) {
 249     extra_headers.SetHeader(net::HttpRequestHeaders::kIfNoneMatch,
 250                             etag_value);
 251   }
 252   if (!extra_headers.IsEmpty())
 253     request_->SetExtraRequestHeaders(extra_headers);
 254 }
 255
 256 void  AppCacheUpdateJob::URLFetcher::OnWriteComplete(int result) {
 257   if (result < 0) {
 258     request_->Cancel();
 259     result_ = DISKCACHE_ERROR;
 260     OnResponseCompleted();
 261     return;
 262   }
 263   ReadResponseData();
 264 }
 265
 266 void AppCacheUpdateJob::URLFetcher::ReadResponseData() {
 267   InternalUpdateState state = job_->internal_state_;
 268   if (state == CACHE_FAILURE || state == CANCELLED || state == COMPLETED)
 269     return;
 270   int bytes_read = 0;
 271   request_->Read(buffer_.get(), kBufferSize, &bytes_read);
 272   OnReadCompleted(request_.get(), bytes_read);
 273 }
 274
 275 // Returns false if response data is processed asynchronously, in which
 276 // case ReadResponseData will be invoked when it is safe to continue
 277 // reading more response data from the request.
 278 bool AppCacheUpdateJob::URLFetcher::ConsumeResponseData(int bytes_read) {
 279   DCHECK_GT(bytes_read, 0);
 280   switch (fetch_type_) {
 281     case MANIFEST_FETCH:
 282     case MANIFEST_REFETCH:
 283       manifest_data_.append(buffer_->data(), bytes_read);
 284       break;
 285     case URL_FETCH:
 286     case MASTER_ENTRY_FETCH:
 287       DCHECK(response_writer_.get());
 288       response_writer_->WriteData(
 289           buffer_.get(),
 290           bytes_read,
 291           base::Bind(&URLFetcher::OnWriteComplete, base::Unretained(this)));
 292       return false;  // wait for async write completion to continue reading
 293     default:
 294       NOTREACHED();
 295   }
 296   return true;
 297 }
 298
 299 void AppCacheUpdateJob::URLFetcher::OnResponseCompleted() {
 300   if (request_->status().is_success())
 301     job_->MadeProgress();
 302
 303   // Retry for 503s where retry-after is 0.
 304   if (request_->status().is_success() &&
 305       request_->GetResponseCode() == 503 &&
 306       MaybeRetryRequest()) {
 307     return;
 308   }
 309
 310   switch (fetch_type_) {
 311     case MANIFEST_FETCH:
 312       job_->HandleManifestFetchCompleted(this);
 313       break;
 314     case URL_FETCH:
 315       job_->HandleUrlFetchCompleted(this);
 316       break;
 317     case MASTER_ENTRY_FETCH:
 318       job_->HandleMasterEntryFetchCompleted(this);
 319       break;
 320     case MANIFEST_REFETCH:
 321       job_->HandleManifestRefetchCompleted(this);
 322       break;
 323     default:
 324       NOTREACHED();
 325   }
 326
 327   delete this;
 328 }
 329
 330 bool AppCacheUpdateJob::URLFetcher::MaybeRetryRequest() {
 331   if (retry_503_attempts_ >= kMax503Retries ||
 332       !request_->response_headers()->HasHeaderValue("retry-after", "0")) {
 333     return false;
 334   }
 335   ++retry_503_attempts_;
 336   result_ = UPDATE_OK;
 337   request_ = job_->service_->request_context()->CreateRequest(
 338       url_, net::DEFAULT_PRIORITY, this, NULL);
 339   Start();
 340   return true;
 341 }
 342
 343 AppCacheUpdateJob::AppCacheUpdateJob(AppCacheServiceImpl* service,
 344                                      AppCacheGroup* group)
 345     : service_(service),
 346       manifest_url_(group->manifest_url()),
 347       group_(group),
 348       update_type_(UNKNOWN_TYPE),
 349       internal_state_(FETCH_MANIFEST),
 350       master_entries_completed_(0),
 351       url_fetches_completed_(0),
 352       manifest_fetcher_(NULL),
 353       manifest_has_valid_mime_type_(false),
 354       stored_state_(UNSTORED),
 355       storage_(service->storage()) {
 356     service_->AddObserver(this);
 357 }
 358
 359 AppCacheUpdateJob::~AppCacheUpdateJob() {
 360   if (service_)
 361     service_->RemoveObserver(this);
 362   if (internal_state_ != COMPLETED)
 363     Cancel();
 364
 365   DCHECK(!manifest_fetcher_);
 366   DCHECK(pending_url_fetches_.empty());
 367   DCHECK(!inprogress_cache_.get());
 368   DCHECK(pending_master_entries_.empty());
 369   DCHECK(master_entry_fetches_.empty());
 370
 371   if (group_)
 372     group_->SetUpdateAppCacheStatus(AppCacheGroup::IDLE);
 373 }
 374
 375 void AppCacheUpdateJob::StartUpdate(AppCacheHost* host,
 376                                     const GURL& new_master_resource) {
 377   DCHECK(group_->update_job() == this);
 378   DCHECK(!group_->is_obsolete());
 379
 380   bool is_new_pending_master_entry = false;
 381   if (!new_master_resource.is_empty()) {
 382     DCHECK(new_master_resource == host->pending_master_entry_url());
 383     DCHECK(!new_master_resource.has_ref());
 384     DCHECK(new_master_resource.GetOrigin() == manifest_url_.GetOrigin());
 385
 386     // Cannot add more to this update if already terminating.
 387     if (IsTerminating()) {
 388       group_->QueueUpdate(host, new_master_resource);
 389       return;
 390     }
 391
 392     std::pair<PendingMasters::iterator, bool> ret =
 393         pending_master_entries_.insert(
 394             PendingMasters::value_type(new_master_resource, PendingHosts()));
 395     is_new_pending_master_entry = ret.second;
 396     ret.first->second.push_back(host);
 397     host->AddObserver(this);
 398   }
 399
 400   // Notify host (if any) if already checking or downloading.
 401   AppCacheGroup::UpdateAppCacheStatus update_status = group_->update_status();
 402   if (update_status == AppCacheGroup::CHECKING ||
 403       update_status == AppCacheGroup::DOWNLOADING) {
 404     if (host) {
 405       NotifySingleHost(host, APPCACHE_CHECKING_EVENT);
 406       if (update_status == AppCacheGroup::DOWNLOADING)
 407         NotifySingleHost(host, APPCACHE_DOWNLOADING_EVENT);
 408
 409       // Add to fetch list or an existing entry if already fetched.
 410       if (!new_master_resource.is_empty()) {
 411         AddMasterEntryToFetchList(host, new_master_resource,
 412                                   is_new_pending_master_entry);
 413       }
 414     }
 415     return;
 416   }
 417
 418   // Begin update process for the group.
 419   MadeProgress();
 420   group_->SetUpdateAppCacheStatus(AppCacheGroup::CHECKING);
 421   if (group_->HasCache()) {
 422     update_type_ = UPGRADE_ATTEMPT;
 423     NotifyAllAssociatedHosts(APPCACHE_CHECKING_EVENT);
 424   } else {
 425     update_type_ = CACHE_ATTEMPT;
 426     DCHECK(host);
 427     NotifySingleHost(host, APPCACHE_CHECKING_EVENT);
 428   }
 429
 430   if (!new_master_resource.is_empty()) {
 431     AddMasterEntryToFetchList(host, new_master_resource,
 432                               is_new_pending_master_entry);
 433   }
 434
 435   FetchManifest(true);
 436 }
 437
 438 AppCacheResponseWriter* AppCacheUpdateJob::CreateResponseWriter() {
 439   AppCacheResponseWriter* writer =
 440       storage_->CreateResponseWriter(manifest_url_,
 441                                                 group_->group_id());
 442   stored_response_ids_.push_back(writer->response_id());
 443   return writer;
 444 }
 445
 446 void AppCacheUpdateJob::HandleCacheFailure(
 447     const AppCacheErrorDetails& error_details,
 448     ResultType result,
 449     const GURL& failed_resource_url) {
 450   // 6.9.4 cache failure steps 2-8.
 451   DCHECK(internal_state_ != CACHE_FAILURE);
 452   DCHECK(!error_details.message.empty());
 453   DCHECK(result != UPDATE_OK);
 454   internal_state_ = CACHE_FAILURE;
 455   LogHistogramStats(result, failed_resource_url);
 456   CancelAllUrlFetches();
 457   CancelAllMasterEntryFetches(error_details);
 458   NotifyAllError(error_details);
 459   DiscardInprogressCache();
 460   internal_state_ = COMPLETED;
 461   DeleteSoon();  // To unwind the stack prior to deletion.
 462 }
 463
 464 void AppCacheUpdateJob::FetchManifest(bool is_first_fetch) {
 465   DCHECK(!manifest_fetcher_);
 466   manifest_fetcher_ = new URLFetcher(
 467      manifest_url_,
 468      is_first_fetch ? URLFetcher::MANIFEST_FETCH :
 469                       URLFetcher::MANIFEST_REFETCH,
 470      this);
 471
 472   // Add any necessary Http headers before sending fetch request.
 473   if (is_first_fetch) {
 474     AppCacheEntry* entry = (update_type_ == UPGRADE_ATTEMPT) ?
 475         group_->newest_complete_cache()->GetEntry(manifest_url_) : NULL;
 476     if (entry) {
 477       // Asynchronously load response info for manifest from newest cache.
 478       storage_->LoadResponseInfo(manifest_url_, group_->group_id(),
 479                                  entry->response_id(), this);
 480     } else {
 481       manifest_fetcher_->Start();
 482     }
 483   } else {
 484     DCHECK(internal_state_ == REFETCH_MANIFEST);
 485     DCHECK(manifest_response_info_.get());
 486     manifest_fetcher_->set_existing_response_headers(
 487         manifest_response_info_->headers.get());
 488     manifest_fetcher_->Start();
 489   }
 490 }
 491
 492
 493 void AppCacheUpdateJob::HandleManifestFetchCompleted(
 494     URLFetcher* fetcher) {
 495   DCHECK_EQ(internal_state_, FETCH_MANIFEST);
 496   DCHECK_EQ(manifest_fetcher_, fetcher);
 497   manifest_fetcher_ = NULL;
 498
 499   net::URLRequest* request = fetcher->request();
 500   int response_code = -1;
 501   bool is_valid_response_code = false;
 502   if (request->status().is_success()) {
 503     response_code = request->GetResponseCode();
 504     is_valid_response_code = (response_code / 100 == 2);
 505
 506     std::string mime_type;
 507     request->GetMimeType(&mime_type);
 508     manifest_has_valid_mime_type_ = (mime_type == "text/cache-manifest");
 509   }
 510
 511   if (is_valid_response_code) {
 512     manifest_data_ = fetcher->manifest_data();
 513     manifest_response_info_.reset(
 514         new net::HttpResponseInfo(request->response_info()));
 515     if (update_type_ == UPGRADE_ATTEMPT)
 516       CheckIfManifestChanged();  // continues asynchronously
 517     else
 518       ContinueHandleManifestFetchCompleted(true);
 519   } else if (response_code == 304 && update_type_ == UPGRADE_ATTEMPT) {
 520     ContinueHandleManifestFetchCompleted(false);
 521   } else if ((response_code == 404 || response_code == 410) &&
 522              update_type_ == UPGRADE_ATTEMPT) {
 523     storage_->MakeGroupObsolete(group_, this, response_code);  // async
 524   } else {
 525     const char* kFormatString = "Manifest fetch failed (%d) %s";
 526     std::string message = FormatUrlErrorMessage(
 527         kFormatString, manifest_url_, fetcher->result(), response_code);
 528     HandleCacheFailure(AppCacheErrorDetails(message,
 529                                     appcache::APPCACHE_MANIFEST_ERROR,
 530                                     manifest_url_,
 531                                     response_code,
 532                                     false /*is_cross_origin*/),
 533                        fetcher->result(),
 534                        GURL());
 535   }
 536 }
 537
 538 void AppCacheUpdateJob::OnGroupMadeObsolete(AppCacheGroup* group,
 539                                             bool success,
 540                                             int response_code) {
 541   DCHECK(master_entry_fetches_.empty());
 542   CancelAllMasterEntryFetches(AppCacheErrorDetails(
 543       "The cache has been made obsolete, "
 544       "the manifest file returned 404 or 410",
 545       appcache::APPCACHE_MANIFEST_ERROR,
 546       GURL(),
 547       response_code,
 548       false /*is_cross_origin*/));
 549   if (success) {
 550     DCHECK(group->is_obsolete());
 551     NotifyAllAssociatedHosts(APPCACHE_OBSOLETE_EVENT);
 552     internal_state_ = COMPLETED;
 553     MaybeCompleteUpdate();
 554   } else {
 555     // Treat failure to mark group obsolete as a cache failure.
 556     HandleCacheFailure(AppCacheErrorDetails(
 557         "Failed to mark the cache as obsolete",
 558         APPCACHE_UNKNOWN_ERROR,
 559         GURL(),
 560         0,
 561         false /*is_cross_origin*/),
 562                        DB_ERROR,
 563                        GURL());
 564   }
 565 }
 566
 567 void AppCacheUpdateJob::ContinueHandleManifestFetchCompleted(bool changed) {
 568   DCHECK(internal_state_ == FETCH_MANIFEST);
 569
 570   if (!changed) {
 571     DCHECK(update_type_ == UPGRADE_ATTEMPT);
 572     internal_state_ = NO_UPDATE;
 573
 574     // Wait for pending master entries to download.
 575     FetchMasterEntries();
 576     MaybeCompleteUpdate();  // if not done, run async 6.9.4 step 7 substeps
 577     return;
 578   }
 579
 580   Manifest manifest;
 581   if (!ParseManifest(manifest_url_, manifest_data_.data(),
 582                      manifest_data_.length(),
 583                      manifest_has_valid_mime_type_ ?
 584                         PARSE_MANIFEST_ALLOWING_INTERCEPTS :
 585                         PARSE_MANIFEST_PER_STANDARD,
 586                      manifest)) {
 587     const char* kFormatString = "Failed to parse manifest %s";
 588     const std::string message = base::StringPrintf(kFormatString,
 589         manifest_url_.spec().c_str());
 590     HandleCacheFailure(
 591         AppCacheErrorDetails(
 592             message, APPCACHE_SIGNATURE_ERROR, GURL(), 0,
 593             false /*is_cross_origin*/),
 594         APPCACHE_MANIFEST_ERROR,
 595         GURL());
 596     VLOG(1) << message;
 597     return;
 598   }
 599
 600   // Proceed with update process. Section 6.9.4 steps 8-20.
 601   internal_state_ = DOWNLOADING;
 602   inprogress_cache_ = new AppCache(storage_, storage_->NewCacheId());
 603   BuildUrlFileList(manifest);
 604   inprogress_cache_->InitializeWithManifest(&manifest);
 605
 606   // Associate all pending master hosts with the newly created cache.
 607   for (PendingMasters::iterator it = pending_master_entries_.begin();
 608        it != pending_master_entries_.end(); ++it) {
 609     PendingHosts& hosts = it->second;
 610     for (PendingHosts::iterator host_it = hosts.begin();
 611          host_it != hosts.end(); ++host_it) {
 612       (*host_it)
 613           ->AssociateIncompleteCache(inprogress_cache_.get(), manifest_url_);
 614     }
 615   }
 616
 617   if (manifest.did_ignore_intercept_namespaces) {
 618     // Must be done after associating all pending master hosts.
 619     std::string message(
 620         "Ignoring the INTERCEPT section of the application cache manifest "
 621         "because the content type is not text/cache-manifest");
 622     LogConsoleMessageToAll(message);
 623   }
 624
 625   group_->SetUpdateAppCacheStatus(AppCacheGroup::DOWNLOADING);
 626   NotifyAllAssociatedHosts(APPCACHE_DOWNLOADING_EVENT);
 627   FetchUrls();
 628   FetchMasterEntries();
 629   MaybeCompleteUpdate();  // if not done, continues when async fetches complete
 630 }
 631
 632 void AppCacheUpdateJob::HandleUrlFetchCompleted(URLFetcher* fetcher) {
 633   DCHECK(internal_state_ == DOWNLOADING);
 634
 635   net::URLRequest* request = fetcher->request();
 636   const GURL& url = request->original_url();
 637   pending_url_fetches_.erase(url);
 638   NotifyAllProgress(url);
 639   ++url_fetches_completed_;
 640
 641   int response_code = request->status().is_success()
 642                           ? request->GetResponseCode()
 643                           : fetcher->redirect_response_code();
 644
 645   AppCacheEntry& entry = url_file_list_.find(url)->second;
 646
 647   if (response_code / 100 == 2) {
 648     // Associate storage with the new entry.
 649     DCHECK(fetcher->response_writer());
 650     entry.set_response_id(fetcher->response_writer()->response_id());
 651     entry.set_response_size(fetcher->response_writer()->amount_written());
 652     if (!inprogress_cache_->AddOrModifyEntry(url, entry))
 653       duplicate_response_ids_.push_back(entry.response_id());
 654
 655     // TODO(michaeln): Check for <html manifest=xxx>
 656     // See http://code.google.com/p/chromium/issues/detail?id=97930
 657     // if (entry.IsMaster() && !(entry.IsExplicit() || fallback || intercept))
 658     //   if (!manifestAttribute) skip it
 659
 660     // Foreign entries will be detected during cache selection.
 661     // Note: 6.9.4, step 17.9 possible optimization: if resource is HTML or XML
 662     // file whose root element is an html element with a manifest attribute
 663     // whose value doesn't match the manifest url of the application cache
 664     // being processed, mark the entry as being foreign.
 665   } else {
 666     VLOG(1) << "Request status: " << request->status().status()
 667             << " error: " << request->status().error()
 668             << " response code: " << response_code;
 669     if (entry.IsExplicit() || entry.IsFallback() || entry.IsIntercept()) {
 670       if (response_code == 304 && fetcher->existing_entry().has_response_id()) {
 671         // Keep the existing response.
 672         entry.set_response_id(fetcher->existing_entry().response_id());
 673         entry.set_response_size(fetcher->existing_entry().response_size());
 674         inprogress_cache_->AddOrModifyEntry(url, entry);
 675       } else {
 676         const char* kFormatString = "Resource fetch failed (%d) %s";
 677         std::string message = FormatUrlErrorMessage(
 678             kFormatString, url, fetcher->result(), response_code);
 679         ResultType result = fetcher->result();
 680         bool is_cross_origin = url.GetOrigin() != manifest_url_.GetOrigin();
 681         switch (result) {
 682           case DISKCACHE_ERROR:
 683             HandleCacheFailure(
 684                 AppCacheErrorDetails(
 685                     message, APPCACHE_UNKNOWN_ERROR, GURL(), 0,
 686                     is_cross_origin),
 687                 result,
 688                 url);
 689             break;
 690           case NETWORK_ERROR:
 691             HandleCacheFailure(
 692                 AppCacheErrorDetails(message, APPCACHE_RESOURCE_ERROR, url, 0,
 693                     is_cross_origin),
 694                 result,
 695                 url);
 696             break;
 697           default:
 698             HandleCacheFailure(AppCacheErrorDetails(message,
 699                                             APPCACHE_RESOURCE_ERROR,
 700                                             url,
 701                                             response_code,
 702                                             is_cross_origin),
 703                                result,
 704                                url);
 705             break;
 706         }
 707         return;
 708       }
 709     } else if (response_code == 404 || response_code == 410) {
 710       // Entry is skipped.  They are dropped from the cache.
 711     } else if (update_type_ == UPGRADE_ATTEMPT &&
 712                fetcher->existing_entry().has_response_id()) {
 713       // Keep the existing response.
 714       // TODO(michaeln): Not sure this is a good idea. This is spec compliant
 715       // but the old resource may or may not be compatible with the new contents
 716       // of the cache. Impossible to know one way or the other.
 717       entry.set_response_id(fetcher->existing_entry().response_id());
 718       entry.set_response_size(fetcher->existing_entry().response_size());
 719       inprogress_cache_->AddOrModifyEntry(url, entry);
 720     }
 721   }
 722
 723   // Fetch another URL now that one request has completed.
 724   DCHECK(internal_state_ != CACHE_FAILURE);
 725   FetchUrls();
 726   MaybeCompleteUpdate();
 727 }
 728
 729 void AppCacheUpdateJob::HandleMasterEntryFetchCompleted(
 730     URLFetcher* fetcher) {
 731   DCHECK(internal_state_ == NO_UPDATE || internal_state_ == DOWNLOADING);
 732
 733   // TODO(jennb): Handle downloads completing during cache failure when update
 734   // no longer fetches master entries directly. For now, we cancel all pending
 735   // master entry fetches when entering cache failure state so this will never
 736   // be called in CACHE_FAILURE state.
 737
 738   net::URLRequest* request = fetcher->request();
 739   const GURL& url = request->original_url();
 740   master_entry_fetches_.erase(url);
 741   ++master_entries_completed_;
 742
 743   int response_code = request->status().is_success()
 744       ? request->GetResponseCode() : -1;
 745
 746   PendingMasters::iterator found = pending_master_entries_.find(url);
 747   DCHECK(found != pending_master_entries_.end());
 748   PendingHosts& hosts = found->second;
 749
 750   // Section 6.9.4. No update case: step 7.3, else step 22.
 751   if (response_code / 100 == 2) {
 752     // Add fetched master entry to the appropriate cache.
 753     AppCache* cache = inprogress_cache_.get() ? inprogress_cache_.get()
 754                                               : group_->newest_complete_cache();
 755     DCHECK(fetcher->response_writer());
 756     AppCacheEntry master_entry(AppCacheEntry::MASTER,
 757                                fetcher->response_writer()->response_id(),
 758                                fetcher->response_writer()->amount_written());
 759     if (cache->AddOrModifyEntry(url, master_entry))
 760       added_master_entries_.push_back(url);
 761     else
 762       duplicate_response_ids_.push_back(master_entry.response_id());
 763
 764     // In no-update case, associate host with the newest cache.
 765     if (!inprogress_cache_.get()) {
 766       // TODO(michaeln): defer until the updated cache has been stored
 767       DCHECK(cache == group_->newest_complete_cache());
 768       for (PendingHosts::iterator host_it = hosts.begin();
 769            host_it != hosts.end(); ++host_it) {
 770         (*host_it)->AssociateCompleteCache(cache);
 771       }
 772     }
 773   } else {
 774     HostNotifier host_notifier;
 775     for (PendingHosts::iterator host_it = hosts.begin();
 776          host_it != hosts.end(); ++host_it) {
 777       AppCacheHost* host = *host_it;
 778       host_notifier.AddHost(host);
 779
 780       // In downloading case, disassociate host from inprogress cache.
 781       if (inprogress_cache_.get())
 782         host->AssociateNoCache(GURL());
 783
 784       host->RemoveObserver(this);
 785     }
 786     hosts.clear();
 787
 788     const char* kFormatString = "Manifest fetch failed (%d) %s";
 789     std::string message = FormatUrlErrorMessage(
 790         kFormatString, request->url(), fetcher->result(), response_code);
 791     host_notifier.SendErrorNotifications(
 792         AppCacheErrorDetails(message,
 793                      appcache::APPCACHE_MANIFEST_ERROR,
 794                      request->url(),
 795                      response_code,
 796                      false /*is_cross_origin*/));
 797
 798     // In downloading case, update result is different if all master entries
 799     // failed vs. only some failing.
 800     if (inprogress_cache_.get()) {
 801       // Only count successful downloads to know if all master entries failed.
 802       pending_master_entries_.erase(found);
 803       --master_entries_completed_;
 804
 805       // Section 6.9.4, step 22.3.
 806       if (update_type_ == CACHE_ATTEMPT && pending_master_entries_.empty()) {
 807         HandleCacheFailure(AppCacheErrorDetails(message,
 808                                         appcache::APPCACHE_MANIFEST_ERROR,
 809                                         request->url(),
 810                                         response_code,
 811                                         false /*is_cross_origin*/),
 812                            fetcher->result(),
 813                            GURL());
 814         return;
 815       }
 816     }
 817   }
 818
 819   DCHECK(internal_state_ != CACHE_FAILURE);
 820   FetchMasterEntries();
 821   MaybeCompleteUpdate();
 822 }
 823
 824 void AppCacheUpdateJob::HandleManifestRefetchCompleted(
 825     URLFetcher* fetcher) {
 826   DCHECK(internal_state_ == REFETCH_MANIFEST);
 827   DCHECK(manifest_fetcher_ == fetcher);
 828   manifest_fetcher_ = NULL;
 829
 830   net::URLRequest* request = fetcher->request();
 831   int response_code = request->status().is_success()
 832       ? request->GetResponseCode() : -1;
 833   if (response_code == 304 || manifest_data_ == fetcher->manifest_data()) {
 834     // Only need to store response in storage if manifest is not already
 835     // an entry in the cache.
 836     AppCacheEntry* entry = inprogress_cache_->GetEntry(manifest_url_);
 837     if (entry) {
 838       entry->add_types(AppCacheEntry::MANIFEST);
 839       StoreGroupAndCache();
 840     } else {
 841       manifest_response_writer_.reset(CreateResponseWriter());
 842       scoped_refptr<HttpResponseInfoIOBuffer> io_buffer(
 843           new HttpResponseInfoIOBuffer(manifest_response_info_.release()));
 844       manifest_response_writer_->WriteInfo(
 845           io_buffer.get(),
 846           base::Bind(&AppCacheUpdateJob::OnManifestInfoWriteComplete,
 847                      base::Unretained(this)));
 848     }
 849   } else {
 850     VLOG(1) << "Request status: " << request->status().status()
 851             << " error: " << request->status().error()
 852             << " response code: " << response_code;
 853     ScheduleUpdateRetry(kRerunDelayMs);
 854     if (response_code == 200) {
 855       HandleCacheFailure(AppCacheErrorDetails("Manifest changed during update",
 856                                       APPCACHE_CHANGED_ERROR,
 857                                       GURL(),
 858                                       0,
 859                                       false /*is_cross_origin*/),
 860                          APPCACHE_MANIFEST_ERROR,
 861                          GURL());
 862     } else {
 863       const char* kFormatString = "Manifest re-fetch failed (%d) %s";
 864       std::string message = FormatUrlErrorMessage(
 865           kFormatString, manifest_url_, fetcher->result(), response_code);
 866       HandleCacheFailure(AppCacheErrorDetails(message,
 867                                       appcache::APPCACHE_MANIFEST_ERROR,
 868                                       GURL(),
 869                                       response_code,
 870                                       false /*is_cross_origin*/),
 871                          fetcher->result(),
 872                          GURL());
 873     }
 874   }
 875 }
 876
 877 void AppCacheUpdateJob::OnManifestInfoWriteComplete(int result) {
 878   if (result > 0) {
 879     scoped_refptr<net::StringIOBuffer> io_buffer(
 880         new net::StringIOBuffer(manifest_data_));
 881     manifest_response_writer_->WriteData(
 882         io_buffer.get(),
 883         manifest_data_.length(),
 884         base::Bind(&AppCacheUpdateJob::OnManifestDataWriteComplete,
 885                    base::Unretained(this)));
 886   } else {
 887     HandleCacheFailure(
 888         AppCacheErrorDetails("Failed to write the manifest headers to storage",
 889                      APPCACHE_UNKNOWN_ERROR,
 890                      GURL(),
 891                      0,
 892                      false /*is_cross_origin*/),
 893         DISKCACHE_ERROR,
 894         GURL());
 895   }
 896 }
 897
 898 void AppCacheUpdateJob::OnManifestDataWriteComplete(int result) {
 899   if (result > 0) {
 900     AppCacheEntry entry(AppCacheEntry::MANIFEST,
 901         manifest_response_writer_->response_id(),
 902         manifest_response_writer_->amount_written());
 903     if (!inprogress_cache_->AddOrModifyEntry(manifest_url_, entry))
 904       duplicate_response_ids_.push_back(entry.response_id());
 905     StoreGroupAndCache();
 906   } else {
 907     HandleCacheFailure(
 908         AppCacheErrorDetails("Failed to write the manifest data to storage",
 909                      APPCACHE_UNKNOWN_ERROR,
 910                      GURL(),
 911                      0,
 912                      false /*is_cross_origin*/),
 913         DISKCACHE_ERROR,
 914         GURL());
 915   }
 916 }
 917
 918 void AppCacheUpdateJob::StoreGroupAndCache() {
 919   DCHECK(stored_state_ == UNSTORED);
 920   stored_state_ = STORING;
 921   scoped_refptr<AppCache> newest_cache;
 922   if (inprogress_cache_.get())
 923     newest_cache.swap(inprogress_cache_);
 924   else
 925     newest_cache = group_->newest_complete_cache();
 926   newest_cache->set_update_time(base::Time::Now());
 927
 928   // TODO(michaeln): dcheck is fishing for clues to crbug/95101
 929   DCHECK_EQ(manifest_url_, group_->manifest_url());
 930   storage_->StoreGroupAndNewestCache(group_, newest_cache.get(), this);
 931 }
 932
 933 void AppCacheUpdateJob::OnGroupAndNewestCacheStored(AppCacheGroup* group,
 934                                                     AppCache* newest_cache,
 935                                                     bool success,
 936                                                     bool would_exceed_quota) {
 937   DCHECK(stored_state_ == STORING);
 938   if (success) {
 939     stored_state_ = STORED;
 940     MaybeCompleteUpdate();  // will definitely complete
 941   } else {
 942     stored_state_ = UNSTORED;
 943
 944     // Restore inprogress_cache_ to get the proper events delivered
 945     // and the proper cleanup to occur.
 946     if (newest_cache != group->newest_complete_cache())
 947       inprogress_cache_ = newest_cache;
 948
 949     ResultType result = DB_ERROR;
 950     AppCacheErrorReason reason = APPCACHE_UNKNOWN_ERROR;
 951     std::string message("Failed to commit new cache to storage");
 952     if (would_exceed_quota) {
 953       message.append(", would exceed quota");
 954       result = APPCACHE_QUOTA_ERROR;
 955       reason = appcache::APPCACHE_QUOTA_ERROR;
 956     }
 957     HandleCacheFailure(
 958         AppCacheErrorDetails(message, reason, GURL(), 0,
 959             false /*is_cross_origin*/),
 960         result,
 961         GURL());
 962   }
 963 }
 964
 965 void AppCacheUpdateJob::NotifySingleHost(AppCacheHost* host,
 966                                          AppCacheEventID event_id) {
 967   std::vector<int> ids(1, host->host_id());
 968   host->frontend()->OnEventRaised(ids, event_id);
 969 }
 970
 971 void AppCacheUpdateJob::NotifyAllAssociatedHosts(AppCacheEventID event_id) {
 972   HostNotifier host_notifier;
 973   AddAllAssociatedHostsToNotifier(&host_notifier);
 974   host_notifier.SendNotifications(event_id);
 975 }
 976
 977 void AppCacheUpdateJob::NotifyAllProgress(const GURL& url) {
 978   HostNotifier host_notifier;
 979   AddAllAssociatedHostsToNotifier(&host_notifier);
 980   host_notifier.SendProgressNotifications(
 981       url, url_file_list_.size(), url_fetches_completed_);
 982 }
 983
 984 void AppCacheUpdateJob::NotifyAllFinalProgress() {
 985   DCHECK(url_file_list_.size() == url_fetches_completed_);
 986   NotifyAllProgress(GURL());
 987 }
 988
 989 void AppCacheUpdateJob::NotifyAllError(const AppCacheErrorDetails& details) {
 990   HostNotifier host_notifier;
 991   AddAllAssociatedHostsToNotifier(&host_notifier);
 992   host_notifier.SendErrorNotifications(details);
 993 }
 994
 995 void AppCacheUpdateJob::LogConsoleMessageToAll(const std::string& message) {
 996   HostNotifier host_notifier;
 997   AddAllAssociatedHostsToNotifier(&host_notifier);
 998   host_notifier.SendLogMessage(message);
 999 }
1000
1001 void AppCacheUpdateJob::AddAllAssociatedHostsToNotifier(
1002     HostNotifier* host_notifier) {
1003   // Collect hosts so we only send one notification per frontend.
1004   // A host can only be associated with a single cache so no need to worry
1005   // about duplicate hosts being added to the notifier.
1006   if (inprogress_cache_.get()) {
1007     DCHECK(internal_state_ == DOWNLOADING || internal_state_ == CACHE_FAILURE);
1008     host_notifier->AddHosts(inprogress_cache_->associated_hosts());
1009   }
1010
1011   AppCacheGroup::Caches old_caches = group_->old_caches();
1012   for (AppCacheGroup::Caches::const_iterator it = old_caches.begin();
1013        it != old_caches.end(); ++it) {
1014     host_notifier->AddHosts((*it)->associated_hosts());
1015   }
1016
1017   AppCache* newest_cache = group_->newest_complete_cache();
1018   if (newest_cache)
1019     host_notifier->AddHosts(newest_cache->associated_hosts());
1020 }
1021
1022 void AppCacheUpdateJob::OnDestructionImminent(AppCacheHost* host) {
1023   // The host is about to be deleted; remove from our collection.
1024   PendingMasters::iterator found =
1025       pending_master_entries_.find(host->pending_master_entry_url());
1026   DCHECK(found != pending_master_entries_.end());
1027   PendingHosts& hosts = found->second;
1028   PendingHosts::iterator it = std::find(hosts.begin(), hosts.end(), host);
1029   DCHECK(it != hosts.end());
1030   hosts.erase(it);
1031 }
1032
1033 void AppCacheUpdateJob::OnServiceReinitialized(
1034     AppCacheStorageReference* old_storage_ref) {
1035   // We continue to use the disabled instance, but arrange for its
1036   // deletion when its no longer needed.
1037   if (old_storage_ref->storage() == storage_)
1038     disabled_storage_reference_ = old_storage_ref;
1039 }
1040
1041 void AppCacheUpdateJob::CheckIfManifestChanged() {
1042   DCHECK(update_type_ == UPGRADE_ATTEMPT);
1043   AppCacheEntry* entry = NULL;
1044   if (group_->newest_complete_cache())
1045     entry = group_->newest_complete_cache()->GetEntry(manifest_url_);
1046   if (!entry) {
1047     // TODO(michaeln): This is just a bandaid to avoid a crash.
1048     // http://code.google.com/p/chromium/issues/detail?id=95101
1049     if (service_->storage() == storage_) {
1050       // Use a local variable because service_ is reset in HandleCacheFailure.
1051       AppCacheServiceImpl* service = service_;
1052       HandleCacheFailure(
1053           AppCacheErrorDetails("Manifest entry not found in existing cache",
1054                        APPCACHE_UNKNOWN_ERROR,
1055                        GURL(),
1056                        0,
1057                        false /*is_cross_origin*/),
1058           DB_ERROR,
1059           GURL());
1060       AppCacheHistograms::AddMissingManifestEntrySample();
1061       service->DeleteAppCacheGroup(manifest_url_, net::CompletionCallback());
1062     }
1063     return;
1064   }
1065
1066   // Load manifest data from storage to compare against fetched manifest.
1067   manifest_response_reader_.reset(
1068       storage_->CreateResponseReader(manifest_url_,
1069                                      group_->group_id(),
1070                                      entry->response_id()));
1071   read_manifest_buffer_ = new net::IOBuffer(kBufferSize);
1072   manifest_response_reader_->ReadData(
1073       read_manifest_buffer_.get(),
1074       kBufferSize,
1075       base::Bind(&AppCacheUpdateJob::OnManifestDataReadComplete,
1076                  base::Unretained(this)));  // async read
1077 }
1078
1079 void AppCacheUpdateJob::OnManifestDataReadComplete(int result) {
1080   if (result > 0) {
1081     loaded_manifest_data_.append(read_manifest_buffer_->data(), result);
1082     manifest_response_reader_->ReadData(
1083         read_manifest_buffer_.get(),
1084         kBufferSize,
1085         base::Bind(&AppCacheUpdateJob::OnManifestDataReadComplete,
1086                    base::Unretained(this)));  // read more
1087   } else {
1088     read_manifest_buffer_ = NULL;
1089     manifest_response_reader_.reset();
1090     ContinueHandleManifestFetchCompleted(
1091         result < 0 || manifest_data_ != loaded_manifest_data_);
1092   }
1093 }
1094
1095 void AppCacheUpdateJob::BuildUrlFileList(const Manifest& manifest) {
1096   for (base::hash_set<std::string>::const_iterator it =
1097            manifest.explicit_urls.begin();
1098        it != manifest.explicit_urls.end(); ++it) {
1099     AddUrlToFileList(GURL(*it), AppCacheEntry::EXPLICIT);
1100   }
1101
1102   const std::vector<Namespace>& intercepts =
1103       manifest.intercept_namespaces;
1104   for (std::vector<Namespace>::const_iterator it = intercepts.begin();
1105        it != intercepts.end(); ++it) {
1106     int flags = AppCacheEntry::INTERCEPT;
1107     if (it->is_executable)
1108       flags |= AppCacheEntry::EXECUTABLE;
1109     AddUrlToFileList(it->target_url, flags);
1110   }
1111
1112   const std::vector<Namespace>& fallbacks =
1113       manifest.fallback_namespaces;
1114   for (std::vector<Namespace>::const_iterator it = fallbacks.begin();
1115        it != fallbacks.end(); ++it) {
1116      AddUrlToFileList(it->target_url, AppCacheEntry::FALLBACK);
1117   }
1118
1119   // Add all master entries from newest complete cache.
1120   if (update_type_ == UPGRADE_ATTEMPT) {
1121     const AppCache::EntryMap& entries =
1122         group_->newest_complete_cache()->entries();
1123     for (AppCache::EntryMap::const_iterator it = entries.begin();
1124          it != entries.end(); ++it) {
1125       const AppCacheEntry& entry = it->second;
1126       if (entry.IsMaster())
1127         AddUrlToFileList(it->first, AppCacheEntry::MASTER);
1128     }
1129   }
1130 }
1131
1132 void AppCacheUpdateJob::AddUrlToFileList(const GURL& url, int type) {
1133   std::pair<AppCache::EntryMap::iterator, bool> ret = url_file_list_.insert(
1134       AppCache::EntryMap::value_type(url, AppCacheEntry(type)));
1135
1136   if (ret.second)
1137     urls_to_fetch_.push_back(UrlToFetch(url, false, NULL));
1138   else
1139     ret.first->second.add_types(type);  // URL already exists. Merge types.
1140 }
1141
1142 void AppCacheUpdateJob::FetchUrls() {
1143   DCHECK(internal_state_ == DOWNLOADING);
1144
1145   // Fetch each URL in the list according to section 6.9.4 step 17.1-17.3.
1146   // Fetch up to the concurrent limit. Other fetches will be triggered as each
1147   // each fetch completes.
1148   while (pending_url_fetches_.size() < kMaxConcurrentUrlFetches &&
1149          !urls_to_fetch_.empty()) {
1150     UrlToFetch url_to_fetch = urls_to_fetch_.front();
1151     urls_to_fetch_.pop_front();
1152
1153     AppCache::EntryMap::iterator it = url_file_list_.find(url_to_fetch.url);
1154     DCHECK(it != url_file_list_.end());
1155     AppCacheEntry& entry = it->second;
1156     if (ShouldSkipUrlFetch(entry)) {
1157       NotifyAllProgress(url_to_fetch.url);
1158       ++url_fetches_completed_;
1159     } else if (AlreadyFetchedEntry(url_to_fetch.url, entry.types())) {
1160       NotifyAllProgress(url_to_fetch.url);
1161       ++url_fetches_completed_;  // saved a URL request
1162     } else if (!url_to_fetch.storage_checked &&
1163                MaybeLoadFromNewestCache(url_to_fetch.url, entry)) {
1164       // Continues asynchronously after data is loaded from newest cache.
1165     } else {
1166       URLFetcher* fetcher = new URLFetcher(
1167           url_to_fetch.url, URLFetcher::URL_FETCH, this);
1168       if (url_to_fetch.existing_response_info.get()) {
1169         DCHECK(group_->newest_complete_cache());
1170         AppCacheEntry* existing_entry =
1171             group_->newest_complete_cache()->GetEntry(url_to_fetch.url);
1172         DCHECK(existing_entry);
1173         DCHECK(existing_entry->response_id() ==
1174                url_to_fetch.existing_response_info->response_id());
1175         fetcher->set_existing_response_headers(
1176             url_to_fetch.existing_response_info->http_response_info()->headers
1177                 .get());
1178         fetcher->set_existing_entry(*existing_entry);
1179       }
1180       fetcher->Start();
1181       pending_url_fetches_.insert(
1182           PendingUrlFetches::value_type(url_to_fetch.url, fetcher));
1183     }
1184   }
1185 }
1186
1187 void AppCacheUpdateJob::CancelAllUrlFetches() {
1188   // Cancel any pending URL requests.
1189   for (PendingUrlFetches::iterator it = pending_url_fetches_.begin();
1190        it != pending_url_fetches_.end(); ++it) {
1191     delete it->second;
1192   }
1193
1194   url_fetches_completed_ +=
1195       pending_url_fetches_.size() + urls_to_fetch_.size();
1196   pending_url_fetches_.clear();
1197   urls_to_fetch_.clear();
1198 }
1199
1200 bool AppCacheUpdateJob::ShouldSkipUrlFetch(const AppCacheEntry& entry) {
1201   // 6.6.4 Step 17
1202   // If the resource URL being processed was flagged as neither an
1203   // "explicit entry" nor or a "fallback entry", then the user agent
1204   // may skip this URL.
1205   if (entry.IsExplicit() || entry.IsFallback() || entry.IsIntercept())
1206     return false;
1207
1208   // TODO(jennb): decide if entry should be skipped to expire it from cache
1209   return false;
1210 }
1211
1212 bool AppCacheUpdateJob::AlreadyFetchedEntry(const GURL& url,
1213                                             int entry_type) {
1214   DCHECK(internal_state_ == DOWNLOADING || internal_state_ == NO_UPDATE);
1215   AppCacheEntry* existing =
1216       inprogress_cache_.get() ? inprogress_cache_->GetEntry(url)
1217                               : group_->newest_complete_cache()->GetEntry(url);
1218   if (existing) {
1219     existing->add_types(entry_type);
1220     return true;
1221   }
1222   return false;
1223 }
1224
1225 void AppCacheUpdateJob::AddMasterEntryToFetchList(AppCacheHost* host,
1226                                                   const GURL& url,
1227                                                   bool is_new) {
1228   DCHECK(!IsTerminating());
1229
1230   if (internal_state_ == DOWNLOADING || internal_state_ == NO_UPDATE) {
1231     AppCache* cache;
1232     if (inprogress_cache_.get()) {
1233       // always associate
1234       host->AssociateIncompleteCache(inprogress_cache_.get(), manifest_url_);
1235       cache = inprogress_cache_.get();
1236     } else {
1237       cache = group_->newest_complete_cache();
1238     }
1239
1240     // Update existing entry if it has already been fetched.
1241     AppCacheEntry* entry = cache->GetEntry(url);
1242     if (entry) {
1243       entry->add_types(AppCacheEntry::MASTER);
1244       if (internal_state_ == NO_UPDATE && !inprogress_cache_.get()) {
1245         // only associate if have entry
1246         host->AssociateCompleteCache(cache);
1247       }
1248       if (is_new)
1249         ++master_entries_completed_;  // pretend fetching completed
1250       return;
1251     }
1252   }
1253
1254   // Add to fetch list if not already fetching.
1255   if (master_entry_fetches_.find(url) == master_entry_fetches_.end()) {
1256     master_entries_to_fetch_.insert(url);
1257     if (internal_state_ == DOWNLOADING || internal_state_ == NO_UPDATE)
1258       FetchMasterEntries();
1259   }
1260 }
1261
1262 void AppCacheUpdateJob::FetchMasterEntries() {
1263   DCHECK(internal_state_ == NO_UPDATE || internal_state_ == DOWNLOADING);
1264
1265   // Fetch each master entry in the list, up to the concurrent limit.
1266   // Additional fetches will be triggered as each fetch completes.
1267   while (master_entry_fetches_.size() < kMaxConcurrentUrlFetches &&
1268          !master_entries_to_fetch_.empty()) {
1269     const GURL& url = *master_entries_to_fetch_.begin();
1270
1271     if (AlreadyFetchedEntry(url, AppCacheEntry::MASTER)) {
1272       ++master_entries_completed_;  // saved a URL request
1273
1274       // In no update case, associate hosts to newest cache in group
1275       // now that master entry has been "successfully downloaded".
1276       if (internal_state_ == NO_UPDATE) {
1277         // TODO(michaeln): defer until the updated cache has been stored.
1278         DCHECK(!inprogress_cache_.get());
1279         AppCache* cache = group_->newest_complete_cache();
1280         PendingMasters::iterator found = pending_master_entries_.find(url);
1281         DCHECK(found != pending_master_entries_.end());
1282         PendingHosts& hosts = found->second;
1283         for (PendingHosts::iterator host_it = hosts.begin();
1284              host_it != hosts.end(); ++host_it) {
1285           (*host_it)->AssociateCompleteCache(cache);
1286         }
1287       }
1288     } else {
1289       URLFetcher* fetcher = new URLFetcher(
1290           url, URLFetcher::MASTER_ENTRY_FETCH, this);
1291       fetcher->Start();
1292       master_entry_fetches_.insert(PendingUrlFetches::value_type(url, fetcher));
1293     }
1294
1295     master_entries_to_fetch_.erase(master_entries_to_fetch_.begin());
1296   }
1297 }
1298
1299 void AppCacheUpdateJob::CancelAllMasterEntryFetches(
1300     const AppCacheErrorDetails& error_details) {
1301   // For now, cancel all in-progress fetches for master entries and pretend
1302   // all master entries fetches have completed.
1303   // TODO(jennb): Delete this when update no longer fetches master entries
1304   // directly.
1305
1306   // Cancel all in-progress fetches.
1307   for (PendingUrlFetches::iterator it = master_entry_fetches_.begin();
1308        it != master_entry_fetches_.end(); ++it) {
1309     delete it->second;
1310     master_entries_to_fetch_.insert(it->first);  // back in unfetched list
1311   }
1312   master_entry_fetches_.clear();
1313
1314   master_entries_completed_ += master_entries_to_fetch_.size();
1315
1316   // Cache failure steps, step 2.
1317   // Pretend all master entries that have not yet been fetched have completed
1318   // downloading. Unassociate hosts from any appcache and send ERROR event.
1319   HostNotifier host_notifier;
1320   while (!master_entries_to_fetch_.empty()) {
1321     const GURL& url = *master_entries_to_fetch_.begin();
1322     PendingMasters::iterator found = pending_master_entries_.find(url);
1323     DCHECK(found != pending_master_entries_.end());
1324     PendingHosts& hosts = found->second;
1325     for (PendingHosts::iterator host_it = hosts.begin();
1326          host_it != hosts.end(); ++host_it) {
1327       AppCacheHost* host = *host_it;
1328       host->AssociateNoCache(GURL());
1329       host_notifier.AddHost(host);
1330       host->RemoveObserver(this);
1331     }
1332     hosts.clear();
1333
1334     master_entries_to_fetch_.erase(master_entries_to_fetch_.begin());
1335   }
1336   host_notifier.SendErrorNotifications(error_details);
1337 }
1338
1339 bool AppCacheUpdateJob::MaybeLoadFromNewestCache(const GURL& url,
1340                                                  AppCacheEntry& entry) {
1341   if (update_type_ != UPGRADE_ATTEMPT)
1342     return false;
1343
1344   AppCache* newest = group_->newest_complete_cache();
1345   AppCacheEntry* copy_me = newest->GetEntry(url);
1346   if (!copy_me || !copy_me->has_response_id())
1347     return false;
1348
1349   // Load HTTP headers for entry from newest cache.
1350   loading_responses_.insert(
1351       LoadingResponses::value_type(copy_me->response_id(), url));
1352   storage_->LoadResponseInfo(manifest_url_, group_->group_id(),
1353                              copy_me->response_id(),
1354                              this);
1355   // Async: wait for OnResponseInfoLoaded to complete.
1356   return true;
1357 }
1358
1359 void AppCacheUpdateJob::OnResponseInfoLoaded(
1360     AppCacheResponseInfo* response_info, int64 response_id) {
1361   const net::HttpResponseInfo* http_info = response_info ?
1362       response_info->http_response_info() : NULL;
1363
1364   // Needed response info for a manifest fetch request.
1365   if (internal_state_ == FETCH_MANIFEST) {
1366     if (http_info)
1367       manifest_fetcher_->set_existing_response_headers(
1368           http_info->headers.get());
1369     manifest_fetcher_->Start();
1370     return;
1371   }
1372
1373   LoadingResponses::iterator found = loading_responses_.find(response_id);
1374   DCHECK(found != loading_responses_.end());
1375   const GURL& url = found->second;
1376
1377   if (!http_info) {
1378     LoadFromNewestCacheFailed(url, NULL);  // no response found
1379   } else {
1380     // Check if response can be re-used according to HTTP caching semantics.
1381     // Responses with a "vary" header get treated as expired.
1382     const std::string name = "vary";
1383     std::string value;
1384     void* iter = NULL;
1385     if (!http_info->headers.get() ||
1386         http_info->headers->RequiresValidation(http_info->request_time,
1387                                                http_info->response_time,
1388                                                base::Time::Now()) ||
1389         http_info->headers->EnumerateHeader(&iter, name, &value)) {
1390       LoadFromNewestCacheFailed(url, response_info);
1391     } else {
1392       DCHECK(group_->newest_complete_cache());
1393       AppCacheEntry* copy_me = group_->newest_complete_cache()->GetEntry(url);
1394       DCHECK(copy_me);
1395       DCHECK(copy_me->response_id() == response_id);
1396
1397       AppCache::EntryMap::iterator it = url_file_list_.find(url);
1398       DCHECK(it != url_file_list_.end());
1399       AppCacheEntry& entry = it->second;
1400       entry.set_response_id(response_id);
1401       entry.set_response_size(copy_me->response_size());
1402       inprogress_cache_->AddOrModifyEntry(url, entry);
1403       NotifyAllProgress(url);
1404       ++url_fetches_completed_;
1405     }
1406   }
1407   loading_responses_.erase(found);
1408
1409   MaybeCompleteUpdate();
1410 }
1411
1412 void AppCacheUpdateJob::LoadFromNewestCacheFailed(
1413     const GURL& url, AppCacheResponseInfo* response_info) {
1414   if (internal_state_ == CACHE_FAILURE)
1415     return;
1416
1417   // Re-insert url at front of fetch list. Indicate storage has been checked.
1418   urls_to_fetch_.push_front(UrlToFetch(url, true, response_info));
1419   FetchUrls();
1420 }
1421
1422 void AppCacheUpdateJob::MaybeCompleteUpdate() {
1423   DCHECK(internal_state_ != CACHE_FAILURE);
1424
1425   // Must wait for any pending master entries or url fetches to complete.
1426   if (master_entries_completed_ != pending_master_entries_.size() ||
1427       url_fetches_completed_ != url_file_list_.size()) {
1428     DCHECK(internal_state_ != COMPLETED);
1429     return;
1430   }
1431
1432   switch (internal_state_) {
1433     case NO_UPDATE:
1434       if (master_entries_completed_ > 0) {
1435         switch (stored_state_) {
1436           case UNSTORED:
1437             StoreGroupAndCache();
1438             return;
1439           case STORING:
1440             return;
1441           case STORED:
1442             break;
1443         }
1444       }
1445       // 6.9.4 steps 7.3-7.7.
1446       NotifyAllAssociatedHosts(APPCACHE_NO_UPDATE_EVENT);
1447       DiscardDuplicateResponses();
1448       internal_state_ = COMPLETED;
1449       break;
1450     case DOWNLOADING:
1451       internal_state_ = REFETCH_MANIFEST;
1452       FetchManifest(false);
1453       break;
1454     case REFETCH_MANIFEST:
1455       DCHECK(stored_state_ == STORED);
1456       NotifyAllFinalProgress();
1457       if (update_type_ == CACHE_ATTEMPT)
1458         NotifyAllAssociatedHosts(APPCACHE_CACHED_EVENT);
1459       else
1460         NotifyAllAssociatedHosts(APPCACHE_UPDATE_READY_EVENT);
1461       DiscardDuplicateResponses();
1462       internal_state_ = COMPLETED;
1463       LogHistogramStats(UPDATE_OK, GURL());
1464       break;
1465     case CACHE_FAILURE:
1466       NOTREACHED();  // See HandleCacheFailure
1467       break;
1468     default:
1469       break;
1470   }
1471
1472   // Let the stack unwind before deletion to make it less risky as this
1473   // method is called from multiple places in this file.
1474   if (internal_state_ == COMPLETED)
1475     DeleteSoon();
1476 }
1477
1478 void AppCacheUpdateJob::ScheduleUpdateRetry(int delay_ms) {
1479   // TODO(jennb): post a delayed task with the "same parameters" as this job
1480   // to retry the update at a later time. Need group, URLs of pending master
1481   // entries and their hosts.
1482 }
1483
1484 void AppCacheUpdateJob::Cancel() {
1485   internal_state_ = CANCELLED;
1486
1487   LogHistogramStats(CANCELLED_ERROR, GURL());
1488
1489   if (manifest_fetcher_) {
1490     delete manifest_fetcher_;
1491     manifest_fetcher_ = NULL;
1492   }
1493
1494   for (PendingUrlFetches::iterator it = pending_url_fetches_.begin();
1495        it != pending_url_fetches_.end(); ++it) {
1496     delete it->second;
1497   }
1498   pending_url_fetches_.clear();
1499
1500   for (PendingUrlFetches::iterator it = master_entry_fetches_.begin();
1501        it != master_entry_fetches_.end(); ++it) {
1502     delete it->second;
1503   }
1504   master_entry_fetches_.clear();
1505
1506   ClearPendingMasterEntries();
1507   DiscardInprogressCache();
1508
1509   // Delete response writer to avoid any callbacks.
1510   if (manifest_response_writer_)
1511     manifest_response_writer_.reset();
1512
1513   storage_->CancelDelegateCallbacks(this);
1514 }
1515
1516 void AppCacheUpdateJob::ClearPendingMasterEntries() {
1517   for (PendingMasters::iterator it = pending_master_entries_.begin();
1518        it != pending_master_entries_.end(); ++it) {
1519     PendingHosts& hosts = it->second;
1520     for (PendingHosts::iterator host_it = hosts.begin();
1521          host_it != hosts.end(); ++host_it) {
1522       (*host_it)->RemoveObserver(this);
1523     }
1524   }
1525
1526   pending_master_entries_.clear();
1527 }
1528
1529 void AppCacheUpdateJob::DiscardInprogressCache() {
1530   if (stored_state_ == STORING) {
1531     // We can make no assumptions about whether the StoreGroupAndCacheTask
1532     // actually completed or not. This condition should only be reachable
1533     // during shutdown. Free things up and return to do no harm.
1534     inprogress_cache_ = NULL;
1535     added_master_entries_.clear();
1536     return;
1537   }
1538
1539   storage_->DoomResponses(manifest_url_, stored_response_ids_);
1540
1541   if (!inprogress_cache_.get()) {
1542     // We have to undo the changes we made, if any, to the existing cache.
1543     if (group_ && group_->newest_complete_cache()) {
1544       for (std::vector<GURL>::iterator iter = added_master_entries_.begin();
1545            iter != added_master_entries_.end(); ++iter) {
1546         group_->newest_complete_cache()->RemoveEntry(*iter);
1547       }
1548     }
1549     added_master_entries_.clear();
1550     return;
1551   }
1552
1553   AppCache::AppCacheHosts& hosts = inprogress_cache_->associated_hosts();
1554   while (!hosts.empty())
1555     (*hosts.begin())->AssociateNoCache(GURL());
1556
1557   inprogress_cache_ = NULL;
1558   added_master_entries_.clear();
1559 }
1560
1561 void AppCacheUpdateJob::DiscardDuplicateResponses() {
1562   storage_->DoomResponses(manifest_url_, duplicate_response_ids_);
1563 }
1564
1565 void AppCacheUpdateJob::LogHistogramStats(
1566       ResultType result, const GURL& failed_resource_url) {
1567   AppCacheHistograms::CountUpdateJobResult(result, manifest_url_.GetOrigin());
1568   if (result == UPDATE_OK)
1569     return;
1570
1571   int percent_complete = 0;
1572   if (url_file_list_.size() > 0) {
1573     size_t actual_fetches_completed = url_fetches_completed_;
1574     if (!failed_resource_url.is_empty() && actual_fetches_completed)
1575       --actual_fetches_completed;
1576     percent_complete = (static_cast<double>(actual_fetches_completed) /
1577                             static_cast<double>(url_file_list_.size())) * 100.0;
1578     percent_complete = std::min(percent_complete, 99);
1579   }
1580
1581   bool was_making_progress =
1582       base::Time::Now() - last_progress_time_ <
1583           base::TimeDelta::FromMinutes(5);
1584
1585   bool off_origin_resource_failure =
1586       !failed_resource_url.is_empty() &&
1587           (failed_resource_url.GetOrigin() != manifest_url_.GetOrigin());
1588
1589   AppCacheHistograms::LogUpdateFailureStats(
1590       manifest_url_.GetOrigin(),
1591       percent_complete,
1592       was_making_progress,
1593       off_origin_resource_failure);
1594 }
1595
1596 void AppCacheUpdateJob::DeleteSoon() {
1597   ClearPendingMasterEntries();
1598   manifest_response_writer_.reset();
1599   storage_->CancelDelegateCallbacks(this);
1600   service_->RemoveObserver(this);
1601   service_ = NULL;
1602
1603   // Break the connection with the group so the group cannot call delete
1604   // on this object after we've posted a task to delete ourselves.
1605   group_->SetUpdateAppCacheStatus(AppCacheGroup::IDLE);
1606   group_ = NULL;
1607
1608   base::MessageLoop::current()->DeleteSoon(FROM_HERE, this);
1609 }
1610
1611 }  // namespace appcache