src/content/browser/appcache/appcache_update_job.cc

   1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "content/browser/appcache/appcache_update_job.h"
   6
   7 #include "base/bind.h"
   8 #include "base/bind_helpers.h"
   9 #include "base/compiler_specific.h"
  10 #include "base/message_loop/message_loop.h"
  11 #include "base/strings/string_util.h"
  12 #include "base/strings/stringprintf.h"
  13 #include "content/browser/appcache/appcache_group.h"
  14 #include "content/browser/appcache/appcache_histograms.h"
  15 #include "net/base/host_port_pair.h"
  16 #include "net/base/io_buffer.h"
  17 #include "net/base/load_flags.h"
  18 #include "net/base/net_errors.h"
  19 #include "net/base/request_priority.h"
  20 #include "net/http/http_request_headers.h"
  21 #include "net/http/http_response_headers.h"
  22 #include "net/url_request/url_request_context.h"
  23
  24 namespace {
  25 bool IsDataReductionProxy(const net::HostPortPair& proxy_server) {
  26   return (
  27       proxy_server.Equals(net::HostPortPair("proxy.googlezip.net", 443)) ||
  28       proxy_server.Equals(net::HostPortPair("compress.googlezip.net", 80)) ||
  29       proxy_server.Equals(net::HostPortPair("proxy-dev.googlezip.net", 80)));
  30 }
  31 }  // namspace
  32
  33 namespace content {
  34
  35 static const int kBufferSize = 32768;
  36 static const size_t kMaxConcurrentUrlFetches = 2;
  37 static const int kMax503Retries = 3;
  38
  39 static std::string FormatUrlErrorMessage(
  40       const char* format, const GURL& url,
  41       AppCacheUpdateJob::ResultType error,
  42       int response_code) {
  43     // Show the net response code if we have one.
  44     int code = response_code;
  45     if (error != AppCacheUpdateJob::SERVER_ERROR)
  46       code = static_cast<int>(error);
  47     return base::StringPrintf(format, code, url.spec().c_str());
  48 }
  49
  50 // Helper class for collecting hosts per frontend when sending notifications
  51 // so that only one notification is sent for all hosts using the same frontend.
  52 class HostNotifier {
  53  public:
  54   typedef std::vector<int> HostIds;
  55   typedef std::map<AppCacheFrontend*, HostIds> NotifyHostMap;
  56
  57   // Caller is responsible for ensuring there will be no duplicate hosts.
  58   void AddHost(AppCacheHost* host) {
  59     std::pair<NotifyHostMap::iterator , bool> ret = hosts_to_notify.insert(
  60         NotifyHostMap::value_type(host->frontend(), HostIds()));
  61     ret.first->second.push_back(host->host_id());
  62   }
  63
  64   void AddHosts(const std::set<AppCacheHost*>& hosts) {
  65     for (std::set<AppCacheHost*>::const_iterator it = hosts.begin();
  66          it != hosts.end(); ++it) {
  67       AddHost(*it);
  68     }
  69   }
  70
  71   void SendNotifications(AppCacheEventID event_id) {
  72     for (NotifyHostMap::iterator it = hosts_to_notify.begin();
  73          it != hosts_to_notify.end(); ++it) {
  74       AppCacheFrontend* frontend = it->first;
  75       frontend->OnEventRaised(it->second, event_id);
  76     }
  77   }
  78
  79   void SendProgressNotifications(
  80       const GURL& url, int num_total, int num_complete) {
  81     for (NotifyHostMap::iterator it = hosts_to_notify.begin();
  82          it != hosts_to_notify.end(); ++it) {
  83       AppCacheFrontend* frontend = it->first;
  84       frontend->OnProgressEventRaised(it->second, url,
  85                                       num_total, num_complete);
  86     }
  87   }
  88
  89   void SendErrorNotifications(const AppCacheErrorDetails& details) {
  90     DCHECK(!details.message.empty());
  91     for (NotifyHostMap::iterator it = hosts_to_notify.begin();
  92          it != hosts_to_notify.end(); ++it) {
  93       AppCacheFrontend* frontend = it->first;
  94       frontend->OnErrorEventRaised(it->second, details);
  95     }
  96   }
  97
  98   void SendLogMessage(const std::string& message) {
  99     for (NotifyHostMap::iterator it = hosts_to_notify.begin();
 100          it != hosts_to_notify.end(); ++it) {
 101       AppCacheFrontend* frontend = it->first;
 102       for (HostIds::iterator id = it->second.begin();
 103            id != it->second.end(); ++id) {
 104         frontend->OnLogMessage(*id, APPCACHE_LOG_WARNING, message);
 105       }
 106     }
 107   }
 108
 109  private:
 110   NotifyHostMap hosts_to_notify;
 111 };
 112
 113 AppCacheUpdateJob::UrlToFetch::UrlToFetch(const GURL& url,
 114                                           bool checked,
 115                                           AppCacheResponseInfo* info)
 116     : url(url),
 117       storage_checked(checked),
 118       existing_response_info(info) {
 119 }
 120
 121 AppCacheUpdateJob::UrlToFetch::~UrlToFetch() {
 122 }
 123
 124 // Helper class to fetch resources. Depending on the fetch type,
 125 // can either fetch to an in-memory string or write the response
 126 // data out to the disk cache.
 127 AppCacheUpdateJob::URLFetcher::URLFetcher(const GURL& url,
 128                                           FetchType fetch_type,
 129                                           AppCacheUpdateJob* job)
 130     : url_(url),
 131       job_(job),
 132       fetch_type_(fetch_type),
 133       retry_503_attempts_(0),
 134       buffer_(new net::IOBuffer(kBufferSize)),
 135       request_(job->service_->request_context()
 136                    ->CreateRequest(url, net::DEFAULT_PRIORITY, this, NULL)),
 137       result_(UPDATE_OK),
 138       redirect_response_code_(-1) {}
 139
 140 AppCacheUpdateJob::URLFetcher::~URLFetcher() {
 141 }
 142
 143 void AppCacheUpdateJob::URLFetcher::Start() {
 144   request_->set_first_party_for_cookies(job_->manifest_url_);
 145   request_->SetLoadFlags(request_->load_flags() |
 146                          net::LOAD_DISABLE_INTERCEPT);
 147   if (existing_response_headers_.get())
 148     AddConditionalHeaders(existing_response_headers_.get());
 149   request_->Start();
 150 }
 151
 152 void AppCacheUpdateJob::URLFetcher::OnReceivedRedirect(
 153     net::URLRequest* request,
 154     const net::RedirectInfo& redirect_info,
 155     bool* defer_redirect) {
 156   DCHECK(request_ == request);
 157   // TODO(bengr): Remove this special case logic when crbug.com/429505 is
 158   // resolved. Until then, the data reduction proxy client logic uses the
 159   // redirect mechanism to resend requests over a direct connection when
 160   // the proxy instructs it to do so. The redirect is to the same location
 161   // as the original URL.
 162   if ((request->load_flags() & net::LOAD_BYPASS_PROXY) &&
 163       IsDataReductionProxy(request->proxy_server())) {
 164     DCHECK_EQ(request->original_url(), request->url());
 165     return;
 166   }
 167   // Redirect is not allowed by the update process.
 168   job_->MadeProgress();
 169   redirect_response_code_ = request->GetResponseCode();
 170   request->Cancel();
 171   result_ = REDIRECT_ERROR;
 172   OnResponseCompleted();
 173 }
 174
 175 void AppCacheUpdateJob::URLFetcher::OnResponseStarted(
 176     net::URLRequest *request) {
 177   DCHECK(request == request_);
 178   int response_code = -1;
 179   if (request->status().is_success()) {
 180     response_code = request->GetResponseCode();
 181     job_->MadeProgress();
 182   }
 183   if ((response_code / 100) == 2) {
 184
 185     // See http://code.google.com/p/chromium/issues/detail?id=69594
 186     // We willfully violate the HTML5 spec at this point in order
 187     // to support the appcaching of cross-origin HTTPS resources.
 188     // We've opted for a milder constraint and allow caching unless
 189     // the resource has a "no-store" header. A spec change has been
 190     // requested on the whatwg list.
 191     // TODO(michaeln): Consider doing this for cross-origin HTTP resources too.
 192     if (url_.SchemeIsSecure() &&
 193         url_.GetOrigin() != job_->manifest_url_.GetOrigin()) {
 194       if (request->response_headers()->
 195               HasHeaderValue("cache-control", "no-store")) {
 196         DCHECK_EQ(-1, redirect_response_code_);
 197         request->Cancel();
 198         result_ = SERVER_ERROR;  // Not the best match?
 199         OnResponseCompleted();
 200         return;
 201       }
 202     }
 203
 204     // Write response info to storage for URL fetches. Wait for async write
 205     // completion before reading any response data.
 206     if (fetch_type_ == URL_FETCH || fetch_type_ == MASTER_ENTRY_FETCH) {
 207       response_writer_.reset(job_->CreateResponseWriter());
 208       scoped_refptr<HttpResponseInfoIOBuffer> io_buffer(
 209           new HttpResponseInfoIOBuffer(
 210               new net::HttpResponseInfo(request->response_info())));
 211       response_writer_->WriteInfo(
 212           io_buffer.get(),
 213           base::Bind(&URLFetcher::OnWriteComplete, base::Unretained(this)));
 214     } else {
 215       ReadResponseData();
 216     }
 217   } else {
 218     if (response_code > 0)
 219       result_ = SERVER_ERROR;
 220     else
 221       result_ = NETWORK_ERROR;
 222     OnResponseCompleted();
 223   }
 224 }
 225
 226 void AppCacheUpdateJob::URLFetcher::OnReadCompleted(
 227     net::URLRequest* request, int bytes_read) {
 228   DCHECK(request_ == request);
 229   bool data_consumed = true;
 230   if (request->status().is_success() && bytes_read > 0) {
 231     job_->MadeProgress();
 232     data_consumed = ConsumeResponseData(bytes_read);
 233     if (data_consumed) {
 234       bytes_read = 0;
 235       while (request->Read(buffer_.get(), kBufferSize, &bytes_read)) {
 236         if (bytes_read > 0) {
 237           data_consumed = ConsumeResponseData(bytes_read);
 238           if (!data_consumed)
 239             break;  // wait for async data processing, then read more
 240         } else {
 241           break;
 242         }
 243       }
 244     }
 245   }
 246   if (data_consumed && !request->status().is_io_pending()) {
 247     DCHECK_EQ(UPDATE_OK, result_);
 248     OnResponseCompleted();
 249   }
 250 }
 251
 252 void AppCacheUpdateJob::URLFetcher::AddConditionalHeaders(
 253     const net::HttpResponseHeaders* headers) {
 254   DCHECK(request_.get() && headers);
 255   net::HttpRequestHeaders extra_headers;
 256
 257   // Add If-Modified-Since header if response info has Last-Modified header.
 258   const std::string last_modified = "Last-Modified";
 259   std::string last_modified_value;
 260   headers->EnumerateHeader(NULL, last_modified, &last_modified_value);
 261   if (!last_modified_value.empty()) {
 262     extra_headers.SetHeader(net::HttpRequestHeaders::kIfModifiedSince,
 263                             last_modified_value);
 264   }
 265
 266   // Add If-None-Match header if response info has ETag header.
 267   const std::string etag = "ETag";
 268   std::string etag_value;
 269   headers->EnumerateHeader(NULL, etag, &etag_value);
 270   if (!etag_value.empty()) {
 271     extra_headers.SetHeader(net::HttpRequestHeaders::kIfNoneMatch,
 272                             etag_value);
 273   }
 274   if (!extra_headers.IsEmpty())
 275     request_->SetExtraRequestHeaders(extra_headers);
 276 }
 277
 278 void  AppCacheUpdateJob::URLFetcher::OnWriteComplete(int result) {
 279   if (result < 0) {
 280     request_->Cancel();
 281     result_ = DISKCACHE_ERROR;
 282     OnResponseCompleted();
 283     return;
 284   }
 285   ReadResponseData();
 286 }
 287
 288 void AppCacheUpdateJob::URLFetcher::ReadResponseData() {
 289   InternalUpdateState state = job_->internal_state_;
 290   if (state == CACHE_FAILURE || state == CANCELLED || state == COMPLETED)
 291     return;
 292   int bytes_read = 0;
 293   request_->Read(buffer_.get(), kBufferSize, &bytes_read);
 294   OnReadCompleted(request_.get(), bytes_read);
 295 }
 296
 297 // Returns false if response data is processed asynchronously, in which
 298 // case ReadResponseData will be invoked when it is safe to continue
 299 // reading more response data from the request.
 300 bool AppCacheUpdateJob::URLFetcher::ConsumeResponseData(int bytes_read) {
 301   DCHECK_GT(bytes_read, 0);
 302   switch (fetch_type_) {
 303     case MANIFEST_FETCH:
 304     case MANIFEST_REFETCH:
 305       manifest_data_.append(buffer_->data(), bytes_read);
 306       break;
 307     case URL_FETCH:
 308     case MASTER_ENTRY_FETCH:
 309       DCHECK(response_writer_.get());
 310       response_writer_->WriteData(
 311           buffer_.get(),
 312           bytes_read,
 313           base::Bind(&URLFetcher::OnWriteComplete, base::Unretained(this)));
 314       return false;  // wait for async write completion to continue reading
 315     default:
 316       NOTREACHED();
 317   }
 318   return true;
 319 }
 320
 321 void AppCacheUpdateJob::URLFetcher::OnResponseCompleted() {
 322   if (request_->status().is_success())
 323     job_->MadeProgress();
 324
 325   // Retry for 503s where retry-after is 0.
 326   if (request_->status().is_success() &&
 327       request_->GetResponseCode() == 503 &&
 328       MaybeRetryRequest()) {
 329     return;
 330   }
 331
 332   switch (fetch_type_) {
 333     case MANIFEST_FETCH:
 334       job_->HandleManifestFetchCompleted(this);
 335       break;
 336     case URL_FETCH:
 337       job_->HandleUrlFetchCompleted(this);
 338       break;
 339     case MASTER_ENTRY_FETCH:
 340       job_->HandleMasterEntryFetchCompleted(this);
 341       break;
 342     case MANIFEST_REFETCH:
 343       job_->HandleManifestRefetchCompleted(this);
 344       break;
 345     default:
 346       NOTREACHED();
 347   }
 348
 349   delete this;
 350 }
 351
 352 bool AppCacheUpdateJob::URLFetcher::MaybeRetryRequest() {
 353   if (retry_503_attempts_ >= kMax503Retries ||
 354       !request_->response_headers()->HasHeaderValue("retry-after", "0")) {
 355     return false;
 356   }
 357   ++retry_503_attempts_;
 358   result_ = UPDATE_OK;
 359   request_ = job_->service_->request_context()->CreateRequest(
 360       url_, net::DEFAULT_PRIORITY, this, NULL);
 361   Start();
 362   return true;
 363 }
 364
 365 AppCacheUpdateJob::AppCacheUpdateJob(AppCacheServiceImpl* service,
 366                                      AppCacheGroup* group)
 367     : service_(service),
 368       manifest_url_(group->manifest_url()),
 369       group_(group),
 370       update_type_(UNKNOWN_TYPE),
 371       internal_state_(FETCH_MANIFEST),
 372       master_entries_completed_(0),
 373       url_fetches_completed_(0),
 374       manifest_fetcher_(NULL),
 375       manifest_has_valid_mime_type_(false),
 376       stored_state_(UNSTORED),
 377       storage_(service->storage()) {
 378     service_->AddObserver(this);
 379 }
 380
 381 AppCacheUpdateJob::~AppCacheUpdateJob() {
 382   if (service_)
 383     service_->RemoveObserver(this);
 384   if (internal_state_ != COMPLETED)
 385     Cancel();
 386
 387   DCHECK(!manifest_fetcher_);
 388   DCHECK(pending_url_fetches_.empty());
 389   DCHECK(!inprogress_cache_.get());
 390   DCHECK(pending_master_entries_.empty());
 391   DCHECK(master_entry_fetches_.empty());
 392
 393   if (group_)
 394     group_->SetUpdateAppCacheStatus(AppCacheGroup::IDLE);
 395 }
 396
 397 void AppCacheUpdateJob::StartUpdate(AppCacheHost* host,
 398                                     const GURL& new_master_resource) {
 399   DCHECK(group_->update_job() == this);
 400   DCHECK(!group_->is_obsolete());
 401
 402   bool is_new_pending_master_entry = false;
 403   if (!new_master_resource.is_empty()) {
 404     DCHECK(new_master_resource == host->pending_master_entry_url());
 405     DCHECK(!new_master_resource.has_ref());
 406     DCHECK(new_master_resource.GetOrigin() == manifest_url_.GetOrigin());
 407
 408     // Cannot add more to this update if already terminating.
 409     if (IsTerminating()) {
 410       group_->QueueUpdate(host, new_master_resource);
 411       return;
 412     }
 413
 414     std::pair<PendingMasters::iterator, bool> ret =
 415         pending_master_entries_.insert(
 416             PendingMasters::value_type(new_master_resource, PendingHosts()));
 417     is_new_pending_master_entry = ret.second;
 418     ret.first->second.push_back(host);
 419     host->AddObserver(this);
 420   }
 421
 422   // Notify host (if any) if already checking or downloading.
 423   AppCacheGroup::UpdateAppCacheStatus update_status = group_->update_status();
 424   if (update_status == AppCacheGroup::CHECKING ||
 425       update_status == AppCacheGroup::DOWNLOADING) {
 426     if (host) {
 427       NotifySingleHost(host, APPCACHE_CHECKING_EVENT);
 428       if (update_status == AppCacheGroup::DOWNLOADING)
 429         NotifySingleHost(host, APPCACHE_DOWNLOADING_EVENT);
 430
 431       // Add to fetch list or an existing entry if already fetched.
 432       if (!new_master_resource.is_empty()) {
 433         AddMasterEntryToFetchList(host, new_master_resource,
 434                                   is_new_pending_master_entry);
 435       }
 436     }
 437     return;
 438   }
 439
 440   // Begin update process for the group.
 441   MadeProgress();
 442   group_->SetUpdateAppCacheStatus(AppCacheGroup::CHECKING);
 443   if (group_->HasCache()) {
 444     update_type_ = UPGRADE_ATTEMPT;
 445     NotifyAllAssociatedHosts(APPCACHE_CHECKING_EVENT);
 446   } else {
 447     update_type_ = CACHE_ATTEMPT;
 448     DCHECK(host);
 449     NotifySingleHost(host, APPCACHE_CHECKING_EVENT);
 450   }
 451
 452   if (!new_master_resource.is_empty()) {
 453     AddMasterEntryToFetchList(host, new_master_resource,
 454                               is_new_pending_master_entry);
 455   }
 456
 457   FetchManifest(true);
 458 }
 459
 460 AppCacheResponseWriter* AppCacheUpdateJob::CreateResponseWriter() {
 461   AppCacheResponseWriter* writer =
 462       storage_->CreateResponseWriter(manifest_url_,
 463                                                 group_->group_id());
 464   stored_response_ids_.push_back(writer->response_id());
 465   return writer;
 466 }
 467
 468 void AppCacheUpdateJob::HandleCacheFailure(
 469     const AppCacheErrorDetails& error_details,
 470     ResultType result,
 471     const GURL& failed_resource_url) {
 472   // 6.9.4 cache failure steps 2-8.
 473   DCHECK(internal_state_ != CACHE_FAILURE);
 474   DCHECK(!error_details.message.empty());
 475   DCHECK(result != UPDATE_OK);
 476   internal_state_ = CACHE_FAILURE;
 477   LogHistogramStats(result, failed_resource_url);
 478   CancelAllUrlFetches();
 479   CancelAllMasterEntryFetches(error_details);
 480   NotifyAllError(error_details);
 481   DiscardInprogressCache();
 482   internal_state_ = COMPLETED;
 483   DeleteSoon();  // To unwind the stack prior to deletion.
 484 }
 485
 486 void AppCacheUpdateJob::FetchManifest(bool is_first_fetch) {
 487   DCHECK(!manifest_fetcher_);
 488   manifest_fetcher_ = new URLFetcher(
 489      manifest_url_,
 490      is_first_fetch ? URLFetcher::MANIFEST_FETCH :
 491                       URLFetcher::MANIFEST_REFETCH,
 492      this);
 493
 494   // Add any necessary Http headers before sending fetch request.
 495   if (is_first_fetch) {
 496     AppCacheEntry* entry = (update_type_ == UPGRADE_ATTEMPT) ?
 497         group_->newest_complete_cache()->GetEntry(manifest_url_) : NULL;
 498     if (entry) {
 499       // Asynchronously load response info for manifest from newest cache.
 500       storage_->LoadResponseInfo(manifest_url_, group_->group_id(),
 501                                  entry->response_id(), this);
 502     } else {
 503       manifest_fetcher_->Start();
 504     }
 505   } else {
 506     DCHECK(internal_state_ == REFETCH_MANIFEST);
 507     DCHECK(manifest_response_info_.get());
 508     manifest_fetcher_->set_existing_response_headers(
 509         manifest_response_info_->headers.get());
 510     manifest_fetcher_->Start();
 511   }
 512 }
 513
 514
 515 void AppCacheUpdateJob::HandleManifestFetchCompleted(
 516     URLFetcher* fetcher) {
 517   DCHECK_EQ(internal_state_, FETCH_MANIFEST);
 518   DCHECK_EQ(manifest_fetcher_, fetcher);
 519   manifest_fetcher_ = NULL;
 520
 521   net::URLRequest* request = fetcher->request();
 522   int response_code = -1;
 523   bool is_valid_response_code = false;
 524   if (request->status().is_success()) {
 525     response_code = request->GetResponseCode();
 526     is_valid_response_code = (response_code / 100 == 2);
 527
 528     std::string mime_type;
 529     request->GetMimeType(&mime_type);
 530     manifest_has_valid_mime_type_ = (mime_type == "text/cache-manifest");
 531   }
 532
 533   if (is_valid_response_code) {
 534     manifest_data_ = fetcher->manifest_data();
 535     manifest_response_info_.reset(
 536         new net::HttpResponseInfo(request->response_info()));
 537     if (update_type_ == UPGRADE_ATTEMPT)
 538       CheckIfManifestChanged();  // continues asynchronously
 539     else
 540       ContinueHandleManifestFetchCompleted(true);
 541   } else if (response_code == 304 && update_type_ == UPGRADE_ATTEMPT) {
 542     ContinueHandleManifestFetchCompleted(false);
 543   } else if ((response_code == 404 || response_code == 410) &&
 544              update_type_ == UPGRADE_ATTEMPT) {
 545     storage_->MakeGroupObsolete(group_, this, response_code);  // async
 546   } else {
 547     const char* kFormatString = "Manifest fetch failed (%d) %s";
 548     std::string message = FormatUrlErrorMessage(
 549         kFormatString, manifest_url_, fetcher->result(), response_code);
 550     HandleCacheFailure(AppCacheErrorDetails(message,
 551                                     APPCACHE_MANIFEST_ERROR,
 552                                     manifest_url_,
 553                                     response_code,
 554                                     false /*is_cross_origin*/),
 555                        fetcher->result(),
 556                        GURL());
 557   }
 558 }
 559
 560 void AppCacheUpdateJob::OnGroupMadeObsolete(AppCacheGroup* group,
 561                                             bool success,
 562                                             int response_code) {
 563   DCHECK(master_entry_fetches_.empty());
 564   CancelAllMasterEntryFetches(AppCacheErrorDetails(
 565       "The cache has been made obsolete, "
 566       "the manifest file returned 404 or 410",
 567       APPCACHE_MANIFEST_ERROR,
 568       GURL(),
 569       response_code,
 570       false /*is_cross_origin*/));
 571   if (success) {
 572     DCHECK(group->is_obsolete());
 573     NotifyAllAssociatedHosts(APPCACHE_OBSOLETE_EVENT);
 574     internal_state_ = COMPLETED;
 575     MaybeCompleteUpdate();
 576   } else {
 577     // Treat failure to mark group obsolete as a cache failure.
 578     HandleCacheFailure(AppCacheErrorDetails(
 579         "Failed to mark the cache as obsolete",
 580         APPCACHE_UNKNOWN_ERROR,
 581         GURL(),
 582         0,
 583         false /*is_cross_origin*/),
 584                        DB_ERROR,
 585                        GURL());
 586   }
 587 }
 588
 589 void AppCacheUpdateJob::ContinueHandleManifestFetchCompleted(bool changed) {
 590   DCHECK(internal_state_ == FETCH_MANIFEST);
 591
 592   if (!changed) {
 593     DCHECK(update_type_ == UPGRADE_ATTEMPT);
 594     internal_state_ = NO_UPDATE;
 595
 596     // Wait for pending master entries to download.
 597     FetchMasterEntries();
 598     MaybeCompleteUpdate();  // if not done, run async 6.9.4 step 7 substeps
 599     return;
 600   }
 601
 602   AppCacheManifest manifest;
 603   if (!ParseManifest(manifest_url_, manifest_data_.data(),
 604                      manifest_data_.length(),
 605                      manifest_has_valid_mime_type_ ?
 606                         PARSE_MANIFEST_ALLOWING_INTERCEPTS :
 607                         PARSE_MANIFEST_PER_STANDARD,
 608                      manifest)) {
 609     const char* kFormatString = "Failed to parse manifest %s";
 610     const std::string message = base::StringPrintf(kFormatString,
 611         manifest_url_.spec().c_str());
 612     HandleCacheFailure(
 613         AppCacheErrorDetails(
 614             message, APPCACHE_SIGNATURE_ERROR, GURL(), 0,
 615             false /*is_cross_origin*/),
 616         MANIFEST_ERROR,
 617         GURL());
 618     VLOG(1) << message;
 619     return;
 620   }
 621
 622   // Proceed with update process. Section 6.9.4 steps 8-20.
 623   internal_state_ = DOWNLOADING;
 624   inprogress_cache_ = new AppCache(storage_, storage_->NewCacheId());
 625   BuildUrlFileList(manifest);
 626   inprogress_cache_->InitializeWithManifest(&manifest);
 627
 628   // Associate all pending master hosts with the newly created cache.
 629   for (PendingMasters::iterator it = pending_master_entries_.begin();
 630        it != pending_master_entries_.end(); ++it) {
 631     PendingHosts& hosts = it->second;
 632     for (PendingHosts::iterator host_it = hosts.begin();
 633          host_it != hosts.end(); ++host_it) {
 634       (*host_it)
 635           ->AssociateIncompleteCache(inprogress_cache_.get(), manifest_url_);
 636     }
 637   }
 638
 639   if (manifest.did_ignore_intercept_namespaces) {
 640     // Must be done after associating all pending master hosts.
 641     std::string message(
 642         "Ignoring the INTERCEPT section of the application cache manifest "
 643         "because the content type is not text/cache-manifest");
 644     LogConsoleMessageToAll(message);
 645   }
 646
 647   group_->SetUpdateAppCacheStatus(AppCacheGroup::DOWNLOADING);
 648   NotifyAllAssociatedHosts(APPCACHE_DOWNLOADING_EVENT);
 649   FetchUrls();
 650   FetchMasterEntries();
 651   MaybeCompleteUpdate();  // if not done, continues when async fetches complete
 652 }
 653
 654 void AppCacheUpdateJob::HandleUrlFetchCompleted(URLFetcher* fetcher) {
 655   DCHECK(internal_state_ == DOWNLOADING);
 656
 657   net::URLRequest* request = fetcher->request();
 658   const GURL& url = request->original_url();
 659   pending_url_fetches_.erase(url);
 660   NotifyAllProgress(url);
 661   ++url_fetches_completed_;
 662
 663   int response_code = request->status().is_success()
 664                           ? request->GetResponseCode()
 665                           : fetcher->redirect_response_code();
 666
 667   AppCacheEntry& entry = url_file_list_.find(url)->second;
 668
 669   if (response_code / 100 == 2) {
 670     // Associate storage with the new entry.
 671     DCHECK(fetcher->response_writer());
 672     entry.set_response_id(fetcher->response_writer()->response_id());
 673     entry.set_response_size(fetcher->response_writer()->amount_written());
 674     if (!inprogress_cache_->AddOrModifyEntry(url, entry))
 675       duplicate_response_ids_.push_back(entry.response_id());
 676
 677     // TODO(michaeln): Check for <html manifest=xxx>
 678     // See http://code.google.com/p/chromium/issues/detail?id=97930
 679     // if (entry.IsMaster() && !(entry.IsExplicit() || fallback || intercept))
 680     //   if (!manifestAttribute) skip it
 681
 682     // Foreign entries will be detected during cache selection.
 683     // Note: 6.9.4, step 17.9 possible optimization: if resource is HTML or XML
 684     // file whose root element is an html element with a manifest attribute
 685     // whose value doesn't match the manifest url of the application cache
 686     // being processed, mark the entry as being foreign.
 687   } else {
 688     VLOG(1) << "Request status: " << request->status().status()
 689             << " error: " << request->status().error()
 690             << " response code: " << response_code;
 691     if (entry.IsExplicit() || entry.IsFallback() || entry.IsIntercept()) {
 692       if (response_code == 304 && fetcher->existing_entry().has_response_id()) {
 693         // Keep the existing response.
 694         entry.set_response_id(fetcher->existing_entry().response_id());
 695         entry.set_response_size(fetcher->existing_entry().response_size());
 696         inprogress_cache_->AddOrModifyEntry(url, entry);
 697       } else {
 698         const char* kFormatString = "Resource fetch failed (%d) %s";
 699         std::string message = FormatUrlErrorMessage(
 700             kFormatString, url, fetcher->result(), response_code);
 701         ResultType result = fetcher->result();
 702         bool is_cross_origin = url.GetOrigin() != manifest_url_.GetOrigin();
 703         switch (result) {
 704           case DISKCACHE_ERROR:
 705             HandleCacheFailure(
 706                 AppCacheErrorDetails(
 707                     message, APPCACHE_UNKNOWN_ERROR, GURL(), 0,
 708                     is_cross_origin),
 709                 result,
 710                 url);
 711             break;
 712           case NETWORK_ERROR:
 713             HandleCacheFailure(
 714                 AppCacheErrorDetails(message, APPCACHE_RESOURCE_ERROR, url, 0,
 715                     is_cross_origin),
 716                 result,
 717                 url);
 718             break;
 719           default:
 720             HandleCacheFailure(AppCacheErrorDetails(message,
 721                                             APPCACHE_RESOURCE_ERROR,
 722                                             url,
 723                                             response_code,
 724                                             is_cross_origin),
 725                                result,
 726                                url);
 727             break;
 728         }
 729         return;
 730       }
 731     } else if (response_code == 404 || response_code == 410) {
 732       // Entry is skipped.  They are dropped from the cache.
 733     } else if (update_type_ == UPGRADE_ATTEMPT &&
 734                fetcher->existing_entry().has_response_id()) {
 735       // Keep the existing response.
 736       // TODO(michaeln): Not sure this is a good idea. This is spec compliant
 737       // but the old resource may or may not be compatible with the new contents
 738       // of the cache. Impossible to know one way or the other.
 739       entry.set_response_id(fetcher->existing_entry().response_id());
 740       entry.set_response_size(fetcher->existing_entry().response_size());
 741       inprogress_cache_->AddOrModifyEntry(url, entry);
 742     }
 743   }
 744
 745   // Fetch another URL now that one request has completed.
 746   DCHECK(internal_state_ != CACHE_FAILURE);
 747   FetchUrls();
 748   MaybeCompleteUpdate();
 749 }
 750
 751 void AppCacheUpdateJob::HandleMasterEntryFetchCompleted(
 752     URLFetcher* fetcher) {
 753   DCHECK(internal_state_ == NO_UPDATE || internal_state_ == DOWNLOADING);
 754
 755   // TODO(jennb): Handle downloads completing during cache failure when update
 756   // no longer fetches master entries directly. For now, we cancel all pending
 757   // master entry fetches when entering cache failure state so this will never
 758   // be called in CACHE_FAILURE state.
 759
 760   net::URLRequest* request = fetcher->request();
 761   const GURL& url = request->original_url();
 762   master_entry_fetches_.erase(url);
 763   ++master_entries_completed_;
 764
 765   int response_code = request->status().is_success()
 766       ? request->GetResponseCode() : -1;
 767
 768   PendingMasters::iterator found = pending_master_entries_.find(url);
 769   DCHECK(found != pending_master_entries_.end());
 770   PendingHosts& hosts = found->second;
 771
 772   // Section 6.9.4. No update case: step 7.3, else step 22.
 773   if (response_code / 100 == 2) {
 774     // Add fetched master entry to the appropriate cache.
 775     AppCache* cache = inprogress_cache_.get() ? inprogress_cache_.get()
 776                                               : group_->newest_complete_cache();
 777     DCHECK(fetcher->response_writer());
 778     AppCacheEntry master_entry(AppCacheEntry::MASTER,
 779                                fetcher->response_writer()->response_id(),
 780                                fetcher->response_writer()->amount_written());
 781     if (cache->AddOrModifyEntry(url, master_entry))
 782       added_master_entries_.push_back(url);
 783     else
 784       duplicate_response_ids_.push_back(master_entry.response_id());
 785
 786     // In no-update case, associate host with the newest cache.
 787     if (!inprogress_cache_.get()) {
 788       // TODO(michaeln): defer until the updated cache has been stored
 789       DCHECK(cache == group_->newest_complete_cache());
 790       for (PendingHosts::iterator host_it = hosts.begin();
 791            host_it != hosts.end(); ++host_it) {
 792         (*host_it)->AssociateCompleteCache(cache);
 793       }
 794     }
 795   } else {
 796     HostNotifier host_notifier;
 797     for (PendingHosts::iterator host_it = hosts.begin();
 798          host_it != hosts.end(); ++host_it) {
 799       AppCacheHost* host = *host_it;
 800       host_notifier.AddHost(host);
 801
 802       // In downloading case, disassociate host from inprogress cache.
 803       if (inprogress_cache_.get())
 804         host->AssociateNoCache(GURL());
 805
 806       host->RemoveObserver(this);
 807     }
 808     hosts.clear();
 809
 810     const char* kFormatString = "Manifest fetch failed (%d) %s";
 811     std::string message = FormatUrlErrorMessage(
 812         kFormatString, request->url(), fetcher->result(), response_code);
 813     host_notifier.SendErrorNotifications(
 814         AppCacheErrorDetails(message,
 815                      APPCACHE_MANIFEST_ERROR,
 816                      request->url(),
 817                      response_code,
 818                      false /*is_cross_origin*/));
 819
 820     // In downloading case, update result is different if all master entries
 821     // failed vs. only some failing.
 822     if (inprogress_cache_.get()) {
 823       // Only count successful downloads to know if all master entries failed.
 824       pending_master_entries_.erase(found);
 825       --master_entries_completed_;
 826
 827       // Section 6.9.4, step 22.3.
 828       if (update_type_ == CACHE_ATTEMPT && pending_master_entries_.empty()) {
 829         HandleCacheFailure(AppCacheErrorDetails(message,
 830                                         APPCACHE_MANIFEST_ERROR,
 831                                         request->url(),
 832                                         response_code,
 833                                         false /*is_cross_origin*/),
 834                            fetcher->result(),
 835                            GURL());
 836         return;
 837       }
 838     }
 839   }
 840
 841   DCHECK(internal_state_ != CACHE_FAILURE);
 842   FetchMasterEntries();
 843   MaybeCompleteUpdate();
 844 }
 845
 846 void AppCacheUpdateJob::HandleManifestRefetchCompleted(
 847     URLFetcher* fetcher) {
 848   DCHECK(internal_state_ == REFETCH_MANIFEST);
 849   DCHECK(manifest_fetcher_ == fetcher);
 850   manifest_fetcher_ = NULL;
 851
 852   net::URLRequest* request = fetcher->request();
 853   int response_code = request->status().is_success()
 854       ? request->GetResponseCode() : -1;
 855   if (response_code == 304 || manifest_data_ == fetcher->manifest_data()) {
 856     // Only need to store response in storage if manifest is not already
 857     // an entry in the cache.
 858     AppCacheEntry* entry = inprogress_cache_->GetEntry(manifest_url_);
 859     if (entry) {
 860       entry->add_types(AppCacheEntry::MANIFEST);
 861       StoreGroupAndCache();
 862     } else {
 863       manifest_response_writer_.reset(CreateResponseWriter());
 864       scoped_refptr<HttpResponseInfoIOBuffer> io_buffer(
 865           new HttpResponseInfoIOBuffer(manifest_response_info_.release()));
 866       manifest_response_writer_->WriteInfo(
 867           io_buffer.get(),
 868           base::Bind(&AppCacheUpdateJob::OnManifestInfoWriteComplete,
 869                      base::Unretained(this)));
 870     }
 871   } else {
 872     VLOG(1) << "Request status: " << request->status().status()
 873             << " error: " << request->status().error()
 874             << " response code: " << response_code;
 875     ScheduleUpdateRetry(kRerunDelayMs);
 876     if (response_code == 200) {
 877       HandleCacheFailure(AppCacheErrorDetails("Manifest changed during update",
 878                                       APPCACHE_CHANGED_ERROR,
 879                                       GURL(),
 880                                       0,
 881                                       false /*is_cross_origin*/),
 882                          MANIFEST_ERROR,
 883                          GURL());
 884     } else {
 885       const char* kFormatString = "Manifest re-fetch failed (%d) %s";
 886       std::string message = FormatUrlErrorMessage(
 887           kFormatString, manifest_url_, fetcher->result(), response_code);
 888       HandleCacheFailure(AppCacheErrorDetails(message,
 889                                       APPCACHE_MANIFEST_ERROR,
 890                                       GURL(),
 891                                       response_code,
 892                                       false /*is_cross_origin*/),
 893                          fetcher->result(),
 894                          GURL());
 895     }
 896   }
 897 }
 898
 899 void AppCacheUpdateJob::OnManifestInfoWriteComplete(int result) {
 900   if (result > 0) {
 901     scoped_refptr<net::StringIOBuffer> io_buffer(
 902         new net::StringIOBuffer(manifest_data_));
 903     manifest_response_writer_->WriteData(
 904         io_buffer.get(),
 905         manifest_data_.length(),
 906         base::Bind(&AppCacheUpdateJob::OnManifestDataWriteComplete,
 907                    base::Unretained(this)));
 908   } else {
 909     HandleCacheFailure(
 910         AppCacheErrorDetails("Failed to write the manifest headers to storage",
 911                      APPCACHE_UNKNOWN_ERROR,
 912                      GURL(),
 913                      0,
 914                      false /*is_cross_origin*/),
 915         DISKCACHE_ERROR,
 916         GURL());
 917   }
 918 }
 919
 920 void AppCacheUpdateJob::OnManifestDataWriteComplete(int result) {
 921   if (result > 0) {
 922     AppCacheEntry entry(AppCacheEntry::MANIFEST,
 923         manifest_response_writer_->response_id(),
 924         manifest_response_writer_->amount_written());
 925     if (!inprogress_cache_->AddOrModifyEntry(manifest_url_, entry))
 926       duplicate_response_ids_.push_back(entry.response_id());
 927     StoreGroupAndCache();
 928   } else {
 929     HandleCacheFailure(
 930         AppCacheErrorDetails("Failed to write the manifest data to storage",
 931                      APPCACHE_UNKNOWN_ERROR,
 932                      GURL(),
 933                      0,
 934                      false /*is_cross_origin*/),
 935         DISKCACHE_ERROR,
 936         GURL());
 937   }
 938 }
 939
 940 void AppCacheUpdateJob::StoreGroupAndCache() {
 941   DCHECK(stored_state_ == UNSTORED);
 942   stored_state_ = STORING;
 943   scoped_refptr<AppCache> newest_cache;
 944   if (inprogress_cache_.get())
 945     newest_cache.swap(inprogress_cache_);
 946   else
 947     newest_cache = group_->newest_complete_cache();
 948   newest_cache->set_update_time(base::Time::Now());
 949
 950   // TODO(michaeln): dcheck is fishing for clues to crbug/95101
 951   DCHECK_EQ(manifest_url_, group_->manifest_url());
 952   storage_->StoreGroupAndNewestCache(group_, newest_cache.get(), this);
 953 }
 954
 955 void AppCacheUpdateJob::OnGroupAndNewestCacheStored(AppCacheGroup* group,
 956                                                     AppCache* newest_cache,
 957                                                     bool success,
 958                                                     bool would_exceed_quota) {
 959   DCHECK(stored_state_ == STORING);
 960   if (success) {
 961     stored_state_ = STORED;
 962     MaybeCompleteUpdate();  // will definitely complete
 963   } else {
 964     stored_state_ = UNSTORED;
 965
 966     // Restore inprogress_cache_ to get the proper events delivered
 967     // and the proper cleanup to occur.
 968     if (newest_cache != group->newest_complete_cache())
 969       inprogress_cache_ = newest_cache;
 970
 971     ResultType result = DB_ERROR;
 972     AppCacheErrorReason reason = APPCACHE_UNKNOWN_ERROR;
 973     std::string message("Failed to commit new cache to storage");
 974     if (would_exceed_quota) {
 975       message.append(", would exceed quota");
 976       result = QUOTA_ERROR;
 977       reason = APPCACHE_QUOTA_ERROR;
 978     }
 979     HandleCacheFailure(
 980         AppCacheErrorDetails(message, reason, GURL(), 0,
 981             false /*is_cross_origin*/),
 982         result,
 983         GURL());
 984   }
 985 }
 986
 987 void AppCacheUpdateJob::NotifySingleHost(AppCacheHost* host,
 988                                          AppCacheEventID event_id) {
 989   std::vector<int> ids(1, host->host_id());
 990   host->frontend()->OnEventRaised(ids, event_id);
 991 }
 992
 993 void AppCacheUpdateJob::NotifyAllAssociatedHosts(AppCacheEventID event_id) {
 994   HostNotifier host_notifier;
 995   AddAllAssociatedHostsToNotifier(&host_notifier);
 996   host_notifier.SendNotifications(event_id);
 997 }
 998
 999 void AppCacheUpdateJob::NotifyAllProgress(const GURL& url) {
1000   HostNotifier host_notifier;
1001   AddAllAssociatedHostsToNotifier(&host_notifier);
1002   host_notifier.SendProgressNotifications(
1003       url, url_file_list_.size(), url_fetches_completed_);
1004 }
1005
1006 void AppCacheUpdateJob::NotifyAllFinalProgress() {
1007   DCHECK(url_file_list_.size() == url_fetches_completed_);
1008   NotifyAllProgress(GURL());
1009 }
1010
1011 void AppCacheUpdateJob::NotifyAllError(const AppCacheErrorDetails& details) {
1012   HostNotifier host_notifier;
1013   AddAllAssociatedHostsToNotifier(&host_notifier);
1014   host_notifier.SendErrorNotifications(details);
1015 }
1016
1017 void AppCacheUpdateJob::LogConsoleMessageToAll(const std::string& message) {
1018   HostNotifier host_notifier;
1019   AddAllAssociatedHostsToNotifier(&host_notifier);
1020   host_notifier.SendLogMessage(message);
1021 }
1022
1023 void AppCacheUpdateJob::AddAllAssociatedHostsToNotifier(
1024     HostNotifier* host_notifier) {
1025   // Collect hosts so we only send one notification per frontend.
1026   // A host can only be associated with a single cache so no need to worry
1027   // about duplicate hosts being added to the notifier.
1028   if (inprogress_cache_.get()) {
1029     DCHECK(internal_state_ == DOWNLOADING || internal_state_ == CACHE_FAILURE);
1030     host_notifier->AddHosts(inprogress_cache_->associated_hosts());
1031   }
1032
1033   AppCacheGroup::Caches old_caches = group_->old_caches();
1034   for (AppCacheGroup::Caches::const_iterator it = old_caches.begin();
1035        it != old_caches.end(); ++it) {
1036     host_notifier->AddHosts((*it)->associated_hosts());
1037   }
1038
1039   AppCache* newest_cache = group_->newest_complete_cache();
1040   if (newest_cache)
1041     host_notifier->AddHosts(newest_cache->associated_hosts());
1042 }
1043
1044 void AppCacheUpdateJob::OnDestructionImminent(AppCacheHost* host) {
1045   // The host is about to be deleted; remove from our collection.
1046   PendingMasters::iterator found =
1047       pending_master_entries_.find(host->pending_master_entry_url());
1048   DCHECK(found != pending_master_entries_.end());
1049   PendingHosts& hosts = found->second;
1050   PendingHosts::iterator it = std::find(hosts.begin(), hosts.end(), host);
1051   DCHECK(it != hosts.end());
1052   hosts.erase(it);
1053 }
1054
1055 void AppCacheUpdateJob::OnServiceReinitialized(
1056     AppCacheStorageReference* old_storage_ref) {
1057   // We continue to use the disabled instance, but arrange for its
1058   // deletion when its no longer needed.
1059   if (old_storage_ref->storage() == storage_)
1060     disabled_storage_reference_ = old_storage_ref;
1061 }
1062
1063 void AppCacheUpdateJob::CheckIfManifestChanged() {
1064   DCHECK(update_type_ == UPGRADE_ATTEMPT);
1065   AppCacheEntry* entry = NULL;
1066   if (group_->newest_complete_cache())
1067     entry = group_->newest_complete_cache()->GetEntry(manifest_url_);
1068   if (!entry) {
1069     // TODO(michaeln): This is just a bandaid to avoid a crash.
1070     // http://code.google.com/p/chromium/issues/detail?id=95101
1071     if (service_->storage() == storage_) {
1072       // Use a local variable because service_ is reset in HandleCacheFailure.
1073       AppCacheServiceImpl* service = service_;
1074       HandleCacheFailure(
1075           AppCacheErrorDetails("Manifest entry not found in existing cache",
1076                        APPCACHE_UNKNOWN_ERROR,
1077                        GURL(),
1078                        0,
1079                        false /*is_cross_origin*/),
1080           DB_ERROR,
1081           GURL());
1082       AppCacheHistograms::AddMissingManifestEntrySample();
1083       service->DeleteAppCacheGroup(manifest_url_, net::CompletionCallback());
1084     }
1085     return;
1086   }
1087
1088   // Load manifest data from storage to compare against fetched manifest.
1089   manifest_response_reader_.reset(
1090       storage_->CreateResponseReader(manifest_url_,
1091                                      group_->group_id(),
1092                                      entry->response_id()));
1093   read_manifest_buffer_ = new net::IOBuffer(kBufferSize);
1094   manifest_response_reader_->ReadData(
1095       read_manifest_buffer_.get(),
1096       kBufferSize,
1097       base::Bind(&AppCacheUpdateJob::OnManifestDataReadComplete,
1098                  base::Unretained(this)));  // async read
1099 }
1100
1101 void AppCacheUpdateJob::OnManifestDataReadComplete(int result) {
1102   if (result > 0) {
1103     loaded_manifest_data_.append(read_manifest_buffer_->data(), result);
1104     manifest_response_reader_->ReadData(
1105         read_manifest_buffer_.get(),
1106         kBufferSize,
1107         base::Bind(&AppCacheUpdateJob::OnManifestDataReadComplete,
1108                    base::Unretained(this)));  // read more
1109   } else {
1110     read_manifest_buffer_ = NULL;
1111     manifest_response_reader_.reset();
1112     ContinueHandleManifestFetchCompleted(
1113         result < 0 || manifest_data_ != loaded_manifest_data_);
1114   }
1115 }
1116
1117 void AppCacheUpdateJob::BuildUrlFileList(const AppCacheManifest& manifest) {
1118   for (base::hash_set<std::string>::const_iterator it =
1119            manifest.explicit_urls.begin();
1120        it != manifest.explicit_urls.end(); ++it) {
1121     AddUrlToFileList(GURL(*it), AppCacheEntry::EXPLICIT);
1122   }
1123
1124   const std::vector<AppCacheNamespace>& intercepts =
1125       manifest.intercept_namespaces;
1126   for (std::vector<AppCacheNamespace>::const_iterator it = intercepts.begin();
1127        it != intercepts.end(); ++it) {
1128     int flags = AppCacheEntry::INTERCEPT;
1129     if (it->is_executable)
1130       flags |= AppCacheEntry::EXECUTABLE;
1131     AddUrlToFileList(it->target_url, flags);
1132   }
1133
1134   const std::vector<AppCacheNamespace>& fallbacks =
1135       manifest.fallback_namespaces;
1136   for (std::vector<AppCacheNamespace>::const_iterator it = fallbacks.begin();
1137        it != fallbacks.end(); ++it) {
1138      AddUrlToFileList(it->target_url, AppCacheEntry::FALLBACK);
1139   }
1140
1141   // Add all master entries from newest complete cache.
1142   if (update_type_ == UPGRADE_ATTEMPT) {
1143     const AppCache::EntryMap& entries =
1144         group_->newest_complete_cache()->entries();
1145     for (AppCache::EntryMap::const_iterator it = entries.begin();
1146          it != entries.end(); ++it) {
1147       const AppCacheEntry& entry = it->second;
1148       if (entry.IsMaster())
1149         AddUrlToFileList(it->first, AppCacheEntry::MASTER);
1150     }
1151   }
1152 }
1153
1154 void AppCacheUpdateJob::AddUrlToFileList(const GURL& url, int type) {
1155   std::pair<AppCache::EntryMap::iterator, bool> ret = url_file_list_.insert(
1156       AppCache::EntryMap::value_type(url, AppCacheEntry(type)));
1157
1158   if (ret.second)
1159     urls_to_fetch_.push_back(UrlToFetch(url, false, NULL));
1160   else
1161     ret.first->second.add_types(type);  // URL already exists. Merge types.
1162 }
1163
1164 void AppCacheUpdateJob::FetchUrls() {
1165   DCHECK(internal_state_ == DOWNLOADING);
1166
1167   // Fetch each URL in the list according to section 6.9.4 step 17.1-17.3.
1168   // Fetch up to the concurrent limit. Other fetches will be triggered as each
1169   // each fetch completes.
1170   while (pending_url_fetches_.size() < kMaxConcurrentUrlFetches &&
1171          !urls_to_fetch_.empty()) {
1172     UrlToFetch url_to_fetch = urls_to_fetch_.front();
1173     urls_to_fetch_.pop_front();
1174
1175     AppCache::EntryMap::iterator it = url_file_list_.find(url_to_fetch.url);
1176     DCHECK(it != url_file_list_.end());
1177     AppCacheEntry& entry = it->second;
1178     if (ShouldSkipUrlFetch(entry)) {
1179       NotifyAllProgress(url_to_fetch.url);
1180       ++url_fetches_completed_;
1181     } else if (AlreadyFetchedEntry(url_to_fetch.url, entry.types())) {
1182       NotifyAllProgress(url_to_fetch.url);
1183       ++url_fetches_completed_;  // saved a URL request
1184     } else if (!url_to_fetch.storage_checked &&
1185                MaybeLoadFromNewestCache(url_to_fetch.url, entry)) {
1186       // Continues asynchronously after data is loaded from newest cache.
1187     } else {
1188       URLFetcher* fetcher = new URLFetcher(
1189           url_to_fetch.url, URLFetcher::URL_FETCH, this);
1190       if (url_to_fetch.existing_response_info.get()) {
1191         DCHECK(group_->newest_complete_cache());
1192         AppCacheEntry* existing_entry =
1193             group_->newest_complete_cache()->GetEntry(url_to_fetch.url);
1194         DCHECK(existing_entry);
1195         DCHECK(existing_entry->response_id() ==
1196                url_to_fetch.existing_response_info->response_id());
1197         fetcher->set_existing_response_headers(
1198             url_to_fetch.existing_response_info->http_response_info()->headers
1199                 .get());
1200         fetcher->set_existing_entry(*existing_entry);
1201       }
1202       fetcher->Start();
1203       pending_url_fetches_.insert(
1204           PendingUrlFetches::value_type(url_to_fetch.url, fetcher));
1205     }
1206   }
1207 }
1208
1209 void AppCacheUpdateJob::CancelAllUrlFetches() {
1210   // Cancel any pending URL requests.
1211   for (PendingUrlFetches::iterator it = pending_url_fetches_.begin();
1212        it != pending_url_fetches_.end(); ++it) {
1213     delete it->second;
1214   }
1215
1216   url_fetches_completed_ +=
1217       pending_url_fetches_.size() + urls_to_fetch_.size();
1218   pending_url_fetches_.clear();
1219   urls_to_fetch_.clear();
1220 }
1221
1222 bool AppCacheUpdateJob::ShouldSkipUrlFetch(const AppCacheEntry& entry) {
1223   // 6.6.4 Step 17
1224   // If the resource URL being processed was flagged as neither an
1225   // "explicit entry" nor or a "fallback entry", then the user agent
1226   // may skip this URL.
1227   if (entry.IsExplicit() || entry.IsFallback() || entry.IsIntercept())
1228     return false;
1229
1230   // TODO(jennb): decide if entry should be skipped to expire it from cache
1231   return false;
1232 }
1233
1234 bool AppCacheUpdateJob::AlreadyFetchedEntry(const GURL& url,
1235                                             int entry_type) {
1236   DCHECK(internal_state_ == DOWNLOADING || internal_state_ == NO_UPDATE);
1237   AppCacheEntry* existing =
1238       inprogress_cache_.get() ? inprogress_cache_->GetEntry(url)
1239                               : group_->newest_complete_cache()->GetEntry(url);
1240   if (existing) {
1241     existing->add_types(entry_type);
1242     return true;
1243   }
1244   return false;
1245 }
1246
1247 void AppCacheUpdateJob::AddMasterEntryToFetchList(AppCacheHost* host,
1248                                                   const GURL& url,
1249                                                   bool is_new) {
1250   DCHECK(!IsTerminating());
1251
1252   if (internal_state_ == DOWNLOADING || internal_state_ == NO_UPDATE) {
1253     AppCache* cache;
1254     if (inprogress_cache_.get()) {
1255       // always associate
1256       host->AssociateIncompleteCache(inprogress_cache_.get(), manifest_url_);
1257       cache = inprogress_cache_.get();
1258     } else {
1259       cache = group_->newest_complete_cache();
1260     }
1261
1262     // Update existing entry if it has already been fetched.
1263     AppCacheEntry* entry = cache->GetEntry(url);
1264     if (entry) {
1265       entry->add_types(AppCacheEntry::MASTER);
1266       if (internal_state_ == NO_UPDATE && !inprogress_cache_.get()) {
1267         // only associate if have entry
1268         host->AssociateCompleteCache(cache);
1269       }
1270       if (is_new)
1271         ++master_entries_completed_;  // pretend fetching completed
1272       return;
1273     }
1274   }
1275
1276   // Add to fetch list if not already fetching.
1277   if (master_entry_fetches_.find(url) == master_entry_fetches_.end()) {
1278     master_entries_to_fetch_.insert(url);
1279     if (internal_state_ == DOWNLOADING || internal_state_ == NO_UPDATE)
1280       FetchMasterEntries();
1281   }
1282 }
1283
1284 void AppCacheUpdateJob::FetchMasterEntries() {
1285   DCHECK(internal_state_ == NO_UPDATE || internal_state_ == DOWNLOADING);
1286
1287   // Fetch each master entry in the list, up to the concurrent limit.
1288   // Additional fetches will be triggered as each fetch completes.
1289   while (master_entry_fetches_.size() < kMaxConcurrentUrlFetches &&
1290          !master_entries_to_fetch_.empty()) {
1291     const GURL& url = *master_entries_to_fetch_.begin();
1292
1293     if (AlreadyFetchedEntry(url, AppCacheEntry::MASTER)) {
1294       ++master_entries_completed_;  // saved a URL request
1295
1296       // In no update case, associate hosts to newest cache in group
1297       // now that master entry has been "successfully downloaded".
1298       if (internal_state_ == NO_UPDATE) {
1299         // TODO(michaeln): defer until the updated cache has been stored.
1300         DCHECK(!inprogress_cache_.get());
1301         AppCache* cache = group_->newest_complete_cache();
1302         PendingMasters::iterator found = pending_master_entries_.find(url);
1303         DCHECK(found != pending_master_entries_.end());
1304         PendingHosts& hosts = found->second;
1305         for (PendingHosts::iterator host_it = hosts.begin();
1306              host_it != hosts.end(); ++host_it) {
1307           (*host_it)->AssociateCompleteCache(cache);
1308         }
1309       }
1310     } else {
1311       URLFetcher* fetcher = new URLFetcher(
1312           url, URLFetcher::MASTER_ENTRY_FETCH, this);
1313       fetcher->Start();
1314       master_entry_fetches_.insert(PendingUrlFetches::value_type(url, fetcher));
1315     }
1316
1317     master_entries_to_fetch_.erase(master_entries_to_fetch_.begin());
1318   }
1319 }
1320
1321 void AppCacheUpdateJob::CancelAllMasterEntryFetches(
1322     const AppCacheErrorDetails& error_details) {
1323   // For now, cancel all in-progress fetches for master entries and pretend
1324   // all master entries fetches have completed.
1325   // TODO(jennb): Delete this when update no longer fetches master entries
1326   // directly.
1327
1328   // Cancel all in-progress fetches.
1329   for (PendingUrlFetches::iterator it = master_entry_fetches_.begin();
1330        it != master_entry_fetches_.end(); ++it) {
1331     delete it->second;
1332     master_entries_to_fetch_.insert(it->first);  // back in unfetched list
1333   }
1334   master_entry_fetches_.clear();
1335
1336   master_entries_completed_ += master_entries_to_fetch_.size();
1337
1338   // Cache failure steps, step 2.
1339   // Pretend all master entries that have not yet been fetched have completed
1340   // downloading. Unassociate hosts from any appcache and send ERROR event.
1341   HostNotifier host_notifier;
1342   while (!master_entries_to_fetch_.empty()) {
1343     const GURL& url = *master_entries_to_fetch_.begin();
1344     PendingMasters::iterator found = pending_master_entries_.find(url);
1345     DCHECK(found != pending_master_entries_.end());
1346     PendingHosts& hosts = found->second;
1347     for (PendingHosts::iterator host_it = hosts.begin();
1348          host_it != hosts.end(); ++host_it) {
1349       AppCacheHost* host = *host_it;
1350       host->AssociateNoCache(GURL());
1351       host_notifier.AddHost(host);
1352       host->RemoveObserver(this);
1353     }
1354     hosts.clear();
1355
1356     master_entries_to_fetch_.erase(master_entries_to_fetch_.begin());
1357   }
1358   host_notifier.SendErrorNotifications(error_details);
1359 }
1360
1361 bool AppCacheUpdateJob::MaybeLoadFromNewestCache(const GURL& url,
1362                                                  AppCacheEntry& entry) {
1363   if (update_type_ != UPGRADE_ATTEMPT)
1364     return false;
1365
1366   AppCache* newest = group_->newest_complete_cache();
1367   AppCacheEntry* copy_me = newest->GetEntry(url);
1368   if (!copy_me || !copy_me->has_response_id())
1369     return false;
1370
1371   // Load HTTP headers for entry from newest cache.
1372   loading_responses_.insert(
1373       LoadingResponses::value_type(copy_me->response_id(), url));
1374   storage_->LoadResponseInfo(manifest_url_, group_->group_id(),
1375                              copy_me->response_id(),
1376                              this);
1377   // Async: wait for OnResponseInfoLoaded to complete.
1378   return true;
1379 }
1380
1381 void AppCacheUpdateJob::OnResponseInfoLoaded(
1382     AppCacheResponseInfo* response_info, int64 response_id) {
1383   const net::HttpResponseInfo* http_info = response_info ?
1384       response_info->http_response_info() : NULL;
1385
1386   // Needed response info for a manifest fetch request.
1387   if (internal_state_ == FETCH_MANIFEST) {
1388     if (http_info)
1389       manifest_fetcher_->set_existing_response_headers(
1390           http_info->headers.get());
1391     manifest_fetcher_->Start();
1392     return;
1393   }
1394
1395   LoadingResponses::iterator found = loading_responses_.find(response_id);
1396   DCHECK(found != loading_responses_.end());
1397   const GURL& url = found->second;
1398
1399   if (!http_info) {
1400     LoadFromNewestCacheFailed(url, NULL);  // no response found
1401   } else {
1402     // Check if response can be re-used according to HTTP caching semantics.
1403     // Responses with a "vary" header get treated as expired.
1404     const std::string name = "vary";
1405     std::string value;
1406     void* iter = NULL;
1407     if (!http_info->headers.get() ||
1408         http_info->headers->RequiresValidation(http_info->request_time,
1409                                                http_info->response_time,
1410                                                base::Time::Now()) ||
1411         http_info->headers->EnumerateHeader(&iter, name, &value)) {
1412       LoadFromNewestCacheFailed(url, response_info);
1413     } else {
1414       DCHECK(group_->newest_complete_cache());
1415       AppCacheEntry* copy_me = group_->newest_complete_cache()->GetEntry(url);
1416       DCHECK(copy_me);
1417       DCHECK(copy_me->response_id() == response_id);
1418
1419       AppCache::EntryMap::iterator it = url_file_list_.find(url);
1420       DCHECK(it != url_file_list_.end());
1421       AppCacheEntry& entry = it->second;
1422       entry.set_response_id(response_id);
1423       entry.set_response_size(copy_me->response_size());
1424       inprogress_cache_->AddOrModifyEntry(url, entry);
1425       NotifyAllProgress(url);
1426       ++url_fetches_completed_;
1427     }
1428   }
1429   loading_responses_.erase(found);
1430
1431   MaybeCompleteUpdate();
1432 }
1433
1434 void AppCacheUpdateJob::LoadFromNewestCacheFailed(
1435     const GURL& url, AppCacheResponseInfo* response_info) {
1436   if (internal_state_ == CACHE_FAILURE)
1437     return;
1438
1439   // Re-insert url at front of fetch list. Indicate storage has been checked.
1440   urls_to_fetch_.push_front(UrlToFetch(url, true, response_info));
1441   FetchUrls();
1442 }
1443
1444 void AppCacheUpdateJob::MaybeCompleteUpdate() {
1445   DCHECK(internal_state_ != CACHE_FAILURE);
1446
1447   // Must wait for any pending master entries or url fetches to complete.
1448   if (master_entries_completed_ != pending_master_entries_.size() ||
1449       url_fetches_completed_ != url_file_list_.size()) {
1450     DCHECK(internal_state_ != COMPLETED);
1451     return;
1452   }
1453
1454   switch (internal_state_) {
1455     case NO_UPDATE:
1456       if (master_entries_completed_ > 0) {
1457         switch (stored_state_) {
1458           case UNSTORED:
1459             StoreGroupAndCache();
1460             return;
1461           case STORING:
1462             return;
1463           case STORED:
1464             break;
1465         }
1466       }
1467       // 6.9.4 steps 7.3-7.7.
1468       NotifyAllAssociatedHosts(APPCACHE_NO_UPDATE_EVENT);
1469       DiscardDuplicateResponses();
1470       internal_state_ = COMPLETED;
1471       break;
1472     case DOWNLOADING:
1473       internal_state_ = REFETCH_MANIFEST;
1474       FetchManifest(false);
1475       break;
1476     case REFETCH_MANIFEST:
1477       DCHECK(stored_state_ == STORED);
1478       NotifyAllFinalProgress();
1479       if (update_type_ == CACHE_ATTEMPT)
1480         NotifyAllAssociatedHosts(APPCACHE_CACHED_EVENT);
1481       else
1482         NotifyAllAssociatedHosts(APPCACHE_UPDATE_READY_EVENT);
1483       DiscardDuplicateResponses();
1484       internal_state_ = COMPLETED;
1485       LogHistogramStats(UPDATE_OK, GURL());
1486       break;
1487     case CACHE_FAILURE:
1488       NOTREACHED();  // See HandleCacheFailure
1489       break;
1490     default:
1491       break;
1492   }
1493
1494   // Let the stack unwind before deletion to make it less risky as this
1495   // method is called from multiple places in this file.
1496   if (internal_state_ == COMPLETED)
1497     DeleteSoon();
1498 }
1499
1500 void AppCacheUpdateJob::ScheduleUpdateRetry(int delay_ms) {
1501   // TODO(jennb): post a delayed task with the "same parameters" as this job
1502   // to retry the update at a later time. Need group, URLs of pending master
1503   // entries and their hosts.
1504 }
1505
1506 void AppCacheUpdateJob::Cancel() {
1507   internal_state_ = CANCELLED;
1508
1509   LogHistogramStats(CANCELLED_ERROR, GURL());
1510
1511   if (manifest_fetcher_) {
1512     delete manifest_fetcher_;
1513     manifest_fetcher_ = NULL;
1514   }
1515
1516   for (PendingUrlFetches::iterator it = pending_url_fetches_.begin();
1517        it != pending_url_fetches_.end(); ++it) {
1518     delete it->second;
1519   }
1520   pending_url_fetches_.clear();
1521
1522   for (PendingUrlFetches::iterator it = master_entry_fetches_.begin();
1523        it != master_entry_fetches_.end(); ++it) {
1524     delete it->second;
1525   }
1526   master_entry_fetches_.clear();
1527
1528   ClearPendingMasterEntries();
1529   DiscardInprogressCache();
1530
1531   // Delete response writer to avoid any callbacks.
1532   if (manifest_response_writer_)
1533     manifest_response_writer_.reset();
1534
1535   storage_->CancelDelegateCallbacks(this);
1536 }
1537
1538 void AppCacheUpdateJob::ClearPendingMasterEntries() {
1539   for (PendingMasters::iterator it = pending_master_entries_.begin();
1540        it != pending_master_entries_.end(); ++it) {
1541     PendingHosts& hosts = it->second;
1542     for (PendingHosts::iterator host_it = hosts.begin();
1543          host_it != hosts.end(); ++host_it) {
1544       (*host_it)->RemoveObserver(this);
1545     }
1546   }
1547
1548   pending_master_entries_.clear();
1549 }
1550
1551 void AppCacheUpdateJob::DiscardInprogressCache() {
1552   if (stored_state_ == STORING) {
1553     // We can make no assumptions about whether the StoreGroupAndCacheTask
1554     // actually completed or not. This condition should only be reachable
1555     // during shutdown. Free things up and return to do no harm.
1556     inprogress_cache_ = NULL;
1557     added_master_entries_.clear();
1558     return;
1559   }
1560
1561   storage_->DoomResponses(manifest_url_, stored_response_ids_);
1562
1563   if (!inprogress_cache_.get()) {
1564     // We have to undo the changes we made, if any, to the existing cache.
1565     if (group_ && group_->newest_complete_cache()) {
1566       for (std::vector<GURL>::iterator iter = added_master_entries_.begin();
1567            iter != added_master_entries_.end(); ++iter) {
1568         group_->newest_complete_cache()->RemoveEntry(*iter);
1569       }
1570     }
1571     added_master_entries_.clear();
1572     return;
1573   }
1574
1575   AppCache::AppCacheHosts& hosts = inprogress_cache_->associated_hosts();
1576   while (!hosts.empty())
1577     (*hosts.begin())->AssociateNoCache(GURL());
1578
1579   inprogress_cache_ = NULL;
1580   added_master_entries_.clear();
1581 }
1582
1583 void AppCacheUpdateJob::DiscardDuplicateResponses() {
1584   storage_->DoomResponses(manifest_url_, duplicate_response_ids_);
1585 }
1586
1587 void AppCacheUpdateJob::LogHistogramStats(
1588       ResultType result, const GURL& failed_resource_url) {
1589   AppCacheHistograms::CountUpdateJobResult(result, manifest_url_.GetOrigin());
1590   if (result == UPDATE_OK)
1591     return;
1592
1593   int percent_complete = 0;
1594   if (url_file_list_.size() > 0) {
1595     size_t actual_fetches_completed = url_fetches_completed_;
1596     if (!failed_resource_url.is_empty() && actual_fetches_completed)
1597       --actual_fetches_completed;
1598     percent_complete = (static_cast<double>(actual_fetches_completed) /
1599                             static_cast<double>(url_file_list_.size())) * 100.0;
1600     percent_complete = std::min(percent_complete, 99);
1601   }
1602
1603   bool was_making_progress =
1604       base::Time::Now() - last_progress_time_ <
1605           base::TimeDelta::FromMinutes(5);
1606
1607   bool off_origin_resource_failure =
1608       !failed_resource_url.is_empty() &&
1609           (failed_resource_url.GetOrigin() != manifest_url_.GetOrigin());
1610
1611   AppCacheHistograms::LogUpdateFailureStats(
1612       manifest_url_.GetOrigin(),
1613       percent_complete,
1614       was_making_progress,
1615       off_origin_resource_failure);
1616 }
1617
1618 void AppCacheUpdateJob::DeleteSoon() {
1619   ClearPendingMasterEntries();
1620   manifest_response_writer_.reset();
1621   storage_->CancelDelegateCallbacks(this);
1622   service_->RemoveObserver(this);
1623   service_ = NULL;
1624
1625   // Break the connection with the group so the group cannot call delete
1626   // on this object after we've posted a task to delete ourselves.
1627   group_->SetUpdateAppCacheStatus(AppCacheGroup::IDLE);
1628   group_ = NULL;
1629
1630   base::MessageLoop::current()->DeleteSoon(FROM_HERE, this);
1631 }
1632
1633 }  // namespace content