1 // Copyright 2013 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
5 #include "chrome/browser/component_updater/background_downloader_win.h"
14 #include "base/file_util.h"
15 #include "base/strings/sys_string_conversions.h"
16 #include "base/win/scoped_co_mem.h"
17 #include "chrome/browser/component_updater/component_updater_utils.h"
18 #include "content/public/browser/browser_thread.h"
19 #include "ui/base/win/atl_module.h"
22 using base::win::ScopedCoMem;
23 using base::win::ScopedComPtr;
24 using content::BrowserThread;
26 // The class BackgroundDownloader in this module is an adapter between
27 // the CrxDownloader interface and the BITS service interfaces.
28 // The interface exposed on the CrxDownloader code runs on the UI thread, while
29 // the BITS specific code runs in a single threaded apartment on the FILE
31 // For every url to download, a BITS job is created, unless there is already
32 // an existing job for that url, in which case, the downloader connects to it.
33 // Once a job is associated with the url, the code looks for changes in the
34 // BITS job state. The checks are triggered by a timer.
35 // The BITS job contains just one file to download. There could only be one
36 // download in progress at a time. If Chrome closes down before the download is
37 // complete, the BITS job remains active and finishes in the background, without
38 // any intervention. The job can be completed next time the code runs, if the
39 // file is still needed, otherwise it will be cleaned up on a periodic basis.
41 // To list the BITS jobs for a user, use the |bitsadmin| tool. The command line
42 // to do that is: "bitsadmin /list /verbose". Another useful command is
43 // "bitsadmin /info" and provide the job id returned by the previous /list
46 // Ignoring the suspend/resume issues since this code is not using them, the
47 // job state machine implemented by BITS is something like this:
49 // Suspended--->Queued--->Connecting---->Transferring--->Transferred
51 // | | V V | (complete)
52 // +----------|---------+-----------------+-----+ V
53 // | | | | Acknowledged
55 // | Transient Error------->Error |
57 // | +-------+---------+--->-+
60 // +------<----------+ +---->Cancelled
62 // The job is created in the "suspended" state. Once |Resume| is called,
63 // BITS queues up the job, then tries to connect, begins transferring the
64 // job bytes, and moves the job to the "transferred state, after the job files
65 // have been transferred. When calling |Complete| for a job, the job files are
66 // made available to the caller, and the job is moved to the "acknowledged"
68 // At any point, the job can be cancelled, in which case, the job is moved
69 // to the "cancelled state" and the job object is removed from the BITS queue.
70 // Along the way, the job can encounter recoverable and non-recoverable errors.
71 // BITS moves the job to "transient error" or "error", depending on which kind
72 // of error has occured.
73 // If the job has reached the "transient error" state, BITS retries the
74 // job after a certain programmable delay. If the job can't be completed in a
75 // certain time interval, BITS stops retrying and errors the job out. This time
76 // interval is also programmable.
77 // If the job is in either of the error states, the job parameters can be
78 // adjusted to handle the error, after which the job can be resumed, and the
79 // whole cycle starts again.
80 // Jobs that are not touched in 90 days (or a value set by group policy) are
81 // automatically disposed off by BITS. This concludes the brief description of
82 // a job lifetime, according to BITS.
84 // In addition to how BITS is managing the life time of the job, there are a
85 // couple of special cases defined by the BackgroundDownloader.
86 // First, if the job encounters any of the 5xx HTTP responses, the job is
87 // not retried, in order to avoid DDOS-ing the servers.
88 // Second, there is a simple mechanism to detect stuck jobs, and allow the rest
89 // of the code to move on to trying other urls or trying other components.
90 // Last, after completing a job, irrespective of the outcome, the jobs older
91 // than a week are proactively cleaned up.
93 namespace component_updater {
97 // All jobs created by this module have a specific description so they can
98 // be found at run-time or by using system administration tools.
99 const base::char16 kJobDescription[] = L"Chrome Component Updater";
101 // How often the code looks for changes in the BITS job state.
102 const int kJobPollingIntervalSec = 10;
104 // How long BITS waits before retrying a job after the job encountered
105 // a transient error. If this value is not set, the BITS default is 10 minutes.
106 const int kMinimumRetryDelayMin = 1;
108 // How long to wait for stuck jobs. Stuck jobs could be queued for too long,
109 // have trouble connecting, could be suspended for any reason, or they have
110 // encountered some transient error.
111 const int kJobStuckTimeoutMin = 15;
113 // How long BITS waits before giving up on a job that could not be completed
114 // since the job has encountered its first transient error. If this value is
115 // not set, the BITS default is 14 days.
116 const int kSetNoProgressTimeoutDays = 1;
118 // How often the jobs which were started but not completed for any reason
119 // are cleaned up. Reasons for jobs to be left behind include browser restarts,
120 // system restarts, etc. Also, the check to purge stale jobs only happens
121 // at most once a day. If the job clean up code is not running, the BITS
122 // default policy is to cancel jobs after 90 days of inactivity.
123 const int kPurgeStaleJobsAfterDays = 7;
124 const int kPurgeStaleJobsIntervalBetweenChecksDays = 1;
126 // Returns the status code from a given BITS error.
127 int GetHttpStatusFromBitsError(HRESULT error) {
128 // BITS errors are defined in bitsmsg.h. Although not documented, it is
129 // clear that all errors corresponding to http status code have the high
130 // word equal to 0x8019 and the low word equal to the http status code.
131 const int kHttpStatusFirst = 100; // Continue.
132 const int kHttpStatusLast = 505; // Version not supported.
133 bool is_valid = HIWORD(error) == 0x8019 &&
134 LOWORD(error) >= kHttpStatusFirst &&
135 LOWORD(error) <= kHttpStatusLast;
136 return is_valid ? LOWORD(error) : 0;
139 // Returns the files in a BITS job.
140 HRESULT GetFilesInJob(IBackgroundCopyJob* job,
141 std::vector<ScopedComPtr<IBackgroundCopyFile> >* files) {
142 ScopedComPtr<IEnumBackgroundCopyFiles> enum_files;
143 HRESULT hr = job->EnumFiles(enum_files.Receive());
148 hr = enum_files->GetCount(&num_files);
152 for (ULONG i = 0; i != num_files; ++i) {
153 ScopedComPtr<IBackgroundCopyFile> file;
154 if (enum_files->Next(1, file.Receive(), NULL) == S_OK)
155 files->push_back(file);
161 // Returns the file name, the url, and some per-file progress information.
162 // The function out parameters can be NULL if that data is not requested.
163 HRESULT GetJobFileProperties(IBackgroundCopyFile* file,
164 base::string16* local_name,
165 base::string16* remote_name,
166 BG_FILE_PROGRESS* progress) {
170 ScopedCoMem<base::char16> name;
171 hr = file->GetLocalName(&name);
174 local_name->assign(name);
178 ScopedCoMem<base::char16> name;
179 hr = file->GetRemoteName(&name);
182 remote_name->assign(name);
186 BG_FILE_PROGRESS bg_file_progress = {};
187 hr = file->GetProgress(&bg_file_progress);
190 *progress = bg_file_progress;
196 // Returns the number of bytes downloaded and bytes to download for all files
197 // in the job. If the values are not known or if an error has occurred,
198 // a value of -1 is reported.
199 HRESULT GetJobByteCount(IBackgroundCopyJob* job,
200 int64* bytes_downloaded,
201 int64* bytes_total) {
202 *bytes_downloaded = -1;
208 BG_JOB_PROGRESS job_progress = {0};
209 HRESULT hr = job->GetProgress(&job_progress);
213 if (job_progress.BytesTransferred <= kint64max)
214 *bytes_downloaded = job_progress.BytesTransferred;
216 if (job_progress.BytesTotal <= kint64max &&
217 job_progress.BytesTotal != BG_SIZE_UNKNOWN)
218 *bytes_total = job_progress.BytesTotal;
223 HRESULT GetJobDescription(IBackgroundCopyJob* job, const base::string16* name) {
224 ScopedCoMem<base::char16> description;
225 return job->GetDescription(&description);
228 // Returns the job error code in |error_code| if the job is in the transient
229 // or the final error state. Otherwise, the job error is not available and
230 // the function fails.
231 HRESULT GetJobError(IBackgroundCopyJob* job, HRESULT* error_code_out) {
232 *error_code_out = S_OK;
233 ScopedComPtr<IBackgroundCopyError> copy_error;
234 HRESULT hr = job->GetError(copy_error.Receive());
238 BG_ERROR_CONTEXT error_context = BG_ERROR_CONTEXT_NONE;
239 HRESULT error_code = S_OK;
240 hr = copy_error->GetError(&error_context, &error_code);
244 *error_code_out = FAILED(error_code) ? error_code : E_FAIL;
248 // Finds the component updater jobs matching the given predicate.
249 // Returns S_OK if the function has found at least one job, returns S_FALSE if
250 // no job was found, and it returns an error otherwise.
251 template<class Predicate>
252 HRESULT FindBitsJobIf(Predicate pred,
253 IBackgroundCopyManager* bits_manager,
254 std::vector<ScopedComPtr<IBackgroundCopyJob> >* jobs) {
255 ScopedComPtr<IEnumBackgroundCopyJobs> enum_jobs;
256 HRESULT hr = bits_manager->EnumJobs(0, enum_jobs.Receive());
261 hr = enum_jobs->GetCount(&job_count);
265 // Iterate over jobs, run the predicate, and select the job only if
266 // the job description matches the component updater jobs.
267 for (ULONG i = 0; i != job_count; ++i) {
268 ScopedComPtr<IBackgroundCopyJob> current_job;
269 if (enum_jobs->Next(1, current_job.Receive(), NULL) == S_OK &&
271 base::string16 job_description;
272 hr = GetJobDescription(current_job, &job_description);
273 if (job_description.compare(kJobDescription) == 0)
274 jobs->push_back(current_job);
278 return jobs->empty() ? S_FALSE : S_OK;
281 // Compares the job creation time and returns true if the job creation time
282 // is older than |num_days|.
283 struct JobCreationOlderThanDays
284 : public std::binary_function<IBackgroundCopyJob*, int, bool> {
285 bool operator()(IBackgroundCopyJob* job, int num_days) const;
288 bool JobCreationOlderThanDays::operator()(IBackgroundCopyJob* job,
289 int num_days) const {
290 BG_JOB_TIMES times = {0};
291 HRESULT hr = job->GetTimes(×);
295 const base::TimeDelta time_delta(base::TimeDelta::FromDays(num_days));
296 const base::Time creation_time(base::Time::FromFileTime(times.CreationTime));
298 return creation_time + time_delta < base::Time::Now();
301 // Compares the url of a file in a job and returns true if the remote name
302 // of any file in a job matches the argument.
303 struct JobFileUrlEqual
304 : public std::binary_function<IBackgroundCopyJob*, const base::string16&,
306 bool operator()(IBackgroundCopyJob* job,
307 const base::string16& remote_name) const;
310 bool JobFileUrlEqual::operator()(IBackgroundCopyJob* job,
311 const base::string16& remote_name) const {
312 std::vector<ScopedComPtr<IBackgroundCopyFile> > files;
313 HRESULT hr = GetFilesInJob(job, &files);
317 for (size_t i = 0; i != files.size(); ++i) {
318 ScopedCoMem<base::char16> name;
319 if (SUCCEEDED(files[i]->GetRemoteName(&name)) &&
320 remote_name.compare(name) == 0)
327 // Creates an instance of the BITS manager.
328 HRESULT GetBitsManager(IBackgroundCopyManager** bits_manager) {
329 ScopedComPtr<IBackgroundCopyManager> object;
330 HRESULT hr = object.CreateInstance(__uuidof(BackgroundCopyManager));
332 VLOG(1) << "Failed to instantiate BITS." << std::hex << hr;
333 // TODO: add UMA pings.
336 *bits_manager = object.Detach();
340 void CleanupJobFiles(IBackgroundCopyJob* job) {
341 std::vector<ScopedComPtr<IBackgroundCopyFile> > files;
342 if (FAILED(GetFilesInJob(job, &files)))
344 for (size_t i = 0; i != files.size(); ++i) {
345 base::string16 local_name;
346 HRESULT hr(GetJobFileProperties(files[i], &local_name, NULL, NULL));
348 DeleteFileAndEmptyParentDirectory(base::FilePath(local_name));
352 // Cleans up incompleted jobs that are too old.
353 HRESULT CleanupStaleJobs(
354 base::win::ScopedComPtr<IBackgroundCopyManager> bits_manager) {
358 static base::Time last_sweep;
360 const base::TimeDelta time_delta(base::TimeDelta::FromDays(
361 kPurgeStaleJobsIntervalBetweenChecksDays));
362 const base::Time current_time(base::Time::Now());
363 if (last_sweep + time_delta > current_time)
366 last_sweep = current_time;
368 std::vector<ScopedComPtr<IBackgroundCopyJob> > jobs;
369 HRESULT hr = FindBitsJobIf(
370 std::bind2nd(JobCreationOlderThanDays(), kPurgeStaleJobsAfterDays),
376 for (size_t i = 0; i != jobs.size(); ++i) {
378 CleanupJobFiles(jobs[i]);
386 BackgroundDownloader::BackgroundDownloader(
387 scoped_ptr<CrxDownloader> successor,
388 net::URLRequestContextGetter* context_getter,
389 scoped_refptr<base::SequencedTaskRunner> task_runner,
390 const DownloadCallback& download_callback)
391 : CrxDownloader(successor.Pass(), download_callback),
392 context_getter_(context_getter),
393 task_runner_(task_runner),
394 is_completed_(false) {
395 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
398 BackgroundDownloader::~BackgroundDownloader() {
399 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
401 // The following objects have thread affinity and can't be destroyed on the
402 // UI thread. The resources managed by these objects are acquired at the
403 // beginning of a download and released at the end of the download. Most of
404 // the time, when this destructor is called, these resources have been already
405 // disposed by. Releasing the ownership here is a NOP. However, if the browser
406 // is shutting down while a download is in progress, the timer is active and
407 // the interface pointers are valid. Releasing the ownership means leaking
408 // these objects and their associated resources.
410 bits_manager_.Detach();
414 void BackgroundDownloader::DoStartDownload(const GURL& url) {
415 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
417 BrowserThread::PostTask(
420 base::Bind(&BackgroundDownloader::BeginDownload,
421 base::Unretained(this),
425 // Called once when this class is asked to do a download. Creates or opens
426 // an existing bits job, hooks up the notifications, and starts the timer.
427 void BackgroundDownloader::BeginDownload(const GURL& url) {
428 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::FILE));
432 is_completed_ = false;
433 download_start_time_ = base::Time::Now();
434 job_stuck_begin_time_ = download_start_time_;
436 HRESULT hr = QueueBitsJob(url);
442 // A repeating timer retains the user task. This timer can be stopped and
443 // reset multiple times.
444 timer_.reset(new base::RepeatingTimer<BackgroundDownloader>);
445 timer_->Start(FROM_HERE,
446 base::TimeDelta::FromSeconds(kJobPollingIntervalSec),
448 &BackgroundDownloader::OnDownloading);
451 // Called any time the timer fires.
452 void BackgroundDownloader::OnDownloading() {
453 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::FILE));
457 DCHECK(!is_completed_);
461 BG_JOB_STATE job_state = BG_JOB_STATE_ERROR;
462 HRESULT hr = job_->GetState(&job_state);
469 case BG_JOB_STATE_TRANSFERRED:
470 OnStateTransferred();
473 case BG_JOB_STATE_ERROR:
477 case BG_JOB_STATE_CANCELLED:
481 case BG_JOB_STATE_ACKNOWLEDGED:
482 OnStateAcknowledged();
485 case BG_JOB_STATE_QUEUED:
487 case BG_JOB_STATE_CONNECTING:
489 case BG_JOB_STATE_SUSPENDED:
493 case BG_JOB_STATE_TRANSIENT_ERROR:
494 OnStateTransientError();
497 case BG_JOB_STATE_TRANSFERRING:
498 OnStateTransferring();
506 // Completes the BITS download, picks up the file path of the response, and
507 // notifies the CrxDownloader. The function should be called only once.
508 void BackgroundDownloader::EndDownload(HRESULT error) {
509 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::FILE));
511 DCHECK(!is_completed_);
512 is_completed_ = true;
516 const base::Time download_end_time(base::Time::Now());
517 const base::TimeDelta download_time =
518 download_end_time >= download_start_time_ ?
519 download_end_time - download_start_time_ : base::TimeDelta();
521 int64 bytes_downloaded = -1;
522 int64 bytes_total = -1;
523 GetJobByteCount(job_, &bytes_downloaded, &bytes_total);
525 base::FilePath response;
526 if (SUCCEEDED(error)) {
528 std::vector<ScopedComPtr<IBackgroundCopyFile> > files;
529 GetFilesInJob(job_, &files);
530 DCHECK(files.size() == 1);
531 base::string16 local_name;
532 BG_FILE_PROGRESS progress = {0};
533 HRESULT hr = GetJobFileProperties(files[0], &local_name, NULL, &progress);
535 // Sanity check the post-conditions of a successful download, including
536 // the file and job invariants. The byte counts for a job and its file
537 // must match as a job only contains one file.
538 DCHECK(progress.Completed);
539 DCHECK(bytes_downloaded == static_cast<int64>(progress.BytesTransferred));
540 DCHECK(bytes_total == static_cast<int64>(progress.BytesTotal));
541 response = base::FilePath(local_name);
547 if (FAILED(error) && job_) {
549 CleanupJobFiles(job_);
554 // Consider the url handled if it has been successfully downloaded or a
555 // 5xx has been received.
556 const bool is_handled = SUCCEEDED(error) ||
557 IsHttpServerError(GetHttpStatusFromBitsError(error));
559 const int error_to_report = SUCCEEDED(error) ? 0 : error;
561 DownloadMetrics download_metrics;
562 download_metrics.url = url();
563 download_metrics.downloader = DownloadMetrics::kBits;
564 download_metrics.error = error_to_report;
565 download_metrics.bytes_downloaded = bytes_downloaded;
566 download_metrics.bytes_total = bytes_total;
567 download_metrics.download_time_ms = download_time.InMilliseconds();
569 // Clean up stale jobs before invoking the callback.
570 CleanupStaleJobs(bits_manager_);
572 bits_manager_ = NULL;
575 result.error = error_to_report;
576 result.response = response;
577 BrowserThread::PostTask(
580 base::Bind(&BackgroundDownloader::OnDownloadComplete,
581 base::Unretained(this),
586 // Once the task is posted to the the UI thread, this object may be deleted
587 // by its owner. It is not safe to access members of this object on the
588 // FILE thread from this point on. The timer is stopped and all BITS
589 // interface pointers have been released.
592 // Called when the BITS job has been transferred successfully. Completes the
593 // BITS job by removing it from the BITS queue and making the download
594 // available to the caller.
595 void BackgroundDownloader::OnStateTransferred() {
596 HRESULT hr = job_->Complete();
597 if (SUCCEEDED(hr) || hr == BG_S_UNABLE_TO_DELETE_FILES)
602 // Called when the job has encountered an error and no further progress can
603 // be made. Cancels this job and removes it from the BITS queue.
604 void BackgroundDownloader::OnStateError() {
605 HRESULT error_code = S_OK;
606 HRESULT hr = GetJobError(job_, &error_code);
609 DCHECK(FAILED(error_code));
610 EndDownload(error_code);
613 // Called when the job has encountered a transient error, such as a
614 // network disconnect, a server error, or some other recoverable error.
615 void BackgroundDownloader::OnStateTransientError() {
616 // If the job appears to be stuck, handle the transient error as if
617 // it were a final error. This causes the job to be cancelled and a specific
618 // error be returned, if the error was available.
624 // Don't retry at all if the transient error was a 5xx.
625 HRESULT error_code = S_OK;
626 HRESULT hr = GetJobError(job_, &error_code);
628 IsHttpServerError(GetHttpStatusFromBitsError(error_code))) {
634 void BackgroundDownloader::OnStateQueued() {
636 EndDownload(E_ABORT); // Return a generic error for now.
639 void BackgroundDownloader::OnStateTransferring() {
640 // Resets the baseline for detecting a stuck job since the job is transferring
641 // data and it is making progress.
642 job_stuck_begin_time_ = base::Time::Now();
645 // Called when the download was cancelled. Since the observer should have
646 // been disconnected by now, this notification must not be seen.
647 void BackgroundDownloader::OnStateCancelled() {
648 EndDownload(E_UNEXPECTED);
651 // Called when the download was completed. Same as above.
652 void BackgroundDownloader::OnStateAcknowledged() {
653 EndDownload(E_UNEXPECTED);
656 // Creates or opens a job for the given url and queues it up. Tries to
657 // install a job observer but continues on if an observer can't be set up.
658 HRESULT BackgroundDownloader::QueueBitsJob(const GURL& url) {
659 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::FILE));
662 if (bits_manager_ == NULL) {
663 hr = GetBitsManager(bits_manager_.Receive());
668 hr = CreateOrOpenJob(url);
673 hr = InitializeNewJob(url);
678 return job_->Resume();
681 HRESULT BackgroundDownloader::CreateOrOpenJob(const GURL& url) {
682 std::vector<ScopedComPtr<IBackgroundCopyJob> > jobs;
683 HRESULT hr = FindBitsJobIf(
684 std::bind2nd(JobFileUrlEqual(), base::SysUTF8ToWide(url.spec())),
687 if (SUCCEEDED(hr) && !jobs.empty()) {
692 // Use kJobDescription as a temporary job display name until the proper
693 // display name is initialized later on.
695 ScopedComPtr<IBackgroundCopyJob> job;
696 hr = bits_manager_->CreateJob(kJobDescription,
697 BG_JOB_TYPE_DOWNLOAD,
707 HRESULT BackgroundDownloader::InitializeNewJob(const GURL& url) {
708 const base::string16 filename(base::SysUTF8ToWide(url.ExtractFileName()));
710 base::FilePath tempdir;
711 if (!base::CreateNewTempDirectory(
712 FILE_PATH_LITERAL("chrome_BITS_"),
716 HRESULT hr = job_->AddFile(
717 base::SysUTF8ToWide(url.spec()).c_str(),
718 tempdir.Append(filename).AsUTF16Unsafe().c_str());
722 hr = job_->SetDisplayName(filename.c_str());
726 hr = job_->SetDescription(kJobDescription);
730 hr = job_->SetPriority(BG_JOB_PRIORITY_NORMAL);
734 hr = job_->SetMinimumRetryDelay(60 * kMinimumRetryDelayMin);
738 const int kSecondsDay = 60 * 60 * 24;
739 hr = job_->SetNoProgressTimeout(kSecondsDay * kSetNoProgressTimeoutDays);
746 bool BackgroundDownloader::IsStuck() {
747 const base::TimeDelta job_stuck_timeout(
748 base::TimeDelta::FromMinutes(kJobStuckTimeoutMin));
749 return job_stuck_begin_time_ + job_stuck_timeout < base::Time::Now();
752 } // namespace component_updater