src/chrome_frame/test/perf/chrome_frame_perftest.cc

   1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "chrome_frame/test/perf/chrome_frame_perftest.h"
   6
   7 #include <atlhost.h>
   8 #include <atlwin.h>
   9
  10 #include <map>
  11 #include <string>
  12 #include <vector>
  13
  14 #include "base/debug/trace_event_win.h"
  15 #include "base/file_util.h"
  16 #include "base/files/file_path.h"
  17 #include "base/memory/scoped_ptr.h"
  18 #include "base/path_service.h"
  19 #include "base/process/kill.h"
  20 #include "base/process/launch.h"
  21 #include "base/process/process_iterator.h"
  22 #include "base/strings/string_util.h"
  23 #include "base/strings/stringprintf.h"
  24 #include "base/strings/utf_string_conversions.h"
  25 #include "base/test/perf_time_logger.h"
  26 #include "base/test/test_file_util.h"
  27 #include "base/threading/platform_thread.h"
  28 #include "base/time/time.h"
  29 #include "base/win/event_trace_consumer.h"
  30 #include "base/win/event_trace_controller.h"
  31 #include "base/win/registry.h"
  32 #include "base/win/scoped_bstr.h"
  33 #include "base/win/scoped_comptr.h"
  34 #include "base/win/scoped_variant.h"
  35 #include "chrome/app/image_pre_reader_win.h"
  36 #include "chrome/common/chrome_constants.h"
  37 #include "chrome/common/chrome_paths.h"
  38 #include "chrome/common/chrome_paths_internal.h"
  39 #include "chrome/test/base/chrome_process_util.h"
  40 #include "chrome/test/ui/ui_perf_test.h"
  41 #include "chrome_frame/chrome_tab.h"
  42 #include "chrome_frame/test_utils.h"
  43 #include "chrome_frame/utils.h"
  44 #include "testing/perf/perf_test.h"
  45
  46 const wchar_t kSilverlightControlKey[] =
  47     L"CLSID\\{DFEAF541-F3E1-4c24-ACAC-99C30715084A}\\InprocServer32";
  48
  49 const wchar_t kFlashControlKey[] =
  50     L"CLSID\\{D27CDB6E-AE6D-11cf-96B8-444553540000}\\InprocServer32";
  51
  52 using base::TimeDelta;
  53 using base::TimeTicks;
  54
  55 // This class implements an ActiveX container which hosts the ChromeFrame
  56 // ActiveX control. It provides hooks which can be implemented by derived
  57 // classes for implementing performance measurement, etc.
  58 class ChromeFrameActiveXContainer
  59     : public CWindowImpl<ChromeFrameActiveXContainer, CWindow, CWinTraits <
  60           WS_OVERLAPPEDWINDOW | WS_VISIBLE | WS_CLIPCHILDREN | WS_CLIPSIBLINGS,
  61           WS_EX_APPWINDOW | WS_EX_WINDOWEDGE> >,
  62       public CComObjectRootEx<CComSingleThreadModel>,
  63       public IPropertyNotifySink {
  64  public:
  65   ~ChromeFrameActiveXContainer() {
  66     if (m_hWnd)
  67       DestroyWindow();
  68   }
  69
  70   DECLARE_WND_CLASS_EX(L"ChromeFrameActiveX_container", 0, 0)
  71
  72   BEGIN_COM_MAP(ChromeFrameActiveXContainer)
  73     COM_INTERFACE_ENTRY(IPropertyNotifySink)
  74   END_COM_MAP()
  75
  76   BEGIN_MSG_MAP(ChromeFrameActiveXContainer)
  77     MESSAGE_HANDLER(WM_CREATE, OnCreate)
  78     MESSAGE_HANDLER(WM_DESTROY, OnDestroy)
  79   END_MSG_MAP()
  80
  81   HRESULT OnMessageCallback(const VARIANT* param) {
  82     DVLOG(1) << __FUNCTION__;
  83     OnMessageCallbackImpl(param);
  84     return S_OK;
  85   }
  86
  87   HRESULT OnLoadErrorCallback(const VARIANT* param) {
  88     DVLOG(1) << __FUNCTION__ << " " << param->bstrVal;
  89     OnLoadErrorCallbackImpl(param);
  90     return S_OK;
  91   }
  92
  93   HRESULT OnLoadCallback(const VARIANT* param) {
  94     DVLOG(1) << __FUNCTION__ << " " << param->bstrVal;
  95     OnLoadCallbackImpl(param);
  96     return S_OK;
  97   }
  98
  99   ChromeFrameActiveXContainer() :
 100       prop_notify_cookie_(0),
 101       onmsg_(this, &ChromeFrameActiveXContainer::OnMessageCallback),
 102       onloaderror_(this, &ChromeFrameActiveXContainer::OnLoadErrorCallback),
 103       onload_(this, &ChromeFrameActiveXContainer::OnLoadCallback) {
 104   }
 105
 106   LRESULT OnCreate(UINT , WPARAM , LPARAM , BOOL& ) {
 107     chromeview_.Attach(m_hWnd);
 108     return 0;
 109   }
 110
 111   // This will be called twice.
 112   // Once from CAxHostWindow::OnDestroy (through DefWindowProc)
 113   // and once more from the ATL since CAxHostWindow::OnDestroy claims the
 114   // message is not handled.
 115   LRESULT OnDestroy(UINT, WPARAM, LPARAM, BOOL& handled) {  // NOLINT
 116     if (prop_notify_cookie_) {
 117       AtlUnadvise(tab_, IID_IPropertyNotifySink, prop_notify_cookie_);
 118       prop_notify_cookie_ = 0;
 119     }
 120
 121     tab_.Release();
 122     return 0;
 123   }
 124
 125   virtual void OnFinalMessage(HWND /*hWnd*/) {
 126     ::PostQuitMessage(6);
 127   }
 128
 129   static const wchar_t* GetWndCaption() {
 130     return L"ChromeFrame Container";
 131   }
 132
 133   // IPropertyNotifySink
 134   STDMETHOD(OnRequestEdit)(DISPID disp_id) {
 135     OnRequestEditImpl(disp_id);
 136     return S_OK;
 137   }
 138
 139   STDMETHOD(OnChanged)(DISPID disp_id) {
 140     if (disp_id != DISPID_READYSTATE)
 141       return S_OK;
 142
 143     long ready_state;
 144     HRESULT hr = tab_->get_readyState(&ready_state);
 145     DCHECK(hr == S_OK);
 146
 147     OnReadyStateChanged(ready_state);
 148
 149     if (ready_state == READYSTATE_COMPLETE) {
 150       if (!starting_url_.empty()) {
 151         Navigate(starting_url_.c_str());
 152       } else {
 153         PostMessage(WM_CLOSE);
 154       }
 155     } else if (ready_state == READYSTATE_UNINITIALIZED) {
 156       DLOG(ERROR) << __FUNCTION__ << " Chrome launch failed.";
 157     }
 158
 159     return S_OK;
 160   }
 161
 162   void CreateChromeFrameWindow(const std::string& starting_url) {
 163     starting_url_ = starting_url;
 164     RECT rc = { 0, 0, 800, 600 };
 165     Create(NULL, rc);
 166     DCHECK(m_hWnd);
 167     ShowWindow(SW_SHOWDEFAULT);
 168   }
 169
 170   void CreateControl(bool setup_event_sinks) {
 171     HRESULT hr = chromeview_.CreateControl(L"ChromeTab.ChromeFrame");
 172     EXPECT_HRESULT_SUCCEEDED(hr);
 173     hr = chromeview_.QueryControl(tab_.Receive());
 174     EXPECT_HRESULT_SUCCEEDED(hr);
 175
 176     if (setup_event_sinks)
 177       SetupEventSinks();
 178   }
 179
 180   void Navigate(const char* url) {
 181     BeforeNavigateImpl(url);
 182
 183     HRESULT hr = tab_->put_src(base::win::ScopedBstr(UTF8ToWide(url).c_str()));
 184     DCHECK(hr == S_OK) << "Chrome frame NavigateToURL(" << url
 185                        << base::StringPrintf(L") failed 0x%08X", hr);
 186   }
 187
 188   void SetupEventSinks() {
 189     HRESULT hr = AtlAdvise(tab_, this, IID_IPropertyNotifySink,
 190                            &prop_notify_cookie_);
 191     DCHECK(hr == S_OK) << "AtlAdvice for IPropertyNotifySink failed " << hr;
 192
 193     base::win::ScopedVariant onmessage(onmsg_.ToDispatch());
 194     base::win::ScopedVariant onloaderror(onloaderror_.ToDispatch());
 195     base::win::ScopedVariant onload(onload_.ToDispatch());
 196     EXPECT_HRESULT_SUCCEEDED(tab_->put_onmessage(onmessage));
 197     EXPECT_HRESULT_SUCCEEDED(tab_->put_onloaderror(onloaderror));
 198     EXPECT_HRESULT_SUCCEEDED(tab_->put_onload(onload));
 199   }
 200
 201  protected:
 202   // These functions are implemented by derived classes for special behavior
 203   // like performance measurement, etc.
 204   virtual void OnReadyStateChanged(long ready_state) {}
 205   virtual void OnRequestEditImpl(DISPID disp_id) {}
 206
 207   virtual void OnMessageCallbackImpl(const VARIANT* param) {}
 208
 209   virtual void OnLoadCallbackImpl(const VARIANT* param) {
 210     PostMessage(WM_CLOSE);
 211   }
 212
 213   virtual void OnLoadErrorCallbackImpl(const VARIANT* param) {
 214     PostMessage(WM_CLOSE);
 215   }
 216   virtual void BeforeNavigateImpl(const char* url) {}
 217
 218   CAxWindow chromeview_;
 219   base::win::ScopedComPtr<IChromeFrame> tab_;
 220   DWORD prop_notify_cookie_;
 221   DispCallback<ChromeFrameActiveXContainer> onmsg_;
 222   DispCallback<ChromeFrameActiveXContainer> onloaderror_;
 223   DispCallback<ChromeFrameActiveXContainer> onload_;
 224   std::string starting_url_;
 225 };
 226
 227 // This class overrides the hooks provided by the ChromeFrameActiveXContainer
 228 // class and measures performance at various stages, like initialzation of
 229 // the Chrome frame widget, navigation, etc.
 230 class ChromeFrameActiveXContainerPerf : public ChromeFrameActiveXContainer {
 231  public:
 232   ChromeFrameActiveXContainerPerf() {}
 233
 234   void CreateControl(bool setup_event_sinks) {
 235     perf_initialize_.reset(new base::PerfTimeLogger("Fully initialized"));
 236     base::PerfTimeLogger perf_create("Create Control");
 237
 238     HRESULT hr = chromeview_.CreateControl(L"ChromeTab.ChromeFrame");
 239     EXPECT_HRESULT_SUCCEEDED(hr);
 240     hr = chromeview_.QueryControl(tab_.Receive());
 241     EXPECT_HRESULT_SUCCEEDED(hr);
 242
 243     perf_create.Done();
 244     if (setup_event_sinks)
 245       SetupEventSinks();
 246   }
 247
 248  protected:
 249   virtual void OnReadyStateChanged(long ready_state) {
 250     // READYSTATE_COMPLETE is fired when the automation server is ready.
 251     if (ready_state == READYSTATE_COMPLETE) {
 252       perf_initialize_->Done();
 253     } else if (ready_state == READYSTATE_INTERACTIVE) {
 254       // Window ready.  Currently we never receive this notification because it
 255       // is fired before we finish setting up our hosting environment.
 256       // This is because of how ATL is written.  Moving forward we might
 257       // have our own hosting classes and then have more control over when we
 258       // set up the prop notify sink.
 259     } else {
 260       DCHECK(ready_state != READYSTATE_UNINITIALIZED) << "failed to initialize";
 261     }
 262   }
 263
 264   virtual void OnLoadCallbackImpl(const VARIANT* param) {
 265     PostMessage(WM_CLOSE);
 266     perf_navigate_->Done();
 267   }
 268
 269   virtual void OnLoadErrorCallbackImpl(const VARIANT* param) {
 270     PostMessage(WM_CLOSE);
 271     perf_navigate_->Done();
 272   }
 273
 274   virtual void BeforeNavigateImpl(const char* url ) {
 275     std::string test_name = "Navigate ";
 276     test_name += url;
 277     perf_navigate_.reset(new base::PerfTimeLogger(test_name.c_str()));
 278   }
 279
 280   scoped_ptr<base::PerfTimeLogger> perf_initialize_;
 281   scoped_ptr<base::PerfTimeLogger> perf_navigate_;
 282 };
 283
 284 // This class provides common functionality which can be used for most of the
 285 // ChromeFrame/Tab performance tests.
 286 class ChromeFramePerfTestBase : public UIPerfTest {
 287  public:
 288   ChromeFramePerfTestBase() {}
 289  protected:
 290   scoped_ptr<ScopedChromeFrameRegistrar> chrome_frame_registrar_;
 291 };
 292
 293 class ChromeFrameStartupTest : public ChromeFramePerfTestBase {
 294  public:
 295   ChromeFrameStartupTest() {}
 296
 297   virtual void SetUp() {
 298     ASSERT_TRUE(PathService::Get(chrome::DIR_APP, &dir_app_));
 299
 300     chrome_dll_ = dir_app_.Append(L"chrome.dll");
 301     chrome_exe_ = dir_app_.Append(chrome::kBrowserProcessExecutableName);
 302     chrome_frame_dll_ = dir_app_.Append(kChromeFrameDllName);
 303     icu_dll_ = dir_app_.Append(L"icudt.dll");
 304     ffmpegsumo_dll_ = dir_app_.Append(L"ffmpegsumo.dll");
 305   }
 306
 307   // TODO(iyengar)
 308   // This function is similar to the RunStartupTest function used in chrome
 309   // startup tests. Refactor into a common implementation.
 310   void RunStartupTest(const char* graph, const char* trace,
 311                       const char* startup_url, bool test_cold,
 312                       int total_binaries,
 313                       const base::FilePath binaries_to_evict[],
 314                       bool important, bool ignore_cache_error) {
 315     const int kNumCycles = 20;
 316
 317     startup_url_ = startup_url;
 318
 319     TimeDelta timings[kNumCycles];
 320
 321     for (int i = 0; i < kNumCycles; ++i) {
 322       if (test_cold) {
 323         for (int binary_index = 0; binary_index < total_binaries;
 324              binary_index++) {
 325           bool result = base::EvictFileFromSystemCacheWithRetry(
 326               binaries_to_evict[binary_index]);
 327           if (!ignore_cache_error) {
 328             ASSERT_TRUE(result);
 329           } else if (!result) {
 330             LOG(ERROR) << GetLastError();
 331             printf("\nFailed to evict file %ls from cache. Not running test\n",
 332                    binaries_to_evict[binary_index].value().c_str());
 333             return;
 334           }
 335         }
 336       }
 337
 338       TimeTicks start_time, end_time;
 339
 340       RunStartupTestImpl(&start_time, &end_time);
 341
 342       timings[i] = end_time - start_time;
 343
 344       CoFreeUnusedLibraries();
 345
 346       // TODO(beng): Can't shut down so quickly. Figure out why, and fix. If we
 347       // do, we crash.
 348       base::PlatformThread::Sleep(base::TimeDelta::FromMilliseconds(50));
 349     }
 350
 351     std::string times;
 352     for (int i = 0; i < kNumCycles; ++i)
 353       base::StringAppendF(&times, "%.2f,", timings[i].InMillisecondsF());
 354
 355     perf_test::PrintResultList(graph, "", trace, times, "ms", important);
 356   }
 357
 358   base::FilePath dir_app_;
 359   base::FilePath chrome_dll_;
 360   base::FilePath chrome_exe_;
 361   base::FilePath chrome_frame_dll_;
 362   base::FilePath icu_dll_;
 363   base::FilePath ffmpegsumo_dll_;
 364
 365  protected:
 366   // Individual startup tests should implement this function.
 367   virtual void RunStartupTestImpl(TimeTicks* start_time,
 368                                   TimeTicks* end_time) {}
 369
 370   // The host is torn down by this function. It should not be used after
 371   // this function returns.
 372   static void ReleaseHostComReferences(CAxWindow& host) {
 373     CComPtr<IAxWinHostWindow> spWinHost;
 374     host.QueryHost(&spWinHost);
 375     ASSERT_TRUE(spWinHost != NULL);
 376
 377     // Hack to get the host to release all interfaces and thus ensure that
 378     // the COM server can be unloaded.
 379     CAxHostWindow* host_window = static_cast<CAxHostWindow*>(spWinHost.p);
 380     host_window->ReleaseAll();
 381     host.DestroyWindow();
 382   }
 383
 384   std::string startup_url_;
 385 };
 386
 387 class ChromeFrameStartupTestActiveX : public ChromeFrameStartupTest {
 388  public:
 389   virtual void SetUp() {
 390     // Register the Chrome Frame DLL in the build directory.
 391     chrome_frame_registrar_.reset(new ScopedChromeFrameRegistrar(
 392         ScopedChromeFrameRegistrar::SYSTEM_LEVEL));
 393
 394     ChromeFrameStartupTest::SetUp();
 395   }
 396
 397  protected:
 398   virtual void RunStartupTestImpl(TimeTicks* start_time,
 399                                   TimeTicks* end_time) {
 400     *start_time = TimeTicks::Now();
 401     SimpleModule module;
 402     AtlAxWinInit();
 403     CComObjectStackEx<ChromeFrameActiveXContainer> wnd;
 404     wnd.CreateChromeFrameWindow(startup_url_);
 405     wnd.CreateControl(true);
 406     module.RunMessageLoop();
 407     *end_time = TimeTicks::Now();
 408   }
 409 };
 410
 411 // This class measures the load time of chrome and chrome frame binaries
 412 class ChromeFrameBinariesLoadTest : public ChromeFrameStartupTestActiveX {
 413   static const size_t kStepSize = 4 * 1024;
 414  public:
 415   enum PreReadType {
 416     kPreReadNone,
 417     kPreReadPartial,
 418     kPreReadFull
 419   };
 420
 421   ChromeFrameBinariesLoadTest()
 422       : pre_read_type_(kPreReadNone),
 423         step_size_(kStepSize),
 424         bytes_to_read_(0),
 425         percentage_to_preread_(25) {}
 426
 427  protected:
 428   virtual void RunStartupTestImpl(TimeTicks* start_time,
 429                                   TimeTicks* end_time) {
 430     *start_time = TimeTicks::Now();
 431
 432     if (pre_read_type_ == kPreReadFull) {
 433       EXPECT_TRUE(ImagePreReader::PreReadImage(chrome_exe_.value().c_str(),
 434                                                bytes_to_read_,
 435                                                step_size_));
 436       EXPECT_TRUE(ImagePreReader::PreReadImage(chrome_dll_.value().c_str(),
 437                                                bytes_to_read_,
 438                                                step_size_));
 439     } else if (pre_read_type_ == kPreReadPartial) {
 440       EXPECT_TRUE(
 441           ImagePreReader::PartialPreReadImage(chrome_exe_.value().c_str(),
 442                                               percentage_to_preread_,
 443                                               step_size_));
 444       EXPECT_TRUE(
 445           ImagePreReader::PartialPreReadImage(chrome_dll_.value().c_str(),
 446                                               percentage_to_preread_,
 447                                               step_size_));
 448     }
 449
 450     HMODULE chrome_exe = LoadLibrary(chrome_exe_.value().c_str());
 451     EXPECT_TRUE(chrome_exe != NULL);
 452
 453     HMODULE chrome_dll = LoadLibrary(chrome_dll_.value().c_str());
 454     EXPECT_TRUE(chrome_dll != NULL);
 455
 456     *end_time = TimeTicks::Now();
 457
 458     FreeLibrary(chrome_exe);
 459     FreeLibrary(chrome_dll);
 460   }
 461
 462   PreReadType pre_read_type_;
 463   size_t bytes_to_read_;
 464   size_t step_size_;
 465   uint8 percentage_to_preread_;
 466 };
 467
 468 // This class provides functionality to run the startup performance test for
 469 // the ChromeFrame ActiveX against a reference build. At this point we only run
 470 // this test in warm mode.
 471 class ChromeFrameStartupTestActiveXReference
 472     : public ChromeFrameStartupTestActiveX {
 473  public:
 474   // override the browser directory to use the reference build instead.
 475   virtual void SetUp() {
 476     // Register the reference build Chrome Frame DLL.
 477     chrome_frame_registrar_.reset(new ScopedChromeFrameRegistrar(
 478         ScopedChromeFrameRegistrar::SYSTEM_LEVEL));
 479     chrome_frame_registrar_->RegisterReferenceChromeFrameBuild();
 480
 481     ChromeFrameStartupTest::SetUp();
 482
 483     chrome_frame_dll_ = base::FilePath(
 484         chrome_frame_registrar_->GetReferenceChromeFrameDllPath());
 485     DVLOG(1) << __FUNCTION__ << ": " << chrome_frame_dll_.value();
 486   }
 487
 488   virtual void TearDown() {
 489     // Reregister the Chrome Frame DLL in the build directory.
 490     chrome_frame_registrar_.reset(NULL);
 491   }
 492 };
 493
 494 // This class provides base functionality to measure ChromeFrame memory
 495 // usage.
 496 // TODO(iyengar)
 497 // Some of the functionality in this class like printing the results, etc
 498 // is based on the chrome\test\memory_test.cc. We need to factor out
 499 // the common code.
 500 class ChromeFrameMemoryTest : public ChromeFramePerfTestBase {
 501   // Contains information about the memory consumption of a process.
 502   class ProcessMemoryInfo {
 503    public:
 504     // Default constructor
 505     // Added to enable us to add ProcessMemoryInfo instances to a map.
 506     ProcessMemoryInfo()
 507         : process_id_(0),
 508           virtual_size_(0),
 509           working_set_size_(0),
 510           chrome_browser_process_(false),
 511           chrome_frame_memory_test_instance_(NULL) {}
 512
 513     ProcessMemoryInfo(base::ProcessId process_id, bool chrome_browser_process,
 514                      ChromeFrameMemoryTest* memory_test_instance)
 515         : process_id_(process_id),
 516           virtual_size_(0),
 517           working_set_size_(0),
 518           chrome_browser_process_(chrome_browser_process),
 519           chrome_frame_memory_test_instance_(memory_test_instance) {}
 520
 521     bool GetMemoryConsumptionDetails() {
 522       base::ProcessHandle process_handle;
 523       if (!base::OpenPrivilegedProcessHandle(process_id_, &process_handle)) {
 524         NOTREACHED();
 525       }
 526
 527       // TODO(sgk):  if/when base::ProcessMetrics can return real memory
 528       // stats on mac, convert to:
 529       //
 530       // scoped_ptr<base::ProcessMetrics> process_metrics;
 531       // process_metrics.reset(
 532       //     base::ProcessMetrics::CreateProcessMetrics(process_handle));
 533       scoped_ptr<ChromeTestProcessMetrics> process_metrics;
 534       process_metrics.reset(
 535           ChromeTestProcessMetrics::CreateProcessMetrics(process_handle));
 536
 537       virtual_size_ = process_metrics->GetPagefileUsage();
 538       working_set_size_ = process_metrics->GetWorkingSetSize();
 539
 540       base::CloseProcessHandle(process_handle);
 541       return true;
 542     }
 543
 544     void Print(const char* test_name) {
 545       std::string trace_name(test_name);
 546
 547       ASSERT_TRUE(chrome_frame_memory_test_instance_ != NULL);
 548
 549       if (chrome_browser_process_) {
 550          perf_test::PrintResult("vm_final_browser", "", trace_name + "_vm_b",
 551              virtual_size_ / 1024, "KB", false /* not important */);
 552          perf_test::PrintResult("ws_final_browser", "", trace_name + "_ws_b",
 553              working_set_size_ / 1024, "KB", false /* not important */);
 554       } else if (process_id_ == base::GetCurrentProcId()) {
 555         perf_test::PrintResult("vm_current_process", "", trace_name + "_vm_c",
 556             virtual_size_ / 1024, "KB", false /* not important */);
 557         perf_test::PrintResult("ws_current_process", "", trace_name + "_ws_c",
 558             working_set_size_ / 1024, "KB", false /* not important */);
 559       }
 560
 561       printf("\n");
 562     }
 563
 564     base::ProcessId process_id_;
 565     size_t virtual_size_;
 566     size_t working_set_size_;
 567     // Set to true if this is the chrome browser process.
 568     bool chrome_browser_process_;
 569
 570     // A reference to the ChromeFrameMemoryTest instance. Used to print memory
 571     // consumption information.
 572     ChromeFrameMemoryTest* chrome_frame_memory_test_instance_;
 573   };
 574
 575   // This map tracks memory usage for a process. It is keyed on the process
 576   // id.
 577   typedef std::map<DWORD, ProcessMemoryInfo> ProcessMemoryConsumptionMap;
 578
 579  public:
 580   ChromeFrameMemoryTest() : current_url_index_(0) {
 581   }
 582
 583   virtual void SetUp() {
 584     // Register the Chrome Frame DLL in the build directory.
 585     chrome_frame_registrar_.reset(new ScopedChromeFrameRegistrar(
 586         ScopedChromeFrameRegistrar::SYSTEM_LEVEL));
 587   }
 588
 589   void RunTest(const char* test_name, char* urls[], int total_urls) {
 590     ASSERT_TRUE(urls != NULL);
 591     ASSERT_GT(total_urls, 0);
 592
 593     // Record the initial CommitCharge.  This is a system-wide measurement,
 594     // so if other applications are running, they can create variance in this
 595     // test.
 596     start_commit_charge_ = base::GetSystemCommitCharge();
 597
 598     for (int i = 0; i < total_urls; i++)
 599       urls_.push_back(urls[i]);
 600
 601     std::string url;
 602     GetNextUrl(&url);
 603     ASSERT_TRUE(!url.empty());
 604
 605     StartTest(url, test_name);
 606   }
 607
 608   void OnNavigationSuccess(const VARIANT* param) {
 609     ASSERT_TRUE(param != NULL);
 610     ASSERT_EQ(VT_BSTR, param->vt);
 611
 612     DVLOG(1) << __FUNCTION__ << " " << param->bstrVal;
 613     InitiateNextNavigation();
 614   }
 615
 616   void OnNavigationFailure(const VARIANT* param) {
 617     ASSERT_TRUE(param != NULL);
 618     ASSERT_EQ(VT_BSTR, param->vt);
 619
 620     DVLOG(1) << __FUNCTION__ << " " << param->bstrVal;
 621     InitiateNextNavigation();
 622   }
 623
 624  protected:
 625   bool GetNextUrl(std::string* url) {
 626     if (current_url_index_ >= urls_.size())
 627       return false;
 628
 629     *url = urls_[current_url_index_++];
 630     return true;
 631   }
 632
 633   void InitiateNextNavigation() {
 634     // Get the memory consumption information for the child processes
 635     // of the chrome browser.
 636     ChromeProcessList child_processes = GetBrowserChildren();
 637     ChromeProcessList::iterator index;
 638     for (index = child_processes.begin(); index != child_processes.end();
 639          ++index) {
 640       AccountProcessMemoryUsage(*index);
 641     }
 642
 643     // TODO(iyengar): Bug 2953
 644     // Need to verify if this is still true.
 645     // The automation crashes periodically if we cycle too quickly.
 646     // To make these tests more reliable, slowing them down a bit.
 647     Sleep(200);
 648
 649     std::string url;
 650     bool next_url = GetNextUrl(&url);
 651     if (!url.empty()) {
 652       NavigateImpl(url);
 653     } else {
 654       TestCompleted();
 655     }
 656   }
 657
 658   void PrintResults(const char* test_name) {
 659     PrintMemoryUsageInfo(test_name);
 660     memory_consumption_map_.clear();
 661
 662     // Added to give the OS some time to flush the used pages for the
 663     // chrome processes which would have exited by now.
 664     Sleep(200);
 665
 666     size_t end_commit_charge = base::GetSystemCommitCharge();
 667     size_t commit_size = (end_commit_charge - start_commit_charge_) * 1024;
 668
 669     std::string trace_name(test_name);
 670     trace_name.append("_cc");
 671
 672     perf_test::PrintResult("commit_charge", "", trace_name,
 673                            commit_size / 1024, "KB", true /* important */);
 674     printf("\n");
 675   }
 676
 677   base::ProcessId chrome_browser_process_id() {
 678     base::NamedProcessIterator iter(L"chrome.exe", NULL);
 679     const base::ProcessEntry* entry = iter.NextProcessEntry();
 680     if (entry) {
 681       return entry->pid();
 682     }
 683     return -1;
 684   }
 685
 686   ChromeProcessList GetBrowserChildren() {
 687     ChromeProcessList list = GetRunningChromeProcesses(
 688         chrome_browser_process_id());
 689     ChromeProcessList::iterator browser =
 690         std::find(list.begin(), list.end(), chrome_browser_process_id());
 691     if (browser != list.end()) {
 692       list.erase(browser);
 693     }
 694     return list;
 695   }
 696
 697   void AccountProcessMemoryUsage(DWORD process_id) {
 698     ProcessMemoryInfo process_memory_info(
 699         process_id, process_id == chrome_browser_process_id(), this);
 700
 701     ASSERT_TRUE(process_memory_info.GetMemoryConsumptionDetails());
 702
 703     memory_consumption_map_[process_id] = process_memory_info;
 704   }
 705
 706   void PrintMemoryUsageInfo(const char* test_name) {
 707     printf("\n");
 708
 709     std::string trace_name(test_name);
 710
 711     ProcessMemoryConsumptionMap::iterator index;
 712     size_t total_virtual_size = 0;
 713     size_t total_working_set_size = 0;
 714
 715     for (index = memory_consumption_map_.begin();
 716          index != memory_consumption_map_.end();
 717          ++index) {
 718       ProcessMemoryInfo& memory_info = (*index).second;
 719       memory_info.Print(test_name);
 720
 721       total_virtual_size += memory_info.virtual_size_;
 722       total_working_set_size += memory_info.working_set_size_;
 723     }
 724
 725     printf("\n");
 726
 727     perf_test::PrintResult("vm_final_total", "", trace_name + "_vm",
 728                            total_virtual_size / 1024, "KB",
 729                            false /* not important */);
 730     perf_test::PrintResult("ws_final_total", "", trace_name + "_ws",
 731                            total_working_set_size / 1024, "KB",
 732                            true /* important */);
 733   }
 734
 735   // Should never get called.
 736   virtual void StartTest(const std::string& url,
 737                          const std::string& test_name) = 0 {
 738     ASSERT_FALSE(false);
 739   }
 740
 741   // Should never get called.
 742   virtual void NavigateImpl(const std::string& url) = 0 {
 743     ASSERT_FALSE(false);
 744   }
 745
 746   virtual void TestCompleted() = 0 {
 747     ASSERT_FALSE(false);
 748   }
 749
 750   // Holds the commit charge in KBytes at the start of the memory test run.
 751   size_t start_commit_charge_;
 752
 753   // The index of the URL being tested.
 754   size_t current_url_index_;
 755
 756   // Contains the list of urls against which the tests are run.
 757   std::vector<std::string> urls_;
 758
 759   ProcessMemoryConsumptionMap memory_consumption_map_;
 760 };
 761
 762 // This class provides functionality to run the memory test against a reference
 763 // chrome frame build.
 764 class ChromeFrameMemoryTestReference : public ChromeFrameMemoryTest {
 765  public:
 766   virtual void SetUp() {
 767     chrome_frame_registrar_.reset(new ScopedChromeFrameRegistrar(
 768         ScopedChromeFrameRegistrar::SYSTEM_LEVEL));
 769     chrome_frame_registrar_->RegisterReferenceChromeFrameBuild();
 770   }
 771
 772   virtual void TearDown() {
 773     // Reregisters the chrome frame DLL in the build directory.
 774     chrome_frame_registrar_.reset(NULL);
 775   }
 776 };
 777
 778 // This class overrides the hooks provided by the ChromeFrameActiveXContainer
 779 // class and calls back into the ChromeFrameMemoryTest object instance,
 780 // which measures ChromeFrame memory usage.
 781 class ChromeFrameActiveXContainerMemory : public ChromeFrameActiveXContainer {
 782  public:
 783   ChromeFrameActiveXContainerMemory()
 784       : delegate_(NULL) {}
 785
 786   ~ChromeFrameActiveXContainerMemory() {}
 787
 788   void Initialize(ChromeFrameMemoryTest* delegate) {
 789     ASSERT_TRUE(delegate != NULL);
 790     delegate_ = delegate;
 791   }
 792
 793  protected:
 794   virtual void OnLoadCallbackImpl(const VARIANT* param) {
 795     delegate_->OnNavigationSuccess(param);
 796   }
 797
 798   virtual void OnLoadErrorCallbackImpl(const VARIANT* param) {
 799     delegate_->OnNavigationFailure(param);
 800   }
 801
 802   ChromeFrameMemoryTest* delegate_;
 803 };
 804
 805 // This class runs memory tests against the ChromeFrame ActiveX.
 806 template<class MemoryTestBase>
 807 class ChromeFrameActiveXMemoryTest : public MemoryTestBase {
 808  public:
 809   ChromeFrameActiveXMemoryTest()
 810       : chrome_frame_container_(NULL),
 811         test_completed_(false) {}
 812
 813   ~ChromeFrameActiveXMemoryTest() {
 814   }
 815
 816   void StartTest(const std::string& url, const std::string& test_name) {
 817     ASSERT_TRUE(chrome_frame_container_ == NULL);
 818
 819     test_name_ = test_name;
 820
 821     SimpleModule module;
 822     AtlAxWinInit();
 823
 824     CComObject<ChromeFrameActiveXContainerMemory>::CreateInstance(
 825         &chrome_frame_container_);
 826     chrome_frame_container_->AddRef();
 827
 828     chrome_frame_container_->Initialize(this);
 829
 830     chrome_frame_container_->CreateChromeFrameWindow(url.c_str());
 831     chrome_frame_container_->CreateControl(true);
 832
 833     module.RunMessageLoop();
 834
 835     chrome_frame_container_->Release();
 836
 837     PrintResults(test_name_.c_str());
 838
 839     CoFreeUnusedLibraries();
 840     base::PlatformThread::Sleep(base::TimeDelta::FromMilliseconds(100));
 841   }
 842
 843   void NavigateImpl(const std::string& url) {
 844     ASSERT_TRUE(chrome_frame_container_ != NULL);
 845     ASSERT_TRUE(!url.empty());
 846     chrome_frame_container_->Navigate(url.c_str());
 847   }
 848
 849   void TestCompleted() {
 850     // This can get called multiple times if the last url results in a
 851     // redirect.
 852     if (!test_completed_) {
 853       // Measure memory usage for the browser process.
 854       AccountProcessMemoryUsage(chrome_browser_process_id());
 855       // Measure memory usage for the current process.
 856       AccountProcessMemoryUsage(GetCurrentProcessId());
 857
 858       test_completed_ = true;
 859       EXPECT_TRUE(PostMessage(static_cast<HWND>(*chrome_frame_container_),
 860                               WM_CLOSE, 0, 0));
 861     }
 862   }
 863
 864  protected:
 865   CComObject<ChromeFrameActiveXContainerMemory>* chrome_frame_container_;
 866   std::string test_name_;
 867   bool test_completed_;
 868 };
 869
 870 // This class runs tests to measure chrome frame creation only. This will help
 871 // track overall page load performance with chrome frame instances.
 872 class ChromeFrameCreationTest : public ChromeFrameStartupTest {
 873  protected:
 874   virtual void RunStartupTestImpl(TimeTicks* start_time,
 875                                   TimeTicks* end_time) {
 876     SimpleModule module;
 877     AtlAxWinInit();
 878     CComObjectStackEx<ChromeFrameActiveXContainer> wnd;
 879     wnd.CreateChromeFrameWindow(startup_url_);
 880     *start_time = TimeTicks::Now();
 881     wnd.CreateControl(false);
 882     *end_time = TimeTicks::Now();
 883   }
 884 };
 885
 886 // This class provides functionality to run the chrome frame
 887 // performance test against a reference build.
 888 class ChromeFrameCreationTestReference : public ChromeFrameCreationTest {
 889  public:
 890   // override the browser directory to use the reference build instead.
 891   virtual void SetUp() {
 892     chrome_frame_registrar_.reset(new ScopedChromeFrameRegistrar(
 893         ScopedChromeFrameRegistrar::SYSTEM_LEVEL));
 894     chrome_frame_registrar_->RegisterReferenceChromeFrameBuild();
 895     ChromeFrameStartupTest::SetUp();
 896   }
 897
 898   virtual void TearDown() {
 899     chrome_frame_registrar_.reset(NULL);
 900   }
 901 };
 902
 903 // This class measures the creation time for Flash, which would be used
 904 // as a baseline to measure chrome frame creation performance.
 905 class FlashCreationTest : public ChromeFrameStartupTest {
 906  protected:
 907   virtual void RunStartupTestImpl(TimeTicks* start_time,
 908                                   TimeTicks* end_time) {
 909     SimpleModule module;
 910     AtlAxWinInit();
 911     CAxWindow host;
 912     RECT rc = {0, 0, 800, 600};
 913     host.Create(NULL, rc, NULL,
 914         WS_OVERLAPPEDWINDOW | WS_VISIBLE | WS_CLIPCHILDREN | WS_CLIPSIBLINGS,
 915         WS_EX_APPWINDOW | WS_EX_WINDOWEDGE);
 916     EXPECT_TRUE(host.m_hWnd != NULL);
 917
 918     *start_time = TimeTicks::Now();
 919     HRESULT hr = host.CreateControl(L"ShockwaveFlash.ShockwaveFlash");
 920     EXPECT_HRESULT_SUCCEEDED(hr);
 921     *end_time = TimeTicks::Now();
 922
 923     ReleaseHostComReferences(host);
 924   }
 925 };
 926
 927 // This class measures the creation time for Silverlight, which would be used
 928 // as a baseline to measure chrome frame creation performance.
 929 class SilverlightCreationTest : public ChromeFrameStartupTest {
 930  protected:
 931   virtual void RunStartupTestImpl(TimeTicks* start_time,
 932                                   TimeTicks* end_time) {
 933     SimpleModule module;
 934     AtlAxWinInit();
 935     CAxWindow host;
 936     RECT rc = {0, 0, 800, 600};
 937     host.Create(NULL, rc, NULL,
 938         WS_OVERLAPPEDWINDOW | WS_VISIBLE | WS_CLIPCHILDREN | WS_CLIPSIBLINGS,
 939         WS_EX_APPWINDOW | WS_EX_WINDOWEDGE);
 940     EXPECT_TRUE(host.m_hWnd != NULL);
 941
 942     *start_time = TimeTicks::Now();
 943     HRESULT hr = host.CreateControl(L"AgControl.AgControl");
 944     EXPECT_HRESULT_SUCCEEDED(hr);
 945     *end_time = TimeTicks::Now();
 946
 947     ReleaseHostComReferences(host);
 948   }
 949 };
 950
 951 // TODO(rogerm): Flesh out the *PreReadImage* tests to validate an observed
 952 //     change in paging behaviour between raw loading and pre-reading.
 953
 954 // TODO(rogerm): Add checks to the *PreReadImage* tests to validate the
 955 //     handling of invalid pe files and paths as input.
 956
 957 TEST(ImagePreReader, PreReadImage) {
 958   base::FilePath current_exe;
 959   ASSERT_TRUE(PathService::Get(base::FILE_EXE, &current_exe));
 960
 961   int64 file_size_64 = 0;
 962   ASSERT_TRUE(file_util::GetFileSize(current_exe, &file_size_64));
 963   ASSERT_TRUE(file_size_64 < std::numeric_limits<std::size_t>::max());
 964   size_t file_size = static_cast<size_t>(file_size_64);
 965
 966   const wchar_t* module_path = current_exe.value().c_str();
 967   const size_t kStepSize = 2 * 1024 * 1024;
 968
 969   ASSERT_TRUE(
 970       ImagePreReader::PreReadImage(module_path, 0, kStepSize));
 971   ASSERT_TRUE(
 972       ImagePreReader::PreReadImage(module_path, file_size / 4, kStepSize));
 973   ASSERT_TRUE(
 974       ImagePreReader::PreReadImage(module_path, file_size / 2, kStepSize));
 975   ASSERT_TRUE(
 976       ImagePreReader::PreReadImage(module_path, file_size, kStepSize));
 977   ASSERT_TRUE(
 978       ImagePreReader::PreReadImage(module_path, file_size * 2, kStepSize));
 979 }
 980
 981 TEST(ImagePreReader, PartialPreReadImage) {
 982   base::FilePath current_exe;
 983   ASSERT_TRUE(PathService::Get(base::FILE_EXE, &current_exe));
 984
 985   const wchar_t* module_path = current_exe.value().c_str();
 986   const size_t kStepSize = 2 * 1024 * 1024;
 987
 988   ASSERT_TRUE(
 989       ImagePreReader::PartialPreReadImage(module_path, 0, kStepSize));
 990   ASSERT_TRUE(
 991       ImagePreReader::PartialPreReadImage(module_path, 25, kStepSize));
 992   ASSERT_TRUE(
 993       ImagePreReader::PartialPreReadImage(module_path, 50, kStepSize));
 994   ASSERT_TRUE(
 995       ImagePreReader::PartialPreReadImage(module_path, 100, kStepSize));
 996   ASSERT_TRUE(
 997       ImagePreReader::PartialPreReadImage(module_path, 150, kStepSize));
 998 }
 999
1000 TEST(ImagePreReader, PartialPreReadImageOnDisk) {
1001   base::FilePath current_exe;
1002   ASSERT_TRUE(PathService::Get(base::FILE_EXE, &current_exe));
1003
1004   const wchar_t* module_path = current_exe.value().c_str();
1005   const size_t kChunkSize = 2 * 1024 * 1024;
1006
1007   ASSERT_TRUE(
1008       ImagePreReader::PartialPreReadImageOnDisk(module_path, 0, kChunkSize));
1009   ASSERT_TRUE(
1010       ImagePreReader::PartialPreReadImageOnDisk(module_path, 25, kChunkSize));
1011   ASSERT_TRUE(
1012       ImagePreReader::PartialPreReadImageOnDisk(module_path, 50, kChunkSize));
1013   ASSERT_TRUE(
1014       ImagePreReader::PartialPreReadImageOnDisk(module_path, 100, kChunkSize));
1015   ASSERT_TRUE(
1016       ImagePreReader::PartialPreReadImageOnDisk(module_path, 150, kChunkSize));
1017 }
1018
1019 TEST(ImagePreReader, PartialPreReadImageInMemory) {
1020   base::FilePath current_exe;
1021   ASSERT_TRUE(PathService::Get(base::FILE_EXE, &current_exe));
1022   const wchar_t* module_path = current_exe.value().c_str();
1023
1024   ASSERT_TRUE(
1025       ImagePreReader::PartialPreReadImageInMemory(module_path, 0));
1026   ASSERT_TRUE(
1027       ImagePreReader::PartialPreReadImageInMemory(module_path, 25));
1028   ASSERT_TRUE(
1029       ImagePreReader::PartialPreReadImageInMemory(module_path, 50));
1030   ASSERT_TRUE(
1031       ImagePreReader::PartialPreReadImageInMemory(module_path, 100));
1032   ASSERT_TRUE(
1033       ImagePreReader::PartialPreReadImageInMemory(module_path, 150));
1034 }
1035
1036 TEST(ChromeFramePerf, DISABLED_HostActiveX) {
1037   // TODO(stoyan): Create a low integrity level thread && perform the test there
1038   SimpleModule module;
1039   AtlAxWinInit();
1040   CComObjectStackEx<ChromeFrameActiveXContainerPerf> wnd;
1041   wnd.CreateChromeFrameWindow("http://www.google.com");
1042   wnd.CreateControl(true);
1043   module.RunMessageLoop();
1044 }
1045
1046 TEST(ChromeFramePerf, DISABLED_HostActiveXInvalidURL) {
1047   // TODO(stoyan): Create a low integrity level thread && perform the test there
1048   SimpleModule module;
1049   AtlAxWinInit();
1050   CComObjectStackEx<ChromeFrameActiveXContainerPerf> wnd;
1051   wnd.CreateChromeFrameWindow("http://non-existent-domain.org/");
1052   wnd.CreateControl(true);
1053   module.RunMessageLoop();
1054 }
1055
1056 TEST_F(ChromeFrameStartupTestActiveX, PerfWarm) {
1057   RunStartupTest("warm", "t", "about:blank", false /* cold */, 0, NULL,
1058                  true /* important */, false);
1059 }
1060
1061 TEST_F(ChromeFrameBinariesLoadTest, PerfWarm) {
1062   RunStartupTest("binary_load_warm", "t", "", false /* cold */, 0, NULL,
1063                  true /* important */, false);
1064 }
1065
1066 TEST_F(ChromeFrameStartupTestActiveX, PerfCold) {
1067   SetConfigInt(L"PreRead", 0);
1068   base::FilePath binaries_to_evict[] = {
1069     ffmpegsumo_dll_, chrome_exe_, chrome_dll_, chrome_frame_dll_
1070   };
1071   RunStartupTest("cold", "t", "about:blank", true /* cold */,
1072                  arraysize(binaries_to_evict), binaries_to_evict,
1073                  false /* not important */, false);
1074   DeleteConfigValue(L"PreRead");
1075 }
1076
1077 TEST_F(ChromeFrameStartupTestActiveX, PerfColdPreRead) {
1078   SetConfigInt(L"PreRead", 1);
1079   base::FilePath binaries_to_evict[] = {
1080     ffmpegsumo_dll_, chrome_exe_, chrome_dll_, chrome_frame_dll_
1081   };
1082   RunStartupTest("cold_preread", "t", "about:blank", true /* cold */,
1083                  arraysize(binaries_to_evict), binaries_to_evict,
1084                  false /* not important */, false);
1085   DeleteConfigValue(L"PreRead");
1086 }
1087
1088 TEST_F(ChromeFrameBinariesLoadTest, PerfCold) {
1089   base::FilePath binaries_to_evict[] = {chrome_exe_, chrome_dll_};
1090   RunStartupTest("binary_load_cold", "t", "", true /* cold */,
1091                  arraysize(binaries_to_evict), binaries_to_evict,
1092                  false /* not important */, false);
1093 }
1094
1095 TEST_F(ChromeFrameBinariesLoadTest, PerfColdPreRead) {
1096   base::FilePath binaries_to_evict[] = {chrome_exe_, chrome_dll_};
1097   pre_read_type_ = kPreReadFull;
1098   RunStartupTest("binary_load_cold_preread", "t", "", true /* cold */,
1099                  arraysize(binaries_to_evict), binaries_to_evict,
1100                  false /* not important */, false);
1101 }
1102
1103 TEST_F(ChromeFrameBinariesLoadTest, PerfColdPartialPreRead15) {
1104   base::FilePath binaries_to_evict[] = {chrome_exe_, chrome_dll_};
1105   pre_read_type_ = kPreReadPartial;
1106   percentage_to_preread_ = 15;
1107   RunStartupTest("binary_load_cold_partial_preread", "t", "", true /* cold */,
1108                  arraysize(binaries_to_evict), binaries_to_evict,
1109                  false /* not important */, false);
1110 }
1111
1112
1113 TEST_F(ChromeFrameBinariesLoadTest, PerfColdPartialPreRead25) {
1114   base::FilePath binaries_to_evict[] = {chrome_exe_, chrome_dll_};
1115   pre_read_type_ = kPreReadPartial;
1116   percentage_to_preread_ = 25;
1117   RunStartupTest("binary_load_cold_partial_preread", "t", "", true /* cold */,
1118                  arraysize(binaries_to_evict), binaries_to_evict,
1119                  false /* not important */, false);
1120 }
1121
1122 TEST_F(ChromeFrameBinariesLoadTest, PerfColdPartialPreRead40) {
1123   base::FilePath binaries_to_evict[] = {chrome_exe_, chrome_dll_};
1124   pre_read_type_ = kPreReadPartial;
1125   percentage_to_preread_ = 40;
1126   RunStartupTest("binary_load_cold_partial_preread", "t", "", true /* cold */,
1127                  arraysize(binaries_to_evict), binaries_to_evict,
1128                  false /* not important */, false);
1129 }
1130
1131 TEST_F(ChromeFrameStartupTestActiveXReference, PerfWarm) {
1132   RunStartupTest("warm", "t_ref", "about:blank", false /* cold */, 0, NULL,
1133                  true /* important */, false);
1134 }
1135
1136 TEST_F(ChromeFrameStartupTestActiveX, PerfChromeFrameInitializationWarm) {
1137   RunStartupTest("ChromeFrame_init_warm", "t", "", false /* cold */, 0,
1138                  NULL, true /* important */, false);
1139 }
1140
1141 TEST_F(ChromeFrameStartupTestActiveX, PerfChromeFrameInitializationCold) {
1142   base::FilePath binaries_to_evict[] = {chrome_frame_dll_};
1143   RunStartupTest("ChromeFrame_init_cold", "t", "", true /* cold */,
1144                  arraysize(binaries_to_evict), binaries_to_evict,
1145                  false /* not important */, false);
1146 }
1147
1148 TEST_F(ChromeFrameStartupTestActiveXReference,
1149        PerfChromeFrameInitializationWarm) {
1150   RunStartupTest("ChromeFrame_init_warm", "t_ref", "", false /* cold */, 0,
1151                  NULL, true /* important */, false);
1152 }
1153
1154 typedef ChromeFrameActiveXMemoryTest<ChromeFrameMemoryTest>
1155     RegularChromeFrameActiveXMemoryTest;
1156
1157 TEST_F(RegularChromeFrameActiveXMemoryTest, MemoryTestAboutBlank) {
1158   char *urls[] = {"about:blank"};
1159   RunTest("memory_about_blank", urls, arraysize(urls));
1160 }
1161
1162 // TODO(iyengar)
1163 // Revisit why the chrome frame dll does not unload correctly when this test is
1164 // run.
1165 // http://code.google.com/p/chromium/issues/detail?id=47812
1166 TEST_F(RegularChromeFrameActiveXMemoryTest, DISABLED_MemoryTestUrls) {
1167   // TODO(iyengar)
1168   // We should use static pages to measure memory usage.
1169   char *urls[] = {
1170     "http://www.youtube.com/watch?v=PN2HAroA12w",
1171     "http://www.youtube.com/watch?v=KmLJDrsaJmk&feature=channel"
1172   };
1173
1174   RunTest("memory", urls, arraysize(urls));
1175 }
1176
1177 typedef ChromeFrameActiveXMemoryTest<ChromeFrameMemoryTestReference>
1178     ReferenceBuildChromeFrameActiveXMemoryTest;
1179
1180 // Disabled to investigate why the chrome frame dll does not unload while
1181 // running this test.
1182 // http://code.google.com/p/chromium/issues/detail?id=47812
1183 TEST_F(ReferenceBuildChromeFrameActiveXMemoryTest,
1184        DISABLED_MemoryTestAboutBlank) {
1185   char *urls[] = {"about:blank"};
1186   RunTest("memory_about_blank_reference", urls, arraysize(urls));
1187 }
1188
1189 // TODO(iyengar)
1190 // Revisit why the chrome frame dll does not unload correctly when this test is
1191 // run.
1192 TEST_F(ReferenceBuildChromeFrameActiveXMemoryTest, DISABLED_MemoryTestUrls) {
1193   // TODO(iyengar)
1194   // We should use static pages to measure memory usage.
1195   char *urls[] = {
1196     "http://www.youtube.com/watch?v=PN2HAroA12w",
1197     "http://www.youtube.com/watch?v=KmLJDrsaJmk&feature=channel"
1198   };
1199
1200   RunTest("memory_reference", urls, arraysize(urls));
1201 }
1202
1203 TEST_F(ChromeFrameCreationTest, PerfWarm) {
1204   RunStartupTest("creation_warm", "t", "", false /* cold */, 0,
1205                  NULL, true /* important */, false);
1206 }
1207
1208 TEST_F(ChromeFrameCreationTestReference, PerfWarm) {
1209   RunStartupTest("creation_warm", "t_ref", "about:blank", false /* cold */, 0,
1210                  NULL, true /* not important */, false);
1211 }
1212
1213 TEST_F(FlashCreationTest, DISABLED_PerfWarm) {
1214   RunStartupTest("creation_warm", "t_flash", "", false /* cold */, 0, NULL,
1215                  true /* not important */, false);
1216 }
1217
1218 TEST_F(SilverlightCreationTest, DISABLED_PerfWarm) {
1219   RunStartupTest("creation_warm", "t_silverlight", "", false /* cold */, 0,
1220                  NULL, false /* not important */, false);
1221 }
1222
1223 TEST_F(ChromeFrameCreationTest, PerfCold) {
1224   base::FilePath binaries_to_evict[] = {chrome_frame_dll_};
1225
1226   RunStartupTest("creation_cold", "t", "", true /* cold */,
1227                  arraysize(binaries_to_evict), binaries_to_evict,
1228                  true /* important */, false);
1229 }
1230
1231 // Attempt to evict the Flash control can fail on the buildbot as the dll
1232 // is marked read only. The test run is aborted if we fail to evict the file
1233 // from the cache. This could also fail if the Flash control is in use.
1234 // On Vista this could fail because of UAC
1235 TEST_F(FlashCreationTest, PerfCold) {
1236   base::win::RegKey flash_key(HKEY_CLASSES_ROOT, kFlashControlKey, KEY_READ);
1237
1238   std::wstring plugin_path;
1239   ASSERT_EQ(ERROR_SUCCESS, flash_key.ReadValue(L"", &plugin_path));
1240   ASSERT_FALSE(plugin_path.empty());
1241
1242   base::FilePath flash_path = base::FilePath(plugin_path);
1243   base::FilePath binaries_to_evict[] = {flash_path};
1244
1245   RunStartupTest("creation_cold", "t_flash", "", true /* cold */,
1246                  arraysize(binaries_to_evict), binaries_to_evict,
1247                  false/* important */, true);
1248 }
1249
1250 // This test would fail on Vista due to UAC or if the Silverlight control is
1251 // in use. The test run is aborted if we fail to evict the file from the cache.
1252 // Disabling this test as the Silverlight dll does not seem to get unloaded
1253 // correctly causing the attempt to evict the dll from the system cache to
1254 // fail.
1255 TEST_F(SilverlightCreationTest, DISABLED_PerfCold) {
1256   base::win::RegKey silverlight_key(HKEY_CLASSES_ROOT, kSilverlightControlKey,
1257                                     KEY_READ);
1258
1259   std::wstring plugin_path;
1260   ASSERT_EQ(ERROR_SUCCESS, silverlight_key.ReadValue(L"", &plugin_path));
1261   ASSERT_FALSE(plugin_path.empty());
1262
1263   base::FilePath silverlight_path = base::FilePath(plugin_path);
1264   base::FilePath binaries_to_evict[] = {silverlight_path};
1265
1266   RunStartupTest("creation_cold", "t_silverlight", "", true /* cold */,
1267                  arraysize(binaries_to_evict), binaries_to_evict,
1268                  false /* important */, true);
1269 }
1270
1271 namespace {
1272
1273 // Derive from this class in order to receive custom events traced
1274 // via TRACE_EVENT_XXXX macros from ChromeFrame/Chrome.
1275 class TracedEvents {
1276  public:
1277   virtual void OnTraceEventBegin(EVENT_TRACE* event) {}
1278   virtual void OnTraceEventEnd(EVENT_TRACE* event) {}
1279   virtual void OnTraceEventInstant(EVENT_TRACE* event) {}
1280 };
1281
1282 // For the time being we pass to delegate only base::kTraceEventClass32
1283 // events i.e. these generated by TRACE_EVENT_XXXX macros.
1284 // We may need to add kernel provider and pass Process Start/Exit events etc,
1285 // but for the time being we stick with base::kChromeTraceProviderName
1286 // provider only.
1287 class EtwConsumer : public base::win::EtwTraceConsumerBase<EtwConsumer> {
1288  public:
1289   EtwConsumer() {
1290     set_delegate(NULL);
1291   }
1292
1293   ~EtwConsumer() {
1294     set_delegate(NULL);
1295   }
1296
1297   void set_delegate(TracedEvents* delegate) {
1298     delegate_ = delegate;
1299   }
1300
1301   static void ProcessEvent(EVENT_TRACE* event) {
1302     DCHECK(delegate_);
1303     if (event->Header.Guid != base::debug::kTraceEventClass32)
1304       return;
1305     if (event->Header.Class.Version != 0)
1306       return;
1307
1308     switch (event->Header.Class.Type) {
1309       case base::debug::kTraceEventTypeBegin:
1310         delegate_->OnTraceEventBegin(event);
1311         break;
1312       case base::debug::kTraceEventTypeEnd:
1313         delegate_->OnTraceEventEnd(event);
1314         break;
1315       case base::debug::kTraceEventTypeInstant:
1316         delegate_->OnTraceEventInstant(event);
1317         break;
1318       default:
1319         NOTREACHED();
1320         break;
1321     }
1322   }
1323
1324   static TracedEvents* delegate_;
1325 };
1326
1327 TracedEvents* EtwConsumer::delegate_ = NULL;
1328 };  // namespace
1329
1330 class EtwPerfSession {
1331  public:
1332   EtwPerfSession() {
1333   }
1334
1335   ~EtwPerfSession() {
1336     base::DeleteFile(etl_log_file_, false);
1337   }
1338
1339   void Start() {
1340     // To ensure there is no session leftover from crashes, previous runs, etc.
1341     base::win::EtwTraceProperties ignore;
1342     base::win::EtwTraceController::Stop(L"cf_perf", &ignore);
1343     ASSERT_TRUE(file_util::CreateTemporaryFile(&etl_log_file_));
1344     ASSERT_HRESULT_SUCCEEDED(controller_.StartFileSession(L"cf_perf",
1345         etl_log_file_.value().c_str(), false));
1346     ASSERT_HRESULT_SUCCEEDED(controller_.EnableProvider(
1347         base::debug::kChromeTraceProviderName,
1348         TRACE_LEVEL_INFORMATION,
1349         ~(base::debug::CAPTURE_STACK_TRACE)));
1350   }
1351
1352   HRESULT Stop() {
1353     return controller_.Stop(NULL);
1354   }
1355
1356   void AnalyzeOutput(TracedEvents* delegate) {
1357     EtwConsumer consumer;
1358     consumer.set_delegate(delegate);
1359     consumer.OpenFileSession(etl_log_file_.value().c_str());
1360     consumer.Consume();
1361     consumer.Close();
1362   }
1363
1364   base::FilePath etl_log_file_;
1365   base::win::EtwTraceController controller_;
1366 };
1367
1368 // Base class for the tracing event helper classes.
1369 class MonitorTraceBase {
1370  public:
1371   static bool IsMatchingEvent(EVENT_TRACE* event,
1372                               const base::StringPiece& event_to_compare) {
1373     return event->MofLength > event_to_compare.size() &&
1374       (memcmp(event_to_compare.data(), event->MofData,
1375               event_to_compare.size() + 1) == 0);
1376   }
1377
1378   bool is_valid() const {
1379     return !start_.is_null() && !end_.is_null() && start_ <= end_;
1380   }
1381
1382   base::TimeDelta duration() const {
1383     return end_ - start_;
1384   }
1385
1386   base::Time start_;
1387   base::Time end_;
1388 };
1389
1390 // This class measures the time between begin and end events of a particular
1391 // type.
1392 class MonitorTracePair : public MonitorTraceBase,
1393                          public TracedEvents {
1394  public:
1395   MonitorTracePair() : event_(NULL) {
1396   }
1397
1398   void set_interesting_event(const char* event) {
1399     event_ = event;
1400   }
1401
1402   virtual void OnTraceEventBegin(EVENT_TRACE* event) {
1403     if (IsMatchingEvent(event, event_)) {
1404       EXPECT_TRUE(start_.is_null());
1405       start_ = base::Time::FromFileTime(
1406           reinterpret_cast<FILETIME&>(event->Header.TimeStamp));
1407     }
1408   }
1409
1410   virtual void OnTraceEventEnd(EVENT_TRACE* event) {
1411     if (IsMatchingEvent(event, event_)) {
1412       EXPECT_FALSE(start_.is_null());
1413       EXPECT_TRUE(end_.is_null());
1414       end_ = base::Time::FromFileTime(
1415         reinterpret_cast<FILETIME&>(event->Header.TimeStamp));
1416     }
1417   }
1418
1419   base::StringPiece event_;
1420 };
1421
1422 // This class measures the time between two events.
1423 class MonitorTraceBetweenEventPair : public MonitorTraceBase,
1424                                      public TracedEvents {
1425  public:
1426   MonitorTraceBetweenEventPair() : event_end_(NULL),
1427                                    event_start_(NULL) {
1428   }
1429
1430   void set_start_event(const char* event) {
1431     event_start_ = event;
1432   }
1433
1434   void set_end_event(const char* event) {
1435     event_end_ = event;
1436   }
1437
1438   virtual void OnTraceEventBegin(EVENT_TRACE* event) {
1439     if (IsMatchingEvent(event, event_start_)) {
1440       EXPECT_TRUE(start_.is_null());
1441       EXPECT_TRUE(end_.is_null());
1442       start_ = base::Time::FromFileTime(
1443         reinterpret_cast<FILETIME&>(event->Header.TimeStamp));
1444     } else if (IsMatchingEvent(event, event_end_)) {
1445       EXPECT_FALSE(start_.is_null());
1446       EXPECT_TRUE(end_.is_null());
1447       end_ = base::Time::FromFileTime(
1448         reinterpret_cast<FILETIME&>(event->Header.TimeStamp));
1449     }
1450   }
1451
1452   virtual void OnTraceEventEnd(EVENT_TRACE* event) {}
1453
1454   base::StringPiece event_start_;
1455   base::StringPiece event_end_;
1456 };
1457
1458 // The very same as UIPerfTest::PrintResultXXXX without the need to
1459 // create an UIPerfTest instance.
1460 void PrintResultsImpl(const std::string& measurement,
1461                       const std::string& modifier,
1462                       const std::string& trace,
1463                       const std::string& values,
1464                       const std::string& prefix,
1465                       const std::string& suffix,
1466                       const std::string& units,
1467                       bool important) {
1468   // <*>RESULT <graph_name>: <trace_name>= <value> <units>
1469   // <*>RESULT <graph_name>: <trace_name>= {<mean>, <std deviation>} <units>
1470   // <*>RESULT <graph_name>: <trace_name>= [<value>,value,value,...,] <units>
1471   printf("%sRESULT %s%s: %s= %s%s%s %s\n",
1472          important ? "*" : "", measurement.c_str(), modifier.c_str(),
1473          trace.c_str(), prefix.c_str(), values.c_str(), suffix.c_str(),
1474          units.c_str());
1475 }
1476
1477 void PrintResultList(const std::string& measurement,
1478                      const std::string& modifier,
1479                      const std::string& trace,
1480                      const std::string& values,
1481                      const std::string& units,
1482                      bool important) {
1483   PrintResultsImpl(measurement, modifier, trace, values,
1484       "[", "]", units, important);
1485 }
1486
1487 bool RunSingleTestOutOfProc(const std::string& test_name) {
1488   base::FilePath path;
1489   if (!PathService::Get(base::DIR_EXE, &path))
1490     return false;
1491   path = path.Append(L"chrome_frame_tests.exe");
1492
1493   CommandLine cmd_line(path);
1494   // Always enable disabled tests.  This method is not called with disabled
1495   // tests unless this flag was specified to the browser test executable.
1496   cmd_line.AppendSwitch("gtest_also_run_disabled_tests");
1497   cmd_line.AppendSwitchASCII("gtest_filter", test_name);
1498
1499   base::ProcessHandle process_handle;
1500   if (!base::LaunchProcess(cmd_line, base::LaunchOptions(), &process_handle))
1501     return false;
1502
1503   base::TimeDelta test_terminate_timeout = base::TimeDelta::FromMinutes(1);
1504   int exit_code = 0;
1505   if (!base::WaitForExitCodeWithTimeout(process_handle, &exit_code,
1506                                         test_terminate_timeout)) {
1507     LOG(ERROR) << "Test timeout (" << test_terminate_timeout.InMilliseconds()
1508                << " ms) exceeded for " << test_name;
1509
1510     exit_code = -1;  // Set a non-zero exit code to signal a failure.
1511
1512     // Ensure that the process terminates.
1513     base::KillProcess(process_handle, -1, true);
1514   }
1515
1516   base::CloseProcessHandle(process_handle);
1517
1518   return exit_code == 0;
1519 }
1520
1521 template <class Monitor>
1522 void PrintPerfTestResults(const Monitor* monitor,
1523                           int num_cycles,
1524                           const char* result_name) {
1525   std::string times;
1526
1527   for (int i = 0; i < num_cycles; ++i) {
1528     ASSERT_TRUE(monitor[i].is_valid());
1529     base::StringAppendF(&times,
1530                         "%.2f,",
1531                         monitor[i].duration().InMillisecondsF());
1532   }
1533
1534   PrintResultList(result_name, "", "t", times, "ms", false);
1535 }
1536
1537 TEST(TestAsPerfTest, MetaTag_createproxy) {
1538   const int kNumCycles = 10;
1539
1540   MonitorTracePair create_proxy_monitor[kNumCycles];
1541   MonitorTraceBetweenEventPair browser_main_start_monitor[kNumCycles];
1542   MonitorTraceBetweenEventPair browser_main_loop_monitor[kNumCycles];
1543   MonitorTraceBetweenEventPair automation_provider_start_monitor[kNumCycles];
1544   MonitorTraceBetweenEventPair automation_provider_connect_monitor[kNumCycles];
1545   MonitorTracePair external_tab_navigate_monitor[kNumCycles];
1546   MonitorTracePair pre_read_chrome_monitor[kNumCycles];
1547   MonitorTraceBetweenEventPair renderer_main_monitor[kNumCycles];
1548
1549   for (int i = 0; i < kNumCycles; ++i) {
1550     EtwPerfSession perf_session;
1551     ASSERT_NO_FATAL_FAILURE(perf_session.Start());
1552     ASSERT_TRUE(RunSingleTestOutOfProc(
1553         "ChromeFrameTestWithWebServer.FullTabModeIE_MetaTag"));
1554     // Since we cannot have array of objects with a non-default constructor,
1555     // dedicated method is used to initialize watched event.
1556     create_proxy_monitor[i].set_interesting_event("chromeframe.createproxy");
1557
1558     browser_main_start_monitor[i].set_start_event("chromeframe.createproxy");
1559     browser_main_start_monitor[i].set_end_event("BrowserMain");
1560
1561     browser_main_loop_monitor[i].set_start_event("BrowserMain");
1562     browser_main_loop_monitor[i].set_end_event("BrowserMain:MESSAGE_LOOP");
1563
1564     automation_provider_start_monitor[i].set_start_event("BrowserMain");
1565     automation_provider_start_monitor[i].set_end_event(
1566         "AutomationProvider::AutomationProvider");
1567
1568     automation_provider_connect_monitor[i].set_start_event(
1569         "AutomationProvider::AutomationProvider");
1570     automation_provider_connect_monitor[i].set_end_event(
1571         "AutomationProvider::InitializeChannel");
1572
1573     external_tab_navigate_monitor[i].set_interesting_event(
1574         "ExternalTabContainerWin::Navigate");
1575
1576     renderer_main_monitor[i].set_start_event(
1577         "ExternalTabContainerWin::Navigate");
1578     renderer_main_monitor[i].set_end_event("RendererMain");
1579
1580     pre_read_chrome_monitor[i].set_interesting_event("PreReadImage");
1581
1582     ASSERT_HRESULT_SUCCEEDED(perf_session.Stop());
1583
1584     perf_session.AnalyzeOutput(&create_proxy_monitor[i]);
1585     perf_session.AnalyzeOutput(&browser_main_start_monitor[i]);
1586     perf_session.AnalyzeOutput(&browser_main_loop_monitor[i]);
1587     perf_session.AnalyzeOutput(&automation_provider_start_monitor[i]);
1588     perf_session.AnalyzeOutput(&automation_provider_connect_monitor[i]);
1589     perf_session.AnalyzeOutput(&external_tab_navigate_monitor[i]);
1590     perf_session.AnalyzeOutput(&pre_read_chrome_monitor[i]);
1591     perf_session.AnalyzeOutput(&renderer_main_monitor[i]);
1592   }
1593
1594   // Print results
1595   PrintPerfTestResults(create_proxy_monitor, kNumCycles, "createproxy");
1596   PrintPerfTestResults(browser_main_start_monitor, kNumCycles,
1597                        "browserstart");
1598   PrintPerfTestResults(browser_main_loop_monitor, kNumCycles,
1599                        "browserloop");
1600   PrintPerfTestResults(automation_provider_start_monitor, kNumCycles,
1601                        "automationproviderstart");
1602   PrintPerfTestResults(automation_provider_connect_monitor, kNumCycles,
1603                        "automationproviderconnect");
1604   PrintPerfTestResults(external_tab_navigate_monitor, kNumCycles,
1605                        "externaltabnavigate");
1606   PrintPerfTestResults(renderer_main_monitor, kNumCycles,
1607                        "beginrenderermain");
1608 #ifdef NDEBUG
1609   PrintPerfTestResults(pre_read_chrome_monitor, kNumCycles, "PreReadImage");
1610 #endif  // NDEBUG
1611 }