src/content/test/gpu/gpu_tests/pixel.py

   1 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
   2 # Use of this source code is governed by a BSD-style license that can be
   3 # found in the LICENSE file.
   4 from datetime import datetime
   5 import glob
   6 import optparse
   7 import os
   8 import re
   9
  10 import cloud_storage_test_base
  11 import pixel_expectations
  12
  13 from telemetry import test
  14 from telemetry.core import bitmap
  15 from telemetry.page import page_test
  16
  17 test_data_dir = os.path.abspath(os.path.join(
  18     os.path.dirname(__file__), '..', '..', 'data', 'gpu'))
  19
  20 default_reference_image_dir = os.path.join(test_data_dir, 'gpu_reference')
  21
  22 test_harness_script = r"""
  23   var domAutomationController = {};
  24
  25   domAutomationController._succeeded = false;
  26   domAutomationController._finished = false;
  27
  28   domAutomationController.setAutomationId = function(id) {}
  29
  30   domAutomationController.send = function(msg) {
  31     domAutomationController._finished = true;
  32
  33     if(msg.toLowerCase() == "success") {
  34       domAutomationController._succeeded = true;
  35     } else {
  36       domAutomationController._succeeded = false;
  37     }
  38   }
  39
  40   window.domAutomationController = domAutomationController;
  41 """
  42
  43 class PixelTestFailure(Exception):
  44   pass
  45
  46 def _DidTestSucceed(tab):
  47   return tab.EvaluateJavaScript('domAutomationController._succeeded')
  48
  49 class _PixelValidator(cloud_storage_test_base.ValidatorBase):
  50   def CustomizeBrowserOptions(self, options):
  51     options.AppendExtraBrowserArgs('--enable-gpu-benchmarking')
  52
  53   def ValidatePage(self, page, tab, results):
  54     if not _DidTestSucceed(tab):
  55       raise page_test.Failure('Page indicated a failure')
  56
  57     if not tab.screenshot_supported:
  58       raise page_test.Failure('Browser does not support screenshot capture')
  59
  60     screenshot = tab.Screenshot(5)
  61
  62     if not screenshot:
  63       raise page_test.Failure('Could not capture screenshot')
  64
  65     if hasattr(page, 'test_rect'):
  66       screenshot = screenshot.Crop(
  67           page.test_rect[0], page.test_rect[1],
  68           page.test_rect[2], page.test_rect[3])
  69
  70     image_name = self._UrlToImageName(page.display_name)
  71
  72     if self.options.upload_refimg_to_cloud_storage:
  73       if self._ConditionallyUploadToCloudStorage(image_name, page, tab,
  74                                                  screenshot):
  75         # This is the new reference image; there's nothing to compare against.
  76         ref_png = screenshot
  77       else:
  78         # There was a preexisting reference image, so we might as well
  79         # compare against it.
  80         ref_png = self._DownloadFromCloudStorage(image_name, page, tab)
  81     elif self.options.download_refimg_from_cloud_storage:
  82       # This bot doesn't have the ability to properly generate a
  83       # reference image, so download it from cloud storage.
  84       ref_png = self._DownloadFromCloudStorage(image_name, page, tab)
  85     else:
  86       # Legacy path using on-disk results.
  87       ref_png = self._GetReferenceImage(self.options.reference_dir,
  88           image_name, page.revision, screenshot)
  89
  90     # Test new snapshot against existing reference image
  91     if not ref_png.IsEqual(screenshot, tolerance=2):
  92       if self.options.test_machine_name:
  93         self._UploadErrorImagesToCloudStorage(image_name, screenshot, ref_png)
  94       else:
  95         self._WriteErrorImages(self.options.generated_dir, image_name,
  96                                screenshot, ref_png)
  97       raise page_test.Failure('Reference image did not match captured screen')
  98
  99   def _DeleteOldReferenceImages(self, ref_image_path, cur_revision):
 100     if not cur_revision:
 101       return
 102
 103     old_revisions = glob.glob(ref_image_path + "_*.png")
 104     for rev_path in old_revisions:
 105       m = re.match(r'^.*_(\d+)\.png$', rev_path)
 106       if m and int(m.group(1)) < cur_revision:
 107         print 'Found deprecated reference image. Deleting rev ' + m.group(1)
 108         os.remove(rev_path)
 109
 110   def _GetReferenceImage(self, img_dir, img_name, cur_revision, screenshot):
 111     if not cur_revision:
 112       cur_revision = 0
 113
 114     image_path = os.path.join(img_dir, img_name)
 115
 116     self._DeleteOldReferenceImages(image_path, cur_revision)
 117
 118     image_path = image_path + '_' + str(cur_revision) + '.png'
 119
 120     try:
 121       ref_png = bitmap.Bitmap.FromPngFile(image_path)
 122     except IOError:
 123       ref_png = None
 124
 125     if ref_png:
 126       return ref_png
 127
 128     print 'Reference image not found. Writing tab contents as reference.'
 129
 130     self._WriteImage(image_path, screenshot)
 131     return screenshot
 132
 133 class Pixel(cloud_storage_test_base.TestBase):
 134   test = _PixelValidator
 135   page_set = 'page_sets/pixel_tests.py'
 136
 137   @classmethod
 138   def AddTestCommandLineArgs(cls, group):
 139     super(Pixel, cls).AddTestCommandLineArgs(group)
 140     group.add_option('--reference-dir',
 141         help='Overrides the default on-disk location for reference images '
 142         '(only used for local testing without a cloud storage account)',
 143         default=default_reference_image_dir)
 144
 145   def CreatePageSet(self, options):
 146     page_set = super(Pixel, self).CreatePageSet(options)
 147     for page in page_set.pages:
 148       page.script_to_evaluate_on_commit = test_harness_script
 149     return page_set
 150
 151   def CreateExpectations(self, page_set):
 152     return pixel_expectations.PixelExpectations()