# System-level imports
import fnmatch
import json
+import logging
import os
import re
import sys
CATEGORIES_TO_SUMMARIZE = [
'builder', 'test', 'config', 'resultType',
]
+RESULTS_ALL = 'all'
+RESULTS_FAILURES = 'failures'
class Results(object):
""" Loads actual and expected results from all builders, supplying combined
- reports as requested. """
+ reports as requested.
+
+ Once this object has been constructed, the results are immutable. If you
+ want to update the results based on updated JSON file contents, you will
+ need to create a new Results object."""
def __init__(self, actuals_root, expected_root):
"""
actuals_root: root directory containing all actual-results.json files
expected_root: root directory containing all expected-results.json files
"""
- self._actual_builder_dicts = Results._GetDictsFromRoot(actuals_root)
- self._expected_builder_dicts = Results._GetDictsFromRoot(expected_root)
- self._all_results = Results._Combine(
- actual_builder_dicts=self._actual_builder_dicts,
- expected_builder_dicts=self._expected_builder_dicts)
+ self._actual_builder_dicts = Results._get_dicts_from_root(actuals_root)
+ self._expected_builder_dicts = Results._get_dicts_from_root(expected_root)
+ self._combine_actual_and_expected()
+
+ def get_results_of_type(self, type):
+ """Return results of some/all tests (depending on 'type' parameter).
+
+ Args:
+ type: string describing which types of results to include; must be one
+ of the RESULTS_* constants
- def GetAll(self):
- """Return results of all tests, as a dictionary in this form:
+ Results are returned as a dictionary in this form:
{
'categories': # dictionary of categories listed in
'testData': # list of test results, with a dictionary for each
[
{
- 'index': 0, # index of this result within testData list
'builder': 'Test-Mac10.6-MacMini4.1-GeForce320M-x86-Debug',
'test': 'bigmatrix',
'config': '8888',
], # end of 'testData' list
}
"""
- return self._all_results
+ return self._results[type]
@staticmethod
- def _GetDictsFromRoot(root, pattern='*.json'):
+ def _get_dicts_from_root(root, pattern='*.json'):
"""Read all JSON dictionaries within a directory tree.
Args:
meta_dict[builder] = gm_json.LoadFromFile(fullpath)
return meta_dict
- @staticmethod
- def _Combine(actual_builder_dicts, expected_builder_dicts):
+ def _combine_actual_and_expected(self):
"""Gathers the results of all tests, across all builders (based on the
- contents of actual_builder_dicts and expected_builder_dicts).
-
- This is a static method, because once we start refreshing results
- asynchronously, we need to make sure we are not corrupting the object's
- member variables.
-
- Args:
- actual_builder_dicts: a meta-dictionary of all actual JSON results,
- as returned by _GetDictsFromRoot().
- actual_builder_dicts: a meta-dictionary of all expected JSON results,
- as returned by _GetDictsFromRoot().
-
- Returns:
- A list of all the results of all tests, in the same form returned by
- self.GetAll().
+ contents of self._actual_builder_dicts and self._expected_builder_dicts),
+ and stores them in self._results.
"""
- test_data = []
- category_dict = {}
- Results._EnsureIncludedInCategoryDict(category_dict, 'resultType', [
+ categories_all = {}
+ categories_failures = {}
+ Results._ensure_included_in_category_dict(categories_all,
+ 'resultType', [
gm_json.JSONKEY_ACTUALRESULTS_FAILED,
gm_json.JSONKEY_ACTUALRESULTS_FAILUREIGNORED,
gm_json.JSONKEY_ACTUALRESULTS_NOCOMPARISON,
gm_json.JSONKEY_ACTUALRESULTS_SUCCEEDED,
])
+ Results._ensure_included_in_category_dict(categories_failures,
+ 'resultType', [
+ gm_json.JSONKEY_ACTUALRESULTS_FAILED,
+ gm_json.JSONKEY_ACTUALRESULTS_FAILUREIGNORED,
+ gm_json.JSONKEY_ACTUALRESULTS_NOCOMPARISON,
+ ])
- for builder in sorted(actual_builder_dicts.keys()):
+ data_all = []
+ data_failures = []
+ for builder in sorted(self._actual_builder_dicts.keys()):
actual_results_for_this_builder = (
- actual_builder_dicts[builder][gm_json.JSONKEY_ACTUALRESULTS])
+ self._actual_builder_dicts[builder][gm_json.JSONKEY_ACTUALRESULTS])
for result_type in sorted(actual_results_for_this_builder.keys()):
results_of_this_type = actual_results_for_this_builder[result_type]
if not results_of_this_type:
try:
# TODO(epoger): assumes a single allowed digest per test
expected_image = (
- expected_builder_dicts
+ self._expected_builder_dicts
[builder][gm_json.JSONKEY_EXPECTEDRESULTS]
[image_name][gm_json.JSONKEY_EXPECTEDRESULTS_ALLOWEDDIGESTS]
[0])
if result_type not in [
gm_json.JSONKEY_ACTUALRESULTS_NOCOMPARISON,
gm_json.JSONKEY_ACTUALRESULTS_FAILUREIGNORED] :
- print 'WARNING: No expectations found for test: %s' % {
+ logging.warning('No expectations found for test: %s' % {
'builder': builder,
'image_name': image_name,
'result_type': result_type,
- }
+ })
expected_image = [None, None]
# If this test was recently rebaselined, it will remain in
(test, config) = IMAGE_FILENAME_RE.match(image_name).groups()
results_for_this_test = {
- 'index': len(test_data),
'builder': builder,
'test': test,
'config': config,
'expectedHashType': expected_image[0],
'expectedHashDigest': str(expected_image[1]),
}
- Results._AddToCategoryDict(category_dict, results_for_this_test)
- test_data.append(results_for_this_test)
- return {'categories': category_dict, 'testData': test_data}
+ Results._add_to_category_dict(categories_all, results_for_this_test)
+ data_all.append(results_for_this_test)
+ if updated_result_type != gm_json.JSONKEY_ACTUALRESULTS_SUCCEEDED:
+ Results._add_to_category_dict(categories_failures,
+ results_for_this_test)
+ data_failures.append(results_for_this_test)
+
+ self._results = {
+ RESULTS_ALL:
+ {'categories': categories_all, 'testData': data_all},
+ RESULTS_FAILURES:
+ {'categories': categories_failures, 'testData': data_failures},
+ }
@staticmethod
- def _AddToCategoryDict(category_dict, test_results):
+ def _add_to_category_dict(category_dict, test_results):
"""Add test_results to the category dictionary we are building.
- (See documentation of self.GetAll() for the format of this dictionary.)
+ (See documentation of self.get_results_of_type() for the format of this
+ dictionary.)
Args:
category_dict: category dict-of-dicts to add to; modify this in-place
category_dict[category][category_value] += 1
@staticmethod
- def _EnsureIncludedInCategoryDict(category_dict,
- category_name, category_values):
+ def _ensure_included_in_category_dict(category_dict,
+ category_name, category_values):
"""Ensure that the category name/value pairs are included in category_dict,
even if there aren't any results with that name/value pair.
- (See documentation of self.GetAll() for the format of this dictionary.)
+ (See documentation of self.get_results_of_type() for the format of this
+ dictionary.)
Args:
category_dict: category dict-of-dicts to modify
import argparse
import BaseHTTPServer
import json
+import logging
import os
import posixpath
import re
import shutil
import sys
+import urlparse
# Imports from within Skia
#
the gm-actuals and expectations will automatically be updated every few
minutes. See discussion in https://codereview.chromium.org/24274003/ .
"""
- print 'Checking out latest actual GM results from %s into %s ...' % (
- ACTUALS_SVN_REPO, self._actuals_dir)
+ logging.info('Checking out latest actual GM results from %s into %s ...' % (
+ ACTUALS_SVN_REPO, self._actuals_dir))
actuals_repo = svn.Svn(self._actuals_dir)
if not os.path.isdir(self._actuals_dir):
os.makedirs(self._actuals_dir)
actuals_repo.Checkout(ACTUALS_SVN_REPO, '.')
else:
actuals_repo.Update('.')
- print 'Parsing results from actuals in %s and expectations in %s ...' % (
- self._actuals_dir, self._expectations_dir)
+ logging.info(
+ 'Parsing results from actuals in %s and expectations in %s ...' % (
+ self._actuals_dir, self._expectations_dir))
self.results = results.Results(
actuals_root=self._actuals_dir,
expected_root=self._expectations_dir)
self.fetch_results()
if self._export:
server_address = ('', self._port)
- print ('WARNING: Running in "export" mode. Users on other machines will '
- 'be able to modify your GM expectations!')
+ logging.warning('Running in "export" mode. Users on other machines will '
+ 'be able to modify your GM expectations!')
else:
server_address = ('127.0.0.1', self._port)
http_server = BaseHTTPServer.HTTPServer(server_address, HTTPRequestHandler)
- print 'Ready for requests on http://%s:%d' % (
- http_server.server_name, http_server.server_port)
+ logging.info('Ready for requests on http://%s:%d' % (
+ http_server.server_name, http_server.server_port))
http_server.serve_forever()
""" Handles all GET requests, forwarding them to the appropriate
do_GET_* dispatcher. """
if self.path == '' or self.path == '/' or self.path == '/index.html' :
- self.redirect_to('/static/view.html')
+ self.redirect_to('/static/view.html?resultsToLoad=all')
return
if self.path == '/favicon.ico' :
self.redirect_to('/static/favicon.ico')
dispatcher = dispatchers[dispatcher_name]
dispatcher(remainder)
- def do_GET_results(self, result_type):
+ def do_GET_results(self, type):
""" Handle a GET request for GM results.
- For now, we ignore the remaining path info, because we only know how to
- return all results.
Args:
- result_type: currently unused
-
- TODO(epoger): Unless we start making use of result_type, remove that
- parameter."""
- print 'do_GET_results: sending results of type "%s"' % result_type
- # TODO(epoger): Cache response_dict rather than the results object, to save
- # time on subsequent fetches (no need to regenerate the header, etc.)
- response_dict = _SERVER.results.GetAll()
- if response_dict:
+ type: string indicating which set of results to return;
+ must be one of the results.RESULTS_* constants
+ """
+ logging.debug('do_GET_results: sending results of type "%s"' % type)
+ try:
+ # TODO(epoger): Rather than using a global variable for the handler
+ # to refer to the Server object, make Server a subclass of
+ # HTTPServer, and then it could be available to the handler via
+ # the handler's .server instance variable.
+ response_dict = _SERVER.results.get_results_of_type(type)
response_dict['header'] = {
# Hash of testData, which the client must return with any edits--
# this ensures that the edits were made to a particular dataset.
'isExported': _SERVER.is_exported(),
}
self.send_json_dict(response_dict)
- else:
+ except:
self.send_error(404)
def do_GET_static(self, path):
Args:
path: path to file (under static directory) to retrieve
"""
- print 'do_GET_static: sending file "%s"' % path
+ # Strip arguments ('?resultsToLoad=all') from the path
+ path = urlparse.urlparse(path).path
+
+ logging.debug('do_GET_static: sending file "%s"' % path)
static_dir = os.path.realpath(os.path.join(PARENT_DIRECTORY, 'static'))
full_path = os.path.realpath(os.path.join(static_dir, path))
if full_path.startswith(static_dir):
self.send_file(full_path)
else:
- print ('Attempted do_GET_static() of path [%s] outside of static dir [%s]'
- % (full_path, static_dir))
+ logging.error(
+ 'Attempted do_GET_static() of path [%s] outside of static dir [%s]'
+ % (full_path, static_dir))
self.send_error(404)
def redirect_to(self, url):
def main():
+ logging.basicConfig(level=logging.INFO)
parser = argparse.ArgumentParser()
parser.add_argument('--actuals-dir',
help=('Directory into which we will check out the latest '
projects / platform / upstream / libSkiaSharp.git / commitdiff