import gm_json
IMAGE_FILENAME_RE = re.compile(gm_json.IMAGE_FILENAME_PATTERN)
+CATEGORIES_TO_SUMMARIZE = [
+ 'builder', 'test', 'config', 'resultType',
+]
class Results(object):
""" Loads actual and expected results from all builders, supplying combined
"""
self._actual_builder_dicts = Results._GetDictsFromRoot(actuals_root)
self._expected_builder_dicts = Results._GetDictsFromRoot(expected_root)
- self._all_results = self._Combine()
+ self._all_results = Results._Combine(
+ actual_builder_dicts=self._actual_builder_dicts,
+ expected_builder_dicts=self._expected_builder_dicts)
def GetAll(self):
- """Return results of all tests, as a list in this form:
+ """Return results of all tests, as a dictionary in this form:
- [
+ {
+ "categories": # dictionary of categories listed in
+ # CATEGORIES_TO_SUMMARIZE, with the number of times
+ # each value appears within its category
{
- "builder": "Test-Mac10.6-MacMini4.1-GeForce320M-x86-Debug",
- "test": "bigmatrix",
- "config": "8888",
- "resultType": "failed",
- "expectedHashType": "bitmap-64bitMD5",
- "expectedHashDigest": "10894408024079689926",
- "actualHashType": "bitmap-64bitMD5",
- "actualHashDigest": "2409857384569",
- },
- ...
- ]
+ "resultType": # category name
+ {
+ "failed": 29, # category value and total number found of that value
+ "failure-ignored": 948,
+ "no-comparison": 4502,
+ "succeeded": 38609,
+ },
+ "builder":
+ {
+ "Test-Mac10.6-MacMini4.1-GeForce320M-x86-Debug": 1286,
+ "Test-Mac10.6-MacMini4.1-GeForce320M-x86-Release": 1134,
+ ...
+ },
+ ... # other categories from CATEGORIES_TO_SUMMARIZE
+ }, # end of "categories" dictionary
+
+ "testData": # list of test results, with a dictionary for each
+ [
+ {
+ "builder": "Test-Mac10.6-MacMini4.1-GeForce320M-x86-Debug",
+ "test": "bigmatrix",
+ "config": "8888",
+ "resultType": "failed",
+ "expectedHashType": "bitmap-64bitMD5",
+ "expectedHashDigest": "10894408024079689926",
+ "actualHashType": "bitmap-64bitMD5",
+ "actualHashDigest": "2409857384569",
+ },
+ ...
+ ], # end of "testData" list
+ }
"""
return self._all_results
meta_dict[builder] = gm_json.LoadFromFile(fullpath)
return meta_dict
- def _Combine(self):
- """Returns a list of all tests, across all builders, based on the
- contents of self._actual_builder_dicts and self._expected_builder_dicts .
- Returns the list in the same form needed for GetAllResults().
+ @staticmethod
+ def _Combine(actual_builder_dicts, expected_builder_dicts):
+ """Gathers the results of all tests, across all builders (based on the
+ contents of actual_builder_dicts and expected_builder_dicts)
+ and returns it in a list in the same form needed for self.GetAll().
+
+ This is a static method, because once we start refreshing results
+ asynchronously, we need to make sure we are not corrupting the object's
+ member variables.
"""
- all_tests = []
- for builder in sorted(self._actual_builder_dicts.keys()):
+ test_data = []
+ category_dict = {}
+ for builder in sorted(actual_builder_dicts.keys()):
actual_results_for_this_builder = (
- self._actual_builder_dicts[builder][gm_json.JSONKEY_ACTUALRESULTS])
+ actual_builder_dicts[builder][gm_json.JSONKEY_ACTUALRESULTS])
for result_type in sorted(actual_results_for_this_builder.keys()):
results_of_this_type = actual_results_for_this_builder[result_type]
if not results_of_this_type:
try:
# TODO(epoger): assumes a single allowed digest per test
expected_image = (
- self._expected_builder_dicts
+ expected_builder_dicts
[builder][gm_json.JSONKEY_EXPECTEDRESULTS]
[image_name][gm_json.JSONKEY_EXPECTEDRESULTS_ALLOWEDDIGESTS]
[0])
else:
updated_result_type = result_type
- # TODO(epoger): For now, don't include succeeded results.
- # There are so many of them that they make the client too slow.
- if updated_result_type == gm_json.JSONKEY_ACTUALRESULTS_SUCCEEDED:
- continue
-
(test, config) = IMAGE_FILENAME_RE.match(image_name).groups()
- all_tests.append({
+ results_for_this_test = {
"builder": builder,
"test": test,
"config": config,
"actualHashDigest": str(actual_image[1]),
"expectedHashType": expected_image[0],
"expectedHashDigest": str(expected_image[1]),
- })
- return all_tests
+ }
+ Results._AddToCategoryDict(category_dict, results_for_this_test)
+
+ # TODO(epoger): For now, don't include succeeded results in the raw
+ # data. There are so many of them that they make the client too slow.
+ if updated_result_type != gm_json.JSONKEY_ACTUALRESULTS_SUCCEEDED:
+ test_data.append(results_for_this_test)
+ return {"categories": category_dict, "testData": test_data}
+
+ @staticmethod
+ def _AddToCategoryDict(category_dict, test_results):
+ """Add test_results to the category dictionary we are building
+ (see documentation of self.GetAll() for the format of this dictionary).
+
+ params:
+ category_dict: category dict-of-dicts to add to; modify this in-place
+ test_results: test data with which to update category_list, in a dict:
+ {
+ "category_name": "category_value",
+ "category_name": "category_value",
+ ...
+ }
+ """
+ for category in CATEGORIES_TO_SUMMARIZE:
+ category_value = test_results.get(category)
+ if not category_value:
+ continue # test_results did not include this category, keep going
+ if not category_dict.get(category):
+ category_dict[category] = {}
+ if not category_dict[category].get(category_value):
+ category_dict[category][category_value] = 0
+ category_dict[category][category_value] += 1
--export mode
-->
+ <!-- TODO(epoger): Add some indication of how old the
+ expected/actual data is -->
+
Settings:
<ul>
<!-- TODO(epoger): Now that we get multiple result types in a single
<li>show results of type
<select ng-model="showResultsOfType"
ng-init="showResultsOfType='failed'">
- <option>failed</option>
- <option>failure-ignored</option>
- <!--
- <option>no-comparison</option>
-
- TODO(epoger): For now, I have disabled viewing the
- no-comparison results because there are so many of them, and
- the browser takes forever to download all the images. Maybe
- we should use some sort of lazy-loading technique
- (e.g. http://www.appelsiini.net/projects/lazyload ), so that
- the images are only loaded as they become viewable...
- -->
- <!--
- <option>succeeded</option>
-
+ <option ng-repeat="(resultType, count) in categories['resultType']"
+ value="{{resultType}}">
+ {{resultType}} ({{count}})
+ </option>
+ </select>
+ <!--
TODO(epoger): See results.py: for now, I have disabled
returning succeeded tests as part of the JSON, because it
makes the returned JSON too big (and slows down the client).
- -->
- </select>
+
+ Also, we should use some sort of lazy-loading technique
+ (e.g. http://www.appelsiini.net/projects/lazyload ), so that
+ the images are only loaded as they become viewable...
+ that will help with long lists like resultType='no-comparison'.
+ -->
+ <br>
+ TODO(epoger): 'no-comparison' will probably take forever;
+ see HTML source for details
+ <br>
+ TODO(epoger): 'succeeded' will not show any results;
+ see HTML source for details
</li>
<li>image size
- <input type="text" ng-model="imageSize" ng-init="imageSize=100"
+ <input type="text" ng-model="imageSizePending"
+ ng-init="imageSizePending=100; imageSize=100"
maxlength="4"/>
+ <button ng:click="imageSize=imageSizePending">apply</button>
</li>
</ul>
<p>
+ Click on the column header radio buttons to re-sort by that column...<br>
<!-- TODO(epoger): Show some sort of "loading" message, instead of
an empty table, while the data is loading. Otherwise, if there are
a lot of failures and it takes a long time to load them, the user
might think there are NO failures and leave the page! -->
<table border="1">
<tr>
- <th ng:click="sortColumn='builder'">Builder</th>
- <th ng:click="sortColumn='test'">Test</th>
- <th ng:click="sortColumn='config'">Config</th>
- <th ng:click="sortColumn='expectedHashDigest'">Expected Image</th>
- <th ng:click="sortColumn='actualHashDigest'">Actual Image</th>
+ <th><input ng-model="sortColumn" name="sortColumnRadio" type="radio" value="builder">Builder</input></th>
+ <th><input ng-model="sortColumn" name="sortColumnRadio" type="radio" value="test">Test</input></th>
+ <th><input ng-model="sortColumn" name="sortColumnRadio" type="radio" value="config">Config</input></th>
+ <th><input ng-model="sortColumn" name="sortColumnRadio" type="radio" value="expectedHashDigest">Expected Image</input></th>
+ <th><input ng-model="sortColumn" name="sortColumnRadio" type="radio" value="actualHashDigest">Actual Image</input></th>
<!-- TODO(epoger): Add more columns, such as...
pixel diff
notes/bugs
</tr>
<!-- TODO(epoger): improve the column sorting, as per
http://jsfiddle.net/vojtajina/js64b/14/ -->
- <tr ng-repeat="result in results | filter: { resultType: showResultsOfType } | orderBy: sortColumn">
+ <tr ng-repeat="result in testData | filter: { resultType: showResultsOfType } | orderBy: sortColumn">
<td>{{result.builder}}</td>
<td>{{result.test}}</td>
<td>{{result.config}}</td>
projects / platform / upstream / libSkiaSharp.git / commitdiff