From: epoger@google.com Date: Mon, 30 Sep 2013 15:06:25 +0000 (+0000) Subject: HTTP GM results viewer: server now returns category summaries along with testData X-Git-Tag: accepted/tizen/5.0/unified/20181102.025319~10656 X-Git-Url: http://review.tizen.org/git/?a=commitdiff_plain;h=afaad3dd7002feb6e69983ddc4e5148d7803baed;p=platform%2Fupstream%2FlibSkiaSharp.git HTTP GM results viewer: server now returns category summaries along with testData (SkipBuildbotRuns) R=borenet@google.com Review URL: https://codereview.chromium.org/25045003 git-svn-id: http://skia.googlecode.com/svn/trunk@11520 2bbb7eff-a529-9590-31e7-b0007b416f81 --- diff --git a/gm/rebaseline_server/results.py b/gm/rebaseline_server/results.py index 0c50d26..1336097 100755 --- a/gm/rebaseline_server/results.py +++ b/gm/rebaseline_server/results.py @@ -31,6 +31,9 @@ if GM_DIRECTORY not in sys.path: import gm_json IMAGE_FILENAME_RE = re.compile(gm_json.IMAGE_FILENAME_PATTERN) +CATEGORIES_TO_SUMMARIZE = [ + 'builder', 'test', 'config', 'resultType', +] class Results(object): """ Loads actual and expected results from all builders, supplying combined @@ -44,24 +47,49 @@ class Results(object): """ self._actual_builder_dicts = Results._GetDictsFromRoot(actuals_root) self._expected_builder_dicts = Results._GetDictsFromRoot(expected_root) - self._all_results = self._Combine() + self._all_results = Results._Combine( + actual_builder_dicts=self._actual_builder_dicts, + expected_builder_dicts=self._expected_builder_dicts) def GetAll(self): - """Return results of all tests, as a list in this form: + """Return results of all tests, as a dictionary in this form: - [ + { + "categories": # dictionary of categories listed in + # CATEGORIES_TO_SUMMARIZE, with the number of times + # each value appears within its category { - "builder": "Test-Mac10.6-MacMini4.1-GeForce320M-x86-Debug", - "test": "bigmatrix", - "config": "8888", - "resultType": "failed", - "expectedHashType": "bitmap-64bitMD5", - "expectedHashDigest": "10894408024079689926", - "actualHashType": "bitmap-64bitMD5", - "actualHashDigest": "2409857384569", - }, - ... - ] + "resultType": # category name + { + "failed": 29, # category value and total number found of that value + "failure-ignored": 948, + "no-comparison": 4502, + "succeeded": 38609, + }, + "builder": + { + "Test-Mac10.6-MacMini4.1-GeForce320M-x86-Debug": 1286, + "Test-Mac10.6-MacMini4.1-GeForce320M-x86-Release": 1134, + ... + }, + ... # other categories from CATEGORIES_TO_SUMMARIZE + }, # end of "categories" dictionary + + "testData": # list of test results, with a dictionary for each + [ + { + "builder": "Test-Mac10.6-MacMini4.1-GeForce320M-x86-Debug", + "test": "bigmatrix", + "config": "8888", + "resultType": "failed", + "expectedHashType": "bitmap-64bitMD5", + "expectedHashDigest": "10894408024079689926", + "actualHashType": "bitmap-64bitMD5", + "actualHashDigest": "2409857384569", + }, + ... + ], # end of "testData" list + } """ return self._all_results @@ -84,15 +112,21 @@ class Results(object): meta_dict[builder] = gm_json.LoadFromFile(fullpath) return meta_dict - def _Combine(self): - """Returns a list of all tests, across all builders, based on the - contents of self._actual_builder_dicts and self._expected_builder_dicts . - Returns the list in the same form needed for GetAllResults(). + @staticmethod + def _Combine(actual_builder_dicts, expected_builder_dicts): + """Gathers the results of all tests, across all builders (based on the + contents of actual_builder_dicts and expected_builder_dicts) + and returns it in a list in the same form needed for self.GetAll(). + + This is a static method, because once we start refreshing results + asynchronously, we need to make sure we are not corrupting the object's + member variables. """ - all_tests = [] - for builder in sorted(self._actual_builder_dicts.keys()): + test_data = [] + category_dict = {} + for builder in sorted(actual_builder_dicts.keys()): actual_results_for_this_builder = ( - self._actual_builder_dicts[builder][gm_json.JSONKEY_ACTUALRESULTS]) + actual_builder_dicts[builder][gm_json.JSONKEY_ACTUALRESULTS]) for result_type in sorted(actual_results_for_this_builder.keys()): results_of_this_type = actual_results_for_this_builder[result_type] if not results_of_this_type: @@ -102,7 +136,7 @@ class Results(object): try: # TODO(epoger): assumes a single allowed digest per test expected_image = ( - self._expected_builder_dicts + expected_builder_dicts [builder][gm_json.JSONKEY_EXPECTEDRESULTS] [image_name][gm_json.JSONKEY_EXPECTEDRESULTS_ALLOWEDDIGESTS] [0]) @@ -159,13 +193,8 @@ class Results(object): else: updated_result_type = result_type - # TODO(epoger): For now, don't include succeeded results. - # There are so many of them that they make the client too slow. - if updated_result_type == gm_json.JSONKEY_ACTUALRESULTS_SUCCEEDED: - continue - (test, config) = IMAGE_FILENAME_RE.match(image_name).groups() - all_tests.append({ + results_for_this_test = { "builder": builder, "test": test, "config": config, @@ -174,5 +203,35 @@ class Results(object): "actualHashDigest": str(actual_image[1]), "expectedHashType": expected_image[0], "expectedHashDigest": str(expected_image[1]), - }) - return all_tests + } + Results._AddToCategoryDict(category_dict, results_for_this_test) + + # TODO(epoger): For now, don't include succeeded results in the raw + # data. There are so many of them that they make the client too slow. + if updated_result_type != gm_json.JSONKEY_ACTUALRESULTS_SUCCEEDED: + test_data.append(results_for_this_test) + return {"categories": category_dict, "testData": test_data} + + @staticmethod + def _AddToCategoryDict(category_dict, test_results): + """Add test_results to the category dictionary we are building + (see documentation of self.GetAll() for the format of this dictionary). + + params: + category_dict: category dict-of-dicts to add to; modify this in-place + test_results: test data with which to update category_list, in a dict: + { + "category_name": "category_value", + "category_name": "category_value", + ... + } + """ + for category in CATEGORIES_TO_SUMMARIZE: + category_value = test_results.get(category) + if not category_value: + continue # test_results did not include this category, keep going + if not category_dict.get(category): + category_dict[category] = {} + if not category_dict[category].get(category_value): + category_dict[category][category_value] = 0 + category_dict[category][category_value] += 1 diff --git a/gm/rebaseline_server/server.py b/gm/rebaseline_server/server.py index 5b81d8c..34c70f4 100755 --- a/gm/rebaseline_server/server.py +++ b/gm/rebaseline_server/server.py @@ -214,10 +214,10 @@ def main(): 'to access this server. WARNING: doing so will ' 'allow users on other hosts to modify your ' 'GM expectations!')) - parser.add_argument('--port', - help=('Which TCP port to listen on for HTTP requests; ' - 'defaults to %(default)s'), - default=DEFAULT_PORT) + parser.add_argument('--port', type=int, + help=('Which TCP port to listen on for HTTP requests; ' + 'defaults to %(default)s'), + default=DEFAULT_PORT) args = parser.parse_args() global _SERVER _SERVER = Server(expectations_dir=args.expectations_dir, diff --git a/gm/rebaseline_server/static/loader.js b/gm/rebaseline_server/static/loader.js index 68da73a..d08fdcc 100644 --- a/gm/rebaseline_server/static/loader.js +++ b/gm/rebaseline_server/static/loader.js @@ -1,7 +1,7 @@ /* * Loader: - * Reads GM result reports written out by results_loader.py, and imports - * their data into $scope.results . + * Reads GM result reports written out by results.py, and imports + * them into $scope.categories and $scope.testData . */ var Loader = angular.module( 'Loader', @@ -12,8 +12,10 @@ Loader.controller( function($scope, $http) { $http.get("/results/all").then( function(response) { - $scope.results = response.data; + $scope.categories = response.data.categories; + $scope.testData = response.data.testData; $scope.sortColumn = 'test'; + $scope.showResultsOfType = 'failed'; } ); } diff --git a/gm/rebaseline_server/static/view.html b/gm/rebaseline_server/static/view.html index 89ef538..5913e5c 100644 --- a/gm/rebaseline_server/static/view.html +++ b/gm/rebaseline_server/static/view.html @@ -15,6 +15,9 @@ --export mode --> + + Settings:

+ Click on the column header radio buttons to re-sort by that column...
- - - - - + + + + + - +
BuilderTestConfigExpected ImageActual ImageBuilderTestConfigExpected ImageActual Image
{{result.builder}} {{result.test}} {{result.config}}