tools/run_tests/python_utils/report_utils.py

   1 # Copyright 2015 gRPC authors.
   2 #
   3 # Licensed under the Apache License, Version 2.0 (the "License");
   4 # you may not use this file except in compliance with the License.
   5 # You may obtain a copy of the License at
   6 #
   7 #     http://www.apache.org/licenses/LICENSE-2.0
   8 #
   9 # Unless required by applicable law or agreed to in writing, software
  10 # distributed under the License is distributed on an "AS IS" BASIS,
  11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12 # See the License for the specific language governing permissions and
  13 # limitations under the License.
  14 """Generate XML and HTML test reports."""
  15
  16 try:
  17     from mako.runtime import Context
  18     from mako.template import Template
  19     from mako import exceptions
  20 except (ImportError):
  21     pass  # Mako not installed but it is ok.
  22 import datetime
  23 import os
  24 import string
  25 import xml.etree.cElementTree as ET
  26 import six
  27
  28
  29 def _filter_msg(msg, output_format):
  30     """Filters out nonprintable and illegal characters from the message."""
  31     if output_format in ['XML', 'HTML']:
  32         # keep whitespaces but remove formfeed and vertical tab characters
  33         # that make XML report unparsable.
  34         filtered_msg = filter(
  35             lambda x: x in string.printable and x != '\f' and x != '\v',
  36             msg.decode('UTF-8', 'ignore'))
  37         if output_format == 'HTML':
  38             filtered_msg = filtered_msg.replace('"', '&quot;')
  39         return filtered_msg
  40     else:
  41         return msg
  42
  43
  44 def new_junit_xml_tree():
  45     return ET.ElementTree(ET.Element('testsuites'))
  46
  47
  48 def render_junit_xml_report(resultset,
  49                             report_file,
  50                             suite_package='grpc',
  51                             suite_name='tests',
  52                             replace_dots=True,
  53                             multi_target=False):
  54     """Generate JUnit-like XML report."""
  55     if not multi_target:
  56         tree = new_junit_xml_tree()
  57         append_junit_xml_results(tree, resultset, suite_package, suite_name,
  58                                  '1', replace_dots)
  59         create_xml_report_file(tree, report_file)
  60     else:
  61         # To have each test result displayed as a separate target by the Resultstore/Sponge UI,
  62         # we generate a separate XML report file for each test result
  63         for shortname, results in six.iteritems(resultset):
  64             one_result = {shortname: results}
  65             tree = new_junit_xml_tree()
  66             append_junit_xml_results(tree, one_result,
  67                                      '%s_%s' % (suite_package, shortname),
  68                                      '%s_%s' % (suite_name, shortname), '1',
  69                                      replace_dots)
  70             per_suite_report_file = os.path.join(os.path.dirname(report_file),
  71                                                  shortname,
  72                                                  os.path.basename(report_file))
  73             create_xml_report_file(tree, per_suite_report_file)
  74
  75
  76 def create_xml_report_file(tree, report_file):
  77     """Generate JUnit-like report file from xml tree ."""
  78     # ensure the report directory exists
  79     report_dir = os.path.dirname(os.path.abspath(report_file))
  80     if not os.path.exists(report_dir):
  81         os.makedirs(report_dir)
  82     tree.write(report_file, encoding='UTF-8')
  83
  84
  85 def append_junit_xml_results(tree,
  86                              resultset,
  87                              suite_package,
  88                              suite_name,
  89                              id,
  90                              replace_dots=True):
  91     """Append a JUnit-like XML report tree with test results as a new suite."""
  92     if replace_dots:
  93         # ResultStore UI displays test suite names containing dots only as the component
  94         # after the last dot, which results bad info being displayed in the UI.
  95         # We replace dots by another character to avoid this problem.
  96         suite_name = suite_name.replace('.', '_')
  97     testsuite = ET.SubElement(tree.getroot(),
  98                               'testsuite',
  99                               id=id,
 100                               package=suite_package,
 101                               name=suite_name,
 102                               timestamp=datetime.datetime.now().isoformat())
 103     failure_count = 0
 104     error_count = 0
 105     for shortname, results in six.iteritems(resultset):
 106         for result in results:
 107             xml_test = ET.SubElement(testsuite, 'testcase', name=shortname)
 108             if result.elapsed_time:
 109                 xml_test.set('time', str(result.elapsed_time))
 110             filtered_msg = _filter_msg(result.message, 'XML')
 111             if result.state == 'FAILED':
 112                 ET.SubElement(xml_test, 'failure',
 113                               message='Failure').text = filtered_msg
 114                 failure_count += 1
 115             elif result.state == 'TIMEOUT':
 116                 ET.SubElement(xml_test, 'error',
 117                               message='Timeout').text = filtered_msg
 118                 error_count += 1
 119             elif result.state == 'SKIPPED':
 120                 ET.SubElement(xml_test, 'skipped', message='Skipped')
 121     testsuite.set('failures', str(failure_count))
 122     testsuite.set('errors', str(error_count))
 123
 124
 125 def render_interop_html_report(client_langs, server_langs, test_cases,
 126                                auth_test_cases, http2_cases, http2_server_cases,
 127                                resultset, num_failures, cloud_to_prod,
 128                                prod_servers, http2_interop):
 129     """Generate HTML report for interop tests."""
 130     template_file = 'tools/run_tests/interop/interop_html_report.template'
 131     try:
 132         mytemplate = Template(filename=template_file, format_exceptions=True)
 133     except NameError:
 134         print(
 135             'Mako template is not installed. Skipping HTML report generation.')
 136         return
 137     except IOError as e:
 138         print('Failed to find the template %s: %s' % (template_file, e))
 139         return
 140
 141     sorted_test_cases = sorted(test_cases)
 142     sorted_auth_test_cases = sorted(auth_test_cases)
 143     sorted_http2_cases = sorted(http2_cases)
 144     sorted_http2_server_cases = sorted(http2_server_cases)
 145     sorted_client_langs = sorted(client_langs)
 146     sorted_server_langs = sorted(server_langs)
 147     sorted_prod_servers = sorted(prod_servers)
 148
 149     args = {
 150         'client_langs': sorted_client_langs,
 151         'server_langs': sorted_server_langs,
 152         'test_cases': sorted_test_cases,
 153         'auth_test_cases': sorted_auth_test_cases,
 154         'http2_cases': sorted_http2_cases,
 155         'http2_server_cases': sorted_http2_server_cases,
 156         'resultset': resultset,
 157         'num_failures': num_failures,
 158         'cloud_to_prod': cloud_to_prod,
 159         'prod_servers': sorted_prod_servers,
 160         'http2_interop': http2_interop
 161     }
 162
 163     html_report_out_dir = 'reports'
 164     if not os.path.exists(html_report_out_dir):
 165         os.mkdir(html_report_out_dir)
 166     html_file_path = os.path.join(html_report_out_dir, 'index.html')
 167     try:
 168         with open(html_file_path, 'w') as output_file:
 169             mytemplate.render_context(Context(output_file, **args))
 170     except:
 171         print(exceptions.text_error_template().render())
 172         raise
 173
 174
 175 def render_perf_profiling_results(output_filepath, profile_names):
 176     with open(output_filepath, 'w') as output_file:
 177         output_file.write('<ul>\n')
 178         for name in profile_names:
 179             output_file.write('<li><a href=%s>%s</a></li>\n' % (name, name))
 180         output_file.write('</ul>\n')