1# Copyright 2015 gRPC authors. 2# 3# Licensed under the Apache License, Version 2.0 (the "License"); 4# you may not use this file except in compliance with the License. 5# You may obtain a copy of the License at 6# 7# http://www.apache.org/licenses/LICENSE-2.0 8# 9# Unless required by applicable law or agreed to in writing, software 10# distributed under the License is distributed on an "AS IS" BASIS, 11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12# See the License for the specific language governing permissions and 13# limitations under the License. 14"""Generate XML and HTML test reports.""" 15 16try: 17 from mako import exceptions 18 from mako.runtime import Context 19 from mako.template import Template 20except ImportError: 21 pass # Mako not installed but it is ok. 22import datetime 23import os 24import string 25import xml.etree.cElementTree as ET 26 27import six 28 29 30def _filter_msg(msg, output_format): 31 """Filters out nonprintable and illegal characters from the message.""" 32 if output_format in ["XML", "HTML"]: 33 if isinstance(msg, bytes): 34 decoded_msg = msg.decode("UTF-8", "ignore") 35 else: 36 decoded_msg = msg 37 # keep whitespaces but remove formfeed and vertical tab characters 38 # that make XML report unparsable. 39 filtered_msg = "".join( 40 filter( 41 lambda x: x in string.printable and x != "\f" and x != "\v", 42 decoded_msg, 43 ) 44 ) 45 if output_format == "HTML": 46 filtered_msg = filtered_msg.replace('"', """) 47 return filtered_msg 48 else: 49 return msg 50 51 52def new_junit_xml_tree(): 53 return ET.ElementTree(ET.Element("testsuites")) 54 55 56def render_junit_xml_report( 57 resultset, 58 report_file, 59 suite_package="grpc", 60 suite_name="tests", 61 replace_dots=True, 62 multi_target=False, 63): 64 """Generate JUnit-like XML report.""" 65 if not multi_target: 66 tree = new_junit_xml_tree() 67 append_junit_xml_results( 68 tree, resultset, suite_package, suite_name, "1", replace_dots 69 ) 70 create_xml_report_file(tree, report_file) 71 else: 72 # To have each test result displayed as a separate target by the Resultstore/Sponge UI, 73 # we generate a separate XML report file for each test result 74 for shortname, results in six.iteritems(resultset): 75 one_result = {shortname: results} 76 tree = new_junit_xml_tree() 77 append_junit_xml_results( 78 tree, 79 one_result, 80 "%s_%s" % (suite_package, shortname), 81 "%s_%s" % (suite_name, shortname), 82 "1", 83 replace_dots, 84 ) 85 per_suite_report_file = os.path.join( 86 os.path.dirname(report_file), 87 shortname, 88 os.path.basename(report_file), 89 ) 90 create_xml_report_file(tree, per_suite_report_file) 91 92 93def create_xml_report_file(tree, report_file): 94 """Generate JUnit-like report file from xml tree .""" 95 # env variable can be used to override the base location for the reports 96 base_dir = os.getenv("GRPC_TEST_REPORT_BASE_DIR", None) 97 if base_dir: 98 report_file = os.path.join(base_dir, report_file) 99 # ensure the report directory exists 100 report_dir = os.path.dirname(os.path.abspath(report_file)) 101 if not os.path.exists(report_dir): 102 os.makedirs(report_dir) 103 tree.write(report_file, encoding="UTF-8") 104 105 106def append_junit_xml_results( 107 tree, resultset, suite_package, suite_name, id, replace_dots=True 108): 109 """Append a JUnit-like XML report tree with test results as a new suite.""" 110 if replace_dots: 111 # ResultStore UI displays test suite names containing dots only as the component 112 # after the last dot, which results bad info being displayed in the UI. 113 # We replace dots by another character to avoid this problem. 114 suite_name = suite_name.replace(".", "_") 115 testsuite = ET.SubElement( 116 tree.getroot(), 117 "testsuite", 118 id=id, 119 package=suite_package, 120 name=suite_name, 121 timestamp=datetime.datetime.now().isoformat(), 122 ) 123 failure_count = 0 124 error_count = 0 125 for shortname, results in six.iteritems(resultset): 126 for result in results: 127 xml_test = ET.SubElement(testsuite, "testcase", name=shortname) 128 if result.elapsed_time: 129 xml_test.set("time", str(result.elapsed_time)) 130 filtered_msg = _filter_msg(result.message, "XML") 131 if result.state == "FAILED": 132 ET.SubElement( 133 xml_test, "failure", message="Failure" 134 ).text = filtered_msg 135 failure_count += 1 136 elif result.state == "TIMEOUT": 137 ET.SubElement( 138 xml_test, "error", message="Timeout" 139 ).text = filtered_msg 140 error_count += 1 141 elif result.state == "SKIPPED": 142 ET.SubElement(xml_test, "skipped", message="Skipped") 143 testsuite.set("failures", str(failure_count)) 144 testsuite.set("errors", str(error_count)) 145 146 147def render_interop_html_report( 148 client_langs, 149 server_langs, 150 test_cases, 151 auth_test_cases, 152 http2_cases, 153 http2_server_cases, 154 resultset, 155 num_failures, 156 cloud_to_prod, 157 prod_servers, 158 http2_interop, 159): 160 """Generate HTML report for interop tests.""" 161 template_file = "tools/run_tests/interop/interop_html_report.template" 162 try: 163 mytemplate = Template(filename=template_file, format_exceptions=True) 164 except NameError: 165 print( 166 "Mako template is not installed. Skipping HTML report generation." 167 ) 168 return 169 except IOError as e: 170 print(("Failed to find the template %s: %s" % (template_file, e))) 171 return 172 173 sorted_test_cases = sorted(test_cases) 174 sorted_auth_test_cases = sorted(auth_test_cases) 175 sorted_http2_cases = sorted(http2_cases) 176 sorted_http2_server_cases = sorted(http2_server_cases) 177 sorted_client_langs = sorted(client_langs) 178 sorted_server_langs = sorted(server_langs) 179 sorted_prod_servers = sorted(prod_servers) 180 181 args = { 182 "client_langs": sorted_client_langs, 183 "server_langs": sorted_server_langs, 184 "test_cases": sorted_test_cases, 185 "auth_test_cases": sorted_auth_test_cases, 186 "http2_cases": sorted_http2_cases, 187 "http2_server_cases": sorted_http2_server_cases, 188 "resultset": resultset, 189 "num_failures": num_failures, 190 "cloud_to_prod": cloud_to_prod, 191 "prod_servers": sorted_prod_servers, 192 "http2_interop": http2_interop, 193 } 194 195 html_report_out_dir = "reports" 196 if not os.path.exists(html_report_out_dir): 197 os.mkdir(html_report_out_dir) 198 html_file_path = os.path.join(html_report_out_dir, "index.html") 199 try: 200 with open(html_file_path, "w") as output_file: 201 mytemplate.render_context(Context(output_file, **args)) 202 except: 203 print((exceptions.text_error_template().render())) 204 raise 205 206 207def render_perf_profiling_results(output_filepath, profile_names): 208 with open(output_filepath, "w") as output_file: 209 output_file.write("<ul>\n") 210 for name in profile_names: 211 output_file.write("<li><a href=%s>%s</a></li>\n" % (name, name)) 212 output_file.write("</ul>\n") 213