1#!/usr/bin/env python 2# Copyright 2017 gRPC authors. 3# 4# Licensed under the Apache License, Version 2.0 (the "License"); 5# you may not use this file except in compliance with the License. 6# You may obtain a copy of the License at 7# 8# http://www.apache.org/licenses/LICENSE-2.0 9# 10# Unless required by applicable law or agreed to in writing, software 11# distributed under the License is distributed on an "AS IS" BASIS, 12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13# See the License for the specific language governing permissions and 14# limitations under the License. 15"""Helper to upload Jenkins test results to BQ""" 16 17from __future__ import print_function 18 19import os 20import six 21import sys 22import time 23import uuid 24 25gcp_utils_dir = os.path.abspath( 26 os.path.join(os.path.dirname(__file__), '../../gcp/utils')) 27sys.path.append(gcp_utils_dir) 28import big_query_utils 29 30_DATASET_ID = 'jenkins_test_results' 31_DESCRIPTION = 'Test results from master job run on Jenkins' 32# 90 days in milliseconds 33_EXPIRATION_MS = 90 * 24 * 60 * 60 * 1000 34_PARTITION_TYPE = 'DAY' 35_PROJECT_ID = 'grpc-testing' 36_RESULTS_SCHEMA = [ 37 ('job_name', 'STRING', 'Name of Jenkins job'), 38 ('build_id', 'INTEGER', 'Build ID of Jenkins job'), 39 ('build_url', 'STRING', 'URL of Jenkins job'), 40 ('test_name', 'STRING', 'Individual test name'), 41 ('language', 'STRING', 'Language of test'), 42 ('platform', 'STRING', 'Platform used for test'), 43 ('config', 'STRING', 'Config used for test'), 44 ('compiler', 'STRING', 'Compiler used for test'), 45 ('iomgr_platform', 'STRING', 'Iomgr used for test'), 46 ('result', 'STRING', 'Test result: PASSED, TIMEOUT, FAILED, or SKIPPED'), 47 ('timestamp', 'TIMESTAMP', 'Timestamp of test run'), 48 ('elapsed_time', 'FLOAT', 'How long test took to run'), 49 ('cpu_estimated', 'FLOAT', 'Estimated CPU usage of test'), 50 ('cpu_measured', 'FLOAT', 'Actual CPU usage of test'), 51 ('return_code', 'INTEGER', 'Exit code of test'), 52] 53_INTEROP_RESULTS_SCHEMA = [ 54 ('job_name', 'STRING', 'Name of Jenkins/Kokoro job'), 55 ('build_id', 'INTEGER', 'Build ID of Jenkins/Kokoro job'), 56 ('build_url', 'STRING', 'URL of Jenkins/Kokoro job'), 57 ('test_name', 'STRING', 58 'Unique test name combining client, server, and test_name'), 59 ('suite', 'STRING', 60 'Test suite: cloud_to_cloud, cloud_to_prod, or cloud_to_prod_auth'), 61 ('client', 'STRING', 'Client language'), 62 ('server', 'STRING', 'Server host name'), 63 ('test_case', 'STRING', 'Name of test case'), 64 ('result', 'STRING', 'Test result: PASSED, TIMEOUT, FAILED, or SKIPPED'), 65 ('timestamp', 'TIMESTAMP', 'Timestamp of test run'), 66 ('elapsed_time', 'FLOAT', 'How long test took to run'), 67] 68 69 70def _get_build_metadata(test_results): 71 """Add Kokoro build metadata to test_results based on environment 72 variables set by Kokoro. 73 """ 74 build_id = os.getenv('KOKORO_BUILD_NUMBER') 75 build_url = 'https://source.cloud.google.com/results/invocations/%s' % os.getenv( 76 'KOKORO_BUILD_ID') 77 job_name = os.getenv('KOKORO_JOB_NAME') 78 79 if build_id: 80 test_results['build_id'] = build_id 81 if build_url: 82 test_results['build_url'] = build_url 83 if job_name: 84 test_results['job_name'] = job_name 85 86 87def _insert_rows_with_retries(bq, bq_table, bq_rows): 88 """Insert rows to bq table. Retry on error.""" 89 # BigQuery sometimes fails with large uploads, so batch 1,000 rows at a time. 90 for i in range((len(bq_rows) / 1000) + 1): 91 max_retries = 3 92 for attempt in range(max_retries): 93 if big_query_utils.insert_rows(bq, _PROJECT_ID, _DATASET_ID, 94 bq_table, 95 bq_rows[i * 1000:(i + 1) * 1000]): 96 break 97 else: 98 if attempt < max_retries - 1: 99 print('Error uploading result to bigquery, will retry.') 100 else: 101 print( 102 'Error uploading result to bigquery, all attempts failed.' 103 ) 104 sys.exit(1) 105 106 107def upload_results_to_bq(resultset, bq_table, extra_fields): 108 """Upload test results to a BQ table. 109 110 Args: 111 resultset: dictionary generated by jobset.run 112 bq_table: string name of table to create/upload results to in BQ 113 extra_fields: dict with extra values that will be uploaded along with the results 114 """ 115 bq = big_query_utils.create_big_query() 116 big_query_utils.create_partitioned_table( 117 bq, 118 _PROJECT_ID, 119 _DATASET_ID, 120 bq_table, 121 _RESULTS_SCHEMA, 122 _DESCRIPTION, 123 partition_type=_PARTITION_TYPE, 124 expiration_ms=_EXPIRATION_MS) 125 126 bq_rows = [] 127 for shortname, results in six.iteritems(resultset): 128 for result in results: 129 test_results = {} 130 _get_build_metadata(test_results) 131 test_results['cpu_estimated'] = result.cpu_estimated 132 test_results['cpu_measured'] = result.cpu_measured 133 test_results['elapsed_time'] = '%.2f' % result.elapsed_time 134 test_results['result'] = result.state 135 test_results['return_code'] = result.returncode 136 test_results['test_name'] = shortname 137 test_results['timestamp'] = time.strftime('%Y-%m-%d %H:%M:%S') 138 for field_name, field_value in six.iteritems(extra_fields): 139 test_results[field_name] = field_value 140 row = big_query_utils.make_row(str(uuid.uuid4()), test_results) 141 bq_rows.append(row) 142 _insert_rows_with_retries(bq, bq_table, bq_rows) 143 144 145def upload_interop_results_to_bq(resultset, bq_table): 146 """Upload interop test results to a BQ table. 147 148 Args: 149 resultset: dictionary generated by jobset.run 150 bq_table: string name of table to create/upload results to in BQ 151 """ 152 bq = big_query_utils.create_big_query() 153 big_query_utils.create_partitioned_table( 154 bq, 155 _PROJECT_ID, 156 _DATASET_ID, 157 bq_table, 158 _INTEROP_RESULTS_SCHEMA, 159 _DESCRIPTION, 160 partition_type=_PARTITION_TYPE, 161 expiration_ms=_EXPIRATION_MS) 162 163 bq_rows = [] 164 for shortname, results in six.iteritems(resultset): 165 for result in results: 166 test_results = {} 167 _get_build_metadata(test_results) 168 test_results['elapsed_time'] = '%.2f' % result.elapsed_time 169 test_results['result'] = result.state 170 test_results['test_name'] = shortname 171 test_results['suite'] = shortname.split(':')[0] 172 test_results['client'] = shortname.split(':')[1] 173 test_results['server'] = shortname.split(':')[2] 174 test_results['test_case'] = shortname.split(':')[3] 175 test_results['timestamp'] = time.strftime('%Y-%m-%d %H:%M:%S') 176 row = big_query_utils.make_row(str(uuid.uuid4()), test_results) 177 bq_rows.append(row) 178 _insert_rows_with_retries(bq, bq_table, bq_rows) 179