• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1#!/usr/bin/env python
2# Copyright 2016 gRPC authors.
3#
4# Licensed under the Apache License, Version 2.0 (the "License");
5# you may not use this file except in compliance with the License.
6# You may obtain a copy of the License at
7#
8#     http://www.apache.org/licenses/LICENSE-2.0
9#
10# Unless required by applicable law or agreed to in writing, software
11# distributed under the License is distributed on an "AS IS" BASIS,
12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13# See the License for the specific language governing permissions and
14# limitations under the License.
15"""Tool to get build statistics from Jenkins and upload to BigQuery."""
16
17from __future__ import print_function
18
19import argparse
20import jenkinsapi
21from jenkinsapi.custom_exceptions import JenkinsAPIException
22from jenkinsapi.jenkins import Jenkins
23import json
24import os
25import re
26import sys
27import urllib
28
29gcp_utils_dir = os.path.abspath(
30    os.path.join(os.path.dirname(__file__), '../gcp/utils'))
31sys.path.append(gcp_utils_dir)
32import big_query_utils
33
34_PROJECT_ID = 'grpc-testing'
35_HAS_MATRIX = True
36_BUILDS = {
37    'gRPC_interop_master': not _HAS_MATRIX,
38    'gRPC_master_linux': not _HAS_MATRIX,
39    'gRPC_master_macos': not _HAS_MATRIX,
40    'gRPC_master_windows': not _HAS_MATRIX,
41    'gRPC_performance_master': not _HAS_MATRIX,
42    'gRPC_portability_master_linux': not _HAS_MATRIX,
43    'gRPC_portability_master_windows': not _HAS_MATRIX,
44    'gRPC_master_asanitizer_c': not _HAS_MATRIX,
45    'gRPC_master_asanitizer_cpp': not _HAS_MATRIX,
46    'gRPC_master_msan_c': not _HAS_MATRIX,
47    'gRPC_master_tsanitizer_c': not _HAS_MATRIX,
48    'gRPC_master_tsan_cpp': not _HAS_MATRIX,
49    'gRPC_interop_pull_requests': not _HAS_MATRIX,
50    'gRPC_performance_pull_requests': not _HAS_MATRIX,
51    'gRPC_portability_pull_requests_linux': not _HAS_MATRIX,
52    'gRPC_portability_pr_win': not _HAS_MATRIX,
53    'gRPC_pull_requests_linux': not _HAS_MATRIX,
54    'gRPC_pull_requests_macos': not _HAS_MATRIX,
55    'gRPC_pr_win': not _HAS_MATRIX,
56    'gRPC_pull_requests_asan_c': not _HAS_MATRIX,
57    'gRPC_pull_requests_asan_cpp': not _HAS_MATRIX,
58    'gRPC_pull_requests_msan_c': not _HAS_MATRIX,
59    'gRPC_pull_requests_tsan_c': not _HAS_MATRIX,
60    'gRPC_pull_requests_tsan_cpp': not _HAS_MATRIX,
61}
62_URL_BASE = 'https://grpc-testing.appspot.com/job'
63
64# This is a dynamic list where known and active issues should be added.
65# Fixed ones should be removed.
66# Also try not to add multiple messages from the same failure.
67_KNOWN_ERRORS = [
68    'Failed to build workspace Tests with scheme AllTests',
69    'Build timed out',
70    'TIMEOUT: tools/run_tests/pre_build_node.sh',
71    'TIMEOUT: tools/run_tests/pre_build_ruby.sh',
72    'FATAL: Unable to produce a script file',
73    'FAILED: build_docker_c\+\+',
74    'cannot find package \"cloud.google.com/go/compute/metadata\"',
75    'LLVM ERROR: IO failure on output stream.',
76    'MSBUILD : error MSB1009: Project file does not exist.',
77    'fatal: git fetch_pack: expected ACK/NAK',
78    'Failed to fetch from http://github.com/grpc/grpc.git',
79    ('hudson.remoting.RemotingSystemException: java.io.IOException: '
80     'Backing channel is disconnected.'),
81    'hudson.remoting.ChannelClosedException',
82    'Could not initialize class hudson.Util',
83    'Too many open files in system',
84    'FAILED: bins/tsan/qps_openloop_test GRPC_POLL_STRATEGY=epoll',
85    'FAILED: bins/tsan/qps_openloop_test GRPC_POLL_STRATEGY=legacy',
86    'FAILED: bins/tsan/qps_openloop_test GRPC_POLL_STRATEGY=poll',
87    ('tests.bins/asan/h2_proxy_test streaming_error_response '
88     'GRPC_POLL_STRATEGY=legacy'),
89    'hudson.plugins.git.GitException',
90    'Couldn\'t find any revision to build',
91    'org.jenkinsci.plugin.Diskcheck.preCheckout',
92    'Something went wrong while deleting Files',
93]
94_NO_REPORT_FILES_FOUND_ERROR = 'No test report files were found.'
95_UNKNOWN_ERROR = 'Unknown error'
96_DATASET_ID = 'build_statistics'
97
98
99def _scrape_for_known_errors(html):
100    error_list = []
101    for known_error in _KNOWN_ERRORS:
102        errors = re.findall(known_error, html)
103        this_error_count = len(errors)
104        if this_error_count > 0:
105            error_list.append({
106                'description': known_error,
107                'count': this_error_count
108            })
109            print('====> %d failures due to %s' % (this_error_count,
110                                                   known_error))
111    return error_list
112
113
114def _no_report_files_found(html):
115    return _NO_REPORT_FILES_FOUND_ERROR in html
116
117
118def _get_last_processed_buildnumber(build_name):
119    query = 'SELECT max(build_number) FROM [%s:%s.%s];' % (_PROJECT_ID,
120                                                           _DATASET_ID,
121                                                           build_name)
122    query_job = big_query_utils.sync_query_job(bq, _PROJECT_ID, query)
123    page = bq.jobs().getQueryResults(
124        pageToken=None, **query_job['jobReference']).execute(num_retries=3)
125    if page['rows'][0]['f'][0]['v']:
126        return int(page['rows'][0]['f'][0]['v'])
127    return 0
128
129
130def _process_matrix(build, url_base):
131    matrix_list = []
132    for matrix in build.get_matrix_runs():
133        matrix_str = re.match('.*\\xc2\\xbb ((?:[^,]+,?)+) #.*',
134                              matrix.name).groups()[0]
135        matrix_tuple = matrix_str.split(',')
136        json_url = '%s/config=%s,language=%s,platform=%s/testReport/api/json' % (
137            url_base, matrix_tuple[0], matrix_tuple[1], matrix_tuple[2])
138        console_url = '%s/config=%s,language=%s,platform=%s/consoleFull' % (
139            url_base, matrix_tuple[0], matrix_tuple[1], matrix_tuple[2])
140        matrix_dict = {
141            'name': matrix_str,
142            'duration': matrix.get_duration().total_seconds()
143        }
144        matrix_dict.update(_process_build(json_url, console_url))
145        matrix_list.append(matrix_dict)
146
147    return matrix_list
148
149
150def _process_build(json_url, console_url):
151    build_result = {}
152    error_list = []
153    try:
154        html = urllib.urlopen(json_url).read()
155        test_result = json.loads(html)
156        print('====> Parsing result from %s' % json_url)
157        failure_count = test_result['failCount']
158        build_result['pass_count'] = test_result['passCount']
159        build_result['failure_count'] = failure_count
160        # This means Jenkins failure occurred.
161        build_result['no_report_files_found'] = _no_report_files_found(html)
162        # Only check errors if Jenkins failure occurred.
163        if build_result['no_report_files_found']:
164            error_list = _scrape_for_known_errors(html)
165    except Exception as e:
166        print('====> Got exception for %s: %s.' % (json_url, str(e)))
167        print('====> Parsing errors from %s.' % console_url)
168        html = urllib.urlopen(console_url).read()
169        build_result['pass_count'] = 0
170        build_result['failure_count'] = 1
171        # In this case, the string doesn't exist in the result html but the fact
172        # that we fail to parse the result html indicates Jenkins failure and hence
173        # no report files were generated.
174        build_result['no_report_files_found'] = True
175        error_list = _scrape_for_known_errors(html)
176
177    if error_list:
178        build_result['error'] = error_list
179    elif build_result['no_report_files_found']:
180        build_result['error'] = [{'description': _UNKNOWN_ERROR, 'count': 1}]
181    else:
182        build_result['error'] = [{'description': '', 'count': 0}]
183
184    return build_result
185
186
187# parse command line
188argp = argparse.ArgumentParser(description='Get build statistics.')
189argp.add_argument('-u', '--username', default='jenkins')
190argp.add_argument(
191    '-b',
192    '--builds',
193    choices=['all'] + sorted(_BUILDS.keys()),
194    nargs='+',
195    default=['all'])
196args = argp.parse_args()
197
198J = Jenkins('https://grpc-testing.appspot.com', args.username, 'apiToken')
199bq = big_query_utils.create_big_query()
200
201for build_name in _BUILDS.keys() if 'all' in args.builds else args.builds:
202    print('====> Build: %s' % build_name)
203    # Since get_last_completed_build() always fails due to malformatted string
204    # error, we use get_build_metadata() instead.
205    job = None
206    try:
207        job = J[build_name]
208    except Exception as e:
209        print('====> Failed to get build %s: %s.' % (build_name, str(e)))
210        continue
211    last_processed_build_number = _get_last_processed_buildnumber(build_name)
212    last_complete_build_number = job.get_last_completed_buildnumber()
213    # To avoid processing all builds for a project never looked at. In this case,
214    # only examine 10 latest builds.
215    starting_build_number = max(last_processed_build_number + 1,
216                                last_complete_build_number - 9)
217    for build_number in xrange(starting_build_number,
218                               last_complete_build_number + 1):
219        print('====> Processing %s build %d.' % (build_name, build_number))
220        build = None
221        try:
222            build = job.get_build_metadata(build_number)
223            print('====> Build status: %s.' % build.get_status())
224            if build.get_status() == 'ABORTED':
225                continue
226            # If any build is still running, stop processing this job. Next time, we
227            # start from where it was left so that all builds are processed
228            # sequentially.
229            if build.is_running():
230                print('====> Build %d is still running.' % build_number)
231                break
232        except KeyError:
233            print('====> Build %s is missing. Skip.' % build_number)
234            continue
235        build_result = {
236            'build_number': build_number,
237            'timestamp': str(build.get_timestamp())
238        }
239        url_base = json_url = '%s/%s/%d' % (_URL_BASE, build_name, build_number)
240        if _BUILDS[build_name]:  # The build has matrix, such as gRPC_master.
241            build_result['matrix'] = _process_matrix(build, url_base)
242        else:
243            json_url = '%s/testReport/api/json' % url_base
244            console_url = '%s/consoleFull' % url_base
245            build_result['duration'] = build.get_duration().total_seconds()
246            build_stat = _process_build(json_url, console_url)
247            build_result.update(build_stat)
248        rows = [big_query_utils.make_row(build_number, build_result)]
249        if not big_query_utils.insert_rows(bq, _PROJECT_ID, _DATASET_ID,
250                                           build_name, rows):
251            print('====> Error uploading result to bigquery.')
252            sys.exit(1)
253