1# Copyright 2024 The Android Open Source Project 2# 3# Licensed under the Apache License, Version 2.0 (the "License"); 4# you may not use this file except in compliance with the License. 5# You may obtain a copy of the License at 6# 7# http://www.apache.org/licenses/LICENSE-2.0 8# 9# Unless required by applicable law or agreed to in writing, software 10# distributed under the License is distributed on an "AS IS" BASIS, 11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12# See the License for the specific language governing permissions and 13# limitations under the License. 14"""Verify that frames from UW and W cameras are not distorted.""" 15 16import collections 17import logging 18import os 19import cv2 20import math 21import numpy as np 22 23from cv2 import aruco 24from mobly import test_runner 25 26import its_base_test 27import camera_properties_utils 28import image_processing_utils 29import its_session_utils 30import opencv_processing_utils 31import preview_processing_utils 32 33_ACCURACY = 0.001 34_ARUCO_COUNT = 8 35_ARUCO_DIST_TOL = 0.15 36_ARUCO_SIZE = (3, 3) 37_ASPECT_RATIO_4_3 = 4/3 38_CH_FULL_SCALE = 255 39_CHESSBOARD_CORNERS = 24 40_CHKR_DIST_TOL = 0.05 41_CROSS_SIZE = 6 42_CROSS_THICKNESS = 1 43_FONT_SCALE = 0.3 44_FONT_THICKNESS = 1 45_GREEN_LIGHT = (80, 255, 80) 46_GREEN_DARK = (0, 190, 0) 47_MAX_ITER = 30 48_NAME = os.path.splitext(os.path.basename(__file__))[0] 49_RED = (255, 0, 0) 50_VALID_CONTROLLERS = ('arduino', 'external') 51_WIDE_ZOOM = 1 52_ZOOM_STEP = 0.5 53_ZOOM_STEP_REDUCTION = 0.1 54_ZOOM_TOL = 0.1 55 56# Note: b/284232490: 1080p could be 1088. 480p could be 704 or 640 too. 57# Use for tests not sensitive to variations of 1080p or 480p. 58# TODO: b/370841141 - Remove usage of VIDEO_PREVIEW_QUALITY_SIZE. 59# Create and use get_supported_video_sizes instead of 60# get_supported_video_qualities. 61_VIDEO_PREVIEW_QUALITY_SIZE = { 62 # 'HIGH' and 'LOW' not included as they are DUT-dependent 63 '4KDC': '4096x2160', 64 '2160P': '3840x2160', 65 'QHD': '2560x1440', 66 '2k': '2048x1080', 67 '1080P': '1920x1080', 68 '720P': '1280x720', 69 '480P': '720x480', 70 'VGA': '640x480', 71 'CIF': '352x288', 72 'QVGA': '320x240', 73 'QCIF': '176x144', 74} 75 76 77def get_largest_video_size(cam, camera_id): 78 """Returns the largest supported video size and its area. 79 80 Determine largest supported video size and its area from 81 get_supported_video_qualities. 82 83 Args: 84 cam: camera object. 85 camera_id: str; camera ID. 86 87 Returns: 88 max_size: str; largest supported video size in the format 'widthxheight'. 89 max_area: int; area of the largest supported video size. 90 """ 91 supported_video_qualities = cam.get_supported_video_qualities(camera_id) 92 logging.debug('Supported video profiles & IDs: %s', 93 supported_video_qualities) 94 95 quality_keys = [ 96 quality.split(':')[0] 97 for quality in supported_video_qualities 98 ] 99 logging.debug('Quality keys: %s', quality_keys) 100 101 supported_video_sizes = [ 102 _VIDEO_PREVIEW_QUALITY_SIZE[key] 103 for key in quality_keys 104 if key in _VIDEO_PREVIEW_QUALITY_SIZE 105 ] 106 logging.debug('Supported video sizes: %s', supported_video_sizes) 107 108 if not supported_video_sizes: 109 raise AssertionError('No supported video sizes found!') 110 111 size_to_area = lambda s: int(s.split('x')[0])*int(s.split('x')[1]) 112 max_size = max(supported_video_sizes, key=size_to_area) 113 114 logging.debug('Largest video size: %s', max_size) 115 return size_to_area(max_size) 116 117 118def get_chart_coverage(image, corners): 119 """Calculates the chart coverage in the image. 120 121 Args: 122 image: image containing chessboard 123 corners: corners of the chart 124 125 Returns: 126 chart_coverage: percentage of the image covered by chart corners 127 chart_diagonal_pixels: pixel count from the first corner to the last corner 128 """ 129 first_corner = corners[0].tolist()[0] 130 logging.debug('first_corner: %s', first_corner) 131 last_corner = corners[-1].tolist()[0] 132 logging.debug('last_corner: %s', last_corner) 133 chart_diagonal_pixels = math.dist(first_corner, last_corner) 134 logging.debug('chart_diagonal_pixels: %s', chart_diagonal_pixels) 135 136 # Calculate chart coverage relative to image diagonal 137 image_diagonal = np.sqrt(image.shape[0]**2 + image.shape[1]**2) 138 logging.debug('image.shape: %s', image.shape) 139 logging.debug('Image diagonal (pixels): %s', image_diagonal) 140 chart_coverage = chart_diagonal_pixels / image_diagonal * 100 141 logging.debug('Chart coverage: %s', chart_coverage) 142 143 return chart_coverage, chart_diagonal_pixels 144 145 146def plot_corners(image, corners, cross_color=_RED, text_color=_RED): 147 """Plot corners to the given image. 148 149 Args: 150 image: image 151 corners: points in the image 152 cross_color: color of cross 153 text_color: color of text 154 155 Returns: 156 image: image with cross and text for each corner 157 """ 158 for i, corner in enumerate(corners): 159 x, y = int(corner.ravel()[0]), int(corner.ravel()[1]) 160 161 # Draw corner index 162 cv2.putText(image, str(i), (x, y - 10), cv2.FONT_HERSHEY_SIMPLEX, 163 _FONT_SCALE, text_color, _FONT_THICKNESS, cv2.LINE_AA) 164 165 for corner in corners: 166 x, y = corner.ravel() 167 168 # Ensure coordinates are integers and within image boundaries 169 x = max(0, min(int(x), image.shape[1] - 1)) 170 y = max(0, min(int(y), image.shape[0] - 1)) 171 172 # Draw horizontal line 173 cv2.line(image, (x - _CROSS_SIZE, y), (x + _CROSS_SIZE, y), cross_color, 174 _CROSS_THICKNESS) 175 # Draw vertical line 176 cv2.line(image, (x, y - _CROSS_SIZE), (x, y + _CROSS_SIZE), cross_color, 177 _CROSS_THICKNESS) 178 179 return image 180 181 182def get_ideal_points(pattern_size): 183 """Calculate the ideal points for pattern. 184 185 These are just corners at unit intervals of the same dimensions 186 as pattern_size. Looks like.. 187 [[ 0. 0. 0.] 188 [ 1. 0. 0.] 189 [ 2. 0. 0.] 190 ... 191 [21. 23. 0.] 192 [22. 23. 0.] 193 [23. 23. 0.]] 194 195 Args: 196 pattern_size: pattern size. Example (24, 24) 197 198 Returns: 199 ideal_points: corners at unit interval. 200 """ 201 ideal_points = np.zeros((pattern_size[0] * pattern_size[1], 3), np.float32) 202 ideal_points[:,:2] = ( 203 np.mgrid[0:pattern_size[0], 0:pattern_size[1]].T.reshape(-1, 2) 204 ) 205 206 return ideal_points 207 208 209def get_distortion_error(image, corners, ideal_points, rotation_vector, 210 translation_vector, camera_matrix): 211 """Get distortion error by comparing corners and ideal points. 212 213 compare corners and ideal points to derive the distortion error 214 215 Args: 216 image: image containing chessboard and ArUco 217 corners: corners of the chart. Shape = (number of corners, 1, 2) 218 ideal_points: corners at unit interval. Shape = (number of corners, 3) 219 rotation_vector: rotation vector based on chart's rotation. Shape = (3, 1) 220 translation_vector: translation vector based on chart's rotation. 221 Shape = (3, 1) 222 camera_matrix: camera intrinsic matrix. Shape = (3, 3) 223 224 Returns: 225 normalized_distortion_error_percentage: normalized distortion error 226 percentage. None if all corners based on pattern_size not found. 227 chart_coverage: percentage of the image covered by corners 228 """ 229 chart_coverage, chart_diagonal_pixels = get_chart_coverage(image, corners) 230 logging.debug('Chart coverage: %s', chart_coverage) 231 232 projected_points = cv2.projectPoints(ideal_points, rotation_vector, 233 translation_vector, camera_matrix, None) 234 # Reshape projected points to 2D array 235 projected = projected_points[0].reshape(-1, 2) 236 corners_reshaped = corners.reshape(-1, 2) 237 logging.debug('projected: %s', projected) 238 239 plot_corners(image, projected, _GREEN_LIGHT, _GREEN_DARK) 240 241 # Calculate the distortion error 242 distortion_errors = [ 243 math.dist(projected_point, corner_point) 244 for projected_point, corner_point in zip(projected, corners_reshaped) 245 ] 246 logging.debug('distortion_error: %s', distortion_errors) 247 248 # Get RMS of error 249 rms_error = math.sqrt(np.mean(np.square(distortion_errors))) 250 logging.debug('RMS distortion error: %s', rms_error) 251 252 # Calculate as a percentage of the chart diagonal 253 normalized_distortion_error_percentage = ( 254 rms_error / chart_diagonal_pixels * 100 255 ) 256 logging.debug('Normalized percent distortion error: %s', 257 normalized_distortion_error_percentage) 258 259 return normalized_distortion_error_percentage, chart_coverage 260 261 262def get_chessboard_corners(pattern_size, image): 263 """Find chessboard corners from image. 264 265 Args: 266 pattern_size: (int, int) chessboard corners. 267 image: image containing chessboard 268 269 Returns: 270 corners: corners of the chessboard chart 271 ideal_points: ideal pattern of chessboard corners 272 i.e. points at unit intervals 273 """ 274 # Convert the image to grayscale 275 gray_image = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY) 276 277 # Find the checkerboard corners 278 found_corners, corners_pass1 = cv2.findChessboardCorners(gray_image, 279 pattern_size) 280 logging.debug('Found corners: %s', found_corners) 281 logging.debug('corners_pass1: %s', corners_pass1) 282 283 if not found_corners: 284 logging.debug('Chessboard pattern not found.') 285 return None, None 286 287 # Refine corners 288 criteria = (cv2.TERM_CRITERIA_EPS + cv2.TERM_CRITERIA_MAX_ITER, _MAX_ITER, 289 _ACCURACY) 290 corners = cv2.cornerSubPix(gray_image, corners_pass1, (11, 11), (-1, -1), 291 criteria) 292 logging.debug('Refined Corners: %s', corners) 293 294 plot_corners(image, corners) 295 296 ideal_points = get_ideal_points(pattern_size) 297 logging.debug('ideal_points: %s', ideal_points) 298 299 return corners, ideal_points 300 301 302def get_aruco_corners(image): 303 """Find ArUco corners from image. 304 305 Args: 306 image: image containing ArUco markers 307 308 Returns: 309 corners: First corner of each ArUco markers in the image. 310 None if expected ArUco corners are not found. 311 ideal_points: ideal pattern of the ArUco marker corners. 312 None if expected ArUco corners are not found. 313 """ 314 # Detect ArUco markers 315 corners, ids, _ = opencv_processing_utils.version_agnostic_detect_markers( 316 image 317 ) 318 319 logging.debug('corners: %s', corners) 320 logging.debug('ids: %s', ids) 321 322 if ids is None: 323 logging.debug('ArUco markers are not found') 324 return None, None 325 326 aruco.drawDetectedMarkers(image, corners, ids, _RED) 327 328 # Convert to numpy array 329 corners = np.concatenate(corners, axis=0).reshape(-1, 4, 2) 330 331 # Extract first corners efficiently 332 corners = corners[:, 0, :] 333 logging.debug('corners: %s', corners) 334 335 # Create marker_dict using efficient vectorization 336 marker_dict = dict(zip(ids.flatten(), corners)) 337 338 if len(marker_dict) != _ARUCO_COUNT: 339 logging.debug('%s arUCO markers found instead of %s', 340 len(ids), _ARUCO_COUNT) 341 return None, None 342 343 # Arrange corners based on ids 344 arranged_corners = np.array([marker_dict[i] for i in range(len(corners))]) 345 346 # Add a dimension to match format for cv2.calibrateCamera 347 corners = np.expand_dims(arranged_corners, axis=1) 348 logging.debug('updated corners: %s', corners) 349 350 plot_corners(image, corners) 351 352 ideal_points = get_ideal_points(_ARUCO_SIZE) 353 354 # No ArUco marker in the center, so remove the middle point 355 middle_index = (_ARUCO_SIZE[0] // 2) * _ARUCO_SIZE[1] + (_ARUCO_SIZE[1] // 2) 356 ideal_points = np.delete(ideal_points, middle_index, axis=0) 357 logging.debug('ideal_points: %s', ideal_points) 358 359 return corners, ideal_points 360 361 362def get_preview_frame(dut, cam, preview_size, zoom, z_range, log_path): 363 """Captures preview frame at given zoom ratio. 364 365 Args: 366 dut: device under test 367 cam: camera object 368 preview_size: str; preview resolution. ex. '1920x1080' 369 zoom: zoom ratio 370 z_range: zoom range 371 log_path: str; path for video file directory 372 373 Returns: 374 img_name: the filename of the first captured image 375 capture_result: total capture results of the preview frame 376 """ 377 logging.debug('zoom: %s', zoom) 378 if not (z_range[0] <= zoom <= z_range[1]): 379 raise ValueError(f'Zoom {zoom} is outside the allowed range {z_range}') 380 381 z_min = zoom 382 z_max = z_min + _ZOOM_STEP - _ZOOM_STEP_REDUCTION 383 if(z_max > z_range[1]): 384 z_max = z_range[1] 385 386 # Capture preview images over zoom range 387 # TODO: b/343200676 - use do_preview_recording instead of 388 # preview_over_zoom_range 389 capture_results, file_list = preview_processing_utils.preview_over_zoom_range( 390 dut, cam, preview_size, z_min, z_max, _ZOOM_STEP, log_path 391 ) 392 393 # Get first captured image 394 img_name = file_list[0] 395 capture_result = capture_results[0] 396 397 return img_name, capture_result 398 399 400def add_update_to_filename(file_name, update_str='_update'): 401 """Adds the provided update string to the base name of a file. 402 403 Args: 404 file_name (str): The full path to the file to be modified. 405 update_str (str, optional): The string to insert before the extension 406 407 Returns: 408 file_name: The full path to the new file with the update string added. 409 """ 410 411 directory, file_with_ext = os.path.split(file_name) 412 base_name, ext = os.path.splitext(file_with_ext) 413 414 new_file_name = os.path.join(directory, f'{base_name}_{update_str}{ext}') 415 416 return new_file_name 417 418 419def get_distortion_errors(props, img_name): 420 """Calculates the distortion error using checkerboard and ArUco markers. 421 422 Args: 423 props: camera properties object. 424 img_name: image name including complete file path 425 426 Returns: 427 chkr_chart_coverage: normalized distortion error percentage for chessboard 428 corners. None if all corners based on pattern_size not found. 429 chkr_chart_coverage: percentage of the image covered by chessboard chart 430 arc_distortion_error: normalized distortion error percentage for ArUco 431 corners. None if all corners based on pattern_size not found. 432 arc_chart_coverage: percentage of the image covered by ArUco corners 433 434 """ 435 image = cv2.imread(img_name) 436 if (props['android.lens.facing'] == 437 camera_properties_utils.LENS_FACING['FRONT']): 438 image = image_processing_utils.mirror_preview_image_by_sensor_orientation( 439 props['android.sensor.orientation'], image) 440 441 pattern_size = (_CHESSBOARD_CORNERS, _CHESSBOARD_CORNERS) 442 443 chess_corners, chess_ideal_points = get_chessboard_corners(pattern_size, 444 image) 445 aruco_corners, aruco_ideal_points = get_aruco_corners(image) 446 447 if chess_corners is None: 448 return None, None, None, None 449 450 ideal_points = [chess_ideal_points] 451 image_corners = [chess_corners] 452 453 if aruco_corners is not None: 454 ideal_points.append(aruco_ideal_points) 455 image_corners.append(aruco_corners) 456 457 # Calculate the distortion error 458 # Do this by: 459 # 1) Calibrate the camera from the detected checkerboard points 460 # 2) Project the ideal points, using the camera calibration data. 461 # 3) Except, do not use distortion coefficients so we model ideal pinhole 462 # 4) Calculate the error of the detected corners relative to the ideal 463 # 5) Normalize the average error by the size of the chart 464 calib_flags = ( 465 cv2.CALIB_FIX_K1 466 + cv2.CALIB_FIX_K2 467 + cv2.CALIB_FIX_K3 468 + cv2.CALIB_FIX_K4 469 + cv2.CALIB_FIX_K5 470 + cv2.CALIB_FIX_K6 471 + cv2.CALIB_ZERO_TANGENT_DIST 472 ) 473 ret, camera_matrix, dist_coeffs, rotation_vectors, translation_vectors = ( 474 cv2.calibrateCamera(ideal_points, image_corners, image.shape[:2], 475 None, None, flags=calib_flags) 476 ) 477 logging.debug('Projection error: %s dist_coeffs: %s', ret, dist_coeffs) 478 logging.debug('rotation_vector: %s', rotation_vectors) 479 logging.debug('translation_vector: %s', translation_vectors) 480 logging.debug('matrix: %s', camera_matrix) 481 482 chkr_distortion_error, chkr_chart_coverage = ( 483 get_distortion_error(image, chess_corners, chess_ideal_points, 484 rotation_vectors[0], translation_vectors[0], 485 camera_matrix) 486 ) 487 488 if aruco_corners is not None: 489 arc_distortion_error, arc_chart_coverage = get_distortion_error( 490 image, aruco_corners, aruco_ideal_points, rotation_vectors[1], 491 translation_vectors[1], camera_matrix 492 ) 493 else: 494 arc_distortion_error, arc_chart_coverage = None, None 495 496 img_name_update = add_update_to_filename(img_name) 497 image_processing_utils.write_image(image / _CH_FULL_SCALE, img_name_update) 498 499 return (chkr_distortion_error, chkr_chart_coverage, 500 arc_distortion_error, arc_chart_coverage) 501 502 503class PreviewDistortionTest(its_base_test.ItsBaseTest): 504 """Test that frames from UW and W cameras are not distorted. 505 506 Captures preview frames at different zoom levels. If whole chart is visible 507 in the frame, detect the distortion error. Pass the test if distortion error 508 is within the pre-determined TOL. 509 """ 510 511 def test_preview_distortion(self): 512 rot_rig = {} 513 log_path = self.log_path 514 515 with its_session_utils.ItsSession( 516 device_id=self.dut.serial, 517 camera_id=self.camera_id, 518 hidden_physical_id=self.hidden_physical_id) as cam: 519 520 props = cam.get_camera_properties() 521 props = cam.override_with_hidden_physical_camera_props(props) 522 camera_properties_utils.skip_unless( 523 camera_properties_utils.zoom_ratio_range(props)) 524 525 # Raise error if not FRONT or REAR facing camera 526 camera_properties_utils.check_front_or_rear_camera(props) 527 528 # Initialize rotation rig 529 rot_rig['cntl'] = self.rotator_cntl 530 rot_rig['ch'] = self.rotator_ch 531 if rot_rig['cntl'].lower() not in _VALID_CONTROLLERS: 532 raise AssertionError( 533 f'You must use the {_VALID_CONTROLLERS} controller for {_NAME}.') 534 535 largest_area = get_largest_video_size(cam, self.camera_id) 536 537 # Determine preview size 538 try: 539 preview_size = preview_processing_utils.get_max_preview_test_size( 540 cam, self.camera_id, 541 aspect_ratio=_ASPECT_RATIO_4_3, 542 max_tested_area=largest_area) 543 logging.debug('preview_size: %s', preview_size) 544 except Exception as e: 545 logging.error('Unable to find supported 4/3 preview size.' 546 'Exception: %s', e) 547 raise AssertionError(f'{its_session_utils.NOT_YET_MANDATED_MESSAGE}' 548 '\n\nUnable to find supported 4/3 preview ' 549 'size') from e 550 551 # Determine test zoom range 552 z_range = props['android.control.zoomRatioRange'] 553 logging.debug('z_range: %s', z_range) 554 555 # Collect preview frames and associated capture results 556 PreviewFrameData = collections.namedtuple( 557 'PreviewFrameData', ['img_name', 'capture_result', 'z_level'] 558 ) 559 preview_frames = [] 560 z_levels = [z_range[0]] # Min zoom 561 if (z_range[0] < _WIDE_ZOOM <= z_range[1]): 562 z_levels.append(_WIDE_ZOOM) 563 564 for z in z_levels: 565 try: 566 img_name, capture_result = get_preview_frame( 567 self.dut, cam, preview_size, z, z_range, log_path 568 ) 569 except Exception as e: 570 logging.error('Failed to capture preview frames' 571 'Exception: %s', e) 572 raise AssertionError(f'{its_session_utils.NOT_YET_MANDATED_MESSAGE}' 573 '\n\nFailed to capture preview frames') from e 574 if img_name: 575 frame_data = PreviewFrameData(img_name, capture_result, z) 576 preview_frames.append(frame_data) 577 578 failure_msg = [] 579 # Determine distortion error and chart coverage for each frames 580 for frame in preview_frames: 581 img_full_name = f'{os.path.join(log_path, frame.img_name)}' 582 (chkr_distortion_err, chkr_chart_coverage, arc_distortion_err, 583 arc_chart_coverage) = get_distortion_errors(props, img_full_name) 584 585 zoom = float(frame.capture_result['android.control.zoomRatio']) 586 if camera_properties_utils.logical_multi_camera(props): 587 cam_id = frame.capture_result[ 588 'android.logicalMultiCamera.activePhysicalId' 589 ] 590 else: 591 cam_id = None 592 logging.debug('Zoom: %.2f, cam_id: %s, img_name: %s', 593 zoom, cam_id, img_name) 594 595 if math.isclose(zoom, z_levels[0], rel_tol=_ZOOM_TOL): 596 z_str = 'min' 597 else: 598 z_str = 'max' 599 600 # Don't change print to logging. Used for KPI. 601 print(f'{_NAME}_{z_str}_zoom: ', zoom) 602 print(f'{_NAME}_{z_str}_physical_id: ', cam_id) 603 print(f'{_NAME}_{z_str}_chkr_distortion_error: ', chkr_distortion_err) 604 print(f'{_NAME}_{z_str}_chkr_chart_coverage: ', chkr_chart_coverage) 605 print(f'{_NAME}_{z_str}_aruco_distortion_error: ', arc_distortion_err) 606 print(f'{_NAME}_{z_str}_aruco_chart_coverage: ', arc_chart_coverage) 607 logging.debug('%s_%s_zoom: %s', _NAME, z_str, zoom) 608 logging.debug('%s_%s_physical_id: %s', _NAME, z_str, cam_id) 609 logging.debug('%s_%s_chkr_distortion_error: %s', _NAME, z_str, 610 chkr_distortion_err) 611 logging.debug('%s_%s_chkr_chart_coverage: %s', _NAME, z_str, 612 chkr_chart_coverage) 613 logging.debug('%s_%s_aruco_distortion_error: %s', _NAME, z_str, 614 arc_distortion_err) 615 logging.debug('%s_%s_aruco_chart_coverage: %s', _NAME, z_str, 616 arc_chart_coverage) 617 618 if arc_distortion_err is None: 619 if zoom < _WIDE_ZOOM: 620 failure_msg.append('Unable to find all ArUco markers in ' 621 f'{img_name}') 622 logging.debug(failure_msg[-1]) 623 else: 624 if arc_distortion_err > _ARUCO_DIST_TOL: 625 failure_msg.append('ArUco Distortion error ' 626 f'{arc_distortion_err:.3f} is greater than ' 627 f'tolerance {_ARUCO_DIST_TOL}') 628 logging.debug(failure_msg[-1]) 629 630 if chkr_distortion_err is None: 631 # Checkerboard corners shall be detected at minimum zoom level 632 failure_msg.append(f'Unable to find full checker board in {img_name}') 633 logging.debug(failure_msg[-1]) 634 else: 635 if chkr_distortion_err > _CHKR_DIST_TOL: 636 failure_msg.append('Chess Distortion error ' 637 f'{chkr_distortion_err:.3f} is greater than ' 638 f'tolerance {_CHKR_DIST_TOL}') 639 logging.debug(failure_msg[-1]) 640 641 if failure_msg: 642 raise AssertionError(f'{its_session_utils.NOT_YET_MANDATED_MESSAGE}' 643 f'\n\n{failure_msg}') 644 645if __name__ == '__main__': 646 test_runner.main() 647 648