1# Copyright 2023 The Android Open Source Project 2# 3# Licensed under the Apache License, Version 2.0 (the "License"); 4# you may not use this file except in compliance with the License. 5# You may obtain a copy of the License at 6# 7# http://www.apache.org/licenses/LICENSE-2.0 8# 9# Unless required by applicable law or agreed to in writing, software 10# distributed under the License is distributed on an "AS IS" BASIS, 11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12# See the License for the specific language governing permissions and 13# limitations under the License. 14"""Verify HDR is activated correctly for extension captures.""" 15 16 17import logging 18import os.path 19import time 20 21import cv2 22from mobly import test_runner 23import numpy as np 24from scipy import ndimage 25 26import its_base_test 27import camera_properties_utils 28import capture_request_utils 29import image_processing_utils 30import its_session_utils 31import lighting_control_utils 32import opencv_processing_utils 33 34_NAME = os.path.splitext(os.path.basename(__file__))[0] 35_EXTENSION_HDR = 3 36_TABLET_BRIGHTNESS = '12' # Highest minimum brightness on a supported tablet 37 38_FMT_NAME = 'jpg' 39_WIDTH = 1920 40_HEIGHT = 1080 41 42_MIN_QRCODE_AREA = 0.01 # Reject squares smaller than 1% of image 43_QR_CODE_VALUE = 'CameraITS' 44_CONTRAST_ARANGE = (1, 10, 0.01) 45_CONTOUR_INDEX = -1 # Draw all contours as per opencv convention 46_BGR_RED = (0, 0, 255) 47_CONTOUR_LINE_THICKNESS = 3 48 49_DURATION_DIFF_TOL = 0.5 # HDR ON captures must take 0.5 seconds longer 50_GRADIENT_TOL = 0.15 # Largest HDR gradient must be at most 15% of non-HDR 51 52 53def extract_tile(img, file_stem_with_suffix): 54 """Extracts a white square from an image and processes it for analysis. 55 56 Args: 57 img: An RGB image 58 file_stem_with_suffix: Filename describing image format and HDR activation. 59 Returns: 60 openCV image representing the QR code 61 """ 62 img *= 255 # openCV needs [0:255] images 63 square = opencv_processing_utils.find_white_square( 64 img, _MIN_QRCODE_AREA) 65 tile = image_processing_utils.get_image_patch( 66 img, 67 square['left']/img.shape[1], 68 square['top']/img.shape[0], 69 square['w']/img.shape[1], 70 square['h']/img.shape[0] 71 ) 72 tile = tile.astype(np.uint8) 73 tile = tile[:, :, ::-1] # RGB --> BGR for cv2 74 tile = cv2.cvtColor(tile, cv2.COLOR_BGR2GRAY) # Convert to grayscale 75 76 # Rotate tile to reduce scene variation 77 h, w = tile.shape[:2] 78 center_x, center_y = w // 2, h // 2 79 rotation_matrix = cv2.getRotationMatrix2D((center_x, center_y), 80 square['angle'], 1.0) 81 tile = cv2.warpAffine(tile, rotation_matrix, (w, h)) 82 cv2.imwrite(f'{file_stem_with_suffix}_tile.png', tile) 83 return tile 84 85 86def analyze_qr_code(img, file_stem_with_suffix): 87 """Analyze gradient across ROI and detect/decode its QR code from an image. 88 89 Attempts to detect and decode a QR code from the image represented by img, 90 after converting to grayscale and rotating the code to be in line with 91 the x and y axes. Then, if even detection fails, modifies the contrast of 92 the image until the QR code is detectable. Measures the gradient across 93 the code by finding the length of the largest contour found by openCV. 94 95 Args: 96 img: An RGB image 97 file_stem_with_suffix: Filename describing image format and HDR activation. 98 99 Returns: 100 detection_object: Union[str, bool], describes decoded data or detection 101 lowest_successful_alpha: float, contrast where QR code was detected/decoded 102 contour_length: int, length of largest contour in gradient image 103 """ 104 tile = extract_tile(img, file_stem_with_suffix) 105 106 # Find gradient 107 sobel_x = ndimage.sobel(tile, axis=0, mode='constant') 108 sobel_y = ndimage.sobel(tile, axis=1, mode='constant') 109 sobel = np.float32(np.hypot(sobel_x, sobel_y)) 110 111 # Find largest contour in gradient image 112 contour = max( 113 opencv_processing_utils.find_all_contours(np.uint8(sobel)), key=len) 114 contour_length = len(contour) 115 116 # Draw contour (need a color image for visibility) 117 sobel_bgr = cv2.cvtColor(sobel, cv2.COLOR_GRAY2BGR) 118 contour_image = cv2.drawContours(sobel_bgr, contour, _CONTOUR_INDEX, 119 _BGR_RED, _CONTOUR_LINE_THICKNESS) 120 cv2.imwrite(f'{file_stem_with_suffix}_sobel_contour.png', contour_image) 121 122 # Try to detect QR code 123 detection_object = None 124 lowest_successful_alpha = None 125 qr_detector = cv2.QRCodeDetector() 126 127 # See if original tile is detectable 128 qr_code, _, _ = qr_detector.detectAndDecode(tile) 129 if qr_code and qr_code == _QR_CODE_VALUE: 130 logging.debug('Decoded correct QR code: %s without contrast changes', 131 _QR_CODE_VALUE) 132 return qr_code, 0.0, contour_length 133 else: 134 qr_code, _ = qr_detector.detect(tile) 135 if qr_code: 136 detection_object = qr_code 137 lowest_successful_alpha = 0.0 138 logging.debug('Detected QR code without contrast changes') 139 140 # Modify contrast (not brightness) to see if QR code detectable/decodable 141 for a in np.arange(*_CONTRAST_ARANGE): 142 qr_tile = cv2.convertScaleAbs(tile, alpha=a, beta=0) 143 qr_code, _, _ = qr_detector.detectAndDecode(qr_tile) 144 if qr_code and qr_code == _QR_CODE_VALUE: 145 logging.debug('Decoded correct QR code: %s at alpha of %.2f', 146 _QR_CODE_VALUE, a) 147 return qr_code, a, contour_length 148 elif qr_code: 149 logging.debug('Decoded other QR code: %s', qr_code) 150 else: 151 # If QR code already detected, only try to decode. 152 if detection_object: 153 continue 154 qr_code, _ = qr_detector.detect(qr_tile) 155 if qr_code: 156 logging.debug('Detected QR code at alpha of %.2f', a) 157 detection_object = qr_code 158 lowest_successful_alpha = a 159 160 return detection_object, lowest_successful_alpha, contour_length 161 162 163class HdrExtensionTest(its_base_test.ItsBaseTest): 164 """Tests HDR extension under dark lighting conditions. 165 166 Takes capture with and without HDR extension activated. 167 Verifies that QR code on the right is lit evenly, 168 or can be decoded/detected with the HDR extension on. 169 """ 170 171 def test_hdr(self): 172 # Handle subdirectory 173 self.scene = 'scene_hdr' 174 with its_session_utils.ItsSession( 175 device_id=self.dut.serial, 176 camera_id=self.camera_id, 177 hidden_physical_id=self.hidden_physical_id) as cam: 178 props = cam.get_camera_properties() 179 props = cam.override_with_hidden_physical_camera_props(props) 180 test_name = os.path.join(self.log_path, _NAME) 181 182 # Determine camera supported extensions 183 supported_extensions = cam.get_supported_extensions(self.camera_id) 184 logging.debug('Supported extensions: %s', supported_extensions) 185 186 # Check SKIP conditions 187 vendor_api_level = its_session_utils.get_vendor_api_level(self.dut.serial) 188 camera_properties_utils.skip_unless( 189 _EXTENSION_HDR in supported_extensions and 190 vendor_api_level >= its_session_utils.ANDROID14_API_LEVEL) 191 192 # Establish connection with lighting controller 193 arduino_serial_port = lighting_control_utils.lighting_control( 194 self.lighting_cntl, self.lighting_ch) 195 196 # Turn OFF lights to darken scene 197 lighting_control_utils.set_lighting_state( 198 arduino_serial_port, self.lighting_ch, 'OFF') 199 200 # Check that tablet is connected and turn it off to validate lighting 201 if self.tablet: 202 lighting_control_utils.turn_off_device(self.tablet) 203 else: 204 raise AssertionError('Test must be run with tablet.') 205 206 # Validate lighting 207 cam.do_3a(do_af=False) 208 cap = cam.do_capture( 209 capture_request_utils.auto_capture_request(), cam.CAP_YUV) 210 y_plane, _, _ = image_processing_utils.convert_capture_to_planes(cap) 211 its_session_utils.validate_lighting( 212 y_plane, self.scene, state='OFF', log_path=self.log_path) 213 214 self.setup_tablet() 215 self.set_screen_brightness(_TABLET_BRIGHTNESS) 216 217 its_session_utils.load_scene( 218 cam, props, self.scene, self.tablet, self.chart_distance, 219 lighting_check=False, log_path=self.log_path) 220 221 file_stem = f'{test_name}_{_FMT_NAME}_{_WIDTH}x{_HEIGHT}' 222 223 # Take capture without HDR extension activated as baseline 224 logging.debug('Taking capture without HDR extension') 225 out_surfaces = {'format': _FMT_NAME, 'width': _WIDTH, 'height': _HEIGHT} 226 cam.do_3a() 227 req = capture_request_utils.auto_capture_request() 228 no_hdr_start_of_capture = time.time() 229 no_hdr_cap = cam.do_capture(req, out_surfaces) 230 no_hdr_end_of_capture = time.time() 231 no_hdr_capture_duration = no_hdr_end_of_capture - no_hdr_start_of_capture 232 logging.debug('no HDR cap duration: %.2f', no_hdr_capture_duration) 233 logging.debug('no HDR cap metadata: %s', no_hdr_cap['metadata']) 234 no_hdr_img = image_processing_utils.convert_capture_to_rgb_image( 235 no_hdr_cap) 236 image_processing_utils.write_image( 237 no_hdr_img, f'{file_stem}_no_HDR.jpg') 238 239 # Take capture with HDR extension 240 logging.debug('Taking capture with HDR extension') 241 out_surfaces = {'format': _FMT_NAME, 'width': _WIDTH, 'height': _HEIGHT} 242 cam.do_3a() 243 req = capture_request_utils.auto_capture_request() 244 hdr_start_of_capture = time.time() 245 hdr_cap = cam.do_capture_with_extensions( 246 req, _EXTENSION_HDR, out_surfaces) 247 hdr_end_of_capture = time.time() 248 hdr_capture_duration = hdr_end_of_capture - hdr_start_of_capture 249 logging.debug('HDR cap duration: %.2f', hdr_capture_duration) 250 logging.debug('HDR cap metadata: %s', hdr_cap['metadata']) 251 hdr_img = image_processing_utils.convert_capture_to_rgb_image( 252 hdr_cap) 253 image_processing_utils.write_image(hdr_img, f'{file_stem}_HDR.jpg') 254 255 # Attempt to decode QR code with and without HDR 256 format_optional_float = lambda x: f'{x:.2f}' if x is not None else 'None' 257 logging.debug('Attempting to detect and decode QR code without HDR') 258 no_hdr_detection_object, no_hdr_alpha, no_hdr_length = analyze_qr_code( 259 no_hdr_img, f'{file_stem}_no_HDR') 260 logging.debug('No HDR code: %s, No HDR alpha: %s, ' 261 'No HDR contour length: %d', 262 no_hdr_detection_object, 263 format_optional_float(no_hdr_alpha), 264 no_hdr_length) 265 logging.debug('Attempting to detect and decode QR code with HDR') 266 hdr_detection_object, hdr_alpha, hdr_length = analyze_qr_code( 267 hdr_img, f'{file_stem}_HDR') 268 logging.debug('HDR code: %s, HDR alpha: %s, HDR contour length: %d', 269 hdr_detection_object, 270 format_optional_float(hdr_alpha), 271 hdr_length) 272 273 # Assert correct behavior 274 failure_messages = [] 275 # Decoding QR code with HDR -> PASS 276 if hdr_detection_object != _QR_CODE_VALUE: 277 if hdr_alpha is None: # Allow hdr_alpha to be falsy (0.0) 278 failure_messages.append( 279 'Unable to detect QR code with HDR extension') 280 if (no_hdr_alpha is not None and 281 hdr_alpha is not None and 282 no_hdr_alpha < hdr_alpha): 283 failure_messages.append('QR code was found at a lower contrast with ' 284 f'HDR off ({no_hdr_alpha}) than with HDR on ' 285 f'({hdr_alpha})') 286 if no_hdr_length > 0 and hdr_length / no_hdr_length > _GRADIENT_TOL: 287 failure_messages.append( 288 ('HDR gradient was not significantly ' 289 'smaller than gradient without HDR. ' 290 'Largest HDR gradient contour perimeter was ' 291 f'{hdr_length / no_hdr_length} of ' 292 'the size of largest non-HDR contour length, ' 293 f'expected to be at least {_GRADIENT_TOL}') 294 ) 295 else: 296 # If HDR gradient is better, allow PASS to account for cv2 flakiness 297 if failure_messages: 298 logging.error('\n'.join(failure_messages)) 299 failure_messages = [] 300 301 # Compare capture durations 302 duration_diff = hdr_capture_duration - no_hdr_capture_duration 303 if duration_diff < _DURATION_DIFF_TOL: 304 failure_messages.append('Capture with HDR did not take ' 305 'significantly more time than ' 306 'capture without HDR! ' 307 f'Difference: {duration_diff:.2f}, ' 308 f'Expected: {_DURATION_DIFF_TOL}') 309 310 if failure_messages: 311 raise AssertionError('\n'.join(failure_messages)) 312 313 314if __name__ == '__main__': 315 test_runner.main() 316