1# Copyright 2013 The Android Open Source Project 2# 3# Licensed under the Apache License, Version 2.0 (the "License"); 4# you may not use this file except in compliance with the License. 5# You may obtain a copy of the License at 6# 7# http://www.apache.org/licenses/LICENSE-2.0 8# 9# Unless required by applicable law or agreed to in writing, software 10# distributed under the License is distributed on an "AS IS" BASIS, 11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12# See the License for the specific language governing permissions and 13# limitations under the License. 14 15import its.error 16import os 17import os.path 18import sys 19import re 20import json 21import time 22import unittest 23import socket 24import subprocess 25import hashlib 26import numpy 27 28class ItsSession(object): 29 """Controls a device over adb to run ITS scripts. 30 31 The script importing this module (on the host machine) prepares JSON 32 objects encoding CaptureRequests, specifying sets of parameters to use 33 when capturing an image using the Camera2 APIs. This class encapsualtes 34 sending the requests to the device, monitoring the device's progress, and 35 copying the resultant captures back to the host machine when done. TCP 36 forwarded over adb is the transport mechanism used. 37 38 The device must have ItsService.apk installed. 39 40 Attributes: 41 sock: The open socket. 42 """ 43 44 # Open a connection to localhost:6000, forwarded to port 6000 on the device. 45 # TODO: Support multiple devices running over different TCP ports. 46 IPADDR = '127.0.0.1' 47 PORT = 6000 48 BUFFER_SIZE = 4096 49 50 # Seconds timeout on each socket operation. 51 SOCK_TIMEOUT = 10.0 52 53 PACKAGE = 'com.android.camera2.its' 54 INTENT_START = 'com.android.camera2.its.START' 55 56 # Definitions for some of the common output format options for do_capture(). 57 # Each gets images of full resolution for each requested format. 58 CAP_RAW = {"format":"raw"} 59 CAP_DNG = {"format":"dng"} 60 CAP_YUV = {"format":"yuv"} 61 CAP_JPEG = {"format":"jpeg"} 62 CAP_RAW_YUV = [{"format":"raw"}, {"format":"yuv"}] 63 CAP_DNG_YUV = [{"format":"dng"}, {"format":"yuv"}] 64 CAP_RAW_JPEG = [{"format":"raw"}, {"format":"jpeg"}] 65 CAP_DNG_JPEG = [{"format":"dng"}, {"format":"jpeg"}] 66 CAP_YUV_JPEG = [{"format":"yuv"}, {"format":"jpeg"}] 67 CAP_RAW_YUV_JPEG = [{"format":"raw"}, {"format":"yuv"}, {"format":"jpeg"}] 68 CAP_DNG_YUV_JPEG = [{"format":"dng"}, {"format":"yuv"}, {"format":"jpeg"}] 69 70 # Method to handle the case where the service isn't already running. 71 # This occurs when a test is invoked directly from the command line, rather 72 # than as a part of a separate test harness which is setting up the device 73 # and the TCP forwarding. 74 def __pre_init(self): 75 # TODO: Handle multiple connected devices. 76 adb = "adb -d" 77 78 # This also includes the optional reboot handling: if the user 79 # provides a "reboot" or "reboot=N" arg, then reboot the device, 80 # waiting for N seconds (default 30) before returning. 81 for s in sys.argv[1:]: 82 if s[:6] == "reboot": 83 duration = 30 84 if len(s) > 7 and s[6] == "=": 85 duration = int(s[7:]) 86 print "Rebooting device" 87 _run("%s reboot" % (adb)); 88 _run("%s wait-for-device" % (adb)) 89 time.sleep(duration) 90 print "Reboot complete" 91 92 # TODO: Figure out why "--user 0" is needed, and fix the problem. 93 _run('%s shell am force-stop --user 0 %s' % (adb, self.PACKAGE)) 94 _run(('%s shell am startservice --user 0 -t text/plain ' 95 '-a %s') % (adb, self.INTENT_START)) 96 97 # Wait until the socket is ready to accept a connection. 98 proc = subprocess.Popen( 99 adb.split() + ["logcat"], 100 stdout=subprocess.PIPE) 101 logcat = proc.stdout 102 while True: 103 line = logcat.readline().strip() 104 if line.find('ItsService ready') >= 0: 105 break 106 proc.kill() 107 108 # Setup the TCP-over-ADB forwarding. 109 _run('%s forward tcp:%d tcp:%d' % (adb,self.PORT,self.PORT)) 110 111 def __init__(self): 112 if "noinit" not in sys.argv: 113 self.__pre_init() 114 self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) 115 self.sock.connect((self.IPADDR, self.PORT)) 116 self.sock.settimeout(self.SOCK_TIMEOUT) 117 self.__close_camera() 118 self.__open_camera() 119 120 def __del__(self): 121 if hasattr(self, 'sock') and self.sock: 122 self.__close_camera() 123 self.sock.close() 124 125 def __enter__(self): 126 return self 127 128 def __exit__(self, type, value, traceback): 129 return False 130 131 def __read_response_from_socket(self): 132 # Read a line (newline-terminated) string serialization of JSON object. 133 chars = [] 134 while len(chars) == 0 or chars[-1] != '\n': 135 ch = self.sock.recv(1) 136 if len(ch) == 0: 137 # Socket was probably closed; otherwise don't get empty strings 138 raise its.error.Error('Problem with socket on device side') 139 chars.append(ch) 140 line = ''.join(chars) 141 jobj = json.loads(line) 142 # Optionally read a binary buffer of a fixed size. 143 buf = None 144 if jobj.has_key("bufValueSize"): 145 n = jobj["bufValueSize"] 146 buf = bytearray(n) 147 view = memoryview(buf) 148 while n > 0: 149 nbytes = self.sock.recv_into(view, n) 150 view = view[nbytes:] 151 n -= nbytes 152 buf = numpy.frombuffer(buf, dtype=numpy.uint8) 153 return jobj, buf 154 155 def __open_camera(self): 156 # Get the camera ID to open as an argument. 157 camera_id = 0 158 for s in sys.argv[1:]: 159 if s[:7] == "camera=" and len(s) > 7: 160 camera_id = int(s[7:]) 161 cmd = {"cmdName":"open", "cameraId":camera_id} 162 self.sock.send(json.dumps(cmd) + "\n") 163 data,_ = self.__read_response_from_socket() 164 if data['tag'] != 'cameraOpened': 165 raise its.error.Error('Invalid command response') 166 167 def __close_camera(self): 168 cmd = {"cmdName":"close"} 169 self.sock.send(json.dumps(cmd) + "\n") 170 data,_ = self.__read_response_from_socket() 171 if data['tag'] != 'cameraClosed': 172 raise its.error.Error('Invalid command response') 173 174 def do_vibrate(self, pattern): 175 """Cause the device to vibrate to a specific pattern. 176 177 Args: 178 pattern: Durations (ms) for which to turn on or off the vibrator. 179 The first value indicates the number of milliseconds to wait 180 before turning the vibrator on. The next value indicates the 181 number of milliseconds for which to keep the vibrator on 182 before turning it off. Subsequent values alternate between 183 durations in milliseconds to turn the vibrator off or to turn 184 the vibrator on. 185 186 Returns: 187 Nothing. 188 """ 189 cmd = {} 190 cmd["cmdName"] = "doVibrate" 191 cmd["pattern"] = pattern 192 self.sock.send(json.dumps(cmd) + "\n") 193 data,_ = self.__read_response_from_socket() 194 if data['tag'] != 'vibrationStarted': 195 raise its.error.Error('Invalid command response') 196 197 def start_sensor_events(self): 198 """Start collecting sensor events on the device. 199 200 See get_sensor_events for more info. 201 202 Returns: 203 Nothing. 204 """ 205 cmd = {} 206 cmd["cmdName"] = "startSensorEvents" 207 self.sock.send(json.dumps(cmd) + "\n") 208 data,_ = self.__read_response_from_socket() 209 if data['tag'] != 'sensorEventsStarted': 210 raise its.error.Error('Invalid command response') 211 212 def get_sensor_events(self): 213 """Get a trace of all sensor events on the device. 214 215 The trace starts when the start_sensor_events function is called. If 216 the test runs for a long time after this call, then the device's 217 internal memory can fill up. Calling get_sensor_events gets all events 218 from the device, and then stops the device from collecting events and 219 clears the internal buffer; to start again, the start_sensor_events 220 call must be used again. 221 222 Events from the accelerometer, compass, and gyro are returned; each 223 has a timestamp and x,y,z values. 224 225 Note that sensor events are only produced if the device isn't in its 226 standby mode (i.e.) if the screen is on. 227 228 Returns: 229 A Python dictionary with three keys ("accel", "mag", "gyro") each 230 of which maps to a list of objects containing "time","x","y","z" 231 keys. 232 """ 233 cmd = {} 234 cmd["cmdName"] = "getSensorEvents" 235 self.sock.send(json.dumps(cmd) + "\n") 236 data,_ = self.__read_response_from_socket() 237 if data['tag'] != 'sensorEvents': 238 raise its.error.Error('Invalid command response') 239 return data['objValue'] 240 241 def get_camera_properties(self): 242 """Get the camera properties object for the device. 243 244 Returns: 245 The Python dictionary object for the CameraProperties object. 246 """ 247 cmd = {} 248 cmd["cmdName"] = "getCameraProperties" 249 self.sock.send(json.dumps(cmd) + "\n") 250 data,_ = self.__read_response_from_socket() 251 if data['tag'] != 'cameraProperties': 252 raise its.error.Error('Invalid command response') 253 return data['objValue']['cameraProperties'] 254 255 def do_3a(self, regions_ae=[[0,0,1,1,1]], 256 regions_awb=[[0,0,1,1,1]], 257 regions_af=[[0,0,1,1,1]], 258 do_ae=True, do_awb=True, do_af=True, 259 lock_ae=False, lock_awb=False, 260 get_results=False): 261 """Perform a 3A operation on the device. 262 263 Triggers some or all of AE, AWB, and AF, and returns once they have 264 converged. Uses the vendor 3A that is implemented inside the HAL. 265 266 Throws an assertion if 3A fails to converge. 267 268 Args: 269 regions_ae: List of weighted AE regions. 270 regions_awb: List of weighted AWB regions. 271 regions_af: List of weighted AF regions. 272 do_ae: Trigger AE and wait for it to converge. 273 do_awb: Wait for AWB to converge. 274 do_af: Trigger AF and wait for it to converge. 275 lock_ae: Request AE lock after convergence, and wait for it. 276 lock_awb: Request AWB lock after convergence, and wait for it. 277 get_results: Return the 3A results from this function. 278 279 Region format in args: 280 Arguments are lists of weighted regions; each weighted region is a 281 list of 5 values, [x,y,w,h, wgt], and each argument is a list of 282 these 5-value lists. The coordinates are given as normalized 283 rectangles (x,y,w,h) specifying the region. For example: 284 [[0.0, 0.0, 1.0, 0.5, 5], [0.0, 0.5, 1.0, 0.5, 10]]. 285 Weights are non-negative integers. 286 287 Returns: 288 Five values are returned if get_results is true:: 289 * AE sensitivity; None if do_ae is False 290 * AE exposure time; None if do_ae is False 291 * AWB gains (list); None if do_awb is False 292 * AWB transform (list); None if do_awb is false 293 * AF focus position; None if do_af is false 294 Otherwise, it returns five None values. 295 """ 296 print "Running vendor 3A on device" 297 cmd = {} 298 cmd["cmdName"] = "do3A" 299 cmd["regions"] = {"ae": sum(regions_ae, []), 300 "awb": sum(regions_awb, []), 301 "af": sum(regions_af, [])} 302 cmd["triggers"] = {"ae": do_ae, "af": do_af} 303 if lock_ae: 304 cmd["aeLock"] = True 305 if lock_awb: 306 cmd["awbLock"] = True 307 self.sock.send(json.dumps(cmd) + "\n") 308 309 # Wait for each specified 3A to converge. 310 ae_sens = None 311 ae_exp = None 312 awb_gains = None 313 awb_transform = None 314 af_dist = None 315 converged = False 316 while True: 317 data,_ = self.__read_response_from_socket() 318 vals = data['strValue'].split() 319 if data['tag'] == 'aeResult': 320 ae_sens, ae_exp = [int(i) for i in vals] 321 elif data['tag'] == 'afResult': 322 af_dist = float(vals[0]) 323 elif data['tag'] == 'awbResult': 324 awb_gains = [float(f) for f in vals[:4]] 325 awb_transform = [float(f) for f in vals[4:]] 326 elif data['tag'] == '3aConverged': 327 converged = True 328 elif data['tag'] == '3aDone': 329 break 330 else: 331 raise its.error.Error('Invalid command response') 332 if converged and not get_results: 333 return None,None,None,None,None 334 if (do_ae and ae_sens == None or do_awb and awb_gains == None 335 or do_af and af_dist == None or not converged): 336 raise its.error.Error('3A failed to converge') 337 return ae_sens, ae_exp, awb_gains, awb_transform, af_dist 338 339 def do_capture(self, cap_request, out_surfaces=None): 340 """Issue capture request(s), and read back the image(s) and metadata. 341 342 The main top-level function for capturing one or more images using the 343 device. Captures a single image if cap_request is a single object, and 344 captures a burst if it is a list of objects. 345 346 The out_surfaces field can specify the width(s), height(s), and 347 format(s) of the captured image. The formats may be "yuv", "jpeg", 348 "dng", "raw", or "raw10". The default is a YUV420 frame ("yuv") 349 corresponding to a full sensor frame. 350 351 Note that one or more surfaces can be specified, allowing a capture to 352 request images back in multiple formats (e.g.) raw+yuv, raw+jpeg, 353 yuv+jpeg, raw+yuv+jpeg. If the size is omitted for a surface, the 354 default is the largest resolution available for the format of that 355 surface. At most one output surface can be specified for a given format, 356 and raw+dng, raw10+dng, and raw+raw10 are not supported as combinations. 357 358 Example of a single capture request: 359 360 { 361 "android.sensor.exposureTime": 100*1000*1000, 362 "android.sensor.sensitivity": 100 363 } 364 365 Example of a list of capture requests: 366 367 [ 368 { 369 "android.sensor.exposureTime": 100*1000*1000, 370 "android.sensor.sensitivity": 100 371 }, 372 { 373 "android.sensor.exposureTime": 100*1000*1000, 374 "android.sensor.sensitivity": 200 375 } 376 ] 377 378 Examples of output surface specifications: 379 380 { 381 "width": 640, 382 "height": 480, 383 "format": "yuv" 384 } 385 386 [ 387 { 388 "format": "jpeg" 389 }, 390 { 391 "format": "raw" 392 } 393 ] 394 395 The following variables defined in this class are shortcuts for 396 specifying one or more formats where each output is the full size for 397 that format; they can be used as values for the out_surfaces arguments: 398 399 CAP_RAW 400 CAP_DNG 401 CAP_YUV 402 CAP_JPEG 403 CAP_RAW_YUV 404 CAP_DNG_YUV 405 CAP_RAW_JPEG 406 CAP_DNG_JPEG 407 CAP_YUV_JPEG 408 CAP_RAW_YUV_JPEG 409 CAP_DNG_YUV_JPEG 410 411 If multiple formats are specified, then this function returns multuple 412 capture objects, one for each requested format. If multiple formats and 413 multiple captures (i.e. a burst) are specified, then this function 414 returns multiple lists of capture objects. In both cases, the order of 415 the returned objects matches the order of the requested formats in the 416 out_surfaces parameter. For example: 417 418 yuv_cap = do_capture( req1 ) 419 yuv_cap = do_capture( req1, yuv_fmt ) 420 yuv_cap, raw_cap = do_capture( req1, [yuv_fmt,raw_fmt] ) 421 yuv_caps = do_capture( [req1,req2], yuv_fmt ) 422 yuv_caps, raw_caps = do_capture( [req1,req2], [yuv_fmt,raw_fmt] ) 423 424 Args: 425 cap_request: The Python dict/list specifying the capture(s), which 426 will be converted to JSON and sent to the device. 427 out_surfaces: (Optional) specifications of the output image formats 428 and sizes to use for each capture. 429 430 Returns: 431 An object, list of objects, or list of lists of objects, where each 432 object contains the following fields: 433 * data: the image data as a numpy array of bytes. 434 * width: the width of the captured image. 435 * height: the height of the captured image. 436 * format: image the format, in ["yuv","jpeg","raw","raw10","dng"]. 437 * metadata: the capture result object (Python dictionaty). 438 """ 439 cmd = {} 440 cmd["cmdName"] = "doCapture" 441 if not isinstance(cap_request, list): 442 cmd["captureRequests"] = [cap_request] 443 else: 444 cmd["captureRequests"] = cap_request 445 if out_surfaces is not None: 446 if not isinstance(out_surfaces, list): 447 cmd["outputSurfaces"] = [out_surfaces] 448 else: 449 cmd["outputSurfaces"] = out_surfaces 450 formats = [c["format"] if c.has_key("format") else "yuv" 451 for c in cmd["outputSurfaces"]] 452 formats = [s if s != "jpg" else "jpeg" for s in formats] 453 else: 454 formats = ['yuv'] 455 ncap = len(cmd["captureRequests"]) 456 nsurf = 1 if out_surfaces is None else len(cmd["outputSurfaces"]) 457 if len(formats) > len(set(formats)): 458 raise its.error.Error('Duplicate format requested') 459 if "dng" in formats and "raw" in formats or \ 460 "dng" in formats and "raw10" in formats or \ 461 "raw" in formats and "raw10" in formats: 462 raise its.error.Error('Different raw formats not supported') 463 print "Capturing %d frame%s with %d format%s [%s]" % ( 464 ncap, "s" if ncap>1 else "", nsurf, "s" if nsurf>1 else "", 465 ",".join(formats)) 466 self.sock.send(json.dumps(cmd) + "\n") 467 468 # Wait for ncap*nsurf images and ncap metadata responses. 469 # Assume that captures come out in the same order as requested in 470 # the burst, however indifidual images of different formats ca come 471 # out in any order for that capture. 472 nbufs = 0 473 bufs = {"yuv":[], "raw":[], "raw10":[], "dng":[], "jpeg":[]} 474 mds = [] 475 widths = None 476 heights = None 477 while nbufs < ncap*nsurf or len(mds) < ncap: 478 jsonObj,buf = self.__read_response_from_socket() 479 if jsonObj['tag'] in ['jpegImage', 'yuvImage', 'rawImage', \ 480 'raw10Image', 'dngImage'] and buf is not None: 481 fmt = jsonObj['tag'][:-5] 482 bufs[fmt].append(buf) 483 nbufs += 1 484 elif jsonObj['tag'] == 'captureResults': 485 mds.append(jsonObj['objValue']['captureResult']) 486 outputs = jsonObj['objValue']['outputs'] 487 widths = [out['width'] for out in outputs] 488 heights = [out['height'] for out in outputs] 489 else: 490 # Just ignore other tags 491 None 492 rets = [] 493 for j,fmt in enumerate(formats): 494 objs = [] 495 for i in range(ncap): 496 obj = {} 497 obj["data"] = bufs[fmt][i] 498 obj["width"] = widths[j] 499 obj["height"] = heights[j] 500 obj["format"] = fmt 501 obj["metadata"] = mds[i] 502 objs.append(obj) 503 rets.append(objs if ncap>1 else objs[0]) 504 return rets if len(rets)>1 else rets[0] 505 506def _run(cmd): 507 """Replacement for os.system, with hiding of stdout+stderr messages. 508 """ 509 with open(os.devnull, 'wb') as devnull: 510 subprocess.check_call( 511 cmd.split(), stdout=devnull, stderr=subprocess.STDOUT) 512 513class __UnitTest(unittest.TestCase): 514 """Run a suite of unit tests on this module. 515 """ 516 517 # TODO: Add some unit tests. 518 None 519 520if __name__ == '__main__': 521 unittest.main() 522 523