1# Lint as: python2, python3 2# Copyright 2016 The Chromium OS Authors. All rights reserved. 3# Use of this source code is governed by a BSD-style license that can be 4# found in the LICENSE file. 5 6"""This module provides the utilities for avsync_probe's data processing. 7 8We will get a lot of raw data from the avsync_probe.Capture(). One data per 9millisecond. 10AVSyncProbeDataParser will help to transform the raw data to more readable 11formats. It also helps to calculate the audio/video sync timing if the 12sound_interval_frames parameter is not None. 13 14Example: 15 capture_data = avsync_probe.Capture(12) 16 parser = avsync_probe_utils.AVSyncProbeDataParser(self.resultsdir, 17 capture_data, 30) 18 19 # Use the following attributes to access data. They can be referenced in 20 # AVSyncProbeDataParser Class. 21 parser.video_duration_average 22 parser.video_duration_std 23 parser.sync_duration_averag 24 parser.sync_duration_std 25 parser.cumulative_frame_count 26 parser.dropped_frame_count 27 parser.corrupted_frame_count 28 parser.binarize_data 29 parser.audio_events 30 parser.video_events 31 32""" 33 34from __future__ import absolute_import 35from __future__ import division 36from __future__ import print_function 37import collections 38import logging 39import math 40import os 41import sys 42from six.moves import range 43 44 45# Indices for binarize_data, audio_events and video_events. 46TIME_INDEX = 0 47VIDEO_INDEX = 1 48AUDIO_INDEX = 2 49# This index is used for video_events and audio_events. 50# The slot contains the time difference to the previous event. 51TIME_DIFF_INDEX = 3 52 53# SyncResult namedtuple of audio and video frame. 54# time_delay < 0 means that audio comes out first. 55SyncResult = collections.namedtuple( 56 'SynResult', ['video_time', 'audio_time', 'time_delay']) 57 58 59class GrayCode(object): 60 """Converts bit patterns between binary and Gray code. 61 62 The bit patterns of Gray code values are packed into an int value. 63 For example, 4 is "110" in Gray code, which reads "6" when interpreted 64 as binary. 65 See "https://en.wikipedia.org/wiki/Gray_code" 66 67 """ 68 69 @staticmethod 70 def binary_to_gray(binary): 71 """Binary code to gray code. 72 73 @param binary: Binary code. 74 @return: gray code. 75 76 """ 77 return binary ^ (binary >> 1) 78 79 @staticmethod 80 def gray_to_binary(gray): 81 """Gray code to binary code. 82 83 @param gray: Gray code. 84 @return: binary code. 85 86 """ 87 result = gray 88 result ^= (result >> 16) 89 result ^= (result >> 8) 90 result ^= (result >> 4) 91 result ^= (result >> 2) 92 result ^= (result >> 1) 93 return result 94 95 96class HysteresisSwitch(object): 97 """ 98 Iteratively binarizes input sequence using hysteresis comparator with a 99 pair of fixed thresholds. 100 101 Hysteresis means to use 2 different thresholds 102 for activating and de-activating output. It is often used for thresholding 103 time-series signal while reducing small noise in the input. 104 105 Note that the low threshold is exclusive but the high threshold is 106 inclusive. 107 When the same values were applied for the both, the object works as a 108 non-hysteresis switch. 109 (i.e. equivalent to the >= operator). 110 111 """ 112 113 def __init__(self, low_threshold, high_threshold, init_state): 114 """Init HysteresisSwitch class. 115 116 @param low_threshold: The threshold value to deactivate the output. 117 The comparison is exclusive. 118 @param high_threshold: The threshold value to activate the output. 119 The comparison is inclusive. 120 @param init_state: True or False of the switch initial state. 121 122 """ 123 if low_threshold > high_threshold: 124 raise Exception('Low threshold %d exceeds the high threshold %d', 125 low_threshold, high_threshold) 126 self._low_threshold = low_threshold 127 self._high_threshold = high_threshold 128 self._last_state = init_state 129 130 def adjust_state(self, value): 131 """Updates the state of the switch by the input value and returns the 132 result. 133 134 @param value: value for updating. 135 @return the state of the switch. 136 137 """ 138 if value < self._low_threshold: 139 self._last_state = False 140 141 if value >= self._high_threshold: 142 self._last_state = True 143 144 return self._last_state 145 146 147class AVSyncProbeDataParser(object): 148 """ Digital information extraction from the raw sensor data sequence. 149 150 This class will transform the raw data to easier understand formats. 151 152 Attributes: 153 binarize_data: Transer the raw data to [Time, video code, is_audio]. 154 video code is from 0-7 repeatedly. 155 video_events: Events of video frame. 156 audio_events: Events of when audio happens. 157 video_duration_average: (ms) The average duration during video frames. 158 video_duration_std: Standard deviation of the video_duration_average. 159 sync_duration_average: (ms) The average duration for audio/video sync. 160 sync_duration_std: Standard deviation of sync_duration_average. 161 cumulative_frame_count: Number of total video frames. 162 dropped_frame_count: Total dropped video frames. 163 corrupted_frame_count: Total corrupted video frames. 164 165 """ 166 # Thresholds for hysteresis binarization of input signals. 167 # Relative to the minumum (0.0) and maximum (1.0) values of the value range 168 # of each input signal. 169 _NORMALIZED_LOW_THRESHOLD = 0.6 170 _NORMALIZED_HIGH_THRESHOLD = 0.7 171 172 _VIDEO_CODE_CYCLE = (1 << 3) 173 174 def __init__(self, log_dir, capture_raw_data, video_fps, 175 sound_interval_frames=None): 176 """Inits AVSyncProbeDataParser class. 177 178 @param log_dir: Directory for dumping each events' contents. 179 @param capture_raw_data: Raw data from avsync_probe device. 180 A list contains the list values of [timestamp, video0, video1, 181 video2, audio]. 182 @param video_fps: Video frames per second. Used to know if the video 183 frame is dropoped or just corrupted. 184 @param sound_interval_frames: The period of sound (beep) in the number 185 of video frames. This class will help to calculate audio/video 186 sync stats if sound_interval_frames is not None. 187 188 """ 189 self.video_duration_average = None 190 self.video_duration_std = None 191 self.sync_duration_average = None 192 self.sync_duration_std = None 193 self.cumulative_frame_count = None 194 self.dropped_frame_count = None 195 196 self._log_dir = log_dir 197 self._raw_data = capture_raw_data 198 # Translate to millisecond for each video frame. 199 self._video_duration = 1000 // video_fps 200 self._sound_interval_frames = sound_interval_frames 201 self._log_list_data_to_file('raw.txt', capture_raw_data) 202 203 self.binarize_data = self._binarize_raw_data() 204 # we need to get audio events before remove video preamble frames. 205 # Because audio event may appear before the preamble frame, if we 206 # remove the preamble frames first, we will lost the audio event. 207 self.audio_events = self._detect_audio_events() 208 self._remove_video_preamble() 209 self.video_events = self._detect_video_events() 210 self._analyze_events() 211 self._calculate_statistics_report() 212 213 def _log_list_data_to_file(self, filename, data): 214 """Log the list data to file. 215 216 It will log under self._log_dir directory. 217 218 @param filename: The file name. 219 @data: Data for logging. 220 221 """ 222 filepath = os.path.join(self._log_dir, filename) 223 with open(filepath, 'w') as f: 224 for v in data: 225 f.write('%s\n' % str(v)) 226 227 def _get_hysteresis_switch(self, index): 228 """Get HysteresisSwitch by the raw data. 229 230 @param index: The index of self._raw_data's element. 231 @return: HysteresisSwitch instance by the value of the raw data. 232 233 """ 234 max_value = max(x[index] for x in self._raw_data) 235 min_value = min(x[index] for x in self._raw_data) 236 scale = max_value - min_value 237 logging.info('index %d, max %d, min %d, scale %d', index, max_value, 238 min_value, scale) 239 return HysteresisSwitch( 240 min_value + scale * self._NORMALIZED_LOW_THRESHOLD, 241 min_value + scale * self._NORMALIZED_HIGH_THRESHOLD, 242 False) 243 244 def _binarize_raw_data(self): 245 """Conducts adaptive thresholding and decoding embedded frame codes. 246 247 Sensors[0] is timestamp. 248 Sensors[1-3] are photo transistors, which outputs lower value for 249 brighter light(=white pixels on screen). These are used to detect black 250 and white pattern on the screen, and decoded as an integer code. 251 252 The final channel is for audio input, which outputs higher voltage for 253 larger sound volume. This will be used for detecting beep sounds added 254 to the video. 255 256 @return Decoded frame codes list for all the input frames. Each entry 257 contains [Timestamp, video code, is_audio]. 258 259 """ 260 decoded_data = [] 261 262 hystersis_switch = [] 263 for i in range(5): 264 hystersis_switch.append(self._get_hysteresis_switch(i)) 265 266 for data in self._raw_data: 267 code = 0 268 # Decode black-and-white pattern on video. 269 # There are 3 black or white boxes sensed by the sensors. 270 # Each square represents a single bit (white = 1, black = 0) coding 271 # an integer in Gray code. 272 for i in range(1, 4): 273 # Lower sensor value for brighter light(square painted white). 274 is_white = not hystersis_switch[i].adjust_state(data[i]) 275 if is_white: 276 code |= (1 << (i - 1)) 277 code = GrayCode.gray_to_binary(code) 278 # The final channel is sound signal. Higher sensor value for 279 # higher sound level. 280 sound = hystersis_switch[4].adjust_state(data[4]) 281 decoded_data.append([data[0], code, sound]) 282 283 self._log_list_data_to_file('binarize_raw.txt', decoded_data) 284 return decoded_data 285 286 def _remove_video_preamble(self): 287 """Remove preamble video frames of self.binarize_data.""" 288 # find preamble frame (code = 0) 289 index = next(i for i, v in enumerate(self.binarize_data) 290 if v[VIDEO_INDEX] == 0) 291 self.binarize_data = self.binarize_data[index:] 292 293 # skip preamble frame (code = 0) 294 index = next(i for i, v in enumerate(self.binarize_data) 295 if v[VIDEO_INDEX] != 0) 296 self.binarize_data = self.binarize_data[index:] 297 298 def _detect_events(self, detect_condition): 299 """Detects events from the binarize data sequence by the 300 detect_condition. 301 302 @param detect_condition: callback function for checking event happens. 303 This API will pass index and element of binarize_data to the 304 callback function. 305 306 @return: The list of events. It's the same as the binarize_data and add 307 additional time_difference information. 308 309 """ 310 detected_events = [] 311 previous_time = self.binarize_data[0][TIME_INDEX] 312 for i, v in enumerate(self.binarize_data): 313 if (detect_condition(i, v)): 314 time = v[TIME_INDEX] 315 time_difference = time - previous_time 316 # Copy a new instance here, because we will append time 317 # difference. 318 event = list(v) 319 event.append(time_difference) 320 detected_events.append(event) 321 previous_time = time 322 323 return detected_events 324 325 def _detect_audio_events(self): 326 """Detects the audio start frame from the binarize data sequence. 327 328 @return: The list of Audio events. It's the same as the binarize_data 329 and add additional time_difference information. 330 331 """ 332 # Only check the first audio happen event. 333 detected_events = self._detect_events( 334 lambda i, v: (v[AUDIO_INDEX] and not 335 self.binarize_data[i - 1][AUDIO_INDEX])) 336 337 self._log_list_data_to_file('audio_events.txt', detected_events) 338 return detected_events 339 340 def _detect_video_events(self): 341 """Detects the video frame from the binarize data sequence. 342 343 @return: The list of Video events. It's the same as the binarize_data 344 and add additional time_difference information. 345 346 """ 347 # remove duplicate frames. (frames in transition state.) 348 detected_events = self._detect_events( 349 lambda i, v: (v[VIDEO_INDEX] != 350 self.binarize_data[i - 1][VIDEO_INDEX])) 351 352 self._log_list_data_to_file('video_events.txt', detected_events) 353 return detected_events 354 355 def _match_sync(self, video_time): 356 """Match the audio/video sync timing. 357 358 This function will find the closest sound in the audio_events to the 359 video_time and returns a audio/video sync tuple. 360 361 @param video_time: the time of the video which have sound. 362 @return A SyncResult namedtuple containing: 363 - timestamp of the video frame which should have audio. 364 - timestamp of nearest audio frame. 365 - time delay between audio and video frame. 366 367 """ 368 closest_difference = sys.maxsize 369 audio_time = 0 370 for audio_event in self.audio_events: 371 difference = audio_event[TIME_INDEX] - video_time 372 if abs(difference) < abs(closest_difference): 373 closest_difference = difference 374 audio_time = audio_event[TIME_INDEX] 375 return SyncResult(video_time, audio_time, closest_difference) 376 377 def _calculate_statistics(self, data): 378 """Calculate average and standard deviation of the list data. 379 380 @param data: The list of values to be calcualted. 381 @return: An tuple with (average, standard_deviation) 382 383 """ 384 if not data: 385 return (None, None) 386 387 total = sum(data) 388 average = total / len(data) 389 variance = sum((v - average)**2 for v in data) / len(data) 390 standard_deviation = math.sqrt(variance) 391 return (average, standard_deviation) 392 393 def _analyze_events(self): 394 """Analyze audio/video events. 395 396 This function will analyze video frame status and audio/video sync 397 status. 398 399 """ 400 sound_interval_frames = self._sound_interval_frames 401 current_code = 0 402 cumulative_frame_count = 0 403 dropped_frame_count = 0 404 corrupted_frame_count = 0 405 sync_events = [] 406 407 for v in self.video_events: 408 code = v[VIDEO_INDEX] 409 time = v[TIME_INDEX] 410 frame_diff = code - current_code 411 # Get difference of the codes. # The code is between 0 - 7. 412 if frame_diff < 0: 413 frame_diff += self._VIDEO_CODE_CYCLE 414 415 if frame_diff != 1: 416 # Check if we dropped frame or just got corrupted frame. 417 # Treat the frame as corrupted frame if the frame duration is 418 # less than 2 video frame duration. 419 if v[TIME_DIFF_INDEX] < 2 * self._video_duration: 420 logging.warning('Corrupted frame near %s', str(v)) 421 # Correct the code. 422 code = current_code + 1 423 corrupted_frame_count += 1 424 frame_diff = 1 425 else: 426 logging.warning('Dropped frame near %s', str(v)) 427 dropped_frame_count += (frame_diff - 1) 428 429 cumulative_frame_count += frame_diff 430 431 if sound_interval_frames is not None: 432 # This frame corresponds to a sound. 433 if cumulative_frame_count % sound_interval_frames == 1: 434 sync_events.append(self._match_sync(time)) 435 436 current_code = code 437 self.cumulative_frame_count = cumulative_frame_count 438 self.dropped_frame_count = dropped_frame_count 439 self.corrupted_frame_count = corrupted_frame_count 440 self._sync_events = sync_events 441 self._log_list_data_to_file('sync.txt', sync_events) 442 443 def _calculate_statistics_report(self): 444 """Calculates statistics report.""" 445 video_duration_average, video_duration_std = self._calculate_statistics( 446 [v[TIME_DIFF_INDEX] for v in self.video_events]) 447 sync_duration_average, sync_duration_std = self._calculate_statistics( 448 [v.time_delay for v in self._sync_events]) 449 self.video_duration_average = video_duration_average 450 self.video_duration_std = video_duration_std 451 self.sync_duration_average = sync_duration_average 452 self.sync_duration_std = sync_duration_std 453