tests/scene2_a/test_num_faces.py

# Copyright 2014 The Android Open Source Project
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Verifies 3 faces with different skin tones are detected."""


import logging
import math
import os.path

import cv2
from mobly import test_runner
from scipy.spatial import distance

import its_base_test
import camera_properties_utils
import capture_request_utils
import image_processing_utils
import its_session_utils
import opencv_processing_utils

_CV2_FACE_SCALE_FACTOR = 1.05  # 5% step for resizing image to find face
_CV2_FACE_MIN_NEIGHBORS = 4  # recommended 3-6: higher for less faces
_CV2_GREEN = (0, 1, 0)
_CV2_RED = (1, 0, 0)
_FACE_CENTER_MATCH_TOL_X = 10  # 10 pixels or ~1.5% in 640x480 image
_FACE_CENTER_MATCH_TOL_Y = 20  # 20 pixels or ~4% in 640x480 image
_FACE_CENTER_MIN_LOGGING_DIST = 50
_FD_MODE_OFF, _FD_MODE_SIMPLE, _FD_MODE_FULL = 0, 1, 2
_MIN_NUM_FACES_ALIGNED = 2
_MIN_CENTER_DELTA = 15
_NAME = os.path.splitext(os.path.basename(__file__))[0]
_NUM_FACES = 3
_NUM_TEST_FRAMES = 20
_TEST_REQUIRED_MPC = 34
_W, _H = 640, 480


def eliminate_duplicate_centers(coordinates_list):
  """Checks center coordinates of OpenCV's face rectangles

  Method makes sure that the list of face rectangles' centers do not
  contain duplicates from the same face.

  Args:
    coordinates_list: list; coordinates of face rectangles' centers
  Returns:
    non_duplicate_list: list; coordinates of face rectangles' centers
    without duplicates on the same face
  """
  output = set()

  for i, xy1 in enumerate(coordinates_list):
    for j, xy2 in enumerate(coordinates_list):
      if distance.euclidean(xy1, xy2) < _MIN_CENTER_DELTA:
        continue
      if xy1 not in output:
        output.add(xy1)
      else:
        output.add(xy2)
  return list(output)


def match_face_locations(faces_cropped, faces_opencv, mode, img, img_name):
  """Assert face locations between two methods.

  Method determines if center of opencv face boxes is within face detection
  face boxes. Using math.hypot to measure the distance between the centers,
  as math.dist is not available for python versions before 3.8.

  Args:
    faces_cropped: list of lists with (l, r, t, b) for each face.
    faces_opencv: list of lists with (x, y, w, h) for each face.
    mode: int indicating face detection mode
    img: np image array
    img_name: text string with path to image file
  """
  # turn faces_opencv into list of center locations
  faces_opencv_center = [(x+w//2, y+h//2) for (x, y, w, h) in faces_opencv]
  cropped_faces_centers = [
      ((l+r)//2, (t+b)//2) for (l, r, t, b) in faces_cropped]
  faces_opencv_center.sort(key=lambda t: [t[1], t[0]])
  cropped_faces_centers.sort(key=lambda t: [t[1], t[0]])
  logging.debug('cropped face centers: %s', str(cropped_faces_centers))
  logging.debug('opencv face center: %s', str(faces_opencv_center))
  faces_opencv_centers = []
  num_centers_aligned = 0

  # eliminate duplicate openCV face rectangles' centers the same face
  faces_opencv_centers = eliminate_duplicate_centers(faces_opencv_center)
  logging.debug('opencv face centers: %s', str(faces_opencv_centers))

  for (x, y) in faces_opencv_centers:
    for (x1, y1) in cropped_faces_centers:
      centers_dist = math.hypot(x-x1, y-y1)
      if centers_dist < _FACE_CENTER_MIN_LOGGING_DIST:
        logging.debug('centers_dist: %.3f', centers_dist)
      if (abs(x-x1) < _FACE_CENTER_MATCH_TOL_X and
          abs(y-y1) < _FACE_CENTER_MATCH_TOL_Y):
        num_centers_aligned += 1

  # If test failed, save image with green AND OpenCV red rectangles
  image_processing_utils.write_image(img, img_name)
  if num_centers_aligned < _MIN_NUM_FACES_ALIGNED:
    for (x, y, w, h) in faces_opencv:
      cv2.rectangle(img, (x, y), (x+w, y+h), _CV2_RED, 2)
      image_processing_utils.write_image(img, img_name)
      logging.debug('centered: %s', str(num_centers_aligned))
    raise AssertionError(f'Mode {mode} face rectangles in wrong location(s)!. '
                         f'Found {num_centers_aligned} rectangles near cropped '
                         f'face centers, expected {_MIN_NUM_FACES_ALIGNED}')


def check_face_bounding_box(rect, aw, ah, index):
  """Checks face bounding box is within the active array area.

  Args:
    rect: dict; with face bounding box information
    aw: int; active array width
    ah: int; active array height
    index: int to designate face number
  """
  logging.debug('Checking bounding box in face %d: %s', index, str(rect))
  if (rect['top'] >= rect['bottom'] or
      rect['left'] >= rect['right']):
    raise AssertionError('Face coordinates incorrect! '
                         f" t: {rect['top']}, b: {rect['bottom']}, "
                         f" l: {rect['left']}, r: {rect['right']}")
  if (not 0 <= rect['top'] <= ah or
      not 0 <= rect['bottom'] <= ah):
    raise AssertionError('Face top/bottom outside of image height! '
                         f"t: {rect['top']}, b: {rect['bottom']}, "
                         f"h: {ah}")
  if (not 0 <= rect['left'] <= aw or
      not 0 <= rect['right'] <= aw):
    raise AssertionError('Face left/right outside of image width! '
                         f"l: {rect['left']}, r: {rect['right']}, "
                         f" w: {aw}")


def check_face_landmarks(face, fd_mode, index):
  """Checks face landmarks fall within face bounding box.

  Face ID should be -1 for SIMPLE and unique for FULL
  Args:
    face: dict from face detection algorithm
    fd_mode: int of face detection mode
    index: int to designate face number
  """
  logging.debug('Checking landmarks in face %d: %s', index, str(face))
  if fd_mode == _FD_MODE_SIMPLE:
    if 'leftEye' in face or 'rightEye' in face:
      raise AssertionError('Eyes not supported in FD_MODE_SIMPLE.')
    if 'mouth' in face:
      raise AssertionError('Mouth not supported in FD_MODE_SIMPLE.')
    if face['id'] != -1:
      raise AssertionError('face_id should be -1 in FD_MODE_SIMPLE.')
  elif fd_mode == _FD_MODE_FULL:
    l, r = face['bounds']['left'], face['bounds']['right']
    t, b = face['bounds']['top'], face['bounds']['bottom']
    l_eye_x, l_eye_y = face['leftEye']['x'], face['leftEye']['y']
    r_eye_x, r_eye_y = face['rightEye']['x'], face['rightEye']['y']
    mouth_x, mouth_y = face['mouth']['x'], face['mouth']['y']
    if not l <= l_eye_x <= r:
      raise AssertionError(f'Face l: {l}, r: {r}, left eye x: {l_eye_x}')
    if not t <= l_eye_y <= b:
      raise AssertionError(f'Face t: {t}, b: {b}, left eye y: {l_eye_y}')
    if not l <= r_eye_x <= r:
      raise AssertionError(f'Face l: {l}, r: {r}, right eye x: {r_eye_x}')
    if not t <= r_eye_y <= b:
      raise AssertionError(f'Face t: {t}, b: {b}, right eye y: {r_eye_y}')
    if not l <= mouth_x <= r:
      raise AssertionError(f'Face l: {l}, r: {r}, mouth x: {mouth_x}')
    if not t <= mouth_y <= b:
      raise AssertionError(f'Face t: {t}, b: {b}, mouth y: {mouth_y}')
  else:
    raise AssertionError(f'Unknown face detection mode: {fd_mode}.')


def correct_faces_for_crop(faces, img, crop):
  """Correct face rectangles for sensor crop.

  Args:
    faces: list of dicts with face information
    img: np image array
    crop: dict of crop region size with 'top, right, left, bottom' as keys
  Returns:
    list of face locations (left, right, top, bottom) corrected
  """
  faces_corrected = []
  cw, ch = crop['right'] - crop['left'], crop['bottom'] - crop['top']
  logging.debug('crop region: %s', str(crop))
  w = img.shape[1]
  h = img.shape[0]
  for rect in [face['bounds'] for face in faces]:
    logging.debug('rect: %s', str(rect))
    left = int(round((rect['left'] - crop['left']) * w / cw))
    right = int(round((rect['right'] - crop['left']) * w / cw))
    top = int(round((rect['top'] - crop['top']) * h / ch))
    bottom = int(round((rect['bottom'] - crop['top']) * h / ch))
    faces_corrected.append([left, right, top, bottom])
  logging.debug('faces_corrected: %s', str(faces_corrected))
  return faces_corrected


class NumFacesTest(its_base_test.ItsBaseTest):
  """Test face detection with different skin tones.
  """

  def test_num_faces(self):
    """Test face detection."""
    with its_session_utils.ItsSession(
        device_id=self.dut.serial,
        camera_id=self.camera_id,
        hidden_physical_id=self.hidden_physical_id) as cam:
      props = cam.get_camera_properties()
      props = cam.override_with_hidden_physical_camera_props(props)

      # Load chart for scene
      its_session_utils.load_scene(
          cam, props, self.scene, self.tablet, self.chart_distance,
          log_path=self.log_path)

      # Check media performance class
      should_run = camera_properties_utils.face_detect(props)
      media_performance_class = its_session_utils.get_media_performance_class(
          self.dut.serial)
      if media_performance_class >= _TEST_REQUIRED_MPC and not should_run:
        its_session_utils.raise_mpc_assertion_error(
            _TEST_REQUIRED_MPC, _NAME, media_performance_class)

      # Check skip conditions
      camera_properties_utils.skip_unless(should_run)
      mono_camera = camera_properties_utils.mono_camera(props)
      fd_modes = props['android.statistics.info.availableFaceDetectModes']
      a = props['android.sensor.info.activeArraySize']
      aw, ah = a['right'] - a['left'], a['bottom'] - a['top']
      logging.debug('active array size: %s', str(a))
      file_name_stem = os.path.join(self.log_path, _NAME)

      cam.do_3a(mono_camera=mono_camera)

      for fd_mode in fd_modes:
        logging.debug('face detection mode: %d', fd_mode)
        if not _FD_MODE_OFF <= fd_mode <= _FD_MODE_FULL:
          raise AssertionError(f'FD mode {fd_mode} not in MODES! '
                               f'OFF: {_FD_MODE_OFF}, FULL: {_FD_MODE_FULL}')
        req = capture_request_utils.auto_capture_request()
        req['android.statistics.faceDetectMode'] = fd_mode
        fmt = {'format': 'yuv', 'width': _W, 'height': _H}
        caps = cam.do_capture([req]*_NUM_TEST_FRAMES, fmt)
        for i, cap in enumerate(caps):
          fd_mode_cap = cap['metadata']['android.statistics.faceDetectMode']
          if fd_mode_cap != fd_mode:
            raise AssertionError(f'metadata {fd_mode_cap} != req {fd_mode}')

          faces = cap['metadata']['android.statistics.faces']
          # 0 faces should be returned for OFF mode
          if fd_mode == _FD_MODE_OFF:
            if faces:
              raise AssertionError(f'Error: faces detected in OFF: {faces}')
            continue
          # Face detection could take several frames to warm up,
          # but should detect the correct number of faces in last frame
          if i == _NUM_TEST_FRAMES - 1:
            img = image_processing_utils.convert_capture_to_rgb_image(
                cap, props=props)
            fnd_faces = len(faces)
            logging.debug('Found %d face(s), expected %d.',
                          fnd_faces, _NUM_FACES)

            # draw boxes around faces in green
            crop_region = cap['metadata']['android.scaler.cropRegion']
            faces_cropped = correct_faces_for_crop(faces, img, crop_region)
            for (l, r, t, b) in faces_cropped:
              cv2.rectangle(img, (l, t), (r, b), _CV2_GREEN, 2)

            # Save image with green rectangles
            img_name = f'{file_name_stem}_fd_mode_{fd_mode}.jpg'
            image_processing_utils.write_image(img, img_name)
            if fnd_faces != _NUM_FACES:
              raise AssertionError('Wrong num of faces found! Found: '
                                   f'{fnd_faces}, expected: {_NUM_FACES}')
            # Reasonable scores for faces
            face_scores = [face['score'] for face in faces]
            for score in face_scores:
              if not 1 <= score <= 100:
                raise AssertionError(f'score not between [1:100]! {score}')

            # Face bounds should be within active array
            face_rectangles = [face['bounds'] for face in faces]
            for j, rect in enumerate(face_rectangles):
              check_face_bounding_box(rect, aw, ah, j)

            # Face landmarks (if provided) are within face bounding box
            vendor_api_level = its_session_utils.get_vendor_api_level(
                self.dut.serial)
            if vendor_api_level >= its_session_utils.ANDROID14_API_LEVEL:
              for k, face in enumerate(faces):
                check_face_landmarks(face, fd_mode, k)

            # Match location of opencv and face detection mode faces
            if self.scene == 'scene2_d':
              logging.debug('Match face centers between opencv & faces')
              faces_opencv = opencv_processing_utils.find_opencv_faces(
                  img, _CV2_FACE_SCALE_FACTOR, _CV2_FACE_MIN_NEIGHBORS)
              if fd_mode:  # non-zero value for ON
                match_face_locations(faces_cropped, faces_opencv,
                                     fd_mode, img, img_name)

          if not faces:
            continue
          logging.debug('Frame %d face metadata:', i)
          logging.debug(' Faces: %s', str(faces))


if __name__ == '__main__':
  test_runner.main()