• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1#!/usr/bin/env vpython3
2# Copyright (c) 2017 The WebRTC project authors. All Rights Reserved.
3#
4# Use of this source code is governed by a BSD-style license
5# that can be found in the LICENSE file in the root of the source
6# tree. An additional intellectual property rights grant can be found
7# in the file PATENTS.  All contributing project authors may
8# be found in the AUTHORS file in the root of the source tree.
9"""
10This script is the wrapper that runs the low-bandwidth audio test.
11
12After running the test, post-process steps for calculating audio quality of the
13output files will be performed.
14"""
15
16import argparse
17import collections
18import json
19import logging
20import os
21import re
22import shutil
23import subprocess
24import sys
25
26SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
27SRC_DIR = os.path.normpath(os.path.join(SCRIPT_DIR, os.pardir, os.pardir))
28
29NO_TOOLS_ERROR_MESSAGE = (
30    'Could not find PESQ or POLQA at %s.\n'
31    '\n'
32    'To fix this run:\n'
33    '  python %s %s\n'
34    '\n'
35    'Note that these tools are Google-internal due to licensing, so in order '
36    'to use them you will have to get your own license and manually put them '
37    'in the right location.\n'
38    'See https://cs.chromium.org/chromium/src/third_party/webrtc/tools_webrtc/'
39    'download_tools.py?rcl=bbceb76f540159e2dba0701ac03c514f01624130&l=13')
40
41
42def _LogCommand(command):
43  logging.info('Running %r', command)
44  return command
45
46
47def _ParseArgs():
48  parser = argparse.ArgumentParser(description='Run low-bandwidth audio tests.')
49  parser.add_argument('build_dir',
50                      help='Path to the build directory (e.g. out/Release).')
51  parser.add_argument('--remove',
52                      action='store_true',
53                      help='Remove output audio files after testing.')
54  parser.add_argument(
55      '--android',
56      action='store_true',
57      help='Perform the test on a connected Android device instead.')
58  parser.add_argument('--adb-path', help='Path to adb binary.', default='adb')
59  parser.add_argument('--num-retries',
60                      default='0',
61                      help='Number of times to retry the test on Android.')
62  parser.add_argument(
63      '--isolated-script-test-perf-output',
64      default=None,
65      help='Path to store perf results in histogram proto format.')
66  parser.add_argument(
67      '--isolated-script-test-output',
68      default=None,
69      help='Path to output an empty JSON file which Chromium infra requires.')
70
71  return parser.parse_known_args()
72
73
74def _GetPlatform():
75  if sys.platform == 'win32':
76    return 'win'
77  if sys.platform == 'darwin':
78    return 'mac'
79  if sys.platform.startswith('linux'):
80    return 'linux'
81  raise AssertionError('Unknown platform %s' % sys.platform)
82
83
84def _GetExtension():
85  return '.exe' if sys.platform == 'win32' else ''
86
87
88def _GetPathToTools():
89  tools_dir = os.path.join(SRC_DIR, 'tools_webrtc')
90  toolchain_dir = os.path.join(tools_dir, 'audio_quality')
91
92  platform = _GetPlatform()
93  ext = _GetExtension()
94
95  pesq_path = os.path.join(toolchain_dir, platform, 'pesq' + ext)
96  if not os.path.isfile(pesq_path):
97    pesq_path = None
98
99  polqa_path = os.path.join(toolchain_dir, platform, 'PolqaOem64' + ext)
100  if not os.path.isfile(polqa_path):
101    polqa_path = None
102
103  if (platform != 'mac' and not polqa_path) or not pesq_path:
104    logging.error(NO_TOOLS_ERROR_MESSAGE, toolchain_dir,
105                  os.path.join(tools_dir, 'download_tools.py'), toolchain_dir)
106
107  return pesq_path, polqa_path
108
109
110def ExtractTestRuns(lines, echo=False):
111  """Extracts information about tests from the output of a test runner.
112
113  Produces tuples
114  (android_device, test_name, reference_file, degraded_file, cur_perf_results).
115  """
116  for line in lines:
117    if echo:
118      sys.stdout.write(line)
119
120    # Output from Android has a prefix with the device name.
121    android_prefix_re = r'(?:I\b.+\brun_tests_on_device\((.+?)\)\s*)?'
122    test_re = r'^' + android_prefix_re + (r'TEST (\w+) ([^ ]+?) ([^\s]+)'
123                                          r' ?([^\s]+)?\s*$')
124
125    match = re.search(test_re, line)
126    if match:
127      yield match.groups()
128
129
130def _GetFile(file_path,
131             out_dir,
132             move=False,
133             android=False,
134             adb_prefix=('adb', )):
135  out_file_name = os.path.basename(file_path)
136  out_file_path = os.path.join(out_dir, out_file_name)
137
138  if android:
139    # Pull the file from the connected Android device.
140    adb_command = adb_prefix + ('pull', file_path, out_dir)
141    subprocess.check_call(_LogCommand(adb_command))
142    if move:
143      # Remove that file.
144      adb_command = adb_prefix + ('shell', 'rm', file_path)
145      subprocess.check_call(_LogCommand(adb_command))
146  elif os.path.abspath(file_path) != os.path.abspath(out_file_path):
147    if move:
148      shutil.move(file_path, out_file_path)
149    else:
150      shutil.copy(file_path, out_file_path)
151
152  return out_file_path
153
154
155def _RunPesq(executable_path,
156             reference_file,
157             degraded_file,
158             sample_rate_hz=16000):
159  directory = os.path.dirname(reference_file)
160  assert os.path.dirname(degraded_file) == directory
161
162  # Analyze audio.
163  command = [
164      executable_path,
165      '+%d' % sample_rate_hz,
166      os.path.basename(reference_file),
167      os.path.basename(degraded_file)
168  ]
169  # Need to provide paths in the current directory due to a bug in PESQ:
170  # On Mac, for some 'path/to/file.wav', if 'file.wav' is longer than
171  # 'path/to', PESQ crashes.
172  out = subprocess.check_output(_LogCommand(command),
173                                cwd=directory,
174                                universal_newlines=True,
175                                stderr=subprocess.STDOUT)
176
177  # Find the scores in stdout of PESQ.
178  match = re.search(
179      r'Prediction \(Raw MOS, MOS-LQO\):\s+=\s+([\d.]+)\s+([\d.]+)', out)
180  if match:
181    raw_mos, _ = match.groups()
182    return {'pesq_mos': (raw_mos, 'unitless')}
183  logging.error('PESQ: %s', out.splitlines()[-1])
184  return {}
185
186
187def _RunPolqa(executable_path, reference_file, degraded_file):
188  # Analyze audio.
189  command = [
190      executable_path, '-q', '-LC', 'NB', '-Ref', reference_file, '-Test',
191      degraded_file
192  ]
193  process = subprocess.Popen(_LogCommand(command),
194                             universal_newlines=True,
195                             stdout=subprocess.PIPE,
196                             stderr=subprocess.PIPE)
197  out, err = process.communicate()
198
199  # Find the scores in stdout of POLQA.
200  match = re.search(r'\bMOS-LQO:\s+([\d.]+)', out)
201
202  if process.returncode != 0 or not match:
203    if process.returncode == 2:
204      logging.warning('%s (2)', err.strip())
205      logging.warning('POLQA license error, skipping test.')
206    else:
207      logging.error('%s (%d)', err.strip(), process.returncode)
208    return {}
209
210  mos_lqo, = match.groups()
211  return {'polqa_mos_lqo': (mos_lqo, 'unitless')}
212
213
214def _MergeInPerfResultsFromCcTests(histograms, run_perf_results_file):
215  from tracing.value import histogram_set
216
217  cc_histograms = histogram_set.HistogramSet()
218  with open(run_perf_results_file, 'rb') as f:
219    contents = f.read()
220    if not contents:
221      return
222
223    cc_histograms.ImportProto(contents)
224
225  histograms.Merge(cc_histograms)
226
227
228Analyzer = collections.namedtuple(
229    'Analyzer', ['name', 'func', 'executable', 'sample_rate_hz'])
230
231
232def _ConfigurePythonPath(args):
233  script_dir = os.path.dirname(os.path.realpath(__file__))
234  checkout_root = os.path.abspath(os.path.join(script_dir, os.pardir,
235                                               os.pardir))
236
237  # TODO(https://crbug.com/1029452): Use a copy rule and add these from the
238  # out dir like for the third_party/protobuf code.
239  sys.path.insert(
240      0, os.path.join(checkout_root, 'third_party', 'catapult', 'tracing'))
241
242  # The low_bandwidth_audio_perf_test gn rule will build the protobuf stub
243  # for python, so put it in the path for this script before we attempt to
244  # import it.
245  histogram_proto_path = os.path.join(os.path.abspath(args.build_dir),
246                                      'pyproto', 'tracing', 'tracing', 'proto')
247  sys.path.insert(0, histogram_proto_path)
248  proto_stub_path = os.path.join(os.path.abspath(args.build_dir), 'pyproto')
249  sys.path.insert(0, proto_stub_path)
250
251  # Fail early in case the proto hasn't been built.
252  try:
253    #pylint: disable=unused-import
254    import histogram_pb2
255  except ImportError as e:
256    raise ImportError('Could not import histogram_pb2. You need to build the '
257                      'low_bandwidth_audio_perf_test target before invoking '
258                      'this script. Expected to find '
259                      'histogram_pb2.py in %s.' % histogram_proto_path) from e
260
261
262def main():
263  logging.basicConfig(format='%(asctime)s %(levelname)-8s %(message)s',
264                      level=logging.INFO,
265                      datefmt='%Y-%m-%d %H:%M:%S')
266  logging.info('Invoked with %s', str(sys.argv))
267
268  args, extra_test_args = _ParseArgs()
269
270  _ConfigurePythonPath(args)
271
272  # Import catapult modules here after configuring the pythonpath.
273  from tracing.value import histogram_set
274  from tracing.value.diagnostics import reserved_infos
275  from tracing.value.diagnostics import generic_set
276
277  pesq_path, polqa_path = _GetPathToTools()
278  if pesq_path is None:
279    return 1
280
281  out_dir = os.path.join(args.build_dir, '..')
282  if args.android:
283    test_command = [
284        os.path.join(args.build_dir, 'bin', 'run_low_bandwidth_audio_test'),
285        '-v', '--num-retries', args.num_retries
286    ]
287  else:
288    test_command = [os.path.join(args.build_dir, 'low_bandwidth_audio_test')]
289
290  analyzers = [Analyzer('pesq', _RunPesq, pesq_path, 16000)]
291  # Check if POLQA can run at all, or skip the 48 kHz tests entirely.
292  example_path = os.path.join(SRC_DIR, 'resources', 'voice_engine',
293                              'audio_tiny48.wav')
294  if polqa_path and _RunPolqa(polqa_path, example_path, example_path):
295    analyzers.append(Analyzer('polqa', _RunPolqa, polqa_path, 48000))
296
297  histograms = histogram_set.HistogramSet()
298  for analyzer in analyzers:
299    # Start the test executable that produces audio files.
300    test_process = subprocess.Popen(_LogCommand(test_command + [
301        '--sample_rate_hz=%d' % analyzer.sample_rate_hz,
302        '--test_case_prefix=%s' % analyzer.name,
303    ] + extra_test_args),
304                                    universal_newlines=True,
305                                    stdout=subprocess.PIPE,
306                                    stderr=subprocess.STDOUT)
307    perf_results_file = None
308    try:
309      lines = iter(test_process.stdout.readline, '')
310      for result in ExtractTestRuns(lines, echo=True):
311        (android_device, test_name, reference_file, degraded_file,
312         perf_results_file) = result
313
314        adb_prefix = (args.adb_path, )
315        if android_device:
316          adb_prefix += ('-s', android_device)
317
318        reference_file = _GetFile(reference_file,
319                                  out_dir,
320                                  android=args.android,
321                                  adb_prefix=adb_prefix)
322        degraded_file = _GetFile(degraded_file,
323                                 out_dir,
324                                 move=True,
325                                 android=args.android,
326                                 adb_prefix=adb_prefix)
327
328        analyzer_results = analyzer.func(analyzer.executable, reference_file,
329                                         degraded_file)
330        for metric, (value, units) in list(analyzer_results.items()):
331          hist = histograms.CreateHistogram(metric, units, [value])
332          user_story = generic_set.GenericSet([test_name])
333          hist.diagnostics[reserved_infos.STORIES.name] = user_story
334
335          # Output human readable results.
336          print('RESULT %s: %s= %s %s' % (metric, test_name, value, units))
337
338        if args.remove:
339          os.remove(reference_file)
340          os.remove(degraded_file)
341    finally:
342      test_process.terminate()
343    if perf_results_file:
344      perf_results_file = _GetFile(perf_results_file,
345                                   out_dir,
346                                   move=True,
347                                   android=args.android,
348                                   adb_prefix=adb_prefix)
349      _MergeInPerfResultsFromCcTests(histograms, perf_results_file)
350      if args.remove:
351        os.remove(perf_results_file)
352
353  if args.isolated_script_test_perf_output:
354    with open(args.isolated_script_test_perf_output, 'wb') as f:
355      f.write(histograms.AsProto().SerializeToString())
356
357  if args.isolated_script_test_output:
358    with open(args.isolated_script_test_output, 'w') as f:
359      json.dump({"version": 3}, f)
360
361  return test_process.wait()
362
363
364if __name__ == '__main__':
365  sys.exit(main())
366