• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1#!/usr/bin/env python
2# Copyright (c) 2017 The WebRTC project authors. All Rights Reserved.
3#
4# Use of this source code is governed by a BSD-style license
5# that can be found in the LICENSE file in the root of the source
6# tree. An additional intellectual property rights grant can be found
7# in the file PATENTS.  All contributing project authors may
8# be found in the AUTHORS file in the root of the source tree.
9
10"""
11This script is the wrapper that runs the low-bandwidth audio test.
12
13After running the test, post-process steps for calculating audio quality of the
14output files will be performed.
15"""
16
17import argparse
18import collections
19import logging
20import os
21import re
22import shutil
23import subprocess
24import sys
25
26
27SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
28SRC_DIR = os.path.normpath(os.path.join(SCRIPT_DIR, os.pardir, os.pardir))
29
30NO_TOOLS_ERROR_MESSAGE = (
31  'Could not find PESQ or POLQA at %s.\n'
32  '\n'
33  'To fix this run:\n'
34  '  python %s %s\n'
35  '\n'
36  'Note that these tools are Google-internal due to licensing, so in order to '
37  'use them you will have to get your own license and manually put them in the '
38  'right location.\n'
39  'See https://cs.chromium.org/chromium/src/third_party/webrtc/tools_webrtc/'
40  'download_tools.py?rcl=bbceb76f540159e2dba0701ac03c514f01624130&l=13')
41
42
43def _LogCommand(command):
44  logging.info('Running %r', command)
45  return command
46
47
48def _ParseArgs():
49  parser = argparse.ArgumentParser(description='Run low-bandwidth audio tests.')
50  parser.add_argument('build_dir',
51      help='Path to the build directory (e.g. out/Release).')
52  parser.add_argument('--remove', action='store_true',
53      help='Remove output audio files after testing.')
54  parser.add_argument('--android', action='store_true',
55      help='Perform the test on a connected Android device instead.')
56  parser.add_argument('--adb-path', help='Path to adb binary.', default='adb')
57  parser.add_argument('--num-retries', default='0',
58                      help='Number of times to retry the test on Android.')
59  parser.add_argument('--isolated-script-test-perf-output', default=None,
60      help='Path to store perf results in histogram proto format.')
61  parser.add_argument('--extra-test-args', default=[], action='append',
62      help='Extra args to path to the test binary.')
63
64  # Ignore Chromium-specific flags
65  parser.add_argument('--test-launcher-summary-output',
66                      type=str, default=None)
67  args = parser.parse_args()
68
69  return args
70
71
72def _GetPlatform():
73  if sys.platform == 'win32':
74    return 'win'
75  elif sys.platform == 'darwin':
76    return 'mac'
77  elif sys.platform.startswith('linux'):
78    return 'linux'
79
80
81def _GetExtension():
82  return '.exe' if sys.platform == 'win32' else ''
83
84
85def _GetPathToTools():
86  tools_dir = os.path.join(SRC_DIR, 'tools_webrtc')
87  toolchain_dir = os.path.join(tools_dir, 'audio_quality')
88
89  platform = _GetPlatform()
90  ext = _GetExtension()
91
92  pesq_path = os.path.join(toolchain_dir, platform, 'pesq' + ext)
93  if not os.path.isfile(pesq_path):
94    pesq_path = None
95
96  polqa_path = os.path.join(toolchain_dir, platform, 'PolqaOem64' + ext)
97  if not os.path.isfile(polqa_path):
98    polqa_path = None
99
100  if (platform != 'mac' and not polqa_path) or not pesq_path:
101    logging.error(NO_TOOLS_ERROR_MESSAGE,
102                  toolchain_dir,
103                  os.path.join(tools_dir, 'download_tools.py'),
104                  toolchain_dir)
105
106  return pesq_path, polqa_path
107
108
109def ExtractTestRuns(lines, echo=False):
110  """Extracts information about tests from the output of a test runner.
111
112  Produces tuples
113  (android_device, test_name, reference_file, degraded_file, cur_perf_results).
114  """
115  for line in lines:
116    if echo:
117      sys.stdout.write(line)
118
119    # Output from Android has a prefix with the device name.
120    android_prefix_re = r'(?:I\b.+\brun_tests_on_device\((.+?)\)\s*)?'
121    test_re = r'^' + android_prefix_re + (r'TEST (\w+) ([^ ]+?) ([^\s]+)'
122                                          r' ?([^\s]+)?\s*$')
123
124    match = re.search(test_re, line)
125    if match:
126      yield match.groups()
127
128
129def _GetFile(file_path, out_dir, move=False,
130             android=False, adb_prefix=('adb',)):
131  out_file_name = os.path.basename(file_path)
132  out_file_path = os.path.join(out_dir, out_file_name)
133
134  if android:
135    # Pull the file from the connected Android device.
136    adb_command = adb_prefix + ('pull', file_path, out_dir)
137    subprocess.check_call(_LogCommand(adb_command))
138    if move:
139      # Remove that file.
140      adb_command = adb_prefix + ('shell', 'rm', file_path)
141      subprocess.check_call(_LogCommand(adb_command))
142  elif os.path.abspath(file_path) != os.path.abspath(out_file_path):
143    if move:
144      shutil.move(file_path, out_file_path)
145    else:
146      shutil.copy(file_path, out_file_path)
147
148  return out_file_path
149
150
151def _RunPesq(executable_path, reference_file, degraded_file,
152             sample_rate_hz=16000):
153  directory = os.path.dirname(reference_file)
154  assert os.path.dirname(degraded_file) == directory
155
156  # Analyze audio.
157  command = [executable_path, '+%d' % sample_rate_hz,
158             os.path.basename(reference_file),
159             os.path.basename(degraded_file)]
160  # Need to provide paths in the current directory due to a bug in PESQ:
161  # On Mac, for some 'path/to/file.wav', if 'file.wav' is longer than
162  # 'path/to', PESQ crashes.
163  out = subprocess.check_output(_LogCommand(command),
164                                cwd=directory, stderr=subprocess.STDOUT)
165
166  # Find the scores in stdout of PESQ.
167  match = re.search(
168      r'Prediction \(Raw MOS, MOS-LQO\):\s+=\s+([\d.]+)\s+([\d.]+)', out)
169  if match:
170    raw_mos, _ = match.groups()
171
172    return {'pesq_mos': (raw_mos, 'unitless')}
173  else:
174    logging.error('PESQ: %s', out.splitlines()[-1])
175    return {}
176
177
178def _RunPolqa(executable_path, reference_file, degraded_file):
179  # Analyze audio.
180  command = [executable_path, '-q', '-LC', 'NB',
181             '-Ref', reference_file, '-Test', degraded_file]
182  process = subprocess.Popen(_LogCommand(command),
183                             stdout=subprocess.PIPE, stderr=subprocess.PIPE)
184  out, err = process.communicate()
185
186  # Find the scores in stdout of POLQA.
187  match = re.search(r'\bMOS-LQO:\s+([\d.]+)', out)
188
189  if process.returncode != 0 or not match:
190    if process.returncode == 2:
191      logging.warning('%s (2)', err.strip())
192      logging.warning('POLQA license error, skipping test.')
193    else:
194      logging.error('%s (%d)', err.strip(), process.returncode)
195    return {}
196
197  mos_lqo, = match.groups()
198  return {'polqa_mos_lqo': (mos_lqo, 'unitless')}
199
200
201def _MergeInPerfResultsFromCcTests(histograms, run_perf_results_file):
202  from tracing.value import histogram_set
203
204  cc_histograms = histogram_set.HistogramSet()
205  with open(run_perf_results_file, 'rb') as f:
206    contents = f.read()
207    if not contents:
208      return
209
210    cc_histograms.ImportProto(contents)
211
212  histograms.Merge(cc_histograms)
213
214
215Analyzer = collections.namedtuple('Analyzer', ['name', 'func', 'executable',
216                                               'sample_rate_hz'])
217
218
219def _ConfigurePythonPath(args):
220  script_dir = os.path.dirname(os.path.realpath(__file__))
221  checkout_root = os.path.abspath(
222      os.path.join(script_dir, os.pardir, os.pardir))
223
224  # TODO(https://crbug.com/1029452): Use a copy rule and add these from the out
225  # dir like for the third_party/protobuf code.
226  sys.path.insert(0, os.path.join(checkout_root, 'third_party', 'catapult',
227                                  'tracing'))
228
229  # The low_bandwidth_audio_perf_test gn rule will build the protobuf stub for
230  # python, so put it in the path for this script before we attempt to import
231  # it.
232  histogram_proto_path = os.path.join(
233      os.path.abspath(args.build_dir), 'pyproto', 'tracing', 'tracing', 'proto')
234  sys.path.insert(0, histogram_proto_path)
235  proto_stub_path = os.path.join(os.path.abspath(args.build_dir), 'pyproto')
236  sys.path.insert(0, proto_stub_path)
237
238  # Fail early in case the proto hasn't been built.
239  try:
240    import histogram_pb2
241  except ImportError as e:
242    logging.exception(e)
243    raise ImportError('Could not import histogram_pb2. You need to build the '
244                      'low_bandwidth_audio_perf_test target before invoking '
245                      'this script. Expected to find '
246                      'histogram_pb2.py in %s.' % histogram_proto_path)
247
248
249def main():
250  # pylint: disable=W0101
251  logging.basicConfig(level=logging.INFO)
252  logging.info('Invoked with %s', str(sys.argv))
253
254  args = _ParseArgs()
255
256  _ConfigurePythonPath(args)
257
258  # Import catapult modules here after configuring the pythonpath.
259  from tracing.value import histogram_set
260  from tracing.value.diagnostics import reserved_infos
261  from tracing.value.diagnostics import generic_set
262
263  pesq_path, polqa_path = _GetPathToTools()
264  if pesq_path is None:
265    return 1
266
267  out_dir = os.path.join(args.build_dir, '..')
268  if args.android:
269    test_command = [os.path.join(args.build_dir, 'bin',
270                                 'run_low_bandwidth_audio_test'),
271                    '-v', '--num-retries', args.num_retries]
272  else:
273    test_command = [os.path.join(args.build_dir, 'low_bandwidth_audio_test')]
274
275  analyzers = [Analyzer('pesq', _RunPesq, pesq_path, 16000)]
276  # Check if POLQA can run at all, or skip the 48 kHz tests entirely.
277  example_path = os.path.join(SRC_DIR, 'resources',
278                              'voice_engine', 'audio_tiny48.wav')
279  if polqa_path and _RunPolqa(polqa_path, example_path, example_path):
280    analyzers.append(Analyzer('polqa', _RunPolqa, polqa_path, 48000))
281
282  histograms = histogram_set.HistogramSet()
283  for analyzer in analyzers:
284    # Start the test executable that produces audio files.
285    test_process = subprocess.Popen(
286        _LogCommand(test_command + [
287            '--sample_rate_hz=%d' % analyzer.sample_rate_hz,
288            '--test_case_prefix=%s' % analyzer.name,
289          ] + args.extra_test_args),
290        stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
291    perf_results_file = None
292    try:
293      lines = iter(test_process.stdout.readline, '')
294      for result in ExtractTestRuns(lines, echo=True):
295        (android_device, test_name, reference_file, degraded_file,
296         perf_results_file) = result
297
298        adb_prefix = (args.adb_path,)
299        if android_device:
300          adb_prefix += ('-s', android_device)
301
302        reference_file = _GetFile(reference_file, out_dir,
303                                  android=args.android, adb_prefix=adb_prefix)
304        degraded_file = _GetFile(degraded_file, out_dir, move=True,
305                                 android=args.android, adb_prefix=adb_prefix)
306
307        analyzer_results = analyzer.func(analyzer.executable,
308                                         reference_file, degraded_file)
309        for metric, (value, units) in analyzer_results.items():
310          hist = histograms.CreateHistogram(metric, units, [value])
311          user_story = generic_set.GenericSet([test_name])
312          hist.diagnostics[reserved_infos.STORIES.name] = user_story
313
314          # Output human readable results.
315          print 'RESULT %s: %s= %s %s' % (metric, test_name, value, units)
316
317        if args.remove:
318          os.remove(reference_file)
319          os.remove(degraded_file)
320    finally:
321      test_process.terminate()
322    if perf_results_file:
323      perf_results_file = _GetFile(perf_results_file, out_dir, move=True,
324                           android=args.android, adb_prefix=adb_prefix)
325      _MergeInPerfResultsFromCcTests(histograms, perf_results_file)
326      if args.remove:
327        os.remove(perf_results_file)
328
329  if args.isolated_script_test_perf_output:
330    with open(args.isolated_script_test_perf_output, 'wb') as f:
331      f.write(histograms.AsProto().SerializeToString())
332
333  return test_process.wait()
334
335
336if __name__ == '__main__':
337  sys.exit(main())
338