• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1#  Copyright (c) 2017 The WebRTC project authors. All Rights Reserved.
2#
3#  Use of this source code is governed by a BSD-style license
4#  that can be found in the LICENSE file in the root of the source
5#  tree. An additional intellectual property rights grant can be found
6#  in the file PATENTS.  All contributing project authors may
7#  be found in the AUTHORS file in the root of the source tree.
8"""Plots statistics from WebRTC integration test logs.
9
10Usage: $ python plot_webrtc_test_logs.py filename.txt
11"""
12
13import numpy
14import sys
15import re
16
17import matplotlib.pyplot as plt
18
19# Log events.
20EVENT_START = 'RUN      ] CodecSettings/VideoCodecTestParameterized.'
21EVENT_END = 'OK ] CodecSettings/VideoCodecTestParameterized.'
22
23# Metrics to plot, tuple: (name to parse in file, label to use when plotting).
24WIDTH = ('width', 'width')
25HEIGHT = ('height', 'height')
26FILENAME = ('filename', 'clip')
27CODEC_TYPE = ('codec_type', 'Codec')
28ENCODER_IMPLEMENTATION_NAME = ('enc_impl_name', 'enc name')
29DECODER_IMPLEMENTATION_NAME = ('dec_impl_name', 'dec name')
30CODEC_IMPLEMENTATION_NAME = ('codec_impl_name', 'codec name')
31CORES = ('num_cores', 'CPU cores used')
32DENOISING = ('denoising', 'denoising')
33RESILIENCE = ('resilience', 'resilience')
34ERROR_CONCEALMENT = ('error_concealment', 'error concealment')
35CPU_USAGE = ('cpu_usage_percent', 'CPU usage (%)')
36BITRATE = ('target_bitrate_kbps', 'target bitrate (kbps)')
37FRAMERATE = ('input_framerate_fps', 'fps')
38QP = ('avg_qp', 'QP avg')
39PSNR = ('avg_psnr', 'PSNR (dB)')
40SSIM = ('avg_ssim', 'SSIM')
41ENC_BITRATE = ('bitrate_kbps', 'encoded bitrate (kbps)')
42NUM_FRAMES = ('num_input_frames', 'num frames')
43NUM_DROPPED_FRAMES = ('num_dropped_frames', 'num dropped frames')
44TIME_TO_TARGET = ('time_to_reach_target_bitrate_sec',
45                  'time to reach target rate (sec)')
46ENCODE_SPEED_FPS = ('enc_speed_fps', 'encode speed (fps)')
47DECODE_SPEED_FPS = ('dec_speed_fps', 'decode speed (fps)')
48AVG_KEY_FRAME_SIZE = ('avg_key_frame_size_bytes', 'avg key frame size (bytes)')
49AVG_DELTA_FRAME_SIZE = ('avg_delta_frame_size_bytes',
50                        'avg delta frame size (bytes)')
51
52# Settings.
53SETTINGS = [
54    WIDTH,
55    HEIGHT,
56    FILENAME,
57    NUM_FRAMES,
58]
59
60# Settings, options for x-axis.
61X_SETTINGS = [
62    CORES,
63    FRAMERATE,
64    DENOISING,
65    RESILIENCE,
66    ERROR_CONCEALMENT,
67    BITRATE,  # TODO(asapersson): Needs to be last.
68]
69
70# Settings, options for subplots.
71SUBPLOT_SETTINGS = [
72    CODEC_TYPE,
73    ENCODER_IMPLEMENTATION_NAME,
74    DECODER_IMPLEMENTATION_NAME,
75    CODEC_IMPLEMENTATION_NAME,
76] + X_SETTINGS
77
78# Results.
79RESULTS = [
80    PSNR,
81    SSIM,
82    ENC_BITRATE,
83    NUM_DROPPED_FRAMES,
84    TIME_TO_TARGET,
85    ENCODE_SPEED_FPS,
86    DECODE_SPEED_FPS,
87    QP,
88    CPU_USAGE,
89    AVG_KEY_FRAME_SIZE,
90    AVG_DELTA_FRAME_SIZE,
91]
92
93METRICS_TO_PARSE = SETTINGS + SUBPLOT_SETTINGS + RESULTS
94
95Y_METRICS = [res[1] for res in RESULTS]
96
97# Parameters for plotting.
98FIG_SIZE_SCALE_FACTOR_X = 1.6
99FIG_SIZE_SCALE_FACTOR_Y = 1.8
100GRID_COLOR = [0.45, 0.45, 0.45]
101
102
103def ParseSetting(filename, setting):
104    """Parses setting from file.
105
106  Args:
107    filename: The name of the file.
108    setting: Name of setting to parse (e.g. width).
109
110  Returns:
111    A list holding parsed settings, e.g. ['width: 128.0', 'width: 160.0'] """
112
113    settings = []
114
115    settings_file = open(filename)
116    while True:
117        line = settings_file.readline()
118        if not line:
119            break
120        if re.search(r'%s' % EVENT_START, line):
121            # Parse event.
122            parsed = {}
123            while True:
124                line = settings_file.readline()
125                if not line:
126                    break
127                if re.search(r'%s' % EVENT_END, line):
128                    # Add parsed setting to list.
129                    if setting in parsed:
130                        s = setting + ': ' + str(parsed[setting])
131                        if s not in settings:
132                            settings.append(s)
133                    break
134
135                TryFindMetric(parsed, line)
136
137    settings_file.close()
138    return settings
139
140
141def ParseMetrics(filename, setting1, setting2):
142    """Parses metrics from file.
143
144  Args:
145    filename: The name of the file.
146    setting1: First setting for sorting metrics (e.g. width).
147    setting2: Second setting for sorting metrics (e.g. CPU cores used).
148
149  Returns:
150    A dictionary holding parsed metrics.
151
152  For example:
153    metrics[key1][key2][measurement]
154
155  metrics = {
156  "width: 352": {
157    "CPU cores used: 1.0": {
158      "encode time (us)": [0.718005, 0.806925, 0.909726, 0.931835, 0.953642],
159      "PSNR (dB)": [25.546029, 29.465518, 34.723535, 36.428493, 38.686551],
160      "bitrate (kbps)": [50, 100, 300, 500, 1000]
161    },
162    "CPU cores used: 2.0": {
163      "encode time (us)": [0.718005, 0.806925, 0.909726, 0.931835, 0.953642],
164      "PSNR (dB)": [25.546029, 29.465518, 34.723535, 36.428493, 38.686551],
165      "bitrate (kbps)": [50, 100, 300, 500, 1000]
166    },
167  },
168  "width: 176": {
169    "CPU cores used: 1.0": {
170      "encode time (us)": [0.857897, 0.91608, 0.959173, 0.971116, 0.980961],
171      "PSNR (dB)": [30.243646, 33.375592, 37.574387, 39.42184, 41.437897],
172      "bitrate (kbps)": [50, 100, 300, 500, 1000]
173    },
174  }
175  } """
176
177    metrics = {}
178
179    # Parse events.
180    settings_file = open(filename)
181    while True:
182        line = settings_file.readline()
183        if not line:
184            break
185        if re.search(r'%s' % EVENT_START, line):
186            # Parse event.
187            parsed = {}
188            while True:
189                line = settings_file.readline()
190                if not line:
191                    break
192                if re.search(r'%s' % EVENT_END, line):
193                    # Add parsed values to metrics.
194                    key1 = setting1 + ': ' + str(parsed[setting1])
195                    key2 = setting2 + ': ' + str(parsed[setting2])
196                    if key1 not in metrics:
197                        metrics[key1] = {}
198                    if key2 not in metrics[key1]:
199                        metrics[key1][key2] = {}
200
201                    for label in parsed:
202                        if label not in metrics[key1][key2]:
203                            metrics[key1][key2][label] = []
204                        metrics[key1][key2][label].append(parsed[label])
205
206                    break
207
208                TryFindMetric(parsed, line)
209
210    settings_file.close()
211    return metrics
212
213
214def TryFindMetric(parsed, line):
215    for metric in METRICS_TO_PARSE:
216        name = metric[0]
217        label = metric[1]
218        if re.search(r'%s' % name, line):
219            found, value = GetMetric(name, line)
220            if found:
221                parsed[label] = value
222            return
223
224
225def GetMetric(name, string):
226    # Float (e.g. bitrate = 98.8253).
227    pattern = r'%s\s*[:=]\s*([+-]?\d+\.*\d*)' % name
228    m = re.search(r'%s' % pattern, string)
229    if m is not None:
230        return StringToFloat(m.group(1))
231
232    # Alphanumeric characters (e.g. codec type : VP8).
233    pattern = r'%s\s*[:=]\s*(\w+)' % name
234    m = re.search(r'%s' % pattern, string)
235    if m is not None:
236        return True, m.group(1)
237
238    return False, -1
239
240
241def StringToFloat(value):
242    try:
243        value = float(value)
244    except ValueError:
245        print "Not a float, skipped %s" % value
246        return False, -1
247
248    return True, value
249
250
251def Plot(y_metric, x_metric, metrics):
252    """Plots y_metric vs x_metric per key in metrics.
253
254  For example:
255    y_metric = 'PSNR (dB)'
256    x_metric = 'bitrate (kbps)'
257    metrics = {
258      "CPU cores used: 1.0": {
259        "PSNR (dB)": [25.546029, 29.465518, 34.723535, 36.428493, 38.686551],
260        "bitrate (kbps)": [50, 100, 300, 500, 1000]
261      },
262      "CPU cores used: 2.0": {
263        "PSNR (dB)": [25.546029, 29.465518, 34.723535, 36.428493, 38.686551],
264        "bitrate (kbps)": [50, 100, 300, 500, 1000]
265      },
266    }
267    """
268    for key in sorted(metrics):
269        data = metrics[key]
270        if y_metric not in data:
271            print "Failed to find metric: %s" % y_metric
272            continue
273
274        y = numpy.array(data[y_metric])
275        x = numpy.array(data[x_metric])
276        if len(y) != len(x):
277            print "Length mismatch for %s, %s" % (y, x)
278            continue
279
280        label = y_metric + ' - ' + str(key)
281
282        plt.plot(x,
283                 y,
284                 label=label,
285                 linewidth=1.5,
286                 marker='o',
287                 markersize=5,
288                 markeredgewidth=0.0)
289
290
291def PlotFigure(settings, y_metrics, x_metric, metrics, title):
292    """Plots metrics in y_metrics list. One figure is plotted and each entry
293  in the list is plotted in a subplot (and sorted per settings).
294
295  For example:
296    settings = ['width: 128.0', 'width: 160.0']. Sort subplot per setting.
297    y_metrics = ['PSNR (dB)', 'PSNR (dB)']. Metric to plot per subplot.
298    x_metric = 'bitrate (kbps)'
299
300  """
301
302    plt.figure()
303    plt.suptitle(title, fontsize='large', fontweight='bold')
304    settings.sort()
305    rows = len(settings)
306    cols = 1
307    pos = 1
308    while pos <= rows:
309        plt.rc('grid', color=GRID_COLOR)
310        ax = plt.subplot(rows, cols, pos)
311        plt.grid()
312        plt.setp(ax.get_xticklabels(), visible=(pos == rows), fontsize='large')
313        plt.setp(ax.get_yticklabels(), fontsize='large')
314        setting = settings[pos - 1]
315        Plot(y_metrics[pos - 1], x_metric, metrics[setting])
316        if setting.startswith(WIDTH[1]):
317            plt.title(setting, fontsize='medium')
318        plt.legend(fontsize='large', loc='best')
319        pos += 1
320
321    plt.xlabel(x_metric, fontsize='large')
322    plt.subplots_adjust(left=0.06,
323                        right=0.98,
324                        bottom=0.05,
325                        top=0.94,
326                        hspace=0.08)
327
328
329def GetTitle(filename, setting):
330    title = ''
331    if setting != CODEC_IMPLEMENTATION_NAME[1] and setting != CODEC_TYPE[1]:
332        codec_types = ParseSetting(filename, CODEC_TYPE[1])
333        for i in range(0, len(codec_types)):
334            title += codec_types[i] + ', '
335
336    if setting != CORES[1]:
337        cores = ParseSetting(filename, CORES[1])
338        for i in range(0, len(cores)):
339            title += cores[i].split('.')[0] + ', '
340
341    if setting != FRAMERATE[1]:
342        framerate = ParseSetting(filename, FRAMERATE[1])
343        for i in range(0, len(framerate)):
344            title += framerate[i].split('.')[0] + ', '
345
346    if (setting != CODEC_IMPLEMENTATION_NAME[1]
347            and setting != ENCODER_IMPLEMENTATION_NAME[1]):
348        enc_names = ParseSetting(filename, ENCODER_IMPLEMENTATION_NAME[1])
349        for i in range(0, len(enc_names)):
350            title += enc_names[i] + ', '
351
352    if (setting != CODEC_IMPLEMENTATION_NAME[1]
353            and setting != DECODER_IMPLEMENTATION_NAME[1]):
354        dec_names = ParseSetting(filename, DECODER_IMPLEMENTATION_NAME[1])
355        for i in range(0, len(dec_names)):
356            title += dec_names[i] + ', '
357
358    filenames = ParseSetting(filename, FILENAME[1])
359    title += filenames[0].split('_')[0]
360
361    num_frames = ParseSetting(filename, NUM_FRAMES[1])
362    for i in range(0, len(num_frames)):
363        title += ' (' + num_frames[i].split('.')[0] + ')'
364
365    return title
366
367
368def ToString(input_list):
369    return ToStringWithoutMetric(input_list, ('', ''))
370
371
372def ToStringWithoutMetric(input_list, metric):
373    i = 1
374    output_str = ""
375    for m in input_list:
376        if m != metric:
377            output_str = output_str + ("%s. %s\n" % (i, m[1]))
378            i += 1
379    return output_str
380
381
382def GetIdx(text_list):
383    return int(raw_input(text_list)) - 1
384
385
386def main():
387    filename = sys.argv[1]
388
389    # Setup.
390    idx_metric = GetIdx("Choose metric:\n0. All\n%s" % ToString(RESULTS))
391    if idx_metric == -1:
392        # Plot all metrics. One subplot for each metric.
393        # Per subplot: metric vs bitrate (per resolution).
394        cores = ParseSetting(filename, CORES[1])
395        setting1 = CORES[1]
396        setting2 = WIDTH[1]
397        sub_keys = [cores[0]] * len(Y_METRICS)
398        y_metrics = Y_METRICS
399        x_metric = BITRATE[1]
400    else:
401        resolutions = ParseSetting(filename, WIDTH[1])
402        idx = GetIdx("Select metric for x-axis:\n%s" % ToString(X_SETTINGS))
403        if X_SETTINGS[idx] == BITRATE:
404            idx = GetIdx("Plot per:\n%s" %
405                         ToStringWithoutMetric(SUBPLOT_SETTINGS, BITRATE))
406            idx_setting = METRICS_TO_PARSE.index(SUBPLOT_SETTINGS[idx])
407            # Plot one metric. One subplot for each resolution.
408            # Per subplot: metric vs bitrate (per setting).
409            setting1 = WIDTH[1]
410            setting2 = METRICS_TO_PARSE[idx_setting][1]
411            sub_keys = resolutions
412            y_metrics = [RESULTS[idx_metric][1]] * len(sub_keys)
413            x_metric = BITRATE[1]
414        else:
415            # Plot one metric. One subplot for each resolution.
416            # Per subplot: metric vs setting (per bitrate).
417            setting1 = WIDTH[1]
418            setting2 = BITRATE[1]
419            sub_keys = resolutions
420            y_metrics = [RESULTS[idx_metric][1]] * len(sub_keys)
421            x_metric = X_SETTINGS[idx][1]
422
423    metrics = ParseMetrics(filename, setting1, setting2)
424
425    # Stretch fig size.
426    figsize = plt.rcParams["figure.figsize"]
427    figsize[0] *= FIG_SIZE_SCALE_FACTOR_X
428    figsize[1] *= FIG_SIZE_SCALE_FACTOR_Y
429    plt.rcParams["figure.figsize"] = figsize
430
431    PlotFigure(sub_keys, y_metrics, x_metric, metrics,
432               GetTitle(filename, setting2))
433
434    plt.show()
435
436
437if __name__ == '__main__':
438    main()
439