1""" 2/* Copyright (c) 2023 Amazon 3 Written by Jan Buethe */ 4/* 5 Redistribution and use in source and binary forms, with or without 6 modification, are permitted provided that the following conditions 7 are met: 8 9 - Redistributions of source code must retain the above copyright 10 notice, this list of conditions and the following disclaimer. 11 12 - Redistributions in binary form must reproduce the above copyright 13 notice, this list of conditions and the following disclaimer in the 14 documentation and/or other materials provided with the distribution. 15 16 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 17 ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 18 LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 19 A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER 20 OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, 21 EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 22 PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR 23 PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF 24 LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING 25 NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 26 SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 27*/ 28""" 29 30import os 31import argparse 32 33import numpy as np 34import matplotlib.pyplot as plt 35from prettytable import PrettyTable 36from matplotlib.patches import Patch 37 38parser = argparse.ArgumentParser() 39parser.add_argument('folder', type=str, help='path to folder with pre-calculated metrics') 40parser.add_argument('--metric', choices=['pesq', 'moc', 'warpq', 'nomad', 'laceloss', 'all'], default='all', help='default: all') 41parser.add_argument('--output', type=str, default=None, help='alternative output folder, default: folder') 42 43def load_data(folder): 44 data = dict() 45 46 if os.path.isfile(os.path.join(folder, 'results_moc.npy')): 47 data['moc'] = np.load(os.path.join(folder, 'results_moc.npy'), allow_pickle=True).item() 48 49 if os.path.isfile(os.path.join(folder, 'results_moc2.npy')): 50 data['moc2'] = np.load(os.path.join(folder, 'results_moc2.npy'), allow_pickle=True).item() 51 52 if os.path.isfile(os.path.join(folder, 'results_pesq.npy')): 53 data['pesq'] = np.load(os.path.join(folder, 'results_pesq.npy'), allow_pickle=True).item() 54 55 if os.path.isfile(os.path.join(folder, 'results_warpq.npy')): 56 data['warpq'] = np.load(os.path.join(folder, 'results_warpq.npy'), allow_pickle=True).item() 57 58 if os.path.isfile(os.path.join(folder, 'results_nomad.npy')): 59 data['nomad'] = np.load(os.path.join(folder, 'results_nomad.npy'), allow_pickle=True).item() 60 61 if os.path.isfile(os.path.join(folder, 'results_laceloss.npy')): 62 data['laceloss'] = np.load(os.path.join(folder, 'results_laceloss.npy'), allow_pickle=True).item() 63 64 return data 65 66def make_table(filename, data, title=None): 67 68 # mean values 69 tbl = PrettyTable() 70 tbl.field_names = ['bitrate (bps)', 'Opus', 'LACE', 'NoLACE'] 71 for br in data.keys(): 72 opus = data[br][:, 0] 73 lace = data[br][:, 1] 74 nolace = data[br][:, 2] 75 tbl.add_row([br, f"{float(opus.mean()):.3f} ({float(opus.std()):.2f})", f"{float(lace.mean()):.3f} ({float(lace.std()):.2f})", f"{float(nolace.mean()):.3f} ({float(nolace.std()):.2f})"]) 76 77 with open(filename + ".txt", "w") as f: 78 f.write(str(tbl)) 79 80 with open(filename + ".html", "w") as f: 81 f.write(tbl.get_html_string()) 82 83 with open(filename + ".csv", "w") as f: 84 f.write(tbl.get_csv_string()) 85 86 print(tbl) 87 88 89def make_diff_table(filename, data, title=None): 90 91 # mean values 92 tbl = PrettyTable() 93 tbl.field_names = ['bitrate (bps)', 'LACE - Opus', 'NoLACE - Opus'] 94 for br in data.keys(): 95 opus = data[br][:, 0] 96 lace = data[br][:, 1] - opus 97 nolace = data[br][:, 2] - opus 98 tbl.add_row([br, f"{float(lace.mean()):.3f} ({float(lace.std()):.2f})", f"{float(nolace.mean()):.3f} ({float(nolace.std()):.2f})"]) 99 100 with open(filename + ".txt", "w") as f: 101 f.write(str(tbl)) 102 103 with open(filename + ".html", "w") as f: 104 f.write(tbl.get_html_string()) 105 106 with open(filename + ".csv", "w") as f: 107 f.write(tbl.get_csv_string()) 108 109 print(tbl) 110 111if __name__ == "__main__": 112 args = parser.parse_args() 113 data = load_data(args.folder) 114 115 metrics = list(data.keys()) if args.metric == 'all' else [args.metric] 116 folder = args.folder if args.output is None else args.output 117 os.makedirs(folder, exist_ok=True) 118 119 for metric in metrics: 120 print(f"Plotting data for {metric} metric...") 121 make_table(os.path.join(folder, f"table_{metric}"), data[metric]) 122 make_diff_table(os.path.join(folder, f"table_diff_{metric}"), data[metric]) 123 124 print("Done.")