1#!/usr/bin/env python3 2# 3# Copyright 2019 - The Android Open Source Project 4# 5# Licensed under the Apache License, Version 2.0 (the "License"); 6# you may not use this file except in compliance with the License. 7# You may obtain a copy of the License at 8# 9# http://www.apache.org/licenses/LICENSE-2.0 10# 11# Unless required by applicable law or agreed to in writing, software 12# distributed under the License is distributed on an "AS IS" BASIS, 13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14# See the License for the specific language governing permissions and 15# limitations under the License. 16'''Python Module for GNSS test log utilities.''' 17 18import re as regex 19import datetime 20import functools as fts 21import numpy as npy 22import pandas as pds 23from acts import logger 24 25# GPS API Log Reading Config 26CONFIG_GPSAPILOG = { 27 'phone_time': 28 r'(?P<date>\d+\/\d+\/\d+)\s+(?P<time>\d+:\d+:\d+)\s+' 29 r'Read:\s+(?P<logsize>\d+)\s+bytes', 30 'SpaceVehicle': 31 r'Fix:\s+(?P<Fix>\w+)\s+Type:\s+(?P<Type>\w+)\s+' 32 r'SV:\s+(?P<SV>\d+)\s+C\/No:\s+(?P<CNo>\d+\.\d+)\s+' 33 r'Elevation:\s+(?P<Elevation>\d+\.\d+)\s+' 34 r'Azimuth:\s+(?P<Azimuth>\d+\.\d+)\s+' 35 r'Signal:\s+(?P<Signal>\w+)\s+' 36 r'Frequency:\s+(?P<Frequency>\d+\.\d+)\s+' 37 r'EPH:\s+(?P<EPH>\w+)\s+ALM:\s+(?P<ALM>\w+)', 38 'HistoryAvgTop4CNo': 39 r'History\s+Avg\s+Top4\s+:\s+(?P<HistoryAvgTop4CNo>\d+\.\d+)', 40 'CurrentAvgTop4CNo': 41 r'Current\s+Avg\s+Top4\s+:\s+(?P<CurrentAvgTop4CNo>\d+\.\d+)', 42 'HistoryAvgCNo': 43 r'History\s+Avg\s+:\s+(?P<HistoryAvgCNo>\d+\.\d+)', 44 'CurrentAvgCNo': 45 r'Current\s+Avg\s+:\s+(?P<CurrentAvgCNo>\d+\.\d+)', 46 'L5inFix': 47 r'L5\s+used\s+in\s+fix:\s+(?P<L5inFix>\w+)', 48 'L5EngagingRate': 49 r'L5\s+engaging\s+rate:\s+(?P<L5EngagingRate>\d+.\d+)%', 50 'Provider': 51 r'Provider:\s+(?P<Provider>\w+)', 52 'Latitude': 53 r'Latitude:\s+(?P<Latitude>-?\d+.\d+)', 54 'Longitude': 55 r'Longitude:\s+(?P<Longitude>-?\d+.\d+)', 56 'Altitude': 57 r'Altitude:\s+(?P<Altitude>-?\d+.\d+)', 58 'GNSSTime': 59 r'Time:\s+(?P<Date>\d+\/\d+\/\d+)\s+' 60 r'(?P<Time>\d+:\d+:\d+)', 61 'Speed': 62 r'Speed:\s+(?P<Speed>\d+.\d+)', 63 'Bearing': 64 r'Bearing:\s+(?P<Bearing>\d+.\d+)', 65} 66 67# Space Vehicle Statistics Dataframe List 68LIST_SVSTAT = [ 69 'HistoryAvgTop4CNo', 'CurrentAvgTop4CNo', 'HistoryAvgCNo', 'CurrentAvgCNo', 70 'L5inFix', 'L5EngagingRate' 71] 72 73# Location Fix Info Dataframe List 74LIST_LOCINFO = [ 75 'Provider', 'Latitude', 'Longitude', 'Altitude', 'GNSSTime', 'Speed', 76 'Bearing' 77] 78 79# GPS TTFF Log Reading Config 80CONFIG_GPSTTFFLOG = { 81 'ttff_info': 82 r'Loop:(?P<loop>\d+)\s+' 83 r'(?P<start_datetime>\d+\/\d+\/\d+-\d+:\d+:\d+.\d+)\s+' 84 r'(?P<stop_datetime>\d+\/\d+\/\d+-\d+:\d+:\d+.\d+)\s+' 85 r'(?P<ttff>\d+.\d+)\s+' 86 r'\[Avg Top4 : (?P<avg_top4_cn0>\d+.\d+)\]\s' 87 r'\[Avg : (?P<avg_cn0>\d+.\d+)\]\s+\[(?P<fix_type>\d+\w+ fix)\]\s+' 88 r'\[Satellites used for fix : (?P<satnum_for_fix>\d+)\]' 89} 90 91LOGPARSE_UTIL_LOGGER = logger.create_logger() 92 93 94def parse_log_to_df(filename, configs, index_rownum=True): 95 r"""Parse log to a dictionary of Pandas dataframes. 96 97 Args: 98 filename: log file name. 99 Type String. 100 configs: configs dictionary of parsed Pandas dataframes. 101 Type dictionary. 102 dict key, the parsed pattern name, such as 'Speed', 103 dict value, regex of the config pattern, 104 Type Raw String. 105 index_rownum: index row number from raw data. 106 Type Boolean. 107 Default, True. 108 109 Returns: 110 parsed_data: dictionary of parsed data. 111 Type dictionary. 112 dict key, the parsed pattern name, such as 'Speed', 113 dict value, the corresponding parsed dataframe. 114 115 Examples: 116 configs = { 117 'GNSSTime': 118 r'Time:\s+(?P<Date>\d+\/\d+\/\d+)\s+ 119 r(?P<Time>\d+:\d+:\d+)')}, 120 'Speed': r'Speed:\s+(?P<Speed>\d+.\d+)', 121 } 122 """ 123 # Init a local config dictionary to hold compiled regex and match dict. 124 configs_local = {} 125 # Construct parsed data dictionary 126 parsed_data = {} 127 128 # Loop the config dictionary to compile regex and init data list 129 for key, regex_string in configs.items(): 130 configs_local[key] = { 131 'cregex': regex.compile(regex_string), 132 'datalist': [], 133 } 134 135 # Open the file, loop and parse 136 with open(filename, 'r') as fid: 137 138 for idx_line, current_line in enumerate(fid): 139 for _, config in configs_local.items(): 140 matched_log_object = config['cregex'].search(current_line) 141 142 if matched_log_object: 143 matched_data = matched_log_object.groupdict() 144 matched_data['rownumber'] = idx_line + 1 145 config['datalist'].append(matched_data) 146 147 # Loop to generate parsed data from configs list 148 for key, config in configs_local.items(): 149 parsed_data[key] = pds.DataFrame(config['datalist']) 150 if index_rownum and not parsed_data[key].empty: 151 parsed_data[key].set_index('rownumber', inplace=True) 152 elif parsed_data[key].empty: 153 LOGPARSE_UTIL_LOGGER.warning( 154 'The parsed dataframe of "%s" is empty.', key) 155 156 # Return parsed data list 157 return parsed_data 158 159 160def parse_gpstool_ttfflog_to_df(filename): 161 """Parse GPSTool ttff log to Pandas dataframes. 162 163 Args: 164 filename: full log file name. 165 Type, String. 166 167 Returns: 168 ttff_df: TTFF Data Frame. 169 Type, Pandas DataFrame. 170 """ 171 # Get parsed dataframe list 172 parsed_data = lputil.parse_log_to_df( 173 filename=filename, 174 configs=CONFIG_GPSTTFFLOG, 175 ) 176 ttff_df = parsed_data['ttff_info'] 177 178 # Data Conversion 179 ttff_df['loop'] = ttff_df['loop'].astype(int) 180 ttff_df['start_datetime'] = pds.to_datetime(ttff_df['start_datetime']) 181 ttff_df['stop_datetime'] = pds.to_datetime(ttff_df['stop_datetime']) 182 ttff_df['ttff'] = ttff_df['ttff'].astype(float) 183 ttff_df['avg_top4_cn0'] = ttff_df['avg_top4_cn0'].astype(float) 184 ttff_df['avg_cn0'] = ttff_df['avg_cn0'].astype(float) 185 ttff_df['satnum_for_fix'] = ttff_df['satnum_for_fix'].astype(int) 186 187 # return ttff dataframe 188 return ttff_df 189 190 191def parse_gpsapilog_to_df(filename): 192 """Parse GPS API log to Pandas dataframes. 193 194 Args: 195 filename: full log file name. 196 Type, String. 197 198 Returns: 199 timestamp_df: Timestamp Data Frame. 200 Type, Pandas DataFrame. 201 sv_info_df: GNSS SV info Data Frame. 202 Type, Pandas DataFrame. 203 loc_info_df: Location Information Data Frame. 204 Type, Pandas DataFrame. 205 include Provider, Latitude, Longitude, Altitude, GNSSTime, Speed, Bearing 206 """ 207 def get_phone_time(target_df_row, timestamp_df): 208 """subfunction to get the phone_time.""" 209 210 try: 211 row_num = timestamp_df[ 212 timestamp_df.index < target_df_row.name].iloc[-1].name 213 phone_time = timestamp_df.loc[row_num]['phone_time'] 214 except IndexError: 215 row_num = npy.NaN 216 phone_time = npy.NaN 217 218 return phone_time, row_num 219 220 # Get parsed dataframe list 221 parsed_data = parse_log_to_df( 222 filename=filename, 223 configs=CONFIG_GPSAPILOG, 224 ) 225 226 # get DUT Timestamp 227 timestamp_df = parsed_data['phone_time'] 228 timestamp_df['phone_time'] = timestamp_df.apply( 229 lambda row: datetime.datetime.strptime(row.date + '-' + row.time, 230 '%Y/%m/%d-%H:%M:%S'), 231 axis=1) 232 233 # Add phone_time from timestamp_df dataframe by row number 234 for key in parsed_data: 235 if key != 'phone_time': 236 current_df = parsed_data[key] 237 time_n_row_num = current_df.apply(get_phone_time, 238 axis=1, 239 timestamp_df=timestamp_df) 240 current_df[['phone_time', 'time_row_num' 241 ]] = pds.DataFrame(time_n_row_num.apply(pds.Series)) 242 243 # Get space vehicle info dataframe 244 sv_info_df = parsed_data['SpaceVehicle'] 245 246 # Get space vehicle statistics dataframe 247 # First merge all dataframe from LIST_SVSTAT[1:], 248 # Drop duplicated 'phone_time', based on time_row_num 249 sv_stat_df = fts.reduce( 250 lambda item1, item2: pds.merge(item1, item2, on='time_row_num'), [ 251 parsed_data[key].drop(['phone_time'], axis=1) 252 for key in LIST_SVSTAT[1:] 253 ]) 254 # Then merge with LIST_SVSTAT[0] 255 sv_stat_df = pds.merge(sv_stat_df, 256 parsed_data[LIST_SVSTAT[0]], 257 on='time_row_num') 258 259 # Get location fix information dataframe 260 # First merge all dataframe from LIST_LOCINFO[1:], 261 # Drop duplicated 'phone_time', based on time_row_num 262 loc_info_df = fts.reduce( 263 lambda item1, item2: pds.merge(item1, item2, on='time_row_num'), [ 264 parsed_data[key].drop(['phone_time'], axis=1) 265 for key in LIST_LOCINFO[1:] 266 ]) 267 # Then merge with LIST_LOCINFO[8] 268 loc_info_df = pds.merge(loc_info_df, 269 parsed_data[LIST_LOCINFO[0]], 270 on='time_row_num') 271 # Convert GNSS Time 272 loc_info_df['gnsstime'] = loc_info_df.apply( 273 lambda row: datetime.datetime.strptime(row.Date + '-' + row.Time, 274 '%Y/%m/%d-%H:%M:%S'), 275 axis=1) 276 277 return timestamp_df, sv_info_df, sv_stat_df, loc_info_df 278 279 280def parse_gpsapilog_to_df_v2(filename): 281 """Parse GPS API log to Pandas dataframes, by using merge_asof. 282 283 Args: 284 filename: full log file name. 285 Type, String. 286 287 Returns: 288 timestamp_df: Timestamp Data Frame. 289 Type, Pandas DataFrame. 290 sv_info_df: GNSS SV info Data Frame. 291 Type, Pandas DataFrame. 292 loc_info_df: Location Information Data Frame. 293 Type, Pandas DataFrame. 294 include Provider, Latitude, Longitude, Altitude, GNSSTime, Speed, Bearing 295 """ 296 # Get parsed dataframe list 297 parsed_data = parse_log_to_df( 298 filename=filename, 299 configs=CONFIG_GPSAPILOG, 300 ) 301 302 # get DUT Timestamp 303 timestamp_df = parsed_data['phone_time'] 304 timestamp_df['phone_time'] = timestamp_df.apply( 305 lambda row: datetime.datetime.strptime(row.date + '-' + row.time, 306 '%Y/%m/%d-%H:%M:%S'), 307 axis=1) 308 # drop logsize, date, time 309 parsed_data['phone_time'] = timestamp_df.drop(['logsize', 'date', 'time'], 310 axis=1) 311 312 # Add phone_time from timestamp dataframe by row number 313 for key in parsed_data: 314 if key != 'phone_time': 315 parsed_data[key] = pds.merge_asof(parsed_data[key], 316 parsed_data['phone_time'], 317 left_index=True, 318 right_index=True) 319 320 # Get space vehicle info dataframe 321 sv_info_df = parsed_data['SpaceVehicle'] 322 323 # Get space vehicle statistics dataframe 324 # First merge all dataframe from LIST_SVSTAT[1:], 325 sv_stat_df = fts.reduce( 326 lambda item1, item2: pds.merge(item1, item2, on='phone_time'), 327 [parsed_data[key] for key in LIST_SVSTAT[1:]]) 328 # Then merge with LIST_SVSTAT[0] 329 sv_stat_df = pds.merge(sv_stat_df, 330 parsed_data[LIST_SVSTAT[0]], 331 on='phone_time') 332 333 # Get location fix information dataframe 334 # First merge all dataframe from LIST_LOCINFO[1:], 335 loc_info_df = fts.reduce( 336 lambda item1, item2: pds.merge(item1, item2, on='phone_time'), 337 [parsed_data[key] for key in LIST_LOCINFO[1:]]) 338 # Then merge with LIST_LOCINFO[8] 339 loc_info_df = pds.merge(loc_info_df, 340 parsed_data[LIST_LOCINFO[0]], 341 on='phone_time') 342 # Convert GNSS Time 343 loc_info_df['gnsstime'] = loc_info_df.apply( 344 lambda row: datetime.datetime.strptime(row.Date + '-' + row.Time, 345 '%Y/%m/%d-%H:%M:%S'), 346 axis=1) 347 348 return timestamp_df, sv_info_df, sv_stat_df, loc_info_df 349