1#!/usr/bin/env python3 2# Copyright (C) 2022 The Android Open Source Project 3# 4# Licensed under the Apache License, Version 2.0 (the "License"); 5# you may not use this file except in compliance with the License. 6# You may obtain a copy of the License at 7# 8# http://www.apache.org/licenses/LICENSE-2.0 9# 10# Unless required by applicable law or agreed to in writing, software 11# disibuted under the License is disibuted on an "AS IS" BASIS, 12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13# See the License for the specific language governing permissions and 14# limitations under the License. 15 16from __future__ import absolute_import 17from __future__ import division 18from __future__ import print_function 19 20import argparse 21import sys 22import json 23from typing import Any, List, Dict 24 25INTRODUCTION = ''' 26# PerfettoSQL standard library 27*This page documents the PerfettoSQL standard library.* 28 29## Introduction 30The PerfettoSQL standard library is a repository of tables, views, functions 31and macros, contributed by domain experts, which make querying traces easier 32Its design is heavily inspired by standard libraries in languages like Python, 33C++ and Java. 34 35Some of the purposes of the standard library include: 361) Acting as a way of sharing and commonly written queries without needing 37to copy/paste large amounts of SQL. 382) Raising the abstraction level when exposing data in the trace. Many 39modules in the standard library convert low-level trace concepts 40e.g. slices, tracks and into concepts developers may be more familar with 41e.g. for Android developers: app startups, binder transactions etc. 42 43Standard library modules can be included as follows: 44``` 45-- Include all tables/views/functions from the android.startup.startups 46-- module in the standard library. 47INCLUDE PERFETTO MODULE android.startup.startups; 48 49-- Use the android_startups table defined in the android.startup.startups 50-- module. 51SELECT * 52FROM android_startups; 53``` 54 55Prelude is a special module is automatically imported. It contains key helper 56tables, views and functions which are universally useful. 57 58More information on importing modules is available in the 59[syntax documentation](/docs/analysis/perfetto-sql-syntax#including-perfettosql-modules) 60for the `INCLUDE PERFETTO MODULE` statement. 61 62<!-- TODO(b/290185551): talk about experimental module and contributions. --> 63 64## Summary 65''' 66 67 68def _escape_in_table(desc: str): 69 """Escapes special characters in a markdown table.""" 70 return desc.replace('|', '\\|') 71 72 73def _md_table(cols: List[str]): 74 col_str = ' | '.join(cols) + '\n' 75 lines = ['-' * len(col) for col in cols] 76 underlines = ' | '.join(lines) 77 return col_str + underlines 78 79 80def _write_summary(sql_type: str, table_cols: List[str], 81 summary_objs: List[str]) -> str: 82 table_data = '\n'.join(s.strip() for s in summary_objs if s) 83 return f""" 84### {sql_type} 85 86{_md_table(table_cols)} 87{table_data} 88 89""" 90 91 92class FileMd: 93 """Responsible for file level markdown generation.""" 94 95 def __init__(self, module_name, file_dict): 96 self.import_key = file_dict['import_key'] 97 import_key_name = self.import_key if module_name != 'prelude' else 'N/A' 98 self.objs, self.funs, self.view_funs, self.macros = [], [], [], [] 99 summary_objs_list, summary_funs_list, summary_view_funs_list, summary_macros_list = [], [], [], [] 100 101 # Add imports if in file. 102 for data in file_dict['imports']: 103 # Anchor 104 anchor = f'''obj/{module_name}/{data['name']}''' 105 106 # Add summary of imported view/table 107 summary_objs_list.append(f'''[{data['name']}](#{anchor})|''' 108 f'''{import_key_name}|''' 109 f'''{_escape_in_table(data['summary_desc'])}''') 110 111 self.objs.append(f'''\n\n<a name="{anchor}"></a>''' 112 f'''**{data['name']}**, {data['type']}\n\n''' 113 f'''{_escape_in_table(data['desc'])}\n''') 114 115 self.objs.append(_md_table(['Column', 'Type', 'Description'])) 116 for name, info in data['cols'].items(): 117 self.objs.append( 118 f'{name} | {info["type"]} | {_escape_in_table(info["desc"])}') 119 120 self.objs.append('\n\n') 121 122 # Add functions if in file 123 for data in file_dict['functions']: 124 # Anchor 125 anchor = f'''fun/{module_name}/{data['name']}''' 126 127 # Add summary of imported function 128 summary_funs_list.append(f'''[{data['name']}](#{anchor})|''' 129 f'''{import_key_name}|''' 130 f'''{data['return_type']}|''' 131 f'''{_escape_in_table(data['summary_desc'])}''') 132 self.funs.append( 133 f'''\n\n<a name="{anchor}"></a>''' 134 f'''**{data['name']}**\n\n''' 135 f'''{data['desc']}\n\n''' 136 f'''Returns: {data['return_type']}, {data['return_desc']}\n\n''') 137 if data['args']: 138 self.funs.append(_md_table(['Argument', 'Type', 'Description'])) 139 for name, arg_dict in data['args'].items(): 140 self.funs.append( 141 f'''{name} | {arg_dict['type']} | {_escape_in_table(arg_dict['desc'])}''' 142 ) 143 144 self.funs.append('\n\n') 145 146 # Add table functions if in file 147 for data in file_dict['table_functions']: 148 # Anchor 149 anchor = rf'''view_fun/{module_name}/{data['name']}''' 150 # Add summary of imported view function 151 summary_view_funs_list.append( 152 f'''[{data['name']}](#{anchor})|''' 153 f'''{import_key_name}|''' 154 f'''{_escape_in_table(data['summary_desc'])}''') 155 156 self.view_funs.append(f'''\n\n<a name="{anchor}"></a>''' 157 f'''**{data['name']}**\n''' 158 f'''{data['desc']}\n\n''') 159 if data['args']: 160 self.funs.append(_md_table(['Argument', 'Type', 'Description'])) 161 for name, arg_dict in data['args'].items(): 162 self.view_funs.append( 163 f'''{name} | {arg_dict['type']} | {_escape_in_table(arg_dict['desc'])}''' 164 ) 165 self.view_funs.append('\n') 166 self.view_funs.append(_md_table(['Column', 'Type', 'Description'])) 167 for name, column in data['cols'].items(): 168 self.view_funs.append(f'{name} | {column["type"]} | {column["desc"]}') 169 170 self.view_funs.append('\n\n') 171 172 # Add macros if in file 173 for data in file_dict['macros']: 174 # Anchor 175 anchor = rf'''macro/{module_name}/{data['name']}''' 176 # Add summary of imported view function 177 summary_macros_list.append( 178 f'''[{data['name']}](#{anchor})|''' 179 f'''{import_key_name}|''' 180 f'''{_escape_in_table(data['summary_desc'])}''') 181 182 self.macros.append( 183 f'''\n\n<a name="{anchor}"></a>''' 184 f'''**{data['name']}**\n''' 185 f'''{data['desc']}\n\n''' 186 f'''Returns: {data['return_type']}, {data['return_desc']}\n\n''') 187 if data['args']: 188 self.macros.append(_md_table(['Argument', 'Type', 'Description'])) 189 for name, arg_dict in data['args'].items(): 190 self.macros.append( 191 f'''{name} | {arg_dict['type']} | {_escape_in_table(arg_dict['desc'])}''' 192 ) 193 self.macros.append('\n') 194 self.macros.append('\n\n') 195 196 self.summary_objs = '\n'.join(summary_objs_list) 197 self.summary_funs = '\n'.join(summary_funs_list) 198 self.summary_view_funs = '\n'.join(summary_view_funs_list) 199 self.summary_macros = '\n'.join(summary_macros_list) 200 201 202class ModuleMd: 203 """Responsible for module level markdown generation.""" 204 205 def __init__(self, module_name: str, module_files: List[Dict[str, 206 Any]]) -> None: 207 self.module_name = module_name 208 self.files_md = sorted( 209 [FileMd(module_name, file_dict) for file_dict in module_files], 210 key=lambda x: x.import_key) 211 self.summary_objs = '\n'.join( 212 file.summary_objs for file in self.files_md if file.summary_objs) 213 self.summary_funs = '\n'.join( 214 file.summary_funs for file in self.files_md if file.summary_funs) 215 self.summary_view_funs = '\n'.join(file.summary_view_funs 216 for file in self.files_md 217 if file.summary_view_funs) 218 self.summary_macros = '\n'.join( 219 file.summary_macros for file in self.files_md if file.summary_macros) 220 221 def get_prelude_description(self) -> str: 222 if not self.module_name == 'prelude': 223 raise ValueError("Only callable on prelude module") 224 225 lines = [] 226 lines.append(f'## Module: {self.module_name}') 227 228 # Prelude is a special module which is automatically imported and doesn't 229 # have any include keys. 230 objs = '\n'.join(obj for file in self.files_md for obj in file.objs) 231 if objs: 232 lines.append('#### Views/Tables') 233 lines.append(objs) 234 235 funs = '\n'.join(fun for file in self.files_md for fun in file.funs) 236 if funs: 237 lines.append('#### Functions') 238 lines.append(funs) 239 240 table_funs = '\n'.join( 241 view_fun for file in self.files_md for view_fun in file.view_funs) 242 if table_funs: 243 lines.append('#### Table Functions') 244 lines.append(table_funs) 245 246 macros = '\n'.join(macro for file in self.files_md for macro in file.macros) 247 if macros: 248 lines.append('#### Macros') 249 lines.append(macros) 250 251 return '\n'.join(lines) 252 253 def get_description(self) -> str: 254 if not self.files_md: 255 return '' 256 257 if self.module_name == 'prelude': 258 raise ValueError("Can't be called with prelude module") 259 260 lines = [] 261 lines.append(f'## Module: {self.module_name}') 262 263 for file in self.files_md: 264 if not any((file.objs, file.funs, file.view_funs, file.macros)): 265 continue 266 267 lines.append(f'### {file.import_key}') 268 if file.objs: 269 lines.append('#### Views/Tables') 270 lines.append('\n'.join(file.objs)) 271 if file.funs: 272 lines.append('#### Functions') 273 lines.append('\n'.join(file.funs)) 274 if file.view_funs: 275 lines.append('#### Table Functions') 276 lines.append('\n'.join(file.view_funs)) 277 if file.macros: 278 lines.append('#### Macros') 279 lines.append('\n'.join(file.macros)) 280 281 return '\n'.join(lines) 282 283 284def main(): 285 parser = argparse.ArgumentParser() 286 parser.add_argument('--input', required=True) 287 parser.add_argument('--output', required=True) 288 args = parser.parse_args() 289 290 with open(args.input) as f: 291 modules_json_dict = json.load(f) 292 293 # Fetch the modules from json documentation. 294 modules_dict: Dict[str, ModuleMd] = {} 295 for module_name, module_files in modules_json_dict.items(): 296 # Remove 'common' when it has been removed from the code. 297 if module_name not in ['deprecated', 'common']: 298 modules_dict[module_name] = ModuleMd(module_name, module_files) 299 300 prelude_module = modules_dict.pop('prelude') 301 302 with open(args.output, 'w') as f: 303 f.write(INTRODUCTION) 304 305 summary_objs = [prelude_module.summary_objs 306 ] if prelude_module.summary_objs else [] 307 summary_objs += [ 308 module.summary_objs 309 for module in modules_dict.values() 310 if (module.summary_objs) 311 ] 312 313 summary_funs = [prelude_module.summary_funs 314 ] if prelude_module.summary_funs else [] 315 summary_funs += [module.summary_funs for module in modules_dict.values()] 316 summary_view_funs = [prelude_module.summary_view_funs 317 ] if prelude_module.summary_view_funs else [] 318 summary_view_funs += [ 319 module.summary_view_funs for module in modules_dict.values() 320 ] 321 summary_macros = [prelude_module.summary_macros 322 ] if prelude_module.summary_macros else [] 323 summary_macros += [ 324 module.summary_macros for module in modules_dict.values() 325 ] 326 327 if summary_objs: 328 f.write( 329 _write_summary('Views/tables', ['Name', 'Import', 'Description'], 330 summary_objs)) 331 332 if summary_funs: 333 f.write( 334 _write_summary('Functions', 335 ['Name', 'Import', 'Return type', 'Description'], 336 summary_funs)) 337 338 if summary_view_funs: 339 f.write( 340 _write_summary('Table functions', ['Name', 'Import', 'Description'], 341 summary_view_funs)) 342 343 if summary_macros: 344 f.write( 345 _write_summary('Macros', ['Name', 'Import', 'Description'], 346 summary_macros)) 347 348 f.write('\n\n') 349 f.write(prelude_module.get_prelude_description()) 350 f.write('\n') 351 f.write('\n'.join( 352 module.get_description() for module in modules_dict.values())) 353 354 return 0 355 356 357if __name__ == '__main__': 358 sys.exit(main()) 359