1# Copyright (C) 2022 The Android Open Source Project 2# 3# Licensed under the Apache License, Version 2.0 (the "License"); 4# you may not use this file except in compliance with the License. 5# You may obtain a copy of the License at 6# 7# http://www.apache.org/licenses/LICENSE-2.0 8# 9# Unless required by applicable law or agreed to in writing, software 10# distributed under the License is distributed on an "AS IS" BASIS, 11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12# See the License for the specific language governing permissions and 13# limitations under the License. 14 15import dataclasses 16from dataclasses import dataclass 17import importlib 18import sys 19from typing import Dict 20from typing import List 21from typing import Set 22from typing import Optional 23from typing import Union 24 25from python.generators.trace_processor_table.public import Alias 26from python.generators.trace_processor_table.public import Column 27from python.generators.trace_processor_table.public import ColumnDoc 28from python.generators.trace_processor_table.public import ColumnFlag 29from python.generators.trace_processor_table.public import CppColumnType 30from python.generators.trace_processor_table.public import CppDouble 31from python.generators.trace_processor_table.public import CppInt32 32from python.generators.trace_processor_table.public import CppInt64 33from python.generators.trace_processor_table.public import CppOptional 34from python.generators.trace_processor_table.public import CppSelfTableId 35from python.generators.trace_processor_table.public import CppString 36from python.generators.trace_processor_table.public import CppTableId 37from python.generators.trace_processor_table.public import CppUint32 38from python.generators.trace_processor_table.public import Table 39 40 41@dataclass 42class ParsedType: 43 """Result of parsing a CppColumnType into its parts.""" 44 cpp_type: str 45 is_optional: bool = False 46 is_alias: bool = False 47 alias_underlying_name: Optional[str] = None 48 is_self_id: bool = False 49 id_table: Optional[Table] = None 50 51 def cpp_type_with_optionality(self) -> str: 52 """Returns the C++ type wrapping with base::Optional if necessary.""" 53 54 # ThreadTable and ProcessTable are special for legacy reasons as they were 55 # around even before the advent of C++ macro tables. Because of this a lot 56 # of code was written assuming that upid and utid were uint32 (e.g. indexing 57 # directly into vectors using them) and it was decided this behaviour was 58 # too expensive in engineering cost to fix given the trivial benefit. For 59 # this reason, continue to maintain this illusion. 60 if self.id_table and self.id_table.class_name in ('ThreadTable', 61 'ProcessTable'): 62 cpp_type = 'uint32_t' 63 else: 64 cpp_type = self.cpp_type 65 if self.is_optional: 66 return f'std::optional<{cpp_type}>' 67 return cpp_type 68 69 70@dataclass(frozen=True) 71class ParsedColumn: 72 """Representation of a column parsed from a Python definition.""" 73 74 column: Column 75 doc: Optional[ColumnDoc] 76 77 # Whether this column is the implicit "id" column which is added by while 78 # parsing the tables rather than by the user. 79 is_implicit_id: bool = False 80 81 # Whether this column is the implicit "type" column which is added by while 82 # parsing the tables rather than by the user. 83 is_implicit_type: bool = False 84 85 # Whether this column comes from copying a column from the ancestor. If this 86 # is set to false, the user explicitly specified it for this table. 87 is_ancestor: bool = False 88 89 90@dataclass(frozen=True) 91class ParsedTable: 92 """Representation of a table parsed from a Python definition.""" 93 94 table: Table 95 columns: List[ParsedColumn] 96 97 98def parse_type_with_cols(table: Table, cols: List[Column], 99 col_type: CppColumnType) -> ParsedType: 100 """Parses a CppColumnType into its constiuent parts.""" 101 102 if isinstance(col_type, CppInt64): 103 return ParsedType('int64_t') 104 if isinstance(col_type, CppInt32): 105 return ParsedType('int32_t') 106 if isinstance(col_type, CppUint32): 107 return ParsedType('uint32_t') 108 if isinstance(col_type, CppDouble): 109 return ParsedType('double') 110 if isinstance(col_type, CppString): 111 return ParsedType('StringPool::Id') 112 113 if isinstance(col_type, Alias): 114 col = next(c for c in cols if c.name == col_type.underlying_column) 115 return ParsedType( 116 parse_type(table, col.type).cpp_type, 117 is_alias=True, 118 alias_underlying_name=col.name) 119 120 if isinstance(col_type, CppTableId): 121 return ParsedType( 122 f'{col_type.table.class_name}::Id', id_table=col_type.table) 123 124 if isinstance(col_type, CppSelfTableId): 125 return ParsedType( 126 f'{table.class_name}::Id', is_self_id=True, id_table=table) 127 128 if isinstance(col_type, CppOptional): 129 inner = parse_type(table, col_type.inner) 130 assert not inner.is_optional, 'Nested optional not allowed' 131 return dataclasses.replace(inner, is_optional=True) 132 133 raise Exception(f'Unknown type {col_type}') 134 135 136def parse_type(table: Table, col_type: CppColumnType) -> ParsedType: 137 """Parses a CppColumnType into its constiuent parts.""" 138 return parse_type_with_cols(table, table.columns, col_type) 139 140 141def typed_column_type(table: Table, col: ParsedColumn) -> str: 142 """Returns the TypedColumn/IdColumn C++ type for a given column.""" 143 144 parsed = parse_type(table, col.column.type) 145 if col.is_implicit_id: 146 return f'IdColumn<{parsed.cpp_type}>' 147 return f'TypedColumn<{parsed.cpp_type_with_optionality()}>' 148 149 150def find_table_deps(table: Table) -> List[Table]: 151 """Finds all the other table class names this table depends on. 152 153 By "depends", we mean this table in C++ would need the dependency to be 154 defined (or included) before this table is defined.""" 155 156 deps: Dict[str, Table] = {} 157 if table.parent: 158 deps[table.parent.class_name] = table.parent 159 for c in table.columns: 160 # Aliases cannot have dependencies so simply ignore them: trying to parse 161 # them before adding implicit columns can cause issues. 162 if isinstance(c.type, Alias): 163 continue 164 id_table = parse_type(table, c.type).id_table 165 if id_table: 166 deps[id_table.class_name] = id_table 167 return list(deps.values()) 168 169 170def public_sql_name(table: Table) -> str: 171 """Extracts SQL name for the table which should be publicised.""" 172 173 wrapping_view = table.wrapping_sql_view 174 return wrapping_view.view_name if wrapping_view else table.sql_name 175 176 177def _create_implicit_columns_for_root(table: Table) -> List[ParsedColumn]: 178 """Given a root table, returns the implicit id and type columns.""" 179 assert table.parent is None 180 181 sql_name = public_sql_name(table) 182 id_doc = table.tabledoc.columns.get('id') if table.tabledoc else None 183 type_doc = table.tabledoc.columns.get('type') if table.tabledoc else None 184 return [ 185 ParsedColumn( 186 Column('id', CppSelfTableId(), ColumnFlag.SORTED), 187 _to_column_doc(id_doc) if id_doc else ColumnDoc( 188 doc=f'Unique idenitifier for this {sql_name}.'), 189 is_implicit_id=True), 190 ParsedColumn( 191 Column('type', CppString(), ColumnFlag.NONE), 192 _to_column_doc(type_doc) if type_doc else ColumnDoc(doc=''' 193 The name of the "most-specific" child table containing this 194 row. 195 '''), 196 is_implicit_type=True, 197 ) 198 ] 199 200 201def _topological_sort_table_and_deps(parsed: List[Table]) -> List[Table]: 202 """Topologically sorts a list of tables (i.e. dependenices appear earlier). 203 204 See [1] for information on a topological sort. We do this to allow 205 dependencies to be processed and appear ealier than their dependents. 206 207 [1] https://en.wikipedia.org/wiki/Topological_sorting""" 208 visited: Set[str] = set() 209 result: List[Table] = [] 210 211 # Topological sorting is really just a DFS where we put the nodes in the list 212 # after any dependencies. 213 def dfs(t: Table): 214 if t.class_name in visited: 215 return 216 visited.add(t.class_name) 217 218 for dep in find_table_deps(t): 219 dfs(dep) 220 result.append(t) 221 222 for p in parsed: 223 dfs(p) 224 return result 225 226 227def _to_column_doc(doc: Union[ColumnDoc, str, None]) -> Optional[ColumnDoc]: 228 """Cooerces a user specified ColumnDoc or string into a ColumnDoc.""" 229 230 if doc is None or isinstance(doc, ColumnDoc): 231 return doc 232 return ColumnDoc(doc=doc) 233 234 235def parse_tables_from_modules(modules: List[str]) -> List[ParsedTable]: 236 """Creates a list of tables with the associated paths.""" 237 238 # Create a mapping from the table to a "parsed" version of the table. 239 tables: Dict[str, Table] = {} 240 for module in modules: 241 imported = importlib.import_module(module) 242 run_tables: List[Table] = imported.__dict__['ALL_TABLES'] 243 for table in run_tables: 244 existing_table = tables.get(table.class_name) 245 assert not existing_table or existing_table == table 246 tables[table.class_name] = table 247 248 # Sort all the tables: note that this list may include tables which are not 249 # in |tables| dictionary due to dependencies on tables which live in a file 250 # not covered by |input_paths|. 251 sorted_tables = _topological_sort_table_and_deps(list(tables.values())) 252 253 parsed_tables: Dict[str, ParsedTable] = {} 254 for table in sorted_tables: 255 parsed_columns: List[ParsedColumn] 256 if table.parent: 257 parsed_parent = parsed_tables[table.parent.class_name] 258 parsed_columns = [ 259 dataclasses.replace(c, is_ancestor=True) 260 for c in parsed_parent.columns 261 ] 262 else: 263 parsed_columns = _create_implicit_columns_for_root(table) 264 265 for c in table.columns: 266 doc = table.tabledoc.columns.get(c.name) if table.tabledoc else None 267 parsed_columns.append(ParsedColumn(c, _to_column_doc(doc))) 268 parsed_tables[table.class_name] = ParsedTable(table, parsed_columns) 269 270 # Only return tables which come directly from |input_paths|. This stops us 271 # generating tables which were not requested. 272 return [ 273 parsed_tables[p.class_name] 274 for p in sorted_tables 275 if p.class_name in tables 276 ] 277