1# Copyright (C) 2022 The Android Open Source Project 2# 3# Licensed under the Apache License, Version 2.0 (the "License"); 4# you may not use this file except in compliance with the License. 5# You may obtain a copy of the License at 6# 7# http://www.apache.org/licenses/LICENSE-2.0 8# 9# Unless required by applicable law or agreed to in writing, software 10# distributed under the License is distributed on an "AS IS" BASIS, 11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12# See the License for the specific language governing permissions and 13# limitations under the License. 14 15import dataclasses 16from dataclasses import dataclass 17import importlib 18import sys 19from typing import Dict 20from typing import List 21from typing import Set 22from typing import Optional 23from typing import Union 24 25from python.generators.trace_processor_table.public import Alias 26from python.generators.trace_processor_table.public import Column 27from python.generators.trace_processor_table.public import ColumnDoc 28from python.generators.trace_processor_table.public import ColumnFlag 29from python.generators.trace_processor_table.public import CppColumnType 30from python.generators.trace_processor_table.public import CppDouble 31from python.generators.trace_processor_table.public import CppInt32 32from python.generators.trace_processor_table.public import CppInt64 33from python.generators.trace_processor_table.public import CppOptional 34from python.generators.trace_processor_table.public import CppSelfTableId 35from python.generators.trace_processor_table.public import CppString 36from python.generators.trace_processor_table.public import CppTableId 37from python.generators.trace_processor_table.public import CppUint32 38from python.generators.trace_processor_table.public import Table 39 40 41@dataclass 42class ParsedType: 43 """Result of parsing a CppColumnType into its parts.""" 44 cpp_type: str 45 is_optional: bool = False 46 is_alias: bool = False 47 alias_underlying_name: Optional[str] = None 48 is_self_id: bool = False 49 id_table: Optional[Table] = None 50 51 def cpp_type_with_optionality(self) -> str: 52 """Returns the C++ type wrapping with base::Optional if necessary.""" 53 54 # ThreadTable and ProcessTable are special for legacy reasons as they were 55 # around even before the advent of C++ macro tables. Because of this a lot 56 # of code was written assuming that upid and utid were uint32 (e.g. indexing 57 # directly into vectors using them) and it was decided this behaviour was 58 # too expensive in engineering cost to fix given the trivial benefit. For 59 # this reason, continue to maintain this illusion. 60 if self.id_table and self.id_table.class_name in ('ThreadTable', 61 'ProcessTable'): 62 cpp_type = 'uint32_t' 63 else: 64 cpp_type = self.cpp_type 65 if self.is_optional: 66 return f'std::optional<{cpp_type}>' 67 return cpp_type 68 69 70@dataclass(frozen=True) 71class ParsedColumn: 72 """Representation of a column parsed from a Python definition.""" 73 74 column: Column 75 doc: Optional[ColumnDoc] 76 77 # Whether this column is the implicit "id" column which is added by while 78 # parsing the tables rather than by the user. 79 is_implicit_id: bool = False 80 81 # Whether this column comes from copying a column from the ancestor. If this 82 # is set to false, the user explicitly specified it for this table. 83 is_ancestor: bool = False 84 85 86@dataclass(frozen=True) 87class ParsedTable: 88 """Representation of a table parsed from a Python definition.""" 89 90 table: Table 91 columns: List[ParsedColumn] 92 93 94def parse_type_with_cols(table: Table, cols: List[Column], 95 col_type: CppColumnType) -> ParsedType: 96 """Parses a CppColumnType into its constiuent parts.""" 97 98 if isinstance(col_type, CppInt64): 99 return ParsedType('int64_t') 100 if isinstance(col_type, CppInt32): 101 return ParsedType('int32_t') 102 if isinstance(col_type, CppUint32): 103 return ParsedType('uint32_t') 104 if isinstance(col_type, CppDouble): 105 return ParsedType('double') 106 if isinstance(col_type, CppString): 107 return ParsedType('StringPool::Id') 108 109 if isinstance(col_type, Alias): 110 col = next(c for c in cols if c.name == col_type.underlying_column) 111 return ParsedType( 112 parse_type(table, col.type).cpp_type, 113 is_alias=True, 114 alias_underlying_name=col.name) 115 116 if isinstance(col_type, CppTableId): 117 return ParsedType( 118 f'{col_type.table.class_name}::Id', id_table=col_type.table) 119 120 if isinstance(col_type, CppSelfTableId): 121 return ParsedType( 122 f'{table.class_name}::Id', is_self_id=True, id_table=table) 123 124 if isinstance(col_type, CppOptional): 125 inner = parse_type(table, col_type.inner) 126 assert not inner.is_optional, 'Nested optional not allowed' 127 return dataclasses.replace(inner, is_optional=True) 128 129 raise Exception(f'Unknown type {col_type}') 130 131 132def parse_type(table: Table, col_type: CppColumnType) -> ParsedType: 133 """Parses a CppColumnType into its constiuent parts.""" 134 return parse_type_with_cols(table, table.columns, col_type) 135 136 137def typed_column_type(table: Table, col: ParsedColumn) -> str: 138 """Returns the TypedColumn/IdColumn C++ type for a given column.""" 139 140 parsed = parse_type(table, col.column.type) 141 if col.is_implicit_id: 142 return f'IdColumn<{parsed.cpp_type}>' 143 return f'TypedColumn<{parsed.cpp_type_with_optionality()}>' 144 145 146def data_layer_type(table: Table, col: ParsedColumn) -> str: 147 """Returns the DataLayer C++ type for a given column.""" 148 149 parsed = parse_type(table, col.column.type) 150 if col.is_implicit_id: 151 return 'column::IdStorage' 152 if parsed.cpp_type == 'StringPool::Id': 153 return 'column::StringStorage' 154 if ColumnFlag.SET_ID in col.column.flags: 155 return 'column::SetIdStorage' 156 return f'column::NumericStorage<ColumnType::{col.column.name}::non_optional_stored_type>' 157 158 159def find_table_deps(table: Table) -> List[Table]: 160 """Finds all the other table class names this table depends on. 161 162 By "depends", we mean this table in C++ would need the dependency to be 163 defined (or included) before this table is defined.""" 164 165 deps: Dict[str, Table] = {} 166 if table.parent: 167 deps[table.parent.class_name] = table.parent 168 for c in table.columns: 169 # Aliases cannot have dependencies so simply ignore them: trying to parse 170 # them before adding implicit columns can cause issues. 171 if isinstance(c.type, Alias): 172 continue 173 id_table = parse_type(table, c.type).id_table 174 if id_table: 175 deps[id_table.class_name] = id_table 176 return list(deps.values()) 177 178 179def public_sql_name(table: Table) -> str: 180 """Extracts SQL name for the table which should be publicised.""" 181 182 wrapping_view = table.wrapping_sql_view 183 return wrapping_view.view_name if wrapping_view else table.sql_name 184 185 186def _create_implicit_columns_for_root(table: Table) -> List[ParsedColumn]: 187 """Given a root table, returns the implicit id and type columns.""" 188 assert table.parent is None 189 190 sql_name = public_sql_name(table) 191 id_doc = table.tabledoc.columns.get('id') if table.tabledoc else None 192 type_doc = table.tabledoc.columns.get('type') if table.tabledoc else None 193 return [ 194 ParsedColumn( 195 Column('id', CppSelfTableId(), ColumnFlag.SORTED), 196 _to_column_doc(id_doc) if id_doc else ColumnDoc( 197 doc=f'Unique identifier for this {sql_name}.'), 198 is_implicit_id=True), 199 ] 200 201 202def _topological_sort_table_and_deps(parsed: List[Table]) -> List[Table]: 203 """Topologically sorts a list of tables (i.e. dependenices appear earlier). 204 205 See [1] for information on a topological sort. We do this to allow 206 dependencies to be processed and appear ealier than their dependents. 207 208 [1] https://en.wikipedia.org/wiki/Topological_sorting""" 209 visited: Set[str] = set() 210 result: List[Table] = [] 211 212 # Topological sorting is really just a DFS where we put the nodes in the list 213 # after any dependencies. 214 def dfs(t: Table): 215 if t.class_name in visited: 216 return 217 visited.add(t.class_name) 218 219 for dep in find_table_deps(t): 220 dfs(dep) 221 result.append(t) 222 223 for p in parsed: 224 dfs(p) 225 return result 226 227 228def _to_column_doc(doc: Union[ColumnDoc, str, None]) -> Optional[ColumnDoc]: 229 """Cooerces a user specified ColumnDoc or string into a ColumnDoc.""" 230 231 if doc is None or isinstance(doc, ColumnDoc): 232 return doc 233 return ColumnDoc(doc=doc) 234 235 236def parse_tables_from_modules(modules: List[str]) -> List[ParsedTable]: 237 """Creates a list of tables with the associated paths.""" 238 239 # Create a mapping from the table to a "parsed" version of the table. 240 tables: Dict[str, Table] = {} 241 for module in modules: 242 imported = importlib.import_module(module) 243 run_tables: List[Table] = imported.__dict__['ALL_TABLES'] 244 for table in run_tables: 245 existing_table = tables.get(table.class_name) 246 assert not existing_table or existing_table == table 247 tables[table.class_name] = table 248 249 # Sort all the tables: note that this list may include tables which are not 250 # in |tables| dictionary due to dependencies on tables which live in a file 251 # not covered by |input_paths|. 252 sorted_tables = _topological_sort_table_and_deps(list(tables.values())) 253 254 parsed_tables: Dict[str, ParsedTable] = {} 255 for table in sorted_tables: 256 parsed_columns: List[ParsedColumn] 257 if table.parent: 258 parsed_parent = parsed_tables[table.parent.class_name] 259 parsed_columns = [ 260 dataclasses.replace(c, is_ancestor=True) 261 for c in parsed_parent.columns 262 ] 263 else: 264 parsed_columns = _create_implicit_columns_for_root(table) 265 266 for c in table.columns: 267 doc = table.tabledoc.columns.get(c.name) if table.tabledoc else None 268 parsed_columns.append(ParsedColumn(c, _to_column_doc(doc))) 269 parsed_tables[table.class_name] = ParsedTable(table, parsed_columns) 270 271 # Only return tables which come directly from |input_paths|. This stops us 272 # generating tables which were not requested. 273 return [ 274 parsed_tables[p.class_name] 275 for p in sorted_tables 276 if p.class_name in tables 277 ] 278