1# Copyright (C) 2022 The Android Open Source Project 2# 3# Licensed under the Apache License, Version 2.0 (the "License"); 4# you may not use this file except in compliance with the License. 5# You may obtain a copy of the License at 6# 7# http://www.apache.org/licenses/LICENSE-2.0 8# 9# Unless required by applicable law or agreed to in writing, software 10# distributed under the License is distributed on an "AS IS" BASIS, 11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12# See the License for the specific language governing permissions and 13# limitations under the License. 14 15from enum import Enum 16import re 17import os 18from typing import Dict, List 19 20NAME = r'[a-zA-Z_\d\{\}]+' 21ANY_WORDS = r'[^\s].*' 22ANY_NON_QUOTE = r'[^\']*.*' 23TYPE = r'[a-zA-Z]+' 24SQL = r'[\s\S]*?' 25WS = r'\s*' 26COMMENT = r' --[^\n]*\n' 27COMMENTS = rf'(?:{COMMENT})*' 28ARG = rf'{COMMENTS} {NAME} {TYPE}' 29ARG_PATTERN = rf'({COMMENTS}) ({NAME}) ({TYPE})' 30ARGS = rf'(?:{ARG})?(?: ,{ARG})*' 31 32 33# Make the pattern more readable by allowing the use of spaces 34# and replace then with a wildcard in a separate step. 35# NOTE: two whitespaces next to each other are really bad for performance. 36# Take special care to avoid them. 37def update_pattern(pattern): 38 return pattern.replace(' ', WS) 39 40 41CREATE_TABLE_VIEW_PATTERN = update_pattern( 42 # Match create table/view and catch type 43 fr'^CREATE (OR REPLACE)? (VIRTUAL|PERFETTO)?' 44 fr' (TABLE|VIEW) (?:IF NOT EXISTS)?' 45 # Catch the name and optional schema. 46 fr' ({NAME}) (?: \( ({ARGS}) \) )? (?:AS|USING)? .*') 47 48CREATE_TABLE_AS_PATTERN = update_pattern(fr'^CREATE TABLE ({NAME}) AS') 49 50CREATE_VIEW_AS_PATTERN = update_pattern(fr'^CREATE VIEW ({NAME}) AS') 51 52DROP_TABLE_VIEW_PATTERN = update_pattern(fr'^DROP (TABLE|VIEW) IF EXISTS ' 53 fr'({NAME});$') 54 55INCLUDE_ALL_PATTERN = update_pattern( 56 fr'^INCLUDE PERFETTO MODULE [a-zA-Z0-9_\.]*\*;') 57 58CREATE_FUNCTION_PATTERN = update_pattern( 59 # Function name. 60 fr"CREATE (OR REPLACE)? PERFETTO FUNCTION ({NAME}) " 61 # Args: anything in the brackets. 62 fr" \( ({ARGS}) \)" 63 # Type: word after RETURNS. 64 fr"({COMMENTS})" 65 fr" RETURNS ({TYPE}) AS ") 66 67CREATE_TABLE_FUNCTION_PATTERN = update_pattern( 68 fr"CREATE (OR REPLACE)? PERFETTO FUNCTION ({NAME}) " 69 # Args: anything in the brackets. 70 fr" \( ({ARGS}) \) " 71 # Type: table definition after RETURNS. 72 fr"({COMMENTS})" 73 fr" RETURNS TABLE \( ({ARGS}) \) AS ") 74 75CREATE_MACRO_PATTERN = update_pattern( 76 fr"CREATE (OR REPLACE)? PERFETTO MACRO ({NAME}) " 77 # Args: anything in the brackets. 78 fr" \( ({ARGS}) \) " 79 # Type: word after RETURNS. 80 fr"({COMMENTS})" 81 fr" RETURNS ({TYPE})") 82 83COLUMN_ANNOTATION_PATTERN = update_pattern(fr'^ ({NAME}) ({ANY_WORDS})') 84 85NAME_AND_TYPE_PATTERN = update_pattern(fr' ({NAME})\s+({TYPE}) ') 86 87ARG_ANNOTATION_PATTERN = fr'\s*{NAME_AND_TYPE_PATTERN}\s+({ANY_WORDS})' 88 89ARG_DEFINITION_PATTERN = update_pattern(ARG_PATTERN) 90 91FUNCTION_RETURN_PATTERN = update_pattern(fr'^ ({TYPE})\s+({ANY_WORDS})') 92 93ANY_PATTERN = r'(?:\s|.)*' 94 95 96class ObjKind(str, Enum): 97 table_view = 'table_view' 98 function = 'function' 99 table_function = 'table_function' 100 macro = 'macro' 101 102 103PATTERN_BY_KIND = { 104 ObjKind.table_view: CREATE_TABLE_VIEW_PATTERN, 105 ObjKind.function: CREATE_FUNCTION_PATTERN, 106 ObjKind.table_function: CREATE_TABLE_FUNCTION_PATTERN, 107 ObjKind.macro: CREATE_MACRO_PATTERN 108} 109 110ALLOWED_PREFIXES = { 111 'counters': 'counter', 112 'chrome/util': 'cr', 113 'intervals': 'interval', 114 'graphs': 'graph', 115 'slices': 'slice', 116} 117 118# Allows for nonstandard object names. 119OBJECT_NAME_ALLOWLIST = { 120 'graphs/partition.sql': ['tree_structural_partition_by_group'], 121 'slices/with_context.sql': ['process_slice', 'thread_slice'], 122 'slices/cpu_time.sql': ['thread_slice_cpu_time'] 123} 124 125 126# Given a regex pattern and a string to match against, returns all the 127# matching positions. Specifically, it returns a dictionary from the line 128# number of the match to the regex match object. 129# Note: this resuts a dict[int, re.Match], but re.Match exists only in later 130# versions of python3, prior to that it was _sre.SRE_Match. 131def match_pattern(pattern: str, file_str: str) -> Dict[int, object]: 132 line_number_to_matches = {} 133 for match in re.finditer(pattern, file_str, re.MULTILINE): 134 line_id = file_str[:match.start()].count('\n') 135 line_number_to_matches[line_id] = match.groups() 136 return line_number_to_matches 137 138 139# Given a list of lines in a text and the line number, scans backwards to find 140# all the comments. 141def extract_comment(lines: List[str], line_number: int) -> List[str]: 142 comments = [] 143 for line in lines[line_number - 1::-1]: 144 # Break on empty line, as that suggests it is no longer a part of 145 # this comment. 146 if not line or not line.startswith('--'): 147 break 148 comments.append(line) 149 150 # Reverse as the above was reversed 151 comments.reverse() 152 return comments 153 154 155# Given SQL string check whether any of the words is used, and create error 156# string if needed. 157def check_banned_words(sql: str, path: str) -> List[str]: 158 lines = [l.strip() for l in sql.split('\n')] 159 errors = [] 160 161 # Ban the use of LIKE in non-comment lines. 162 for line in lines: 163 if line.startswith('--'): 164 continue 165 166 if 'like' in line.casefold(): 167 errors.append( 168 'LIKE is banned in trace processor metrics. Prefer GLOB instead.\n' 169 f'Offending file: {path}\n') 170 continue 171 172 if 'create_function' in line.casefold(): 173 errors.append('CREATE_FUNCTION is deprecated in trace processor. ' 174 'Use CREATE PERFETTO FUNCTION instead.\n' 175 f'Offending file: {path}') 176 177 if 'create_view_function' in line.casefold(): 178 errors.append( 179 'CREATE_VIEW_FUNCTION is deprecated in trace processor. ' 180 'Use CREATE PERFETTO FUNCTION $name RETURNS TABLE instead.\n' 181 f'Offending file: {path}') 182 183 if 'import(' in line.casefold(): 184 errors.append('SELECT IMPORT is deprecated in trace processor. ' 185 'Use INCLUDE PERFETTO MODULE instead.\n' 186 f'Offending file: {path}') 187 188 return errors 189 190 191# Given SQL string check whether there is (not allowlisted) usage of 192# CREATE TABLE {name} AS. 193def check_banned_create_table_as(sql: str, filename: str, stdlib_path: str, 194 allowlist: Dict[str, List[str]]) -> List[str]: 195 errors = [] 196 for _, matches in match_pattern(CREATE_TABLE_AS_PATTERN, sql).items(): 197 name = matches[0] 198 # Normalize paths before checking presence in the allowlist so it will 199 # work on Windows for the Chrome stdlib presubmit. 200 allowlist_normpath = dict( 201 (os.path.normpath(path), tables) for path, tables in allowlist.items()) 202 allowlist_key = os.path.normpath(filename[len(stdlib_path):]) 203 if allowlist_key not in allowlist_normpath: 204 errors.append(f"CREATE TABLE '{name}' is deprecated. " 205 "Use CREATE PERFETTO TABLE instead.\n" 206 f"Offending file: {filename}\n") 207 continue 208 if name not in allowlist_normpath[allowlist_key]: 209 errors.append( 210 f"Table '{name}' uses CREATE TABLE which is deprecated " 211 "and this table is not allowlisted. Use CREATE PERFETTO TABLE.\n" 212 f"Offending file: {filename}\n") 213 return errors 214 215 216# Given SQL string check whether there is usage of CREATE VIEW {name} AS. 217def check_banned_create_view_as(sql: str, filename: str) -> List[str]: 218 errors = [] 219 for _, matches in match_pattern(CREATE_VIEW_AS_PATTERN, sql).items(): 220 name = matches[0] 221 errors.append(f"CREATE VIEW '{name}' is deprecated. " 222 "Use CREATE PERFETTO VIEW instead.\n" 223 f"Offending file: {filename}\n") 224 return errors 225 226 227# Given SQL string check whether there is usage of CREATE VIEW {name} AS. 228def check_banned_include_all(sql: str, filename: str) -> List[str]: 229 errors = [] 230 for _, matches in match_pattern(INCLUDE_ALL_PATTERN, sql).items(): 231 errors.append( 232 f"INCLUDE PERFETTO MODULE with wildcards is not allowed in stdlib. " 233 f"Import specific modules instead. Offending file: {filename}") 234 return errors 235