# Copyright (C) 2022 The Android Open Source Project # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. from enum import Enum import re import os from typing import Dict, List NAME = r'[a-zA-Z_\d\{\}]+' ANY_WORDS = r'[^\s].*' ANY_NON_QUOTE = r'[^\']*.*' TYPE = r'[a-zA-Z]+' SQL = r'[\s\S]*?' WS = r'\s*' COMMENT = r' --[^\n]*\n' COMMENTS = rf'(?:{COMMENT})*' ARG = rf'{COMMENTS} {NAME} {TYPE}' ARG_PATTERN = rf'({COMMENTS}) ({NAME}) ({TYPE})' ARGS = rf'(?:{ARG})?(?: ,{ARG})*' # Make the pattern more readable by allowing the use of spaces # and replace then with a wildcard in a separate step. # NOTE: two whitespaces next to each other are really bad for performance. # Take special care to avoid them. def update_pattern(pattern): return pattern.replace(' ', WS) CREATE_TABLE_VIEW_PATTERN = update_pattern( # Match create table/view and catch type fr'^CREATE (OR REPLACE)? (VIRTUAL|PERFETTO)?' fr' (TABLE|VIEW) (?:IF NOT EXISTS)?' # Catch the name and optional schema. fr' ({NAME}) (?: \( ({ARGS}) \) )? (?:AS|USING)? .*') CREATE_TABLE_AS_PATTERN = update_pattern(fr'^CREATE TABLE ({NAME}) AS') CREATE_VIEW_AS_PATTERN = update_pattern(fr'^CREATE VIEW ({NAME}) AS') DROP_TABLE_VIEW_PATTERN = update_pattern(fr'^DROP (TABLE|VIEW) IF EXISTS ' fr'({NAME});$') INCLUDE_ALL_PATTERN = update_pattern( fr'^INCLUDE PERFETTO MODULE [a-zA-Z0-9_\.]*\*;') CREATE_FUNCTION_PATTERN = update_pattern( # Function name. fr"CREATE (OR REPLACE)? PERFETTO FUNCTION ({NAME}) " # Args: anything in the brackets. fr" \( ({ARGS}) \)" # Type: word after RETURNS. fr"({COMMENTS})" fr" RETURNS ({TYPE}) AS ") CREATE_TABLE_FUNCTION_PATTERN = update_pattern( fr"CREATE (OR REPLACE)? PERFETTO FUNCTION ({NAME}) " # Args: anything in the brackets. fr" \( ({ARGS}) \) " # Type: table definition after RETURNS. fr"({COMMENTS})" fr" RETURNS TABLE \( ({ARGS}) \) AS ") CREATE_MACRO_PATTERN = update_pattern( fr"CREATE (OR REPLACE)? PERFETTO MACRO ({NAME}) " # Args: anything in the brackets. fr" \( ({ARGS}) \) " # Type: word after RETURNS. fr"({COMMENTS})" fr" RETURNS ({TYPE})") COLUMN_ANNOTATION_PATTERN = update_pattern(fr'^ ({NAME}) ({ANY_WORDS})') NAME_AND_TYPE_PATTERN = update_pattern(fr' ({NAME})\s+({TYPE}) ') ARG_ANNOTATION_PATTERN = fr'\s*{NAME_AND_TYPE_PATTERN}\s+({ANY_WORDS})' ARG_DEFINITION_PATTERN = update_pattern(ARG_PATTERN) FUNCTION_RETURN_PATTERN = update_pattern(fr'^ ({TYPE})\s+({ANY_WORDS})') ANY_PATTERN = r'(?:\s|.)*' class ObjKind(str, Enum): table_view = 'table_view' function = 'function' table_function = 'table_function' macro = 'macro' PATTERN_BY_KIND = { ObjKind.table_view: CREATE_TABLE_VIEW_PATTERN, ObjKind.function: CREATE_FUNCTION_PATTERN, ObjKind.table_function: CREATE_TABLE_FUNCTION_PATTERN, ObjKind.macro: CREATE_MACRO_PATTERN } ALLOWED_PREFIXES = { 'counters': 'counter', 'chrome/util': 'cr', 'intervals': 'interval', 'graphs': 'graph', 'slices': 'slice', } # Allows for nonstandard object names. OBJECT_NAME_ALLOWLIST = { 'graphs/partition.sql': ['tree_structural_partition_by_group'], 'slices/with_context.sql': ['process_slice', 'thread_slice'], 'slices/cpu_time.sql': ['thread_slice_cpu_time'] } # Given a regex pattern and a string to match against, returns all the # matching positions. Specifically, it returns a dictionary from the line # number of the match to the regex match object. # Note: this resuts a dict[int, re.Match], but re.Match exists only in later # versions of python3, prior to that it was _sre.SRE_Match. def match_pattern(pattern: str, file_str: str) -> Dict[int, object]: line_number_to_matches = {} for match in re.finditer(pattern, file_str, re.MULTILINE): line_id = file_str[:match.start()].count('\n') line_number_to_matches[line_id] = match.groups() return line_number_to_matches # Given a list of lines in a text and the line number, scans backwards to find # all the comments. def extract_comment(lines: List[str], line_number: int) -> List[str]: comments = [] for line in lines[line_number - 1::-1]: # Break on empty line, as that suggests it is no longer a part of # this comment. if not line or not line.startswith('--'): break comments.append(line) # Reverse as the above was reversed comments.reverse() return comments # Given SQL string check whether any of the words is used, and create error # string if needed. def check_banned_words(sql: str, path: str) -> List[str]: lines = [l.strip() for l in sql.split('\n')] errors = [] # Ban the use of LIKE in non-comment lines. for line in lines: if line.startswith('--'): continue if 'like' in line.casefold(): errors.append( 'LIKE is banned in trace processor metrics. Prefer GLOB instead.\n' f'Offending file: {path}\n') continue if 'create_function' in line.casefold(): errors.append('CREATE_FUNCTION is deprecated in trace processor. ' 'Use CREATE PERFETTO FUNCTION instead.\n' f'Offending file: {path}') if 'create_view_function' in line.casefold(): errors.append( 'CREATE_VIEW_FUNCTION is deprecated in trace processor. ' 'Use CREATE PERFETTO FUNCTION $name RETURNS TABLE instead.\n' f'Offending file: {path}') if 'import(' in line.casefold(): errors.append('SELECT IMPORT is deprecated in trace processor. ' 'Use INCLUDE PERFETTO MODULE instead.\n' f'Offending file: {path}') return errors # Given SQL string check whether there is (not allowlisted) usage of # CREATE TABLE {name} AS. def check_banned_create_table_as(sql: str, filename: str, stdlib_path: str, allowlist: Dict[str, List[str]]) -> List[str]: errors = [] for _, matches in match_pattern(CREATE_TABLE_AS_PATTERN, sql).items(): name = matches[0] # Normalize paths before checking presence in the allowlist so it will # work on Windows for the Chrome stdlib presubmit. allowlist_normpath = dict( (os.path.normpath(path), tables) for path, tables in allowlist.items()) allowlist_key = os.path.normpath(filename[len(stdlib_path):]) if allowlist_key not in allowlist_normpath: errors.append(f"CREATE TABLE '{name}' is deprecated. " "Use CREATE PERFETTO TABLE instead.\n" f"Offending file: {filename}\n") continue if name not in allowlist_normpath[allowlist_key]: errors.append( f"Table '{name}' uses CREATE TABLE which is deprecated " "and this table is not allowlisted. Use CREATE PERFETTO TABLE.\n" f"Offending file: {filename}\n") return errors # Given SQL string check whether there is usage of CREATE VIEW {name} AS. def check_banned_create_view_as(sql: str, filename: str) -> List[str]: errors = [] for _, matches in match_pattern(CREATE_VIEW_AS_PATTERN, sql).items(): name = matches[0] errors.append(f"CREATE VIEW '{name}' is deprecated. " "Use CREATE PERFETTO VIEW instead.\n" f"Offending file: {filename}\n") return errors # Given SQL string check whether there is usage of CREATE VIEW {name} AS. def check_banned_include_all(sql: str, filename: str) -> List[str]: errors = [] for _, matches in match_pattern(INCLUDE_ALL_PATTERN, sql).items(): errors.append( f"INCLUDE PERFETTO MODULE with wildcards is not allowed in stdlib. " f"Import specific modules instead. Offending file: {filename}") return errors