• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1# Copyright (C) 2022 The Android Open Source Project
2#
3# Licensed under the Apache License, Version 2.0 (the "License");
4# you may not use this file except in compliance with the License.
5# You may obtain a copy of the License at
6#
7#      http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS,
11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12# See the License for the specific language governing permissions and
13# limitations under the License.
14
15from enum import Enum
16import re
17import os
18from typing import Dict, List
19
20NAME = r'[a-zA-Z_\d\{\}]+'
21ANY_WORDS = r'[^\s].*'
22ANY_NON_QUOTE = r'[^\']*.*'
23TYPE = r'[a-zA-Z]+'
24SQL = r'[\s\S]*?'
25WS = r'\s*'
26COMMENT = r' --[^\n]*\n'
27COMMENTS = rf'(?:{COMMENT})*'
28ARG = rf'{COMMENTS} {NAME} {TYPE}'
29ARG_PATTERN = rf'({COMMENTS}) ({NAME}) ({TYPE})'
30ARGS = rf'(?:{ARG})?(?: ,{ARG})*'
31
32
33# Make the pattern more readable by allowing the use of spaces
34# and replace then with a wildcard in a separate step.
35# NOTE: two whitespaces next to each other are really bad for performance.
36# Take special care to avoid them.
37def update_pattern(pattern):
38  return pattern.replace(' ', WS)
39
40
41CREATE_TABLE_VIEW_PATTERN = update_pattern(
42    # Match create table/view and catch type
43    fr'^CREATE (OR REPLACE)? (VIRTUAL|PERFETTO)?'
44    fr' (TABLE|VIEW) (?:IF NOT EXISTS)?'
45    # Catch the name and optional schema.
46    fr' ({NAME}) (?: \( ({ARGS}) \) )? (?:AS|USING)? .*')
47
48CREATE_TABLE_AS_PATTERN = update_pattern(fr'^CREATE TABLE ({NAME}) AS')
49
50CREATE_VIEW_AS_PATTERN = update_pattern(fr'^CREATE VIEW ({NAME}) AS')
51
52DROP_TABLE_VIEW_PATTERN = update_pattern(fr'^DROP (TABLE|VIEW) IF EXISTS '
53                                         fr'({NAME});$')
54
55INCLUDE_ALL_PATTERN = update_pattern(
56    fr'^INCLUDE PERFETTO MODULE [a-zA-Z0-9_\.]*\*;')
57
58CREATE_FUNCTION_PATTERN = update_pattern(
59    # Function name.
60    fr"CREATE (OR REPLACE)? PERFETTO FUNCTION ({NAME}) "
61    # Args: anything in the brackets.
62    fr" \( ({ARGS}) \)"
63    # Type: word after RETURNS.
64    fr"({COMMENTS})"
65    fr" RETURNS ({TYPE}) AS ")
66
67CREATE_TABLE_FUNCTION_PATTERN = update_pattern(
68    fr"CREATE (OR REPLACE)? PERFETTO FUNCTION ({NAME}) "
69    # Args: anything in the brackets.
70    fr" \( ({ARGS}) \) "
71    # Type: table definition after RETURNS.
72    fr"({COMMENTS})"
73    fr" RETURNS TABLE \( ({ARGS}) \) AS ")
74
75CREATE_MACRO_PATTERN = update_pattern(
76    fr"CREATE (OR REPLACE)? PERFETTO MACRO ({NAME}) "
77    # Args: anything in the brackets.
78    fr" \( ({ARGS}) \) "
79    # Type: word after RETURNS.
80    fr"({COMMENTS})"
81    fr" RETURNS ({TYPE})")
82
83COLUMN_ANNOTATION_PATTERN = update_pattern(fr'^ ({NAME}) ({ANY_WORDS})')
84
85NAME_AND_TYPE_PATTERN = update_pattern(fr' ({NAME})\s+({TYPE}) ')
86
87ARG_ANNOTATION_PATTERN = fr'\s*{NAME_AND_TYPE_PATTERN}\s+({ANY_WORDS})'
88
89ARG_DEFINITION_PATTERN = update_pattern(ARG_PATTERN)
90
91FUNCTION_RETURN_PATTERN = update_pattern(fr'^ ({TYPE})\s+({ANY_WORDS})')
92
93ANY_PATTERN = r'(?:\s|.)*'
94
95
96class ObjKind(str, Enum):
97  table_view = 'table_view'
98  function = 'function'
99  table_function = 'table_function'
100  macro = 'macro'
101
102
103PATTERN_BY_KIND = {
104    ObjKind.table_view: CREATE_TABLE_VIEW_PATTERN,
105    ObjKind.function: CREATE_FUNCTION_PATTERN,
106    ObjKind.table_function: CREATE_TABLE_FUNCTION_PATTERN,
107    ObjKind.macro: CREATE_MACRO_PATTERN
108}
109
110ALLOWED_PREFIXES = {
111    'counters': 'counter',
112    'chrome/util': 'cr',
113    'intervals': 'interval',
114    'graphs': 'graph',
115    'slices': 'slice',
116}
117
118# Allows for nonstandard object names.
119OBJECT_NAME_ALLOWLIST = {
120    'graphs/partition.sql': ['tree_structural_partition_by_group'],
121    'slices/with_context.sql': ['process_slice', 'thread_slice'],
122    'slices/cpu_time.sql': ['thread_slice_cpu_time']
123}
124
125
126# Given a regex pattern and a string to match against, returns all the
127# matching positions. Specifically, it returns a dictionary from the line
128# number of the match to the regex match object.
129# Note: this resuts a dict[int, re.Match], but re.Match exists only in later
130# versions of python3, prior to that it was _sre.SRE_Match.
131def match_pattern(pattern: str, file_str: str) -> Dict[int, object]:
132  line_number_to_matches = {}
133  for match in re.finditer(pattern, file_str, re.MULTILINE):
134    line_id = file_str[:match.start()].count('\n')
135    line_number_to_matches[line_id] = match.groups()
136  return line_number_to_matches
137
138
139# Given a list of lines in a text and the line number, scans backwards to find
140# all the comments.
141def extract_comment(lines: List[str], line_number: int) -> List[str]:
142  comments = []
143  for line in lines[line_number - 1::-1]:
144    # Break on empty line, as that suggests it is no longer a part of
145    # this comment.
146    if not line or not line.startswith('--'):
147      break
148    comments.append(line)
149
150  # Reverse as the above was reversed
151  comments.reverse()
152  return comments
153
154
155# Given SQL string check whether any of the words is used, and create error
156# string if needed.
157def check_banned_words(sql: str, path: str) -> List[str]:
158  lines = [l.strip() for l in sql.split('\n')]
159  errors = []
160
161  # Ban the use of LIKE in non-comment lines.
162  for line in lines:
163    if line.startswith('--'):
164      continue
165
166    if 'like' in line.casefold():
167      errors.append(
168          'LIKE is banned in trace processor metrics. Prefer GLOB instead.\n'
169          f'Offending file: {path}\n')
170      continue
171
172    if 'create_function' in line.casefold():
173      errors.append('CREATE_FUNCTION is deprecated in trace processor. '
174                    'Use CREATE PERFETTO FUNCTION instead.\n'
175                    f'Offending file: {path}')
176
177    if 'create_view_function' in line.casefold():
178      errors.append(
179          'CREATE_VIEW_FUNCTION is deprecated in trace processor. '
180          'Use CREATE PERFETTO FUNCTION $name RETURNS TABLE instead.\n'
181          f'Offending file: {path}')
182
183    if 'import(' in line.casefold():
184      errors.append('SELECT IMPORT is deprecated in trace processor. '
185                    'Use INCLUDE PERFETTO MODULE instead.\n'
186                    f'Offending file: {path}')
187
188  return errors
189
190
191# Given SQL string check whether there is (not allowlisted) usage of
192# CREATE TABLE {name} AS.
193def check_banned_create_table_as(sql: str, filename: str, stdlib_path: str,
194                                 allowlist: Dict[str, List[str]]) -> List[str]:
195  errors = []
196  for _, matches in match_pattern(CREATE_TABLE_AS_PATTERN, sql).items():
197    name = matches[0]
198    # Normalize paths before checking presence in the allowlist so it will
199    # work on Windows for the Chrome stdlib presubmit.
200    allowlist_normpath = dict(
201        (os.path.normpath(path), tables) for path, tables in allowlist.items())
202    allowlist_key = os.path.normpath(filename[len(stdlib_path):])
203    if allowlist_key not in allowlist_normpath:
204      errors.append(f"CREATE TABLE '{name}' is deprecated. "
205                    "Use CREATE PERFETTO TABLE instead.\n"
206                    f"Offending file: {filename}\n")
207      continue
208    if name not in allowlist_normpath[allowlist_key]:
209      errors.append(
210          f"Table '{name}' uses CREATE TABLE which is deprecated "
211          "and this table is not allowlisted. Use CREATE PERFETTO TABLE.\n"
212          f"Offending file: {filename}\n")
213  return errors
214
215
216# Given SQL string check whether there is usage of CREATE VIEW {name} AS.
217def check_banned_create_view_as(sql: str, filename: str) -> List[str]:
218  errors = []
219  for _, matches in match_pattern(CREATE_VIEW_AS_PATTERN, sql).items():
220    name = matches[0]
221    errors.append(f"CREATE VIEW '{name}' is deprecated. "
222                  "Use CREATE PERFETTO VIEW instead.\n"
223                  f"Offending file: {filename}\n")
224  return errors
225
226
227# Given SQL string check whether there is usage of CREATE VIEW {name} AS.
228def check_banned_include_all(sql: str, filename: str) -> List[str]:
229  errors = []
230  for _, matches in match_pattern(INCLUDE_ALL_PATTERN, sql).items():
231    errors.append(
232        f"INCLUDE PERFETTO MODULE with wildcards is not allowed in stdlib. "
233        f"Import specific modules instead. Offending file: {filename}")
234  return errors
235