1#!/usr/bin/env python3 2# 3# Copyright 2019 - The Android Open Source Project 4# 5# Licensed under the Apache License, Version 2.0 (the "License"); 6# you may not use this file except in compliance with the License. 7# You may obtain a copy of the License at 8# 9# http://www.apache.org/licenses/LICENSE-2.0 10# 11# Unless required by applicable law or agreed to in writing, software 12# distributed under the License is distributed on an "AS IS" BASIS, 13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14# See the License for the specific language governing permissions and 15# limitations under the License. 16 17import filecmp 18import logging 19import os 20import re 21import shutil 22import subprocess 23import tempfile 24 25from acts.libs.proc import job 26 27COMMIT_ID_ENV_KEY = 'PREUPLOAD_COMMIT' 28GIT_FILE_NAMES_CMD = 'git diff-tree --no-commit-id --name-status -r %s' 29 30 31def compile_proto(proto_path, output_dir): 32 """Invoke Protocol Compiler to generate python from given source .proto.""" 33 # Find compiler path 34 protoc = None 35 if 'PROTOC' in os.environ and os.path.exists(os.environ['PROTOC']): 36 protoc = os.environ['PROTOC'] 37 if not protoc: 38 protoc = shutil.which('protoc') 39 if not protoc: 40 logging.error( 41 "Cannot find protobuf compiler (>=3.0.0), please install" 42 "protobuf-compiler package. Prefer copying from <top>/prebuilts/tools" 43 ) 44 logging.error(" prebuilts/tools/linux-x86_64/protoc/bin/protoc") 45 logging.error("If prebuilts are not available, use apt-get:") 46 logging.error(" sudo apt-get install protobuf-compiler") 47 return None 48 # Validate input proto path 49 if not os.path.exists(proto_path): 50 logging.error('Can\'t find required file: %s\n' % proto_path) 51 return None 52 # Validate output py-proto path 53 if not os.path.exists(output_dir): 54 os.makedirs(output_dir) 55 elif not os.path.isdir(output_dir): 56 logging.error("Output path is not a valid directory: %s" % 57 (output_dir)) 58 return None 59 input_dir = os.path.dirname(proto_path) 60 output_filename = os.path.basename(proto_path).replace('.proto', '_pb2.py') 61 output_path = os.path.join(output_dir, output_filename) 62 # Compiling proto 63 logging.debug('Generating %s' % output_path) 64 protoc_command = [ 65 protoc, '-I=%s' % (input_dir), '--python_out=%s' % (output_dir), 66 proto_path 67 ] 68 logging.debug('Running command %s' % protoc_command) 69 if subprocess.call(protoc_command, stderr=subprocess.STDOUT) != 0: 70 logging.error("Fail to compile proto") 71 return None 72 output_module_name = os.path.splitext(output_filename)[0] 73 return output_module_name 74 75 76def proto_generates_gen_file(proto_file, proto_gen_file): 77 """Check that the proto_gen_file matches the compilation result of the 78 proto_file. 79 80 Params: 81 proto_file: The proto file. 82 proto_gen_file: The generated file. 83 84 Returns: True if the compiled proto matches the given proto_gen_file. 85 """ 86 with tempfile.TemporaryDirectory() as tmp_dir: 87 module_name = compile_proto(proto_file, tmp_dir) 88 if not module_name: 89 return False 90 tmp_proto_gen_file = os.path.join(tmp_dir, '%s.py' % module_name) 91 return filecmp.cmp(tmp_proto_gen_file, proto_gen_file) 92 93 94def verify_protos_update_generated_files(proto_files, proto_gen_files): 95 """For each proto file in proto_files, find the corresponding generated 96 file in either proto_gen_files, or in the directory tree of the proto. 97 Verify that the generated file matches the compilation result of the proto. 98 99 Params: 100 proto_files: List of proto files. 101 proto_gen_files: List of generated files. 102 """ 103 success = True 104 gen_files = set(proto_gen_files) 105 for proto_file in proto_files: 106 gen_filename = os.path.basename(proto_file).replace('.proto', '_pb2.py') 107 gen_file = '' 108 # search the gen_files set first 109 for path in gen_files: 110 if (os.path.basename(path) == gen_filename 111 and path.startswith(os.path.dirname(proto_file))): 112 gen_file = path 113 gen_files.remove(path) 114 break 115 else: 116 # search the proto file's directory 117 for root, _, filenames in os.walk(os.path.dirname(proto_file)): 118 for filename in filenames: 119 if filename == gen_filename: 120 gen_file = os.path.join(root, filename) 121 break 122 if gen_file: 123 break 124 125 # check that the generated file matches the compiled proto 126 if gen_file and not proto_generates_gen_file(proto_file, gen_file): 127 logging.error('Proto file %s does not compile to %s' 128 % (proto_file, gen_file)) 129 protoc = shutil.which('protoc') 130 if protoc: 131 protoc_command = ' '.join([ 132 protoc, '-I=%s' % os.path.dirname(proto_file), 133 '--python_out=%s' % os.path.dirname(gen_file), proto_file]) 134 logging.error('Run this command to re-generate file from proto' 135 '\n%s' % protoc_command) 136 success = False 137 138 return success 139 140 141def main(): 142 if COMMIT_ID_ENV_KEY not in os.environ: 143 logging.error('Missing commit id in environment.') 144 exit(1) 145 146 # get list of *.proto and *_pb2.py files from commit, then compare 147 proto_files = [] 148 proto_gen_files = [] 149 git_cmd = GIT_FILE_NAMES_CMD % os.environ[COMMIT_ID_ENV_KEY] 150 lines = job.run(git_cmd).stdout.splitlines() 151 for line in lines: 152 match = re.match(r'(\S+)\s+(.*)', line) 153 status, f = match.group(1), match.group(2) 154 if status != 'D': 155 if f.endswith('.proto'): 156 proto_files.append(os.path.abspath(f)) 157 if f.endswith('_pb2.py'): 158 proto_gen_files.append(os.path.abspath(f)) 159 exit(not verify_protos_update_generated_files(proto_files, proto_gen_files)) 160 161 162if __name__ == '__main__': 163 main() 164