1# Copyright (c) 2011 The Chromium OS Authors. All rights reserved. 2# Use of this source code is governed by a BSD-style license that can be 3# found in the LICENSE file. 4 5import fnmatch 6import glob 7import logging 8import os 9 10from autotest_lib.client.bin import test, utils 11from autotest_lib.client.common_lib import error 12from optparse import OptionParser 13 14FILE_CMD="file -m /usr/local/share/misc/magic.mgc" 15 16class ToolchainOptionSet: 17 """ 18 Handles a set of hits, along with potential whitelists to ignore. 19 """ 20 def __init__(self, description, bad_files, whitelist_file): 21 self.description = description 22 self.bad_set = set(bad_files.splitlines()) 23 self.whitelist_set = set([]) 24 self.process_whitelist_with_private(whitelist_file) 25 26 27 def process_whitelist_with_private(self, whitelist_file): 28 """ 29 Filter out hits found on non-comment lines in the whitelist and 30 and private whitelist. 31 32 @param whitelist_file: path to whitelist file 33 """ 34 whitelist_files = [whitelist_file] 35 private_file = os.path.join(os.path.dirname(whitelist_file), 36 "private_" + 37 os.path.basename(whitelist_file)) 38 whitelist_files.append(private_file) 39 self.process_whitelists(whitelist_files) 40 41 42 def process_whitelist(self, whitelist_file): 43 """ 44 Filter out hits found on non-comment lines in the whitelist. 45 46 @param whitelist_file: path to whitelist file 47 """ 48 if os.path.isfile(whitelist_file): 49 f = open(whitelist_file) 50 whitelist = [x for x in f.read().splitlines() 51 if not x.startswith('#')] 52 f.close() 53 self.whitelist_set = self.whitelist_set.union(set(whitelist)) 54 55 filtered_list = [] 56 for bad_file in self.bad_set: 57 # Does |bad_file| match any entry in the whitelist? 58 in_whitelist = any([fnmatch.fnmatch(bad_file, whitelist_entry) 59 for whitelist_entry in self.whitelist_set]) 60 if not in_whitelist: 61 filtered_list.append(bad_file) 62 63 self.filtered_set = set(filtered_list) 64 # TODO(jorgelo): remove glob patterns from |new_passes|. 65 self.new_passes = self.whitelist_set.difference(self.bad_set) 66 67 68 def process_whitelists(self, whitelist_files): 69 """ 70 Filter out hits found in a list of whitelist files. 71 72 @param whitelist_files: list of paths to whitelist files 73 """ 74 for whitelist_file in whitelist_files: 75 self.process_whitelist(whitelist_file) 76 77 78 def get_fail_summary_message(self): 79 m = "Test %s " % self.description 80 m += "%d failures" % len(self.filtered_set) 81 return m 82 83 84 def get_fail_message(self): 85 m = self.get_fail_summary_message() 86 sorted_list = list(self.filtered_set) 87 sorted_list.sort() 88 m += "\nFAILED:\n%s\n\n" % "\n".join(sorted_list) 89 return m 90 91 92 def __str__(self): 93 m = "Test %s " % self.description 94 m += ("%d failures, %d in whitelist, %d in filtered, %d new passes " % 95 (len(self.bad_set), 96 len(self.whitelist_set), 97 len(self.filtered_set), 98 len(self.new_passes))) 99 100 if len(self.filtered_set): 101 sorted_list = list(self.filtered_set) 102 sorted_list.sort() 103 m += "FAILED:\n%s" % "\n".join(sorted_list) 104 else: 105 m += "PASSED!" 106 107 if len(self.new_passes): 108 sorted_list = list(self.new_passes) 109 sorted_list.sort() 110 m += ("\nNew passes (remove these from the whitelist):\n%s" % 111 "\n".join(sorted_list)) 112 logging.debug(m) 113 return m 114 115 116class platform_ToolchainOptions(test.test): 117 """ 118 Tests for various expected conditions on ELF binaries in the image. 119 """ 120 version = 2 121 122 def get_cmd(self, test_cmd, find_options=""): 123 base_cmd = ("find '%s' -wholename %s -prune -o " 124 " -wholename /proc -prune -o " 125 " -wholename /dev -prune -o " 126 " -wholename /sys -prune -o " 127 " -wholename /mnt/stateful_partition -prune -o " 128 " -wholename /usr/local -prune -o " 129 # There are files in /home/chronos that cause false 130 # positives, and since that's noexec anyways, it should 131 # be skipped. 132 " -wholename '/home/chronos' -prune -o " 133 " -wholename " 134 "/opt/google/containers/android/rootfs/root/vendor" 135 " -prune -o " 136 " %s " 137 " -not -name 'libstdc++.so.*' " 138 " -not -name 'libgcc_s.so.*' " 139 " -type f -executable -exec " 140 "sh -c '%s " 141 "{} | grep -q ELF && " 142 "(%s || echo {})' ';'") 143 rootdir = "/" 144 cmd = base_cmd % (rootdir, self.autodir, find_options, FILE_CMD, 145 test_cmd) 146 return cmd 147 148 149 def create_and_filter(self, description, cmd, whitelist_file, 150 find_options=""): 151 """ 152 Runs a command, with "{}" replaced (via "find -exec") with the 153 target ELF binary. If the command fails, the file is marked as 154 failing the test. Results are filtered against the provided 155 whitelist file. 156 157 @param description: text name of the check being done 158 @param cmd: command to run via find's -exec option 159 @param whitelist_file: list of failures to ignore 160 @param find_options: additional options for find to limit the scope 161 """ 162 full_cmd = self.get_cmd(cmd, find_options) 163 bad_files = utils.system_output(full_cmd) 164 cso = ToolchainOptionSet(description, bad_files, whitelist_file) 165 cso.process_whitelist_with_private(whitelist_file) 166 return cso 167 168 169 def run_once(self, rootdir="/", args=[]): 170 """ 171 Do a find for all the ELF files on the system. 172 For each one, test for compiler options that should have been used 173 when compiling the file. 174 175 For missing compiler options, print the files. 176 """ 177 178 parser = OptionParser() 179 parser.add_option('--hardfp', 180 dest='enable_hardfp', 181 default=False, 182 action='store_true', 183 help='Whether to check for hardfp binaries.') 184 (options, args) = parser.parse_args(args) 185 186 option_sets = [] 187 188 libc_glob = "/lib/libc-[0-9]*" 189 190 readelf_cmd = glob.glob("/usr/local/*/binutils-bin/*/readelf")[0] 191 192 # We do not test binaries if they are built with Address Sanitizer 193 # because it is a separate testing tool. 194 no_asan_used = utils.system_output("%s -s " 195 "/opt/google/chrome/chrome | " 196 "egrep -q \"__asan_init\" || " 197 "echo no ASAN" % readelf_cmd) 198 if not no_asan_used: 199 logging.debug("ASAN detected on /opt/google/chrome/chrome. " 200 "Will skip all checks.") 201 return 202 203 # Check that gold was used to build binaries. 204 # TODO(jorgelo): re-enable this check once crbug.com/417912 is fixed. 205 # gold_cmd = ("%s -S {} 2>&1 | " 206 # "egrep -q \".note.gnu.gold-ve\"" % readelf_cmd) 207 # gold_find_options = "" 208 # if utils.get_cpu_arch() == "arm": 209 # # gold is only enabled for Chrome on ARM. 210 # gold_find_options = "-path \"/opt/google/chrome/chrome\"" 211 # gold_whitelist = os.path.join(self.bindir, "gold_whitelist") 212 # option_sets.append(self.create_and_filter("gold", 213 # gold_cmd, 214 # gold_whitelist, 215 # gold_find_options)) 216 217 # Verify non-static binaries have BIND_NOW in dynamic section. 218 now_cmd = ("(%s {} | grep -q statically) ||" 219 "%s -d {} 2>&1 | " 220 "egrep -q \"BIND_NOW\"" % (FILE_CMD, readelf_cmd)) 221 now_whitelist = os.path.join(self.bindir, "now_whitelist") 222 option_sets.append(self.create_and_filter("-Wl,-z,now", 223 now_cmd, 224 now_whitelist)) 225 226 # Verify non-static binaries have RELRO program header. 227 relro_cmd = ("(%s {} | grep -q statically) ||" 228 "%s -l {} 2>&1 | " 229 "egrep -q \"GNU_RELRO\"" % (FILE_CMD, readelf_cmd)) 230 relro_whitelist = os.path.join(self.bindir, "relro_whitelist") 231 option_sets.append(self.create_and_filter("-Wl,-z,relro", 232 relro_cmd, 233 relro_whitelist)) 234 235 # Verify non-static binaries are dynamic (built PIE). 236 pie_cmd = ("(%s {} | grep -q statically) ||" 237 "%s -l {} 2>&1 | " 238 "egrep -q \"Elf file type is DYN\"" % (FILE_CMD, 239 readelf_cmd)) 240 pie_whitelist = os.path.join(self.bindir, "pie_whitelist") 241 option_sets.append(self.create_and_filter("-fPIE", 242 pie_cmd, 243 pie_whitelist)) 244 245 # Verify ELFs don't include TEXTRELs. 246 # FIXME: Remove the i?86 filter after the bug is fixed. 247 # crbug.com/686926 248 if (utils.get_current_kernel_arch() not in 249 ('i%d86' % i for i in xrange(3,7))): 250 textrel_cmd = ("(%s {} | grep -q statically) ||" 251 "%s -d {} 2>&1 | " 252 "(egrep -q \"0x0+16..TEXTREL\"; [ $? -ne 0 ])" 253 % (FILE_CMD, readelf_cmd)) 254 textrel_whitelist = os.path.join(self.bindir, "textrel_whitelist") 255 option_sets.append(self.create_and_filter("TEXTREL", 256 textrel_cmd, 257 textrel_whitelist)) 258 259 # Verify all binaries have non-exec STACK program header. 260 stack_cmd = ("%s -lW {} 2>&1 | " 261 "egrep -q \"GNU_STACK.*RW \"" % readelf_cmd) 262 stack_whitelist = os.path.join(self.bindir, "stack_whitelist") 263 option_sets.append(self.create_and_filter("Executable Stack", 264 stack_cmd, 265 stack_whitelist)) 266 267 # Verify all binaries have W^X LOAD program headers. 268 loadwx_cmd = ("%s -lW {} 2>&1 | " 269 "grep \"LOAD\" | egrep -v \"(RW |R E)\" | " 270 "wc -l | grep -q \"^0$\"" % readelf_cmd) 271 loadwx_whitelist = os.path.join(self.bindir, "loadwx_whitelist") 272 option_sets.append(self.create_and_filter("LOAD Writable and Exec", 273 loadwx_cmd, 274 loadwx_whitelist)) 275 276 # Verify ARM binaries are all using VFP registers. 277 if (options.enable_hardfp and utils.get_cpu_arch() == 'arm'): 278 hardfp_cmd = ("%s -A {} 2>&1 | " 279 "egrep -q \"Tag_ABI_VFP_args: VFP registers\"" % 280 readelf_cmd) 281 hardfp_whitelist = os.path.join(self.bindir, "hardfp_whitelist") 282 option_sets.append(self.create_and_filter("hardfp", hardfp_cmd, 283 hardfp_whitelist)) 284 285 fail_msg = "" 286 287 # There is currently no way to clear binary prebuilts for all devs. 288 # Thus, when a new check is added to this test, the test might fail 289 # for users who have old prebuilts which have not been compiled 290 # in the correct manner. 291 fail_summaries = [] 292 full_msg = "Test results:" 293 num_fails = 0 294 for cos in option_sets: 295 if len(cos.filtered_set): 296 num_fails += 1 297 fail_msg += cos.get_fail_message() + "\n" 298 fail_summaries.append(cos.get_fail_summary_message()) 299 full_msg += str(cos) + "\n\n" 300 fail_summary_msg = ", ".join(fail_summaries) 301 302 logging.error(fail_msg) 303 logging.debug(full_msg) 304 if num_fails: 305 raise error.TestFail(fail_summary_msg) 306