1#!/usr/bin/env ruby 2# Copyright (c) 2021-2024 Huawei Device Co., Ltd. 3# Licensed under the Apache License, Version 2.0 (the "License"); 4# you may not use this file except in compliance with the License. 5# You may obtain a copy of the License at 6# 7# http://www.apache.org/licenses/LICENSE-2.0 8# 9# Unless required by applicable law or agreed to in writing, software 10# distributed under the License is distributed on an "AS IS" BASIS, 11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12# See the License for the specific language governing permissions and 13# limitations under the License. 14 15require 'optparse' 16require 'ostruct' 17require 'logger' 18require 'fileutils' 19require 'open3' 20 21options = OpenStruct.new 22OptionParser.new do |opts| 23 opts.banner = 'Usage: checker.rb [options] TEST_FILE' 24 25 opts.on('--run-prefix=PREFIX', 'Prefix that will be inserted before panda run command') do |v| 26 options.run_prefix = v 27 end 28 opts.on('--source=FILE', 'Path to source file') 29 opts.on('--test-file=FILE', 'Path to test file') do |v| 30 options.test_file = v 31 end 32 opts.on('--panda=PANDA', 'Path to panda') 33 opts.on('--paoc=PAOC', 'Path to paoc') do |v| 34 options.paoc = v 35 end 36 opts.on('--frontend=FRONTEND', 'Path to frontend binary') 37 opts.on('--panda-options=OPTIONS', 'Default options for panda run') do |v| 38 options.panda_options = v 39 end 40 opts.on('--paoc-options=OPTIONS', 'Default options for paoc run') do |v| 41 options.paoc_options = v 42 end 43 opts.on('--frontend-options=OPTIONS', 'Default options for frontend+bco run') do |v| 44 options.frontend_options = v 45 end 46 opts.on('--method=METHOD', 'Method to optimize') 47 opts.on('--command-token=STRING', 'String that is recognized as command start') do |v| 48 options.command_token = v 49 end 50 opts.on('--release', 'Run in release mode. EVENT, INST and other will not be checked') 51 opts.on('-v', '--verbose', 'Verbose logging') 52 opts.on('--arch=ARCHITECTURE', 'Architecture of system where start panda') 53 opts.on("--keep-data", "Do not remove generated data from disk") { |v| options.keep_data = true } 54 opts.on("--with-llvm", "Tells checker that ARK was built with LLVM support") do |v| 55 options.with_llvm = true 56 end 57end.parse!(into: options) 58 59$LOG_LEVEL = options.verbose ? Logger::DEBUG : Logger::ERROR 60$curr_cmd = nil 61 62def log 63 @log ||= Logger.new($stdout, level: $LOG_LEVEL) 64end 65 66def raise_error(msg) 67 log.error "Test failed: #{$checker_name}" 68 if !$current_method.nil? 69 log.error "Method: \"#{$current_method}\"" 70 end 71 if !$current_pass.nil? 72 log.error $current_pass 73 end 74 log.error msg 75 log.error "Command to reproduce: #{$curr_cmd}" 76 raise msg 77end 78 79def match_str(match) 80 match.is_a?(Regexp) ? "/#{match.source}/" : match 81end 82 83def contains?(str, match) 84 return str =~ match if match.is_a? Regexp 85 86 raise_error "Wrong type for search: #{match.class}" unless match.is_a? String 87 str.include? match 88end 89 90# Provides methods to search lines in a given array 91class SearchScope 92 93 attr_reader :lines 94 attr_reader :current_index 95 96 def initialize(lines, name) 97 @lines = lines 98 @name = name 99 @current_index = 0 100 end 101 102 def find_method_dump(match) 103 @lines = @lines.drop(@current_index) 104 @current_index = 0 105 find(match) 106 @lines = @lines.drop(@current_index - 1) 107 @current_index = 0 108 find(/}$/) 109 @lines = @lines.slice(0, @current_index) 110 @current_index = 0 111 end 112 113 def find_block(match) 114 @lines = @lines.drop(@current_index) 115 @current_index = 0 116 find(match) 117 @lines = @lines.drop(@current_index - 1) 118 @current_index = 0 119 find(/succs:/) 120 @lines = @lines.slice(0, @current_index) 121 @current_index = 0 122 end 123 124 def self.from_file(fname, name) 125 SearchScope.new(File.readlines(fname), name) 126 end 127 128 def find(match) 129 return if match.nil? 130 131 @current_index = @lines.index { |line| contains?(line, match) } 132 raise_error "#{@name} not found: #{match_str(match)}" if @current_index.nil? 133 @current_index += 1 134 end 135 136 def find_next(match) 137 return if match.nil? 138 139 index = @lines.drop(@current_index).index { |line| contains?(line, match) } 140 raise_error "#{@name} not found: #{match_str(match)}" if index.nil? 141 @current_index += index + 1 142 end 143 144 def find_not(match) 145 return if match.nil? 146 147 @lines.each do |line| 148 raise_error "#{@name} should not occur: #{match_str(match)}" if contains?(line, match) 149 end 150 end 151 152 def find_next_not(match) 153 return if match.nil? 154 155 @lines.drop(@current_index).each do |line| 156 raise_error "#{@name} should not occur: #{match_str(match)}" if contains?(line, match) 157 end 158 end 159 160 def to_s 161 "Scope '#{@name}', current=#{@current_index}\n#{@lines.join}" 162 end 163end 164 165class Checker 166 attr_reader :name 167 168 def initialize(options, name) 169 @name = name 170 @lines = [] 171 @code = "" 172 @cwd = "#{Dir.getwd}/#{name.gsub(/[ -:()]/, '_')}" 173 @options = options 174 @args = '' 175 @ir_files = [] 176 @architecture = options.arch 177 @aot_file = '' 178 @llvm_paoc = false 179 180 # Events scope for 'events.csv' 181 @events_scope = nil 182 # IR scope for IR dumps files 'ir_dump/*.ir' 183 @ir_scope = nil 184 185 # Disassembly file lines, that were read from 'disasm.txt' 186 @disasm_lines = nil 187 # Currently processing disasm method 188 @disasm_method_scope = nil 189 # Current search scope 190 @disasm_scope = nil 191 192 Dir.mkdir(@cwd) unless File.exists?(@cwd) 193 clear_data 194 end 195 196 def set_llvm_paoc() 197 @llvm_paoc = true 198 end 199 200 def append_line(line) 201 @code << line + "\n" 202 end 203 204 def RUN(**args) 205 expected_result = 0 206 aborted_sig = 0 207 entry = '_GLOBAL::main' 208 env = '' 209 @args = '' 210 args.each do |name, value| 211 if name == :force_jit and value 212 @args << '--compiler-hotness-threshold=0 --no-async-jit=true --compiler-enable-jit=true ' 213 elsif name == :options 214 @args << value 215 elsif name == :entry 216 entry = value 217 elsif name == :result 218 expected_result = value 219 elsif name == :abort 220 aborted_sig = value 221 elsif name == :env 222 env = value 223 end 224 end 225 raise ":abort and :result cannot be set at the same time, :abort = #{aborted_sig}, :result = #{expected_result}" if aborted_sig != 0 && expected_result != 0 226 227 clear_data 228 aot_arg = @aot_file.empty? ? '' : "--aot-file #{@aot_file}" 229 230 cmd = "#{@options.run_prefix} #{@options.panda} --compiler-queue-type=simple --compiler-ignore-failures=false #{@options.panda_options} \ 231 #{aot_arg} #{@args} --events-output=csv --compiler-dump --compiler-disasm-dump:single-file #{@options.test_file} #{entry}" 232 $curr_cmd = "#{env} #{cmd}" 233 log.debug "Panda command: #{$curr_cmd}" 234 235 # See note on exec in RUN_PAOC 236 output, status = Open3.capture2e("#{env} exec #{cmd}", chdir: @cwd.to_s) 237 if aborted_sig != 0 && !status.signaled? 238 puts output 239 log.error "Expected ark to abort with signal #{aborted_sig}, but ark did not signal" 240 raise_error "Test '#{@name}' failed" 241 end 242 if status.signaled? 243 if status.termsig != aborted_sig 244 puts output 245 log.error "ark aborted with signal #{status.termsig}, but expected #{aborted_sig}" 246 raise_error "Test '#{@name}' failed" 247 end 248 elsif status.exitstatus != expected_result 249 puts output 250 log.error "ark returns code #{status.exitstatus}, but expected #{expected_result}" 251 raise_error "Test '#{@name}' failed" 252 end 253 log.debug output 254 File.open("#{@cwd}/console.out", "w") { |file| file.write(output) } 255 256 @events_scope = SearchScope.from_file("#{@cwd}/events.csv", 'Events') 257 end 258 259 def RUN_PAOC(**args) 260 @aot_file = "#{Dir.getwd}/#{File.basename(@options.test_file, File.extname(@options.test_file))}.an" 261 262 inputs = @options.test_file 263 aot_output_option = '--paoc-output' 264 output = @aot_file 265 options = '' 266 env = '' 267 aborted_sig = 0 268 result = 0 269 270 args.each do |name, value| 271 case name 272 when :options 273 options = value 274 when :boot 275 aot_output_option = '--paoc-boot-output' 276 when :env 277 env = value 278 when :inputs 279 inputs = value 280 when :abort 281 aborted_sig = value 282 when :output 283 output = value 284 when :result 285 result = value 286 end 287 end 288 raise ":abort and :result cannot be set at the same time, :abort = #{aborted_sig}, :result = #{result}" if aborted_sig != 0 && result != 0 289 290 paoc_args = "--paoc-panda-files #{inputs} --events-output=csv --compiler-dump #{options} #{aot_output_option} #{output}" 291 292 clear_data 293 294 cmd = "#{@options.run_prefix} #{@options.paoc} --compiler-ignore-failures=false --compiler-disasm-dump:single-file --compiler-dump #{@options.paoc_options} #{paoc_args}" 295 $curr_cmd = "#{env} #{cmd}" 296 log.debug "Paoc command: #{$curr_cmd}" 297 298 # Using exec to pass signal info to the parent process. 299 # Ruby invokes a process using /bin/sh if the curr_cmd has a metacharacter in it, for example '*', '?', '$'. 300 # If an invoked process signals, then the status.signaled? check below returns different values depending on the shell. 301 # For bash it is true, for dash it is false, because bash propagates a flag, whether the process has signalled or not. 302 # When we use 'exec' we will propagate the signal too 303 output, status = Open3.capture2e("#{env} exec #{cmd}", chdir: @cwd.to_s) 304 if aborted_sig != 0 && !status.signaled? 305 puts output 306 log.error "Expected ark_aot to abort with signal #{aborted_sig}, but ark_aot did not signal" 307 raise_error "Test '#{@name}' failed" 308 end 309 if status.signaled? 310 if status.termsig != aborted_sig 311 puts output 312 log.error "ark_aot aborted with signal #{status.termsig}, but expected #{aborted_sig}" 313 raise_error "Test '#{@name}' failed" 314 end 315 elsif status.exitstatus != result 316 puts output 317 log.error "ark_aot returns code #{status.exitstatus}, but expected #{result}" 318 raise_error "Test '#{@name}' failed" 319 end 320 log.debug output 321 File.open("#{@cwd}/console.out", "w") { |file| file.write(output) } 322 323 @events_scope = SearchScope.from_file("#{@cwd}/events.csv", 'Events') 324 end 325 326 def RUN_AOT(**args) 327 if @llvm_paoc 328 RUN_LLVM(**args) 329 else 330 RUN_PAOC(**args) 331 end 332 end 333 334 def RUN_BCO(**args) 335 inputs = @options.test_file 336 output = "#{@cwd}/#{File.basename(@options.test_file, '.*')}.abc" 337 @args = '' 338 339 args.each do |name, value| 340 case name 341 when :options 342 @args << value 343 when :inputs 344 inputs = value 345 when :output 346 output = value 347 when :method 348 @args << "--bco-optimizer --method-regex=#{value}:.*" 349 end 350 end 351 352 clear_data 353 $curr_cmd = "#{@options.frontend} --opt-level=2 --dump-assembly --bco-compiler --compiler-dump \ 354 #{@options.frontend_options} #{@args} --output=#{output} #{@options.source}" 355 log.debug "Frontend command: #{$curr_cmd}" 356 357 # See note on exec in RUN_PAOC 358 output, err_output, status = Open3.capture3("exec #{$curr_cmd}", chdir: @cwd.to_s) 359 if status.signaled? 360 if status.termsig != 0 361 puts output 362 log.error "#{@options.frontend} aborted with signal #{status.termsig}, but expected 0" 363 raise_error "Test '#{@name}' failed" 364 end 365 elsif status.exitstatus != 0 366 puts output 367 log.error "#{@options.frontend} returns code #{status.exitstatus}, but expected 0" 368 raise_error "Test '#{@name}' failed" 369 elsif !err_output.empty? 370 log.error "Bytecode optimizer failed, logs:" 371 puts err_output 372 raise_error "Test '#{@name}' failed" 373 end 374 File.open("#{@cwd}/console.out", "w") { |file| file.write(output) } 375 Open3.capture2e("cat #{@cwd}/console.out") 376 FileUtils.touch("#{@cwd}/events.csv") 377 end 378 379 def RUN_LLVM(**args) 380 raise SkipException unless @options.with_llvm 381 382 args[:options] << " --paoc-mode=llvm " 383 RUN_PAOC(**args) 384 end 385 386 def EVENT(match) 387 return if @options.release 388 389 @events_scope.find(match) 390 end 391 392 def EVENT_NEXT(match) 393 return if @options.release 394 395 @events_scope.find_next(match) 396 end 397 398 def EVENT_COUNT(match) 399 return 0 if @options.release 400 401 @events_scope.lines.count { |event| contains?(event, match) } 402 end 403 404 def EVENT_NOT(match) 405 return if @options.release 406 407 @events_scope.find_not(match) 408 end 409 410 def EVENT_NEXT_NOT(match) 411 return if @options.release 412 413 @events_scope.find_next_not(match) 414 end 415 416 def EVENTS_COUNT(match, count) 417 return if @options.release 418 419 res = @events_scope.lines.count { |event| contains?(event, match) } 420 raise_error "Events count missmatch for #{match}, expected: #{count}, real: #{res}" unless res == count 421 end 422 423 def TRUE(condition) 424 return if @options.release 425 426 raise_error "Not true condition: \"#{condition}\"" unless condition 427 end 428 429 class SkipException < StandardError 430 end 431 432 def SKIP_IF(condition) 433 raise SkipException if condition 434 end 435 436 def IR_COUNT(match) 437 return 0 if @options.release 438 439 @ir_scope.lines.count { |inst| contains?(inst, match) && !contains?(inst, /^Method:/) } 440 end 441 442 def BLOCK_COUNT 443 IR_COUNT('BB ') 444 end 445 446 def INST(match) 447 return if @options.release 448 449 @ir_scope.find(match) 450 end 451 452 def INST_NEXT(match) 453 return if @options.release 454 455 @ir_scope.find_next(match) 456 end 457 458 def INST_NOT(match) 459 return if @options.release 460 @ir_scope.find_not(match) 461 end 462 463 def INST_NEXT_NOT(match) 464 return if @options.release 465 466 @ir_scope.find_next_not(match) 467 end 468 469 def INST_COUNT(match, count) 470 return if @options.release 471 472 real_count = IR_COUNT(match) 473 raise_error "IR_COUNT mismatch for #{match}: expected=#{count}, real=#{real_count}" unless real_count == count 474 end 475 476 def IN_BLOCK(match) 477 return if @options.release 478 479 @ir_scope.find_block(/prop: #{match}/) 480 end 481 482 def LLVM_METHOD(match) 483 return if @options.release 484 485 @ir_scope.find_method_dump(match) 486 end 487 488 def BC_METHOD(match) 489 return if @options.release 490 491 READ_FILE "console.out" 492 @ir_scope.find_method_dump(/^\.function.*#{match.gsub('.', '-')}/) 493 end 494 495 module SearchState 496 NONE = 0 497 SEARCH_BODY = 1 498 SEARCH_END = 2 499 end 500 501 def ASM_METHOD(match) 502 ensure_disasm 503 state = SearchState::NONE 504 start_index = nil 505 end_index = -1 506 @disasm_lines.each_with_index do |line, index| 507 case state 508 when SearchState::NONE 509 if line.start_with?('METHOD_INFO:') && contains?(@disasm_lines[index + 1].split(':', 2)[1].strip, match) 510 state = SearchState::SEARCH_BODY 511 end 512 when SearchState::SEARCH_BODY 513 if line.start_with?('DISASSEMBLY') 514 start_index = index + 1 515 state = SearchState::SEARCH_END 516 end 517 when SearchState::SEARCH_END 518 if line.start_with?('METHOD_INFO:') 519 end_index = index - 1 520 break 521 end 522 end 523 end 524 raise "Method not found: #{match_str(match)}" if start_index.nil? 525 526 @disasm_method_scope = SearchScope.new(@disasm_lines[start_index..end_index], "Method: #{match_str(match)}") 527 @disasm_scope = @disasm_method_scope 528 end 529 530 def ASM_INST(match) 531 ensure_disasm 532 state = SearchState::NONE 533 start_index = nil 534 end_index = -1 535 prefix = nil 536 @disasm_method_scope.lines.each_with_index do |line, index| 537 case state 538 when SearchState::NONE 539 if contains?(line, match) 540 prefix = line.sub(/#.*/, '#').gsub("\n", '') 541 start_index = index + 1 542 state = SearchState::SEARCH_END 543 end 544 when SearchState::SEARCH_END 545 if line.start_with?(prefix) 546 end_index = index - 1 547 break 548 end 549 end 550 end 551 raise "Can not find asm instruction: #{match}" if start_index.nil? 552 553 @disasm_scope = SearchScope.new(@disasm_method_scope.lines[start_index..end_index], "Inst: #{match_str(match)}") 554 end 555 556 def ASM_RESET 557 @disasm_scope = @disasm_method_scope 558 end 559 560 def ASM(**kwargs) 561 ensure_disasm 562 @disasm_scope.find(select_asm(kwargs)) 563 end 564 565 def ASM_NEXT(**kwargs) 566 ensure_disasm 567 @disasm_scope.find_next(select_asm(kwargs)) 568 end 569 570 def ASM_NOT(**kwargs) 571 ensure_disasm 572 @disasm_scope.find_not(select_asm(kwargs)) 573 end 574 575 def ASM_NEXT_NOT(**kwargs) 576 ensure_disasm 577 @disasm_scope.find_next_not(select_asm(kwargs)) 578 end 579 580 def select_asm(kwargs) 581 kwargs[@options.arch.to_sym] 582 end 583 584 def ensure_disasm 585 @disasm_lines ||= File.readlines("#{@cwd}/disasm.txt") 586 end 587 588 def METHOD(method) 589 return if @options.release 590 @ir_files = Dir["#{@cwd}/ir_dump/*#{method.gsub(/::|[<>]/, '_')}*.ir"] 591 @ir_files.sort! 592 raise_error "IR dumps not found for method: #{method.gsub(/::|[<>]/, '_')}" if @ir_files.empty? 593 $current_method = method 594 @current_file_index = 0 595 end 596 597 def PASS_AFTER(pass) 598 return if @options.release 599 600 $current_pass = "Pass after: #{pass}" 601 @current_file_index = @ir_files.index { |x| File.basename(x).include? pass } 602 raise_error "IR file not found for pass: #{pass}. Possible cause: you forgot to select METHOD first" unless @current_file_index 603 @ir_scope = SearchScope.from_file(@ir_files[@current_file_index], 'IR') 604 end 605 606 def PASS_AFTER_NEXT(pass) 607 return if @options.release 608 609 $current_pass = "Pass after next: #{pass}" 610 index = @ir_files[(@current_file_index + 1)..-1].index { |x| File.basename(x).include? pass } 611 raise_error "IR file not found for pass: #{pass}. Possible cause: you forgot to select METHOD first" unless index 612 @current_file_index += 1 + index 613 @ir_scope = SearchScope.from_file(@ir_files[@current_file_index], 'IR') 614 end 615 616 def PASS_BEFORE(pass) 617 return if @options.release 618 619 $current_pass = "Pass before: #{pass}" 620 @current_file_index = @ir_files.index { |x| File.basename(x).include? pass } 621 raise_error "IR file not found for pass: #{pass}. Possible cause: you forgot to select METHOD first" unless @current_file_index 622 @ir_scope = SearchScope.from_file(@ir_files[@current_file_index - 1], 'IR') 623 end 624 625 def READ_FILE(filename) 626 path = "#{@cwd}/#{filename}" 627 raise_error "File `#{filename}` not found" unless File.file?(path) 628 @ir_scope = SearchScope.from_file(path, 'Plain text') 629 end 630 631 def run 632 log.info "Running \"#{@name}\"" 633 $checker_name = @name 634 begin 635 self.instance_eval @code 636 rescue SkipException 637 log.info "Skipped: \"#{@name}\"" 638 else 639 log.info "Success: \"#{@name}\"" 640 end 641 clear_data 642 end 643 644 def clear_data 645 $current_method = nil 646 $current_pass = nil 647 if !@options.keep_data 648 FileUtils.rm_rf("#{@cwd}/ir_dump") 649 FileUtils.rm_rf("#{@cwd}/events.csv") 650 FileUtils.rm_rf("#{@cwd}/disasm.txt") 651 FileUtils.rm_rf("#{@cwd}/console.out") 652 end 653 end 654end 655 656def read_checks(options) 657 checks = [] 658 check = nil 659 check_llvm = nil 660 command_token = /[ ]*#{options.command_token}(.*)/ 661 checker_start = /[ ]*#{options.command_token} CHECKER[ ]*(.*)/ 662 disabled_checker_start = /[ ]*#{options.command_token} DISABLED_CHECKER[ ]*(.*)/ 663 File.readlines(options.source).each do |line| 664 if check 665 unless line.start_with? command_token 666 check = nil 667 check_llvm = nil 668 next 669 end 670 raise "No space between two checkers: '#{line.strip}'" if line.start_with? checker_start 671 if line.include? "RUN_AOT" 672 checks << check_llvm 673 end 674 check.append_line(command_token.match(line)[1]) unless check == :disabled_check 675 check_llvm.append_line(command_token.match(line)[1]) unless check == :disabled_check 676 else 677 next unless line.start_with? command_token 678 if line.start_with? checker_start 679 name = command_token.match(line)[1] 680 raise "Checker with name '#{name}'' already exists" if checks.any? { |x| x.name == name } 681 682 check = Checker.new(options, name) 683 check_llvm = Checker.new(options, "#{name} LLVMAOT") 684 check_llvm.set_llvm_paoc() 685 checks << check 686 else 687 raise "Line '#{line.strip}' does not belong to any checker" unless line.start_with? disabled_checker_start 688 check = :disabled_check 689 next 690 end 691 end 692 end 693 checks 694end 695 696def main(options) 697 read_checks(options).each(&:run) 698 0 699end 700 701if __FILE__ == $PROGRAM_NAME 702 main(options) 703end 704 705# Somehow ruby resolves `Checker` name to another class in a Testing scope, so make this global 706# variable to refer to it from unit tests. I believe there is more proper way to do it, but I 707# didn't find it at first glance. 708$CheckerForTest = Checker 709