compiler/tests/check_output.py

#
# Copyright (c) 2020 Valve Corporation
#
# Permission is hereby granted, free of charge, to any person obtaining a
# copy of this software and associated documentation files (the "Software"),
# to deal in the Software without restriction, including without limitation
# the rights to use, copy, modify, merge, publish, distribute, sublicense,
# and/or sell copies of the Software, and to permit persons to whom the
# Software is furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice (including the next
# paragraph) shall be included in all copies or substantial portions of the
# Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
# IN THE SOFTWARE.
import re
import sys
import os.path
import struct
import string
import copy
from math import floor

if os.isatty(sys.stdout.fileno()):
    set_red = "\033[31m"
    set_green = "\033[1;32m"
    set_normal = "\033[0m"
else:
    set_red = ''
    set_green = ''
    set_normal = ''

initial_code = '''
def insert_code(code):
    insert_queue.append(CodeCheck(code))

def insert_pattern(pattern):
    insert_queue.append(PatternCheck(pattern))

def vector_gpr(prefix, name, size, align):
    insert_code(f'{name} = {name}0')
    for i in range(size):
        insert_code(f'{name}{i} = {name}0 + {i}')
    insert_code(f'success = {name}0 + {size - 1} == {name}{size - 1}')
    insert_code(f'success = {name}0 % {align} == 0')
    return f'{prefix}[#{name}0:#{name}{size - 1}]'

def sgpr_vector(name, size, align):
    return vector_gpr('s', name, size, align)

funcs.update({
    's64': lambda name: vector_gpr('s', name, 2, 2),
    's96': lambda name: vector_gpr('s', name, 3, 2),
    's128': lambda name: vector_gpr('s', name, 4, 4),
    's256': lambda name: vector_gpr('s', name, 8, 4),
    's512': lambda name: vector_gpr('s', name, 16, 4),
})
for i in range(2, 14):
    funcs['v%d' % (i * 32)] = lambda name: vector_gpr('v', name, i, 1)
'''

class Check:
    def __init__(self, data):
        self.data = data.rstrip()

    def run(self, state):
        pass

class CodeCheck(Check):
    def run(self, state):
        indent = 0
        first_line = [l for l in self.data.split('\n') if l.strip() != ''][0]
        indent_amount = len(first_line) - len(first_line.lstrip())
        indent = first_line[:indent_amount]
        new_lines = []
        for line in self.data.split('\n'):
            if line.strip() == '':
                new_lines.append('')
                continue
            if line[:indent_amount] != indent:
                state.result.log += 'unexpected indent in code check:\n'
                state.result.log += self.data + '\n'
                return False
            new_lines.append(line[indent_amount:])
        code = '\n'.join(new_lines)

        try:
            exec(code, state.g)
            state.result.log += state.g['log']
            state.g['log'] = ''
        except BaseException as e:
            state.result.log += 'code check raised exception:\n'
            state.result.log += code + '\n'
            state.result.log += str(e)
            return False
        if not state.g['success']:
            state.result.log += 'code check failed:\n'
            state.result.log += code + '\n'
            return False
        return True

class StringStream:
    class Pos:
        def __init__(self):
            self.line = 1
            self.column = 1

    def __init__(self, data, name):
        self.name = name
        self.data = data
        self.offset = 0
        self.pos = StringStream.Pos()

    def reset(self):
        self.offset = 0
        self.pos = StringStream.Pos()

    def peek(self, num=1):
        return self.data[self.offset:self.offset+num]

    def peek_test(self, chars):
        c = self.peek(1)
        return c != '' and c in chars

    def read(self, num=4294967296):
        res = self.peek(num)
        self.offset += len(res)
        for c in res:
            if c == '\n':
                self.pos.line += 1
                self.pos.column = 1
            else:
                self.pos.column += 1
        return res

    def get_line(self, num):
        return self.data.split('\n')[num - 1].rstrip()

    def skip_line(self):
        while self.peek(1) not in ['\n', '']:
            self.read(1)
        self.read(1)

    def skip_whitespace(self, inc_line):
        chars = [' ', '\t'] + (['\n'] if inc_line else [])
        while self.peek(1) in chars:
            self.read(1)

    def get_number(self):
        num = ''
        while self.peek() in string.digits:
            num += self.read(1)
        return num

    def check_identifier(self):
        return self.peek_test(string.ascii_letters + '_')

    def get_identifier(self):
        res = ''
        if self.check_identifier():
            while self.peek_test(string.ascii_letters + string.digits + '_'):
                res += self.read(1)
        return res

def format_error_lines(at, line_num, column_num, ctx, line):
    pred = '%s line %d, column %d of %s: "' % (at, line_num, column_num, ctx)
    return [pred + line + '"',
            '-' * (column_num - 1 + len(pred)) + '^']

class MatchResult:
    def __init__(self, pattern):
        self.success = True
        self.func_res = None
        self.pattern = pattern
        self.pattern_pos = StringStream.Pos()
        self.output_pos = StringStream.Pos()
        self.fail_message = ''

    def set_pos(self, pattern, output):
        self.pattern_pos.line = pattern.pos.line
        self.pattern_pos.column = pattern.pos.column
        self.output_pos.line = output.pos.line
        self.output_pos.column = output.pos.column

    def fail(self, msg):
        self.success = False
        self.fail_message = msg

    def format_pattern_pos(self):
        pat_pos = self.pattern_pos
        pat_line = self.pattern.get_line(pat_pos.line)
        res = format_error_lines('at', pat_pos.line, pat_pos.column, 'pattern', pat_line)
        func_res = self.func_res
        while func_res:
            pat_pos = func_res.pattern_pos
            pat_line = func_res.pattern.get_line(pat_pos.line)
            res += format_error_lines('in', pat_pos.line, pat_pos.column, func_res.pattern.name, pat_line)
            func_res = func_res.func_res
        return '\n'.join(res)

def do_match(g, pattern, output, skip_lines, in_func=False):
    assert(not in_func or not skip_lines)

    if not in_func:
        output.skip_whitespace(False)
    pattern.skip_whitespace(False)

    old_g = copy.copy(g)
    old_g_keys = list(g.keys())
    res = MatchResult(pattern)
    escape = False
    while True:
        res.set_pos(pattern, output)

        c = pattern.read(1)
        fail = False
        if c == '':
            break
        elif output.peek() == '':
            res.fail('unexpected end of output')
        elif c == '\\':
            escape = True
            continue
        elif c == '\n':
            old_line = output.pos.line
            output.skip_whitespace(True)
            if output.pos.line == old_line:
                res.fail('expected newline in output')
        elif not escape and c == '#':
            num = output.get_number()
            if num == '':
                res.fail('expected number in output')
            elif pattern.check_identifier():
                name = pattern.get_identifier()
                if name in g and int(num) != g[name]:
                    res.fail('unexpected number for \'%s\': %d (expected %d)' % (name, int(num), g[name]))
                elif name != '_':
                    g[name] = int(num)
        elif not escape and c == '$':
            name = pattern.get_identifier()

            val = ''
            while not output.peek_test(string.whitespace):
                val += output.read(1)

            if name in g and val != g[name]:
                res.fail('unexpected value for \'%s\': \'%s\' (expected \'%s\')' % (name, val, g[name]))
            elif name != '_':
                g[name] = val
        elif not escape and c == '%' and pattern.check_identifier():
            if output.read(1) != '%':
                res.fail('expected \'%\' in output')
            else:
                num = output.get_number()
                if num == '':
                    res.fail('expected number in output')
                else:
                    name = pattern.get_identifier()
                    if name in g and int(num) != g[name]:
                        res.fail('unexpected number for \'%s\': %d (expected %d)' % (name, int(num), g[name]))
                    elif name != '_':
                        g[name] = int(num)
        elif not escape and c == '@' and pattern.check_identifier():
            name = pattern.get_identifier()
            args = ''
            if pattern.peek_test('('):
                pattern.read(1)
                while pattern.peek() not in ['', ')']:
                    args += pattern.read(1)
                assert(pattern.read(1) == ')')
            func_res = g['funcs'][name](args)
            match_res = do_match(g, StringStream(func_res, 'expansion of "%s(%s)"' % (name, args)), output, False, True)
            if not match_res.success:
                res.func_res = match_res
                res.output_pos = match_res.output_pos
                res.fail(match_res.fail_message)
        elif not escape and c == ' ':
            while pattern.peek_test(' '):
                pattern.read(1)

            read_whitespace = False
            while output.peek_test(' \t'):
                output.read(1)
                read_whitespace = True
            if not read_whitespace:
                res.fail('expected whitespace in output, got %r' % (output.peek(1)))
        else:
            outc = output.peek(1)
            if outc != c:
                res.fail('expected %r in output, got %r' % (c, outc))
            else:
                output.read(1)
        if not res.success:
            if skip_lines and output.peek() != '':
                g.clear()
                g.update(old_g)
                res.success = True
                output.skip_line()
                pattern.reset()
                output.skip_whitespace(False)
                pattern.skip_whitespace(False)
            else:
                return res

        escape = False

    if not in_func:
        while output.peek() in [' ', '\t']:
            output.read(1)

        if output.read(1) not in ['', '\n']:
            res.fail('expected end of output')
            return res

    return res

class PatternCheck(Check):
    def __init__(self, data, search, position):
        Check.__init__(self, data)
        self.search = search
        self.position = position

    def run(self, state):
        pattern_stream = StringStream(self.data.rstrip(), 'pattern')
        res = do_match(state.g, pattern_stream, state.g['output'], self.search)
        if not res.success:
            state.result.log += 'pattern at %s failed: %s\n' % (self.position, res.fail_message)
            state.result.log += res.format_pattern_pos() + '\n\n'
            if not self.search:
                out_line = state.g['output'].get_line(res.output_pos.line)
                state.result.log += '\n'.join(format_error_lines('at', res.output_pos.line, res.output_pos.column, 'output', out_line))
            else:
                state.result.log += 'output was:\n'
                state.result.log += state.g['output'].data.rstrip() + '\n'
            return False
        return True

class CheckState:
    def __init__(self, result, variant, checks, output):
        self.result = result
        self.variant = variant
        self.checks = checks

        self.checks.insert(0, CodeCheck(initial_code))
        self.insert_queue = []

        self.g = {'success': True, 'funcs': {}, 'insert_queue': self.insert_queue,
                  'variant': variant, 'log': '', 'output': StringStream(output, 'output'),
                  'CodeCheck': CodeCheck, 'PatternCheck': PatternCheck}

class TestResult:
    def __init__(self, expected):
        self.result = ''
        self.expected = expected
        self.log = ''

def check_output(result, variant, checks, output):
    state = CheckState(result, variant, checks, output)

    while len(state.checks):
        check = state.checks.pop(0)
        if not check.run(state):
            result.result = 'failed'
            return

        for check in state.insert_queue[::-1]:
            state.checks.insert(0, check)
        state.insert_queue.clear()

    result.result = 'passed'
    return

def parse_check(variant, line, checks, pos):
    if line.startswith(';'):
        line = line[1:]
        if len(checks) and isinstance(checks[-1], CodeCheck):
            checks[-1].data += '\n' + line
        else:
            checks.append(CodeCheck(line))
    elif line.startswith('!'):
        checks.append(PatternCheck(line[1:], False, pos))
    elif line.startswith('>>'):
        checks.append(PatternCheck(line[2:], True, pos))
    elif line.startswith('~'):
        end = len(line)
        start = len(line)
        for c in [';', '!', '>>']:
            if line.find(c) != -1 and line.find(c) < end:
                end = line.find(c)
        if end != len(line):
            match = re.match(line[1:end], variant)
            if match and match.end() == len(variant):
                parse_check(variant, line[end:], checks, pos)

def parse_test_source(test_name, variant, fname):
    in_test = False
    test = []
    expected_result = 'passed'
    line_num = 1
    for line in open(fname, 'r').readlines():
        if line.startswith('BEGIN_TEST(%s)' % test_name):
            in_test = True
        elif line.startswith('BEGIN_TEST_TODO(%s)' % test_name):
            in_test = True
            expected_result = 'todo'
        elif line.startswith('BEGIN_TEST_FAIL(%s)' % test_name):
            in_test = True
            expected_result = 'failed'
        elif line.startswith('END_TEST'):
            in_test = False
        elif in_test:
            test.append((line_num, line.strip()))
        line_num += 1

    checks = []
    for line_num, check in [(line_num, l[2:]) for line_num, l in test if l.startswith('//')]:
         parse_check(variant, check, checks, 'line %d of %s' % (line_num, os.path.split(fname)[1]))

    return checks, expected_result

def parse_and_check_test(test_name, variant, test_file, output, current_result):
    checks, expected = parse_test_source(test_name, variant, test_file)

    result = TestResult(expected)
    if len(checks) == 0:
        result.result = 'empty'
        result.log = 'no checks found'
    elif current_result != None:
        result.result, result.log = current_result
    else:
        check_output(result, variant, checks, output)
        if result.result == 'failed' and expected == 'todo':
            result.result = 'todo'

    return result

def print_results(results, output, expected):
    results = {name: result for name, result in results.items() if result.result == output}
    results = {name: result for name, result in results.items() if (result.result == result.expected) == expected}

    if not results:
        return 0

    print('%s tests (%s):' % (output, 'expected' if expected else 'unexpected'))
    for test, result in results.items():
        color = '' if expected else set_red
        print('   %s%s%s' % (color, test, set_normal))
        if result.log.strip() != '':
            for line in result.log.rstrip().split('\n'):
                print('      ' + line.rstrip())
    print('')

    return len(results)

def get_cstr(fp):
    res = b''
    while True:
        c = fp.read(1)
        if c == b'\x00':
            return res.decode('utf-8')
        else:
            res += c

if __name__ == "__main__":
   results = {}

   stdin = sys.stdin.buffer
   while True:
       packet_type = stdin.read(4)
       if packet_type == b'':
           break;

       test_name = get_cstr(stdin)
       test_variant = get_cstr(stdin)
       if test_variant != '':
           full_name = test_name + '/' + test_variant
       else:
           full_name = test_name

       test_source_file = get_cstr(stdin)
       current_result = None
       if ord(stdin.read(1)):
           current_result = (get_cstr(stdin), get_cstr(stdin))
       code_size = struct.unpack("=L", stdin.read(4))[0]
       code = stdin.read(code_size).decode('utf-8')

       results[full_name] = parse_and_check_test(test_name, test_variant, test_source_file, code, current_result)

   result_types = ['passed', 'failed', 'todo', 'empty']
   num_expected = 0
   num_unexpected = 0
   for t in result_types:
       num_expected += print_results(results, t, True)
   for t in result_types:
       num_unexpected += print_results(results, t, False)
   num_expected_skipped = print_results(results, 'skipped', True)
   num_unexpected_skipped = print_results(results, 'skipped', False)

   num_unskipped = len(results) - num_expected_skipped - num_unexpected_skipped
   color = set_red if num_unexpected else set_green
   print('%s%d (%.0f%%) of %d unskipped tests had an expected result%s' % (color, num_expected, floor(num_expected / num_unskipped * 100), num_unskipped, set_normal))
   if num_unexpected_skipped:
       print('%s%d tests had been unexpectedly skipped%s' % (set_red, num_unexpected_skipped, set_normal))

   if num_unexpected:
       sys.exit(1)