1#!/usr/bin/env bash 2# shellcheck disable=SC1091 # paths only become valid at runtime 3 4. "${SCRIPTS_DIR}/setup-test-env.sh" 5 6set -e 7 8comma_separated() { 9 local IFS=, 10 echo "$*" 11} 12 13if [[ -z "$VK_DRIVER" ]]; then 14 printf "VK_DRIVER is not defined\n" 15 exit 1 16fi 17 18INSTALL=$(realpath -s "$PWD"/install) 19 20# Set up the driver environment. 21# Modifiying here directly LD_LIBRARY_PATH may cause problems when 22# using a command wrapper. Hence, we will just set it when running the 23# command. 24export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:$INSTALL/lib/:/vkd3d-proton-tests/x64/" 25 26 27# Set the Vulkan driver to use. 28ARCH=$(uname -m) 29export VK_DRIVER_FILES="$INSTALL/share/vulkan/icd.d/${VK_DRIVER}_icd.$ARCH.json" 30 31# Set environment for Wine. 32export WINEDEBUG="-all" 33export WINEPREFIX="/vkd3d-proton-wine64" 34export WINEESYNC=1 35 36if [ -f "$INSTALL/$GPU_VERSION-vkd3d-skips.txt" ]; then 37 mapfile -t skips < <(grep -vE '^#|^$' "$INSTALL/$GPU_VERSION-vkd3d-skips.txt") 38 VKD3D_TEST_EXCLUDE=$(comma_separated "${skips[@]}") 39 printf 'VKD3D_TEST_EXCLUDE=%s\n' "$VKD3D_TEST_EXCLUDE" 40 export VKD3D_TEST_EXCLUDE 41fi 42 43# Sanity check to ensure that our environment is sufficient to make our tests 44# run against the Mesa built by CI, rather than any installed distro version. 45MESA_VERSION=$(cat "$INSTALL/VERSION") 46if ! vulkaninfo | grep driverInfo | tee /tmp/version.txt | grep -F "Mesa $MESA_VERSION"; then 47 printf "%s\n" "Found $(cat /tmp/version.txt), expected $MESA_VERSION" 48 exit 1 49fi 50 51# Gather the list expected failures 52EXPECTATIONFILE="$RESULTS_DIR/$GPU_VERSION-vkd3d-fails.txt" 53if [ -f "$INSTALL/$GPU_VERSION-vkd3d-fails.txt" ]; then 54 grep -vE '^(#|$)' "$INSTALL/$GPU_VERSION-vkd3d-fails.txt" | sort > "$EXPECTATIONFILE" 55else 56 printf "%s\n" "$GPU_VERSION-vkd3d-fails.txt not found, assuming a \"no failures\" baseline." 57 touch "$EXPECTATIONFILE" 58fi 59 60if [ -f "$INSTALL/$GPU_VERSION-vkd3d-flakes.txt" ]; then 61 mapfile -t flakes < <(grep -vE '^#|^$' "$INSTALL/$GPU_VERSION-vkd3d-flakes.txt") 62else 63 flakes=() 64fi 65 66# Some sanity checks before we start 67mapfile -t flakes_dups < <( 68 [ ${#flakes[@]} -eq 0 ] || 69 printf '%s\n' "${flakes[@]}" | sort | uniq -d 70) 71if [ ${#flakes_dups[@]} -gt 0 ]; then 72 printf >&2 'Duplicate flakes lines:\n' 73 printf >&2 ' %s\n' "${flakes_dups[@]}" 74 exit 1 75fi 76 77flakes_in_baseline=() 78for flake in "${flakes[@]}"; do 79 if grep -qF "$flake" "$EXPECTATIONFILE"; then 80 flakes_in_baseline+=("$flake") 81 fi 82done 83if [ ${#flakes_in_baseline[@]} -gt 0 ]; then 84 printf >&2 "Flakes found in %s:\n" "$EXPECTATIONFILE" 85 printf >&2 ' %s\n' "${flakes_in_baseline[@]}" 86 exit 1 87fi 88 89printf "%s\n" "Running vkd3d-proton testsuite..." 90 91LOGFILE="$RESULTS_DIR/vkd3d-proton-log.txt" 92TEST_LOGS="$RESULTS_DIR/test-logs" 93(cd /vkd3d-proton-tests && tests/test-runner.sh x64/bin/d3d12 --jobs "${FDO_CI_CONCURRENT:-4}" --output-dir "$TEST_LOGS" | tee "$LOGFILE") 94 95printf '\n\n' 96 97# Check if the executable finished (ie. no segfault). 98if ! grep -E "^Finished" "$LOGFILE" > /dev/null; then 99 error "Failed, see ${ARTIFACTS_BASE_URL}/results/vkd3d-proton-log.txt" 100 exit 1 101fi 102 103# Print list of flakes seen this time 104flakes_seen=() 105for flake in "${flakes[@]}"; do 106 if grep -qF "FAILED $flake" "$LOGFILE"; then 107 flakes_seen+=("$flake") 108 fi 109done 110if [ ${#flakes_seen[@]} -gt 0 ]; then 111 # Keep this string and output format in line with the corresponding 112 # deqp-runner message 113 printf >&2 '\nSome known flakes found:\n' 114 printf >&2 ' %s\n' "${flakes_seen[@]}" 115fi 116 117# Collect all the failures 118mapfile -t fails < <(grep -oE "^FAILED .+$" "$LOGFILE" | cut -d' ' -f2 | sort) 119 120# Save test output for failed tests (before excluding flakes) 121for failed_test in "${fails[@]}"; do 122 cp "$TEST_LOGS/$failed_test.log" "$RESULTS/$failed_test.log" 123done 124 125# Ignore flakes when comparing 126for flake in "${flakes[@]}"; do 127 for idx in "${!fails[@]}"; do 128 grep -qF "$flake" <<< "${fails[$idx]}" && unset -v 'fails[$idx]' 129 done 130done 131 132RESULTSFILE="$RESULTS/$GPU_VERSION.txt" 133for failed_test in "${fails[@]}"; do 134 if ! grep -qE "$failed_test end" "$RESULTS/$failed_test.log"; then 135 test_status=Crash 136 elif grep -qE "Test failed:" "$RESULTS/$failed_test.log"; then 137 test_status=Fail 138 else 139 test_status=Unknown 140 fi 141 printf '%s,%s\n' "$failed_test" "$test_status" 142done > "$RESULTSFILE" 143 144# Catch tests listed but not executed or not failing 145mapfile -t expected_fail_lines < "$EXPECTATIONFILE" 146for expected_fail_line in "${expected_fail_lines[@]}"; do 147 test_name=$(cut -d, -f1 <<< "$expected_fail_line") 148 if [ ! -f "$TEST_LOGS/$test_name.log" ]; then 149 test_status='UnexpectedImprovement(Skip)' 150 elif [ ! -f "$RESULTS/$test_name.log" ]; then 151 test_status='UnexpectedImprovement(Pass)' 152 else 153 continue 154 fi 155 printf '%s,%s\n' "$test_name" "$test_status" 156done >> "$RESULTSFILE" 157 158mapfile -t unexpected_results < <(comm -23 "$RESULTSFILE" "$EXPECTATIONFILE") 159if [ ${#unexpected_results[@]} -gt 0 ]; then 160 printf >&2 '\nUnexpected results:\n' 161 printf >&2 ' %s\n' "${unexpected_results[@]}" 162 exit 1 163fi 164 165exit 0 166