xref: /art/test/run-test

#!/bin/bash
#
# Copyright (C) 2007 The Android Open Source Project
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# Set up prog to be the path of this script, including following symlinks,
# and set up progdir to be the fully-qualified pathname of its directory.
prog="$0"
args="$@"
while [ -h "${prog}" ]; do
    newProg=`/bin/ls -ld "${prog}"`
    newProg=`expr "${newProg}" : ".* -> \(.*\)$"`
    if expr "x${newProg}" : 'x/' >/dev/null; then
        prog="${newProg}"
    else
        progdir=`dirname "${prog}"`
        prog="${progdir}/${newProg}"
    fi
done
oldwd=`pwd`
progdir=`dirname "${prog}"`
cd "${progdir}"
progdir=`pwd`
prog="${progdir}"/`basename "${prog}"`
test_dir="test-$$"
if [ -z "$TMPDIR" ]; then
  tmp_dir="/tmp/$USER/${test_dir}"
else
  tmp_dir="${TMPDIR}/$USER/${test_dir}"
fi
checker="${progdir}/../tools/checker.py"

export JAVA="java"
export JAVAC="javac -g"
export RUN="${progdir}/etc/run-test-jar"
export DEX_LOCATION=/data/run-test/${test_dir}
export NEED_DEX="true"
export USE_JACK="false"

# If dx was not set by the environment variable, assume it is in the path.
if [ -z "$DX" ]; then
  export DX="dx"
fi

# If jasmin was not set by the environment variable, assume it is in the path.
if [ -z "$JASMIN" ]; then
  export JASMIN="jasmin"
fi

# If smali was not set by the environment variable, assume it is in the path.
if [ -z "$SMALI" ]; then
  export SMALI="smali"
fi

# If dexmerger was not set by the environment variable, assume it is in the path.
if [ -z "$DXMERGER" ]; then
  export DXMERGER="dexmerger"
fi

# If jack was not set by the environment variable, assume it is in the path.
if [ -z "$JACK" ]; then
  export JACK="jack"
fi

# If the tree is compiled with Jack, build test with Jack by default.
if [ "$ANDROID_COMPILE_WITH_JACK" = "true" ]; then
  USE_JACK="true"
fi

# ANDROID_BUILD_TOP is not set in a build environment.
if [ -z "$ANDROID_BUILD_TOP" ]; then
    export ANDROID_BUILD_TOP=$oldwd
fi

# If JACK_VM_COMMAND is not set, assume it launches the prebuilt jack-launcher.
if [ -z "$JACK_VM_COMMAND" ]; then
  if [ ! -z "$TMPDIR" ]; then
    jack_temp_dir="-Djava.io.tmpdir=$TMPDIR"
  fi
  export JACK_VM_COMMAND="java -Dfile.encoding=UTF-8 -Xms2560m -XX:+TieredCompilation $jack_temp_dir -jar $ANDROID_BUILD_TOP/prebuilts/sdk/tools/jack-launcher.jar"
fi

# If JACK_CLASSPATH is not set, assume it only contains core-libart.
if [ -z "$JACK_CLASSPATH" ]; then
  export JACK_CLASSPATH="$ANDROID_BUILD_TOP/out/host/common/obj/JAVA_LIBRARIES/core-libart-hostdex_intermediates/classes.jack"
fi

# If JACK_JAR is not set, assume it is located in the prebuilts directory.
if [ -z "$JACK_JAR" ]; then
  export JACK_JAR="$ANDROID_BUILD_TOP/prebuilts/sdk/tools/jack.jar"
fi

# If JILL_JAR is not set, assume it is located in the prebuilts directory.
if [ -z "$JILL_JAR" ]; then
  export JILL_JAR="$ANDROID_BUILD_TOP/prebuilts/sdk/tools/jill.jar"
fi

export JACK="$JACK -g -cp $JACK_CLASSPATH"
export JILL="java -jar $JILL_JAR"

info="info.txt"
build="build"
run="run"
expected="expected.txt"
check_cmd="check"
output="output.txt"
build_output="build-output.txt"
cfg_output="cfg-output.txt"
lib="libartd.so"
run_args="--quiet"
build_args=""

debuggable="no"
prebuild_mode="yes"
target_mode="yes"
dev_mode="no"
update_mode="no"
debug_mode="no"
relocate="yes"
runtime="art"
usage="no"
build_only="no"
suffix64=""
trace="false"
trace_stream="false"
basic_verify="false"
gc_verify="false"
gc_stress="false"
always_clean="no"
have_dex2oat="yes"
have_patchoat="yes"
have_image="yes"
image_suffix=""
pic_image_suffix=""
android_root="/system"

while true; do
    if [ "x$1" = "x--host" ]; then
        target_mode="no"
        DEX_LOCATION=$tmp_dir
        run_args="${run_args} --host"
        shift
    elif [ "x$1" = "x--jvm" ]; then
        target_mode="no"
        runtime="jvm"
        prebuild_mode="no"
        NEED_DEX="false"
        USE_JACK="false"
        run_args="${run_args} --jvm"
        shift
    elif [ "x$1" = "x-O" ]; then
        lib="libart.so"
        shift
    elif [ "x$1" = "x--dalvik" ]; then
        lib="libdvm.so"
        runtime="dalvik"
        shift
    elif [ "x$1" = "x--no-dex2oat" ]; then
        have_dex2oat="no"
        shift
    elif [ "x$1" = "x--no-patchoat" ]; then
        have_patchoat="no"
        shift
    elif [ "x$1" = "x--no-image" ]; then
        have_image="no"
        shift
    elif [ "x$1" = "x--pic-image" ]; then
        pic_image_suffix="-pic"
        shift
    elif [ "x$1" = "x--pic-test" ]; then
        run_args="${run_args} --pic-test"
        shift
    elif [ "x$1" = "x--relocate" ]; then
        relocate="yes"
        shift
    elif [ "x$1" = "x--no-relocate" ]; then
        relocate="no"
        shift
    elif [ "x$1" = "x--prebuild" ]; then
        run_args="${run_args} --prebuild"
        prebuild_mode="yes"
        shift;
    elif [ "x$1" = "x--debuggable" ]; then
        run_args="${run_args} -Xcompiler-option --debuggable"
        debuggable="yes"
        shift;
    elif [ "x$1" = "x--no-prebuild" ]; then
        run_args="${run_args} --no-prebuild"
        prebuild_mode="no"
        shift;
    elif [ "x$1" = "x--gcverify" ]; then
        basic_verify="true"
        gc_verify="true"
        shift
    elif [ "x$1" = "x--gcstress" ]; then
        basic_verify="true"
        gc_stress="true"
        shift
    elif [ "x$1" = "x--image" ]; then
        shift
        image="$1"
        run_args="${run_args} --image $image"
        shift
    elif [ "x$1" = "x-Xcompiler-option" ]; then
        shift
        option="$1"
        run_args="${run_args} -Xcompiler-option $option"
        shift
    elif [ "x$1" = "x--runtime-option" ]; then
        shift
        option="$1"
        run_args="${run_args} --runtime-option $option"
        shift
    elif [ "x$1" = "x--gdb-arg" ]; then
        shift
        gdb_arg="$1"
        run_args="${run_args} --gdb-arg $gdb_arg"
        shift
    elif [ "x$1" = "x--debug" ]; then
        run_args="${run_args} --debug"
        shift
    elif [ "x$1" = "x--gdb" ]; then
        run_args="${run_args} --gdb"
        dev_mode="yes"
        shift
    elif [ "x$1" = "x--zygote" ]; then
        run_args="${run_args} --zygote"
        shift
    elif [ "x$1" = "x--interpreter" ]; then
        run_args="${run_args} --interpreter"
        image_suffix="-interpreter"
        shift
    elif [ "x$1" = "x--jit" ]; then
        run_args="${run_args} --jit"
        image_suffix="-interpreter"
        shift
    elif [ "x$1" = "x--optimizing" ]; then
        run_args="${run_args} -Xcompiler-option --compiler-backend=Optimizing"
        image_suffix="-optimizing"
        shift
    elif [ "x$1" = "x--quick" ]; then
        run_args="${run_args} -Xcompiler-option --compiler-backend=Quick"
        shift
    elif [ "x$1" = "x--no-verify" ]; then
        run_args="${run_args} --no-verify"
        shift
    elif [ "x$1" = "x--no-optimize" ]; then
        run_args="${run_args} --no-optimize"
        shift
    elif [ "x$1" = "x--no-precise" ]; then
        run_args="${run_args} --no-precise"
        shift
    elif [ "x$1" = "x--invoke-with" ]; then
        shift
        what="$1"
        if [ "x$what" = "x" ]; then
            echo "$0 missing argument to --invoke-with" 1>&2
            usage="yes"
            break
        fi
        run_args="${run_args} --invoke-with ${what}"
        shift
    elif [ "x$1" = "x--dev" ]; then
        run_args="${run_args} --dev"
        dev_mode="yes"
        shift
    elif [ "x$1" = "x--build-only" ]; then
        build_only="yes"
        shift
    elif [ "x$1" = "x--build-with-javac-dx" ]; then
        USE_JACK="false"
        shift
    elif [ "x$1" = "x--build-with-jack" ]; then
        USE_JACK="true"
        shift
    elif [ "x$1" = "x--output-path" ]; then
        shift
        tmp_dir=$1
        if [ "x$tmp_dir" = "x" ]; then
            echo "$0 missing argument to --output-path" 1>&2
            usage="yes"
            break
        fi
        shift
    elif [ "x$1" = "x--android-root" ]; then
        shift
        if [ "x$1" = "x" ]; then
            echo "$0 missing argument to --android-root" 1>&2
            usage="yes"
            break
        fi
        android_root="$1"
        run_args="${run_args} --android-root $1"
        shift
    elif [ "x$1" = "x--update" ]; then
        update_mode="yes"
        shift
    elif [ "x$1" = "x--help" ]; then
        usage="yes"
        shift
    elif [ "x$1" = "x--64" ]; then
        run_args="${run_args} --64"
        suffix64="64"
        shift
    elif [ "x$1" = "x--trace" ]; then
        trace="true"
        shift
    elif [ "x$1" = "x--stream" ]; then
        trace_stream="true"
        shift
    elif [ "x$1" = "x--always-clean" ]; then
        always_clean="yes"
        shift
    elif [ "x$1" = "x--dex2oat-swap" ]; then
        run_args="${run_args} --dex2oat-swap"
        shift
    elif expr "x$1" : "x--" >/dev/null 2>&1; then
        echo "unknown $0 option: $1" 1>&2
        usage="yes"
        break
    else
        break
    fi
done

# tmp_dir may be relative, resolve.
#
# Cannot use realpath, as it does not exist on Mac.
# Cannot us a simple "cd", as the path might not be created yet.
# Cannot use readlink -m, as it does not exist on Mac.
# Fallback to nuclear option:
noncanonical_tmp_dir=$tmp_dir
tmp_dir="`cd $oldwd ; python -c "import os; print os.path.realpath('$tmp_dir')"`"
mkdir -p $tmp_dir

if [ "$basic_verify" = "true" ]; then
  # Set HspaceCompactForOOMMinIntervalMs to zero to run hspace compaction for OOM more frequently in tests.
  run_args="${run_args} --runtime-option -Xgc:preverify --runtime-option -Xgc:postverify --runtime-option -XX:HspaceCompactForOOMMinIntervalMs=0"
fi
if [ "$gc_verify" = "true" ]; then
  run_args="${run_args} --runtime-option -Xgc:preverify_rosalloc --runtime-option -Xgc:postverify_rosalloc"
fi
if [ "$gc_stress" = "true" ]; then
  run_args="${run_args} --runtime-option -Xgc:SS --runtime-option -Xms2m --runtime-option -Xmx2m"
fi
if [ "$trace" = "true" ]; then
    run_args="${run_args} --runtime-option -Xmethod-trace --runtime-option -Xmethod-trace-file-size:2000000"
    if [ "$trace_stream" = "true" ]; then
        # Streaming mode uses the file size as the buffer size. So output gets really large. Drop
        # the ability to analyze the file and just write to /dev/null.
        run_args="${run_args} --runtime-option -Xmethod-trace-file:/dev/null"
        # Enable streaming mode.
        run_args="${run_args} --runtime-option -Xmethod-trace-stream"
    else
        run_args="${run_args} --runtime-option -Xmethod-trace-file:${DEX_LOCATION}/trace.bin"
    fi
elif [ "$trace_stream" = "true" ]; then
    echo "Cannot use --stream without --trace."
    exit 1
fi

# Most interesting target architecture variables are Makefile variables, not environment variables.
# Try to map the suffix64 flag and what we find in ${ANDROID_PRODUCT_OUT}/data/art-test to an architecture name.
function guess_arch_name() {
    grep32bit=`ls ${ANDROID_PRODUCT_OUT}/data/art-test | grep -E '^(arm|x86|mips)$'`
    grep64bit=`ls ${ANDROID_PRODUCT_OUT}/data/art-test | grep -E '^(arm64|x86_64|mips64)$'`
    if [ "x${suffix64}" = "x64" ]; then
        target_arch_name=${grep64bit}
    else
        target_arch_name=${grep32bit}
    fi
}

if [ "$target_mode" = "no" ]; then
    if [ "$runtime" = "jvm" ]; then
        if [ "$prebuild_mode" = "yes" ]; then
            echo "--prebuild with --jvm is unsupported";
            exit 1;
        fi
    fi
fi

if [ "$have_patchoat" = "no" ]; then
  run_args="${run_args} --no-patchoat"
fi

if [ "$have_dex2oat" = "no" ]; then
  run_args="${run_args} --no-dex2oat"
fi

if [ ! "$runtime" = "jvm" ]; then
  run_args="${run_args} --lib $lib"
fi

if [ "$runtime" = "dalvik" ]; then
    if [ "$target_mode" = "no" ]; then
        framework="${ANDROID_PRODUCT_OUT}/system/framework"
        bpath="${framework}/core.jar:${framework}/conscrypt.jar:${framework}/okhttp.jar:${framework}/core-junit.jar:${framework}/bouncycastle.jar:${framework}/ext.jar"
        run_args="${run_args} --boot -Xbootclasspath:${bpath}"
    else
        true # defaults to using target BOOTCLASSPATH
    fi
elif [ "$runtime" = "art" ]; then
    if [ "$target_mode" = "no" ]; then
        # ANDROID_HOST_OUT is not set in a build environment.
        if [ -z "$ANDROID_HOST_OUT" ]; then
            export ANDROID_HOST_OUT=$ANDROID_BUILD_TOP/out/host/linux-x86
        fi
        run_args="${run_args} --boot ${ANDROID_HOST_OUT}/framework/core${image_suffix}${pic_image_suffix}.art"
        run_args="${run_args} --runtime-option -Djava.library.path=${ANDROID_HOST_OUT}/lib${suffix64}"
    else
        guess_arch_name
        run_args="${run_args} --runtime-option -Djava.library.path=/data/art-test/${target_arch_name}"
        run_args="${run_args} --boot /data/art-test/core${image_suffix}${pic_image_suffix}.art"
    fi
    if [ "$relocate" = "yes" ]; then
      run_args="${run_args} --relocate"
    else
      run_args="${run_args} --no-relocate"
    fi
elif [ "$runtime" = "jvm" ]; then
    # TODO: Detect whether the host is 32-bit or 64-bit.
    run_args="${run_args} --runtime-option -Djava.library.path=${ANDROID_HOST_OUT}/lib64"
fi

if [ "$have_image" = "no" ]; then
    if [ "$runtime" != "art" ]; then
        echo "--no-image is only supported on the art runtime"
        exit 1
    fi
    if [ "$target_mode" = "no" ]; then
        framework="${ANDROID_HOST_OUT}/framework"
        bpath_suffix="-hostdex"
    else
        framework="${android_root}/framework"
        bpath_suffix=""
    fi
    # TODO If the target was compiled WITH_DEXPREOPT=true then these tests will
    # fail since these jar files will be stripped.
    bpath="${framework}/core-libart${bpath_suffix}.jar"
    bpath="${bpath}:${framework}/conscrypt${bpath_suffix}.jar"
    bpath="${bpath}:${framework}/okhttp${bpath_suffix}.jar"
    bpath="${bpath}:${framework}/core-junit${bpath_suffix}.jar"
    bpath="${bpath}:${framework}/bouncycastle${bpath_suffix}.jar"
    # Pass down the bootclasspath
    run_args="${run_args} --runtime-option -Xbootclasspath:${bpath}"
    run_args="${run_args} --no-image"
fi

if [ "$dev_mode" = "yes" -a "$update_mode" = "yes" ]; then
    echo "--dev and --update are mutually exclusive" 1>&2
    usage="yes"
fi

if [ "$usage" = "no" ]; then
    if [ "x$1" = "x" -o "x$1" = "x-" ]; then
        test_dir=`basename "$oldwd"`
    else
        test_dir="$1"
    fi

    if [ '!' -d "$test_dir" ]; then
        td2=`echo ${test_dir}-*`
        if [ '!' -d "$td2" ]; then
            echo "${test_dir}: no such test directory" 1>&2
            usage="yes"
        fi
        test_dir="$td2"
    fi
    # Shift to get rid of the test name argument. The rest of the arguments
    # will get passed to the test run.
    shift
fi

if [ "$usage" = "yes" ]; then
    prog=`basename $prog`
    (
        echo "usage:"
        echo "  $prog --help                          Print this message."
        echo "  $prog [options] [test-name]           Run test normally."
        echo "  $prog --dev [options] [test-name]     Development mode" \
             "(dumps to stdout)."
        echo "  $prog --update [options] [test-name]  Update mode" \
             "(replaces expected.txt)."
        echo '  Omitting the test name or specifying "-" will use the' \
             "current directory."
        echo "  Runtime Options:"
        echo "    -O                    Run non-debug rather than debug build (off by default)."
        echo "    -Xcompiler-option     Pass an option to the compiler."
        echo "    --runtime-option      Pass an option to the runtime."
        echo "    --debug               Wait for a debugger to attach."
        echo "    --debuggable          Whether to compile Java code for a debugger."
        echo "    --gdb                 Run under gdb; incompatible with some tests."
        echo "    --build-only          Build test files only (off by default)."
        echo "    --build-with-javac-dx Build test files with javac and dx (on by default)."
        echo "    --build-with-jack     Build test files with jack and jill (off by default)."
        echo "    --interpreter         Enable interpreter only mode (off by default)."
        echo "    --jit                 Enable jit (off by default)."
        echo "    --optimizing          Enable optimizing compiler (default)."
        echo "    --quick               Use Quick compiler (off by default)."
        echo "    --no-verify           Turn off verification (on by default)."
        echo "    --no-optimize         Turn off optimization (on by default)."
        echo "    --no-precise          Turn off precise GC (on by default)."
        echo "    --zygote              Spawn the process from the Zygote." \
             "If used, then the"
        echo "                          other runtime options are ignored."
        echo "    --no-dex2oat          Run as though dex2oat was failing."
        echo "    --no-patchoat         Run as though patchoat was failing."
        echo "    --prebuild            Run dex2oat on the files before starting test. (default)"
        echo "    --no-prebuild         Do not run dex2oat on the files before starting"
        echo "                          the test."
        echo "    --relocate            Force the use of relocating in the test, making"
        echo "                          the image and oat files be relocated to a random"
        echo "                          address before running. (default)"
        echo "    --no-relocate         Force the use of no relocating in the test"
        echo "    --host                Use the host-mode virtual machine."
        echo "    --invoke-with         Pass --invoke-with option to runtime."
        echo "    --dalvik              Use Dalvik (off by default)."
        echo "    --jvm                 Use a host-local RI virtual machine."
        echo "    --output-path [path]  Location where to store the build" \
             "files."
        echo "    --64                  Run the test in 64-bit mode"
        echo "    --trace               Run with method tracing"
        echo "    --stream              Run method tracing in streaming mode (requires --trace)"
        echo "    --gcstress            Run with gc stress testing"
        echo "    --gcverify            Run with gc verification"
        echo "    --always-clean        Delete the test files even if the test fails."
        echo "    --android-root [path] The path on target for the android root. (/system by default)."
        echo "    --dex2oat-swap        Use a dex2oat swap file."
    ) 1>&2
    exit 1
fi

cd "$test_dir"
test_dir=`pwd`

td_info="${test_dir}/${info}"
td_expected="${test_dir}/${expected}"

if [ ! -r $td_info ]; then
    echo "${test_dir}: missing file $td_info" 1>&2
    exit 1
fi

if [ ! -r $td_expected ]; then
    echo "${test_dir}: missing file $td_expected" 1>&2
    exit 1
fi

# copy the test to a temp dir and run it

echo "${test_dir}: building..." 1>&2

rm -rf "$tmp_dir"
cp -Rp "$test_dir" "$tmp_dir"
cd "$tmp_dir"

if [ '!' -r "$build" ]; then
    cp "${progdir}/etc/default-build" build
fi

if [ '!' -r "$run" ]; then
    cp "${progdir}/etc/default-run" run
fi

if [ '!' -r "$check_cmd" ]; then
    cp "${progdir}/etc/default-check" check
fi

chmod 755 "$build"
chmod 755 "$run"
chmod 755 "$check_cmd"

export TEST_NAME=`basename ${test_dir}`

# Tests named '<number>-checker-*' will also have their CFGs verified with
# Checker when compiled with Optimizing on host.
if [[ "$TEST_NAME" =~ ^[0-9]+-checker- ]]; then
  # Build Checker DEX files without dx's optimizations so the input to dex2oat
  # better resembles the Java source. We always build the DEX the same way, even
  # if Checker is not invoked and the test only runs the program.
  build_args="${build_args} --dx-option --no-optimize"

  # Jack does not necessarily generate the same DEX output than dx. Because these tests depend
  # on a particular DEX output, keep building them with dx for now (b/19467889).
  USE_JACK="false"

  if [ "$runtime" = "art" -a "$image_suffix" = "-optimizing" -a "$target_mode" = "no" -a "$debuggable" = "no" ]; then
    run_checker="yes"
    run_args="${run_args} -Xcompiler-option --dump-cfg=$tmp_dir/$cfg_output \
                          -Xcompiler-option -j1"
  fi
fi

# To cause tests to fail fast, limit the file sizes created by dx, dex2oat and ART output to 2MB.
build_file_size_limit=2048
run_file_size_limit=2048
if echo "$test_dir" | grep 089; then
  build_file_size_limit=5120
  run_file_size_limit=5120
elif echo "$test_dir" | grep 083; then
  build_file_size_limit=5120
  run_file_size_limit=5120
fi
if [ ${USE_JACK} = "false" ]; then
  # Set ulimit if we build with dx only, Jack can generate big temp files.
  if ! ulimit -S "$build_file_size_limit"; then
    echo "ulimit file size setting failed"
  fi
fi

good="no"
good_build="yes"
good_run="yes"
if [ "$dev_mode" = "yes" ]; then
    "./${build}" $build_args 2>&1
    build_exit="$?"
    echo "build exit status: $build_exit" 1>&2
    if [ "$build_exit" = '0' ]; then
        if ! ulimit -S "$run_file_size_limit"; then
          echo "ulimit file size setting failed"
        fi
        echo "${test_dir}: running..." 1>&2
        "./${run}" $run_args "$@" 2>&1
        run_exit="$?"

        if [ "$run_exit" = "0" ]; then
            if [ "$run_checker" = "yes" ]; then
                "$checker" "$cfg_output" "$tmp_dir" 2>&1
                checker_exit="$?"
                if [ "$checker_exit" = "0" ]; then
                    good="yes"
                fi
                echo "checker exit status: $checker_exit" 1>&2
            else
                good="yes"
            fi
        fi
        echo "run exit status: $run_exit" 1>&2
    fi
elif [ "$update_mode" = "yes" ]; then
    "./${build}" $build_args >"$build_output" 2>&1
    build_exit="$?"
    if [ "$build_exit" = '0' ]; then
        if ! ulimit -S "$run_file_size_limit"; then
          echo "ulimit file size setting failed"
        fi
        echo "${test_dir}: running..." 1>&2
        "./${run}" $run_args "$@" >"$output" 2>&1
        if [ "$run_checker" = "yes" ]; then
          "$checker" -q "$cfg_output" "$tmp_dir" >> "$output" 2>&1
        fi
        sed -e 's/[[:cntrl:]]$//g' < "$output" >"${td_expected}"
        good="yes"
    else
        cat "$build_output" 1>&2
        echo "build exit status: $build_exit" 1>&2
    fi
elif [ "$build_only" = "yes" ]; then
    good="yes"
    "./${build}" $build_args >"$build_output" 2>&1
    build_exit="$?"
    if [ "$build_exit" '!=' '0' ]; then
        cp "$build_output" "$output"
        echo "build exit status: $build_exit" >>"$output"
        diff --strip-trailing-cr -q "$expected" "$output" >/dev/null
        if [ "$?" '!=' "0" ]; then
            good="no"
            echo "BUILD FAILED For ${TEST_NAME}"
        fi
    fi
    # Clean up extraneous files that are not used by tests.
    find $tmp_dir -mindepth 1  ! -regex ".*/\(.*jar\|$output\|$expected\)" | xargs rm -rf
    exit 0
else
    "./${build}" $build_args >"$build_output" 2>&1
    build_exit="$?"
    if [ "$build_exit" = '0' ]; then
        if ! ulimit -S "$run_file_size_limit"; then
          echo "ulimit file size setting failed"
        fi
        echo "${test_dir}: running..." 1>&2
        "./${run}" $run_args "$@" >"$output" 2>&1
        run_exit="$?"
        if [ "$run_exit" != "0" ]; then
            echo "run exit status: $run_exit" 1>&2
            good_run="no"
        elif [ "$run_checker" = "yes" ]; then
            "$checker" -q "$cfg_output" "$tmp_dir" >> "$output" 2>&1
            checker_exit="$?"
            if [ "$checker_exit" != "0" ]; then
                echo "checker exit status: $checker_exit" 1>&2
                good_run="no"
            else
                good_run="yes"
            fi
        else
            good_run="yes"
        fi
    else
        good_build="no"
        cp "$build_output" "$output"
        echo "Failed to build in tmpdir=${tmp_dir} from oldwd=${oldwd} and cwd=`pwd`" >> "$output"
        echo "Non-canonical tmpdir was ${noncanonical_tmp_dir}" >> "$output"
        echo "Args: ${args}" >> "$output"
        echo "build exit status: $build_exit" >> "$output"
        max_name_length=$(getconf NAME_MAX ${tmp_dir})
        echo "Max filename (NAME_MAX): ${max_name_length}" >> "$output"
        max_path_length=$(getconf PATH_MAX ${tmp_dir})
        echo "Max pathlength (PATH_MAX): ${max_path_length}" >> "$output"
    fi
    ./$check_cmd "$expected" "$output"
    if [ "$?" = "0" ]; then
        if [ "$good_build" = "no" -o "$good_run" = "yes" ]; then
          # output == expected
          good="yes"
          echo "${test_dir}: succeeded!" 1>&2
        fi
    fi
fi

(
    if [ "$good" != "yes" -a "$update_mode" != "yes" ]; then
        echo "${test_dir}: FAILED!"
        echo ' '
        echo '#################### info'
        cat "${td_info}" | sed 's/^/# /g'
        echo '#################### diffs'
        diff --strip-trailing-cr -u "$expected" "$output" | tail -n 2000
        echo '####################'
        echo ' '
    fi

) 1>&2

# Clean up test files.
if [ "$always_clean" = "yes" -o "$good" = "yes" ]; then
    cd "$oldwd"
    rm -rf "$tmp_dir"
    if [ "$target_mode" = "yes" -a "$build_exit" = "0" ]; then
        adb shell rm -rf $DEX_LOCATION
    fi
    if [ "$good" = "yes" ]; then
        exit 0
    fi
fi


(
    if [ "$always_clean" = "yes" ]; then
        echo "${TEST_NAME} files deleted from host "
        if [ "$target_mode" == "yes" ]; then
            echo "and from target"
        fi
    else
        echo "${TEST_NAME} files left in ${tmp_dir} on host"
        if [ "$target_mode" == "yes" ]; then
            echo "and in ${DEX_LOCATION} on target"
        fi
    fi

) 1>&2

exit 1