/* * Copyright (C) 2012 The Android Open Source Project * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.android.monkey; import com.android.ddmlib.CollectingOutputReceiver; import com.android.ddmlib.IShellOutputReceiver; import com.android.loganalysis.item.AnrItem; import com.android.loganalysis.item.BugreportItem; import com.android.loganalysis.item.MiscKernelLogItem; import com.android.loganalysis.item.MonkeyLogItem; import com.android.loganalysis.parser.BugreportParser; import com.android.loganalysis.parser.KernelLogParser; import com.android.loganalysis.parser.MonkeyLogParser; import com.android.tradefed.config.Option; import com.android.tradefed.config.Option.Importance; import com.android.tradefed.device.DeviceNotAvailableException; import com.android.tradefed.device.ITestDevice; import com.android.tradefed.log.LogUtil.CLog; import com.android.tradefed.metrics.proto.MetricMeasurement.Metric; import com.android.tradefed.result.ByteArrayInputStreamSource; import com.android.tradefed.result.DeviceFileReporter; import com.android.tradefed.result.FileInputStreamSource; import com.android.tradefed.result.ITestInvocationListener; import com.android.tradefed.result.InputStreamSource; import com.android.tradefed.result.LogDataType; import com.android.tradefed.result.TestDescription; import com.android.tradefed.testtype.IDeviceTest; import com.android.tradefed.testtype.IRemoteTest; import com.android.tradefed.testtype.IRetriableTest; import com.android.tradefed.util.ArrayUtil; import com.android.tradefed.util.Bugreport; import com.android.tradefed.util.CircularAtraceUtil; import com.android.tradefed.util.FileUtil; import com.android.tradefed.util.IRunUtil; import com.android.tradefed.util.RunUtil; import com.android.tradefed.util.StreamUtil; import org.junit.Assert; import java.io.BufferedReader; import java.io.File; import java.io.FileReader; import java.io.IOException; import java.io.InputStreamReader; import java.util.ArrayList; import java.util.Collection; import java.util.Date; import java.util.HashMap; import java.util.HashSet; import java.util.LinkedHashMap; import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.Random; import java.util.concurrent.TimeUnit; /** Runner for stress tests which use the monkey command. */ public class MonkeyBase implements IDeviceTest, IRemoteTest, IRetriableTest { public static final String MONKEY_LOG_NAME = "monkey_log"; public static final String BUGREPORT_NAME = "bugreport"; /** Allow a 15 second buffer between the monkey run time and the delta uptime. */ public static final long UPTIME_BUFFER = 15 * 1000; private static final String DEVICE_WHITELIST_PATH = "/data/local/tmp/monkey_whitelist.txt"; /** * am command template to launch app intent with same action, category and task flags as if user * started it from the app's launcher icon */ private static final String LAUNCH_APP_CMD = "am start -W -n '%s' " + "-a android.intent.action.MAIN -c android.intent.category.LAUNCHER -f 0x10200000"; private static final String NULL_UPTIME = "0.00"; /** * Helper to run a monkey command with an absolute timeout. * *

This is used so that the command can be stopped after a set timeout, since the timeout * that {@link ITestDevice#executeShellCommand(String, IShellOutputReceiver, long, TimeUnit, * int)} takes applies to the time between output, not the overall time of the command. */ private class CommandHelper { private DeviceNotAvailableException mException = null; private String mOutput = null; public void runCommand(final ITestDevice device, final String command, long timeout) throws DeviceNotAvailableException { final CollectingOutputReceiver receiver = new CollectingOutputReceiver(); Thread t = new Thread() { @Override public void run() { try { device.executeShellCommand(command, receiver); } catch (DeviceNotAvailableException e) { mException = e; } } }; t.start(); try { t.join(timeout); } catch (InterruptedException e) { // Ignore and log. The thread should terminate once receiver.cancel() is called. CLog.e("Thread was interrupted while running %s", command); } mOutput = receiver.getOutput(); receiver.cancel(); if (mException != null) { throw mException; } } public String getOutput() { return mOutput; } } @Option(name = "package", description = "Package name to send events to. May be repeated.") private Collection mPackages = new LinkedList<>(); @Option( name = "exclude-package", description = "Substring of package names to exclude from " + "the package list. May be repeated.", importance = Importance.IF_UNSET) private Collection mExcludePackages = new HashSet<>(); @Option(name = "category", description = "App Category. May be repeated.") private Collection mCategories = new LinkedList<>(); @Option(name = "option", description = "Option to pass to monkey command. May be repeated.") private Collection mOptions = new LinkedList<>(); @Option( name = "launch-extras-int", description = "Launch int extras. May be repeated. " + "Format: --launch-extras-i key value. Note: this will be applied to all components.") private Map mIntegerExtras = new HashMap<>(); @Option( name = "launch-extras-str", description = "Launch string extras. May be repeated. " + "Format: --launch-extras-s key value. Note: this will be applied to all components.") private Map mStringExtras = new HashMap<>(); @Option( name = "target-count", description = "Target number of events to send.", importance = Importance.ALWAYS) private int mTargetCount = 125000; @Option(name = "random-seed", description = "Random seed to use for the monkey.") private Long mRandomSeed = null; @Option( name = "throttle", description = "How much time to wait between sending successive " + "events, in msecs. Default is 0ms.") private int mThrottle = 0; @Option( name = "ignore-crashes", description = "Monkey should keep going after encountering " + "an app crash") private boolean mIgnoreCrashes = false; @Option( name = "ignore-timeout", description = "Monkey should keep going after encountering " + "an app timeout (ANR)") private boolean mIgnoreTimeouts = false; @Option( name = "reboot-device", description = "Reboot device before running monkey. Defaults " + "to true.") private boolean mRebootDevice = true; @Option(name = "idle-time", description = "How long to sleep before running monkey, in secs") private int mIdleTimeSecs = 5 * 60; @Option( name = "monkey-arg", description = "Extra parameters to pass onto monkey. Key/value " + "pairs should be passed as key:value. May be repeated.") private Collection mMonkeyArgs = new LinkedList<>(); @Option( name = "use-pkg-whitelist-file", description = "Whether to use the monkey " + "--pkg-whitelist-file option to work around cmdline length limits") private boolean mUseWhitelistFile = false; @Option( name = "monkey-timeout", description = "How long to wait for the monkey to " + "complete, in minutes. Default is 4 hours.") private int mMonkeyTimeout = 4 * 60; @Option( name = "warmup-component", description = "Component name of app to launch for " + "\"warming up\" before monkey test, will be used in an intent together with standard " + "flags and parameters as launched from Launcher. May be repeated") private List mLaunchComponents = new ArrayList<>(); @Option(name = "retry-on-failure", description = "Retry the test on failure") private boolean mRetryOnFailure = false; // FIXME: Remove this once traces.txt is no longer needed. @Option( name = "upload-file-pattern", description = "File glob of on-device files to upload " + "if found. Takes two arguments: the glob, and the file type " + "(text/xml/zip/gzip/png/unknown). May be repeated.") private Map mUploadFilePatterns = new LinkedHashMap<>(); @Option(name = "screenshot", description = "Take a device screenshot on monkey completion") private boolean mScreenshot = false; @Option( name = "ignore-security-exceptions", description = "Ignore SecurityExceptions while injecting events") private boolean mIgnoreSecurityExceptions = true; @Option( name = "collect-atrace", description = "Enable a continuous circular buffer to collect atrace information") private boolean mAtraceEnabled = false; // options for generating ANR report via post processing script @Option(name = "generate-anr-report", description = "Generate ANR report via post-processing") private boolean mGenerateAnrReport = false; @Option( name = "anr-report-script", description = "Path to the script for monkey ANR " + "report generation.") private String mAnrReportScriptPath = null; @Option( name = "anr-report-storage-backend-base-path", description = "Base path to the storage " + "backend used for saving the reports") private String mAnrReportBasePath = null; @Option( name = "anr-report-storage-backend-url-prefix", description = "URL prefix for the " + "storage backend that would enable web acess to the stored reports.") private String mAnrReportUrlPrefix = null; @Option( name = "anr-report-storage-path", description = "Sub path under the base storage " + "location for generated monkey ANR reports.") private String mAnrReportPath = null; private ITestDevice mTestDevice = null; private MonkeyLogItem mMonkeyLog = null; private BugreportItem mBugreport = null; private AnrReportGenerator mAnrGen = null; /** {@inheritDoc} */ @Override public void run(ITestInvocationListener listener) throws DeviceNotAvailableException { Assert.assertNotNull(getDevice()); TestDescription id = new TestDescription(getClass().getCanonicalName(), "monkey"); long startTime = System.currentTimeMillis(); listener.testRunStarted(getClass().getCanonicalName(), 1); listener.testStarted(id); try { runMonkey(listener); } finally { listener.testEnded(id, new HashMap()); listener.testRunEnded( System.currentTimeMillis() - startTime, new HashMap()); } } /** Returns the command that should be used to launch the app, */ private String getAppCmdWithExtras() { String extras = ""; for (Map.Entry sEntry : mStringExtras.entrySet()) { extras += String.format(" -e %s %s", sEntry.getKey(), sEntry.getValue()); } for (Map.Entry sEntry : mIntegerExtras.entrySet()) { extras += String.format(" --ei %s %d", sEntry.getKey(), sEntry.getValue()); } return LAUNCH_APP_CMD + extras; } /** Run the monkey one time and return a {@link MonkeyLogItem} for the run. */ protected void runMonkey(ITestInvocationListener listener) throws DeviceNotAvailableException { ITestDevice device = getDevice(); if (mRebootDevice) { CLog.v("Rebooting device prior to running Monkey"); device.reboot(); } else { CLog.v("Pre-run reboot disabled; skipping..."); } if (mIdleTimeSecs > 0) { CLog.i("Sleeping for %d seconds to allow device to settle...", mIdleTimeSecs); getRunUtil().sleep(mIdleTimeSecs * 1000); CLog.i("Done sleeping."); } // launch the list of apps that needs warm-up for (String componentName : mLaunchComponents) { getDevice().executeShellCommand(String.format(getAppCmdWithExtras(), componentName)); // give it some more time to settle down getRunUtil().sleep(5000); } if (mUseWhitelistFile) { // Use \r\n for new lines on the device. String whitelist = ArrayUtil.join("\r\n", setSubtract(mPackages, mExcludePackages)); device.pushString(whitelist.toString(), DEVICE_WHITELIST_PATH); } // Generate the monkey command to run, given the options String command = buildMonkeyCommand(); CLog.i("About to run monkey with at %d minute timeout: %s", mMonkeyTimeout, command); StringBuilder outputBuilder = new StringBuilder(); CommandHelper commandHelper = new CommandHelper(); long start = System.currentTimeMillis(); long duration = 0; Date dateAfter = null; String uptimeAfter = NULL_UPTIME; FileInputStreamSource atraceStream = null; // Generate the monkey log prefix, which includes the device uptime outputBuilder.append( String.format( "# %s - device uptime = %s: Monkey command used " + "for this test:\nadb shell %s\n\n", new Date().toString(), getUptime(), command)); // Start atrace before running the monkey command, but after reboot if (mAtraceEnabled) { CircularAtraceUtil.startTrace(getDevice(), null, 10); } if (mGenerateAnrReport) { mAnrGen = new AnrReportGenerator( mAnrReportScriptPath, mAnrReportBasePath, mAnrReportUrlPrefix, mAnrReportPath, mTestDevice.getBuildId(), mTestDevice.getBuildFlavor(), mTestDevice.getSerialNumber()); } try { onMonkeyStart(); commandHelper.runCommand(mTestDevice, command, getMonkeyTimeoutMs()); } finally { // Wait for device to recover if it's not online. If it hasn't recovered, ignore. try { mTestDevice.waitForDeviceOnline(); mTestDevice.enableAdbRoot(); duration = System.currentTimeMillis() - start; dateAfter = new Date(); uptimeAfter = getUptime(); onMonkeyFinish(); takeScreenshot(listener, "screenshot"); if (mAtraceEnabled) { atraceStream = CircularAtraceUtil.endTrace(getDevice()); } mBugreport = takeBugreport(listener, BUGREPORT_NAME); // FIXME: Remove this once traces.txt is no longer needed. takeTraces(listener); } finally { // @@@ DO NOT add anything that requires device interaction into this block @@@ // @@@ logging that no longer requires device interaction MUST be in this block @@@ outputBuilder.append(commandHelper.getOutput()); if (dateAfter == null) { dateAfter = new Date(); } // Generate the monkey log suffix, which includes the device uptime. outputBuilder.append( String.format( "\n# %s - device uptime = %s: Monkey command " + "ran for: %d:%02d (mm:ss)\n", dateAfter.toString(), uptimeAfter, duration / 1000 / 60, duration / 1000 % 60)); mMonkeyLog = createMonkeyLog(listener, MONKEY_LOG_NAME, outputBuilder.toString()); boolean isAnr = mMonkeyLog.getCrash() instanceof AnrItem; if (mAtraceEnabled && isAnr) { // This was identified as an ANR; post atrace data listener.testLog("circular-atrace", LogDataType.TEXT, atraceStream); } if (mAnrGen != null) { if (isAnr) { if (!mAnrGen.genereateAnrReport(listener)) { CLog.w("Failed to post-process ANR."); } else { CLog.i("Successfully post-processed ANR."); } mAnrGen.cleanTempFiles(); } else { CLog.d("ANR post-processing enabled but no ANR detected."); } } StreamUtil.cancel(atraceStream); } } // Extra logs for what was found if (mBugreport != null && mBugreport.getLastKmsg() != null) { List kernelErrors = mBugreport.getLastKmsg().getMiscEvents(KernelLogParser.KERNEL_ERROR); List kernelResets = mBugreport.getLastKmsg().getMiscEvents(KernelLogParser.KERNEL_ERROR); CLog.d( "Found %d kernel errors and %d kernel resets in last kmsg", kernelErrors.size(), kernelResets.size()); for (int i = 0; i < kernelErrors.size(); i++) { String stack = kernelErrors.get(i).getStack(); if (stack != null) { CLog.d("Kernel Error #%d: %s", i + 1, stack.split("\n")[0].trim()); } } for (int i = 0; i < kernelResets.size(); i++) { String stack = kernelResets.get(i).getStack(); if (stack != null) { CLog.d("Kernel Reset #%d: %s", i + 1, stack.split("\n")[0].trim()); } } } checkResults(); } /** A hook to allow subclasses to perform actions just before the monkey starts. */ protected void onMonkeyStart() { // empty } /** A hook to allow sublaccess to perform actions just after the monkey finished. */ protected void onMonkeyFinish() { // empty } /** * If enabled, capture a screenshot and send it to a listener. * * @throws DeviceNotAvailableException */ protected void takeScreenshot(ITestInvocationListener listener, String screenshotName) throws DeviceNotAvailableException { if (mScreenshot) { try (InputStreamSource screenshot = mTestDevice.getScreenshot("JPEG")) { listener.testLog(screenshotName, LogDataType.JPEG, screenshot); } } } /** Capture a bugreport and send it to a listener. */ protected BugreportItem takeBugreport(ITestInvocationListener listener, String bugreportName) { Bugreport bugreport = mTestDevice.takeBugreport(); if (bugreport == null) { CLog.e("Could not take bugreport"); return null; } bugreport.log(bugreportName, listener); File main = null; InputStreamSource is = null; try { main = bugreport.getMainFile(); if (main == null) { CLog.e("Bugreport has no main file"); return null; } if (mAnrGen != null) { is = new FileInputStreamSource(main); mAnrGen.setBugReportInfo(is); } return new BugreportParser().parse(new BufferedReader(new FileReader(main))); } catch (IOException e) { CLog.e("Could not process bugreport"); CLog.e(e); return null; } finally { StreamUtil.close(bugreport); StreamUtil.cancel(is); FileUtil.deleteFile(main); } } protected void takeTraces(ITestInvocationListener listener) { DeviceFileReporter dfr = new DeviceFileReporter(mTestDevice, listener); dfr.addPatterns(mUploadFilePatterns); try { dfr.run(); } catch (DeviceNotAvailableException e) { // Log but don't throw CLog.e( "Device %s became unresponsive while pulling files", mTestDevice.getSerialNumber()); } } /** Create the monkey log, parse it, and send it to a listener. */ protected MonkeyLogItem createMonkeyLog( ITestInvocationListener listener, String monkeyLogName, String log) { try (InputStreamSource source = new ByteArrayInputStreamSource(log.getBytes())) { if (mAnrGen != null) { mAnrGen.setMonkeyLogInfo(source); } listener.testLog(monkeyLogName, LogDataType.MONKEY_LOG, source); return new MonkeyLogParser() .parse(new BufferedReader(new InputStreamReader(source.createInputStream()))); } catch (IOException e) { CLog.e("Could not process monkey log."); CLog.e(e); return null; } } /** * A helper method to build a monkey command given the specified arguments. * *

Actual output argument order is: {@code monkey [-p PACKAGE]... [-c CATEGORY]... * [--OPTION]... -s SEED -v -v -v COUNT} * * @return a {@link String} containing the command with the arguments assembled in the proper * order. */ protected String buildMonkeyCommand() { List cmdList = new LinkedList<>(); cmdList.add("monkey"); if (!mUseWhitelistFile) { for (String pkg : setSubtract(mPackages, mExcludePackages)) { cmdList.add("-p"); cmdList.add(pkg); } } for (String cat : mCategories) { cmdList.add("-c"); cmdList.add(cat); } if (mIgnoreSecurityExceptions) { cmdList.add("--ignore-security-exceptions"); } if (mThrottle >= 1) { cmdList.add("--throttle"); cmdList.add(Integer.toString(mThrottle)); } if (mIgnoreCrashes) { cmdList.add("--ignore-crashes"); } if (mIgnoreTimeouts) { cmdList.add("--ignore-timeouts"); } if (mUseWhitelistFile) { cmdList.add("--pkg-whitelist-file"); cmdList.add(DEVICE_WHITELIST_PATH); } for (String arg : mMonkeyArgs) { String[] args = arg.split(":"); cmdList.add(String.format("--%s", args[0])); if (args.length > 1) { cmdList.add(args[1]); } } cmdList.addAll(mOptions); cmdList.add("-s"); if (mRandomSeed == null) { // Pick a number that is random, but in a small enough range that some seeds are likely // to be repeated cmdList.add(Long.toString(new Random().nextInt(1000))); } else { cmdList.add(Long.toString(mRandomSeed)); } // verbose cmdList.add("-v"); cmdList.add("-v"); cmdList.add("-v"); cmdList.add(Integer.toString(mTargetCount)); return ArrayUtil.join(" ", cmdList); } /** * Get a {@link String} containing the number seconds since the device was booted. * *

{@code NULL_UPTIME} is returned if the device becomes unresponsive. Used in the monkey log * prefix and suffix. */ protected String getUptime() { try { // make two attempts to get valid uptime for (int i = 0; i < 2; i++) { // uptime will typically have a format like "5278.73 1866.80". Use the first one // (which is wall-time) String uptime = mTestDevice.executeShellCommand("cat /proc/uptime").split(" ")[0]; try { Float.parseFloat(uptime); // if this parsed, its a valid uptime return uptime; } catch (NumberFormatException e) { CLog.w( "failed to get valid uptime from %s. Received: '%s'", mTestDevice.getSerialNumber(), uptime); } } } catch (DeviceNotAvailableException e) { CLog.e( "Device %s became unresponsive while getting the uptime.", mTestDevice.getSerialNumber()); } return NULL_UPTIME; } /** * Perform set subtraction between two {@link Collection} objects. * *

The return value will consist of all of the elements of {@code keep}, excluding the * elements that are also in {@code exclude}. Exposed for unit testing. * * @param keep the minuend in the subtraction * @param exclude the subtrahend * @return the collection of elements in {@code keep} that are not also in {@code exclude}. If * {@code keep} is an ordered {@link Collection}, the remaining elements in the return value * will remain in their original order. */ static Collection setSubtract(Collection keep, Collection exclude) { if (exclude.isEmpty()) { return keep; } Collection output = new ArrayList<>(keep); output.removeAll(exclude); return output; } /** Get {@link IRunUtil} to use. Exposed for unit testing. */ IRunUtil getRunUtil() { return RunUtil.getDefault(); } /** {@inheritDoc} */ @Override public void setDevice(ITestDevice device) { mTestDevice = device; } /** {@inheritDoc} */ @Override public ITestDevice getDevice() { return mTestDevice; } /** * {@inheritDoc} * * @return {@code false} if retry-on-failure is not set, if the monkey ran to completion, * crashed in an understood way, or if there were no packages to run, {@code true} * otherwise. */ @Override public boolean isRetriable() { return mRetryOnFailure; } /** Check the results and return if valid or throw an assertion error if not valid. */ private void checkResults() { Assert.assertNotNull("Monkey log is null", mMonkeyLog); Assert.assertNotNull("Bugreport is null", mBugreport); Assert.assertNotNull("Bugreport is empty", mBugreport.getTime()); // If there are no activities, retrying the test won't matter. if (mMonkeyLog.getNoActivities()) { return; } Assert.assertNotNull("Start uptime is missing", mMonkeyLog.getStartUptimeDuration()); Assert.assertNotNull("Stop uptime is missing", mMonkeyLog.getStopUptimeDuration()); Assert.assertNotNull("Total duration is missing", mMonkeyLog.getTotalDuration()); long startUptime = mMonkeyLog.getStartUptimeDuration(); long stopUptime = mMonkeyLog.getStopUptimeDuration(); long totalDuration = mMonkeyLog.getTotalDuration(); Assert.assertTrue( "Uptime failure", stopUptime - startUptime > totalDuration - UPTIME_BUFFER); // False count Assert.assertFalse( "False count", mMonkeyLog.getIsFinished() && mMonkeyLog.getTargetCount() - mMonkeyLog.getIntermediateCount() > 100); // Monkey finished or crashed, so don't fail if (mMonkeyLog.getIsFinished() || mMonkeyLog.getFinalCount() != null) { return; } // Missing count Assert.fail("Missing count"); } /** Get the monkey timeout in milliseconds */ protected long getMonkeyTimeoutMs() { return mMonkeyTimeout * 60 * 1000; } }