1 /* 2 * Copyright (C) 2017 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package com.android.tradefed.result; 18 19 import com.android.ddmlib.Log; 20 import com.android.ddmlib.Log.LogLevel; 21 import com.android.ddmlib.testrunner.TestResult.TestStatus; 22 import com.android.tradefed.config.Option; 23 import com.android.tradefed.config.OptionClass; 24 import com.android.tradefed.util.FileUtil; 25 import com.android.tradefed.util.StreamUtil; 26 27 import com.google.common.annotations.VisibleForTesting; 28 29 import org.kxml2.io.KXmlSerializer; 30 31 import java.io.BufferedOutputStream; 32 import java.io.File; 33 import java.io.FileOutputStream; 34 import java.io.IOException; 35 import java.io.OutputStream; 36 import java.text.SimpleDateFormat; 37 import java.util.Date; 38 import java.util.Locale; 39 import java.util.Map; 40 import java.util.TimeZone; 41 42 /** 43 * MetricsXMLResultReporter writes test metrics and run metrics to an XML file in a folder specified 44 * by metrics-folder parameter at the invocationEnded phase of the test. The XML file will be piped 45 * into an algorithm to detect regression. 46 * 47 * <p>All k-v paris in run metrics map will be formatted into: <runmetric name="name" value="value" 48 * /> and placed under <testsuite/> tag 49 * 50 * <p>All k-v paris in run metrics map will be formatted into: <testmetric name="name" value="value" 51 * /> and placed under <testcase/> tag, a tag nested under <testsuite/>. 52 * 53 * <p>A sample XML format: <testsuite name="suite" tests="1" failures="0" time="10" 54 * timestamp="2017-01-01T01:00:00"> <runmetric name="sample" value="1.0" /> <testcase 55 * testname="test" classname="classname" time="2"> <testmetric name="sample" value="1.0" /> 56 * </testcase> </testsuite> 57 */ 58 @OptionClass(alias = "metricsreporter") 59 public class MetricsXMLResultReporter extends CollectingTestListener { 60 61 private static final String TAG = "MetricsXMLResultReporter"; 62 private static final String METRICS_PREFIX = "metrics-"; 63 private static final String TAG_TESTSUITE = "testsuite"; 64 private static final String TAG_TESTCASE = "testcase"; 65 private static final String TAG_RUN_METRIC = "runmetric"; 66 private static final String TAG_TEST_METRIC = "testmetric"; 67 private static final String ATTR_NAME = "name"; 68 private static final String ATTR_VALUE = "value"; 69 private static final String ATTR_TESTNAME = "testname"; 70 private static final String ATTR_TIME = "time"; 71 private static final String ATTR_FAILURES = "failures"; 72 private static final String ATTR_TESTS = "tests"; 73 private static final String ATTR_CLASSNAME = "classname"; 74 private static final String ATTR_TIMESTAMP = "timestamp"; 75 /** the XML namespace */ 76 private static final String NS = null; 77 78 @Option(name = "metrics-folder", description = "The folder to save metrics files") 79 private File mFolder; 80 81 private File mLog; 82 83 @Override invocationEnded(long elapsedTime)84 public void invocationEnded(long elapsedTime) { 85 super.invocationEnded(elapsedTime); 86 if (mFolder == null) { 87 Log.w(TAG, "metrics-folder not specified, unable to record metrics"); 88 return; 89 } 90 generateResults(elapsedTime); 91 } 92 generateResults(long elapsedTime)93 private void generateResults(long elapsedTime) { 94 String timestamp = getTimeStamp(); 95 OutputStream os = null; 96 97 try { 98 os = createOutputStream(); 99 if (os == null) { 100 return; 101 } 102 KXmlSerializer serializer = new KXmlSerializer(); 103 serializer.setOutput(os, "UTF-8"); 104 serializer.startDocument("UTF-8", null); 105 serializer.setFeature("http://xmlpull.org/v1/doc/features.html#indent-output", true); 106 printRunResults(serializer, timestamp, elapsedTime); 107 serializer.endDocument(); 108 if (mLog != null) { 109 String msg = 110 String.format( 111 Locale.US, 112 "XML metrics report generated at %s. " 113 + "Total tests %d, Failed %d", 114 mLog.getPath(), 115 getNumTotalTests(), 116 getNumAllFailedTests()); 117 Log.logAndDisplay(LogLevel.INFO, TAG, msg); 118 } 119 } catch (IOException e) { 120 Log.e(TAG, "Failed to generate XML metric report"); 121 throw new RuntimeException(e); 122 } finally { 123 StreamUtil.close(os); 124 } 125 } 126 printRunResults(KXmlSerializer serializer, String timestamp, long elapsedTime)127 private void printRunResults(KXmlSerializer serializer, String timestamp, long elapsedTime) 128 throws IOException { 129 serializer.startTag(NS, TAG_TESTSUITE); 130 serializer.attribute(NS, ATTR_NAME, getInvocationContext().getTestTag()); 131 serializer.attribute(NS, ATTR_TESTS, Integer.toString(getNumTotalTests())); 132 serializer.attribute(NS, ATTR_FAILURES, Integer.toString(getNumAllFailedTests())); 133 serializer.attribute(NS, ATTR_TIME, Long.toString(elapsedTime)); 134 serializer.attribute(NS, ATTR_TIMESTAMP, timestamp); 135 136 for (TestRunResult runResult : getMergedTestRunResults()) { 137 printRunMetrics(serializer, runResult.getRunMetrics()); 138 Map<TestDescription, TestResult> testResults = runResult.getTestResults(); 139 for (TestDescription test : testResults.keySet()) { 140 printTestResults(serializer, test, testResults.get(test)); 141 } 142 } 143 144 serializer.endTag(NS, TAG_TESTSUITE); 145 } 146 printTestResults( KXmlSerializer serializer, TestDescription testId, TestResult testResult)147 private void printTestResults( 148 KXmlSerializer serializer, TestDescription testId, TestResult testResult) 149 throws IOException { 150 serializer.startTag(NS, TAG_TESTCASE); 151 serializer.attribute(NS, ATTR_TESTNAME, testId.getTestName()); 152 serializer.attribute(NS, ATTR_CLASSNAME, testId.getClassName()); 153 long elapsedTime = testResult.getEndTime() - testResult.getStartTime(); 154 serializer.attribute(NS, ATTR_TIME, Long.toString(elapsedTime)); 155 156 printTestMetrics(serializer, testResult.getMetrics()); 157 158 if (!TestStatus.PASSED.equals(testResult.getStatus())) { 159 String result = testResult.getStatus().name(); 160 serializer.startTag(NS, result); 161 String stackText = sanitize(testResult.getStackTrace()); 162 serializer.text(stackText); 163 serializer.endTag(NS, result); 164 } 165 166 serializer.endTag(NS, TAG_TESTCASE); 167 } 168 printRunMetrics(KXmlSerializer serializer, Map<String, String> metrics)169 private void printRunMetrics(KXmlSerializer serializer, Map<String, String> metrics) 170 throws IOException { 171 for (String key : metrics.keySet()) { 172 serializer.startTag(NS, TAG_RUN_METRIC); 173 serializer.attribute(NS, ATTR_NAME, key); 174 serializer.attribute(NS, ATTR_VALUE, metrics.get(key)); 175 serializer.endTag(NS, TAG_RUN_METRIC); 176 } 177 } 178 printTestMetrics(KXmlSerializer serializer, Map<String, String> metrics)179 private void printTestMetrics(KXmlSerializer serializer, Map<String, String> metrics) 180 throws IOException { 181 for (String key : metrics.keySet()) { 182 serializer.startTag(NS, TAG_TEST_METRIC); 183 serializer.attribute(NS, ATTR_NAME, key); 184 serializer.attribute(NS, ATTR_VALUE, metrics.get(key)); 185 serializer.endTag(NS, TAG_TEST_METRIC); 186 } 187 } 188 189 @VisibleForTesting createOutputStream()190 public OutputStream createOutputStream() throws IOException { 191 if (!mFolder.exists() && !mFolder.mkdirs()) { 192 throw new IOException(String.format("Unable to create metrics directory: %s", mFolder)); 193 } 194 mLog = FileUtil.createTempFile(METRICS_PREFIX, ".xml", mFolder); 195 return new BufferedOutputStream(new FileOutputStream(mLog)); 196 } 197 198 /** Returns the text in a format that is safe for use in an XML document. */ sanitize(String text)199 private String sanitize(String text) { 200 return text.replace("\0", "<\\0>"); 201 } 202 203 /** Return the current timestamp as a {@link String}. */ 204 @VisibleForTesting getTimeStamp()205 public String getTimeStamp() { 206 SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss"); 207 dateFormat.setTimeZone(TimeZone.getTimeZone("UTC")); 208 dateFormat.setLenient(true); 209 return dateFormat.format(new Date()); 210 } 211 } 212