1 /* 2 * Copyright (C) 2017 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package com.android.tradefed.result; 18 19 import com.android.tradefed.config.Option; 20 import com.android.tradefed.config.OptionClass; 21 import com.android.tradefed.log.LogUtil.CLog; 22 import com.android.tradefed.util.FileUtil; 23 import com.android.tradefed.util.StreamUtil; 24 25 import com.google.common.annotations.VisibleForTesting; 26 27 import org.kxml2.io.KXmlSerializer; 28 29 import java.io.BufferedOutputStream; 30 import java.io.File; 31 import java.io.FileOutputStream; 32 import java.io.IOException; 33 import java.io.OutputStream; 34 import java.text.SimpleDateFormat; 35 import java.util.Date; 36 import java.util.Map; 37 import java.util.TimeZone; 38 39 /** 40 * MetricsXMLResultReporter writes test metrics and run metrics to an XML file in a folder specified 41 * by metrics-folder parameter at the invocationEnded phase of the test. The XML file will be piped 42 * into an algorithm to detect regression. 43 * 44 * <p>All k-v paris in run metrics map will be formatted into: <runmetric name="name" value="value" 45 * /> and placed under <testsuite/> tag 46 * 47 * <p>All k-v paris in run metrics map will be formatted into: <testmetric name="name" value="value" 48 * /> and placed under <testcase/> tag, a tag nested under <testsuite/>. 49 * 50 * <p>A sample XML format: <testsuite name="suite" tests="1" failures="0" time="10" 51 * timestamp="2017-01-01T01:00:00"> <runmetric name="sample" value="1.0" /> <testcase 52 * testname="test" classname="classname" time="2"> <testmetric name="sample" value="1.0" /> 53 * </testcase> </testsuite> 54 */ 55 @OptionClass(alias = "metricsreporter") 56 public class MetricsXMLResultReporter extends CollectingTestListener { 57 58 private static final String METRICS_PREFIX = "metrics-"; 59 private static final String TAG_TESTSUITE = "testsuite"; 60 private static final String TAG_TESTCASE = "testcase"; 61 private static final String TAG_RUN_METRIC = "runmetric"; 62 private static final String TAG_TEST_METRIC = "testmetric"; 63 private static final String ATTR_NAME = "name"; 64 private static final String ATTR_VALUE = "value"; 65 private static final String ATTR_TESTNAME = "testname"; 66 private static final String ATTR_TIME = "time"; 67 private static final String ATTR_FAILURES = "failures"; 68 private static final String ATTR_TESTS = "tests"; 69 private static final String ATTR_CLASSNAME = "classname"; 70 private static final String ATTR_TIMESTAMP = "timestamp"; 71 /** the XML namespace */ 72 private static final String NS = null; 73 74 @Option(name = "metrics-folder", description = "The folder to save metrics files") 75 private File mFolder; 76 77 private File mLog; 78 79 @Override invocationEnded(long elapsedTime)80 public void invocationEnded(long elapsedTime) { 81 super.invocationEnded(elapsedTime); 82 if (mFolder == null) { 83 CLog.w("metrics-folder not specified, unable to record metrics"); 84 return; 85 } 86 generateResults(elapsedTime); 87 } 88 generateResults(long elapsedTime)89 private void generateResults(long elapsedTime) { 90 String timestamp = getTimeStamp(); 91 OutputStream os = null; 92 93 try { 94 os = createOutputStream(); 95 if (os == null) { 96 return; 97 } 98 KXmlSerializer serializer = new KXmlSerializer(); 99 serializer.setOutput(os, "UTF-8"); 100 serializer.startDocument("UTF-8", null); 101 serializer.setFeature("http://xmlpull.org/v1/doc/features.html#indent-output", true); 102 printRunResults(serializer, timestamp, elapsedTime); 103 serializer.endDocument(); 104 if (mLog != null) { 105 CLog.i( 106 "XML metrics report generated at %s. " + "Total tests %d, Failed %d", 107 mLog.getPath(), getNumTotalTests(), getNumAllFailedTests()); 108 } 109 } catch (IOException e) { 110 CLog.e("Failed to generate XML metric report"); 111 throw new RuntimeException(e); 112 } finally { 113 StreamUtil.close(os); 114 } 115 } 116 printRunResults(KXmlSerializer serializer, String timestamp, long elapsedTime)117 private void printRunResults(KXmlSerializer serializer, String timestamp, long elapsedTime) 118 throws IOException { 119 serializer.startTag(NS, TAG_TESTSUITE); 120 serializer.attribute(NS, ATTR_NAME, getInvocationContext().getTestTag()); 121 serializer.attribute(NS, ATTR_TESTS, Integer.toString(getNumTotalTests())); 122 serializer.attribute(NS, ATTR_FAILURES, Integer.toString(getNumAllFailedTests())); 123 serializer.attribute(NS, ATTR_TIME, Long.toString(elapsedTime)); 124 serializer.attribute(NS, ATTR_TIMESTAMP, timestamp); 125 126 for (TestRunResult runResult : getMergedTestRunResults()) { 127 printRunMetrics(serializer, runResult.getRunMetrics()); 128 Map<TestDescription, TestResult> testResults = runResult.getTestResults(); 129 for (TestDescription test : testResults.keySet()) { 130 printTestResults(serializer, test, testResults.get(test)); 131 } 132 } 133 134 serializer.endTag(NS, TAG_TESTSUITE); 135 } 136 printTestResults( KXmlSerializer serializer, TestDescription testId, TestResult testResult)137 private void printTestResults( 138 KXmlSerializer serializer, TestDescription testId, TestResult testResult) 139 throws IOException { 140 serializer.startTag(NS, TAG_TESTCASE); 141 serializer.attribute(NS, ATTR_TESTNAME, testId.getTestName()); 142 serializer.attribute(NS, ATTR_CLASSNAME, testId.getClassName()); 143 long elapsedTime = testResult.getEndTime() - testResult.getStartTime(); 144 serializer.attribute(NS, ATTR_TIME, Long.toString(elapsedTime)); 145 146 printTestMetrics(serializer, testResult.getMetrics()); 147 148 if (!TestStatus.PASSED.equals(testResult.getResultStatus())) { 149 String result = testResult.getStatus().name(); 150 serializer.startTag(NS, result); 151 String stackText = sanitize(testResult.getStackTrace()); 152 serializer.text(stackText); 153 serializer.endTag(NS, result); 154 } 155 156 serializer.endTag(NS, TAG_TESTCASE); 157 } 158 printRunMetrics(KXmlSerializer serializer, Map<String, String> metrics)159 private void printRunMetrics(KXmlSerializer serializer, Map<String, String> metrics) 160 throws IOException { 161 for (String key : metrics.keySet()) { 162 serializer.startTag(NS, TAG_RUN_METRIC); 163 serializer.attribute(NS, ATTR_NAME, key); 164 serializer.attribute(NS, ATTR_VALUE, metrics.get(key)); 165 serializer.endTag(NS, TAG_RUN_METRIC); 166 } 167 } 168 printTestMetrics(KXmlSerializer serializer, Map<String, String> metrics)169 private void printTestMetrics(KXmlSerializer serializer, Map<String, String> metrics) 170 throws IOException { 171 for (String key : metrics.keySet()) { 172 serializer.startTag(NS, TAG_TEST_METRIC); 173 serializer.attribute(NS, ATTR_NAME, key); 174 serializer.attribute(NS, ATTR_VALUE, metrics.get(key)); 175 serializer.endTag(NS, TAG_TEST_METRIC); 176 } 177 } 178 179 @VisibleForTesting createOutputStream()180 public OutputStream createOutputStream() throws IOException { 181 if (!mFolder.exists() && !mFolder.mkdirs()) { 182 throw new IOException(String.format("Unable to create metrics directory: %s", mFolder)); 183 } 184 mLog = FileUtil.createTempFile(METRICS_PREFIX, ".xml", mFolder); 185 return new BufferedOutputStream(new FileOutputStream(mLog)); 186 } 187 188 /** Returns the text in a format that is safe for use in an XML document. */ sanitize(String text)189 private String sanitize(String text) { 190 return text == null ? "" : text.replace("\0", "<\\0>"); 191 } 192 193 /** Return the current timestamp as a {@link String}. */ 194 @VisibleForTesting getTimeStamp()195 public String getTimeStamp() { 196 SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss"); 197 dateFormat.setTimeZone(TimeZone.getTimeZone("UTC")); 198 dateFormat.setLenient(true); 199 return dateFormat.format(new Date()); 200 } 201 } 202