1 /*******************************************************************************
2 * Copyright (c) 2000, 2009 IBM Corporation and others.
3 * All rights reserved. This program and the accompanying materials
4 * are made available under the terms of the Eclipse Public License v1.0
5 * which accompanies this distribution, and is available at
6 * http://www.eclipse.org/legal/epl-v10.html
7 *
8 * Contributors:
9 * IBM Corporation - initial API and implementation
10 *******************************************************************************/
11 package org.eclipse.test.performance.ui;
12
13 import java.io.BufferedOutputStream;
14 import java.io.File;
15 import java.io.FileNotFoundException;
16 import java.io.FileOutputStream;
17 import java.io.IOException;
18 import java.io.OutputStream;
19 import java.io.PrintStream;
20 import java.util.ArrayList;
21 import java.util.Iterator;
22 import java.util.List;
23
24 import junit.framework.AssertionFailedError;
25
26 import org.eclipse.core.runtime.OperationCanceledException;
27 import org.eclipse.core.runtime.SubMonitor;
28 import org.eclipse.swt.SWT;
29 import org.eclipse.swt.graphics.Color;
30 import org.eclipse.swt.graphics.Image;
31 import org.eclipse.swt.graphics.ImageData;
32 import org.eclipse.swt.graphics.ImageLoader;
33 import org.eclipse.swt.widgets.Display;
34 import org.eclipse.test.internal.performance.data.Dim;
35 import org.eclipse.test.internal.performance.results.db.BuildResults;
36 import org.eclipse.test.internal.performance.results.db.ComponentResults;
37 import org.eclipse.test.internal.performance.results.db.ConfigResults;
38 import org.eclipse.test.internal.performance.results.db.DB_Results;
39 import org.eclipse.test.internal.performance.results.db.PerformanceResults;
40 import org.eclipse.test.internal.performance.results.db.ScenarioResults;
41 import org.eclipse.test.internal.performance.results.utils.Util;
42
43 /**
44 * Class used to print scenario all builds data.
45 */
46 public class ScenarioData {
47 private String baselinePrefix = null;
48 private List pointsOfInterest;
49 private List buildIDStreamPatterns;
50 private File rootDir;
51 private static final int GRAPH_WIDTH = 600;
52 private static final int GRAPH_HEIGHT = 200;
53 private Dim[] dimensions = DB_Results.getResultsDimensions();
54
55 /**
56 * Summary of results for a scenario for a given build compared to a
57 * reference.
58 *
59 * @param baselinePrefix The prefix of the baseline build names
60 * @param pointsOfInterest A list of buildId's to highlight on line graphs
61 * @param buildIDPatterns
62 * @param outputDir The directory root where the files are generated
63 *
64 */
ScenarioData(String baselinePrefix, List pointsOfInterest, List buildIDPatterns, File outputDir)65 public ScenarioData(String baselinePrefix, List pointsOfInterest, List buildIDPatterns, File outputDir) {
66 this.baselinePrefix = baselinePrefix;
67 this.pointsOfInterest = pointsOfInterest;
68 this.buildIDStreamPatterns = buildIDPatterns;
69 this.rootDir = outputDir;
70 }
71
72 /*
73 * Create a file handle verifying that its name does not go over
74 * the maximum authorized length.
75 */
createFile(File outputDir, String subdir, String name, String extension)76 private File createFile(File outputDir, String subdir, String name, String extension) {
77 File dir = outputDir;
78 if (subdir != null) {
79 dir = new File(outputDir, subdir);
80 if (!dir.exists()) {
81 dir.mkdir();
82 }
83 }
84 return new File(dir, name + '.' + extension);
85 }
86
87 /*
88 * Returns a LineGraph object representing measurements for a scenario over builds.
89 */
getLineGraph(ScenarioResults scenarioResults, ConfigResults configResults, Dim dim, List highlightedPoints, List currentBuildIdPrefixes)90 private TimeLineGraph getLineGraph(ScenarioResults scenarioResults, ConfigResults configResults, Dim dim, List highlightedPoints, List currentBuildIdPrefixes) {
91 Display display = Display.getDefault();
92
93 Color black = display.getSystemColor(SWT.COLOR_BLACK);
94 Color yellow = display.getSystemColor(SWT.COLOR_DARK_YELLOW);
95 Color magenta = display.getSystemColor(SWT.COLOR_MAGENTA);
96
97 String scenarioName = scenarioResults.getName();
98 TimeLineGraph graph = new TimeLineGraph(scenarioName + ": " + dim.getName(), dim);
99 String baseline = configResults.getBaselineBuildName();
100 String current = configResults.getCurrentBuildName();
101
102 final String defaultBaselinePrefix = DB_Results.getDbBaselinePrefix();
103 Iterator builds = configResults.getResults();
104 List lastSevenNightlyBuilds = configResults.lastNightlyBuildNames(7);
105 buildLoop: while (builds.hasNext()) {
106 BuildResults buildResults = (BuildResults) builds.next();
107 String buildID = buildResults.getName();
108 int underscoreIndex = buildID.indexOf('_');
109 String label = (underscoreIndex != -1 && buildID.equals(current)) ? buildID.substring(0, underscoreIndex) : buildID;
110 if (buildID.startsWith(defaultBaselinePrefix)) {
111 label = defaultBaselinePrefix+buildID.charAt(defaultBaselinePrefix.length())+buildID.substring(underscoreIndex);
112 }
113
114 double value = buildResults.getValue(dim.getId());
115
116 if (buildID.equals(current)) {
117 Color color = black;
118 if (buildID.startsWith("N"))
119 color = yellow;
120
121 graph.addItem("main", label, dim.getDisplayValue(value), value, color, true, Utils.getDateFromBuildID(buildID), true);
122 continue;
123 }
124 if (highlightedPoints.contains(buildID)) {
125 graph.addItem("main", label, dim.getDisplayValue(value), value, black, false, Utils.getDateFromBuildID(buildID, false), true);
126 continue;
127 }
128 if (buildID.charAt(0) == 'N') {
129 if (lastSevenNightlyBuilds.contains(buildID)) {
130 graph.addItem("main", buildID, dim.getDisplayValue(value), value, yellow, false, Utils.getDateFromBuildID(buildID), false);
131 }
132 continue;
133 }
134 for (int i=0;i<currentBuildIdPrefixes.size();i++){
135 if (buildID.startsWith(currentBuildIdPrefixes.get(i).toString())) {
136 graph.addItem("main", buildID, dim.getDisplayValue(value), value, black, false, Utils.getDateFromBuildID(buildID), false);
137 continue buildLoop;
138 }
139 }
140 if (buildID.equals(baseline)) {
141 boolean drawBaseline = (this.baselinePrefix != null) ? false : true;
142 graph.addItem("reference", label, dim.getDisplayValue(value), value, magenta, true, Utils.getDateFromBuildID(buildID, true), true, drawBaseline);
143 continue;
144 }
145 if (this.baselinePrefix != null) {
146 if (buildID.startsWith(this.baselinePrefix) && !buildID.equals(baseline) && Utils.getDateFromBuildID(buildID, true) <= Utils.getDateFromBuildID(baseline, true)) {
147 graph.addItem("reference", label, dim.getDisplayValue(value), value, magenta, false, Utils.getDateFromBuildID(buildID, true), false);
148 continue;
149 }
150 }
151 }
152 return graph;
153 }
154
155 /**
156 * Print the scenario all builds data from the given performance results.
157 *
158 * @param performanceResults The needed information to generate scenario data
159 */
print(PerformanceResults performanceResults, PrintStream printStream, final SubMonitor subMonitor)160 public void print(PerformanceResults performanceResults, PrintStream printStream, final SubMonitor subMonitor) {
161 String[] configNames = performanceResults.getConfigNames(false/*not sorted*/);
162 String[] configBoxes = performanceResults.getConfigBoxes(false/*not sorted*/);
163 int length = configNames.length;
164 int size = performanceResults.size();
165 double total = length * size;
166 subMonitor.setWorkRemaining(length*size);
167 int progress = 0;
168 for (int i=0; i<length; i++) {
169 final String configName = configNames[i];
170 final String configBox = configBoxes[i];
171
172 // Manage monitor
173 subMonitor.setTaskName("Generating data for "+configBox);
174 if (subMonitor.isCanceled()) throw new OperationCanceledException();
175
176 long start = System.currentTimeMillis();
177 if (printStream != null) printStream.print(" + "+configName);
178 final File outputDir = new File(this.rootDir, configName);
179 outputDir.mkdir();
180 Iterator components = performanceResults.getResults();
181 while (components.hasNext()) {
182 if (printStream != null) printStream.print(".");
183 final ComponentResults componentResults = (ComponentResults) components.next();
184
185 // Manage monitor
186 int percentage = (int) ((progress++ / total) * 100);
187 subMonitor.setTaskName("Generating data for "+configBox+": "+percentage+"%");
188 subMonitor.subTask("Component "+componentResults.getName()+"...");
189
190 Display display = Display.getDefault();
191 display.syncExec(
192 new Runnable() {
193 public void run(){
194 printSummary(configName, configBox, componentResults, outputDir, subMonitor);
195 }
196 }
197 );
198 // printSummary(configName, configBox, componentResults, outputDir, monitor);
199 printDetails(configName, configBoxes[i], componentResults, outputDir);
200
201 subMonitor.worked(1);
202 if (subMonitor.isCanceled()) throw new OperationCanceledException();
203 }
204 if (printStream != null) {
205 String duration = Util.timeString(System.currentTimeMillis()-start);
206 printStream.println(" done in "+duration);
207 }
208 }
209 }
210
211 /*
212 * Print the summary file of the builds data.
213 */
printSummary(String configName, String configBox, ComponentResults componentResults, File outputDir, SubMonitor subMonitor)214 void printSummary(String configName, String configBox, ComponentResults componentResults, File outputDir, SubMonitor subMonitor) {
215 Iterator scenarios = componentResults.getResults();
216 while (scenarios.hasNext()) {
217 List highlightedPoints = new ArrayList();
218 ScenarioResults scenarioResults = (ScenarioResults) scenarios.next();
219 ConfigResults configResults = scenarioResults.getConfigResults(configName);
220 if (configResults == null || !configResults.isValid()) continue;
221
222 // get latest points of interest matching
223 if (this.pointsOfInterest != null) {
224 Iterator buildPrefixes = this.pointsOfInterest.iterator();
225 while (buildPrefixes.hasNext()) {
226 String buildPrefix = (String) buildPrefixes.next();
227 List builds = configResults.getBuilds(buildPrefix);
228 if (buildPrefix.indexOf('*') <0 && buildPrefix.indexOf('?') < 0) {
229 if (builds.size() > 0) {
230 highlightedPoints.add(builds.get(builds.size()-1));
231 }
232 } else {
233 highlightedPoints.addAll(builds);
234 }
235 }
236 }
237
238 String scenarioFileName = scenarioResults.getFileName();
239 File outputFile = new File(outputDir, scenarioFileName+".html");
240 PrintStream stream = null;
241 try {
242 stream = new PrintStream(new BufferedOutputStream(new FileOutputStream(outputFile)));
243 } catch (FileNotFoundException e) {
244 System.err.println("can't create output file" + outputFile); //$NON-NLS-1$
245 }
246 if (stream == null) {
247 stream = System.out;
248 }
249 stream.print(Utils.HTML_OPEN);
250 stream.print(Utils.HTML_DEFAULT_CSS);
251
252 stream.print("<title>" + scenarioResults.getName() + "(" + configBox + ")" + "</title></head>\n"); //$NON-NLS-1$
253 stream.print("<h4>Scenario: " + scenarioResults.getName() + " (" + configBox + ")</h4><br>\n"); //$NON-NLS-1$ //$NON-NLS-2$
254
255 String failureMessage = Utils.failureMessage(configResults.getCurrentBuildDeltaInfo(), true);
256 if (failureMessage != null){
257 stream.print("<table><tr><td><b>"+failureMessage+"</td></tr></table>\n");
258 }
259
260 BuildResults currentBuildResults = configResults.getCurrentBuildResults();
261 String comment = currentBuildResults.getComment();
262 if (comment != null) {
263 stream.print("<p><b>Note:</b><br>\n");
264 stream.print(comment + "</p>\n");
265 }
266
267 // Print link to raw data.
268 String rawDataFile = "raw/" + scenarioFileName+".html";
269 stream.print("<br><br><b><a href=\""+rawDataFile+"\">Raw data and Stats</a></b><br><br>\n");
270 stream.print("<b>Click measurement name to view line graph of measured values over builds.</b><br><br>\n");
271 if (subMonitor.isCanceled()) throw new OperationCanceledException();
272
273 try {
274 // Print build result table
275 stream.print("<table border=\"1\">\n"); //$NON-NLS-1$
276 stream.print("<tr><td><b>Build Id</b></td>"); //$NON-NLS-1$
277 int dimLength = this.dimensions.length;
278 for (int d=0; d<dimLength; d++) {
279 Dim dim = this.dimensions[d];
280 stream.print("<td><a href=\"#" + dim.getLabel() + "\"><b>" + dim.getName() + "</b></a></td>");
281 }
282 stream.print("</tr>\n");
283
284 // Write build lines
285 printTableLine(stream, currentBuildResults);
286 printTableLine(stream, configResults.getBaselineBuildResults());
287
288 // Write difference line
289 printDifferenceLine(stream, configResults);
290
291 // End of table
292 stream.print("</table>\n");
293 stream.print("*Delta values in red and green indicate degradation > 10% and improvement > 10%,respectively.<br><br>\n");
294 stream.print("<br><hr>\n\n");
295
296 // print text legend.
297 stream.print("Black and yellow points plot values measured in integration and last seven nightly builds.<br>\n" + "Magenta points plot the repeated baseline measurement over time.<br>\n"
298 + "Boxed points represent previous releases, milestone builds, current reference and current build.<br><br>\n"
299 + "Hover over any point for build id and value.\n");
300
301 // print image maps of historical
302 for (int d=0; d<dimLength; d++) {
303 Dim dim = this.dimensions[d];
304 TimeLineGraph lineGraph = getLineGraph(scenarioResults, configResults, dim, highlightedPoints, this.buildIDStreamPatterns);
305 if (subMonitor.isCanceled()) throw new OperationCanceledException();
306
307 String dimShortName = dim.getLabel();
308 String imgFileName = scenarioFileName + "_" + dimShortName;
309 File imgFile = createFile(outputDir, "graphs", imgFileName, "gif");
310 saveGraph(lineGraph, imgFile);
311 stream.print("<br><a name=\"" + dimShortName + "\"></a>\n");
312 stream.print("<br><b>" + dim.getName() + "</b><br>\n");
313 stream.print(dim.getDescription() + "<br><br>\n");
314 stream.print("<img src=\"graphs/");
315 stream.print(imgFile.getName());
316 stream.print("\" usemap=\"#" + lineGraph.fTitle + "\">");
317 stream.print("<map name=\"" + lineGraph.fTitle + "\">");
318 stream.print(lineGraph.getAreas());
319 stream.print("</map>\n");
320 if (subMonitor.isCanceled()) throw new OperationCanceledException();
321 }
322 stream.print("<br><br></body>\n");
323 stream.print(Utils.HTML_CLOSE);
324 if (stream != System.out)
325 stream.close();
326
327 } catch (AssertionFailedError e) {
328 e.printStackTrace();
329 continue;
330 }
331 }
332 }
333
334 /*
335 * Print the data for a build results.
336 */
printTableLine(PrintStream stream, BuildResults buildResults)337 private void printTableLine(PrintStream stream, BuildResults buildResults) {
338 stream.print("<tr><td>");
339 stream.print(buildResults.getName());
340 if (buildResults.isBaseline()) stream.print(" (reference)");
341 stream.print("</td>");
342 int dimLength = this.dimensions.length;
343 for (int d=0; d<dimLength; d++) {
344 Dim dim = this.dimensions[d];
345 int dim_id = dim.getId();
346 double stddev = buildResults.getDeviation(dim_id);
347 String displayValue = dim.getDisplayValue(buildResults.getValue(dim_id));
348 stream.print("<td>");
349 stream.print(displayValue);
350 if (stddev < 0) {
351 stream.print(" [n/a]\n");
352 } else if (stddev > 0) {
353 stream.print(" [");
354 stream.print(dim.getDisplayValue(stddev));
355 stream.print("]");
356 }
357 stream.print( "</td>");
358 }
359 stream.print("</tr>\n");
360 }
361
362 /*
363 * Print the line showing the difference between current and baseline builds.
364 */
printDifferenceLine(PrintStream stream, ConfigResults configResults)365 private void printDifferenceLine(PrintStream stream, ConfigResults configResults) {
366 stream.print("<tr><td>*Delta</td>");
367 int dimLength = this.dimensions.length;
368 for (int d=0; d<dimLength; d++) {
369 Dim currentDim = this.dimensions[d];
370 int dim_id = currentDim.getId();
371 BuildResults currentBuild = configResults.getCurrentBuildResults();
372 BuildResults baselineBuild = configResults.getBaselineBuildResults();
373
374 // Compute difference values
375 double baselineValue = baselineBuild.getValue(dim_id);
376 double diffValue = baselineValue - currentBuild.getValue(dim_id);
377 double diffPercentage = baselineValue == 0 ? 0 : Math.round(diffValue / baselineValue * 1000) / 10.0;
378 String diffDisplayValue = currentDim.getDisplayValue(diffValue);
379
380 // Set colors
381 String fontColor = "";
382 if (diffPercentage > 10) {
383 fontColor = "#006600"; // green
384 }
385 if (diffPercentage < -10) {
386 fontColor = "#FF0000"; // red
387 }
388
389 // Print line
390 String percentage = (diffPercentage == 0) ? "" : "<br>" + diffPercentage + " %";
391 if (diffPercentage > 10 || diffPercentage < -10) {
392 stream.print("<td><FONT COLOR=\"" + fontColor + "\"><b>" + diffDisplayValue + percentage + "</b></FONT></td>");
393 } else {
394 stream.print("<td>" + diffDisplayValue + percentage + "</td>");
395 }
396 }
397 stream.print("</tr></font>");
398 }
399
400 /*
401 * Print details file of the scenario builds data.
402 */
printDetails(String configName, String configBox, ComponentResults componentResults, File outputDir)403 private void printDetails(String configName, String configBox, ComponentResults componentResults, File outputDir) {
404 Iterator scenarios = componentResults.getResults();
405 while (scenarios.hasNext()) {
406 ScenarioResults scenarioResults = (ScenarioResults) scenarios.next();
407 ConfigResults configResults = scenarioResults.getConfigResults(configName);
408 if (configResults == null || !configResults.isValid()) continue;
409 String scenarioName= scenarioResults.getName();
410 String scenarioFileName = scenarioResults.getFileName();
411 File outputFile = createFile(outputDir, "raw", scenarioFileName, "html");
412 PrintStream stream = null;
413 try {
414 stream = new PrintStream(new BufferedOutputStream(new FileOutputStream(outputFile)));
415 } catch (FileNotFoundException e) {
416 System.err.println("can't create output file" + outputFile); //$NON-NLS-1$
417 }
418 if (stream == null) stream = System.out;
419 RawDataTable currentResultsTable = new RawDataTable(configResults, this.buildIDStreamPatterns, stream);
420 RawDataTable baselineResultsTable = new RawDataTable(configResults, this.baselinePrefix, stream);
421 stream.print(Utils.HTML_OPEN);
422 stream.print(Utils.HTML_DEFAULT_CSS);
423 stream.print("<title>" + scenarioName + "(" + configBox + ")" + " - Details</title></head>\n"); //$NON-NLS-1$
424 stream.print("<h4>Scenario: " + scenarioName + " (" + configBox + ")</h4>\n"); //$NON-NLS-1$
425 stream.print("<a href=\"../"+scenarioFileName+".html\">VIEW GRAPH</a><br><br>\n"); //$NON-NLS-1$
426 stream.print("<table><td><b>Current Stream Test Runs</b></td><td><b>Baseline Test Runs</b></td></tr>\n");
427 stream.print("<tr valign=\"top\">\n");
428 stream.print("<td>");
429 currentResultsTable.print();
430 stream.print("</td>\n");
431 stream.print("<td>");
432 baselineResultsTable.print();
433 stream.print("</td>\n");
434 stream.print("</tr>\n");
435 stream.print("</table>\n");
436 stream.close();
437 }
438 }
439
440 /*
441 * Prints a LineGraph object as a gif file.
442 */
saveGraph(LineGraph p, File outputFile)443 private void saveGraph(LineGraph p, File outputFile) {
444 Image image = new Image(Display.getDefault(), GRAPH_WIDTH, GRAPH_HEIGHT);
445 p.paint(image);
446
447 /* Downscale to 8 bit depth palette to save to gif */
448 ImageData data = Utils.downSample(image);
449 ImageLoader il = new ImageLoader();
450 il.data = new ImageData[] { data };
451 OutputStream out = null;
452 try {
453 out = new BufferedOutputStream(new FileOutputStream(outputFile));
454 il.save(out, SWT.IMAGE_GIF);
455
456 } catch (FileNotFoundException e) {
457 e.printStackTrace();
458 } finally {
459 image.dispose();
460 if (out != null) {
461 try {
462 out.close();
463 } catch (IOException e1) {
464 // silently ignored
465 }
466 }
467 }
468 }
469 }
470