1 /*******************************************************************************
2 * Copyright (c) 2000, 2009 IBM Corporation and others.
3 * All rights reserved. This program and the accompanying materials
4 * are made available under the terms of the Eclipse Public License v1.0
5 * which accompanies this distribution, and is available at
6 * http://www.eclipse.org/legal/epl-v10.html
7 *
8 * Contributors:
9 * IBM Corporation - initial API and implementation
10 *******************************************************************************/
11 package org.eclipse.test.internal.performance.results.model;
12
13 import java.util.ArrayList;
14 import java.util.List;
15 import java.util.Vector;
16
17 import org.eclipse.core.runtime.IAdaptable;
18 import org.eclipse.core.runtime.preferences.IEclipsePreferences;
19 import org.eclipse.core.runtime.preferences.InstanceScope;
20 import org.eclipse.jface.resource.ImageDescriptor;
21 import org.eclipse.swt.graphics.Image;
22 import org.eclipse.test.internal.performance.eval.StatisticsUtil;
23 import org.eclipse.test.internal.performance.results.db.AbstractResults;
24 import org.eclipse.test.internal.performance.results.db.BuildResults;
25 import org.eclipse.test.internal.performance.results.db.ConfigResults;
26 import org.eclipse.test.internal.performance.results.db.DB_Results;
27 import org.eclipse.test.internal.performance.results.utils.IPerformancesConstants;
28 import org.eclipse.test.internal.performance.results.utils.Util;
29 import org.eclipse.ui.ISharedImages;
30 import org.eclipse.ui.PlatformUI;
31 import org.eclipse.ui.model.IWorkbenchAdapter;
32 import org.eclipse.ui.views.properties.ComboBoxPropertyDescriptor;
33 import org.eclipse.ui.views.properties.IPropertyDescriptor;
34 import org.eclipse.ui.views.properties.IPropertySource;
35 import org.eclipse.ui.views.properties.PropertyDescriptor;
36 import org.eclipse.ui.views.properties.TextPropertyDescriptor;
37 import org.osgi.service.prefs.BackingStoreException;
38
39 /**
40 * An Organization Element
41 */
42 public abstract class ResultsElement implements IAdaptable, IPropertySource, IWorkbenchAdapter, Comparable {
43
44 // Image descriptors
45 private static final ISharedImages WORKBENCH_SHARED_IMAGES = PlatformUI.getWorkbench().getSharedImages();
46 public static final Image ERROR_IMAGE = WORKBENCH_SHARED_IMAGES.getImage(ISharedImages.IMG_OBJS_ERROR_TSK);
47 public static final ImageDescriptor ERROR_IMAGE_DESCRIPTOR = WORKBENCH_SHARED_IMAGES.getImageDescriptor(ISharedImages.IMG_OBJS_ERROR_TSK);
48 public static final Image WARN_IMAGE = WORKBENCH_SHARED_IMAGES.getImage(ISharedImages.IMG_OBJS_WARN_TSK);
49 public static final ImageDescriptor WARN_IMAGE_DESCRIPTOR = WORKBENCH_SHARED_IMAGES.getImageDescriptor(ISharedImages.IMG_OBJS_WARN_TSK);
50 public static final Image INFO_IMAGE = WORKBENCH_SHARED_IMAGES.getImage(ISharedImages.IMG_OBJS_INFO_TSK);
51 public static final ImageDescriptor INFO_IMAGE_DESCRIPTOR = WORKBENCH_SHARED_IMAGES.getImageDescriptor(ISharedImages.IMG_OBJS_INFO_TSK);
52 public static final Image HELP_IMAGE = WORKBENCH_SHARED_IMAGES.getImage(ISharedImages.IMG_LCL_LINKTO_HELP);
53 public static final ImageDescriptor HELP_IMAGE_DESCRIPTOR = WORKBENCH_SHARED_IMAGES.getImageDescriptor(ISharedImages.IMG_LCL_LINKTO_HELP);
54 public static final ImageDescriptor FOLDER_IMAGE_DESCRIPTOR = WORKBENCH_SHARED_IMAGES.getImageDescriptor(ISharedImages.IMG_OBJ_FOLDER);
55 public static final ImageDescriptor CONNECT_IMAGE_DESCRIPTOR = WORKBENCH_SHARED_IMAGES.getImageDescriptor(ISharedImages.IMG_ELCL_SYNCED);
56
57 // Model
58 ResultsElement parent;
59 AbstractResults results;
60 ResultsElement[] children;
61 String name;
62 int status = -1;
63
64 // Stats
65 double[] statistics;
66
67 // Status constants
68 // state
69 static final int UNKNOWN = 0x01;
70 static final int UNREAD = 0x02;
71 static final int READ = 0x04;
72 static final int MISSING = 0x08;
73 public static final int STATE_MASK = 0x0F;
74 // info
75 static final int SMALL_VALUE = 0x0010;
76 static final int STUDENT_TTEST = 0x0020;
77 public static final int INFO_MASK = 0x0030;
78 // warning
79 static final int NO_BASELINE = 0x0040;
80 static final int SINGLE_RUN = 0x0080;
81 static final int BIG_ERROR = 0x0100;
82 static final int NOT_STABLE = 0x0200;
83 static final int NOT_RELIABLE = 0x0400;
84 public static final int WARNING_MASK = 0x0FC0;
85 // error
86 static final int BIG_DELTA = 0x1000;
87 public static final int ERROR_MASK = 0xF000;
88
89 // Property descriptors
90 static final String P_ID_STATUS_INFO = "ResultsElement.status_info"; //$NON-NLS-1$
91 static final String P_ID_STATUS_WARNING = "ResultsElement.status_warning"; //$NON-NLS-1$
92 static final String P_ID_STATUS_ERROR = "ResultsElement.status_error"; //$NON-NLS-1$
93 static final String P_ID_STATUS_COMMENT = "ResultsElement.status_comment"; //$NON-NLS-1$
94
95 static final String P_STR_STATUS_INFO = " info"; //$NON-NLS-1$
96 static final String P_STR_STATUS_WARNING = "warning"; //$NON-NLS-1$
97 static final String P_STR_STATUS_ERROR = "error"; //$NON-NLS-1$
98 static final String P_STR_STATUS_COMMENT = "comment"; //$NON-NLS-1$
99 static final String[] NO_VALUES = new String[0];
100
101 private static Vector DESCRIPTORS;
102 static final TextPropertyDescriptor COMMENT_DESCRIPTOR = new TextPropertyDescriptor(P_ID_STATUS_COMMENT, P_STR_STATUS_COMMENT);
103 static final TextPropertyDescriptor ERROR_DESCRIPTOR = new TextPropertyDescriptor(P_ID_STATUS_ERROR, P_STR_STATUS_ERROR);
initDescriptors(int status)104 static Vector initDescriptors(int status) {
105 DESCRIPTORS = new Vector();
106 // Status category
107 DESCRIPTORS.add(getInfosDescriptor(status));
108 DESCRIPTORS.add(getWarningsDescriptor(status));
109 DESCRIPTORS.add(ERROR_DESCRIPTOR);
110 ERROR_DESCRIPTOR.setCategory("Status");
111 // Survey category
112 DESCRIPTORS.add(COMMENT_DESCRIPTOR);
113 COMMENT_DESCRIPTOR.setCategory("Survey");
114 return DESCRIPTORS;
115 }
getDescriptors()116 static Vector getDescriptors() {
117 return DESCRIPTORS;
118 }
getInfosDescriptor(int status)119 static ComboBoxPropertyDescriptor getInfosDescriptor(int status) {
120 List list = new ArrayList();
121 if ((status & SMALL_VALUE) != 0) {
122 list.add("Some builds have tests with small values");
123 }
124 if ((status & STUDENT_TTEST) != 0) {
125 list.add("Some builds have student-t test error over the threshold");
126 }
127 String[] infos = new String[list.size()];
128 if (list.size() > 0) {
129 list.toArray(infos);
130 }
131 ComboBoxPropertyDescriptor infoDescriptor = new ComboBoxPropertyDescriptor(P_ID_STATUS_INFO, P_STR_STATUS_INFO, infos);
132 infoDescriptor.setCategory("Status");
133 return infoDescriptor;
134 }
getWarningsDescriptor(int status)135 static PropertyDescriptor getWarningsDescriptor(int status) {
136 List list = new ArrayList();
137 if ((status & BIG_ERROR) != 0) {
138 list.add("Some builds have tests with error over 3%");
139 }
140 if ((status & NOT_RELIABLE) != 0) {
141 list.add("Some builds have no reliable tests");
142 }
143 if ((status & NOT_STABLE) != 0) {
144 list.add("Some builds have no stable tests");
145 }
146 if ((status & NO_BASELINE) != 0) {
147 list.add("Some builds have no baseline to compare with");
148 }
149 if ((status & SINGLE_RUN) != 0) {
150 list.add("Some builds have single run tests");
151 }
152 String[] warnings = new String[list.size()];
153 if (list.size() > 0) {
154 list.toArray(warnings);
155 }
156 ComboBoxPropertyDescriptor warningDescriptor = new ComboBoxPropertyDescriptor(P_ID_STATUS_WARNING, P_STR_STATUS_WARNING, warnings);
157 warningDescriptor.setCategory("Status");
158 return warningDescriptor;
159 }
160
ResultsElement()161 ResultsElement() {
162 }
163
ResultsElement(AbstractResults results, ResultsElement parent)164 ResultsElement(AbstractResults results, ResultsElement parent) {
165 this.parent = parent;
166 this.results = results;
167 }
168
ResultsElement(String name, ResultsElement parent)169 ResultsElement(String name, ResultsElement parent) {
170 this.parent = parent;
171 this.name = name;
172 }
173
compareTo(Object o)174 public int compareTo(Object o) {
175 if (this.results == null) {
176 if (o instanceof ResultsElement && this.name != null) {
177 ResultsElement element = (ResultsElement) o;
178 return this.name.compareTo(element.getName());
179 }
180 return -1;
181 }
182 if (o instanceof ResultsElement) {
183 return this.results.compareTo(((ResultsElement)o).results);
184 }
185 return -1;
186 }
187
createChild(AbstractResults testResults)188 abstract ResultsElement createChild(AbstractResults testResults);
189
190 /* (non-Javadoc)
191 * Method declared on IAdaptable
192 */
getAdapter(Class adapter)193 public Object getAdapter(Class adapter) {
194 if (adapter == IPropertySource.class) {
195 return this;
196 }
197 if (adapter == IWorkbenchAdapter.class) {
198 return this;
199 }
200 return null;
201 }
202
203 /**
204 * Iterate the element children.
205 */
getChildren()206 public ResultsElement[] getChildren() {
207 if (this.results == null) {
208 return new ResultsElement[0];
209 }
210 if (this.children == null) {
211 initChildren();
212 }
213 return this.children;
214 }
215
216 /* (non-Javadoc)
217 * Method declared on IWorkbenchAdapter
218 */
getChildren(Object o)219 public Object[] getChildren(Object o) {
220 if (this.results == null) {
221 return new Object[0];
222 }
223 if (this.children == null) {
224 initChildren();
225 }
226 return this.children;
227 }
228
229 /* (non-Javadoc)
230 * Method declared on IPropertySource
231 */
getEditableValue()232 public Object getEditableValue() {
233 return this;
234 }
235
getId()236 final String getId() {
237 return getId(new StringBuffer()).toString();
238 }
239
getId(StringBuffer buffer)240 private StringBuffer getId(StringBuffer buffer) {
241 if (this.parent != null) {
242 return this.parent.getId(buffer).append('/').append(getName());
243 }
244 return buffer.append(DB_Results.getDbName());
245 }
246
247 /* (non-Javadoc)
248 * Method declared on IWorkbenchAdapter
249 */
getImageDescriptor(Object object)250 public ImageDescriptor getImageDescriptor(Object object) {
251 if (object instanceof ResultsElement) {
252 ResultsElement resultsElement = (ResultsElement) object;
253 // DEBUG
254 // if (resultsElement.getName().equals("I20090806-0100")) {
255 // if (resultsElement.results != null) {
256 // String toString = resultsElement.results.getParent().toString();
257 // String toString = resultsElement.results.toString();
258 // if (toString.indexOf("testStoreExists")>0 && toString.indexOf("eplnx2")>0) {
259 // System.out.println("stop");
260 // }
261 // }
262 // }
263 int elementStatus = resultsElement.getStatus();
264 if (elementStatus == MISSING) {
265 return HELP_IMAGE_DESCRIPTOR;
266 }
267 if ((elementStatus & ResultsElement.ERROR_MASK) != 0) {
268 return ERROR_IMAGE_DESCRIPTOR;
269 }
270 if ((elementStatus & ResultsElement.WARNING_MASK) != 0) {
271 return WARN_IMAGE_DESCRIPTOR;
272 }
273 if ((elementStatus & ResultsElement.INFO_MASK) != 0) {
274 return INFO_IMAGE_DESCRIPTOR;
275 }
276 }
277 return null;
278 }
279
280 /* (non-Javadoc)
281 * Method declared on IWorkbenchAdapter
282 */
getLabel(Object o)283 public String getLabel(Object o) {
284 return getName();
285 }
286
287 /**
288 * Returns the name
289 */
getName()290 public String getName() {
291 if (this.name == null && this.results != null) {
292 this.name = this.results.getName();
293 }
294 return this.name;
295 }
296
297 /**
298 * Returns the parent
299 */
getParent(Object o)300 public Object getParent(Object o) {
301 return this.parent;
302 }
303
304 /* (non-Javadoc)
305 * @see org.eclipse.ui.views.properties.IPropertySource#getPropertyDescriptors()
306 */
getPropertyDescriptors()307 public IPropertyDescriptor[] getPropertyDescriptors() {
308 Vector descriptors = getDescriptors();
309 if (descriptors == null) {
310 descriptors = initDescriptors(getStatus());
311 }
312 int size = descriptors.size();
313 IPropertyDescriptor[] descriptorsArray = new IPropertyDescriptor[size];
314 descriptorsArray[0] = getInfosDescriptor(getStatus());
315 descriptorsArray[1] = getWarningsDescriptor(getStatus());
316 for (int i=2; i<size; i++) {
317 descriptorsArray[i] = (IPropertyDescriptor) descriptors.get(i);
318 }
319 return descriptorsArray;
320 }
321
322 /* (non-Javadoc)
323 * @see org.eclipse.ui.views.properties.IPropertySource#getPropertyValue(java.lang.Object)
324 */
getPropertyValue(Object propKey)325 public Object getPropertyValue(Object propKey) {
326 if (propKey.equals(P_ID_STATUS_INFO)) {
327 if ((getStatus() & INFO_MASK) != 0) {
328 return new Integer(0);
329 }
330 }
331 if (propKey.equals(P_ID_STATUS_WARNING)) {
332 if ((getStatus() & WARNING_MASK) != 0) {
333 return new Integer(0);
334 }
335 }
336 if (propKey.equals(P_ID_STATUS_ERROR)) {
337 if ((getStatus() & BIG_DELTA) != 0) {
338 return "Some builds have tests with regression";
339 }
340 }
341 if (propKey.equals(P_ID_STATUS_COMMENT)) {
342 IEclipsePreferences preferences = new InstanceScope().getNode(IPerformancesConstants.PLUGIN_ID);
343 return preferences.get(getId(), "");
344 }
345 return null;
346 }
347
getResultsElement(String resultName)348 public ResultsElement getResultsElement(String resultName) {
349 int length = getChildren(null).length;
350 for (int i=0; i<length; i++) {
351 ResultsElement searchedResults = this.children[i];
352 if (searchedResults.getName().equals(resultName)) {
353 return searchedResults;
354 }
355 }
356 return null;
357 }
358
359 /**
360 * Return the status of the element.
361 *
362 * The status is a bit mask pattern where digits are
363 * allowed as follow:
364 * <ul>
365 * <li>0-3: bits for state showing whether the element is
366 * <ul>
367 * <li>{@link #UNKNOWN} : not connected to a db</li>
368 * <li>{@link #UNREAD} : is not valid (e.g. NaN results)</li>
369 * <li>{@link #MISSING} : no results (e.g. the perf machine crashed and didn't store any results)</li>
370 * <li>{@link #READ} : has valid results</li>
371 * </ul>
372 * </li>
373 * <li>4-5: bits for information. Current possible information are
374 * <ul>
375 * <li>{@link #SMALL_VALUE} : build results or delta with baseline value is under 100ms</li>
376 * <li>{@link #STUDENT_TTEST} : the Student T-test is over the threshold (old yellow color for test results)</li>
377 * </ul>
378 * </li>
379 * <li>6-11: bits for warnings. Current possible warnings are
380 * <ul>
381 * <li>{@link #NO_BASELINE} : no baseline for the current build</li>
382 * <li>{@link #SINGLE_RUN} : the test has only one run (i.e. no error could be computed), hence its reliability cannot be evaluated</li>
383 * <li>{@link #BIG_ERROR} : the test result is over the 3% threshold</li>
384 * <li>{@link #NOT_STABLE} : the test history shows a deviation between 10% and 20% (may mean that this test is not so reliable)</li>
385 * <li>{@link #NOT_RELIABLE} : the test history shows a deviation over 20% (surely means that this test is too erratic to be reliable)</li>
386 * </ul>
387 * </li>
388 * <li>12-15: bits for errors. Current possible errors are
389 * <ul>
390 * <li>{@link #BIG_DELTA} : the delta for the test is over the 10% threshold</li>
391 * </ul>
392 * </li>
393 * </ul>
394 *
395 * Note that these explanation applied to {@link BuildResultsElement}, and {@link DimResultsElement}.
396 * For {@link ComponentResultsElement}, and {@link ScenarioResultsElement}, it's the merge of all the children status
397 * and means "Some tests have..." instead of "The test has...". For {@link ConfigResultsElement}, it means the status
398 * of the most recent build compared to its most recent baseline.
399 *
400 * @return An int with each bit set when the corresponding symptom applies.
401 */
getStatus()402 public final int getStatus() {
403 if (this.status < 0) {
404 initStatus();
405 }
406 return this.status;
407 }
408
409 /**
410 * Return the statistics of the build along its history.
411 *
412 * @return An array of double built as follows:
413 * <ul>
414 * <li>0: numbers of values</li>
415 * <li>1: mean of values</li>
416 * <li>2: standard deviation of these values</li>
417 * <li>3: coefficient of variation of these values</li>
418 * </ul>
419 */
getStatistics()420 double[] getStatistics() {
421 return this.statistics;
422 }
423
424 /**
425 * Returns whether the element (or one in its hierarchy) has an error.
426 *
427 * @return <code> true</code> if the element or one in its hierarchy has an error,
428 * <code> false</code> otherwise
429 */
hasError()430 public final boolean hasError() {
431 return (getStatus() & ERROR_MASK) != 0;
432 }
433
initChildren()434 void initChildren() {
435 AbstractResults[] resultsChildren = this.results.getChildren();
436 int length = resultsChildren.length;
437 this.children = new ResultsElement[length];
438 int count = 0;
439 for (int i=0; i<length; i++) {
440 ResultsElement childElement = createChild(resultsChildren[i]);
441 if (childElement != null) {
442 this.children[count++] = childElement;
443 }
444 }
445 if (count < length) {
446 System.arraycopy(this.children, 0, this.children = new ResultsElement[count], 0, count);
447 }
448 }
initStatus()449 void initStatus() {
450 this.status = READ;
451 if (this.results != null) {
452 if (this.children == null) initChildren();
453 int length = this.children.length;
454 for (int i=0; i<length; i++) {
455 this.status |= this.children[i].getStatus();
456 }
457 }
458 }
459
initStatus(BuildResults buildResults)460 int initStatus(BuildResults buildResults) {
461 this.status = READ;
462
463 // Get values
464 double buildValue = buildResults.getValue();
465 ConfigResults configResults = (ConfigResults) buildResults.getParent();
466 BuildResults baselineResults = configResults.getBaselineBuildResults(buildResults.getName());
467 double baselineValue = baselineResults.getValue();
468 double delta = (baselineValue - buildValue) / baselineValue;
469
470 // Store if there's no baseline
471 if (Double.isNaN(delta)) {
472 this.status |= NO_BASELINE;
473 }
474
475 // Store if there's only one run
476 long baselineCount = baselineResults.getCount();
477 long currentCount = buildResults.getCount();
478 double error = Double.NaN;
479 if (baselineCount == 1 || currentCount == 1) {
480 this.status |= SINGLE_RUN;
481 }
482
483 // Store if the T-test is not good
484 double ttestValue = Util.computeTTest(baselineResults, buildResults);
485 int degreeOfFreedom = (int) (baselineResults.getCount()+buildResults.getCount()-2);
486 if (ttestValue >= 0 && StatisticsUtil.getStudentsT(degreeOfFreedom, StatisticsUtil.T90) >= ttestValue) {
487 this.status |= STUDENT_TTEST;
488 }
489
490 // Store if there's a big error (over 3%)
491 double baselineError = baselineResults.getError();
492 double currentError = buildResults.getError();
493 error = Double.isNaN(baselineError)
494 ? currentError / baselineValue
495 : Math.sqrt(baselineError*baselineError + currentError*currentError) / baselineValue;
496 if (error > 0.03) {
497 this.status |= BIG_ERROR;
498 }
499
500 // Store if there's a big delta (over 10%)
501 if (delta <= -0.1) {
502 this.status |= BIG_DELTA;
503 double currentBuildValue = buildResults.getValue();
504 double diff = Math.abs(baselineValue - currentBuildValue);
505 if (currentBuildValue < 100 || diff < 100) { // moderate the status when
506 // diff is less than 100ms
507 this.status |= SMALL_VALUE;
508 } else {
509 double[] stats = getStatistics();
510 if (stats != null) {
511 if (stats[3] > 0.2) { // invalidate the status when the test
512 // historical deviation is over 20%
513 this.status |= NOT_RELIABLE;
514 } else if (stats[3] > 0.1) { // moderate the status when the test
515 // historical deviation is between 10%
516 // and 20%
517 this.status |= NOT_STABLE;
518 }
519 }
520 }
521 }
522
523 return this.status;
524 }
525
isInitialized()526 public boolean isInitialized() {
527 return this.results != null;
528 }
529
530 /* (non-Javadoc)
531 * Method declared on IPropertySource
532 */
isPropertySet(Object property)533 public boolean isPropertySet(Object property) {
534 return false;
535 }
536
onlyFingerprints()537 boolean onlyFingerprints() {
538 if (this.parent != null) {
539 return this.parent.onlyFingerprints();
540 }
541 return ((PerformanceResultsElement)this).fingerprints;
542 }
543
544 /* (non-Javadoc)
545 * Method declared on IPropertySource
546 */
resetPropertyValue(Object property)547 public void resetPropertyValue(Object property) {
548 }
549
resetStatus()550 void resetStatus() {
551 this.status = -1;
552 if (this.results != null) {
553 if (this.children == null) initChildren();
554 int length = this.children.length;
555 for (int i=0; i<length; i++) {
556 this.children[i].resetStatus();
557 }
558 }
559 }
560
setPropertyValue(Object name, Object value)561 public void setPropertyValue(Object name, Object value) {
562 if (name.equals(P_ID_STATUS_COMMENT)) {
563 IEclipsePreferences preferences = new InstanceScope().getNode(IPerformancesConstants.PLUGIN_ID);
564 preferences.put(getId(), (String) value);
565 try {
566 preferences.flush();
567 } catch (BackingStoreException e) {
568 // skip
569 }
570 }
571 }
572
573 /**
574 * Sets the image descriptor
575 */
setImageDescriptor(ImageDescriptor desc)576 void setImageDescriptor(ImageDescriptor desc) {
577 // this.imageDescriptor = desc;
578 }
579
toString()580 public String toString() {
581 if (this.results == null) {
582 return getName();
583 }
584 return this.results.toString();
585 }
586
587 /*
588 * Write the element status in the given stream
589 */
writableStatus(StringBuffer buffer, int kind, StringBuffer excluded)590 StringBuffer writableStatus(StringBuffer buffer, int kind, StringBuffer excluded) {
591 int length = this.children.length;
592 for (int i=0; i<length; i++) {
593 this.children[i].writableStatus(buffer, kind, excluded);
594 }
595 return buffer;
596 }
597
598
599 }
600