1 package org.testng.internal; 2 3 import java.lang.annotation.Annotation; 4 import java.lang.reflect.InvocationTargetException; 5 import java.lang.reflect.Method; 6 import java.util.Collection; 7 import java.util.Collections; 8 import java.util.Iterator; 9 import java.util.List; 10 import java.util.Map; 11 import java.util.Set; 12 13 import org.testng.IClass; 14 import org.testng.IClassListener; 15 import org.testng.IConfigurable; 16 import org.testng.IConfigurationListener; 17 import org.testng.IConfigurationListener2; 18 import org.testng.IHookable; 19 import org.testng.IInvokedMethod; 20 import org.testng.IInvokedMethodListener; 21 import org.testng.IRetryAnalyzer; 22 import org.testng.ITestClass; 23 import org.testng.ITestContext; 24 import org.testng.ITestListener; 25 import org.testng.ITestNGMethod; 26 import org.testng.ITestResult; 27 import org.testng.Reporter; 28 import org.testng.SkipException; 29 import org.testng.SuiteRunState; 30 import org.testng.TestException; 31 import org.testng.TestNGException; 32 import org.testng.annotations.IConfigurationAnnotation; 33 import org.testng.annotations.NoInjection; 34 import org.testng.collections.Lists; 35 import org.testng.collections.Maps; 36 import org.testng.collections.Sets; 37 import org.testng.internal.InvokeMethodRunnable.TestNGRuntimeException; 38 import org.testng.internal.ParameterHolder.ParameterOrigin; 39 import org.testng.internal.annotations.AnnotationHelper; 40 import org.testng.internal.annotations.IAnnotationFinder; 41 import org.testng.internal.invokers.InvokedMethodListenerInvoker; 42 import org.testng.internal.invokers.InvokedMethodListenerMethod; 43 import org.testng.internal.thread.ThreadExecutionException; 44 import org.testng.internal.thread.ThreadUtil; 45 import org.testng.internal.thread.graph.IWorker; 46 import org.testng.xml.XmlClass; 47 import org.testng.xml.XmlSuite; 48 import org.testng.xml.XmlTest; 49 50 import static org.testng.internal.invokers.InvokedMethodListenerMethod.AFTER_INVOCATION; 51 import static org.testng.internal.invokers.InvokedMethodListenerMethod.BEFORE_INVOCATION; 52 53 /** 54 * This class is responsible for invoking methods: 55 * - test methods 56 * - configuration methods 57 * - possibly in a separate thread 58 * and then for notifying the result listeners. 59 * 60 * @author <a href="mailto:cedric@beust.com">Cedric Beust</a> 61 * @author <a href='mailto:the_mindstorm@evolva.ro'>Alexandru Popescu</a> 62 */ 63 public class Invoker implements IInvoker { 64 private final ITestContext m_testContext; 65 private final ITestResultNotifier m_notifier; 66 private final IAnnotationFinder m_annotationFinder; 67 private final SuiteRunState m_suiteState; 68 private final boolean m_skipFailedInvocationCounts; 69 private final Collection<IInvokedMethodListener> m_invokedMethodListeners; 70 private final boolean m_continueOnFailedConfiguration; 71 private final List<IClassListener> m_classListeners; 72 73 /** Group failures must be synced as the Invoker is accessed concurrently */ 74 private Map<String, Boolean> m_beforegroupsFailures = Maps.newHashtable(); 75 76 /** Class failures must be synced as the Invoker is accessed concurrently */ 77 private Map<Class<?>, Set<Object>> m_classInvocationResults = Maps.newHashtable(); 78 79 /** Test methods whose configuration methods have failed. */ 80 private Map<ITestNGMethod, Set<Object>> m_methodInvocationResults = Maps.newHashtable(); 81 private IConfiguration m_configuration; 82 83 /** Predicate to filter methods */ 84 private static Predicate<ITestNGMethod, IClass> CAN_RUN_FROM_CLASS = new CanRunFromClassPredicate(); 85 /** Predicate to filter methods */ 86 private static final Predicate<ITestNGMethod, IClass> SAME_CLASS = new SameClassNamePredicate(); 87 setClassInvocationFailure(Class<?> clazz, Object instance)88 private void setClassInvocationFailure(Class<?> clazz, Object instance) { 89 Set<Object> instances = m_classInvocationResults.get( clazz ); 90 if (instances == null) { 91 instances = Sets.newHashSet(); 92 m_classInvocationResults.put(clazz, instances); 93 } 94 instances.add(instance); 95 } 96 setMethodInvocationFailure(ITestNGMethod method, Object instance)97 private void setMethodInvocationFailure(ITestNGMethod method, Object instance) { 98 Set<Object> instances = m_methodInvocationResults.get(method); 99 if (instances == null) { 100 instances = Sets.newHashSet(); 101 m_methodInvocationResults.put(method, instances); 102 } 103 instances.add(getMethodInvocationToken(method, instance)); 104 } 105 Invoker(IConfiguration configuration, ITestContext testContext, ITestResultNotifier notifier, SuiteRunState state, boolean skipFailedInvocationCounts, Collection<IInvokedMethodListener> invokedMethodListeners, List<IClassListener> classListeners)106 public Invoker(IConfiguration configuration, 107 ITestContext testContext, 108 ITestResultNotifier notifier, 109 SuiteRunState state, 110 boolean skipFailedInvocationCounts, 111 Collection<IInvokedMethodListener> invokedMethodListeners, 112 List<IClassListener> classListeners) { 113 m_configuration = configuration; 114 m_testContext= testContext; 115 m_suiteState= state; 116 m_notifier= notifier; 117 m_annotationFinder= configuration.getAnnotationFinder(); 118 m_skipFailedInvocationCounts = skipFailedInvocationCounts; 119 m_invokedMethodListeners = invokedMethodListeners; 120 m_continueOnFailedConfiguration = XmlSuite.CONTINUE.equals(testContext.getSuite().getXmlSuite().getConfigFailurePolicy()); 121 m_classListeners = classListeners; 122 } 123 124 /** 125 * Invoke configuration methods if they belong to the same TestClass passed 126 * in parameter.. <p/>TODO: Calculate ahead of time which methods should be 127 * invoked for each class. Might speed things up for users who invoke the 128 * same test class with different parameters in the same suite run. 129 * 130 * If instance is non-null, the configuration will be run on it. If it is null, 131 * the configuration methods will be run on all the instances retrieved 132 * from the ITestClass. 133 */ 134 @Override invokeConfigurations(IClass testClass, ITestNGMethod[] allMethods, XmlSuite suite, Map<String, String> params, Object[] parameterValues, Object instance)135 public void invokeConfigurations(IClass testClass, 136 ITestNGMethod[] allMethods, 137 XmlSuite suite, 138 Map<String, String> params, 139 Object[] parameterValues, 140 Object instance) 141 { 142 invokeConfigurations(testClass, null, allMethods, suite, params, parameterValues, instance, 143 null); 144 } 145 invokeConfigurations(IClass testClass, ITestNGMethod currentTestMethod, ITestNGMethod[] allMethods, XmlSuite suite, Map<String, String> params, Object[] parameterValues, Object instance, ITestResult testMethodResult)146 private void invokeConfigurations(IClass testClass, 147 ITestNGMethod currentTestMethod, 148 ITestNGMethod[] allMethods, 149 XmlSuite suite, 150 Map<String, String> params, 151 Object[] parameterValues, 152 Object instance, 153 ITestResult testMethodResult) 154 { 155 if(null == allMethods) { 156 log(5, "No configuration methods found"); 157 158 return; 159 } 160 161 ITestNGMethod[] methods= filterMethods(testClass, allMethods, SAME_CLASS); 162 163 for(ITestNGMethod tm : methods) { 164 if(null == testClass) { 165 testClass= tm.getTestClass(); 166 } 167 168 ITestResult testResult= new TestResult(testClass, 169 instance, 170 tm, 171 null, 172 System.currentTimeMillis(), 173 System.currentTimeMillis(), 174 m_testContext); 175 176 IConfigurationAnnotation configurationAnnotation= null; 177 try { 178 Object inst = tm.getInstance(); 179 if (inst == null) { 180 inst = instance; 181 } 182 Class<?> objectClass= inst.getClass(); 183 Method method= tm.getMethod(); 184 185 // Only run the configuration if 186 // - the test is enabled and 187 // - the Configuration method belongs to the same class or a parent 188 configurationAnnotation = AnnotationHelper.findConfiguration(m_annotationFinder, method); 189 boolean alwaysRun= isAlwaysRun(configurationAnnotation); 190 if(MethodHelper.isEnabled(objectClass, m_annotationFinder) || alwaysRun) { 191 192 if (MethodHelper.isEnabled(configurationAnnotation)) { 193 194 if (!confInvocationPassed(tm, currentTestMethod, testClass, instance) && !alwaysRun) { 195 handleConfigurationSkip(tm, testResult, configurationAnnotation, currentTestMethod, instance, suite); 196 continue; 197 } 198 199 log(3, "Invoking " + Utils.detailedMethodName(tm, true)); 200 201 Object[] parameters = Parameters.createConfigurationParameters(tm.getMethod(), 202 params, 203 parameterValues, 204 currentTestMethod, 205 m_annotationFinder, 206 suite, 207 m_testContext, 208 testMethodResult); 209 testResult.setParameters(parameters); 210 211 Object newInstance = null != instance ? instance: inst; 212 213 runConfigurationListeners(testResult, true /* before */); 214 215 invokeConfigurationMethod(newInstance, tm, 216 parameters, testResult); 217 218 // TODO: probably we should trigger the event for each instance??? 219 testResult.setEndMillis(System.currentTimeMillis()); 220 runConfigurationListeners(testResult, false /* after */); 221 } 222 else { 223 log(3, 224 "Skipping " 225 + Utils.detailedMethodName(tm, true) 226 + " because it is not enabled"); 227 } 228 } // if is enabled 229 else { 230 log(3, 231 "Skipping " 232 + Utils.detailedMethodName(tm, true) 233 + " because " 234 + objectClass.getName() 235 + " is not enabled"); 236 } 237 } 238 catch(InvocationTargetException ex) { 239 handleConfigurationFailure(ex, tm, testResult, configurationAnnotation, currentTestMethod, instance, suite); 240 } catch(Throwable ex) { // covers the non-wrapper exceptions 241 handleConfigurationFailure(ex, tm, testResult, configurationAnnotation, currentTestMethod, instance, suite); 242 } 243 } // for methods 244 } 245 246 /** 247 * Marks the current <code>TestResult</code> as skipped and invokes the listeners. 248 */ handleConfigurationSkip(ITestNGMethod tm, ITestResult testResult, IConfigurationAnnotation annotation, ITestNGMethod currentTestMethod, Object instance, XmlSuite suite)249 private void handleConfigurationSkip(ITestNGMethod tm, 250 ITestResult testResult, 251 IConfigurationAnnotation annotation, 252 ITestNGMethod currentTestMethod, 253 Object instance, 254 XmlSuite suite) { 255 recordConfigurationInvocationFailed(tm, testResult.getTestClass(), annotation, currentTestMethod, instance, suite); 256 testResult.setStatus(ITestResult.SKIP); 257 runConfigurationListeners(testResult, false /* after */); 258 } 259 260 /** 261 * Is the <code>IConfiguration</code> marked as alwaysRun. 262 */ isAlwaysRun(IConfigurationAnnotation configurationAnnotation)263 private boolean isAlwaysRun(IConfigurationAnnotation configurationAnnotation) { 264 if(null == configurationAnnotation) { 265 return false; 266 } 267 268 boolean alwaysRun= false; 269 if ((configurationAnnotation.getAfterSuite() 270 || configurationAnnotation.getAfterTest() 271 || configurationAnnotation.getAfterTestClass() 272 || configurationAnnotation.getAfterTestMethod() 273 || configurationAnnotation.getBeforeTestMethod() 274 || configurationAnnotation.getBeforeTestClass() 275 || configurationAnnotation.getBeforeTest() 276 || configurationAnnotation.getBeforeSuite()) 277 && configurationAnnotation.getAlwaysRun()) 278 { 279 alwaysRun= true; 280 } 281 282 return alwaysRun; 283 } 284 handleConfigurationFailure(Throwable ite, ITestNGMethod tm, ITestResult testResult, IConfigurationAnnotation annotation, ITestNGMethod currentTestMethod, Object instance, XmlSuite suite)285 private void handleConfigurationFailure(Throwable ite, 286 ITestNGMethod tm, 287 ITestResult testResult, 288 IConfigurationAnnotation annotation, 289 ITestNGMethod currentTestMethod, 290 Object instance, 291 XmlSuite suite) 292 { 293 Throwable cause= ite.getCause() != null ? ite.getCause() : ite; 294 295 if(isSkipExceptionAndSkip(cause)) { 296 testResult.setThrowable(cause); 297 handleConfigurationSkip(tm, testResult, annotation, currentTestMethod, instance, suite); 298 return; 299 } 300 Utils.log("", 3, "Failed to invoke configuration method " 301 + tm.getRealClass().getName() + "." + tm.getMethodName() + ":" + cause.getMessage()); 302 handleException(cause, tm, testResult, 1); 303 runConfigurationListeners(testResult, false /* after */); 304 305 // 306 // If in TestNG mode, need to take a look at the annotation to figure out 307 // what kind of @Configuration method we're dealing with 308 // 309 if (null != annotation) { 310 recordConfigurationInvocationFailed(tm, testResult.getTestClass(), annotation, currentTestMethod, instance, suite); 311 } 312 } 313 314 /** 315 * @return All the classes that belong to the same <test> tag as @param cls 316 */ findClassesInSameTest(Class<?> cls, XmlSuite suite)317 private XmlClass[] findClassesInSameTest(Class<?> cls, XmlSuite suite) { 318 Map<String, XmlClass> vResult= Maps.newHashMap(); 319 String className= cls.getName(); 320 for(XmlTest test : suite.getTests()) { 321 for(XmlClass testClass : test.getXmlClasses()) { 322 if(testClass.getName().equals(className)) { 323 324 // Found it, add all the classes in this test in the result 325 for(XmlClass thisClass : test.getXmlClasses()) { 326 vResult.put(thisClass.getName(), thisClass); 327 } 328 // Note: we need to iterate through the entire suite since the same 329 // class might appear in several <test> tags 330 } 331 } 332 } 333 334 XmlClass[] result= vResult.values().toArray(new XmlClass[vResult.size()]); 335 336 return result; 337 } 338 339 /** 340 * Record internally the failure of a Configuration, so that we can determine 341 * later if @Test should be skipped. 342 */ recordConfigurationInvocationFailed(ITestNGMethod tm, IClass testClass, IConfigurationAnnotation annotation, ITestNGMethod currentTestMethod, Object instance, XmlSuite suite)343 private void recordConfigurationInvocationFailed(ITestNGMethod tm, 344 IClass testClass, 345 IConfigurationAnnotation annotation, 346 ITestNGMethod currentTestMethod, 347 Object instance, 348 XmlSuite suite) { 349 // If beforeTestClass or afterTestClass failed, mark either the config method's 350 // entire class as failed, or the class under tests as failed, depending on 351 // the configuration failure policy 352 if (annotation.getBeforeTestClass() || annotation.getAfterTestClass()) { 353 // tm is the configuration method, and currentTestMethod is null for BeforeClass 354 // methods, so we need testClass 355 if (m_continueOnFailedConfiguration) { 356 setClassInvocationFailure(testClass.getRealClass(), instance); 357 } else { 358 setClassInvocationFailure(tm.getRealClass(), instance); 359 } 360 } 361 362 // If before/afterTestMethod failed, mark either the config method's entire 363 // class as failed, or just the current test method as failed, depending on 364 // the configuration failure policy 365 else if (annotation.getBeforeTestMethod() || annotation.getAfterTestMethod()) { 366 if (m_continueOnFailedConfiguration) { 367 setMethodInvocationFailure(currentTestMethod, instance); 368 } else { 369 setClassInvocationFailure(tm.getRealClass(), instance); 370 } 371 } 372 373 // If beforeSuite or afterSuite failed, mark *all* the classes as failed 374 // for configurations. At this point, the entire Suite is screwed 375 else if (annotation.getBeforeSuite() || annotation.getAfterSuite()) { 376 m_suiteState.failed(); 377 } 378 379 // beforeTest or afterTest: mark all the classes in the same 380 // <test> stanza as failed for configuration 381 else if (annotation.getBeforeTest() || annotation.getAfterTest()) { 382 setClassInvocationFailure(tm.getRealClass(), instance); 383 XmlClass[] classes= findClassesInSameTest(tm.getRealClass(), suite); 384 for(XmlClass xmlClass : classes) { 385 setClassInvocationFailure(xmlClass.getSupportClass(), instance); 386 } 387 } 388 String[] beforeGroups= annotation.getBeforeGroups(); 389 if(null != beforeGroups && beforeGroups.length > 0) { 390 for(String group: beforeGroups) { 391 m_beforegroupsFailures.put(group, Boolean.FALSE); 392 } 393 } 394 } 395 396 /** 397 * @return true if this class or a parent class failed to initialize. 398 */ classConfigurationFailed(Class<?> cls)399 private boolean classConfigurationFailed(Class<?> cls) { 400 for (Class<?> c : m_classInvocationResults.keySet()) { 401 if (c == cls || c.isAssignableFrom(cls)) { 402 return true; 403 } 404 } 405 return false; 406 } 407 408 /** 409 * @return true if this class has successfully run all its @Configuration 410 * method or false if at least one of these methods failed. 411 */ confInvocationPassed(ITestNGMethod method, ITestNGMethod currentTestMethod, IClass testClass, Object instance)412 private boolean confInvocationPassed(ITestNGMethod method, ITestNGMethod currentTestMethod, 413 IClass testClass, Object instance) { 414 boolean result= true; 415 416 Class<?> cls = testClass.getRealClass(); 417 418 if(m_suiteState.isFailed()) { 419 result= false; 420 } 421 else { 422 if (classConfigurationFailed(cls)) { 423 if (! m_continueOnFailedConfiguration) { 424 result = !classConfigurationFailed(cls); 425 } else { 426 result = !m_classInvocationResults.get(cls).contains(instance); 427 } 428 } 429 // if method is BeforeClass, currentTestMethod will be null 430 else if (m_continueOnFailedConfiguration && 431 currentTestMethod != null && 432 m_methodInvocationResults.containsKey(currentTestMethod)) { 433 result = !m_methodInvocationResults.get(currentTestMethod).contains(getMethodInvocationToken(currentTestMethod, instance)); 434 } 435 else if (! m_continueOnFailedConfiguration) { 436 for(Class<?> clazz: m_classInvocationResults.keySet()) { 437 // if (clazz == cls) { 438 if(clazz.isAssignableFrom(cls)) { 439 result= false; 440 break; 441 } 442 } 443 } 444 } 445 446 // check if there are failed @BeforeGroups 447 String[] groups= method.getGroups(); 448 if(null != groups && groups.length > 0) { 449 for(String group: groups) { 450 if(m_beforegroupsFailures.containsKey(group)) { 451 result= false; 452 break; 453 } 454 } 455 } 456 return result; 457 } 458 459 // Creates a token for tracking a unique invocation of a method on an instance. 460 // Is used when configFailurePolicy=continue. getMethodInvocationToken(ITestNGMethod method, Object instance)461 private Object getMethodInvocationToken(ITestNGMethod method, Object instance) { 462 return String.format("%s+%d", instance.toString(), method.getCurrentInvocationCount()); 463 } 464 465 /** 466 * Effectively invokes a configuration method on all passed in instances. 467 * TODO: Should change this method to be more like invokeMethod() so that we can 468 * handle calls to {@code IInvokedMethodListener} better. 469 * 470 * @param targetInstance the instance to invoke the configuration method on 471 * @param tm the configuration method 472 * @param params the parameters needed for method invocation 473 * @param testResult 474 * @throws InvocationTargetException 475 * @throws IllegalAccessException 476 */ invokeConfigurationMethod(Object targetInstance, ITestNGMethod tm, Object[] params, ITestResult testResult)477 private void invokeConfigurationMethod(Object targetInstance, 478 ITestNGMethod tm, 479 Object[] params, 480 ITestResult testResult) 481 throws InvocationTargetException, IllegalAccessException 482 { 483 // Mark this method with the current thread id 484 tm.setId(ThreadUtil.currentThreadInfo()); 485 486 { 487 InvokedMethod invokedMethod= new InvokedMethod(targetInstance, 488 tm, 489 params, 490 System.currentTimeMillis(), 491 testResult); 492 493 runInvokedMethodListeners(BEFORE_INVOCATION, invokedMethod, testResult); 494 m_notifier.addInvokedMethod(invokedMethod); 495 try { 496 Reporter.setCurrentTestResult(testResult); 497 Method method = tm.getMethod(); 498 499 // 500 // If this method is a IConfigurable, invoke its run() method 501 // 502 IConfigurable configurableInstance = 503 IConfigurable.class.isAssignableFrom(tm.getMethod().getDeclaringClass()) ? 504 (IConfigurable) targetInstance : m_configuration.getConfigurable(); 505 if (configurableInstance != null) { 506 MethodInvocationHelper.invokeConfigurable(targetInstance, params, configurableInstance, method, 507 testResult); 508 } 509 else { 510 // 511 // Not a IConfigurable, invoke directly 512 // 513 if (MethodHelper.calculateTimeOut(tm) <= 0) { 514 MethodInvocationHelper.invokeMethod(method, targetInstance, params); 515 } 516 else { 517 MethodInvocationHelper.invokeWithTimeout(tm, targetInstance, params, testResult); 518 if (!testResult.isSuccess()) { 519 // A time out happened 520 throwConfigurationFailure(testResult, testResult.getThrowable()); 521 throw testResult.getThrowable(); 522 } 523 } 524 } 525 } 526 catch (InvocationTargetException | IllegalAccessException ex) { 527 throwConfigurationFailure(testResult, ex); 528 throw ex; 529 } catch (Throwable ex) { 530 throwConfigurationFailure(testResult, ex); 531 throw new TestNGException(ex); 532 } 533 finally { 534 Reporter.setCurrentTestResult(testResult); 535 runInvokedMethodListeners(AFTER_INVOCATION, invokedMethod, testResult); 536 Reporter.setCurrentTestResult(null); 537 } 538 } 539 } 540 throwConfigurationFailure(ITestResult testResult, Throwable ex)541 private void throwConfigurationFailure(ITestResult testResult, Throwable ex) 542 { 543 testResult.setStatus(ITestResult.FAILURE);; 544 testResult.setThrowable(ex.getCause() == null ? ex : ex.getCause()); 545 } 546 runInvokedMethodListeners(InvokedMethodListenerMethod listenerMethod, IInvokedMethod invokedMethod, ITestResult testResult)547 private void runInvokedMethodListeners(InvokedMethodListenerMethod listenerMethod, IInvokedMethod invokedMethod, 548 ITestResult testResult) 549 { 550 if ( noListenersPresent() ) { 551 return; 552 } 553 554 InvokedMethodListenerInvoker invoker = new InvokedMethodListenerInvoker(listenerMethod, testResult, m_testContext); 555 for (IInvokedMethodListener currentListener : m_invokedMethodListeners) { 556 invoker.invokeListener(currentListener, invokedMethod); 557 } 558 } 559 noListenersPresent()560 private boolean noListenersPresent() { 561 return (m_invokedMethodListeners == null) || (m_invokedMethodListeners.size() == 0); 562 } 563 564 // pass both paramValues and paramIndex to be thread safe in case parallel=true + dataprovider. invokeMethod(Object instance, final ITestNGMethod tm, Object[] parameterValues, int parametersIndex, XmlSuite suite, Map<String, String> params, ITestClass testClass, ITestNGMethod[] beforeMethods, ITestNGMethod[] afterMethods, ConfigurationGroupMethods groupMethods, FailureContext failureContext)565 private ITestResult invokeMethod(Object instance, 566 final ITestNGMethod tm, 567 Object[] parameterValues, 568 int parametersIndex, 569 XmlSuite suite, 570 Map<String, String> params, 571 ITestClass testClass, 572 ITestNGMethod[] beforeMethods, 573 ITestNGMethod[] afterMethods, 574 ConfigurationGroupMethods groupMethods, 575 FailureContext failureContext) { 576 TestResult testResult = new TestResult(); 577 578 // 579 // Invoke beforeGroups configurations 580 // 581 invokeBeforeGroupsConfigurations(testClass, tm, groupMethods, suite, params, 582 instance); 583 584 // 585 // Invoke beforeMethods only if 586 // - firstTimeOnly is not set 587 // - firstTimeOnly is set, and we are reaching at the first invocationCount 588 // 589 invokeConfigurations(testClass, tm, 590 filterConfigurationMethods(tm, beforeMethods, true /* beforeMethods */), 591 suite, params, parameterValues, 592 instance, testResult); 593 594 // 595 // Create the ExtraOutput for this method 596 // 597 InvokedMethod invokedMethod = null; 598 try { 599 testResult.init(testClass, instance, 600 tm, 601 null, 602 System.currentTimeMillis(), 603 0, 604 m_testContext); 605 testResult.setParameters(parameterValues); 606 testResult.setHost(m_testContext.getHost()); 607 testResult.setStatus(ITestResult.STARTED); 608 609 invokedMethod= new InvokedMethod(instance, 610 tm, 611 parameterValues, 612 System.currentTimeMillis(), 613 testResult); 614 615 // Fix from ansgarkonermann 616 // invokedMethod is used in the finally, which can be invoked if 617 // any of the test listeners throws an exception, therefore, 618 // invokedMethod must have a value before we get here 619 runTestListeners(testResult); 620 621 runInvokedMethodListeners(BEFORE_INVOCATION, invokedMethod, testResult); 622 623 m_notifier.addInvokedMethod(invokedMethod); 624 625 Method thisMethod = tm.getConstructorOrMethod().getMethod(); 626 627 if(confInvocationPassed(tm, tm, testClass, instance)) { 628 log(3, "Invoking " + tm.getRealClass().getName() + "." + tm.getMethodName()); 629 630 Reporter.setCurrentTestResult(testResult); 631 632 // If this method is a IHookable, invoke its run() method 633 IHookable hookableInstance = 634 IHookable.class.isAssignableFrom(tm.getRealClass()) ? 635 (IHookable) instance : m_configuration.getHookable(); 636 637 if (MethodHelper.calculateTimeOut(tm) <= 0) { 638 if (hookableInstance != null) { 639 MethodInvocationHelper.invokeHookable(instance, 640 parameterValues, hookableInstance, thisMethod, testResult); 641 } else { 642 // Not a IHookable, invoke directly 643 MethodInvocationHelper.invokeMethod(thisMethod, instance, 644 parameterValues); 645 } 646 testResult.setStatus(ITestResult.SUCCESS); 647 } else { 648 // Method with a timeout 649 MethodInvocationHelper.invokeWithTimeout(tm, instance, parameterValues, testResult, hookableInstance); 650 } 651 } 652 else { 653 testResult.setStatus(ITestResult.SKIP); 654 } 655 } 656 catch(InvocationTargetException ite) { 657 testResult.setThrowable(ite.getCause()); 658 testResult.setStatus(ITestResult.FAILURE); 659 } 660 catch(ThreadExecutionException tee) { // wrapper for TestNGRuntimeException 661 Throwable cause= tee.getCause(); 662 if(TestNGRuntimeException.class.equals(cause.getClass())) { 663 testResult.setThrowable(cause.getCause()); 664 } 665 else { 666 testResult.setThrowable(cause); 667 } 668 testResult.setStatus(ITestResult.FAILURE); 669 } 670 catch(Throwable thr) { // covers the non-wrapper exceptions 671 testResult.setThrowable(thr); 672 testResult.setStatus(ITestResult.FAILURE); 673 } 674 finally { 675 // Set end time ASAP 676 testResult.setEndMillis(System.currentTimeMillis()); 677 678 ExpectedExceptionsHolder expectedExceptionClasses 679 = new ExpectedExceptionsHolder(m_annotationFinder, tm, new RegexpExpectedExceptionsHolder(m_annotationFinder, tm)); 680 List<ITestResult> results = Lists.<ITestResult>newArrayList(testResult); 681 handleInvocationResults(tm, results, expectedExceptionClasses, failureContext); 682 683 // If this method has a data provider and just failed, memorize the number 684 // at which it failed. 685 // Note: we're not exactly testing that this method has a data provider, just 686 // that it has parameters, so might have to revisit this if bugs get reported 687 // for the case where this method has parameters that don't come from a data 688 // provider 689 if (testResult.getThrowable() != null && parameterValues.length > 0) { 690 tm.addFailedInvocationNumber(parametersIndex); 691 } 692 693 // 694 // Increment the invocation count for this method 695 // 696 tm.incrementCurrentInvocationCount(); 697 698 // Run invokedMethodListeners after updating TestResult 699 runInvokedMethodListeners(AFTER_INVOCATION, invokedMethod, testResult); 700 runTestListeners(testResult); 701 702 // 703 // Invoke afterMethods only if 704 // - lastTimeOnly is not set 705 // - lastTimeOnly is set, and we are reaching the last invocationCount 706 // 707 invokeConfigurations(testClass, tm, 708 filterConfigurationMethods(tm, afterMethods, false /* beforeMethods */), 709 suite, params, parameterValues, 710 instance, 711 testResult); 712 713 // 714 // Invoke afterGroups configurations 715 // 716 invokeAfterGroupsConfigurations(testClass, tm, groupMethods, suite, 717 params, instance); 718 719 // Reset the test result last. If we do this too early, Reporter.log() 720 // invocations from listeners will be discarded 721 Reporter.setCurrentTestResult(null); 722 } 723 724 return testResult; 725 } 726 collectResults(ITestNGMethod testMethod, Collection<ITestResult> results)727 void collectResults(ITestNGMethod testMethod, Collection<ITestResult> results) { 728 for (ITestResult result : results) { 729 // Collect the results 730 final int status = result.getStatus(); 731 if(ITestResult.SUCCESS == status) { 732 m_notifier.addPassedTest(testMethod, result); 733 } 734 else if(ITestResult.SKIP == status) { 735 m_notifier.addSkippedTest(testMethod, result); 736 } 737 else if(ITestResult.FAILURE == status) { 738 m_notifier.addFailedTest(testMethod, result); 739 } 740 else if(ITestResult.SUCCESS_PERCENTAGE_FAILURE == status) { 741 m_notifier.addFailedButWithinSuccessPercentageTest(testMethod, result); 742 } 743 else { 744 assert false : "UNKNOWN STATUS:" + status; 745 } 746 } 747 } 748 749 /** 750 * The array of methods contains @BeforeMethods if isBefore if true, @AfterMethods 751 * otherwise. This function removes all the methods that should not be run at this 752 * point because they are either firstTimeOnly or lastTimeOnly and we haven't reached 753 * the current invocationCount yet 754 */ filterConfigurationMethods(ITestNGMethod tm, ITestNGMethod[] methods, boolean isBefore)755 private ITestNGMethod[] filterConfigurationMethods(ITestNGMethod tm, 756 ITestNGMethod[] methods, boolean isBefore) 757 { 758 List<ITestNGMethod> result = Lists.newArrayList(); 759 for (ITestNGMethod m : methods) { 760 ConfigurationMethod cm = (ConfigurationMethod) m; 761 if (isBefore) { 762 if (! cm.isFirstTimeOnly() || 763 (cm.isFirstTimeOnly() && tm.getCurrentInvocationCount() == 0)) 764 { 765 result.add(m); 766 } 767 } 768 else { 769 int current = tm.getCurrentInvocationCount(); 770 boolean isLast = false; 771 // If we have parameters, set the boolean if we are about to run 772 // the last invocation 773 if (tm.getParameterInvocationCount() > 0) { 774 isLast = current == tm.getParameterInvocationCount() * tm.getTotalInvocationCount(); 775 } 776 // If we have invocationCount > 1, set the boolean if we are about to 777 // run the last invocation 778 else if (tm.getTotalInvocationCount() > 1) { 779 isLast = current == tm.getTotalInvocationCount(); 780 } 781 if (! cm.isLastTimeOnly() || (cm.isLastTimeOnly() && isLast)) { 782 result.add(m); 783 } 784 } 785 } 786 787 return result.toArray(new ITestNGMethod[result.size()]); 788 } 789 790 /** 791 * invokeTestMethods() eventually converge here to invoke a single @Test method. 792 * <p/> 793 * This method is responsible for actually invoking the method. It decides if the invocation 794 * must be done: 795 * <ul> 796 * <li>through an <code>IHookable</code></li> 797 * <li>directly (through reflection)</li> 798 * <li>in a separate thread (in case it needs to timeout) 799 * </ul> 800 * 801 * <p/> 802 * This method is also responsible for invoking @BeforeGroup, @BeforeMethod, @AfterMethod, @AfterGroup 803 * if it is the case for the passed in @Test method. 804 */ invokeTestMethod(Object instance, final ITestNGMethod tm, Object[] parameterValues, int parametersIndex, XmlSuite suite, Map<String, String> params, ITestClass testClass, ITestNGMethod[] beforeMethods, ITestNGMethod[] afterMethods, ConfigurationGroupMethods groupMethods, FailureContext failureContext)805 protected ITestResult invokeTestMethod(Object instance, 806 final ITestNGMethod tm, 807 Object[] parameterValues, 808 int parametersIndex, 809 XmlSuite suite, 810 Map<String, String> params, 811 ITestClass testClass, 812 ITestNGMethod[] beforeMethods, 813 ITestNGMethod[] afterMethods, 814 ConfigurationGroupMethods groupMethods, 815 FailureContext failureContext) 816 { 817 // Mark this method with the current thread id 818 tm.setId(ThreadUtil.currentThreadInfo()); 819 820 ITestResult result = invokeMethod(instance, tm, parameterValues, parametersIndex, suite, params, 821 testClass, beforeMethods, afterMethods, groupMethods, 822 failureContext); 823 824 return result; 825 } 826 827 /** 828 * Filter all the beforeGroups methods and invoke only those that apply 829 * to the current test method 830 */ invokeBeforeGroupsConfigurations(ITestClass testClass, ITestNGMethod currentTestMethod, ConfigurationGroupMethods groupMethods, XmlSuite suite, Map<String, String> params, Object instance)831 private void invokeBeforeGroupsConfigurations(ITestClass testClass, 832 ITestNGMethod currentTestMethod, 833 ConfigurationGroupMethods groupMethods, 834 XmlSuite suite, 835 Map<String, String> params, 836 Object instance) 837 { 838 synchronized(groupMethods) { 839 List<ITestNGMethod> filteredMethods = Lists.newArrayList(); 840 String[] groups = currentTestMethod.getGroups(); 841 Map<String, List<ITestNGMethod>> beforeGroupMap = groupMethods.getBeforeGroupsMap(); 842 843 for (String group : groups) { 844 List<ITestNGMethod> methods = beforeGroupMap.get(group); 845 if (methods != null) { 846 filteredMethods.addAll(methods); 847 } 848 } 849 850 ITestNGMethod[] beforeMethodsArray = filteredMethods.toArray(new ITestNGMethod[filteredMethods.size()]); 851 // 852 // Invoke the right groups methods 853 // 854 if(beforeMethodsArray.length > 0) { 855 // don't pass the IClass or the instance as the method may be external 856 // the invocation must be similar to @BeforeTest/@BeforeSuite 857 invokeConfigurations(null, beforeMethodsArray, suite, params, 858 null, /* no parameter values */ 859 null); 860 } 861 862 // 863 // Remove them so they don't get run again 864 // 865 groupMethods.removeBeforeGroups(groups); 866 } 867 } 868 invokeAfterGroupsConfigurations(ITestClass testClass, ITestNGMethod currentTestMethod, ConfigurationGroupMethods groupMethods, XmlSuite suite, Map<String, String> params, Object instance)869 private void invokeAfterGroupsConfigurations(ITestClass testClass, 870 ITestNGMethod currentTestMethod, 871 ConfigurationGroupMethods groupMethods, 872 XmlSuite suite, 873 Map<String, String> params, 874 Object instance) 875 { 876 // Skip this if the current method doesn't belong to any group 877 // (only a method that belongs to a group can trigger the invocation 878 // of afterGroups methods) 879 if (currentTestMethod.getGroups().length == 0) { 880 return; 881 } 882 883 // See if the currentMethod is the last method in any of the groups 884 // it belongs to 885 Map<String, String> filteredGroups = Maps.newHashMap(); 886 String[] groups = currentTestMethod.getGroups(); 887 synchronized(groupMethods) { 888 for (String group : groups) { 889 if (groupMethods.isLastMethodForGroup(group, currentTestMethod)) { 890 filteredGroups.put(group, group); 891 } 892 } 893 894 if(filteredGroups.isEmpty()) { 895 return; 896 } 897 898 // The list of afterMethods to run 899 Map<ITestNGMethod, ITestNGMethod> afterMethods = Maps.newHashMap(); 900 901 // Now filteredGroups contains all the groups for which we need to run the afterGroups 902 // method. Find all the methods that correspond to these groups and invoke them. 903 Map<String, List<ITestNGMethod>> map = groupMethods.getAfterGroupsMap(); 904 for (String g : filteredGroups.values()) { 905 List<ITestNGMethod> methods = map.get(g); 906 // Note: should put them in a map if we want to make sure the same afterGroups 907 // doesn't get run twice 908 if (methods != null) { 909 for (ITestNGMethod m : methods) { 910 afterMethods.put(m, m); 911 } 912 } 913 } 914 915 // Got our afterMethods, invoke them 916 ITestNGMethod[] afterMethodsArray = afterMethods.keySet().toArray(new ITestNGMethod[afterMethods.size()]); 917 // don't pass the IClass or the instance as the method may be external 918 // the invocation must be similar to @BeforeTest/@BeforeSuite 919 invokeConfigurations(null, afterMethodsArray, suite, params, 920 null, /* no parameter values */ 921 null); 922 923 // Remove the groups so they don't get run again 924 groupMethods.removeAfterGroups(filteredGroups.keySet()); 925 } 926 } 927 getParametersFromIndex(Iterator<Object[]> parametersValues, int index)928 private Object[] getParametersFromIndex(Iterator<Object[]> parametersValues, int index) { 929 while (parametersValues.hasNext()) { 930 Object[] parameters = parametersValues.next(); 931 932 if (index == 0) { 933 return parameters; 934 } 935 index--; 936 } 937 return null; 938 } 939 retryFailed(Object instance, final ITestNGMethod tm, XmlSuite suite, ITestClass testClass, ITestNGMethod[] beforeMethods, ITestNGMethod[] afterMethods, ConfigurationGroupMethods groupMethods, List<ITestResult> result, int failureCount, ExpectedExceptionsHolder expectedExceptionHolder, ITestContext testContext, Map<String, String> parameters, int parametersIndex)940 int retryFailed(Object instance, 941 final ITestNGMethod tm, 942 XmlSuite suite, 943 ITestClass testClass, 944 ITestNGMethod[] beforeMethods, 945 ITestNGMethod[] afterMethods, 946 ConfigurationGroupMethods groupMethods, 947 List<ITestResult> result, 948 int failureCount, 949 ExpectedExceptionsHolder expectedExceptionHolder, 950 ITestContext testContext, 951 Map<String, String> parameters, 952 int parametersIndex) { 953 final FailureContext failure = new FailureContext(); 954 failure.count = failureCount; 955 do { 956 failure.instances = Lists.newArrayList (); 957 Map<String, String> allParameters = Maps.newHashMap(); 958 /** 959 * TODO: This recreates all the parameters every time when we only need 960 * one specific set. Should optimize it by only recreating the set needed. 961 */ 962 ParameterBag bag = createParameters(tm, parameters, 963 allParameters, suite, testContext, null /* fedInstance */); 964 Object[] parameterValues = 965 getParametersFromIndex(bag.parameterHolder.parameters, parametersIndex); 966 967 result.add(invokeMethod(instance, tm, parameterValues, parametersIndex, suite, 968 allParameters, testClass, beforeMethods, afterMethods, groupMethods, failure)); 969 } 970 while (!failure.instances.isEmpty()); 971 return failure.count; 972 } 973 createParameters(ITestNGMethod testMethod, Map<String, String> parameters, Map<String, String> allParameterNames, XmlSuite suite, ITestContext testContext, Object fedInstance)974 private ParameterBag createParameters(ITestNGMethod testMethod, 975 Map<String, String> parameters, 976 Map<String, String> allParameterNames, 977 XmlSuite suite, 978 ITestContext testContext, 979 Object fedInstance) 980 { 981 Object instance; 982 if (fedInstance != null) { 983 instance = fedInstance; 984 } 985 else { 986 instance = testMethod.getInstance(); 987 } 988 989 ParameterBag bag = handleParameters(testMethod, 990 instance, allParameterNames, parameters, null, suite, testContext, fedInstance, null); 991 992 return bag; 993 } 994 995 /** 996 * Invoke all the test methods. Note the plural: the method passed in 997 * parameter might be invoked several times if the test class it belongs 998 * to has more than one instance (i.e., if an @Factory method has been 999 * declared somewhere that returns several instances of this TestClass). 1000 * If no @Factory method was specified, testMethod will only be invoked 1001 * once. 1002 * <p/> 1003 * Note that this method also takes care of invoking the beforeTestMethod 1004 * and afterTestMethod, if any. 1005 * 1006 * Note (alex): this method can be refactored to use a SingleTestMethodWorker that 1007 * directly invokes 1008 * {@link #invokeTestMethod(Object, ITestNGMethod, Object[], int, XmlSuite, Map, ITestClass, ITestNGMethod[], ITestNGMethod[], ConfigurationGroupMethods, FailureContext)} 1009 * and this would simplify the implementation (see how DataTestMethodWorker is used) 1010 */ 1011 @Override invokeTestMethods(ITestNGMethod testMethod, XmlSuite suite, Map<String, String> testParameters, ConfigurationGroupMethods groupMethods, Object instance, ITestContext testContext)1012 public List<ITestResult> invokeTestMethods(ITestNGMethod testMethod, 1013 XmlSuite suite, 1014 Map<String, String> testParameters, 1015 ConfigurationGroupMethods groupMethods, 1016 Object instance, 1017 ITestContext testContext) 1018 { 1019 // Potential bug here if the test method was declared on a parent class 1020 assert null != testMethod.getTestClass() 1021 : "COULDN'T FIND TESTCLASS FOR " + testMethod.getRealClass(); 1022 1023 if (!MethodHelper.isEnabled(testMethod.getMethod(), m_annotationFinder)) { 1024 // return if the method is not enabled. No need to do any more calculations 1025 return Collections.emptyList(); 1026 } 1027 1028 // By the time this testMethod to be invoked, 1029 // all dependencies should be already run or we need to skip this method, 1030 // so invocation count should not affect dependencies check 1031 final String okToProceed = checkDependencies(testMethod, testContext.getAllTestMethods()); 1032 1033 if (okToProceed != null) { 1034 // 1035 // Not okToProceed. Test is being skipped 1036 // 1037 ITestResult result = registerSkippedTestResult(testMethod, null, System.currentTimeMillis(), 1038 new Throwable(okToProceed)); 1039 m_notifier.addSkippedTest(testMethod, result); 1040 return Collections.singletonList(result); 1041 } 1042 1043 1044 final Map<String, String> parameters = 1045 testMethod.findMethodParameters(testContext.getCurrentXmlTest()); 1046 1047 // For invocationCount > 1 and threadPoolSize > 1 run this method in its own pool thread. 1048 if (testMethod.getInvocationCount() > 1 && testMethod.getThreadPoolSize() > 1) { 1049 return invokePooledTestMethods(testMethod, suite, parameters, groupMethods, testContext); 1050 } 1051 1052 long timeOutInvocationCount = testMethod.getInvocationTimeOut(); 1053 //FIXME: Is this correct? 1054 boolean onlyOne = testMethod.getThreadPoolSize() > 1 || 1055 timeOutInvocationCount > 0; 1056 1057 int invocationCount = onlyOne ? 1 : testMethod.getInvocationCount(); 1058 1059 ExpectedExceptionsHolder expectedExceptionHolder = 1060 new ExpectedExceptionsHolder(m_annotationFinder, testMethod, 1061 new RegexpExpectedExceptionsHolder(m_annotationFinder, testMethod)); 1062 final ITestClass testClass= testMethod.getTestClass(); 1063 final List<ITestResult> result = Lists.newArrayList(); 1064 final FailureContext failure = new FailureContext(); 1065 final ITestNGMethod[] beforeMethods = filterMethods(testClass, testClass.getBeforeTestMethods(), CAN_RUN_FROM_CLASS); 1066 final ITestNGMethod[] afterMethods = filterMethods(testClass, testClass.getAfterTestMethods(), CAN_RUN_FROM_CLASS); 1067 while(invocationCount-- > 0) { 1068 if(false) { 1069 // Prevent code formatting 1070 } 1071 // 1072 // No threads, regular invocation 1073 // 1074 else { 1075 // Used in catch statement 1076 long start = System.currentTimeMillis(); 1077 1078 Map<String, String> allParameterNames = Maps.newHashMap(); 1079 ParameterBag bag = createParameters(testMethod, 1080 parameters, allParameterNames, suite, testContext, instance); 1081 1082 if (bag.hasErrors()) { 1083 final ITestResult tr = bag.errorResult; 1084 tr.setStatus(ITestResult.SKIP); 1085 runTestListeners(tr); 1086 m_notifier.addSkippedTest(testMethod, tr); 1087 result.add(tr); 1088 continue; 1089 } 1090 1091 Iterator<Object[]> allParameterValues = bag.parameterHolder.parameters; 1092 int parametersIndex = 0; 1093 1094 try { 1095 List<TestMethodWithDataProviderMethodWorker> workers = Lists.newArrayList(); 1096 1097 if (bag.parameterHolder.origin == ParameterOrigin.ORIGIN_DATA_PROVIDER && 1098 bag.parameterHolder.dataProviderHolder.annotation.isParallel()) { 1099 while (allParameterValues.hasNext()) { 1100 Object[] parameterValues = injectParameters(allParameterValues.next(), 1101 testMethod.getMethod(), testContext, null /* test result */); 1102 TestMethodWithDataProviderMethodWorker w = 1103 new TestMethodWithDataProviderMethodWorker(this, 1104 testMethod, parametersIndex, 1105 parameterValues, instance, suite, parameters, testClass, 1106 beforeMethods, afterMethods, groupMethods, 1107 expectedExceptionHolder, testContext, m_skipFailedInvocationCounts, 1108 invocationCount, failure.count, m_notifier); 1109 workers.add(w); 1110 // testng387: increment the param index in the bag. 1111 parametersIndex++; 1112 } 1113 PoolService<List<ITestResult>> ps = 1114 new PoolService<>(suite.getDataProviderThreadCount()); 1115 List<List<ITestResult>> r = ps.submitTasksAndWait(workers); 1116 for (List<ITestResult> l2 : r) { 1117 result.addAll(l2); 1118 } 1119 1120 } else { 1121 while (allParameterValues.hasNext()) { 1122 Object[] parameterValues = injectParameters(allParameterValues.next(), 1123 testMethod.getMethod(), testContext, null /* test result */); 1124 1125 List<ITestResult> tmpResults = Lists.newArrayList(); 1126 1127 try { 1128 tmpResults.add(invokeTestMethod(instance, 1129 testMethod, 1130 parameterValues, 1131 parametersIndex, 1132 suite, 1133 parameters, 1134 testClass, 1135 beforeMethods, 1136 afterMethods, 1137 groupMethods, failure)); 1138 } 1139 finally { 1140 if (failure.instances.isEmpty()) { 1141 result.addAll(tmpResults); 1142 } else { 1143 for (Object failedInstance : failure.instances) { 1144 List<ITestResult> retryResults = Lists.newArrayList(); 1145 1146 failure.count = retryFailed( 1147 failedInstance, testMethod, suite, testClass, beforeMethods, 1148 afterMethods, groupMethods, retryResults, 1149 failure.count, expectedExceptionHolder, 1150 testContext, parameters, parametersIndex); 1151 result.addAll(retryResults); 1152 } 1153 } 1154 1155 // 1156 // If we have a failure, skip all the 1157 // other invocationCounts 1158 // 1159 if (failure.count > 0 1160 && (m_skipFailedInvocationCounts 1161 || testMethod.skipFailedInvocations())) { 1162 while (invocationCount-- > 0) { 1163 result.add(registerSkippedTestResult(testMethod, instance, System.currentTimeMillis(), null)); 1164 } 1165 break; 1166 } 1167 }// end finally 1168 parametersIndex++; 1169 } 1170 } 1171 } 1172 catch (Throwable cause) { 1173 ITestResult r = 1174 new TestResult(testMethod.getTestClass(), 1175 instance, 1176 testMethod, 1177 cause, 1178 start, 1179 System.currentTimeMillis(), 1180 m_testContext); 1181 r.setStatus(TestResult.FAILURE); 1182 result.add(r); 1183 runTestListeners(r); 1184 m_notifier.addFailedTest(testMethod, r); 1185 } // catch 1186 } 1187 } 1188 1189 return result; 1190 1191 } // invokeTestMethod 1192 registerSkippedTestResult(ITestNGMethod testMethod, Object instance, long start, Throwable throwable)1193 private ITestResult registerSkippedTestResult(ITestNGMethod testMethod, Object instance, 1194 long start, Throwable throwable) { 1195 ITestResult result = 1196 new TestResult(testMethod.getTestClass(), 1197 instance, 1198 testMethod, 1199 throwable, 1200 start, 1201 System.currentTimeMillis(), 1202 m_testContext); 1203 result.setStatus(TestResult.SKIP); 1204 runTestListeners(result); 1205 1206 return result; 1207 } 1208 1209 /** 1210 * Gets an array of parameter values returned by data provider or the ones that 1211 * are injected based on parameter type. The method also checks for {@code NoInjection} 1212 * annotation 1213 * @param parameterValues parameter values from a data provider 1214 * @param method method to be invoked 1215 * @param context test context 1216 * @param testResult test result 1217 */ injectParameters(Object[] parameterValues, Method method, ITestContext context, ITestResult testResult)1218 private Object[] injectParameters(Object[] parameterValues, Method method, 1219 ITestContext context, ITestResult testResult) 1220 throws TestNGException { 1221 List<Object> vResult = Lists.newArrayList(); 1222 int i = 0; 1223 int numValues = parameterValues.length; 1224 int numParams = method.getParameterTypes().length; 1225 1226 if (numValues > numParams && ! method.isVarArgs()) { 1227 throw new TestNGException("The data provider is trying to pass " + numValues 1228 + " parameters but the method " 1229 + method.getDeclaringClass().getName() + "#" + method.getName() 1230 + " takes " + numParams); 1231 } 1232 1233 // beyond this, numValues <= numParams 1234 for (Class<?> cls : method.getParameterTypes()) { 1235 Annotation[] annotations = method.getParameterAnnotations()[i]; 1236 boolean noInjection = false; 1237 for (Annotation a : annotations) { 1238 if (a instanceof NoInjection) { 1239 noInjection = true; 1240 break; 1241 } 1242 } 1243 Object injected = Parameters.getInjectedParameter(cls, method, context, testResult); 1244 if (injected != null && ! noInjection) { 1245 vResult.add(injected); 1246 } else { 1247 try { 1248 if (method.isVarArgs()) vResult.add(parameterValues); 1249 else vResult.add(parameterValues[i++]); 1250 } catch (ArrayIndexOutOfBoundsException ex) { 1251 throw new TestNGException("The data provider is trying to pass " + numValues 1252 + " parameters but the method " 1253 + method.getDeclaringClass().getName() + "#" + method.getName() 1254 + " takes " + numParams 1255 + " and TestNG is unable in inject a suitable object", ex); 1256 } 1257 } 1258 } 1259 return vResult.toArray(new Object[vResult.size()]); 1260 } 1261 handleParameters(ITestNGMethod testMethod, Object instance, Map<String, String> allParameterNames, Map<String, String> parameters, Object[] parameterValues, XmlSuite suite, ITestContext testContext, Object fedInstance, ITestResult testResult)1262 private ParameterBag handleParameters(ITestNGMethod testMethod, 1263 Object instance, 1264 Map<String, String> allParameterNames, 1265 Map<String, String> parameters, 1266 Object[] parameterValues, 1267 XmlSuite suite, 1268 ITestContext testContext, 1269 Object fedInstance, 1270 ITestResult testResult) 1271 { 1272 try { 1273 return new ParameterBag( 1274 Parameters.handleParameters(testMethod, 1275 allParameterNames, 1276 instance, 1277 new Parameters.MethodParameters(parameters, 1278 testMethod.findMethodParameters(testContext.getCurrentXmlTest()), 1279 parameterValues, 1280 testMethod.getMethod(), testContext, testResult), 1281 suite, 1282 m_annotationFinder, 1283 fedInstance)); 1284 } 1285 // catch(TestNGException ex) { 1286 // throw ex; 1287 // } 1288 catch(Throwable cause) { 1289 return new ParameterBag( 1290 new TestResult( 1291 testMethod.getTestClass(), 1292 instance, 1293 testMethod, 1294 cause, 1295 System.currentTimeMillis(), 1296 System.currentTimeMillis(), 1297 m_testContext)); 1298 } 1299 } 1300 1301 /** 1302 * Invokes a method that has a specified threadPoolSize. 1303 */ invokePooledTestMethods(ITestNGMethod testMethod, XmlSuite suite, Map<String, String> parameters, ConfigurationGroupMethods groupMethods, ITestContext testContext)1304 private List<ITestResult> invokePooledTestMethods(ITestNGMethod testMethod, 1305 XmlSuite suite, 1306 Map<String, String> parameters, 1307 ConfigurationGroupMethods groupMethods, 1308 ITestContext testContext) 1309 { 1310 // 1311 // Create the workers 1312 // 1313 List<IWorker<ITestNGMethod>> workers = Lists.newArrayList(); 1314 1315 // Create one worker per invocationCount 1316 for (int i = 0; i < testMethod.getInvocationCount(); i++) { 1317 // we use clones for reporting purposes 1318 ITestNGMethod clonedMethod= testMethod.clone(); 1319 clonedMethod.setInvocationCount(1); 1320 clonedMethod.setThreadPoolSize(1); 1321 1322 MethodInstance mi = new MethodInstance(clonedMethod); 1323 workers.add(new SingleTestMethodWorker(this, 1324 mi, 1325 suite, 1326 parameters, 1327 testContext, 1328 m_classListeners)); 1329 } 1330 1331 return runWorkers(testMethod, workers, testMethod.getThreadPoolSize(), groupMethods, suite, 1332 parameters); 1333 } 1334 1335 static class FailureContext { 1336 int count = 0; 1337 List<Object> instances = Lists.newArrayList(); 1338 } 1339 handleInvocationResults(ITestNGMethod testMethod, List<ITestResult> result, ExpectedExceptionsHolder expectedExceptionsHolder, FailureContext failure)1340 void handleInvocationResults(ITestNGMethod testMethod, 1341 List<ITestResult> result, 1342 ExpectedExceptionsHolder expectedExceptionsHolder, 1343 FailureContext failure) 1344 { 1345 // 1346 // Go through all the results and create a TestResult for each of them 1347 // 1348 List<ITestResult> resultsToRetry = Lists.newArrayList(); 1349 1350 for (ITestResult testResult : result) { 1351 Throwable ite= testResult.getThrowable(); 1352 int status= testResult.getStatus(); 1353 1354 boolean handled = false; 1355 1356 // Exception thrown? 1357 if (ite != null) { 1358 1359 // Invocation caused an exception, see if the method was annotated with @ExpectedException 1360 if (expectedExceptionsHolder != null) { 1361 if (expectedExceptionsHolder.isExpectedException(ite)) { 1362 testResult.setStatus(ITestResult.SUCCESS); 1363 status = ITestResult.SUCCESS; 1364 } else { 1365 if (isSkipExceptionAndSkip(ite)){ 1366 status = ITestResult.SKIP; 1367 } else { 1368 testResult.setThrowable(expectedExceptionsHolder.wrongException(ite)); 1369 status = ITestResult.FAILURE; 1370 } 1371 } 1372 } else { 1373 handleException(ite, testMethod, testResult, failure.count++); 1374 handled = true; 1375 status = testResult.getStatus(); 1376 } 1377 } 1378 1379 // No exception thrown, make sure we weren't expecting one 1380 else if(status != ITestResult.SKIP && expectedExceptionsHolder != null) { 1381 TestException exception = expectedExceptionsHolder.noException(testMethod); 1382 if (exception != null) { 1383 testResult.setThrowable(exception); 1384 status= ITestResult.FAILURE; 1385 } 1386 } 1387 1388 IRetryAnalyzer retryAnalyzer = testMethod.getRetryAnalyzer(); 1389 boolean willRetry = retryAnalyzer != null && status == ITestResult.FAILURE && failure.instances != null && retryAnalyzer.retry(testResult); 1390 1391 if (willRetry) { 1392 resultsToRetry.add(testResult); 1393 failure.count++; 1394 failure.instances.add(testResult.getInstance()); 1395 testResult.setStatus(ITestResult.SKIP); 1396 } else { 1397 testResult.setStatus(status); 1398 if (status == ITestResult.FAILURE && !handled) { 1399 handleException(ite, testMethod, testResult, failure.count++); 1400 } 1401 } 1402 collectResults(testMethod, Collections.singleton(testResult)); 1403 } // for results 1404 1405 removeResultsToRetryFromResult(resultsToRetry, result, failure); 1406 } 1407 isSkipExceptionAndSkip(Throwable ite)1408 private boolean isSkipExceptionAndSkip(Throwable ite) { 1409 return SkipException.class.isAssignableFrom(ite.getClass()) && ((SkipException) ite).isSkip(); 1410 } 1411 removeResultsToRetryFromResult(List<ITestResult> resultsToRetry, List<ITestResult> result, FailureContext failure)1412 private void removeResultsToRetryFromResult(List<ITestResult> resultsToRetry, 1413 List<ITestResult> result, FailureContext failure) { 1414 if (resultsToRetry != null) { 1415 for (ITestResult res : resultsToRetry) { 1416 result.remove(res); 1417 failure.count--; 1418 } 1419 } 1420 } 1421 1422 /** 1423 * To reduce thread contention and also to correctly handle thread-confinement 1424 * this method invokes the @BeforeGroups and @AfterGroups corresponding to the current @Test method. 1425 */ runWorkers(ITestNGMethod testMethod, List<IWorker<ITestNGMethod>> workers, int threadPoolSize, ConfigurationGroupMethods groupMethods, XmlSuite suite, Map<String, String> parameters)1426 private List<ITestResult> runWorkers(ITestNGMethod testMethod, 1427 List<IWorker<ITestNGMethod>> workers, 1428 int threadPoolSize, 1429 ConfigurationGroupMethods groupMethods, 1430 XmlSuite suite, 1431 Map<String, String> parameters) 1432 { 1433 // Invoke @BeforeGroups on the original method (reduce thread contention, 1434 // and also solve thread confinement) 1435 ITestClass testClass= testMethod.getTestClass(); 1436 Object[] instances = testClass.getInstances(true); 1437 for(Object instance: instances) { 1438 invokeBeforeGroupsConfigurations(testClass, testMethod, groupMethods, suite, parameters, instance); 1439 } 1440 1441 1442 long maxTimeOut= -1; // 10 seconds 1443 1444 for(IWorker<ITestNGMethod> tmw : workers) { 1445 long mt= tmw.getTimeOut(); 1446 if(mt > maxTimeOut) { 1447 maxTimeOut= mt; 1448 } 1449 } 1450 1451 ThreadUtil.execute(workers, threadPoolSize, maxTimeOut, true); 1452 1453 // 1454 // Collect all the TestResults 1455 // 1456 List<ITestResult> result = Lists.newArrayList(); 1457 for (IWorker<ITestNGMethod> tmw : workers) { 1458 if (tmw instanceof TestMethodWorker) { 1459 result.addAll(((TestMethodWorker)tmw).getTestResults()); 1460 } 1461 } 1462 1463 for(Object instance: instances) { 1464 invokeAfterGroupsConfigurations(testClass, testMethod, groupMethods, suite, parameters, instance); 1465 } 1466 1467 return result; 1468 } 1469 1470 /** 1471 * Checks to see of the test method has certain dependencies that prevents 1472 * TestNG from executing it 1473 * @param testMethod test method being checked for 1474 * @return error message or null if dependencies have been run successfully 1475 */ checkDependencies(ITestNGMethod testMethod, ITestNGMethod[] allTestMethods)1476 private String checkDependencies(ITestNGMethod testMethod, 1477 ITestNGMethod[] allTestMethods) 1478 { 1479 // If this method is marked alwaysRun, no need to check for its dependencies 1480 if (testMethod.isAlwaysRun()) { 1481 return null; 1482 } 1483 1484 // Any missing group? 1485 if (testMethod.getMissingGroup() != null 1486 && !testMethod.ignoreMissingDependencies()) { 1487 return "Method " + testMethod + " depends on nonexistent group \"" + testMethod.getMissingGroup() + "\""; 1488 } 1489 1490 // If this method depends on groups, collect all the methods that 1491 // belong to these groups and make sure they have been run successfully 1492 final String[] groups = testMethod.getGroupsDependedUpon(); 1493 if (null != groups && groups.length > 0) { 1494 // Get all the methods that belong to the group depended upon 1495 for (String element : groups) { 1496 ITestNGMethod[] methods = 1497 MethodGroupsHelper.findMethodsThatBelongToGroup(testMethod, 1498 m_testContext.getAllTestMethods(), 1499 element); 1500 if (methods.length == 0 && !testMethod.ignoreMissingDependencies()) { 1501 // Group is missing 1502 return "Method " + testMethod + " depends on nonexistent group \"" + element + "\""; 1503 } 1504 if (!haveBeenRunSuccessfully(testMethod, methods)) { 1505 return "Method " + testMethod + 1506 " depends on not successfully finished methods in group \"" + element + "\""; 1507 } 1508 } 1509 } // depends on groups 1510 1511 // If this method depends on other methods, make sure all these other 1512 // methods have been run successfully 1513 if (dependsOnMethods(testMethod)) { 1514 ITestNGMethod[] methods = 1515 MethodHelper.findDependedUponMethods(testMethod, allTestMethods); 1516 1517 if (!haveBeenRunSuccessfully(testMethod, methods)) { 1518 return "Method " + testMethod + " depends on not successfully finished methods"; 1519 } 1520 } 1521 1522 return null; 1523 } 1524 1525 /** 1526 * @return the test results that apply to one of the instances of the testMethod. 1527 */ keepSameInstances(ITestNGMethod method, Set<ITestResult> results)1528 private Set<ITestResult> keepSameInstances(ITestNGMethod method, Set<ITestResult> results) { 1529 Set<ITestResult> result = Sets.newHashSet(); 1530 for (ITestResult r : results) { 1531 final Object o = method.getInstance(); 1532 // Keep this instance if 1) It's on a different class or 2) It's on the same class 1533 // and on the same instance 1534 Object instance = r.getInstance() != null 1535 ? r.getInstance() : r.getMethod().getInstance(); 1536 if (r.getTestClass() != method.getTestClass() || instance == o) result.add(r); 1537 } 1538 return result; 1539 } 1540 1541 /** 1542 * @return true if all the methods have been run successfully 1543 */ haveBeenRunSuccessfully(ITestNGMethod testMethod, ITestNGMethod[] methods)1544 private boolean haveBeenRunSuccessfully(ITestNGMethod testMethod, ITestNGMethod[] methods) { 1545 // Make sure the method has been run successfully 1546 for (ITestNGMethod method : methods) { 1547 Set<ITestResult> results = keepSameInstances(testMethod, m_notifier.getPassedTests(method)); 1548 Set<ITestResult> failedAndSkippedMethods = Sets.newHashSet(); 1549 failedAndSkippedMethods.addAll(m_notifier.getFailedTests(method)); 1550 failedAndSkippedMethods.addAll(m_notifier.getSkippedTests(method)); 1551 Set<ITestResult> failedresults = keepSameInstances(testMethod, failedAndSkippedMethods); 1552 1553 // If failed results were returned on the same instance, then these tests didn't pass 1554 if (failedresults != null && failedresults.size() > 0) { 1555 return false; 1556 } 1557 1558 for (ITestResult result : results) { 1559 if(!result.isSuccess()) { 1560 return false; 1561 } 1562 } 1563 } 1564 1565 return true; 1566 } 1567 1568 // private boolean containsInstance(Set<ITestResult> failedresults, Object[] instances) { 1569 // for (ITestResult tr : failedresults) { 1570 // for (Object o : instances) { 1571 // if (o == tr.getInstance()) { 1572 // return true; 1573 // } 1574 // } 1575 // } 1576 // return false; 1577 // } 1578 1579 /** 1580 * An exception was thrown by the test, determine if this method 1581 * should be marked as a failure or as failure_but_within_successPercentage 1582 */ handleException(Throwable throwable, ITestNGMethod testMethod, ITestResult testResult, int failureCount)1583 private void handleException(Throwable throwable, 1584 ITestNGMethod testMethod, 1585 ITestResult testResult, 1586 int failureCount) { 1587 if (throwable != null) { 1588 testResult.setThrowable(throwable); 1589 } 1590 int successPercentage= testMethod.getSuccessPercentage(); 1591 int invocationCount= testMethod.getInvocationCount(); 1592 float numberOfTestsThatCanFail= ((100 - successPercentage) * invocationCount) / 100f; 1593 1594 if(failureCount < numberOfTestsThatCanFail) { 1595 testResult.setStatus(ITestResult.SUCCESS_PERCENTAGE_FAILURE); 1596 } 1597 else { 1598 testResult.setStatus(ITestResult.FAILURE); 1599 } 1600 1601 } 1602 1603 static interface Predicate<K, T> { isTrue(K k, T v)1604 boolean isTrue(K k, T v); 1605 } 1606 1607 static class CanRunFromClassPredicate implements Predicate <ITestNGMethod, IClass> { 1608 @Override isTrue(ITestNGMethod m, IClass v)1609 public boolean isTrue(ITestNGMethod m, IClass v) { 1610 return m.canRunFromClass(v); 1611 } 1612 } 1613 1614 static class SameClassNamePredicate implements Predicate<ITestNGMethod, IClass> { 1615 @Override isTrue(ITestNGMethod m, IClass c)1616 public boolean isTrue(ITestNGMethod m, IClass c) { 1617 return c == null || m.getTestClass().getName().equals(c.getName()); 1618 } 1619 } 1620 1621 /** 1622 * @return Only the ITestNGMethods applicable for this testClass 1623 */ filterMethods(IClass testClass, ITestNGMethod[] methods, Predicate<ITestNGMethod, IClass> predicate)1624 private ITestNGMethod[] filterMethods(IClass testClass, ITestNGMethod[] methods, 1625 Predicate<ITestNGMethod, IClass> predicate) { 1626 List<ITestNGMethod> vResult= Lists.newArrayList(); 1627 1628 for(ITestNGMethod tm : methods) { 1629 if (predicate.isTrue(tm, testClass)) { 1630 log(10, "Keeping method " + tm + " for class " + testClass); 1631 vResult.add(tm); 1632 } else { 1633 log(10, "Filtering out method " + tm + " for class " + testClass); 1634 } 1635 } 1636 1637 ITestNGMethod[] result= vResult.toArray(new ITestNGMethod[vResult.size()]); 1638 1639 return result; 1640 } 1641 1642 /** 1643 * @return true if this method depends on certain methods. 1644 */ dependsOnMethods(ITestNGMethod tm)1645 private boolean dependsOnMethods(ITestNGMethod tm) { 1646 String[] methods = tm.getMethodsDependedUpon(); 1647 return null != methods && methods.length > 0; 1648 } 1649 runConfigurationListeners(ITestResult tr, boolean before)1650 private void runConfigurationListeners(ITestResult tr, boolean before) { 1651 if (before) { 1652 for(IConfigurationListener icl: m_notifier.getConfigurationListeners()) { 1653 if (icl instanceof IConfigurationListener2) { 1654 ((IConfigurationListener2) icl).beforeConfiguration(tr); 1655 } 1656 } 1657 } else { 1658 for(IConfigurationListener icl: m_notifier.getConfigurationListeners()) { 1659 switch(tr.getStatus()) { 1660 case ITestResult.SKIP: 1661 icl.onConfigurationSkip(tr); 1662 break; 1663 case ITestResult.FAILURE: 1664 icl.onConfigurationFailure(tr); 1665 break; 1666 case ITestResult.SUCCESS: 1667 icl.onConfigurationSuccess(tr); 1668 break; 1669 } 1670 } 1671 } 1672 } 1673 runTestListeners(ITestResult tr)1674 void runTestListeners(ITestResult tr) { 1675 runTestListeners(tr, m_notifier.getTestListeners()); 1676 } 1677 1678 // TODO: move this from here as it is directly called from TestNG runTestListeners(ITestResult tr, List<ITestListener> listeners)1679 public static void runTestListeners(ITestResult tr, List<ITestListener> listeners) { 1680 for (ITestListener itl : listeners) { 1681 switch(tr.getStatus()) { 1682 case ITestResult.SKIP: { 1683 itl.onTestSkipped(tr); 1684 break; 1685 } 1686 case ITestResult.SUCCESS_PERCENTAGE_FAILURE: { 1687 itl.onTestFailedButWithinSuccessPercentage(tr); 1688 break; 1689 } 1690 case ITestResult.FAILURE: { 1691 itl.onTestFailure(tr); 1692 break; 1693 } 1694 case ITestResult.SUCCESS: { 1695 itl.onTestSuccess(tr); 1696 break; 1697 } 1698 1699 case ITestResult.STARTED: { 1700 itl.onTestStart(tr); 1701 break; 1702 } 1703 1704 default: { 1705 assert false : "UNKNOWN STATUS:" + tr; 1706 } 1707 } 1708 } 1709 } 1710 log(int level, String s)1711 private void log(int level, String s) { 1712 Utils.log("Invoker " + Thread.currentThread().hashCode(), level, s); 1713 } 1714 1715 /** 1716 * This class holds a {@code ParameterHolder} or in case of an error, a non-null 1717 * {@code TestResult} containing the cause 1718 */ 1719 private static class ParameterBag { 1720 final ParameterHolder parameterHolder; 1721 final ITestResult errorResult; 1722 ParameterBag(ParameterHolder parameterHolder)1723 public ParameterBag(ParameterHolder parameterHolder) { 1724 this.parameterHolder = parameterHolder; 1725 this.errorResult = null; 1726 } 1727 ParameterBag(ITestResult errorResult)1728 public ParameterBag(ITestResult errorResult) { 1729 this.parameterHolder = null; 1730 this.errorResult = errorResult; 1731 } 1732 hasErrors()1733 public boolean hasErrors() { 1734 return errorResult != null; 1735 } 1736 } 1737 1738 } 1739