import logging, os, re from autotest_lib.client.common_lib import error from autotest_lib.client.virt import virt_utils, rss_client, aexpect def run_whql_submission(test, params, env): """ WHQL submission test: 1) Log into the client machines and into a DTM server machine 2) Copy the automation program binary (dsso_test_binary) to the server machine 3) Run the automation program 4) Pass the program all relevant parameters (e.g. device_data) 5) Wait for the program to terminate 6) Parse and report job results (logs and HTML reports are placed in test.debugdir) @param test: kvm test object @param params: Dictionary with the test parameters @param env: Dictionary with test environment. """ # Log into all client VMs login_timeout = int(params.get("login_timeout", 360)) vms = [] sessions = [] for vm_name in params.objects("vms"): vms.append(env.get_vm(vm_name)) vms[-1].verify_alive() sessions.append(vms[-1].wait_for_login(timeout=login_timeout)) # Make sure all NICs of all client VMs are up for vm in vms: nics = vm.params.objects("nics") for nic_index in range(len(nics)): s = vm.wait_for_login(nic_index, 600) s.close() # Collect parameters server_address = params.get("server_address") server_shell_port = int(params.get("server_shell_port")) server_file_transfer_port = int(params.get("server_file_transfer_port")) server_studio_path = params.get("server_studio_path", "%programfiles%\\ " "Microsoft Driver Test Manager\\Studio") dsso_test_binary = params.get("dsso_test_binary", "deps/whql_submission_15.exe") dsso_test_binary = virt_utils.get_path(test.bindir, dsso_test_binary) dsso_delete_machine_binary = params.get("dsso_delete_machine_binary", "deps/whql_delete_machine_15.exe") dsso_delete_machine_binary = virt_utils.get_path(test.bindir, dsso_delete_machine_binary) test_timeout = float(params.get("test_timeout", 600)) # Copy dsso binaries to the server for filename in dsso_test_binary, dsso_delete_machine_binary: rss_client.upload(server_address, server_file_transfer_port, filename, server_studio_path, timeout=60) # Open a shell session with the server server_session = virt_utils.remote_login("nc", server_address, server_shell_port, "", "", sessions[0].prompt, sessions[0].linesep) server_session.set_status_test_command(sessions[0].status_test_command) # Get the computer names of the server and clients cmd = "echo %computername%" server_name = server_session.cmd_output(cmd).strip() client_names = [session.cmd_output(cmd).strip() for session in sessions] # Delete all client machines from the server's data store server_session.cmd("cd %s" % server_studio_path) for client_name in client_names: cmd = "%s %s %s" % (os.path.basename(dsso_delete_machine_binary), server_name, client_name) server_session.cmd(cmd, print_func=logging.debug) # Reboot the client machines sessions = virt_utils.parallel((vm.reboot, (session,)) for vm, session in zip(vms, sessions)) # Check the NICs again for vm in vms: nics = vm.params.objects("nics") for nic_index in range(len(nics)): s = vm.wait_for_login(nic_index, 600) s.close() # Run whql_pre_command and close the sessions if params.get("whql_pre_command"): for session in sessions: session.cmd(params.get("whql_pre_command"), int(params.get("whql_pre_command_timeout", 600))) session.close() # Run the automation program on the server pool_name = "%s_pool" % client_names[0] submission_name = "%s_%s" % (client_names[0], params.get("submission_name")) cmd = "%s %s %s %s %s %s" % (os.path.basename(dsso_test_binary), server_name, pool_name, submission_name, test_timeout, " ".join(client_names)) server_session.sendline(cmd) # Helper function: wait for a given prompt and raise an exception if an # error occurs def find_prompt(prompt): m, o = server_session.read_until_last_line_matches( [prompt, server_session.prompt], print_func=logging.info, timeout=600) if m != 0: errors = re.findall("^Error:.*$", o, re.I | re.M) if errors: raise error.TestError(errors[0]) else: raise error.TestError("Error running automation program: " "could not find '%s' prompt" % prompt) # Tell the automation program which device to test find_prompt("Device to test:") server_session.sendline(params.get("test_device")) # Tell the automation program which jobs to run find_prompt("Jobs to run:") server_session.sendline(params.get("job_filter", ".*")) # Set submission DeviceData find_prompt("DeviceData name:") for dd in params.objects("device_data"): dd_params = params.object_params(dd) if dd_params.get("dd_name") and dd_params.get("dd_data"): server_session.sendline(dd_params.get("dd_name")) server_session.sendline(dd_params.get("dd_data")) server_session.sendline() # Set submission descriptors find_prompt("Descriptor path:") for desc in params.objects("descriptors"): desc_params = params.object_params(desc) if desc_params.get("desc_path"): server_session.sendline(desc_params.get("desc_path")) server_session.sendline() # Set machine dimensions for each client machine for vm_name in params.objects("vms"): vm_params = params.object_params(vm_name) find_prompt(r"Dimension name\b.*:") for dp in vm_params.objects("dimensions"): dp_params = vm_params.object_params(dp) if dp_params.get("dim_name") and dp_params.get("dim_value"): server_session.sendline(dp_params.get("dim_name")) server_session.sendline(dp_params.get("dim_value")) server_session.sendline() # Set extra parameters for tests that require them (e.g. NDISTest) for vm_name in params.objects("vms"): vm_params = params.object_params(vm_name) find_prompt(r"Parameter name\b.*:") for dp in vm_params.objects("device_params"): dp_params = vm_params.object_params(dp) if dp_params.get("dp_name") and dp_params.get("dp_regex"): server_session.sendline(dp_params.get("dp_name")) server_session.sendline(dp_params.get("dp_regex")) # Make sure the prompt appears again (if the device isn't found # the automation program will terminate) find_prompt(r"Parameter name\b.*:") server_session.sendline() # Wait for the automation program to terminate try: o = server_session.read_up_to_prompt(print_func=logging.info, timeout=test_timeout + 300) # (test_timeout + 300 is used here because the automation program is # supposed to terminate cleanly on its own when test_timeout expires) done = True except aexpect.ExpectError, e: o = e.output done = False server_session.close() # Look for test results in the automation program's output result_summaries = re.findall(r"---- \[.*?\] ----", o, re.DOTALL) if not result_summaries: raise error.TestError("The automation program did not return any " "results") results = result_summaries[-1].strip("-") results = eval("".join(results.splitlines())) # Download logs and HTML reports from the server for i, r in enumerate(results): if "report" in r: try: rss_client.download(server_address, server_file_transfer_port, r["report"], test.debugdir) except rss_client.FileTransferNotFoundError: pass if "logs" in r: try: rss_client.download(server_address, server_file_transfer_port, r["logs"], test.debugdir) except rss_client.FileTransferNotFoundError: pass else: try: # Create symlinks to test log dirs to make it easier # to access them (their original names are not human # readable) link_name = "logs_%s" % r["report"].split("\\")[-1] link_name = link_name.replace(" ", "_") link_name = link_name.replace("/", "_") os.symlink(r["logs"].split("\\")[-1], os.path.join(test.debugdir, link_name)) except (KeyError, OSError): pass # Print result summary (both to the regular logs and to a file named # 'summary' in test.debugdir) def print_summary_line(f, line): logging.info(line) f.write(line + "\n") if results: # Make sure all results have the required keys for r in results: r["id"] = str(r.get("id")) r["job"] = str(r.get("job")) r["status"] = str(r.get("status")) r["pass"] = int(r.get("pass", 0)) r["fail"] = int(r.get("fail", 0)) r["notrun"] = int(r.get("notrun", 0)) r["notapplicable"] = int(r.get("notapplicable", 0)) # Sort the results by failures and total test count in descending order results = [(r["fail"], r["pass"] + r["fail"] + r["notrun"] + r["notapplicable"], r) for r in results] results.sort(reverse=True) results = [r[-1] for r in results] # Print results logging.info("") logging.info("Result summary:") name_length = max(len(r["job"]) for r in results) fmt = "%%-6s %%-%ds %%-15s %%-8s %%-8s %%-8s %%-15s" % name_length f = open(os.path.join(test.debugdir, "summary"), "w") print_summary_line(f, fmt % ("ID", "Job", "Status", "Pass", "Fail", "NotRun", "NotApplicable")) print_summary_line(f, fmt % ("--", "---", "------", "----", "----", "------", "-------------")) for r in results: print_summary_line(f, fmt % (r["id"], r["job"], r["status"], r["pass"], r["fail"], r["notrun"], r["notapplicable"])) f.close() logging.info("(see logs and HTML reports in %s)", test.debugdir) # Kill the client VMs and fail if the automation program did not terminate # on time if not done: virt_utils.parallel(vm.destroy for vm in vms) raise error.TestFail("The automation program did not terminate " "on time") # Fail if there are failed or incomplete jobs (kill the client VMs if there # are incomplete jobs) failed_jobs = [r["job"] for r in results if r["status"].lower() == "investigate"] running_jobs = [r["job"] for r in results if r["status"].lower() == "inprogress"] errors = [] if failed_jobs: errors += ["Jobs failed: %s." % failed_jobs] if running_jobs: for vm in vms: vm.destroy() errors += ["Jobs did not complete on time: %s." % running_jobs] if errors: raise error.TestFail(" ".join(errors))