275 lines
12 KiB
Python
275 lines
12 KiB
Python
import logging, os, re
|
|
from autotest_lib.client.common_lib import error
|
|
from autotest_lib.client.virt import virt_utils, rss_client, aexpect
|
|
|
|
|
|
def run_whql_submission(test, params, env):
|
|
"""
|
|
WHQL submission test:
|
|
1) Log into the client machines and into a DTM server machine
|
|
2) Copy the automation program binary (dsso_test_binary) to the server machine
|
|
3) Run the automation program
|
|
4) Pass the program all relevant parameters (e.g. device_data)
|
|
5) Wait for the program to terminate
|
|
6) Parse and report job results
|
|
(logs and HTML reports are placed in test.debugdir)
|
|
|
|
@param test: kvm test object
|
|
@param params: Dictionary with the test parameters
|
|
@param env: Dictionary with test environment.
|
|
"""
|
|
# Log into all client VMs
|
|
login_timeout = int(params.get("login_timeout", 360))
|
|
vms = []
|
|
sessions = []
|
|
for vm_name in params.objects("vms"):
|
|
vms.append(env.get_vm(vm_name))
|
|
vms[-1].verify_alive()
|
|
sessions.append(vms[-1].wait_for_login(timeout=login_timeout))
|
|
|
|
# Make sure all NICs of all client VMs are up
|
|
for vm in vms:
|
|
nics = vm.params.objects("nics")
|
|
for nic_index in range(len(nics)):
|
|
s = vm.wait_for_login(nic_index, 600)
|
|
s.close()
|
|
|
|
# Collect parameters
|
|
server_address = params.get("server_address")
|
|
server_shell_port = int(params.get("server_shell_port"))
|
|
server_file_transfer_port = int(params.get("server_file_transfer_port"))
|
|
server_studio_path = params.get("server_studio_path", "%programfiles%\\ "
|
|
"Microsoft Driver Test Manager\\Studio")
|
|
dsso_test_binary = params.get("dsso_test_binary",
|
|
"deps/whql_submission_15.exe")
|
|
dsso_test_binary = virt_utils.get_path(test.bindir, dsso_test_binary)
|
|
dsso_delete_machine_binary = params.get("dsso_delete_machine_binary",
|
|
"deps/whql_delete_machine_15.exe")
|
|
dsso_delete_machine_binary = virt_utils.get_path(test.bindir,
|
|
dsso_delete_machine_binary)
|
|
test_timeout = float(params.get("test_timeout", 600))
|
|
|
|
# Copy dsso binaries to the server
|
|
for filename in dsso_test_binary, dsso_delete_machine_binary:
|
|
rss_client.upload(server_address, server_file_transfer_port,
|
|
filename, server_studio_path, timeout=60)
|
|
|
|
# Open a shell session with the server
|
|
server_session = virt_utils.remote_login("nc", server_address,
|
|
server_shell_port, "", "",
|
|
sessions[0].prompt,
|
|
sessions[0].linesep)
|
|
server_session.set_status_test_command(sessions[0].status_test_command)
|
|
|
|
# Get the computer names of the server and clients
|
|
cmd = "echo %computername%"
|
|
server_name = server_session.cmd_output(cmd).strip()
|
|
client_names = [session.cmd_output(cmd).strip() for session in sessions]
|
|
|
|
# Delete all client machines from the server's data store
|
|
server_session.cmd("cd %s" % server_studio_path)
|
|
for client_name in client_names:
|
|
cmd = "%s %s %s" % (os.path.basename(dsso_delete_machine_binary),
|
|
server_name, client_name)
|
|
server_session.cmd(cmd, print_func=logging.debug)
|
|
|
|
# Reboot the client machines
|
|
sessions = virt_utils.parallel((vm.reboot, (session,))
|
|
for vm, session in zip(vms, sessions))
|
|
|
|
# Check the NICs again
|
|
for vm in vms:
|
|
nics = vm.params.objects("nics")
|
|
for nic_index in range(len(nics)):
|
|
s = vm.wait_for_login(nic_index, 600)
|
|
s.close()
|
|
|
|
# Run whql_pre_command and close the sessions
|
|
if params.get("whql_pre_command"):
|
|
for session in sessions:
|
|
session.cmd(params.get("whql_pre_command"),
|
|
int(params.get("whql_pre_command_timeout", 600)))
|
|
session.close()
|
|
|
|
# Run the automation program on the server
|
|
pool_name = "%s_pool" % client_names[0]
|
|
submission_name = "%s_%s" % (client_names[0],
|
|
params.get("submission_name"))
|
|
cmd = "%s %s %s %s %s %s" % (os.path.basename(dsso_test_binary),
|
|
server_name, pool_name, submission_name,
|
|
test_timeout, " ".join(client_names))
|
|
server_session.sendline(cmd)
|
|
|
|
# Helper function: wait for a given prompt and raise an exception if an
|
|
# error occurs
|
|
def find_prompt(prompt):
|
|
m, o = server_session.read_until_last_line_matches(
|
|
[prompt, server_session.prompt], print_func=logging.info,
|
|
timeout=600)
|
|
if m != 0:
|
|
errors = re.findall("^Error:.*$", o, re.I | re.M)
|
|
if errors:
|
|
raise error.TestError(errors[0])
|
|
else:
|
|
raise error.TestError("Error running automation program: "
|
|
"could not find '%s' prompt" % prompt)
|
|
|
|
# Tell the automation program which device to test
|
|
find_prompt("Device to test:")
|
|
server_session.sendline(params.get("test_device"))
|
|
|
|
# Tell the automation program which jobs to run
|
|
find_prompt("Jobs to run:")
|
|
server_session.sendline(params.get("job_filter", ".*"))
|
|
|
|
# Set submission DeviceData
|
|
find_prompt("DeviceData name:")
|
|
for dd in params.objects("device_data"):
|
|
dd_params = params.object_params(dd)
|
|
if dd_params.get("dd_name") and dd_params.get("dd_data"):
|
|
server_session.sendline(dd_params.get("dd_name"))
|
|
server_session.sendline(dd_params.get("dd_data"))
|
|
server_session.sendline()
|
|
|
|
# Set submission descriptors
|
|
find_prompt("Descriptor path:")
|
|
for desc in params.objects("descriptors"):
|
|
desc_params = params.object_params(desc)
|
|
if desc_params.get("desc_path"):
|
|
server_session.sendline(desc_params.get("desc_path"))
|
|
server_session.sendline()
|
|
|
|
# Set machine dimensions for each client machine
|
|
for vm_name in params.objects("vms"):
|
|
vm_params = params.object_params(vm_name)
|
|
find_prompt(r"Dimension name\b.*:")
|
|
for dp in vm_params.objects("dimensions"):
|
|
dp_params = vm_params.object_params(dp)
|
|
if dp_params.get("dim_name") and dp_params.get("dim_value"):
|
|
server_session.sendline(dp_params.get("dim_name"))
|
|
server_session.sendline(dp_params.get("dim_value"))
|
|
server_session.sendline()
|
|
|
|
# Set extra parameters for tests that require them (e.g. NDISTest)
|
|
for vm_name in params.objects("vms"):
|
|
vm_params = params.object_params(vm_name)
|
|
find_prompt(r"Parameter name\b.*:")
|
|
for dp in vm_params.objects("device_params"):
|
|
dp_params = vm_params.object_params(dp)
|
|
if dp_params.get("dp_name") and dp_params.get("dp_regex"):
|
|
server_session.sendline(dp_params.get("dp_name"))
|
|
server_session.sendline(dp_params.get("dp_regex"))
|
|
# Make sure the prompt appears again (if the device isn't found
|
|
# the automation program will terminate)
|
|
find_prompt(r"Parameter name\b.*:")
|
|
server_session.sendline()
|
|
|
|
# Wait for the automation program to terminate
|
|
try:
|
|
o = server_session.read_up_to_prompt(print_func=logging.info,
|
|
timeout=test_timeout + 300)
|
|
# (test_timeout + 300 is used here because the automation program is
|
|
# supposed to terminate cleanly on its own when test_timeout expires)
|
|
done = True
|
|
except aexpect.ExpectError, e:
|
|
o = e.output
|
|
done = False
|
|
server_session.close()
|
|
|
|
# Look for test results in the automation program's output
|
|
result_summaries = re.findall(r"---- \[.*?\] ----", o, re.DOTALL)
|
|
if not result_summaries:
|
|
raise error.TestError("The automation program did not return any "
|
|
"results")
|
|
results = result_summaries[-1].strip("-")
|
|
results = eval("".join(results.splitlines()))
|
|
|
|
# Download logs and HTML reports from the server
|
|
for i, r in enumerate(results):
|
|
if "report" in r:
|
|
try:
|
|
rss_client.download(server_address,
|
|
server_file_transfer_port,
|
|
r["report"], test.debugdir)
|
|
except rss_client.FileTransferNotFoundError:
|
|
pass
|
|
if "logs" in r:
|
|
try:
|
|
rss_client.download(server_address,
|
|
server_file_transfer_port,
|
|
r["logs"], test.debugdir)
|
|
except rss_client.FileTransferNotFoundError:
|
|
pass
|
|
else:
|
|
try:
|
|
# Create symlinks to test log dirs to make it easier
|
|
# to access them (their original names are not human
|
|
# readable)
|
|
link_name = "logs_%s" % r["report"].split("\\")[-1]
|
|
link_name = link_name.replace(" ", "_")
|
|
link_name = link_name.replace("/", "_")
|
|
os.symlink(r["logs"].split("\\")[-1],
|
|
os.path.join(test.debugdir, link_name))
|
|
except (KeyError, OSError):
|
|
pass
|
|
|
|
# Print result summary (both to the regular logs and to a file named
|
|
# 'summary' in test.debugdir)
|
|
def print_summary_line(f, line):
|
|
logging.info(line)
|
|
f.write(line + "\n")
|
|
if results:
|
|
# Make sure all results have the required keys
|
|
for r in results:
|
|
r["id"] = str(r.get("id"))
|
|
r["job"] = str(r.get("job"))
|
|
r["status"] = str(r.get("status"))
|
|
r["pass"] = int(r.get("pass", 0))
|
|
r["fail"] = int(r.get("fail", 0))
|
|
r["notrun"] = int(r.get("notrun", 0))
|
|
r["notapplicable"] = int(r.get("notapplicable", 0))
|
|
# Sort the results by failures and total test count in descending order
|
|
results = [(r["fail"],
|
|
r["pass"] + r["fail"] + r["notrun"] + r["notapplicable"],
|
|
r) for r in results]
|
|
results.sort(reverse=True)
|
|
results = [r[-1] for r in results]
|
|
# Print results
|
|
logging.info("")
|
|
logging.info("Result summary:")
|
|
name_length = max(len(r["job"]) for r in results)
|
|
fmt = "%%-6s %%-%ds %%-15s %%-8s %%-8s %%-8s %%-15s" % name_length
|
|
f = open(os.path.join(test.debugdir, "summary"), "w")
|
|
print_summary_line(f, fmt % ("ID", "Job", "Status", "Pass", "Fail",
|
|
"NotRun", "NotApplicable"))
|
|
print_summary_line(f, fmt % ("--", "---", "------", "----", "----",
|
|
"------", "-------------"))
|
|
for r in results:
|
|
print_summary_line(f, fmt % (r["id"], r["job"], r["status"],
|
|
r["pass"], r["fail"], r["notrun"],
|
|
r["notapplicable"]))
|
|
f.close()
|
|
logging.info("(see logs and HTML reports in %s)", test.debugdir)
|
|
|
|
# Kill the client VMs and fail if the automation program did not terminate
|
|
# on time
|
|
if not done:
|
|
virt_utils.parallel(vm.destroy for vm in vms)
|
|
raise error.TestFail("The automation program did not terminate "
|
|
"on time")
|
|
|
|
# Fail if there are failed or incomplete jobs (kill the client VMs if there
|
|
# are incomplete jobs)
|
|
failed_jobs = [r["job"] for r in results
|
|
if r["status"].lower() == "investigate"]
|
|
running_jobs = [r["job"] for r in results
|
|
if r["status"].lower() == "inprogress"]
|
|
errors = []
|
|
if failed_jobs:
|
|
errors += ["Jobs failed: %s." % failed_jobs]
|
|
if running_jobs:
|
|
for vm in vms:
|
|
vm.destroy()
|
|
errors += ["Jobs did not complete on time: %s." % running_jobs]
|
|
if errors:
|
|
raise error.TestFail(" ".join(errors))
|