317 lines
11 KiB
Python
317 lines
11 KiB
Python
import datetime
|
|
import getpass
|
|
import glob
|
|
import os
|
|
import pickle
|
|
import re
|
|
import threading
|
|
import time
|
|
import image_chromeos
|
|
import machine_manager_singleton
|
|
import table_formatter
|
|
from cros_utils import command_executer
|
|
from cros_utils import logger
|
|
|
|
SCRATCH_DIR = '/home/%s/cros_scratch' % getpass.getuser()
|
|
PICKLE_FILE = 'pickle.txt'
|
|
VERSION = '1'
|
|
|
|
|
|
def ConvertToFilename(text):
|
|
ret = text
|
|
ret = re.sub('/', '__', ret)
|
|
ret = re.sub(' ', '_', ret)
|
|
ret = re.sub('=', '', ret)
|
|
ret = re.sub("\"", '', ret)
|
|
return ret
|
|
|
|
|
|
class AutotestRun(threading.Thread):
|
|
|
|
def __init__(self,
|
|
autotest,
|
|
chromeos_root='',
|
|
chromeos_image='',
|
|
board='',
|
|
remote='',
|
|
iteration=0,
|
|
image_checksum='',
|
|
exact_remote=False,
|
|
rerun=False,
|
|
rerun_if_failed=False):
|
|
self.autotest = autotest
|
|
self.chromeos_root = chromeos_root
|
|
self.chromeos_image = chromeos_image
|
|
self.board = board
|
|
self.remote = remote
|
|
self.iteration = iteration
|
|
l = logger.GetLogger()
|
|
l.LogFatalIf(not image_checksum, "Checksum shouldn't be None")
|
|
self.image_checksum = image_checksum
|
|
self.results = {}
|
|
threading.Thread.__init__(self)
|
|
self.terminate = False
|
|
self.retval = None
|
|
self.status = 'PENDING'
|
|
self.run_completed = False
|
|
self.exact_remote = exact_remote
|
|
self.rerun = rerun
|
|
self.rerun_if_failed = rerun_if_failed
|
|
self.results_dir = None
|
|
self.full_name = None
|
|
|
|
@staticmethod
|
|
def MeanExcludingSlowest(array):
|
|
mean = sum(array) / len(array)
|
|
array2 = []
|
|
|
|
for v in array:
|
|
if mean != 0 and abs(v - mean) / mean < 0.2:
|
|
array2.append(v)
|
|
|
|
if array2:
|
|
return sum(array2) / len(array2)
|
|
else:
|
|
return mean
|
|
|
|
@staticmethod
|
|
def AddComposite(results_dict):
|
|
composite_keys = []
|
|
composite_dict = {}
|
|
for key in results_dict:
|
|
mo = re.match('(.*){\d+}', key)
|
|
if mo:
|
|
composite_keys.append(mo.group(1))
|
|
for key in results_dict:
|
|
for composite_key in composite_keys:
|
|
if (key.count(composite_key) != 0 and
|
|
table_formatter.IsFloat(results_dict[key])):
|
|
if composite_key not in composite_dict:
|
|
composite_dict[composite_key] = []
|
|
composite_dict[composite_key].append(float(results_dict[key]))
|
|
break
|
|
|
|
for composite_key in composite_dict:
|
|
v = composite_dict[composite_key]
|
|
results_dict['%s[c]' % composite_key] = sum(v) / len(v)
|
|
mean_excluding_slowest = AutotestRun.MeanExcludingSlowest(v)
|
|
results_dict['%s[ce]' % composite_key] = mean_excluding_slowest
|
|
|
|
return results_dict
|
|
|
|
def ParseOutput(self):
|
|
p = re.compile('^-+.*?^-+', re.DOTALL | re.MULTILINE)
|
|
matches = p.findall(self.out)
|
|
for i in range(len(matches)):
|
|
results = matches[i]
|
|
results_dict = {}
|
|
for line in results.splitlines()[1:-1]:
|
|
mo = re.match('(.*\S)\s+\[\s+(PASSED|FAILED)\s+\]', line)
|
|
if mo:
|
|
results_dict[mo.group(1)] = mo.group(2)
|
|
continue
|
|
mo = re.match('(.*\S)\s+(.*)', line)
|
|
if mo:
|
|
results_dict[mo.group(1)] = mo.group(2)
|
|
|
|
# Add a composite keyval for tests like startup.
|
|
results_dict = AutotestRun.AddComposite(results_dict)
|
|
|
|
self.results = results_dict
|
|
|
|
# This causes it to not parse the table again
|
|
# Autotest recently added a secondary table
|
|
# That reports errors and screws up the final pretty output.
|
|
break
|
|
mo = re.search('Results placed in (\S+)', self.out)
|
|
if mo:
|
|
self.results_dir = mo.group(1)
|
|
self.full_name = os.path.basename(self.results_dir)
|
|
|
|
def GetCacheHashBase(self):
|
|
ret = ('%s %s %s' %
|
|
(self.image_checksum, self.autotest.name, self.iteration))
|
|
if self.autotest.args:
|
|
ret += ' %s' % self.autotest.args
|
|
ret += '-%s' % VERSION
|
|
return ret
|
|
|
|
def GetLabel(self):
|
|
ret = '%s %s remote:%s' % (self.chromeos_image, self.autotest.name,
|
|
self.remote)
|
|
return ret
|
|
|
|
def TryToLoadFromCache(self):
|
|
base = self.GetCacheHashBase()
|
|
if self.exact_remote:
|
|
if not self.remote:
|
|
return False
|
|
cache_dir_glob = '%s_%s' % (ConvertToFilename(base), self.remote)
|
|
else:
|
|
cache_dir_glob = '%s*' % ConvertToFilename(base)
|
|
cache_path_glob = os.path.join(SCRATCH_DIR, cache_dir_glob)
|
|
matching_dirs = glob.glob(cache_path_glob)
|
|
if matching_dirs:
|
|
matching_dir = matching_dirs[0]
|
|
cache_file = os.path.join(matching_dir, PICKLE_FILE)
|
|
assert os.path.isfile(cache_file)
|
|
self._logger.LogOutput('Trying to read from cache file: %s' % cache_file)
|
|
return self.ReadFromCache(cache_file)
|
|
self._logger.LogOutput('Cache miss. AM going to run: %s for: %s' %
|
|
(self.autotest.name, self.chromeos_image))
|
|
return False
|
|
|
|
def ReadFromCache(self, cache_file):
|
|
with open(cache_file, 'rb') as f:
|
|
self.retval = pickle.load(f)
|
|
self.out = pickle.load(f)
|
|
self.err = pickle.load(f)
|
|
self._logger.LogOutput(self.out)
|
|
return True
|
|
return False
|
|
|
|
def StoreToCache(self):
|
|
base = self.GetCacheHashBase()
|
|
self.cache_dir = os.path.join(SCRATCH_DIR,
|
|
'%s_%s' % (ConvertToFilename(base),
|
|
self.remote))
|
|
cache_file = os.path.join(self.cache_dir, PICKLE_FILE)
|
|
command = 'mkdir -p %s' % os.path.dirname(cache_file)
|
|
ret = self._ce.RunCommand(command)
|
|
assert ret == 0, "Couldn't create cache dir"
|
|
with open(cache_file, 'wb') as f:
|
|
pickle.dump(self.retval, f)
|
|
pickle.dump(self.out, f)
|
|
pickle.dump(self.err, f)
|
|
|
|
def run(self):
|
|
self._logger = logger.Logger(
|
|
os.path.dirname(__file__), '%s.%s' % (os.path.basename(__file__),
|
|
self.name), True)
|
|
self._ce = command_executer.GetCommandExecuter(self._logger)
|
|
self.RunCached()
|
|
|
|
def RunCached(self):
|
|
self.status = 'WAITING'
|
|
cache_hit = False
|
|
if not self.rerun:
|
|
cache_hit = self.TryToLoadFromCache()
|
|
else:
|
|
self._logger.LogOutput('--rerun passed. Not using cached results.')
|
|
if self.rerun_if_failed and self.retval:
|
|
self._logger.LogOutput('--rerun_if_failed passed and existing test '
|
|
'failed. Rerunning...')
|
|
cache_hit = False
|
|
if not cache_hit:
|
|
# Get machine
|
|
while True:
|
|
if self.terminate:
|
|
return 1
|
|
self.machine = (machine_manager_singleton.MachineManagerSingleton(
|
|
).AcquireMachine(self.image_checksum))
|
|
if self.machine:
|
|
self._logger.LogOutput('%s: Machine %s acquired at %s' %
|
|
(self.name, self.machine.name,
|
|
datetime.datetime.now()))
|
|
break
|
|
else:
|
|
sleep_duration = 10
|
|
time.sleep(sleep_duration)
|
|
try:
|
|
self.remote = self.machine.name
|
|
|
|
if self.machine.checksum != self.image_checksum:
|
|
self.retval = self.ImageTo(self.machine.name)
|
|
if self.retval:
|
|
return self.retval
|
|
self.machine.checksum = self.image_checksum
|
|
self.machine.image = self.chromeos_image
|
|
self.status = 'RUNNING: %s' % self.autotest.name
|
|
[self.retval, self.out, self.err] = self.RunTestOn(self.machine.name)
|
|
self.run_completed = True
|
|
|
|
finally:
|
|
self._logger.LogOutput('Releasing machine: %s' % self.machine.name)
|
|
machine_manager_singleton.MachineManagerSingleton().ReleaseMachine(
|
|
self.machine)
|
|
self._logger.LogOutput('Released machine: %s' % self.machine.name)
|
|
|
|
self.StoreToCache()
|
|
|
|
if not self.retval:
|
|
self.status = 'SUCCEEDED'
|
|
else:
|
|
self.status = 'FAILED'
|
|
|
|
self.ParseOutput()
|
|
# Copy results directory to the scratch dir
|
|
if (not cache_hit and not self.retval and self.autotest.args and
|
|
'--profile' in self.autotest.args):
|
|
results_dir = os.path.join(self.chromeos_root, 'chroot',
|
|
self.results_dir.lstrip('/'))
|
|
tarball = os.path.join(
|
|
self.cache_dir, os.path.basename(os.path.dirname(self.results_dir)))
|
|
command = ('cd %s && tar cjf %s.tbz2 .' % (results_dir, tarball))
|
|
self._ce.RunCommand(command)
|
|
perf_data_file = os.path.join(self.results_dir, self.full_name,
|
|
'profiling/iteration.1/perf.data')
|
|
|
|
# Attempt to build a perf report and keep it with the results.
|
|
command = ('cd %s/src/scripts &&'
|
|
' cros_sdk -- /usr/sbin/perf report --symfs=/build/%s'
|
|
' -i %s --stdio' % (self.chromeos_root, self.board,
|
|
perf_data_file))
|
|
ret, out, err = self._ce.RunCommandWOutput(command)
|
|
with open(os.path.join(self.cache_dir, 'perf.report'), 'wb') as f:
|
|
f.write(out)
|
|
return self.retval
|
|
|
|
def ImageTo(self, machine_name):
|
|
image_args = [image_chromeos.__file__, '--chromeos_root=%s' %
|
|
self.chromeos_root, '--image=%s' % self.chromeos_image,
|
|
'--remote=%s' % machine_name]
|
|
if self.board:
|
|
image_args.append('--board=%s' % self.board)
|
|
|
|
### devserver_port = 8080
|
|
### mo = re.search("\d+", self.name)
|
|
### if mo:
|
|
### to_add = int(mo.group(0))
|
|
### assert to_add < 100, "Too many threads launched!"
|
|
### devserver_port += to_add
|
|
|
|
### # I tried --noupdate_stateful, but that still fails when run in parallel.
|
|
### image_args.append("--image_to_live_args=\"--devserver_port=%s"
|
|
### " --noupdate_stateful\"" % devserver_port)
|
|
### image_args.append("--image_to_live_args=--devserver_port=%s" %
|
|
### devserver_port)
|
|
|
|
# Currently can't image two machines at once.
|
|
# So have to serialized on this lock.
|
|
self.status = 'WAITING ON IMAGE_LOCK'
|
|
with machine_manager_singleton.MachineManagerSingleton().image_lock:
|
|
self.status = 'IMAGING'
|
|
retval = self._ce.RunCommand(' '.join(['python'] + image_args))
|
|
machine_manager_singleton.MachineManagerSingleton().num_reimages += 1
|
|
if retval:
|
|
self.status = 'ABORTED DUE TO IMAGE FAILURE'
|
|
return retval
|
|
|
|
def DoPowerdHack(self):
|
|
command = 'sudo initctl stop powerd'
|
|
self._ce.CrosRunCommand(command,
|
|
machine=self.machine.name,
|
|
chromeos_root=self.chromeos_root)
|
|
|
|
def RunTestOn(self, machine_name):
|
|
command = 'cd %s/src/scripts' % self.chromeos_root
|
|
options = ''
|
|
if self.board:
|
|
options += ' --board=%s' % self.board
|
|
if self.autotest.args:
|
|
options += " --args='%s'" % self.autotest.args
|
|
if 'tegra2' in self.board:
|
|
self.DoPowerdHack()
|
|
command += ('&& cros_sdk -- /usr/bin/test_that %s %s %s' %
|
|
(options, machine_name, self.autotest.name))
|
|
return self._ce.RunCommand(command, True)
|