365 lines
12 KiB
Python
Executable file
365 lines
12 KiB
Python
Executable file
#!/usr/bin/python2
|
|
|
|
# Script to test different toolchains against ChromeOS benchmarks.
|
|
"""Toolchain team nightly performance test script (local builds)."""
|
|
|
|
from __future__ import print_function
|
|
|
|
import argparse
|
|
import datetime
|
|
import os
|
|
import sys
|
|
import build_chromeos
|
|
import setup_chromeos
|
|
import time
|
|
from cros_utils import command_executer
|
|
from cros_utils import misc
|
|
from cros_utils import logger
|
|
|
|
CROSTC_ROOT = '/usr/local/google/crostc'
|
|
MAIL_PROGRAM = '~/var/bin/mail-sheriff'
|
|
PENDING_ARCHIVES_DIR = os.path.join(CROSTC_ROOT, 'pending_archives')
|
|
NIGHTLY_TESTS_DIR = os.path.join(CROSTC_ROOT, 'nightly_test_reports')
|
|
|
|
|
|
class GCCConfig(object):
|
|
"""GCC configuration class."""
|
|
|
|
def __init__(self, githash):
|
|
self.githash = githash
|
|
|
|
|
|
class ToolchainConfig(object):
|
|
"""Toolchain configuration class."""
|
|
|
|
def __init__(self, gcc_config=None):
|
|
self.gcc_config = gcc_config
|
|
|
|
|
|
class ChromeOSCheckout(object):
|
|
"""Main class for checking out, building and testing ChromeOS."""
|
|
|
|
def __init__(self, board, chromeos_root):
|
|
self._board = board
|
|
self._chromeos_root = chromeos_root
|
|
self._ce = command_executer.GetCommandExecuter()
|
|
self._l = logger.GetLogger()
|
|
self._build_num = None
|
|
|
|
def _DeleteChroot(self):
|
|
command = 'cd %s; cros_sdk --delete' % self._chromeos_root
|
|
return self._ce.RunCommand(command)
|
|
|
|
def _DeleteCcahe(self):
|
|
# crosbug.com/34956
|
|
command = 'sudo rm -rf %s' % os.path.join(self._chromeos_root, '.cache')
|
|
return self._ce.RunCommand(command)
|
|
|
|
def _GetBuildNumber(self):
|
|
"""Get the build number of the ChromeOS image from the chroot.
|
|
|
|
This function assumes a ChromeOS image has been built in the chroot.
|
|
It translates the 'latest' symlink in the
|
|
<chroot>/src/build/images/<board> directory, to find the actual
|
|
ChromeOS build number for the image that was built. For example, if
|
|
src/build/image/lumpy/latest -> R37-5982.0.2014_06_23_0454-a1, then
|
|
This function would parse it out and assign 'R37-5982' to self._build_num.
|
|
This is used to determine the official, vanilla build to use for
|
|
comparison tests.
|
|
"""
|
|
# Get the path to 'latest'
|
|
sym_path = os.path.join(
|
|
misc.GetImageDir(self._chromeos_root, self._board), 'latest')
|
|
# Translate the symlink to its 'real' path.
|
|
real_path = os.path.realpath(sym_path)
|
|
# Break up the path and get the last piece
|
|
# (e.g. 'R37-5982.0.2014_06_23_0454-a1"
|
|
path_pieces = real_path.split('/')
|
|
last_piece = path_pieces[-1]
|
|
# Break this piece into the image number + other pieces, and get the
|
|
# image number [ 'R37-5982', '0', '2014_06_23_0454-a1']
|
|
image_parts = last_piece.split('.')
|
|
self._build_num = image_parts[0]
|
|
|
|
def _BuildLabelName(self, config):
|
|
pieces = config.split('/')
|
|
compiler_version = pieces[-1]
|
|
label = compiler_version + '_tot_afdo'
|
|
return label
|
|
|
|
def _BuildAndImage(self, label=''):
|
|
if (not label or
|
|
not misc.DoesLabelExist(self._chromeos_root, self._board, label)):
|
|
build_chromeos_args = [
|
|
build_chromeos.__file__, '--chromeos_root=%s' % self._chromeos_root,
|
|
'--board=%s' % self._board, '--rebuild'
|
|
]
|
|
if self._public:
|
|
build_chromeos_args.append('--env=USE=-chrome_internal')
|
|
|
|
ret = build_chromeos.Main(build_chromeos_args)
|
|
if ret != 0:
|
|
raise RuntimeError("Couldn't build ChromeOS!")
|
|
|
|
if not self._build_num:
|
|
self._GetBuildNumber()
|
|
# Check to see if we need to create the symbolic link for the vanilla
|
|
# image, and do so if appropriate.
|
|
if not misc.DoesLabelExist(self._chromeos_root, self._board, 'vanilla'):
|
|
build_name = '%s-release/%s.0.0' % (self._board, self._build_num)
|
|
full_vanilla_path = os.path.join(os.getcwd(), self._chromeos_root,
|
|
'chroot/tmp', build_name)
|
|
misc.LabelLatestImage(self._chromeos_root, self._board, label,
|
|
full_vanilla_path)
|
|
else:
|
|
misc.LabelLatestImage(self._chromeos_root, self._board, label)
|
|
return label
|
|
|
|
def _SetupBoard(self, env_dict, usepkg_flag, clobber_flag):
|
|
env_string = misc.GetEnvStringFromDict(env_dict)
|
|
command = ('%s %s' % (env_string, misc.GetSetupBoardCommand(
|
|
self._board, usepkg=usepkg_flag, force=clobber_flag)))
|
|
ret = self._ce.ChrootRunCommand(self._chromeos_root, command)
|
|
error_str = "Could not setup board: '%s'" % command
|
|
assert ret == 0, error_str
|
|
|
|
def _UnInstallToolchain(self):
|
|
command = ('sudo CLEAN_DELAY=0 emerge -C cross-%s/gcc' %
|
|
misc.GetCtargetFromBoard(self._board, self._chromeos_root))
|
|
ret = self._ce.ChrootRunCommand(self._chromeos_root, command)
|
|
if ret != 0:
|
|
raise RuntimeError("Couldn't uninstall the toolchain!")
|
|
|
|
def _CheckoutChromeOS(self):
|
|
# TODO(asharif): Setup a fixed ChromeOS version (quarterly snapshot).
|
|
if not os.path.exists(self._chromeos_root):
|
|
setup_chromeos_args = ['--dir=%s' % self._chromeos_root]
|
|
if self._public:
|
|
setup_chromeos_args.append('--public')
|
|
ret = setup_chromeos.Main(setup_chromeos_args)
|
|
if ret != 0:
|
|
raise RuntimeError("Couldn't run setup_chromeos!")
|
|
|
|
def _BuildToolchain(self, config):
|
|
# Call setup_board for basic, vanilla setup.
|
|
self._SetupBoard({}, usepkg_flag=True, clobber_flag=False)
|
|
# Now uninstall the vanilla compiler and setup/build our custom
|
|
# compiler.
|
|
self._UnInstallToolchain()
|
|
envdict = {
|
|
'USE': 'git_gcc',
|
|
'GCC_GITHASH': config.gcc_config.githash,
|
|
'EMERGE_DEFAULT_OPTS': '--exclude=gcc'
|
|
}
|
|
self._SetupBoard(envdict, usepkg_flag=False, clobber_flag=False)
|
|
|
|
|
|
class ToolchainComparator(ChromeOSCheckout):
|
|
"""Main class for running tests and generating reports."""
|
|
|
|
def __init__(self,
|
|
board,
|
|
remotes,
|
|
configs,
|
|
clean,
|
|
public,
|
|
force_mismatch,
|
|
noschedv2=False):
|
|
self._board = board
|
|
self._remotes = remotes
|
|
self._chromeos_root = 'chromeos'
|
|
self._configs = configs
|
|
self._clean = clean
|
|
self._public = public
|
|
self._force_mismatch = force_mismatch
|
|
self._ce = command_executer.GetCommandExecuter()
|
|
self._l = logger.GetLogger()
|
|
timestamp = datetime.datetime.strftime(datetime.datetime.now(),
|
|
'%Y-%m-%d_%H:%M:%S')
|
|
self._reports_dir = os.path.join(
|
|
NIGHTLY_TESTS_DIR,
|
|
'%s.%s' % (timestamp, board),)
|
|
self._noschedv2 = noschedv2
|
|
ChromeOSCheckout.__init__(self, board, self._chromeos_root)
|
|
|
|
def _FinishSetup(self):
|
|
# Get correct .boto file
|
|
current_dir = os.getcwd()
|
|
src = '/usr/local/google/home/mobiletc-prebuild/.boto'
|
|
dest = os.path.join(current_dir, self._chromeos_root,
|
|
'src/private-overlays/chromeos-overlay/'
|
|
'googlestorage_account.boto')
|
|
# Copy the file to the correct place
|
|
copy_cmd = 'cp %s %s' % (src, dest)
|
|
retv = self._ce.RunCommand(copy_cmd)
|
|
if retv != 0:
|
|
raise RuntimeError("Couldn't copy .boto file for google storage.")
|
|
|
|
# Fix protections on ssh key
|
|
command = ('chmod 600 /var/cache/chromeos-cache/distfiles/target'
|
|
'/chrome-src-internal/src/third_party/chromite/ssh_keys'
|
|
'/testing_rsa')
|
|
retv = self._ce.ChrootRunCommand(self._chromeos_root, command)
|
|
if retv != 0:
|
|
raise RuntimeError('chmod for testing_rsa failed')
|
|
|
|
def _TestLabels(self, labels):
|
|
experiment_file = 'toolchain_experiment.txt'
|
|
image_args = ''
|
|
if self._force_mismatch:
|
|
image_args = '--force-mismatch'
|
|
experiment_header = """
|
|
board: %s
|
|
remote: %s
|
|
retries: 1
|
|
""" % (self._board, self._remotes)
|
|
experiment_tests = """
|
|
benchmark: all_toolchain_perf {
|
|
suite: telemetry_Crosperf
|
|
iterations: 3
|
|
}
|
|
"""
|
|
|
|
with open(experiment_file, 'w') as f:
|
|
f.write(experiment_header)
|
|
f.write(experiment_tests)
|
|
for label in labels:
|
|
# TODO(asharif): Fix crosperf so it accepts labels with symbols
|
|
crosperf_label = label
|
|
crosperf_label = crosperf_label.replace('-', '_')
|
|
crosperf_label = crosperf_label.replace('+', '_')
|
|
crosperf_label = crosperf_label.replace('.', '')
|
|
|
|
# Use the official build instead of building vanilla ourselves.
|
|
if label == 'vanilla':
|
|
build_name = '%s-release/%s.0.0' % (self._board, self._build_num)
|
|
|
|
# Now add 'official build' to test file.
|
|
official_image = """
|
|
official_image {
|
|
chromeos_root: %s
|
|
build: %s
|
|
}
|
|
""" % (self._chromeos_root, build_name)
|
|
f.write(official_image)
|
|
|
|
else:
|
|
experiment_image = """
|
|
%s {
|
|
chromeos_image: %s
|
|
image_args: %s
|
|
}
|
|
""" % (crosperf_label, os.path.join(
|
|
misc.GetImageDir(self._chromeos_root, self._board), label,
|
|
'chromiumos_test_image.bin'), image_args)
|
|
f.write(experiment_image)
|
|
|
|
crosperf = os.path.join(os.path.dirname(__file__), 'crosperf', 'crosperf')
|
|
noschedv2_opts = '--noschedv2' if self._noschedv2 else ''
|
|
command = ('{crosperf} --no_email=True --results_dir={r_dir} '
|
|
'--json_report=True {noschedv2_opts} {exp_file}').format(
|
|
crosperf=crosperf,
|
|
r_dir=self._reports_dir,
|
|
noschedv2_opts=noschedv2_opts,
|
|
exp_file=experiment_file)
|
|
|
|
ret = self._ce.RunCommand(command)
|
|
if ret != 0:
|
|
raise RuntimeError('Crosperf execution error!')
|
|
else:
|
|
# Copy json report to pending archives directory.
|
|
command = 'cp %s/*.json %s/.' % (self._reports_dir, PENDING_ARCHIVES_DIR)
|
|
ret = self._ce.RunCommand(command)
|
|
return
|
|
|
|
def _SendEmail(self):
|
|
"""Find email msesage generated by crosperf and send it."""
|
|
filename = os.path.join(self._reports_dir, 'msg_body.html')
|
|
if (os.path.exists(filename) and
|
|
os.path.exists(os.path.expanduser(MAIL_PROGRAM))):
|
|
command = ('cat %s | %s -s "Nightly test results, %s" -team -html' %
|
|
(filename, MAIL_PROGRAM, self._board))
|
|
self._ce.RunCommand(command)
|
|
|
|
def DoAll(self):
|
|
self._CheckoutChromeOS()
|
|
labels = []
|
|
labels.append('vanilla')
|
|
for config in self._configs:
|
|
label = self._BuildLabelName(config.gcc_config.githash)
|
|
if not misc.DoesLabelExist(self._chromeos_root, self._board, label):
|
|
self._BuildToolchain(config)
|
|
label = self._BuildAndImage(label)
|
|
labels.append(label)
|
|
self._FinishSetup()
|
|
self._TestLabels(labels)
|
|
self._SendEmail()
|
|
if self._clean:
|
|
ret = self._DeleteChroot()
|
|
if ret != 0:
|
|
return ret
|
|
ret = self._DeleteCcahe()
|
|
if ret != 0:
|
|
return ret
|
|
return 0
|
|
|
|
|
|
def Main(argv):
|
|
"""The main function."""
|
|
# Common initializations
|
|
### command_executer.InitCommandExecuter(True)
|
|
command_executer.InitCommandExecuter()
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument(
|
|
'--remote', dest='remote', help='Remote machines to run tests on.')
|
|
parser.add_argument(
|
|
'--board', dest='board', default='x86-alex', help='The target board.')
|
|
parser.add_argument(
|
|
'--githashes',
|
|
dest='githashes',
|
|
default='master',
|
|
help='The gcc githashes to test.')
|
|
parser.add_argument(
|
|
'--clean',
|
|
dest='clean',
|
|
default=False,
|
|
action='store_true',
|
|
help='Clean the chroot after testing.')
|
|
parser.add_argument(
|
|
'--public',
|
|
dest='public',
|
|
default=False,
|
|
action='store_true',
|
|
help='Use the public checkout/build.')
|
|
parser.add_argument(
|
|
'--force-mismatch',
|
|
dest='force_mismatch',
|
|
default='',
|
|
help='Force the image regardless of board mismatch')
|
|
parser.add_argument(
|
|
'--noschedv2',
|
|
dest='noschedv2',
|
|
action='store_true',
|
|
default=False,
|
|
help='Pass --noschedv2 to crosperf.')
|
|
options = parser.parse_args(argv)
|
|
if not options.board:
|
|
print('Please give a board.')
|
|
return 1
|
|
if not options.remote:
|
|
print('Please give at least one remote machine.')
|
|
return 1
|
|
toolchain_configs = []
|
|
for githash in options.githashes.split(','):
|
|
gcc_config = GCCConfig(githash=githash)
|
|
toolchain_config = ToolchainConfig(gcc_config=gcc_config)
|
|
toolchain_configs.append(toolchain_config)
|
|
fc = ToolchainComparator(options.board, options.remote, toolchain_configs,
|
|
options.clean, options.public,
|
|
options.force_mismatch, options.noschedv2)
|
|
return fc.DoAll()
|
|
|
|
|
|
if __name__ == '__main__':
|
|
retval = Main(sys.argv[1:])
|
|
sys.exit(retval)
|