385 lines
13 KiB
Python
385 lines
13 KiB
Python
# Copyright 2016 The Chromium OS Authors. All rights reserved.
|
|
# Use of this source code is governed by a BSD-style license that can be
|
|
# found in the LICENSE file.
|
|
"""Utilities for launching and accessing ChromeOS buildbots."""
|
|
|
|
from __future__ import print_function
|
|
|
|
import base64
|
|
import json
|
|
import os
|
|
import time
|
|
import urllib2
|
|
|
|
# pylint: disable=no-name-in-module
|
|
from oauth2client.service_account import ServiceAccountCredentials
|
|
|
|
from cros_utils import command_executer
|
|
from cros_utils import logger
|
|
from cros_utils import buildbot_json
|
|
|
|
INITIAL_SLEEP_TIME = 7200 # 2 hours; wait time before polling buildbot.
|
|
SLEEP_TIME = 600 # 10 minutes; time between polling of buildbot.
|
|
TIME_OUT = 28800 # Decide the build is dead or will never finish
|
|
# after this time (8 hours).
|
|
OK_STATUS = [ # List of result status values that are 'ok'.
|
|
# This was obtained from:
|
|
# https://chromium.googlesource.com/chromium/tools/build/+/
|
|
# master/third_party/buildbot_8_4p1/buildbot/status/results.py
|
|
0, # "success"
|
|
1, # "warnings"
|
|
6, # "retry"
|
|
]
|
|
|
|
|
|
class BuildbotTimeout(Exception):
|
|
"""Exception to throw when a buildbot operation timesout."""
|
|
pass
|
|
|
|
|
|
def ParseReportLog(url, build):
|
|
"""Scrape the trybot image name off the Reports log page.
|
|
|
|
This takes the URL for a trybot Reports Stage web page,
|
|
and a trybot build type, such as 'daisy-release'. It
|
|
opens the web page and parses it looking for the trybot
|
|
artifact name (e.g. something like
|
|
'trybot-daisy-release/R40-6394.0.0-b1389'). It returns the
|
|
artifact name, if found.
|
|
"""
|
|
trybot_image = ''
|
|
url += '/text'
|
|
newurl = url.replace('uberchromegw', 'chromegw')
|
|
webpage = urllib2.urlopen(newurl)
|
|
data = webpage.read()
|
|
lines = data.split('\n')
|
|
for l in lines:
|
|
if l.find('Artifacts') > 0 and l.find('trybot') > 0:
|
|
trybot_name = 'trybot-%s' % build
|
|
start_pos = l.find(trybot_name)
|
|
end_pos = l.find('@https://storage')
|
|
trybot_image = l[start_pos:end_pos]
|
|
|
|
return trybot_image
|
|
|
|
|
|
def GetBuildData(buildbot_queue, build_id):
|
|
"""Find the Reports stage web page for a trybot build.
|
|
|
|
This takes the name of a buildbot_queue, such as 'daisy-release'
|
|
and a build id (the build number), and uses the json buildbot api to
|
|
find the Reports stage web page for that build, if it exists.
|
|
"""
|
|
builder = buildbot_json.Buildbot(
|
|
'http://chromegw/p/tryserver.chromiumos/').builders[buildbot_queue]
|
|
build_data = builder.builds[build_id].data
|
|
logs = build_data['logs']
|
|
for l in logs:
|
|
fname = l[1]
|
|
if 'steps/Report/' in fname:
|
|
return fname
|
|
|
|
return ''
|
|
|
|
|
|
def FindBuildRecordFromLog(description, build_info):
|
|
"""Find the right build record in the build logs.
|
|
|
|
Get the first build record from build log with a reason field
|
|
that matches 'description'. ('description' is a special tag we
|
|
created when we launched the buildbot, so we could find it at this
|
|
point.)
|
|
"""
|
|
for build_log in build_info:
|
|
if description in build_log['reason']:
|
|
return build_log
|
|
return {}
|
|
|
|
|
|
def GetBuildInfo(file_dir, waterfall_builder):
|
|
"""Get all the build records for the trybot builds."""
|
|
|
|
builder = ''
|
|
if waterfall_builder.endswith('-release'):
|
|
builder = 'release'
|
|
elif waterfall_builder.endswith('-gcc-toolchain'):
|
|
builder = 'gcc_toolchain'
|
|
elif waterfall_builder.endswith('-llvm-toolchain'):
|
|
builder = 'llvm_toolchain'
|
|
elif waterfall_builder.endswith('-llvm-next-toolchain'):
|
|
builder = 'llvm_next_toolchain'
|
|
|
|
sa_file = os.path.expanduser(
|
|
os.path.join(file_dir, 'cros_utils',
|
|
'chromeos-toolchain-credentials.json'))
|
|
scopes = ['https://www.googleapis.com/auth/userinfo.email']
|
|
|
|
credentials = ServiceAccountCredentials.from_json_keyfile_name(
|
|
sa_file, scopes=scopes)
|
|
url = (
|
|
'https://luci-milo.appspot.com/prpc/milo.Buildbot/GetBuildbotBuildsJSON')
|
|
|
|
# NOTE: If we want to get build logs for the main waterfall builders, the
|
|
# 'master' field below should be 'chromeos' instead of 'chromiumos.tryserver'.
|
|
# Builder would be 'amd64-gcc-toolchain' or 'arm-llvm-toolchain', etc.
|
|
|
|
body = json.dumps({
|
|
'master': 'chromiumos.tryserver',
|
|
'builder': builder,
|
|
'include_current': True,
|
|
'limit': 100
|
|
})
|
|
access_token = credentials.get_access_token()
|
|
headers = {
|
|
'Accept': 'application/json',
|
|
'Content-Type': 'application/json',
|
|
'Authorization': 'Bearer %s' % access_token.access_token
|
|
}
|
|
r = urllib2.Request(url, body, headers)
|
|
u = urllib2.urlopen(r, timeout=60)
|
|
u.read(4)
|
|
o = json.load(u)
|
|
data = [base64.b64decode(item['data']) for item in o['builds']]
|
|
result = []
|
|
for d in data:
|
|
tmp = json.loads(d)
|
|
result.append(tmp)
|
|
return result
|
|
|
|
|
|
def FindArchiveImage(chromeos_root, build, build_id):
|
|
"""Returns name of the trybot artifact for board/build_id."""
|
|
ce = command_executer.GetCommandExecuter()
|
|
command = ('gsutil ls gs://chromeos-image-archive/trybot-%s/*b%s'
|
|
'/chromiumos_test_image.tar.xz' % (build, build_id))
|
|
_, out, _ = ce.ChrootRunCommandWOutput(
|
|
chromeos_root, command, print_to_console=False)
|
|
#
|
|
# If build_id is not unique, there may be multiple archive images
|
|
# to choose from; sort them & pick the first (newest).
|
|
#
|
|
# If there are multiple archive images found, out will look something
|
|
# like this:
|
|
#
|
|
# 'gs://.../R35-5692.0.0-b105/chromiumos_test_image.tar.xz
|
|
# gs://.../R46-7339.0.0-b105/chromiumos_test_image.tar.xz'
|
|
#
|
|
out = out.rstrip('\n')
|
|
tmp_list = out.split('\n')
|
|
# After stripping the final '\n' and splitting on any other '\n', we get
|
|
# something like this:
|
|
# tmp_list = [ 'gs://.../R35-5692.0.0-b105/chromiumos_test_image.tar.xz' ,
|
|
# 'gs://.../R46-7339.0.0-b105/chromiumos_test_image.tar.xz' ]
|
|
#
|
|
# If we sort this in descending order, we should end up with the most
|
|
# recent test image first, so that's what we do here.
|
|
#
|
|
if len(tmp_list) > 1:
|
|
tmp_list = sorted(tmp_list, reverse=True)
|
|
out = tmp_list[0]
|
|
|
|
trybot_image = ''
|
|
trybot_name = 'trybot-%s' % build
|
|
if out and out.find(trybot_name) > 0:
|
|
start_pos = out.find(trybot_name)
|
|
end_pos = out.find('/chromiumos_test_image')
|
|
trybot_image = out[start_pos:end_pos]
|
|
|
|
return trybot_image
|
|
|
|
|
|
def GetTrybotImage(chromeos_root,
|
|
buildbot_name,
|
|
patch_list,
|
|
build_tag,
|
|
other_flags=None,
|
|
build_toolchain=False,
|
|
async=False):
|
|
"""Launch buildbot and get resulting trybot artifact name.
|
|
|
|
This function launches a buildbot with the appropriate flags to
|
|
build the test ChromeOS image, with the current ToT mobile compiler. It
|
|
checks every 10 minutes to see if the trybot has finished. When the trybot
|
|
has finished, it parses the resulting report logs to find the trybot
|
|
artifact (if one was created), and returns that artifact name.
|
|
|
|
chromeos_root is the path to the ChromeOS root, needed for finding chromite
|
|
and launching the buildbot.
|
|
|
|
buildbot_name is the name of the buildbot queue, such as lumpy-release or
|
|
daisy-paladin.
|
|
|
|
patch_list a python list of the patches, if any, for the buildbot to use.
|
|
|
|
build_tag is a (unique) string to be used to look up the buildbot results
|
|
from among all the build records.
|
|
"""
|
|
ce = command_executer.GetCommandExecuter()
|
|
cbuildbot_path = os.path.join(chromeos_root, 'chromite/cbuildbot')
|
|
base_dir = os.getcwd()
|
|
patch_arg = ''
|
|
if patch_list:
|
|
for p in patch_list:
|
|
patch_arg = patch_arg + ' -g ' + repr(p)
|
|
toolchain_flags = ''
|
|
if build_toolchain:
|
|
toolchain_flags += '--latest-toolchain'
|
|
os.chdir(cbuildbot_path)
|
|
if other_flags:
|
|
optional_flags = ' '.join(other_flags)
|
|
else:
|
|
optional_flags = ''
|
|
|
|
# Launch buildbot with appropriate flags.
|
|
build = buildbot_name
|
|
description = build_tag
|
|
command_prefix = ''
|
|
if not patch_arg:
|
|
command_prefix = 'yes | '
|
|
command = ('%s ./cbuildbot --remote --nochromesdk %s'
|
|
' --remote-description=%s %s %s %s' % (command_prefix,
|
|
optional_flags, description,
|
|
toolchain_flags, patch_arg,
|
|
build))
|
|
_, out, _ = ce.RunCommandWOutput(command)
|
|
if 'Tryjob submitted!' not in out:
|
|
logger.GetLogger().LogFatal('Error occurred while launching trybot job: '
|
|
'%s' % command)
|
|
|
|
os.chdir(base_dir)
|
|
|
|
build_id = 0
|
|
build_status = None
|
|
# Wait for buildbot to finish running (check every 10 minutes). Wait
|
|
# 10 minutes before the first check to give the buildbot time to launch
|
|
# (so we don't start looking for build data before it's out there).
|
|
time.sleep(SLEEP_TIME)
|
|
done = False
|
|
pending = True
|
|
# pending_time is the time between when we submit the job and when the
|
|
# buildbot actually launches the build. running_time is the time between
|
|
# when the buildbot job launches and when it finishes. The job is
|
|
# considered 'pending' until we can find an entry for it in the buildbot
|
|
# logs.
|
|
pending_time = SLEEP_TIME
|
|
running_time = 0
|
|
long_slept = False
|
|
while not done:
|
|
done = True
|
|
build_info = GetBuildInfo(base_dir, build)
|
|
if not build_info:
|
|
if pending_time > TIME_OUT:
|
|
logger.GetLogger().LogFatal('Unable to get build logs for target %s.' %
|
|
build)
|
|
else:
|
|
pending_message = 'Unable to find build log; job may be pending.'
|
|
done = False
|
|
|
|
if done:
|
|
data_dict = FindBuildRecordFromLog(description, build_info)
|
|
if not data_dict:
|
|
# Trybot job may be pending (not actually launched yet).
|
|
if pending_time > TIME_OUT:
|
|
logger.GetLogger().LogFatal('Unable to find build record for trybot'
|
|
' %s.' % description)
|
|
else:
|
|
pending_message = 'Unable to find build record; job may be pending.'
|
|
done = False
|
|
|
|
else:
|
|
# Now that we have actually found the entry for the build
|
|
# job in the build log, we know the job is actually
|
|
# runnning, not pending, so we flip the 'pending' flag. We
|
|
# still have to wait for the buildbot job to finish running
|
|
# however.
|
|
pending = False
|
|
build_id = data_dict['number']
|
|
|
|
if async:
|
|
# Do not wait for trybot job to finish; return immediately
|
|
return build_id, ' '
|
|
|
|
if not long_slept:
|
|
# The trybot generally takes more than 2 hours to finish.
|
|
# Wait two hours before polling the status.
|
|
long_slept = True
|
|
time.sleep(INITIAL_SLEEP_TIME)
|
|
pending_time += INITIAL_SLEEP_TIME
|
|
if True == data_dict['finished']:
|
|
build_status = data_dict['results']
|
|
else:
|
|
done = False
|
|
|
|
if not done:
|
|
if pending:
|
|
logger.GetLogger().LogOutput(pending_message)
|
|
logger.GetLogger().LogOutput('Current pending time: %d minutes.' %
|
|
(pending_time / 60))
|
|
pending_time += SLEEP_TIME
|
|
else:
|
|
logger.GetLogger().LogOutput('{0} minutes passed.'.format(running_time /
|
|
60))
|
|
logger.GetLogger().LogOutput('Sleeping {0} seconds.'.format(SLEEP_TIME))
|
|
running_time += SLEEP_TIME
|
|
|
|
time.sleep(SLEEP_TIME)
|
|
if running_time > TIME_OUT:
|
|
done = True
|
|
|
|
trybot_image = ''
|
|
|
|
if build.endswith('-toolchain'):
|
|
# For rotating testers, we don't care about their build_status
|
|
# result, because if any HWTest failed it will be non-zero.
|
|
trybot_image = FindArchiveImage(chromeos_root, build, build_id)
|
|
else:
|
|
# The nightly performance tests do not run HWTests, so if
|
|
# their build_status is non-zero, we do care. In this case
|
|
# non-zero means the image itself probably did not build.
|
|
if build_status in OK_STATUS:
|
|
trybot_image = FindArchiveImage(chromeos_root, build, build_id)
|
|
if not trybot_image:
|
|
logger.GetLogger().LogError('Trybot job %s failed with status %d;'
|
|
' no trybot image generated.' %
|
|
(description, build_status))
|
|
|
|
logger.GetLogger().LogOutput("trybot_image is '%s'" % trybot_image)
|
|
logger.GetLogger().LogOutput('build_status is %d' % build_status)
|
|
return build_id, trybot_image
|
|
|
|
|
|
def GetGSContent(chromeos_root, path):
|
|
"""gsutil cat path"""
|
|
|
|
ce = command_executer.GetCommandExecuter()
|
|
command = ('gsutil cat gs://chromeos-image-archive/%s' % path)
|
|
_, out, _ = ce.ChrootRunCommandWOutput(
|
|
chromeos_root, command, print_to_console=False)
|
|
return out
|
|
|
|
|
|
def DoesImageExist(chromeos_root, build):
|
|
"""Check if the image for the given build exists."""
|
|
|
|
ce = command_executer.GetCommandExecuter()
|
|
command = ('gsutil ls gs://chromeos-image-archive/%s'
|
|
'/chromiumos_test_image.tar.xz' % (build))
|
|
ret = ce.ChrootRunCommand(chromeos_root, command, print_to_console=False)
|
|
return not ret
|
|
|
|
|
|
def WaitForImage(chromeos_root, build):
|
|
"""Wait for an image to be ready."""
|
|
|
|
elapsed_time = 0
|
|
while elapsed_time < TIME_OUT:
|
|
if DoesImageExist(chromeos_root, build):
|
|
return
|
|
logger.GetLogger().LogOutput('Image %s not ready, waiting for 10 minutes' %
|
|
build)
|
|
time.sleep(SLEEP_TIME)
|
|
elapsed_time += SLEEP_TIME
|
|
|
|
logger.GetLogger().LogOutput('Image %s not found, waited for %d hours' %
|
|
(build, (TIME_OUT / 3600)))
|
|
raise BuildbotTimeout('Timeout while waiting for image %s' % build)
|