235 lines
6.9 KiB
Python
235 lines
6.9 KiB
Python
# Copyright 2015 The Chromium Authors. All rights reserved.
|
|
# Use of this source code is governed by a BSD-style license that can be
|
|
# found in the LICENSE file.
|
|
|
|
import collections
|
|
import logging
|
|
import re
|
|
import time
|
|
|
|
from google.appengine.api import urlfetch
|
|
import webapp2
|
|
|
|
from base import bigquery
|
|
from base import constants
|
|
from common import buildbot
|
|
|
|
|
|
class Builds(webapp2.RequestHandler):
|
|
|
|
def get(self):
|
|
urlfetch.set_default_fetch_deadline(300)
|
|
|
|
bq = bigquery.BigQuery()
|
|
|
|
current_events = []
|
|
events = []
|
|
for master_name in constants.MASTER_NAMES:
|
|
builders = buildbot.Builders(master_name)
|
|
available_builds = _AvailableBuilds(builders)
|
|
recorded_builds = _RecordedBuilds(bq, builders, available_builds)
|
|
for builder in builders:
|
|
# Filter out recorded builds from available builds.
|
|
build_numbers = (available_builds[builder.name] -
|
|
recorded_builds[builder.name])
|
|
builder_current_events, builder_events = _TraceEventsForBuilder(
|
|
builder, build_numbers)
|
|
current_events += builder_current_events
|
|
events += builder_events
|
|
|
|
jobs = []
|
|
if current_events:
|
|
jobs += bq.InsertRowsAsync(
|
|
constants.DATASET, constants.CURRENT_BUILDS_TABLE,
|
|
current_events, truncate=True)
|
|
if events:
|
|
jobs += bq.InsertRowsAsync(constants.DATASET, constants.BUILDS_TABLE,
|
|
events)
|
|
|
|
for job in jobs:
|
|
bq.PollJob(job, 60 * 20) # 20 minutes.
|
|
|
|
|
|
def _AvailableBuilds(builders):
|
|
available_builds = {}
|
|
for builder in builders:
|
|
if not builder.cached_builds:
|
|
available_builds[builder.name] = frozenset()
|
|
continue
|
|
|
|
max_build = max(builder.cached_builds)
|
|
# Buildbot on tryserver.chromium.perf is occasionally including build 0 in
|
|
# its list of cached builds. That results in more builds than we want.
|
|
# Limit the list to the last 100 builds, because the urlfetch URL limit is
|
|
# 2048 bytes, and "&select=100000" * 100 is 1400 bytes.
|
|
builds = frozenset(build for build in builder.cached_builds
|
|
if build >= max_build - 100)
|
|
available_builds[builder.name] = builds
|
|
return available_builds
|
|
|
|
|
|
def _RecordedBuilds(bq, builders, available_builds):
|
|
# 105 days / 15 weeks. Must be some number greater than 100 days, because
|
|
# we request up to 100 builds (see above comment), and the slowest cron bots
|
|
# run one job every day.
|
|
start_time_ms = -1000 * 60 * 60 * 24 * 105
|
|
table = '%s.%s@%d-' % (constants.DATASET, constants.BUILDS_TABLE,
|
|
start_time_ms)
|
|
|
|
conditions = []
|
|
for builder in builders:
|
|
if not available_builds[builder.name]:
|
|
continue
|
|
max_build = max(available_builds[builder.name])
|
|
min_build = min(available_builds[builder.name])
|
|
conditions.append('WHEN builder = "%s" THEN build >= %d AND build <= %d' %
|
|
(builder.name, min_build, max_build))
|
|
|
|
query = (
|
|
'SELECT builder, build '
|
|
'FROM [%s] ' % table +
|
|
'WHERE CASE %s END ' % ' '.join(conditions) +
|
|
'GROUP BY builder, build'
|
|
)
|
|
query_result = bq.QuerySync(query, 600)
|
|
|
|
builds = collections.defaultdict(set)
|
|
for row in query_result:
|
|
builds[row['f'][0]['v']].add(int(row['f'][1]['v']))
|
|
return builds
|
|
|
|
|
|
def _TraceEventsForBuilder(builder, build_numbers):
|
|
if not build_numbers:
|
|
return (), ()
|
|
|
|
build_numbers_string = ', '.join(map(str, sorted(build_numbers)))
|
|
logging.info('Getting %s: %s', builder.name, build_numbers_string)
|
|
|
|
# Fetch build information and generate trace events.
|
|
current_events = []
|
|
events = []
|
|
|
|
builder_builds = builder.builds.Fetch(build_numbers)
|
|
query_time = time.time()
|
|
for build in builder_builds:
|
|
if build.complete:
|
|
events += _TraceEventsFromBuild(builder, build, query_time)
|
|
else:
|
|
current_events += _TraceEventsFromBuild(builder, build, query_time)
|
|
|
|
return current_events, events
|
|
|
|
|
|
def _TraceEventsFromBuild(builder, build, query_time):
|
|
match = re.match(r'(.+) \(([0-9]+)\)', builder.name)
|
|
if match:
|
|
configuration, host_shard = match.groups()
|
|
host_shard = int(host_shard)
|
|
else:
|
|
configuration = builder.name
|
|
host_shard = 0
|
|
|
|
# Build trace event.
|
|
if build.end_time:
|
|
build_end_time = build.end_time
|
|
else:
|
|
build_end_time = query_time
|
|
os, os_version, role = _ParseBuilderName(builder.master_name, builder.name)
|
|
yield {
|
|
'name': 'Build %d' % build.number,
|
|
'start_time': build.start_time,
|
|
'end_time': build_end_time,
|
|
|
|
'build': build.number,
|
|
'builder': builder.name,
|
|
'configuration': configuration,
|
|
'host_shard': host_shard,
|
|
'hostname': build.slave_name,
|
|
'master': builder.master_name,
|
|
'os': os,
|
|
'os_version': os_version,
|
|
'role': role,
|
|
'status': build.status,
|
|
'url': build.url,
|
|
}
|
|
|
|
# Step trace events.
|
|
for step in build.steps:
|
|
if not step.start_time:
|
|
continue
|
|
|
|
if step.name == 'steps':
|
|
continue
|
|
|
|
if step.end_time:
|
|
step_end_time = step.end_time
|
|
else:
|
|
step_end_time = query_time
|
|
yield {
|
|
'name': step.name,
|
|
'start_time': step.start_time,
|
|
'end_time': step_end_time,
|
|
|
|
'benchmark': step.name, # TODO(dtu): This isn't always right.
|
|
'build': build.number,
|
|
'builder': builder.name,
|
|
'configuration': configuration,
|
|
'host_shard': host_shard,
|
|
'hostname': build.slave_name,
|
|
'master': builder.master_name,
|
|
'os': os,
|
|
'os_version': os_version,
|
|
'role': role,
|
|
'status': step.status,
|
|
'url': step.url,
|
|
}
|
|
|
|
|
|
def _ParseBuilderName(master_name, builder_name):
|
|
if master_name == 'chromium.perf':
|
|
match = re.match(r'^([A-Za-z]+)(?: ([0-9\.]+|XP))?([A-Za-z0-9-\. ]+)? '
|
|
r'(Builder|Perf)(?: \([0-9]+\))?$', builder_name).groups()
|
|
os = match[0]
|
|
if match[1]:
|
|
os_version = match[1]
|
|
else:
|
|
os_version = None
|
|
if match[3] == 'Builder':
|
|
role = 'builder'
|
|
elif match[3] == 'Perf':
|
|
role = 'tester'
|
|
else:
|
|
raise NotImplementedError()
|
|
elif master_name == 'client.catapult':
|
|
match = re.match(r'^Catapult(?: ([A-Za-z])+)? ([A-Za-z]+)$',
|
|
builder_name).groups()
|
|
os = match[1]
|
|
os_version = None
|
|
role = match[0]
|
|
if not role:
|
|
role = 'tester'
|
|
elif master_name == 'tryserver.chromium.perf':
|
|
match = re.match(r'^(android|linux|mac|win).*_([a-z]+)$',
|
|
builder_name).groups()
|
|
os = match[0]
|
|
os_version = None
|
|
role = match[1]
|
|
elif master_name == 'tryserver.client.catapult':
|
|
match = re.match(r'^Catapult(?: (Android|Linux|Mac|Windows))? ([A-Za-z]+)$',
|
|
builder_name).groups()
|
|
os = match[0]
|
|
os_version = None
|
|
role = match[1]
|
|
else:
|
|
raise NotImplementedError()
|
|
|
|
if os:
|
|
os = os.lower()
|
|
if os == 'windows':
|
|
os = 'win'
|
|
if os_version:
|
|
os_version = os_version.lower()
|
|
role = role.lower()
|
|
|
|
return (os, os_version, role)
|