786 lines
32 KiB
Python
786 lines
32 KiB
Python
# tko/nightly.py code shared by various tko/*.cgi graphing scripts
|
|
|
|
import cgi, cgitb
|
|
import os, sys
|
|
import common
|
|
from autotest_lib.tko import db, plotgraph, perf
|
|
from autotest_lib.client.common_lib import kernel_versions
|
|
|
|
|
|
def add_kernel_jobs(label_pattern):
|
|
cmd = "select job_idx from tko_jobs where label like '%s'" % label_pattern
|
|
nrows = perf.db_cur.execute(cmd)
|
|
return [row[0] for row in perf.db_cur.fetchall()]
|
|
|
|
|
|
def is_filtered_platform(platform, platforms_filter):
|
|
if not platforms_filter:
|
|
return True
|
|
for p in platforms_filter:
|
|
if platform.startswith(p):
|
|
return True
|
|
return False
|
|
|
|
|
|
def get_test_attributes(testrunx):
|
|
cmd = ( "select attribute, value from tko_test_attributes"
|
|
" where test_idx = %d" % testrunx )
|
|
nrows = perf.db_cur.execute(cmd)
|
|
return dict(perf.db_cur.fetchall())
|
|
|
|
|
|
def get_antag(testrunx):
|
|
attrs = get_test_attributes(testrunx)
|
|
return attrs.get('antag', None)
|
|
|
|
|
|
def matching_test_attributes(attrs, required_test_attributes):
|
|
if not required_test_attributes:
|
|
return True
|
|
matches = [attrs[key] == required_test_attributes[key]
|
|
for key in attrs if key in required_test_attributes]
|
|
return min(matches+[True]) # True if all jointly-existing keys matched
|
|
|
|
|
|
def collect_testruns(jobs, test, test_attributes,
|
|
platforms_filter, by_hosts, no_antag):
|
|
# get test_runs run #s for 1 test on 1 kernel and some platforms
|
|
# TODO: Is jobs list short enough to use directly in 1 sql cmd?
|
|
# TODO: add filtering on test series?
|
|
runs = {} # platform --> list of test runs
|
|
for jobx in jobs:
|
|
cmd = ( "select test_idx, machine_idx from tko_tests"
|
|
" where job_idx = %s and test = %s" )
|
|
args = [jobx, test]
|
|
nrows = perf.db_cur.execute(cmd, args)
|
|
for testrunx, machx in perf.db_cur.fetchall():
|
|
platform, host = perf.machine_idx_to_platform_host(machx)
|
|
if by_hosts:
|
|
platform += '.'+host
|
|
if ( is_filtered_platform(platform, platforms_filter) and
|
|
matching_test_attributes(get_test_attributes(testrunx),
|
|
test_attributes) and
|
|
(not no_antag or get_antag(testrunx) == '') ):
|
|
runs.setdefault(platform, []).append(testrunx)
|
|
return runs
|
|
|
|
|
|
def all_tested_platforms(test_runs):
|
|
# extract list of all tested platforms from test_runs table
|
|
platforms = set()
|
|
for kernel in test_runs:
|
|
platforms.update(set(test_runs[kernel].keys()))
|
|
return sorted(platforms)
|
|
|
|
|
|
def divide_twoway_testruns(test_runs, platform):
|
|
# partition all twoway runs based on name of antagonist progs
|
|
twoway_runs = {}
|
|
antagonists = set()
|
|
for kernel in test_runs:
|
|
runs = {}
|
|
for testrunx in test_runs[kernel].get(platform, []):
|
|
antag = get_antag(testrunx)
|
|
if antag is not None:
|
|
runs.setdefault(antag, []).append(testrunx)
|
|
antagonists.add(antag)
|
|
twoway_runs[kernel] = runs
|
|
return twoway_runs, sorted(antagonists)
|
|
|
|
|
|
def collect_raw_scores(runs, metric):
|
|
# get unscaled scores of test runs for 1 test on certain jobs
|
|
# arrange them by platform type
|
|
platform_scores = {} # platform --> list of perf scores
|
|
for platform in runs:
|
|
vals = perf.get_metric_at_point(runs[platform], metric)
|
|
if vals:
|
|
platform_scores[platform] = vals
|
|
return platform_scores
|
|
|
|
|
|
def collect_scaled_scores(metric, test_runs, regressed_platforms, relative):
|
|
# get scores of test runs for 1 test on some kernels and platforms
|
|
# optionally make relative to oldest (?) kernel on that platform
|
|
# arrange by plotline (ie platform) for gnuplot
|
|
plot_data = {} # platform --> (kernel --> list of perf scores)
|
|
baseline = {}
|
|
for kernel in sorted(test_runs.keys()):
|
|
for platform in test_runs[kernel]:
|
|
if not (regressed_platforms is None or
|
|
platform in regressed_platforms):
|
|
continue # delete results for uninteresting platforms
|
|
vals = perf.get_metric_at_point(test_runs[kernel][platform],
|
|
metric)
|
|
if vals:
|
|
if relative:
|
|
if platform not in baseline:
|
|
baseline[platform], std = plotgraph.avg_dev(vals)
|
|
vals = [v/baseline[platform] for v in vals]
|
|
pdp = plot_data.setdefault(platform, {})
|
|
pdp.setdefault(kernel, []).extend(vals)
|
|
return plot_data
|
|
|
|
|
|
def collect_twoway_scores(metric, antagonists, twoway_runs, relative):
|
|
alone = ''
|
|
plot_data = {}
|
|
for kernel in twoway_runs:
|
|
for test2 in antagonists:
|
|
runs = twoway_runs[kernel].get(test2, [])
|
|
vals = perf.get_metric_at_point(runs, metric)
|
|
plot_data.setdefault(test2, {})
|
|
if vals:
|
|
plot_data[test2][kernel] = vals
|
|
if relative:
|
|
vals = plot_data[alone].get(kernel, [])
|
|
if vals:
|
|
baseline = perf.average(vals)
|
|
for test2 in antagonists:
|
|
vals = plot_data[test2].get(kernel, [])
|
|
vals = [val/baseline for val in vals]
|
|
if vals:
|
|
plot_data[test2][kernel] = vals
|
|
else:
|
|
for test2 in antagonists:
|
|
if kernel in plot_data[test2]:
|
|
del plot_data[test2][kernel]
|
|
return plot_data
|
|
|
|
|
|
def find_regressions(kernels, test_runs, metric):
|
|
# A test is regressed on some platform if its latest results are
|
|
# definitely lower than on the reference kernel.
|
|
# Runs for the latest kernel may be underway and incomplete.
|
|
# In that case, selectively use next-latest kernel.
|
|
# TODO: the next-latest method hurts if latest run is not sorted last,
|
|
# or if there are several dev threads
|
|
ref = kernels[0]
|
|
latest = kernels[-1]
|
|
prev = kernels[-2:][0]
|
|
scores = {} # kernel --> (platform --> list of perf scores)
|
|
for k in [ref, prev, latest]:
|
|
if k in test_runs:
|
|
scores[k] = collect_raw_scores(test_runs[k], metric)
|
|
regressed_platforms = []
|
|
for platform in scores[ref]:
|
|
if latest in scores and platform in scores[latest]:
|
|
k = latest
|
|
elif prev in scores and platform in scores[prev]:
|
|
k = prev
|
|
else: # perhaps due to decay of test machines
|
|
k = ref # no regression info avail
|
|
ref_avg, ref_std = plotgraph.avg_dev(scores[ref][platform])
|
|
avg, std = plotgraph.avg_dev(scores[ k ][platform])
|
|
if avg+std < ref_avg-ref_std:
|
|
regressed_platforms.append(platform)
|
|
return sorted(regressed_platforms)
|
|
|
|
|
|
def get_testrun_context(testrun):
|
|
cmd = ( 'select tko_jobs.label, tko_jobs.tag, tko_tests.subdir,'
|
|
' tko_tests.started_time'
|
|
' from tko_jobs, tko_tests'
|
|
' where tko_jobs.job_idx = tko_tests.job_idx'
|
|
' and tko_tests.test_idx = %d' % testrun )
|
|
nrows = perf.db_cur.execute(cmd)
|
|
assert nrows == 1
|
|
row = perf.db_cur.fetchone()
|
|
row = [row[0], row[1], row[2], row[3].strftime('%m/%d/%y %H:%M')]
|
|
return row
|
|
|
|
|
|
def html_top():
|
|
print "Content-Type: text/html\n\n<html><body>"
|
|
|
|
|
|
def abs_rel_link(myurl, passthru):
|
|
# link redraws current page with opposite absolute/relative choice
|
|
mod_passthru = passthru[:]
|
|
if 'absolute' in passthru:
|
|
mod_passthru.remove('absolute')
|
|
opposite = 'relative'
|
|
else:
|
|
mod_passthru.append('absolute')
|
|
opposite = 'absolute'
|
|
url = '%s?%s' % (myurl, '&'.join(mod_passthru))
|
|
return "<a href='%s'> %s </a>" % (url, opposite)
|
|
|
|
|
|
def table_1_metric_all_kernels(plot_data, columns, column_argname,
|
|
kernels, kernel_dates,
|
|
myurl, filtered_passthru):
|
|
# generate html table of graph's numbers
|
|
# for 1 benchmark metric over all kernels (rows),
|
|
# over various platforms or various antagonists etc (cols).
|
|
ref_thresholds = {}
|
|
print "<table border=1 cellpadding=3 cellspacing=0>"
|
|
print "<tr> <td><b> Kernel </b></td>",
|
|
for label in columns:
|
|
if not label and column_argname == 'antag':
|
|
label = 'no antag'
|
|
print "<td><b>", label.replace('_', '<br>_'), "</b></td>"
|
|
print "</tr>"
|
|
for kernel in kernels:
|
|
print "<tr> <td><b>", kernel, "</b>",
|
|
if kernel in kernel_dates:
|
|
print "<br><small>", kernel_dates[kernel], "</small>"
|
|
print "</td>"
|
|
for col in columns:
|
|
print "<td",
|
|
vals = plot_data[col].get(kernel, [])
|
|
if not vals:
|
|
print "> ?",
|
|
else:
|
|
(avg, std_dev) = plotgraph.avg_dev(vals)
|
|
if col not in ref_thresholds:
|
|
ref_thresholds[col] = avg - std_dev
|
|
if avg+std_dev < ref_thresholds[col]:
|
|
print "bgcolor=pink",
|
|
print "> ",
|
|
args = filtered_passthru[:]
|
|
perf.append_cgi_args(args,
|
|
{column_argname:col, 'kernel':kernel})
|
|
print "<a href='%s?%s&runs&attrs'>" % (myurl,
|
|
'&'.join(args))
|
|
print "<b>%.4g</b>" % avg, "</a><br>",
|
|
print " <small> %dr </small>" % len(vals),
|
|
print " <small> %.3g </small>" % std_dev,
|
|
print "</td>"
|
|
print "</tr>\n"
|
|
print "</table>"
|
|
print "<p> <b>Bold value:</b> Average of this metric, then <br>"
|
|
print "number of good test runs, then standard deviation of those runs"
|
|
print "<br> Pink if regressed from reference kernel"
|
|
|
|
|
|
def table_all_metrics_1_platform(test_runs, platform, relative):
|
|
# TODO: show std dev in cells
|
|
# can't mark regressions, since some metrics improve downwards
|
|
kernels = perf.sort_kernels(test_runs.keys())
|
|
scores = {}
|
|
attrs = set()
|
|
for kernel in kernels:
|
|
testruns = test_runs[kernel].get(platform, [])
|
|
if testruns:
|
|
d = perf.collect_all_metrics_scores(testruns)
|
|
scores[kernel] = d
|
|
attrs.update(set(d.keys()))
|
|
else:
|
|
print "No runs completed on", kernel, "<br>"
|
|
attrs = sorted(list(attrs))[:100]
|
|
|
|
print "<table border=1 cellpadding=4 cellspacing=0>"
|
|
print "<tr><td> Metric </td>"
|
|
for kernel in kernels:
|
|
kernel = kernel.replace("_", "_<br>")
|
|
print "<td>", kernel, "</td>"
|
|
print "</tr>"
|
|
for attr in attrs:
|
|
print "<tr>"
|
|
print "<td>", attr, "</td>"
|
|
baseline = None
|
|
for kernel in kernels:
|
|
print "<td>",
|
|
if kernel in scores and attr in scores[kernel]:
|
|
(avg, dev) = plotgraph.avg_dev(scores[kernel][attr])
|
|
if baseline and relative:
|
|
percent = (avg/baseline - 1)*100
|
|
print "%+.1f%%" % percent,
|
|
else:
|
|
baseline = avg
|
|
print "%.4g" % avg,
|
|
else:
|
|
print "?"
|
|
print "</td>"
|
|
print "</tr>"
|
|
print "</table>"
|
|
|
|
|
|
def table_variants_all_tests(plot_data, columns, colkeys, benchmarks,
|
|
myurl, filtered_passthru):
|
|
# generate html table of graph's numbers
|
|
# for primary metric over all benchmarks (rows),
|
|
# on one platform and one kernel,
|
|
# over various combos of test run attribute constraints (cols).
|
|
ref_thresholds = {}
|
|
print "<table border=1 cellpadding=3 cellspacing=0>"
|
|
print "<tr> <td><b> Benchmark </b></td>",
|
|
for col in columns:
|
|
print "<td><b>", colkeys[col].replace(',', ',<br>'), "</b></td>"
|
|
print "</tr>"
|
|
for benchmark in benchmarks:
|
|
print "<tr> <td><b>", benchmark, "</b></td>"
|
|
for col in columns:
|
|
print "<td>",
|
|
vals = plot_data[col].get(benchmark, [])
|
|
if not vals:
|
|
print "?",
|
|
else:
|
|
(avg, std_dev) = plotgraph.avg_dev(vals)
|
|
args = filtered_passthru[:]
|
|
perf.append_cgi_args(args, {'test':benchmark})
|
|
for keyval in colkeys[col].split(','):
|
|
key, val = keyval.split('=', 1)
|
|
perf.append_cgi_args(args, {key:val})
|
|
print "<a href='%s?%s&runs&attrs'>" % (myurl,
|
|
'&'.join(args))
|
|
print "<b>%.4g</b>" % avg, "</a><br>",
|
|
print " <small> %dr </small>" % len(vals),
|
|
print " <small> %.3g </small>" % std_dev,
|
|
print "</td>"
|
|
print "</tr>\n"
|
|
print "</table>"
|
|
print "<p> <b>Bold value:</b> Average of this metric, then <br>"
|
|
print "number of good test runs, then standard deviation of those runs"
|
|
|
|
|
|
def table_testrun_details(runs, metric, tko_server, show_attrs):
|
|
print "<table border=1 cellpadding=4 cellspacing=0>"
|
|
print "<tr><td> %s metric </td>" % metric
|
|
print "<td> Job label </td> <td> Job tag </td> <td> Run results </td>"
|
|
print "<td> Started_time </td>"
|
|
if show_attrs:
|
|
print "<td> Test attributes </td>"
|
|
print "</tr>\n"
|
|
|
|
for testrunx in runs:
|
|
print "<tr> <td>",
|
|
vals = perf.get_metric_at_point([testrunx], metric)
|
|
for v in vals:
|
|
print "%.4g " % v,
|
|
print "</td>"
|
|
row = get_testrun_context(testrunx)
|
|
row[2] = ( "<a href='//%s/results/%s/%s/results/keyval'> %s </a>"
|
|
% (tko_server, row[1], row[2], row[2]) )
|
|
for v in row:
|
|
print "<td> %s </td>" % v
|
|
if show_attrs:
|
|
attrs = get_test_attributes(testrunx)
|
|
print "<td>",
|
|
for attr in sorted(attrs.keys()):
|
|
if attr == "sysinfo-cmdline": continue
|
|
if attr[:4] == "svs-": continue
|
|
val = attrs[attr]
|
|
if len(val) > 40:
|
|
val = val[:40-3] + "..."
|
|
print "%s=%s " % (attr, val)
|
|
print "</td>"
|
|
print "</tr>\n"
|
|
print "</table>"
|
|
|
|
|
|
def overview_thumb(test, metric, myurl, passthru):
|
|
pass_ = passthru + ['test=%s' % test]
|
|
if metric:
|
|
pass_ += ['metric=%s' % metric]
|
|
pass_ = '&'.join(pass_)
|
|
print "<a href='%s?%s&table'>" % (myurl, pass_)
|
|
print " <img src='%s?%s&size=450,500'> </a>" % (myurl, pass_)
|
|
# embedded graphs fit 3 across on 1400x1050 laptop
|
|
|
|
|
|
def graph_1_test(title, metric, plot_data, line_argname, lines,
|
|
kernel_legend, relative, size, dark=False):
|
|
# generate graph image for one benchmark, showing avg and
|
|
# std dev of one metric, over various kernels (X columns),
|
|
# over various platforms or antagonists etc (graphed lines)
|
|
xlegend = kernel_legend
|
|
ylegend = metric.capitalize()
|
|
if relative:
|
|
ylegend += ', Relative'
|
|
ymin = 0.8
|
|
else:
|
|
ymin = None
|
|
if len(lines) > 1:
|
|
keytitle = line_argname.capitalize() + ':'
|
|
else:
|
|
keytitle = ''
|
|
graph = plotgraph.gnuplot(title, xlegend, ylegend, size=size,
|
|
xsort=perf.sort_kernels, keytitle=keytitle)
|
|
for line in lines:
|
|
label = line
|
|
if not label and line_argname == 'antag':
|
|
label = 'no antag'
|
|
graph.add_dataset(label, plot_data[line])
|
|
graph.plot(cgi_header=True, ymin=ymin, dark=dark)
|
|
|
|
|
|
def graph_variants_all_tests(title, plot_data, linekeys, size, dark):
|
|
# generate graph image showing all benchmarks
|
|
# on one platform and one kernel,
|
|
# over various combos of test run attribute constraints (lines).
|
|
xlegend = "Benchmark"
|
|
ylegend = "Relative Perf"
|
|
graph = plotgraph.gnuplot(title, xlegend, ylegend, size=size)
|
|
for i in linekeys:
|
|
graph.add_dataset(linekeys[i], plot_data[i])
|
|
graph.plot(cgi_header=True, dark=dark, ymin=0.8)
|
|
|
|
|
|
class generate_views(object):
|
|
|
|
|
|
def __init__(self, kernel_legend, benchmarks, test_group,
|
|
site_benchmark_metrics, tko_server,
|
|
jobs_selector, no_antag):
|
|
self.kernel_legend = kernel_legend
|
|
self.benchmarks = benchmarks
|
|
self.test_group = test_group
|
|
self.tko_server = tko_server
|
|
self.jobs_selector = jobs_selector
|
|
self.no_antag = no_antag
|
|
|
|
cgitb.enable()
|
|
test, antagonists = self.parse_most_cgi_args()
|
|
|
|
perf.init(tko_server=tko_server)
|
|
for b in site_benchmark_metrics:
|
|
perf.add_benchmark_main_metric(b, site_benchmark_metrics[b])
|
|
|
|
self.test_runs = {} # kernel --> (platform --> list of test runs)
|
|
self.job_table = {} # kernel id --> list of job idxs
|
|
self.kernel_dates = {} # kernel id --> date of nightly test
|
|
|
|
vary = self.cgiform.getlist('vary')
|
|
if vary:
|
|
platform = self.platforms_filter[0]
|
|
self.analyze_variants_all_tests_1_platform(platform, vary)
|
|
elif test:
|
|
self.analyze_1_test(test, antagonists)
|
|
else:
|
|
self.overview_page_all_tests(self.benchmarks, antagonists)
|
|
|
|
|
|
def collect_all_testruns(self, trimmed_kernels, test):
|
|
# get test_runs run #s for 1 test on some kernels and platforms
|
|
for kernel in trimmed_kernels:
|
|
runs = collect_testruns(self.job_table[kernel], test,
|
|
self.test_attributes, self.platforms_filter,
|
|
'by_hosts' in self.toggles, self.no_antag)
|
|
if runs:
|
|
self.test_runs[kernel] = runs
|
|
|
|
|
|
def table_for_graph_1_test(self, title, metric, plot_data,
|
|
column_argname, columns, filtered_passthru):
|
|
# generate detailed html page with 1 graph and corresp numbers
|
|
# for 1 benchmark metric over all kernels (rows),
|
|
# over various platforms or various antagonists etc (cols).
|
|
html_top()
|
|
print '<h3> %s </h3>' % title
|
|
print ('%s, machine group %s on //%s server <br>' %
|
|
(self.kernel_legend, self.test_group, self.tko_server))
|
|
if self.test_tag:
|
|
print '%s test script series <br>' % self.test_tag[1:]
|
|
|
|
print "<img src='%s?%s'>" % (self.myurl, '&'.join(self.passthru))
|
|
|
|
link = abs_rel_link(self.myurl, self.passthru+['table'])
|
|
print "<p><p> <h4> Redraw this with %s performance? </h4>" % link
|
|
|
|
heading = "%s, %s metric" % (title, metric)
|
|
if self.relative:
|
|
heading += ", relative"
|
|
print "<p><p> <h3> %s: </h3>" % heading
|
|
table_1_metric_all_kernels(plot_data, columns, column_argname,
|
|
self.kernels, self.kernel_dates,
|
|
self.myurl, filtered_passthru)
|
|
print "</body></html>"
|
|
|
|
|
|
def graph_1_test_all_platforms(self, test, metric, platforms, plot_data):
|
|
# generate graph image for one benchmark
|
|
title = test.capitalize()
|
|
if 'regress' in self.toggles:
|
|
title += ' Regressions'
|
|
if 'table' in self.cgiform:
|
|
self.table_for_graph_1_test(title, metric, plot_data,
|
|
'platforms', platforms,
|
|
filtered_passthru=self.passthru)
|
|
else:
|
|
graph_1_test(title, metric, plot_data, 'platforms', platforms,
|
|
self.kernel_legend, self.relative,
|
|
self.size, 'dark' in self.toggles)
|
|
|
|
|
|
def testrun_details(self, title, runs, metric):
|
|
html_top()
|
|
print '<h3> %s </h3>' % title
|
|
print ('%s, machine group %s on //%s server' %
|
|
(self.kernel_legend, self.test_group, self.tko_server))
|
|
if self.test_tag:
|
|
print '<br> %s test script series' % self.test_tag[1:]
|
|
print '<p>'
|
|
table_testrun_details(runs, metric,
|
|
self.tko_server, 'attrs' in self.cgiform)
|
|
print "</body></html>"
|
|
|
|
|
|
def testrun_details_for_1_test_kernel_platform(self, test,
|
|
metric, platform):
|
|
default_kernel = min(self.test_runs.keys())
|
|
kernel = self.cgiform.getvalue('kernel', default_kernel)
|
|
title = '%s on %s using %s' % (test.capitalize(), platform, kernel)
|
|
runs = self.test_runs[kernel].get(platform, [])
|
|
self.testrun_details(title, runs, metric)
|
|
|
|
|
|
def analyze_1_metric_all_platforms(self, test, metric):
|
|
if 'regress' in self.toggles:
|
|
regressed_platforms = find_regressions(self.kernels, self.test_runs,
|
|
metric)
|
|
else:
|
|
regressed_platforms = None
|
|
plot_data = collect_scaled_scores(metric, self.test_runs,
|
|
regressed_platforms, self.relative)
|
|
platforms = sorted(plot_data.keys())
|
|
if not plot_data:
|
|
html_top()
|
|
print 'No runs'
|
|
elif 'runs' in self.cgiform:
|
|
self.testrun_details_for_1_test_kernel_platform(test, metric,
|
|
platforms[0])
|
|
else:
|
|
self.graph_1_test_all_platforms(test, metric, platforms, plot_data)
|
|
|
|
|
|
def analyze_all_metrics_1_platform(self, test, platform):
|
|
# TODO: show #runs in header
|
|
html_top()
|
|
heading = "%s %s:  %s" % (self.test_group, self.kernel_legend,
|
|
test.capitalize())
|
|
print "<h2> %s </h2>" % heading
|
|
print "platform=%s <br>" % platform
|
|
for attr in self.test_attributes:
|
|
print "%s=%s " % (attr, self.test_attributes[attr])
|
|
print "<p>"
|
|
table_all_metrics_1_platform(self.test_runs, platform, self.relative)
|
|
print "</body></html>"
|
|
|
|
|
|
def table_for_variants_all_tests(self, title, plot_data, colkeys, columns,
|
|
filtered_passthru, test_tag):
|
|
# generate detailed html page with 1 graph and corresp numbers
|
|
# for primary metric over all benchmarks (rows),
|
|
# on one platform and one kernel,
|
|
# over various combos of test run attribute constraints (cols).
|
|
html_top()
|
|
print '<h3> %s </h3>' % title
|
|
print ('%s, machine group %s on //%s server <br>' %
|
|
(self.kernel_legend, self.test_group, self.tko_server))
|
|
if test_tag:
|
|
print '%s test script series <br>' % test_tag[1:]
|
|
|
|
varies = ['vary='+colkeys[col] for col in columns]
|
|
print "<img src='%s?%s'>" % (self.myurl, '&'.join(self.passthru+varies))
|
|
|
|
print "<p><p> <h3> %s: </h3>" % title
|
|
table_variants_all_tests(plot_data, columns, colkeys, self.benchmarks,
|
|
self.myurl, filtered_passthru)
|
|
print "</body></html>"
|
|
|
|
|
|
def analyze_variants_all_tests_1_platform(self, platform, vary):
|
|
# generate one graph image for results of all benchmarks
|
|
# on one platform and one kernel, comparing effects of
|
|
# two or more combos of kernel options (test run attributes)
|
|
# (numa_fake,stale_page,kswapd_merge,sched_idle, etc)
|
|
kernel = self.cgiform.getvalue('kernel', 'some_kernel')
|
|
self.passthru.append('kernel=%s' % kernel)
|
|
|
|
# two or more vary_groups, one for each plotted line,
|
|
# each group begins with vary= and ends with next &
|
|
# each group has comma-separated list of test attribute key=val pairs
|
|
# eg vary=keyval1,keyval2&vary=keyval3,keyval4
|
|
vary_groups = [dict(pair.split('=',1) for pair
|
|
in vary_group.split(','))
|
|
for vary_group in vary]
|
|
|
|
test = self.benchmarks[0] # pick any test in all jobs
|
|
kernels, test_tag = self.jobs_selector(test, self.job_table,
|
|
self.kernel_dates)
|
|
|
|
linekeys = {}
|
|
plot_data = {}
|
|
baselines = {}
|
|
for i, vary_group in enumerate(vary_groups):
|
|
group_attributes = self.test_attributes.copy()
|
|
group_attributes.update(vary_group)
|
|
linekey = ','.join('%s=%s' % (attr, vary_group[attr])
|
|
for attr in vary_group)
|
|
linekeys[i] = linekey
|
|
data = {}
|
|
for benchmark in self.benchmarks:
|
|
metric = perf.benchmark_main_metric(benchmark)
|
|
runs = collect_testruns(self.job_table[kernel],
|
|
benchmark+test_tag,
|
|
group_attributes,
|
|
self.platforms_filter,
|
|
'by_hosts' in self.toggles,
|
|
self.no_antag)
|
|
vals = []
|
|
for testrunx in runs[platform]:
|
|
vals += perf.get_metric_at_point([testrunx], metric)
|
|
if vals:
|
|
if benchmark not in baselines:
|
|
baselines[benchmark], stddev = plotgraph.avg_dev(vals)
|
|
vals = [val/baselines[benchmark] for val in vals]
|
|
data[benchmark] = vals
|
|
plot_data[i] = data
|
|
|
|
title = "%s on %s" % (kernel, platform)
|
|
for attr in self.test_attributes:
|
|
title += ', %s=%s' % (attr, self.test_attributes[attr])
|
|
if 'table' in self.cgiform:
|
|
self.table_for_variants_all_tests(title, plot_data, linekeys,
|
|
range(len(linekeys)),
|
|
filtered_passthru=self.passthru,
|
|
test_tag=test_tag)
|
|
else:
|
|
graph_variants_all_tests(title, plot_data, linekeys,
|
|
self.size, 'dark' in self.toggles)
|
|
|
|
|
|
def graph_twoway_antagonists_1_test_1_platform(
|
|
self, test, metric, platform, antagonists, twoway_runs):
|
|
# generate graph of one benchmark's performance paired with
|
|
# various antagonists, with one plotted line per antagonist,
|
|
# over most kernels (X axis), all on one machine type
|
|
# performance is relative to the no-antag baseline case
|
|
plot_data = collect_twoway_scores(metric, antagonists,
|
|
twoway_runs, self.relative)
|
|
title = "%s vs. an Antagonist on %s:" % (test.capitalize(), platform)
|
|
if 'table' in self.cgiform:
|
|
filtered_passthru = [arg for arg in self.passthru
|
|
if not arg.startswith('antag=')]
|
|
self.table_for_graph_1_test(title, metric, plot_data,
|
|
'antag', antagonists,
|
|
filtered_passthru=filtered_passthru)
|
|
else:
|
|
graph_1_test(title, metric, plot_data, 'antag', antagonists,
|
|
self.kernel_legend, self.relative,
|
|
self.size, 'dark' in self.toggles)
|
|
|
|
|
|
def testrun_details_for_twoway_test(self, test, metric, platform,
|
|
antagonist, twoway_runs):
|
|
default_kernel = min(twoway_runs.keys())
|
|
kernel = self.cgiform.getvalue('kernel', default_kernel)
|
|
title = '%s vs. Antagonist %s on %s using %s' % (
|
|
test.capitalize(), antagonist.capitalize(), platform, kernel)
|
|
runs = twoway_runs[kernel].get(antagonist, [])
|
|
self.testrun_details(title, runs, metric)
|
|
|
|
|
|
def analyze_twoway_antagonists_1_test_1_platform(
|
|
self, test, metric, platform, antagonists):
|
|
twoway_runs, all_antagonists = divide_twoway_testruns(self.test_runs,
|
|
platform)
|
|
if antagonists == ['*']:
|
|
antagonists = all_antagonists
|
|
if not twoway_runs:
|
|
html_top()
|
|
print 'No runs'
|
|
elif 'runs' in self.cgiform:
|
|
self.testrun_details_for_twoway_test(
|
|
test, metric, platform, antagonists[0], twoway_runs)
|
|
else:
|
|
self.graph_twoway_antagonists_1_test_1_platform(
|
|
test, metric, platform, antagonists, twoway_runs)
|
|
|
|
|
|
def get_twoway_default_platform(self):
|
|
if self.platforms_filter:
|
|
return self.platforms_filter[0]
|
|
test = 'unixbench'
|
|
kernels, test_tag = self.jobs_selector(test, self.job_table,
|
|
self.kernel_dates)
|
|
self.collect_all_testruns(kernels, test+test_tag)
|
|
return all_tested_platforms(self.test_runs)[0]
|
|
|
|
|
|
def overview_page_all_tests(self, benchmarks, antagonists):
|
|
# generate overview html page with small graphs for each benchmark
|
|
# linking to detailed html page for that benchmark
|
|
# recursively link to this same cgi to generate each image
|
|
html_top()
|
|
if antagonists is not None:
|
|
heading = ('Twoway Container Isolation using %s on %s' %
|
|
(self.kernel_legend, self.get_twoway_default_platform()))
|
|
else:
|
|
heading = '%s, %s Benchmarks' % (self.kernel_legend,
|
|
self.test_group)
|
|
if 'regress' in self.toggles:
|
|
heading += ", Regressions Only"
|
|
print "<h3> %s </h3>" % heading
|
|
for test in benchmarks:
|
|
overview_thumb(test, '', self.myurl, self.passthru)
|
|
if test == 'unixbench':
|
|
overview_thumb('unixbench', 'Process_creation',
|
|
self.myurl, self.passthru)
|
|
|
|
link = abs_rel_link(self.myurl, self.passthru)
|
|
print "<p><p> <h4> Redraw this with %s performance? </h4>" % link
|
|
print "</body></html>"
|
|
|
|
|
|
def analyze_1_test(self, test, antagonists):
|
|
self.passthru.append('test=%s' % test)
|
|
metric = self.cgiform.getvalue('metric', '')
|
|
if metric:
|
|
self.passthru.append('metric=%s' % metric)
|
|
else:
|
|
metric = perf.benchmark_main_metric(test)
|
|
assert metric, "no default metric for test %s" % test
|
|
self.kernels, self.test_tag = self.jobs_selector(test, self.job_table,
|
|
self.kernel_dates)
|
|
self.collect_all_testruns(self.kernels, test+self.test_tag)
|
|
if not self.platforms_filter and (metric == '*' or
|
|
antagonists is not None):
|
|
# choose default platform
|
|
self.platforms_filter = all_tested_platforms(self.test_runs)[0:1]
|
|
self.passthru.append('platforms=%s' %
|
|
','.join(self.platforms_filter))
|
|
if antagonists is not None:
|
|
antagonists = antagonists.split(',')
|
|
if len(antagonists) == 1 and antagonists != ['*']:
|
|
self.relative = False
|
|
self.analyze_twoway_antagonists_1_test_1_platform(
|
|
test, metric, self.platforms_filter[0], antagonists)
|
|
elif metric == '*':
|
|
platform = self.platforms_filter[0]
|
|
self.analyze_all_metrics_1_platform(test, platform)
|
|
else:
|
|
self.analyze_1_metric_all_platforms(test, metric)
|
|
|
|
|
|
def parse_most_cgi_args(self):
|
|
self.myurl = os.path.basename(sys.argv[0])
|
|
self.cgiform = cgi.FieldStorage(keep_blank_values=True)
|
|
self.size = self.cgiform.getvalue('size', '1200,850')
|
|
all_toggles = set(('absolute', 'regress', 'dark', 'by_hosts'))
|
|
self.toggles = set(tog for tog in all_toggles if tog in self.cgiform)
|
|
platforms = self.cgiform.getvalue('platforms', '')
|
|
if '.' in platforms:
|
|
self.toggles.add('by_hosts')
|
|
self.passthru = list(self.toggles)
|
|
self.relative = 'absolute' not in self.toggles
|
|
if platforms:
|
|
self.passthru.append('platforms=%s' % platforms)
|
|
self.platforms_filter = platforms.split(',')
|
|
else:
|
|
self.platforms_filter = []
|
|
self.test_attributes = perf.parse_test_attr_args(self.cgiform)
|
|
perf.append_cgi_args(self.passthru, self.test_attributes)
|
|
test = self.cgiform.getvalue('test', '')
|
|
if 'antag' in self.cgiform:
|
|
antagonists = ','.join(self.cgiform.getlist('antag'))
|
|
# antag=*
|
|
# or antag=test1,test2,test3,...
|
|
# or antag=test1&antag=test2&...
|
|
# testN is empty for solo case of no antagonist
|
|
self.passthru.append('antag=%s' % antagonists)
|
|
else:
|
|
antagonists = None # not same as ''
|
|
return test, antagonists
|