861 lines
32 KiB
Python
861 lines
32 KiB
Python
# pylint: disable-msg=C0111
|
|
|
|
import base64, os, tempfile, pickle, datetime, django.db
|
|
import os.path, getpass
|
|
from math import sqrt
|
|
|
|
# When you import matplotlib, it tries to write some temp files for better
|
|
# performance, and it does that to the directory in MPLCONFIGDIR, or, if that
|
|
# doesn't exist, the home directory. Problem is, the home directory is not
|
|
# writable when running under Apache, and matplotlib's not smart enough to
|
|
# handle that. It does appear smart enough to handle the files going
|
|
# away after they are written, though.
|
|
|
|
temp_dir = os.path.join(tempfile.gettempdir(),
|
|
'.matplotlib-%s' % getpass.getuser())
|
|
if not os.path.exists(temp_dir):
|
|
os.mkdir(temp_dir)
|
|
os.environ['MPLCONFIGDIR'] = temp_dir
|
|
|
|
try:
|
|
import matplotlib
|
|
matplotlib.use('Agg')
|
|
import matplotlib.figure, matplotlib.backends.backend_agg
|
|
import StringIO, colorsys, PIL.Image, PIL.ImageChops
|
|
except ImportError:
|
|
# Do nothing, in case this is part of a unit test, so the unit test
|
|
# can proceed.
|
|
pass
|
|
|
|
from autotest_lib.frontend.afe import readonly_connection
|
|
from autotest_lib.frontend.afe.model_logic import ValidationError
|
|
from json import encoder
|
|
from autotest_lib.client.common_lib import global_config
|
|
from autotest_lib.frontend.tko import models, tko_rpc_utils
|
|
|
|
_FIGURE_DPI = 100
|
|
_FIGURE_WIDTH_IN = 10
|
|
_FIGURE_BOTTOM_PADDING_IN = 2 # for x-axis labels
|
|
|
|
_SINGLE_PLOT_HEIGHT = 6
|
|
_MULTIPLE_PLOT_HEIGHT_PER_PLOT = 4
|
|
|
|
_MULTIPLE_PLOT_MARKER_TYPE = 'o'
|
|
_MULTIPLE_PLOT_MARKER_SIZE = 4
|
|
_SINGLE_PLOT_STYLE = 'bs-' # blue squares with lines connecting
|
|
_SINGLE_PLOT_ERROR_BAR_COLOR = 'r'
|
|
|
|
_LEGEND_FONT_SIZE = 'xx-small'
|
|
_LEGEND_HANDLE_LENGTH = 0.03
|
|
_LEGEND_NUM_POINTS = 3
|
|
_LEGEND_MARKER_TYPE = 'o'
|
|
|
|
_LINE_XTICK_LABELS_SIZE = 'x-small'
|
|
_BAR_XTICK_LABELS_SIZE = 8
|
|
|
|
_json_encoder = encoder.JSONEncoder()
|
|
|
|
class NoDataError(Exception):
|
|
"""\
|
|
Exception to raise if the graphing query returned an empty resultset.
|
|
"""
|
|
|
|
|
|
def _colors(n):
|
|
"""\
|
|
Generator function for creating n colors. The return value is a tuple
|
|
representing the RGB of the color.
|
|
"""
|
|
for i in xrange(n):
|
|
yield colorsys.hsv_to_rgb(float(i) / n, 1.0, 1.0)
|
|
|
|
|
|
def _resort(kernel_labels, list_to_sort):
|
|
"""\
|
|
Resorts a list, using a list of kernel strings as the keys. Returns the
|
|
resorted list.
|
|
"""
|
|
|
|
labels = [tko_rpc_utils.KernelString(label) for label in kernel_labels]
|
|
resorted_pairs = sorted(zip(labels, list_to_sort))
|
|
|
|
# We only want the resorted list; we are not interested in the kernel
|
|
# strings.
|
|
return [pair[1] for pair in resorted_pairs]
|
|
|
|
|
|
def _quote(string):
|
|
return "%s%s%s" % ("'", string.replace("'", r"\'"), "'")
|
|
|
|
|
|
_HTML_TEMPLATE = """\
|
|
<html><head></head><body>
|
|
<img src="data:image/png;base64,%s" usemap="#%s"
|
|
border="0" alt="graph">
|
|
<map name="%s">%s</map>
|
|
</body></html>"""
|
|
|
|
_AREA_TEMPLATE = """\
|
|
<area shape="rect" coords="%i,%i,%i,%i" title="%s"
|
|
href="#"
|
|
onclick="%s(%s); return false;">"""
|
|
|
|
|
|
class MetricsPlot(object):
|
|
def __init__(self, query_dict, plot_type, inverted_series, normalize_to,
|
|
drilldown_callback):
|
|
"""
|
|
query_dict: dictionary containing the main query and the drilldown
|
|
queries. The main query returns a row for each x value. The first
|
|
column contains the x-axis label. Subsequent columns contain data
|
|
for each series, named by the column names. A column named
|
|
'errors-<x>' will be interpreted as errors for the series named <x>.
|
|
|
|
plot_type: 'Line' or 'Bar', depending on the plot type the user wants
|
|
|
|
inverted_series: list of series that should be plotted on an inverted
|
|
y-axis
|
|
|
|
normalize_to:
|
|
None - do not normalize
|
|
'first' - normalize against the first data point
|
|
'x__%s' - normalize against the x-axis value %s
|
|
'series__%s' - normalize against the series %s
|
|
|
|
drilldown_callback: name of drilldown callback method.
|
|
"""
|
|
self.query_dict = query_dict
|
|
if plot_type == 'Line':
|
|
self.is_line = True
|
|
elif plot_type == 'Bar':
|
|
self.is_line = False
|
|
else:
|
|
raise ValidationError({'plot' : 'Plot must be either Line or Bar'})
|
|
self.plot_type = plot_type
|
|
self.inverted_series = inverted_series
|
|
self.normalize_to = normalize_to
|
|
if self.normalize_to is None:
|
|
self.normalize_to = ''
|
|
self.drilldown_callback = drilldown_callback
|
|
|
|
|
|
class QualificationHistogram(object):
|
|
def __init__(self, query, filter_string, interval, drilldown_callback):
|
|
"""
|
|
query: the main query to retrieve the pass rate information. The first
|
|
column contains the hostnames of all the machines that satisfied the
|
|
global filter. The second column (titled 'total') contains the total
|
|
number of tests that ran on that machine and satisfied the global
|
|
filter. The third column (titled 'good') contains the number of
|
|
those tests that passed on that machine.
|
|
|
|
filter_string: filter to apply to the common global filter to show the
|
|
Table View drilldown of a histogram bucket
|
|
|
|
interval: interval for each bucket. E.g., 10 means that buckets should
|
|
be 0-10%, 10%-20%, ...
|
|
|
|
"""
|
|
self.query = query
|
|
self.filter_string = filter_string
|
|
self.interval = interval
|
|
self.drilldown_callback = drilldown_callback
|
|
|
|
|
|
def _create_figure(height_inches):
|
|
"""\
|
|
Creates an instance of matplotlib.figure.Figure, given the height in inches.
|
|
Returns the figure and the height in pixels.
|
|
"""
|
|
|
|
fig = matplotlib.figure.Figure(
|
|
figsize=(_FIGURE_WIDTH_IN, height_inches + _FIGURE_BOTTOM_PADDING_IN),
|
|
dpi=_FIGURE_DPI, facecolor='white')
|
|
fig.subplots_adjust(bottom=float(_FIGURE_BOTTOM_PADDING_IN) / height_inches)
|
|
return (fig, fig.get_figheight() * _FIGURE_DPI)
|
|
|
|
|
|
def _create_line(plots, labels, plot_info):
|
|
"""\
|
|
Given all the data for the metrics, create a line plot.
|
|
|
|
plots: list of dicts containing the plot data. Each dict contains:
|
|
x: list of x-values for the plot
|
|
y: list of corresponding y-values
|
|
errors: errors for each data point, or None if no error information
|
|
available
|
|
label: plot title
|
|
labels: list of x-tick labels
|
|
plot_info: a MetricsPlot
|
|
"""
|
|
# when we're doing any kind of normalization, all series get put into a
|
|
# single plot
|
|
single = bool(plot_info.normalize_to)
|
|
|
|
area_data = []
|
|
lines = []
|
|
if single:
|
|
plot_height = _SINGLE_PLOT_HEIGHT
|
|
else:
|
|
plot_height = _MULTIPLE_PLOT_HEIGHT_PER_PLOT * len(plots)
|
|
figure, height = _create_figure(plot_height)
|
|
|
|
if single:
|
|
subplot = figure.add_subplot(1, 1, 1)
|
|
|
|
# Plot all the data
|
|
for plot_index, (plot, color) in enumerate(zip(plots, _colors(len(plots)))):
|
|
needs_invert = (plot['label'] in plot_info.inverted_series)
|
|
|
|
# Add a new subplot, if user wants multiple subplots
|
|
# Also handle axis inversion for subplots here
|
|
if not single:
|
|
subplot = figure.add_subplot(len(plots), 1, plot_index + 1)
|
|
subplot.set_title(plot['label'])
|
|
if needs_invert:
|
|
# for separate plots, just invert the y-axis
|
|
subplot.set_ylim(1, 0)
|
|
elif needs_invert:
|
|
# for a shared plot (normalized data), need to invert the y values
|
|
# manually, since all plots share a y-axis
|
|
plot['y'] = [-y for y in plot['y']]
|
|
|
|
# Plot the series
|
|
subplot.set_xticks(range(0, len(labels)))
|
|
subplot.set_xlim(-1, len(labels))
|
|
if single:
|
|
lines += subplot.plot(plot['x'], plot['y'], label=plot['label'],
|
|
marker=_MULTIPLE_PLOT_MARKER_TYPE,
|
|
markersize=_MULTIPLE_PLOT_MARKER_SIZE)
|
|
error_bar_color = lines[-1].get_color()
|
|
else:
|
|
lines += subplot.plot(plot['x'], plot['y'], _SINGLE_PLOT_STYLE,
|
|
label=plot['label'])
|
|
error_bar_color = _SINGLE_PLOT_ERROR_BAR_COLOR
|
|
if plot['errors']:
|
|
subplot.errorbar(plot['x'], plot['y'], linestyle='None',
|
|
yerr=plot['errors'], color=error_bar_color)
|
|
subplot.set_xticklabels([])
|
|
|
|
# Construct the information for the drilldowns.
|
|
# We need to do this in a separate loop so that all the data is in
|
|
# matplotlib before we start calling transform(); otherwise, it will return
|
|
# incorrect data because it hasn't finished adjusting axis limits.
|
|
for line in lines:
|
|
|
|
# Get the pixel coordinates of each point on the figure
|
|
x = line.get_xdata()
|
|
y = line.get_ydata()
|
|
label = line.get_label()
|
|
icoords = line.get_transform().transform(zip(x,y))
|
|
|
|
# Get the appropriate drilldown query
|
|
drill = plot_info.query_dict['__' + label + '__']
|
|
|
|
# Set the title attributes (hover-over tool-tips)
|
|
x_labels = [labels[x_val] for x_val in x]
|
|
titles = ['%s - %s: %f' % (label, x_label, y_val)
|
|
for x_label, y_val in zip(x_labels, y)]
|
|
|
|
# Get the appropriate parameters for the drilldown query
|
|
params = [dict(query=drill, series=line.get_label(), param=x_label)
|
|
for x_label in x_labels]
|
|
|
|
area_data += [dict(left=ix - 5, top=height - iy - 5,
|
|
right=ix + 5, bottom=height - iy + 5,
|
|
title= title,
|
|
callback=plot_info.drilldown_callback,
|
|
callback_arguments=param_dict)
|
|
for (ix, iy), title, param_dict
|
|
in zip(icoords, titles, params)]
|
|
|
|
subplot.set_xticklabels(labels, rotation=90, size=_LINE_XTICK_LABELS_SIZE)
|
|
|
|
# Show the legend if there are not multiple subplots
|
|
if single:
|
|
font_properties = matplotlib.font_manager.FontProperties(
|
|
size=_LEGEND_FONT_SIZE)
|
|
legend = figure.legend(lines, [plot['label'] for plot in plots],
|
|
prop=font_properties,
|
|
handlelen=_LEGEND_HANDLE_LENGTH,
|
|
numpoints=_LEGEND_NUM_POINTS)
|
|
# Workaround for matplotlib not keeping all line markers in the legend -
|
|
# it seems if we don't do this, matplotlib won't keep all the line
|
|
# markers in the legend.
|
|
for line in legend.get_lines():
|
|
line.set_marker(_LEGEND_MARKER_TYPE)
|
|
|
|
return (figure, area_data)
|
|
|
|
|
|
def _get_adjusted_bar(x, bar_width, series_index, num_plots):
|
|
"""\
|
|
Adjust the list 'x' to take the multiple series into account. Each series
|
|
should be shifted such that the middle series lies at the appropriate x-axis
|
|
tick with the other bars around it. For example, if we had four series
|
|
(i.e. four bars per x value), we want to shift the left edges of the bars as
|
|
such:
|
|
Bar 1: -2 * width
|
|
Bar 2: -width
|
|
Bar 3: none
|
|
Bar 4: width
|
|
"""
|
|
adjust = (-0.5 * num_plots - 1 + series_index) * bar_width
|
|
return [x_val + adjust for x_val in x]
|
|
|
|
|
|
# TODO(showard): merge much of this function with _create_line by extracting and
|
|
# parameterizing methods
|
|
def _create_bar(plots, labels, plot_info):
|
|
"""\
|
|
Given all the data for the metrics, create a line plot.
|
|
|
|
plots: list of dicts containing the plot data.
|
|
x: list of x-values for the plot
|
|
y: list of corresponding y-values
|
|
errors: errors for each data point, or None if no error information
|
|
available
|
|
label: plot title
|
|
labels: list of x-tick labels
|
|
plot_info: a MetricsPlot
|
|
"""
|
|
|
|
area_data = []
|
|
bars = []
|
|
figure, height = _create_figure(_SINGLE_PLOT_HEIGHT)
|
|
|
|
# Set up the plot
|
|
subplot = figure.add_subplot(1, 1, 1)
|
|
subplot.set_xticks(range(0, len(labels)))
|
|
subplot.set_xlim(-1, len(labels))
|
|
subplot.set_xticklabels(labels, rotation=90, size=_BAR_XTICK_LABELS_SIZE)
|
|
# draw a bold line at y=0, making it easier to tell if bars are dipping
|
|
# below the axis or not.
|
|
subplot.axhline(linewidth=2, color='black')
|
|
|
|
# width here is the width for each bar in the plot. Matplotlib default is
|
|
# 0.8.
|
|
width = 0.8 / len(plots)
|
|
|
|
# Plot the data
|
|
for plot_index, (plot, color) in enumerate(zip(plots, _colors(len(plots)))):
|
|
# Invert the y-axis if needed
|
|
if plot['label'] in plot_info.inverted_series:
|
|
plot['y'] = [-y for y in plot['y']]
|
|
|
|
adjusted_x = _get_adjusted_bar(plot['x'], width, plot_index + 1,
|
|
len(plots))
|
|
bar_data = subplot.bar(adjusted_x, plot['y'],
|
|
width=width, yerr=plot['errors'],
|
|
facecolor=color,
|
|
label=plot['label'])
|
|
bars.append(bar_data[0])
|
|
|
|
# Construct the information for the drilldowns.
|
|
# See comment in _create_line for why we need a separate loop to do this.
|
|
for plot_index, plot in enumerate(plots):
|
|
adjusted_x = _get_adjusted_bar(plot['x'], width, plot_index + 1,
|
|
len(plots))
|
|
|
|
# Let matplotlib plot the data, so that we can get the data-to-image
|
|
# coordinate transforms
|
|
line = subplot.plot(adjusted_x, plot['y'], linestyle='None')[0]
|
|
label = plot['label']
|
|
upper_left_coords = line.get_transform().transform(zip(adjusted_x,
|
|
plot['y']))
|
|
bottom_right_coords = line.get_transform().transform(
|
|
[(x + width, 0) for x in adjusted_x])
|
|
|
|
# Get the drilldown query
|
|
drill = plot_info.query_dict['__' + label + '__']
|
|
|
|
# Set the title attributes
|
|
x_labels = [labels[x] for x in plot['x']]
|
|
titles = ['%s - %s: %f' % (plot['label'], label, y)
|
|
for label, y in zip(x_labels, plot['y'])]
|
|
params = [dict(query=drill, series=plot['label'], param=x_label)
|
|
for x_label in x_labels]
|
|
area_data += [dict(left=ulx, top=height - uly,
|
|
right=brx, bottom=height - bry,
|
|
title=title,
|
|
callback=plot_info.drilldown_callback,
|
|
callback_arguments=param_dict)
|
|
for (ulx, uly), (brx, bry), title, param_dict
|
|
in zip(upper_left_coords, bottom_right_coords, titles,
|
|
params)]
|
|
|
|
figure.legend(bars, [plot['label'] for plot in plots])
|
|
return (figure, area_data)
|
|
|
|
|
|
def _normalize(data_values, data_errors, base_values, base_errors):
|
|
"""\
|
|
Normalize the data against a baseline.
|
|
|
|
data_values: y-values for the to-be-normalized data
|
|
data_errors: standard deviations for the to-be-normalized data
|
|
base_values: list of values normalize against
|
|
base_errors: list of standard deviations for those base values
|
|
"""
|
|
values = []
|
|
for value, base in zip(data_values, base_values):
|
|
try:
|
|
values.append(100 * (value - base) / base)
|
|
except ZeroDivisionError:
|
|
# Base is 0.0 so just simplify:
|
|
# If value < base: append -100.0;
|
|
# If value == base: append 0.0 (obvious); and
|
|
# If value > base: append 100.0.
|
|
values.append(100 * float(cmp(value, base)))
|
|
|
|
# Based on error for f(x,y) = 100 * (x - y) / y
|
|
if data_errors:
|
|
if not base_errors:
|
|
base_errors = [0] * len(data_errors)
|
|
errors = []
|
|
for data, error, base_value, base_error in zip(
|
|
data_values, data_errors, base_values, base_errors):
|
|
try:
|
|
errors.append(sqrt(error**2 * (100 / base_value)**2
|
|
+ base_error**2 * (100 * data / base_value**2)**2
|
|
+ error * base_error * (100 / base_value**2)**2))
|
|
except ZeroDivisionError:
|
|
# Again, base is 0.0 so do the simple thing.
|
|
errors.append(100 * abs(error))
|
|
else:
|
|
errors = None
|
|
|
|
return (values, errors)
|
|
|
|
|
|
def _create_png(figure):
|
|
"""\
|
|
Given the matplotlib figure, generate the PNG data for it.
|
|
"""
|
|
|
|
# Draw the image
|
|
canvas = matplotlib.backends.backend_agg.FigureCanvasAgg(figure)
|
|
canvas.draw()
|
|
size = canvas.get_renderer().get_canvas_width_height()
|
|
image_as_string = canvas.tostring_rgb()
|
|
image = PIL.Image.fromstring('RGB', size, image_as_string, 'raw', 'RGB', 0,
|
|
1)
|
|
image_background = PIL.Image.new(image.mode, image.size,
|
|
figure.get_facecolor())
|
|
|
|
# Crop the image to remove surrounding whitespace
|
|
non_whitespace = PIL.ImageChops.difference(image, image_background)
|
|
bounding_box = non_whitespace.getbbox()
|
|
image = image.crop(bounding_box)
|
|
|
|
image_data = StringIO.StringIO()
|
|
image.save(image_data, format='PNG')
|
|
|
|
return image_data.getvalue(), bounding_box
|
|
|
|
|
|
def _create_image_html(figure, area_data, plot_info):
|
|
"""\
|
|
Given the figure and drilldown data, construct the HTML that will render the
|
|
graph as a PNG image, and attach the image map to that image.
|
|
|
|
figure: figure containing the drawn plot(s)
|
|
area_data: list of parameters for each area of the image map. See the
|
|
definition of the template string '_AREA_TEMPLATE'
|
|
plot_info: a MetricsPlot or QualHistogram
|
|
"""
|
|
|
|
png, bbox = _create_png(figure)
|
|
|
|
# Construct the list of image map areas
|
|
areas = [_AREA_TEMPLATE %
|
|
(data['left'] - bbox[0], data['top'] - bbox[1],
|
|
data['right'] - bbox[0], data['bottom'] - bbox[1],
|
|
data['title'], data['callback'],
|
|
_json_encoder.encode(data['callback_arguments'])
|
|
.replace('"', '"'))
|
|
for data in area_data]
|
|
|
|
map_name = plot_info.drilldown_callback + '_map'
|
|
return _HTML_TEMPLATE % (base64.b64encode(png), map_name, map_name,
|
|
'\n'.join(areas))
|
|
|
|
|
|
def _find_plot_by_label(plots, label):
|
|
for index, plot in enumerate(plots):
|
|
if plot['label'] == label:
|
|
return index
|
|
raise ValueError('no plot labeled "%s" found' % label)
|
|
|
|
|
|
def _normalize_to_series(plots, base_series):
|
|
base_series_index = _find_plot_by_label(plots, base_series)
|
|
base_plot = plots[base_series_index]
|
|
base_xs = base_plot['x']
|
|
base_values = base_plot['y']
|
|
base_errors = base_plot['errors']
|
|
del plots[base_series_index]
|
|
|
|
for plot in plots:
|
|
old_xs, old_values, old_errors = plot['x'], plot['y'], plot['errors']
|
|
new_xs, new_values, new_errors = [], [], []
|
|
new_base_values, new_base_errors = [], []
|
|
# Select only points in the to-be-normalized data that have a
|
|
# corresponding baseline value
|
|
for index, x_value in enumerate(old_xs):
|
|
try:
|
|
base_index = base_xs.index(x_value)
|
|
except ValueError:
|
|
continue
|
|
|
|
new_xs.append(x_value)
|
|
new_values.append(old_values[index])
|
|
new_base_values.append(base_values[base_index])
|
|
if old_errors:
|
|
new_errors.append(old_errors[index])
|
|
new_base_errors.append(base_errors[base_index])
|
|
|
|
if not new_xs:
|
|
raise NoDataError('No normalizable data for series ' +
|
|
plot['label'])
|
|
plot['x'] = new_xs
|
|
plot['y'] = new_values
|
|
if old_errors:
|
|
plot['errors'] = new_errors
|
|
|
|
plot['y'], plot['errors'] = _normalize(plot['y'], plot['errors'],
|
|
new_base_values,
|
|
new_base_errors)
|
|
|
|
|
|
def _create_metrics_plot_helper(plot_info, extra_text=None):
|
|
"""
|
|
Create a metrics plot of the given plot data.
|
|
plot_info: a MetricsPlot object.
|
|
extra_text: text to show at the uppper-left of the graph
|
|
|
|
TODO(showard): move some/all of this logic into methods on MetricsPlot
|
|
"""
|
|
query = plot_info.query_dict['__main__']
|
|
cursor = readonly_connection.cursor()
|
|
cursor.execute(query)
|
|
|
|
if not cursor.rowcount:
|
|
raise NoDataError('query did not return any data')
|
|
rows = cursor.fetchall()
|
|
# "transpose" rows, so columns[0] is all the values from the first column,
|
|
# etc.
|
|
columns = zip(*rows)
|
|
|
|
plots = []
|
|
labels = [str(label) for label in columns[0]]
|
|
needs_resort = (cursor.description[0][0] == 'kernel')
|
|
|
|
# Collect all the data for the plot
|
|
col = 1
|
|
while col < len(cursor.description):
|
|
y = columns[col]
|
|
label = cursor.description[col][0]
|
|
col += 1
|
|
if (col < len(cursor.description) and
|
|
'errors-' + label == cursor.description[col][0]):
|
|
errors = columns[col]
|
|
col += 1
|
|
else:
|
|
errors = None
|
|
if needs_resort:
|
|
y = _resort(labels, y)
|
|
if errors:
|
|
errors = _resort(labels, errors)
|
|
|
|
x = [index for index, value in enumerate(y) if value is not None]
|
|
if not x:
|
|
raise NoDataError('No data for series ' + label)
|
|
y = [y[i] for i in x]
|
|
if errors:
|
|
errors = [errors[i] for i in x]
|
|
plots.append({
|
|
'label': label,
|
|
'x': x,
|
|
'y': y,
|
|
'errors': errors
|
|
})
|
|
|
|
if needs_resort:
|
|
labels = _resort(labels, labels)
|
|
|
|
# Normalize the data if necessary
|
|
normalize_to = plot_info.normalize_to
|
|
if normalize_to == 'first' or normalize_to.startswith('x__'):
|
|
if normalize_to != 'first':
|
|
baseline = normalize_to[3:]
|
|
try:
|
|
baseline_index = labels.index(baseline)
|
|
except ValueError:
|
|
raise ValidationError({
|
|
'Normalize' : 'Invalid baseline %s' % baseline
|
|
})
|
|
for plot in plots:
|
|
if normalize_to == 'first':
|
|
plot_index = 0
|
|
else:
|
|
try:
|
|
plot_index = plot['x'].index(baseline_index)
|
|
# if the value is not found, then we cannot normalize
|
|
except ValueError:
|
|
raise ValidationError({
|
|
'Normalize' : ('%s does not have a value for %s'
|
|
% (plot['label'], normalize_to[3:]))
|
|
})
|
|
base_values = [plot['y'][plot_index]] * len(plot['y'])
|
|
if plot['errors']:
|
|
base_errors = [plot['errors'][plot_index]] * len(plot['errors'])
|
|
plot['y'], plot['errors'] = _normalize(plot['y'], plot['errors'],
|
|
base_values,
|
|
None or base_errors)
|
|
|
|
elif normalize_to.startswith('series__'):
|
|
base_series = normalize_to[8:]
|
|
_normalize_to_series(plots, base_series)
|
|
|
|
# Call the appropriate function to draw the line or bar plot
|
|
if plot_info.is_line:
|
|
figure, area_data = _create_line(plots, labels, plot_info)
|
|
else:
|
|
figure, area_data = _create_bar(plots, labels, plot_info)
|
|
|
|
# TODO(showard): extract these magic numbers to named constants
|
|
if extra_text:
|
|
text_y = .95 - .0075 * len(plots)
|
|
figure.text(.1, text_y, extra_text, size='xx-small')
|
|
|
|
return (figure, area_data)
|
|
|
|
|
|
def create_metrics_plot(query_dict, plot_type, inverted_series, normalize_to,
|
|
drilldown_callback, extra_text=None):
|
|
plot_info = MetricsPlot(query_dict, plot_type, inverted_series,
|
|
normalize_to, drilldown_callback)
|
|
figure, area_data = _create_metrics_plot_helper(plot_info, extra_text)
|
|
return _create_image_html(figure, area_data, plot_info)
|
|
|
|
|
|
def _get_hostnames_in_bucket(hist_data, bucket):
|
|
"""\
|
|
Get all the hostnames that constitute a particular bucket in the histogram.
|
|
|
|
hist_data: list containing tuples of (hostname, pass_rate)
|
|
bucket: tuple containing the (low, high) values of the target bucket
|
|
"""
|
|
|
|
return [hostname for hostname, pass_rate in hist_data
|
|
if bucket[0] <= pass_rate < bucket[1]]
|
|
|
|
|
|
def _create_qual_histogram_helper(plot_info, extra_text=None):
|
|
"""\
|
|
Create a machine qualification histogram of the given data.
|
|
|
|
plot_info: a QualificationHistogram
|
|
extra_text: text to show at the upper-left of the graph
|
|
|
|
TODO(showard): move much or all of this into methods on
|
|
QualificationHistogram
|
|
"""
|
|
cursor = readonly_connection.cursor()
|
|
cursor.execute(plot_info.query)
|
|
|
|
if not cursor.rowcount:
|
|
raise NoDataError('query did not return any data')
|
|
|
|
# Lists to store the plot data.
|
|
# hist_data store tuples of (hostname, pass_rate) for machines that have
|
|
# pass rates between 0 and 100%, exclusive.
|
|
# no_tests is a list of machines that have run none of the selected tests
|
|
# no_pass is a list of machines with 0% pass rate
|
|
# perfect is a list of machines with a 100% pass rate
|
|
hist_data = []
|
|
no_tests = []
|
|
no_pass = []
|
|
perfect = []
|
|
|
|
# Construct the lists of data to plot
|
|
for hostname, total, good in cursor.fetchall():
|
|
if total == 0:
|
|
no_tests.append(hostname)
|
|
continue
|
|
|
|
if good == 0:
|
|
no_pass.append(hostname)
|
|
elif good == total:
|
|
perfect.append(hostname)
|
|
else:
|
|
percentage = 100.0 * good / total
|
|
hist_data.append((hostname, percentage))
|
|
|
|
interval = plot_info.interval
|
|
bins = range(0, 100, interval)
|
|
if bins[-1] != 100:
|
|
bins.append(bins[-1] + interval)
|
|
|
|
figure, height = _create_figure(_SINGLE_PLOT_HEIGHT)
|
|
subplot = figure.add_subplot(1, 1, 1)
|
|
|
|
# Plot the data and get all the bars plotted
|
|
_,_, bars = subplot.hist([data[1] for data in hist_data],
|
|
bins=bins, align='left')
|
|
bars += subplot.bar([-interval], len(no_pass),
|
|
width=interval, align='center')
|
|
bars += subplot.bar([bins[-1]], len(perfect),
|
|
width=interval, align='center')
|
|
bars += subplot.bar([-3 * interval], len(no_tests),
|
|
width=interval, align='center')
|
|
|
|
buckets = [(bin, min(bin + interval, 100)) for bin in bins[:-1]]
|
|
# set the x-axis range to cover all the normal bins plus the three "special"
|
|
# ones - N/A (3 intervals left), 0% (1 interval left) ,and 100% (far right)
|
|
subplot.set_xlim(-4 * interval, bins[-1] + interval)
|
|
subplot.set_xticks([-3 * interval, -interval] + bins + [100 + interval])
|
|
subplot.set_xticklabels(['N/A', '0%'] +
|
|
['%d%% - <%d%%' % bucket for bucket in buckets] +
|
|
['100%'], rotation=90, size='small')
|
|
|
|
# Find the coordinates on the image for each bar
|
|
x = []
|
|
y = []
|
|
for bar in bars:
|
|
x.append(bar.get_x())
|
|
y.append(bar.get_height())
|
|
f = subplot.plot(x, y, linestyle='None')[0]
|
|
upper_left_coords = f.get_transform().transform(zip(x, y))
|
|
bottom_right_coords = f.get_transform().transform(
|
|
[(x_val + interval, 0) for x_val in x])
|
|
|
|
# Set the title attributes
|
|
titles = ['%d%% - <%d%%: %d machines' % (bucket[0], bucket[1], y_val)
|
|
for bucket, y_val in zip(buckets, y)]
|
|
titles.append('0%%: %d machines' % len(no_pass))
|
|
titles.append('100%%: %d machines' % len(perfect))
|
|
titles.append('N/A: %d machines' % len(no_tests))
|
|
|
|
# Get the hostnames for each bucket in the histogram
|
|
names_list = [_get_hostnames_in_bucket(hist_data, bucket)
|
|
for bucket in buckets]
|
|
names_list += [no_pass, perfect]
|
|
|
|
if plot_info.filter_string:
|
|
plot_info.filter_string += ' AND '
|
|
|
|
# Construct the list of drilldown parameters to be passed when the user
|
|
# clicks on the bar.
|
|
params = []
|
|
for names in names_list:
|
|
if names:
|
|
hostnames = ','.join(_quote(hostname) for hostname in names)
|
|
hostname_filter = 'hostname IN (%s)' % hostnames
|
|
full_filter = plot_info.filter_string + hostname_filter
|
|
params.append({'type': 'normal',
|
|
'filterString': full_filter})
|
|
else:
|
|
params.append({'type': 'empty'})
|
|
|
|
params.append({'type': 'not_applicable',
|
|
'hosts': '<br />'.join(no_tests)})
|
|
|
|
area_data = [dict(left=ulx, top=height - uly,
|
|
right=brx, bottom=height - bry,
|
|
title=title, callback=plot_info.drilldown_callback,
|
|
callback_arguments=param_dict)
|
|
for (ulx, uly), (brx, bry), title, param_dict
|
|
in zip(upper_left_coords, bottom_right_coords, titles, params)]
|
|
|
|
# TODO(showard): extract these magic numbers to named constants
|
|
if extra_text:
|
|
figure.text(.1, .95, extra_text, size='xx-small')
|
|
|
|
return (figure, area_data)
|
|
|
|
|
|
def create_qual_histogram(query, filter_string, interval, drilldown_callback,
|
|
extra_text=None):
|
|
plot_info = QualificationHistogram(query, filter_string, interval,
|
|
drilldown_callback)
|
|
figure, area_data = _create_qual_histogram_helper(plot_info, extra_text)
|
|
return _create_image_html(figure, area_data, plot_info)
|
|
|
|
|
|
def create_embedded_plot(model, update_time):
|
|
"""\
|
|
Given an EmbeddedGraphingQuery object, generate the PNG image for it.
|
|
|
|
model: EmbeddedGraphingQuery object
|
|
update_time: 'Last updated' time
|
|
"""
|
|
|
|
params = pickle.loads(model.params)
|
|
extra_text = 'Last updated: %s' % update_time
|
|
|
|
if model.graph_type == 'metrics':
|
|
plot_info = MetricsPlot(query_dict=params['queries'],
|
|
plot_type=params['plot'],
|
|
inverted_series=params['invert'],
|
|
normalize_to=None,
|
|
drilldown_callback='')
|
|
figure, areas_unused = _create_metrics_plot_helper(plot_info,
|
|
extra_text)
|
|
elif model.graph_type == 'qual':
|
|
plot_info = QualificationHistogram(
|
|
query=params['query'], filter_string=params['filter_string'],
|
|
interval=params['interval'], drilldown_callback='')
|
|
figure, areas_unused = _create_qual_histogram_helper(plot_info,
|
|
extra_text)
|
|
else:
|
|
raise ValueError('Invalid graph_type %s' % model.graph_type)
|
|
|
|
image, bounding_box_unused = _create_png(figure)
|
|
return image
|
|
|
|
|
|
_cache_timeout = global_config.global_config.get_config_value(
|
|
'AUTOTEST_WEB', 'graph_cache_creation_timeout_minutes')
|
|
|
|
|
|
def handle_plot_request(id, max_age):
|
|
"""\
|
|
Given the embedding id of a graph, generate a PNG of the embedded graph
|
|
associated with that id.
|
|
|
|
id: id of the embedded graph
|
|
max_age: maximum age, in minutes, that a cached version should be held
|
|
"""
|
|
model = models.EmbeddedGraphingQuery.objects.get(id=id)
|
|
|
|
# Check if the cached image needs to be updated
|
|
now = datetime.datetime.now()
|
|
update_time = model.last_updated + datetime.timedelta(minutes=int(max_age))
|
|
if now > update_time:
|
|
cursor = django.db.connection.cursor()
|
|
|
|
# We want this query to update the refresh_time only once, even if
|
|
# multiple threads are running it at the same time. That is, only the
|
|
# first thread will win the race, and it will be the one to update the
|
|
# cached image; all other threads will show that they updated 0 rows
|
|
query = """
|
|
UPDATE embedded_graphing_queries
|
|
SET refresh_time = NOW()
|
|
WHERE id = %s AND (
|
|
refresh_time IS NULL OR
|
|
refresh_time + INTERVAL %s MINUTE < NOW()
|
|
)
|
|
"""
|
|
cursor.execute(query, (id, _cache_timeout))
|
|
|
|
# Only refresh the cached image if we were successful in updating the
|
|
# refresh time
|
|
if cursor.rowcount:
|
|
model.cached_png = create_embedded_plot(model, now.ctime())
|
|
model.last_updated = now
|
|
model.refresh_time = None
|
|
model.save()
|
|
|
|
return model.cached_png
|