257 lines
11 KiB
Python
Executable file
257 lines
11 KiB
Python
Executable file
#!/usr/bin/env python
|
|
|
|
# Copyright (c) 2014 The Chromium OS Authors. All rights reserved.
|
|
# Use of this source code is governed by a BSD-style license that can be
|
|
# found in the LICENSE file.
|
|
|
|
# Script to check the history of stage calls made to devserver.
|
|
# Following are some sample use cases:
|
|
#
|
|
# 1. Find all stage request for autotest and image nyan_big-release/R38-6055.0.0
|
|
# in the last 10 days across all devservers.
|
|
# ./devserver_history.py --image_filters nyan_big 38 6055.0.0 -l 240 \
|
|
# --artifact_filters autotest -v
|
|
# output:
|
|
# ==============================================================================
|
|
# 170.21.64.22
|
|
# ==============================================================================
|
|
# Number of calls: 1
|
|
# Number of unique images: 1
|
|
# 2014-08-23 12:45:00: nyan_big-release/R38-6055.0.0 autotest
|
|
# ==============================================================================
|
|
# 170.21.64.23
|
|
# ==============================================================================
|
|
# Number of calls: 2
|
|
# Number of unique images: 1
|
|
# 2014-08-23 12:45:00: nyan_big-release/R38-6055.0.0 autotest, test_suites
|
|
# 2014-08-23 12:55:00: nyan_big-release/R38-6055.0.0 autotest, test_suites
|
|
#
|
|
# 2. Find all duplicated stage request for the last 10 days.
|
|
# ./devserver_history.py -d -l 240
|
|
# output:
|
|
# Detecting artifacts staged in multiple devservers.
|
|
# ==============================================================================
|
|
# nyan_big-release/R38-6055.0.0
|
|
# ==============================================================================
|
|
# 170.21.64.22: 23 requests 2014-09-04 22:44:28 -- 2014-09-05 00:03:23
|
|
# 170.21.64.23: 6 requests 2014-09-04 22:48:58 -- 2014-09-04 22:49:42
|
|
#
|
|
# Count of images with duplicated stages on each devserver:
|
|
# 170.21.64.22 : 22
|
|
# 170.21.64.23 : 11
|
|
|
|
|
|
import argparse
|
|
import datetime
|
|
import logging
|
|
import operator
|
|
import re
|
|
import time
|
|
from itertools import groupby
|
|
|
|
import common
|
|
from autotest_lib.client.common_lib import global_config
|
|
from autotest_lib.client.common_lib import time_utils
|
|
from autotest_lib.client.common_lib.cros.graphite import autotest_es
|
|
|
|
|
|
class devserver_call(object):
|
|
"""A container to store the information of devserver stage call.
|
|
"""
|
|
|
|
def __init__(self, hit):
|
|
"""Retrieve information from a ES query hit.
|
|
"""
|
|
self.devserver = hit['devserver']
|
|
self.subname = hit['subname']
|
|
self.artifacts = hit['artifacts'].split(' ')
|
|
self.image = hit['image']
|
|
self.value = hit['value']
|
|
self.time_recorded = time_utils.epoch_time_to_date_string(
|
|
hit['time_recorded'])
|
|
|
|
|
|
def __str__(self):
|
|
pairs = ['%-20s: %s' % (attr, getattr(self, attr)) for attr in dir(self)
|
|
if not attr.startswith('__') and
|
|
not callable(getattr(self, attr))]
|
|
return '\n'.join(pairs)
|
|
|
|
|
|
def get_calls(time_start, time_end, artifact_filters=None,
|
|
regex_constraints=None, devserver=None, size=1e7):
|
|
"""Gets all devserver calls from es db with the given constraints.
|
|
|
|
@param time_start: Earliest time entry was recorded.
|
|
@param time_end: Latest time entry was recorded.
|
|
@param artifact_filters: A list of names to match artifacts.
|
|
@param regex_constraints: A list of regex constraints for ES query.
|
|
@param devserver: name of devserver to query for. If it's set to None,
|
|
return calls for all devservers. Default is set to None.
|
|
@param size: Max number of entries to return, default to 1 million.
|
|
|
|
@returns: Entries from esdb.
|
|
"""
|
|
eqs = [('_type', 'devserver')]
|
|
if devserver:
|
|
eqs.append(('devserver', devserver))
|
|
if artifact_filters:
|
|
for artifact in artifact_filters:
|
|
eqs.append(('artifacts', artifact))
|
|
time_start_epoch = time_utils.to_epoch_time(time_start)
|
|
time_end_epoch = time_utils.to_epoch_time(time_end)
|
|
results = autotest_es.query(
|
|
fields_returned=None,
|
|
equality_constraints=eqs,
|
|
range_constraints=[('time_recorded', time_start_epoch,
|
|
time_end_epoch)],
|
|
size=size,
|
|
sort_specs=[{'time_recorded': 'desc'}],
|
|
regex_constraints=regex_constraints)
|
|
devserver_calls = []
|
|
for hit in results.hits:
|
|
devserver_calls.append(devserver_call(hit))
|
|
logging.info('Found %d calls.', len(devserver_calls))
|
|
return devserver_calls
|
|
|
|
|
|
def print_call_details(calls, verbose):
|
|
"""Print details of each call to devserver to stage artifacts.
|
|
|
|
@param calls: A list of devserver stage requests.
|
|
@param verbose: Set to True to print out all devserver calls.
|
|
"""
|
|
calls = sorted(calls, key=lambda c: c.devserver)
|
|
for devserver,calls_for_devserver in groupby(calls, lambda c: c.devserver):
|
|
calls_for_devserver = list(calls_for_devserver)
|
|
print '='*80
|
|
print devserver
|
|
print '='*80
|
|
print 'Number of calls: %d' % len(calls_for_devserver)
|
|
print ('Number of unique images: %d' %
|
|
len(set([call.image for call in calls_for_devserver])))
|
|
if verbose:
|
|
for call in sorted(calls_for_devserver,
|
|
key=lambda c: c.time_recorded):
|
|
print ('%s %s %s' % (call.time_recorded, call.image,
|
|
', '.join(call.artifacts)))
|
|
|
|
|
|
def detect_duplicated_stage(calls):
|
|
"""Detect any artifact for same build was staged in multiple devservers.
|
|
|
|
@param calls: A list of devserver stage requests.
|
|
"""
|
|
print '\nDetecting artifacts staged in multiple devservers.'
|
|
calls = sorted(calls, key=lambda c: c.image)
|
|
# Count how many times a devserver staged duplicated artifacts. A number
|
|
# significantly larger then others can indicate that the devserver failed
|
|
# check_health too often and needs to be removed from production.
|
|
duplicated_stage_count = {}
|
|
for image,calls_for_image in groupby(calls, lambda c: c.image):
|
|
calls_for_image = list(calls_for_image)
|
|
devservers = set([call.devserver for call in calls_for_image])
|
|
if len(devservers) > 1:
|
|
print '='*80
|
|
print image
|
|
print '='*80
|
|
calls_for_image = sorted(calls_for_image, key=lambda c: c.devserver)
|
|
for devserver,calls_for_devserver in groupby(calls_for_image,
|
|
lambda c: c.devserver):
|
|
timestamps = [c.time_recorded for c in calls_for_devserver]
|
|
print ('%s: %-3d requests %s -- %s' %
|
|
(devserver, len(timestamps), min(timestamps),
|
|
max(timestamps)))
|
|
duplicated_stage_count[devserver] = (
|
|
duplicated_stage_count.get(devserver, 0) + 1)
|
|
print '\nCount of images with duplicated stages on each devserver:'
|
|
counts = sorted(duplicated_stage_count.iteritems(),
|
|
key=operator.itemgetter(1), reverse=True)
|
|
for k,v in counts:
|
|
print '%-15s: %d' % (k, v)
|
|
|
|
|
|
def main():
|
|
"""main script. """
|
|
t_now = time.time()
|
|
t_now_minus_one_day = t_now - 3600 * 24
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument('-l', type=float, dest='last',
|
|
help='last hours to search results across',
|
|
default=None)
|
|
parser.add_argument('--start', type=str, dest='start',
|
|
help=('Enter start time as: yyyy-mm-dd hh-mm-ss,'
|
|
'defualts to 24h ago. This option is ignored when'
|
|
' -l is used.'),
|
|
default=time_utils.epoch_time_to_date_string(
|
|
t_now_minus_one_day))
|
|
parser.add_argument('--end', type=str, dest='end',
|
|
help=('Enter end time in as: yyyy-mm-dd hh-mm-ss,'
|
|
'defualts to current time. This option is ignored'
|
|
' when -l is used.'),
|
|
default=time_utils.epoch_time_to_date_string(t_now))
|
|
parser.add_argument('--devservers', nargs='+', dest='devservers',
|
|
help=('Enter space deliminated devservers. Default are'
|
|
' all devservers specified in global config.'),
|
|
default=[])
|
|
parser.add_argument('--artifact_filters', nargs='+',
|
|
dest='artifact_filters',
|
|
help=('Enter space deliminated filters on artifact '
|
|
'name. For example "autotest test_suites". The '
|
|
'filter does not support regex.'),
|
|
default=[])
|
|
parser.add_argument('--image_filters', nargs='+', dest='image_filters',
|
|
help=('Enter space deliminated filters on image name. '
|
|
'For example "nyan 38 6566", search will use '
|
|
'regex to match each filter. Do not use filters '
|
|
'with mixed letter and number, e.g., R38.'),
|
|
default=[])
|
|
parser.add_argument('-d', '--detect_duplicated_stage', action='store_true',
|
|
dest='detect_duplicated_stage',
|
|
help=('Set to True to detect if an artifacts for a same'
|
|
' build was staged in multiple devservers. '
|
|
'Default is True.'),
|
|
default=False)
|
|
parser.add_argument('-v', action='store_true', dest='verbose',
|
|
default=False,
|
|
help='-v to print out ALL entries.')
|
|
options = parser.parse_args()
|
|
if options.verbose:
|
|
logging.getLogger().setLevel(logging.INFO)
|
|
|
|
if options.last:
|
|
end_time = datetime.datetime.now()
|
|
start_time = end_time - datetime.timedelta(seconds=3600 * options.last)
|
|
else:
|
|
start_time = datetime.datetime.strptime(options.start,
|
|
time_utils.TIME_FMT)
|
|
end_time = datetime.datetime.strptime(options.end, time_utils.TIME_FMT)
|
|
logging.info('Searching devserver calls from %s to %s', start_time,
|
|
end_time)
|
|
|
|
devservers = options.devservers
|
|
if not devservers:
|
|
devserver_urls = global_config.global_config.get_config_value(
|
|
'CROS', 'dev_server', type=list, default=[])
|
|
devservers = []
|
|
for url in devserver_urls:
|
|
match = re.match('http://([^:]*):*\d*', url)
|
|
devservers.append(match.groups(0)[0] if match else url)
|
|
logging.info('Found devservers: %s', devservers)
|
|
|
|
regex_constraints = []
|
|
for filter in options.image_filters:
|
|
regex_constraints.append(('image', '.*%s.*' % filter))
|
|
calls = []
|
|
for devserver in devservers:
|
|
calls.extend(get_calls(start_time, end_time, options.artifact_filters,
|
|
regex_constraints, devserver=devserver))
|
|
|
|
print_call_details(calls, options.verbose)
|
|
|
|
if options.detect_duplicated_stage:
|
|
detect_duplicated_stage(calls)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|