145 lines
4.4 KiB
Python
Executable file
145 lines
4.4 KiB
Python
Executable file
#!/usr/bin/python2
|
|
#
|
|
# Copyright 2010 Google Inc. All Rights Reserved.
|
|
"""Script to compare a baseline results file to a new results file."""
|
|
|
|
from __future__ import print_function
|
|
|
|
__author__ = 'raymes@google.com (Raymes Khoury)'
|
|
|
|
import sys
|
|
from cros_utils import logger
|
|
from cros_utils import html_tools
|
|
|
|
PASS = 'pass'
|
|
FAIL = 'fail'
|
|
NOT_EXECUTED = 'not executed'
|
|
|
|
|
|
class ResultsReport(object):
|
|
"""Class for holding report results."""
|
|
|
|
def __init__(self, report, num_tests_executed, num_passes, num_failures,
|
|
num_regressions):
|
|
self.report = report
|
|
self.num_tests_executed = num_tests_executed
|
|
self.num_passes = num_passes
|
|
self.num_failures = num_failures
|
|
self.num_regressions = num_regressions
|
|
|
|
def GetReport(self):
|
|
return self.report
|
|
|
|
def GetNumExecuted(self):
|
|
return self.num_tests_executed
|
|
|
|
def GetNumPasses(self):
|
|
return self.num_passes
|
|
|
|
def GetNumFailures(self):
|
|
return self.num_failures
|
|
|
|
def GetNumRegressions(self):
|
|
return self.num_regressions
|
|
|
|
def GetSummary(self):
|
|
summary = 'Tests executed: %s\n' % str(self.num_tests_executed)
|
|
summary += 'Tests Passing: %s\n' % str(self.num_passes)
|
|
summary += 'Tests Failing: %s\n' % str(self.num_failures)
|
|
summary += 'Regressions: %s\n' % str(self.num_regressions)
|
|
return summary
|
|
|
|
|
|
def Usage():
|
|
print('Usage: %s baseline_results new_results' % sys.argv[0])
|
|
sys.exit(1)
|
|
|
|
|
|
def ParseResults(results_filename):
|
|
results = []
|
|
try:
|
|
results_file = open(results_filename, 'rb')
|
|
for line in results_file:
|
|
if line.strip() != '':
|
|
results.append(line.strip().split('\t'))
|
|
results_file.close()
|
|
except IOError:
|
|
logger.GetLogger().LogWarning('Could not open results file: ' +
|
|
results_filename)
|
|
return results
|
|
|
|
|
|
def GenerateResultsReport(baseline_file, new_result_file):
|
|
baseline_results = ParseResults(baseline_file)
|
|
new_results = ParseResults(new_result_file)
|
|
|
|
test_status = {}
|
|
|
|
for new_result in new_results:
|
|
new_test_name = new_result[0]
|
|
new_test_result = new_result[1]
|
|
test_status[new_test_name] = (new_test_result, NOT_EXECUTED)
|
|
|
|
for baseline_result in baseline_results:
|
|
baseline_test_name = baseline_result[0]
|
|
baseline_test_result = baseline_result[1]
|
|
if baseline_test_name in test_status:
|
|
new_test_result = test_status[baseline_test_name][0]
|
|
test_status[baseline_test_name] = (new_test_result, baseline_test_result)
|
|
else:
|
|
test_status[baseline_test_name] = (NOT_EXECUTED, baseline_test_result)
|
|
|
|
regressions = []
|
|
for result in test_status.keys():
|
|
if test_status[result][0] != test_status[result][1]:
|
|
regressions.append(result)
|
|
|
|
num_tests_executed = len(new_results)
|
|
num_regressions = len(regressions)
|
|
num_passes = 0
|
|
num_failures = 0
|
|
for result in new_results:
|
|
if result[1] == PASS:
|
|
num_passes += 1
|
|
else:
|
|
num_failures += 1
|
|
|
|
report = html_tools.GetPageHeader('Test Summary')
|
|
report += html_tools.GetHeader('Test Summary')
|
|
report += html_tools.GetListHeader()
|
|
report += html_tools.GetListItem('Tests executed: ' + str(num_tests_executed))
|
|
report += html_tools.GetListItem('Passes: ' + str(num_passes))
|
|
report += html_tools.GetListItem('Failures: ' + str(num_failures))
|
|
report += html_tools.GetListItem('Regressions: ' + str(num_regressions))
|
|
report += html_tools.GetListFooter()
|
|
report += html_tools.GetHeader('Regressions', 2)
|
|
report += html_tools.GetTableHeader(['Test name', 'Expected result',
|
|
'Actual result'])
|
|
|
|
for regression in regressions:
|
|
report += html_tools.GetTableRow([regression[:150], test_status[regression][
|
|
1], test_status[regression][0]])
|
|
report += '\n'
|
|
report += html_tools.GetTableFooter()
|
|
report += html_tools.GetHeader('All Tests', 2)
|
|
report += html_tools.GetTableHeader(['Test name', 'Expected result',
|
|
'Actual result'])
|
|
for result in test_status.keys():
|
|
report += html_tools.GetTableRow([result[:150], test_status[result][1],
|
|
test_status[result][0]])
|
|
report += '\n'
|
|
report += html_tools.GetTableFooter()
|
|
report += html_tools.GetFooter()
|
|
return ResultsReport(report, num_tests_executed, num_passes, num_failures,
|
|
num_regressions)
|
|
|
|
|
|
def Main(argv):
|
|
if len(argv) < 2:
|
|
Usage()
|
|
|
|
print(GenerateResultsReport(argv[1], argv[2])[0])
|
|
|
|
|
|
if __name__ == '__main__':
|
|
Main(sys.argv)
|