296 lines
10 KiB
Python
Executable file
296 lines
10 KiB
Python
Executable file
#!/usr/bin/python
|
|
#
|
|
# Copyright (c) 2013 The Chromium OS Authors. All rights reserved.
|
|
# Use of this source code is governed by a BSD-style license that can be
|
|
# found in the LICENSE file.
|
|
|
|
import datetime, unittest
|
|
|
|
import mox
|
|
|
|
import common
|
|
# This must come before the import of complete_failures in order to use the
|
|
# in memory database.
|
|
from autotest_lib.frontend import setup_django_readonly_environment
|
|
from autotest_lib.frontend import setup_test_environment
|
|
import complete_failures
|
|
from autotest_lib.client.common_lib import mail
|
|
from autotest_lib.frontend.tko import models
|
|
from django import test
|
|
|
|
|
|
GOOD_STATUS_IDX = 6
|
|
|
|
# See complte_failurs_functional_tests.py for why we need this.
|
|
class MockDatetime(datetime.datetime):
|
|
"""Used to mock out parts of datetime.datetime."""
|
|
pass
|
|
|
|
|
|
class EmailAboutTestFailureTests(mox.MoxTestBase):
|
|
"""
|
|
Tests that emails are sent about failed tests.
|
|
"""
|
|
def setUp(self):
|
|
super(EmailAboutTestFailureTests, self).setUp()
|
|
|
|
# We need to mock out the send function in all tests or else the
|
|
# emails will be sent out during tests.
|
|
self.mox.StubOutWithMock(mail, 'send')
|
|
|
|
self._orig_too_long = complete_failures._DAYS_TO_BE_FAILING_TOO_LONG
|
|
|
|
|
|
def tearDown(self):
|
|
complete_failures._DAYS_TO_BE_FAILING_TOO_LONG = self._orig_too_long
|
|
super(EmailAboutTestFailureTests, self).tearDown()
|
|
|
|
|
|
def test_email_sent_about_all_failed_tests(self):
|
|
"""Test that the email report mentions all the failed_tests."""
|
|
complete_failures._DAYS_TO_BE_FAILING_TOO_LONG = 60
|
|
|
|
mail.send(
|
|
'chromeos-test-health@google.com',
|
|
['chromeos-lab-infrastructure@google.com'],
|
|
[],
|
|
'Long Failing Tests',
|
|
'1/1 tests have been failing for at least %d days.\n'
|
|
'They are the following:\n\ntest'
|
|
% complete_failures._DAYS_TO_BE_FAILING_TOO_LONG)
|
|
|
|
failures = ['test']
|
|
all_tests = set(failures)
|
|
|
|
self.mox.ReplayAll()
|
|
complete_failures.email_about_test_failure(failures, all_tests)
|
|
|
|
|
|
def test_email_has_test_names_sorted_alphabetically(self):
|
|
"""Test that the email report has entries sorted alphabetically."""
|
|
complete_failures._DAYS_TO_BE_FAILING_TOO_LONG = 60
|
|
|
|
mail.send(
|
|
'chromeos-test-health@google.com',
|
|
['chromeos-lab-infrastructure@google.com'],
|
|
[],
|
|
'Long Failing Tests',
|
|
'2/2 tests have been failing for at least %d days.\n'
|
|
'They are the following:\n\ntest1\ntest2'
|
|
% complete_failures._DAYS_TO_BE_FAILING_TOO_LONG)
|
|
|
|
# We use an OrderedDict to gurantee that the elements would be out of
|
|
# order if we did a simple traversal.
|
|
failures = ['test2', 'test1']
|
|
all_tests = set(failures)
|
|
|
|
self.mox.ReplayAll()
|
|
complete_failures.email_about_test_failure(failures, all_tests)
|
|
|
|
|
|
def test_email_count_of_total_number_of_tests(self):
|
|
"""Test that the email report displays total number of tests."""
|
|
complete_failures._DAYS_TO_BE_FAILING_TOO_LONG = 60
|
|
|
|
mail.send(
|
|
'chromeos-test-health@google.com',
|
|
['chromeos-lab-infrastructure@google.com'],
|
|
[],
|
|
'Long Failing Tests',
|
|
'1/2 tests have been failing for at least %d days.\n'
|
|
'They are the following:\n\ntest'
|
|
% complete_failures._DAYS_TO_BE_FAILING_TOO_LONG)
|
|
|
|
failures = ['test']
|
|
all_tests = set(failures) | {'not_failure'}
|
|
|
|
self.mox.ReplayAll()
|
|
complete_failures.email_about_test_failure(failures, all_tests)
|
|
|
|
|
|
class IsValidTestNameTests(test.TestCase):
|
|
"""Tests the is_valid_test_name function."""
|
|
|
|
def test_returns_true_for_valid_test_name(self):
|
|
"""Test that a valid test name returns True."""
|
|
name = 'TestName.TestName'
|
|
self.assertTrue(complete_failures.is_valid_test_name(name))
|
|
|
|
|
|
def test_returns_false_if_name_has_slash_in_it(self):
|
|
"""Test that a name with a slash in it returns False."""
|
|
name = 'path/to/test'
|
|
self.assertFalse(complete_failures.is_valid_test_name(name))
|
|
|
|
|
|
def test_returns_false_for_try_new_image_entries(self):
|
|
"""Test that a name that starts with try_new_image returns False."""
|
|
name = 'try_new_image-blah'
|
|
self.assertFalse(complete_failures.is_valid_test_name(name))
|
|
|
|
|
|
class PrepareLastPassesTests(test.TestCase):
|
|
"""Tests the prepare_last_passes function."""
|
|
|
|
def setUp(self):
|
|
super(PrepareLastPassesTests, self).setUp()
|
|
|
|
def tearDown(self):
|
|
super(PrepareLastPassesTests, self).tearDown()
|
|
|
|
def test_does_not_return_invalid_test_names(self):
|
|
"""Tests that tests with invalid test names are not returned."""
|
|
results = complete_failures.prepare_last_passes(['invalid_test/name'])
|
|
|
|
self.assertEqual(results, {})
|
|
|
|
|
|
class GetRecentlyRanTestNamesTests(mox.MoxTestBase, test.TestCase):
|
|
"""Tests the get_recently_ran_test_names function."""
|
|
|
|
def setUp(self):
|
|
super(GetRecentlyRanTestNamesTests, self).setUp()
|
|
self.mox.StubOutWithMock(MockDatetime, 'today')
|
|
self.datetime = datetime.datetime
|
|
datetime.datetime = MockDatetime
|
|
setup_test_environment.set_up()
|
|
self._orig_cutoff = complete_failures._DAYS_NOT_RUNNING_CUTOFF
|
|
|
|
|
|
def tearDown(self):
|
|
datetime.datetime = self.datetime
|
|
complete_failures._DAYS_NOT_RUNNING_CUTOFF = self._orig_cutoff
|
|
setup_test_environment.tear_down()
|
|
super(GetRecentlyRanTestNamesTests, self).tearDown()
|
|
|
|
|
|
def test_return_all_recently_ran_tests(self):
|
|
"""Test that the function does as it says it does."""
|
|
job = models.Job(job_idx=1)
|
|
kernel = models.Kernel(kernel_idx=1)
|
|
machine = models.Machine(machine_idx=1)
|
|
success_status = models.Status(status_idx=GOOD_STATUS_IDX)
|
|
|
|
recent = models.Test(job=job, status=success_status,
|
|
kernel=kernel, machine=machine,
|
|
test='recent',
|
|
started_time=self.datetime(2012, 1, 1))
|
|
recent.save()
|
|
old = models.Test(job=job, status=success_status,
|
|
kernel=kernel, machine=machine,
|
|
test='old',
|
|
started_time=self.datetime(2011, 1, 2))
|
|
old.save()
|
|
|
|
datetime.datetime.today().AndReturn(self.datetime(2012, 1, 4))
|
|
complete_failures._DAYS_NOT_RUNNING_CUTOFF = 60
|
|
|
|
self.mox.ReplayAll()
|
|
results = complete_failures.get_recently_ran_test_names()
|
|
|
|
self.assertEqual(set(results), set(['recent']))
|
|
|
|
|
|
def test_returns_no_duplicate_names(self):
|
|
"""Test that each test name appears only once."""
|
|
job = models.Job(job_idx=1)
|
|
kernel = models.Kernel(kernel_idx=1)
|
|
machine = models.Machine(machine_idx=1)
|
|
success_status = models.Status(status_idx=GOOD_STATUS_IDX)
|
|
|
|
test = models.Test(job=job, status=success_status,
|
|
kernel=kernel, machine=machine,
|
|
test='test',
|
|
started_time=self.datetime(2012, 1, 1))
|
|
test.save()
|
|
duplicate = models.Test(job=job, status=success_status,
|
|
kernel=kernel, machine=machine,
|
|
test='test',
|
|
started_time=self.datetime(2012, 1, 2))
|
|
duplicate.save()
|
|
|
|
datetime.datetime.today().AndReturn(self.datetime(2012, 1, 3))
|
|
complete_failures._DAYS_NOT_RUNNING_CUTOFF = 60
|
|
|
|
self.mox.ReplayAll()
|
|
results = complete_failures.get_recently_ran_test_names()
|
|
|
|
self.assertEqual(len(results), 1)
|
|
|
|
|
|
class GetTestsToAnalyzeTests(mox.MoxTestBase):
|
|
"""Tests the get_tests_to_analyze function."""
|
|
|
|
def test_returns_recent_test_names(self):
|
|
"""Test should return all the test names in the database."""
|
|
|
|
recent_tests = {'passing_test', 'failing_test'}
|
|
last_passes = {'passing_test': datetime.datetime(2012, 1 ,1),
|
|
'old_passing_test': datetime.datetime(2011, 1, 1)}
|
|
|
|
results = complete_failures.get_tests_to_analyze(recent_tests,
|
|
last_passes)
|
|
|
|
self.assertEqual(results,
|
|
{'passing_test': datetime.datetime(2012, 1, 1),
|
|
'failing_test': datetime.datetime.min})
|
|
|
|
|
|
def test_returns_failing_tests_with_min_datetime(self):
|
|
"""Test that never-passed tests are paired with datetime.min."""
|
|
|
|
recent_tests = {'test'}
|
|
last_passes = {}
|
|
|
|
self.mox.ReplayAll()
|
|
results = complete_failures.get_tests_to_analyze(recent_tests,
|
|
last_passes)
|
|
|
|
self.assertEqual(results, {'test': datetime.datetime.min})
|
|
|
|
|
|
class FilterOutGoodTestsTests(mox.MoxTestBase):
|
|
"""Tests the filter_our_good_tests function."""
|
|
|
|
def setUp(self):
|
|
super(FilterOutGoodTestsTests, self).setUp()
|
|
self.mox.StubOutWithMock(MockDatetime, 'today')
|
|
self.datetime = datetime.datetime
|
|
datetime.datetime = MockDatetime
|
|
self._orig_too_long = complete_failures._DAYS_TO_BE_FAILING_TOO_LONG
|
|
|
|
|
|
def tearDown(self):
|
|
datetime.datetime = self.datetime
|
|
complete_failures._DAYS_TO_BE_FAILING_TOO_LONG = self._orig_too_long
|
|
super(FilterOutGoodTestsTests, self).tearDown()
|
|
|
|
|
|
def test_remove_all_tests_that_have_passed_recently_enough(self):
|
|
"""Test that all recently passing tests are not returned."""
|
|
|
|
complete_failures._DAYS_TO_BE_FAILING_TOO_LONG = 10
|
|
datetime.datetime.today().AndReturn(self.datetime(2012, 1, 21))
|
|
|
|
self.mox.ReplayAll()
|
|
result = complete_failures.filter_out_good_tests(
|
|
{'test': self.datetime(2012, 1, 20)})
|
|
|
|
self.assertEqual(result, [])
|
|
|
|
|
|
def test_keep_all_tests_that_have_not_passed_recently_enough(self):
|
|
"""Test that the tests that have not recently passed are returned."""
|
|
|
|
complete_failures._DAYS_TO_BE_FAILING_TOO_LONG = 10
|
|
datetime.datetime.today().AndReturn(self.datetime(2012, 1, 21))
|
|
|
|
self.mox.ReplayAll()
|
|
result = complete_failures.filter_out_good_tests(
|
|
{'test': self.datetime(2012, 1, 10)})
|
|
|
|
self.assertEqual(result, ['test'])
|
|
|
|
|
|
if __name__ == '__main__':
|
|
unittest.main()
|