Stop tests at the first failure

  1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
__doc__ = """
In your settings, use

  TEST_RUNNER = 'ambidjangolib.test.simple.run_tests_until_fail'

to make `manage.py test` stop after the first test suite with failures, and
only show the first failing test in the suite.
"""


from unittest import \
     TestSuite, TextTestRunner, _TextTestResult, defaultTestLoader
from django.test import _doctest as doctest
from django.conf import settings
from django.db import transaction
from django.db.models import get_app, get_apps
from django.test.utils import \
     setup_test_environment, teardown_test_environment, \
     create_test_db, destroy_test_db
from django.test.simple import build_test, get_tests, doctestOutputChecker
from django.test.testcases import DocTestRunner


class DocTestRunner(doctest.DocTestRunner):
    """
    Replacement for django.test.testcases.DocTestRunner which unfortunately
    overrides any supplied `optionflags=` kwarg with only `doctest.ELLIPSIS`.
    We need to pass on optionflags.
    """
    def __init__(self, *args, **kwargs):
        doctest.DocTestRunner.__init__(self, *args, **kwargs)
        self.optionflags |= doctest.ELLIPSIS
        # Django's original has `=` instead of `|=` here

    def report_unexpected_exception(self, out, test, example, exc_info):
        doctest.DocTestRunner.report_unexpected_exception(self, out, test,
                                                          example, exc_info)
        # Rollback, in case of database errors. Otherwise they'd have
        # side effects on other tests.
        transaction.rollback_unless_managed()


def _add_tests_for_module(suite, module):
    """
    Repeated code from inside `build_suite()` is refactored here.
    """
    # Load unit and doctests in the given module. If module has a suite()
    # method, use it. Otherwise build the test suite ourselves.
    if hasattr(module, 'suite'):
        suite.addTest(module.suite())
    else:
        suite.addTest(defaultTestLoader.loadTestsFromModule(module))
        try:
            suite.addTest(doctest.DocTestSuite(
                module,
                checker=doctestOutputChecker,
                runner=DocTestRunner,
                optionflags=doctest.REPORT_ONLY_FIRST_FAILURE))
        except ValueError:
            # No doc tests in models.py
            pass


def build_suite(app_module):
    """
    Create a complete Django test suite for the provided application module.

    This overrides Django's original `django.test.simple.build_suite()` because
    we need to pass the `REPORT_ONLY_FIRST_FAILURE` option flag to
    `DocTestSuite` instances.
    """
    suite = TestSuite()

    _add_tests_for_module(suite, app_module)

    # Check to see if a separate 'tests' module exists parallel to the
    # models module
    test_module = get_tests(app_module)
    if test_module:
        _add_tests_for_module(suite, test_module)

    return suite


class _FailStopTextTestResult(_TextTestResult):
    def addError(self, test, err):
        _TextTestResult.addError(self, test, err)
        self.shouldStop = True

    def addFailure(self, test, err):
        _TextTestResult.addFailure(self, test, err)
        self.shouldStop = True


class FailStopTextTestRunner(TextTestRunner):
    def _makeResult(self):
        return _FailStopTextTestResult(
            self.stream, self.descriptions, self.verbosity)


def run_tests_until_fail(test_labels, verbosity=1, interactive=True, extra_tests=[]):
    """
    Run the unit tests for all the test labels in the provided list.
    Labels must be of the form:
     - app.TestClass.test_method
        Run a single specific test method
     - app.TestClass
        Run all the test methods in a given class
     - app
        Search for doctests and unittests in the named application.

    When looking for tests, the test runner will look in the models and
    tests modules for the application.

    A list of 'extra' tests may also be provided; these tests
    will be added to the test suite.

    Stops the tests at the first failure and returns 1.  If all test pass,
    returns 0.

    Also displays only the first failure in the failing test suite.
    """
    setup_test_environment()

    settings.DEBUG = False
    suite = TestSuite()

    if test_labels:
        for label in test_labels:
            if '.' in label:
                suite.addTest(build_test(label))
            else:
                app = get_app(label)
                suite.addTest(build_suite(app))
    else:
        for app in get_apps():
            suite.addTest(build_suite(app))

    for test in extra_tests:
        suite.addTest(test)

    old_name = settings.DATABASE_NAME
    create_test_db(verbosity, autoclobber=not interactive)
    result = FailStopTextTestRunner(verbosity=verbosity).run(suite)
    destroy_test_db(old_name, verbosity)

    teardown_test_environment()

    return len(result.failures) + len(result.errors)

More like this

  1. Forcing unit test runner to abort after failed test by simonbun 6 years, 11 months ago
  2. Variable._resolve_lookup monkeypatch by showell 4 years, 5 months ago
  3. Zope testing django layer by grahamcarlyle 6 years, 3 months ago
  4. Bulk Insert - updated 5/9/2008 by coolie 6 years, 6 months ago
  5. Breaking tests.py into multiple files by gsakkis 4 years ago

Comments

(Forgotten your password?)