[3] | 1 | """ |
---|
| 2 | XML Test Runner for PyUnit |
---|
| 3 | """ |
---|
| 4 | |
---|
| 5 | # Written by Sebastian Rittau <srittau@jroger.in-berlin.de> and placed in |
---|
| 6 | # the Public Domain. With contributions by Paolo Borelli. |
---|
| 7 | |
---|
| 8 | __revision__ = "$Id: /private/python/stdlib/xmlrunner.py 16654 2007-11-12T12:46:35.368945Z srittau $" |
---|
| 9 | |
---|
| 10 | import os.path |
---|
| 11 | import re |
---|
| 12 | import sys |
---|
| 13 | import time |
---|
| 14 | import traceback |
---|
| 15 | import unittest |
---|
| 16 | from StringIO import StringIO |
---|
| 17 | from xml.sax.saxutils import escape |
---|
| 18 | |
---|
| 19 | from StringIO import StringIO |
---|
| 20 | |
---|
| 21 | |
---|
| 22 | |
---|
| 23 | class _TestInfo(object): |
---|
| 24 | |
---|
| 25 | """Information about a particular test. |
---|
| 26 | |
---|
| 27 | Used by _XMLTestResult. |
---|
| 28 | |
---|
| 29 | """ |
---|
| 30 | |
---|
| 31 | def __init__(self, test, time): |
---|
| 32 | _pieces = test.id().split('.') |
---|
| 33 | (self._class, self._method) = ('.'.join(_pieces[:-1]), _pieces[-1]) |
---|
| 34 | self._time = time |
---|
| 35 | self._error = None |
---|
| 36 | self._failure = None |
---|
| 37 | |
---|
| 38 | |
---|
| 39 | def print_report(self, stream): |
---|
| 40 | """Print information about this test case in XML format to the |
---|
| 41 | supplied stream. |
---|
| 42 | |
---|
| 43 | """ |
---|
| 44 | stream.write(' <testcase classname="%(class)s" name="%(method)s" time="%(time).4f">' % \ |
---|
| 45 | { |
---|
| 46 | "class": self._class, |
---|
| 47 | "method": self._method, |
---|
| 48 | "time": self._time, |
---|
| 49 | }) |
---|
| 50 | if self._failure != None: |
---|
| 51 | self._print_error(stream, 'failure', self._failure) |
---|
| 52 | if self._error != None: |
---|
| 53 | self._print_error(stream, 'error', self._error) |
---|
| 54 | stream.write('</testcase>\n') |
---|
| 55 | |
---|
| 56 | def _print_error(self, stream, tagname, error): |
---|
| 57 | """Print information from a failure or error to the supplied stream.""" |
---|
| 58 | text = escape(str(error[1])) |
---|
| 59 | stream.write('\n') |
---|
| 60 | stream.write(' <%s type="%s">%s\n' \ |
---|
| 61 | % (tagname, issubclass(error[0], Exception) and error[0].__name__ or str(error[0]), text)) |
---|
| 62 | tb_stream = StringIO() |
---|
| 63 | traceback.print_tb(error[2], None, tb_stream) |
---|
| 64 | stream.write(escape(tb_stream.getvalue())) |
---|
| 65 | stream.write(' </%s>\n' % tagname) |
---|
| 66 | stream.write(' ') |
---|
| 67 | |
---|
| 68 | # Module level functions since Python 2.3 doesn't grok decorators |
---|
| 69 | def create_success(test, time): |
---|
| 70 | """Create a _TestInfo instance for a successful test.""" |
---|
| 71 | return _TestInfo(test, time) |
---|
| 72 | |
---|
| 73 | def create_failure(test, time, failure): |
---|
| 74 | """Create a _TestInfo instance for a failed test.""" |
---|
| 75 | info = _TestInfo(test, time) |
---|
| 76 | info._failure = failure |
---|
| 77 | return info |
---|
| 78 | |
---|
| 79 | def create_error(test, time, error): |
---|
| 80 | """Create a _TestInfo instance for an erroneous test.""" |
---|
| 81 | info = _TestInfo(test, time) |
---|
| 82 | info._error = error |
---|
| 83 | return info |
---|
| 84 | |
---|
| 85 | class _XMLTestResult(unittest.TestResult): |
---|
| 86 | |
---|
| 87 | """A test result class that stores result as XML. |
---|
| 88 | |
---|
| 89 | Used by XMLTestRunner. |
---|
| 90 | |
---|
| 91 | """ |
---|
| 92 | |
---|
| 93 | def __init__(self, classname): |
---|
| 94 | unittest.TestResult.__init__(self) |
---|
| 95 | self._test_name = classname |
---|
| 96 | self._start_time = None |
---|
| 97 | self._tests = [] |
---|
| 98 | self._error = None |
---|
| 99 | self._failure = None |
---|
| 100 | |
---|
| 101 | def startTest(self, test): |
---|
| 102 | unittest.TestResult.startTest(self, test) |
---|
| 103 | self._error = None |
---|
| 104 | self._failure = None |
---|
| 105 | self._start_time = time.time() |
---|
| 106 | |
---|
| 107 | def stopTest(self, test): |
---|
| 108 | time_taken = time.time() - self._start_time |
---|
| 109 | unittest.TestResult.stopTest(self, test) |
---|
| 110 | if self._error: |
---|
| 111 | info = create_error(test, time_taken, self._error) |
---|
| 112 | elif self._failure: |
---|
| 113 | info = create_failure(test, time_taken, self._failure) |
---|
| 114 | else: |
---|
| 115 | info = create_success(test, time_taken) |
---|
| 116 | self._tests.append(info) |
---|
| 117 | |
---|
| 118 | def addError(self, test, err): |
---|
| 119 | unittest.TestResult.addError(self, test, err) |
---|
| 120 | self._error = err |
---|
| 121 | |
---|
| 122 | def addFailure(self, test, err): |
---|
| 123 | unittest.TestResult.addFailure(self, test, err) |
---|
| 124 | self._failure = err |
---|
| 125 | |
---|
| 126 | def print_report(self, stream, time_taken, out, err): |
---|
| 127 | """Prints the XML report to the supplied stream. |
---|
| 128 | |
---|
| 129 | The time the tests took to perform as well as the captured standard |
---|
| 130 | output and standard error streams must be passed in.a |
---|
| 131 | |
---|
| 132 | """ |
---|
| 133 | stream.write('<testsuite errors="%(e)d" failures="%(f)d" ' % \ |
---|
| 134 | { "e": len(self.errors), "f": len(self.failures) }) |
---|
| 135 | stream.write('name="%(n)s" tests="%(t)d" time="%(time).3f">\n' % \ |
---|
| 136 | { |
---|
| 137 | "n": self._test_name, |
---|
| 138 | "t": self.testsRun, |
---|
| 139 | "time": time_taken, |
---|
| 140 | }) |
---|
| 141 | for info in self._tests: |
---|
| 142 | info.print_report(stream) |
---|
| 143 | stream.write(' <system-out><![CDATA[%s]]></system-out>\n' % out) |
---|
| 144 | stream.write(' <system-err><![CDATA[%s]]></system-err>\n' % err) |
---|
| 145 | stream.write('</testsuite>\n') |
---|
| 146 | |
---|
| 147 | |
---|
| 148 | class XMLTestRunner(object): |
---|
| 149 | |
---|
| 150 | """A test runner that stores results in XML format compatible with JUnit. |
---|
| 151 | |
---|
| 152 | XMLTestRunner(stream=None) -> XML test runner |
---|
| 153 | |
---|
| 154 | The XML file is written to the supplied stream. If stream is None, the |
---|
| 155 | results are stored in a file called TEST-<module>.<class>.xml in the |
---|
| 156 | current working directory (if not overridden with the path property), |
---|
| 157 | where <module> and <class> are the module and class name of the test class. |
---|
| 158 | |
---|
| 159 | """ |
---|
| 160 | |
---|
| 161 | def __init__(self, *args, **kwargs): |
---|
| 162 | self._stream = kwargs.get('stream') |
---|
| 163 | self._filename = kwargs.get('filename') |
---|
| 164 | self._path = "." |
---|
| 165 | |
---|
| 166 | def run(self, test): |
---|
| 167 | """Run the given test case or test suite.""" |
---|
| 168 | class_ = test.__class__ |
---|
| 169 | classname = class_.__module__ + "." + class_.__name__ |
---|
| 170 | if self._stream == None: |
---|
| 171 | filename = "TEST-%s.xml" % classname |
---|
| 172 | if self._filename: |
---|
| 173 | filename = self._filename |
---|
| 174 | stream = file(os.path.join(self._path, filename), "w") |
---|
| 175 | stream.write('<?xml version="1.0" encoding="utf-8"?>\n') |
---|
| 176 | else: |
---|
| 177 | stream = self._stream |
---|
| 178 | |
---|
| 179 | result = _XMLTestResult(classname) |
---|
| 180 | start_time = time.time() |
---|
| 181 | |
---|
| 182 | # TODO: Python 2.5: Use the with statement |
---|
| 183 | old_stdout = sys.stdout |
---|
| 184 | old_stderr = sys.stderr |
---|
| 185 | sys.stdout = StringIO() |
---|
| 186 | sys.stderr = StringIO() |
---|
| 187 | |
---|
| 188 | try: |
---|
| 189 | test(result) |
---|
| 190 | try: |
---|
| 191 | out_s = sys.stdout.getvalue() |
---|
| 192 | except AttributeError: |
---|
| 193 | out_s = "" |
---|
| 194 | try: |
---|
| 195 | err_s = sys.stderr.getvalue() |
---|
| 196 | except AttributeError: |
---|
| 197 | err_s = "" |
---|
| 198 | finally: |
---|
| 199 | sys.stdout = old_stdout |
---|
| 200 | sys.stderr = old_stderr |
---|
| 201 | |
---|
| 202 | time_taken = time.time() - start_time |
---|
| 203 | result.print_report(stream, time_taken, out_s, err_s) |
---|
| 204 | if self._stream == None: |
---|
| 205 | stream.close() |
---|
| 206 | |
---|
| 207 | return result |
---|
| 208 | |
---|
| 209 | def _set_path(self, path): |
---|
| 210 | self._path = path |
---|
| 211 | |
---|
| 212 | path = property(lambda self: self._path, _set_path, None, |
---|
| 213 | """The path where the XML files are stored. |
---|
| 214 | |
---|
| 215 | This property is ignored when the XML file is written to a file |
---|
| 216 | stream.""") |
---|
| 217 | |
---|
| 218 | |
---|
| 219 | class XMLTestRunnerTest(unittest.TestCase): |
---|
| 220 | def setUp(self): |
---|
| 221 | self._stream = StringIO() |
---|
| 222 | |
---|
| 223 | def _try_test_run(self, test_class, expected): |
---|
| 224 | |
---|
| 225 | """Run the test suite against the supplied test class and compare the |
---|
| 226 | XML result against the expected XML string. Fail if the expected |
---|
| 227 | string doesn't match the actual string. All time attribute in the |
---|
| 228 | expected string should have the value "0.000". All error and failure |
---|
| 229 | messages are reduced to "Foobar". |
---|
| 230 | |
---|
| 231 | """ |
---|
| 232 | |
---|
| 233 | runner = XMLTestRunner(self._stream) |
---|
| 234 | runner.run(unittest.makeSuite(test_class)) |
---|
| 235 | |
---|
| 236 | got = self._stream.getvalue() |
---|
| 237 | # Replace all time="X.YYY" attributes by time="0.000" to enable a |
---|
| 238 | # simple string comparison. |
---|
| 239 | got = re.sub(r'time="\d+\.\d+"', 'time="0.000"', got) |
---|
| 240 | # Likewise, replace all failure and error messages by a simple "Foobar" |
---|
| 241 | # string. |
---|
| 242 | got = re.sub(r'(?s)<failure (.*?)>.*?</failure>', r'<failure \1>Foobar</failure>', got) |
---|
| 243 | got = re.sub(r'(?s)<error (.*?)>.*?</error>', r'<error \1>Foobar</error>', got) |
---|
| 244 | |
---|
| 245 | self.assertEqual(expected, got) |
---|
| 246 | |
---|
| 247 | def test_no_tests(self): |
---|
| 248 | """Regression test: Check whether a test run without any tests |
---|
| 249 | matches a previous run. |
---|
| 250 | |
---|
| 251 | """ |
---|
| 252 | class TestTest(unittest.TestCase): |
---|
| 253 | pass |
---|
| 254 | self._try_test_run(TestTest, """<testsuite errors="0" failures="0" name="unittest.TestSuite" tests="0" time="0.000"> |
---|
| 255 | <system-out><![CDATA[]]></system-out> |
---|
| 256 | <system-err><![CDATA[]]></system-err> |
---|
| 257 | </testsuite> |
---|
| 258 | """) |
---|
| 259 | |
---|
| 260 | def test_success(self): |
---|
| 261 | """Regression test: Check whether a test run with a successful test |
---|
| 262 | matches a previous run. |
---|
| 263 | |
---|
| 264 | """ |
---|
| 265 | class TestTest(unittest.TestCase): |
---|
| 266 | def test_foo(self): |
---|
| 267 | pass |
---|
| 268 | self._try_test_run(TestTest, """<testsuite errors="0" failures="0" name="unittest.TestSuite" tests="1" time="0.000"> |
---|
| 269 | <testcase classname="__main__.TestTest" name="test_foo" time="0.000"></testcase> |
---|
| 270 | <system-out><![CDATA[]]></system-out> |
---|
| 271 | <system-err><![CDATA[]]></system-err> |
---|
| 272 | </testsuite> |
---|
| 273 | """) |
---|
| 274 | |
---|
| 275 | def test_failure(self): |
---|
| 276 | """Regression test: Check whether a test run with a failing test |
---|
| 277 | matches a previous run. |
---|
| 278 | |
---|
| 279 | """ |
---|
| 280 | class TestTest(unittest.TestCase): |
---|
| 281 | def test_foo(self): |
---|
| 282 | self.assert_(False) |
---|
| 283 | self._try_test_run(TestTest, """<testsuite errors="0" failures="1" name="unittest.TestSuite" tests="1" time="0.000"> |
---|
| 284 | <testcase classname="__main__.TestTest" name="test_foo" time="0.000"> |
---|
| 285 | <failure type="exceptions.AssertionError">Foobar</failure> |
---|
| 286 | </testcase> |
---|
| 287 | <system-out><![CDATA[]]></system-out> |
---|
| 288 | <system-err><![CDATA[]]></system-err> |
---|
| 289 | </testsuite> |
---|
| 290 | """) |
---|
| 291 | |
---|
| 292 | def test_error(self): |
---|
| 293 | """Regression test: Check whether a test run with a erroneous test |
---|
| 294 | matches a previous run. |
---|
| 295 | |
---|
| 296 | """ |
---|
| 297 | class TestTest(unittest.TestCase): |
---|
| 298 | def test_foo(self): |
---|
| 299 | raise IndexError() |
---|
| 300 | self._try_test_run(TestTest, """<testsuite errors="1" failures="0" name="unittest.TestSuite" tests="1" time="0.000"> |
---|
| 301 | <testcase classname="__main__.TestTest" name="test_foo" time="0.000"> |
---|
| 302 | <error type="exceptions.IndexError">Foobar</error> |
---|
| 303 | </testcase> |
---|
| 304 | <system-out><![CDATA[]]></system-out> |
---|
| 305 | <system-err><![CDATA[]]></system-err> |
---|
| 306 | </testsuite> |
---|
| 307 | """) |
---|
| 308 | |
---|
| 309 | def test_stdout_capture(self): |
---|
| 310 | """Regression test: Check whether a test run with output to stdout |
---|
| 311 | matches a previous run. |
---|
| 312 | |
---|
| 313 | """ |
---|
| 314 | class TestTest(unittest.TestCase): |
---|
| 315 | def test_foo(self): |
---|
| 316 | print "Test" |
---|
| 317 | self._try_test_run(TestTest, """<testsuite errors="0" failures="0" name="unittest.TestSuite" tests="1" time="0.000"> |
---|
| 318 | <testcase classname="__main__.TestTest" name="test_foo" time="0.000"></testcase> |
---|
| 319 | <system-out><![CDATA[Test |
---|
| 320 | ]]></system-out> |
---|
| 321 | <system-err><![CDATA[]]></system-err> |
---|
| 322 | </testsuite> |
---|
| 323 | """) |
---|
| 324 | |
---|
| 325 | def test_stderr_capture(self): |
---|
| 326 | """Regression test: Check whether a test run with output to stderr |
---|
| 327 | matches a previous run. |
---|
| 328 | |
---|
| 329 | """ |
---|
| 330 | class TestTest(unittest.TestCase): |
---|
| 331 | def test_foo(self): |
---|
| 332 | print >>sys.stderr, "Test" |
---|
| 333 | self._try_test_run(TestTest, """<testsuite errors="0" failures="0" name="unittest.TestSuite" tests="1" time="0.000"> |
---|
| 334 | <testcase classname="__main__.TestTest" name="test_foo" time="0.000"></testcase> |
---|
| 335 | <system-out><![CDATA[]]></system-out> |
---|
| 336 | <system-err><![CDATA[Test |
---|
| 337 | ]]></system-err> |
---|
| 338 | </testsuite> |
---|
| 339 | """) |
---|
| 340 | |
---|
| 341 | class NullStream(object): |
---|
| 342 | """A file-like object that discards everything written to it.""" |
---|
| 343 | def write(self, buffer): |
---|
| 344 | pass |
---|
| 345 | |
---|
| 346 | def test_unittests_changing_stdout(self): |
---|
| 347 | """Check whether the XMLTestRunner recovers gracefully from unit tests |
---|
| 348 | that change stdout, but don't change it back properly. |
---|
| 349 | |
---|
| 350 | """ |
---|
| 351 | class TestTest(unittest.TestCase): |
---|
| 352 | def test_foo(self): |
---|
| 353 | sys.stdout = XMLTestRunnerTest.NullStream() |
---|
| 354 | |
---|
| 355 | runner = XMLTestRunner(self._stream) |
---|
| 356 | runner.run(unittest.makeSuite(TestTest)) |
---|
| 357 | |
---|
| 358 | def test_unittests_changing_stderr(self): |
---|
| 359 | """Check whether the XMLTestRunner recovers gracefully from unit tests |
---|
| 360 | that change stderr, but don't change it back properly. |
---|
| 361 | |
---|
| 362 | """ |
---|
| 363 | class TestTest(unittest.TestCase): |
---|
| 364 | def test_foo(self): |
---|
| 365 | sys.stderr = XMLTestRunnerTest.NullStream() |
---|
| 366 | |
---|
| 367 | runner = XMLTestRunner(self._stream) |
---|
| 368 | runner.run(unittest.makeSuite(TestTest)) |
---|
| 369 | |
---|
| 370 | |
---|
| 371 | class XMLTestProgram(unittest.TestProgram): |
---|
| 372 | def runTests(self): |
---|
| 373 | if self.testRunner is None: |
---|
| 374 | self.testRunner = XMLTestRunner() |
---|
| 375 | unittest.TestProgram.runTests(self) |
---|
| 376 | |
---|
| 377 | main = XMLTestProgram |
---|
| 378 | |
---|
| 379 | |
---|
| 380 | if __name__ == "__main__": |
---|
| 381 | main(module=None) |
---|