[3] | 1 | """ |
---|
| 2 | This plugin adds a test id (like #1) to each test name output. After |
---|
| 3 | you've run once to generate test ids, you can re-run individual |
---|
| 4 | tests by activating the plugin and passing the ids (with or |
---|
| 5 | without the # prefix) instead of test names. |
---|
| 6 | |
---|
| 7 | For example, if your normal test run looks like:: |
---|
| 8 | |
---|
| 9 | % nosetests -v |
---|
| 10 | tests.test_a ... ok |
---|
| 11 | tests.test_b ... ok |
---|
| 12 | tests.test_c ... ok |
---|
| 13 | |
---|
| 14 | When adding ``--with-id`` you'll see:: |
---|
| 15 | |
---|
| 16 | % nosetests -v --with-id |
---|
| 17 | #1 tests.test_a ... ok |
---|
| 18 | #2 tests.test_b ... ok |
---|
| 19 | #2 tests.test_c ... ok |
---|
| 20 | |
---|
| 21 | Then you can re-run individual tests by supplying just an id number:: |
---|
| 22 | |
---|
| 23 | % nosetests -v --with-id 2 |
---|
| 24 | #2 tests.test_b ... ok |
---|
| 25 | |
---|
| 26 | You can also pass multiple id numbers:: |
---|
| 27 | |
---|
| 28 | % nosetests -v --with-id 2 3 |
---|
| 29 | #2 tests.test_b ... ok |
---|
| 30 | #3 tests.test_c ... ok |
---|
| 31 | |
---|
| 32 | Since most shells consider '#' a special character, you can leave it out when |
---|
| 33 | specifying a test id. |
---|
| 34 | |
---|
| 35 | Note that when run without the -v switch, no special output is displayed, but |
---|
| 36 | the ids file is still written. |
---|
| 37 | |
---|
| 38 | Looping over failed tests |
---|
| 39 | ------------------------- |
---|
| 40 | |
---|
| 41 | This plugin also adds a mode that will direct the test runner to record |
---|
| 42 | failed tests. Subsequent test runs will then run only the tests that failed |
---|
| 43 | last time. Activate this mode with the ``--failed`` switch:: |
---|
| 44 | |
---|
| 45 | % nosetests -v --failed |
---|
| 46 | #1 test.test_a ... ok |
---|
| 47 | #2 test.test_b ... ERROR |
---|
| 48 | #3 test.test_c ... FAILED |
---|
| 49 | #4 test.test_d ... ok |
---|
| 50 | |
---|
| 51 | On the second run, only tests #2 and #3 will run:: |
---|
| 52 | |
---|
| 53 | % nosetests -v --failed |
---|
| 54 | #2 test.test_b ... ERROR |
---|
| 55 | #3 test.test_c ... FAILED |
---|
| 56 | |
---|
| 57 | As you correct errors and tests pass, they'll drop out of subsequent runs. |
---|
| 58 | |
---|
| 59 | First:: |
---|
| 60 | |
---|
| 61 | % nosetests -v --failed |
---|
| 62 | #2 test.test_b ... ok |
---|
| 63 | #3 test.test_c ... FAILED |
---|
| 64 | |
---|
| 65 | Second:: |
---|
| 66 | |
---|
| 67 | % nosetests -v --failed |
---|
| 68 | #3 test.test_c ... FAILED |
---|
| 69 | |
---|
| 70 | When all tests pass, the full set will run on the next invocation. |
---|
| 71 | |
---|
| 72 | First:: |
---|
| 73 | |
---|
| 74 | % nosetests -v --failed |
---|
| 75 | #3 test.test_c ... ok |
---|
| 76 | |
---|
| 77 | Second:: |
---|
| 78 | |
---|
| 79 | % nosetests -v --failed |
---|
| 80 | #1 test.test_a ... ok |
---|
| 81 | #2 test.test_b ... ok |
---|
| 82 | #3 test.test_c ... ok |
---|
| 83 | #4 test.test_d ... ok |
---|
| 84 | |
---|
| 85 | .. note :: |
---|
| 86 | |
---|
| 87 | If you expect to use ``--failed`` regularly, it's a good idea to always run |
---|
| 88 | run using the ``--with-id`` option. This will ensure that an id file is |
---|
| 89 | always created, allowing you to add ``--failed`` to the command line as soon |
---|
| 90 | as you have failing tests. Otherwise, your first run using ``--failed`` will |
---|
| 91 | (perhaps surprisingly) run *all* tests, because there won't be an id file |
---|
| 92 | containing the record of failed tests from your previous run. |
---|
| 93 | |
---|
| 94 | """ |
---|
| 95 | __test__ = False |
---|
| 96 | |
---|
| 97 | import logging |
---|
| 98 | import os |
---|
| 99 | from nose.plugins import Plugin |
---|
| 100 | from nose.util import src, set |
---|
| 101 | |
---|
| 102 | try: |
---|
| 103 | from cPickle import dump, load |
---|
| 104 | except ImportError: |
---|
| 105 | from pickle import dump, load |
---|
| 106 | |
---|
| 107 | log = logging.getLogger(__name__) |
---|
| 108 | |
---|
| 109 | |
---|
| 110 | class TestId(Plugin): |
---|
| 111 | """ |
---|
| 112 | Activate to add a test id (like #1) to each test name output. Activate |
---|
| 113 | with --failed to rerun failing tests only. |
---|
| 114 | """ |
---|
| 115 | name = 'id' |
---|
| 116 | idfile = None |
---|
| 117 | collecting = True |
---|
| 118 | loopOnFailed = False |
---|
| 119 | |
---|
| 120 | def options(self, parser, env): |
---|
| 121 | """Register commandline options. |
---|
| 122 | """ |
---|
| 123 | Plugin.options(self, parser, env) |
---|
| 124 | parser.add_option('--id-file', action='store', dest='testIdFile', |
---|
| 125 | default='.noseids', metavar="FILE", |
---|
| 126 | help="Store test ids found in test runs in this " |
---|
| 127 | "file. Default is the file .noseids in the " |
---|
| 128 | "working directory.") |
---|
| 129 | parser.add_option('--failed', action='store_true', |
---|
| 130 | dest='failed', default=False, |
---|
| 131 | help="Run the tests that failed in the last " |
---|
| 132 | "test run.") |
---|
| 133 | |
---|
| 134 | def configure(self, options, conf): |
---|
| 135 | """Configure plugin. |
---|
| 136 | """ |
---|
| 137 | Plugin.configure(self, options, conf) |
---|
| 138 | if options.failed: |
---|
| 139 | self.enabled = True |
---|
| 140 | self.loopOnFailed = True |
---|
| 141 | log.debug("Looping on failed tests") |
---|
| 142 | self.idfile = os.path.expanduser(options.testIdFile) |
---|
| 143 | if not os.path.isabs(self.idfile): |
---|
| 144 | self.idfile = os.path.join(conf.workingDir, self.idfile) |
---|
| 145 | self.id = 1 |
---|
| 146 | # Ids and tests are mirror images: ids are {id: test address} and |
---|
| 147 | # tests are {test address: id} |
---|
| 148 | self.ids = {} |
---|
| 149 | self.tests = {} |
---|
| 150 | self.failed = [] |
---|
| 151 | self.source_names = [] |
---|
| 152 | # used to track ids seen when tests is filled from |
---|
| 153 | # loaded ids file |
---|
| 154 | self._seen = {} |
---|
| 155 | self._write_hashes = options.verbosity >= 2 |
---|
| 156 | |
---|
| 157 | def finalize(self, result): |
---|
| 158 | """Save new ids file, if needed. |
---|
| 159 | """ |
---|
| 160 | if result.wasSuccessful(): |
---|
| 161 | self.failed = [] |
---|
| 162 | if self.collecting: |
---|
| 163 | ids = dict(zip(self.tests.values(), self.tests.keys())) |
---|
| 164 | else: |
---|
| 165 | ids = self.ids |
---|
| 166 | fh = open(self.idfile, 'w') |
---|
| 167 | dump({'ids': ids, |
---|
| 168 | 'failed': self.failed, |
---|
| 169 | 'source_names': self.source_names}, fh) |
---|
| 170 | fh.close() |
---|
| 171 | log.debug('Saved test ids: %s, failed %s to %s', |
---|
| 172 | ids, self.failed, self.idfile) |
---|
| 173 | |
---|
| 174 | def loadTestsFromNames(self, names, module=None): |
---|
| 175 | """Translate ids in the list of requested names into their |
---|
| 176 | test addresses, if they are found in my dict of tests. |
---|
| 177 | """ |
---|
| 178 | log.debug('ltfn %s %s', names, module) |
---|
| 179 | try: |
---|
| 180 | fh = open(self.idfile, 'r') |
---|
| 181 | data = load(fh) |
---|
| 182 | if 'ids' in data: |
---|
| 183 | self.ids = data['ids'] |
---|
| 184 | self.failed = data['failed'] |
---|
| 185 | self.source_names = data['source_names'] |
---|
| 186 | else: |
---|
| 187 | # old ids field |
---|
| 188 | self.ids = data |
---|
| 189 | self.failed = [] |
---|
| 190 | self.source_names = names |
---|
| 191 | if self.ids: |
---|
| 192 | self.id = max(self.ids) + 1 |
---|
| 193 | self.tests = dict(zip(self.ids.values(), self.ids.keys())) |
---|
| 194 | else: |
---|
| 195 | self.id = 1 |
---|
| 196 | log.debug( |
---|
| 197 | 'Loaded test ids %s tests %s failed %s sources %s from %s', |
---|
| 198 | self.ids, self.tests, self.failed, self.source_names, |
---|
| 199 | self.idfile) |
---|
| 200 | fh.close() |
---|
| 201 | except IOError: |
---|
| 202 | log.debug('IO error reading %s', self.idfile) |
---|
| 203 | |
---|
| 204 | if self.loopOnFailed and self.failed: |
---|
| 205 | self.collecting = False |
---|
| 206 | names = self.failed |
---|
| 207 | self.failed = [] |
---|
| 208 | # I don't load any tests myself, only translate names like '#2' |
---|
| 209 | # into the associated test addresses |
---|
| 210 | translated = [] |
---|
| 211 | new_source = [] |
---|
| 212 | really_new = [] |
---|
| 213 | for name in names: |
---|
| 214 | trans = self.tr(name) |
---|
| 215 | if trans != name: |
---|
| 216 | translated.append(trans) |
---|
| 217 | else: |
---|
| 218 | new_source.append(name) |
---|
| 219 | # names that are not ids and that are not in the current |
---|
| 220 | # list of source names go into the list for next time |
---|
| 221 | if new_source: |
---|
| 222 | new_set = set(new_source) |
---|
| 223 | old_set = set(self.source_names) |
---|
| 224 | log.debug("old: %s new: %s", old_set, new_set) |
---|
| 225 | really_new = [s for s in new_source |
---|
| 226 | if not s in old_set] |
---|
| 227 | if really_new: |
---|
| 228 | # remember new sources |
---|
| 229 | self.source_names.extend(really_new) |
---|
| 230 | if not translated: |
---|
| 231 | # new set of source names, no translations |
---|
| 232 | # means "run the requested tests" |
---|
| 233 | names = new_source |
---|
| 234 | else: |
---|
| 235 | # no new names to translate and add to id set |
---|
| 236 | self.collecting = False |
---|
| 237 | log.debug("translated: %s new sources %s names %s", |
---|
| 238 | translated, really_new, names) |
---|
| 239 | return (None, translated + really_new or names) |
---|
| 240 | |
---|
| 241 | def makeName(self, addr): |
---|
| 242 | log.debug("Make name %s", addr) |
---|
| 243 | filename, module, call = addr |
---|
| 244 | if filename is not None: |
---|
| 245 | head = src(filename) |
---|
| 246 | else: |
---|
| 247 | head = module |
---|
| 248 | if call is not None: |
---|
| 249 | return "%s:%s" % (head, call) |
---|
| 250 | return head |
---|
| 251 | |
---|
| 252 | def setOutputStream(self, stream): |
---|
| 253 | """Get handle on output stream so the plugin can print id #s |
---|
| 254 | """ |
---|
| 255 | self.stream = stream |
---|
| 256 | |
---|
| 257 | def startTest(self, test): |
---|
| 258 | """Maybe output an id # before the test name. |
---|
| 259 | |
---|
| 260 | Example output:: |
---|
| 261 | |
---|
| 262 | #1 test.test ... ok |
---|
| 263 | #2 test.test_two ... ok |
---|
| 264 | |
---|
| 265 | """ |
---|
| 266 | adr = test.address() |
---|
| 267 | log.debug('start test %s (%s)', adr, adr in self.tests) |
---|
| 268 | if adr in self.tests: |
---|
| 269 | if adr in self._seen: |
---|
| 270 | self.write(' ') |
---|
| 271 | else: |
---|
| 272 | self.write('#%s ' % self.tests[adr]) |
---|
| 273 | self._seen[adr] = 1 |
---|
| 274 | return |
---|
| 275 | self.tests[adr] = self.id |
---|
| 276 | self.write('#%s ' % self.id) |
---|
| 277 | self.id += 1 |
---|
| 278 | |
---|
| 279 | def afterTest(self, test): |
---|
| 280 | # None means test never ran, False means failed/err |
---|
| 281 | if test.passed is False: |
---|
| 282 | try: |
---|
| 283 | key = str(self.tests[test.address()]) |
---|
| 284 | except KeyError: |
---|
| 285 | # never saw this test -- startTest didn't run |
---|
| 286 | pass |
---|
| 287 | else: |
---|
| 288 | if key not in self.failed: |
---|
| 289 | self.failed.append(key) |
---|
| 290 | |
---|
| 291 | def tr(self, name): |
---|
| 292 | log.debug("tr '%s'", name) |
---|
| 293 | try: |
---|
| 294 | key = int(name.replace('#', '')) |
---|
| 295 | except ValueError: |
---|
| 296 | return name |
---|
| 297 | log.debug("Got key %s", key) |
---|
| 298 | # I'm running tests mapped from the ids file, |
---|
| 299 | # not collecting new ones |
---|
| 300 | if key in self.ids: |
---|
| 301 | return self.makeName(self.ids[key]) |
---|
| 302 | return name |
---|
| 303 | |
---|
| 304 | def write(self, output): |
---|
| 305 | if self._write_hashes: |
---|
| 306 | self.stream.write(output) |
---|