[3] | 1 | # (c) 2005 Ian Bicking and contributors; written for Paste (http://pythonpaste.org) |
---|
| 2 | # Licensed under the MIT license: http://www.opensource.org/licenses/mit-license.php |
---|
| 3 | #!/usr/bin/env python2.4 |
---|
| 4 | # (c) 2005 Ian Bicking and contributors; written for Paste (http://pythonpaste.org) |
---|
| 5 | # Licensed under the MIT license: http://www.opensource.org/licenses/mit-license.php |
---|
| 6 | |
---|
| 7 | """ |
---|
| 8 | These are functions for use when doctest-testing a document. |
---|
| 9 | """ |
---|
| 10 | |
---|
| 11 | try: |
---|
| 12 | import subprocess |
---|
| 13 | except ImportError: |
---|
| 14 | from paste.util import subprocess24 as subprocess |
---|
| 15 | import doctest |
---|
| 16 | import os |
---|
| 17 | import sys |
---|
| 18 | import shutil |
---|
| 19 | import re |
---|
| 20 | import cgi |
---|
| 21 | import rfc822 |
---|
| 22 | from cStringIO import StringIO |
---|
| 23 | from paste.util import PySourceColor |
---|
| 24 | |
---|
| 25 | |
---|
| 26 | here = os.path.abspath(__file__) |
---|
| 27 | paste_parent = os.path.dirname( |
---|
| 28 | os.path.dirname(os.path.dirname(here))) |
---|
| 29 | |
---|
| 30 | def run(command): |
---|
| 31 | data = run_raw(command) |
---|
| 32 | if data: |
---|
| 33 | print data |
---|
| 34 | |
---|
| 35 | def run_raw(command): |
---|
| 36 | """ |
---|
| 37 | Runs the string command, returns any output. |
---|
| 38 | """ |
---|
| 39 | proc = subprocess.Popen(command, shell=True, |
---|
| 40 | stderr=subprocess.STDOUT, |
---|
| 41 | stdout=subprocess.PIPE, env=_make_env()) |
---|
| 42 | data = proc.stdout.read() |
---|
| 43 | proc.wait() |
---|
| 44 | while data.endswith('\n') or data.endswith('\r'): |
---|
| 45 | data = data[:-1] |
---|
| 46 | if data: |
---|
| 47 | data = '\n'.join( |
---|
| 48 | [l for l in data.splitlines() if l]) |
---|
| 49 | return data |
---|
| 50 | else: |
---|
| 51 | return '' |
---|
| 52 | |
---|
| 53 | def run_command(command, name, and_print=False): |
---|
| 54 | output = run_raw(command) |
---|
| 55 | data = '$ %s\n%s' % (command, output) |
---|
| 56 | show_file('shell-command', name, description='shell transcript', |
---|
| 57 | data=data) |
---|
| 58 | if and_print and output: |
---|
| 59 | print output |
---|
| 60 | |
---|
| 61 | def _make_env(): |
---|
| 62 | env = os.environ.copy() |
---|
| 63 | env['PATH'] = (env.get('PATH', '') |
---|
| 64 | + ':' |
---|
| 65 | + os.path.join(paste_parent, 'scripts') |
---|
| 66 | + ':' |
---|
| 67 | + os.path.join(paste_parent, 'paste', '3rd-party', |
---|
| 68 | 'sqlobject-files', 'scripts')) |
---|
| 69 | env['PYTHONPATH'] = (env.get('PYTHONPATH', '') |
---|
| 70 | + ':' |
---|
| 71 | + paste_parent) |
---|
| 72 | return env |
---|
| 73 | |
---|
| 74 | def clear_dir(dir): |
---|
| 75 | """ |
---|
| 76 | Clears (deletes) the given directory |
---|
| 77 | """ |
---|
| 78 | shutil.rmtree(dir, True) |
---|
| 79 | |
---|
| 80 | def ls(dir=None, recurse=False, indent=0): |
---|
| 81 | """ |
---|
| 82 | Show a directory listing |
---|
| 83 | """ |
---|
| 84 | dir = dir or os.getcwd() |
---|
| 85 | fns = os.listdir(dir) |
---|
| 86 | fns.sort() |
---|
| 87 | for fn in fns: |
---|
| 88 | full = os.path.join(dir, fn) |
---|
| 89 | if os.path.isdir(full): |
---|
| 90 | fn = fn + '/' |
---|
| 91 | print ' '*indent + fn |
---|
| 92 | if os.path.isdir(full) and recurse: |
---|
| 93 | ls(dir=full, recurse=True, indent=indent+2) |
---|
| 94 | |
---|
| 95 | default_app = None |
---|
| 96 | default_url = None |
---|
| 97 | |
---|
| 98 | def set_default_app(app, url): |
---|
| 99 | global default_app |
---|
| 100 | global default_url |
---|
| 101 | default_app = app |
---|
| 102 | default_url = url |
---|
| 103 | |
---|
| 104 | def resource_filename(fn): |
---|
| 105 | """ |
---|
| 106 | Returns the filename of the resource -- generally in the directory |
---|
| 107 | resources/DocumentName/fn |
---|
| 108 | """ |
---|
| 109 | return os.path.join( |
---|
| 110 | os.path.dirname(sys.testing_document_filename), |
---|
| 111 | 'resources', |
---|
| 112 | os.path.splitext(os.path.basename(sys.testing_document_filename))[0], |
---|
| 113 | fn) |
---|
| 114 | |
---|
| 115 | def show(path_info, example_name): |
---|
| 116 | fn = resource_filename(example_name + '.html') |
---|
| 117 | out = StringIO() |
---|
| 118 | assert default_app is not None, ( |
---|
| 119 | "No default_app set") |
---|
| 120 | url = default_url + path_info |
---|
| 121 | out.write('<span class="doctest-url"><a href="%s">%s</a></span><br>\n' |
---|
| 122 | % (url, url)) |
---|
| 123 | out.write('<div class="doctest-example">\n') |
---|
| 124 | proc = subprocess.Popen( |
---|
| 125 | ['paster', 'serve' '--server=console', '--no-verbose', |
---|
| 126 | '--url=' + path_info], |
---|
| 127 | stderr=subprocess.PIPE, |
---|
| 128 | stdout=subprocess.PIPE, |
---|
| 129 | env=_make_env()) |
---|
| 130 | stdout, errors = proc.communicate() |
---|
| 131 | stdout = StringIO(stdout) |
---|
| 132 | headers = rfc822.Message(stdout) |
---|
| 133 | content = stdout.read() |
---|
| 134 | for header, value in headers.items(): |
---|
| 135 | if header.lower() == 'status' and int(value.split()[0]) == 200: |
---|
| 136 | continue |
---|
| 137 | if header.lower() in ('content-type', 'content-length'): |
---|
| 138 | continue |
---|
| 139 | if (header.lower() == 'set-cookie' |
---|
| 140 | and value.startswith('_SID_')): |
---|
| 141 | continue |
---|
| 142 | out.write('<span class="doctest-header">%s: %s</span><br>\n' |
---|
| 143 | % (header, value)) |
---|
| 144 | lines = [l for l in content.splitlines() if l.strip()] |
---|
| 145 | for line in lines: |
---|
| 146 | out.write(line + '\n') |
---|
| 147 | if errors: |
---|
| 148 | out.write('<pre class="doctest-errors">%s</pre>' |
---|
| 149 | % errors) |
---|
| 150 | out.write('</div>\n') |
---|
| 151 | result = out.getvalue() |
---|
| 152 | if not os.path.exists(fn): |
---|
| 153 | f = open(fn, 'wb') |
---|
| 154 | f.write(result) |
---|
| 155 | f.close() |
---|
| 156 | else: |
---|
| 157 | f = open(fn, 'rb') |
---|
| 158 | expected = f.read() |
---|
| 159 | f.close() |
---|
| 160 | if not html_matches(expected, result): |
---|
| 161 | print 'Pages did not match. Expected from %s:' % fn |
---|
| 162 | print '-'*60 |
---|
| 163 | print expected |
---|
| 164 | print '='*60 |
---|
| 165 | print 'Actual output:' |
---|
| 166 | print '-'*60 |
---|
| 167 | print result |
---|
| 168 | |
---|
| 169 | def html_matches(pattern, text): |
---|
| 170 | regex = re.escape(pattern) |
---|
| 171 | regex = regex.replace(r'\.\.\.', '.*') |
---|
| 172 | regex = re.sub(r'0x[0-9a-f]+', '.*', regex) |
---|
| 173 | regex = '^%s$' % regex |
---|
| 174 | return re.search(regex, text) |
---|
| 175 | |
---|
| 176 | def convert_docstring_string(data): |
---|
| 177 | if data.startswith('\n'): |
---|
| 178 | data = data[1:] |
---|
| 179 | lines = data.splitlines() |
---|
| 180 | new_lines = [] |
---|
| 181 | for line in lines: |
---|
| 182 | if line.rstrip() == '.': |
---|
| 183 | new_lines.append('') |
---|
| 184 | else: |
---|
| 185 | new_lines.append(line) |
---|
| 186 | data = '\n'.join(new_lines) + '\n' |
---|
| 187 | return data |
---|
| 188 | |
---|
| 189 | def create_file(path, version, data): |
---|
| 190 | data = convert_docstring_string(data) |
---|
| 191 | write_data(path, data) |
---|
| 192 | show_file(path, version) |
---|
| 193 | |
---|
| 194 | def append_to_file(path, version, data): |
---|
| 195 | data = convert_docstring_string(data) |
---|
| 196 | f = open(path, 'a') |
---|
| 197 | f.write(data) |
---|
| 198 | f.close() |
---|
| 199 | # I think these appends can happen so quickly (in less than a second) |
---|
| 200 | # that the .pyc file doesn't appear to be expired, even though it |
---|
| 201 | # is after we've made this change; so we have to get rid of the .pyc |
---|
| 202 | # file: |
---|
| 203 | if path.endswith('.py'): |
---|
| 204 | pyc_file = path + 'c' |
---|
| 205 | if os.path.exists(pyc_file): |
---|
| 206 | os.unlink(pyc_file) |
---|
| 207 | show_file(path, version, description='added to %s' % path, |
---|
| 208 | data=data) |
---|
| 209 | |
---|
| 210 | def show_file(path, version, description=None, data=None): |
---|
| 211 | ext = os.path.splitext(path)[1] |
---|
| 212 | if data is None: |
---|
| 213 | f = open(path, 'rb') |
---|
| 214 | data = f.read() |
---|
| 215 | f.close() |
---|
| 216 | if ext == '.py': |
---|
| 217 | html = ('<div class="source-code">%s</div>' |
---|
| 218 | % PySourceColor.str2html(data, PySourceColor.dark)) |
---|
| 219 | else: |
---|
| 220 | html = '<pre class="source-code">%s</pre>' % cgi.escape(data, 1) |
---|
| 221 | html = '<span class="source-filename">%s</span><br>%s' % ( |
---|
| 222 | description or path, html) |
---|
| 223 | write_data(resource_filename('%s.%s.gen.html' % (path, version)), |
---|
| 224 | html) |
---|
| 225 | |
---|
| 226 | def call_source_highlight(input, format): |
---|
| 227 | proc = subprocess.Popen(['source-highlight', '--out-format=html', |
---|
| 228 | '--no-doc', '--css=none', |
---|
| 229 | '--src-lang=%s' % format], shell=False, |
---|
| 230 | stdout=subprocess.PIPE) |
---|
| 231 | stdout, stderr = proc.communicate(input) |
---|
| 232 | result = stdout |
---|
| 233 | proc.wait() |
---|
| 234 | return result |
---|
| 235 | |
---|
| 236 | |
---|
| 237 | def write_data(path, data): |
---|
| 238 | dir = os.path.dirname(os.path.abspath(path)) |
---|
| 239 | if not os.path.exists(dir): |
---|
| 240 | os.makedirs(dir) |
---|
| 241 | f = open(path, 'wb') |
---|
| 242 | f.write(data) |
---|
| 243 | f.close() |
---|
| 244 | |
---|
| 245 | |
---|
| 246 | def change_file(path, changes): |
---|
| 247 | f = open(os.path.abspath(path), 'rb') |
---|
| 248 | lines = f.readlines() |
---|
| 249 | f.close() |
---|
| 250 | for change_type, line, text in changes: |
---|
| 251 | if change_type == 'insert': |
---|
| 252 | lines[line:line] = [text] |
---|
| 253 | elif change_type == 'delete': |
---|
| 254 | lines[line:text] = [] |
---|
| 255 | else: |
---|
| 256 | assert 0, ( |
---|
| 257 | "Unknown change_type: %r" % change_type) |
---|
| 258 | f = open(path, 'wb') |
---|
| 259 | f.write(''.join(lines)) |
---|
| 260 | f.close() |
---|
| 261 | |
---|
| 262 | class LongFormDocTestParser(doctest.DocTestParser): |
---|
| 263 | |
---|
| 264 | """ |
---|
| 265 | This parser recognizes some reST comments as commands, without |
---|
| 266 | prompts or expected output, like: |
---|
| 267 | |
---|
| 268 | .. run: |
---|
| 269 | |
---|
| 270 | do_this(... |
---|
| 271 | ...) |
---|
| 272 | """ |
---|
| 273 | |
---|
| 274 | _EXAMPLE_RE = re.compile(r""" |
---|
| 275 | # Source consists of a PS1 line followed by zero or more PS2 lines. |
---|
| 276 | (?: (?P<source> |
---|
| 277 | (?:^(?P<indent> [ ]*) >>> .*) # PS1 line |
---|
| 278 | (?:\n [ ]* \.\.\. .*)*) # PS2 lines |
---|
| 279 | \n? |
---|
| 280 | # Want consists of any non-blank lines that do not start with PS1. |
---|
| 281 | (?P<want> (?:(?![ ]*$) # Not a blank line |
---|
| 282 | (?![ ]*>>>) # Not a line starting with PS1 |
---|
| 283 | .*$\n? # But any other line |
---|
| 284 | )*)) |
---|
| 285 | | |
---|
| 286 | (?: # This is for longer commands that are prefixed with a reST |
---|
| 287 | # comment like '.. run:' (two colons makes that a directive). |
---|
| 288 | # These commands cannot have any output. |
---|
| 289 | |
---|
| 290 | (?:^\.\.[ ]*(?P<run>run):[ ]*\n) # Leading command/command |
---|
| 291 | (?:[ ]*\n)? # Blank line following |
---|
| 292 | (?P<runsource> |
---|
| 293 | (?:(?P<runindent> [ ]+)[^ ].*$) |
---|
| 294 | (?:\n [ ]+ .*)*) |
---|
| 295 | ) |
---|
| 296 | | |
---|
| 297 | (?: # This is for shell commands |
---|
| 298 | |
---|
| 299 | (?P<shellsource> |
---|
| 300 | (?:^(P<shellindent> [ ]*) [$] .*) # Shell line |
---|
| 301 | (?:\n [ ]* [>] .*)*) # Continuation |
---|
| 302 | \n? |
---|
| 303 | # Want consists of any non-blank lines that do not start with $ |
---|
| 304 | (?P<shellwant> (?:(?![ ]*$) |
---|
| 305 | (?![ ]*[$]$) |
---|
| 306 | .*$\n? |
---|
| 307 | )*)) |
---|
| 308 | """, re.MULTILINE | re.VERBOSE) |
---|
| 309 | |
---|
| 310 | def _parse_example(self, m, name, lineno): |
---|
| 311 | r""" |
---|
| 312 | Given a regular expression match from `_EXAMPLE_RE` (`m`), |
---|
| 313 | return a pair `(source, want)`, where `source` is the matched |
---|
| 314 | example's source code (with prompts and indentation stripped); |
---|
| 315 | and `want` is the example's expected output (with indentation |
---|
| 316 | stripped). |
---|
| 317 | |
---|
| 318 | `name` is the string's name, and `lineno` is the line number |
---|
| 319 | where the example starts; both are used for error messages. |
---|
| 320 | |
---|
| 321 | >>> def parseit(s): |
---|
| 322 | ... p = LongFormDocTestParser() |
---|
| 323 | ... return p._parse_example(p._EXAMPLE_RE.search(s), '<string>', 1) |
---|
| 324 | >>> parseit('>>> 1\n1') |
---|
| 325 | ('1', {}, '1', None) |
---|
| 326 | >>> parseit('>>> (1\n... +1)\n2') |
---|
| 327 | ('(1\n+1)', {}, '2', None) |
---|
| 328 | >>> parseit('.. run:\n\n test1\n test2\n') |
---|
| 329 | ('test1\ntest2', {}, '', None) |
---|
| 330 | """ |
---|
| 331 | # Get the example's indentation level. |
---|
| 332 | runner = m.group('run') or '' |
---|
| 333 | indent = len(m.group('%sindent' % runner)) |
---|
| 334 | |
---|
| 335 | # Divide source into lines; check that they're properly |
---|
| 336 | # indented; and then strip their indentation & prompts. |
---|
| 337 | source_lines = m.group('%ssource' % runner).split('\n') |
---|
| 338 | if runner: |
---|
| 339 | self._check_prefix(source_lines[1:], ' '*indent, name, lineno) |
---|
| 340 | else: |
---|
| 341 | self._check_prompt_blank(source_lines, indent, name, lineno) |
---|
| 342 | self._check_prefix(source_lines[2:], ' '*indent + '.', name, lineno) |
---|
| 343 | if runner: |
---|
| 344 | source = '\n'.join([sl[indent:] for sl in source_lines]) |
---|
| 345 | else: |
---|
| 346 | source = '\n'.join([sl[indent+4:] for sl in source_lines]) |
---|
| 347 | |
---|
| 348 | if runner: |
---|
| 349 | want = '' |
---|
| 350 | exc_msg = None |
---|
| 351 | else: |
---|
| 352 | # Divide want into lines; check that it's properly indented; and |
---|
| 353 | # then strip the indentation. Spaces before the last newline should |
---|
| 354 | # be preserved, so plain rstrip() isn't good enough. |
---|
| 355 | want = m.group('want') |
---|
| 356 | want_lines = want.split('\n') |
---|
| 357 | if len(want_lines) > 1 and re.match(r' *$', want_lines[-1]): |
---|
| 358 | del want_lines[-1] # forget final newline & spaces after it |
---|
| 359 | self._check_prefix(want_lines, ' '*indent, name, |
---|
| 360 | lineno + len(source_lines)) |
---|
| 361 | want = '\n'.join([wl[indent:] for wl in want_lines]) |
---|
| 362 | |
---|
| 363 | # If `want` contains a traceback message, then extract it. |
---|
| 364 | m = self._EXCEPTION_RE.match(want) |
---|
| 365 | if m: |
---|
| 366 | exc_msg = m.group('msg') |
---|
| 367 | else: |
---|
| 368 | exc_msg = None |
---|
| 369 | |
---|
| 370 | # Extract options from the source. |
---|
| 371 | options = self._find_options(source, name, lineno) |
---|
| 372 | |
---|
| 373 | return source, options, want, exc_msg |
---|
| 374 | |
---|
| 375 | |
---|
| 376 | def parse(self, string, name='<string>'): |
---|
| 377 | """ |
---|
| 378 | Divide the given string into examples and intervening text, |
---|
| 379 | and return them as a list of alternating Examples and strings. |
---|
| 380 | Line numbers for the Examples are 0-based. The optional |
---|
| 381 | argument `name` is a name identifying this string, and is only |
---|
| 382 | used for error messages. |
---|
| 383 | """ |
---|
| 384 | string = string.expandtabs() |
---|
| 385 | # If all lines begin with the same indentation, then strip it. |
---|
| 386 | min_indent = self._min_indent(string) |
---|
| 387 | if min_indent > 0: |
---|
| 388 | string = '\n'.join([l[min_indent:] for l in string.split('\n')]) |
---|
| 389 | |
---|
| 390 | output = [] |
---|
| 391 | charno, lineno = 0, 0 |
---|
| 392 | # Find all doctest examples in the string: |
---|
| 393 | for m in self._EXAMPLE_RE.finditer(string): |
---|
| 394 | # Add the pre-example text to `output`. |
---|
| 395 | output.append(string[charno:m.start()]) |
---|
| 396 | # Update lineno (lines before this example) |
---|
| 397 | lineno += string.count('\n', charno, m.start()) |
---|
| 398 | # Extract info from the regexp match. |
---|
| 399 | (source, options, want, exc_msg) = \ |
---|
| 400 | self._parse_example(m, name, lineno) |
---|
| 401 | # Create an Example, and add it to the list. |
---|
| 402 | if not self._IS_BLANK_OR_COMMENT(source): |
---|
| 403 | # @@: Erg, this is the only line I need to change... |
---|
| 404 | output.append(doctest.Example( |
---|
| 405 | source, want, exc_msg, |
---|
| 406 | lineno=lineno, |
---|
| 407 | indent=min_indent+len(m.group('indent') or m.group('runindent')), |
---|
| 408 | options=options)) |
---|
| 409 | # Update lineno (lines inside this example) |
---|
| 410 | lineno += string.count('\n', m.start(), m.end()) |
---|
| 411 | # Update charno. |
---|
| 412 | charno = m.end() |
---|
| 413 | # Add any remaining post-example text to `output`. |
---|
| 414 | output.append(string[charno:]) |
---|
| 415 | return output |
---|
| 416 | |
---|
| 417 | |
---|
| 418 | |
---|
| 419 | if __name__ == '__main__': |
---|
| 420 | if sys.argv[1:] and sys.argv[1] == 'doctest': |
---|
| 421 | doctest.testmod() |
---|
| 422 | sys.exit() |
---|
| 423 | if not paste_parent in sys.path: |
---|
| 424 | sys.path.append(paste_parent) |
---|
| 425 | for fn in sys.argv[1:]: |
---|
| 426 | fn = os.path.abspath(fn) |
---|
| 427 | # @@: OK, ick; but this module gets loaded twice |
---|
| 428 | sys.testing_document_filename = fn |
---|
| 429 | doctest.testfile( |
---|
| 430 | fn, module_relative=False, |
---|
| 431 | optionflags=doctest.ELLIPSIS|doctest.REPORT_ONLY_FIRST_FAILURE, |
---|
| 432 | parser=LongFormDocTestParser()) |
---|
| 433 | new = os.path.splitext(fn)[0] + '.html' |
---|
| 434 | assert new != fn |
---|
| 435 | os.system('rst2html.py %s > %s' % (fn, new)) |
---|