1 # Module doctest.
2 # Released to the public domain 16-Jan-2001, by Tim Peters (tim@python.org).
3 # Major enhancements and refactoring by:
4 # Jim Fulton
5 # Edward Loper
6
7 # Provided as-is; use at your own risk; no warranty; no promises; enjoy!
8
9 r"""Module doctest -- a framework for running examples in docstrings.
10
11 In simplest use, end each module M to be tested with:
12
13 def _test():
14 import doctest
15 doctest.testmod()
16
17 if __name__ == "__main__":
18 _test()
19
20 Then running the module as a script will cause the examples in the
21 docstrings to get executed and verified:
22
23 python M.py
24
25 This won't display anything unless an example fails, in which case the
26 failing example(s) and the cause(s) of the failure(s) are printed to stdout
27 (why not stderr? because stderr is a lame hack <0.2 wink>), and the final
28 line of output is "Test failed.".
29
30 Run it with the -v switch instead:
31
32 python M.py -v
33
34 and a detailed report of all examples tried is printed to stdout, along
35 with assorted summaries at the end.
36
37 You can force verbose mode by passing "verbose=True" to testmod, or prohibit
38 it by passing "verbose=False". In either of those cases, sys.argv is not
39 examined by testmod.
40
41 There are a variety of other ways to run doctests, including integration
42 with the unittest framework, and support for running non-Python text
43 files containing doctests. There are also many ways to override parts
44 of doctest's default behaviors. See the Library Reference Manual for
45 details.
46 """
47
48 __docformat__ = 'reStructuredText en'
49
50 __all__ = [
51 # 0, Option Flags
52 'register_optionflag',
53 'DONT_ACCEPT_TRUE_FOR_1',
54 'DONT_ACCEPT_BLANKLINE',
55 'NORMALIZE_WHITESPACE',
56 'ELLIPSIS',
57 'SKIP',
58 'IGNORE_EXCEPTION_DETAIL',
59 'COMPARISON_FLAGS',
60 'REPORT_UDIFF',
61 'REPORT_CDIFF',
62 'REPORT_NDIFF',
63 'REPORT_ONLY_FIRST_FAILURE',
64 'REPORTING_FLAGS',
65 'FAIL_FAST',
66 # 1. Utility Functions
67 # 2. Example & DocTest
68 'Example',
69 'DocTest',
70 # 3. Doctest Parser
71 'DocTestParser',
72 # 4. Doctest Finder
73 'DocTestFinder',
74 # 5. Doctest Runner
75 'DocTestRunner',
76 'OutputChecker',
77 'DocTestFailure',
78 'UnexpectedException',
79 'DebugRunner',
80 # 6. Test Functions
81 'testmod',
82 'testfile',
83 'run_docstring_examples',
84 # 7. Unittest Support
85 'DocTestSuite',
86 'DocFileSuite',
87 'set_unittest_reportflags',
88 # 8. Debugging Support
89 'script_from_examples',
90 'testsource',
91 'debug_src',
92 'debug',
93 ]
94
95 import __future__
96 import difflib
97 import inspect
98 import linecache
99 import os
100 import pdb
101 import re
102 import sys
103 import traceback
104 import unittest
105 from io import StringIO, IncrementalNewlineDecoder
106 from collections import namedtuple
107
108 TestResults = namedtuple('TestResults', 'failed attempted')
109
110 # There are 4 basic classes:
111 # - Example: a <source, want> pair, plus an intra-docstring line number.
112 # - DocTest: a collection of examples, parsed from a docstring, plus
113 # info about where the docstring came from (name, filename, lineno).
114 # - DocTestFinder: extracts DocTests from a given object's docstring and
115 # its contained objects' docstrings.
116 # - DocTestRunner: runs DocTest cases, and accumulates statistics.
117 #
118 # So the basic picture is:
119 #
120 # list of:
121 # +------+ +---------+ +-------+
122 # |object| --DocTestFinder-> | DocTest | --DocTestRunner-> |results|
123 # +------+ +---------+ +-------+
124 # | Example |
125 # | ... |
126 # | Example |
127 # +---------+
128
129 # Option constants.
130
131 OPTIONFLAGS_BY_NAME = {}
132 def register_optionflag(name):
133 # Create a new flag unless `name` is already known.
134 return OPTIONFLAGS_BY_NAME.setdefault(name, 1 << len(OPTIONFLAGS_BY_NAME))
135
136 DONT_ACCEPT_TRUE_FOR_1 = register_optionflag('DONT_ACCEPT_TRUE_FOR_1')
137 DONT_ACCEPT_BLANKLINE = register_optionflag('DONT_ACCEPT_BLANKLINE')
138 NORMALIZE_WHITESPACE = register_optionflag('NORMALIZE_WHITESPACE')
139 ELLIPSIS = register_optionflag('ELLIPSIS')
140 SKIP = register_optionflag('SKIP')
141 IGNORE_EXCEPTION_DETAIL = register_optionflag('IGNORE_EXCEPTION_DETAIL')
142
143 COMPARISON_FLAGS = (DONT_ACCEPT_TRUE_FOR_1 |
144 DONT_ACCEPT_BLANKLINE |
145 NORMALIZE_WHITESPACE |
146 ELLIPSIS |
147 SKIP |
148 IGNORE_EXCEPTION_DETAIL)
149
150 REPORT_UDIFF = register_optionflag('REPORT_UDIFF')
151 REPORT_CDIFF = register_optionflag('REPORT_CDIFF')
152 REPORT_NDIFF = register_optionflag('REPORT_NDIFF')
153 REPORT_ONLY_FIRST_FAILURE = register_optionflag('REPORT_ONLY_FIRST_FAILURE')
154 FAIL_FAST = register_optionflag('FAIL_FAST')
155
156 REPORTING_FLAGS = (REPORT_UDIFF |
157 REPORT_CDIFF |
158 REPORT_NDIFF |
159 REPORT_ONLY_FIRST_FAILURE |
160 FAIL_FAST)
161
162 # Special string markers for use in `want` strings:
163 BLANKLINE_MARKER = '<BLANKLINE>'
164 ELLIPSIS_MARKER = '...'
165
166 ######################################################################
167 ## Table of Contents
168 ######################################################################
169 # 1. Utility Functions
170 # 2. Example & DocTest -- store test cases
171 # 3. DocTest Parser -- extracts examples from strings
172 # 4. DocTest Finder -- extracts test cases from objects
173 # 5. DocTest Runner -- runs test cases
174 # 6. Test Functions -- convenient wrappers for testing
175 # 7. Unittest Support
176 # 8. Debugging Support
177 # 9. Example Usage
178
179 ######################################################################
180 ## 1. Utility Functions
181 ######################################################################
182
183 def _extract_future_flags(globs):
184 """
185 Return the compiler-flags associated with the future features that
186 have been imported into the given namespace (globs).
187 """
188 flags = 0
189 for fname in __future__.all_feature_names:
190 feature = globs.get(fname, None)
191 if feature is getattr(__future__, fname):
192 flags |= feature.compiler_flag
193 return flags
194
195 def _normalize_module(module, depth=2):
196 """
197 Return the module specified by `module`. In particular:
198 - If `module` is a module, then return module.
199 - If `module` is a string, then import and return the
200 module with that name.
201 - If `module` is None, then return the calling module.
202 The calling module is assumed to be the module of
203 the stack frame at the given depth in the call stack.
204 """
205 if inspect.ismodule(module):
206 return module
207 elif isinstance(module, str):
208 return __import__(module, globals(), locals(), ["*"])
209 elif module is None:
210 try:
211 try:
212 return sys.modules[sys._getframemodulename(depth)]
213 except AttributeError:
214 return sys.modules[sys._getframe(depth).f_globals['__name__']]
215 except KeyError:
216 pass
217 else:
218 raise TypeError("Expected a module, string, or None")
219
220 def _newline_convert(data):
221 # The IO module provides a handy decoder for universal newline conversion
222 return IncrementalNewlineDecoder(None, True).decode(data, True)
223
224 def _load_testfile(filename, package, module_relative, encoding):
225 if module_relative:
226 package = _normalize_module(package, 3)
227 filename = _module_relative_path(package, filename)
228 if (loader := getattr(package, '__loader__', None)) is None:
229 try:
230 loader = package.__spec__.loader
231 except AttributeError:
232 pass
233 if hasattr(loader, 'get_data'):
234 file_contents = loader.get_data(filename)
235 file_contents = file_contents.decode(encoding)
236 # get_data() opens files as 'rb', so one must do the equivalent
237 # conversion as universal newlines would do.
238 return _newline_convert(file_contents), filename
239 with open(filename, encoding=encoding) as f:
240 return f.read(), filename
241
242 def _indent(s, indent=4):
243 """
244 Add the given number of space characters to the beginning of
245 every non-blank line in `s`, and return the result.
246 """
247 # This regexp matches the start of non-blank lines:
248 return re.sub('(?m)^(?!$)', indent*' ', s)
249
250 def _exception_traceback(exc_info):
251 """
252 Return a string containing a traceback message for the given
253 exc_info tuple (as returned by sys.exc_info()).
254 """
255 # Get a traceback message.
256 excout = StringIO()
257 exc_type, exc_val, exc_tb = exc_info
258 traceback.print_exception(exc_type, exc_val, exc_tb, file=excout)
259 return excout.getvalue()
260
261 # Override some StringIO methods.
262 class ESC[4;38;5;81m_SpoofOut(ESC[4;38;5;149mStringIO):
263 def getvalue(self):
264 result = StringIO.getvalue(self)
265 # If anything at all was written, make sure there's a trailing
266 # newline. There's no way for the expected output to indicate
267 # that a trailing newline is missing.
268 if result and not result.endswith("\n"):
269 result += "\n"
270 return result
271
272 def truncate(self, size=None):
273 self.seek(size)
274 StringIO.truncate(self)
275
276 # Worst-case linear-time ellipsis matching.
277 def _ellipsis_match(want, got):
278 """
279 Essentially the only subtle case:
280 >>> _ellipsis_match('aa...aa', 'aaa')
281 False
282 """
283 if ELLIPSIS_MARKER not in want:
284 return want == got
285
286 # Find "the real" strings.
287 ws = want.split(ELLIPSIS_MARKER)
288 assert len(ws) >= 2
289
290 # Deal with exact matches possibly needed at one or both ends.
291 startpos, endpos = 0, len(got)
292 w = ws[0]
293 if w: # starts with exact match
294 if got.startswith(w):
295 startpos = len(w)
296 del ws[0]
297 else:
298 return False
299 w = ws[-1]
300 if w: # ends with exact match
301 if got.endswith(w):
302 endpos -= len(w)
303 del ws[-1]
304 else:
305 return False
306
307 if startpos > endpos:
308 # Exact end matches required more characters than we have, as in
309 # _ellipsis_match('aa...aa', 'aaa')
310 return False
311
312 # For the rest, we only need to find the leftmost non-overlapping
313 # match for each piece. If there's no overall match that way alone,
314 # there's no overall match period.
315 for w in ws:
316 # w may be '' at times, if there are consecutive ellipses, or
317 # due to an ellipsis at the start or end of `want`. That's OK.
318 # Search for an empty string succeeds, and doesn't change startpos.
319 startpos = got.find(w, startpos, endpos)
320 if startpos < 0:
321 return False
322 startpos += len(w)
323
324 return True
325
326 def _comment_line(line):
327 "Return a commented form of the given line"
328 line = line.rstrip()
329 if line:
330 return '# '+line
331 else:
332 return '#'
333
334 def _strip_exception_details(msg):
335 # Support for IGNORE_EXCEPTION_DETAIL.
336 # Get rid of everything except the exception name; in particular, drop
337 # the possibly dotted module path (if any) and the exception message (if
338 # any). We assume that a colon is never part of a dotted name, or of an
339 # exception name.
340 # E.g., given
341 # "foo.bar.MyError: la di da"
342 # return "MyError"
343 # Or for "abc.def" or "abc.def:\n" return "def".
344
345 start, end = 0, len(msg)
346 # The exception name must appear on the first line.
347 i = msg.find("\n")
348 if i >= 0:
349 end = i
350 # retain up to the first colon (if any)
351 i = msg.find(':', 0, end)
352 if i >= 0:
353 end = i
354 # retain just the exception name
355 i = msg.rfind('.', 0, end)
356 if i >= 0:
357 start = i+1
358 return msg[start: end]
359
360 class ESC[4;38;5;81m_OutputRedirectingPdb(ESC[4;38;5;149mpdbESC[4;38;5;149m.ESC[4;38;5;149mPdb):
361 """
362 A specialized version of the python debugger that redirects stdout
363 to a given stream when interacting with the user. Stdout is *not*
364 redirected when traced code is executed.
365 """
366 def __init__(self, out):
367 self.__out = out
368 self.__debugger_used = False
369 # do not play signal games in the pdb
370 pdb.Pdb.__init__(self, stdout=out, nosigint=True)
371 # still use input() to get user input
372 self.use_rawinput = 1
373
374 def set_trace(self, frame=None):
375 self.__debugger_used = True
376 if frame is None:
377 frame = sys._getframe().f_back
378 pdb.Pdb.set_trace(self, frame)
379
380 def set_continue(self):
381 # Calling set_continue unconditionally would break unit test
382 # coverage reporting, as Bdb.set_continue calls sys.settrace(None).
383 if self.__debugger_used:
384 pdb.Pdb.set_continue(self)
385
386 def trace_dispatch(self, *args):
387 # Redirect stdout to the given stream.
388 save_stdout = sys.stdout
389 sys.stdout = self.__out
390 # Call Pdb's trace dispatch method.
391 try:
392 return pdb.Pdb.trace_dispatch(self, *args)
393 finally:
394 sys.stdout = save_stdout
395
396 # [XX] Normalize with respect to os.path.pardir?
397 def _module_relative_path(module, test_path):
398 if not inspect.ismodule(module):
399 raise TypeError('Expected a module: %r' % module)
400 if test_path.startswith('/'):
401 raise ValueError('Module-relative files may not have absolute paths')
402
403 # Normalize the path. On Windows, replace "/" with "\".
404 test_path = os.path.join(*(test_path.split('/')))
405
406 # Find the base directory for the path.
407 if hasattr(module, '__file__'):
408 # A normal module/package
409 basedir = os.path.split(module.__file__)[0]
410 elif module.__name__ == '__main__':
411 # An interactive session.
412 if len(sys.argv)>0 and sys.argv[0] != '':
413 basedir = os.path.split(sys.argv[0])[0]
414 else:
415 basedir = os.curdir
416 else:
417 if hasattr(module, '__path__'):
418 for directory in module.__path__:
419 fullpath = os.path.join(directory, test_path)
420 if os.path.exists(fullpath):
421 return fullpath
422
423 # A module w/o __file__ (this includes builtins)
424 raise ValueError("Can't resolve paths relative to the module "
425 "%r (it has no __file__)"
426 % module.__name__)
427
428 # Combine the base directory and the test path.
429 return os.path.join(basedir, test_path)
430
431 ######################################################################
432 ## 2. Example & DocTest
433 ######################################################################
434 ## - An "example" is a <source, want> pair, where "source" is a
435 ## fragment of source code, and "want" is the expected output for
436 ## "source." The Example class also includes information about
437 ## where the example was extracted from.
438 ##
439 ## - A "doctest" is a collection of examples, typically extracted from
440 ## a string (such as an object's docstring). The DocTest class also
441 ## includes information about where the string was extracted from.
442
443 class ESC[4;38;5;81mExample:
444 """
445 A single doctest example, consisting of source code and expected
446 output. `Example` defines the following attributes:
447
448 - source: A single Python statement, always ending with a newline.
449 The constructor adds a newline if needed.
450
451 - want: The expected output from running the source code (either
452 from stdout, or a traceback in case of exception). `want` ends
453 with a newline unless it's empty, in which case it's an empty
454 string. The constructor adds a newline if needed.
455
456 - exc_msg: The exception message generated by the example, if
457 the example is expected to generate an exception; or `None` if
458 it is not expected to generate an exception. This exception
459 message is compared against the return value of
460 `traceback.format_exception_only()`. `exc_msg` ends with a
461 newline unless it's `None`. The constructor adds a newline
462 if needed.
463
464 - lineno: The line number within the DocTest string containing
465 this Example where the Example begins. This line number is
466 zero-based, with respect to the beginning of the DocTest.
467
468 - indent: The example's indentation in the DocTest string.
469 I.e., the number of space characters that precede the
470 example's first prompt.
471
472 - options: A dictionary mapping from option flags to True or
473 False, which is used to override default options for this
474 example. Any option flags not contained in this dictionary
475 are left at their default value (as specified by the
476 DocTestRunner's optionflags). By default, no options are set.
477 """
478 def __init__(self, source, want, exc_msg=None, lineno=0, indent=0,
479 options=None):
480 # Normalize inputs.
481 if not source.endswith('\n'):
482 source += '\n'
483 if want and not want.endswith('\n'):
484 want += '\n'
485 if exc_msg is not None and not exc_msg.endswith('\n'):
486 exc_msg += '\n'
487 # Store properties.
488 self.source = source
489 self.want = want
490 self.lineno = lineno
491 self.indent = indent
492 if options is None: options = {}
493 self.options = options
494 self.exc_msg = exc_msg
495
496 def __eq__(self, other):
497 if type(self) is not type(other):
498 return NotImplemented
499
500 return self.source == other.source and \
501 self.want == other.want and \
502 self.lineno == other.lineno and \
503 self.indent == other.indent and \
504 self.options == other.options and \
505 self.exc_msg == other.exc_msg
506
507 def __hash__(self):
508 return hash((self.source, self.want, self.lineno, self.indent,
509 self.exc_msg))
510
511 class ESC[4;38;5;81mDocTest:
512 """
513 A collection of doctest examples that should be run in a single
514 namespace. Each `DocTest` defines the following attributes:
515
516 - examples: the list of examples.
517
518 - globs: The namespace (aka globals) that the examples should
519 be run in.
520
521 - name: A name identifying the DocTest (typically, the name of
522 the object whose docstring this DocTest was extracted from).
523
524 - filename: The name of the file that this DocTest was extracted
525 from, or `None` if the filename is unknown.
526
527 - lineno: The line number within filename where this DocTest
528 begins, or `None` if the line number is unavailable. This
529 line number is zero-based, with respect to the beginning of
530 the file.
531
532 - docstring: The string that the examples were extracted from,
533 or `None` if the string is unavailable.
534 """
535 def __init__(self, examples, globs, name, filename, lineno, docstring):
536 """
537 Create a new DocTest containing the given examples. The
538 DocTest's globals are initialized with a copy of `globs`.
539 """
540 assert not isinstance(examples, str), \
541 "DocTest no longer accepts str; use DocTestParser instead"
542 self.examples = examples
543 self.docstring = docstring
544 self.globs = globs.copy()
545 self.name = name
546 self.filename = filename
547 self.lineno = lineno
548
549 def __repr__(self):
550 if len(self.examples) == 0:
551 examples = 'no examples'
552 elif len(self.examples) == 1:
553 examples = '1 example'
554 else:
555 examples = '%d examples' % len(self.examples)
556 return ('<%s %s from %s:%s (%s)>' %
557 (self.__class__.__name__,
558 self.name, self.filename, self.lineno, examples))
559
560 def __eq__(self, other):
561 if type(self) is not type(other):
562 return NotImplemented
563
564 return self.examples == other.examples and \
565 self.docstring == other.docstring and \
566 self.globs == other.globs and \
567 self.name == other.name and \
568 self.filename == other.filename and \
569 self.lineno == other.lineno
570
571 def __hash__(self):
572 return hash((self.docstring, self.name, self.filename, self.lineno))
573
574 # This lets us sort tests by name:
575 def __lt__(self, other):
576 if not isinstance(other, DocTest):
577 return NotImplemented
578 return ((self.name, self.filename, self.lineno, id(self))
579 <
580 (other.name, other.filename, other.lineno, id(other)))
581
582 ######################################################################
583 ## 3. DocTestParser
584 ######################################################################
585
586 class ESC[4;38;5;81mDocTestParser:
587 """
588 A class used to parse strings containing doctest examples.
589 """
590 # This regular expression is used to find doctest examples in a
591 # string. It defines three groups: `source` is the source code
592 # (including leading indentation and prompts); `indent` is the
593 # indentation of the first (PS1) line of the source code; and
594 # `want` is the expected output (including leading indentation).
595 _EXAMPLE_RE = re.compile(r'''
596 # Source consists of a PS1 line followed by zero or more PS2 lines.
597 (?P<source>
598 (?:^(?P<indent> [ ]*) >>> .*) # PS1 line
599 (?:\n [ ]* \.\.\. .*)*) # PS2 lines
600 \n?
601 # Want consists of any non-blank lines that do not start with PS1.
602 (?P<want> (?:(?![ ]*$) # Not a blank line
603 (?![ ]*>>>) # Not a line starting with PS1
604 .+$\n? # But any other line
605 )*)
606 ''', re.MULTILINE | re.VERBOSE)
607
608 # A regular expression for handling `want` strings that contain
609 # expected exceptions. It divides `want` into three pieces:
610 # - the traceback header line (`hdr`)
611 # - the traceback stack (`stack`)
612 # - the exception message (`msg`), as generated by
613 # traceback.format_exception_only()
614 # `msg` may have multiple lines. We assume/require that the
615 # exception message is the first non-indented line starting with a word
616 # character following the traceback header line.
617 _EXCEPTION_RE = re.compile(r"""
618 # Grab the traceback header. Different versions of Python have
619 # said different things on the first traceback line.
620 ^(?P<hdr> Traceback\ \(
621 (?: most\ recent\ call\ last
622 | innermost\ last
623 ) \) :
624 )
625 \s* $ # toss trailing whitespace on the header.
626 (?P<stack> .*?) # don't blink: absorb stuff until...
627 ^ (?P<msg> \w+ .*) # a line *starts* with alphanum.
628 """, re.VERBOSE | re.MULTILINE | re.DOTALL)
629
630 # A callable returning a true value iff its argument is a blank line
631 # or contains a single comment.
632 _IS_BLANK_OR_COMMENT = re.compile(r'^[ ]*(#.*)?$').match
633
634 def parse(self, string, name='<string>'):
635 """
636 Divide the given string into examples and intervening text,
637 and return them as a list of alternating Examples and strings.
638 Line numbers for the Examples are 0-based. The optional
639 argument `name` is a name identifying this string, and is only
640 used for error messages.
641 """
642 string = string.expandtabs()
643 # If all lines begin with the same indentation, then strip it.
644 min_indent = self._min_indent(string)
645 if min_indent > 0:
646 string = '\n'.join([l[min_indent:] for l in string.split('\n')])
647
648 output = []
649 charno, lineno = 0, 0
650 # Find all doctest examples in the string:
651 for m in self._EXAMPLE_RE.finditer(string):
652 # Add the pre-example text to `output`.
653 output.append(string[charno:m.start()])
654 # Update lineno (lines before this example)
655 lineno += string.count('\n', charno, m.start())
656 # Extract info from the regexp match.
657 (source, options, want, exc_msg) = \
658 self._parse_example(m, name, lineno)
659 # Create an Example, and add it to the list.
660 if not self._IS_BLANK_OR_COMMENT(source):
661 output.append( Example(source, want, exc_msg,
662 lineno=lineno,
663 indent=min_indent+len(m.group('indent')),
664 options=options) )
665 # Update lineno (lines inside this example)
666 lineno += string.count('\n', m.start(), m.end())
667 # Update charno.
668 charno = m.end()
669 # Add any remaining post-example text to `output`.
670 output.append(string[charno:])
671 return output
672
673 def get_doctest(self, string, globs, name, filename, lineno):
674 """
675 Extract all doctest examples from the given string, and
676 collect them into a `DocTest` object.
677
678 `globs`, `name`, `filename`, and `lineno` are attributes for
679 the new `DocTest` object. See the documentation for `DocTest`
680 for more information.
681 """
682 return DocTest(self.get_examples(string, name), globs,
683 name, filename, lineno, string)
684
685 def get_examples(self, string, name='<string>'):
686 """
687 Extract all doctest examples from the given string, and return
688 them as a list of `Example` objects. Line numbers are
689 0-based, because it's most common in doctests that nothing
690 interesting appears on the same line as opening triple-quote,
691 and so the first interesting line is called \"line 1\" then.
692
693 The optional argument `name` is a name identifying this
694 string, and is only used for error messages.
695 """
696 return [x for x in self.parse(string, name)
697 if isinstance(x, Example)]
698
699 def _parse_example(self, m, name, lineno):
700 """
701 Given a regular expression match from `_EXAMPLE_RE` (`m`),
702 return a pair `(source, want)`, where `source` is the matched
703 example's source code (with prompts and indentation stripped);
704 and `want` is the example's expected output (with indentation
705 stripped).
706
707 `name` is the string's name, and `lineno` is the line number
708 where the example starts; both are used for error messages.
709 """
710 # Get the example's indentation level.
711 indent = len(m.group('indent'))
712
713 # Divide source into lines; check that they're properly
714 # indented; and then strip their indentation & prompts.
715 source_lines = m.group('source').split('\n')
716 self._check_prompt_blank(source_lines, indent, name, lineno)
717 self._check_prefix(source_lines[1:], ' '*indent + '.', name, lineno)
718 source = '\n'.join([sl[indent+4:] for sl in source_lines])
719
720 # Divide want into lines; check that it's properly indented; and
721 # then strip the indentation. Spaces before the last newline should
722 # be preserved, so plain rstrip() isn't good enough.
723 want = m.group('want')
724 want_lines = want.split('\n')
725 if len(want_lines) > 1 and re.match(r' *$', want_lines[-1]):
726 del want_lines[-1] # forget final newline & spaces after it
727 self._check_prefix(want_lines, ' '*indent, name,
728 lineno + len(source_lines))
729 want = '\n'.join([wl[indent:] for wl in want_lines])
730
731 # If `want` contains a traceback message, then extract it.
732 m = self._EXCEPTION_RE.match(want)
733 if m:
734 exc_msg = m.group('msg')
735 else:
736 exc_msg = None
737
738 # Extract options from the source.
739 options = self._find_options(source, name, lineno)
740
741 return source, options, want, exc_msg
742
743 # This regular expression looks for option directives in the
744 # source code of an example. Option directives are comments
745 # starting with "doctest:". Warning: this may give false
746 # positives for string-literals that contain the string
747 # "#doctest:". Eliminating these false positives would require
748 # actually parsing the string; but we limit them by ignoring any
749 # line containing "#doctest:" that is *followed* by a quote mark.
750 _OPTION_DIRECTIVE_RE = re.compile(r'#\s*doctest:\s*([^\n\'"]*)$',
751 re.MULTILINE)
752
753 def _find_options(self, source, name, lineno):
754 """
755 Return a dictionary containing option overrides extracted from
756 option directives in the given source string.
757
758 `name` is the string's name, and `lineno` is the line number
759 where the example starts; both are used for error messages.
760 """
761 options = {}
762 # (note: with the current regexp, this will match at most once:)
763 for m in self._OPTION_DIRECTIVE_RE.finditer(source):
764 option_strings = m.group(1).replace(',', ' ').split()
765 for option in option_strings:
766 if (option[0] not in '+-' or
767 option[1:] not in OPTIONFLAGS_BY_NAME):
768 raise ValueError('line %r of the doctest for %s '
769 'has an invalid option: %r' %
770 (lineno+1, name, option))
771 flag = OPTIONFLAGS_BY_NAME[option[1:]]
772 options[flag] = (option[0] == '+')
773 if options and self._IS_BLANK_OR_COMMENT(source):
774 raise ValueError('line %r of the doctest for %s has an option '
775 'directive on a line with no example: %r' %
776 (lineno, name, source))
777 return options
778
779 # This regular expression finds the indentation of every non-blank
780 # line in a string.
781 _INDENT_RE = re.compile(r'^([ ]*)(?=\S)', re.MULTILINE)
782
783 def _min_indent(self, s):
784 "Return the minimum indentation of any non-blank line in `s`"
785 indents = [len(indent) for indent in self._INDENT_RE.findall(s)]
786 if len(indents) > 0:
787 return min(indents)
788 else:
789 return 0
790
791 def _check_prompt_blank(self, lines, indent, name, lineno):
792 """
793 Given the lines of a source string (including prompts and
794 leading indentation), check to make sure that every prompt is
795 followed by a space character. If any line is not followed by
796 a space character, then raise ValueError.
797 """
798 for i, line in enumerate(lines):
799 if len(line) >= indent+4 and line[indent+3] != ' ':
800 raise ValueError('line %r of the docstring for %s '
801 'lacks blank after %s: %r' %
802 (lineno+i+1, name,
803 line[indent:indent+3], line))
804
805 def _check_prefix(self, lines, prefix, name, lineno):
806 """
807 Check that every line in the given list starts with the given
808 prefix; if any line does not, then raise a ValueError.
809 """
810 for i, line in enumerate(lines):
811 if line and not line.startswith(prefix):
812 raise ValueError('line %r of the docstring for %s has '
813 'inconsistent leading whitespace: %r' %
814 (lineno+i+1, name, line))
815
816
817 ######################################################################
818 ## 4. DocTest Finder
819 ######################################################################
820
821 class ESC[4;38;5;81mDocTestFinder:
822 """
823 A class used to extract the DocTests that are relevant to a given
824 object, from its docstring and the docstrings of its contained
825 objects. Doctests can currently be extracted from the following
826 object types: modules, functions, classes, methods, staticmethods,
827 classmethods, and properties.
828 """
829
830 def __init__(self, verbose=False, parser=DocTestParser(),
831 recurse=True, exclude_empty=True):
832 """
833 Create a new doctest finder.
834
835 The optional argument `parser` specifies a class or
836 function that should be used to create new DocTest objects (or
837 objects that implement the same interface as DocTest). The
838 signature for this factory function should match the signature
839 of the DocTest constructor.
840
841 If the optional argument `recurse` is false, then `find` will
842 only examine the given object, and not any contained objects.
843
844 If the optional argument `exclude_empty` is false, then `find`
845 will include tests for objects with empty docstrings.
846 """
847 self._parser = parser
848 self._verbose = verbose
849 self._recurse = recurse
850 self._exclude_empty = exclude_empty
851
852 def find(self, obj, name=None, module=None, globs=None, extraglobs=None):
853 """
854 Return a list of the DocTests that are defined by the given
855 object's docstring, or by any of its contained objects'
856 docstrings.
857
858 The optional parameter `module` is the module that contains
859 the given object. If the module is not specified or is None, then
860 the test finder will attempt to automatically determine the
861 correct module. The object's module is used:
862
863 - As a default namespace, if `globs` is not specified.
864 - To prevent the DocTestFinder from extracting DocTests
865 from objects that are imported from other modules.
866 - To find the name of the file containing the object.
867 - To help find the line number of the object within its
868 file.
869
870 Contained objects whose module does not match `module` are ignored.
871
872 If `module` is False, no attempt to find the module will be made.
873 This is obscure, of use mostly in tests: if `module` is False, or
874 is None but cannot be found automatically, then all objects are
875 considered to belong to the (non-existent) module, so all contained
876 objects will (recursively) be searched for doctests.
877
878 The globals for each DocTest is formed by combining `globs`
879 and `extraglobs` (bindings in `extraglobs` override bindings
880 in `globs`). A new copy of the globals dictionary is created
881 for each DocTest. If `globs` is not specified, then it
882 defaults to the module's `__dict__`, if specified, or {}
883 otherwise. If `extraglobs` is not specified, then it defaults
884 to {}.
885
886 """
887 # If name was not specified, then extract it from the object.
888 if name is None:
889 name = getattr(obj, '__name__', None)
890 if name is None:
891 raise ValueError("DocTestFinder.find: name must be given "
892 "when obj.__name__ doesn't exist: %r" %
893 (type(obj),))
894
895 # Find the module that contains the given object (if obj is
896 # a module, then module=obj.). Note: this may fail, in which
897 # case module will be None.
898 if module is False:
899 module = None
900 elif module is None:
901 module = inspect.getmodule(obj)
902
903 # Read the module's source code. This is used by
904 # DocTestFinder._find_lineno to find the line number for a
905 # given object's docstring.
906 try:
907 file = inspect.getsourcefile(obj)
908 except TypeError:
909 source_lines = None
910 else:
911 if not file:
912 # Check to see if it's one of our special internal "files"
913 # (see __patched_linecache_getlines).
914 file = inspect.getfile(obj)
915 if not file[0]+file[-2:] == '<]>': file = None
916 if file is None:
917 source_lines = None
918 else:
919 if module is not None:
920 # Supply the module globals in case the module was
921 # originally loaded via a PEP 302 loader and
922 # file is not a valid filesystem path
923 source_lines = linecache.getlines(file, module.__dict__)
924 else:
925 # No access to a loader, so assume it's a normal
926 # filesystem path
927 source_lines = linecache.getlines(file)
928 if not source_lines:
929 source_lines = None
930
931 # Initialize globals, and merge in extraglobs.
932 if globs is None:
933 if module is None:
934 globs = {}
935 else:
936 globs = module.__dict__.copy()
937 else:
938 globs = globs.copy()
939 if extraglobs is not None:
940 globs.update(extraglobs)
941 if '__name__' not in globs:
942 globs['__name__'] = '__main__' # provide a default module name
943
944 # Recursively explore `obj`, extracting DocTests.
945 tests = []
946 self._find(tests, obj, name, module, source_lines, globs, {})
947 # Sort the tests by alpha order of names, for consistency in
948 # verbose-mode output. This was a feature of doctest in Pythons
949 # <= 2.3 that got lost by accident in 2.4. It was repaired in
950 # 2.4.4 and 2.5.
951 tests.sort()
952 return tests
953
954 def _from_module(self, module, object):
955 """
956 Return true if the given object is defined in the given
957 module.
958 """
959 if module is None:
960 return True
961 elif inspect.getmodule(object) is not None:
962 return module is inspect.getmodule(object)
963 elif inspect.isfunction(object):
964 return module.__dict__ is object.__globals__
965 elif (inspect.ismethoddescriptor(object) or
966 inspect.ismethodwrapper(object)):
967 if hasattr(object, '__objclass__'):
968 obj_mod = object.__objclass__.__module__
969 elif hasattr(object, '__module__'):
970 obj_mod = object.__module__
971 else:
972 return True # [XX] no easy way to tell otherwise
973 return module.__name__ == obj_mod
974 elif inspect.isclass(object):
975 return module.__name__ == object.__module__
976 elif hasattr(object, '__module__'):
977 return module.__name__ == object.__module__
978 elif isinstance(object, property):
979 return True # [XX] no way not be sure.
980 else:
981 raise ValueError("object must be a class or function")
982
983 def _is_routine(self, obj):
984 """
985 Safely unwrap objects and determine if they are functions.
986 """
987 maybe_routine = obj
988 try:
989 maybe_routine = inspect.unwrap(maybe_routine)
990 except ValueError:
991 pass
992 return inspect.isroutine(maybe_routine)
993
994 def _find(self, tests, obj, name, module, source_lines, globs, seen):
995 """
996 Find tests for the given object and any contained objects, and
997 add them to `tests`.
998 """
999 if self._verbose:
1000 print('Finding tests in %s' % name)
1001
1002 # If we've already processed this object, then ignore it.
1003 if id(obj) in seen:
1004 return
1005 seen[id(obj)] = 1
1006
1007 # Find a test for this object, and add it to the list of tests.
1008 test = self._get_test(obj, name, module, globs, source_lines)
1009 if test is not None:
1010 tests.append(test)
1011
1012 # Look for tests in a module's contained objects.
1013 if inspect.ismodule(obj) and self._recurse:
1014 for valname, val in obj.__dict__.items():
1015 valname = '%s.%s' % (name, valname)
1016
1017 # Recurse to functions & classes.
1018 if ((self._is_routine(val) or inspect.isclass(val)) and
1019 self._from_module(module, val)):
1020 self._find(tests, val, valname, module, source_lines,
1021 globs, seen)
1022
1023 # Look for tests in a module's __test__ dictionary.
1024 if inspect.ismodule(obj) and self._recurse:
1025 for valname, val in getattr(obj, '__test__', {}).items():
1026 if not isinstance(valname, str):
1027 raise ValueError("DocTestFinder.find: __test__ keys "
1028 "must be strings: %r" %
1029 (type(valname),))
1030 if not (inspect.isroutine(val) or inspect.isclass(val) or
1031 inspect.ismodule(val) or isinstance(val, str)):
1032 raise ValueError("DocTestFinder.find: __test__ values "
1033 "must be strings, functions, methods, "
1034 "classes, or modules: %r" %
1035 (type(val),))
1036 valname = '%s.__test__.%s' % (name, valname)
1037 self._find(tests, val, valname, module, source_lines,
1038 globs, seen)
1039
1040 # Look for tests in a class's contained objects.
1041 if inspect.isclass(obj) and self._recurse:
1042 for valname, val in obj.__dict__.items():
1043 # Special handling for staticmethod/classmethod.
1044 if isinstance(val, (staticmethod, classmethod)):
1045 val = val.__func__
1046
1047 # Recurse to methods, properties, and nested classes.
1048 if ((inspect.isroutine(val) or inspect.isclass(val) or
1049 isinstance(val, property)) and
1050 self._from_module(module, val)):
1051 valname = '%s.%s' % (name, valname)
1052 self._find(tests, val, valname, module, source_lines,
1053 globs, seen)
1054
1055 def _get_test(self, obj, name, module, globs, source_lines):
1056 """
1057 Return a DocTest for the given object, if it defines a docstring;
1058 otherwise, return None.
1059 """
1060 # Extract the object's docstring. If it doesn't have one,
1061 # then return None (no test for this object).
1062 if isinstance(obj, str):
1063 docstring = obj
1064 else:
1065 try:
1066 if obj.__doc__ is None:
1067 docstring = ''
1068 else:
1069 docstring = obj.__doc__
1070 if not isinstance(docstring, str):
1071 docstring = str(docstring)
1072 except (TypeError, AttributeError):
1073 docstring = ''
1074
1075 # Find the docstring's location in the file.
1076 lineno = self._find_lineno(obj, source_lines)
1077
1078 # Don't bother if the docstring is empty.
1079 if self._exclude_empty and not docstring:
1080 return None
1081
1082 # Return a DocTest for this object.
1083 if module is None:
1084 filename = None
1085 else:
1086 # __file__ can be None for namespace packages.
1087 filename = getattr(module, '__file__', None) or module.__name__
1088 if filename[-4:] == ".pyc":
1089 filename = filename[:-1]
1090 return self._parser.get_doctest(docstring, globs, name,
1091 filename, lineno)
1092
1093 def _find_lineno(self, obj, source_lines):
1094 """
1095 Return a line number of the given object's docstring.
1096
1097 Returns `None` if the given object does not have a docstring.
1098 """
1099 lineno = None
1100 docstring = getattr(obj, '__doc__', None)
1101
1102 # Find the line number for modules.
1103 if inspect.ismodule(obj) and docstring is not None:
1104 lineno = 0
1105
1106 # Find the line number for classes.
1107 # Note: this could be fooled if a class is defined multiple
1108 # times in a single file.
1109 if inspect.isclass(obj) and docstring is not None:
1110 if source_lines is None:
1111 return None
1112 pat = re.compile(r'^\s*class\s*%s\b' %
1113 re.escape(getattr(obj, '__name__', '-')))
1114 for i, line in enumerate(source_lines):
1115 if pat.match(line):
1116 lineno = i
1117 break
1118
1119 # Find the line number for functions & methods.
1120 if inspect.ismethod(obj): obj = obj.__func__
1121 if inspect.isfunction(obj) and getattr(obj, '__doc__', None):
1122 # We don't use `docstring` var here, because `obj` can be changed.
1123 obj = obj.__code__
1124 if inspect.istraceback(obj): obj = obj.tb_frame
1125 if inspect.isframe(obj): obj = obj.f_code
1126 if inspect.iscode(obj):
1127 lineno = obj.co_firstlineno - 1
1128
1129 # Find the line number where the docstring starts. Assume
1130 # that it's the first line that begins with a quote mark.
1131 # Note: this could be fooled by a multiline function
1132 # signature, where a continuation line begins with a quote
1133 # mark.
1134 if lineno is not None:
1135 if source_lines is None:
1136 return lineno+1
1137 pat = re.compile(r'(^|.*:)\s*\w*("|\')')
1138 for lineno in range(lineno, len(source_lines)):
1139 if pat.match(source_lines[lineno]):
1140 return lineno
1141
1142 # We couldn't find the line number.
1143 return None
1144
1145 ######################################################################
1146 ## 5. DocTest Runner
1147 ######################################################################
1148
1149 class ESC[4;38;5;81mDocTestRunner:
1150 """
1151 A class used to run DocTest test cases, and accumulate statistics.
1152 The `run` method is used to process a single DocTest case. It
1153 returns a tuple `(f, t)`, where `t` is the number of test cases
1154 tried, and `f` is the number of test cases that failed.
1155
1156 >>> tests = DocTestFinder().find(_TestClass)
1157 >>> runner = DocTestRunner(verbose=False)
1158 >>> tests.sort(key = lambda test: test.name)
1159 >>> for test in tests:
1160 ... print(test.name, '->', runner.run(test))
1161 _TestClass -> TestResults(failed=0, attempted=2)
1162 _TestClass.__init__ -> TestResults(failed=0, attempted=2)
1163 _TestClass.get -> TestResults(failed=0, attempted=2)
1164 _TestClass.square -> TestResults(failed=0, attempted=1)
1165
1166 The `summarize` method prints a summary of all the test cases that
1167 have been run by the runner, and returns an aggregated `(f, t)`
1168 tuple:
1169
1170 >>> runner.summarize(verbose=1)
1171 4 items passed all tests:
1172 2 tests in _TestClass
1173 2 tests in _TestClass.__init__
1174 2 tests in _TestClass.get
1175 1 tests in _TestClass.square
1176 7 tests in 4 items.
1177 7 passed and 0 failed.
1178 Test passed.
1179 TestResults(failed=0, attempted=7)
1180
1181 The aggregated number of tried examples and failed examples is
1182 also available via the `tries` and `failures` attributes:
1183
1184 >>> runner.tries
1185 7
1186 >>> runner.failures
1187 0
1188
1189 The comparison between expected outputs and actual outputs is done
1190 by an `OutputChecker`. This comparison may be customized with a
1191 number of option flags; see the documentation for `testmod` for
1192 more information. If the option flags are insufficient, then the
1193 comparison may also be customized by passing a subclass of
1194 `OutputChecker` to the constructor.
1195
1196 The test runner's display output can be controlled in two ways.
1197 First, an output function (`out) can be passed to
1198 `TestRunner.run`; this function will be called with strings that
1199 should be displayed. It defaults to `sys.stdout.write`. If
1200 capturing the output is not sufficient, then the display output
1201 can be also customized by subclassing DocTestRunner, and
1202 overriding the methods `report_start`, `report_success`,
1203 `report_unexpected_exception`, and `report_failure`.
1204 """
1205 # This divider string is used to separate failure messages, and to
1206 # separate sections of the summary.
1207 DIVIDER = "*" * 70
1208
1209 def __init__(self, checker=None, verbose=None, optionflags=0):
1210 """
1211 Create a new test runner.
1212
1213 Optional keyword arg `checker` is the `OutputChecker` that
1214 should be used to compare the expected outputs and actual
1215 outputs of doctest examples.
1216
1217 Optional keyword arg 'verbose' prints lots of stuff if true,
1218 only failures if false; by default, it's true iff '-v' is in
1219 sys.argv.
1220
1221 Optional argument `optionflags` can be used to control how the
1222 test runner compares expected output to actual output, and how
1223 it displays failures. See the documentation for `testmod` for
1224 more information.
1225 """
1226 self._checker = checker or OutputChecker()
1227 if verbose is None:
1228 verbose = '-v' in sys.argv
1229 self._verbose = verbose
1230 self.optionflags = optionflags
1231 self.original_optionflags = optionflags
1232
1233 # Keep track of the examples we've run.
1234 self.tries = 0
1235 self.failures = 0
1236 self._name2ft = {}
1237
1238 # Create a fake output target for capturing doctest output.
1239 self._fakeout = _SpoofOut()
1240
1241 #/////////////////////////////////////////////////////////////////
1242 # Reporting methods
1243 #/////////////////////////////////////////////////////////////////
1244
1245 def report_start(self, out, test, example):
1246 """
1247 Report that the test runner is about to process the given
1248 example. (Only displays a message if verbose=True)
1249 """
1250 if self._verbose:
1251 if example.want:
1252 out('Trying:\n' + _indent(example.source) +
1253 'Expecting:\n' + _indent(example.want))
1254 else:
1255 out('Trying:\n' + _indent(example.source) +
1256 'Expecting nothing\n')
1257
1258 def report_success(self, out, test, example, got):
1259 """
1260 Report that the given example ran successfully. (Only
1261 displays a message if verbose=True)
1262 """
1263 if self._verbose:
1264 out("ok\n")
1265
1266 def report_failure(self, out, test, example, got):
1267 """
1268 Report that the given example failed.
1269 """
1270 out(self._failure_header(test, example) +
1271 self._checker.output_difference(example, got, self.optionflags))
1272
1273 def report_unexpected_exception(self, out, test, example, exc_info):
1274 """
1275 Report that the given example raised an unexpected exception.
1276 """
1277 out(self._failure_header(test, example) +
1278 'Exception raised:\n' + _indent(_exception_traceback(exc_info)))
1279
1280 def _failure_header(self, test, example):
1281 out = [self.DIVIDER]
1282 if test.filename:
1283 if test.lineno is not None and example.lineno is not None:
1284 lineno = test.lineno + example.lineno + 1
1285 else:
1286 lineno = '?'
1287 out.append('File "%s", line %s, in %s' %
1288 (test.filename, lineno, test.name))
1289 else:
1290 out.append('Line %s, in %s' % (example.lineno+1, test.name))
1291 out.append('Failed example:')
1292 source = example.source
1293 out.append(_indent(source))
1294 return '\n'.join(out)
1295
1296 #/////////////////////////////////////////////////////////////////
1297 # DocTest Running
1298 #/////////////////////////////////////////////////////////////////
1299
1300 def __run(self, test, compileflags, out):
1301 """
1302 Run the examples in `test`. Write the outcome of each example
1303 with one of the `DocTestRunner.report_*` methods, using the
1304 writer function `out`. `compileflags` is the set of compiler
1305 flags that should be used to execute examples. Return a tuple
1306 `(f, t)`, where `t` is the number of examples tried, and `f`
1307 is the number of examples that failed. The examples are run
1308 in the namespace `test.globs`.
1309 """
1310 # Keep track of the number of failures and tries.
1311 failures = tries = 0
1312
1313 # Save the option flags (since option directives can be used
1314 # to modify them).
1315 original_optionflags = self.optionflags
1316
1317 SUCCESS, FAILURE, BOOM = range(3) # `outcome` state
1318
1319 check = self._checker.check_output
1320
1321 # Process each example.
1322 for examplenum, example in enumerate(test.examples):
1323
1324 # If REPORT_ONLY_FIRST_FAILURE is set, then suppress
1325 # reporting after the first failure.
1326 quiet = (self.optionflags & REPORT_ONLY_FIRST_FAILURE and
1327 failures > 0)
1328
1329 # Merge in the example's options.
1330 self.optionflags = original_optionflags
1331 if example.options:
1332 for (optionflag, val) in example.options.items():
1333 if val:
1334 self.optionflags |= optionflag
1335 else:
1336 self.optionflags &= ~optionflag
1337
1338 # If 'SKIP' is set, then skip this example.
1339 if self.optionflags & SKIP:
1340 continue
1341
1342 # Record that we started this example.
1343 tries += 1
1344 if not quiet:
1345 self.report_start(out, test, example)
1346
1347 # Use a special filename for compile(), so we can retrieve
1348 # the source code during interactive debugging (see
1349 # __patched_linecache_getlines).
1350 filename = '<doctest %s[%d]>' % (test.name, examplenum)
1351
1352 # Run the example in the given context (globs), and record
1353 # any exception that gets raised. (But don't intercept
1354 # keyboard interrupts.)
1355 try:
1356 # Don't blink! This is where the user's code gets run.
1357 exec(compile(example.source, filename, "single",
1358 compileflags, True), test.globs)
1359 self.debugger.set_continue() # ==== Example Finished ====
1360 exception = None
1361 except KeyboardInterrupt:
1362 raise
1363 except:
1364 exception = sys.exc_info()
1365 self.debugger.set_continue() # ==== Example Finished ====
1366
1367 got = self._fakeout.getvalue() # the actual output
1368 self._fakeout.truncate(0)
1369 outcome = FAILURE # guilty until proved innocent or insane
1370
1371 # If the example executed without raising any exceptions,
1372 # verify its output.
1373 if exception is None:
1374 if check(example.want, got, self.optionflags):
1375 outcome = SUCCESS
1376
1377 # The example raised an exception: check if it was expected.
1378 else:
1379 exc_msg = traceback.format_exception_only(*exception[:2])[-1]
1380 if not quiet:
1381 got += _exception_traceback(exception)
1382
1383 # If `example.exc_msg` is None, then we weren't expecting
1384 # an exception.
1385 if example.exc_msg is None:
1386 outcome = BOOM
1387
1388 # We expected an exception: see whether it matches.
1389 elif check(example.exc_msg, exc_msg, self.optionflags):
1390 outcome = SUCCESS
1391
1392 # Another chance if they didn't care about the detail.
1393 elif self.optionflags & IGNORE_EXCEPTION_DETAIL:
1394 if check(_strip_exception_details(example.exc_msg),
1395 _strip_exception_details(exc_msg),
1396 self.optionflags):
1397 outcome = SUCCESS
1398
1399 # Report the outcome.
1400 if outcome is SUCCESS:
1401 if not quiet:
1402 self.report_success(out, test, example, got)
1403 elif outcome is FAILURE:
1404 if not quiet:
1405 self.report_failure(out, test, example, got)
1406 failures += 1
1407 elif outcome is BOOM:
1408 if not quiet:
1409 self.report_unexpected_exception(out, test, example,
1410 exception)
1411 failures += 1
1412 else:
1413 assert False, ("unknown outcome", outcome)
1414
1415 if failures and self.optionflags & FAIL_FAST:
1416 break
1417
1418 # Restore the option flags (in case they were modified)
1419 self.optionflags = original_optionflags
1420
1421 # Record and return the number of failures and tries.
1422 self.__record_outcome(test, failures, tries)
1423 return TestResults(failures, tries)
1424
1425 def __record_outcome(self, test, f, t):
1426 """
1427 Record the fact that the given DocTest (`test`) generated `f`
1428 failures out of `t` tried examples.
1429 """
1430 f2, t2 = self._name2ft.get(test.name, (0,0))
1431 self._name2ft[test.name] = (f+f2, t+t2)
1432 self.failures += f
1433 self.tries += t
1434
1435 __LINECACHE_FILENAME_RE = re.compile(r'<doctest '
1436 r'(?P<name>.+)'
1437 r'\[(?P<examplenum>\d+)\]>$')
1438 def __patched_linecache_getlines(self, filename, module_globals=None):
1439 m = self.__LINECACHE_FILENAME_RE.match(filename)
1440 if m and m.group('name') == self.test.name:
1441 example = self.test.examples[int(m.group('examplenum'))]
1442 return example.source.splitlines(keepends=True)
1443 else:
1444 return self.save_linecache_getlines(filename, module_globals)
1445
1446 def run(self, test, compileflags=None, out=None, clear_globs=True):
1447 """
1448 Run the examples in `test`, and display the results using the
1449 writer function `out`.
1450
1451 The examples are run in the namespace `test.globs`. If
1452 `clear_globs` is true (the default), then this namespace will
1453 be cleared after the test runs, to help with garbage
1454 collection. If you would like to examine the namespace after
1455 the test completes, then use `clear_globs=False`.
1456
1457 `compileflags` gives the set of flags that should be used by
1458 the Python compiler when running the examples. If not
1459 specified, then it will default to the set of future-import
1460 flags that apply to `globs`.
1461
1462 The output of each example is checked using
1463 `DocTestRunner.check_output`, and the results are formatted by
1464 the `DocTestRunner.report_*` methods.
1465 """
1466 self.test = test
1467
1468 if compileflags is None:
1469 compileflags = _extract_future_flags(test.globs)
1470
1471 save_stdout = sys.stdout
1472 if out is None:
1473 encoding = save_stdout.encoding
1474 if encoding is None or encoding.lower() == 'utf-8':
1475 out = save_stdout.write
1476 else:
1477 # Use backslashreplace error handling on write
1478 def out(s):
1479 s = str(s.encode(encoding, 'backslashreplace'), encoding)
1480 save_stdout.write(s)
1481 sys.stdout = self._fakeout
1482
1483 # Patch pdb.set_trace to restore sys.stdout during interactive
1484 # debugging (so it's not still redirected to self._fakeout).
1485 # Note that the interactive output will go to *our*
1486 # save_stdout, even if that's not the real sys.stdout; this
1487 # allows us to write test cases for the set_trace behavior.
1488 save_trace = sys.gettrace()
1489 save_set_trace = pdb.set_trace
1490 self.debugger = _OutputRedirectingPdb(save_stdout)
1491 self.debugger.reset()
1492 pdb.set_trace = self.debugger.set_trace
1493
1494 # Patch linecache.getlines, so we can see the example's source
1495 # when we're inside the debugger.
1496 self.save_linecache_getlines = linecache.getlines
1497 linecache.getlines = self.__patched_linecache_getlines
1498
1499 # Make sure sys.displayhook just prints the value to stdout
1500 save_displayhook = sys.displayhook
1501 sys.displayhook = sys.__displayhook__
1502
1503 try:
1504 return self.__run(test, compileflags, out)
1505 finally:
1506 sys.stdout = save_stdout
1507 pdb.set_trace = save_set_trace
1508 sys.settrace(save_trace)
1509 linecache.getlines = self.save_linecache_getlines
1510 sys.displayhook = save_displayhook
1511 if clear_globs:
1512 test.globs.clear()
1513 import builtins
1514 builtins._ = None
1515
1516 #/////////////////////////////////////////////////////////////////
1517 # Summarization
1518 #/////////////////////////////////////////////////////////////////
1519 def summarize(self, verbose=None):
1520 """
1521 Print a summary of all the test cases that have been run by
1522 this DocTestRunner, and return a tuple `(f, t)`, where `f` is
1523 the total number of failed examples, and `t` is the total
1524 number of tried examples.
1525
1526 The optional `verbose` argument controls how detailed the
1527 summary is. If the verbosity is not specified, then the
1528 DocTestRunner's verbosity is used.
1529 """
1530 if verbose is None:
1531 verbose = self._verbose
1532 notests = []
1533 passed = []
1534 failed = []
1535 totalt = totalf = 0
1536 for x in self._name2ft.items():
1537 name, (f, t) = x
1538 assert f <= t
1539 totalt += t
1540 totalf += f
1541 if t == 0:
1542 notests.append(name)
1543 elif f == 0:
1544 passed.append( (name, t) )
1545 else:
1546 failed.append(x)
1547 if verbose:
1548 if notests:
1549 print(len(notests), "items had no tests:")
1550 notests.sort()
1551 for thing in notests:
1552 print(" ", thing)
1553 if passed:
1554 print(len(passed), "items passed all tests:")
1555 passed.sort()
1556 for thing, count in passed:
1557 print(" %3d tests in %s" % (count, thing))
1558 if failed:
1559 print(self.DIVIDER)
1560 print(len(failed), "items had failures:")
1561 failed.sort()
1562 for thing, (f, t) in failed:
1563 print(" %3d of %3d in %s" % (f, t, thing))
1564 if verbose:
1565 print(totalt, "tests in", len(self._name2ft), "items.")
1566 print(totalt - totalf, "passed and", totalf, "failed.")
1567 if totalf:
1568 print("***Test Failed***", totalf, "failures.")
1569 elif verbose:
1570 print("Test passed.")
1571 return TestResults(totalf, totalt)
1572
1573 #/////////////////////////////////////////////////////////////////
1574 # Backward compatibility cruft to maintain doctest.master.
1575 #/////////////////////////////////////////////////////////////////
1576 def merge(self, other):
1577 d = self._name2ft
1578 for name, (f, t) in other._name2ft.items():
1579 if name in d:
1580 # Don't print here by default, since doing
1581 # so breaks some of the buildbots
1582 #print("*** DocTestRunner.merge: '" + name + "' in both" \
1583 # " testers; summing outcomes.")
1584 f2, t2 = d[name]
1585 f = f + f2
1586 t = t + t2
1587 d[name] = f, t
1588
1589 class ESC[4;38;5;81mOutputChecker:
1590 """
1591 A class used to check the whether the actual output from a doctest
1592 example matches the expected output. `OutputChecker` defines two
1593 methods: `check_output`, which compares a given pair of outputs,
1594 and returns true if they match; and `output_difference`, which
1595 returns a string describing the differences between two outputs.
1596 """
1597 def _toAscii(self, s):
1598 """
1599 Convert string to hex-escaped ASCII string.
1600 """
1601 return str(s.encode('ASCII', 'backslashreplace'), "ASCII")
1602
1603 def check_output(self, want, got, optionflags):
1604 """
1605 Return True iff the actual output from an example (`got`)
1606 matches the expected output (`want`). These strings are
1607 always considered to match if they are identical; but
1608 depending on what option flags the test runner is using,
1609 several non-exact match types are also possible. See the
1610 documentation for `TestRunner` for more information about
1611 option flags.
1612 """
1613
1614 # If `want` contains hex-escaped character such as "\u1234",
1615 # then `want` is a string of six characters(e.g. [\,u,1,2,3,4]).
1616 # On the other hand, `got` could be another sequence of
1617 # characters such as [\u1234], so `want` and `got` should
1618 # be folded to hex-escaped ASCII string to compare.
1619 got = self._toAscii(got)
1620 want = self._toAscii(want)
1621
1622 # Handle the common case first, for efficiency:
1623 # if they're string-identical, always return true.
1624 if got == want:
1625 return True
1626
1627 # The values True and False replaced 1 and 0 as the return
1628 # value for boolean comparisons in Python 2.3.
1629 if not (optionflags & DONT_ACCEPT_TRUE_FOR_1):
1630 if (got,want) == ("True\n", "1\n"):
1631 return True
1632 if (got,want) == ("False\n", "0\n"):
1633 return True
1634
1635 # <BLANKLINE> can be used as a special sequence to signify a
1636 # blank line, unless the DONT_ACCEPT_BLANKLINE flag is used.
1637 if not (optionflags & DONT_ACCEPT_BLANKLINE):
1638 # Replace <BLANKLINE> in want with a blank line.
1639 want = re.sub(r'(?m)^%s\s*?$' % re.escape(BLANKLINE_MARKER),
1640 '', want)
1641 # If a line in got contains only spaces, then remove the
1642 # spaces.
1643 got = re.sub(r'(?m)^[^\S\n]+$', '', got)
1644 if got == want:
1645 return True
1646
1647 # This flag causes doctest to ignore any differences in the
1648 # contents of whitespace strings. Note that this can be used
1649 # in conjunction with the ELLIPSIS flag.
1650 if optionflags & NORMALIZE_WHITESPACE:
1651 got = ' '.join(got.split())
1652 want = ' '.join(want.split())
1653 if got == want:
1654 return True
1655
1656 # The ELLIPSIS flag says to let the sequence "..." in `want`
1657 # match any substring in `got`.
1658 if optionflags & ELLIPSIS:
1659 if _ellipsis_match(want, got):
1660 return True
1661
1662 # We didn't find any match; return false.
1663 return False
1664
1665 # Should we do a fancy diff?
1666 def _do_a_fancy_diff(self, want, got, optionflags):
1667 # Not unless they asked for a fancy diff.
1668 if not optionflags & (REPORT_UDIFF |
1669 REPORT_CDIFF |
1670 REPORT_NDIFF):
1671 return False
1672
1673 # If expected output uses ellipsis, a meaningful fancy diff is
1674 # too hard ... or maybe not. In two real-life failures Tim saw,
1675 # a diff was a major help anyway, so this is commented out.
1676 # [todo] _ellipsis_match() knows which pieces do and don't match,
1677 # and could be the basis for a kick-ass diff in this case.
1678 ##if optionflags & ELLIPSIS and ELLIPSIS_MARKER in want:
1679 ## return False
1680
1681 # ndiff does intraline difference marking, so can be useful even
1682 # for 1-line differences.
1683 if optionflags & REPORT_NDIFF:
1684 return True
1685
1686 # The other diff types need at least a few lines to be helpful.
1687 return want.count('\n') > 2 and got.count('\n') > 2
1688
1689 def output_difference(self, example, got, optionflags):
1690 """
1691 Return a string describing the differences between the
1692 expected output for a given example (`example`) and the actual
1693 output (`got`). `optionflags` is the set of option flags used
1694 to compare `want` and `got`.
1695 """
1696 want = example.want
1697 # If <BLANKLINE>s are being used, then replace blank lines
1698 # with <BLANKLINE> in the actual output string.
1699 if not (optionflags & DONT_ACCEPT_BLANKLINE):
1700 got = re.sub('(?m)^[ ]*(?=\n)', BLANKLINE_MARKER, got)
1701
1702 # Check if we should use diff.
1703 if self._do_a_fancy_diff(want, got, optionflags):
1704 # Split want & got into lines.
1705 want_lines = want.splitlines(keepends=True)
1706 got_lines = got.splitlines(keepends=True)
1707 # Use difflib to find their differences.
1708 if optionflags & REPORT_UDIFF:
1709 diff = difflib.unified_diff(want_lines, got_lines, n=2)
1710 diff = list(diff)[2:] # strip the diff header
1711 kind = 'unified diff with -expected +actual'
1712 elif optionflags & REPORT_CDIFF:
1713 diff = difflib.context_diff(want_lines, got_lines, n=2)
1714 diff = list(diff)[2:] # strip the diff header
1715 kind = 'context diff with expected followed by actual'
1716 elif optionflags & REPORT_NDIFF:
1717 engine = difflib.Differ(charjunk=difflib.IS_CHARACTER_JUNK)
1718 diff = list(engine.compare(want_lines, got_lines))
1719 kind = 'ndiff with -expected +actual'
1720 else:
1721 assert 0, 'Bad diff option'
1722 return 'Differences (%s):\n' % kind + _indent(''.join(diff))
1723
1724 # If we're not using diff, then simply list the expected
1725 # output followed by the actual output.
1726 if want and got:
1727 return 'Expected:\n%sGot:\n%s' % (_indent(want), _indent(got))
1728 elif want:
1729 return 'Expected:\n%sGot nothing\n' % _indent(want)
1730 elif got:
1731 return 'Expected nothing\nGot:\n%s' % _indent(got)
1732 else:
1733 return 'Expected nothing\nGot nothing\n'
1734
1735 class ESC[4;38;5;81mDocTestFailure(ESC[4;38;5;149mException):
1736 """A DocTest example has failed in debugging mode.
1737
1738 The exception instance has variables:
1739
1740 - test: the DocTest object being run
1741
1742 - example: the Example object that failed
1743
1744 - got: the actual output
1745 """
1746 def __init__(self, test, example, got):
1747 self.test = test
1748 self.example = example
1749 self.got = got
1750
1751 def __str__(self):
1752 return str(self.test)
1753
1754 class ESC[4;38;5;81mUnexpectedException(ESC[4;38;5;149mException):
1755 """A DocTest example has encountered an unexpected exception
1756
1757 The exception instance has variables:
1758
1759 - test: the DocTest object being run
1760
1761 - example: the Example object that failed
1762
1763 - exc_info: the exception info
1764 """
1765 def __init__(self, test, example, exc_info):
1766 self.test = test
1767 self.example = example
1768 self.exc_info = exc_info
1769
1770 def __str__(self):
1771 return str(self.test)
1772
1773 class ESC[4;38;5;81mDebugRunner(ESC[4;38;5;149mDocTestRunner):
1774 r"""Run doc tests but raise an exception as soon as there is a failure.
1775
1776 If an unexpected exception occurs, an UnexpectedException is raised.
1777 It contains the test, the example, and the original exception:
1778
1779 >>> runner = DebugRunner(verbose=False)
1780 >>> test = DocTestParser().get_doctest('>>> raise KeyError\n42',
1781 ... {}, 'foo', 'foo.py', 0)
1782 >>> try:
1783 ... runner.run(test)
1784 ... except UnexpectedException as f:
1785 ... failure = f
1786
1787 >>> failure.test is test
1788 True
1789
1790 >>> failure.example.want
1791 '42\n'
1792
1793 >>> exc_info = failure.exc_info
1794 >>> raise exc_info[1] # Already has the traceback
1795 Traceback (most recent call last):
1796 ...
1797 KeyError
1798
1799 We wrap the original exception to give the calling application
1800 access to the test and example information.
1801
1802 If the output doesn't match, then a DocTestFailure is raised:
1803
1804 >>> test = DocTestParser().get_doctest('''
1805 ... >>> x = 1
1806 ... >>> x
1807 ... 2
1808 ... ''', {}, 'foo', 'foo.py', 0)
1809
1810 >>> try:
1811 ... runner.run(test)
1812 ... except DocTestFailure as f:
1813 ... failure = f
1814
1815 DocTestFailure objects provide access to the test:
1816
1817 >>> failure.test is test
1818 True
1819
1820 As well as to the example:
1821
1822 >>> failure.example.want
1823 '2\n'
1824
1825 and the actual output:
1826
1827 >>> failure.got
1828 '1\n'
1829
1830 If a failure or error occurs, the globals are left intact:
1831
1832 >>> del test.globs['__builtins__']
1833 >>> test.globs
1834 {'x': 1}
1835
1836 >>> test = DocTestParser().get_doctest('''
1837 ... >>> x = 2
1838 ... >>> raise KeyError
1839 ... ''', {}, 'foo', 'foo.py', 0)
1840
1841 >>> runner.run(test)
1842 Traceback (most recent call last):
1843 ...
1844 doctest.UnexpectedException: <DocTest foo from foo.py:0 (2 examples)>
1845
1846 >>> del test.globs['__builtins__']
1847 >>> test.globs
1848 {'x': 2}
1849
1850 But the globals are cleared if there is no error:
1851
1852 >>> test = DocTestParser().get_doctest('''
1853 ... >>> x = 2
1854 ... ''', {}, 'foo', 'foo.py', 0)
1855
1856 >>> runner.run(test)
1857 TestResults(failed=0, attempted=1)
1858
1859 >>> test.globs
1860 {}
1861
1862 """
1863
1864 def run(self, test, compileflags=None, out=None, clear_globs=True):
1865 r = DocTestRunner.run(self, test, compileflags, out, False)
1866 if clear_globs:
1867 test.globs.clear()
1868 return r
1869
1870 def report_unexpected_exception(self, out, test, example, exc_info):
1871 raise UnexpectedException(test, example, exc_info)
1872
1873 def report_failure(self, out, test, example, got):
1874 raise DocTestFailure(test, example, got)
1875
1876 ######################################################################
1877 ## 6. Test Functions
1878 ######################################################################
1879 # These should be backwards compatible.
1880
1881 # For backward compatibility, a global instance of a DocTestRunner
1882 # class, updated by testmod.
1883 master = None
1884
1885 def testmod(m=None, name=None, globs=None, verbose=None,
1886 report=True, optionflags=0, extraglobs=None,
1887 raise_on_error=False, exclude_empty=False):
1888 """m=None, name=None, globs=None, verbose=None, report=True,
1889 optionflags=0, extraglobs=None, raise_on_error=False,
1890 exclude_empty=False
1891
1892 Test examples in docstrings in functions and classes reachable
1893 from module m (or the current module if m is not supplied), starting
1894 with m.__doc__.
1895
1896 Also test examples reachable from dict m.__test__ if it exists and is
1897 not None. m.__test__ maps names to functions, classes and strings;
1898 function and class docstrings are tested even if the name is private;
1899 strings are tested directly, as if they were docstrings.
1900
1901 Return (#failures, #tests).
1902
1903 See help(doctest) for an overview.
1904
1905 Optional keyword arg "name" gives the name of the module; by default
1906 use m.__name__.
1907
1908 Optional keyword arg "globs" gives a dict to be used as the globals
1909 when executing examples; by default, use m.__dict__. A copy of this
1910 dict is actually used for each docstring, so that each docstring's
1911 examples start with a clean slate.
1912
1913 Optional keyword arg "extraglobs" gives a dictionary that should be
1914 merged into the globals that are used to execute examples. By
1915 default, no extra globals are used. This is new in 2.4.
1916
1917 Optional keyword arg "verbose" prints lots of stuff if true, prints
1918 only failures if false; by default, it's true iff "-v" is in sys.argv.
1919
1920 Optional keyword arg "report" prints a summary at the end when true,
1921 else prints nothing at the end. In verbose mode, the summary is
1922 detailed, else very brief (in fact, empty if all tests passed).
1923
1924 Optional keyword arg "optionflags" or's together module constants,
1925 and defaults to 0. This is new in 2.3. Possible values (see the
1926 docs for details):
1927
1928 DONT_ACCEPT_TRUE_FOR_1
1929 DONT_ACCEPT_BLANKLINE
1930 NORMALIZE_WHITESPACE
1931 ELLIPSIS
1932 SKIP
1933 IGNORE_EXCEPTION_DETAIL
1934 REPORT_UDIFF
1935 REPORT_CDIFF
1936 REPORT_NDIFF
1937 REPORT_ONLY_FIRST_FAILURE
1938
1939 Optional keyword arg "raise_on_error" raises an exception on the
1940 first unexpected exception or failure. This allows failures to be
1941 post-mortem debugged.
1942
1943 Advanced tomfoolery: testmod runs methods of a local instance of
1944 class doctest.Tester, then merges the results into (or creates)
1945 global Tester instance doctest.master. Methods of doctest.master
1946 can be called directly too, if you want to do something unusual.
1947 Passing report=0 to testmod is especially useful then, to delay
1948 displaying a summary. Invoke doctest.master.summarize(verbose)
1949 when you're done fiddling.
1950 """
1951 global master
1952
1953 # If no module was given, then use __main__.
1954 if m is None:
1955 # DWA - m will still be None if this wasn't invoked from the command
1956 # line, in which case the following TypeError is about as good an error
1957 # as we should expect
1958 m = sys.modules.get('__main__')
1959
1960 # Check that we were actually given a module.
1961 if not inspect.ismodule(m):
1962 raise TypeError("testmod: module required; %r" % (m,))
1963
1964 # If no name was given, then use the module's name.
1965 if name is None:
1966 name = m.__name__
1967
1968 # Find, parse, and run all tests in the given module.
1969 finder = DocTestFinder(exclude_empty=exclude_empty)
1970
1971 if raise_on_error:
1972 runner = DebugRunner(verbose=verbose, optionflags=optionflags)
1973 else:
1974 runner = DocTestRunner(verbose=verbose, optionflags=optionflags)
1975
1976 for test in finder.find(m, name, globs=globs, extraglobs=extraglobs):
1977 runner.run(test)
1978
1979 if report:
1980 runner.summarize()
1981
1982 if master is None:
1983 master = runner
1984 else:
1985 master.merge(runner)
1986
1987 return TestResults(runner.failures, runner.tries)
1988
1989 def testfile(filename, module_relative=True, name=None, package=None,
1990 globs=None, verbose=None, report=True, optionflags=0,
1991 extraglobs=None, raise_on_error=False, parser=DocTestParser(),
1992 encoding=None):
1993 """
1994 Test examples in the given file. Return (#failures, #tests).
1995
1996 Optional keyword arg "module_relative" specifies how filenames
1997 should be interpreted:
1998
1999 - If "module_relative" is True (the default), then "filename"
2000 specifies a module-relative path. By default, this path is
2001 relative to the calling module's directory; but if the
2002 "package" argument is specified, then it is relative to that
2003 package. To ensure os-independence, "filename" should use
2004 "/" characters to separate path segments, and should not
2005 be an absolute path (i.e., it may not begin with "/").
2006
2007 - If "module_relative" is False, then "filename" specifies an
2008 os-specific path. The path may be absolute or relative (to
2009 the current working directory).
2010
2011 Optional keyword arg "name" gives the name of the test; by default
2012 use the file's basename.
2013
2014 Optional keyword argument "package" is a Python package or the
2015 name of a Python package whose directory should be used as the
2016 base directory for a module relative filename. If no package is
2017 specified, then the calling module's directory is used as the base
2018 directory for module relative filenames. It is an error to
2019 specify "package" if "module_relative" is False.
2020
2021 Optional keyword arg "globs" gives a dict to be used as the globals
2022 when executing examples; by default, use {}. A copy of this dict
2023 is actually used for each docstring, so that each docstring's
2024 examples start with a clean slate.
2025
2026 Optional keyword arg "extraglobs" gives a dictionary that should be
2027 merged into the globals that are used to execute examples. By
2028 default, no extra globals are used.
2029
2030 Optional keyword arg "verbose" prints lots of stuff if true, prints
2031 only failures if false; by default, it's true iff "-v" is in sys.argv.
2032
2033 Optional keyword arg "report" prints a summary at the end when true,
2034 else prints nothing at the end. In verbose mode, the summary is
2035 detailed, else very brief (in fact, empty if all tests passed).
2036
2037 Optional keyword arg "optionflags" or's together module constants,
2038 and defaults to 0. Possible values (see the docs for details):
2039
2040 DONT_ACCEPT_TRUE_FOR_1
2041 DONT_ACCEPT_BLANKLINE
2042 NORMALIZE_WHITESPACE
2043 ELLIPSIS
2044 SKIP
2045 IGNORE_EXCEPTION_DETAIL
2046 REPORT_UDIFF
2047 REPORT_CDIFF
2048 REPORT_NDIFF
2049 REPORT_ONLY_FIRST_FAILURE
2050
2051 Optional keyword arg "raise_on_error" raises an exception on the
2052 first unexpected exception or failure. This allows failures to be
2053 post-mortem debugged.
2054
2055 Optional keyword arg "parser" specifies a DocTestParser (or
2056 subclass) that should be used to extract tests from the files.
2057
2058 Optional keyword arg "encoding" specifies an encoding that should
2059 be used to convert the file to unicode.
2060
2061 Advanced tomfoolery: testmod runs methods of a local instance of
2062 class doctest.Tester, then merges the results into (or creates)
2063 global Tester instance doctest.master. Methods of doctest.master
2064 can be called directly too, if you want to do something unusual.
2065 Passing report=0 to testmod is especially useful then, to delay
2066 displaying a summary. Invoke doctest.master.summarize(verbose)
2067 when you're done fiddling.
2068 """
2069 global master
2070
2071 if package and not module_relative:
2072 raise ValueError("Package may only be specified for module-"
2073 "relative paths.")
2074
2075 # Relativize the path
2076 text, filename = _load_testfile(filename, package, module_relative,
2077 encoding or "utf-8")
2078
2079 # If no name was given, then use the file's name.
2080 if name is None:
2081 name = os.path.basename(filename)
2082
2083 # Assemble the globals.
2084 if globs is None:
2085 globs = {}
2086 else:
2087 globs = globs.copy()
2088 if extraglobs is not None:
2089 globs.update(extraglobs)
2090 if '__name__' not in globs:
2091 globs['__name__'] = '__main__'
2092
2093 if raise_on_error:
2094 runner = DebugRunner(verbose=verbose, optionflags=optionflags)
2095 else:
2096 runner = DocTestRunner(verbose=verbose, optionflags=optionflags)
2097
2098 # Read the file, convert it to a test, and run it.
2099 test = parser.get_doctest(text, globs, name, filename, 0)
2100 runner.run(test)
2101
2102 if report:
2103 runner.summarize()
2104
2105 if master is None:
2106 master = runner
2107 else:
2108 master.merge(runner)
2109
2110 return TestResults(runner.failures, runner.tries)
2111
2112 def run_docstring_examples(f, globs, verbose=False, name="NoName",
2113 compileflags=None, optionflags=0):
2114 """
2115 Test examples in the given object's docstring (`f`), using `globs`
2116 as globals. Optional argument `name` is used in failure messages.
2117 If the optional argument `verbose` is true, then generate output
2118 even if there are no failures.
2119
2120 `compileflags` gives the set of flags that should be used by the
2121 Python compiler when running the examples. If not specified, then
2122 it will default to the set of future-import flags that apply to
2123 `globs`.
2124
2125 Optional keyword arg `optionflags` specifies options for the
2126 testing and output. See the documentation for `testmod` for more
2127 information.
2128 """
2129 # Find, parse, and run all tests in the given module.
2130 finder = DocTestFinder(verbose=verbose, recurse=False)
2131 runner = DocTestRunner(verbose=verbose, optionflags=optionflags)
2132 for test in finder.find(f, name, globs=globs):
2133 runner.run(test, compileflags=compileflags)
2134
2135 ######################################################################
2136 ## 7. Unittest Support
2137 ######################################################################
2138
2139 _unittest_reportflags = 0
2140
2141 def set_unittest_reportflags(flags):
2142 """Sets the unittest option flags.
2143
2144 The old flag is returned so that a runner could restore the old
2145 value if it wished to:
2146
2147 >>> import doctest
2148 >>> old = doctest._unittest_reportflags
2149 >>> doctest.set_unittest_reportflags(REPORT_NDIFF |
2150 ... REPORT_ONLY_FIRST_FAILURE) == old
2151 True
2152
2153 >>> doctest._unittest_reportflags == (REPORT_NDIFF |
2154 ... REPORT_ONLY_FIRST_FAILURE)
2155 True
2156
2157 Only reporting flags can be set:
2158
2159 >>> doctest.set_unittest_reportflags(ELLIPSIS)
2160 Traceback (most recent call last):
2161 ...
2162 ValueError: ('Only reporting flags allowed', 8)
2163
2164 >>> doctest.set_unittest_reportflags(old) == (REPORT_NDIFF |
2165 ... REPORT_ONLY_FIRST_FAILURE)
2166 True
2167 """
2168 global _unittest_reportflags
2169
2170 if (flags & REPORTING_FLAGS) != flags:
2171 raise ValueError("Only reporting flags allowed", flags)
2172 old = _unittest_reportflags
2173 _unittest_reportflags = flags
2174 return old
2175
2176
2177 class ESC[4;38;5;81mDocTestCase(ESC[4;38;5;149munittestESC[4;38;5;149m.ESC[4;38;5;149mTestCase):
2178
2179 def __init__(self, test, optionflags=0, setUp=None, tearDown=None,
2180 checker=None):
2181
2182 unittest.TestCase.__init__(self)
2183 self._dt_optionflags = optionflags
2184 self._dt_checker = checker
2185 self._dt_globs = test.globs.copy()
2186 self._dt_test = test
2187 self._dt_setUp = setUp
2188 self._dt_tearDown = tearDown
2189
2190 def setUp(self):
2191 test = self._dt_test
2192
2193 if self._dt_setUp is not None:
2194 self._dt_setUp(test)
2195
2196 def tearDown(self):
2197 test = self._dt_test
2198
2199 if self._dt_tearDown is not None:
2200 self._dt_tearDown(test)
2201
2202 # restore the original globs
2203 test.globs.clear()
2204 test.globs.update(self._dt_globs)
2205
2206 def runTest(self):
2207 test = self._dt_test
2208 old = sys.stdout
2209 new = StringIO()
2210 optionflags = self._dt_optionflags
2211
2212 if not (optionflags & REPORTING_FLAGS):
2213 # The option flags don't include any reporting flags,
2214 # so add the default reporting flags
2215 optionflags |= _unittest_reportflags
2216
2217 runner = DocTestRunner(optionflags=optionflags,
2218 checker=self._dt_checker, verbose=False)
2219
2220 try:
2221 runner.DIVIDER = "-"*70
2222 failures, tries = runner.run(
2223 test, out=new.write, clear_globs=False)
2224 finally:
2225 sys.stdout = old
2226
2227 if failures:
2228 raise self.failureException(self.format_failure(new.getvalue()))
2229
2230 def format_failure(self, err):
2231 test = self._dt_test
2232 if test.lineno is None:
2233 lineno = 'unknown line number'
2234 else:
2235 lineno = '%s' % test.lineno
2236 lname = '.'.join(test.name.split('.')[-1:])
2237 return ('Failed doctest test for %s\n'
2238 ' File "%s", line %s, in %s\n\n%s'
2239 % (test.name, test.filename, lineno, lname, err)
2240 )
2241
2242 def debug(self):
2243 r"""Run the test case without results and without catching exceptions
2244
2245 The unit test framework includes a debug method on test cases
2246 and test suites to support post-mortem debugging. The test code
2247 is run in such a way that errors are not caught. This way a
2248 caller can catch the errors and initiate post-mortem debugging.
2249
2250 The DocTestCase provides a debug method that raises
2251 UnexpectedException errors if there is an unexpected
2252 exception:
2253
2254 >>> test = DocTestParser().get_doctest('>>> raise KeyError\n42',
2255 ... {}, 'foo', 'foo.py', 0)
2256 >>> case = DocTestCase(test)
2257 >>> try:
2258 ... case.debug()
2259 ... except UnexpectedException as f:
2260 ... failure = f
2261
2262 The UnexpectedException contains the test, the example, and
2263 the original exception:
2264
2265 >>> failure.test is test
2266 True
2267
2268 >>> failure.example.want
2269 '42\n'
2270
2271 >>> exc_info = failure.exc_info
2272 >>> raise exc_info[1] # Already has the traceback
2273 Traceback (most recent call last):
2274 ...
2275 KeyError
2276
2277 If the output doesn't match, then a DocTestFailure is raised:
2278
2279 >>> test = DocTestParser().get_doctest('''
2280 ... >>> x = 1
2281 ... >>> x
2282 ... 2
2283 ... ''', {}, 'foo', 'foo.py', 0)
2284 >>> case = DocTestCase(test)
2285
2286 >>> try:
2287 ... case.debug()
2288 ... except DocTestFailure as f:
2289 ... failure = f
2290
2291 DocTestFailure objects provide access to the test:
2292
2293 >>> failure.test is test
2294 True
2295
2296 As well as to the example:
2297
2298 >>> failure.example.want
2299 '2\n'
2300
2301 and the actual output:
2302
2303 >>> failure.got
2304 '1\n'
2305
2306 """
2307
2308 self.setUp()
2309 runner = DebugRunner(optionflags=self._dt_optionflags,
2310 checker=self._dt_checker, verbose=False)
2311 runner.run(self._dt_test, clear_globs=False)
2312 self.tearDown()
2313
2314 def id(self):
2315 return self._dt_test.name
2316
2317 def __eq__(self, other):
2318 if type(self) is not type(other):
2319 return NotImplemented
2320
2321 return self._dt_test == other._dt_test and \
2322 self._dt_optionflags == other._dt_optionflags and \
2323 self._dt_setUp == other._dt_setUp and \
2324 self._dt_tearDown == other._dt_tearDown and \
2325 self._dt_checker == other._dt_checker
2326
2327 def __hash__(self):
2328 return hash((self._dt_optionflags, self._dt_setUp, self._dt_tearDown,
2329 self._dt_checker))
2330
2331 def __repr__(self):
2332 name = self._dt_test.name.split('.')
2333 return "%s (%s)" % (name[-1], '.'.join(name[:-1]))
2334
2335 __str__ = object.__str__
2336
2337 def shortDescription(self):
2338 return "Doctest: " + self._dt_test.name
2339
2340 class ESC[4;38;5;81mSkipDocTestCase(ESC[4;38;5;149mDocTestCase):
2341 def __init__(self, module):
2342 self.module = module
2343 DocTestCase.__init__(self, None)
2344
2345 def setUp(self):
2346 self.skipTest("DocTestSuite will not work with -O2 and above")
2347
2348 def test_skip(self):
2349 pass
2350
2351 def shortDescription(self):
2352 return "Skipping tests from %s" % self.module.__name__
2353
2354 __str__ = shortDescription
2355
2356
2357 class ESC[4;38;5;81m_DocTestSuite(ESC[4;38;5;149munittestESC[4;38;5;149m.ESC[4;38;5;149mTestSuite):
2358
2359 def _removeTestAtIndex(self, index):
2360 pass
2361
2362
2363 def DocTestSuite(module=None, globs=None, extraglobs=None, test_finder=None,
2364 **options):
2365 """
2366 Convert doctest tests for a module to a unittest test suite.
2367
2368 This converts each documentation string in a module that
2369 contains doctest tests to a unittest test case. If any of the
2370 tests in a doc string fail, then the test case fails. An exception
2371 is raised showing the name of the file containing the test and a
2372 (sometimes approximate) line number.
2373
2374 The `module` argument provides the module to be tested. The argument
2375 can be either a module or a module name.
2376
2377 If no argument is given, the calling module is used.
2378
2379 A number of options may be provided as keyword arguments:
2380
2381 setUp
2382 A set-up function. This is called before running the
2383 tests in each file. The setUp function will be passed a DocTest
2384 object. The setUp function can access the test globals as the
2385 globs attribute of the test passed.
2386
2387 tearDown
2388 A tear-down function. This is called after running the
2389 tests in each file. The tearDown function will be passed a DocTest
2390 object. The tearDown function can access the test globals as the
2391 globs attribute of the test passed.
2392
2393 globs
2394 A dictionary containing initial global variables for the tests.
2395
2396 optionflags
2397 A set of doctest option flags expressed as an integer.
2398 """
2399
2400 if test_finder is None:
2401 test_finder = DocTestFinder()
2402
2403 module = _normalize_module(module)
2404 tests = test_finder.find(module, globs=globs, extraglobs=extraglobs)
2405
2406 if not tests and sys.flags.optimize >=2:
2407 # Skip doctests when running with -O2
2408 suite = _DocTestSuite()
2409 suite.addTest(SkipDocTestCase(module))
2410 return suite
2411
2412 tests.sort()
2413 suite = _DocTestSuite()
2414
2415 for test in tests:
2416 if len(test.examples) == 0:
2417 continue
2418 if not test.filename:
2419 filename = module.__file__
2420 if filename[-4:] == ".pyc":
2421 filename = filename[:-1]
2422 test.filename = filename
2423 suite.addTest(DocTestCase(test, **options))
2424
2425 return suite
2426
2427 class ESC[4;38;5;81mDocFileCase(ESC[4;38;5;149mDocTestCase):
2428
2429 def id(self):
2430 return '_'.join(self._dt_test.name.split('.'))
2431
2432 def __repr__(self):
2433 return self._dt_test.filename
2434
2435 def format_failure(self, err):
2436 return ('Failed doctest test for %s\n File "%s", line 0\n\n%s'
2437 % (self._dt_test.name, self._dt_test.filename, err)
2438 )
2439
2440 def DocFileTest(path, module_relative=True, package=None,
2441 globs=None, parser=DocTestParser(),
2442 encoding=None, **options):
2443 if globs is None:
2444 globs = {}
2445 else:
2446 globs = globs.copy()
2447
2448 if package and not module_relative:
2449 raise ValueError("Package may only be specified for module-"
2450 "relative paths.")
2451
2452 # Relativize the path.
2453 doc, path = _load_testfile(path, package, module_relative,
2454 encoding or "utf-8")
2455
2456 if "__file__" not in globs:
2457 globs["__file__"] = path
2458
2459 # Find the file and read it.
2460 name = os.path.basename(path)
2461
2462 # Convert it to a test, and wrap it in a DocFileCase.
2463 test = parser.get_doctest(doc, globs, name, path, 0)
2464 return DocFileCase(test, **options)
2465
2466 def DocFileSuite(*paths, **kw):
2467 """A unittest suite for one or more doctest files.
2468
2469 The path to each doctest file is given as a string; the
2470 interpretation of that string depends on the keyword argument
2471 "module_relative".
2472
2473 A number of options may be provided as keyword arguments:
2474
2475 module_relative
2476 If "module_relative" is True, then the given file paths are
2477 interpreted as os-independent module-relative paths. By
2478 default, these paths are relative to the calling module's
2479 directory; but if the "package" argument is specified, then
2480 they are relative to that package. To ensure os-independence,
2481 "filename" should use "/" characters to separate path
2482 segments, and may not be an absolute path (i.e., it may not
2483 begin with "/").
2484
2485 If "module_relative" is False, then the given file paths are
2486 interpreted as os-specific paths. These paths may be absolute
2487 or relative (to the current working directory).
2488
2489 package
2490 A Python package or the name of a Python package whose directory
2491 should be used as the base directory for module relative paths.
2492 If "package" is not specified, then the calling module's
2493 directory is used as the base directory for module relative
2494 filenames. It is an error to specify "package" if
2495 "module_relative" is False.
2496
2497 setUp
2498 A set-up function. This is called before running the
2499 tests in each file. The setUp function will be passed a DocTest
2500 object. The setUp function can access the test globals as the
2501 globs attribute of the test passed.
2502
2503 tearDown
2504 A tear-down function. This is called after running the
2505 tests in each file. The tearDown function will be passed a DocTest
2506 object. The tearDown function can access the test globals as the
2507 globs attribute of the test passed.
2508
2509 globs
2510 A dictionary containing initial global variables for the tests.
2511
2512 optionflags
2513 A set of doctest option flags expressed as an integer.
2514
2515 parser
2516 A DocTestParser (or subclass) that should be used to extract
2517 tests from the files.
2518
2519 encoding
2520 An encoding that will be used to convert the files to unicode.
2521 """
2522 suite = _DocTestSuite()
2523
2524 # We do this here so that _normalize_module is called at the right
2525 # level. If it were called in DocFileTest, then this function
2526 # would be the caller and we might guess the package incorrectly.
2527 if kw.get('module_relative', True):
2528 kw['package'] = _normalize_module(kw.get('package'))
2529
2530 for path in paths:
2531 suite.addTest(DocFileTest(path, **kw))
2532
2533 return suite
2534
2535 ######################################################################
2536 ## 8. Debugging Support
2537 ######################################################################
2538
2539 def script_from_examples(s):
2540 r"""Extract script from text with examples.
2541
2542 Converts text with examples to a Python script. Example input is
2543 converted to regular code. Example output and all other words
2544 are converted to comments:
2545
2546 >>> text = '''
2547 ... Here are examples of simple math.
2548 ...
2549 ... Python has super accurate integer addition
2550 ...
2551 ... >>> 2 + 2
2552 ... 5
2553 ...
2554 ... And very friendly error messages:
2555 ...
2556 ... >>> 1/0
2557 ... To Infinity
2558 ... And
2559 ... Beyond
2560 ...
2561 ... You can use logic if you want:
2562 ...
2563 ... >>> if 0:
2564 ... ... blah
2565 ... ... blah
2566 ... ...
2567 ...
2568 ... Ho hum
2569 ... '''
2570
2571 >>> print(script_from_examples(text))
2572 # Here are examples of simple math.
2573 #
2574 # Python has super accurate integer addition
2575 #
2576 2 + 2
2577 # Expected:
2578 ## 5
2579 #
2580 # And very friendly error messages:
2581 #
2582 1/0
2583 # Expected:
2584 ## To Infinity
2585 ## And
2586 ## Beyond
2587 #
2588 # You can use logic if you want:
2589 #
2590 if 0:
2591 blah
2592 blah
2593 #
2594 # Ho hum
2595 <BLANKLINE>
2596 """
2597 output = []
2598 for piece in DocTestParser().parse(s):
2599 if isinstance(piece, Example):
2600 # Add the example's source code (strip trailing NL)
2601 output.append(piece.source[:-1])
2602 # Add the expected output:
2603 want = piece.want
2604 if want:
2605 output.append('# Expected:')
2606 output += ['## '+l for l in want.split('\n')[:-1]]
2607 else:
2608 # Add non-example text.
2609 output += [_comment_line(l)
2610 for l in piece.split('\n')[:-1]]
2611
2612 # Trim junk on both ends.
2613 while output and output[-1] == '#':
2614 output.pop()
2615 while output and output[0] == '#':
2616 output.pop(0)
2617 # Combine the output, and return it.
2618 # Add a courtesy newline to prevent exec from choking (see bug #1172785)
2619 return '\n'.join(output) + '\n'
2620
2621 def testsource(module, name):
2622 """Extract the test sources from a doctest docstring as a script.
2623
2624 Provide the module (or dotted name of the module) containing the
2625 test to be debugged and the name (within the module) of the object
2626 with the doc string with tests to be debugged.
2627 """
2628 module = _normalize_module(module)
2629 tests = DocTestFinder().find(module)
2630 test = [t for t in tests if t.name == name]
2631 if not test:
2632 raise ValueError(name, "not found in tests")
2633 test = test[0]
2634 testsrc = script_from_examples(test.docstring)
2635 return testsrc
2636
2637 def debug_src(src, pm=False, globs=None):
2638 """Debug a single doctest docstring, in argument `src`'"""
2639 testsrc = script_from_examples(src)
2640 debug_script(testsrc, pm, globs)
2641
2642 def debug_script(src, pm=False, globs=None):
2643 "Debug a test script. `src` is the script, as a string."
2644 import pdb
2645
2646 if globs:
2647 globs = globs.copy()
2648 else:
2649 globs = {}
2650
2651 if pm:
2652 try:
2653 exec(src, globs, globs)
2654 except:
2655 print(sys.exc_info()[1])
2656 p = pdb.Pdb(nosigint=True)
2657 p.reset()
2658 p.interaction(None, sys.exc_info()[2])
2659 else:
2660 pdb.Pdb(nosigint=True).run("exec(%r)" % src, globs, globs)
2661
2662 def debug(module, name, pm=False):
2663 """Debug a single doctest docstring.
2664
2665 Provide the module (or dotted name of the module) containing the
2666 test to be debugged and the name (within the module) of the object
2667 with the docstring with tests to be debugged.
2668 """
2669 module = _normalize_module(module)
2670 testsrc = testsource(module, name)
2671 debug_script(testsrc, pm, module.__dict__)
2672
2673 ######################################################################
2674 ## 9. Example Usage
2675 ######################################################################
2676 class ESC[4;38;5;81m_TestClass:
2677 """
2678 A pointless class, for sanity-checking of docstring testing.
2679
2680 Methods:
2681 square()
2682 get()
2683
2684 >>> _TestClass(13).get() + _TestClass(-12).get()
2685 1
2686 >>> hex(_TestClass(13).square().get())
2687 '0xa9'
2688 """
2689
2690 def __init__(self, val):
2691 """val -> _TestClass object with associated value val.
2692
2693 >>> t = _TestClass(123)
2694 >>> print(t.get())
2695 123
2696 """
2697
2698 self.val = val
2699
2700 def square(self):
2701 """square() -> square TestClass's associated value
2702
2703 >>> _TestClass(13).square().get()
2704 169
2705 """
2706
2707 self.val = self.val ** 2
2708 return self
2709
2710 def get(self):
2711 """get() -> return TestClass's associated value.
2712
2713 >>> x = _TestClass(-42)
2714 >>> print(x.get())
2715 -42
2716 """
2717
2718 return self.val
2719
2720 __test__ = {"_TestClass": _TestClass,
2721 "string": r"""
2722 Example of a string object, searched as-is.
2723 >>> x = 1; y = 2
2724 >>> x + y, x * y
2725 (3, 2)
2726 """,
2727
2728 "bool-int equivalence": r"""
2729 In 2.2, boolean expressions displayed
2730 0 or 1. By default, we still accept
2731 them. This can be disabled by passing
2732 DONT_ACCEPT_TRUE_FOR_1 to the new
2733 optionflags argument.
2734 >>> 4 == 4
2735 1
2736 >>> 4 == 4
2737 True
2738 >>> 4 > 4
2739 0
2740 >>> 4 > 4
2741 False
2742 """,
2743
2744 "blank lines": r"""
2745 Blank lines can be marked with <BLANKLINE>:
2746 >>> print('foo\n\nbar\n')
2747 foo
2748 <BLANKLINE>
2749 bar
2750 <BLANKLINE>
2751 """,
2752
2753 "ellipsis": r"""
2754 If the ellipsis flag is used, then '...' can be used to
2755 elide substrings in the desired output:
2756 >>> print(list(range(1000))) #doctest: +ELLIPSIS
2757 [0, 1, 2, ..., 999]
2758 """,
2759
2760 "whitespace normalization": r"""
2761 If the whitespace normalization flag is used, then
2762 differences in whitespace are ignored.
2763 >>> print(list(range(30))) #doctest: +NORMALIZE_WHITESPACE
2764 [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14,
2765 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26,
2766 27, 28, 29]
2767 """,
2768 }
2769
2770
2771 def _test():
2772 import argparse
2773
2774 parser = argparse.ArgumentParser(description="doctest runner")
2775 parser.add_argument('-v', '--verbose', action='store_true', default=False,
2776 help='print very verbose output for all tests')
2777 parser.add_argument('-o', '--option', action='append',
2778 choices=OPTIONFLAGS_BY_NAME.keys(), default=[],
2779 help=('specify a doctest option flag to apply'
2780 ' to the test run; may be specified more'
2781 ' than once to apply multiple options'))
2782 parser.add_argument('-f', '--fail-fast', action='store_true',
2783 help=('stop running tests after first failure (this'
2784 ' is a shorthand for -o FAIL_FAST, and is'
2785 ' in addition to any other -o options)'))
2786 parser.add_argument('file', nargs='+',
2787 help='file containing the tests to run')
2788 args = parser.parse_args()
2789 testfiles = args.file
2790 # Verbose used to be handled by the "inspect argv" magic in DocTestRunner,
2791 # but since we are using argparse we are passing it manually now.
2792 verbose = args.verbose
2793 options = 0
2794 for option in args.option:
2795 options |= OPTIONFLAGS_BY_NAME[option]
2796 if args.fail_fast:
2797 options |= FAIL_FAST
2798 for filename in testfiles:
2799 if filename.endswith(".py"):
2800 # It is a module -- insert its dir into sys.path and try to
2801 # import it. If it is part of a package, that possibly
2802 # won't work because of package imports.
2803 dirname, filename = os.path.split(filename)
2804 sys.path.insert(0, dirname)
2805 m = __import__(filename[:-3])
2806 del sys.path[0]
2807 failures, _ = testmod(m, verbose=verbose, optionflags=options)
2808 else:
2809 failures, _ = testfile(filename, module_relative=False,
2810 verbose=verbose, optionflags=options)
2811 if failures:
2812 return 1
2813 return 0
2814
2815
2816 if __name__ == "__main__":
2817 sys.exit(_test())