1 # Module doctest.
2 # Released to the public domain 16-Jan-2001, by Tim Peters (tim@python.org).
3 # Major enhancements and refactoring by:
4 # Jim Fulton
5 # Edward Loper
6
7 # Provided as-is; use at your own risk; no warranty; no promises; enjoy!
8
9 r"""Module doctest -- a framework for running examples in docstrings.
10
11 In simplest use, end each module M to be tested with:
12
13 def _test():
14 import doctest
15 doctest.testmod()
16
17 if __name__ == "__main__":
18 _test()
19
20 Then running the module as a script will cause the examples in the
21 docstrings to get executed and verified:
22
23 python M.py
24
25 This won't display anything unless an example fails, in which case the
26 failing example(s) and the cause(s) of the failure(s) are printed to stdout
27 (why not stderr? because stderr is a lame hack <0.2 wink>), and the final
28 line of output is "Test failed.".
29
30 Run it with the -v switch instead:
31
32 python M.py -v
33
34 and a detailed report of all examples tried is printed to stdout, along
35 with assorted summaries at the end.
36
37 You can force verbose mode by passing "verbose=True" to testmod, or prohibit
38 it by passing "verbose=False". In either of those cases, sys.argv is not
39 examined by testmod.
40
41 There are a variety of other ways to run doctests, including integration
42 with the unittest framework, and support for running non-Python text
43 files containing doctests. There are also many ways to override parts
44 of doctest's default behaviors. See the Library Reference Manual for
45 details.
46 """
47
48 __docformat__ = 'reStructuredText en'
49
50 __all__ = [
51 # 0, Option Flags
52 'register_optionflag',
53 'DONT_ACCEPT_TRUE_FOR_1',
54 'DONT_ACCEPT_BLANKLINE',
55 'NORMALIZE_WHITESPACE',
56 'ELLIPSIS',
57 'SKIP',
58 'IGNORE_EXCEPTION_DETAIL',
59 'COMPARISON_FLAGS',
60 'REPORT_UDIFF',
61 'REPORT_CDIFF',
62 'REPORT_NDIFF',
63 'REPORT_ONLY_FIRST_FAILURE',
64 'REPORTING_FLAGS',
65 'FAIL_FAST',
66 # 1. Utility Functions
67 # 2. Example & DocTest
68 'Example',
69 'DocTest',
70 # 3. Doctest Parser
71 'DocTestParser',
72 # 4. Doctest Finder
73 'DocTestFinder',
74 # 5. Doctest Runner
75 'DocTestRunner',
76 'OutputChecker',
77 'DocTestFailure',
78 'UnexpectedException',
79 'DebugRunner',
80 # 6. Test Functions
81 'testmod',
82 'testfile',
83 'run_docstring_examples',
84 # 7. Unittest Support
85 'DocTestSuite',
86 'DocFileSuite',
87 'set_unittest_reportflags',
88 # 8. Debugging Support
89 'script_from_examples',
90 'testsource',
91 'debug_src',
92 'debug',
93 ]
94
95 import __future__
96 import difflib
97 import inspect
98 import linecache
99 import os
100 import pdb
101 import re
102 import sys
103 import traceback
104 import unittest
105 from io import StringIO, IncrementalNewlineDecoder
106 from collections import namedtuple
107
108 TestResults = namedtuple('TestResults', 'failed attempted')
109
110 # There are 4 basic classes:
111 # - Example: a <source, want> pair, plus an intra-docstring line number.
112 # - DocTest: a collection of examples, parsed from a docstring, plus
113 # info about where the docstring came from (name, filename, lineno).
114 # - DocTestFinder: extracts DocTests from a given object's docstring and
115 # its contained objects' docstrings.
116 # - DocTestRunner: runs DocTest cases, and accumulates statistics.
117 #
118 # So the basic picture is:
119 #
120 # list of:
121 # +------+ +---------+ +-------+
122 # |object| --DocTestFinder-> | DocTest | --DocTestRunner-> |results|
123 # +------+ +---------+ +-------+
124 # | Example |
125 # | ... |
126 # | Example |
127 # +---------+
128
129 # Option constants.
130
131 OPTIONFLAGS_BY_NAME = {}
132 def register_optionflag(name):
133 # Create a new flag unless `name` is already known.
134 return OPTIONFLAGS_BY_NAME.setdefault(name, 1 << len(OPTIONFLAGS_BY_NAME))
135
136 DONT_ACCEPT_TRUE_FOR_1 = register_optionflag('DONT_ACCEPT_TRUE_FOR_1')
137 DONT_ACCEPT_BLANKLINE = register_optionflag('DONT_ACCEPT_BLANKLINE')
138 NORMALIZE_WHITESPACE = register_optionflag('NORMALIZE_WHITESPACE')
139 ELLIPSIS = register_optionflag('ELLIPSIS')
140 SKIP = register_optionflag('SKIP')
141 IGNORE_EXCEPTION_DETAIL = register_optionflag('IGNORE_EXCEPTION_DETAIL')
142
143 COMPARISON_FLAGS = (DONT_ACCEPT_TRUE_FOR_1 |
144 DONT_ACCEPT_BLANKLINE |
145 NORMALIZE_WHITESPACE |
146 ELLIPSIS |
147 SKIP |
148 IGNORE_EXCEPTION_DETAIL)
149
150 REPORT_UDIFF = register_optionflag('REPORT_UDIFF')
151 REPORT_CDIFF = register_optionflag('REPORT_CDIFF')
152 REPORT_NDIFF = register_optionflag('REPORT_NDIFF')
153 REPORT_ONLY_FIRST_FAILURE = register_optionflag('REPORT_ONLY_FIRST_FAILURE')
154 FAIL_FAST = register_optionflag('FAIL_FAST')
155
156 REPORTING_FLAGS = (REPORT_UDIFF |
157 REPORT_CDIFF |
158 REPORT_NDIFF |
159 REPORT_ONLY_FIRST_FAILURE |
160 FAIL_FAST)
161
162 # Special string markers for use in `want` strings:
163 BLANKLINE_MARKER = '<BLANKLINE>'
164 ELLIPSIS_MARKER = '...'
165
166 ######################################################################
167 ## Table of Contents
168 ######################################################################
169 # 1. Utility Functions
170 # 2. Example & DocTest -- store test cases
171 # 3. DocTest Parser -- extracts examples from strings
172 # 4. DocTest Finder -- extracts test cases from objects
173 # 5. DocTest Runner -- runs test cases
174 # 6. Test Functions -- convenient wrappers for testing
175 # 7. Unittest Support
176 # 8. Debugging Support
177 # 9. Example Usage
178
179 ######################################################################
180 ## 1. Utility Functions
181 ######################################################################
182
183 def _extract_future_flags(globs):
184 """
185 Return the compiler-flags associated with the future features that
186 have been imported into the given namespace (globs).
187 """
188 flags = 0
189 for fname in __future__.all_feature_names:
190 feature = globs.get(fname, None)
191 if feature is getattr(__future__, fname):
192 flags |= feature.compiler_flag
193 return flags
194
195 def _normalize_module(module, depth=2):
196 """
197 Return the module specified by `module`. In particular:
198 - If `module` is a module, then return module.
199 - If `module` is a string, then import and return the
200 module with that name.
201 - If `module` is None, then return the calling module.
202 The calling module is assumed to be the module of
203 the stack frame at the given depth in the call stack.
204 """
205 if inspect.ismodule(module):
206 return module
207 elif isinstance(module, str):
208 return __import__(module, globals(), locals(), ["*"])
209 elif module is None:
210 return sys.modules[sys._getframe(depth).f_globals['__name__']]
211 else:
212 raise TypeError("Expected a module, string, or None")
213
214 def _newline_convert(data):
215 # The IO module provides a handy decoder for universal newline conversion
216 return IncrementalNewlineDecoder(None, True).decode(data, True)
217
218 def _load_testfile(filename, package, module_relative, encoding):
219 if module_relative:
220 package = _normalize_module(package, 3)
221 filename = _module_relative_path(package, filename)
222 if (loader := getattr(package, '__loader__', None)) is None:
223 try:
224 loader = package.__spec__.loader
225 except AttributeError:
226 pass
227 if hasattr(loader, 'get_data'):
228 file_contents = loader.get_data(filename)
229 file_contents = file_contents.decode(encoding)
230 # get_data() opens files as 'rb', so one must do the equivalent
231 # conversion as universal newlines would do.
232 return _newline_convert(file_contents), filename
233 with open(filename, encoding=encoding) as f:
234 return f.read(), filename
235
236 def _indent(s, indent=4):
237 """
238 Add the given number of space characters to the beginning of
239 every non-blank line in `s`, and return the result.
240 """
241 # This regexp matches the start of non-blank lines:
242 return re.sub('(?m)^(?!$)', indent*' ', s)
243
244 def _exception_traceback(exc_info):
245 """
246 Return a string containing a traceback message for the given
247 exc_info tuple (as returned by sys.exc_info()).
248 """
249 # Get a traceback message.
250 excout = StringIO()
251 exc_type, exc_val, exc_tb = exc_info
252 traceback.print_exception(exc_type, exc_val, exc_tb, file=excout)
253 return excout.getvalue()
254
255 # Override some StringIO methods.
256 class ESC[4;38;5;81m_SpoofOut(ESC[4;38;5;149mStringIO):
257 def getvalue(self):
258 result = StringIO.getvalue(self)
259 # If anything at all was written, make sure there's a trailing
260 # newline. There's no way for the expected output to indicate
261 # that a trailing newline is missing.
262 if result and not result.endswith("\n"):
263 result += "\n"
264 return result
265
266 def truncate(self, size=None):
267 self.seek(size)
268 StringIO.truncate(self)
269
270 # Worst-case linear-time ellipsis matching.
271 def _ellipsis_match(want, got):
272 """
273 Essentially the only subtle case:
274 >>> _ellipsis_match('aa...aa', 'aaa')
275 False
276 """
277 if ELLIPSIS_MARKER not in want:
278 return want == got
279
280 # Find "the real" strings.
281 ws = want.split(ELLIPSIS_MARKER)
282 assert len(ws) >= 2
283
284 # Deal with exact matches possibly needed at one or both ends.
285 startpos, endpos = 0, len(got)
286 w = ws[0]
287 if w: # starts with exact match
288 if got.startswith(w):
289 startpos = len(w)
290 del ws[0]
291 else:
292 return False
293 w = ws[-1]
294 if w: # ends with exact match
295 if got.endswith(w):
296 endpos -= len(w)
297 del ws[-1]
298 else:
299 return False
300
301 if startpos > endpos:
302 # Exact end matches required more characters than we have, as in
303 # _ellipsis_match('aa...aa', 'aaa')
304 return False
305
306 # For the rest, we only need to find the leftmost non-overlapping
307 # match for each piece. If there's no overall match that way alone,
308 # there's no overall match period.
309 for w in ws:
310 # w may be '' at times, if there are consecutive ellipses, or
311 # due to an ellipsis at the start or end of `want`. That's OK.
312 # Search for an empty string succeeds, and doesn't change startpos.
313 startpos = got.find(w, startpos, endpos)
314 if startpos < 0:
315 return False
316 startpos += len(w)
317
318 return True
319
320 def _comment_line(line):
321 "Return a commented form of the given line"
322 line = line.rstrip()
323 if line:
324 return '# '+line
325 else:
326 return '#'
327
328 def _strip_exception_details(msg):
329 # Support for IGNORE_EXCEPTION_DETAIL.
330 # Get rid of everything except the exception name; in particular, drop
331 # the possibly dotted module path (if any) and the exception message (if
332 # any). We assume that a colon is never part of a dotted name, or of an
333 # exception name.
334 # E.g., given
335 # "foo.bar.MyError: la di da"
336 # return "MyError"
337 # Or for "abc.def" or "abc.def:\n" return "def".
338
339 start, end = 0, len(msg)
340 # The exception name must appear on the first line.
341 i = msg.find("\n")
342 if i >= 0:
343 end = i
344 # retain up to the first colon (if any)
345 i = msg.find(':', 0, end)
346 if i >= 0:
347 end = i
348 # retain just the exception name
349 i = msg.rfind('.', 0, end)
350 if i >= 0:
351 start = i+1
352 return msg[start: end]
353
354 class ESC[4;38;5;81m_OutputRedirectingPdb(ESC[4;38;5;149mpdbESC[4;38;5;149m.ESC[4;38;5;149mPdb):
355 """
356 A specialized version of the python debugger that redirects stdout
357 to a given stream when interacting with the user. Stdout is *not*
358 redirected when traced code is executed.
359 """
360 def __init__(self, out):
361 self.__out = out
362 self.__debugger_used = False
363 # do not play signal games in the pdb
364 pdb.Pdb.__init__(self, stdout=out, nosigint=True)
365 # still use input() to get user input
366 self.use_rawinput = 1
367
368 def set_trace(self, frame=None):
369 self.__debugger_used = True
370 if frame is None:
371 frame = sys._getframe().f_back
372 pdb.Pdb.set_trace(self, frame)
373
374 def set_continue(self):
375 # Calling set_continue unconditionally would break unit test
376 # coverage reporting, as Bdb.set_continue calls sys.settrace(None).
377 if self.__debugger_used:
378 pdb.Pdb.set_continue(self)
379
380 def trace_dispatch(self, *args):
381 # Redirect stdout to the given stream.
382 save_stdout = sys.stdout
383 sys.stdout = self.__out
384 # Call Pdb's trace dispatch method.
385 try:
386 return pdb.Pdb.trace_dispatch(self, *args)
387 finally:
388 sys.stdout = save_stdout
389
390 # [XX] Normalize with respect to os.path.pardir?
391 def _module_relative_path(module, test_path):
392 if not inspect.ismodule(module):
393 raise TypeError('Expected a module: %r' % module)
394 if test_path.startswith('/'):
395 raise ValueError('Module-relative files may not have absolute paths')
396
397 # Normalize the path. On Windows, replace "/" with "\".
398 test_path = os.path.join(*(test_path.split('/')))
399
400 # Find the base directory for the path.
401 if hasattr(module, '__file__'):
402 # A normal module/package
403 basedir = os.path.split(module.__file__)[0]
404 elif module.__name__ == '__main__':
405 # An interactive session.
406 if len(sys.argv)>0 and sys.argv[0] != '':
407 basedir = os.path.split(sys.argv[0])[0]
408 else:
409 basedir = os.curdir
410 else:
411 if hasattr(module, '__path__'):
412 for directory in module.__path__:
413 fullpath = os.path.join(directory, test_path)
414 if os.path.exists(fullpath):
415 return fullpath
416
417 # A module w/o __file__ (this includes builtins)
418 raise ValueError("Can't resolve paths relative to the module "
419 "%r (it has no __file__)"
420 % module.__name__)
421
422 # Combine the base directory and the test path.
423 return os.path.join(basedir, test_path)
424
425 ######################################################################
426 ## 2. Example & DocTest
427 ######################################################################
428 ## - An "example" is a <source, want> pair, where "source" is a
429 ## fragment of source code, and "want" is the expected output for
430 ## "source." The Example class also includes information about
431 ## where the example was extracted from.
432 ##
433 ## - A "doctest" is a collection of examples, typically extracted from
434 ## a string (such as an object's docstring). The DocTest class also
435 ## includes information about where the string was extracted from.
436
437 class ESC[4;38;5;81mExample:
438 """
439 A single doctest example, consisting of source code and expected
440 output. `Example` defines the following attributes:
441
442 - source: A single Python statement, always ending with a newline.
443 The constructor adds a newline if needed.
444
445 - want: The expected output from running the source code (either
446 from stdout, or a traceback in case of exception). `want` ends
447 with a newline unless it's empty, in which case it's an empty
448 string. The constructor adds a newline if needed.
449
450 - exc_msg: The exception message generated by the example, if
451 the example is expected to generate an exception; or `None` if
452 it is not expected to generate an exception. This exception
453 message is compared against the return value of
454 `traceback.format_exception_only()`. `exc_msg` ends with a
455 newline unless it's `None`. The constructor adds a newline
456 if needed.
457
458 - lineno: The line number within the DocTest string containing
459 this Example where the Example begins. This line number is
460 zero-based, with respect to the beginning of the DocTest.
461
462 - indent: The example's indentation in the DocTest string.
463 I.e., the number of space characters that precede the
464 example's first prompt.
465
466 - options: A dictionary mapping from option flags to True or
467 False, which is used to override default options for this
468 example. Any option flags not contained in this dictionary
469 are left at their default value (as specified by the
470 DocTestRunner's optionflags). By default, no options are set.
471 """
472 def __init__(self, source, want, exc_msg=None, lineno=0, indent=0,
473 options=None):
474 # Normalize inputs.
475 if not source.endswith('\n'):
476 source += '\n'
477 if want and not want.endswith('\n'):
478 want += '\n'
479 if exc_msg is not None and not exc_msg.endswith('\n'):
480 exc_msg += '\n'
481 # Store properties.
482 self.source = source
483 self.want = want
484 self.lineno = lineno
485 self.indent = indent
486 if options is None: options = {}
487 self.options = options
488 self.exc_msg = exc_msg
489
490 def __eq__(self, other):
491 if type(self) is not type(other):
492 return NotImplemented
493
494 return self.source == other.source and \
495 self.want == other.want and \
496 self.lineno == other.lineno and \
497 self.indent == other.indent and \
498 self.options == other.options and \
499 self.exc_msg == other.exc_msg
500
501 def __hash__(self):
502 return hash((self.source, self.want, self.lineno, self.indent,
503 self.exc_msg))
504
505 class ESC[4;38;5;81mDocTest:
506 """
507 A collection of doctest examples that should be run in a single
508 namespace. Each `DocTest` defines the following attributes:
509
510 - examples: the list of examples.
511
512 - globs: The namespace (aka globals) that the examples should
513 be run in.
514
515 - name: A name identifying the DocTest (typically, the name of
516 the object whose docstring this DocTest was extracted from).
517
518 - filename: The name of the file that this DocTest was extracted
519 from, or `None` if the filename is unknown.
520
521 - lineno: The line number within filename where this DocTest
522 begins, or `None` if the line number is unavailable. This
523 line number is zero-based, with respect to the beginning of
524 the file.
525
526 - docstring: The string that the examples were extracted from,
527 or `None` if the string is unavailable.
528 """
529 def __init__(self, examples, globs, name, filename, lineno, docstring):
530 """
531 Create a new DocTest containing the given examples. The
532 DocTest's globals are initialized with a copy of `globs`.
533 """
534 assert not isinstance(examples, str), \
535 "DocTest no longer accepts str; use DocTestParser instead"
536 self.examples = examples
537 self.docstring = docstring
538 self.globs = globs.copy()
539 self.name = name
540 self.filename = filename
541 self.lineno = lineno
542
543 def __repr__(self):
544 if len(self.examples) == 0:
545 examples = 'no examples'
546 elif len(self.examples) == 1:
547 examples = '1 example'
548 else:
549 examples = '%d examples' % len(self.examples)
550 return ('<%s %s from %s:%s (%s)>' %
551 (self.__class__.__name__,
552 self.name, self.filename, self.lineno, examples))
553
554 def __eq__(self, other):
555 if type(self) is not type(other):
556 return NotImplemented
557
558 return self.examples == other.examples and \
559 self.docstring == other.docstring and \
560 self.globs == other.globs and \
561 self.name == other.name and \
562 self.filename == other.filename and \
563 self.lineno == other.lineno
564
565 def __hash__(self):
566 return hash((self.docstring, self.name, self.filename, self.lineno))
567
568 # This lets us sort tests by name:
569 def __lt__(self, other):
570 if not isinstance(other, DocTest):
571 return NotImplemented
572 self_lno = self.lineno if self.lineno is not None else -1
573 other_lno = other.lineno if other.lineno is not None else -1
574 return ((self.name, self.filename, self_lno, id(self))
575 <
576 (other.name, other.filename, other_lno, id(other)))
577
578 ######################################################################
579 ## 3. DocTestParser
580 ######################################################################
581
582 class ESC[4;38;5;81mDocTestParser:
583 """
584 A class used to parse strings containing doctest examples.
585 """
586 # This regular expression is used to find doctest examples in a
587 # string. It defines three groups: `source` is the source code
588 # (including leading indentation and prompts); `indent` is the
589 # indentation of the first (PS1) line of the source code; and
590 # `want` is the expected output (including leading indentation).
591 _EXAMPLE_RE = re.compile(r'''
592 # Source consists of a PS1 line followed by zero or more PS2 lines.
593 (?P<source>
594 (?:^(?P<indent> [ ]*) >>> .*) # PS1 line
595 (?:\n [ ]* \.\.\. .*)*) # PS2 lines
596 \n?
597 # Want consists of any non-blank lines that do not start with PS1.
598 (?P<want> (?:(?![ ]*$) # Not a blank line
599 (?![ ]*>>>) # Not a line starting with PS1
600 .+$\n? # But any other line
601 )*)
602 ''', re.MULTILINE | re.VERBOSE)
603
604 # A regular expression for handling `want` strings that contain
605 # expected exceptions. It divides `want` into three pieces:
606 # - the traceback header line (`hdr`)
607 # - the traceback stack (`stack`)
608 # - the exception message (`msg`), as generated by
609 # traceback.format_exception_only()
610 # `msg` may have multiple lines. We assume/require that the
611 # exception message is the first non-indented line starting with a word
612 # character following the traceback header line.
613 _EXCEPTION_RE = re.compile(r"""
614 # Grab the traceback header. Different versions of Python have
615 # said different things on the first traceback line.
616 ^(?P<hdr> Traceback\ \(
617 (?: most\ recent\ call\ last
618 | innermost\ last
619 ) \) :
620 )
621 \s* $ # toss trailing whitespace on the header.
622 (?P<stack> .*?) # don't blink: absorb stuff until...
623 ^ (?P<msg> \w+ .*) # a line *starts* with alphanum.
624 """, re.VERBOSE | re.MULTILINE | re.DOTALL)
625
626 # A callable returning a true value iff its argument is a blank line
627 # or contains a single comment.
628 _IS_BLANK_OR_COMMENT = re.compile(r'^[ ]*(#.*)?$').match
629
630 def parse(self, string, name='<string>'):
631 """
632 Divide the given string into examples and intervening text,
633 and return them as a list of alternating Examples and strings.
634 Line numbers for the Examples are 0-based. The optional
635 argument `name` is a name identifying this string, and is only
636 used for error messages.
637 """
638 string = string.expandtabs()
639 # If all lines begin with the same indentation, then strip it.
640 min_indent = self._min_indent(string)
641 if min_indent > 0:
642 string = '\n'.join([l[min_indent:] for l in string.split('\n')])
643
644 output = []
645 charno, lineno = 0, 0
646 # Find all doctest examples in the string:
647 for m in self._EXAMPLE_RE.finditer(string):
648 # Add the pre-example text to `output`.
649 output.append(string[charno:m.start()])
650 # Update lineno (lines before this example)
651 lineno += string.count('\n', charno, m.start())
652 # Extract info from the regexp match.
653 (source, options, want, exc_msg) = \
654 self._parse_example(m, name, lineno)
655 # Create an Example, and add it to the list.
656 if not self._IS_BLANK_OR_COMMENT(source):
657 output.append( Example(source, want, exc_msg,
658 lineno=lineno,
659 indent=min_indent+len(m.group('indent')),
660 options=options) )
661 # Update lineno (lines inside this example)
662 lineno += string.count('\n', m.start(), m.end())
663 # Update charno.
664 charno = m.end()
665 # Add any remaining post-example text to `output`.
666 output.append(string[charno:])
667 return output
668
669 def get_doctest(self, string, globs, name, filename, lineno):
670 """
671 Extract all doctest examples from the given string, and
672 collect them into a `DocTest` object.
673
674 `globs`, `name`, `filename`, and `lineno` are attributes for
675 the new `DocTest` object. See the documentation for `DocTest`
676 for more information.
677 """
678 return DocTest(self.get_examples(string, name), globs,
679 name, filename, lineno, string)
680
681 def get_examples(self, string, name='<string>'):
682 """
683 Extract all doctest examples from the given string, and return
684 them as a list of `Example` objects. Line numbers are
685 0-based, because it's most common in doctests that nothing
686 interesting appears on the same line as opening triple-quote,
687 and so the first interesting line is called \"line 1\" then.
688
689 The optional argument `name` is a name identifying this
690 string, and is only used for error messages.
691 """
692 return [x for x in self.parse(string, name)
693 if isinstance(x, Example)]
694
695 def _parse_example(self, m, name, lineno):
696 """
697 Given a regular expression match from `_EXAMPLE_RE` (`m`),
698 return a pair `(source, want)`, where `source` is the matched
699 example's source code (with prompts and indentation stripped);
700 and `want` is the example's expected output (with indentation
701 stripped).
702
703 `name` is the string's name, and `lineno` is the line number
704 where the example starts; both are used for error messages.
705 """
706 # Get the example's indentation level.
707 indent = len(m.group('indent'))
708
709 # Divide source into lines; check that they're properly
710 # indented; and then strip their indentation & prompts.
711 source_lines = m.group('source').split('\n')
712 self._check_prompt_blank(source_lines, indent, name, lineno)
713 self._check_prefix(source_lines[1:], ' '*indent + '.', name, lineno)
714 source = '\n'.join([sl[indent+4:] for sl in source_lines])
715
716 # Divide want into lines; check that it's properly indented; and
717 # then strip the indentation. Spaces before the last newline should
718 # be preserved, so plain rstrip() isn't good enough.
719 want = m.group('want')
720 want_lines = want.split('\n')
721 if len(want_lines) > 1 and re.match(r' *$', want_lines[-1]):
722 del want_lines[-1] # forget final newline & spaces after it
723 self._check_prefix(want_lines, ' '*indent, name,
724 lineno + len(source_lines))
725 want = '\n'.join([wl[indent:] for wl in want_lines])
726
727 # If `want` contains a traceback message, then extract it.
728 m = self._EXCEPTION_RE.match(want)
729 if m:
730 exc_msg = m.group('msg')
731 else:
732 exc_msg = None
733
734 # Extract options from the source.
735 options = self._find_options(source, name, lineno)
736
737 return source, options, want, exc_msg
738
739 # This regular expression looks for option directives in the
740 # source code of an example. Option directives are comments
741 # starting with "doctest:". Warning: this may give false
742 # positives for string-literals that contain the string
743 # "#doctest:". Eliminating these false positives would require
744 # actually parsing the string; but we limit them by ignoring any
745 # line containing "#doctest:" that is *followed* by a quote mark.
746 _OPTION_DIRECTIVE_RE = re.compile(r'#\s*doctest:\s*([^\n\'"]*)$',
747 re.MULTILINE)
748
749 def _find_options(self, source, name, lineno):
750 """
751 Return a dictionary containing option overrides extracted from
752 option directives in the given source string.
753
754 `name` is the string's name, and `lineno` is the line number
755 where the example starts; both are used for error messages.
756 """
757 options = {}
758 # (note: with the current regexp, this will match at most once:)
759 for m in self._OPTION_DIRECTIVE_RE.finditer(source):
760 option_strings = m.group(1).replace(',', ' ').split()
761 for option in option_strings:
762 if (option[0] not in '+-' or
763 option[1:] not in OPTIONFLAGS_BY_NAME):
764 raise ValueError('line %r of the doctest for %s '
765 'has an invalid option: %r' %
766 (lineno+1, name, option))
767 flag = OPTIONFLAGS_BY_NAME[option[1:]]
768 options[flag] = (option[0] == '+')
769 if options and self._IS_BLANK_OR_COMMENT(source):
770 raise ValueError('line %r of the doctest for %s has an option '
771 'directive on a line with no example: %r' %
772 (lineno, name, source))
773 return options
774
775 # This regular expression finds the indentation of every non-blank
776 # line in a string.
777 _INDENT_RE = re.compile(r'^([ ]*)(?=\S)', re.MULTILINE)
778
779 def _min_indent(self, s):
780 "Return the minimum indentation of any non-blank line in `s`"
781 indents = [len(indent) for indent in self._INDENT_RE.findall(s)]
782 if len(indents) > 0:
783 return min(indents)
784 else:
785 return 0
786
787 def _check_prompt_blank(self, lines, indent, name, lineno):
788 """
789 Given the lines of a source string (including prompts and
790 leading indentation), check to make sure that every prompt is
791 followed by a space character. If any line is not followed by
792 a space character, then raise ValueError.
793 """
794 for i, line in enumerate(lines):
795 if len(line) >= indent+4 and line[indent+3] != ' ':
796 raise ValueError('line %r of the docstring for %s '
797 'lacks blank after %s: %r' %
798 (lineno+i+1, name,
799 line[indent:indent+3], line))
800
801 def _check_prefix(self, lines, prefix, name, lineno):
802 """
803 Check that every line in the given list starts with the given
804 prefix; if any line does not, then raise a ValueError.
805 """
806 for i, line in enumerate(lines):
807 if line and not line.startswith(prefix):
808 raise ValueError('line %r of the docstring for %s has '
809 'inconsistent leading whitespace: %r' %
810 (lineno+i+1, name, line))
811
812
813 ######################################################################
814 ## 4. DocTest Finder
815 ######################################################################
816
817 class ESC[4;38;5;81mDocTestFinder:
818 """
819 A class used to extract the DocTests that are relevant to a given
820 object, from its docstring and the docstrings of its contained
821 objects. Doctests can currently be extracted from the following
822 object types: modules, functions, classes, methods, staticmethods,
823 classmethods, and properties.
824 """
825
826 def __init__(self, verbose=False, parser=DocTestParser(),
827 recurse=True, exclude_empty=True):
828 """
829 Create a new doctest finder.
830
831 The optional argument `parser` specifies a class or
832 function that should be used to create new DocTest objects (or
833 objects that implement the same interface as DocTest). The
834 signature for this factory function should match the signature
835 of the DocTest constructor.
836
837 If the optional argument `recurse` is false, then `find` will
838 only examine the given object, and not any contained objects.
839
840 If the optional argument `exclude_empty` is false, then `find`
841 will include tests for objects with empty docstrings.
842 """
843 self._parser = parser
844 self._verbose = verbose
845 self._recurse = recurse
846 self._exclude_empty = exclude_empty
847
848 def find(self, obj, name=None, module=None, globs=None, extraglobs=None):
849 """
850 Return a list of the DocTests that are defined by the given
851 object's docstring, or by any of its contained objects'
852 docstrings.
853
854 The optional parameter `module` is the module that contains
855 the given object. If the module is not specified or is None, then
856 the test finder will attempt to automatically determine the
857 correct module. The object's module is used:
858
859 - As a default namespace, if `globs` is not specified.
860 - To prevent the DocTestFinder from extracting DocTests
861 from objects that are imported from other modules.
862 - To find the name of the file containing the object.
863 - To help find the line number of the object within its
864 file.
865
866 Contained objects whose module does not match `module` are ignored.
867
868 If `module` is False, no attempt to find the module will be made.
869 This is obscure, of use mostly in tests: if `module` is False, or
870 is None but cannot be found automatically, then all objects are
871 considered to belong to the (non-existent) module, so all contained
872 objects will (recursively) be searched for doctests.
873
874 The globals for each DocTest is formed by combining `globs`
875 and `extraglobs` (bindings in `extraglobs` override bindings
876 in `globs`). A new copy of the globals dictionary is created
877 for each DocTest. If `globs` is not specified, then it
878 defaults to the module's `__dict__`, if specified, or {}
879 otherwise. If `extraglobs` is not specified, then it defaults
880 to {}.
881
882 """
883 # If name was not specified, then extract it from the object.
884 if name is None:
885 name = getattr(obj, '__name__', None)
886 if name is None:
887 raise ValueError("DocTestFinder.find: name must be given "
888 "when obj.__name__ doesn't exist: %r" %
889 (type(obj),))
890
891 # Find the module that contains the given object (if obj is
892 # a module, then module=obj.). Note: this may fail, in which
893 # case module will be None.
894 if module is False:
895 module = None
896 elif module is None:
897 module = inspect.getmodule(obj)
898
899 # Read the module's source code. This is used by
900 # DocTestFinder._find_lineno to find the line number for a
901 # given object's docstring.
902 try:
903 file = inspect.getsourcefile(obj)
904 except TypeError:
905 source_lines = None
906 else:
907 if not file:
908 # Check to see if it's one of our special internal "files"
909 # (see __patched_linecache_getlines).
910 file = inspect.getfile(obj)
911 if not file[0]+file[-2:] == '<]>': file = None
912 if file is None:
913 source_lines = None
914 else:
915 if module is not None:
916 # Supply the module globals in case the module was
917 # originally loaded via a PEP 302 loader and
918 # file is not a valid filesystem path
919 source_lines = linecache.getlines(file, module.__dict__)
920 else:
921 # No access to a loader, so assume it's a normal
922 # filesystem path
923 source_lines = linecache.getlines(file)
924 if not source_lines:
925 source_lines = None
926
927 # Initialize globals, and merge in extraglobs.
928 if globs is None:
929 if module is None:
930 globs = {}
931 else:
932 globs = module.__dict__.copy()
933 else:
934 globs = globs.copy()
935 if extraglobs is not None:
936 globs.update(extraglobs)
937 if '__name__' not in globs:
938 globs['__name__'] = '__main__' # provide a default module name
939
940 # Recursively explore `obj`, extracting DocTests.
941 tests = []
942 self._find(tests, obj, name, module, source_lines, globs, {})
943 # Sort the tests by alpha order of names, for consistency in
944 # verbose-mode output. This was a feature of doctest in Pythons
945 # <= 2.3 that got lost by accident in 2.4. It was repaired in
946 # 2.4.4 and 2.5.
947 tests.sort()
948 return tests
949
950 def _from_module(self, module, object):
951 """
952 Return true if the given object is defined in the given
953 module.
954 """
955 if module is None:
956 return True
957 elif inspect.getmodule(object) is not None:
958 return module is inspect.getmodule(object)
959 elif inspect.isfunction(object):
960 return module.__dict__ is object.__globals__
961 elif (inspect.ismethoddescriptor(object) or
962 inspect.ismethodwrapper(object)):
963 if hasattr(object, '__objclass__'):
964 obj_mod = object.__objclass__.__module__
965 elif hasattr(object, '__module__'):
966 obj_mod = object.__module__
967 else:
968 return True # [XX] no easy way to tell otherwise
969 return module.__name__ == obj_mod
970 elif inspect.isclass(object):
971 return module.__name__ == object.__module__
972 elif hasattr(object, '__module__'):
973 return module.__name__ == object.__module__
974 elif isinstance(object, property):
975 return True # [XX] no way not be sure.
976 else:
977 raise ValueError("object must be a class or function")
978
979 def _is_routine(self, obj):
980 """
981 Safely unwrap objects and determine if they are functions.
982 """
983 maybe_routine = obj
984 try:
985 maybe_routine = inspect.unwrap(maybe_routine)
986 except ValueError:
987 pass
988 return inspect.isroutine(maybe_routine)
989
990 def _find(self, tests, obj, name, module, source_lines, globs, seen):
991 """
992 Find tests for the given object and any contained objects, and
993 add them to `tests`.
994 """
995 if self._verbose:
996 print('Finding tests in %s' % name)
997
998 # If we've already processed this object, then ignore it.
999 if id(obj) in seen:
1000 return
1001 seen[id(obj)] = 1
1002
1003 # Find a test for this object, and add it to the list of tests.
1004 test = self._get_test(obj, name, module, globs, source_lines)
1005 if test is not None:
1006 tests.append(test)
1007
1008 # Look for tests in a module's contained objects.
1009 if inspect.ismodule(obj) and self._recurse:
1010 for valname, val in obj.__dict__.items():
1011 valname = '%s.%s' % (name, valname)
1012
1013 # Recurse to functions & classes.
1014 if ((self._is_routine(val) or inspect.isclass(val)) and
1015 self._from_module(module, val)):
1016 self._find(tests, val, valname, module, source_lines,
1017 globs, seen)
1018
1019 # Look for tests in a module's __test__ dictionary.
1020 if inspect.ismodule(obj) and self._recurse:
1021 for valname, val in getattr(obj, '__test__', {}).items():
1022 if not isinstance(valname, str):
1023 raise ValueError("DocTestFinder.find: __test__ keys "
1024 "must be strings: %r" %
1025 (type(valname),))
1026 if not (inspect.isroutine(val) or inspect.isclass(val) or
1027 inspect.ismodule(val) or isinstance(val, str)):
1028 raise ValueError("DocTestFinder.find: __test__ values "
1029 "must be strings, functions, methods, "
1030 "classes, or modules: %r" %
1031 (type(val),))
1032 valname = '%s.__test__.%s' % (name, valname)
1033 self._find(tests, val, valname, module, source_lines,
1034 globs, seen)
1035
1036 # Look for tests in a class's contained objects.
1037 if inspect.isclass(obj) and self._recurse:
1038 for valname, val in obj.__dict__.items():
1039 # Special handling for staticmethod/classmethod.
1040 if isinstance(val, (staticmethod, classmethod)):
1041 val = val.__func__
1042
1043 # Recurse to methods, properties, and nested classes.
1044 if ((inspect.isroutine(val) or inspect.isclass(val) or
1045 isinstance(val, property)) and
1046 self._from_module(module, val)):
1047 valname = '%s.%s' % (name, valname)
1048 self._find(tests, val, valname, module, source_lines,
1049 globs, seen)
1050
1051 def _get_test(self, obj, name, module, globs, source_lines):
1052 """
1053 Return a DocTest for the given object, if it defines a docstring;
1054 otherwise, return None.
1055 """
1056 # Extract the object's docstring. If it doesn't have one,
1057 # then return None (no test for this object).
1058 if isinstance(obj, str):
1059 docstring = obj
1060 else:
1061 try:
1062 if obj.__doc__ is None:
1063 docstring = ''
1064 else:
1065 docstring = obj.__doc__
1066 if not isinstance(docstring, str):
1067 docstring = str(docstring)
1068 except (TypeError, AttributeError):
1069 docstring = ''
1070
1071 # Find the docstring's location in the file.
1072 lineno = self._find_lineno(obj, source_lines)
1073
1074 # Don't bother if the docstring is empty.
1075 if self._exclude_empty and not docstring:
1076 return None
1077
1078 # Return a DocTest for this object.
1079 if module is None:
1080 filename = None
1081 else:
1082 # __file__ can be None for namespace packages.
1083 filename = getattr(module, '__file__', None) or module.__name__
1084 if filename[-4:] == ".pyc":
1085 filename = filename[:-1]
1086 return self._parser.get_doctest(docstring, globs, name,
1087 filename, lineno)
1088
1089 def _find_lineno(self, obj, source_lines):
1090 """
1091 Return a line number of the given object's docstring.
1092
1093 Returns `None` if the given object does not have a docstring.
1094 """
1095 lineno = None
1096 docstring = getattr(obj, '__doc__', None)
1097
1098 # Find the line number for modules.
1099 if inspect.ismodule(obj) and docstring is not None:
1100 lineno = 0
1101
1102 # Find the line number for classes.
1103 # Note: this could be fooled if a class is defined multiple
1104 # times in a single file.
1105 if inspect.isclass(obj) and docstring is not None:
1106 if source_lines is None:
1107 return None
1108 pat = re.compile(r'^\s*class\s*%s\b' %
1109 re.escape(getattr(obj, '__name__', '-')))
1110 for i, line in enumerate(source_lines):
1111 if pat.match(line):
1112 lineno = i
1113 break
1114
1115 # Find the line number for functions & methods.
1116 if inspect.ismethod(obj): obj = obj.__func__
1117 if inspect.isfunction(obj) and getattr(obj, '__doc__', None):
1118 # We don't use `docstring` var here, because `obj` can be changed.
1119 obj = obj.__code__
1120 if inspect.istraceback(obj): obj = obj.tb_frame
1121 if inspect.isframe(obj): obj = obj.f_code
1122 if inspect.iscode(obj):
1123 lineno = obj.co_firstlineno - 1
1124
1125 # Find the line number where the docstring starts. Assume
1126 # that it's the first line that begins with a quote mark.
1127 # Note: this could be fooled by a multiline function
1128 # signature, where a continuation line begins with a quote
1129 # mark.
1130 if lineno is not None:
1131 if source_lines is None:
1132 return lineno+1
1133 pat = re.compile(r'(^|.*:)\s*\w*("|\')')
1134 for lineno in range(lineno, len(source_lines)):
1135 if pat.match(source_lines[lineno]):
1136 return lineno
1137
1138 # We couldn't find the line number.
1139 return None
1140
1141 ######################################################################
1142 ## 5. DocTest Runner
1143 ######################################################################
1144
1145 class ESC[4;38;5;81mDocTestRunner:
1146 """
1147 A class used to run DocTest test cases, and accumulate statistics.
1148 The `run` method is used to process a single DocTest case. It
1149 returns a tuple `(f, t)`, where `t` is the number of test cases
1150 tried, and `f` is the number of test cases that failed.
1151
1152 >>> tests = DocTestFinder().find(_TestClass)
1153 >>> runner = DocTestRunner(verbose=False)
1154 >>> tests.sort(key = lambda test: test.name)
1155 >>> for test in tests:
1156 ... print(test.name, '->', runner.run(test))
1157 _TestClass -> TestResults(failed=0, attempted=2)
1158 _TestClass.__init__ -> TestResults(failed=0, attempted=2)
1159 _TestClass.get -> TestResults(failed=0, attempted=2)
1160 _TestClass.square -> TestResults(failed=0, attempted=1)
1161
1162 The `summarize` method prints a summary of all the test cases that
1163 have been run by the runner, and returns an aggregated `(f, t)`
1164 tuple:
1165
1166 >>> runner.summarize(verbose=1)
1167 4 items passed all tests:
1168 2 tests in _TestClass
1169 2 tests in _TestClass.__init__
1170 2 tests in _TestClass.get
1171 1 tests in _TestClass.square
1172 7 tests in 4 items.
1173 7 passed and 0 failed.
1174 Test passed.
1175 TestResults(failed=0, attempted=7)
1176
1177 The aggregated number of tried examples and failed examples is
1178 also available via the `tries` and `failures` attributes:
1179
1180 >>> runner.tries
1181 7
1182 >>> runner.failures
1183 0
1184
1185 The comparison between expected outputs and actual outputs is done
1186 by an `OutputChecker`. This comparison may be customized with a
1187 number of option flags; see the documentation for `testmod` for
1188 more information. If the option flags are insufficient, then the
1189 comparison may also be customized by passing a subclass of
1190 `OutputChecker` to the constructor.
1191
1192 The test runner's display output can be controlled in two ways.
1193 First, an output function (`out) can be passed to
1194 `TestRunner.run`; this function will be called with strings that
1195 should be displayed. It defaults to `sys.stdout.write`. If
1196 capturing the output is not sufficient, then the display output
1197 can be also customized by subclassing DocTestRunner, and
1198 overriding the methods `report_start`, `report_success`,
1199 `report_unexpected_exception`, and `report_failure`.
1200 """
1201 # This divider string is used to separate failure messages, and to
1202 # separate sections of the summary.
1203 DIVIDER = "*" * 70
1204
1205 def __init__(self, checker=None, verbose=None, optionflags=0):
1206 """
1207 Create a new test runner.
1208
1209 Optional keyword arg `checker` is the `OutputChecker` that
1210 should be used to compare the expected outputs and actual
1211 outputs of doctest examples.
1212
1213 Optional keyword arg 'verbose' prints lots of stuff if true,
1214 only failures if false; by default, it's true iff '-v' is in
1215 sys.argv.
1216
1217 Optional argument `optionflags` can be used to control how the
1218 test runner compares expected output to actual output, and how
1219 it displays failures. See the documentation for `testmod` for
1220 more information.
1221 """
1222 self._checker = checker or OutputChecker()
1223 if verbose is None:
1224 verbose = '-v' in sys.argv
1225 self._verbose = verbose
1226 self.optionflags = optionflags
1227 self.original_optionflags = optionflags
1228
1229 # Keep track of the examples we've run.
1230 self.tries = 0
1231 self.failures = 0
1232 self._name2ft = {}
1233
1234 # Create a fake output target for capturing doctest output.
1235 self._fakeout = _SpoofOut()
1236
1237 #/////////////////////////////////////////////////////////////////
1238 # Reporting methods
1239 #/////////////////////////////////////////////////////////////////
1240
1241 def report_start(self, out, test, example):
1242 """
1243 Report that the test runner is about to process the given
1244 example. (Only displays a message if verbose=True)
1245 """
1246 if self._verbose:
1247 if example.want:
1248 out('Trying:\n' + _indent(example.source) +
1249 'Expecting:\n' + _indent(example.want))
1250 else:
1251 out('Trying:\n' + _indent(example.source) +
1252 'Expecting nothing\n')
1253
1254 def report_success(self, out, test, example, got):
1255 """
1256 Report that the given example ran successfully. (Only
1257 displays a message if verbose=True)
1258 """
1259 if self._verbose:
1260 out("ok\n")
1261
1262 def report_failure(self, out, test, example, got):
1263 """
1264 Report that the given example failed.
1265 """
1266 out(self._failure_header(test, example) +
1267 self._checker.output_difference(example, got, self.optionflags))
1268
1269 def report_unexpected_exception(self, out, test, example, exc_info):
1270 """
1271 Report that the given example raised an unexpected exception.
1272 """
1273 out(self._failure_header(test, example) +
1274 'Exception raised:\n' + _indent(_exception_traceback(exc_info)))
1275
1276 def _failure_header(self, test, example):
1277 out = [self.DIVIDER]
1278 if test.filename:
1279 if test.lineno is not None and example.lineno is not None:
1280 lineno = test.lineno + example.lineno + 1
1281 else:
1282 lineno = '?'
1283 out.append('File "%s", line %s, in %s' %
1284 (test.filename, lineno, test.name))
1285 else:
1286 out.append('Line %s, in %s' % (example.lineno+1, test.name))
1287 out.append('Failed example:')
1288 source = example.source
1289 out.append(_indent(source))
1290 return '\n'.join(out)
1291
1292 #/////////////////////////////////////////////////////////////////
1293 # DocTest Running
1294 #/////////////////////////////////////////////////////////////////
1295
1296 def __run(self, test, compileflags, out):
1297 """
1298 Run the examples in `test`. Write the outcome of each example
1299 with one of the `DocTestRunner.report_*` methods, using the
1300 writer function `out`. `compileflags` is the set of compiler
1301 flags that should be used to execute examples. Return a tuple
1302 `(f, t)`, where `t` is the number of examples tried, and `f`
1303 is the number of examples that failed. The examples are run
1304 in the namespace `test.globs`.
1305 """
1306 # Keep track of the number of failures and tries.
1307 failures = tries = 0
1308
1309 # Save the option flags (since option directives can be used
1310 # to modify them).
1311 original_optionflags = self.optionflags
1312
1313 SUCCESS, FAILURE, BOOM = range(3) # `outcome` state
1314
1315 check = self._checker.check_output
1316
1317 # Process each example.
1318 for examplenum, example in enumerate(test.examples):
1319
1320 # If REPORT_ONLY_FIRST_FAILURE is set, then suppress
1321 # reporting after the first failure.
1322 quiet = (self.optionflags & REPORT_ONLY_FIRST_FAILURE and
1323 failures > 0)
1324
1325 # Merge in the example's options.
1326 self.optionflags = original_optionflags
1327 if example.options:
1328 for (optionflag, val) in example.options.items():
1329 if val:
1330 self.optionflags |= optionflag
1331 else:
1332 self.optionflags &= ~optionflag
1333
1334 # If 'SKIP' is set, then skip this example.
1335 if self.optionflags & SKIP:
1336 continue
1337
1338 # Record that we started this example.
1339 tries += 1
1340 if not quiet:
1341 self.report_start(out, test, example)
1342
1343 # Use a special filename for compile(), so we can retrieve
1344 # the source code during interactive debugging (see
1345 # __patched_linecache_getlines).
1346 filename = '<doctest %s[%d]>' % (test.name, examplenum)
1347
1348 # Run the example in the given context (globs), and record
1349 # any exception that gets raised. (But don't intercept
1350 # keyboard interrupts.)
1351 try:
1352 # Don't blink! This is where the user's code gets run.
1353 exec(compile(example.source, filename, "single",
1354 compileflags, True), test.globs)
1355 self.debugger.set_continue() # ==== Example Finished ====
1356 exception = None
1357 except KeyboardInterrupt:
1358 raise
1359 except:
1360 exception = sys.exc_info()
1361 self.debugger.set_continue() # ==== Example Finished ====
1362
1363 got = self._fakeout.getvalue() # the actual output
1364 self._fakeout.truncate(0)
1365 outcome = FAILURE # guilty until proved innocent or insane
1366
1367 # If the example executed without raising any exceptions,
1368 # verify its output.
1369 if exception is None:
1370 if check(example.want, got, self.optionflags):
1371 outcome = SUCCESS
1372
1373 # The example raised an exception: check if it was expected.
1374 else:
1375 formatted_ex = traceback.format_exception_only(*exception[:2])
1376 if issubclass(exception[0], SyntaxError):
1377 # SyntaxError / IndentationError is special:
1378 # we don't care about the carets / suggestions / etc
1379 # We only care about the error message and notes.
1380 # They start with `SyntaxError:` (or any other class name)
1381 exception_line_prefixes = (
1382 f"{exception[0].__qualname__}:",
1383 f"{exception[0].__module__}.{exception[0].__qualname__}:",
1384 )
1385 exc_msg_index = next(
1386 index
1387 for index, line in enumerate(formatted_ex)
1388 if line.startswith(exception_line_prefixes)
1389 )
1390 formatted_ex = formatted_ex[exc_msg_index:]
1391
1392 exc_msg = "".join(formatted_ex)
1393 if not quiet:
1394 got += _exception_traceback(exception)
1395
1396 # If `example.exc_msg` is None, then we weren't expecting
1397 # an exception.
1398 if example.exc_msg is None:
1399 outcome = BOOM
1400
1401 # We expected an exception: see whether it matches.
1402 elif check(example.exc_msg, exc_msg, self.optionflags):
1403 outcome = SUCCESS
1404
1405 # Another chance if they didn't care about the detail.
1406 elif self.optionflags & IGNORE_EXCEPTION_DETAIL:
1407 if check(_strip_exception_details(example.exc_msg),
1408 _strip_exception_details(exc_msg),
1409 self.optionflags):
1410 outcome = SUCCESS
1411
1412 # Report the outcome.
1413 if outcome is SUCCESS:
1414 if not quiet:
1415 self.report_success(out, test, example, got)
1416 elif outcome is FAILURE:
1417 if not quiet:
1418 self.report_failure(out, test, example, got)
1419 failures += 1
1420 elif outcome is BOOM:
1421 if not quiet:
1422 self.report_unexpected_exception(out, test, example,
1423 exception)
1424 failures += 1
1425 else:
1426 assert False, ("unknown outcome", outcome)
1427
1428 if failures and self.optionflags & FAIL_FAST:
1429 break
1430
1431 # Restore the option flags (in case they were modified)
1432 self.optionflags = original_optionflags
1433
1434 # Record and return the number of failures and tries.
1435 self.__record_outcome(test, failures, tries)
1436 return TestResults(failures, tries)
1437
1438 def __record_outcome(self, test, f, t):
1439 """
1440 Record the fact that the given DocTest (`test`) generated `f`
1441 failures out of `t` tried examples.
1442 """
1443 f2, t2 = self._name2ft.get(test.name, (0,0))
1444 self._name2ft[test.name] = (f+f2, t+t2)
1445 self.failures += f
1446 self.tries += t
1447
1448 __LINECACHE_FILENAME_RE = re.compile(r'<doctest '
1449 r'(?P<name>.+)'
1450 r'\[(?P<examplenum>\d+)\]>$')
1451 def __patched_linecache_getlines(self, filename, module_globals=None):
1452 m = self.__LINECACHE_FILENAME_RE.match(filename)
1453 if m and m.group('name') == self.test.name:
1454 example = self.test.examples[int(m.group('examplenum'))]
1455 return example.source.splitlines(keepends=True)
1456 else:
1457 return self.save_linecache_getlines(filename, module_globals)
1458
1459 def run(self, test, compileflags=None, out=None, clear_globs=True):
1460 """
1461 Run the examples in `test`, and display the results using the
1462 writer function `out`.
1463
1464 The examples are run in the namespace `test.globs`. If
1465 `clear_globs` is true (the default), then this namespace will
1466 be cleared after the test runs, to help with garbage
1467 collection. If you would like to examine the namespace after
1468 the test completes, then use `clear_globs=False`.
1469
1470 `compileflags` gives the set of flags that should be used by
1471 the Python compiler when running the examples. If not
1472 specified, then it will default to the set of future-import
1473 flags that apply to `globs`.
1474
1475 The output of each example is checked using
1476 `DocTestRunner.check_output`, and the results are formatted by
1477 the `DocTestRunner.report_*` methods.
1478 """
1479 self.test = test
1480
1481 if compileflags is None:
1482 compileflags = _extract_future_flags(test.globs)
1483
1484 save_stdout = sys.stdout
1485 if out is None:
1486 encoding = save_stdout.encoding
1487 if encoding is None or encoding.lower() == 'utf-8':
1488 out = save_stdout.write
1489 else:
1490 # Use backslashreplace error handling on write
1491 def out(s):
1492 s = str(s.encode(encoding, 'backslashreplace'), encoding)
1493 save_stdout.write(s)
1494 sys.stdout = self._fakeout
1495
1496 # Patch pdb.set_trace to restore sys.stdout during interactive
1497 # debugging (so it's not still redirected to self._fakeout).
1498 # Note that the interactive output will go to *our*
1499 # save_stdout, even if that's not the real sys.stdout; this
1500 # allows us to write test cases for the set_trace behavior.
1501 save_trace = sys.gettrace()
1502 save_set_trace = pdb.set_trace
1503 self.debugger = _OutputRedirectingPdb(save_stdout)
1504 self.debugger.reset()
1505 pdb.set_trace = self.debugger.set_trace
1506
1507 # Patch linecache.getlines, so we can see the example's source
1508 # when we're inside the debugger.
1509 self.save_linecache_getlines = linecache.getlines
1510 linecache.getlines = self.__patched_linecache_getlines
1511
1512 # Make sure sys.displayhook just prints the value to stdout
1513 save_displayhook = sys.displayhook
1514 sys.displayhook = sys.__displayhook__
1515
1516 try:
1517 return self.__run(test, compileflags, out)
1518 finally:
1519 sys.stdout = save_stdout
1520 pdb.set_trace = save_set_trace
1521 sys.settrace(save_trace)
1522 linecache.getlines = self.save_linecache_getlines
1523 sys.displayhook = save_displayhook
1524 if clear_globs:
1525 test.globs.clear()
1526 import builtins
1527 builtins._ = None
1528
1529 #/////////////////////////////////////////////////////////////////
1530 # Summarization
1531 #/////////////////////////////////////////////////////////////////
1532 def summarize(self, verbose=None):
1533 """
1534 Print a summary of all the test cases that have been run by
1535 this DocTestRunner, and return a tuple `(f, t)`, where `f` is
1536 the total number of failed examples, and `t` is the total
1537 number of tried examples.
1538
1539 The optional `verbose` argument controls how detailed the
1540 summary is. If the verbosity is not specified, then the
1541 DocTestRunner's verbosity is used.
1542 """
1543 if verbose is None:
1544 verbose = self._verbose
1545 notests = []
1546 passed = []
1547 failed = []
1548 totalt = totalf = 0
1549 for x in self._name2ft.items():
1550 name, (f, t) = x
1551 assert f <= t
1552 totalt += t
1553 totalf += f
1554 if t == 0:
1555 notests.append(name)
1556 elif f == 0:
1557 passed.append( (name, t) )
1558 else:
1559 failed.append(x)
1560 if verbose:
1561 if notests:
1562 print(len(notests), "items had no tests:")
1563 notests.sort()
1564 for thing in notests:
1565 print(" ", thing)
1566 if passed:
1567 print(len(passed), "items passed all tests:")
1568 passed.sort()
1569 for thing, count in passed:
1570 print(" %3d tests in %s" % (count, thing))
1571 if failed:
1572 print(self.DIVIDER)
1573 print(len(failed), "items had failures:")
1574 failed.sort()
1575 for thing, (f, t) in failed:
1576 print(" %3d of %3d in %s" % (f, t, thing))
1577 if verbose:
1578 print(totalt, "tests in", len(self._name2ft), "items.")
1579 print(totalt - totalf, "passed and", totalf, "failed.")
1580 if totalf:
1581 print("***Test Failed***", totalf, "failures.")
1582 elif verbose:
1583 print("Test passed.")
1584 return TestResults(totalf, totalt)
1585
1586 #/////////////////////////////////////////////////////////////////
1587 # Backward compatibility cruft to maintain doctest.master.
1588 #/////////////////////////////////////////////////////////////////
1589 def merge(self, other):
1590 d = self._name2ft
1591 for name, (f, t) in other._name2ft.items():
1592 if name in d:
1593 # Don't print here by default, since doing
1594 # so breaks some of the buildbots
1595 #print("*** DocTestRunner.merge: '" + name + "' in both" \
1596 # " testers; summing outcomes.")
1597 f2, t2 = d[name]
1598 f = f + f2
1599 t = t + t2
1600 d[name] = f, t
1601
1602 class ESC[4;38;5;81mOutputChecker:
1603 """
1604 A class used to check the whether the actual output from a doctest
1605 example matches the expected output. `OutputChecker` defines two
1606 methods: `check_output`, which compares a given pair of outputs,
1607 and returns true if they match; and `output_difference`, which
1608 returns a string describing the differences between two outputs.
1609 """
1610 def _toAscii(self, s):
1611 """
1612 Convert string to hex-escaped ASCII string.
1613 """
1614 return str(s.encode('ASCII', 'backslashreplace'), "ASCII")
1615
1616 def check_output(self, want, got, optionflags):
1617 """
1618 Return True iff the actual output from an example (`got`)
1619 matches the expected output (`want`). These strings are
1620 always considered to match if they are identical; but
1621 depending on what option flags the test runner is using,
1622 several non-exact match types are also possible. See the
1623 documentation for `TestRunner` for more information about
1624 option flags.
1625 """
1626
1627 # If `want` contains hex-escaped character such as "\u1234",
1628 # then `want` is a string of six characters(e.g. [\,u,1,2,3,4]).
1629 # On the other hand, `got` could be another sequence of
1630 # characters such as [\u1234], so `want` and `got` should
1631 # be folded to hex-escaped ASCII string to compare.
1632 got = self._toAscii(got)
1633 want = self._toAscii(want)
1634
1635 # Handle the common case first, for efficiency:
1636 # if they're string-identical, always return true.
1637 if got == want:
1638 return True
1639
1640 # The values True and False replaced 1 and 0 as the return
1641 # value for boolean comparisons in Python 2.3.
1642 if not (optionflags & DONT_ACCEPT_TRUE_FOR_1):
1643 if (got,want) == ("True\n", "1\n"):
1644 return True
1645 if (got,want) == ("False\n", "0\n"):
1646 return True
1647
1648 # <BLANKLINE> can be used as a special sequence to signify a
1649 # blank line, unless the DONT_ACCEPT_BLANKLINE flag is used.
1650 if not (optionflags & DONT_ACCEPT_BLANKLINE):
1651 # Replace <BLANKLINE> in want with a blank line.
1652 want = re.sub(r'(?m)^%s\s*?$' % re.escape(BLANKLINE_MARKER),
1653 '', want)
1654 # If a line in got contains only spaces, then remove the
1655 # spaces.
1656 got = re.sub(r'(?m)^[^\S\n]+$', '', got)
1657 if got == want:
1658 return True
1659
1660 # This flag causes doctest to ignore any differences in the
1661 # contents of whitespace strings. Note that this can be used
1662 # in conjunction with the ELLIPSIS flag.
1663 if optionflags & NORMALIZE_WHITESPACE:
1664 got = ' '.join(got.split())
1665 want = ' '.join(want.split())
1666 if got == want:
1667 return True
1668
1669 # The ELLIPSIS flag says to let the sequence "..." in `want`
1670 # match any substring in `got`.
1671 if optionflags & ELLIPSIS:
1672 if _ellipsis_match(want, got):
1673 return True
1674
1675 # We didn't find any match; return false.
1676 return False
1677
1678 # Should we do a fancy diff?
1679 def _do_a_fancy_diff(self, want, got, optionflags):
1680 # Not unless they asked for a fancy diff.
1681 if not optionflags & (REPORT_UDIFF |
1682 REPORT_CDIFF |
1683 REPORT_NDIFF):
1684 return False
1685
1686 # If expected output uses ellipsis, a meaningful fancy diff is
1687 # too hard ... or maybe not. In two real-life failures Tim saw,
1688 # a diff was a major help anyway, so this is commented out.
1689 # [todo] _ellipsis_match() knows which pieces do and don't match,
1690 # and could be the basis for a kick-ass diff in this case.
1691 ##if optionflags & ELLIPSIS and ELLIPSIS_MARKER in want:
1692 ## return False
1693
1694 # ndiff does intraline difference marking, so can be useful even
1695 # for 1-line differences.
1696 if optionflags & REPORT_NDIFF:
1697 return True
1698
1699 # The other diff types need at least a few lines to be helpful.
1700 return want.count('\n') > 2 and got.count('\n') > 2
1701
1702 def output_difference(self, example, got, optionflags):
1703 """
1704 Return a string describing the differences between the
1705 expected output for a given example (`example`) and the actual
1706 output (`got`). `optionflags` is the set of option flags used
1707 to compare `want` and `got`.
1708 """
1709 want = example.want
1710 # If <BLANKLINE>s are being used, then replace blank lines
1711 # with <BLANKLINE> in the actual output string.
1712 if not (optionflags & DONT_ACCEPT_BLANKLINE):
1713 got = re.sub('(?m)^[ ]*(?=\n)', BLANKLINE_MARKER, got)
1714
1715 # Check if we should use diff.
1716 if self._do_a_fancy_diff(want, got, optionflags):
1717 # Split want & got into lines.
1718 want_lines = want.splitlines(keepends=True)
1719 got_lines = got.splitlines(keepends=True)
1720 # Use difflib to find their differences.
1721 if optionflags & REPORT_UDIFF:
1722 diff = difflib.unified_diff(want_lines, got_lines, n=2)
1723 diff = list(diff)[2:] # strip the diff header
1724 kind = 'unified diff with -expected +actual'
1725 elif optionflags & REPORT_CDIFF:
1726 diff = difflib.context_diff(want_lines, got_lines, n=2)
1727 diff = list(diff)[2:] # strip the diff header
1728 kind = 'context diff with expected followed by actual'
1729 elif optionflags & REPORT_NDIFF:
1730 engine = difflib.Differ(charjunk=difflib.IS_CHARACTER_JUNK)
1731 diff = list(engine.compare(want_lines, got_lines))
1732 kind = 'ndiff with -expected +actual'
1733 else:
1734 assert 0, 'Bad diff option'
1735 return 'Differences (%s):\n' % kind + _indent(''.join(diff))
1736
1737 # If we're not using diff, then simply list the expected
1738 # output followed by the actual output.
1739 if want and got:
1740 return 'Expected:\n%sGot:\n%s' % (_indent(want), _indent(got))
1741 elif want:
1742 return 'Expected:\n%sGot nothing\n' % _indent(want)
1743 elif got:
1744 return 'Expected nothing\nGot:\n%s' % _indent(got)
1745 else:
1746 return 'Expected nothing\nGot nothing\n'
1747
1748 class ESC[4;38;5;81mDocTestFailure(ESC[4;38;5;149mException):
1749 """A DocTest example has failed in debugging mode.
1750
1751 The exception instance has variables:
1752
1753 - test: the DocTest object being run
1754
1755 - example: the Example object that failed
1756
1757 - got: the actual output
1758 """
1759 def __init__(self, test, example, got):
1760 self.test = test
1761 self.example = example
1762 self.got = got
1763
1764 def __str__(self):
1765 return str(self.test)
1766
1767 class ESC[4;38;5;81mUnexpectedException(ESC[4;38;5;149mException):
1768 """A DocTest example has encountered an unexpected exception
1769
1770 The exception instance has variables:
1771
1772 - test: the DocTest object being run
1773
1774 - example: the Example object that failed
1775
1776 - exc_info: the exception info
1777 """
1778 def __init__(self, test, example, exc_info):
1779 self.test = test
1780 self.example = example
1781 self.exc_info = exc_info
1782
1783 def __str__(self):
1784 return str(self.test)
1785
1786 class ESC[4;38;5;81mDebugRunner(ESC[4;38;5;149mDocTestRunner):
1787 r"""Run doc tests but raise an exception as soon as there is a failure.
1788
1789 If an unexpected exception occurs, an UnexpectedException is raised.
1790 It contains the test, the example, and the original exception:
1791
1792 >>> runner = DebugRunner(verbose=False)
1793 >>> test = DocTestParser().get_doctest('>>> raise KeyError\n42',
1794 ... {}, 'foo', 'foo.py', 0)
1795 >>> try:
1796 ... runner.run(test)
1797 ... except UnexpectedException as f:
1798 ... failure = f
1799
1800 >>> failure.test is test
1801 True
1802
1803 >>> failure.example.want
1804 '42\n'
1805
1806 >>> exc_info = failure.exc_info
1807 >>> raise exc_info[1] # Already has the traceback
1808 Traceback (most recent call last):
1809 ...
1810 KeyError
1811
1812 We wrap the original exception to give the calling application
1813 access to the test and example information.
1814
1815 If the output doesn't match, then a DocTestFailure is raised:
1816
1817 >>> test = DocTestParser().get_doctest('''
1818 ... >>> x = 1
1819 ... >>> x
1820 ... 2
1821 ... ''', {}, 'foo', 'foo.py', 0)
1822
1823 >>> try:
1824 ... runner.run(test)
1825 ... except DocTestFailure as f:
1826 ... failure = f
1827
1828 DocTestFailure objects provide access to the test:
1829
1830 >>> failure.test is test
1831 True
1832
1833 As well as to the example:
1834
1835 >>> failure.example.want
1836 '2\n'
1837
1838 and the actual output:
1839
1840 >>> failure.got
1841 '1\n'
1842
1843 If a failure or error occurs, the globals are left intact:
1844
1845 >>> del test.globs['__builtins__']
1846 >>> test.globs
1847 {'x': 1}
1848
1849 >>> test = DocTestParser().get_doctest('''
1850 ... >>> x = 2
1851 ... >>> raise KeyError
1852 ... ''', {}, 'foo', 'foo.py', 0)
1853
1854 >>> runner.run(test)
1855 Traceback (most recent call last):
1856 ...
1857 doctest.UnexpectedException: <DocTest foo from foo.py:0 (2 examples)>
1858
1859 >>> del test.globs['__builtins__']
1860 >>> test.globs
1861 {'x': 2}
1862
1863 But the globals are cleared if there is no error:
1864
1865 >>> test = DocTestParser().get_doctest('''
1866 ... >>> x = 2
1867 ... ''', {}, 'foo', 'foo.py', 0)
1868
1869 >>> runner.run(test)
1870 TestResults(failed=0, attempted=1)
1871
1872 >>> test.globs
1873 {}
1874
1875 """
1876
1877 def run(self, test, compileflags=None, out=None, clear_globs=True):
1878 r = DocTestRunner.run(self, test, compileflags, out, False)
1879 if clear_globs:
1880 test.globs.clear()
1881 return r
1882
1883 def report_unexpected_exception(self, out, test, example, exc_info):
1884 raise UnexpectedException(test, example, exc_info)
1885
1886 def report_failure(self, out, test, example, got):
1887 raise DocTestFailure(test, example, got)
1888
1889 ######################################################################
1890 ## 6. Test Functions
1891 ######################################################################
1892 # These should be backwards compatible.
1893
1894 # For backward compatibility, a global instance of a DocTestRunner
1895 # class, updated by testmod.
1896 master = None
1897
1898 def testmod(m=None, name=None, globs=None, verbose=None,
1899 report=True, optionflags=0, extraglobs=None,
1900 raise_on_error=False, exclude_empty=False):
1901 """m=None, name=None, globs=None, verbose=None, report=True,
1902 optionflags=0, extraglobs=None, raise_on_error=False,
1903 exclude_empty=False
1904
1905 Test examples in docstrings in functions and classes reachable
1906 from module m (or the current module if m is not supplied), starting
1907 with m.__doc__.
1908
1909 Also test examples reachable from dict m.__test__ if it exists and is
1910 not None. m.__test__ maps names to functions, classes and strings;
1911 function and class docstrings are tested even if the name is private;
1912 strings are tested directly, as if they were docstrings.
1913
1914 Return (#failures, #tests).
1915
1916 See help(doctest) for an overview.
1917
1918 Optional keyword arg "name" gives the name of the module; by default
1919 use m.__name__.
1920
1921 Optional keyword arg "globs" gives a dict to be used as the globals
1922 when executing examples; by default, use m.__dict__. A copy of this
1923 dict is actually used for each docstring, so that each docstring's
1924 examples start with a clean slate.
1925
1926 Optional keyword arg "extraglobs" gives a dictionary that should be
1927 merged into the globals that are used to execute examples. By
1928 default, no extra globals are used. This is new in 2.4.
1929
1930 Optional keyword arg "verbose" prints lots of stuff if true, prints
1931 only failures if false; by default, it's true iff "-v" is in sys.argv.
1932
1933 Optional keyword arg "report" prints a summary at the end when true,
1934 else prints nothing at the end. In verbose mode, the summary is
1935 detailed, else very brief (in fact, empty if all tests passed).
1936
1937 Optional keyword arg "optionflags" or's together module constants,
1938 and defaults to 0. This is new in 2.3. Possible values (see the
1939 docs for details):
1940
1941 DONT_ACCEPT_TRUE_FOR_1
1942 DONT_ACCEPT_BLANKLINE
1943 NORMALIZE_WHITESPACE
1944 ELLIPSIS
1945 SKIP
1946 IGNORE_EXCEPTION_DETAIL
1947 REPORT_UDIFF
1948 REPORT_CDIFF
1949 REPORT_NDIFF
1950 REPORT_ONLY_FIRST_FAILURE
1951
1952 Optional keyword arg "raise_on_error" raises an exception on the
1953 first unexpected exception or failure. This allows failures to be
1954 post-mortem debugged.
1955
1956 Advanced tomfoolery: testmod runs methods of a local instance of
1957 class doctest.Tester, then merges the results into (or creates)
1958 global Tester instance doctest.master. Methods of doctest.master
1959 can be called directly too, if you want to do something unusual.
1960 Passing report=0 to testmod is especially useful then, to delay
1961 displaying a summary. Invoke doctest.master.summarize(verbose)
1962 when you're done fiddling.
1963 """
1964 global master
1965
1966 # If no module was given, then use __main__.
1967 if m is None:
1968 # DWA - m will still be None if this wasn't invoked from the command
1969 # line, in which case the following TypeError is about as good an error
1970 # as we should expect
1971 m = sys.modules.get('__main__')
1972
1973 # Check that we were actually given a module.
1974 if not inspect.ismodule(m):
1975 raise TypeError("testmod: module required; %r" % (m,))
1976
1977 # If no name was given, then use the module's name.
1978 if name is None:
1979 name = m.__name__
1980
1981 # Find, parse, and run all tests in the given module.
1982 finder = DocTestFinder(exclude_empty=exclude_empty)
1983
1984 if raise_on_error:
1985 runner = DebugRunner(verbose=verbose, optionflags=optionflags)
1986 else:
1987 runner = DocTestRunner(verbose=verbose, optionflags=optionflags)
1988
1989 for test in finder.find(m, name, globs=globs, extraglobs=extraglobs):
1990 runner.run(test)
1991
1992 if report:
1993 runner.summarize()
1994
1995 if master is None:
1996 master = runner
1997 else:
1998 master.merge(runner)
1999
2000 return TestResults(runner.failures, runner.tries)
2001
2002 def testfile(filename, module_relative=True, name=None, package=None,
2003 globs=None, verbose=None, report=True, optionflags=0,
2004 extraglobs=None, raise_on_error=False, parser=DocTestParser(),
2005 encoding=None):
2006 """
2007 Test examples in the given file. Return (#failures, #tests).
2008
2009 Optional keyword arg "module_relative" specifies how filenames
2010 should be interpreted:
2011
2012 - If "module_relative" is True (the default), then "filename"
2013 specifies a module-relative path. By default, this path is
2014 relative to the calling module's directory; but if the
2015 "package" argument is specified, then it is relative to that
2016 package. To ensure os-independence, "filename" should use
2017 "/" characters to separate path segments, and should not
2018 be an absolute path (i.e., it may not begin with "/").
2019
2020 - If "module_relative" is False, then "filename" specifies an
2021 os-specific path. The path may be absolute or relative (to
2022 the current working directory).
2023
2024 Optional keyword arg "name" gives the name of the test; by default
2025 use the file's basename.
2026
2027 Optional keyword argument "package" is a Python package or the
2028 name of a Python package whose directory should be used as the
2029 base directory for a module relative filename. If no package is
2030 specified, then the calling module's directory is used as the base
2031 directory for module relative filenames. It is an error to
2032 specify "package" if "module_relative" is False.
2033
2034 Optional keyword arg "globs" gives a dict to be used as the globals
2035 when executing examples; by default, use {}. A copy of this dict
2036 is actually used for each docstring, so that each docstring's
2037 examples start with a clean slate.
2038
2039 Optional keyword arg "extraglobs" gives a dictionary that should be
2040 merged into the globals that are used to execute examples. By
2041 default, no extra globals are used.
2042
2043 Optional keyword arg "verbose" prints lots of stuff if true, prints
2044 only failures if false; by default, it's true iff "-v" is in sys.argv.
2045
2046 Optional keyword arg "report" prints a summary at the end when true,
2047 else prints nothing at the end. In verbose mode, the summary is
2048 detailed, else very brief (in fact, empty if all tests passed).
2049
2050 Optional keyword arg "optionflags" or's together module constants,
2051 and defaults to 0. Possible values (see the docs for details):
2052
2053 DONT_ACCEPT_TRUE_FOR_1
2054 DONT_ACCEPT_BLANKLINE
2055 NORMALIZE_WHITESPACE
2056 ELLIPSIS
2057 SKIP
2058 IGNORE_EXCEPTION_DETAIL
2059 REPORT_UDIFF
2060 REPORT_CDIFF
2061 REPORT_NDIFF
2062 REPORT_ONLY_FIRST_FAILURE
2063
2064 Optional keyword arg "raise_on_error" raises an exception on the
2065 first unexpected exception or failure. This allows failures to be
2066 post-mortem debugged.
2067
2068 Optional keyword arg "parser" specifies a DocTestParser (or
2069 subclass) that should be used to extract tests from the files.
2070
2071 Optional keyword arg "encoding" specifies an encoding that should
2072 be used to convert the file to unicode.
2073
2074 Advanced tomfoolery: testmod runs methods of a local instance of
2075 class doctest.Tester, then merges the results into (or creates)
2076 global Tester instance doctest.master. Methods of doctest.master
2077 can be called directly too, if you want to do something unusual.
2078 Passing report=0 to testmod is especially useful then, to delay
2079 displaying a summary. Invoke doctest.master.summarize(verbose)
2080 when you're done fiddling.
2081 """
2082 global master
2083
2084 if package and not module_relative:
2085 raise ValueError("Package may only be specified for module-"
2086 "relative paths.")
2087
2088 # Relativize the path
2089 text, filename = _load_testfile(filename, package, module_relative,
2090 encoding or "utf-8")
2091
2092 # If no name was given, then use the file's name.
2093 if name is None:
2094 name = os.path.basename(filename)
2095
2096 # Assemble the globals.
2097 if globs is None:
2098 globs = {}
2099 else:
2100 globs = globs.copy()
2101 if extraglobs is not None:
2102 globs.update(extraglobs)
2103 if '__name__' not in globs:
2104 globs['__name__'] = '__main__'
2105
2106 if raise_on_error:
2107 runner = DebugRunner(verbose=verbose, optionflags=optionflags)
2108 else:
2109 runner = DocTestRunner(verbose=verbose, optionflags=optionflags)
2110
2111 # Read the file, convert it to a test, and run it.
2112 test = parser.get_doctest(text, globs, name, filename, 0)
2113 runner.run(test)
2114
2115 if report:
2116 runner.summarize()
2117
2118 if master is None:
2119 master = runner
2120 else:
2121 master.merge(runner)
2122
2123 return TestResults(runner.failures, runner.tries)
2124
2125 def run_docstring_examples(f, globs, verbose=False, name="NoName",
2126 compileflags=None, optionflags=0):
2127 """
2128 Test examples in the given object's docstring (`f`), using `globs`
2129 as globals. Optional argument `name` is used in failure messages.
2130 If the optional argument `verbose` is true, then generate output
2131 even if there are no failures.
2132
2133 `compileflags` gives the set of flags that should be used by the
2134 Python compiler when running the examples. If not specified, then
2135 it will default to the set of future-import flags that apply to
2136 `globs`.
2137
2138 Optional keyword arg `optionflags` specifies options for the
2139 testing and output. See the documentation for `testmod` for more
2140 information.
2141 """
2142 # Find, parse, and run all tests in the given module.
2143 finder = DocTestFinder(verbose=verbose, recurse=False)
2144 runner = DocTestRunner(verbose=verbose, optionflags=optionflags)
2145 for test in finder.find(f, name, globs=globs):
2146 runner.run(test, compileflags=compileflags)
2147
2148 ######################################################################
2149 ## 7. Unittest Support
2150 ######################################################################
2151
2152 _unittest_reportflags = 0
2153
2154 def set_unittest_reportflags(flags):
2155 """Sets the unittest option flags.
2156
2157 The old flag is returned so that a runner could restore the old
2158 value if it wished to:
2159
2160 >>> import doctest
2161 >>> old = doctest._unittest_reportflags
2162 >>> doctest.set_unittest_reportflags(REPORT_NDIFF |
2163 ... REPORT_ONLY_FIRST_FAILURE) == old
2164 True
2165
2166 >>> doctest._unittest_reportflags == (REPORT_NDIFF |
2167 ... REPORT_ONLY_FIRST_FAILURE)
2168 True
2169
2170 Only reporting flags can be set:
2171
2172 >>> doctest.set_unittest_reportflags(ELLIPSIS)
2173 Traceback (most recent call last):
2174 ...
2175 ValueError: ('Only reporting flags allowed', 8)
2176
2177 >>> doctest.set_unittest_reportflags(old) == (REPORT_NDIFF |
2178 ... REPORT_ONLY_FIRST_FAILURE)
2179 True
2180 """
2181 global _unittest_reportflags
2182
2183 if (flags & REPORTING_FLAGS) != flags:
2184 raise ValueError("Only reporting flags allowed", flags)
2185 old = _unittest_reportflags
2186 _unittest_reportflags = flags
2187 return old
2188
2189
2190 class ESC[4;38;5;81mDocTestCase(ESC[4;38;5;149munittestESC[4;38;5;149m.ESC[4;38;5;149mTestCase):
2191
2192 def __init__(self, test, optionflags=0, setUp=None, tearDown=None,
2193 checker=None):
2194
2195 unittest.TestCase.__init__(self)
2196 self._dt_optionflags = optionflags
2197 self._dt_checker = checker
2198 self._dt_globs = test.globs.copy()
2199 self._dt_test = test
2200 self._dt_setUp = setUp
2201 self._dt_tearDown = tearDown
2202
2203 def setUp(self):
2204 test = self._dt_test
2205
2206 if self._dt_setUp is not None:
2207 self._dt_setUp(test)
2208
2209 def tearDown(self):
2210 test = self._dt_test
2211
2212 if self._dt_tearDown is not None:
2213 self._dt_tearDown(test)
2214
2215 # restore the original globs
2216 test.globs.clear()
2217 test.globs.update(self._dt_globs)
2218
2219 def runTest(self):
2220 test = self._dt_test
2221 old = sys.stdout
2222 new = StringIO()
2223 optionflags = self._dt_optionflags
2224
2225 if not (optionflags & REPORTING_FLAGS):
2226 # The option flags don't include any reporting flags,
2227 # so add the default reporting flags
2228 optionflags |= _unittest_reportflags
2229
2230 runner = DocTestRunner(optionflags=optionflags,
2231 checker=self._dt_checker, verbose=False)
2232
2233 try:
2234 runner.DIVIDER = "-"*70
2235 failures, tries = runner.run(
2236 test, out=new.write, clear_globs=False)
2237 finally:
2238 sys.stdout = old
2239
2240 if failures:
2241 raise self.failureException(self.format_failure(new.getvalue()))
2242
2243 def format_failure(self, err):
2244 test = self._dt_test
2245 if test.lineno is None:
2246 lineno = 'unknown line number'
2247 else:
2248 lineno = '%s' % test.lineno
2249 lname = '.'.join(test.name.split('.')[-1:])
2250 return ('Failed doctest test for %s\n'
2251 ' File "%s", line %s, in %s\n\n%s'
2252 % (test.name, test.filename, lineno, lname, err)
2253 )
2254
2255 def debug(self):
2256 r"""Run the test case without results and without catching exceptions
2257
2258 The unit test framework includes a debug method on test cases
2259 and test suites to support post-mortem debugging. The test code
2260 is run in such a way that errors are not caught. This way a
2261 caller can catch the errors and initiate post-mortem debugging.
2262
2263 The DocTestCase provides a debug method that raises
2264 UnexpectedException errors if there is an unexpected
2265 exception:
2266
2267 >>> test = DocTestParser().get_doctest('>>> raise KeyError\n42',
2268 ... {}, 'foo', 'foo.py', 0)
2269 >>> case = DocTestCase(test)
2270 >>> try:
2271 ... case.debug()
2272 ... except UnexpectedException as f:
2273 ... failure = f
2274
2275 The UnexpectedException contains the test, the example, and
2276 the original exception:
2277
2278 >>> failure.test is test
2279 True
2280
2281 >>> failure.example.want
2282 '42\n'
2283
2284 >>> exc_info = failure.exc_info
2285 >>> raise exc_info[1] # Already has the traceback
2286 Traceback (most recent call last):
2287 ...
2288 KeyError
2289
2290 If the output doesn't match, then a DocTestFailure is raised:
2291
2292 >>> test = DocTestParser().get_doctest('''
2293 ... >>> x = 1
2294 ... >>> x
2295 ... 2
2296 ... ''', {}, 'foo', 'foo.py', 0)
2297 >>> case = DocTestCase(test)
2298
2299 >>> try:
2300 ... case.debug()
2301 ... except DocTestFailure as f:
2302 ... failure = f
2303
2304 DocTestFailure objects provide access to the test:
2305
2306 >>> failure.test is test
2307 True
2308
2309 As well as to the example:
2310
2311 >>> failure.example.want
2312 '2\n'
2313
2314 and the actual output:
2315
2316 >>> failure.got
2317 '1\n'
2318
2319 """
2320
2321 self.setUp()
2322 runner = DebugRunner(optionflags=self._dt_optionflags,
2323 checker=self._dt_checker, verbose=False)
2324 runner.run(self._dt_test, clear_globs=False)
2325 self.tearDown()
2326
2327 def id(self):
2328 return self._dt_test.name
2329
2330 def __eq__(self, other):
2331 if type(self) is not type(other):
2332 return NotImplemented
2333
2334 return self._dt_test == other._dt_test and \
2335 self._dt_optionflags == other._dt_optionflags and \
2336 self._dt_setUp == other._dt_setUp and \
2337 self._dt_tearDown == other._dt_tearDown and \
2338 self._dt_checker == other._dt_checker
2339
2340 def __hash__(self):
2341 return hash((self._dt_optionflags, self._dt_setUp, self._dt_tearDown,
2342 self._dt_checker))
2343
2344 def __repr__(self):
2345 name = self._dt_test.name.split('.')
2346 return "%s (%s)" % (name[-1], '.'.join(name[:-1]))
2347
2348 __str__ = object.__str__
2349
2350 def shortDescription(self):
2351 return "Doctest: " + self._dt_test.name
2352
2353 class ESC[4;38;5;81mSkipDocTestCase(ESC[4;38;5;149mDocTestCase):
2354 def __init__(self, module):
2355 self.module = module
2356 DocTestCase.__init__(self, None)
2357
2358 def setUp(self):
2359 self.skipTest("DocTestSuite will not work with -O2 and above")
2360
2361 def test_skip(self):
2362 pass
2363
2364 def shortDescription(self):
2365 return "Skipping tests from %s" % self.module.__name__
2366
2367 __str__ = shortDescription
2368
2369
2370 class ESC[4;38;5;81m_DocTestSuite(ESC[4;38;5;149munittestESC[4;38;5;149m.ESC[4;38;5;149mTestSuite):
2371
2372 def _removeTestAtIndex(self, index):
2373 pass
2374
2375
2376 def DocTestSuite(module=None, globs=None, extraglobs=None, test_finder=None,
2377 **options):
2378 """
2379 Convert doctest tests for a module to a unittest test suite.
2380
2381 This converts each documentation string in a module that
2382 contains doctest tests to a unittest test case. If any of the
2383 tests in a doc string fail, then the test case fails. An exception
2384 is raised showing the name of the file containing the test and a
2385 (sometimes approximate) line number.
2386
2387 The `module` argument provides the module to be tested. The argument
2388 can be either a module or a module name.
2389
2390 If no argument is given, the calling module is used.
2391
2392 A number of options may be provided as keyword arguments:
2393
2394 setUp
2395 A set-up function. This is called before running the
2396 tests in each file. The setUp function will be passed a DocTest
2397 object. The setUp function can access the test globals as the
2398 globs attribute of the test passed.
2399
2400 tearDown
2401 A tear-down function. This is called after running the
2402 tests in each file. The tearDown function will be passed a DocTest
2403 object. The tearDown function can access the test globals as the
2404 globs attribute of the test passed.
2405
2406 globs
2407 A dictionary containing initial global variables for the tests.
2408
2409 optionflags
2410 A set of doctest option flags expressed as an integer.
2411 """
2412
2413 if test_finder is None:
2414 test_finder = DocTestFinder()
2415
2416 module = _normalize_module(module)
2417 tests = test_finder.find(module, globs=globs, extraglobs=extraglobs)
2418
2419 if not tests and sys.flags.optimize >=2:
2420 # Skip doctests when running with -O2
2421 suite = _DocTestSuite()
2422 suite.addTest(SkipDocTestCase(module))
2423 return suite
2424
2425 tests.sort()
2426 suite = _DocTestSuite()
2427
2428 for test in tests:
2429 if len(test.examples) == 0:
2430 continue
2431 if not test.filename:
2432 filename = module.__file__
2433 if filename[-4:] == ".pyc":
2434 filename = filename[:-1]
2435 test.filename = filename
2436 suite.addTest(DocTestCase(test, **options))
2437
2438 return suite
2439
2440 class ESC[4;38;5;81mDocFileCase(ESC[4;38;5;149mDocTestCase):
2441
2442 def id(self):
2443 return '_'.join(self._dt_test.name.split('.'))
2444
2445 def __repr__(self):
2446 return self._dt_test.filename
2447
2448 def format_failure(self, err):
2449 return ('Failed doctest test for %s\n File "%s", line 0\n\n%s'
2450 % (self._dt_test.name, self._dt_test.filename, err)
2451 )
2452
2453 def DocFileTest(path, module_relative=True, package=None,
2454 globs=None, parser=DocTestParser(),
2455 encoding=None, **options):
2456 if globs is None:
2457 globs = {}
2458 else:
2459 globs = globs.copy()
2460
2461 if package and not module_relative:
2462 raise ValueError("Package may only be specified for module-"
2463 "relative paths.")
2464
2465 # Relativize the path.
2466 doc, path = _load_testfile(path, package, module_relative,
2467 encoding or "utf-8")
2468
2469 if "__file__" not in globs:
2470 globs["__file__"] = path
2471
2472 # Find the file and read it.
2473 name = os.path.basename(path)
2474
2475 # Convert it to a test, and wrap it in a DocFileCase.
2476 test = parser.get_doctest(doc, globs, name, path, 0)
2477 return DocFileCase(test, **options)
2478
2479 def DocFileSuite(*paths, **kw):
2480 """A unittest suite for one or more doctest files.
2481
2482 The path to each doctest file is given as a string; the
2483 interpretation of that string depends on the keyword argument
2484 "module_relative".
2485
2486 A number of options may be provided as keyword arguments:
2487
2488 module_relative
2489 If "module_relative" is True, then the given file paths are
2490 interpreted as os-independent module-relative paths. By
2491 default, these paths are relative to the calling module's
2492 directory; but if the "package" argument is specified, then
2493 they are relative to that package. To ensure os-independence,
2494 "filename" should use "/" characters to separate path
2495 segments, and may not be an absolute path (i.e., it may not
2496 begin with "/").
2497
2498 If "module_relative" is False, then the given file paths are
2499 interpreted as os-specific paths. These paths may be absolute
2500 or relative (to the current working directory).
2501
2502 package
2503 A Python package or the name of a Python package whose directory
2504 should be used as the base directory for module relative paths.
2505 If "package" is not specified, then the calling module's
2506 directory is used as the base directory for module relative
2507 filenames. It is an error to specify "package" if
2508 "module_relative" is False.
2509
2510 setUp
2511 A set-up function. This is called before running the
2512 tests in each file. The setUp function will be passed a DocTest
2513 object. The setUp function can access the test globals as the
2514 globs attribute of the test passed.
2515
2516 tearDown
2517 A tear-down function. This is called after running the
2518 tests in each file. The tearDown function will be passed a DocTest
2519 object. The tearDown function can access the test globals as the
2520 globs attribute of the test passed.
2521
2522 globs
2523 A dictionary containing initial global variables for the tests.
2524
2525 optionflags
2526 A set of doctest option flags expressed as an integer.
2527
2528 parser
2529 A DocTestParser (or subclass) that should be used to extract
2530 tests from the files.
2531
2532 encoding
2533 An encoding that will be used to convert the files to unicode.
2534 """
2535 suite = _DocTestSuite()
2536
2537 # We do this here so that _normalize_module is called at the right
2538 # level. If it were called in DocFileTest, then this function
2539 # would be the caller and we might guess the package incorrectly.
2540 if kw.get('module_relative', True):
2541 kw['package'] = _normalize_module(kw.get('package'))
2542
2543 for path in paths:
2544 suite.addTest(DocFileTest(path, **kw))
2545
2546 return suite
2547
2548 ######################################################################
2549 ## 8. Debugging Support
2550 ######################################################################
2551
2552 def script_from_examples(s):
2553 r"""Extract script from text with examples.
2554
2555 Converts text with examples to a Python script. Example input is
2556 converted to regular code. Example output and all other words
2557 are converted to comments:
2558
2559 >>> text = '''
2560 ... Here are examples of simple math.
2561 ...
2562 ... Python has super accurate integer addition
2563 ...
2564 ... >>> 2 + 2
2565 ... 5
2566 ...
2567 ... And very friendly error messages:
2568 ...
2569 ... >>> 1/0
2570 ... To Infinity
2571 ... And
2572 ... Beyond
2573 ...
2574 ... You can use logic if you want:
2575 ...
2576 ... >>> if 0:
2577 ... ... blah
2578 ... ... blah
2579 ... ...
2580 ...
2581 ... Ho hum
2582 ... '''
2583
2584 >>> print(script_from_examples(text))
2585 # Here are examples of simple math.
2586 #
2587 # Python has super accurate integer addition
2588 #
2589 2 + 2
2590 # Expected:
2591 ## 5
2592 #
2593 # And very friendly error messages:
2594 #
2595 1/0
2596 # Expected:
2597 ## To Infinity
2598 ## And
2599 ## Beyond
2600 #
2601 # You can use logic if you want:
2602 #
2603 if 0:
2604 blah
2605 blah
2606 #
2607 # Ho hum
2608 <BLANKLINE>
2609 """
2610 output = []
2611 for piece in DocTestParser().parse(s):
2612 if isinstance(piece, Example):
2613 # Add the example's source code (strip trailing NL)
2614 output.append(piece.source[:-1])
2615 # Add the expected output:
2616 want = piece.want
2617 if want:
2618 output.append('# Expected:')
2619 output += ['## '+l for l in want.split('\n')[:-1]]
2620 else:
2621 # Add non-example text.
2622 output += [_comment_line(l)
2623 for l in piece.split('\n')[:-1]]
2624
2625 # Trim junk on both ends.
2626 while output and output[-1] == '#':
2627 output.pop()
2628 while output and output[0] == '#':
2629 output.pop(0)
2630 # Combine the output, and return it.
2631 # Add a courtesy newline to prevent exec from choking (see bug #1172785)
2632 return '\n'.join(output) + '\n'
2633
2634 def testsource(module, name):
2635 """Extract the test sources from a doctest docstring as a script.
2636
2637 Provide the module (or dotted name of the module) containing the
2638 test to be debugged and the name (within the module) of the object
2639 with the doc string with tests to be debugged.
2640 """
2641 module = _normalize_module(module)
2642 tests = DocTestFinder().find(module)
2643 test = [t for t in tests if t.name == name]
2644 if not test:
2645 raise ValueError(name, "not found in tests")
2646 test = test[0]
2647 testsrc = script_from_examples(test.docstring)
2648 return testsrc
2649
2650 def debug_src(src, pm=False, globs=None):
2651 """Debug a single doctest docstring, in argument `src`'"""
2652 testsrc = script_from_examples(src)
2653 debug_script(testsrc, pm, globs)
2654
2655 def debug_script(src, pm=False, globs=None):
2656 "Debug a test script. `src` is the script, as a string."
2657 import pdb
2658
2659 if globs:
2660 globs = globs.copy()
2661 else:
2662 globs = {}
2663
2664 if pm:
2665 try:
2666 exec(src, globs, globs)
2667 except:
2668 print(sys.exc_info()[1])
2669 p = pdb.Pdb(nosigint=True)
2670 p.reset()
2671 p.interaction(None, sys.exc_info()[2])
2672 else:
2673 pdb.Pdb(nosigint=True).run("exec(%r)" % src, globs, globs)
2674
2675 def debug(module, name, pm=False):
2676 """Debug a single doctest docstring.
2677
2678 Provide the module (or dotted name of the module) containing the
2679 test to be debugged and the name (within the module) of the object
2680 with the docstring with tests to be debugged.
2681 """
2682 module = _normalize_module(module)
2683 testsrc = testsource(module, name)
2684 debug_script(testsrc, pm, module.__dict__)
2685
2686 ######################################################################
2687 ## 9. Example Usage
2688 ######################################################################
2689 class ESC[4;38;5;81m_TestClass:
2690 """
2691 A pointless class, for sanity-checking of docstring testing.
2692
2693 Methods:
2694 square()
2695 get()
2696
2697 >>> _TestClass(13).get() + _TestClass(-12).get()
2698 1
2699 >>> hex(_TestClass(13).square().get())
2700 '0xa9'
2701 """
2702
2703 def __init__(self, val):
2704 """val -> _TestClass object with associated value val.
2705
2706 >>> t = _TestClass(123)
2707 >>> print(t.get())
2708 123
2709 """
2710
2711 self.val = val
2712
2713 def square(self):
2714 """square() -> square TestClass's associated value
2715
2716 >>> _TestClass(13).square().get()
2717 169
2718 """
2719
2720 self.val = self.val ** 2
2721 return self
2722
2723 def get(self):
2724 """get() -> return TestClass's associated value.
2725
2726 >>> x = _TestClass(-42)
2727 >>> print(x.get())
2728 -42
2729 """
2730
2731 return self.val
2732
2733 __test__ = {"_TestClass": _TestClass,
2734 "string": r"""
2735 Example of a string object, searched as-is.
2736 >>> x = 1; y = 2
2737 >>> x + y, x * y
2738 (3, 2)
2739 """,
2740
2741 "bool-int equivalence": r"""
2742 In 2.2, boolean expressions displayed
2743 0 or 1. By default, we still accept
2744 them. This can be disabled by passing
2745 DONT_ACCEPT_TRUE_FOR_1 to the new
2746 optionflags argument.
2747 >>> 4 == 4
2748 1
2749 >>> 4 == 4
2750 True
2751 >>> 4 > 4
2752 0
2753 >>> 4 > 4
2754 False
2755 """,
2756
2757 "blank lines": r"""
2758 Blank lines can be marked with <BLANKLINE>:
2759 >>> print('foo\n\nbar\n')
2760 foo
2761 <BLANKLINE>
2762 bar
2763 <BLANKLINE>
2764 """,
2765
2766 "ellipsis": r"""
2767 If the ellipsis flag is used, then '...' can be used to
2768 elide substrings in the desired output:
2769 >>> print(list(range(1000))) #doctest: +ELLIPSIS
2770 [0, 1, 2, ..., 999]
2771 """,
2772
2773 "whitespace normalization": r"""
2774 If the whitespace normalization flag is used, then
2775 differences in whitespace are ignored.
2776 >>> print(list(range(30))) #doctest: +NORMALIZE_WHITESPACE
2777 [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14,
2778 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26,
2779 27, 28, 29]
2780 """,
2781 }
2782
2783
2784 def _test():
2785 import argparse
2786
2787 parser = argparse.ArgumentParser(description="doctest runner")
2788 parser.add_argument('-v', '--verbose', action='store_true', default=False,
2789 help='print very verbose output for all tests')
2790 parser.add_argument('-o', '--option', action='append',
2791 choices=OPTIONFLAGS_BY_NAME.keys(), default=[],
2792 help=('specify a doctest option flag to apply'
2793 ' to the test run; may be specified more'
2794 ' than once to apply multiple options'))
2795 parser.add_argument('-f', '--fail-fast', action='store_true',
2796 help=('stop running tests after first failure (this'
2797 ' is a shorthand for -o FAIL_FAST, and is'
2798 ' in addition to any other -o options)'))
2799 parser.add_argument('file', nargs='+',
2800 help='file containing the tests to run')
2801 args = parser.parse_args()
2802 testfiles = args.file
2803 # Verbose used to be handled by the "inspect argv" magic in DocTestRunner,
2804 # but since we are using argparse we are passing it manually now.
2805 verbose = args.verbose
2806 options = 0
2807 for option in args.option:
2808 options |= OPTIONFLAGS_BY_NAME[option]
2809 if args.fail_fast:
2810 options |= FAIL_FAST
2811 for filename in testfiles:
2812 if filename.endswith(".py"):
2813 # It is a module -- insert its dir into sys.path and try to
2814 # import it. If it is part of a package, that possibly
2815 # won't work because of package imports.
2816 dirname, filename = os.path.split(filename)
2817 sys.path.insert(0, dirname)
2818 m = __import__(filename[:-3])
2819 del sys.path[0]
2820 failures, _ = testmod(m, verbose=verbose, optionflags=options)
2821 else:
2822 failures, _ = testfile(filename, module_relative=False,
2823 verbose=verbose, optionflags=options)
2824 if failures:
2825 return 1
2826 return 0
2827
2828
2829 if __name__ == "__main__":
2830 sys.exit(_test())