python (3.11.7)
1 """
2 pygments.scanner
3 ~~~~~~~~~~~~~~~~
4
5 This library implements a regex based scanner. Some languages
6 like Pascal are easy to parse but have some keywords that
7 depend on the context. Because of this it's impossible to lex
8 that just by using a regular expression lexer like the
9 `RegexLexer`.
10
11 Have a look at the `DelphiLexer` to get an idea of how to use
12 this scanner.
13
14 :copyright: Copyright 2006-2023 by the Pygments team, see AUTHORS.
15 :license: BSD, see LICENSE for details.
16 """
17 import re
18
19
20 class ESC[4;38;5;81mEndOfText(ESC[4;38;5;149mRuntimeError):
21 """
22 Raise if end of text is reached and the user
23 tried to call a match function.
24 """
25
26
27 class ESC[4;38;5;81mScanner:
28 """
29 Simple scanner
30
31 All method patterns are regular expression strings (not
32 compiled expressions!)
33 """
34
35 def __init__(self, text, flags=0):
36 """
37 :param text: The text which should be scanned
38 :param flags: default regular expression flags
39 """
40 self.data = text
41 self.data_length = len(text)
42 self.start_pos = 0
43 self.pos = 0
44 self.flags = flags
45 self.last = None
46 self.match = None
47 self._re_cache = {}
48
49 def eos(self):
50 """`True` if the scanner reached the end of text."""
51 return self.pos >= self.data_length
52 eos = property(eos, eos.__doc__)
53
54 def check(self, pattern):
55 """
56 Apply `pattern` on the current position and return
57 the match object. (Doesn't touch pos). Use this for
58 lookahead.
59 """
60 if self.eos:
61 raise EndOfText()
62 if pattern not in self._re_cache:
63 self._re_cache[pattern] = re.compile(pattern, self.flags)
64 return self._re_cache[pattern].match(self.data, self.pos)
65
66 def test(self, pattern):
67 """Apply a pattern on the current position and check
68 if it patches. Doesn't touch pos.
69 """
70 return self.check(pattern) is not None
71
72 def scan(self, pattern):
73 """
74 Scan the text for the given pattern and update pos/match
75 and related fields. The return value is a boolean that
76 indicates if the pattern matched. The matched value is
77 stored on the instance as ``match``, the last value is
78 stored as ``last``. ``start_pos`` is the position of the
79 pointer before the pattern was matched, ``pos`` is the
80 end position.
81 """
82 if self.eos:
83 raise EndOfText()
84 if pattern not in self._re_cache:
85 self._re_cache[pattern] = re.compile(pattern, self.flags)
86 self.last = self.match
87 m = self._re_cache[pattern].match(self.data, self.pos)
88 if m is None:
89 return False
90 self.start_pos = m.start()
91 self.pos = m.end()
92 self.match = m.group()
93 return True
94
95 def get_char(self):
96 """Scan exactly one char."""
97 self.scan('.')
98
99 def __repr__(self):
100 return '<%s %d/%d>' % (
101 self.__class__.__name__,
102 self.pos,
103 self.data_length
104 )
Browse
Usage
Versions