python (3.11.7)
1 """
2 pygments.token
3 ~~~~~~~~~~~~~~
4
5 Basic token types and the standard tokens.
6
7 :copyright: Copyright 2006-2023 by the Pygments team, see AUTHORS.
8 :license: BSD, see LICENSE for details.
9 """
10
11
12 class ESC[4;38;5;81m_TokenType(ESC[4;38;5;149mtuple):
13 parent = None
14
15 def split(self):
16 buf = []
17 node = self
18 while node is not None:
19 buf.append(node)
20 node = node.parent
21 buf.reverse()
22 return buf
23
24 def __init__(self, *args):
25 # no need to call super.__init__
26 self.subtypes = set()
27
28 def __contains__(self, val):
29 return self is val or (
30 type(val) is self.__class__ and
31 val[:len(self)] == self
32 )
33
34 def __getattr__(self, val):
35 if not val or not val[0].isupper():
36 return tuple.__getattribute__(self, val)
37 new = _TokenType(self + (val,))
38 setattr(self, val, new)
39 self.subtypes.add(new)
40 new.parent = self
41 return new
42
43 def __repr__(self):
44 return 'Token' + (self and '.' or '') + '.'.join(self)
45
46 def __copy__(self):
47 # These instances are supposed to be singletons
48 return self
49
50 def __deepcopy__(self, memo):
51 # These instances are supposed to be singletons
52 return self
53
54
55 Token = _TokenType()
56
57 # Special token types
58 Text = Token.Text
59 Whitespace = Text.Whitespace
60 Escape = Token.Escape
61 Error = Token.Error
62 # Text that doesn't belong to this lexer (e.g. HTML in PHP)
63 Other = Token.Other
64
65 # Common token types for source code
66 Keyword = Token.Keyword
67 Name = Token.Name
68 Literal = Token.Literal
69 String = Literal.String
70 Number = Literal.Number
71 Punctuation = Token.Punctuation
72 Operator = Token.Operator
73 Comment = Token.Comment
74
75 # Generic types for non-source code
76 Generic = Token.Generic
77
78 # String and some others are not direct children of Token.
79 # alias them:
80 Token.Token = Token
81 Token.String = String
82 Token.Number = Number
83
84
85 def is_token_subtype(ttype, other):
86 """
87 Return True if ``ttype`` is a subtype of ``other``.
88
89 exists for backwards compatibility. use ``ttype in other`` now.
90 """
91 return ttype in other
92
93
94 def string_to_tokentype(s):
95 """
96 Convert a string into a token type::
97
98 >>> string_to_token('String.Double')
99 Token.Literal.String.Double
100 >>> string_to_token('Token.Literal.Number')
101 Token.Literal.Number
102 >>> string_to_token('')
103 Token
104
105 Tokens that are already tokens are returned unchanged:
106
107 >>> string_to_token(String)
108 Token.Literal.String
109 """
110 if isinstance(s, _TokenType):
111 return s
112 if not s:
113 return Token
114 node = Token
115 for item in s.split('.'):
116 node = getattr(node, item)
117 return node
118
119
120 # Map standard token types to short names, used in CSS class naming.
121 # If you add a new item, please be sure to run this file to perform
122 # a consistency check for duplicate values.
123 STANDARD_TYPES = {
124 Token: '',
125
126 Text: '',
127 Whitespace: 'w',
128 Escape: 'esc',
129 Error: 'err',
130 Other: 'x',
131
132 Keyword: 'k',
133 Keyword.Constant: 'kc',
134 Keyword.Declaration: 'kd',
135 Keyword.Namespace: 'kn',
136 Keyword.Pseudo: 'kp',
137 Keyword.Reserved: 'kr',
138 Keyword.Type: 'kt',
139
140 Name: 'n',
141 Name.Attribute: 'na',
142 Name.Builtin: 'nb',
143 Name.Builtin.Pseudo: 'bp',
144 Name.Class: 'nc',
145 Name.Constant: 'no',
146 Name.Decorator: 'nd',
147 Name.Entity: 'ni',
148 Name.Exception: 'ne',
149 Name.Function: 'nf',
150 Name.Function.Magic: 'fm',
151 Name.Property: 'py',
152 Name.Label: 'nl',
153 Name.Namespace: 'nn',
154 Name.Other: 'nx',
155 Name.Tag: 'nt',
156 Name.Variable: 'nv',
157 Name.Variable.Class: 'vc',
158 Name.Variable.Global: 'vg',
159 Name.Variable.Instance: 'vi',
160 Name.Variable.Magic: 'vm',
161
162 Literal: 'l',
163 Literal.Date: 'ld',
164
165 String: 's',
166 String.Affix: 'sa',
167 String.Backtick: 'sb',
168 String.Char: 'sc',
169 String.Delimiter: 'dl',
170 String.Doc: 'sd',
171 String.Double: 's2',
172 String.Escape: 'se',
173 String.Heredoc: 'sh',
174 String.Interpol: 'si',
175 String.Other: 'sx',
176 String.Regex: 'sr',
177 String.Single: 's1',
178 String.Symbol: 'ss',
179
180 Number: 'm',
181 Number.Bin: 'mb',
182 Number.Float: 'mf',
183 Number.Hex: 'mh',
184 Number.Integer: 'mi',
185 Number.Integer.Long: 'il',
186 Number.Oct: 'mo',
187
188 Operator: 'o',
189 Operator.Word: 'ow',
190
191 Punctuation: 'p',
192 Punctuation.Marker: 'pm',
193
194 Comment: 'c',
195 Comment.Hashbang: 'ch',
196 Comment.Multiline: 'cm',
197 Comment.Preproc: 'cp',
198 Comment.PreprocFile: 'cpf',
199 Comment.Single: 'c1',
200 Comment.Special: 'cs',
201
202 Generic: 'g',
203 Generic.Deleted: 'gd',
204 Generic.Emph: 'ge',
205 Generic.Error: 'gr',
206 Generic.Heading: 'gh',
207 Generic.Inserted: 'gi',
208 Generic.Output: 'go',
209 Generic.Prompt: 'gp',
210 Generic.Strong: 'gs',
211 Generic.Subheading: 'gu',
212 Generic.Traceback: 'gt',
213 }
Browse
Usage
Versions