1 import builtins
2 import keyword
3 import re
4 import time
5
6 from idlelib.config import idleConf
7 from idlelib.delegator import Delegator
8
9 DEBUG = False
10
11
12 def any(name, alternates):
13 "Return a named group pattern matching list of alternates."
14 return "(?P<%s>" % name + "|".join(alternates) + ")"
15
16
17 def make_pat():
18 kw = r"\b" + any("KEYWORD", keyword.kwlist) + r"\b"
19 match_softkw = (
20 r"^[ \t]*" + # at beginning of line + possible indentation
21 r"(?P<MATCH_SOFTKW>match)\b" +
22 r"(?![ \t]*(?:" + "|".join([ # not followed by ...
23 r"[:,;=^&|@~)\]}]", # a character which means it can't be a
24 # pattern-matching statement
25 r"\b(?:" + r"|".join(keyword.kwlist) + r")\b", # a keyword
26 ]) +
27 r"))"
28 )
29 case_default = (
30 r"^[ \t]*" + # at beginning of line + possible indentation
31 r"(?P<CASE_SOFTKW>case)" +
32 r"[ \t]+(?P<CASE_DEFAULT_UNDERSCORE>_\b)"
33 )
34 case_softkw_and_pattern = (
35 r"^[ \t]*" + # at beginning of line + possible indentation
36 r"(?P<CASE_SOFTKW2>case)\b" +
37 r"(?![ \t]*(?:" + "|".join([ # not followed by ...
38 r"_\b", # a lone underscore
39 r"[:,;=^&|@~)\]}]", # a character which means it can't be a
40 # pattern-matching case
41 r"\b(?:" + r"|".join(keyword.kwlist) + r")\b", # a keyword
42 ]) +
43 r"))"
44 )
45 builtinlist = [str(name) for name in dir(builtins)
46 if not name.startswith('_') and
47 name not in keyword.kwlist]
48 builtin = r"([^.'\"\\#]\b|^)" + any("BUILTIN", builtinlist) + r"\b"
49 comment = any("COMMENT", [r"#[^\n]*"])
50 stringprefix = r"(?i:r|u|f|fr|rf|b|br|rb)?"
51 sqstring = stringprefix + r"'[^'\\\n]*(\\.[^'\\\n]*)*'?"
52 dqstring = stringprefix + r'"[^"\\\n]*(\\.[^"\\\n]*)*"?'
53 sq3string = stringprefix + r"'''[^'\\]*((\\.|'(?!''))[^'\\]*)*(''')?"
54 dq3string = stringprefix + r'"""[^"\\]*((\\.|"(?!""))[^"\\]*)*(""")?'
55 string = any("STRING", [sq3string, dq3string, sqstring, dqstring])
56 prog = re.compile("|".join([
57 builtin, comment, string, kw,
58 match_softkw, case_default,
59 case_softkw_and_pattern,
60 any("SYNC", [r"\n"]),
61 ]),
62 re.DOTALL | re.MULTILINE)
63 return prog
64
65
66 prog = make_pat()
67 idprog = re.compile(r"\s+(\w+)")
68 prog_group_name_to_tag = {
69 "MATCH_SOFTKW": "KEYWORD",
70 "CASE_SOFTKW": "KEYWORD",
71 "CASE_DEFAULT_UNDERSCORE": "KEYWORD",
72 "CASE_SOFTKW2": "KEYWORD",
73 }
74
75
76 def matched_named_groups(re_match):
77 "Get only the non-empty named groups from an re.Match object."
78 return ((k, v) for (k, v) in re_match.groupdict().items() if v)
79
80
81 def color_config(text):
82 """Set color options of Text widget.
83
84 If ColorDelegator is used, this should be called first.
85 """
86 # Called from htest, TextFrame, Editor, and Turtledemo.
87 # Not automatic because ColorDelegator does not know 'text'.
88 theme = idleConf.CurrentTheme()
89 normal_colors = idleConf.GetHighlight(theme, 'normal')
90 cursor_color = idleConf.GetHighlight(theme, 'cursor')['foreground']
91 select_colors = idleConf.GetHighlight(theme, 'hilite')
92 text.config(
93 foreground=normal_colors['foreground'],
94 background=normal_colors['background'],
95 insertbackground=cursor_color,
96 selectforeground=select_colors['foreground'],
97 selectbackground=select_colors['background'],
98 inactiveselectbackground=select_colors['background'], # new in 8.5
99 )
100
101
102 class ESC[4;38;5;81mColorDelegator(ESC[4;38;5;149mDelegator):
103 """Delegator for syntax highlighting (text coloring).
104
105 Instance variables:
106 delegate: Delegator below this one in the stack, meaning the
107 one this one delegates to.
108
109 Used to track state:
110 after_id: Identifier for scheduled after event, which is a
111 timer for colorizing the text.
112 allow_colorizing: Boolean toggle for applying colorizing.
113 colorizing: Boolean flag when colorizing is in process.
114 stop_colorizing: Boolean flag to end an active colorizing
115 process.
116 """
117
118 def __init__(self):
119 Delegator.__init__(self)
120 self.init_state()
121 self.prog = prog
122 self.idprog = idprog
123 self.LoadTagDefs()
124
125 def init_state(self):
126 "Initialize variables that track colorizing state."
127 self.after_id = None
128 self.allow_colorizing = True
129 self.stop_colorizing = False
130 self.colorizing = False
131
132 def setdelegate(self, delegate):
133 """Set the delegate for this instance.
134
135 A delegate is an instance of a Delegator class and each
136 delegate points to the next delegator in the stack. This
137 allows multiple delegators to be chained together for a
138 widget. The bottom delegate for a colorizer is a Text
139 widget.
140
141 If there is a delegate, also start the colorizing process.
142 """
143 if self.delegate is not None:
144 self.unbind("<<toggle-auto-coloring>>")
145 Delegator.setdelegate(self, delegate)
146 if delegate is not None:
147 self.config_colors()
148 self.bind("<<toggle-auto-coloring>>", self.toggle_colorize_event)
149 self.notify_range("1.0", "end")
150 else:
151 # No delegate - stop any colorizing.
152 self.stop_colorizing = True
153 self.allow_colorizing = False
154
155 def config_colors(self):
156 "Configure text widget tags with colors from tagdefs."
157 for tag, cnf in self.tagdefs.items():
158 self.tag_configure(tag, **cnf)
159 self.tag_raise('sel')
160
161 def LoadTagDefs(self):
162 "Create dictionary of tag names to text colors."
163 theme = idleConf.CurrentTheme()
164 self.tagdefs = {
165 "COMMENT": idleConf.GetHighlight(theme, "comment"),
166 "KEYWORD": idleConf.GetHighlight(theme, "keyword"),
167 "BUILTIN": idleConf.GetHighlight(theme, "builtin"),
168 "STRING": idleConf.GetHighlight(theme, "string"),
169 "DEFINITION": idleConf.GetHighlight(theme, "definition"),
170 "SYNC": {'background': None, 'foreground': None},
171 "TODO": {'background': None, 'foreground': None},
172 "ERROR": idleConf.GetHighlight(theme, "error"),
173 # "hit" is used by ReplaceDialog to mark matches. It shouldn't be changed by Colorizer, but
174 # that currently isn't technically possible. This should be moved elsewhere in the future
175 # when fixing the "hit" tag's visibility, or when the replace dialog is replaced with a
176 # non-modal alternative.
177 "hit": idleConf.GetHighlight(theme, "hit"),
178 }
179 if DEBUG: print('tagdefs', self.tagdefs)
180
181 def insert(self, index, chars, tags=None):
182 "Insert chars into widget at index and mark for colorizing."
183 index = self.index(index)
184 self.delegate.insert(index, chars, tags)
185 self.notify_range(index, index + "+%dc" % len(chars))
186
187 def delete(self, index1, index2=None):
188 "Delete chars between indexes and mark for colorizing."
189 index1 = self.index(index1)
190 self.delegate.delete(index1, index2)
191 self.notify_range(index1)
192
193 def notify_range(self, index1, index2=None):
194 "Mark text changes for processing and restart colorizing, if active."
195 self.tag_add("TODO", index1, index2)
196 if self.after_id:
197 if DEBUG: print("colorizing already scheduled")
198 return
199 if self.colorizing:
200 self.stop_colorizing = True
201 if DEBUG: print("stop colorizing")
202 if self.allow_colorizing:
203 if DEBUG: print("schedule colorizing")
204 self.after_id = self.after(1, self.recolorize)
205 return
206
207 def close(self):
208 if self.after_id:
209 after_id = self.after_id
210 self.after_id = None
211 if DEBUG: print("cancel scheduled recolorizer")
212 self.after_cancel(after_id)
213 self.allow_colorizing = False
214 self.stop_colorizing = True
215
216 def toggle_colorize_event(self, event=None):
217 """Toggle colorizing on and off.
218
219 When toggling off, if colorizing is scheduled or is in
220 process, it will be cancelled and/or stopped.
221
222 When toggling on, colorizing will be scheduled.
223 """
224 if self.after_id:
225 after_id = self.after_id
226 self.after_id = None
227 if DEBUG: print("cancel scheduled recolorizer")
228 self.after_cancel(after_id)
229 if self.allow_colorizing and self.colorizing:
230 if DEBUG: print("stop colorizing")
231 self.stop_colorizing = True
232 self.allow_colorizing = not self.allow_colorizing
233 if self.allow_colorizing and not self.colorizing:
234 self.after_id = self.after(1, self.recolorize)
235 if DEBUG:
236 print("auto colorizing turned",
237 "on" if self.allow_colorizing else "off")
238 return "break"
239
240 def recolorize(self):
241 """Timer event (every 1ms) to colorize text.
242
243 Colorizing is only attempted when the text widget exists,
244 when colorizing is toggled on, and when the colorizing
245 process is not already running.
246
247 After colorizing is complete, some cleanup is done to
248 make sure that all the text has been colorized.
249 """
250 self.after_id = None
251 if not self.delegate:
252 if DEBUG: print("no delegate")
253 return
254 if not self.allow_colorizing:
255 if DEBUG: print("auto colorizing is off")
256 return
257 if self.colorizing:
258 if DEBUG: print("already colorizing")
259 return
260 try:
261 self.stop_colorizing = False
262 self.colorizing = True
263 if DEBUG: print("colorizing...")
264 t0 = time.perf_counter()
265 self.recolorize_main()
266 t1 = time.perf_counter()
267 if DEBUG: print("%.3f seconds" % (t1-t0))
268 finally:
269 self.colorizing = False
270 if self.allow_colorizing and self.tag_nextrange("TODO", "1.0"):
271 if DEBUG: print("reschedule colorizing")
272 self.after_id = self.after(1, self.recolorize)
273
274 def recolorize_main(self):
275 "Evaluate text and apply colorizing tags."
276 next = "1.0"
277 while todo_tag_range := self.tag_nextrange("TODO", next):
278 self.tag_remove("SYNC", todo_tag_range[0], todo_tag_range[1])
279 sync_tag_range = self.tag_prevrange("SYNC", todo_tag_range[0])
280 head = sync_tag_range[1] if sync_tag_range else "1.0"
281
282 chars = ""
283 next = head
284 lines_to_get = 1
285 ok = False
286 while not ok:
287 mark = next
288 next = self.index(mark + "+%d lines linestart" %
289 lines_to_get)
290 lines_to_get = min(lines_to_get * 2, 100)
291 ok = "SYNC" in self.tag_names(next + "-1c")
292 line = self.get(mark, next)
293 ##print head, "get", mark, next, "->", repr(line)
294 if not line:
295 return
296 for tag in self.tagdefs:
297 self.tag_remove(tag, mark, next)
298 chars += line
299 self._add_tags_in_section(chars, head)
300 if "SYNC" in self.tag_names(next + "-1c"):
301 head = next
302 chars = ""
303 else:
304 ok = False
305 if not ok:
306 # We're in an inconsistent state, and the call to
307 # update may tell us to stop. It may also change
308 # the correct value for "next" (since this is a
309 # line.col string, not a true mark). So leave a
310 # crumb telling the next invocation to resume here
311 # in case update tells us to leave.
312 self.tag_add("TODO", next)
313 self.update_idletasks()
314 if self.stop_colorizing:
315 if DEBUG: print("colorizing stopped")
316 return
317
318 def _add_tag(self, start, end, head, matched_group_name):
319 """Add a tag to a given range in the text widget.
320
321 This is a utility function, receiving the range as `start` and
322 `end` positions, each of which is a number of characters
323 relative to the given `head` index in the text widget.
324
325 The tag to add is determined by `matched_group_name`, which is
326 the name of a regular expression "named group" as matched by
327 by the relevant highlighting regexps.
328 """
329 tag = prog_group_name_to_tag.get(matched_group_name,
330 matched_group_name)
331 self.tag_add(tag,
332 f"{head}+{start:d}c",
333 f"{head}+{end:d}c")
334
335 def _add_tags_in_section(self, chars, head):
336 """Parse and add highlighting tags to a given part of the text.
337
338 `chars` is a string with the text to parse and to which
339 highlighting is to be applied.
340
341 `head` is the index in the text widget where the text is found.
342 """
343 for m in self.prog.finditer(chars):
344 for name, matched_text in matched_named_groups(m):
345 a, b = m.span(name)
346 self._add_tag(a, b, head, name)
347 if matched_text in ("def", "class"):
348 if m1 := self.idprog.match(chars, b):
349 a, b = m1.span(1)
350 self._add_tag(a, b, head, "DEFINITION")
351
352 def removecolors(self):
353 "Remove all colorizing tags."
354 for tag in self.tagdefs:
355 self.tag_remove(tag, "1.0", "end")
356
357
358 def _color_delegator(parent): # htest #
359 from tkinter import Toplevel, Text
360 from idlelib.idle_test.test_colorizer import source
361 from idlelib.percolator import Percolator
362
363 top = Toplevel(parent)
364 top.title("Test ColorDelegator")
365 x, y = map(int, parent.geometry().split('+')[1:])
366 top.geometry("700x550+%d+%d" % (x + 20, y + 175))
367
368 text = Text(top, background="white")
369 text.pack(expand=1, fill="both")
370 text.insert("insert", source)
371 text.focus_set()
372
373 color_config(text)
374 p = Percolator(text)
375 d = ColorDelegator()
376 p.insertfilter(d)
377
378
379 if __name__ == "__main__":
380 from unittest import main
381 main('idlelib.idle_test.test_colorizer', verbosity=2, exit=False)
382
383 from idlelib.idle_test.htest import run
384 run(_color_delegator)