python (3.11.7)
1 # unicode.py
2
3 import sys
4 from itertools import filterfalse
5 from typing import List, Tuple, Union
6
7
8 class ESC[4;38;5;81m_lazyclassproperty:
9 def __init__(self, fn):
10 self.fn = fn
11 self.__doc__ = fn.__doc__
12 self.__name__ = fn.__name__
13
14 def __get__(self, obj, cls):
15 if cls is None:
16 cls = type(obj)
17 if not hasattr(cls, "_intern") or any(
18 cls._intern is getattr(superclass, "_intern", [])
19 for superclass in cls.__mro__[1:]
20 ):
21 cls._intern = {}
22 attrname = self.fn.__name__
23 if attrname not in cls._intern:
24 cls._intern[attrname] = self.fn(cls)
25 return cls._intern[attrname]
26
27
28 UnicodeRangeList = List[Union[Tuple[int, int], Tuple[int]]]
29
30
31 class ESC[4;38;5;81municode_set:
32 """
33 A set of Unicode characters, for language-specific strings for
34 ``alphas``, ``nums``, ``alphanums``, and ``printables``.
35 A unicode_set is defined by a list of ranges in the Unicode character
36 set, in a class attribute ``_ranges``. Ranges can be specified using
37 2-tuples or a 1-tuple, such as::
38
39 _ranges = [
40 (0x0020, 0x007e),
41 (0x00a0, 0x00ff),
42 (0x0100,),
43 ]
44
45 Ranges are left- and right-inclusive. A 1-tuple of (x,) is treated as (x, x).
46
47 A unicode set can also be defined using multiple inheritance of other unicode sets::
48
49 class CJK(Chinese, Japanese, Korean):
50 pass
51 """
52
53 _ranges: UnicodeRangeList = []
54
55 @_lazyclassproperty
56 def _chars_for_ranges(cls):
57 ret = []
58 for cc in cls.__mro__:
59 if cc is unicode_set:
60 break
61 for rr in getattr(cc, "_ranges", ()):
62 ret.extend(range(rr[0], rr[-1] + 1))
63 return [chr(c) for c in sorted(set(ret))]
64
65 @_lazyclassproperty
66 def printables(cls):
67 "all non-whitespace characters in this range"
68 return "".join(filterfalse(str.isspace, cls._chars_for_ranges))
69
70 @_lazyclassproperty
71 def alphas(cls):
72 "all alphabetic characters in this range"
73 return "".join(filter(str.isalpha, cls._chars_for_ranges))
74
75 @_lazyclassproperty
76 def nums(cls):
77 "all numeric digit characters in this range"
78 return "".join(filter(str.isdigit, cls._chars_for_ranges))
79
80 @_lazyclassproperty
81 def alphanums(cls):
82 "all alphanumeric characters in this range"
83 return cls.alphas + cls.nums
84
85 @_lazyclassproperty
86 def identchars(cls):
87 "all characters in this range that are valid identifier characters, plus underscore '_'"
88 return "".join(
89 sorted(
90 set(
91 "".join(filter(str.isidentifier, cls._chars_for_ranges))
92 + "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyzªµº"
93 + "ÀÁÂÃÄÅÆÇÈÉÊËÌÍÎÏÐÑÒÓÔÕÖØÙÚÛÜÝÞßàáâãäåæçèéêëìíîïðñòóôõöøùúûüýþÿ"
94 + "_"
95 )
96 )
97 )
98
99 @_lazyclassproperty
100 def identbodychars(cls):
101 """
102 all characters in this range that are valid identifier body characters,
103 plus the digits 0-9
104 """
105 return "".join(
106 sorted(
107 set(
108 cls.identchars
109 + "0123456789"
110 + "".join(
111 [c for c in cls._chars_for_ranges if ("_" + c).isidentifier()]
112 )
113 )
114 )
115 )
116
117
118 class ESC[4;38;5;81mpyparsing_unicode(ESC[4;38;5;149municode_set):
119 """
120 A namespace class for defining common language unicode_sets.
121 """
122
123 # fmt: off
124
125 # define ranges in language character sets
126 _ranges: UnicodeRangeList = [
127 (0x0020, sys.maxunicode),
128 ]
129
130 class ESC[4;38;5;81mBasicMultilingualPlane(ESC[4;38;5;149municode_set):
131 "Unicode set for the Basic Multilingual Plane"
132 _ranges: UnicodeRangeList = [
133 (0x0020, 0xFFFF),
134 ]
135
136 class ESC[4;38;5;81mLatin1(ESC[4;38;5;149municode_set):
137 "Unicode set for Latin-1 Unicode Character Range"
138 _ranges: UnicodeRangeList = [
139 (0x0020, 0x007E),
140 (0x00A0, 0x00FF),
141 ]
142
143 class ESC[4;38;5;81mLatinA(ESC[4;38;5;149municode_set):
144 "Unicode set for Latin-A Unicode Character Range"
145 _ranges: UnicodeRangeList = [
146 (0x0100, 0x017F),
147 ]
148
149 class ESC[4;38;5;81mLatinB(ESC[4;38;5;149municode_set):
150 "Unicode set for Latin-B Unicode Character Range"
151 _ranges: UnicodeRangeList = [
152 (0x0180, 0x024F),
153 ]
154
155 class ESC[4;38;5;81mGreek(ESC[4;38;5;149municode_set):
156 "Unicode set for Greek Unicode Character Ranges"
157 _ranges: UnicodeRangeList = [
158 (0x0342, 0x0345),
159 (0x0370, 0x0377),
160 (0x037A, 0x037F),
161 (0x0384, 0x038A),
162 (0x038C,),
163 (0x038E, 0x03A1),
164 (0x03A3, 0x03E1),
165 (0x03F0, 0x03FF),
166 (0x1D26, 0x1D2A),
167 (0x1D5E,),
168 (0x1D60,),
169 (0x1D66, 0x1D6A),
170 (0x1F00, 0x1F15),
171 (0x1F18, 0x1F1D),
172 (0x1F20, 0x1F45),
173 (0x1F48, 0x1F4D),
174 (0x1F50, 0x1F57),
175 (0x1F59,),
176 (0x1F5B,),
177 (0x1F5D,),
178 (0x1F5F, 0x1F7D),
179 (0x1F80, 0x1FB4),
180 (0x1FB6, 0x1FC4),
181 (0x1FC6, 0x1FD3),
182 (0x1FD6, 0x1FDB),
183 (0x1FDD, 0x1FEF),
184 (0x1FF2, 0x1FF4),
185 (0x1FF6, 0x1FFE),
186 (0x2129,),
187 (0x2719, 0x271A),
188 (0xAB65,),
189 (0x10140, 0x1018D),
190 (0x101A0,),
191 (0x1D200, 0x1D245),
192 (0x1F7A1, 0x1F7A7),
193 ]
194
195 class ESC[4;38;5;81mCyrillic(ESC[4;38;5;149municode_set):
196 "Unicode set for Cyrillic Unicode Character Range"
197 _ranges: UnicodeRangeList = [
198 (0x0400, 0x052F),
199 (0x1C80, 0x1C88),
200 (0x1D2B,),
201 (0x1D78,),
202 (0x2DE0, 0x2DFF),
203 (0xA640, 0xA672),
204 (0xA674, 0xA69F),
205 (0xFE2E, 0xFE2F),
206 ]
207
208 class ESC[4;38;5;81mChinese(ESC[4;38;5;149municode_set):
209 "Unicode set for Chinese Unicode Character Range"
210 _ranges: UnicodeRangeList = [
211 (0x2E80, 0x2E99),
212 (0x2E9B, 0x2EF3),
213 (0x31C0, 0x31E3),
214 (0x3400, 0x4DB5),
215 (0x4E00, 0x9FEF),
216 (0xA700, 0xA707),
217 (0xF900, 0xFA6D),
218 (0xFA70, 0xFAD9),
219 (0x16FE2, 0x16FE3),
220 (0x1F210, 0x1F212),
221 (0x1F214, 0x1F23B),
222 (0x1F240, 0x1F248),
223 (0x20000, 0x2A6D6),
224 (0x2A700, 0x2B734),
225 (0x2B740, 0x2B81D),
226 (0x2B820, 0x2CEA1),
227 (0x2CEB0, 0x2EBE0),
228 (0x2F800, 0x2FA1D),
229 ]
230
231 class ESC[4;38;5;81mJapanese(ESC[4;38;5;149municode_set):
232 "Unicode set for Japanese Unicode Character Range, combining Kanji, Hiragana, and Katakana ranges"
233 _ranges: UnicodeRangeList = []
234
235 class ESC[4;38;5;81mKanji(ESC[4;38;5;149municode_set):
236 "Unicode set for Kanji Unicode Character Range"
237 _ranges: UnicodeRangeList = [
238 (0x4E00, 0x9FBF),
239 (0x3000, 0x303F),
240 ]
241
242 class ESC[4;38;5;81mHiragana(ESC[4;38;5;149municode_set):
243 "Unicode set for Hiragana Unicode Character Range"
244 _ranges: UnicodeRangeList = [
245 (0x3041, 0x3096),
246 (0x3099, 0x30A0),
247 (0x30FC,),
248 (0xFF70,),
249 (0x1B001,),
250 (0x1B150, 0x1B152),
251 (0x1F200,),
252 ]
253
254 class ESC[4;38;5;81mKatakana(ESC[4;38;5;149municode_set):
255 "Unicode set for Katakana Unicode Character Range"
256 _ranges: UnicodeRangeList = [
257 (0x3099, 0x309C),
258 (0x30A0, 0x30FF),
259 (0x31F0, 0x31FF),
260 (0x32D0, 0x32FE),
261 (0xFF65, 0xFF9F),
262 (0x1B000,),
263 (0x1B164, 0x1B167),
264 (0x1F201, 0x1F202),
265 (0x1F213,),
266 ]
267
268 class ESC[4;38;5;81mHangul(ESC[4;38;5;149municode_set):
269 "Unicode set for Hangul (Korean) Unicode Character Range"
270 _ranges: UnicodeRangeList = [
271 (0x1100, 0x11FF),
272 (0x302E, 0x302F),
273 (0x3131, 0x318E),
274 (0x3200, 0x321C),
275 (0x3260, 0x327B),
276 (0x327E,),
277 (0xA960, 0xA97C),
278 (0xAC00, 0xD7A3),
279 (0xD7B0, 0xD7C6),
280 (0xD7CB, 0xD7FB),
281 (0xFFA0, 0xFFBE),
282 (0xFFC2, 0xFFC7),
283 (0xFFCA, 0xFFCF),
284 (0xFFD2, 0xFFD7),
285 (0xFFDA, 0xFFDC),
286 ]
287
288 Korean = Hangul
289
290 class ESC[4;38;5;81mCJK(ESC[4;38;5;149mChinese, ESC[4;38;5;149mJapanese, ESC[4;38;5;149mHangul):
291 "Unicode set for combined Chinese, Japanese, and Korean (CJK) Unicode Character Range"
292
293 class ESC[4;38;5;81mThai(ESC[4;38;5;149municode_set):
294 "Unicode set for Thai Unicode Character Range"
295 _ranges: UnicodeRangeList = [
296 (0x0E01, 0x0E3A),
297 (0x0E3F, 0x0E5B)
298 ]
299
300 class ESC[4;38;5;81mArabic(ESC[4;38;5;149municode_set):
301 "Unicode set for Arabic Unicode Character Range"
302 _ranges: UnicodeRangeList = [
303 (0x0600, 0x061B),
304 (0x061E, 0x06FF),
305 (0x0700, 0x077F),
306 ]
307
308 class ESC[4;38;5;81mHebrew(ESC[4;38;5;149municode_set):
309 "Unicode set for Hebrew Unicode Character Range"
310 _ranges: UnicodeRangeList = [
311 (0x0591, 0x05C7),
312 (0x05D0, 0x05EA),
313 (0x05EF, 0x05F4),
314 (0xFB1D, 0xFB36),
315 (0xFB38, 0xFB3C),
316 (0xFB3E,),
317 (0xFB40, 0xFB41),
318 (0xFB43, 0xFB44),
319 (0xFB46, 0xFB4F),
320 ]
321
322 class ESC[4;38;5;81mDevanagari(ESC[4;38;5;149municode_set):
323 "Unicode set for Devanagari Unicode Character Range"
324 _ranges: UnicodeRangeList = [
325 (0x0900, 0x097F),
326 (0xA8E0, 0xA8FF)
327 ]
328
329 # fmt: on
330
331
332 pyparsing_unicode.Japanese._ranges = (
333 pyparsing_unicode.Japanese.Kanji._ranges
334 + pyparsing_unicode.Japanese.Hiragana._ranges
335 + pyparsing_unicode.Japanese.Katakana._ranges
336 )
337
338 pyparsing_unicode.BMP = pyparsing_unicode.BasicMultilingualPlane
339
340 # add language identifiers using language Unicode
341 pyparsing_unicode.العربية = pyparsing_unicode.Arabic
342 pyparsing_unicode.中文 = pyparsing_unicode.Chinese
343 pyparsing_unicode.кириллица = pyparsing_unicode.Cyrillic
344 pyparsing_unicode.Ελληνικά = pyparsing_unicode.Greek
345 pyparsing_unicode.עִברִית = pyparsing_unicode.Hebrew
346 pyparsing_unicode.日本語 = pyparsing_unicode.Japanese
347 pyparsing_unicode.Japanese.漢字 = pyparsing_unicode.Japanese.Kanji
348 pyparsing_unicode.Japanese.カタカナ = pyparsing_unicode.Japanese.Katakana
349 pyparsing_unicode.Japanese.ひらがな = pyparsing_unicode.Japanese.Hiragana
350 pyparsing_unicode.한국어 = pyparsing_unicode.Korean
351 pyparsing_unicode.ไทย = pyparsing_unicode.Thai
352 pyparsing_unicode.देवनागरी = pyparsing_unicode.Devanagari
Browse
Usage
Versions