1 from test.test_importlib import util
2
3 machinery = util.import_importlib('importlib.machinery')
4
5 import codecs
6 import importlib.util
7 import re
8 import types
9 # Because sys.path gets essentially blanked, need to have unicodedata already
10 # imported for the parser to use.
11 import unicodedata
12 import unittest
13 import warnings
14
15
16 CODING_RE = re.compile(r'^[ \t\f]*#.*?coding[:=][ \t]*([-\w.]+)', re.ASCII)
17
18
19 class ESC[4;38;5;81mEncodingTest:
20
21 """PEP 3120 makes UTF-8 the default encoding for source code
22 [default encoding].
23
24 PEP 263 specifies how that can change on a per-file basis. Either the first
25 or second line can contain the encoding line [encoding first line]
26 [encoding second line]. If the file has the BOM marker it is considered UTF-8
27 implicitly [BOM]. If any encoding is specified it must be UTF-8, else it is
28 an error [BOM and utf-8][BOM conflict].
29
30 """
31
32 variable = '\u00fc'
33 character = '\u00c9'
34 source_line = "{0} = '{1}'\n".format(variable, character)
35 module_name = '_temp'
36
37 def run_test(self, source):
38 with util.create_modules(self.module_name) as mapping:
39 with open(mapping[self.module_name], 'wb') as file:
40 file.write(source)
41 loader = self.machinery.SourceFileLoader(self.module_name,
42 mapping[self.module_name])
43 return self.load(loader)
44
45 def create_source(self, encoding):
46 encoding_line = "# coding={0}".format(encoding)
47 assert CODING_RE.match(encoding_line)
48 source_lines = [encoding_line.encode('utf-8')]
49 source_lines.append(self.source_line.encode(encoding))
50 return b'\n'.join(source_lines)
51
52 def test_non_obvious_encoding(self):
53 # Make sure that an encoding that has never been a standard one for
54 # Python works.
55 encoding_line = "# coding=koi8-r"
56 assert CODING_RE.match(encoding_line)
57 source = "{0}\na=42\n".format(encoding_line).encode("koi8-r")
58 self.run_test(source)
59
60 # [default encoding]
61 def test_default_encoding(self):
62 self.run_test(self.source_line.encode('utf-8'))
63
64 # [encoding first line]
65 def test_encoding_on_first_line(self):
66 encoding = 'Latin-1'
67 source = self.create_source(encoding)
68 self.run_test(source)
69
70 # [encoding second line]
71 def test_encoding_on_second_line(self):
72 source = b"#/usr/bin/python\n" + self.create_source('Latin-1')
73 self.run_test(source)
74
75 # [BOM]
76 def test_bom(self):
77 self.run_test(codecs.BOM_UTF8 + self.source_line.encode('utf-8'))
78
79 # [BOM and utf-8]
80 def test_bom_and_utf_8(self):
81 source = codecs.BOM_UTF8 + self.create_source('utf-8')
82 self.run_test(source)
83
84 # [BOM conflict]
85 def test_bom_conflict(self):
86 source = codecs.BOM_UTF8 + self.create_source('latin-1')
87 with self.assertRaises(SyntaxError):
88 self.run_test(source)
89
90
91 class ESC[4;38;5;81mEncodingTestPEP451(ESC[4;38;5;149mEncodingTest):
92
93 def load(self, loader):
94 module = types.ModuleType(self.module_name)
95 module.__spec__ = importlib.util.spec_from_loader(self.module_name, loader)
96 loader.exec_module(module)
97 return module
98
99
100 (Frozen_EncodingTestPEP451,
101 Source_EncodingTestPEP451
102 ) = util.test_both(EncodingTestPEP451, machinery=machinery)
103
104
105 class ESC[4;38;5;81mEncodingTestPEP302(ESC[4;38;5;149mEncodingTest):
106
107 def load(self, loader):
108 with warnings.catch_warnings():
109 warnings.simplefilter('ignore', DeprecationWarning)
110 return loader.load_module(self.module_name)
111
112
113 (Frozen_EncodingTestPEP302,
114 Source_EncodingTestPEP302
115 ) = util.test_both(EncodingTestPEP302, machinery=machinery)
116
117
118 class ESC[4;38;5;81mLineEndingTest:
119
120 r"""Source written with the three types of line endings (\n, \r\n, \r)
121 need to be readable [cr][crlf][lf]."""
122
123 def run_test(self, line_ending):
124 module_name = '_temp'
125 source_lines = [b"a = 42", b"b = -13", b'']
126 source = line_ending.join(source_lines)
127 with util.create_modules(module_name) as mapping:
128 with open(mapping[module_name], 'wb') as file:
129 file.write(source)
130 loader = self.machinery.SourceFileLoader(module_name,
131 mapping[module_name])
132 return self.load(loader, module_name)
133
134 # [cr]
135 def test_cr(self):
136 self.run_test(b'\r')
137
138 # [crlf]
139 def test_crlf(self):
140 self.run_test(b'\r\n')
141
142 # [lf]
143 def test_lf(self):
144 self.run_test(b'\n')
145
146
147 class ESC[4;38;5;81mLineEndingTestPEP451(ESC[4;38;5;149mLineEndingTest):
148
149 def load(self, loader, module_name):
150 module = types.ModuleType(module_name)
151 module.__spec__ = importlib.util.spec_from_loader(module_name, loader)
152 loader.exec_module(module)
153 return module
154
155
156 (Frozen_LineEndingTestPEP451,
157 Source_LineEndingTestPEP451
158 ) = util.test_both(LineEndingTestPEP451, machinery=machinery)
159
160
161 class ESC[4;38;5;81mLineEndingTestPEP302(ESC[4;38;5;149mLineEndingTest):
162
163 def load(self, loader, module_name):
164 with warnings.catch_warnings():
165 warnings.simplefilter('ignore', DeprecationWarning)
166 return loader.load_module(module_name)
167
168
169 (Frozen_LineEndingTestPEP302,
170 Source_LineEndingTestPEP302
171 ) = util.test_both(LineEndingTestPEP302, machinery=machinery)
172
173
174 if __name__ == '__main__':
175 unittest.main()