1 """Tests to cover the Tools/i18n package"""
2
3 import os
4 import sys
5 import unittest
6 from textwrap import dedent
7
8 from test.support.script_helper import assert_python_ok
9 from test.test_tools import skip_if_missing, toolsdir
10 from test.support.os_helper import temp_cwd, temp_dir
11
12
13 skip_if_missing()
14
15
16 class ESC[4;38;5;81mTest_pygettext(ESC[4;38;5;149munittestESC[4;38;5;149m.ESC[4;38;5;149mTestCase):
17 """Tests for the pygettext.py tool"""
18
19 script = os.path.join(toolsdir,'i18n', 'pygettext.py')
20
21 def get_header(self, data):
22 """ utility: return the header of a .po file as a dictionary """
23 headers = {}
24 for line in data.split('\n'):
25 if not line or line.startswith(('#', 'msgid','msgstr')):
26 continue
27 line = line.strip('"')
28 key, val = line.split(':',1)
29 headers[key] = val.strip()
30 return headers
31
32 def get_msgids(self, data):
33 """ utility: return all msgids in .po file as a list of strings """
34 msgids = []
35 reading_msgid = False
36 cur_msgid = []
37 for line in data.split('\n'):
38 if reading_msgid:
39 if line.startswith('"'):
40 cur_msgid.append(line.strip('"'))
41 else:
42 msgids.append('\n'.join(cur_msgid))
43 cur_msgid = []
44 reading_msgid = False
45 continue
46 if line.startswith('msgid '):
47 line = line[len('msgid '):]
48 cur_msgid.append(line.strip('"'))
49 reading_msgid = True
50 else:
51 if reading_msgid:
52 msgids.append('\n'.join(cur_msgid))
53
54 return msgids
55
56 def extract_docstrings_from_str(self, module_content):
57 """ utility: return all msgids extracted from module_content """
58 filename = 'test_docstrings.py'
59 with temp_cwd(None) as cwd:
60 with open(filename, 'w', encoding='utf-8') as fp:
61 fp.write(module_content)
62 assert_python_ok(self.script, '-D', filename)
63 with open('messages.pot', encoding='utf-8') as fp:
64 data = fp.read()
65 return self.get_msgids(data)
66
67 def test_header(self):
68 """Make sure the required fields are in the header, according to:
69 http://www.gnu.org/software/gettext/manual/gettext.html#Header-Entry
70 """
71 with temp_cwd(None) as cwd:
72 assert_python_ok(self.script)
73 with open('messages.pot', encoding='utf-8') as fp:
74 data = fp.read()
75 header = self.get_header(data)
76
77 self.assertIn("Project-Id-Version", header)
78 self.assertIn("POT-Creation-Date", header)
79 self.assertIn("PO-Revision-Date", header)
80 self.assertIn("Last-Translator", header)
81 self.assertIn("Language-Team", header)
82 self.assertIn("MIME-Version", header)
83 self.assertIn("Content-Type", header)
84 self.assertIn("Content-Transfer-Encoding", header)
85 self.assertIn("Generated-By", header)
86
87 # not clear if these should be required in POT (template) files
88 #self.assertIn("Report-Msgid-Bugs-To", header)
89 #self.assertIn("Language", header)
90
91 #"Plural-Forms" is optional
92
93 @unittest.skipIf(sys.platform.startswith('aix'),
94 'bpo-29972: broken test on AIX')
95 def test_POT_Creation_Date(self):
96 """ Match the date format from xgettext for POT-Creation-Date """
97 from datetime import datetime
98 with temp_cwd(None) as cwd:
99 assert_python_ok(self.script)
100 with open('messages.pot', encoding='utf-8') as fp:
101 data = fp.read()
102 header = self.get_header(data)
103 creationDate = header['POT-Creation-Date']
104
105 # peel off the escaped newline at the end of string
106 if creationDate.endswith('\\n'):
107 creationDate = creationDate[:-len('\\n')]
108
109 # This will raise if the date format does not exactly match.
110 datetime.strptime(creationDate, '%Y-%m-%d %H:%M%z')
111
112 def test_funcdocstring(self):
113 for doc in ('"""doc"""', "r'''doc'''", "R'doc'", 'u"doc"'):
114 with self.subTest(doc):
115 msgids = self.extract_docstrings_from_str(dedent('''\
116 def foo(bar):
117 %s
118 ''' % doc))
119 self.assertIn('doc', msgids)
120
121 def test_funcdocstring_bytes(self):
122 msgids = self.extract_docstrings_from_str(dedent('''\
123 def foo(bar):
124 b"""doc"""
125 '''))
126 self.assertFalse([msgid for msgid in msgids if 'doc' in msgid])
127
128 def test_funcdocstring_fstring(self):
129 msgids = self.extract_docstrings_from_str(dedent('''\
130 def foo(bar):
131 f"""doc"""
132 '''))
133 self.assertFalse([msgid for msgid in msgids if 'doc' in msgid])
134
135 def test_classdocstring(self):
136 for doc in ('"""doc"""', "r'''doc'''", "R'doc'", 'u"doc"'):
137 with self.subTest(doc):
138 msgids = self.extract_docstrings_from_str(dedent('''\
139 class C:
140 %s
141 ''' % doc))
142 self.assertIn('doc', msgids)
143
144 def test_classdocstring_bytes(self):
145 msgids = self.extract_docstrings_from_str(dedent('''\
146 class C:
147 b"""doc"""
148 '''))
149 self.assertFalse([msgid for msgid in msgids if 'doc' in msgid])
150
151 def test_classdocstring_fstring(self):
152 msgids = self.extract_docstrings_from_str(dedent('''\
153 class C:
154 f"""doc"""
155 '''))
156 self.assertFalse([msgid for msgid in msgids if 'doc' in msgid])
157
158 def test_moduledocstring(self):
159 for doc in ('"""doc"""', "r'''doc'''", "R'doc'", 'u"doc"'):
160 with self.subTest(doc):
161 msgids = self.extract_docstrings_from_str(dedent('''\
162 %s
163 ''' % doc))
164 self.assertIn('doc', msgids)
165
166 def test_moduledocstring_bytes(self):
167 msgids = self.extract_docstrings_from_str(dedent('''\
168 b"""doc"""
169 '''))
170 self.assertFalse([msgid for msgid in msgids if 'doc' in msgid])
171
172 def test_moduledocstring_fstring(self):
173 msgids = self.extract_docstrings_from_str(dedent('''\
174 f"""doc"""
175 '''))
176 self.assertFalse([msgid for msgid in msgids if 'doc' in msgid])
177
178 def test_msgid(self):
179 msgids = self.extract_docstrings_from_str(
180 '''_("""doc""" r'str' u"ing")''')
181 self.assertIn('docstring', msgids)
182
183 def test_msgid_bytes(self):
184 msgids = self.extract_docstrings_from_str('_(b"""doc""")')
185 self.assertFalse([msgid for msgid in msgids if 'doc' in msgid])
186
187 def test_msgid_fstring(self):
188 msgids = self.extract_docstrings_from_str('_(f"""doc""")')
189 self.assertFalse([msgid for msgid in msgids if 'doc' in msgid])
190
191 def test_funcdocstring_annotated_args(self):
192 """ Test docstrings for functions with annotated args """
193 msgids = self.extract_docstrings_from_str(dedent('''\
194 def foo(bar: str):
195 """doc"""
196 '''))
197 self.assertIn('doc', msgids)
198
199 def test_funcdocstring_annotated_return(self):
200 """ Test docstrings for functions with annotated return type """
201 msgids = self.extract_docstrings_from_str(dedent('''\
202 def foo(bar) -> str:
203 """doc"""
204 '''))
205 self.assertIn('doc', msgids)
206
207 def test_funcdocstring_defvalue_args(self):
208 """ Test docstring for functions with default arg values """
209 msgids = self.extract_docstrings_from_str(dedent('''\
210 def foo(bar=()):
211 """doc"""
212 '''))
213 self.assertIn('doc', msgids)
214
215 def test_funcdocstring_multiple_funcs(self):
216 """ Test docstring extraction for multiple functions combining
217 annotated args, annotated return types and default arg values
218 """
219 msgids = self.extract_docstrings_from_str(dedent('''\
220 def foo1(bar: tuple=()) -> str:
221 """doc1"""
222
223 def foo2(bar: List[1:2]) -> (lambda x: x):
224 """doc2"""
225
226 def foo3(bar: 'func'=lambda x: x) -> {1: 2}:
227 """doc3"""
228 '''))
229 self.assertIn('doc1', msgids)
230 self.assertIn('doc2', msgids)
231 self.assertIn('doc3', msgids)
232
233 def test_classdocstring_early_colon(self):
234 """ Test docstring extraction for a class with colons occurring within
235 the parentheses.
236 """
237 msgids = self.extract_docstrings_from_str(dedent('''\
238 class D(L[1:2], F({1: 2}), metaclass=M(lambda x: x)):
239 """doc"""
240 '''))
241 self.assertIn('doc', msgids)
242
243 def test_calls_in_fstrings(self):
244 msgids = self.extract_docstrings_from_str(dedent('''\
245 f"{_('foo bar')}"
246 '''))
247 self.assertIn('foo bar', msgids)
248
249 def test_calls_in_fstrings_raw(self):
250 msgids = self.extract_docstrings_from_str(dedent('''\
251 rf"{_('foo bar')}"
252 '''))
253 self.assertIn('foo bar', msgids)
254
255 def test_calls_in_fstrings_nested(self):
256 msgids = self.extract_docstrings_from_str(dedent('''\
257 f"""{f'{_("foo bar")}'}"""
258 '''))
259 self.assertIn('foo bar', msgids)
260
261 def test_calls_in_fstrings_attribute(self):
262 msgids = self.extract_docstrings_from_str(dedent('''\
263 f"{obj._('foo bar')}"
264 '''))
265 self.assertIn('foo bar', msgids)
266
267 def test_calls_in_fstrings_with_call_on_call(self):
268 msgids = self.extract_docstrings_from_str(dedent('''\
269 f"{type(str)('foo bar')}"
270 '''))
271 self.assertNotIn('foo bar', msgids)
272
273 def test_calls_in_fstrings_with_format(self):
274 msgids = self.extract_docstrings_from_str(dedent('''\
275 f"{_('foo {bar}').format(bar='baz')}"
276 '''))
277 self.assertIn('foo {bar}', msgids)
278
279 def test_calls_in_fstrings_with_wrong_input_1(self):
280 msgids = self.extract_docstrings_from_str(dedent('''\
281 f"{_(f'foo {bar}')}"
282 '''))
283 self.assertFalse([msgid for msgid in msgids if 'foo {bar}' in msgid])
284
285 def test_calls_in_fstrings_with_wrong_input_2(self):
286 msgids = self.extract_docstrings_from_str(dedent('''\
287 f"{_(1)}"
288 '''))
289 self.assertNotIn(1, msgids)
290
291 def test_calls_in_fstring_with_multiple_args(self):
292 msgids = self.extract_docstrings_from_str(dedent('''\
293 f"{_('foo', 'bar')}"
294 '''))
295 self.assertNotIn('foo', msgids)
296 self.assertNotIn('bar', msgids)
297
298 def test_calls_in_fstring_with_keyword_args(self):
299 msgids = self.extract_docstrings_from_str(dedent('''\
300 f"{_('foo', bar='baz')}"
301 '''))
302 self.assertNotIn('foo', msgids)
303 self.assertNotIn('bar', msgids)
304 self.assertNotIn('baz', msgids)
305
306 def test_calls_in_fstring_with_partially_wrong_expression(self):
307 msgids = self.extract_docstrings_from_str(dedent('''\
308 f"{_(f'foo') + _('bar')}"
309 '''))
310 self.assertNotIn('foo', msgids)
311 self.assertIn('bar', msgids)
312
313 def test_files_list(self):
314 """Make sure the directories are inspected for source files
315 bpo-31920
316 """
317 text1 = 'Text to translate1'
318 text2 = 'Text to translate2'
319 text3 = 'Text to ignore'
320 with temp_cwd(None), temp_dir(None) as sdir:
321 os.mkdir(os.path.join(sdir, 'pypkg'))
322 with open(os.path.join(sdir, 'pypkg', 'pymod.py'), 'w',
323 encoding='utf-8') as sfile:
324 sfile.write(f'_({text1!r})')
325 os.mkdir(os.path.join(sdir, 'pkg.py'))
326 with open(os.path.join(sdir, 'pkg.py', 'pymod2.py'), 'w',
327 encoding='utf-8') as sfile:
328 sfile.write(f'_({text2!r})')
329 os.mkdir(os.path.join(sdir, 'CVS'))
330 with open(os.path.join(sdir, 'CVS', 'pymod3.py'), 'w',
331 encoding='utf-8') as sfile:
332 sfile.write(f'_({text3!r})')
333 assert_python_ok(self.script, sdir)
334 with open('messages.pot', encoding='utf-8') as fp:
335 data = fp.read()
336 self.assertIn(f'msgid "{text1}"', data)
337 self.assertIn(f'msgid "{text2}"', data)
338 self.assertNotIn(text3, data)