1 #!/usr/bin/env python3
2 import sys
3 import time
4 import os
5 sys.path.insert(0, "python")
6 import libxml2
7
8 test_nr = 0
9 test_succeed = 0
10 test_failed = 0
11 test_error = 0
12
13 #
14 # the testsuite description
15 #
16 CONF="xml-test-suite/xmlconf/xmlconf.xml"
17 LOG="check-xml-test-suite.log"
18
19 log = open(LOG, "w")
20
21 #
22 # Error and warning handlers
23 #
24 error_nr = 0
25 error_msg = ''
26 def errorHandler(ctx, str):
27 global error_nr
28 global error_msg
29
30 error_nr = error_nr + 1
31 if len(error_msg) < 300:
32 if len(error_msg) == 0 or error_msg[-1] == '\n':
33 error_msg = error_msg + " >>" + str
34 else:
35 error_msg = error_msg + str
36
37 libxml2.registerErrorHandler(errorHandler, None)
38
39 #warning_nr = 0
40 #warning = ''
41 #def warningHandler(ctx, str):
42 # global warning_nr
43 # global warning
44 #
45 # warning_nr = warning_nr + 1
46 # warning = warning + str
47 #
48 #libxml2.registerWarningHandler(warningHandler, None)
49
50 #
51 # Used to load the XML testsuite description
52 #
53 def loadNoentDoc(filename):
54 ctxt = libxml2.createFileParserCtxt(filename)
55 if ctxt is None:
56 return None
57 ctxt.replaceEntities(1)
58 ctxt.parseDocument()
59 try:
60 doc = ctxt.doc()
61 except:
62 doc = None
63 if ctxt.wellFormed() != 1:
64 doc.freeDoc()
65 return None
66 return doc
67
68 #
69 # The conformance testing routines
70 #
71
72 def testNotWf(filename, id):
73 global error_nr
74 global error_msg
75 global log
76
77 error_nr = 0
78 error_msg = ''
79
80 ctxt = libxml2.createFileParserCtxt(filename)
81 if ctxt is None:
82 return -1
83 ret = ctxt.parseDocument()
84
85 try:
86 doc = ctxt.doc()
87 except:
88 doc = None
89 if doc != None:
90 doc.freeDoc()
91 if ret == 0 or ctxt.wellFormed() != 0:
92 print("%s: error: Well Formedness error not detected" % (id))
93 log.write("%s: error: Well Formedness error not detected\n" % (id))
94 return 0
95 return 1
96
97 def testNotWfEnt(filename, id):
98 global error_nr
99 global error_msg
100 global log
101
102 error_nr = 0
103 error_msg = ''
104
105 ctxt = libxml2.createFileParserCtxt(filename)
106 if ctxt is None:
107 return -1
108 ctxt.replaceEntities(1)
109 ret = ctxt.parseDocument()
110
111 try:
112 doc = ctxt.doc()
113 except:
114 doc = None
115 if doc != None:
116 doc.freeDoc()
117 if ret == 0 or ctxt.wellFormed() != 0:
118 print("%s: error: Well Formedness error not detected" % (id))
119 log.write("%s: error: Well Formedness error not detected\n" % (id))
120 return 0
121 return 1
122
123 def testNotWfEntDtd(filename, id):
124 global error_nr
125 global error_msg
126 global log
127
128 error_nr = 0
129 error_msg = ''
130
131 ctxt = libxml2.createFileParserCtxt(filename)
132 if ctxt is None:
133 return -1
134 ctxt.replaceEntities(1)
135 ctxt.loadSubset(1)
136 ret = ctxt.parseDocument()
137
138 try:
139 doc = ctxt.doc()
140 except:
141 doc = None
142 if doc != None:
143 doc.freeDoc()
144 if ret == 0 or ctxt.wellFormed() != 0:
145 print("%s: error: Well Formedness error not detected" % (id))
146 log.write("%s: error: Well Formedness error not detected\n" % (id))
147 return 0
148 return 1
149
150 def testWfEntDtd(filename, id):
151 global error_nr
152 global error_msg
153 global log
154
155 error_nr = 0
156 error_msg = ''
157
158 ctxt = libxml2.createFileParserCtxt(filename)
159 if ctxt is None:
160 return -1
161 ctxt.replaceEntities(1)
162 ctxt.loadSubset(1)
163 ret = ctxt.parseDocument()
164
165 try:
166 doc = ctxt.doc()
167 except:
168 doc = None
169 if doc is None or ret != 0 or ctxt.wellFormed() == 0:
170 print("%s: error: wrongly failed to parse the document" % (id))
171 log.write("%s: error: wrongly failed to parse the document\n" % (id))
172 if doc != None:
173 doc.freeDoc()
174 return 0
175 if error_nr != 0:
176 print("%s: warning: WF document generated an error msg" % (id))
177 log.write("%s: error: WF document generated an error msg\n" % (id))
178 doc.freeDoc()
179 return 2
180 doc.freeDoc()
181 return 1
182
183 def testError(filename, id):
184 global error_nr
185 global error_msg
186 global log
187
188 error_nr = 0
189 error_msg = ''
190
191 ctxt = libxml2.createFileParserCtxt(filename)
192 if ctxt is None:
193 return -1
194 ctxt.replaceEntities(1)
195 ctxt.loadSubset(1)
196 ret = ctxt.parseDocument()
197
198 try:
199 doc = ctxt.doc()
200 except:
201 doc = None
202 if doc != None:
203 doc.freeDoc()
204 if ctxt.wellFormed() == 0:
205 print("%s: warning: failed to parse the document but accepted" % (id))
206 log.write("%s: warning: failed to parse the document but accepte\n" % (id))
207 return 2
208 if error_nr != 0:
209 print("%s: warning: WF document generated an error msg" % (id))
210 log.write("%s: error: WF document generated an error msg\n" % (id))
211 return 2
212 return 1
213
214 def testInvalid(filename, id):
215 global error_nr
216 global error_msg
217 global log
218
219 error_nr = 0
220 error_msg = ''
221
222 ctxt = libxml2.createFileParserCtxt(filename)
223 if ctxt is None:
224 return -1
225 ctxt.validate(1)
226 ret = ctxt.parseDocument()
227
228 try:
229 doc = ctxt.doc()
230 except:
231 doc = None
232 valid = ctxt.isValid()
233 if doc is None:
234 print("%s: error: wrongly failed to parse the document" % (id))
235 log.write("%s: error: wrongly failed to parse the document\n" % (id))
236 return 0
237 if valid == 1:
238 print("%s: error: Validity error not detected" % (id))
239 log.write("%s: error: Validity error not detected\n" % (id))
240 doc.freeDoc()
241 return 0
242 if error_nr == 0:
243 print("%s: warning: Validity error not reported" % (id))
244 log.write("%s: warning: Validity error not reported\n" % (id))
245 doc.freeDoc()
246 return 2
247
248 doc.freeDoc()
249 return 1
250
251 def testValid(filename, id):
252 global error_nr
253 global error_msg
254
255 error_nr = 0
256 error_msg = ''
257
258 ctxt = libxml2.createFileParserCtxt(filename)
259 if ctxt is None:
260 return -1
261 ctxt.validate(1)
262 ctxt.parseDocument()
263
264 try:
265 doc = ctxt.doc()
266 except:
267 doc = None
268 valid = ctxt.isValid()
269 if doc is None:
270 print("%s: error: wrongly failed to parse the document" % (id))
271 log.write("%s: error: wrongly failed to parse the document\n" % (id))
272 return 0
273 if valid != 1:
274 print("%s: error: Validity check failed" % (id))
275 log.write("%s: error: Validity check failed\n" % (id))
276 doc.freeDoc()
277 return 0
278 if error_nr != 0 or valid != 1:
279 print("%s: warning: valid document reported an error" % (id))
280 log.write("%s: warning: valid document reported an error\n" % (id))
281 doc.freeDoc()
282 return 2
283 doc.freeDoc()
284 return 1
285
286 def runTest(test):
287 global test_nr
288 global test_succeed
289 global test_failed
290 global error_msg
291 global log
292
293 uri = test.prop('URI')
294 id = test.prop('ID')
295 if uri is None:
296 print("Test without ID:", uri)
297 return -1
298 if id is None:
299 print("Test without URI:", id)
300 return -1
301 base = test.getBase(None)
302 URI = libxml2.buildURI(uri, base)
303 if os.access(URI, os.R_OK) == 0:
304 print("Test %s missing: base %s uri %s" % (URI, base, uri))
305 return -1
306 type = test.prop('TYPE')
307 if type is None:
308 print("Test %s missing TYPE" % (id))
309 return -1
310
311 extra = None
312 if type == "invalid":
313 res = testInvalid(URI, id)
314 elif type == "valid":
315 res = testValid(URI, id)
316 elif type == "not-wf":
317 extra = test.prop('ENTITIES')
318 # print(URI)
319 #if extra is None:
320 # res = testNotWfEntDtd(URI, id)
321 #elif extra == 'none':
322 # res = testNotWf(URI, id)
323 #elif extra == 'general':
324 # res = testNotWfEnt(URI, id)
325 #elif extra == 'both' or extra == 'parameter':
326 res = testNotWfEntDtd(URI, id)
327 #else:
328 # print("Unknown value %s for an ENTITIES test value" % (extra))
329 # return -1
330 elif type == "error":
331 res = testError(URI, id)
332 else:
333 # TODO skipped for now
334 return -1
335
336 test_nr = test_nr + 1
337 if res > 0:
338 test_succeed = test_succeed + 1
339 elif res == 0:
340 test_failed = test_failed + 1
341 elif res < 0:
342 test_error = test_error + 1
343
344 # Log the ontext
345 if res != 1:
346 log.write(" File: %s\n" % (URI))
347 content = test.content.strip()
348 while content[-1] == '\n':
349 content = content[0:-1]
350 if extra != None:
351 log.write(" %s:%s:%s\n" % (type, extra, content))
352 else:
353 log.write(" %s:%s\n\n" % (type, content))
354 if error_msg != '':
355 log.write(" ----\n%s ----\n" % (error_msg))
356 error_msg = ''
357 log.write("\n")
358
359 return 0
360
361
362 def runTestCases(case):
363 profile = case.prop('PROFILE')
364 if profile != None and \
365 profile.find("IBM XML Conformance Test Suite - Production") < 0:
366 print("=>", profile)
367 test = case.children
368 while test != None:
369 if test.name == 'TEST':
370 runTest(test)
371 if test.name == 'TESTCASES':
372 runTestCases(test)
373 test = test.next
374
375 conf = loadNoentDoc(CONF)
376 if conf is None:
377 print("Unable to load %s" % CONF)
378 sys.exit(1)
379
380 testsuite = conf.getRootElement()
381 if testsuite.name != 'TESTSUITE':
382 print("Expecting TESTSUITE root element: aborting")
383 sys.exit(1)
384
385 profile = testsuite.prop('PROFILE')
386 if profile != None:
387 print(profile)
388
389 start = time.time()
390
391 case = testsuite.children
392 while case != None:
393 if case.name == 'TESTCASES':
394 old_test_nr = test_nr
395 old_test_succeed = test_succeed
396 old_test_failed = test_failed
397 old_test_error = test_error
398 runTestCases(case)
399 print(" Ran %d tests: %d succeeded, %d failed and %d generated an error" % (
400 test_nr - old_test_nr, test_succeed - old_test_succeed,
401 test_failed - old_test_failed, test_error - old_test_error))
402 case = case.next
403
404 conf.freeDoc()
405 log.close()
406
407 print("Ran %d tests: %d succeeded, %d failed and %d generated an error in %.2f s." % (
408 test_nr, test_succeed, test_failed, test_error, time.time() - start))