1 #!/usr/bin/env python3
2
3 #
4 # This is the MS subset of the W3C test suite for XML Schemas.
5 # This file is generated from the MS W3c test suite description file.
6 #
7
8 import sys, os
9 import optparse
10 import libxml2
11
12 opa = optparse.OptionParser()
13
14 opa.add_option("-b", "--base", action="store", type="string", dest="baseDir",
15 default="",
16 help="""The base directory; i.e. the parent folder of the
17 "nisttest", "suntest" and "msxsdtest" directories.""")
18
19 opa.add_option("-o", "--out", action="store", type="string", dest="logFile",
20 default="test.log",
21 help="The filepath of the log file to be created")
22
23 opa.add_option("--log", action="store_true", dest="enableLog",
24 default=False,
25 help="Create the log file")
26
27 opa.add_option("--no-test-out", action="store_true", dest="disableTestStdOut",
28 default=False,
29 help="Don't output test results")
30
31 opa.add_option("-s", "--silent", action="store_true", dest="silent", default=False,
32 help="Disables display of all tests")
33
34 opa.add_option("-v", "--verbose", action="store_true", dest="verbose",
35 default=False,
36 help="Displays all tests (only if --silent is not set)")
37
38 opa.add_option("-x", "--max", type="int", dest="maxTestCount",
39 default="-1",
40 help="The maximum number of tests to be run")
41
42 opa.add_option("-t", "--test", type="string", dest="singleTest",
43 default=None,
44 help="Runs the specified test only")
45
46 opa.add_option("--tsw", "--test-starts-with", type="string", dest="testStartsWith",
47 default=None,
48 help="Runs the specified test(s), starting with the given string")
49
50 opa.add_option("--rieo", "--report-internal-errors-only", action="store_true",
51 dest="reportInternalErrOnly", default=False,
52 help="Display erroneous tests of type 'internal' only")
53
54 opa.add_option("--rueo", "--report-unimplemented-errors-only", action="store_true",
55 dest="reportUnimplErrOnly", default=False,
56 help="Display erroneous tests of type 'unimplemented' only")
57
58 opa.add_option("--rmleo", "--report-mem-leak-errors-only", action="store_true",
59 dest="reportMemLeakErrOnly", default=False,
60 help="Display erroneous tests of type 'memory leak' only")
61
62 opa.add_option("-c", "--combines", type="string", dest="combines",
63 default=None,
64 help="Combines to be run (all if omitted)")
65
66 opa.add_option("--csw", "--csw", type="string", dest="combineStartsWith",
67 default=None,
68 help="Combines to be run (all if omitted)")
69
70 opa.add_option("--rc", "--report-combines", action="store_true",
71 dest="reportCombines", default=False,
72 help="Display combine reports")
73
74 opa.add_option("--rec", "--report-err-combines", action="store_true",
75 dest="reportErrCombines", default=False,
76 help="Display erroneous combine reports only")
77
78 opa.add_option("--debug", action="store_true",
79 dest="debugEnabled", default=False,
80 help="Displays debug messages")
81
82 opa.add_option("--info", action="store_true",
83 dest="info", default=False,
84 help="Displays info on the suite only. Does not run any test.")
85 opa.add_option("--sax", action="store_true",
86 dest="validationSAX", default=False,
87 help="Use SAX2-driven validation.")
88 opa.add_option("--tn", action="store_true",
89 dest="displayTestName", default=False,
90 help="Display the test name in every case.")
91
92 (options, args) = opa.parse_args()
93
94 if options.combines is not None:
95 options.combines = options.combines.split()
96
97 ################################################
98 # The vars below are not intended to be changed.
99 #
100
101 msgSchemaNotValidButShould = "The schema should be valid."
102 msgSchemaValidButShouldNot = "The schema should be invalid."
103 msgInstanceNotValidButShould = "The instance should be valid."
104 msgInstanceValidButShouldNot = "The instance should be invalid."
105 vendorNIST = "NIST"
106 vendorNIST_2 = "NIST-2"
107 vendorSUN = "SUN"
108 vendorMS = "MS"
109
110 ###################
111 # Helper functions.
112 #
113 vendor = None
114
115 def handleError(test, msg):
116 global options
117 if not options.silent:
118 test.addLibLog("'%s' LIB: %s" % (test.name, msg))
119 if msg.find("Unimplemented") > -1:
120 test.failUnimplemented()
121 elif msg.find("Internal") > -1:
122 test.failInternal()
123
124
125 def fixFileNames(fileName):
126 if (fileName is None) or (fileName == ""):
127 return ""
128 dirs = fileName.split("/")
129 if dirs[1] != "Tests":
130 fileName = os.path.join(".", "Tests")
131 for dir in dirs[1:]:
132 fileName = os.path.join(fileName, dir)
133 return fileName
134
135 class ESC[4;38;5;81mXSTCTestGroup:
136 def __init__(self, name, schemaFileName, descr):
137 global vendor, vendorNIST_2
138 self.name = name
139 self.descr = descr
140 self.mainSchema = True
141 self.schemaFileName = fixFileNames(schemaFileName)
142 self.schemaParsed = False
143 self.schemaTried = False
144
145 def setSchema(self, schemaFileName, parsed):
146 if not self.mainSchema:
147 return
148 self.mainSchema = False
149 self.schemaParsed = parsed
150 self.schemaTried = True
151
152 class ESC[4;38;5;81mXSTCTestCase:
153
154 # <!-- groupName, Name, Accepted, File, Val, Descr
155 def __init__(self, isSchema, groupName, name, accepted, file, val, descr):
156 global options
157 #
158 # Constructor.
159 #
160 self.testRunner = None
161 self.isSchema = isSchema
162 self.groupName = groupName
163 self.name = name
164 self.accepted = accepted
165 self.fileName = fixFileNames(file)
166 self.val = val
167 self.descr = descr
168 self.failed = False
169 self.combineName = None
170
171 self.log = []
172 self.libLog = []
173 self.initialMemUsed = 0
174 self.memLeak = 0
175 self.excepted = False
176 self.bad = False
177 self.unimplemented = False
178 self.internalErr = False
179 self.noSchemaErr = False
180 self.failed = False
181 #
182 # Init the log.
183 #
184 if not options.silent:
185 if self.descr is not None:
186 self.log.append("'%s' descr: %s\n" % (self.name, self.descr))
187 self.log.append("'%s' exp validity: %d\n" % (self.name, self.val))
188
189 def initTest(self, runner):
190 global vendorNIST, vendorSUN, vendorMS, vendorNIST_2, options, vendor
191 #
192 # Get the test-group.
193 #
194 self.runner = runner
195 self.group = runner.getGroup(self.groupName)
196 if vendor == vendorMS or vendor == vendorSUN:
197 #
198 # Use the last given directory for the combine name.
199 #
200 dirs = self.fileName.split("/")
201 self.combineName = dirs[len(dirs) -2]
202 elif vendor == vendorNIST:
203 #
204 # NIST files are named in the following form:
205 # "NISTSchema-short-pattern-1.xsd"
206 #
207 tokens = self.name.split("-")
208 self.combineName = tokens[1]
209 elif vendor == vendorNIST_2:
210 #
211 # Group-names have the form: "atomic-normalizedString-length-1"
212 #
213 tokens = self.groupName.split("-")
214 self.combineName = "%s-%s" % (tokens[0], tokens[1])
215 else:
216 self.combineName = "unknown"
217 raise Exception("Could not compute the combine name of a test.")
218 if (not options.silent) and (self.group.descr is not None):
219 self.log.append("'%s' group-descr: %s\n" % (self.name, self.group.descr))
220
221
222 def addLibLog(self, msg):
223 """This one is intended to be used by the error handler
224 function"""
225 global options
226 if not options.silent:
227 self.libLog.append(msg)
228
229 def fail(self, msg):
230 global options
231 self.failed = True
232 if not options.silent:
233 self.log.append("'%s' ( FAILED: %s\n" % (self.name, msg))
234
235 def failNoSchema(self):
236 global options
237 self.failed = True
238 self.noSchemaErr = True
239 if not options.silent:
240 self.log.append("'%s' X NO-SCHEMA\n" % (self.name))
241
242 def failInternal(self):
243 global options
244 self.failed = True
245 self.internalErr = True
246 if not options.silent:
247 self.log.append("'%s' * INTERNAL\n" % self.name)
248
249 def failUnimplemented(self):
250 global options
251 self.failed = True
252 self.unimplemented = True
253 if not options.silent:
254 self.log.append("'%s' ? UNIMPLEMENTED\n" % self.name)
255
256 def failCritical(self, msg):
257 global options
258 self.failed = True
259 self.bad = True
260 if not options.silent:
261 self.log.append("'%s' ! BAD: %s\n" % (self.name, msg))
262
263 def failExcept(self, e):
264 global options
265 self.failed = True
266 self.excepted = True
267 if not options.silent:
268 self.log.append("'%s' # EXCEPTION: %s\n" % (self.name, e.__str__()))
269
270 def setUp(self):
271 #
272 # Set up Libxml2.
273 #
274 self.initialMemUsed = libxml2.debugMemory(1)
275 libxml2.initParser()
276 libxml2.lineNumbersDefault(1)
277 libxml2.registerErrorHandler(handleError, self)
278
279 def tearDown(self):
280 libxml2.schemaCleanupTypes()
281 libxml2.cleanupParser()
282 self.memLeak = libxml2.debugMemory(1) - self.initialMemUsed
283
284 def isIOError(self, file, docType):
285 err = None
286 try:
287 err = libxml2.lastError()
288 except:
289 # Suppress exceptions.
290 pass
291 if (err is None):
292 return False
293 if err.domain() == libxml2.XML_FROM_IO:
294 self.failCritical("failed to access the %s resource '%s'\n" % (docType, file))
295
296 def debugMsg(self, msg):
297 global options
298 if options.debugEnabled:
299 sys.stdout.write("'%s' DEBUG: %s\n" % (self.name, msg))
300
301 def finalize(self):
302 global options
303 """Adds additional info to the log."""
304 #
305 # Add libxml2 messages.
306 #
307 if not options.silent:
308 self.log.extend(self.libLog)
309 #
310 # Add memory leaks.
311 #
312 if self.memLeak != 0:
313 self.log.append("%s + memory leak: %d bytes\n" % (self.name, self.memLeak))
314
315 def run(self):
316 """Runs a test."""
317 global options
318
319 ##filePath = os.path.join(options.baseDir, self.fileName)
320 # filePath = "%s/%s/%s/%s" % (options.baseDir, self.test_Folder, self.schema_Folder, self.schema_File)
321 if options.displayTestName:
322 sys.stdout.write("'%s'\n" % self.name)
323 try:
324 self.validate()
325 except (Exception, libxml2.parserError, libxml2.treeError) as e:
326 self.failExcept(e)
327
328 def parseSchema(fileName):
329 schema = None
330 ctxt = libxml2.schemaNewParserCtxt(fileName)
331 try:
332 try:
333 schema = ctxt.schemaParse()
334 except:
335 pass
336 finally:
337 del ctxt
338 return schema
339
340
341 class ESC[4;38;5;81mXSTCSchemaTest(ESC[4;38;5;149mXSTCTestCase):
342
343 def __init__(self, groupName, name, accepted, file, val, descr):
344 XSTCTestCase.__init__(self, 1, groupName, name, accepted, file, val, descr)
345
346 def validate(self):
347 global msgSchemaNotValidButShould, msgSchemaValidButShouldNot
348 schema = None
349 filePath = self.fileName
350 # os.path.join(options.baseDir, self.fileName)
351 valid = 0
352 try:
353 #
354 # Parse the schema.
355 #
356 self.debugMsg("loading schema: %s" % filePath)
357 schema = parseSchema(filePath)
358 self.debugMsg("after loading schema")
359 if schema is None:
360 self.debugMsg("schema is None")
361 self.debugMsg("checking for IO errors...")
362 if self.isIOError(filePath, "schema"):
363 return
364 self.debugMsg("checking schema result")
365 if (schema is None and self.val) or (schema is not None and self.val == 0):
366 self.debugMsg("schema result is BAD")
367 if (schema == None):
368 self.fail(msgSchemaNotValidButShould)
369 else:
370 self.fail(msgSchemaValidButShouldNot)
371 else:
372 self.debugMsg("schema result is OK")
373 finally:
374 self.group.setSchema(self.fileName, schema is not None)
375 del schema
376
377 class ESC[4;38;5;81mXSTCInstanceTest(ESC[4;38;5;149mXSTCTestCase):
378
379 def __init__(self, groupName, name, accepted, file, val, descr):
380 XSTCTestCase.__init__(self, 0, groupName, name, accepted, file, val, descr)
381
382 def validate(self):
383 instance = None
384 schema = None
385 filePath = self.fileName
386 # os.path.join(options.baseDir, self.fileName)
387
388 if not self.group.schemaParsed and self.group.schemaTried:
389 self.failNoSchema()
390 return
391
392 self.debugMsg("loading instance: %s" % filePath)
393 parserCtxt = libxml2.newParserCtxt()
394 if (parserCtxt is None):
395 # TODO: Is this one necessary, or will an exception
396 # be already raised?
397 raise Exception("Could not create the instance parser context.")
398 if not options.validationSAX:
399 try:
400 try:
401 instance = parserCtxt.ctxtReadFile(filePath, None, libxml2.XML_PARSE_NOWARNING)
402 except:
403 # Suppress exceptions.
404 pass
405 finally:
406 del parserCtxt
407 self.debugMsg("after loading instance")
408 if instance is None:
409 self.debugMsg("instance is None")
410 self.failCritical("Failed to parse the instance for unknown reasons.")
411 return
412 try:
413 #
414 # Validate the instance.
415 #
416 self.debugMsg("loading schema: %s" % self.group.schemaFileName)
417 schema = parseSchema(self.group.schemaFileName)
418 try:
419 validationCtxt = schema.schemaNewValidCtxt()
420 #validationCtxt = libxml2.schemaNewValidCtxt(None)
421 if (validationCtxt is None):
422 self.failCritical("Could not create the validation context.")
423 return
424 try:
425 self.debugMsg("validating instance")
426 if options.validationSAX:
427 instance_Err = validationCtxt.schemaValidateFile(filePath, 0)
428 else:
429 instance_Err = validationCtxt.schemaValidateDoc(instance)
430 self.debugMsg("after instance validation")
431 self.debugMsg("instance-err: %d" % instance_Err)
432 if (instance_Err != 0 and self.val == 1) or (instance_Err == 0 and self.val == 0):
433 self.debugMsg("instance result is BAD")
434 if (instance_Err != 0):
435 self.fail(msgInstanceNotValidButShould)
436 else:
437 self.fail(msgInstanceValidButShouldNot)
438
439 else:
440 self.debugMsg("instance result is OK")
441 finally:
442 del validationCtxt
443 finally:
444 del schema
445 finally:
446 if instance is not None:
447 instance.freeDoc()
448
449
450 ####################
451 # Test runner class.
452 #
453
454 class ESC[4;38;5;81mXSTCTestRunner:
455
456 CNT_TOTAL = 0
457 CNT_RAN = 1
458 CNT_SUCCEEDED = 2
459 CNT_FAILED = 3
460 CNT_UNIMPLEMENTED = 4
461 CNT_INTERNAL = 5
462 CNT_BAD = 6
463 CNT_EXCEPTED = 7
464 CNT_MEMLEAK = 8
465 CNT_NOSCHEMA = 9
466 CNT_NOTACCEPTED = 10
467 CNT_SCHEMA_TEST = 11
468
469 def __init__(self):
470 self.logFile = None
471 self.counters = self.createCounters()
472 self.testList = []
473 self.combinesRan = {}
474 self.groups = {}
475 self.curGroup = None
476
477 def createCounters(self):
478 counters = {self.CNT_TOTAL:0, self.CNT_RAN:0, self.CNT_SUCCEEDED:0,
479 self.CNT_FAILED:0, self.CNT_UNIMPLEMENTED:0, self.CNT_INTERNAL:0, self.CNT_BAD:0,
480 self.CNT_EXCEPTED:0, self.CNT_MEMLEAK:0, self.CNT_NOSCHEMA:0, self.CNT_NOTACCEPTED:0,
481 self.CNT_SCHEMA_TEST:0}
482
483 return counters
484
485 def addTest(self, test):
486 self.testList.append(test)
487 test.initTest(self)
488
489 def getGroup(self, groupName):
490 return self.groups[groupName]
491
492 def addGroup(self, group):
493 self.groups[group.name] = group
494
495 def updateCounters(self, test, counters):
496 if test.memLeak != 0:
497 counters[self.CNT_MEMLEAK] += 1
498 if not test.failed:
499 counters[self.CNT_SUCCEEDED] +=1
500 if test.failed:
501 counters[self.CNT_FAILED] += 1
502 if test.bad:
503 counters[self.CNT_BAD] += 1
504 if test.unimplemented:
505 counters[self.CNT_UNIMPLEMENTED] += 1
506 if test.internalErr:
507 counters[self.CNT_INTERNAL] += 1
508 if test.noSchemaErr:
509 counters[self.CNT_NOSCHEMA] += 1
510 if test.excepted:
511 counters[self.CNT_EXCEPTED] += 1
512 if not test.accepted:
513 counters[self.CNT_NOTACCEPTED] += 1
514 if test.isSchema:
515 counters[self.CNT_SCHEMA_TEST] += 1
516 return counters
517
518 def displayResults(self, out, all, combName, counters):
519 out.write("\n")
520 if all:
521 if options.combines is not None:
522 out.write("combine(s): %s\n" % str(options.combines))
523 elif combName is not None:
524 out.write("combine : %s\n" % combName)
525 out.write(" total : %d\n" % counters[self.CNT_TOTAL])
526 if all or options.combines is not None:
527 out.write(" ran : %d\n" % counters[self.CNT_RAN])
528 out.write(" (schemata) : %d\n" % counters[self.CNT_SCHEMA_TEST])
529 # out.write(" succeeded : %d\n" % counters[self.CNT_SUCCEEDED])
530 out.write(" not accepted : %d\n" % counters[self.CNT_NOTACCEPTED])
531 if counters[self.CNT_FAILED] > 0:
532 out.write(" failed : %d\n" % counters[self.CNT_FAILED])
533 out.write(" -> internal : %d\n" % counters[self.CNT_INTERNAL])
534 out.write(" -> unimpl. : %d\n" % counters[self.CNT_UNIMPLEMENTED])
535 out.write(" -> skip-invalid-schema : %d\n" % counters[self.CNT_NOSCHEMA])
536 out.write(" -> bad : %d\n" % counters[self.CNT_BAD])
537 out.write(" -> exceptions : %d\n" % counters[self.CNT_EXCEPTED])
538 out.write(" memory leaks : %d\n" % counters[self.CNT_MEMLEAK])
539
540 def displayShortResults(self, out, all, combName, counters):
541 out.write("Ran %d of %d tests (%d schemata):" % (counters[self.CNT_RAN],
542 counters[self.CNT_TOTAL], counters[self.CNT_SCHEMA_TEST]))
543 # out.write(" succeeded : %d\n" % counters[self.CNT_SUCCEEDED])
544 if counters[self.CNT_NOTACCEPTED] > 0:
545 out.write(" %d not accepted" % (counters[self.CNT_NOTACCEPTED]))
546 if counters[self.CNT_FAILED] > 0 or counters[self.CNT_MEMLEAK] > 0:
547 if counters[self.CNT_FAILED] > 0:
548 out.write(" %d failed" % (counters[self.CNT_FAILED]))
549 out.write(" (")
550 if counters[self.CNT_INTERNAL] > 0:
551 out.write(" %d internal" % (counters[self.CNT_INTERNAL]))
552 if counters[self.CNT_UNIMPLEMENTED] > 0:
553 out.write(" %d unimplemented" % (counters[self.CNT_UNIMPLEMENTED]))
554 if counters[self.CNT_NOSCHEMA] > 0:
555 out.write(" %d skip-invalid-schema" % (counters[self.CNT_NOSCHEMA]))
556 if counters[self.CNT_BAD] > 0:
557 out.write(" %d bad" % (counters[self.CNT_BAD]))
558 if counters[self.CNT_EXCEPTED] > 0:
559 out.write(" %d exception" % (counters[self.CNT_EXCEPTED]))
560 out.write(" )")
561 if counters[self.CNT_MEMLEAK] > 0:
562 out.write(" %d leaks" % (counters[self.CNT_MEMLEAK]))
563 out.write("\n")
564 else:
565 out.write(" all passed\n")
566
567 def reportCombine(self, combName):
568 global options
569
570 counters = self.createCounters()
571 #
572 # Compute evaluation counters.
573 #
574 for test in self.combinesRan[combName]:
575 counters[self.CNT_TOTAL] += 1
576 counters[self.CNT_RAN] += 1
577 counters = self.updateCounters(test, counters)
578 if options.reportErrCombines and (counters[self.CNT_FAILED] == 0) and (counters[self.CNT_MEMLEAK] == 0):
579 pass
580 else:
581 if options.enableLog:
582 self.displayResults(self.logFile, False, combName, counters)
583 self.displayResults(sys.stdout, False, combName, counters)
584
585 def displayTestLog(self, test):
586 sys.stdout.writelines(test.log)
587 sys.stdout.write("~~~~~~~~~~\n")
588
589 def reportTest(self, test):
590 global options
591
592 error = test.failed or test.memLeak != 0
593 #
594 # Only erroneous tests will be written to the log,
595 # except @verbose is switched on.
596 #
597 if options.enableLog and (options.verbose or error):
598 self.logFile.writelines(test.log)
599 self.logFile.write("~~~~~~~~~~\n")
600 #
601 # if not @silent, only erroneous tests will be
602 # written to stdout, except @verbose is switched on.
603 #
604 if not options.silent:
605 if options.reportInternalErrOnly and test.internalErr:
606 self.displayTestLog(test)
607 if options.reportMemLeakErrOnly and test.memLeak != 0:
608 self.displayTestLog(test)
609 if options.reportUnimplErrOnly and test.unimplemented:
610 self.displayTestLog(test)
611 if (options.verbose or error) and (not options.reportInternalErrOnly) and (not options.reportMemLeakErrOnly) and (not options.reportUnimplErrOnly):
612 self.displayTestLog(test)
613
614
615 def addToCombines(self, test):
616 found = False
617 if test.combineName in self.combinesRan:
618 self.combinesRan[test.combineName].append(test)
619 else:
620 self.combinesRan[test.combineName] = [test]
621
622 def run(self):
623
624 global options
625
626 if options.info:
627 for test in self.testList:
628 self.addToCombines(test)
629 sys.stdout.write("Combines: %d\n" % len(self.combinesRan))
630 sys.stdout.write("%s\n" % self.combinesRan.keys())
631 return
632
633 if options.enableLog:
634 self.logFile = open(options.logFile, "w")
635 try:
636 for test in self.testList:
637 self.counters[self.CNT_TOTAL] += 1
638 #
639 # Filter tests.
640 #
641 if options.singleTest is not None and options.singleTest != "":
642 if (test.name != options.singleTest):
643 continue
644 elif options.combines is not None:
645 if not options.combines.__contains__(test.combineName):
646 continue
647 elif options.testStartsWith is not None:
648 if not test.name.startswith(options.testStartsWith):
649 continue
650 elif options.combineStartsWith is not None:
651 if not test.combineName.startswith(options.combineStartsWith):
652 continue
653
654 if options.maxTestCount != -1 and self.counters[self.CNT_RAN] >= options.maxTestCount:
655 break
656 self.counters[self.CNT_RAN] += 1
657 #
658 # Run the thing, dammit.
659 #
660 try:
661 test.setUp()
662 try:
663 test.run()
664 finally:
665 test.tearDown()
666 finally:
667 #
668 # Evaluate.
669 #
670 test.finalize()
671 self.reportTest(test)
672 if options.reportCombines or options.reportErrCombines:
673 self.addToCombines(test)
674 self.counters = self.updateCounters(test, self.counters)
675 finally:
676 if options.reportCombines or options.reportErrCombines:
677 #
678 # Build a report for every single combine.
679 #
680 # TODO: How to sort a dict?
681 #
682 self.combinesRan.keys().sort(None)
683 for key in self.combinesRan.keys():
684 self.reportCombine(key)
685
686 #
687 # Display the final report.
688 #
689 if options.silent:
690 self.displayShortResults(sys.stdout, True, None, self.counters)
691 else:
692 sys.stdout.write("===========================\n")
693 self.displayResults(sys.stdout, True, None, self.counters)