1 #!/usr/bin/env python3
2
3 # Script to compare testsuite failures against a list of known-to-fail
4 # tests.
5
6 # Contributed by Diego Novillo <dnovillo@google.com>
7 #
8 # Copyright (C) 2011-2023 Free Software Foundation, Inc.
9 #
10 # This file is part of GCC.
11 #
12 # GCC is free software; you can redistribute it and/or modify
13 # it under the terms of the GNU General Public License as published by
14 # the Free Software Foundation; either version 3, or (at your option)
15 # any later version.
16 #
17 # GCC is distributed in the hope that it will be useful,
18 # but WITHOUT ANY WARRANTY; without even the implied warranty of
19 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
20 # GNU General Public License for more details.
21 #
22 # You should have received a copy of the GNU General Public License
23 # along with GCC; see the file COPYING. If not, write to
24 # the Free Software Foundation, 51 Franklin Street, Fifth Floor,
25 # Boston, MA 02110-1301, USA.
26
27 """This script provides a coarser XFAILing mechanism that requires no
28 detailed DejaGNU markings. This is useful in a variety of scenarios:
29
30 - Development branches with many known failures waiting to be fixed.
31 - Release branches with known failures that are not considered
32 important for the particular release criteria used in that branch.
33
34 The script must be executed from the toplevel build directory. When
35 executed it will:
36
37 1- Determine the target built: TARGET
38 2- Determine the source directory: SRCDIR
39 3- Look for a failure manifest file in
40 <SRCDIR>/<MANIFEST_SUBDIR>/<MANIFEST_NAME>.xfail
41 4- Collect all the <tool>.sum files from the build tree.
42 5- Produce a report stating:
43 a- Failures expected in the manifest but not present in the build.
44 b- Failures in the build not expected in the manifest.
45 6- If all the build failures are expected in the manifest, it exits
46 with exit code 0. Otherwise, it exits with error code 1.
47
48 Manifest files contain expected DejaGNU results that are otherwise
49 treated as failures.
50 They may also contain additional text:
51
52 # This is a comment. - self explanatory
53 @include file - the file is a path relative to the includer
54 @remove result text - result text is removed from the expected set
55 """
56
57 import datetime
58 import optparse
59 import os
60 import re
61 import sys
62
63 # Handled test results.
64 _VALID_TEST_RESULTS = [ 'FAIL', 'UNRESOLVED', 'XPASS', 'ERROR' ]
65 _VALID_TEST_RESULTS_REX = re.compile("%s" % "|".join(_VALID_TEST_RESULTS))
66
67 # Subdirectory of srcdir in which to find the manifest file.
68 _MANIFEST_SUBDIR = 'contrib/testsuite-management'
69
70 # Pattern for naming manifest files.
71 # The first argument should be the toplevel GCC(/GNU tool) source directory.
72 # The second argument is the manifest subdir.
73 # The third argument is the manifest target, which defaults to the target
74 # triplet used during the build.
75 _MANIFEST_PATH_PATTERN = '%s/%s/%s.xfail'
76
77 # The options passed to the program.
78 _OPTIONS = None
79
80 def Error(msg):
81 print('error: %s' % msg, file=sys.stderr)
82 sys.exit(1)
83
84
85 class ESC[4;38;5;81mTestResult(ESC[4;38;5;149mobject):
86 """Describes a single DejaGNU test result as emitted in .sum files.
87
88 We are only interested in representing unsuccessful tests. So, only
89 a subset of all the tests are loaded.
90
91 The summary line used to build the test result should have this format:
92
93 attrlist | XPASS: gcc.dg/unroll_1.c (test for excess errors)
94 ^^^^^^^^ ^^^^^ ^^^^^^^^^^^^^^^^^ ^^^^^^^^^^^^^^^^^^^^^^^^
95 optional state name description
96 attributes
97
98 Attributes:
99 attrlist: A comma separated list of attributes.
100 Valid values:
101 flaky Indicates that this test may not always fail. These
102 tests are reported, but their presence does not affect
103 the results.
104
105 expire=YYYYMMDD After this date, this test will produce an error
106 whether it is in the manifest or not.
107
108 state: One of UNRESOLVED, XPASS or FAIL.
109 name: File name for the test.
110 description: String describing the test (flags used, dejagnu message, etc)
111 ordinal: Monotonically increasing integer.
112 It is used to keep results for one .exp file sorted
113 by the order the tests were run.
114 """
115
116 def __init__(self, summary_line, ordinal=-1):
117 try:
118 (self.attrs, summary_line) = SplitAttributesFromSummaryLine(summary_line)
119 try:
120 (self.state,
121 self.name,
122 self.description) = re.match(r'([A-Z]+):\s*(\S+)\s*(.*)',
123 summary_line).groups()
124 except:
125 print('Failed to parse summary line: "%s"' % summary_line)
126 raise
127 self.ordinal = ordinal
128 except ValueError:
129 Error('Cannot parse summary line "%s"' % summary_line)
130
131 if self.state not in _VALID_TEST_RESULTS:
132 Error('Invalid test result %s in "%s" (parsed as "%s")' % (
133 self.state, summary_line, self))
134
135 def __lt__(self, other):
136 return (self.name < other.name or
137 (self.name == other.name and self.ordinal < other.ordinal))
138
139 def __hash__(self):
140 return hash(self.state) ^ hash(self.name) ^ hash(self.description)
141
142 def __eq__(self, other):
143 return (self.state == other.state and
144 self.name == other.name and
145 self.description == other.description)
146
147 def __ne__(self, other):
148 return not (self == other)
149
150 def __str__(self):
151 attrs = ''
152 if self.attrs:
153 attrs = '%s | ' % self.attrs
154 return '%s%s: %s %s' % (attrs, self.state, self.name, self.description)
155
156 def ExpirationDate(self):
157 # Return a datetime.date object with the expiration date for this
158 # test result. Return None, if no expiration has been set.
159 if re.search(r'expire=', self.attrs):
160 expiration = re.search(r'expire=(\d\d\d\d)(\d\d)(\d\d)', self.attrs)
161 if not expiration:
162 Error('Invalid expire= format in "%s". Must be of the form '
163 '"expire=YYYYMMDD"' % self)
164 return datetime.date(int(expiration.group(1)),
165 int(expiration.group(2)),
166 int(expiration.group(3)))
167 return None
168
169 def HasExpired(self):
170 # Return True if the expiration date of this result has passed.
171 expiration_date = self.ExpirationDate()
172 if expiration_date:
173 now = datetime.date.today()
174 return now > expiration_date
175
176
177 def GetMakefileValue(makefile_name, value_name):
178 if os.path.exists(makefile_name):
179 makefile = open(makefile_name, encoding='latin-1', mode='r')
180 for line in makefile:
181 if line.startswith(value_name):
182 (_, value) = line.split('=', 1)
183 value = value.strip()
184 makefile.close()
185 return value
186 makefile.close()
187 return None
188
189
190 def ValidBuildDirectory(builddir):
191 if (not os.path.exists(builddir) or
192 not os.path.exists('%s/Makefile' % builddir)):
193 return False
194 return True
195
196
197 def IsComment(line):
198 """Return True if line is a comment."""
199 return line.startswith('#')
200
201
202 def SplitAttributesFromSummaryLine(line):
203 """Splits off attributes from a summary line, if present."""
204 if '|' in line and not _VALID_TEST_RESULTS_REX.match(line):
205 (attrs, line) = line.split('|', 1)
206 attrs = attrs.strip()
207 else:
208 attrs = ''
209 line = line.strip()
210 return (attrs, line)
211
212
213 def IsInterestingResult(line):
214 """Return True if line is one of the summary lines we care about."""
215 (_, line) = SplitAttributesFromSummaryLine(line)
216 return bool(_VALID_TEST_RESULTS_REX.match(line))
217
218
219 def IsInclude(line):
220 """Return True if line is an include of another file."""
221 return line.startswith("@include ")
222
223
224 def GetIncludeFile(line, includer):
225 """Extract the name of the include file from line."""
226 includer_dir = os.path.dirname(includer)
227 include_file = line[len("@include "):]
228 return os.path.join(includer_dir, include_file.strip())
229
230
231 def IsNegativeResult(line):
232 """Return True if line should be removed from the expected results."""
233 return line.startswith("@remove ")
234
235
236 def GetNegativeResult(line):
237 """Extract the name of the negative result from line."""
238 line = line[len("@remove "):]
239 return line.strip()
240
241
242 def ParseManifestWorker(result_set, manifest_path):
243 """Read manifest_path, adding the contents to result_set."""
244 if _OPTIONS.verbosity >= 1:
245 print('Parsing manifest file %s.' % manifest_path)
246 manifest_file = open(manifest_path, encoding='latin-1', mode='r')
247 for line in manifest_file:
248 line = line.strip()
249 if line == "":
250 pass
251 elif IsComment(line):
252 pass
253 elif IsNegativeResult(line):
254 result_set.remove(TestResult(GetNegativeResult(line)))
255 elif IsInclude(line):
256 ParseManifestWorker(result_set, GetIncludeFile(line, manifest_path))
257 elif IsInterestingResult(line):
258 result_set.add(TestResult(line))
259 else:
260 Error('Unrecognized line in manifest file: %s' % line)
261 manifest_file.close()
262
263
264 def ParseManifest(manifest_path):
265 """Create a set of TestResult instances from the given manifest file."""
266 result_set = set()
267 ParseManifestWorker(result_set, manifest_path)
268 return result_set
269
270
271 def ParseSummary(sum_fname):
272 """Create a set of TestResult instances from the given summary file."""
273 result_set = set()
274 # ordinal is used when sorting the results so that tests within each
275 # .exp file are kept sorted.
276 ordinal=0
277 sum_file = open(sum_fname, encoding='latin-1', mode='r')
278 for line in sum_file:
279 if IsInterestingResult(line):
280 result = TestResult(line, ordinal)
281 ordinal += 1
282 if result.HasExpired():
283 # Tests that have expired are not added to the set of expected
284 # results. If they are still present in the set of actual results,
285 # they will cause an error to be reported.
286 print('WARNING: Expected failure "%s" has expired.' % line.strip())
287 continue
288 result_set.add(result)
289 sum_file.close()
290 return result_set
291
292
293 def GetManifest(manifest_path):
294 """Build a set of expected failures from the manifest file.
295
296 Each entry in the manifest file should have the format understood
297 by the TestResult constructor.
298
299 If no manifest file exists for this target, it returns an empty set.
300 """
301 if os.path.exists(manifest_path):
302 return ParseManifest(manifest_path)
303 else:
304 return set()
305
306
307 def CollectSumFiles(builddir):
308 sum_files = []
309 for root, dirs, files in os.walk(builddir):
310 for ignored in ('.svn', '.git'):
311 if ignored in dirs:
312 dirs.remove(ignored)
313 for fname in files:
314 if fname.endswith('.sum'):
315 sum_files.append(os.path.join(root, fname))
316 return sum_files
317
318
319 def GetResults(sum_files):
320 """Collect all the test results from the given .sum files."""
321 build_results = set()
322 for sum_fname in sum_files:
323 print('\t%s' % sum_fname)
324 build_results |= ParseSummary(sum_fname)
325 return build_results
326
327
328 def CompareResults(manifest, actual):
329 """Compare sets of results and return two lists:
330 - List of results present in ACTUAL but missing from MANIFEST.
331 - List of results present in MANIFEST but missing from ACTUAL.
332 """
333 # Collect all the actual results not present in the manifest.
334 # Results in this set will be reported as errors.
335 actual_vs_manifest = set()
336 for actual_result in actual:
337 if actual_result not in manifest:
338 actual_vs_manifest.add(actual_result)
339
340 # Collect all the tests in the manifest that were not found
341 # in the actual results.
342 # Results in this set will be reported as warnings (since
343 # they are expected failures that are not failing anymore).
344 manifest_vs_actual = set()
345 for expected_result in manifest:
346 # Ignore tests marked flaky.
347 if 'flaky' in expected_result.attrs:
348 continue
349 if expected_result not in actual:
350 manifest_vs_actual.add(expected_result)
351
352 return actual_vs_manifest, manifest_vs_actual
353
354
355 def GetManifestPath(srcdir, target, user_provided_must_exist):
356 """Return the full path to the manifest file."""
357 manifest_path = _OPTIONS.manifest
358 if manifest_path:
359 if user_provided_must_exist and not os.path.exists(manifest_path):
360 Error('Manifest does not exist: %s' % manifest_path)
361 return manifest_path
362 else:
363 if not srcdir:
364 Error('Could not determine the location of GCC\'s source tree. '
365 'The Makefile does not contain a definition for "srcdir".')
366 if not target:
367 Error('Could not determine the target triplet for this build. '
368 'The Makefile does not contain a definition for "target_alias".')
369 return _MANIFEST_PATH_PATTERN % (srcdir, _MANIFEST_SUBDIR, target)
370
371
372 def GetBuildData():
373 if not ValidBuildDirectory(_OPTIONS.build_dir):
374 # If we have been given a set of results to use, we may
375 # not be inside a valid GCC build directory. In that case,
376 # the user must provide both a manifest file and a set
377 # of results to check against it.
378 if not _OPTIONS.results or not _OPTIONS.manifest:
379 Error('%s is not a valid GCC top level build directory. '
380 'You must use --manifest and --results to do the validation.' %
381 _OPTIONS.build_dir)
382 else:
383 return None, None
384 srcdir = GetMakefileValue('%s/Makefile' % _OPTIONS.build_dir, 'srcdir =')
385 target = GetMakefileValue('%s/Makefile' % _OPTIONS.build_dir, 'target_alias=')
386 print('Source directory: %s' % srcdir)
387 print('Build target: %s' % target)
388 return srcdir, target
389
390
391 def PrintSummary(msg, summary):
392 print('\n\n%s' % msg)
393 for result in sorted(summary):
394 print(result)
395
396
397 def GetSumFiles(results, build_dir):
398 if not results:
399 print('Getting actual results from build directory %s' % build_dir)
400 sum_files = CollectSumFiles(build_dir)
401 else:
402 print('Getting actual results from user-provided results')
403 sum_files = results.split()
404 return sum_files
405
406
407 def PerformComparison(expected, actual, ignore_missing_failures):
408 actual_vs_expected, expected_vs_actual = CompareResults(expected, actual)
409
410 tests_ok = True
411 if len(actual_vs_expected) > 0:
412 PrintSummary('Unexpected results in this build (new failures)',
413 actual_vs_expected)
414 tests_ok = False
415
416 if not ignore_missing_failures and len(expected_vs_actual) > 0:
417 PrintSummary('Expected results not present in this build (fixed tests)'
418 '\n\nNOTE: This is not a failure. It just means that these '
419 'tests were expected\nto fail, but either they worked in '
420 'this configuration or they were not\npresent at all.\n',
421 expected_vs_actual)
422
423 if tests_ok:
424 print('\nSUCCESS: No unexpected failures.')
425
426 return tests_ok
427
428
429 def CheckExpectedResults():
430 srcdir, target = GetBuildData()
431 manifest_path = GetManifestPath(srcdir, target, True)
432 print('Manifest: %s' % manifest_path)
433 manifest = GetManifest(manifest_path)
434 sum_files = GetSumFiles(_OPTIONS.results, _OPTIONS.build_dir)
435 actual = GetResults(sum_files)
436
437 if _OPTIONS.verbosity >= 1:
438 PrintSummary('Tests expected to fail', manifest)
439 PrintSummary('\nActual test results', actual)
440
441 return PerformComparison(manifest, actual, _OPTIONS.ignore_missing_failures)
442
443
444 def ProduceManifest():
445 (srcdir, target) = GetBuildData()
446 manifest_path = GetManifestPath(srcdir, target, False)
447 print('Manifest: %s' % manifest_path)
448 if os.path.exists(manifest_path) and not _OPTIONS.force:
449 Error('Manifest file %s already exists.\nUse --force to overwrite.' %
450 manifest_path)
451
452 sum_files = GetSumFiles(_OPTIONS.results, _OPTIONS.build_dir)
453 actual = GetResults(sum_files)
454 manifest_file = open(manifest_path, encoding='latin-1', mode='w')
455 for result in sorted(actual):
456 print(result)
457 manifest_file.write('%s\n' % result)
458 manifest_file.close()
459
460 return True
461
462
463 def CompareBuilds():
464 (srcdir, target) = GetBuildData()
465
466 sum_files = GetSumFiles(_OPTIONS.results, _OPTIONS.build_dir)
467 actual = GetResults(sum_files)
468
469 clean_sum_files = GetSumFiles(_OPTIONS.results, _OPTIONS.clean_build)
470 clean = GetResults(clean_sum_files)
471
472 return PerformComparison(clean, actual, _OPTIONS.ignore_missing_failures)
473
474
475 def Main(argv):
476 parser = optparse.OptionParser(usage=__doc__)
477
478 # Keep the following list sorted by option name.
479 parser.add_option('--build_dir', action='store', type='string',
480 dest='build_dir', default='.',
481 help='Build directory to check (default = .)')
482 parser.add_option('--clean_build', action='store', type='string',
483 dest='clean_build', default=None,
484 help='Compare test results from this build against '
485 'those of another (clean) build. Use this option '
486 'when comparing the test results of your patch versus '
487 'the test results of a clean build without your patch. '
488 'You must provide the path to the top directory of your '
489 'clean build.')
490 parser.add_option('--force', action='store_true', dest='force',
491 default=False, help='When used with --produce_manifest, '
492 'it will overwrite an existing manifest file '
493 '(default = False)')
494 parser.add_option('--ignore_missing_failures', action='store_true',
495 dest='ignore_missing_failures', default=False,
496 help='When a failure is expected in the manifest but '
497 'it is not found in the actual results, the script '
498 'produces a note alerting to this fact. This means '
499 'that the expected failure has been fixed, or '
500 'it did not run, or it may simply be flaky '
501 '(default = False)')
502 parser.add_option('--manifest', action='store', type='string',
503 dest='manifest', default=None,
504 help='Name of the manifest file to use (default = '
505 'taken from '
506 'contrib/testsuite-managment/<target_alias>.xfail)')
507 parser.add_option('--produce_manifest', action='store_true',
508 dest='produce_manifest', default=False,
509 help='Produce the manifest for the current '
510 'build (default = False)')
511 parser.add_option('--results', action='store', type='string',
512 dest='results', default=None, help='Space-separated list '
513 'of .sum files with the testing results to check. The '
514 'only content needed from these files are the lines '
515 'starting with FAIL, XPASS or UNRESOLVED (default = '
516 '.sum files collected from the build directory).')
517 parser.add_option('--verbosity', action='store', dest='verbosity',
518 type='int', default=0, help='Verbosity level (default = 0)')
519 global _OPTIONS
520 (_OPTIONS, _) = parser.parse_args(argv[1:])
521
522 if _OPTIONS.produce_manifest:
523 retval = ProduceManifest()
524 elif _OPTIONS.clean_build:
525 retval = CompareBuilds()
526 else:
527 retval = CheckExpectedResults()
528
529 if retval:
530 return 0
531 else:
532 return 1
533
534
535 if __name__ == '__main__':
536 retval = Main(sys.argv)
537 sys.exit(retval)