1 #!/usr/bin/python
2 # Copyright (C) 2017-2023 Free Software Foundation, Inc.
3 # This file is part of the GNU C Library.
4 #
5 # The GNU C Library is free software; you can redistribute it and/or
6 # modify it under the terms of the GNU Lesser General Public
7 # License as published by the Free Software Foundation; either
8 # version 2.1 of the License, or (at your option) any later version.
9 #
10 # The GNU C Library is distributed in the hope that it will be useful,
11 # but WITHOUT ANY WARRANTY; without even the implied warranty of
12 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 # Lesser General Public License for more details.
14 #
15 # You should have received a copy of the GNU Lesser General Public
16 # License along with the GNU C Library; if not, see
17 # <https://www.gnu.org/licenses/>.
18 """Compare results of string functions
19
20 Given a string benchmark result file, print a table with comparisons with a
21 baseline. The baseline is the first function, which typically is the builtin
22 function.
23 """
24 import math
25 import matplotlib as mpl
26 mpl.use('Agg')
27
28 import sys
29 import os
30 import json
31 import pylab
32 import argparse
33 import traceback
34
35 try:
36 import jsonschema as validator
37 except ImportError:
38 print('Could not find jsonschema module.')
39 raise
40
41
42 def parse_file(filename, schema_filename):
43 try:
44 with open(schema_filename, 'r') as schemafile:
45 schema = json.load(schemafile)
46 with open(filename, 'r') as benchfile:
47 bench = json.load(benchfile)
48 validator.validate(bench, schema)
49 return bench
50 except:
51 print(traceback.format_exc(limit=1))
52 sys.exit(os.EX_NOINPUT)
53
54 def draw_graph(f, v, ifuncs, results):
55 """Plot graphs for functions
56
57 Plot line graphs for each of the ifuncs
58
59 Args:
60 f: Function name
61 v: Benchmark variant for the function.
62 ifuncs: List of ifunc names
63 results: Dictionary of results for each test criterion
64 """
65 print('Generating graph for %s, variant \'%s\'' % (f, v))
66 xkeys = results.keys()
67
68 pylab.clf()
69 fig = pylab.figure(frameon=False)
70 fig.set_size_inches(32, 18)
71 pylab.ylabel('Performance improvement from base')
72 X = range(len(xkeys))
73 pylab.xticks(X, xkeys)
74
75 i = 0
76
77 while i < len(ifuncs):
78 Y = [results[k][i] for k in xkeys]
79 lines = pylab.plot(X, Y, label=':'+ifuncs[i])
80 i = i + 1
81
82 pylab.legend()
83 pylab.grid()
84 pylab.savefig('%s-%s.png' % (f, v), bbox_inches='tight')
85
86
87 def process_results(results, attrs, funcs, base_func, graph, no_diff,
88 no_header, gmean):
89 """ Process results and print them
90
91 Args:
92 results: JSON dictionary of results
93 attrs: Attributes that form the test criteria
94 funcs: Functions that are selected
95 """
96
97 for f in results['functions'].keys():
98
99 v = results['functions'][f]['bench-variant']
100
101 selected = {}
102 index = 0
103 base_index = 0
104 if funcs:
105 ifuncs = []
106 first_func = True
107 for i in results['functions'][f]['ifuncs']:
108 if i in funcs:
109 if first_func:
110 base_index = index
111 first_func = False
112 selected[index] = 1
113 ifuncs.append(i)
114 else:
115 selected[index] = 0
116 index += 1
117 else:
118 ifuncs = results['functions'][f]['ifuncs']
119 for i in ifuncs:
120 selected[index] = 1
121 index += 1
122
123 if base_func:
124 try:
125 base_index = results['functions'][f]['ifuncs'].index(base_func)
126 except ValueError:
127 sys.stderr.write('Invalid -b "%s" parameter. Options: %s.\n' %
128 (base_func, ', '.join(results['functions'][f]['ifuncs'])))
129 sys.exit(os.EX_DATAERR)
130
131 if not no_header:
132 print('Function: %s' % f)
133 print('Variant: %s' % v)
134 print("%36s%s" % (' ', '\t'.join(ifuncs)))
135 print("=" * 120)
136
137 mean_row = [0 for i in range(len(ifuncs))]
138 total=0
139 graph_res = {}
140 for res in results['functions'][f]['results']:
141 try:
142 attr_list = ['%s=%s' % (a, res[a]) for a in attrs]
143 except KeyError as ke:
144 sys.stderr.write('Invalid -a %s parameter. Options: %s.\n'
145 % (ke, ', '.join([a for a in res.keys() if a != 'timings'])))
146 sys.exit(os.EX_DATAERR)
147 i = 0
148 key = ', '.join(attr_list)
149 sys.stdout.write('%36s: ' % key)
150 graph_res[key] = res['timings']
151
152 for t in res['timings']:
153 if selected[i]:
154 if gmean:
155 mean_row[i] = mean_row[i]+math.log(t)
156 sys.stdout.write ('%12.2f' % t)
157 if not no_diff:
158 if i != base_index:
159 base = res['timings'][base_index]
160 diff = (base - t) * 100 / base
161 sys.stdout.write (' (%6.2f%%)' % diff)
162 sys.stdout.write('\t')
163 i = i + 1
164 print('')
165
166 if graph:
167 draw_graph(f, v, results['functions'][f]['ifuncs'], graph_res)
168
169 if gmean:
170 print("=" * 120)
171 total = len(results['functions'][f]['results'])
172 sys.stdout.write ('Geo-mean (for %s inputs)'%total)
173 for m in mean_row:
174 sys.stdout.write ('%12.2f' % (math.exp(m/total)))
175
176 def main(args):
177 """Program Entry Point
178
179 Take a string benchmark output file and compare timings.
180 """
181
182 base_func = None
183 filename = args.input
184 schema_filename = args.schema
185 base_func = args.base
186 attrs = args.attributes.split(',')
187 if args.functions:
188 funcs = args.functions.split(',')
189 if base_func and not base_func in funcs:
190 print('Baseline function (%s) not found.' % base_func)
191 sys.exit(os.EX_DATAERR)
192 else:
193 funcs = None
194
195 results = parse_file(args.input, args.schema)
196 process_results(results, attrs, funcs, base_func, args.graph, args.no_diff,
197 args.no_header, args.gmean)
198 return os.EX_OK
199
200
201 if __name__ == '__main__':
202 parser = argparse.ArgumentParser()
203
204 # The required arguments.
205 req = parser.add_argument_group(title='required arguments')
206 req.add_argument('-a', '--attributes', required=True,
207 help='Comma separated list of benchmark attributes.')
208 req.add_argument('-i', '--input', required=True,
209 help='Input JSON benchmark result file.')
210 req.add_argument('-s', '--schema', required=True,
211 help='Schema file to validate the result file.')
212
213 # Optional arguments.
214 parser.add_argument('-f', '--functions',
215 help='Comma separated list of functions.')
216 parser.add_argument('-b', '--base',
217 help='IFUNC variant to set as baseline.')
218 parser.add_argument('-g', '--graph', action='store_true',
219 help='Generate a graph from results.')
220 parser.add_argument('--no-diff', action='store_true',
221 help='Do not print the difference from baseline.')
222 parser.add_argument('--no-header', action='store_true',
223 help='Do not print the header.')
224 parser.add_argument('--gmean', action='store_true',
225 help='Print the geometric mean at the end of the output.')
226
227 args = parser.parse_args()
228 sys.exit(main(args))