1 /* x86 CPU feature tuning.
2 This file is part of the GNU C Library.
3 Copyright (C) 2017-2023 Free Software Foundation, Inc.
4
5 The GNU C Library is free software; you can redistribute it and/or
6 modify it under the terms of the GNU Lesser General Public
7 License as published by the Free Software Foundation; either
8 version 2.1 of the License, or (at your option) any later version.
9
10 The GNU C Library is distributed in the hope that it will be useful,
11 but WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 Lesser General Public License for more details.
14
15 You should have received a copy of the GNU Lesser General Public
16 License along with the GNU C Library; if not, see
17 <https://www.gnu.org/licenses/>. */
18
19 #define TUNABLE_NAMESPACE cpu
20 #include <stdbool.h>
21 #include <stdint.h>
22 #include <unistd.h> /* Get STDOUT_FILENO for _dl_printf. */
23 #include <elf/dl-tunables.h>
24 #include <string.h>
25 #include <cpu-features.h>
26 #include <ldsodefs.h>
27
28 /* We can't use IFUNC memcmp nor strlen in init_cpu_features from libc.a
29 since IFUNC must be set up by init_cpu_features. */
30 #if defined USE_MULTIARCH && !defined SHARED
31 # ifdef __x86_64__
32 /* DEFAULT_MEMCMP by sysdeps/x86_64/memcmp-isa-default-impl.h. */
33 # include <sysdeps/x86_64/memcmp-isa-default-impl.h>
34 # else
35 # define DEFAULT_MEMCMP __memcmp_ia32
36 # endif
37 extern __typeof (memcmp) DEFAULT_MEMCMP;
38 #else
39 # define DEFAULT_MEMCMP memcmp
40 #endif
41
42 #define CHECK_GLIBC_IFUNC_CPU_OFF(f, cpu_features, name, len) \
43 _Static_assert (sizeof (#name) - 1 == len, #name " != " #len); \
44 if (!DEFAULT_MEMCMP (f, #name, len)) \
45 { \
46 CPU_FEATURE_UNSET (cpu_features, name) \
47 break; \
48 }
49
50 /* Disable a preferred feature NAME. We don't enable a preferred feature
51 which isn't available. */
52 #define CHECK_GLIBC_IFUNC_PREFERRED_OFF(f, cpu_features, name, len) \
53 _Static_assert (sizeof (#name) - 1 == len, #name " != " #len); \
54 if (!DEFAULT_MEMCMP (f, #name, len)) \
55 { \
56 cpu_features->preferred[index_arch_##name] \
57 &= ~bit_arch_##name; \
58 break; \
59 }
60
61 /* Enable/disable a preferred feature NAME. */
62 #define CHECK_GLIBC_IFUNC_PREFERRED_BOTH(f, cpu_features, name, \
63 disable, len) \
64 _Static_assert (sizeof (#name) - 1 == len, #name " != " #len); \
65 if (!DEFAULT_MEMCMP (f, #name, len)) \
66 { \
67 if (disable) \
68 cpu_features->preferred[index_arch_##name] &= ~bit_arch_##name; \
69 else \
70 cpu_features->preferred[index_arch_##name] |= bit_arch_##name; \
71 break; \
72 }
73
74 /* Enable/disable a preferred feature NAME. Enable a preferred feature
75 only if the feature NEED is usable. */
76 #define CHECK_GLIBC_IFUNC_PREFERRED_NEED_BOTH(f, cpu_features, name, \
77 need, disable, len) \
78 _Static_assert (sizeof (#name) - 1 == len, #name " != " #len); \
79 if (!DEFAULT_MEMCMP (f, #name, len)) \
80 { \
81 if (disable) \
82 cpu_features->preferred[index_arch_##name] &= ~bit_arch_##name; \
83 else if (CPU_FEATURE_USABLE_P (cpu_features, need)) \
84 cpu_features->preferred[index_arch_##name] |= bit_arch_##name; \
85 break; \
86 }
87
88 attribute_hidden
89 void
90 TUNABLE_CALLBACK (set_hwcaps) (tunable_val_t *valp)
91 {
92 /* The current IFUNC selection is based on microbenchmarks in glibc.
93 It should give the best performance for most workloads. But other
94 choices may have better performance for a particular workload or on
95 the hardware which wasn't available when the selection was made.
96 The environment variable:
97
98 GLIBC_TUNABLES=glibc.cpu.hwcaps=-xxx,yyy,-zzz,....
99
100 can be used to enable CPU/ARCH feature yyy, disable CPU/ARCH feature
101 yyy and zzz, where the feature name is case-sensitive and has to
102 match the ones in cpu-features.h. It can be used by glibc developers
103 to tune for a new processor or override the IFUNC selection to
104 improve performance for a particular workload.
105
106 NOTE: the IFUNC selection may change over time. Please check all
107 multiarch implementations when experimenting. */
108
109 const char *p = valp->strval, *c;
110 struct cpu_features *cpu_features = &GLRO(dl_x86_cpu_features);
111 size_t len;
112
113 do
114 {
115 const char *n;
116 bool disable;
117 size_t nl;
118
119 for (c = p; *c != ','; c++)
120 if (*c == '\0')
121 break;
122
123 len = c - p;
124 disable = *p == '-';
125 if (disable)
126 {
127 n = p + 1;
128 nl = len - 1;
129 }
130 else
131 {
132 n = p;
133 nl = len;
134 }
135 switch (nl)
136 {
137 default:
138 break;
139 case 3:
140 if (disable)
141 {
142 CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, AVX, 3);
143 CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, CX8, 3);
144 CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, FMA, 3);
145 CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, HTT, 3);
146 CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, IBT, 3);
147 CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, RTM, 3);
148 }
149 break;
150 case 4:
151 if (disable)
152 {
153 CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, AVX2, 4);
154 CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, BMI1, 4);
155 CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, BMI2, 4);
156 CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, CMOV, 4);
157 CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, ERMS, 4);
158 CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, FMA4, 4);
159 CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, SSE2, 4);
160 CHECK_GLIBC_IFUNC_PREFERRED_OFF (n, cpu_features, I586, 4);
161 CHECK_GLIBC_IFUNC_PREFERRED_OFF (n, cpu_features, I686, 4);
162 }
163 break;
164 case 5:
165 if (disable)
166 {
167 CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, LZCNT, 5);
168 CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, MOVBE, 5);
169 CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, SHSTK, 5);
170 CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, SSSE3, 5);
171 CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, XSAVE, 5);
172 }
173 break;
174 case 6:
175 if (disable)
176 {
177 CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, POPCNT, 6);
178 CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, SSE4_1, 6);
179 CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, SSE4_2, 6);
180 if (!DEFAULT_MEMCMP (n, "XSAVEC", 6))
181 {
182 /* Update xsave_state_size to XSAVE state size. */
183 cpu_features->xsave_state_size
184 = cpu_features->xsave_state_full_size;
185 CPU_FEATURE_UNSET (cpu_features, XSAVEC);
186 }
187 }
188 break;
189 case 7:
190 if (disable)
191 {
192 CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, AVX512F, 7);
193 CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, OSXSAVE, 7);
194 }
195 break;
196 case 8:
197 if (disable)
198 {
199 CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, AVX512CD, 8);
200 CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, AVX512BW, 8);
201 CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, AVX512DQ, 8);
202 CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, AVX512ER, 8);
203 CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, AVX512PF, 8);
204 CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, AVX512VL, 8);
205 }
206 CHECK_GLIBC_IFUNC_PREFERRED_BOTH (n, cpu_features, Slow_BSF,
207 disable, 8);
208 break;
209 case 11:
210 {
211 CHECK_GLIBC_IFUNC_PREFERRED_BOTH (n, cpu_features,
212 Prefer_ERMS,
213 disable, 11);
214 CHECK_GLIBC_IFUNC_PREFERRED_BOTH (n, cpu_features,
215 Prefer_FSRM,
216 disable, 11);
217 CHECK_GLIBC_IFUNC_PREFERRED_NEED_BOTH (n, cpu_features,
218 Slow_SSE4_2,
219 SSE4_2,
220 disable, 11);
221 }
222 break;
223 case 15:
224 {
225 CHECK_GLIBC_IFUNC_PREFERRED_BOTH (n, cpu_features,
226 Fast_Rep_String,
227 disable, 15);
228 }
229 break;
230 case 16:
231 {
232 CHECK_GLIBC_IFUNC_PREFERRED_NEED_BOTH
233 (n, cpu_features, Prefer_No_AVX512, AVX512F,
234 disable, 16);
235 }
236 break;
237 case 18:
238 {
239 CHECK_GLIBC_IFUNC_PREFERRED_BOTH (n, cpu_features,
240 Fast_Copy_Backward,
241 disable, 18);
242 }
243 break;
244 case 19:
245 {
246 CHECK_GLIBC_IFUNC_PREFERRED_BOTH (n, cpu_features,
247 Fast_Unaligned_Load,
248 disable, 19);
249 CHECK_GLIBC_IFUNC_PREFERRED_BOTH (n, cpu_features,
250 Fast_Unaligned_Copy,
251 disable, 19);
252 }
253 break;
254 case 20:
255 {
256 CHECK_GLIBC_IFUNC_PREFERRED_NEED_BOTH
257 (n, cpu_features, Prefer_No_VZEROUPPER, AVX, disable,
258 20);
259 }
260 break;
261 case 23:
262 {
263 CHECK_GLIBC_IFUNC_PREFERRED_NEED_BOTH
264 (n, cpu_features, AVX_Fast_Unaligned_Load, AVX,
265 disable, 23);
266 }
267 break;
268 case 24:
269 {
270 CHECK_GLIBC_IFUNC_PREFERRED_NEED_BOTH
271 (n, cpu_features, MathVec_Prefer_No_AVX512, AVX512F,
272 disable, 24);
273 }
274 break;
275 case 26:
276 {
277 CHECK_GLIBC_IFUNC_PREFERRED_NEED_BOTH
278 (n, cpu_features, Prefer_PMINUB_for_stringop, SSE2,
279 disable, 26);
280 }
281 break;
282 }
283 p += len + 1;
284 }
285 while (*c != '\0');
286 }
287
288 #if CET_ENABLED
289 attribute_hidden
290 void
291 TUNABLE_CALLBACK (set_x86_ibt) (tunable_val_t *valp)
292 {
293 if (DEFAULT_MEMCMP (valp->strval, "on", sizeof ("on")) == 0)
294 GL(dl_x86_feature_control).ibt = cet_always_on;
295 else if (DEFAULT_MEMCMP (valp->strval, "off", sizeof ("off")) == 0)
296 GL(dl_x86_feature_control).ibt = cet_always_off;
297 else if (DEFAULT_MEMCMP (valp->strval, "permissive",
298 sizeof ("permissive")) == 0)
299 GL(dl_x86_feature_control).ibt = cet_permissive;
300 }
301
302 attribute_hidden
303 void
304 TUNABLE_CALLBACK (set_x86_shstk) (tunable_val_t *valp)
305 {
306 if (DEFAULT_MEMCMP (valp->strval, "on", sizeof ("on")) == 0)
307 GL(dl_x86_feature_control).shstk = cet_always_on;
308 else if (DEFAULT_MEMCMP (valp->strval, "off", sizeof ("off")) == 0)
309 GL(dl_x86_feature_control).shstk = cet_always_off;
310 else if (DEFAULT_MEMCMP (valp->strval, "permissive",
311 sizeof ("permissive")) == 0)
312 GL(dl_x86_feature_control).shstk = cet_permissive;
313 }
314 #endif