1 /* Initialize CPU feature data.
2 This file is part of the GNU C Library.
3 Copyright (C) 2008-2023 Free Software Foundation, Inc.
4
5 The GNU C Library is free software; you can redistribute it and/or
6 modify it under the terms of the GNU Lesser General Public
7 License as published by the Free Software Foundation; either
8 version 2.1 of the License, or (at your option) any later version.
9
10 The GNU C Library is distributed in the hope that it will be useful,
11 but WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 Lesser General Public License for more details.
14
15 You should have received a copy of the GNU Lesser General Public
16 License along with the GNU C Library; if not, see
17 <https://www.gnu.org/licenses/>. */
18
19 #include <dl-hwcap.h>
20 #include <libc-pointer-arith.h>
21 #include <get-isa-level.h>
22 #include <cacheinfo.h>
23 #include <dl-cacheinfo.h>
24 #include <dl-minsigstacksize.h>
25 #include <dl-hwcap2.h>
26
27 extern void TUNABLE_CALLBACK (set_hwcaps) (tunable_val_t *)
28 attribute_hidden;
29
30 #ifdef __LP64__
31 static void
32 TUNABLE_CALLBACK (set_prefer_map_32bit_exec) (tunable_val_t *valp)
33 {
34 if (valp->numval)
35 GLRO(dl_x86_cpu_features).preferred[index_arch_Prefer_MAP_32BIT_EXEC]
36 |= bit_arch_Prefer_MAP_32BIT_EXEC;
37 }
38 #endif
39
40 #if CET_ENABLED
41 extern void TUNABLE_CALLBACK (set_x86_ibt) (tunable_val_t *)
42 attribute_hidden;
43 extern void TUNABLE_CALLBACK (set_x86_shstk) (tunable_val_t *)
44 attribute_hidden;
45
46 # include <dl-cet.h>
47 #endif
48
49 static void
50 update_active (struct cpu_features *cpu_features)
51 {
52 /* Copy the cpuid bits to active bits for CPU featuress whose usability
53 in user space can be detected without additional OS support. */
54 CPU_FEATURE_SET_ACTIVE (cpu_features, SSE3);
55 CPU_FEATURE_SET_ACTIVE (cpu_features, PCLMULQDQ);
56 CPU_FEATURE_SET_ACTIVE (cpu_features, SSSE3);
57 CPU_FEATURE_SET_ACTIVE (cpu_features, CMPXCHG16B);
58 CPU_FEATURE_SET_ACTIVE (cpu_features, SSE4_1);
59 CPU_FEATURE_SET_ACTIVE (cpu_features, SSE4_2);
60 CPU_FEATURE_SET_ACTIVE (cpu_features, MOVBE);
61 CPU_FEATURE_SET_ACTIVE (cpu_features, POPCNT);
62 CPU_FEATURE_SET_ACTIVE (cpu_features, AES);
63 CPU_FEATURE_SET_ACTIVE (cpu_features, OSXSAVE);
64 CPU_FEATURE_SET_ACTIVE (cpu_features, TSC);
65 CPU_FEATURE_SET_ACTIVE (cpu_features, CX8);
66 CPU_FEATURE_SET_ACTIVE (cpu_features, CMOV);
67 CPU_FEATURE_SET_ACTIVE (cpu_features, CLFSH);
68 CPU_FEATURE_SET_ACTIVE (cpu_features, MMX);
69 CPU_FEATURE_SET_ACTIVE (cpu_features, FXSR);
70 CPU_FEATURE_SET_ACTIVE (cpu_features, SSE);
71 CPU_FEATURE_SET_ACTIVE (cpu_features, SSE2);
72 CPU_FEATURE_SET_ACTIVE (cpu_features, HTT);
73 CPU_FEATURE_SET_ACTIVE (cpu_features, BMI1);
74 CPU_FEATURE_SET_ACTIVE (cpu_features, HLE);
75 CPU_FEATURE_SET_ACTIVE (cpu_features, BMI2);
76 CPU_FEATURE_SET_ACTIVE (cpu_features, ERMS);
77 CPU_FEATURE_SET_ACTIVE (cpu_features, RDSEED);
78 CPU_FEATURE_SET_ACTIVE (cpu_features, ADX);
79 CPU_FEATURE_SET_ACTIVE (cpu_features, CLFLUSHOPT);
80 CPU_FEATURE_SET_ACTIVE (cpu_features, CLWB);
81 CPU_FEATURE_SET_ACTIVE (cpu_features, SHA);
82 CPU_FEATURE_SET_ACTIVE (cpu_features, PREFETCHWT1);
83 CPU_FEATURE_SET_ACTIVE (cpu_features, OSPKE);
84 CPU_FEATURE_SET_ACTIVE (cpu_features, WAITPKG);
85 CPU_FEATURE_SET_ACTIVE (cpu_features, GFNI);
86 CPU_FEATURE_SET_ACTIVE (cpu_features, RDPID);
87 CPU_FEATURE_SET_ACTIVE (cpu_features, RDRAND);
88 CPU_FEATURE_SET_ACTIVE (cpu_features, CLDEMOTE);
89 CPU_FEATURE_SET_ACTIVE (cpu_features, MOVDIRI);
90 CPU_FEATURE_SET_ACTIVE (cpu_features, MOVDIR64B);
91 CPU_FEATURE_SET_ACTIVE (cpu_features, FSRM);
92 CPU_FEATURE_SET_ACTIVE (cpu_features, RTM_ALWAYS_ABORT);
93 CPU_FEATURE_SET_ACTIVE (cpu_features, SERIALIZE);
94 CPU_FEATURE_SET_ACTIVE (cpu_features, TSXLDTRK);
95 CPU_FEATURE_SET_ACTIVE (cpu_features, LAHF64_SAHF64);
96 CPU_FEATURE_SET_ACTIVE (cpu_features, LZCNT);
97 CPU_FEATURE_SET_ACTIVE (cpu_features, SSE4A);
98 CPU_FEATURE_SET_ACTIVE (cpu_features, PREFETCHW);
99 CPU_FEATURE_SET_ACTIVE (cpu_features, TBM);
100 CPU_FEATURE_SET_ACTIVE (cpu_features, RDTSCP);
101 CPU_FEATURE_SET_ACTIVE (cpu_features, WBNOINVD);
102 CPU_FEATURE_SET_ACTIVE (cpu_features, RAO_INT);
103 CPU_FEATURE_SET_ACTIVE (cpu_features, CMPCCXADD);
104 CPU_FEATURE_SET_ACTIVE (cpu_features, FZLRM);
105 CPU_FEATURE_SET_ACTIVE (cpu_features, FSRS);
106 CPU_FEATURE_SET_ACTIVE (cpu_features, FSRCS);
107 CPU_FEATURE_SET_ACTIVE (cpu_features, PREFETCHI);
108 CPU_FEATURE_SET_ACTIVE (cpu_features, PTWRITE);
109
110 if (!CPU_FEATURES_CPU_P (cpu_features, RTM_ALWAYS_ABORT))
111 CPU_FEATURE_SET_ACTIVE (cpu_features, RTM);
112
113 #if CET_ENABLED
114 CPU_FEATURE_SET_ACTIVE (cpu_features, IBT);
115 CPU_FEATURE_SET_ACTIVE (cpu_features, SHSTK);
116 #endif
117
118 /* Can we call xgetbv? */
119 if (CPU_FEATURES_CPU_P (cpu_features, OSXSAVE))
120 {
121 unsigned int xcrlow;
122 unsigned int xcrhigh;
123 asm ("xgetbv" : "=a" (xcrlow), "=d" (xcrhigh) : "c" (0));
124 /* Is YMM and XMM state usable? */
125 if ((xcrlow & (bit_YMM_state | bit_XMM_state))
126 == (bit_YMM_state | bit_XMM_state))
127 {
128 /* Determine if AVX is usable. */
129 if (CPU_FEATURES_CPU_P (cpu_features, AVX))
130 {
131 CPU_FEATURE_SET (cpu_features, AVX);
132 /* The following features depend on AVX being usable. */
133 /* Determine if AVX2 is usable. */
134 if (CPU_FEATURES_CPU_P (cpu_features, AVX2))
135 {
136 CPU_FEATURE_SET (cpu_features, AVX2);
137
138 /* Unaligned load with 256-bit AVX registers are faster
139 on Intel/AMD processors with AVX2. */
140 cpu_features->preferred[index_arch_AVX_Fast_Unaligned_Load]
141 |= bit_arch_AVX_Fast_Unaligned_Load;
142 }
143 /* Determine if AVX-IFMA is usable. */
144 CPU_FEATURE_SET_ACTIVE (cpu_features, AVX_IFMA);
145 /* Determine if AVX-NE-CONVERT is usable. */
146 CPU_FEATURE_SET_ACTIVE (cpu_features, AVX_NE_CONVERT);
147 /* Determine if AVX-VNNI is usable. */
148 CPU_FEATURE_SET_ACTIVE (cpu_features, AVX_VNNI);
149 /* Determine if AVX-VNNI-INT8 is usable. */
150 CPU_FEATURE_SET_ACTIVE (cpu_features, AVX_VNNI_INT8);
151 /* Determine if FMA is usable. */
152 CPU_FEATURE_SET_ACTIVE (cpu_features, FMA);
153 /* Determine if VAES is usable. */
154 CPU_FEATURE_SET_ACTIVE (cpu_features, VAES);
155 /* Determine if VPCLMULQDQ is usable. */
156 CPU_FEATURE_SET_ACTIVE (cpu_features, VPCLMULQDQ);
157 /* Determine if XOP is usable. */
158 CPU_FEATURE_SET_ACTIVE (cpu_features, XOP);
159 /* Determine if F16C is usable. */
160 CPU_FEATURE_SET_ACTIVE (cpu_features, F16C);
161 }
162
163 /* Check if OPMASK state, upper 256-bit of ZMM0-ZMM15 and
164 ZMM16-ZMM31 state are enabled. */
165 if ((xcrlow & (bit_Opmask_state | bit_ZMM0_15_state
166 | bit_ZMM16_31_state))
167 == (bit_Opmask_state | bit_ZMM0_15_state | bit_ZMM16_31_state))
168 {
169 /* Determine if AVX512F is usable. */
170 if (CPU_FEATURES_CPU_P (cpu_features, AVX512F))
171 {
172 CPU_FEATURE_SET (cpu_features, AVX512F);
173 /* Determine if AVX512CD is usable. */
174 CPU_FEATURE_SET_ACTIVE (cpu_features, AVX512CD);
175 /* Determine if AVX512ER is usable. */
176 CPU_FEATURE_SET_ACTIVE (cpu_features, AVX512ER);
177 /* Determine if AVX512PF is usable. */
178 CPU_FEATURE_SET_ACTIVE (cpu_features, AVX512PF);
179 /* Determine if AVX512VL is usable. */
180 CPU_FEATURE_SET_ACTIVE (cpu_features, AVX512VL);
181 /* Determine if AVX512DQ is usable. */
182 CPU_FEATURE_SET_ACTIVE (cpu_features, AVX512DQ);
183 /* Determine if AVX512BW is usable. */
184 CPU_FEATURE_SET_ACTIVE (cpu_features, AVX512BW);
185 /* Determine if AVX512_4FMAPS is usable. */
186 CPU_FEATURE_SET_ACTIVE (cpu_features, AVX512_4FMAPS);
187 /* Determine if AVX512_4VNNIW is usable. */
188 CPU_FEATURE_SET_ACTIVE (cpu_features, AVX512_4VNNIW);
189 /* Determine if AVX512_BITALG is usable. */
190 CPU_FEATURE_SET_ACTIVE (cpu_features, AVX512_BITALG);
191 /* Determine if AVX512_IFMA is usable. */
192 CPU_FEATURE_SET_ACTIVE (cpu_features, AVX512_IFMA);
193 /* Determine if AVX512_VBMI is usable. */
194 CPU_FEATURE_SET_ACTIVE (cpu_features, AVX512_VBMI);
195 /* Determine if AVX512_VBMI2 is usable. */
196 CPU_FEATURE_SET_ACTIVE (cpu_features, AVX512_VBMI2);
197 /* Determine if is AVX512_VNNI usable. */
198 CPU_FEATURE_SET_ACTIVE (cpu_features, AVX512_VNNI);
199 /* Determine if AVX512_VPOPCNTDQ is usable. */
200 CPU_FEATURE_SET_ACTIVE (cpu_features,
201 AVX512_VPOPCNTDQ);
202 /* Determine if AVX512_VP2INTERSECT is usable. */
203 CPU_FEATURE_SET_ACTIVE (cpu_features,
204 AVX512_VP2INTERSECT);
205 /* Determine if AVX512_BF16 is usable. */
206 CPU_FEATURE_SET_ACTIVE (cpu_features, AVX512_BF16);
207 /* Determine if AVX512_FP16 is usable. */
208 CPU_FEATURE_SET_ACTIVE (cpu_features, AVX512_FP16);
209 }
210 }
211 }
212
213 /* Are XTILECFG and XTILEDATA states usable? */
214 if ((xcrlow & (bit_XTILECFG_state | bit_XTILEDATA_state))
215 == (bit_XTILECFG_state | bit_XTILEDATA_state))
216 {
217 /* Determine if AMX_BF16 is usable. */
218 CPU_FEATURE_SET_ACTIVE (cpu_features, AMX_BF16);
219 /* Determine if AMX_TILE is usable. */
220 CPU_FEATURE_SET_ACTIVE (cpu_features, AMX_TILE);
221 /* Determine if AMX_INT8 is usable. */
222 CPU_FEATURE_SET_ACTIVE (cpu_features, AMX_INT8);
223 /* Determine if AMX_FP16 is usable. */
224 CPU_FEATURE_SET_ACTIVE (cpu_features, AMX_FP16);
225 /* Determine if AMX_COMPLEX is usable. */
226 CPU_FEATURE_SET_ACTIVE (cpu_features, AMX_COMPLEX);
227 }
228
229 /* APX is usable only if the APX state is supported by kernel. */
230 if ((xcrlow & bit_APX_state) != 0)
231 CPU_FEATURE_SET_ACTIVE (cpu_features, APX_F);
232
233 /* These features are usable only when OSXSAVE is enabled. */
234 CPU_FEATURE_SET (cpu_features, XSAVE);
235 CPU_FEATURE_SET_ACTIVE (cpu_features, XSAVEOPT);
236 CPU_FEATURE_SET_ACTIVE (cpu_features, XSAVEC);
237 CPU_FEATURE_SET_ACTIVE (cpu_features, XGETBV_ECX_1);
238 CPU_FEATURE_SET_ACTIVE (cpu_features, XFD);
239
240 /* For _dl_runtime_resolve, set xsave_state_size to xsave area
241 size + integer register save size and align it to 64 bytes. */
242 if (cpu_features->basic.max_cpuid >= 0xd)
243 {
244 unsigned int eax, ebx, ecx, edx;
245
246 __cpuid_count (0xd, 0, eax, ebx, ecx, edx);
247 if (ebx != 0)
248 {
249 unsigned int xsave_state_full_size
250 = ALIGN_UP (ebx + STATE_SAVE_OFFSET, 64);
251
252 cpu_features->xsave_state_size
253 = xsave_state_full_size;
254 cpu_features->xsave_state_full_size
255 = xsave_state_full_size;
256
257 /* Check if XSAVEC is available. */
258 if (CPU_FEATURES_CPU_P (cpu_features, XSAVEC))
259 {
260 unsigned int xstate_comp_offsets[32];
261 unsigned int xstate_comp_sizes[32];
262 unsigned int i;
263
264 xstate_comp_offsets[0] = 0;
265 xstate_comp_offsets[1] = 160;
266 xstate_comp_offsets[2] = 576;
267 xstate_comp_sizes[0] = 160;
268 xstate_comp_sizes[1] = 256;
269
270 for (i = 2; i < 32; i++)
271 {
272 if ((STATE_SAVE_MASK & (1 << i)) != 0)
273 {
274 __cpuid_count (0xd, i, eax, ebx, ecx, edx);
275 xstate_comp_sizes[i] = eax;
276 }
277 else
278 {
279 ecx = 0;
280 xstate_comp_sizes[i] = 0;
281 }
282
283 if (i > 2)
284 {
285 xstate_comp_offsets[i]
286 = (xstate_comp_offsets[i - 1]
287 + xstate_comp_sizes[i -1]);
288 if ((ecx & (1 << 1)) != 0)
289 xstate_comp_offsets[i]
290 = ALIGN_UP (xstate_comp_offsets[i], 64);
291 }
292 }
293
294 /* Use XSAVEC. */
295 unsigned int size
296 = xstate_comp_offsets[31] + xstate_comp_sizes[31];
297 if (size)
298 {
299 cpu_features->xsave_state_size
300 = ALIGN_UP (size + STATE_SAVE_OFFSET, 64);
301 CPU_FEATURE_SET (cpu_features, XSAVEC);
302 }
303 }
304 }
305 }
306 }
307
308 /* Determine if PKU is usable. */
309 if (CPU_FEATURES_CPU_P (cpu_features, OSPKE))
310 CPU_FEATURE_SET (cpu_features, PKU);
311
312 /* Determine if Key Locker instructions are usable. */
313 if (CPU_FEATURES_CPU_P (cpu_features, AESKLE))
314 {
315 CPU_FEATURE_SET (cpu_features, AESKLE);
316 CPU_FEATURE_SET_ACTIVE (cpu_features, KL);
317 CPU_FEATURE_SET_ACTIVE (cpu_features, WIDE_KL);
318 }
319
320 dl_check_hwcap2 (cpu_features);
321
322 cpu_features->isa_1 = get_isa_level (cpu_features);
323 }
324
325 static void
326 get_extended_indices (struct cpu_features *cpu_features)
327 {
328 unsigned int eax, ebx, ecx, edx;
329 __cpuid (0x80000000, eax, ebx, ecx, edx);
330 if (eax >= 0x80000001)
331 __cpuid (0x80000001,
332 cpu_features->features[CPUID_INDEX_80000001].cpuid.eax,
333 cpu_features->features[CPUID_INDEX_80000001].cpuid.ebx,
334 cpu_features->features[CPUID_INDEX_80000001].cpuid.ecx,
335 cpu_features->features[CPUID_INDEX_80000001].cpuid.edx);
336 if (eax >= 0x80000007)
337 __cpuid (0x80000007,
338 cpu_features->features[CPUID_INDEX_80000007].cpuid.eax,
339 cpu_features->features[CPUID_INDEX_80000007].cpuid.ebx,
340 cpu_features->features[CPUID_INDEX_80000007].cpuid.ecx,
341 cpu_features->features[CPUID_INDEX_80000007].cpuid.edx);
342 if (eax >= 0x80000008)
343 __cpuid (0x80000008,
344 cpu_features->features[CPUID_INDEX_80000008].cpuid.eax,
345 cpu_features->features[CPUID_INDEX_80000008].cpuid.ebx,
346 cpu_features->features[CPUID_INDEX_80000008].cpuid.ecx,
347 cpu_features->features[CPUID_INDEX_80000008].cpuid.edx);
348 }
349
350 static void
351 get_common_indices (struct cpu_features *cpu_features,
352 unsigned int *family, unsigned int *model,
353 unsigned int *extended_model, unsigned int *stepping)
354 {
355 if (family)
356 {
357 unsigned int eax;
358 __cpuid (1, eax,
359 cpu_features->features[CPUID_INDEX_1].cpuid.ebx,
360 cpu_features->features[CPUID_INDEX_1].cpuid.ecx,
361 cpu_features->features[CPUID_INDEX_1].cpuid.edx);
362 cpu_features->features[CPUID_INDEX_1].cpuid.eax = eax;
363 *family = (eax >> 8) & 0x0f;
364 *model = (eax >> 4) & 0x0f;
365 *extended_model = (eax >> 12) & 0xf0;
366 *stepping = eax & 0x0f;
367 if (*family == 0x0f)
368 {
369 *family += (eax >> 20) & 0xff;
370 *model += *extended_model;
371 }
372 }
373
374 if (cpu_features->basic.max_cpuid >= 7)
375 {
376 __cpuid_count (7, 0,
377 cpu_features->features[CPUID_INDEX_7].cpuid.eax,
378 cpu_features->features[CPUID_INDEX_7].cpuid.ebx,
379 cpu_features->features[CPUID_INDEX_7].cpuid.ecx,
380 cpu_features->features[CPUID_INDEX_7].cpuid.edx);
381 __cpuid_count (7, 1,
382 cpu_features->features[CPUID_INDEX_7_ECX_1].cpuid.eax,
383 cpu_features->features[CPUID_INDEX_7_ECX_1].cpuid.ebx,
384 cpu_features->features[CPUID_INDEX_7_ECX_1].cpuid.ecx,
385 cpu_features->features[CPUID_INDEX_7_ECX_1].cpuid.edx);
386 }
387
388 if (cpu_features->basic.max_cpuid >= 0xd)
389 __cpuid_count (0xd, 1,
390 cpu_features->features[CPUID_INDEX_D_ECX_1].cpuid.eax,
391 cpu_features->features[CPUID_INDEX_D_ECX_1].cpuid.ebx,
392 cpu_features->features[CPUID_INDEX_D_ECX_1].cpuid.ecx,
393 cpu_features->features[CPUID_INDEX_D_ECX_1].cpuid.edx);
394
395 if (cpu_features->basic.max_cpuid >= 0x14)
396 __cpuid_count (0x14, 0,
397 cpu_features->features[CPUID_INDEX_14_ECX_0].cpuid.eax,
398 cpu_features->features[CPUID_INDEX_14_ECX_0].cpuid.ebx,
399 cpu_features->features[CPUID_INDEX_14_ECX_0].cpuid.ecx,
400 cpu_features->features[CPUID_INDEX_14_ECX_0].cpuid.edx);
401
402 if (cpu_features->basic.max_cpuid >= 0x19)
403 __cpuid_count (0x19, 0,
404 cpu_features->features[CPUID_INDEX_19].cpuid.eax,
405 cpu_features->features[CPUID_INDEX_19].cpuid.ebx,
406 cpu_features->features[CPUID_INDEX_19].cpuid.ecx,
407 cpu_features->features[CPUID_INDEX_19].cpuid.edx);
408
409 dl_check_minsigstacksize (cpu_features);
410 }
411
412 _Static_assert (((index_arch_Fast_Unaligned_Load
413 == index_arch_Fast_Unaligned_Copy)
414 && (index_arch_Fast_Unaligned_Load
415 == index_arch_Prefer_PMINUB_for_stringop)
416 && (index_arch_Fast_Unaligned_Load
417 == index_arch_Slow_SSE4_2)
418 && (index_arch_Fast_Unaligned_Load
419 == index_arch_Fast_Rep_String)
420 && (index_arch_Fast_Unaligned_Load
421 == index_arch_Fast_Copy_Backward)),
422 "Incorrect index_arch_Fast_Unaligned_Load");
423
424
425 /* Intel Family-6 microarch list. */
426 enum
427 {
428 /* Atom processors. */
429 INTEL_ATOM_BONNELL,
430 INTEL_ATOM_SILVERMONT,
431 INTEL_ATOM_AIRMONT,
432 INTEL_ATOM_GOLDMONT,
433 INTEL_ATOM_GOLDMONT_PLUS,
434 INTEL_ATOM_SIERRAFOREST,
435 INTEL_ATOM_GRANDRIDGE,
436 INTEL_ATOM_TREMONT,
437
438 /* Bigcore processors. */
439 INTEL_BIGCORE_MEROM,
440 INTEL_BIGCORE_PENRYN,
441 INTEL_BIGCORE_DUNNINGTON,
442 INTEL_BIGCORE_NEHALEM,
443 INTEL_BIGCORE_WESTMERE,
444 INTEL_BIGCORE_SANDYBRIDGE,
445 INTEL_BIGCORE_IVYBRIDGE,
446 INTEL_BIGCORE_HASWELL,
447 INTEL_BIGCORE_BROADWELL,
448 INTEL_BIGCORE_SKYLAKE,
449 INTEL_BIGCORE_KABYLAKE,
450 INTEL_BIGCORE_COMETLAKE,
451 INTEL_BIGCORE_SKYLAKE_AVX512,
452 INTEL_BIGCORE_CANNONLAKE,
453 INTEL_BIGCORE_ICELAKE,
454 INTEL_BIGCORE_TIGERLAKE,
455 INTEL_BIGCORE_ROCKETLAKE,
456 INTEL_BIGCORE_SAPPHIRERAPIDS,
457 INTEL_BIGCORE_RAPTORLAKE,
458 INTEL_BIGCORE_EMERALDRAPIDS,
459 INTEL_BIGCORE_METEORLAKE,
460 INTEL_BIGCORE_LUNARLAKE,
461 INTEL_BIGCORE_ARROWLAKE,
462 INTEL_BIGCORE_GRANITERAPIDS,
463
464 /* Mixed (bigcore + atom SOC). */
465 INTEL_MIXED_LAKEFIELD,
466 INTEL_MIXED_ALDERLAKE,
467
468 /* KNL. */
469 INTEL_KNIGHTS_MILL,
470 INTEL_KNIGHTS_LANDING,
471
472 /* Unknown. */
473 INTEL_UNKNOWN,
474 };
475
476 static unsigned int
477 intel_get_fam6_microarch (unsigned int model,
478 __attribute__ ((unused)) unsigned int stepping)
479 {
480 switch (model)
481 {
482 case 0x1C:
483 case 0x26:
484 return INTEL_ATOM_BONNELL;
485 case 0x27:
486 case 0x35:
487 case 0x36:
488 /* Really Saltwell, but Saltwell is just a die shrink of Bonnell
489 (microarchitecturally identical). */
490 return INTEL_ATOM_BONNELL;
491 case 0x37:
492 case 0x4A:
493 case 0x4D:
494 case 0x5D:
495 return INTEL_ATOM_SILVERMONT;
496 case 0x4C:
497 case 0x5A:
498 case 0x75:
499 return INTEL_ATOM_AIRMONT;
500 case 0x5C:
501 case 0x5F:
502 return INTEL_ATOM_GOLDMONT;
503 case 0x7A:
504 return INTEL_ATOM_GOLDMONT_PLUS;
505 case 0xAF:
506 return INTEL_ATOM_SIERRAFOREST;
507 case 0xB6:
508 return INTEL_ATOM_GRANDRIDGE;
509 case 0x86:
510 case 0x96:
511 case 0x9C:
512 return INTEL_ATOM_TREMONT;
513 case 0x0F:
514 case 0x16:
515 return INTEL_BIGCORE_MEROM;
516 case 0x17:
517 return INTEL_BIGCORE_PENRYN;
518 case 0x1D:
519 return INTEL_BIGCORE_DUNNINGTON;
520 case 0x1A:
521 case 0x1E:
522 case 0x1F:
523 case 0x2E:
524 return INTEL_BIGCORE_NEHALEM;
525 case 0x25:
526 case 0x2C:
527 case 0x2F:
528 return INTEL_BIGCORE_WESTMERE;
529 case 0x2A:
530 case 0x2D:
531 return INTEL_BIGCORE_SANDYBRIDGE;
532 case 0x3A:
533 case 0x3E:
534 return INTEL_BIGCORE_IVYBRIDGE;
535 case 0x3C:
536 case 0x3F:
537 case 0x45:
538 case 0x46:
539 return INTEL_BIGCORE_HASWELL;
540 case 0x3D:
541 case 0x47:
542 case 0x4F:
543 case 0x56:
544 return INTEL_BIGCORE_BROADWELL;
545 case 0x4E:
546 case 0x5E:
547 return INTEL_BIGCORE_SKYLAKE;
548 case 0x8E:
549 /*
550 Stepping = {9}
551 -> Amberlake
552 Stepping = {10}
553 -> Coffeelake
554 Stepping = {11, 12}
555 -> Whiskeylake
556 else
557 -> Kabylake
558
559 All of these are derivatives of Kabylake (Skylake client).
560 */
561 return INTEL_BIGCORE_KABYLAKE;
562 case 0x9E:
563 /*
564 Stepping = {10, 11, 12, 13}
565 -> Coffeelake
566 else
567 -> Kabylake
568
569 Coffeelake is a derivatives of Kabylake (Skylake client).
570 */
571 return INTEL_BIGCORE_KABYLAKE;
572 case 0xA5:
573 case 0xA6:
574 return INTEL_BIGCORE_COMETLAKE;
575 case 0x66:
576 return INTEL_BIGCORE_CANNONLAKE;
577 case 0x55:
578 /*
579 Stepping = {6, 7}
580 -> Cascadelake
581 Stepping = {11}
582 -> Cooperlake
583 else
584 -> Skylake-avx512
585
586 These are all microarchitecturally identical, so use
587 Skylake-avx512 for all of them.
588 */
589 return INTEL_BIGCORE_SKYLAKE_AVX512;
590 case 0x6A:
591 case 0x6C:
592 case 0x7D:
593 case 0x7E:
594 case 0x9D:
595 return INTEL_BIGCORE_ICELAKE;
596 case 0x8C:
597 case 0x8D:
598 return INTEL_BIGCORE_TIGERLAKE;
599 case 0xA7:
600 return INTEL_BIGCORE_ROCKETLAKE;
601 case 0x8F:
602 return INTEL_BIGCORE_SAPPHIRERAPIDS;
603 case 0xB7:
604 case 0xBA:
605 case 0xBF:
606 return INTEL_BIGCORE_RAPTORLAKE;
607 case 0xCF:
608 return INTEL_BIGCORE_EMERALDRAPIDS;
609 case 0xAA:
610 case 0xAC:
611 return INTEL_BIGCORE_METEORLAKE;
612 case 0xbd:
613 return INTEL_BIGCORE_LUNARLAKE;
614 case 0xc6:
615 return INTEL_BIGCORE_ARROWLAKE;
616 case 0xAD:
617 case 0xAE:
618 return INTEL_BIGCORE_GRANITERAPIDS;
619 case 0x8A:
620 return INTEL_MIXED_LAKEFIELD;
621 case 0x97:
622 case 0x9A:
623 case 0xBE:
624 return INTEL_MIXED_ALDERLAKE;
625 case 0x85:
626 return INTEL_KNIGHTS_MILL;
627 case 0x57:
628 return INTEL_KNIGHTS_LANDING;
629 default:
630 return INTEL_UNKNOWN;
631 }
632 }
633
634 static inline void
635 init_cpu_features (struct cpu_features *cpu_features)
636 {
637 unsigned int ebx, ecx, edx;
638 unsigned int family = 0;
639 unsigned int model = 0;
640 unsigned int stepping = 0;
641 enum cpu_features_kind kind;
642
643 cpu_features->cachesize_non_temporal_divisor = 4;
644 #if !HAS_CPUID
645 if (__get_cpuid_max (0, 0) == 0)
646 {
647 kind = arch_kind_other;
648 goto no_cpuid;
649 }
650 #endif
651
652 __cpuid (0, cpu_features->basic.max_cpuid, ebx, ecx, edx);
653
654 /* This spells out "GenuineIntel". */
655 if (ebx == 0x756e6547 && ecx == 0x6c65746e && edx == 0x49656e69)
656 {
657 unsigned int extended_model;
658
659 kind = arch_kind_intel;
660
661 get_common_indices (cpu_features, &family, &model, &extended_model,
662 &stepping);
663
664 get_extended_indices (cpu_features);
665
666 update_active (cpu_features);
667
668 if (family == 0x06)
669 {
670 model += extended_model;
671 unsigned int microarch
672 = intel_get_fam6_microarch (model, stepping);
673
674 switch (microarch)
675 {
676 /* Atom / KNL tuning. */
677 case INTEL_ATOM_BONNELL:
678 /* BSF is slow on Bonnell. */
679 cpu_features->preferred[index_arch_Slow_BSF]
680 |= bit_arch_Slow_BSF;
681 break;
682
683 /* Unaligned load versions are faster than SSSE3
684 on Airmont, Silvermont, Goldmont, and Goldmont Plus. */
685 case INTEL_ATOM_AIRMONT:
686 case INTEL_ATOM_SILVERMONT:
687 case INTEL_ATOM_GOLDMONT:
688 case INTEL_ATOM_GOLDMONT_PLUS:
689
690 /* Knights Landing. Enable Silvermont optimizations. */
691 case INTEL_KNIGHTS_LANDING:
692
693 cpu_features->preferred[index_arch_Fast_Unaligned_Load]
694 |= (bit_arch_Fast_Unaligned_Load
695 | bit_arch_Fast_Unaligned_Copy
696 | bit_arch_Prefer_PMINUB_for_stringop
697 | bit_arch_Slow_SSE4_2);
698 break;
699
700 case INTEL_ATOM_TREMONT:
701 /* Enable rep string instructions, unaligned load, unaligned
702 copy, pminub and avoid SSE 4.2 on Tremont. */
703 cpu_features->preferred[index_arch_Fast_Rep_String]
704 |= (bit_arch_Fast_Rep_String
705 | bit_arch_Fast_Unaligned_Load
706 | bit_arch_Fast_Unaligned_Copy
707 | bit_arch_Prefer_PMINUB_for_stringop
708 | bit_arch_Slow_SSE4_2);
709 break;
710
711 /*
712 Default tuned Knights microarch.
713 case INTEL_KNIGHTS_MILL:
714 */
715
716 /*
717 Default tuned atom microarch.
718 case INTEL_ATOM_SIERRAFOREST:
719 case INTEL_ATOM_GRANDRIDGE:
720 */
721
722 /* Bigcore/Default Tuning. */
723 default:
724 default_tuning:
725 /* Unknown family 0x06 processors. Assuming this is one
726 of Core i3/i5/i7 processors if AVX is available. */
727 if (!CPU_FEATURES_CPU_P (cpu_features, AVX))
728 break;
729
730 enable_modern_features:
731 /* Rep string instructions, unaligned load, unaligned copy,
732 and pminub are fast on Intel Core i3, i5 and i7. */
733 cpu_features->preferred[index_arch_Fast_Rep_String]
734 |= (bit_arch_Fast_Rep_String
735 | bit_arch_Fast_Unaligned_Load
736 | bit_arch_Fast_Unaligned_Copy
737 | bit_arch_Prefer_PMINUB_for_stringop);
738 break;
739
740 case INTEL_BIGCORE_NEHALEM:
741 case INTEL_BIGCORE_WESTMERE:
742 /* Older CPUs prefer non-temporal stores at lower threshold. */
743 cpu_features->cachesize_non_temporal_divisor = 8;
744 goto enable_modern_features;
745
746 /* Older Bigcore microarch (smaller non-temporal store
747 threshold). */
748 case INTEL_BIGCORE_SANDYBRIDGE:
749 case INTEL_BIGCORE_IVYBRIDGE:
750 case INTEL_BIGCORE_HASWELL:
751 case INTEL_BIGCORE_BROADWELL:
752 cpu_features->cachesize_non_temporal_divisor = 8;
753 goto default_tuning;
754
755 /* Newer Bigcore microarch (larger non-temporal store
756 threshold). */
757 case INTEL_BIGCORE_SKYLAKE:
758 case INTEL_BIGCORE_KABYLAKE:
759 case INTEL_BIGCORE_COMETLAKE:
760 case INTEL_BIGCORE_SKYLAKE_AVX512:
761 case INTEL_BIGCORE_CANNONLAKE:
762 case INTEL_BIGCORE_ICELAKE:
763 case INTEL_BIGCORE_TIGERLAKE:
764 case INTEL_BIGCORE_ROCKETLAKE:
765 case INTEL_BIGCORE_RAPTORLAKE:
766 case INTEL_BIGCORE_METEORLAKE:
767 case INTEL_BIGCORE_LUNARLAKE:
768 case INTEL_BIGCORE_ARROWLAKE:
769 case INTEL_BIGCORE_SAPPHIRERAPIDS:
770 case INTEL_BIGCORE_EMERALDRAPIDS:
771 case INTEL_BIGCORE_GRANITERAPIDS:
772 cpu_features->cachesize_non_temporal_divisor = 2;
773 goto default_tuning;
774
775 /* Default tuned Mixed (bigcore + atom SOC). */
776 case INTEL_MIXED_LAKEFIELD:
777 case INTEL_MIXED_ALDERLAKE:
778 cpu_features->cachesize_non_temporal_divisor = 2;
779 goto default_tuning;
780 }
781
782 /* Disable TSX on some processors to avoid TSX on kernels that
783 weren't updated with the latest microcode package (which
784 disables broken feature by default). */
785 switch (microarch)
786 {
787 case INTEL_BIGCORE_SKYLAKE_AVX512:
788 /* 0x55 (Skylake-avx512) && stepping <= 5 disable TSX. */
789 if (stepping <= 5)
790 goto disable_tsx;
791 break;
792
793 case INTEL_BIGCORE_KABYLAKE:
794 /* NB: Although the errata documents that for model == 0x8e
795 (kabylake skylake client), only 0xb stepping or lower are
796 impacted, the intention of the errata was to disable TSX on
797 all client processors on all steppings. Include 0xc
798 stepping which is an Intel Core i7-8665U, a client mobile
799 processor. */
800 if (stepping > 0xc)
801 break;
802 /* Fall through. */
803 case INTEL_BIGCORE_SKYLAKE:
804 /* Disable Intel TSX and enable RTM_ALWAYS_ABORT for
805 processors listed in:
806
807 https://www.intel.com/content/www/us/en/support/articles/000059422/processors.html
808 */
809 disable_tsx:
810 CPU_FEATURE_UNSET (cpu_features, HLE);
811 CPU_FEATURE_UNSET (cpu_features, RTM);
812 CPU_FEATURE_SET (cpu_features, RTM_ALWAYS_ABORT);
813 break;
814
815 case INTEL_BIGCORE_HASWELL:
816 /* Xeon E7 v3 (model == 0x3f) with stepping >= 4 has working
817 TSX. Haswell also include other model numbers that have
818 working TSX. */
819 if (model == 0x3f && stepping >= 4)
820 break;
821
822 CPU_FEATURE_UNSET (cpu_features, RTM);
823 break;
824 }
825 }
826
827
828 /* Since AVX512ER is unique to Xeon Phi, set Prefer_No_VZEROUPPER
829 if AVX512ER is available. Don't use AVX512 to avoid lower CPU
830 frequency if AVX512ER isn't available. */
831 if (CPU_FEATURES_CPU_P (cpu_features, AVX512ER))
832 cpu_features->preferred[index_arch_Prefer_No_VZEROUPPER]
833 |= bit_arch_Prefer_No_VZEROUPPER;
834 else
835 {
836 /* Processors with AVX512 and AVX-VNNI won't lower CPU frequency
837 when ZMM load and store instructions are used. */
838 if (!CPU_FEATURES_CPU_P (cpu_features, AVX_VNNI))
839 cpu_features->preferred[index_arch_Prefer_No_AVX512]
840 |= bit_arch_Prefer_No_AVX512;
841
842 /* Avoid RTM abort triggered by VZEROUPPER inside a
843 transactionally executing RTM region. */
844 if (CPU_FEATURE_USABLE_P (cpu_features, RTM))
845 cpu_features->preferred[index_arch_Prefer_No_VZEROUPPER]
846 |= bit_arch_Prefer_No_VZEROUPPER;
847 }
848
849 /* Avoid avoid short distance REP MOVSB on processor with FSRM. */
850 if (CPU_FEATURES_CPU_P (cpu_features, FSRM))
851 cpu_features->preferred[index_arch_Avoid_Short_Distance_REP_MOVSB]
852 |= bit_arch_Avoid_Short_Distance_REP_MOVSB;
853 }
854 /* This spells out "AuthenticAMD" or "HygonGenuine". */
855 else if ((ebx == 0x68747541 && ecx == 0x444d4163 && edx == 0x69746e65)
856 || (ebx == 0x6f677948 && ecx == 0x656e6975 && edx == 0x6e65476e))
857 {
858 unsigned int extended_model;
859
860 kind = arch_kind_amd;
861
862 get_common_indices (cpu_features, &family, &model, &extended_model,
863 &stepping);
864
865 get_extended_indices (cpu_features);
866
867 update_active (cpu_features);
868
869 ecx = cpu_features->features[CPUID_INDEX_1].cpuid.ecx;
870
871 if (CPU_FEATURE_USABLE_P (cpu_features, AVX))
872 {
873 /* Since the FMA4 bit is in CPUID_INDEX_80000001 and
874 FMA4 requires AVX, determine if FMA4 is usable here. */
875 CPU_FEATURE_SET_ACTIVE (cpu_features, FMA4);
876 }
877
878 if (family == 0x15)
879 {
880 /* "Excavator" */
881 if (model >= 0x60 && model <= 0x7f)
882 {
883 cpu_features->preferred[index_arch_Fast_Unaligned_Load]
884 |= (bit_arch_Fast_Unaligned_Load
885 | bit_arch_Fast_Copy_Backward);
886
887 /* Unaligned AVX loads are slower.*/
888 cpu_features->preferred[index_arch_AVX_Fast_Unaligned_Load]
889 &= ~bit_arch_AVX_Fast_Unaligned_Load;
890 }
891 }
892 }
893 /* This spells out "CentaurHauls" or " Shanghai ". */
894 else if ((ebx == 0x746e6543 && ecx == 0x736c7561 && edx == 0x48727561)
895 || (ebx == 0x68532020 && ecx == 0x20206961 && edx == 0x68676e61))
896 {
897 unsigned int extended_model, stepping;
898
899 kind = arch_kind_zhaoxin;
900
901 get_common_indices (cpu_features, &family, &model, &extended_model,
902 &stepping);
903
904 get_extended_indices (cpu_features);
905
906 update_active (cpu_features);
907
908 model += extended_model;
909 if (family == 0x6)
910 {
911 if (model == 0xf || model == 0x19)
912 {
913 CPU_FEATURE_UNSET (cpu_features, AVX);
914 CPU_FEATURE_UNSET (cpu_features, AVX2);
915
916 cpu_features->preferred[index_arch_Slow_SSE4_2]
917 |= bit_arch_Slow_SSE4_2;
918
919 cpu_features->preferred[index_arch_AVX_Fast_Unaligned_Load]
920 &= ~bit_arch_AVX_Fast_Unaligned_Load;
921 }
922 }
923 else if (family == 0x7)
924 {
925 if (model == 0x1b)
926 {
927 CPU_FEATURE_UNSET (cpu_features, AVX);
928 CPU_FEATURE_UNSET (cpu_features, AVX2);
929
930 cpu_features->preferred[index_arch_Slow_SSE4_2]
931 |= bit_arch_Slow_SSE4_2;
932
933 cpu_features->preferred[index_arch_AVX_Fast_Unaligned_Load]
934 &= ~bit_arch_AVX_Fast_Unaligned_Load;
935 }
936 else if (model == 0x3b)
937 {
938 CPU_FEATURE_UNSET (cpu_features, AVX);
939 CPU_FEATURE_UNSET (cpu_features, AVX2);
940
941 cpu_features->preferred[index_arch_AVX_Fast_Unaligned_Load]
942 &= ~bit_arch_AVX_Fast_Unaligned_Load;
943 }
944 }
945 }
946 else
947 {
948 kind = arch_kind_other;
949 get_common_indices (cpu_features, NULL, NULL, NULL, NULL);
950 update_active (cpu_features);
951 }
952
953 /* Support i586 if CX8 is available. */
954 if (CPU_FEATURES_CPU_P (cpu_features, CX8))
955 cpu_features->preferred[index_arch_I586] |= bit_arch_I586;
956
957 /* Support i686 if CMOV is available. */
958 if (CPU_FEATURES_CPU_P (cpu_features, CMOV))
959 cpu_features->preferred[index_arch_I686] |= bit_arch_I686;
960
961 #if !HAS_CPUID
962 no_cpuid:
963 #endif
964
965 cpu_features->basic.kind = kind;
966 cpu_features->basic.family = family;
967 cpu_features->basic.model = model;
968 cpu_features->basic.stepping = stepping;
969
970 dl_init_cacheinfo (cpu_features);
971
972 TUNABLE_GET (hwcaps, tunable_val_t *, TUNABLE_CALLBACK (set_hwcaps));
973
974 #ifdef __LP64__
975 TUNABLE_GET (prefer_map_32bit_exec, tunable_val_t *,
976 TUNABLE_CALLBACK (set_prefer_map_32bit_exec));
977 #endif
978
979 bool disable_xsave_features = false;
980
981 if (!CPU_FEATURE_USABLE_P (cpu_features, OSXSAVE))
982 {
983 /* These features are usable only if OSXSAVE is usable. */
984 CPU_FEATURE_UNSET (cpu_features, XSAVE);
985 CPU_FEATURE_UNSET (cpu_features, XSAVEOPT);
986 CPU_FEATURE_UNSET (cpu_features, XSAVEC);
987 CPU_FEATURE_UNSET (cpu_features, XGETBV_ECX_1);
988 CPU_FEATURE_UNSET (cpu_features, XFD);
989
990 disable_xsave_features = true;
991 }
992
993 if (disable_xsave_features
994 || (!CPU_FEATURE_USABLE_P (cpu_features, XSAVE)
995 && !CPU_FEATURE_USABLE_P (cpu_features, XSAVEC)))
996 {
997 /* Clear xsave_state_size if both XSAVE and XSAVEC aren't usable. */
998 cpu_features->xsave_state_size = 0;
999
1000 CPU_FEATURE_UNSET (cpu_features, AVX);
1001 CPU_FEATURE_UNSET (cpu_features, AVX2);
1002 CPU_FEATURE_UNSET (cpu_features, AVX_VNNI);
1003 CPU_FEATURE_UNSET (cpu_features, FMA);
1004 CPU_FEATURE_UNSET (cpu_features, VAES);
1005 CPU_FEATURE_UNSET (cpu_features, VPCLMULQDQ);
1006 CPU_FEATURE_UNSET (cpu_features, XOP);
1007 CPU_FEATURE_UNSET (cpu_features, F16C);
1008 CPU_FEATURE_UNSET (cpu_features, AVX512F);
1009 CPU_FEATURE_UNSET (cpu_features, AVX512CD);
1010 CPU_FEATURE_UNSET (cpu_features, AVX512ER);
1011 CPU_FEATURE_UNSET (cpu_features, AVX512PF);
1012 CPU_FEATURE_UNSET (cpu_features, AVX512VL);
1013 CPU_FEATURE_UNSET (cpu_features, AVX512DQ);
1014 CPU_FEATURE_UNSET (cpu_features, AVX512BW);
1015 CPU_FEATURE_UNSET (cpu_features, AVX512_4FMAPS);
1016 CPU_FEATURE_UNSET (cpu_features, AVX512_4VNNIW);
1017 CPU_FEATURE_UNSET (cpu_features, AVX512_BITALG);
1018 CPU_FEATURE_UNSET (cpu_features, AVX512_IFMA);
1019 CPU_FEATURE_UNSET (cpu_features, AVX512_VBMI);
1020 CPU_FEATURE_UNSET (cpu_features, AVX512_VBMI2);
1021 CPU_FEATURE_UNSET (cpu_features, AVX512_VNNI);
1022 CPU_FEATURE_UNSET (cpu_features, AVX512_VPOPCNTDQ);
1023 CPU_FEATURE_UNSET (cpu_features, AVX512_VP2INTERSECT);
1024 CPU_FEATURE_UNSET (cpu_features, AVX512_BF16);
1025 CPU_FEATURE_UNSET (cpu_features, AVX512_FP16);
1026 CPU_FEATURE_UNSET (cpu_features, AMX_BF16);
1027 CPU_FEATURE_UNSET (cpu_features, AMX_TILE);
1028 CPU_FEATURE_UNSET (cpu_features, AMX_INT8);
1029
1030 CPU_FEATURE_UNSET (cpu_features, FMA4);
1031 }
1032
1033 #ifdef __x86_64__
1034 GLRO(dl_hwcap) = HWCAP_X86_64;
1035 if (cpu_features->basic.kind == arch_kind_intel)
1036 {
1037 const char *platform = NULL;
1038
1039 if (CPU_FEATURE_USABLE_P (cpu_features, AVX512CD))
1040 {
1041 if (CPU_FEATURE_USABLE_P (cpu_features, AVX512ER))
1042 {
1043 if (CPU_FEATURE_USABLE_P (cpu_features, AVX512PF))
1044 platform = "xeon_phi";
1045 }
1046 else
1047 {
1048 if (CPU_FEATURE_USABLE_P (cpu_features, AVX512BW)
1049 && CPU_FEATURE_USABLE_P (cpu_features, AVX512DQ)
1050 && CPU_FEATURE_USABLE_P (cpu_features, AVX512VL))
1051 GLRO(dl_hwcap) |= HWCAP_X86_AVX512_1;
1052 }
1053 }
1054
1055 if (platform == NULL
1056 && CPU_FEATURE_USABLE_P (cpu_features, AVX2)
1057 && CPU_FEATURE_USABLE_P (cpu_features, FMA)
1058 && CPU_FEATURE_USABLE_P (cpu_features, BMI1)
1059 && CPU_FEATURE_USABLE_P (cpu_features, BMI2)
1060 && CPU_FEATURE_USABLE_P (cpu_features, LZCNT)
1061 && CPU_FEATURE_USABLE_P (cpu_features, MOVBE)
1062 && CPU_FEATURE_USABLE_P (cpu_features, POPCNT))
1063 platform = "haswell";
1064
1065 if (platform != NULL)
1066 GLRO(dl_platform) = platform;
1067 }
1068 #else
1069 GLRO(dl_hwcap) = 0;
1070 if (CPU_FEATURE_USABLE_P (cpu_features, SSE2))
1071 GLRO(dl_hwcap) |= HWCAP_X86_SSE2;
1072
1073 if (CPU_FEATURES_ARCH_P (cpu_features, I686))
1074 GLRO(dl_platform) = "i686";
1075 else if (CPU_FEATURES_ARCH_P (cpu_features, I586))
1076 GLRO(dl_platform) = "i586";
1077 #endif
1078
1079 #if CET_ENABLED
1080 TUNABLE_GET (x86_ibt, tunable_val_t *,
1081 TUNABLE_CALLBACK (set_x86_ibt));
1082 TUNABLE_GET (x86_shstk, tunable_val_t *,
1083 TUNABLE_CALLBACK (set_x86_shstk));
1084
1085 /* Check CET status. */
1086 unsigned int cet_status = get_cet_status ();
1087
1088 if ((cet_status & GNU_PROPERTY_X86_FEATURE_1_IBT) == 0)
1089 CPU_FEATURE_UNSET (cpu_features, IBT)
1090 if ((cet_status & GNU_PROPERTY_X86_FEATURE_1_SHSTK) == 0)
1091 CPU_FEATURE_UNSET (cpu_features, SHSTK)
1092
1093 if (cet_status)
1094 {
1095 GL(dl_x86_feature_1) = cet_status;
1096
1097 # ifndef SHARED
1098 /* Check if IBT and SHSTK are enabled by kernel. */
1099 if ((cet_status & GNU_PROPERTY_X86_FEATURE_1_IBT)
1100 || (cet_status & GNU_PROPERTY_X86_FEATURE_1_SHSTK))
1101 {
1102 /* Disable IBT and/or SHSTK if they are enabled by kernel, but
1103 disabled by environment variable:
1104
1105 GLIBC_TUNABLES=glibc.cpu.hwcaps=-IBT,-SHSTK
1106 */
1107 unsigned int cet_feature = 0;
1108 if (!CPU_FEATURE_USABLE (IBT))
1109 cet_feature |= GNU_PROPERTY_X86_FEATURE_1_IBT;
1110 if (!CPU_FEATURE_USABLE (SHSTK))
1111 cet_feature |= GNU_PROPERTY_X86_FEATURE_1_SHSTK;
1112
1113 if (cet_feature)
1114 {
1115 int res = dl_cet_disable_cet (cet_feature);
1116
1117 /* Clear the disabled bits in dl_x86_feature_1. */
1118 if (res == 0)
1119 GL(dl_x86_feature_1) &= ~cet_feature;
1120 }
1121
1122 /* Lock CET if IBT or SHSTK is enabled in executable. Don't
1123 lock CET if IBT or SHSTK is enabled permissively. */
1124 if (GL(dl_x86_feature_control).ibt != cet_permissive
1125 && GL(dl_x86_feature_control).shstk != cet_permissive)
1126 dl_cet_lock_cet ();
1127 }
1128 # endif
1129 }
1130 #endif
1131
1132 #ifndef SHARED
1133 /* NB: In libc.a, call init_cacheinfo. */
1134 init_cacheinfo ();
1135 #endif
1136 }