1 /* Initialize cpu feature data. s390x version.
2 Copyright (C) 2023 Free Software Foundation, Inc.
3 This file is part of the GNU C Library.
4
5 The GNU C Library is free software; you can redistribute it and/or
6 modify it under the terms of the GNU Lesser General Public
7 License as published by the Free Software Foundation; either
8 version 2.1 of the License, or (at your option) any later version.
9
10 The GNU C Library is distributed in the hope that it will be useful,
11 but WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 Lesser General Public License for more details.
14
15 You should have received a copy of the GNU Lesser General Public
16 License along with the GNU C Library; if not, see
17 <https://www.gnu.org/licenses/>. */
18
19 #include <cpu-features.h>
20
21 #include <elf/dl-tunables.h>
22 #include <ifunc-memcmp.h>
23 #include <string.h>
24 extern __typeof (memcmp) MEMCMP_DEFAULT;
25
26 #define S390_COPY_CPU_FEATURES(SRC_PTR, DEST_PTR) \
27 (DEST_PTR)->hwcap = (SRC_PTR)->hwcap; \
28 (DEST_PTR)->stfle_bits[0] = (SRC_PTR)->stfle_bits[0];
29
30 static void
31 TUNABLE_CALLBACK (set_hwcaps) (tunable_val_t *valp)
32 {
33 /* The current IFUNC selection is always using the most recent
34 features which are available via AT_HWCAP or STFLE-bits. But in
35 some scenarios it is useful to adjust this selection.
36
37 The environment variable:
38
39 GLIBC_TUNABLES=glibc.cpu.hwcaps=-xxx,yyy,zzz,....
40
41 can be used to enable HWCAP/STFLE feature yyy, disable HWCAP/STFLE feature
42 xxx, where the feature name is case-sensitive and has to match the ones
43 used below. Furthermore, the ARCH-level zzz can be used to set various
44 HWCAP/STFLE features at once. */
45
46 /* Copy the features from dl_s390_cpu_features, which contains the features
47 provided by AT_HWCAP and stfle-instruction. */
48 struct cpu_features *cpu_features = &GLRO(dl_s390_cpu_features);
49 struct cpu_features cpu_features_orig;
50 S390_COPY_CPU_FEATURES (cpu_features, &cpu_features_orig);
51 struct cpu_features cpu_features_curr;
52 S390_COPY_CPU_FEATURES (cpu_features, &cpu_features_curr);
53
54 const char *token = valp->strval;
55 do
56 {
57 const char *token_end, *feature;
58 bool disable;
59 size_t token_len;
60 size_t feature_len;
61
62 /* Find token separator or end of string. */
63 for (token_end = token; *token_end != ','; token_end++)
64 if (*token_end == '\0')
65 break;
66
67 /* Determine feature. */
68 token_len = token_end - token;
69 if (*token == '-')
70 {
71 disable = true;
72 feature = token + 1;
73 feature_len = token_len - 1;
74 }
75 else
76 {
77 disable = false;
78 feature = token;
79 feature_len = token_len;
80 }
81
82 /* Handle only the features here which are really used in the
83 IFUNC-resolvers. All others are ignored as the values are only used
84 inside glibc. */
85 bool reset_features = false;
86 unsigned long int hwcap_mask = 0UL;
87 unsigned long long stfle_bits0_mask = 0ULL;
88
89 if ((*feature == 'z' || *feature == 'a'))
90 {
91 if ((feature_len == 5 && *feature == 'z'
92 && MEMCMP_DEFAULT (feature, "zEC12", 5) == 0)
93 || (feature_len == 6 && *feature == 'a'
94 && MEMCMP_DEFAULT (feature, "arch10", 6) == 0))
95 {
96 reset_features = true;
97 disable = true;
98 hwcap_mask = HWCAP_S390_VXRS | HWCAP_S390_VXRS_EXT
99 | HWCAP_S390_VXRS_EXT2;
100 stfle_bits0_mask = S390_STFLE_MASK_ARCH13_MIE3;
101 }
102 else if ((feature_len == 3 && *feature == 'z'
103 && MEMCMP_DEFAULT (feature, "z13", 3) == 0)
104 || (feature_len == 6 && *feature == 'a'
105 && MEMCMP_DEFAULT (feature, "arch11", 6) == 0))
106 {
107 reset_features = true;
108 disable = true;
109 hwcap_mask = HWCAP_S390_VXRS_EXT | HWCAP_S390_VXRS_EXT2;
110 stfle_bits0_mask = S390_STFLE_MASK_ARCH13_MIE3;
111 }
112 else if ((feature_len == 3 && *feature == 'z'
113 && MEMCMP_DEFAULT (feature, "z14", 3) == 0)
114 || (feature_len == 6 && *feature == 'a'
115 && MEMCMP_DEFAULT (feature, "arch12", 6) == 0))
116 {
117 reset_features = true;
118 disable = true;
119 hwcap_mask = HWCAP_S390_VXRS_EXT2;
120 stfle_bits0_mask = S390_STFLE_MASK_ARCH13_MIE3;
121 }
122 else if ((feature_len == 3 && *feature == 'z'
123 && (MEMCMP_DEFAULT (feature, "z15", 3) == 0
124 || MEMCMP_DEFAULT (feature, "z16", 3) == 0))
125 || (feature_len == 6
126 && (MEMCMP_DEFAULT (feature, "arch13", 6) == 0
127 || MEMCMP_DEFAULT (feature, "arch14", 6) == 0)))
128 {
129 /* For z15 or newer we don't have to disable something,
130 but we have to reset to the original values. */
131 reset_features = true;
132 }
133 }
134 else if (*feature == 'H')
135 {
136 if (feature_len == 15
137 && MEMCMP_DEFAULT (feature, "HWCAP_S390_VXRS", 15) == 0)
138 {
139 hwcap_mask = HWCAP_S390_VXRS;
140 if (disable)
141 hwcap_mask |= HWCAP_S390_VXRS_EXT | HWCAP_S390_VXRS_EXT2;
142 }
143 else if (feature_len == 19
144 && MEMCMP_DEFAULT (feature, "HWCAP_S390_VXRS_EXT", 19) == 0)
145 {
146 hwcap_mask = HWCAP_S390_VXRS_EXT;
147 if (disable)
148 hwcap_mask |= HWCAP_S390_VXRS_EXT2;
149 else
150 hwcap_mask |= HWCAP_S390_VXRS;
151 }
152 else if (feature_len == 20
153 && MEMCMP_DEFAULT (feature, "HWCAP_S390_VXRS_EXT2", 20) == 0)
154 {
155 hwcap_mask = HWCAP_S390_VXRS_EXT2;
156 if (!disable)
157 hwcap_mask |= HWCAP_S390_VXRS | HWCAP_S390_VXRS_EXT;
158 }
159 }
160 else if (*feature == 'S')
161 {
162 if (feature_len == 10
163 && MEMCMP_DEFAULT (feature, "STFLE_MIE3", 10) == 0)
164 {
165 stfle_bits0_mask = S390_STFLE_MASK_ARCH13_MIE3;
166 }
167 }
168
169 /* Perform the actions determined above. */
170 if (reset_features)
171 {
172 S390_COPY_CPU_FEATURES (&cpu_features_orig, &cpu_features_curr);
173 }
174
175 if (hwcap_mask != 0UL)
176 {
177 if (disable)
178 cpu_features_curr.hwcap &= ~hwcap_mask;
179 else
180 cpu_features_curr.hwcap |= hwcap_mask;
181 }
182
183 if (stfle_bits0_mask != 0ULL)
184 {
185 if (disable)
186 cpu_features_curr.stfle_bits[0] &= ~stfle_bits0_mask;
187 else
188 cpu_features_curr.stfle_bits[0] |= stfle_bits0_mask;
189 }
190
191 /* Jump over current token ... */
192 token += token_len;
193
194 /* ... and skip token separator for next round. */
195 if (*token == ',') token++;
196 }
197 while (*token != '\0');
198
199 /* Copy back the features after checking that no unsupported features were
200 enabled by user. */
201 cpu_features->hwcap = cpu_features_curr.hwcap & cpu_features_orig.hwcap;
202 cpu_features->stfle_bits[0] = cpu_features_curr.stfle_bits[0]
203 & cpu_features_orig.stfle_bits[0];
204 }
205
206 static inline void
207 init_cpu_features (struct cpu_features *cpu_features)
208 {
209 /* Fill cpu_features as passed by kernel and machine. */
210 cpu_features->hwcap = GLRO(dl_hwcap);
211
212 /* We want just 1 double word to be returned. */
213 if (__glibc_likely ((cpu_features->hwcap & HWCAP_S390_STFLE)
214 && (cpu_features->hwcap & HWCAP_S390_ZARCH)
215 && (cpu_features->hwcap & HWCAP_S390_HIGH_GPRS)))
216 {
217 register unsigned long reg0 __asm__("0") = 0;
218 __asm__ __volatile__(".machine push" "\n\t"
219 ".machine \"z9-109\"" "\n\t"
220 ".machinemode \"zarch_nohighgprs\"\n\t"
221 "stfle %0" "\n\t"
222 ".machine pop" "\n"
223 : "=QS" (cpu_features->stfle_bits[0]),
224 "+d" (reg0)
225 : : "cc");
226 }
227 else
228 {
229 cpu_features->stfle_bits[0] = 0ULL;
230 }
231
232 TUNABLE_GET (glibc, cpu, hwcaps, tunable_val_t *, TUNABLE_CALLBACK (set_hwcaps));
233 }