1 /* Manage function descriptors. Generic version.
2 Copyright (C) 1999-2023 Free Software Foundation, Inc.
3 This file is part of the GNU C Library.
4
5 The GNU C Library is free software; you can redistribute it and/or
6 modify it under the terms of the GNU Lesser General Public
7 License as published by the Free Software Foundation; either
8 version 2.1 of the License, or (at your option) any later version.
9
10 The GNU C Library is distributed in the hope that it will be useful,
11 but WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 Lesser General Public License for more details.
14
15 You should have received a copy of the GNU Lesser General Public
16 License along with the GNU C Library; if not, write to the Free
17 Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA
18 02111-1307 USA. */
19
20 #include <libintl.h>
21 #include <unistd.h>
22 #include <string.h>
23 #include <sys/param.h>
24 #include <sys/mman.h>
25 #include <link.h>
26 #include <ldsodefs.h>
27 #include <elf/dynamic-link.h>
28 #include <dl-fptr.h>
29 #include <dl-runtime.h>
30 #include <dl-unmap-segments.h>
31 #include <atomic.h>
32 #include <libc-pointer-arith.h>
33
34 #ifndef ELF_MACHINE_BOOT_FPTR_TABLE_LEN
35 /* ELF_MACHINE_BOOT_FPTR_TABLE_LEN should be greater than the number of
36 dynamic symbols in ld.so. */
37 # define ELF_MACHINE_BOOT_FPTR_TABLE_LEN 256
38 #endif
39
40 #ifndef ELF_MACHINE_LOAD_ADDRESS
41 # error "ELF_MACHINE_LOAD_ADDRESS is not defined."
42 #endif
43
44 #ifndef COMPARE_AND_SWAP
45 # define COMPARE_AND_SWAP(ptr, old, new) \
46 (catomic_compare_and_exchange_bool_acq (ptr, new, old) == 0)
47 #endif
48
49 ElfW(Addr) _dl_boot_fptr_table [ELF_MACHINE_BOOT_FPTR_TABLE_LEN];
50
51 static struct local
52 {
53 struct fdesc_table *root;
54 struct fdesc *free_list;
55 unsigned int npages; /* # of pages to allocate */
56 /* the next to members MUST be consecutive! */
57 struct fdesc_table boot_table;
58 struct fdesc boot_fdescs[1024];
59 }
60 local =
61 {
62 #ifdef SHARED
63 /* Address of .boot_table is not known until runtime. */
64 .root = 0,
65 #else
66 .root = &local.boot_table,
67 #endif
68 .npages = 2,
69 .boot_table =
70 {
71 .len = sizeof (local.boot_fdescs) / sizeof (local.boot_fdescs[0]),
72 .first_unused = 0
73 }
74 };
75
76 /* Create a new fdesc table and return a pointer to the first fdesc
77 entry. The fdesc lock must have been acquired already. */
78
79 static struct fdesc_table *
80 new_fdesc_table (struct local *l, size_t *size)
81 {
82 size_t old_npages = l->npages;
83 size_t new_npages = old_npages + old_npages;
84 struct fdesc_table *new_table;
85
86 /* If someone has just created a new table, we return NULL to tell
87 the caller to use the new table. */
88 if (! COMPARE_AND_SWAP (&l->npages, old_npages, new_npages))
89 return (struct fdesc_table *) NULL;
90
91 *size = old_npages * GLRO(dl_pagesize);
92 new_table = __mmap (NULL, *size,
93 PROT_READ | PROT_WRITE, MAP_ANON | MAP_PRIVATE, -1, 0);
94 if (new_table == MAP_FAILED)
95 _dl_signal_error (errno, NULL, NULL,
96 N_("cannot map pages for fdesc table"));
97
98 new_table->len
99 = (*size - sizeof (*new_table)) / sizeof (struct fdesc);
100 new_table->first_unused = 1;
101 return new_table;
102 }
103
104 /* Must call _dl_fptr_init before using any other function. */
105 void
106 _dl_fptr_init (void)
107 {
108 struct local *l;
109
110 ELF_MACHINE_LOAD_ADDRESS (l, local);
111 l->root = &l->boot_table;
112 }
113
114 static ElfW(Addr)
115 make_fdesc (ElfW(Addr) ip, ElfW(Addr) gp)
116 {
117 struct fdesc *fdesc = NULL;
118 struct fdesc_table *root;
119 unsigned int old;
120 struct local *l;
121
122 ELF_MACHINE_LOAD_ADDRESS (l, local);
123
124 retry:
125 root = l->root;
126 while (1)
127 {
128 old = root->first_unused;
129 if (old >= root->len)
130 break;
131 else if (COMPARE_AND_SWAP (&root->first_unused, old, old + 1))
132 {
133 fdesc = &root->fdesc[old];
134 goto install;
135 }
136 }
137
138 if (l->free_list)
139 {
140 /* Get it from free-list. */
141 do
142 {
143 fdesc = l->free_list;
144 if (fdesc == NULL)
145 goto retry;
146 }
147 while (! COMPARE_AND_SWAP ((ElfW(Addr) *) &l->free_list,
148 (ElfW(Addr)) fdesc, fdesc->ip));
149 }
150 else
151 {
152 /* Create a new fdesc table. */
153 size_t size;
154 struct fdesc_table *new_table = new_fdesc_table (l, &size);
155
156 if (new_table == NULL)
157 goto retry;
158
159 new_table->next = root;
160 if (! COMPARE_AND_SWAP ((ElfW(Addr) *) &l->root,
161 (ElfW(Addr)) root,
162 (ElfW(Addr)) new_table))
163 {
164 /* Someone has just installed a new table. Return NULL to
165 tell the caller to use the new table. */
166 __munmap (new_table, size);
167 goto retry;
168 }
169
170 /* Note that the first entry was reserved while allocating the
171 memory for the new page. */
172 fdesc = &new_table->fdesc[0];
173 }
174
175 install:
176 fdesc->gp = gp;
177 fdesc->ip = ip;
178
179 return (ElfW(Addr)) fdesc;
180 }
181
182
183 static inline ElfW(Addr) * __attribute__ ((always_inline))
184 make_fptr_table (struct link_map *map)
185 {
186 const ElfW(Sym) *symtab = (const void *) D_PTR (map, l_info[DT_SYMTAB]);
187 const char *strtab = (const void *) D_PTR (map, l_info[DT_STRTAB]);
188 ElfW(Addr) *fptr_table;
189 size_t size;
190 size_t len;
191 const ElfW(Sym) *symtabend;
192
193 /* Determine the end of the dynamic symbol table using the hash. */
194 if (map->l_info[DT_HASH] != NULL)
195 symtabend = (symtab + ((Elf_Symndx *) D_PTR (map, l_info[DT_HASH]))[1]);
196 else
197 /* There is no direct way to determine the number of symbols in the
198 dynamic symbol table and no hash table is present. The ELF
199 binary is ill-formed but what shall we do? Use the beginning of
200 the string table which generally follows the symbol table. */
201 symtabend = (const ElfW(Sym) *) strtab;
202
203 len = (((char *) symtabend - (char *) symtab)
204 / map->l_info[DT_SYMENT]->d_un.d_val);
205 size = ALIGN_UP (len * sizeof (fptr_table[0]), GLRO(dl_pagesize));
206
207 /* We don't support systems without MAP_ANON. We avoid using malloc
208 because this might get called before malloc is setup. */
209 fptr_table = __mmap (NULL, size,
210 PROT_READ | PROT_WRITE, MAP_ANON | MAP_PRIVATE,
211 -1, 0);
212 if (fptr_table == MAP_FAILED)
213 _dl_signal_error (errno, NULL, NULL,
214 N_("cannot map pages for fptr table"));
215
216 if (COMPARE_AND_SWAP ((ElfW(Addr) *) &map->l_mach.fptr_table,
217 (ElfW(Addr)) NULL, (ElfW(Addr)) fptr_table))
218 map->l_mach.fptr_table_len = len;
219 else
220 __munmap (fptr_table, len * sizeof (fptr_table[0]));
221
222 return map->l_mach.fptr_table;
223 }
224
225
226 ElfW(Addr)
227 _dl_make_fptr (struct link_map *map, const ElfW(Sym) *sym,
228 ElfW(Addr) ip)
229 {
230 ElfW(Addr) *ftab = map->l_mach.fptr_table;
231 const ElfW(Sym) *symtab;
232 Elf_Symndx symidx;
233 struct local *l;
234
235 if (__builtin_expect (ftab == NULL, 0))
236 ftab = make_fptr_table (map);
237
238 symtab = (const void *) D_PTR (map, l_info[DT_SYMTAB]);
239 symidx = sym - symtab;
240
241 if (symidx >= map->l_mach.fptr_table_len)
242 _dl_signal_error (0, NULL, NULL,
243 N_("internal error: symidx out of range of fptr table"));
244
245 while (ftab[symidx] == 0)
246 {
247 /* GOT has already been relocated in elf_get_dynamic_info -
248 don't try to relocate it again. */
249 ElfW(Addr) fdesc
250 = make_fdesc (ip, map->l_info[DT_PLTGOT]->d_un.d_ptr);
251
252 if (__builtin_expect (COMPARE_AND_SWAP (&ftab[symidx], (ElfW(Addr)) NULL,
253 fdesc), 1))
254 {
255 /* No one has updated the entry and the new function
256 descriptor has been installed. */
257 #if 0
258 const char *strtab
259 = (const void *) D_PTR (map, l_info[DT_STRTAB]);
260
261 ELF_MACHINE_LOAD_ADDRESS (l, local);
262 if (l->root != &l->boot_table
263 || l->boot_table.first_unused > 20)
264 _dl_debug_printf ("created fdesc symbol `%s' at %lx\n",
265 strtab + sym->st_name, ftab[symidx]);
266 #endif
267 break;
268 }
269 else
270 {
271 /* We created a duplicated function descriptor. We put it on
272 free-list. */
273 struct fdesc *f = (struct fdesc *) fdesc;
274
275 ELF_MACHINE_LOAD_ADDRESS (l, local);
276
277 do
278 f->ip = (ElfW(Addr)) l->free_list;
279 while (! COMPARE_AND_SWAP ((ElfW(Addr) *) &l->free_list,
280 f->ip, fdesc));
281 }
282 }
283
284 return ftab[symidx];
285 }
286
287
288 void
289 _dl_unmap (struct link_map *map)
290 {
291 ElfW(Addr) *ftab = map->l_mach.fptr_table;
292 struct fdesc *head = NULL, *tail = NULL;
293 size_t i;
294
295 _dl_unmap_segments (map);
296
297 if (ftab == NULL)
298 return;
299
300 /* String together the fdesc structures that are being freed. */
301 for (i = 0; i < map->l_mach.fptr_table_len; ++i)
302 {
303 if (ftab[i])
304 {
305 *(struct fdesc **) ftab[i] = head;
306 head = (struct fdesc *) ftab[i];
307 if (tail == NULL)
308 tail = head;
309 }
310 }
311
312 /* Prepend the new list to the free_list: */
313 if (tail)
314 do
315 tail->ip = (ElfW(Addr)) local.free_list;
316 while (! COMPARE_AND_SWAP ((ElfW(Addr) *) &local.free_list,
317 tail->ip, (ElfW(Addr)) head));
318
319 __munmap (ftab, (map->l_mach.fptr_table_len
320 * sizeof (map->l_mach.fptr_table[0])));
321
322 map->l_mach.fptr_table = NULL;
323 }
324
325 extern ElfW(Addr) _dl_fixup (struct link_map *, ElfW(Word)) attribute_hidden;
326
327 static inline Elf32_Addr
328 elf_machine_resolve (void)
329 {
330 Elf32_Addr addr;
331
332 asm ("b,l 1f,%0\n"
333 " addil L'_dl_runtime_resolve - ($PIC_pcrel$0 - 1),%0\n"
334 "1: ldo R'_dl_runtime_resolve - ($PIC_pcrel$0 - 5)(%%r1),%0\n"
335 : "=r" (addr) : : "r1");
336
337 return addr;
338 }
339
340 static inline int
341 _dl_read_access_allowed (unsigned int *addr)
342 {
343 int result;
344
345 asm ("proberi (%1),3,%0" : "=r" (result) : "r" (addr) : );
346
347 return result;
348 }
349
350 ElfW(Addr)
351 _dl_lookup_address (const void *address)
352 {
353 ElfW(Addr) addr = (ElfW(Addr)) address;
354 ElfW(Word) reloc_arg;
355 unsigned int *desc, *gptr;
356
357 /* Return ADDR if the least-significant two bits of ADDR are not consistent
358 with ADDR being a linker defined function pointer. The normal value for
359 a code address in a backtrace is 3. */
360 if (((uintptr_t) addr & 3) != 2)
361 return addr;
362
363 /* Handle special case where ADDR points to page 0. */
364 if ((uintptr_t) addr < 4096)
365 return addr;
366
367 /* Clear least-significant two bits from descriptor address. */
368 desc = (unsigned int *) ((uintptr_t) addr & ~3);
369 if (!_dl_read_access_allowed (desc))
370 return addr;
371
372 /* First load the relocation offset. */
373 reloc_arg = (ElfW(Word)) desc[1];
374 atomic_full_barrier();
375
376 /* Then load first word of candidate descriptor. It should be a pointer
377 with word alignment and point to memory that can be read. */
378 gptr = (unsigned int *) desc[0];
379 if (((uintptr_t) gptr & 3) != 0
380 || !_dl_read_access_allowed (gptr))
381 return addr;
382
383 /* See if descriptor requires resolution. The following trampoline is
384 used in each global offset table for function resolution:
385
386 ldw 0(r20),r21
387 bv r0(r21)
388 ldw 4(r20),r21
389 tramp: b,l .-12,r20
390 depwi 0,31,2,r20
391 .word _dl_runtime_resolve
392 .word "_dl_runtime_resolve ltp"
393 got: .word _DYNAMIC
394 .word "struct link map address" */
395 if (gptr[0] == 0xea9f1fdd /* b,l .-12,r20 */
396 && gptr[1] == 0xd6801c1e /* depwi 0,31,2,r20 */
397 && (ElfW(Addr)) gptr[2] == elf_machine_resolve ())
398 {
399 struct link_map *l = (struct link_map *) gptr[5];
400
401 /* If gp has been resolved, we need to hunt for relocation offset. */
402 if (!(reloc_arg & PA_GP_RELOC))
403 reloc_arg = _dl_fix_reloc_arg ((struct fdesc *) addr, l);
404
405 _dl_fixup (l, reloc_arg);
406 }
407
408 return (ElfW(Addr)) desc[0];
409 }
410 rtld_hidden_def (_dl_lookup_address)