1 /* Manage function descriptors. Generic version.
2 Copyright (C) 1999-2023 Free Software Foundation, Inc.
3 This file is part of the GNU C Library.
4
5 The GNU C Library is free software; you can redistribute it and/or
6 modify it under the terms of the GNU Lesser General Public
7 License as published by the Free Software Foundation; either
8 version 2.1 of the License, or (at your option) any later version.
9
10 The GNU C Library is distributed in the hope that it will be useful,
11 but WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 Lesser General Public License for more details.
14
15 You should have received a copy of the GNU Lesser General Public
16 License along with the GNU C Library; if not, see
17 <https://www.gnu.org/licenses/>. */
18
19 #include <libintl.h>
20 #include <unistd.h>
21 #include <string.h>
22 #include <sys/param.h>
23 #include <sys/mman.h>
24 #include <link.h>
25 #include <ldsodefs.h>
26 #include <elf/dynamic-link.h>
27 #include <dl-fptr.h>
28 #include <dl-unmap-segments.h>
29 #include <atomic.h>
30
31 #ifndef ELF_MACHINE_BOOT_FPTR_TABLE_LEN
32 /* ELF_MACHINE_BOOT_FPTR_TABLE_LEN should be greater than the number of
33 dynamic symbols in ld.so. */
34 # define ELF_MACHINE_BOOT_FPTR_TABLE_LEN 256
35 #endif
36
37 #ifndef ELF_MACHINE_LOAD_ADDRESS
38 # error "ELF_MACHINE_LOAD_ADDRESS is not defined."
39 #endif
40
41 #ifndef COMPARE_AND_SWAP
42 # define COMPARE_AND_SWAP(ptr, old, new) \
43 (catomic_compare_and_exchange_bool_acq (ptr, new, old) == 0)
44 #endif
45
46 ElfW(Addr) _dl_boot_fptr_table [ELF_MACHINE_BOOT_FPTR_TABLE_LEN];
47
48 static struct local
49 {
50 struct fdesc_table *root;
51 struct fdesc *free_list;
52 unsigned int npages; /* # of pages to allocate */
53 /* the next to members MUST be consecutive! */
54 struct fdesc_table boot_table;
55 struct fdesc boot_fdescs[1024];
56 }
57 local =
58 {
59 .root = &local.boot_table,
60 .npages = 2,
61 .boot_table =
62 {
63 .len = sizeof (local.boot_fdescs) / sizeof (local.boot_fdescs[0]),
64 .first_unused = 0
65 }
66 };
67
68 /* Create a new fdesc table and return a pointer to the first fdesc
69 entry. The fdesc lock must have been acquired already. */
70
71 static struct fdesc_table *
72 new_fdesc_table (struct local *l, size_t *size)
73 {
74 size_t old_npages = l->npages;
75 size_t new_npages = old_npages + old_npages;
76 struct fdesc_table *new_table;
77
78 /* If someone has just created a new table, we return NULL to tell
79 the caller to use the new table. */
80 if (! COMPARE_AND_SWAP (&l->npages, old_npages, new_npages))
81 return (struct fdesc_table *) NULL;
82
83 *size = old_npages * GLRO(dl_pagesize);
84 new_table = __mmap (NULL, *size,
85 PROT_READ | PROT_WRITE, MAP_ANON | MAP_PRIVATE, -1, 0);
86 if (new_table == MAP_FAILED)
87 _dl_signal_error (errno, NULL, NULL,
88 N_("cannot map pages for fdesc table"));
89
90 new_table->len
91 = (*size - sizeof (*new_table)) / sizeof (struct fdesc);
92 new_table->first_unused = 1;
93 return new_table;
94 }
95
96
97 static ElfW(Addr)
98 make_fdesc (ElfW(Addr) ip, ElfW(Addr) gp)
99 {
100 struct fdesc *fdesc = NULL;
101 struct fdesc_table *root;
102 unsigned int old;
103 struct local *l;
104
105 ELF_MACHINE_LOAD_ADDRESS (l, local);
106
107 retry:
108 root = l->root;
109 while (1)
110 {
111 old = root->first_unused;
112 if (old >= root->len)
113 break;
114 else if (COMPARE_AND_SWAP (&root->first_unused, old, old + 1))
115 {
116 fdesc = &root->fdesc[old];
117 goto install;
118 }
119 }
120
121 if (l->free_list)
122 {
123 /* Get it from free-list. */
124 do
125 {
126 fdesc = l->free_list;
127 if (fdesc == NULL)
128 goto retry;
129 }
130 while (! COMPARE_AND_SWAP ((ElfW(Addr) *) &l->free_list,
131 (ElfW(Addr)) fdesc, fdesc->ip));
132 }
133 else
134 {
135 /* Create a new fdesc table. */
136 size_t size;
137 struct fdesc_table *new_table = new_fdesc_table (l, &size);
138
139 if (new_table == NULL)
140 goto retry;
141
142 new_table->next = root;
143 if (! COMPARE_AND_SWAP ((ElfW(Addr) *) &l->root,
144 (ElfW(Addr)) root,
145 (ElfW(Addr)) new_table))
146 {
147 /* Someone has just installed a new table. Return NULL to
148 tell the caller to use the new table. */
149 __munmap (new_table, size);
150 goto retry;
151 }
152
153 /* Note that the first entry was reserved while allocating the
154 memory for the new page. */
155 fdesc = &new_table->fdesc[0];
156 }
157
158 install:
159 fdesc->ip = ip;
160 fdesc->gp = gp;
161
162 return (ElfW(Addr)) fdesc;
163 }
164
165
166 static inline ElfW(Addr) * __attribute__ ((always_inline))
167 make_fptr_table (struct link_map *map)
168 {
169 const ElfW(Sym) *symtab
170 = (const void *) D_PTR (map, l_info[DT_SYMTAB]);
171 const char *strtab = (const void *) D_PTR (map, l_info[DT_STRTAB]);
172 ElfW(Addr) *fptr_table;
173 size_t size;
174 size_t len;
175
176 /* XXX Apparently the only way to find out the size of the dynamic
177 symbol section is to assume that the string table follows right
178 afterwards... */
179 len = ((strtab - (char *) symtab)
180 / map->l_info[DT_SYMENT]->d_un.d_val);
181 size = ((len * sizeof (fptr_table[0]) + GLRO(dl_pagesize) - 1)
182 & -GLRO(dl_pagesize));
183 /* XXX We don't support here in the moment systems without MAP_ANON.
184 There probably are none for IA-64. In case this is proven wrong
185 we will have to open /dev/null here and use the file descriptor
186 instead of the hard-coded -1. */
187 fptr_table = __mmap (NULL, size,
188 PROT_READ | PROT_WRITE, MAP_ANON | MAP_PRIVATE,
189 -1, 0);
190 if (fptr_table == MAP_FAILED)
191 _dl_signal_error (errno, NULL, NULL,
192 N_("cannot map pages for fptr table"));
193
194 if (COMPARE_AND_SWAP ((ElfW(Addr) *) &map->l_mach.fptr_table,
195 (ElfW(Addr)) NULL, (ElfW(Addr)) fptr_table))
196 map->l_mach.fptr_table_len = len;
197 else
198 __munmap (fptr_table, len * sizeof (fptr_table[0]));
199
200 return map->l_mach.fptr_table;
201 }
202
203
204 ElfW(Addr)
205 _dl_make_fptr (struct link_map *map, const ElfW(Sym) *sym,
206 ElfW(Addr) ip)
207 {
208 ElfW(Addr) *ftab = map->l_mach.fptr_table;
209 const ElfW(Sym) *symtab;
210 Elf_Symndx symidx;
211 struct local *l;
212
213 if (__glibc_unlikely (ftab == NULL))
214 ftab = make_fptr_table (map);
215
216 symtab = (const void *) D_PTR (map, l_info[DT_SYMTAB]);
217 symidx = sym - symtab;
218
219 if (symidx >= map->l_mach.fptr_table_len)
220 _dl_signal_error (0, NULL, NULL,
221 N_("internal error: symidx out of range of fptr table"));
222
223 while (ftab[symidx] == 0)
224 {
225 /* GOT has already been relocated in elf_get_dynamic_info -
226 don't try to relocate it again. */
227 ElfW(Addr) fdesc
228 = make_fdesc (ip, map->l_info[DT_PLTGOT]->d_un.d_ptr);
229
230 if (__builtin_expect (COMPARE_AND_SWAP (&ftab[symidx], (ElfW(Addr)) NULL,
231 fdesc), 1))
232 {
233 /* No one has updated the entry and the new function
234 descriptor has been installed. */
235 #if 0
236 const char *strtab
237 = (const void *) D_PTR (map, l_info[DT_STRTAB]);
238
239 ELF_MACHINE_LOAD_ADDRESS (l, local);
240 if (l->root != &l->boot_table
241 || l->boot_table.first_unused > 20)
242 _dl_debug_printf ("created fdesc symbol `%s' at %lx\n",
243 strtab + sym->st_name, ftab[symidx]);
244 #endif
245 break;
246 }
247 else
248 {
249 /* We created a duplicated function descriptor. We put it on
250 free-list. */
251 struct fdesc *f = (struct fdesc *) fdesc;
252
253 ELF_MACHINE_LOAD_ADDRESS (l, local);
254
255 do
256 f->ip = (ElfW(Addr)) l->free_list;
257 while (! COMPARE_AND_SWAP ((ElfW(Addr) *) &l->free_list,
258 f->ip, fdesc));
259 }
260 }
261
262 return ftab[symidx];
263 }
264
265
266 void
267 _dl_unmap (struct link_map *map)
268 {
269 ElfW(Addr) *ftab = map->l_mach.fptr_table;
270 struct fdesc *head = NULL, *tail = NULL;
271 size_t i;
272
273 _dl_unmap_segments (map);
274
275 if (ftab == NULL)
276 return;
277
278 /* String together the fdesc structures that are being freed. */
279 for (i = 0; i < map->l_mach.fptr_table_len; ++i)
280 {
281 if (ftab[i])
282 {
283 *(struct fdesc **) ftab[i] = head;
284 head = (struct fdesc *) ftab[i];
285 if (tail == NULL)
286 tail = head;
287 }
288 }
289
290 /* Prepend the new list to the free_list: */
291 if (tail)
292 do
293 tail->ip = (ElfW(Addr)) local.free_list;
294 while (! COMPARE_AND_SWAP ((ElfW(Addr) *) &local.free_list,
295 tail->ip, (ElfW(Addr)) head));
296
297 __munmap (ftab, (map->l_mach.fptr_table_len
298 * sizeof (map->l_mach.fptr_table[0])));
299
300 map->l_mach.fptr_table = NULL;
301 }
302
303
304 ElfW(Addr)
305 _dl_lookup_address (const void *address)
306 {
307 ElfW(Addr) addr = (ElfW(Addr)) address;
308 struct fdesc_table *t;
309 unsigned long int i;
310
311 for (t = local.root; t != NULL; t = t->next)
312 {
313 i = (struct fdesc *) addr - &t->fdesc[0];
314 if (i < t->first_unused && addr == (ElfW(Addr)) &t->fdesc[i])
315 {
316 addr = t->fdesc[i].ip;
317 break;
318 }
319 }
320
321 return addr;
322 }