1 /* Huge Page support. Linux implementation.
2 Copyright (C) 2021-2023 Free Software Foundation, Inc.
3 This file is part of the GNU C Library.
4
5 The GNU C Library is free software; you can redistribute it and/or
6 modify it under the terms of the GNU Lesser General Public License as
7 published by the Free Software Foundation; either version 2.1 of the
8 License, or (at your option) any later version.
9
10 The GNU C Library is distributed in the hope that it will be useful,
11 but WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 Lesser General Public License for more details.
14
15 You should have received a copy of the GNU Lesser General Public
16 License along with the GNU C Library; see the file COPYING.LIB. If
17 not, see <https://www.gnu.org/licenses/>. */
18
19 #include <intprops.h>
20 #include <dirent.h>
21 #include <malloc-hugepages.h>
22 #include <not-cancel.h>
23 #include <sys/mman.h>
24
25 unsigned long int
26 __malloc_default_thp_pagesize (void)
27 {
28 int fd = __open64_nocancel (
29 "/sys/kernel/mm/transparent_hugepage/hpage_pmd_size", O_RDONLY);
30 if (fd == -1)
31 return 0;
32
33 char str[INT_BUFSIZE_BOUND (unsigned long int)];
34 ssize_t s = __read_nocancel (fd, str, sizeof (str));
35 __close_nocancel (fd);
36 if (s < 0)
37 return 0;
38
39 unsigned long int r = 0;
40 for (ssize_t i = 0; i < s; i++)
41 {
42 if (str[i] == '\n')
43 break;
44 r *= 10;
45 r += str[i] - '0';
46 }
47 return r;
48 }
49
50 enum malloc_thp_mode_t
51 __malloc_thp_mode (void)
52 {
53 int fd = __open64_nocancel ("/sys/kernel/mm/transparent_hugepage/enabled",
54 O_RDONLY);
55 if (fd == -1)
56 return malloc_thp_mode_not_supported;
57
58 static const char mode_always[] = "[always] madvise never\n";
59 static const char mode_madvise[] = "always [madvise] never\n";
60 static const char mode_never[] = "always madvise [never]\n";
61
62 char str[sizeof(mode_always)];
63 ssize_t s = __read_nocancel (fd, str, sizeof (str));
64 if (s >= sizeof str || s < 0)
65 return malloc_thp_mode_not_supported;
66 str[s] = '\0';
67 __close_nocancel (fd);
68
69 if (s == sizeof (mode_always) - 1)
70 {
71 if (strcmp (str, mode_always) == 0)
72 return malloc_thp_mode_always;
73 else if (strcmp (str, mode_madvise) == 0)
74 return malloc_thp_mode_madvise;
75 else if (strcmp (str, mode_never) == 0)
76 return malloc_thp_mode_never;
77 }
78 return malloc_thp_mode_not_supported;
79 }
80
81 static size_t
82 malloc_default_hugepage_size (void)
83 {
84 int fd = __open64_nocancel ("/proc/meminfo", O_RDONLY);
85 if (fd == -1)
86 return 0;
87
88 size_t hpsize = 0;
89
90 char buf[512];
91 off64_t off = 0;
92 while (1)
93 {
94 ssize_t r = __pread64_nocancel (fd, buf, sizeof (buf) - 1, off);
95 if (r < 0)
96 break;
97 buf[r] = '\0';
98
99 /* If the tag is not found, read the last line again. */
100 const char *s = strstr (buf, "Hugepagesize:");
101 if (s == NULL)
102 {
103 char *nl = strrchr (buf, '\n');
104 if (nl == NULL)
105 break;
106 off += (nl + 1) - buf;
107 continue;
108 }
109
110 /* The default huge page size is in the form:
111 Hugepagesize: NUMBER kB */
112 s += sizeof ("Hugepagesize: ") - 1;
113 for (int i = 0; (s[i] >= '0' && s[i] <= '9') || s[i] == ' '; i++)
114 {
115 if (s[i] == ' ')
116 continue;
117 hpsize *= 10;
118 hpsize += s[i] - '0';
119 }
120 hpsize *= 1024;
121 break;
122 }
123
124 __close_nocancel (fd);
125
126 return hpsize;
127 }
128
129 static inline int
130 hugepage_flags (size_t pagesize)
131 {
132 return MAP_HUGETLB | (__builtin_ctzll (pagesize) << MAP_HUGE_SHIFT);
133 }
134
135 void
136 __malloc_hugepage_config (size_t requested, size_t *pagesize, int *flags)
137 {
138 *pagesize = 0;
139 *flags = 0;
140
141 if (requested == 0)
142 {
143 *pagesize = malloc_default_hugepage_size ();
144 if (*pagesize != 0)
145 *flags = hugepage_flags (*pagesize);
146 return;
147 }
148
149 /* Each entry represents a supported huge page in the form of:
150 hugepages-<size>kB. */
151 int dirfd = __open64_nocancel ("/sys/kernel/mm/hugepages",
152 O_RDONLY | O_DIRECTORY, 0);
153 if (dirfd == -1)
154 return;
155
156 char buffer[1024];
157 while (true)
158 {
159 #if !IS_IN(libc)
160 # define __getdents64 getdents64
161 #endif
162 ssize_t ret = __getdents64 (dirfd, buffer, sizeof (buffer));
163 if (ret == -1)
164 break;
165 else if (ret == 0)
166 break;
167
168 bool found = false;
169 char *begin = buffer, *end = buffer + ret;
170 while (begin != end)
171 {
172 unsigned short int d_reclen;
173 memcpy (&d_reclen, begin + offsetof (struct dirent64, d_reclen),
174 sizeof (d_reclen));
175 const char *dname = begin + offsetof (struct dirent64, d_name);
176 begin += d_reclen;
177
178 if (dname[0] == '.'
179 || strncmp (dname, "hugepages-", sizeof ("hugepages-") - 1) != 0)
180 continue;
181
182 size_t hpsize = 0;
183 const char *sizestr = dname + sizeof ("hugepages-") - 1;
184 for (int i = 0; sizestr[i] >= '0' && sizestr[i] <= '9'; i++)
185 {
186 hpsize *= 10;
187 hpsize += sizestr[i] - '0';
188 }
189 hpsize *= 1024;
190
191 if (hpsize == requested)
192 {
193 *pagesize = hpsize;
194 *flags = hugepage_flags (*pagesize);
195 found = true;
196 break;
197 }
198 }
199 if (found)
200 break;
201 }
202
203 __close_nocancel (dirfd);
204 }