1 ///////////////////////////////////////////////////////////////////////////////
2 //
3 /// \file hardware.c
4 /// \brief Detection of available hardware resources
5 //
6 // Author: Lasse Collin
7 //
8 // This file has been put into the public domain.
9 // You can do whatever you want with this file.
10 //
11 ///////////////////////////////////////////////////////////////////////////////
12
13 #include "private.h"
14
15
16 /// Maximum number of worker threads. This can be set with
17 /// the --threads=NUM command line option.
18 static uint32_t threads_max = 1;
19
20 /// True when the number of threads is automatically determined based
21 /// on the available hardware threads.
22 static bool threads_are_automatic = false;
23
24 /// If true, then try to use multi-threaded mode (if memlimit allows)
25 /// even if only one thread was requested explicitly (-T+1).
26 static bool use_mt_mode_with_one_thread = false;
27
28 /// Memory usage limit for compression
29 static uint64_t memlimit_compress = 0;
30
31 /// Memory usage limit for decompression
32 static uint64_t memlimit_decompress = 0;
33
34 /// Default memory usage for multithreaded modes:
35 ///
36 /// - Default value for --memlimit-compress when automatic number of threads
37 /// is used. However, if the limit wouldn't allow even one thread then
38 /// the limit is ignored in coder.c and one thread will be used anyway.
39 /// This mess is a compromise: we wish to prevent -T0 from using too
40 /// many threads but we also don't want xz to give an error due to
41 /// a memlimit that the user didn't explicitly set.
42 ///
43 /// - Default value for --memlimit-mt-decompress
44 ///
45 /// This value is calculated in hardware_init() and cannot be changed later.
46 static uint64_t memlimit_mt_default;
47
48 /// Memory usage limit for multithreaded decompression. This is a soft limit:
49 /// if reducing the number of threads to one isn't enough to keep memory
50 /// usage below this limit, then one thread is used and this limit is ignored.
51 /// memlimit_decompress is still obeyed.
52 ///
53 /// This can be set with --memlimit-mt-decompress. The default value for
54 /// this is memlimit_mt_default.
55 static uint64_t memlimit_mtdec;
56
57 /// Total amount of physical RAM
58 static uint64_t total_ram;
59
60
61 extern void
62 hardware_threads_set(uint32_t n)
63 {
64 // Reset these to false first and set them to true when appropriate.
65 threads_are_automatic = false;
66 use_mt_mode_with_one_thread = false;
67
68 if (n == 0) {
69 // Automatic number of threads was requested.
70 // If there is only one hardware thread, multi-threaded
71 // mode will still be used if memory limit allows.
72 threads_are_automatic = true;
73 use_mt_mode_with_one_thread = true;
74
75 // If threading support was enabled at build time,
76 // use the number of available CPU cores. Otherwise
77 // use one thread since disabling threading support
78 // omits lzma_cputhreads() from liblzma.
79 #ifdef MYTHREAD_ENABLED
80 threads_max = lzma_cputhreads();
81 if (threads_max == 0)
82 threads_max = 1;
83 #else
84 threads_max = 1;
85 #endif
86 } else if (n == UINT32_MAX) {
87 use_mt_mode_with_one_thread = true;
88 threads_max = 1;
89 } else {
90 threads_max = n;
91 }
92
93 return;
94 }
95
96
97 extern uint32_t
98 hardware_threads_get(void)
99 {
100 return threads_max;
101 }
102
103
104 extern bool
105 hardware_threads_is_mt(void)
106 {
107 #ifdef MYTHREAD_ENABLED
108 return threads_max > 1 || use_mt_mode_with_one_thread;
109 #else
110 return false;
111 #endif
112 }
113
114
115 extern void
116 hardware_memlimit_set(uint64_t new_memlimit,
117 bool set_compress, bool set_decompress, bool set_mtdec,
118 bool is_percentage)
119 {
120 if (is_percentage) {
121 assert(new_memlimit > 0);
122 assert(new_memlimit <= 100);
123 new_memlimit = (uint32_t)new_memlimit * total_ram / 100;
124 }
125
126 if (set_compress) {
127 memlimit_compress = new_memlimit;
128
129 #if SIZE_MAX == UINT32_MAX
130 // FIXME?
131 //
132 // When running a 32-bit xz on a system with a lot of RAM and
133 // using a percentage-based memory limit, the result can be
134 // bigger than the 32-bit address space. Limiting the limit
135 // below SIZE_MAX for compression (not decompression) makes
136 // xz lower the compression settings (or number of threads)
137 // to a level that *might* work. In practice it has worked
138 // when using a 64-bit kernel that gives full 4 GiB address
139 // space to 32-bit programs. In other situations this might
140 // still be too high, like 32-bit kernels that may give much
141 // less than 4 GiB to a single application.
142 //
143 // So this is an ugly hack but I will keep it here while
144 // it does more good than bad.
145 //
146 // Use a value less than SIZE_MAX so that there's some room
147 // for the xz program and so on. Don't use 4000 MiB because
148 // it could look like someone mixed up base-2 and base-10.
149 #ifdef __mips__
150 // For MIPS32, due to architectural peculiarities,
151 // the limit is even lower.
152 const uint64_t limit_max = UINT64_C(2000) << 20;
153 #else
154 const uint64_t limit_max = UINT64_C(4020) << 20;
155 #endif
156
157 // UINT64_MAX is a special case for the string "max" so
158 // that has to be handled specially.
159 if (memlimit_compress != UINT64_MAX
160 && memlimit_compress > limit_max)
161 memlimit_compress = limit_max;
162 #endif
163 }
164
165 if (set_decompress)
166 memlimit_decompress = new_memlimit;
167
168 if (set_mtdec)
169 memlimit_mtdec = new_memlimit;
170
171 return;
172 }
173
174
175 extern uint64_t
176 hardware_memlimit_get(enum operation_mode mode)
177 {
178 // 0 is a special value that indicates the default.
179 // It disables the limit in single-threaded mode.
180 //
181 // NOTE: For multithreaded decompression, this is the hard limit
182 // (memlimit_stop). hardware_memlimit_mtdec_get() gives the
183 // soft limit (memlimit_threaded).
184 const uint64_t memlimit = mode == MODE_COMPRESS
185 ? memlimit_compress : memlimit_decompress;
186 return memlimit != 0 ? memlimit : UINT64_MAX;
187 }
188
189
190 extern uint64_t
191 hardware_memlimit_mtenc_get(void)
192 {
193 return hardware_memlimit_mtenc_is_default()
194 ? memlimit_mt_default
195 : hardware_memlimit_get(MODE_COMPRESS);
196 }
197
198
199 extern bool
200 hardware_memlimit_mtenc_is_default(void)
201 {
202 return memlimit_compress == 0 && threads_are_automatic;
203 }
204
205
206 extern uint64_t
207 hardware_memlimit_mtdec_get(void)
208 {
209 uint64_t m = memlimit_mtdec != 0
210 ? memlimit_mtdec
211 : memlimit_mt_default;
212
213 // Cap the value to memlimit_decompress if it has been specified.
214 // This is nice for --info-memory. It wouldn't be needed for liblzma
215 // since it does this anyway.
216 if (memlimit_decompress != 0 && m > memlimit_decompress)
217 m = memlimit_decompress;
218
219 return m;
220 }
221
222
223 /// Helper for hardware_memlimit_show() to print one human-readable info line.
224 static void
225 memlimit_show(const char *str, size_t str_columns, uint64_t value)
226 {
227 // Calculate the field width so that str will be padded to take
228 // str_columns on the terminal.
229 //
230 // NOTE: If the string is invalid, this will be -1. Using -1 as
231 // the field width is fine here so it's not handled specially.
232 const int fw = tuklib_mbstr_fw(str, (int)(str_columns));
233
234 // The memory usage limit is considered to be disabled if value
235 // is 0 or UINT64_MAX. This might get a bit more complex once there
236 // is threading support. See the comment in hardware_memlimit_get().
237 if (value == 0 || value == UINT64_MAX)
238 printf(" %-*s %s\n", fw, str, _("Disabled"));
239 else
240 printf(" %-*s %s MiB (%s B)\n", fw, str,
241 uint64_to_str(round_up_to_mib(value), 0),
242 uint64_to_str(value, 1));
243
244 return;
245 }
246
247
248 extern void
249 hardware_memlimit_show(void)
250 {
251 uint32_t cputhreads = 1;
252 #ifdef MYTHREAD_ENABLED
253 cputhreads = lzma_cputhreads();
254 if (cputhreads == 0)
255 cputhreads = 1;
256 #endif
257
258 if (opt_robot) {
259 printf("%" PRIu64 "\t%" PRIu64 "\t%" PRIu64 "\t%" PRIu64
260 "\t%" PRIu64 "\t%" PRIu32 "\n",
261 total_ram,
262 memlimit_compress,
263 memlimit_decompress,
264 hardware_memlimit_mtdec_get(),
265 memlimit_mt_default,
266 cputhreads);
267 } else {
268 const char *msgs[] = {
269 _("Amount of physical memory (RAM):"),
270 _("Number of processor threads:"),
271 _("Compression:"),
272 _("Decompression:"),
273 _("Multi-threaded decompression:"),
274 _("Default for -T0:"),
275 };
276
277 size_t width_max = 1;
278 for (unsigned i = 0; i < ARRAY_SIZE(msgs); ++i) {
279 size_t w = tuklib_mbstr_width(msgs[i], NULL);
280
281 // When debugging, catch invalid strings with
282 // an assertion. Otherwise fallback to 1 so
283 // that the columns just won't be aligned.
284 assert(w != (size_t)-1);
285 if (w == (size_t)-1)
286 w = 1;
287
288 if (width_max < w)
289 width_max = w;
290 }
291
292 puts(_("Hardware information:"));
293 memlimit_show(msgs[0], width_max, total_ram);
294 printf(" %-*s %" PRIu32 "\n",
295 tuklib_mbstr_fw(msgs[1], (int)(width_max)),
296 msgs[1], cputhreads);
297
298 putchar('\n');
299 puts(_("Memory usage limits:"));
300 memlimit_show(msgs[2], width_max, memlimit_compress);
301 memlimit_show(msgs[3], width_max, memlimit_decompress);
302 memlimit_show(msgs[4], width_max,
303 hardware_memlimit_mtdec_get());
304 memlimit_show(msgs[5], width_max, memlimit_mt_default);
305 }
306
307 tuklib_exit(E_SUCCESS, E_ERROR, message_verbosity_get() != V_SILENT);
308 }
309
310
311 extern void
312 hardware_init(void)
313 {
314 // Get the amount of RAM. If we cannot determine it,
315 // use the assumption defined by the configure script.
316 total_ram = lzma_physmem();
317 if (total_ram == 0)
318 total_ram = (uint64_t)(ASSUME_RAM) * 1024 * 1024;
319
320 // FIXME? There may be better methods to determine the default value.
321 // One Linux-specific suggestion is to use MemAvailable from
322 // /proc/meminfo as the starting point.
323 memlimit_mt_default = total_ram / 4;
324
325 #if SIZE_MAX == UINT32_MAX
326 // A too high value may cause 32-bit xz to run out of address space.
327 // Use a conservative maximum value here. A few typical address space
328 // sizes with Linux:
329 // - x86-64 with 32-bit xz: 4 GiB
330 // - x86: 3 GiB
331 // - MIPS32: 2 GiB
332 const size_t mem_ceiling = 1400U << 20;
333 if (memlimit_mt_default > mem_ceiling)
334 memlimit_mt_default = mem_ceiling;
335 #endif
336
337 return;
338 }