1 // Copyright 2018 Ulf Adams
2 //
3 // The contents of this file may be used under the terms of the Apache License,
4 // Version 2.0.
5 //
6 // (See accompanying file LICENSE-Apache or copy at
7 // http://www.apache.org/licenses/LICENSE-2.0)
8 //
9 // Alternatively, the contents of this file may be used under the terms of
10 // the Boost Software License, Version 1.0.
11 // (See accompanying file LICENSE-Boost or copy at
12 // https://www.boost.org/LICENSE_1_0.txt)
13 //
14 // Unless required by applicable law or agreed to in writing, this software
15 // is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
16 // KIND, either express or implied.
17
18 // Runtime compiler options:
19 // -DRYU_DEBUG Generate verbose debugging output to stdout.
20 //
21 // -DRYU_ONLY_64_BIT_OPS Avoid using uint128_t or 64-bit intrinsics. Slower,
22 // depending on your compiler.
23 //
24 // -DRYU_OPTIMIZE_SIZE Use smaller lookup tables. Instead of storing every
25 // required power of 5, only store every 26th entry, and compute
26 // intermediate values with a multiplication. This reduces the lookup table
27 // size by about 10x (only one case, and only double) at the cost of some
28 // performance. Currently requires MSVC intrinsics.
29
30
31
32 #ifdef RYU_DEBUG
33 #endif
34
35
36 // Include either the small or the full lookup tables depending on the mode.
37 #if defined(RYU_OPTIMIZE_SIZE)
38 #else
39 #endif
40
41 #define DOUBLE_MANTISSA_BITS 52
42 #define DOUBLE_EXPONENT_BITS 11
43 #define DOUBLE_BIAS 1023
44
45 static inline uint32_t decimalLength17(const uint64_t v) {
46 // This is slightly faster than a loop.
47 // The average output length is 16.38 digits, so we check high-to-low.
48 // Function precondition: v is not an 18, 19, or 20-digit number.
49 // (17 digits are sufficient for round-tripping.)
50 assert(v < 100000000000000000L);
51 if (v >= 10000000000000000L) { return 17; }
52 if (v >= 1000000000000000L) { return 16; }
53 if (v >= 100000000000000L) { return 15; }
54 if (v >= 10000000000000L) { return 14; }
55 if (v >= 1000000000000L) { return 13; }
56 if (v >= 100000000000L) { return 12; }
57 if (v >= 10000000000L) { return 11; }
58 if (v >= 1000000000L) { return 10; }
59 if (v >= 100000000L) { return 9; }
60 if (v >= 10000000L) { return 8; }
61 if (v >= 1000000L) { return 7; }
62 if (v >= 100000L) { return 6; }
63 if (v >= 10000L) { return 5; }
64 if (v >= 1000L) { return 4; }
65 if (v >= 100L) { return 3; }
66 if (v >= 10L) { return 2; }
67 return 1;
68 }
69
70 // A floating decimal representing m * 10^e.
71 typedef struct floating_decimal_64 {
72 uint64_t mantissa;
73 // Decimal exponent's range is -324 to 308
74 // inclusive, and can fit in a short if needed.
75 int32_t exponent;
76 bool sign;
77 } floating_decimal_64;
78
79 static inline floating_decimal_64 d2d(const uint64_t ieeeMantissa, const uint32_t ieeeExponent, const bool ieeeSign) {
80 int32_t e2;
81 uint64_t m2;
82 if (ieeeExponent == 0) {
83 // We subtract 2 so that the bounds computation has 2 additional bits.
84 e2 = 1 - DOUBLE_BIAS - DOUBLE_MANTISSA_BITS - 2;
85 m2 = ieeeMantissa;
86 } else {
87 e2 = (int32_t) ieeeExponent - DOUBLE_BIAS - DOUBLE_MANTISSA_BITS - 2;
88 m2 = (1ull << DOUBLE_MANTISSA_BITS) | ieeeMantissa;
89 }
90 const bool even = (m2 & 1) == 0;
91 const bool acceptBounds = even;
92
93 #ifdef RYU_DEBUG
94 printf("-> %" PRIu64 " * 2^%d\n", m2, e2 + 2);
95 #endif
96
97 // Step 2: Determine the interval of valid decimal representations.
98 const uint64_t mv = 4 * m2;
99 // Implicit bool -> int conversion. True is 1, false is 0.
100 const uint32_t mmShift = ieeeMantissa != 0 || ieeeExponent <= 1;
101 // We would compute mp and mm like this:
102 // uint64_t mp = 4 * m2 + 2;
103 // uint64_t mm = mv - 1 - mmShift;
104
105 // Step 3: Convert to a decimal power base using 128-bit arithmetic.
106 uint64_t vr, vp, vm;
107 int32_t e10;
108 bool vmIsTrailingZeros = false;
109 bool vrIsTrailingZeros = false;
110 if (e2 >= 0) {
111 // I tried special-casing q == 0, but there was no effect on performance.
112 // This expression is slightly faster than max(0, log10Pow2(e2) - 1).
113 const uint32_t q = log10Pow2(e2) - (e2 > 3);
114 e10 = (int32_t) q;
115 const int32_t k = DOUBLE_POW5_INV_BITCOUNT + pow5bits((int32_t) q) - 1;
116 const int32_t i = -e2 + (int32_t) q + k;
117 #if defined(RYU_OPTIMIZE_SIZE)
118 uint64_t pow5[2];
119 double_computeInvPow5(q, pow5);
120 vr = mulShiftAll64(m2, pow5, i, &vp, &vm, mmShift);
121 #else
122 vr = mulShiftAll64(m2, DOUBLE_POW5_INV_SPLIT[q], i, &vp, &vm, mmShift);
123 #endif
124 #ifdef RYU_DEBUG
125 printf("%" PRIu64 " * 2^%d / 10^%u\n", mv, e2, q);
126 printf("V+=%" PRIu64 "\nV =%" PRIu64 "\nV-=%" PRIu64 "\n", vp, vr, vm);
127 #endif
128 if (q <= 21) {
129 // This should use q <= 22, but I think 21 is also safe. Smaller values
130 // may still be safe, but it's more difficult to reason about them.
131 // Only one of mp, mv, and mm can be a multiple of 5, if any.
132 const uint32_t mvMod5 = ((uint32_t) mv) - 5 * ((uint32_t) div5(mv));
133 if (mvMod5 == 0) {
134 vrIsTrailingZeros = multipleOfPowerOf5(mv, q);
135 } else if (acceptBounds) {
136 // Same as min(e2 + (~mm & 1), pow5Factor(mm)) >= q
137 // <=> e2 + (~mm & 1) >= q && pow5Factor(mm) >= q
138 // <=> true && pow5Factor(mm) >= q, since e2 >= q.
139 vmIsTrailingZeros = multipleOfPowerOf5(mv - 1 - mmShift, q);
140 } else {
141 // Same as min(e2 + 1, pow5Factor(mp)) >= q.
142 vp -= multipleOfPowerOf5(mv + 2, q);
143 }
144 }
145 } else {
146 // This expression is slightly faster than max(0, log10Pow5(-e2) - 1).
147 const uint32_t q = log10Pow5(-e2) - (-e2 > 1);
148 e10 = (int32_t) q + e2;
149 const int32_t i = -e2 - (int32_t) q;
150 const int32_t k = pow5bits(i) - DOUBLE_POW5_BITCOUNT;
151 const int32_t j = (int32_t) q - k;
152 #if defined(RYU_OPTIMIZE_SIZE)
153 uint64_t pow5[2];
154 double_computePow5(i, pow5);
155 vr = mulShiftAll64(m2, pow5, j, &vp, &vm, mmShift);
156 #else
157 vr = mulShiftAll64(m2, DOUBLE_POW5_SPLIT[i], j, &vp, &vm, mmShift);
158 #endif
159 #ifdef RYU_DEBUG
160 printf("%" PRIu64 " * 5^%d / 10^%u\n", mv, -e2, q);
161 printf("%u %d %d %d\n", q, i, k, j);
162 printf("V+=%" PRIu64 "\nV =%" PRIu64 "\nV-=%" PRIu64 "\n", vp, vr, vm);
163 #endif
164 if (q <= 1) {
165 // {vr,vp,vm} is trailing zeros if {mv,mp,mm} has at least q trailing 0 bits.
166 // mv = 4 * m2, so it always has at least two trailing 0 bits.
167 vrIsTrailingZeros = true;
168 if (acceptBounds) {
169 // mm = mv - 1 - mmShift, so it has 1 trailing 0 bit iff mmShift == 1.
170 vmIsTrailingZeros = mmShift == 1;
171 } else {
172 // mp = mv + 2, so it always has at least one trailing 0 bit.
173 --vp;
174 }
175 } else if (q < 63) { // TODO(ulfjack): Use a tighter bound here.
176 // We want to know if the full product has at least q trailing zeros.
177 // We need to compute min(p2(mv), p5(mv) - e2) >= q
178 // <=> p2(mv) >= q && p5(mv) - e2 >= q
179 // <=> p2(mv) >= q (because -e2 >= q)
180 vrIsTrailingZeros = multipleOfPowerOf2(mv, q);
181 #ifdef RYU_DEBUG
182 printf("vr is trailing zeros=%s\n", vrIsTrailingZeros ? "true" : "false");
183 #endif
184 }
185 }
186 #ifdef RYU_DEBUG
187 printf("e10=%d\n", e10);
188 printf("V+=%" PRIu64 "\nV =%" PRIu64 "\nV-=%" PRIu64 "\n", vp, vr, vm);
189 printf("vm is trailing zeros=%s\n", vmIsTrailingZeros ? "true" : "false");
190 printf("vr is trailing zeros=%s\n", vrIsTrailingZeros ? "true" : "false");
191 #endif
192
193 // Step 4: Find the shortest decimal representation in the interval of valid representations.
194 int32_t removed = 0;
195 uint8_t lastRemovedDigit = 0;
196 uint64_t output;
197 // On average, we remove ~2 digits.
198 if (vmIsTrailingZeros || vrIsTrailingZeros) {
199 // General case, which happens rarely (~0.7%).
200 for (;;) {
201 const uint64_t vpDiv10 = div10(vp);
202 const uint64_t vmDiv10 = div10(vm);
203 if (vpDiv10 <= vmDiv10) {
204 break;
205 }
206 const uint32_t vmMod10 = ((uint32_t) vm) - 10 * ((uint32_t) vmDiv10);
207 const uint64_t vrDiv10 = div10(vr);
208 const uint32_t vrMod10 = ((uint32_t) vr) - 10 * ((uint32_t) vrDiv10);
209 vmIsTrailingZeros &= vmMod10 == 0;
210 vrIsTrailingZeros &= lastRemovedDigit == 0;
211 lastRemovedDigit = (uint8_t) vrMod10;
212 vr = vrDiv10;
213 vp = vpDiv10;
214 vm = vmDiv10;
215 ++removed;
216 }
217 #ifdef RYU_DEBUG
218 printf("V+=%" PRIu64 "\nV =%" PRIu64 "\nV-=%" PRIu64 "\n", vp, vr, vm);
219 printf("d-10=%s\n", vmIsTrailingZeros ? "true" : "false");
220 #endif
221 if (vmIsTrailingZeros) {
222 for (;;) {
223 const uint64_t vmDiv10 = div10(vm);
224 const uint32_t vmMod10 = ((uint32_t) vm) - 10 * ((uint32_t) vmDiv10);
225 if (vmMod10 != 0) {
226 break;
227 }
228 const uint64_t vpDiv10 = div10(vp);
229 const uint64_t vrDiv10 = div10(vr);
230 const uint32_t vrMod10 = ((uint32_t) vr) - 10 * ((uint32_t) vrDiv10);
231 vrIsTrailingZeros &= lastRemovedDigit == 0;
232 lastRemovedDigit = (uint8_t) vrMod10;
233 vr = vrDiv10;
234 vp = vpDiv10;
235 vm = vmDiv10;
236 ++removed;
237 }
238 }
239 #ifdef RYU_DEBUG
240 printf("%" PRIu64 " %d\n", vr, lastRemovedDigit);
241 printf("vr is trailing zeros=%s\n", vrIsTrailingZeros ? "true" : "false");
242 #endif
243 if (vrIsTrailingZeros && lastRemovedDigit == 5 && vr % 2 == 0) {
244 // Round even if the exact number is .....50..0.
245 lastRemovedDigit = 4;
246 }
247 // We need to take vr + 1 if vr is outside bounds or we need to round up.
248 output = vr + ((vr == vm && (!acceptBounds || !vmIsTrailingZeros)) || lastRemovedDigit >= 5);
249 } else {
250 // Specialized for the common case (~99.3%). Percentages below are relative to this.
251 bool roundUp = false;
252 const uint64_t vpDiv100 = div100(vp);
253 const uint64_t vmDiv100 = div100(vm);
254 if (vpDiv100 > vmDiv100) { // Optimization: remove two digits at a time (~86.2%).
255 const uint64_t vrDiv100 = div100(vr);
256 const uint32_t vrMod100 = ((uint32_t) vr) - 100 * ((uint32_t) vrDiv100);
257 roundUp = vrMod100 >= 50;
258 vr = vrDiv100;
259 vp = vpDiv100;
260 vm = vmDiv100;
261 removed += 2;
262 }
263 // Loop iterations below (approximately), without optimization above:
264 // 0: 0.03%, 1: 13.8%, 2: 70.6%, 3: 14.0%, 4: 1.40%, 5: 0.14%, 6+: 0.02%
265 // Loop iterations below (approximately), with optimization above:
266 // 0: 70.6%, 1: 27.8%, 2: 1.40%, 3: 0.14%, 4+: 0.02%
267 for (;;) {
268 const uint64_t vpDiv10 = div10(vp);
269 const uint64_t vmDiv10 = div10(vm);
270 if (vpDiv10 <= vmDiv10) {
271 break;
272 }
273 const uint64_t vrDiv10 = div10(vr);
274 const uint32_t vrMod10 = ((uint32_t) vr) - 10 * ((uint32_t) vrDiv10);
275 roundUp = vrMod10 >= 5;
276 vr = vrDiv10;
277 vp = vpDiv10;
278 vm = vmDiv10;
279 ++removed;
280 }
281 #ifdef RYU_DEBUG
282 printf("%" PRIu64 " roundUp=%s\n", vr, roundUp ? "true" : "false");
283 printf("vr is trailing zeros=%s\n", vrIsTrailingZeros ? "true" : "false");
284 #endif
285 // We need to take vr + 1 if vr is outside bounds or we need to round up.
286 output = vr + (vr == vm || roundUp);
287 }
288 const int32_t exp = e10 + removed;
289
290 #ifdef RYU_DEBUG
291 printf("V+=%" PRIu64 "\nV =%" PRIu64 "\nV-=%" PRIu64 "\n", vp, vr, vm);
292 printf("O=%" PRIu64 "\n", output);
293 printf("EXP=%d\n", exp);
294 #endif
295
296 floating_decimal_64 fd;
297 fd.exponent = exp;
298 fd.mantissa = output;
299 fd.sign = ieeeSign;
300 return fd;
301 }
302
303 static inline int to_chars(const floating_decimal_64 v, char* const result) {
304 // Step 5: Print the decimal representation.
305 int index = 0;
306 if (v.sign) {
307 result[index++] = '-';
308 }
309
310 uint64_t output = v.mantissa;
311 const uint32_t olength = decimalLength17(output);
312
313 #ifdef RYU_DEBUG
314 printf("DIGITS=%" PRIu64 "\n", v.mantissa);
315 printf("OLEN=%u\n", olength);
316 printf("EXP=%u\n", v.exponent + olength);
317 #endif
318
319 // Print the decimal digits.
320 // The following code is equivalent to:
321 // for (uint32_t i = 0; i < olength - 1; ++i) {
322 // const uint32_t c = output % 10; output /= 10;
323 // result[index + olength - i] = (char) ('0' + c);
324 // }
325 // result[index] = '0' + output % 10;
326
327 uint32_t i = 0;
328 // We prefer 32-bit operations, even on 64-bit platforms.
329 // We have at most 17 digits, and uint32_t can store 9 digits.
330 // If output doesn't fit into uint32_t, we cut off 8 digits,
331 // so the rest will fit into uint32_t.
332 if ((output >> 32) != 0) {
333 // Expensive 64-bit division.
334 const uint64_t q = div1e8(output);
335 uint32_t output2 = ((uint32_t) output) - 100000000 * ((uint32_t) q);
336 output = q;
337
338 const uint32_t c = output2 % 10000;
339 output2 /= 10000;
340 const uint32_t d = output2 % 10000;
341 const uint32_t c0 = (c % 100) << 1;
342 const uint32_t c1 = (c / 100) << 1;
343 const uint32_t d0 = (d % 100) << 1;
344 const uint32_t d1 = (d / 100) << 1;
345 memcpy(result + index + olength - i - 1, DIGIT_TABLE + c0, 2);
346 memcpy(result + index + olength - i - 3, DIGIT_TABLE + c1, 2);
347 memcpy(result + index + olength - i - 5, DIGIT_TABLE + d0, 2);
348 memcpy(result + index + olength - i - 7, DIGIT_TABLE + d1, 2);
349 i += 8;
350 }
351 uint32_t output2 = (uint32_t) output;
352 while (output2 >= 10000) {
353 #ifdef __clang__ // https://bugs.llvm.org/show_bug.cgi?id=38217
354 const uint32_t c = output2 - 10000 * (output2 / 10000);
355 #else
356 const uint32_t c = output2 % 10000;
357 #endif
358 output2 /= 10000;
359 const uint32_t c0 = (c % 100) << 1;
360 const uint32_t c1 = (c / 100) << 1;
361 memcpy(result + index + olength - i - 1, DIGIT_TABLE + c0, 2);
362 memcpy(result + index + olength - i - 3, DIGIT_TABLE + c1, 2);
363 i += 4;
364 }
365 if (output2 >= 100) {
366 const uint32_t c = (output2 % 100) << 1;
367 output2 /= 100;
368 memcpy(result + index + olength - i - 1, DIGIT_TABLE + c, 2);
369 i += 2;
370 }
371 if (output2 >= 10) {
372 const uint32_t c = output2 << 1;
373 // We can't use memcpy here: the decimal dot goes between these two digits.
374 result[index + olength - i] = DIGIT_TABLE[c + 1];
375 result[index] = DIGIT_TABLE[c];
376 } else {
377 result[index] = (char) ('0' + output2);
378 }
379
380 // Print decimal point if needed.
381 if (olength > 1) {
382 result[index + 1] = '.';
383 index += olength + 1;
384 } else {
385 ++index;
386 }
387
388 // Print the exponent.
389 result[index++] = 'e';
390 int32_t exp = v.exponent + (int32_t) olength - 1;
391 if (exp < 0) {
392 result[index++] = '-';
393 exp = -exp;
394 } else
395 result[index++] = '+';
396
397 if (exp >= 100) {
398 const int32_t c = exp % 10;
399 memcpy(result + index, DIGIT_TABLE + 2 * (exp / 10), 2);
400 result[index + 2] = (char) ('0' + c);
401 index += 3;
402 } else {
403 memcpy(result + index, DIGIT_TABLE + 2 * exp, 2);
404 index += 2;
405 }
406
407 return index;
408 }
409
410 static inline bool d2d_small_int(const uint64_t ieeeMantissa, const uint32_t ieeeExponent, const bool ieeeSign,
411 floating_decimal_64* const v) {
412 const uint64_t m2 = (1ull << DOUBLE_MANTISSA_BITS) | ieeeMantissa;
413 const int32_t e2 = (int32_t) ieeeExponent - DOUBLE_BIAS - DOUBLE_MANTISSA_BITS;
414
415 if (e2 > 0) {
416 // f = m2 * 2^e2 >= 2^53 is an integer.
417 // Ignore this case for now.
418 return false;
419 }
420
421 if (e2 < -52) {
422 // f < 1.
423 return false;
424 }
425
426 // Since 2^52 <= m2 < 2^53 and 0 <= -e2 <= 52: 1 <= f = m2 / 2^-e2 < 2^53.
427 // Test if the lower -e2 bits of the significand are 0, i.e. whether the fraction is 0.
428 const uint64_t mask = (1ull << -e2) - 1;
429 const uint64_t fraction = m2 & mask;
430 if (fraction != 0) {
431 return false;
432 }
433
434 // f is an integer in the range [1, 2^53).
435 // Note: mantissa might contain trailing (decimal) 0's.
436 // Note: since 2^53 < 10^16, there is no need to adjust decimalLength17().
437 v->mantissa = m2 >> -e2;
438 v->exponent = 0;
439 v->sign = ieeeSign;
440 return true;
441 }
442
443 floating_decimal_64 floating_to_fd64(double f) {
444 // Step 1: Decode the floating-point number, and unify normalized and subnormal cases.
445 const uint64_t bits = double_to_bits(f);
446
447 #ifdef RYU_DEBUG
448 printf("IN=");
449 for (int32_t bit = 63; bit >= 0; --bit) {
450 printf("%d", (int) ((bits >> bit) & 1));
451 }
452 printf("\n");
453 #endif
454
455 // Decode bits into sign, mantissa, and exponent.
456 const bool ieeeSign = ((bits >> (DOUBLE_MANTISSA_BITS + DOUBLE_EXPONENT_BITS)) & 1) != 0;
457 const uint64_t ieeeMantissa = bits & ((1ull << DOUBLE_MANTISSA_BITS) - 1);
458 const uint32_t ieeeExponent = (uint32_t) ((bits >> DOUBLE_MANTISSA_BITS) & ((1u << DOUBLE_EXPONENT_BITS) - 1));
459 // Case distinction; exit early for the easy cases.
460 if (ieeeExponent == ((1u << DOUBLE_EXPONENT_BITS) - 1u) || (ieeeExponent == 0 && ieeeMantissa == 0)) {
461 __builtin_abort();
462 }
463
464 floating_decimal_64 v;
465 const bool isSmallInt = d2d_small_int(ieeeMantissa, ieeeExponent, ieeeSign, &v);
466 if (isSmallInt) {
467 // For small integers in the range [1, 2^53), v.mantissa might contain trailing (decimal) zeros.
468 // For scientific notation we need to move these zeros into the exponent.
469 // (This is not needed for fixed-point notation, so it might be beneficial to trim
470 // trailing zeros in to_chars only if needed - once fixed-point notation output is implemented.)
471 for (;;) {
472 const uint64_t q = div10(v.mantissa);
473 const uint32_t r = ((uint32_t) v.mantissa) - 10 * ((uint32_t) q);
474 if (r != 0) {
475 break;
476 }
477 v.mantissa = q;
478 ++v.exponent;
479 }
480 } else {
481 v = d2d(ieeeMantissa, ieeeExponent, ieeeSign);
482 }
483
484 return v;
485 }