1 /* Conversion between UTF-8 and UTF-32 BE/internal.
2
3 This module uses the Z9-109 variants of the Convert Unicode
4 instructions.
5 Copyright (C) 1997-2023 Free Software Foundation, Inc.
6
7 This is free software; you can redistribute it and/or
8 modify it under the terms of the GNU Lesser General Public
9 License as published by the Free Software Foundation; either
10 version 2.1 of the License, or (at your option) any later version.
11
12 This is distributed in the hope that it will be useful,
13 but WITHOUT ANY WARRANTY; without even the implied warranty of
14 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
15 Lesser General Public License for more details.
16
17 You should have received a copy of the GNU Lesser General Public
18 License along with the GNU C Library; if not, see
19 <https://www.gnu.org/licenses/>. */
20
21 #include <dlfcn.h>
22 #include <stdint.h>
23 #include <unistd.h>
24 #include <gconv.h>
25 #include <string.h>
26
27 /* Select which versions should be defined depending on support
28 for multiarch, vector and used minimum architecture level. */
29 #ifdef HAVE_S390_MIN_Z196_ZARCH_ASM_SUPPORT
30 # define HAVE_FROM_C 0
31 # define FROM_LOOP_DEFAULT FROM_LOOP_CU
32 #else
33 # define HAVE_FROM_C 1
34 # define FROM_LOOP_DEFAULT FROM_LOOP_C
35 #endif
36
37 #define HAVE_TO_C 1
38 #define TO_LOOP_DEFAULT TO_LOOP_C
39
40 #if defined HAVE_S390_MIN_Z196_ZARCH_ASM_SUPPORT || defined USE_MULTIARCH
41 # define HAVE_FROM_CU 1
42 #else
43 # define HAVE_FROM_CU 0
44 #endif
45
46 #if defined HAVE_S390_VX_ASM_SUPPORT && defined USE_MULTIARCH
47 # define HAVE_FROM_VX 1
48 # define HAVE_TO_VX 1
49 # define HAVE_TO_VX_CU 1
50 #else
51 # define HAVE_FROM_VX 0
52 # define HAVE_TO_VX 0
53 # define HAVE_TO_VX_CU 0
54 #endif
55
56 #if defined HAVE_S390_VX_GCC_SUPPORT
57 # define ASM_CLOBBER_VR(NR) , NR
58 #else
59 # define ASM_CLOBBER_VR(NR)
60 #endif
61
62 #if defined __s390x__
63 # define CONVERT_32BIT_SIZE_T(REG)
64 #else
65 # define CONVERT_32BIT_SIZE_T(REG) "llgfr %" #REG ",%" #REG "\n\t"
66 #endif
67
68 /* Defines for skeleton.c. */
69 #define DEFINE_INIT 0
70 #define DEFINE_FINI 0
71 #define MIN_NEEDED_FROM 1
72 #define MAX_NEEDED_FROM 6
73 #define MIN_NEEDED_TO 4
74 #define FROM_LOOP FROM_LOOP_DEFAULT
75 #define TO_LOOP TO_LOOP_DEFAULT
76 #define FROM_DIRECTION (dir == from_utf8)
77 #define ONE_DIRECTION 0
78
79 /* UTF-32 big endian byte order mark. */
80 #define BOM 0x0000feffu
81
82 /* Direction of the transformation. */
83 enum direction
84 {
85 illegal_dir,
86 to_utf8,
87 from_utf8
88 };
89
90 struct utf8_data
91 {
92 enum direction dir;
93 int emit_bom;
94 };
95
96
97 extern int gconv_init (struct __gconv_step *step);
98 int
99 gconv_init (struct __gconv_step *step)
100 {
101 /* Determine which direction. */
102 struct utf8_data *new_data;
103 enum direction dir = illegal_dir;
104 int emit_bom;
105 int result;
106
107 emit_bom = (__strcasecmp (step->__to_name, "UTF-32//") == 0);
108
109 if (__strcasecmp (step->__from_name, "ISO-10646/UTF8/") == 0
110 && (__strcasecmp (step->__to_name, "UTF-32//") == 0
111 || __strcasecmp (step->__to_name, "UTF-32BE//") == 0
112 || __strcasecmp (step->__to_name, "INTERNAL") == 0))
113 {
114 dir = from_utf8;
115 }
116 else if (__strcasecmp (step->__to_name, "ISO-10646/UTF8/") == 0
117 && (__strcasecmp (step->__from_name, "UTF-32BE//") == 0
118 || __strcasecmp (step->__from_name, "INTERNAL") == 0))
119 {
120 dir = to_utf8;
121 }
122
123 result = __GCONV_NOCONV;
124 if (dir != illegal_dir)
125 {
126 new_data = (struct utf8_data *) malloc (sizeof (struct utf8_data));
127
128 result = __GCONV_NOMEM;
129 if (new_data != NULL)
130 {
131 new_data->dir = dir;
132 new_data->emit_bom = emit_bom;
133 step->__data = new_data;
134
135 if (dir == from_utf8)
136 {
137 step->__min_needed_from = MIN_NEEDED_FROM;
138 step->__max_needed_from = MIN_NEEDED_FROM;
139 step->__min_needed_to = MIN_NEEDED_TO;
140 step->__max_needed_to = MIN_NEEDED_TO;
141 }
142 else
143 {
144 step->__min_needed_from = MIN_NEEDED_TO;
145 step->__max_needed_from = MIN_NEEDED_TO;
146 step->__min_needed_to = MIN_NEEDED_FROM;
147 step->__max_needed_to = MIN_NEEDED_FROM;
148 }
149
150 step->__stateful = 0;
151
152 result = __GCONV_OK;
153 }
154 }
155
156 return result;
157 }
158
159
160 extern void gconv_end (struct __gconv_step *data);
161 void
162 gconv_end (struct __gconv_step *data)
163 {
164 free (data->__data);
165 }
166
167 /* The macro for the hardware loop. This is used for both
168 directions. */
169 #define HARDWARE_CONVERT(INSTRUCTION) \
170 { \
171 register const unsigned char* pInput __asm__ ("8") = inptr; \
172 register size_t inlen __asm__ ("9") = inend - inptr; \
173 register unsigned char* pOutput __asm__ ("10") = outptr; \
174 register size_t outlen __asm__("11") = outend - outptr; \
175 unsigned long cc = 0; \
176 \
177 __asm__ __volatile__ (".machine push \n\t" \
178 ".machine \"z9-109\" \n\t" \
179 ".machinemode \"zarch_nohighgprs\"\n\t" \
180 "0: " INSTRUCTION " \n\t" \
181 ".machine pop \n\t" \
182 " jo 0b \n\t" \
183 " ipm %2 \n" \
184 : "+a" (pOutput), "+a" (pInput), "+d" (cc), \
185 "+d" (outlen), "+d" (inlen) \
186 : \
187 : "cc", "memory"); \
188 \
189 inptr = pInput; \
190 outptr = pOutput; \
191 cc >>= 28; \
192 \
193 if (cc == 1) \
194 { \
195 result = __GCONV_FULL_OUTPUT; \
196 } \
197 else if (cc == 2) \
198 { \
199 result = __GCONV_ILLEGAL_INPUT; \
200 } \
201 }
202
203 #define PREPARE_LOOP \
204 enum direction dir = ((struct utf8_data *) step->__data)->dir; \
205 int emit_bom = ((struct utf8_data *) step->__data)->emit_bom; \
206 \
207 if (emit_bom && !data->__internal_use \
208 && data->__invocation_counter == 0) \
209 { \
210 /* Emit the Byte Order Mark. */ \
211 if (__glibc_unlikely (outbuf + 4 > outend)) \
212 return __GCONV_FULL_OUTPUT; \
213 \
214 put32 (outbuf, BOM); \
215 outbuf += 4; \
216 }
217
218 /* Conversion function from UTF-8 to UTF-32 internal/BE. */
219
220 #define STORE_REST_COMMON \
221 { \
222 /* We store the remaining bytes while converting them into the UCS4 \
223 format. We can assume that the first byte in the buffer is \
224 correct and that it requires a larger number of bytes than there \
225 are in the input buffer. */ \
226 wint_t ch = **inptrp; \
227 size_t cnt, r; \
228 \
229 state->__count = inend - *inptrp; \
230 \
231 assert (ch != 0xc0 && ch != 0xc1); \
232 if (ch >= 0xc2 && ch < 0xe0) \
233 { \
234 /* We expect two bytes. The first byte cannot be 0xc0 or \
235 0xc1, otherwise the wide character could have been \
236 represented using a single byte. */ \
237 cnt = 2; \
238 ch &= 0x1f; \
239 } \
240 else if (__glibc_likely ((ch & 0xf0) == 0xe0)) \
241 { \
242 /* We expect three bytes. */ \
243 cnt = 3; \
244 ch &= 0x0f; \
245 } \
246 else if (__glibc_likely ((ch & 0xf8) == 0xf0)) \
247 { \
248 /* We expect four bytes. */ \
249 cnt = 4; \
250 ch &= 0x07; \
251 } \
252 else if (__glibc_likely ((ch & 0xfc) == 0xf8)) \
253 { \
254 /* We expect five bytes. */ \
255 cnt = 5; \
256 ch &= 0x03; \
257 } \
258 else \
259 { \
260 /* We expect six bytes. */ \
261 cnt = 6; \
262 ch &= 0x01; \
263 } \
264 \
265 /* The first byte is already consumed. */ \
266 r = cnt - 1; \
267 while (++(*inptrp) < inend) \
268 { \
269 ch <<= 6; \
270 ch |= **inptrp & 0x3f; \
271 --r; \
272 } \
273 \
274 /* Shift for the so far missing bytes. */ \
275 ch <<= r * 6; \
276 \
277 /* Store the number of bytes expected for the entire sequence. */ \
278 state->__count |= cnt << 8; \
279 \
280 /* Store the value. */ \
281 state->__value.__wch = ch; \
282 }
283
284 #define UNPACK_BYTES_COMMON \
285 { \
286 static const unsigned char inmask[5] = { 0xc0, 0xe0, 0xf0, 0xf8, 0xfc }; \
287 wint_t wch = state->__value.__wch; \
288 size_t ntotal = state->__count >> 8; \
289 \
290 inlen = state->__count & 255; \
291 \
292 bytebuf[0] = inmask[ntotal - 2]; \
293 \
294 do \
295 { \
296 if (--ntotal < inlen) \
297 bytebuf[ntotal] = 0x80 | (wch & 0x3f); \
298 wch >>= 6; \
299 } \
300 while (ntotal > 1); \
301 \
302 bytebuf[0] |= wch; \
303 }
304
305 #define CLEAR_STATE_COMMON \
306 state->__count = 0
307
308 #define BODY_FROM_HW(ASM) \
309 { \
310 ASM; \
311 if (__glibc_likely (inptr == inend) \
312 || result == __GCONV_FULL_OUTPUT) \
313 break; \
314 \
315 int i; \
316 for (i = 1; inptr + i < inend && i < 5; ++i) \
317 if ((inptr[i] & 0xc0) != 0x80) \
318 break; \
319 \
320 if (__glibc_likely (inptr + i == inend \
321 && result == __GCONV_EMPTY_INPUT)) \
322 { \
323 result = __GCONV_INCOMPLETE_INPUT; \
324 break; \
325 } \
326 STANDARD_FROM_LOOP_ERR_HANDLER (i); \
327 }
328
329 #if HAVE_FROM_C == 1
330 /* The software routine is copied from gconv_simple.c. */
331 # define BODY_FROM_C \
332 { \
333 /* Next input byte. */ \
334 uint32_t ch = *inptr; \
335 \
336 if (__glibc_likely (ch < 0x80)) \
337 { \
338 /* One byte sequence. */ \
339 ++inptr; \
340 } \
341 else \
342 { \
343 uint_fast32_t cnt; \
344 uint_fast32_t i; \
345 \
346 if (ch >= 0xc2 && ch < 0xe0) \
347 { \
348 /* We expect two bytes. The first byte cannot be 0xc0 or \
349 0xc1, otherwise the wide character could have been \
350 represented using a single byte. */ \
351 cnt = 2; \
352 ch &= 0x1f; \
353 } \
354 else if (__glibc_likely ((ch & 0xf0) == 0xe0)) \
355 { \
356 /* We expect three bytes. */ \
357 cnt = 3; \
358 ch &= 0x0f; \
359 } \
360 else if (__glibc_likely ((ch & 0xf8) == 0xf0)) \
361 { \
362 /* We expect four bytes. */ \
363 cnt = 4; \
364 ch &= 0x07; \
365 } \
366 else \
367 { \
368 /* Search the end of this ill-formed UTF-8 character. This \
369 is the next byte with (x & 0xc0) != 0x80. */ \
370 i = 0; \
371 do \
372 ++i; \
373 while (inptr + i < inend \
374 && (*(inptr + i) & 0xc0) == 0x80 \
375 && i < 5); \
376 \
377 errout: \
378 STANDARD_FROM_LOOP_ERR_HANDLER (i); \
379 } \
380 \
381 if (__glibc_unlikely (inptr + cnt > inend)) \
382 { \
383 /* We don't have enough input. But before we report \
384 that check that all the bytes are correct. */ \
385 for (i = 1; inptr + i < inend; ++i) \
386 if ((inptr[i] & 0xc0) != 0x80) \
387 break; \
388 \
389 if (__glibc_likely (inptr + i == inend)) \
390 { \
391 result = __GCONV_INCOMPLETE_INPUT; \
392 break; \
393 } \
394 \
395 goto errout; \
396 } \
397 \
398 /* Read the possible remaining bytes. */ \
399 for (i = 1; i < cnt; ++i) \
400 { \
401 uint32_t byte = inptr[i]; \
402 \
403 if ((byte & 0xc0) != 0x80) \
404 /* This is an illegal encoding. */ \
405 break; \
406 \
407 ch <<= 6; \
408 ch |= byte & 0x3f; \
409 } \
410 \
411 /* If i < cnt, some trail byte was not >= 0x80, < 0xc0. \
412 If cnt > 2 and ch < 2^(5*cnt-4), the wide character ch could \
413 have been represented with fewer than cnt bytes. */ \
414 if (i < cnt || (cnt > 2 && (ch >> (5 * cnt - 4)) == 0) \
415 /* Do not accept UTF-16 surrogates. */ \
416 || (ch >= 0xd800 && ch <= 0xdfff) \
417 || (ch > 0x10ffff)) \
418 { \
419 /* This is an illegal encoding. */ \
420 goto errout; \
421 } \
422 \
423 inptr += cnt; \
424 } \
425 \
426 /* Now adjust the pointers and store the result. */ \
427 *((uint32_t *) outptr) = ch; \
428 outptr += sizeof (uint32_t); \
429 }
430
431 /* These definitions apply to the UTF-8 to UTF-32 direction. The
432 software implementation for UTF-8 still supports multibyte
433 characters up to 6 bytes whereas the hardware variant does not. */
434 # define MIN_NEEDED_INPUT MIN_NEEDED_FROM
435 # define MAX_NEEDED_INPUT MAX_NEEDED_FROM
436 # define MIN_NEEDED_OUTPUT MIN_NEEDED_TO
437 # define FROM_LOOP_C __from_utf8_loop_c
438 # define LOOPFCT FROM_LOOP_C
439
440 # define LOOP_NEED_FLAGS
441
442 # define STORE_REST STORE_REST_COMMON
443 # define UNPACK_BYTES UNPACK_BYTES_COMMON
444 # define CLEAR_STATE CLEAR_STATE_COMMON
445 # define BODY BODY_FROM_C
446 # include <iconv/loop.c>
447 #else
448 # define FROM_LOOP_C NULL
449 #endif /* HAVE_FROM_C != 1 */
450
451 #if HAVE_FROM_CU == 1
452 /* This hardware routine uses the Convert UTF8 to UTF32 (cu14) instruction. */
453 # define BODY_FROM_ETF3EH BODY_FROM_HW (HARDWARE_CONVERT ("cu14 %0, %1, 1"))
454
455 /* Generate loop-function with hardware utf-convert instruction. */
456 # define MIN_NEEDED_INPUT MIN_NEEDED_FROM
457 # define MAX_NEEDED_INPUT MAX_NEEDED_FROM
458 # define MIN_NEEDED_OUTPUT MIN_NEEDED_TO
459 # define FROM_LOOP_CU __from_utf8_loop_etf3eh
460 # define LOOPFCT FROM_LOOP_CU
461
462 # define LOOP_NEED_FLAGS
463
464 # define STORE_REST STORE_REST_COMMON
465 # define UNPACK_BYTES UNPACK_BYTES_COMMON
466 # define CLEAR_STATE CLEAR_STATE_COMMON
467 # define BODY BODY_FROM_ETF3EH
468 # include <iconv/loop.c>
469 #else
470 # define FROM_LOOP_CU NULL
471 #endif /* HAVE_FROM_CU != 1 */
472
473 #if HAVE_FROM_VX == 1
474 # define HW_FROM_VX \
475 { \
476 register const unsigned char* pInput asm ("8") = inptr; \
477 register size_t inlen asm ("9") = inend - inptr; \
478 register unsigned char* pOutput asm ("10") = outptr; \
479 register size_t outlen asm("11") = outend - outptr; \
480 unsigned long tmp, tmp2, tmp3; \
481 asm volatile (".machine push\n\t" \
482 ".machine \"z13\"\n\t" \
483 ".machinemode \"zarch_nohighgprs\"\n\t" \
484 " vrepib %%v30,0x7f\n\t" /* For compare > 0x7f. */ \
485 " vrepib %%v31,0x20\n\t" \
486 CONVERT_32BIT_SIZE_T ([R_INLEN]) \
487 CONVERT_32BIT_SIZE_T ([R_OUTLEN]) \
488 /* Loop which handles UTF-8 chars <=0x7f. */ \
489 "0: clgijl %[R_INLEN],16,20f\n\t" \
490 " clgijl %[R_OUTLEN],64,20f\n\t" \
491 "1: vl %%v16,0(%[R_IN])\n\t" \
492 " vstrcbs %%v17,%%v16,%%v30,%%v31\n\t" \
493 " jno 10f\n\t" /* Jump away if not all bytes are 1byte \
494 UTF8 chars. */ \
495 /* Enlarge to UCS4. */ \
496 " vuplhb %%v18,%%v16\n\t" \
497 " vupllb %%v19,%%v16\n\t" \
498 " la %[R_IN],16(%[R_IN])\n\t" \
499 " vuplhh %%v20,%%v18\n\t" \
500 " aghi %[R_INLEN],-16\n\t" \
501 " vupllh %%v21,%%v18\n\t" \
502 " aghi %[R_OUTLEN],-64\n\t" \
503 " vuplhh %%v22,%%v19\n\t" \
504 " vupllh %%v23,%%v19\n\t" \
505 /* Store 64 bytes to buf_out. */ \
506 " vstm %%v20,%%v23,0(%[R_OUT])\n\t" \
507 " la %[R_OUT],64(%[R_OUT])\n\t" \
508 " clgijl %[R_INLEN],16,20f\n\t" \
509 " clgijl %[R_OUTLEN],64,20f\n\t" \
510 " j 1b\n\t" \
511 "10: \n\t" \
512 /* At least one byte is > 0x7f. \
513 Store the preceding 1-byte chars. */ \
514 " vlgvb %[R_TMP],%%v17,7\n\t" \
515 " sllk %[R_TMP2],%[R_TMP],2\n\t" /* Compute highest \
516 index to store. */ \
517 " llgfr %[R_TMP3],%[R_TMP2]\n\t" \
518 " ahi %[R_TMP2],-1\n\t" \
519 " jl 20f\n\t" \
520 " vuplhb %%v18,%%v16\n\t" \
521 " vuplhh %%v20,%%v18\n\t" \
522 " vstl %%v20,%[R_TMP2],0(%[R_OUT])\n\t" \
523 " ahi %[R_TMP2],-16\n\t" \
524 " jl 11f\n\t" \
525 " vupllh %%v21,%%v18\n\t" \
526 " vstl %%v21,%[R_TMP2],16(%[R_OUT])\n\t" \
527 " ahi %[R_TMP2],-16\n\t" \
528 " jl 11f\n\t" \
529 " vupllb %%v19,%%v16\n\t" \
530 " vuplhh %%v22,%%v19\n\t" \
531 " vstl %%v22,%[R_TMP2],32(%[R_OUT])\n\t" \
532 " ahi %[R_TMP2],-16\n\t" \
533 " jl 11f\n\t" \
534 " vupllh %%v23,%%v19\n\t" \
535 " vstl %%v23,%[R_TMP2],48(%[R_OUT])\n\t" \
536 "11: \n\t" \
537 /* Update pointers. */ \
538 " la %[R_IN],0(%[R_TMP],%[R_IN])\n\t" \
539 " slgr %[R_INLEN],%[R_TMP]\n\t" \
540 " la %[R_OUT],0(%[R_TMP3],%[R_OUT])\n\t" \
541 " slgr %[R_OUTLEN],%[R_TMP3]\n\t" \
542 /* Handle multibyte utf8-char with convert instruction. */ \
543 "20: cu14 %[R_OUT],%[R_IN],1\n\t" \
544 " jo 0b\n\t" /* Try vector implementation again. */ \
545 " lochil %[R_RES],%[RES_OUT_FULL]\n\t" /* cc == 1. */ \
546 " lochih %[R_RES],%[RES_IN_ILL]\n\t" /* cc == 2. */ \
547 ".machine pop" \
548 : /* outputs */ [R_IN] "+a" (pInput) \
549 , [R_INLEN] "+d" (inlen), [R_OUT] "+a" (pOutput) \
550 , [R_OUTLEN] "+d" (outlen), [R_TMP] "=a" (tmp) \
551 , [R_TMP2] "=d" (tmp2), [R_TMP3] "=a" (tmp3) \
552 , [R_RES] "+d" (result) \
553 : /* inputs */ \
554 [RES_OUT_FULL] "i" (__GCONV_FULL_OUTPUT) \
555 , [RES_IN_ILL] "i" (__GCONV_ILLEGAL_INPUT) \
556 : /* clobber list */ "memory", "cc" \
557 ASM_CLOBBER_VR ("v16") ASM_CLOBBER_VR ("v17") \
558 ASM_CLOBBER_VR ("v18") ASM_CLOBBER_VR ("v19") \
559 ASM_CLOBBER_VR ("v20") ASM_CLOBBER_VR ("v21") \
560 ASM_CLOBBER_VR ("v22") ASM_CLOBBER_VR ("v30") \
561 ASM_CLOBBER_VR ("v31") \
562 ); \
563 inptr = pInput; \
564 outptr = pOutput; \
565 }
566 # define BODY_FROM_VX BODY_FROM_HW (HW_FROM_VX)
567
568 /* Generate loop-function with hardware vector and utf-convert instructions. */
569 # define MIN_NEEDED_INPUT MIN_NEEDED_FROM
570 # define MAX_NEEDED_INPUT MAX_NEEDED_FROM
571 # define MIN_NEEDED_OUTPUT MIN_NEEDED_TO
572 # define FROM_LOOP_VX __from_utf8_loop_vx
573 # define LOOPFCT FROM_LOOP_VX
574
575 # define LOOP_NEED_FLAGS
576
577 # define STORE_REST STORE_REST_COMMON
578 # define UNPACK_BYTES UNPACK_BYTES_COMMON
579 # define CLEAR_STATE CLEAR_STATE_COMMON
580 # define BODY BODY_FROM_VX
581 # include <iconv/loop.c>
582 #else
583 # define FROM_LOOP_VX NULL
584 #endif /* HAVE_FROM_VX != 1 */
585
586 #if HAVE_TO_C == 1
587 /* The software routine mimics the S/390 cu41 instruction. */
588 # define BODY_TO_C \
589 { \
590 uint32_t wc = *((const uint32_t *) inptr); \
591 \
592 if (__glibc_likely (wc <= 0x7f)) \
593 { \
594 /* Single UTF-8 char. */ \
595 *outptr = (uint8_t)wc; \
596 outptr++; \
597 } \
598 else if (wc <= 0x7ff) \
599 { \
600 /* Two UTF-8 chars. */ \
601 if (__glibc_unlikely (outptr + 2 > outend)) \
602 { \
603 /* Overflow in the output buffer. */ \
604 result = __GCONV_FULL_OUTPUT; \
605 break; \
606 } \
607 \
608 outptr[0] = 0xc0; \
609 outptr[0] |= wc >> 6; \
610 \
611 outptr[1] = 0x80; \
612 outptr[1] |= wc & 0x3f; \
613 \
614 outptr += 2; \
615 } \
616 else if (wc <= 0xffff) \
617 { \
618 /* Three UTF-8 chars. */ \
619 if (__glibc_unlikely (outptr + 3 > outend)) \
620 { \
621 /* Overflow in the output buffer. */ \
622 result = __GCONV_FULL_OUTPUT; \
623 break; \
624 } \
625 if (wc >= 0xd800 && wc <= 0xdfff) \
626 { \
627 /* Do not accept UTF-16 surrogates. */ \
628 result = __GCONV_ILLEGAL_INPUT; \
629 STANDARD_TO_LOOP_ERR_HANDLER (4); \
630 } \
631 outptr[0] = 0xe0; \
632 outptr[0] |= wc >> 12; \
633 \
634 outptr[1] = 0x80; \
635 outptr[1] |= (wc >> 6) & 0x3f; \
636 \
637 outptr[2] = 0x80; \
638 outptr[2] |= wc & 0x3f; \
639 \
640 outptr += 3; \
641 } \
642 else if (wc <= 0x10ffff) \
643 { \
644 /* Four UTF-8 chars. */ \
645 if (__glibc_unlikely (outptr + 4 > outend)) \
646 { \
647 /* Overflow in the output buffer. */ \
648 result = __GCONV_FULL_OUTPUT; \
649 break; \
650 } \
651 outptr[0] = 0xf0; \
652 outptr[0] |= wc >> 18; \
653 \
654 outptr[1] = 0x80; \
655 outptr[1] |= (wc >> 12) & 0x3f; \
656 \
657 outptr[2] = 0x80; \
658 outptr[2] |= (wc >> 6) & 0x3f; \
659 \
660 outptr[3] = 0x80; \
661 outptr[3] |= wc & 0x3f; \
662 \
663 outptr += 4; \
664 } \
665 else \
666 { \
667 STANDARD_TO_LOOP_ERR_HANDLER (4); \
668 } \
669 inptr += 4; \
670 }
671
672 /* Generate loop-function with software routing. */
673 # define MIN_NEEDED_INPUT MIN_NEEDED_TO
674 # define MIN_NEEDED_OUTPUT MIN_NEEDED_FROM
675 # define MAX_NEEDED_OUTPUT MAX_NEEDED_FROM
676 # define TO_LOOP_C __to_utf8_loop_c
677 # define LOOPFCT TO_LOOP_C
678 # define BODY BODY_TO_C
679 # define LOOP_NEED_FLAGS
680 # include <iconv/loop.c>
681 #else
682 # define TO_LOOP_C NULL
683 #endif /* HAVE_TO_C != 1 */
684
685 #if HAVE_TO_VX == 1
686 /* The hardware routine uses the S/390 vector instructions. */
687 # define BODY_TO_VX \
688 { \
689 size_t inlen = inend - inptr; \
690 size_t outlen = outend - outptr; \
691 unsigned long tmp, tmp2, tmp3; \
692 asm volatile (".machine push\n\t" \
693 ".machine \"z13\"\n\t" \
694 ".machinemode \"zarch_nohighgprs\"\n\t" \
695 " vleif %%v20,127,0\n\t" /* element 0: 127 */ \
696 " vzero %%v21\n\t" \
697 " vleih %%v21,8192,0\n\t" /* element 0: > */ \
698 " vleih %%v21,-8192,2\n\t" /* element 1: =<> */ \
699 CONVERT_32BIT_SIZE_T ([R_INLEN]) \
700 CONVERT_32BIT_SIZE_T ([R_OUTLEN]) \
701 /* Loop which handles UTF-32 chars <=0x7f. */ \
702 "0: clgijl %[R_INLEN],64,2f\n\t" \
703 " clgijl %[R_OUTLEN],16,2f\n\t" \
704 "1: vlm %%v16,%%v19,0(%[R_IN])\n\t" \
705 " lghi %[R_TMP2],0\n\t" \
706 /* Shorten to byte values. */ \
707 " vpkf %%v23,%%v16,%%v17\n\t" \
708 " vpkf %%v24,%%v18,%%v19\n\t" \
709 " vpkh %%v23,%%v23,%%v24\n\t" \
710 /* Checking for values > 0x7f. */ \
711 " vstrcfs %%v22,%%v16,%%v20,%%v21\n\t" \
712 " jno 10f\n\t" \
713 " vstrcfs %%v22,%%v17,%%v20,%%v21\n\t" \
714 " jno 11f\n\t" \
715 " vstrcfs %%v22,%%v18,%%v20,%%v21\n\t" \
716 " jno 12f\n\t" \
717 " vstrcfs %%v22,%%v19,%%v20,%%v21\n\t" \
718 " jno 13f\n\t" \
719 /* Store 16bytes to outptr. */ \
720 " vst %%v23,0(%[R_OUT])\n\t" \
721 " aghi %[R_INLEN],-64\n\t" \
722 " aghi %[R_OUTLEN],-16\n\t" \
723 " la %[R_IN],64(%[R_IN])\n\t" \
724 " la %[R_OUT],16(%[R_OUT])\n\t" \
725 " clgijl %[R_INLEN],64,2f\n\t" \
726 " clgijl %[R_OUTLEN],16,2f\n\t" \
727 " j 1b\n\t" \
728 /* Found a value > 0x7f. */ \
729 "13: ahi %[R_TMP2],4\n\t" \
730 "12: ahi %[R_TMP2],4\n\t" \
731 "11: ahi %[R_TMP2],4\n\t" \
732 "10: vlgvb %[R_TMP],%%v22,7\n\t" \
733 " srlg %[R_TMP],%[R_TMP],2\n\t" \
734 " agr %[R_TMP],%[R_TMP2]\n\t" \
735 " je 16f\n\t" \
736 /* Store characters before invalid one... */ \
737 " slgr %[R_OUTLEN],%[R_TMP]\n\t" \
738 "15: aghi %[R_TMP],-1\n\t" \
739 " vstl %%v23,%[R_TMP],0(%[R_OUT])\n\t" \
740 /* ... and update pointers. */ \
741 " aghi %[R_TMP],1\n\t" \
742 " la %[R_OUT],0(%[R_TMP],%[R_OUT])\n\t" \
743 " sllg %[R_TMP2],%[R_TMP],2\n\t" \
744 " la %[R_IN],0(%[R_TMP2],%[R_IN])\n\t" \
745 " slgr %[R_INLEN],%[R_TMP2]\n\t" \
746 /* Calculate remaining uint32_t values in loaded vrs. */ \
747 "16: lghi %[R_TMP2],16\n\t" \
748 " sgr %[R_TMP2],%[R_TMP]\n\t" \
749 " l %[R_TMP],0(%[R_IN])\n\t" \
750 " aghi %[R_INLEN],-4\n\t" \
751 " j 22f\n\t" \
752 /* Handle remaining bytes. */ \
753 "2: clgije %[R_INLEN],0,99f\n\t" \
754 " clgijl %[R_INLEN],4,92f\n\t" \
755 /* Calculate remaining uint32_t values in inptr. */ \
756 " srlg %[R_TMP2],%[R_INLEN],2\n\t" \
757 /* Handle multibyte utf8-char. */ \
758 "20: l %[R_TMP],0(%[R_IN])\n\t" \
759 " aghi %[R_INLEN],-4\n\t" \
760 /* Test if ch is 1byte UTF-8 char. */ \
761 "21: clijh %[R_TMP],0x7f,22f\n\t" \
762 /* Handle 1-byte UTF-8 char. */ \
763 "31: slgfi %[R_OUTLEN],1\n\t" \
764 " jl 90f \n\t" \
765 " stc %[R_TMP],0(%[R_OUT])\n\t" \
766 " la %[R_IN],4(%[R_IN])\n\t" \
767 " la %[R_OUT],1(%[R_OUT])\n\t" \
768 " brctg %[R_TMP2],20b\n\t" \
769 " j 0b\n\t" /* Switch to vx-loop. */ \
770 /* Test if ch is 2byte UTF-8 char. */ \
771 "22: clfi %[R_TMP],0x7ff\n\t" \
772 " jh 23f\n\t" \
773 /* Handle 2-byte UTF-8 char. */ \
774 "32: slgfi %[R_OUTLEN],2\n\t" \
775 " jl 90f \n\t" \
776 " llill %[R_TMP3],0xc080\n\t" \
777 " risbgn %[R_TMP3],%[R_TMP],51,55,2\n\t" /* 1. byte. */ \
778 " risbgn %[R_TMP3],%[R_TMP],58,63,0\n\t" /* 2. byte. */ \
779 " sth %[R_TMP3],0(%[R_OUT])\n\t" \
780 " la %[R_IN],4(%[R_IN])\n\t" \
781 " la %[R_OUT],2(%[R_OUT])\n\t" \
782 " brctg %[R_TMP2],20b\n\t" \
783 " j 0b\n\t" /* Switch to vx-loop. */ \
784 /* Test if ch is 3-byte UTF-8 char. */ \
785 "23: clfi %[R_TMP],0xffff\n\t" \
786 " jh 24f\n\t" \
787 /* Handle 3-byte UTF-8 char. */ \
788 "33: slgfi %[R_OUTLEN],3\n\t" \
789 " jl 90f \n\t" \
790 " llilf %[R_TMP3],0xe08080\n\t" \
791 " risbgn %[R_TMP3],%[R_TMP],44,47,4\n\t" /* 1. byte. */ \
792 " risbgn %[R_TMP3],%[R_TMP],50,55,2\n\t" /* 2. byte. */ \
793 " risbgn %[R_TMP3],%[R_TMP],58,63,0\n\t" /* 3. byte. */ \
794 /* Test if ch is a UTF-16 surrogate: ch & 0xf800 == 0xd800 */ \
795 " nilf %[R_TMP],0xf800\n\t" \
796 " clfi %[R_TMP],0xd800\n\t" \
797 " je 91f\n\t" /* Do not accept UTF-16 surrogates. */ \
798 " stcm %[R_TMP3],7,0(%[R_OUT])\n\t" \
799 " la %[R_IN],4(%[R_IN])\n\t" \
800 " la %[R_OUT],3(%[R_OUT])\n\t" \
801 " brctg %[R_TMP2],20b\n\t" \
802 " j 0b\n\t" /* Switch to vx-loop. */ \
803 /* Test if ch is 4-byte UTF-8 char. */ \
804 "24: clfi %[R_TMP],0x10ffff\n\t" \
805 " jh 91f\n\t" /* ch > 0x10ffff is not allowed! */ \
806 /* Handle 4-byte UTF-8 char. */ \
807 "34: slgfi %[R_OUTLEN],4\n\t" \
808 " jl 90f \n\t" \
809 " llilf %[R_TMP3],0xf0808080\n\t" \
810 " risbgn %[R_TMP3],%[R_TMP],37,39,6\n\t" /* 1. byte. */ \
811 " risbgn %[R_TMP3],%[R_TMP],42,47,4\n\t" /* 2. byte. */ \
812 " risbgn %[R_TMP3],%[R_TMP],50,55,2\n\t" /* 3. byte. */ \
813 " risbgn %[R_TMP3],%[R_TMP],58,63,0\n\t" /* 4. byte. */ \
814 " st %[R_TMP3],0(%[R_OUT])\n\t" \
815 " la %[R_IN],4(%[R_IN])\n\t" \
816 " la %[R_OUT],4(%[R_OUT])\n\t" \
817 " brctg %[R_TMP2],20b\n\t" \
818 " j 0b\n\t" /* Switch to vx-loop. */ \
819 "92: lghi %[R_RES],%[RES_IN_FULL]\n\t" \
820 " j 99f\n\t" \
821 "91: lghi %[R_RES],%[RES_IN_ILL]\n\t" \
822 " j 99f\n\t" \
823 "90: lghi %[R_RES],%[RES_OUT_FULL]\n\t" \
824 "99: \n\t" \
825 ".machine pop" \
826 : /* outputs */ [R_IN] "+a" (inptr) \
827 , [R_INLEN] "+d" (inlen), [R_OUT] "+a" (outptr) \
828 , [R_OUTLEN] "+d" (outlen), [R_TMP] "=a" (tmp) \
829 , [R_TMP2] "=a" (tmp2), [R_TMP3] "=d" (tmp3) \
830 , [R_RES] "+d" (result) \
831 : /* inputs */ \
832 [RES_OUT_FULL] "i" (__GCONV_FULL_OUTPUT) \
833 , [RES_IN_ILL] "i" (__GCONV_ILLEGAL_INPUT) \
834 , [RES_IN_FULL] "i" (__GCONV_INCOMPLETE_INPUT) \
835 : /* clobber list */ "memory", "cc" \
836 ASM_CLOBBER_VR ("v16") ASM_CLOBBER_VR ("v17") \
837 ASM_CLOBBER_VR ("v18") ASM_CLOBBER_VR ("v19") \
838 ASM_CLOBBER_VR ("v20") ASM_CLOBBER_VR ("v21") \
839 ASM_CLOBBER_VR ("v22") ASM_CLOBBER_VR ("v23") \
840 ASM_CLOBBER_VR ("v24") \
841 ); \
842 if (__glibc_likely (inptr == inend) \
843 || result != __GCONV_ILLEGAL_INPUT) \
844 break; \
845 \
846 STANDARD_TO_LOOP_ERR_HANDLER (4); \
847 }
848
849 /* Generate loop-function with hardware vector instructions. */
850 # define MIN_NEEDED_INPUT MIN_NEEDED_TO
851 # define MIN_NEEDED_OUTPUT MIN_NEEDED_FROM
852 # define MAX_NEEDED_OUTPUT MAX_NEEDED_FROM
853 # define TO_LOOP_VX __to_utf8_loop_vx
854 # define LOOPFCT TO_LOOP_VX
855 # define BODY BODY_TO_VX
856 # define LOOP_NEED_FLAGS
857 # include <iconv/loop.c>
858 #else
859 # define TO_LOOP_VX NULL
860 #endif /* HAVE_TO_VX != 1 */
861
862 #if HAVE_TO_VX_CU == 1
863 #define BODY_TO_VX_CU \
864 { \
865 register const unsigned char* pInput asm ("8") = inptr; \
866 register size_t inlen asm ("9") = inend - inptr; \
867 register unsigned char* pOutput asm ("10") = outptr; \
868 register size_t outlen asm ("11") = outend - outptr; \
869 unsigned long tmp, tmp2; \
870 asm volatile (".machine push\n\t" \
871 ".machine \"z13\"\n\t" \
872 ".machinemode \"zarch_nohighgprs\"\n\t" \
873 " vleif %%v20,127,0\n\t" /* element 0: 127 */ \
874 " vzero %%v21\n\t" \
875 " vleih %%v21,8192,0\n\t" /* element 0: > */ \
876 " vleih %%v21,-8192,2\n\t" /* element 1: =<> */ \
877 CONVERT_32BIT_SIZE_T ([R_INLEN]) \
878 CONVERT_32BIT_SIZE_T ([R_OUTLEN]) \
879 /* Loop which handles UTF-32 chars <= 0x7f. */ \
880 "0: clgijl %[R_INLEN],64,20f\n\t" \
881 " clgijl %[R_OUTLEN],16,20f\n\t" \
882 "1: vlm %%v16,%%v19,0(%[R_IN])\n\t" \
883 " lghi %[R_TMP],0\n\t" \
884 /* Shorten to byte values. */ \
885 " vpkf %%v23,%%v16,%%v17\n\t" \
886 " vpkf %%v24,%%v18,%%v19\n\t" \
887 " vpkh %%v23,%%v23,%%v24\n\t" \
888 /* Checking for values > 0x7f. */ \
889 " vstrcfs %%v22,%%v16,%%v20,%%v21\n\t" \
890 " jno 10f\n\t" \
891 " vstrcfs %%v22,%%v17,%%v20,%%v21\n\t" \
892 " jno 11f\n\t" \
893 " vstrcfs %%v22,%%v18,%%v20,%%v21\n\t" \
894 " jno 12f\n\t" \
895 " vstrcfs %%v22,%%v19,%%v20,%%v21\n\t" \
896 " jno 13f\n\t" \
897 /* Store 16bytes to outptr. */ \
898 " vst %%v23,0(%[R_OUT])\n\t" \
899 " aghi %[R_INLEN],-64\n\t" \
900 " aghi %[R_OUTLEN],-16\n\t" \
901 " la %[R_IN],64(%[R_IN])\n\t" \
902 " la %[R_OUT],16(%[R_OUT])\n\t" \
903 " clgijl %[R_INLEN],64,20f\n\t" \
904 " clgijl %[R_OUTLEN],16,20f\n\t" \
905 " j 1b\n\t" \
906 /* Found a value > 0x7f. */ \
907 "13: ahi %[R_TMP],4\n\t" \
908 "12: ahi %[R_TMP],4\n\t" \
909 "11: ahi %[R_TMP],4\n\t" \
910 "10: vlgvb %[R_I],%%v22,7\n\t" \
911 " srlg %[R_I],%[R_I],2\n\t" \
912 " agr %[R_I],%[R_TMP]\n\t" \
913 " je 20f\n\t" \
914 /* Store characters before invalid one... */ \
915 " slgr %[R_OUTLEN],%[R_I]\n\t" \
916 "15: aghi %[R_I],-1\n\t" \
917 " vstl %%v23,%[R_I],0(%[R_OUT])\n\t" \
918 /* ... and update pointers. */ \
919 " aghi %[R_I],1\n\t" \
920 " la %[R_OUT],0(%[R_I],%[R_OUT])\n\t" \
921 " sllg %[R_I],%[R_I],2\n\t" \
922 " la %[R_IN],0(%[R_I],%[R_IN])\n\t" \
923 " slgr %[R_INLEN],%[R_I]\n\t" \
924 /* Handle multibyte utf8-char with convert instruction. */ \
925 "20: cu41 %[R_OUT],%[R_IN]\n\t" \
926 " jo 0b\n\t" /* Try vector implementation again. */ \
927 " lochil %[R_RES],%[RES_OUT_FULL]\n\t" /* cc == 1. */ \
928 " lochih %[R_RES],%[RES_IN_ILL]\n\t" /* cc == 2. */ \
929 ".machine pop" \
930 : /* outputs */ [R_IN] "+a" (pInput) \
931 , [R_INLEN] "+d" (inlen), [R_OUT] "+a" (pOutput) \
932 , [R_OUTLEN] "+d" (outlen), [R_TMP] "=d" (tmp) \
933 , [R_I] "=a" (tmp2) \
934 , [R_RES] "+d" (result) \
935 : /* inputs */ \
936 [RES_OUT_FULL] "i" (__GCONV_FULL_OUTPUT) \
937 , [RES_IN_ILL] "i" (__GCONV_ILLEGAL_INPUT) \
938 : /* clobber list */ "memory", "cc" \
939 ASM_CLOBBER_VR ("v16") ASM_CLOBBER_VR ("v17") \
940 ASM_CLOBBER_VR ("v18") ASM_CLOBBER_VR ("v19") \
941 ASM_CLOBBER_VR ("v20") ASM_CLOBBER_VR ("v21") \
942 ASM_CLOBBER_VR ("v22") ASM_CLOBBER_VR ("v23") \
943 ASM_CLOBBER_VR ("v24") \
944 ); \
945 inptr = pInput; \
946 outptr = pOutput; \
947 \
948 if (__glibc_likely (inptr == inend) \
949 || result == __GCONV_FULL_OUTPUT) \
950 break; \
951 if (inptr + 4 > inend) \
952 { \
953 result = __GCONV_INCOMPLETE_INPUT; \
954 break; \
955 } \
956 STANDARD_TO_LOOP_ERR_HANDLER (4); \
957 }
958
959 /* Generate loop-function with hardware vector and utf-convert instructions. */
960 # define MIN_NEEDED_INPUT MIN_NEEDED_TO
961 # define MIN_NEEDED_OUTPUT MIN_NEEDED_FROM
962 # define MAX_NEEDED_OUTPUT MAX_NEEDED_FROM
963 # define TO_LOOP_VX_CU __to_utf8_loop_vx_cu
964 # define LOOPFCT TO_LOOP_VX_CU
965 # define BODY BODY_TO_VX_CU
966 # define LOOP_NEED_FLAGS
967 # include <iconv/loop.c>
968 #else
969 # define TO_LOOP_VX_CU NULL
970 #endif /* HAVE_TO_VX_CU != 1 */
971
972 /* This file also exists in sysdeps/s390/multiarch/ which
973 generates ifunc resolvers for FROM/TO_LOOP functions
974 and includes iconv/skeleton.c afterwards. */
975 #if ! defined USE_MULTIARCH
976 # include <iconv/skeleton.c>
977 #endif