1 /* Unicode character output to streams with locale dependent encoding.
2
3 Copyright (C) 2000-2003, 2006, 2008-2021 Free Software Foundation, Inc.
4
5 This program is free software: you can redistribute it and/or modify
6 it under the terms of the GNU General Public License as published by
7 the Free Software Foundation; either version 3 of the License, or
8 (at your option) any later version.
9
10 This program is distributed in the hope that it will be useful,
11 but WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 GNU General Public License for more details.
14
15 You should have received a copy of the GNU General Public License
16 along with this program. If not, see <https://www.gnu.org/licenses/>. */
17
18 /* Written by Bruno Haible <haible@clisp.cons.org>. */
19
20 #include <config.h>
21
22 /* Specification. */
23 #include "unicodeio.h"
24
25 #include <stdio.h>
26 #include <string.h>
27 #include <errno.h>
28
29 #if HAVE_ICONV
30 # include <iconv.h>
31 #endif
32
33 #include <error.h>
34
35 #include "gettext.h"
36 #define _(msgid) gettext (msgid)
37 #define N_(msgid) msgid
38
39 #include "localcharset.h"
40 #include "unistr.h"
41
42 /* When we pass a Unicode character to iconv(), we must pass it in a
43 suitable encoding. The standardized Unicode encodings are
44 UTF-8, UCS-2, UCS-4, UTF-16, UTF-16BE, UTF-16LE, UTF-7.
45 UCS-2 supports only characters up to \U0000FFFF.
46 UTF-16 and variants support only characters up to \U0010FFFF.
47 UTF-7 is way too complex and not supported by glibc-2.1.
48 UCS-4 specification leaves doubts about endianness and byte order
49 mark. glibc currently interprets it as big endian without byte order
50 mark, but this is not backed by an RFC.
51 So we use UTF-8. It supports characters up to \U7FFFFFFF and is
52 unambiguously defined. */
53
54 /* Luckily, the encoding's name is platform independent. */
55 #define UTF8_NAME "UTF-8"
56
57 /* Converts the Unicode character CODE to its multibyte representation
58 in the current locale and calls the SUCCESS callback on the resulting
59 byte sequence. If an error occurs, invokes the FAILURE callback instead,
60 passing it CODE and an English error string.
61 Returns whatever the callback returned.
62 Assumes that the locale doesn't change between two calls. */
63 long
64 unicode_to_mb (unsigned int code,
65 long (*success) (const char *buf, size_t buflen,
66 void *callback_arg),
67 long (*failure) (unsigned int code, const char *msg,
68 void *callback_arg),
69 void *callback_arg)
70 {
71 static int initialized;
72 static int is_utf8;
73 #if HAVE_ICONV
74 static iconv_t utf8_to_local;
75 #endif
76
77 char inbuf[6];
78 int count;
79
80 if (!initialized)
81 {
82 const char *charset = locale_charset ();
83
84 is_utf8 = !strcmp (charset, UTF8_NAME);
85 #if HAVE_ICONV
86 if (!is_utf8)
87 {
88 utf8_to_local = iconv_open (charset, UTF8_NAME);
89 if (utf8_to_local == (iconv_t)(-1))
90 /* For an unknown encoding, assume ASCII. */
91 utf8_to_local = iconv_open ("ASCII", UTF8_NAME);
92 }
93 #endif
94 initialized = 1;
95 }
96
97 /* Test whether the utf8_to_local converter is available at all. */
98 if (!is_utf8)
99 {
100 #if HAVE_ICONV
101 if (utf8_to_local == (iconv_t)(-1))
102 return failure (code, N_("iconv function not usable"), callback_arg);
103 #else
104 return failure (code, N_("iconv function not available"), callback_arg);
105 #endif
106 }
107
108 /* Convert the character to UTF-8. */
109 count = u8_uctomb ((unsigned char *) inbuf, code, sizeof (inbuf));
110 if (count < 0)
111 return failure (code, N_("character out of range"), callback_arg);
112
113 #if HAVE_ICONV
114 if (!is_utf8)
115 {
116 char outbuf[25];
117 const char *inptr;
118 size_t inbytesleft;
119 char *outptr;
120 size_t outbytesleft;
121 size_t res;
122
123 inptr = inbuf;
124 inbytesleft = count;
125 outptr = outbuf;
126 outbytesleft = sizeof (outbuf);
127
128 /* Convert the character from UTF-8 to the locale's charset. */
129 res = iconv (utf8_to_local,
130 (ICONV_CONST char **)&inptr, &inbytesleft,
131 &outptr, &outbytesleft);
132 /* Analyze what iconv() actually did and distinguish replacements
133 that are OK (no need to invoke the FAILURE callback), such as
134 - replacing GREEK SMALL LETTER MU with MICRO SIGN, or
135 - replacing FULLWIDTH COLON with ':', or
136 - replacing a Unicode TAG character (U+E00xx) with an empty string,
137 from replacements that are worse than the FAILURE callback, such as
138 - replacing 'รง' with '?' (NetBSD, Solaris 11) or '*' (musl) or
139 NUL (IRIX). */
140 if (inbytesleft > 0 || res == (size_t)(-1)
141 /* Irix iconv() inserts a NUL byte if it cannot convert. */
142 # if !defined _LIBICONV_VERSION && (defined sgi || defined __sgi)
143 || (res > 0 && code != 0 && outptr - outbuf == 1 && *outbuf == '\0')
144 # endif
145 /* FreeBSD iconv(), NetBSD iconv(), and Solaris 11 iconv() insert
146 a '?' if they cannot convert. */
147 # if !defined _LIBICONV_VERSION
148 || (res > 0 && outptr - outbuf == 1 && *outbuf == '?')
149 # endif
150 /* musl libc iconv() inserts a '*' if it cannot convert. */
151 # if !defined _LIBICONV_VERSION && MUSL_LIBC
152 || (res > 0 && outptr - outbuf == 1 && *outbuf == '*')
153 # endif
154 )
155 return failure (code, NULL, callback_arg);
156
157 /* Avoid glibc-2.1 bug and Solaris 7 bug. */
158 # if defined _LIBICONV_VERSION \
159 || !(((__GLIBC__ - 0 == 2 && __GLIBC_MINOR__ - 0 <= 1) \
160 && !defined __UCLIBC__) \
161 || defined __sun)
162
163 /* Get back to the initial shift state. */
164 res = iconv (utf8_to_local, NULL, NULL, &outptr, &outbytesleft);
165 if (res == (size_t)(-1))
166 return failure (code, NULL, callback_arg);
167 # endif
168
169 return success (outbuf, outptr - outbuf, callback_arg);
170 }
171 #endif
172
173 /* At this point, is_utf8 is true, so no conversion is needed. */
174 return success (inbuf, count, callback_arg);
175 }
176
177 /* Simple success callback that outputs the converted string.
178 The STREAM is passed as callback_arg. */
179 long
180 fwrite_success_callback (const char *buf, size_t buflen, void *callback_arg)
181 {
182 FILE *stream = (FILE *) callback_arg;
183
184 /* The return value of fwrite can be ignored here, because under normal
185 conditions (STREAM is an open stream and not wide-character oriented)
186 when fwrite() returns a value != buflen it also sets STREAM's error
187 indicator. */
188 fwrite (buf, 1, buflen, stream);
189 return 0;
190 }
191
192 /* Simple failure callback that displays an error and exits. */
193 static long
194 exit_failure_callback (unsigned int code, const char *msg,
195 _GL_UNUSED void *callback_arg)
196 {
197 if (msg == NULL)
198 error (1, 0, _("cannot convert U+%04X to local character set"), code);
199 else
200 error (1, 0, _("cannot convert U+%04X to local character set: %s"), code,
201 gettext (msg));
202 return -1;
203 }
204
205 /* Simple failure callback that displays a fallback representation in plain
206 ASCII, using the same notation as ISO C99 strings. */
207 static long
208 fallback_failure_callback (unsigned int code,
209 _GL_UNUSED const char *msg,
210 void *callback_arg)
211 {
212 FILE *stream = (FILE *) callback_arg;
213
214 if (code < 0x10000)
215 fprintf (stream, "\\u%04X", code);
216 else
217 fprintf (stream, "\\U%08X", code);
218 return -1;
219 }
220
221 /* Outputs the Unicode character CODE to the output stream STREAM.
222 Upon failure, exit if exit_on_error is true, otherwise output a fallback
223 notation. */
224 void
225 print_unicode_char (FILE *stream, unsigned int code, int exit_on_error)
226 {
227 unicode_to_mb (code, fwrite_success_callback,
228 exit_on_error
229 ? exit_failure_callback
230 : fallback_failure_callback,
231 stream);
232 }