1 /* Convert using charmaps and possibly iconv().
2 Copyright (C) 2001-2023 Free Software Foundation, Inc.
3 This file is part of the GNU C Library.
4
5 This program is free software; you can redistribute it and/or modify
6 it under the terms of the GNU General Public License as published
7 by the Free Software Foundation; version 2 of the License, or
8 (at your option) any later version.
9
10 This program is distributed in the hope that it will be useful,
11 but WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 GNU General Public License for more details.
14
15 You should have received a copy of the GNU General Public License
16 along with this program; if not, see <https://www.gnu.org/licenses/>. */
17
18 #include <assert.h>
19 #include <errno.h>
20 #include <error.h>
21 #include <fcntl.h>
22 #include <iconv.h>
23 #include <libintl.h>
24 #include <stdio.h>
25 #include <stdlib.h>
26 #include <unistd.h>
27 #include <stdint.h>
28 #include <sys/mman.h>
29 #include <sys/stat.h>
30
31 #include "iconv_prog.h"
32
33
34 /* Prototypes for a few program-wide used functions. */
35 #include <programs/xmalloc.h>
36
37
38 struct convtable
39 {
40 int term[256 / 8];
41 union
42 {
43 struct convtable *sub;
44 struct charseq *out;
45 } val[256];
46 };
47
48
49 static inline struct convtable *
50 allocate_table (void)
51 {
52 return (struct convtable *) xcalloc (1, sizeof (struct convtable));
53 }
54
55 static inline void
56 free_table (struct convtable *tbl)
57 {
58 free (tbl);
59 }
60
61
62 static inline int
63 is_term (struct convtable *tbl, unsigned int idx)
64 {
65 return tbl->term[idx / 8] & (1 << (idx % 8));
66 }
67
68
69 static inline void
70 clear_term (struct convtable *tbl, unsigned int idx)
71 {
72 tbl->term[idx / 8] &= ~(1 << (idx % 8));
73 }
74
75
76 static inline void
77 set_term (struct convtable *tbl, unsigned int idx)
78 {
79 tbl->term[idx / 8] |= 1 << (idx % 8);
80 }
81
82
83 /* Generate the conversion table. */
84 static struct convtable *use_from_charmap (struct charmap_t *from_charmap,
85 const char *to_code);
86 static struct convtable *use_to_charmap (const char *from_code,
87 struct charmap_t *to_charmap);
88 static struct convtable *use_both_charmaps (struct charmap_t *from_charmap,
89 struct charmap_t *to_charmap);
90
91 /* Prototypes for the functions doing the actual work. */
92 static int process_block (struct convtable *tbl, char *addr, size_t len,
93 FILE *output);
94 static int process_fd (struct convtable *tbl, int fd, FILE *output);
95 static int process_file (struct convtable *tbl, FILE *input, FILE *output);
96
97
98 int
99 charmap_conversion (const char *from_code, struct charmap_t *from_charmap,
100 const char *to_code, struct charmap_t *to_charmap,
101 int argc, int remaining, char *argv[],
102 const char *output_file)
103 {
104 struct convtable *cvtbl;
105 int status = EXIT_SUCCESS;
106
107 /* We have three different cases to handle:
108
109 - both, from_charmap and to_charmap, are available. This means we
110 can assume that the symbolic names match and use them to create
111 the mapping.
112
113 - only from_charmap is available. In this case we can only hope that
114 the symbolic names used are of the <Uxxxx> form in which case we
115 can use a UCS4->"to_code" iconv() conversion for the second step.
116
117 - only to_charmap is available. This is similar, only that we would
118 use iconv() for the "to_code"->UCS4 conversion.
119
120 We first create a table which maps input bytes into output bytes.
121 Once this is done we can handle all three of the cases above
122 equally. */
123 if (from_charmap != NULL)
124 {
125 if (to_charmap == NULL)
126 cvtbl = use_from_charmap (from_charmap, to_code);
127 else
128 cvtbl = use_both_charmaps (from_charmap, to_charmap);
129 }
130 else
131 {
132 assert (to_charmap != NULL);
133 cvtbl = use_to_charmap (from_code, to_charmap);
134 }
135
136 /* If we couldn't generate a table stop now. */
137 if (cvtbl == NULL)
138 return EXIT_FAILURE;
139
140 /* Determine output file. */
141 FILE *output;
142 if (output_file != NULL && strcmp (output_file, "-") != 0)
143 {
144 output = fopen (output_file, "w");
145 if (output == NULL)
146 error (EXIT_FAILURE, errno, _("cannot open output file"));
147 }
148 else
149 output = stdout;
150
151 /* We can now start the conversion. */
152 if (remaining == argc)
153 {
154 if (process_file (cvtbl, stdin, output) != 0)
155 status = EXIT_FAILURE;
156 }
157 else
158 do
159 {
160 int fd;
161
162 if (verbose)
163 printf ("%s:\n", argv[remaining]);
164 if (strcmp (argv[remaining], "-") == 0)
165 fd = 0;
166 else
167 {
168 fd = open (argv[remaining], O_RDONLY);
169
170 if (fd == -1)
171 {
172 error (0, errno, _("cannot open input file `%s'"),
173 argv[remaining]);
174 status = EXIT_FAILURE;
175 continue;
176 }
177 }
178
179 #ifdef _POSIX_MAPPED_FILES
180 struct stat64 st;
181 char *addr;
182 /* We have possibilities for reading the input file. First try
183 to mmap() it since this will provide the fastest solution. */
184 if (fstat64 (fd, &st) == 0
185 && ((addr = mmap (NULL, st.st_size, PROT_READ, MAP_PRIVATE,
186 fd, 0)) != MAP_FAILED))
187 {
188 /* Yes, we can use mmap(). The descriptor is not needed
189 anymore. */
190 if (close (fd) != 0)
191 error (EXIT_FAILURE, errno,
192 _("error while closing input `%s'"), argv[remaining]);
193
194 if (process_block (cvtbl, addr, st.st_size, output) < 0)
195 {
196 /* Something went wrong. */
197 status = EXIT_FAILURE;
198
199 /* We don't need the input data anymore. */
200 munmap ((void *) addr, st.st_size);
201
202 /* We cannot go on with producing output since it might
203 lead to problem because the last output might leave
204 the output stream in an undefined state. */
205 break;
206 }
207
208 /* We don't need the input data anymore. */
209 munmap ((void *) addr, st.st_size);
210 }
211 else
212 #endif /* _POSIX_MAPPED_FILES */
213 {
214 /* Read the file in pieces. */
215 if (process_fd (cvtbl, fd, output) != 0)
216 {
217 /* Something went wrong. */
218 status = EXIT_FAILURE;
219
220 /* We don't need the input file anymore. */
221 close (fd);
222
223 /* We cannot go on with producing output since it might
224 lead to problem because the last output might leave
225 the output stream in an undefined state. */
226 break;
227 }
228
229 /* Now close the file. */
230 close (fd);
231 }
232 }
233 while (++remaining < argc);
234
235 /* All done. */
236 if (output != stdout)
237 fclose (output);
238 free_table (cvtbl);
239 return status;
240 }
241
242
243 /* Add the IN->OUT mapping to TBL. OUT is potentially stored in the table.
244 IN is used only here, so it need not be kept live afterwards. */
245 static void
246 add_bytes (struct convtable *tbl, const struct charseq *in, struct charseq *out)
247 {
248 int n = 0;
249 unsigned int byte;
250
251 assert (in->nbytes > 0);
252
253 byte = ((unsigned char *) in->bytes)[n];
254 while (n + 1 < in->nbytes)
255 {
256 if (is_term (tbl, byte) || tbl->val[byte].sub == NULL)
257 {
258 /* Note that we simply ignore a definition for a byte sequence
259 which is also the prefix for a longer one. */
260 clear_term (tbl, byte);
261 tbl->val[byte].sub =
262 (struct convtable *) xcalloc (1, sizeof (struct convtable));
263 }
264
265 tbl = tbl->val[byte].sub;
266
267 byte = ((unsigned char *) in->bytes)[++n];
268 }
269
270 /* Only add the new sequence if there is none yet and the byte sequence
271 is not part of an even longer one. */
272 if (! is_term (tbl, byte) && tbl->val[byte].sub == NULL)
273 {
274 set_term (tbl, byte);
275 tbl->val[byte].out = out;
276 }
277 }
278
279 /* Try to convert SEQ from WCHAR_T format using CD.
280 Returns a malloc'd struct or NULL. */
281 static struct charseq *
282 convert_charseq (iconv_t cd, const struct charseq *seq)
283 {
284 struct charseq *result = NULL;
285
286 if (seq->ucs4 != UNINITIALIZED_CHAR_VALUE)
287 {
288 /* There is a chance. Try the iconv module. */
289 wchar_t inbuf[1] = { seq->ucs4 };
290 unsigned char outbuf[64];
291 char *inptr = (char *) inbuf;
292 size_t inlen = sizeof (inbuf);
293 char *outptr = (char *) outbuf;
294 size_t outlen = sizeof (outbuf);
295
296 (void) iconv (cd, &inptr, &inlen, &outptr, &outlen);
297
298 if (outptr != (char *) outbuf)
299 {
300 /* We got some output. Good, use it. */
301 outlen = sizeof (outbuf) - outlen;
302 assert ((char *) outbuf + outlen == outptr);
303
304 result = xmalloc (sizeof (struct charseq) + outlen);
305 result->name = seq->name;
306 result->ucs4 = seq->ucs4;
307 result->nbytes = outlen;
308 memcpy (result->bytes, outbuf, outlen);
309 }
310
311 /* Clear any possible state left behind. */
312 (void) iconv (cd, NULL, NULL, NULL, NULL);
313 }
314
315 return result;
316 }
317
318
319 static struct convtable *
320 use_from_charmap (struct charmap_t *from_charmap, const char *to_code)
321 {
322 /* We iterate over all entries in the from_charmap and for those which
323 have a known UCS4 representation we use an iconv() call to determine
324 the mapping to the to_code charset. */
325 struct convtable *rettbl;
326 iconv_t cd;
327 void *ptr = NULL;
328 const void *key;
329 size_t keylen;
330 void *data;
331
332 cd = iconv_open (to_code, "WCHAR_T");
333 if (cd == (iconv_t) -1)
334 /* We cannot do anything. */
335 return NULL;
336
337 rettbl = allocate_table ();
338
339 while (iterate_table (&from_charmap->char_table, &ptr, &key, &keylen, &data)
340 >= 0)
341 {
342 struct charseq *in = data;
343 struct charseq *newp = convert_charseq (cd, in);
344 if (newp != NULL)
345 add_bytes (rettbl, in, newp);
346 }
347
348 iconv_close (cd);
349
350 return rettbl;
351 }
352
353
354 static struct convtable *
355 use_to_charmap (const char *from_code, struct charmap_t *to_charmap)
356 {
357 /* We iterate over all entries in the to_charmap and for those which
358 have a known UCS4 representation we use an iconv() call to determine
359 the mapping to the from_code charset. */
360 struct convtable *rettbl;
361 iconv_t cd;
362 void *ptr = NULL;
363 const void *key;
364 size_t keylen;
365 void *data;
366
367 /* Note that the conversion we use here is the reverse direction. Without
368 exhaustive search we cannot figure out which input yields the UCS4
369 character we are looking for. Therefore we determine it the other
370 way round. */
371 cd = iconv_open (from_code, "WCHAR_T");
372 if (cd == (iconv_t) -1)
373 /* We cannot do anything. */
374 return NULL;
375
376 rettbl = allocate_table ();
377
378 while (iterate_table (&to_charmap->char_table, &ptr, &key, &keylen, &data)
379 >= 0)
380 {
381 struct charseq *out = data;
382 struct charseq *newp = convert_charseq (cd, out);
383 if (newp != NULL)
384 {
385 add_bytes (rettbl, newp, out);
386 free (newp);
387 }
388 }
389
390 iconv_close (cd);
391
392 return rettbl;
393 }
394
395
396 static struct convtable *
397 use_both_charmaps (struct charmap_t *from_charmap,
398 struct charmap_t *to_charmap)
399 {
400 /* In this case we iterate over all the entries in the from_charmap,
401 determine the internal name, and find an appropriate entry in the
402 to_charmap (if it exists). */
403 struct convtable *rettbl = allocate_table ();
404 void *ptr = NULL;
405 const void *key;
406 size_t keylen;
407 void *data;
408
409 while (iterate_table (&from_charmap->char_table, &ptr, &key, &keylen, &data)
410 >= 0)
411 {
412 struct charseq *in = (struct charseq *) data;
413 struct charseq *out = charmap_find_value (to_charmap, key, keylen);
414
415 if (out != NULL)
416 add_bytes (rettbl, in, out);
417 }
418
419 return rettbl;
420 }
421
422
423 static int
424 process_block (struct convtable *tbl, char *addr, size_t len, FILE *output)
425 {
426 size_t n = 0;
427
428 while (n < len)
429 {
430 struct convtable *cur = tbl;
431 unsigned char *curp = (unsigned char *) addr;
432 unsigned int byte = *curp;
433 int cnt;
434 struct charseq *out;
435
436 while (! is_term (cur, byte))
437 if (cur->val[byte].sub == NULL)
438 {
439 /* This is an invalid sequence. Skip the first byte if we are
440 ignoring errors. Otherwise punt. */
441 if (! omit_invalid)
442 {
443 error (0, 0, _("illegal input sequence at position %zd"), n);
444 return -1;
445 }
446
447 n -= curp - (unsigned char *) addr;
448
449 byte = *(curp = (unsigned char *) ++addr);
450 if (++n >= len)
451 /* All converted. */
452 return 0;
453
454 cur = tbl;
455 }
456 else
457 {
458 cur = cur->val[byte].sub;
459
460 if (++n >= len)
461 {
462 error (0, 0, _("\
463 incomplete character or shift sequence at end of buffer"));
464 return -1;
465 }
466
467 byte = *++curp;
468 }
469
470 /* We found a final byte. Write the output bytes. */
471 out = cur->val[byte].out;
472 for (cnt = 0; cnt < out->nbytes; ++cnt)
473 fputc_unlocked (out->bytes[cnt], output);
474
475 addr = (char *) curp + 1;
476 ++n;
477 }
478
479 return 0;
480 }
481
482
483 static int
484 process_fd (struct convtable *tbl, int fd, FILE *output)
485 {
486 /* We have a problem with reading from a descriptor since we must not
487 provide the iconv() function an incomplete character or shift
488 sequence at the end of the buffer. Since we have to deal with
489 arbitrary encodings we must read the whole text in a buffer and
490 process it in one step. */
491 static char *inbuf = NULL;
492 static size_t maxlen = 0;
493 char *inptr = inbuf;
494 size_t actlen = 0;
495
496 while (actlen < maxlen)
497 {
498 ssize_t n = read (fd, inptr, maxlen - actlen);
499
500 if (n == 0)
501 /* No more text to read. */
502 break;
503
504 if (n == -1)
505 {
506 /* Error while reading. */
507 error (0, errno, _("error while reading the input"));
508 return -1;
509 }
510
511 inptr += n;
512 actlen += n;
513 }
514
515 if (actlen == maxlen)
516 while (1)
517 {
518 ssize_t n;
519 char *new_inbuf;
520
521 /* Increase the buffer. */
522 new_inbuf = (char *) realloc (inbuf, maxlen + 32768);
523 if (new_inbuf == NULL)
524 {
525 error (0, errno, _("unable to allocate buffer for input"));
526 return -1;
527 }
528 inbuf = new_inbuf;
529 maxlen += 32768;
530 inptr = inbuf + actlen;
531
532 do
533 {
534 n = read (fd, inptr, maxlen - actlen);
535
536 if (n == 0)
537 /* No more text to read. */
538 break;
539
540 if (n == -1)
541 {
542 /* Error while reading. */
543 error (0, errno, _("error while reading the input"));
544 return -1;
545 }
546
547 inptr += n;
548 actlen += n;
549 }
550 while (actlen < maxlen);
551
552 if (n == 0)
553 /* Break again so we leave both loops. */
554 break;
555 }
556
557 /* Now we have all the input in the buffer. Process it in one run. */
558 return process_block (tbl, inbuf, actlen, output);
559 }
560
561
562 static int
563 process_file (struct convtable *tbl, FILE *input, FILE *output)
564 {
565 /* This should be safe since we use this function only for `stdin' and
566 we haven't read anything so far. */
567 return process_fd (tbl, fileno (input), output);
568 }