1 /* Copyright 2010-2023 Free Software Foundation, Inc.
2
3 This program is free software: you can redistribute it and/or modify
4 it under the terms of the GNU General Public License as published by
5 the Free Software Foundation, either version 3 of the License, or
6 (at your option) any later version.
7
8 This program is distributed in the hope that it will be useful,
9 but WITHOUT ANY WARRANTY; without even the implied warranty of
10 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 GNU General Public License for more details.
12
13 You should have received a copy of the GNU General Public License
14 along with this program. If not, see <http://www.gnu.org/licenses/>. */
15
16 #include <config.h>
17
18 #include <stdlib.h>
19 #include <stdio.h>
20 #include <string.h>
21 #include <iconv.h>
22 #include <errno.h>
23 #include <sys/stat.h>
24
25 #include "errors.h"
26 #include "debug.h"
27 #include "input.h"
28 #include "text.h"
29 #include "commands.h"
30 #include "source_marks.h"
31
32 enum input_type { IN_file, IN_text };
33
34 typedef struct {
35 enum input_type type;
36
37 FILE *file;
38 SOURCE_INFO source_info;
39 char *input_file_path; /* for IN_file type, the full input file path */
40
41 char *text; /* Input text to be parsed as Texinfo. */
42 char *ptext; /* How far we are through 'text'. Used to split 'text'
43 into lines. */
44 char *value_flag; /* value flag if the input text is a @value
45 expansion */
46 char *macro_name; /* macro name if the input text is a user-defined
47 macro expansion */
48 SOURCE_MARK *input_source_mark;
49 } INPUT;
50
51 static char *input_pushback_string;
52
53 static iconv_t reverse_iconv; /* used in encode_file_name */
54
55 typedef struct {
56 char *encoding_name;
57 iconv_t iconv;
58 } ENCODING_CONVERSION;
59
60 static ENCODING_CONVERSION *encodings_list = 0;
61 int encoding_number = 0;
62 int encoding_space = 0;
63 char *global_input_encoding_name = 0;
64
65 static ENCODING_CONVERSION *current_encoding_conversion = 0;
66
67 /* ENCODING should always be lower cased */
68 /* WARNING: it is very important for the first call to
69 set_input_encoding to be for "utf-8" as the codes assume
70 a conversion to UTF-8 in encodings_list[0]. */
71 int
72 set_input_encoding (char *encoding)
73 {
74 int encoding_index = -1;
75 int encoding_set = 0;
76 char *conversion_encoding = encoding;
77
78 /* should correspond to
79 Texinfo::Common::encoding_name_conversion_map.
80 Thoughts on this mapping are available near
81 Texinfo::Common::encoding_name_conversion_map definition
82 */
83 if (!strcmp (encoding, "us-ascii"))
84 conversion_encoding = "iso-8859-1";
85
86 if (reverse_iconv)
87 {
88 iconv_close (reverse_iconv);
89 reverse_iconv = (iconv_t) 0;
90 }
91
92 if (!strcmp (encoding, "utf-8"))
93 {
94 if (encoding_number > 0)
95 encoding_index = 0;
96 }
97 else if (encoding_number > 1)
98 {
99 int i;
100 for (i = 1; i < encoding_number; i++)
101 {
102 if (!strcmp (encoding, encodings_list[i].encoding_name))
103 {
104 encoding_index = i;
105 break;
106 }
107 }
108 }
109
110 if (encoding_index == -1)
111 {
112 if (encoding_number >= encoding_space)
113 {
114 encodings_list = realloc (encodings_list,
115 (encoding_space += 3) * sizeof (ENCODING_CONVERSION));
116 }
117 encodings_list[encoding_number].encoding_name
118 = strdup (conversion_encoding);
119 /* Initialize conversions for the first time. iconv_open returns
120 (iconv_t) -1 on failure so these should only be called once. */
121 encodings_list[encoding_number].iconv
122 = iconv_open ("UTF-8", conversion_encoding);
123 encoding_index = encoding_number;
124 encoding_number++;
125 }
126
127 if (encodings_list[encoding_index].iconv == (iconv_t) -1)
128 current_encoding_conversion = 0;
129 else
130 {
131 current_encoding_conversion = &encodings_list[encoding_index];
132 encoding_set = 1;
133 free (global_input_encoding_name);
134 global_input_encoding_name = strdup (encoding);
135 }
136
137 return encoding_set;
138 }
139
140
141 static INPUT *input_stack = 0;
142 int input_number = 0;
143 int input_space = 0;
144 int macro_expansion_nr = 0;
145 int value_expansion_nr = 0;
146
147 /* Current filename and line number. Used for reporting. */
148 SOURCE_INFO current_source_info;
149
150 /* Collect text from the input sources until a newline is found. This is used
151 instead of next_text when we need to be sure we get an entire line of
152 Texinfo input (for example as a line argument to a command), which might not
153 be the case if the input is the result of a macro expansion.
154
155 Return value should not be freed by caller, and becomes invalid after
156 a subsequent call. */
157 /* CURRENT is the current container that can be used for source marks. */
158 char *
159 new_line (ELEMENT *current)
160 {
161 static TEXT t;
162 char *new = 0;
163
164 t.end = 0;
165
166 while (1)
167 {
168 new = next_text (current);
169 if (!new)
170 break;
171 text_append (&t, new);
172 free (new);
173
174 if (t.text[t.end - 1] == '\n')
175 break;
176 }
177
178 if (t.end > 0)
179 return t.text;
180 else
181 return 0;
182 }
183
184
185 /* Run iconv using text buffer as output buffer. */
186 size_t
187 text_buffer_iconv (TEXT *buf, iconv_t iconv_state,
188 ICONV_CONST char **inbuf, size_t *inbytesleft)
189 {
190 size_t out_bytes_left;
191 char *outptr;
192 size_t iconv_ret;
193
194 outptr = buf->text + buf->end;
195 if (buf->end == buf->space - 1)
196 {
197 errno = E2BIG;
198 return (size_t) -1;
199 }
200 out_bytes_left = buf->space - buf->end - 1;
201 iconv_ret = iconv (iconv_state, inbuf, inbytesleft,
202 &outptr, &out_bytes_left);
203
204 buf->end = outptr - buf->text;
205
206 return iconv_ret;
207 }
208
209
210 static char *
211 encode_with_iconv (iconv_t our_iconv, char *s)
212 {
213 static TEXT t;
214 ICONV_CONST char *inptr; size_t bytes_left;
215 size_t iconv_ret;
216
217 t.end = 0; /* reset internal TEXT buffer */
218 inptr = s;
219 bytes_left = strlen (s);
220 text_alloc (&t, 10);
221
222 while (1)
223 {
224 iconv_ret = text_buffer_iconv (&t, our_iconv,
225 &inptr, &bytes_left);
226
227 /* Make sure libiconv flushes out the last converted character.
228 This is required when the conversion is stateful, in which
229 case libiconv might not output the last character, waiting to
230 see whether it should be combined with the next one. */
231 if (iconv_ret != (size_t) -1
232 && text_buffer_iconv (&t, our_iconv, 0, 0) != (size_t) -1)
233 /* Success: all of input converted. */
234 break;
235
236 if (bytes_left == 0)
237 break;
238
239 switch (errno)
240 {
241 case E2BIG:
242 text_alloc (&t, t.space + 20);
243 break;
244 case EILSEQ:
245 default:
246 fprintf(stderr, "%s:%d: encoding error at byte 0x%2x\n",
247 current_source_info.file_name, current_source_info.line_nr,
248 *(unsigned char *)inptr);
249 inptr++; bytes_left--;
250 break;
251 }
252 }
253
254 t.text[t.end] = '\0';
255 return strdup (t.text);
256 }
257
258 /* Return conversion of S according to input_encoding. This function
259 frees S. */
260 char *
261 convert_to_utf8 (char *s)
262 {
263 char *ret;
264
265 /* Convert from @documentencoding to UTF-8.
266 It might be possible not to convert to UTF-8 and use an 8-bit encoding
267 throughout, but then we'd have to not set the UTF-8 flag on the Perl
268 strings in api.c. If multiple character encodings were used in a single
269 file, then we'd have to keep track of which strings needed the UTF-8 flag
270 and which didn't. */
271
272 if (current_encoding_conversion == 0)
273 {
274 /* In case the converter couldn't be initialised.
275 Danger: this will cause problems if the input is not in UTF-8 as
276 the Perl strings that are created are flagged as being UTF-8. */
277 return s;
278 }
279
280 ret = encode_with_iconv (current_encoding_conversion->iconv, s);
281 free (s);
282 return ret;
283 }
284
285
286 int doc_encoding_for_input_file_name = 1;
287 char *input_file_name_encoding = 0;
288 char *locale_encoding = 0;
289
290 void
291 set_input_file_name_encoding (char *value)
292 {
293 free (input_file_name_encoding);
294 input_file_name_encoding = value ? strdup (value) : 0;
295 }
296
297 void
298 set_locale_encoding (char *value)
299 {
300 free (locale_encoding);
301 locale_encoding = value ? strdup (value) : 0;
302 }
303
304 /* Reverse the decoding of the filename to the input encoding, to retrieve
305 the bytes that were present in the original Texinfo file. Return
306 value is freed by free_small_strings. */
307 char *
308 encode_file_name (char *filename)
309 {
310 if (!reverse_iconv)
311 {
312 if (input_file_name_encoding)
313 {
314 reverse_iconv = iconv_open (input_file_name_encoding, "UTF-8");
315 }
316 else if (doc_encoding_for_input_file_name)
317 {
318 if (current_encoding_conversion
319 && strcmp (global_input_encoding_name, "utf-8"))
320 {
321 char *conversion_encoding
322 = current_encoding_conversion->encoding_name;
323 reverse_iconv = iconv_open (conversion_encoding, "UTF-8");
324 }
325 }
326 else if (locale_encoding)
327 {
328 reverse_iconv = iconv_open (locale_encoding, "UTF-8");
329 }
330 }
331 if (reverse_iconv && reverse_iconv != (iconv_t) -1)
332 {
333 char *s, *conv;
334 conv = encode_with_iconv (reverse_iconv, filename);
335 s = save_string (conv);
336 free (conv);
337 return s;
338 }
339 else
340 {
341 return save_string (filename);
342 }
343 }
344
345 /* Change the line number of filename of the top input source. Used to
346 record a #line directive. */
347 void
348 save_line_directive (int line_nr, char *filename)
349 {
350 char *f = 0;
351 INPUT *top;
352
353 if (filename)
354 f = encode_file_name (filename);
355
356 top = &input_stack[input_number - 1];
357 if (line_nr)
358 top->source_info.line_nr = line_nr;
359 if (filename)
360 top->source_info.file_name = f;
361 }
362
363
364
365 int
366 expanding_macro (char *macro)
367 {
368 int i;
369 for (i = 0; i < input_number; i++)
370 {
371 if (input_stack[i].source_info.macro
372 && !strcmp (input_stack[i].source_info.macro, macro))
373 {
374 return 1;
375 }
376 }
377 return 0;
378 }
379
380 char *save_string (char *string);
381
382 void
383 input_pushback (char *string)
384 {
385 if (input_pushback_string)
386 fprintf (stderr,
387 "texi2any (XS module): bug: input_pushback called twice\n");
388 input_pushback_string = string;
389 }
390
391 /* Return value to be freed by caller. Return null if we are out of input. */
392 /* CURRENT is the current container that can be used for source marks. */
393 char *
394 next_text (ELEMENT *current)
395 {
396 ssize_t status;
397 char *line = 0;
398 size_t n = 1;
399 /* Note: n needs to be a positive value, rather than 0, to work around
400 a bug in getline on MinGW. This appears to be allowed by POSIX. */
401 FILE *input_file;
402
403 if (input_pushback_string)
404 {
405 char *s;
406 s = input_pushback_string;
407 input_pushback_string = 0;
408 return s;
409 }
410
411 while (input_number > 0)
412 {
413 /* Check for pending input. */
414 INPUT *input = &input_stack[input_number - 1];
415
416 switch (input->type)
417 {
418 char *p, *new;
419 case IN_text:
420 /*
421 debug_nonl ("IN_TEXT '"); debug_print_protected_string (input->ptext);
422 debug ("'");
423 */
424 if (!*input->ptext)
425 break;
426 /* Split off a line of input. */
427 p = strchrnul (input->ptext, '\n');
428 new = strndup (input->ptext, p - input->ptext + 1);
429 if (*p)
430 input->ptext = p + 1;
431 else
432 input->ptext = p; /* The next time, we will pop the input source. */
433 /*
434 debug_nonl ("NEW IN_TEXT '"); debug_print_protected_string (new);
435 debug_nonl ("' next: '");
436 debug_print_protected_string (input->ptext); debug ("'");
437 */
438 if (!input->source_info.macro && !input->value_flag)
439 input->source_info.line_nr++;
440
441 current_source_info = input->source_info;
442
443 return new;
444
445 break;
446 case IN_file:
447 input_file = input->file;
448 status = getline (&line, &n, input_file);
449 if (status != -1)
450 {
451 char *comment;
452 if (feof (input_file))
453 {
454 /* Add a newline at the end of the file if one is missing. */
455 char *line2;
456 xasprintf (&line2, "%s\n", line);
457 free (line);
458 line = line2;
459 }
460
461 /* Strip off a comment. */
462 comment = strchr (line, '\x7F');
463 if (comment)
464 {
465 SOURCE_MARK *source_mark
466 = new_source_mark (SM_type_delcomment);
467 *comment = '\0';
468 if (*(comment+1) != '\0')
469 source_mark->line = convert_to_utf8 (strdup (comment+1));
470 else
471 source_mark->line = 0;
472 input_push_text(strdup (""),
473 input->source_info.line_nr, 0, 0);
474 /* if the input_stack was reallocated in input_push_text,
475 the input pointer for the file may have been freed and
476 re-created at another address. Therefore we reset it.
477 input_number has been increased too, so the input file
478 being processed is now at input_number - 2 */
479 input = &input_stack[input_number - 2];
480 set_input_source_mark (source_mark);
481 }
482
483 input->source_info.line_nr++;
484 current_source_info = input->source_info;
485
486 return convert_to_utf8 (line);
487 }
488 free (line); line = 0;
489 break;
490 default:
491 fatal ("unknown input source type");
492 }
493
494 /* Top input source failed. Close, pop, and try the next one. */
495 if (input->type == IN_file)
496 {
497 FILE *file = input->file;
498
499 if (file != stdin)
500 {
501 if (fclose (input->file) == EOF)
502 {
503 /* convert to UTF-8 for the messages, to have character strings in perl
504 that will be encoded on output to the locale encoding.
505 Done differently for the file names in source_info
506 which are byte strings and end up unmodified in output error
507 messages.
508 */
509 char *decoded_file_name
510 = convert_to_utf8 (strdup(input->input_file_path));
511 line_warn ("error on closing %s: %s",
512 decoded_file_name,
513 strerror (errno));
514 free (decoded_file_name);
515 }
516 }
517 }
518 else
519 {
520 /* End of text reached. */
521 free (input->text);
522 if (input->value_flag)
523 {
524 value_expansion_nr--;
525 free (input->value_flag);
526 }
527 else if (input->macro_name)
528 {
529 macro_expansion_nr--;
530 }
531 }
532
533 if (input->input_source_mark)
534 {
535 if (current)
536 {
537 SOURCE_MARK *input_source_mark = input->input_source_mark;
538 SOURCE_MARK *end_include_source_mark;
539 if (input_source_mark->type == SM_type_delcomment)
540 end_include_source_mark = input_source_mark;
541 else
542 {
543 end_include_source_mark
544 = new_source_mark (input_source_mark->type);
545 end_include_source_mark->counter = input_source_mark->counter;
546 end_include_source_mark->status = SM_status_end;
547 }
548 register_source_mark (current, end_include_source_mark);
549 }
550 else
551 debug ("INPUT MARK MISSED");
552
553 input->input_source_mark = 0;
554 }
555 input_number--;
556 }
557 debug ("INPUT FINISHED");
558 return 0;
559 }
560
561 /* Store TEXT as a source for Texinfo content. TEXT should be a UTF-8
562 string. TEXT will be later free'd and must be allocated on the heap.
563 MACRO_NAME is the name of the macro expanded as text. It should only be
564 given if this is the text corresponds to a new macro expansion.
565 If already within a macro expansion, but not from a macro expansion
566 (from a value expansion, for instance), the macro name will be taken
567 from the input stack.
568 VALUE_FLAG is the name of the value flag expanded as text.
569 VALUE_FLAG will be later free'd, but not MACRO_NAME.
570 */
571 void
572 input_push_text (char *text, int line_number, char *macro_name,
573 char *value_flag)
574 {
575 char *filename = 0;
576 char *in_macro = 0;
577
578 if (!text)
579 return;
580
581 if (input_number == input_space)
582 {
583 input_space++; input_space *= 1.5;
584 input_stack = realloc (input_stack, input_space * sizeof (INPUT));
585 if (!input_stack)
586 fatal ("realloc failed");
587 }
588
589 input_stack[input_number].type = IN_text;
590 input_stack[input_number].file = 0;
591 input_stack[input_number].input_file_path = 0;
592 input_stack[input_number].text = text;
593 input_stack[input_number].ptext = text;
594
595 if (input_number > 0)
596 {
597 filename = input_stack[input_number - 1].source_info.file_name;
598 /* context macro expansion */
599 in_macro = input_stack[input_number - 1].source_info.macro;
600 }
601 if (macro_name) {
602 /* new macro expansion */
603 in_macro = macro_name;
604 }
605 if (!in_macro && !value_flag)
606 line_number--;
607 input_stack[input_number].source_info.line_nr = line_number;
608 input_stack[input_number].source_info.file_name = save_string (filename);
609 input_stack[input_number].source_info.macro = save_string (in_macro);
610 input_stack[input_number].macro_name = save_string (macro_name);
611 input_stack[input_number].value_flag = value_flag;
612 input_stack[input_number].input_source_mark = 0;
613 input_number++;
614 }
615
616 void
617 set_input_source_mark (SOURCE_MARK *source_mark)
618 {
619 input_stack[input_number - 1].input_source_mark = source_mark;
620 }
621
622 /* For filenames and macro names, it is possible that they won't be referenced
623 in the line number of any element. It would be too much work to keep track,
624 so just keep them all here, and free them all together at the end. */
625 static char **small_strings;
626 static size_t small_strings_num;
627 static size_t small_strings_space;
628
629 char *
630 save_string (char *string)
631 {
632 char *ret = string ? strdup (string) : 0;
633 if (ret)
634 {
635 if (small_strings_num == small_strings_space)
636 {
637 small_strings_space++;
638 small_strings_space += (small_strings_space >> 2);
639 small_strings = realloc (small_strings, small_strings_space
640 * sizeof (char *));
641 if (!small_strings)
642 fatal ("realloc failed");
643 }
644 small_strings[small_strings_num++] = ret;
645 }
646 return ret;
647 }
648
649 /* Called in reset_parser. */
650 void
651 free_small_strings (void)
652 {
653 size_t i;
654 for (i = 0; i < small_strings_num; i++)
655 {
656 free (small_strings[i]);
657 }
658 small_strings_num = 0;
659 }
660
661 void
662 input_reset_input_stack (void)
663 {
664 int i;
665 for (i = 0; i < input_number; i++)
666 {
667 switch (input_stack[i].type)
668 {
669 case IN_file:
670 if (input_stack[i].file != stdin)
671 fclose (input_stack[i].file);
672 break;
673 case IN_text:
674 free (input_stack[i].text);
675 break;
676 }
677 }
678 input_number = 0;
679 macro_expansion_nr = 0;
680 value_expansion_nr = 0;
681 }
682
683 void
684 reset_encoding_list (void)
685 {
686 int i;
687 /* never reset the utf-8 encoding in position 0 */
688 if (encoding_number > 1)
689 {
690 for (i = 1; i < encoding_number; i++)
691 {
692 free (encodings_list[i].encoding_name);
693 if (encodings_list[i].iconv != (iconv_t) -1)
694 iconv_close (encodings_list[i].iconv);
695 }
696 encoding_number = 1;
697 }
698 /* could be named global_encoding_conversion and reset in wipe_global_info,
699 but we prefer to keep it static as long as it is only used in one
700 file */
701 current_encoding_conversion = 0;
702 }
703
704 int
705 top_file_index (void)
706 {
707 int i = input_number - 1;
708 while (i >= 0 && input_stack[i].type != IN_file)
709 i--;
710 return i;
711 }
712
713
714 static char **include_dirs;
715 static size_t include_dirs_number;
716 static size_t include_dirs_space;
717
718 void
719 add_include_directory (char *filename)
720 {
721 int len;
722 if (include_dirs_number == include_dirs_space)
723 {
724 include_dirs = realloc (include_dirs,
725 sizeof (char *) * (include_dirs_space += 5));
726 }
727 filename = strdup (filename);
728 include_dirs[include_dirs_number++] = filename;
729 len = strlen (filename);
730 if (len > 0 && filename[len - 1] == '/')
731 filename[len - 1] = '\0';
732 }
733
734 void
735 clear_include_directories (void)
736 {
737 int i;
738 for (i = 0; i < include_dirs_number; i++)
739 {
740 free (include_dirs[i]);
741 }
742 include_dirs_number = 0;
743 }
744
745 /* Return value to be freed by caller. */
746 char *
747 locate_include_file (char *filename)
748 {
749 char *fullpath;
750 struct stat dummy;
751 int i, status;
752
753 /* Checks if filename is absolute or relative to current directory. */
754 /* Note: the Perl code (in Common.pm, 'locate_include_file') handles
755 a volume in a path (like "A:") using the File::Spec module. */
756 if (!memcmp (filename, "/", 1)
757 || !memcmp (filename, "../", 3)
758 || !memcmp (filename, "./", 2))
759 {
760 status = stat (filename, &dummy);
761 if (status == 0)
762 return strdup (filename);
763 }
764 else
765 {
766 for (i = 0; i < include_dirs_number; i++)
767 {
768 xasprintf (&fullpath, "%s/%s", include_dirs[i], filename);
769 status = stat (fullpath, &dummy);
770 if (status == 0)
771 return fullpath;
772 free (fullpath);
773 }
774 }
775 return 0;
776 }
777
778 /* Try to open a file called FILENAME, looking for it in the list of include
779 directories. */
780 int
781 input_push_file (char *filename)
782 {
783 FILE *stream = 0;
784 char *p, *q;
785 char *base_filename;
786
787 if (!strcmp (filename, "-"))
788 stream = stdin;
789 else
790 {
791 stream = fopen (filename, "r");
792 if (!stream)
793 return errno;
794 }
795
796 if (input_number == input_space)
797 {
798 input_stack = realloc (input_stack, (input_space += 5) * sizeof (INPUT));
799 if (!input_stack)
800 fatal ("realloc failed");
801 }
802
803 /* Strip off a leading directory path. */
804 p = 0;
805 q = strchr (filename, '/');
806 while (q)
807 {
808 p = q;
809 q = strchr (q + 1, '/');
810 }
811 if (p)
812 base_filename = save_string (p+1);
813 else
814 base_filename = save_string (filename);
815
816 input_stack[input_number].type = IN_file;
817 input_stack[input_number].file = stream;
818 input_stack[input_number].input_file_path = filename;
819 input_stack[input_number].source_info.file_name = base_filename;
820 input_stack[input_number].source_info.line_nr = 0;
821 input_stack[input_number].source_info.macro = 0;
822 input_stack[input_number].input_source_mark = 0;
823 input_stack[input_number].text = 0;
824 input_stack[input_number].ptext = 0;
825 input_number++;
826
827 return 0;
828 }
829