1 /* readtokens.c -- Functions for reading tokens from an input stream.
2
3 Copyright (C) 1990-1991, 1999-2004, 2006, 2009-2023 Free Software
4 Foundation, Inc.
5
6 This program is free software: you can redistribute it and/or modify
7 it under the terms of the GNU General Public License as published by
8 the Free Software Foundation, either version 3 of the License, or
9 (at your option) any later version.
10
11 This program is distributed in the hope that it will be useful,
12 but WITHOUT ANY WARRANTY; without even the implied warranty of
13 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 GNU General Public License for more details.
15
16 You should have received a copy of the GNU General Public License
17 along with this program. If not, see <https://www.gnu.org/licenses/>.
18
19 Written by Jim Meyering. */
20
21 /* This almost supersedes xreadline stuff -- using delim="\n"
22 gives the same functionality, except that these functions
23 would never return empty lines. */
24
25 #include <config.h>
26
27 #include "readtokens.h"
28
29 #include <limits.h>
30 #include <stdio.h>
31 #include <stdlib.h>
32 #include <string.h>
33
34 #include "xalloc.h"
35
36 #if USE_UNLOCKED_IO
37 # include "unlocked-io.h"
38 #endif
39
40 /* Initialize a tokenbuffer. */
41
42 void
43 init_tokenbuffer (token_buffer *tokenbuffer)
44 {
45 tokenbuffer->size = 0;
46 tokenbuffer->buffer = NULL;
47 }
48
49 typedef size_t word;
50 enum { bits_per_word = sizeof (word) * CHAR_BIT };
51
52 static bool
53 get_nth_bit (size_t n, word const *bitset)
54 {
55 return bitset[n / bits_per_word] >> n % bits_per_word & 1;
56 }
57
58 static void
59 set_nth_bit (size_t n, word *bitset)
60 {
61 size_t one = 1;
62 bitset[n / bits_per_word] |= one << n % bits_per_word;
63 }
64
65 /* Read a token from STREAM into TOKENBUFFER.
66 A token is delimited by any of the N_DELIM bytes in DELIM.
67 Upon return, the token is in tokenbuffer->buffer and
68 has a trailing '\0' instead of any original delimiter.
69 The function value is the length of the token not including
70 the final '\0'. Upon EOF (i.e. on the call after the last
71 token is read) or error, return -1 without modifying tokenbuffer.
72 The EOF and error conditions may be distinguished in the caller
73 by testing ferror (STREAM).
74
75 This function works properly on lines containing NUL bytes
76 and on files that do not end with a delimiter. */
77
78 size_t
79 readtoken (FILE *stream,
80 const char *delim,
81 size_t n_delim,
82 token_buffer *tokenbuffer)
83 {
84 int c;
85 idx_t i;
86 word isdelim[(UCHAR_MAX + bits_per_word) / bits_per_word];
87
88 memset (isdelim, 0, sizeof isdelim);
89 for (i = 0; i < n_delim; i++)
90 {
91 unsigned char ch = delim[i];
92 set_nth_bit (ch, isdelim);
93 }
94
95 /* skip over any leading delimiters */
96 for (c = getc (stream); c >= 0 && get_nth_bit (c, isdelim); c = getc (stream))
97 {
98 /* empty */
99 }
100
101 char *p = tokenbuffer->buffer;
102 idx_t n = tokenbuffer->size;
103 i = 0;
104 for (;;)
105 {
106 if (c < 0 && i == 0)
107 return -1;
108
109 if (i == n)
110 p = xpalloc (p, &n, 1, -1, sizeof *p);
111
112 if (c < 0)
113 {
114 p[i] = 0;
115 break;
116 }
117 if (get_nth_bit (c, isdelim))
118 {
119 p[i] = 0;
120 break;
121 }
122 p[i++] = c;
123 c = getc (stream);
124 }
125
126 tokenbuffer->buffer = p;
127 tokenbuffer->size = n;
128 return i;
129 }
130
131 /* Build a NULL-terminated array of pointers to tokens
132 read from STREAM. Return the number of tokens read.
133 All storage is obtained through calls to xmalloc-like functions.
134
135 %%% Question: is it worth it to do a single
136 %%% realloc() of 'tokens' just before returning? */
137
138 size_t
139 readtokens (FILE *stream,
140 size_t projected_n_tokens,
141 const char *delim,
142 size_t n_delim,
143 char ***tokens_out,
144 size_t **token_lengths)
145 {
146 token_buffer tb, *token = &tb;
147 char **tokens;
148 size_t *lengths;
149 idx_t sz, n_tokens;
150
151 if (projected_n_tokens == 0)
152 projected_n_tokens = 64;
153 else
154 projected_n_tokens++; /* add one for trailing NULL pointer */
155
156 sz = projected_n_tokens;
157 tokens = xnmalloc (sz, sizeof *tokens);
158 lengths = xnmalloc (sz, sizeof *lengths);
159
160 n_tokens = 0;
161 init_tokenbuffer (token);
162 for (;;)
163 {
164 char *tmp;
165 size_t token_length = readtoken (stream, delim, n_delim, token);
166 if (n_tokens >= sz)
167 {
168 tokens = xpalloc (tokens, &sz, 1, -1, sizeof *tokens);
169 lengths = xreallocarray (lengths, sz, sizeof *lengths);
170 }
171
172 if (token_length == (size_t) -1)
173 {
174 /* don't increment n_tokens for NULL entry */
175 tokens[n_tokens] = NULL;
176 lengths[n_tokens] = 0;
177 break;
178 }
179 tmp = xnmalloc (token_length + 1, sizeof *tmp);
180 lengths[n_tokens] = token_length;
181 tokens[n_tokens] = memcpy (tmp, token->buffer, token_length + 1);
182 n_tokens++;
183 }
184
185 free (token->buffer);
186 *tokens_out = tokens;
187 if (token_lengths != NULL)
188 *token_lengths = lengths;
189 else
190 free (lengths);
191 return n_tokens;
192 }