1 #include <stdarg.h>
2 #include <stdio.h>
3 #include <stdlib.h>
4 #include <string.h>
5 #include <glib.h>
6
7 typedef enum
8 {
9 VALID,
10 INCOMPLETE,
11 NOTUNICODE,
12 OVERLONG,
13 MALFORMED
14 } Status;
15
16 static gboolean
17 ucs4_equal (gunichar *a, gunichar *b)
18 {
19 while (*a && *b && (*a == *b))
20 {
21 a++;
22 b++;
23 }
24
25 return (*a == *b);
26 }
27
28 static gboolean
29 utf16_equal (gunichar2 *a, gunichar2 *b)
30 {
31 while (*a && *b && (*a == *b))
32 {
33 a++;
34 b++;
35 }
36
37 return (*a == *b);
38 }
39
40 static gint
41 utf16_count (gunichar2 *a)
42 {
43 gint result = 0;
44
45 while (a[result])
46 result++;
47
48 return result;
49 }
50
51 static void
52 process (gint line,
53 gchar *utf8,
54 Status status,
55 gunichar *ucs4,
56 gint ucs4_len)
57 {
58 const gchar *end;
59 gboolean is_valid = g_utf8_validate (utf8, -1, &end);
60 GError *error = NULL;
61 glong items_read, items_written;
62
63 switch (status)
64 {
65 case VALID:
66 g_assert_true (is_valid);
67 break;
68
69 case NOTUNICODE:
70 case INCOMPLETE:
71 case OVERLONG:
72 case MALFORMED:
73 g_assert_false (is_valid);
74 break;
75 }
76
77 if (status == INCOMPLETE)
78 {
79 gunichar *ucs4_result;
80
81 ucs4_result = g_utf8_to_ucs4 (utf8, -1, NULL, NULL, &error);
82
83 g_assert_error (error, G_CONVERT_ERROR, G_CONVERT_ERROR_PARTIAL_INPUT);
84
85 g_clear_error (&error);
86
87 ucs4_result = g_utf8_to_ucs4 (utf8, -1, &items_read, NULL, &error);
88 g_assert_no_error (error);
89 g_assert_nonnull (ucs4_result);
90 g_assert_cmpint (items_read, !=, strlen (utf8));
91
92 g_free (ucs4_result);
93 }
94
95 if (status == VALID || status == NOTUNICODE)
96 {
97 gunichar *ucs4_result;
98
99 ucs4_result = g_utf8_to_ucs4 (utf8, -1, &items_read, &items_written, &error);
100 g_assert_no_error (error);
101 g_assert_nonnull (ucs4_result);
102
103 g_assert_true (ucs4_equal (ucs4_result, ucs4));
104 g_assert_cmpint (items_read, ==, strlen (utf8));
105 g_assert_cmpint (items_written, ==, ucs4_len);
106
107 g_free (ucs4_result);
108 }
109
110 if (status == VALID)
111 {
112 gunichar *ucs4_result;
113 gchar *utf8_result;
114
115 ucs4_result = g_utf8_to_ucs4_fast (utf8, -1, &items_written);
116 g_assert_nonnull (ucs4_result);
117
118 g_assert_true (ucs4_equal (ucs4_result, ucs4));
119 g_assert_cmpint (items_written, ==, ucs4_len);
120
121 utf8_result = g_ucs4_to_utf8 (ucs4_result, -1, &items_read, &items_written, &error);
122 g_assert_no_error (error);
123 g_assert_nonnull (utf8_result);
124
125 g_assert_cmpstr ((char *) utf8_result, ==, (char *) utf8);
126 g_assert_cmpint (items_read, ==, ucs4_len);
127 g_assert_cmpint (items_written, ==, strlen (utf8));
128
129 g_free (utf8_result);
130 g_free (ucs4_result);
131 }
132
133 if (status == VALID)
134 {
135 gunichar2 *utf16_expected_tmp;
136 gunichar2 *utf16_expected;
137 gunichar2 *utf16_from_utf8;
138 gunichar2 *utf16_from_ucs4;
139 gunichar *ucs4_result;
140 gsize bytes_written;
141 gint n_chars;
142 gchar *utf8_result;
143
144 #if G_BYTE_ORDER == G_LITTLE_ENDIAN
145 #define TARGET "UTF-16LE"
146 #else
147 #define TARGET "UTF-16"
148 #endif
149
150 utf16_expected_tmp =
151 (gunichar2 *) g_convert (utf8, -1, TARGET, "UTF-8", NULL, &bytes_written, NULL);
152 g_assert_nonnull (utf16_expected_tmp);
153
154 /* zero-terminate and remove BOM */
155 n_chars = bytes_written / 2;
156 if (utf16_expected_tmp[0] == 0xfeff) /* BOM */
157 {
158 n_chars--;
159 utf16_expected = g_new (gunichar2, n_chars + 1);
160 memcpy (utf16_expected, utf16_expected_tmp + 1, sizeof(gunichar2) * n_chars);
161 }
162 else
163 {
164 /* We expect the result of the conversion
165 via iconv() to be native-endian. */
166 g_assert_false (utf16_expected_tmp[0] == 0xfffe);
167 utf16_expected = g_new (gunichar2, n_chars + 1);
168 memcpy (utf16_expected, utf16_expected_tmp, sizeof(gunichar2) * n_chars);
169 }
170
171 utf16_expected[n_chars] = '\0';
172
173 utf16_from_utf8 =
174 g_utf8_to_utf16 (utf8, -1, &items_read, &items_written, &error);
175 g_assert_no_error (error);
176 g_assert_nonnull (utf16_from_utf8);
177
178 g_assert_cmpint (items_read, ==, (glong) strlen (utf8));
179 g_assert_cmpint (utf16_count (utf16_from_utf8), ==, items_written);
180
181 utf16_from_ucs4 =
182 g_ucs4_to_utf16 (ucs4, -1, &items_read, &items_written, &error);
183 g_assert_no_error (error);
184 g_assert_nonnull (utf16_from_ucs4);
185
186 g_assert_cmpint (items_read, ==, ucs4_len);
187 g_assert_cmpint (utf16_count (utf16_from_ucs4), ==, items_written);
188
189 g_assert_true (utf16_equal (utf16_from_utf8, utf16_expected));
190 g_assert_true (utf16_equal (utf16_from_ucs4, utf16_expected));
191 g_assert_cmpstr ((char *) utf16_from_utf8, ==, (char *) utf16_expected);
192 g_assert_cmpstr ((char *) utf16_from_ucs4, ==, (char *) utf16_expected);
193
194 utf8_result =
195 g_utf16_to_utf8 (utf16_from_utf8, -1, &items_read, &items_written, &error);
196 g_assert_no_error (error);
197 g_assert_nonnull (utf8_result);
198
199 g_assert_cmpint (items_read, ==, utf16_count (utf16_from_utf8));
200 g_assert_cmpint (items_written, ==, (glong) strlen (utf8));
201
202 ucs4_result =
203 g_utf16_to_ucs4 (utf16_from_ucs4, -1, &items_read, &items_written, &error);
204 g_assert_no_error (error);
205 g_assert_nonnull (ucs4_result);
206
207 g_assert_cmpint (items_read, ==, utf16_count (utf16_from_utf8));
208 g_assert_cmpint (items_written, ==, ucs4_len);
209
210 g_assert_cmpstr (utf8, ==, utf8_result);
211 g_assert_cmpstr ((char *) ucs4, ==, (char *) ucs4_result);
212
213 g_free (utf16_expected_tmp);
214 g_free (utf16_expected);
215 g_free (utf16_from_utf8);
216 g_free (utf16_from_ucs4);
217 g_free (utf8_result);
218 g_free (ucs4_result);
219 }
220 }
221
222 static void
223 test_unicode_encoding (void)
224 {
225 gchar *testfile;
226 gchar *contents;
227 GError *error = NULL;
228 gchar *p, *end;
229 char *tmp;
230 gint state = 0;
231 gint line = 1;
232 gint start_line = 0; /* Quiet GCC */
233 gchar *utf8 = NULL; /* Quiet GCC */
234 GArray *ucs4;
235 Status status = VALID; /* Quiet GCC */
236
237 testfile = g_test_build_filename (G_TEST_DIST, "utf8.txt", NULL);
238
239 g_file_get_contents (testfile, &contents, NULL, &error);
240 g_assert_no_error (error);
241
242 ucs4 = g_array_new (TRUE, FALSE, sizeof(gunichar));
243
244 p = contents;
245
246 /* Loop over lines */
247 while (*p)
248 {
249 while (*p && (*p == ' ' || *p == '\t'))
250 p++;
251
252 end = p;
253 while (*end && (*end != '\r' && *end != '\n'))
254 end++;
255
256 if (!*p || *p == '#' || *p == '\r' || *p == '\n')
257 goto next_line;
258
259 tmp = g_strstrip (g_strndup (p, end - p));
260
261 switch (state)
262 {
263 case 0:
264 /* UTF-8 string */
265 start_line = line;
266 utf8 = tmp;
267 tmp = NULL;
268 break;
269
270 case 1:
271 /* Status */
272 if (!strcmp (tmp, "VALID"))
273 status = VALID;
274 else if (!strcmp (tmp, "INCOMPLETE"))
275 status = INCOMPLETE;
276 else if (!strcmp (tmp, "NOTUNICODE"))
277 status = NOTUNICODE;
278 else if (!strcmp (tmp, "OVERLONG"))
279 status = OVERLONG;
280 else if (!strcmp (tmp, "MALFORMED"))
281 status = MALFORMED;
282 else
283 g_assert_not_reached ();
284
285 if (status != VALID && status != NOTUNICODE)
286 state++; /* No UCS-4 data */
287 break;
288
289 case 2:
290 /* UCS-4 version */
291 p = strtok (tmp, " \t");
292 while (p)
293 {
294 gchar *endptr;
295 gunichar ch = strtoul (p, &endptr, 16);
296 g_assert_cmpint (*endptr, == ,'\0');
297
298 g_array_append_val (ucs4, ch);
299
300 p = strtok (NULL, " \t");
301 }
302 break;
303 }
304
305 g_free (tmp);
306 state = (state + 1) % 3;
307
308 if (state == 0)
309 {
310 process (start_line, utf8, status, (gunichar *)ucs4->data, ucs4->len);
311 g_array_set_size (ucs4, 0);
312 g_free (utf8);
313 }
314
315 next_line:
316 p = end;
317 if (*p && *p == '\r')
318 p++;
319 if (*p && *p == '\n')
320 p++;
321 line++;
322 }
323
324 g_free (testfile);
325 g_array_free (ucs4, TRUE);
326 g_free (contents);
327 }
328
329 int
330 main (int argc, char **argv)
331 {
332 g_test_init (&argc, &argv, NULL);
333
334 g_test_add_func ("/unicode/encoding", test_unicode_encoding);
335
336 return g_test_run ();
337 }