1 /****************************************************
2 * PCRE maintainers' helper program: UTF-8 converter *
3 ****************************************************/
4
5 /* This is a test program for converting character code points to UTF-8 and
6 vice versa. Note that this program conforms to the original definition of
7 UTF-8, which allows codepoints up to 7fffffff. The more recent definition
8 limits the validity of Unicode UTF-8 codepoints to a maximum of 10ffffff, and
9 forbids the "surrogate" code points. This program now gives warnings for these
10 invalid code points.
11
12 The arguments are either single code point values written as U+hh.. or 0xhh..
13 for conversion to UTF-8, or sequences of hex values, written without 0x and
14 optionally including spaces (but such arguments must be quoted), for conversion
15 from UTF-8 to codepoints. For example:
16
17 ./utf8 0x1234
18 U+00001234 => e1 88 b4
19
20 ./utf8 "e1 88 b4"
21 U+00001234 <= e1 88 b4
22
23 In the second case, a number of UTF-8 characters can be present in one
24 argument. In other words, each such argument is interpreted (after ignoring
25 spaces) as a string of UTF-8 bytes representing a string of characters:
26
27 ./utf8 "65 e188b4 77"
28 0x00000065 <= 65
29 0x00001234 <= e1 88 b4
30 0x00000077 <= 77
31
32 If the option -s is given, the sequence of UTF-bytes is written out between
33 angle brackets at the end of the line. On a UTF-8 terminal, this will show the
34 appropriate graphic for the code point.
35
36 Errors provoke error messages, but the program carries on with the next
37 argument. The return code is always zero.
38
39 Philip Hazel
40 Original creation data: unknown
41 Code extended and tidied to avoid compiler warnings: 26 March 2020
42 */
43
44
45 #include <stdio.h>
46 #include <stdlib.h>
47 #include <ctype.h>
48 #include <string.h>
49
50 /* The valid ranges for UTF-8 characters are:
51
52 0000 0000 to 0000 007f 1 byte (ascii)
53 0000 0080 to 0000 07ff 2 bytes
54 0000 0800 to 0000 ffff 3 bytes
55 0001 0000 to 001f ffff 4 bytes
56 0020 0000 to 03ff ffff 5 bytes
57 0400 0000 to 7fff ffff 6 bytes
58 */
59
60
61 static const unsigned int utf8_table1[] = {
62 0x0000007f, 0x000007ff, 0x0000ffff, 0x001fffff, 0x03ffffff, 0x7fffffff};
63
64 static const int utf8_table2[] = {
65 0, 0xc0, 0xe0, 0xf0, 0xf8, 0xfc};
66
67 static const int utf8_table3[] = {
68 0xff, 0x1f, 0x0f, 0x07, 0x03, 0x01};
69
70
71 /*************************************************
72 * Convert character value to UTF-8 *
73 *************************************************/
74
75 /* This function takes an unsigned long integer value in the range 0 -
76 0x7fffffff and encodes it as a UTF-8 character in 1 to 6 bytes.
77
78 Arguments:
79 cvalue the character value
80 buffer pointer to buffer for result - at least 6 bytes long
81
82 Returns: number of bytes placed in the buffer
83 0 if input code point is too big
84 */
85
86 static size_t
ord2utf8(unsigned long int cvalue,unsigned char * buffer)87 ord2utf8(unsigned long int cvalue, unsigned char *buffer)
88 {
89 size_t i, j;
90 for (i = 0; i < sizeof(utf8_table1)/sizeof(int); i++)
91 if (cvalue <= utf8_table1[i]) break;
92 if (i >= sizeof(utf8_table1)/sizeof(int)) return 0;
93 buffer += i;
94 for (j = i; j > 0; j--)
95 {
96 *buffer-- = 0x80 | (cvalue & 0x3f);
97 cvalue >>= 6;
98 }
99 *buffer = utf8_table2[i] | cvalue;
100 return i + 1;
101 }
102
103
104
105 /*************************************************
106 * Convert UTF-8 string to value *
107 *************************************************/
108
109 /* This function takes one or more bytes that represent a UTF-8 character from
110 the start of a string of bytes. It returns the value of the character, or the
111 offset of a malformation. For an overlong encoding that works but is not the
112 correct (shortest) one, the error offset is just after the last byte.
113
114 Argument:
115 buffer a pointer to the byte vector
116 buffend a pointer to the end of the buffer
117 vptr a pointer to a variable to receive the value
118 lenptr a pointer to a variable to receive the offset when error detected
119
120 Returns: > 0 => the number of bytes consumed
121 0 => invalid UTF-8: first byte missing 0x40 bit
122 -1 => invalid UTF-8: first byte has too many high-order 1-bits
123 -2 => incomplete sequence at end of string
124 -3 => incomplete sequence within string
125 -4 => overlong code sequence
126 */
127
128 static int
utf82ord(unsigned char * buffer,unsigned char * buffend,long unsigned int * vptr,int * lenptr)129 utf82ord(unsigned char *buffer, unsigned char *buffend,
130 long unsigned int *vptr, int *lenptr)
131 {
132 unsigned int c = *buffer++;
133 unsigned int d = c;
134 int i, j, s;
135
136 /* Check for an ASCII character, or find the number of additional bytes in a
137 multibyte character. */
138
139 for (i = -1; i < 6; i++)
140 {
141 if ((d & 0x80) == 0) break;
142 d <<= 1;
143 }
144
145 switch (i)
146 {
147 case -1: /* ASCII character; first byte does not have 0x80 bit */
148 *vptr = c;
149 return 1;
150
151 case 0: /* First byte has 0x80 but is missing 0x40 bit */
152 *lenptr = 0;
153 return 0;
154
155 case 6:
156 *lenptr = 0; /* Too many high bits */
157 return -1;
158
159 default:
160 break;
161 }
162
163 /* i now has a value in the range 1-5 */
164
165 s = 6*i;
166 d = (c & utf8_table3[i]) << s;
167
168 for (j = 0; j < i; j++)
169 {
170 if (buffer >= buffend)
171 {
172 *lenptr = j + 1;
173 return -2;
174 }
175 c = *buffer++;
176 if ((c & 0xc0) != 0x80)
177 {
178 *lenptr = j + 1;
179 return -3;
180 }
181 s -= 6;
182 d |= (c & 0x3f) << s;
183 }
184
185 /* Valid UTF-8 syntax */
186
187 *vptr = d;
188
189 /* Check that encoding was the correct one, not overlong */
190
191 for (j = 0; j < (int)(sizeof(utf8_table1)/sizeof(int)); j++)
192 if (d <= utf8_table1[j]) break;
193 if (j != i)
194 {
195 *lenptr = i + 1;
196 return -4;
197 }
198
199 /* Valid value */
200
201 return i + 1;
202 }
203
204
205
206 /*************************************************
207 * Main Program *
208 *************************************************/
209
210 int
main(int argc,char ** argv)211 main(int argc, char **argv)
212 {
213 int i = 1;
214 int show = 0;
215 unsigned char buffer[64];
216
217 if (argc > 1 && strcmp(argv[1], "-s") == 0)
218 {
219 show = 1;
220 i = 2;
221 }
222
223 for (; i < argc; i++)
224 {
225 char *x = argv[i];
226 char *endptr;
227 if (strncmp(x, "0x", 2) == 0 || strncmp(x, "U+", 2) == 0)
228 {
229 size_t rc, j;
230 unsigned long int d = strtoul(x+2, &endptr, 16);
231 if (*endptr != 0)
232 {
233 printf("** Invalid hex number %s\n", x);
234 continue; /* With next argument */
235 }
236 rc = ord2utf8(d, buffer);
237 printf("U+%08lx => ", d);
238 if (rc == 0)
239 printf("** Code point greater than 0x7fffffff cannot be encoded");
240 else
241 {
242 for (j = 0; j < rc; j++) printf("%02x ", buffer[j]);
243 if (show)
244 {
245 printf(">");
246 for (j = 0; j < rc; j++) printf("%c", buffer[j]);
247 printf("< ");
248 }
249 if (d >= 0xd800 && d <= 0xdfff)
250 printf("** Invalid Unicode (surrogate)");
251 else if (d > 0x10ffff)
252 printf("** Invalid Unicode (greater than U+10ffff)");
253 }
254 printf("\n");
255 }
256 else
257 {
258 unsigned char *bptr;
259 unsigned char *buffend;
260 int len = 0;
261 int y = 0;
262 int z = 0;
263
264 for (;;)
265 {
266 while (*x == ' ') x++;
267 if (*x == 0 && !z) break;
268 if (!isxdigit(*x))
269 {
270 printf("** Malformed hex string: %s\n", argv[i]);
271 len = -1;
272 break;
273 }
274 y = y * 16 + tolower(*x) - ((isdigit(*x))? '0' : 'W');
275 x++;
276 if (z)
277 {
278 buffer[len++] = y;
279 y = 0;
280 }
281 z ^= 1;
282 }
283
284 if (len < 0) continue; /* With next argument after malformation */
285
286 bptr = buffer;
287 buffend = buffer + len;
288
289 while (bptr < buffend)
290 {
291 unsigned long int d;
292 int j;
293 int offset;
294 int rc = utf82ord(bptr, buffend, &d, &offset);
295
296 if (rc > 0)
297 {
298 printf("U+%08lx <= ", d);
299 for (j = 0; j < rc; j++) printf("%02x ", bptr[j]);
300 if (show)
301 {
302 printf(">");
303 for (j = 0; j < rc; j++) printf("%c", bptr[j]);
304 printf("<");
305 }
306 printf("\n");
307 bptr += rc;
308 }
309 else if (rc == -4)
310 {
311 printf("U+%08lx <= ", d);
312 for (j = 0; j < offset; j++) printf("%02x ", bptr[j]);
313 printf("** Overlong UTF-8 sequence\n");
314 bptr += offset;
315 }
316 else
317 {
318 switch (rc)
319 {
320 case 0: printf("** First byte missing 0x40 bit");
321 break;
322
323 case -1: printf("** First byte has too many high-order bits");
324 break;
325
326 case -2: printf("** Incomplete UTF-8 sequence at end of string");
327 break;
328
329 case -3: printf("** Incomplete UTF-8 sequence");
330 break;
331
332 default: printf("** Unexpected return %d from utf82ord()", rc);
333 break;
334 }
335 printf(" at offset %d in string ", offset);
336 while (bptr < buffend) printf("%02x ", *bptr++);
337 printf("\n");
338 break;
339 }
340 }
341 }
342 }
343
344 return 0;
345 }
346
347 /* End */
348