• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 /*
2  * Copyright (C) 2005 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 #include <log/log.h>
18 #include <utils/Unicode.h>
19 
20 #include <stddef.h>
21 
22 #ifdef HAVE_WINSOCK
23 # undef  nhtol
24 # undef  htonl
25 # undef  nhtos
26 # undef  htons
27 
28 # define ntohl(x)    ( ((x) << 24) | (((x) >> 24) & 255) | (((x) << 8) & 0xff0000) | (((x) >> 8) & 0xff00) )
29 # define htonl(x)    ntohl(x)
30 # define ntohs(x)    ( (((x) << 8) & 0xff00) | (((x) >> 8) & 255) )
31 # define htons(x)    ntohs(x)
32 #else
33 # include <netinet/in.h>
34 #endif
35 
36 extern "C" {
37 
38 static const char32_t kByteMask = 0x000000BF;
39 static const char32_t kByteMark = 0x00000080;
40 
41 // Surrogates aren't valid for UTF-32 characters, so define some
42 // constants that will let us screen them out.
43 static const char32_t kUnicodeSurrogateHighStart  = 0x0000D800;
44 // Unused, here for completeness:
45 // static const char32_t kUnicodeSurrogateHighEnd = 0x0000DBFF;
46 // static const char32_t kUnicodeSurrogateLowStart = 0x0000DC00;
47 static const char32_t kUnicodeSurrogateLowEnd     = 0x0000DFFF;
48 static const char32_t kUnicodeSurrogateStart      = kUnicodeSurrogateHighStart;
49 static const char32_t kUnicodeSurrogateEnd        = kUnicodeSurrogateLowEnd;
50 static const char32_t kUnicodeMaxCodepoint        = 0x0010FFFF;
51 
52 // Mask used to set appropriate bits in first byte of UTF-8 sequence,
53 // indexed by number of bytes in the sequence.
54 // 0xxxxxxx
55 // -> (00-7f) 7bit. Bit mask for the first byte is 0x00000000
56 // 110yyyyx 10xxxxxx
57 // -> (c0-df)(80-bf) 11bit. Bit mask is 0x000000C0
58 // 1110yyyy 10yxxxxx 10xxxxxx
59 // -> (e0-ef)(80-bf)(80-bf) 16bit. Bit mask is 0x000000E0
60 // 11110yyy 10yyxxxx 10xxxxxx 10xxxxxx
61 // -> (f0-f7)(80-bf)(80-bf)(80-bf) 21bit. Bit mask is 0x000000F0
62 static const char32_t kFirstByteMark[] = {
63     0x00000000, 0x00000000, 0x000000C0, 0x000000E0, 0x000000F0
64 };
65 
66 // --------------------------------------------------------------------------
67 // UTF-32
68 // --------------------------------------------------------------------------
69 
70 /**
71  * Return number of UTF-8 bytes required for the character. If the character
72  * is invalid, return size of 0.
73  */
utf32_codepoint_utf8_length(char32_t srcChar)74 static inline size_t utf32_codepoint_utf8_length(char32_t srcChar)
75 {
76     // Figure out how many bytes the result will require.
77     if (srcChar < 0x00000080) {
78         return 1;
79     } else if (srcChar < 0x00000800) {
80         return 2;
81     } else if (srcChar < 0x00010000) {
82         if ((srcChar < kUnicodeSurrogateStart) || (srcChar > kUnicodeSurrogateEnd)) {
83             return 3;
84         } else {
85             // Surrogates are invalid UTF-32 characters.
86             return 0;
87         }
88     }
89     // Max code point for Unicode is 0x0010FFFF.
90     else if (srcChar <= kUnicodeMaxCodepoint) {
91         return 4;
92     } else {
93         // Invalid UTF-32 character.
94         return 0;
95     }
96 }
97 
98 // Write out the source character to <dstP>.
99 
utf32_codepoint_to_utf8(uint8_t * dstP,char32_t srcChar,size_t bytes)100 static inline void utf32_codepoint_to_utf8(uint8_t* dstP, char32_t srcChar, size_t bytes)
101 {
102     dstP += bytes;
103     switch (bytes)
104     {   /* note: everything falls through. */
105         case 4: *--dstP = (uint8_t)((srcChar | kByteMark) & kByteMask); srcChar >>= 6;
106         case 3: *--dstP = (uint8_t)((srcChar | kByteMark) & kByteMask); srcChar >>= 6;
107         case 2: *--dstP = (uint8_t)((srcChar | kByteMark) & kByteMask); srcChar >>= 6;
108         case 1: *--dstP = (uint8_t)(srcChar | kFirstByteMark[bytes]);
109     }
110 }
111 
strlen32(const char32_t * s)112 size_t strlen32(const char32_t *s)
113 {
114   const char32_t *ss = s;
115   while ( *ss )
116     ss++;
117   return ss-s;
118 }
119 
strnlen32(const char32_t * s,size_t maxlen)120 size_t strnlen32(const char32_t *s, size_t maxlen)
121 {
122   const char32_t *ss = s;
123   while ((maxlen > 0) && *ss) {
124     ss++;
125     maxlen--;
126   }
127   return ss-s;
128 }
129 
utf32_at_internal(const char * cur,size_t * num_read)130 static inline int32_t utf32_at_internal(const char* cur, size_t *num_read)
131 {
132     const char first_char = *cur;
133     if ((first_char & 0x80) == 0) { // ASCII
134         *num_read = 1;
135         return *cur;
136     }
137     cur++;
138     char32_t mask, to_ignore_mask;
139     size_t num_to_read = 0;
140     char32_t utf32 = first_char;
141     for (num_to_read = 1, mask = 0x40, to_ignore_mask = 0xFFFFFF80;
142          (first_char & mask);
143          num_to_read++, to_ignore_mask |= mask, mask >>= 1) {
144         // 0x3F == 00111111
145         utf32 = (utf32 << 6) + (*cur++ & 0x3F);
146     }
147     to_ignore_mask |= mask;
148     utf32 &= ~(to_ignore_mask << (6 * (num_to_read - 1)));
149 
150     *num_read = num_to_read;
151     return static_cast<int32_t>(utf32);
152 }
153 
utf32_from_utf8_at(const char * src,size_t src_len,size_t index,size_t * next_index)154 int32_t utf32_from_utf8_at(const char *src, size_t src_len, size_t index, size_t *next_index)
155 {
156     if (index >= src_len) {
157         return -1;
158     }
159     size_t dummy_index;
160     if (next_index == NULL) {
161         next_index = &dummy_index;
162     }
163     size_t num_read;
164     int32_t ret = utf32_at_internal(src + index, &num_read);
165     if (ret >= 0) {
166         *next_index = index + num_read;
167     }
168 
169     return ret;
170 }
171 
utf32_to_utf8_length(const char32_t * src,size_t src_len)172 ssize_t utf32_to_utf8_length(const char32_t *src, size_t src_len)
173 {
174     if (src == NULL || src_len == 0) {
175         return -1;
176     }
177 
178     size_t ret = 0;
179     const char32_t *end = src + src_len;
180     while (src < end) {
181         ret += utf32_codepoint_utf8_length(*src++);
182     }
183     return ret;
184 }
185 
utf32_to_utf8(const char32_t * src,size_t src_len,char * dst,size_t dst_len)186 void utf32_to_utf8(const char32_t* src, size_t src_len, char* dst, size_t dst_len)
187 {
188     if (src == NULL || src_len == 0 || dst == NULL) {
189         return;
190     }
191 
192     const char32_t *cur_utf32 = src;
193     const char32_t *end_utf32 = src + src_len;
194     char *cur = dst;
195     while (cur_utf32 < end_utf32) {
196         size_t len = utf32_codepoint_utf8_length(*cur_utf32);
197         LOG_ALWAYS_FATAL_IF(dst_len < len, "%zu < %zu", dst_len, len);
198         utf32_codepoint_to_utf8((uint8_t *)cur, *cur_utf32++, len);
199         cur += len;
200         dst_len -= len;
201     }
202     LOG_ALWAYS_FATAL_IF(dst_len < 1, "dst_len < 1: %zu < 1", dst_len);
203     *cur = '\0';
204 }
205 
206 // --------------------------------------------------------------------------
207 // UTF-16
208 // --------------------------------------------------------------------------
209 
strcmp16(const char16_t * s1,const char16_t * s2)210 int strcmp16(const char16_t *s1, const char16_t *s2)
211 {
212   char16_t ch;
213   int d = 0;
214 
215   while ( 1 ) {
216     d = (int)(ch = *s1++) - (int)*s2++;
217     if ( d || !ch )
218       break;
219   }
220 
221   return d;
222 }
223 
strncmp16(const char16_t * s1,const char16_t * s2,size_t n)224 int strncmp16(const char16_t *s1, const char16_t *s2, size_t n)
225 {
226   char16_t ch;
227   int d = 0;
228 
229   while ( n-- ) {
230     d = (int)(ch = *s1++) - (int)*s2++;
231     if ( d || !ch )
232       break;
233   }
234 
235   return d;
236 }
237 
strcpy16(char16_t * dst,const char16_t * src)238 char16_t *strcpy16(char16_t *dst, const char16_t *src)
239 {
240   char16_t *q = dst;
241   const char16_t *p = src;
242   char16_t ch;
243 
244   do {
245     *q++ = ch = *p++;
246   } while ( ch );
247 
248   return dst;
249 }
250 
strlen16(const char16_t * s)251 size_t strlen16(const char16_t *s)
252 {
253   const char16_t *ss = s;
254   while ( *ss )
255     ss++;
256   return ss-s;
257 }
258 
259 
strncpy16(char16_t * dst,const char16_t * src,size_t n)260 char16_t *strncpy16(char16_t *dst, const char16_t *src, size_t n)
261 {
262   char16_t *q = dst;
263   const char16_t *p = src;
264   char ch;
265 
266   while (n) {
267     n--;
268     *q++ = ch = *p++;
269     if ( !ch )
270       break;
271   }
272 
273   *q = 0;
274 
275   return dst;
276 }
277 
strnlen16(const char16_t * s,size_t maxlen)278 size_t strnlen16(const char16_t *s, size_t maxlen)
279 {
280   const char16_t *ss = s;
281 
282   /* Important: the maxlen test must precede the reference through ss;
283      since the byte beyond the maximum may segfault */
284   while ((maxlen > 0) && *ss) {
285     ss++;
286     maxlen--;
287   }
288   return ss-s;
289 }
290 
strzcmp16(const char16_t * s1,size_t n1,const char16_t * s2,size_t n2)291 int strzcmp16(const char16_t *s1, size_t n1, const char16_t *s2, size_t n2)
292 {
293     const char16_t* e1 = s1+n1;
294     const char16_t* e2 = s2+n2;
295 
296     while (s1 < e1 && s2 < e2) {
297         const int d = (int)*s1++ - (int)*s2++;
298         if (d) {
299             return d;
300         }
301     }
302 
303     return n1 < n2
304         ? (0 - (int)*s2)
305         : (n1 > n2
306            ? ((int)*s1 - 0)
307            : 0);
308 }
309 
strzcmp16_h_n(const char16_t * s1H,size_t n1,const char16_t * s2N,size_t n2)310 int strzcmp16_h_n(const char16_t *s1H, size_t n1, const char16_t *s2N, size_t n2)
311 {
312     const char16_t* e1 = s1H+n1;
313     const char16_t* e2 = s2N+n2;
314 
315     while (s1H < e1 && s2N < e2) {
316         const char16_t c2 = ntohs(*s2N);
317         const int d = (int)*s1H++ - (int)c2;
318         s2N++;
319         if (d) {
320             return d;
321         }
322     }
323 
324     return n1 < n2
325         ? (0 - (int)ntohs(*s2N))
326         : (n1 > n2
327            ? ((int)*s1H - 0)
328            : 0);
329 }
330 
utf16_to_utf8(const char16_t * src,size_t src_len,char * dst,size_t dst_len)331 void utf16_to_utf8(const char16_t* src, size_t src_len, char* dst, size_t dst_len)
332 {
333     if (src == NULL || src_len == 0 || dst == NULL) {
334         return;
335     }
336 
337     const char16_t* cur_utf16 = src;
338     const char16_t* const end_utf16 = src + src_len;
339     char *cur = dst;
340     while (cur_utf16 < end_utf16) {
341         char32_t utf32;
342         // surrogate pairs
343         if((*cur_utf16 & 0xFC00) == 0xD800 && (cur_utf16 + 1) < end_utf16
344                 && (*(cur_utf16 + 1) & 0xFC00) == 0xDC00) {
345             utf32 = (*cur_utf16++ - 0xD800) << 10;
346             utf32 |= *cur_utf16++ - 0xDC00;
347             utf32 += 0x10000;
348         } else {
349             utf32 = (char32_t) *cur_utf16++;
350         }
351         const size_t len = utf32_codepoint_utf8_length(utf32);
352         LOG_ALWAYS_FATAL_IF(dst_len < len, "%zu < %zu", dst_len, len);
353         utf32_codepoint_to_utf8((uint8_t*)cur, utf32, len);
354         cur += len;
355         dst_len -= len;
356     }
357     LOG_ALWAYS_FATAL_IF(dst_len < 1, "%zu < 1", dst_len);
358     *cur = '\0';
359 }
360 
361 // --------------------------------------------------------------------------
362 // UTF-8
363 // --------------------------------------------------------------------------
364 
utf8_length(const char * src)365 ssize_t utf8_length(const char *src)
366 {
367     const char *cur = src;
368     size_t ret = 0;
369     while (*cur != '\0') {
370         const char first_char = *cur++;
371         if ((first_char & 0x80) == 0) { // ASCII
372             ret += 1;
373             continue;
374         }
375         // (UTF-8's character must not be like 10xxxxxx,
376         //  but 110xxxxx, 1110xxxx, ... or 1111110x)
377         if ((first_char & 0x40) == 0) {
378             return -1;
379         }
380 
381         int32_t mask, to_ignore_mask;
382         size_t num_to_read = 0;
383         char32_t utf32 = 0;
384         for (num_to_read = 1, mask = 0x40, to_ignore_mask = 0x80;
385              num_to_read < 5 && (first_char & mask);
386              num_to_read++, to_ignore_mask |= mask, mask >>= 1) {
387             if ((*cur & 0xC0) != 0x80) { // must be 10xxxxxx
388                 return -1;
389             }
390             // 0x3F == 00111111
391             utf32 = (utf32 << 6) + (*cur++ & 0x3F);
392         }
393         // "first_char" must be (110xxxxx - 11110xxx)
394         if (num_to_read == 5) {
395             return -1;
396         }
397         to_ignore_mask |= mask;
398         utf32 |= ((~to_ignore_mask) & first_char) << (6 * (num_to_read - 1));
399         if (utf32 > kUnicodeMaxCodepoint) {
400             return -1;
401         }
402 
403         ret += num_to_read;
404     }
405     return ret;
406 }
407 
utf16_to_utf8_length(const char16_t * src,size_t src_len)408 ssize_t utf16_to_utf8_length(const char16_t *src, size_t src_len)
409 {
410     if (src == NULL || src_len == 0) {
411         return -1;
412     }
413 
414     size_t ret = 0;
415     const char16_t* const end = src + src_len;
416     while (src < end) {
417         if ((*src & 0xFC00) == 0xD800 && (src + 1) < end
418                 && (*(src + 1) & 0xFC00) == 0xDC00) {
419             // surrogate pairs are always 4 bytes.
420             ret += 4;
421             src += 2;
422         } else {
423             ret += utf32_codepoint_utf8_length((char32_t) *src++);
424         }
425     }
426     return ret;
427 }
428 
429 /**
430  * Returns 1-4 based on the number of leading bits.
431  *
432  * 1111 -> 4
433  * 1110 -> 3
434  * 110x -> 2
435  * 10xx -> 1
436  * 0xxx -> 1
437  */
utf8_codepoint_len(uint8_t ch)438 static inline size_t utf8_codepoint_len(uint8_t ch)
439 {
440     return ((0xe5000000 >> ((ch >> 3) & 0x1e)) & 3) + 1;
441 }
442 
utf8_shift_and_mask(uint32_t * codePoint,const uint8_t byte)443 static inline void utf8_shift_and_mask(uint32_t* codePoint, const uint8_t byte)
444 {
445     *codePoint <<= 6;
446     *codePoint |= 0x3F & byte;
447 }
448 
utf8_to_utf32_length(const char * src,size_t src_len)449 size_t utf8_to_utf32_length(const char *src, size_t src_len)
450 {
451     if (src == NULL || src_len == 0) {
452         return 0;
453     }
454     size_t ret = 0;
455     const char* cur;
456     const char* end;
457     size_t num_to_skip;
458     for (cur = src, end = src + src_len, num_to_skip = 1;
459          cur < end;
460          cur += num_to_skip, ret++) {
461         const char first_char = *cur;
462         num_to_skip = 1;
463         if ((first_char & 0x80) == 0) {  // ASCII
464             continue;
465         }
466         int32_t mask;
467 
468         for (mask = 0x40; (first_char & mask); num_to_skip++, mask >>= 1) {
469         }
470     }
471     return ret;
472 }
473 
utf8_to_utf32(const char * src,size_t src_len,char32_t * dst)474 void utf8_to_utf32(const char* src, size_t src_len, char32_t* dst)
475 {
476     if (src == NULL || src_len == 0 || dst == NULL) {
477         return;
478     }
479 
480     const char* cur = src;
481     const char* const end = src + src_len;
482     char32_t* cur_utf32 = dst;
483     while (cur < end) {
484         size_t num_read;
485         *cur_utf32++ = static_cast<char32_t>(utf32_at_internal(cur, &num_read));
486         cur += num_read;
487     }
488     *cur_utf32 = 0;
489 }
490 
utf8_to_utf32_codepoint(const uint8_t * src,size_t length)491 static inline uint32_t utf8_to_utf32_codepoint(const uint8_t *src, size_t length)
492 {
493     uint32_t unicode;
494 
495     switch (length)
496     {
497         case 1:
498             return src[0];
499         case 2:
500             unicode = src[0] & 0x1f;
501             utf8_shift_and_mask(&unicode, src[1]);
502             return unicode;
503         case 3:
504             unicode = src[0] & 0x0f;
505             utf8_shift_and_mask(&unicode, src[1]);
506             utf8_shift_and_mask(&unicode, src[2]);
507             return unicode;
508         case 4:
509             unicode = src[0] & 0x07;
510             utf8_shift_and_mask(&unicode, src[1]);
511             utf8_shift_and_mask(&unicode, src[2]);
512             utf8_shift_and_mask(&unicode, src[3]);
513             return unicode;
514         default:
515             return 0xffff;
516     }
517 
518     //printf("Char at %p: len=%d, utf-16=%p\n", src, length, (void*)result);
519 }
520 
utf8_to_utf16_length(const uint8_t * u8str,size_t u8len)521 ssize_t utf8_to_utf16_length(const uint8_t* u8str, size_t u8len)
522 {
523     const uint8_t* const u8end = u8str + u8len;
524     const uint8_t* u8cur = u8str;
525 
526     /* Validate that the UTF-8 is the correct len */
527     size_t u16measuredLen = 0;
528     while (u8cur < u8end) {
529         u16measuredLen++;
530         int u8charLen = utf8_codepoint_len(*u8cur);
531         uint32_t codepoint = utf8_to_utf32_codepoint(u8cur, u8charLen);
532         if (codepoint > 0xFFFF) u16measuredLen++; // this will be a surrogate pair in utf16
533         u8cur += u8charLen;
534     }
535 
536     /**
537      * Make sure that we ended where we thought we would and the output UTF-16
538      * will be exactly how long we were told it would be.
539      */
540     if (u8cur != u8end) {
541         return -1;
542     }
543 
544     return u16measuredLen;
545 }
546 
utf8_to_utf16_no_null_terminator(const uint8_t * u8str,size_t u8len,char16_t * u16str)547 char16_t* utf8_to_utf16_no_null_terminator(const uint8_t* u8str, size_t u8len, char16_t* u16str)
548 {
549     const uint8_t* const u8end = u8str + u8len;
550     const uint8_t* u8cur = u8str;
551     char16_t* u16cur = u16str;
552 
553     while (u8cur < u8end) {
554         size_t u8len = utf8_codepoint_len(*u8cur);
555         uint32_t codepoint = utf8_to_utf32_codepoint(u8cur, u8len);
556 
557         // Convert the UTF32 codepoint to one or more UTF16 codepoints
558         if (codepoint <= 0xFFFF) {
559             // Single UTF16 character
560             *u16cur++ = (char16_t) codepoint;
561         } else {
562             // Multiple UTF16 characters with surrogates
563             codepoint = codepoint - 0x10000;
564             *u16cur++ = (char16_t) ((codepoint >> 10) + 0xD800);
565             *u16cur++ = (char16_t) ((codepoint & 0x3FF) + 0xDC00);
566         }
567 
568         u8cur += u8len;
569     }
570     return u16cur;
571 }
572 
utf8_to_utf16(const uint8_t * u8str,size_t u8len,char16_t * u16str)573 void utf8_to_utf16(const uint8_t* u8str, size_t u8len, char16_t* u16str) {
574     char16_t* end = utf8_to_utf16_no_null_terminator(u8str, u8len, u16str);
575     *end = 0;
576 }
577 
utf8_to_utf16_n(const uint8_t * src,size_t srcLen,char16_t * dst,size_t dstLen)578 char16_t* utf8_to_utf16_n(const uint8_t* src, size_t srcLen, char16_t* dst, size_t dstLen) {
579     const uint8_t* const u8end = src + srcLen;
580     const uint8_t* u8cur = src;
581     const char16_t* const u16end = dst + dstLen;
582     char16_t* u16cur = dst;
583 
584     while (u8cur < u8end && u16cur < u16end) {
585         size_t u8len = utf8_codepoint_len(*u8cur);
586         uint32_t codepoint = utf8_to_utf32_codepoint(u8cur, u8len);
587 
588         // Convert the UTF32 codepoint to one or more UTF16 codepoints
589         if (codepoint <= 0xFFFF) {
590             // Single UTF16 character
591             *u16cur++ = (char16_t) codepoint;
592         } else {
593             // Multiple UTF16 characters with surrogates
594             codepoint = codepoint - 0x10000;
595             *u16cur++ = (char16_t) ((codepoint >> 10) + 0xD800);
596             if (u16cur >= u16end) {
597                 // Ooops...  not enough room for this surrogate pair.
598                 return u16cur-1;
599             }
600             *u16cur++ = (char16_t) ((codepoint & 0x3FF) + 0xDC00);
601         }
602 
603         u8cur += u8len;
604     }
605     return u16cur;
606 }
607 
608 }
609