1 /*
2 * Copyright (C) 2018 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17 #ifndef LIBTEXTCLASSIFIER_UTILS_STRINGS_UTF8_H_
18 #define LIBTEXTCLASSIFIER_UTILS_STRINGS_UTF8_H_
19
20 namespace libtextclassifier3 {
21
22 // Returns the length (number of bytes) of the Unicode code point starting at
23 // src, based on inspecting just that one byte. Preconditions: src != NULL,
24 // *src can be read, and *src is not '\0', and src points to a well-formed UTF-8
25 // std::string.
GetNumBytesForNonZeroUTF8Char(const char * src)26 static inline int GetNumBytesForNonZeroUTF8Char(const char *src) {
27 // On most platforms, char is unsigned by default, but iOS is an exception.
28 // The cast below makes sure we always interpret *src as an unsigned char.
29 return "\1\1\1\1\1\1\1\1\1\1\1\1\2\2\3\4"
30 [(*(reinterpret_cast<const unsigned char *>(src)) & 0xFF) >> 4];
31 }
32
33 // Like GetNumBytesForNonZeroUTF8Char, but *src may be '\0'; returns 0 in that
34 // case.
GetNumBytesForUTF8Char(const char * src)35 static inline int GetNumBytesForUTF8Char(const char *src) {
36 if (*src == '\0') return 0;
37 return GetNumBytesForNonZeroUTF8Char(src);
38 }
39
40 // Returns true if this byte is a trailing UTF-8 byte (10xx xxxx)
IsTrailByte(char x)41 static inline bool IsTrailByte(char x) {
42 // return (x & 0xC0) == 0x80;
43 // Since trail bytes are always in [0x80, 0xBF], we can optimize:
44 return static_cast<signed char>(x) < -0x40;
45 }
46
47 // Returns true iff src points to a well-formed UTF-8 string.
48 bool IsValidUTF8(const char *src, int size);
49
50 // Returns byte length of the first valid codepoint in the string, otherwise -1
51 // if pointing to an ill-formed UTF-8 character.
52 int ValidUTF8CharLength(const char *src, int size);
53
54 } // namespace libtextclassifier3
55
56 #endif // LIBTEXTCLASSIFIER_UTILS_STRINGS_UTF8_H_
57