1 /*
2 * Copyright (C) 2011 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17 #ifndef ART_LIBDEXFILE_DEX_UTF_H_
18 #define ART_LIBDEXFILE_DEX_UTF_H_
19
20 #include <stddef.h>
21 #include <stdint.h>
22
23 #include <string>
24 #include <string_view>
25 #include <type_traits>
26
27 #include "base/macros.h"
28
29 /*
30 * All UTF-8 in art is actually modified UTF-8. Mostly, this distinction
31 * doesn't matter.
32 *
33 * See http://en.wikipedia.org/wiki/UTF-8#Modified_UTF-8 for the details.
34 */
35 namespace art {
36
37 /*
38 * Returns the number of UTF-16 characters in the given modified UTF-8 string.
39 */
40 size_t CountModifiedUtf8Chars(const char* utf8);
41 size_t CountModifiedUtf8Chars(const char* utf8, size_t byte_count);
42
43 /*
44 * Convert from Modified UTF-8 to UTF-16.
45 */
46 void ConvertModifiedUtf8ToUtf16(uint16_t* utf16_out, const char* utf8_in);
47 void ConvertModifiedUtf8ToUtf16(uint16_t* utf16_out, size_t out_chars,
48 const char* utf8_in, size_t in_bytes);
49
50 /*
51 * Compare two modified UTF-8 strings as UTF-16 code point values in a non-locale sensitive manner
52 */
53 ALWAYS_INLINE int CompareModifiedUtf8ToModifiedUtf8AsUtf16CodePointValues(const char* utf8_1,
54 const char* utf8_2);
55
56 /*
57 * Compare a null-terminated modified UTF-8 string with a UTF-16 string (not null-terminated)
58 * as code point values in a non-locale sensitive manner.
59 */
60 int CompareModifiedUtf8ToUtf16AsCodePointValues(const char* utf8, const uint16_t* utf16,
61 size_t utf16_length);
62
63 /*
64 * Helper template for converting UTF-16 to UTF-8 and similar encodings.
65 *
66 * Template arguments:
67 * kUseShortZero: Encode U+0000 as a single byte with value 0 (otherwise emit 0xc0 0x80).
68 * kUse4ByteSequence: Encode valid surrogate pairs as a 4-byte sequence.
69 * kReplaceBadSurrogates: Replace unmatched surrogates with '?' (otherwise use 3-byte sequence).
70 * Must be false if kUse4ByteSequence is false.
71 * Append: The type of the `append` functor. Should be deduced automatically.
72 *
73 * Encoding kUseShortZero kUse4ByteSequence kReplaceBadSurrogates
74 * UTF-8 true true true
75 * Modified UTF8 false false n/a
76 * JNI GetStringUTFChars false true false
77 */
78 template <bool kUseShortZero, bool kUse4ByteSequence, bool kReplaceBadSurrogates, typename Append>
79 void ConvertUtf16ToUtf8(const uint16_t* utf16, size_t char_count, Append&& append);
80
81 /*
82 * Returns the number of modified UTF-8 bytes needed to represent the given
83 * UTF-16 string.
84 */
85 size_t CountModifiedUtf8BytesInUtf16(const uint16_t* chars, size_t char_count);
86
87 /*
88 * Convert from UTF-16 to Modified UTF-8. Note that the output is _not_
89 * NUL-terminated. You probably need to call CountModifiedUtf8BytesInUtf16 before calling
90 * this anyway, so if you want a NUL-terminated string, you know where to
91 * put the NUL byte.
92 */
93 void ConvertUtf16ToModifiedUtf8(char* utf8_out, size_t byte_count,
94 const uint16_t* utf16_in, size_t char_count);
95
96 /*
97 * The java.lang.String hashCode() algorithm.
98 */
99 template<typename MemoryType>
ComputeUtf16Hash(const MemoryType * chars,size_t char_count)100 int32_t ComputeUtf16Hash(const MemoryType* chars, size_t char_count) {
101 static_assert(std::is_same_v<MemoryType, char> ||
102 std::is_same_v<MemoryType, uint8_t> ||
103 std::is_same_v<MemoryType, uint16_t>);
104 using UnsignedMemoryType = std::make_unsigned_t<MemoryType>;
105 uint32_t hash = 0;
106 while (char_count--) {
107 hash = hash * 31 + static_cast<UnsignedMemoryType>(*chars++);
108 }
109 return static_cast<int32_t>(hash);
110 }
111
112 int32_t ComputeUtf16HashFromModifiedUtf8(const char* utf8, size_t utf16_length);
113
114 // Compute a hash code of a modified UTF-8 string. Not the standard java hash since it returns a
115 // uint32_t and hashes individual chars instead of codepoint words.
116 uint32_t ComputeModifiedUtf8Hash(const char* chars);
117 uint32_t ComputeModifiedUtf8Hash(std::string_view chars);
118
119 // The starting value of a modified UTF-8 hash.
StartModifiedUtf8Hash()120 constexpr uint32_t StartModifiedUtf8Hash() {
121 return 0u;
122 }
123
124 // Update a modified UTF-8 hash with one character.
125 ALWAYS_INLINE
UpdateModifiedUtf8Hash(uint32_t hash,char c)126 inline uint32_t UpdateModifiedUtf8Hash(uint32_t hash, char c) {
127 return hash * 31u + static_cast<uint8_t>(c);
128 }
129
130 // Update a modified UTF-8 hash with characters of a `std::string_view`.
131 ALWAYS_INLINE
UpdateModifiedUtf8Hash(uint32_t hash,std::string_view chars)132 inline uint32_t UpdateModifiedUtf8Hash(uint32_t hash, std::string_view chars) {
133 for (char c : chars) {
134 hash = UpdateModifiedUtf8Hash(hash, c);
135 }
136 return hash;
137 }
138
139 /*
140 * Retrieve the next UTF-16 character or surrogate pair from a UTF-8 string.
141 * single byte, 2-byte and 3-byte UTF-8 sequences result in a single UTF-16
142 * character (possibly one half of a surrogate) whereas 4-byte UTF-8 sequences
143 * result in a surrogate pair. Use GetLeadingUtf16Char and GetTrailingUtf16Char
144 * to process the return value of this function.
145 *
146 * Advances "*utf8_data_in" to the start of the next character.
147 *
148 * WARNING: If a string is corrupted by dropping a '\0' in the middle
149 * of a multi byte sequence, you can end up overrunning the buffer with
150 * reads (and possibly with the writes if the length was computed and
151 * cached before the damage). For performance reasons, this function
152 * assumes that the string being parsed is known to be valid (e.g., by
153 * already being verified). Most strings we process here are coming
154 * out of dex files or other internal translations, so the only real
155 * risk comes from the JNI NewStringUTF call.
156 */
157 uint32_t GetUtf16FromUtf8(const char** utf8_data_in);
158
159 /**
160 * Gets the leading UTF-16 character from a surrogate pair, or the sole
161 * UTF-16 character from the return value of GetUtf16FromUtf8.
162 */
163 ALWAYS_INLINE uint16_t GetLeadingUtf16Char(uint32_t maybe_pair);
164
165 /**
166 * Gets the trailing UTF-16 character from a surrogate pair, or 0 otherwise
167 * from the return value of GetUtf16FromUtf8.
168 */
169 ALWAYS_INLINE uint16_t GetTrailingUtf16Char(uint32_t maybe_pair);
170
171 // Returns a printable (escaped) version of a character.
172 std::string PrintableChar(uint16_t ch);
173
174 // Returns an ASCII string corresponding to the given UTF-8 string.
175 // Java escapes are used for non-ASCII characters.
176 std::string PrintableString(const char* utf8);
177
178 } // namespace art
179
180 #endif // ART_LIBDEXFILE_DEX_UTF_H_
181