1 /* 2 * Copyright (C) 2018 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 #ifndef LIBTEXTCLASSIFIER_UTILS_UTF8_UNICODETEXT_H_ 18 #define LIBTEXTCLASSIFIER_UTILS_UTF8_UNICODETEXT_H_ 19 20 #include <iterator> 21 #include <string> 22 #include <utility> 23 24 #include "utils/base/integral_types.h" 25 #include "utils/base/logging.h" 26 #include "utils/strings/stringpiece.h" 27 28 namespace libtextclassifier3 { 29 30 // ***************************** UnicodeText ************************** 31 // 32 // A UnicodeText object is a wrapper around a sequence of Unicode 33 // codepoint values that allows iteration over these values. 34 // 35 // The internal representation of the text is UTF-8. Since UTF-8 is a 36 // variable-width format, UnicodeText does not provide random access 37 // to the text, and changes to the text are permitted only at the end. 38 // 39 // The UnicodeText class defines a const_iterator. The dereferencing 40 // operator (*) returns a codepoint (int32). The iterator is a 41 // read-only iterator. It becomes invalid if the text is changed. 42 // 43 // Codepoints are integers in the range [0, 0xD7FF] or [0xE000, 44 // 0x10FFFF], but UnicodeText has the additional restriction that it 45 // can contain only those characters that are valid for interchange on 46 // the Web. This excludes all of the control codes except for carriage 47 // return, line feed, and horizontal tab. It also excludes 48 // non-characters, but codepoints that are in the Private Use regions 49 // are allowed, as are codepoints that are unassigned. (See the 50 // Unicode reference for details.) 51 // 52 // MEMORY MANAGEMENT: 53 // 54 // PointToUTF8(buffer, size) creates an alias pointing to buffer. 55 // 56 // The purpose of an alias is to avoid making an unnecessary copy of a 57 // UTF-8 buffer while still providing access to the Unicode values 58 // within that text through iterators. The lifetime of an alias must not 59 // exceed the lifetime of the buffer from which it was constructed. 60 // 61 // Aliases should be used with care. If the source from which an alias 62 // was created is freed, or if the contents are changed, while the 63 // alias is still in use, fatal errors could result. But it can be 64 // quite useful to have a UnicodeText "window" through which to see a 65 // UTF-8 buffer without having to pay the price of making a copy. 66 67 class UnicodeText { 68 public: 69 class const_iterator; 70 71 UnicodeText(); // Create an empty text. 72 UnicodeText(const UnicodeText& src, bool do_copy = true); 73 UnicodeText& operator=(UnicodeText&& src); 74 ~UnicodeText(); 75 76 class const_iterator { 77 typedef const_iterator CI; 78 79 public: 80 typedef std::bidirectional_iterator_tag iterator_category; 81 typedef char32 value_type; 82 typedef int difference_type; 83 typedef void pointer; // (Not needed.) 84 typedef const char32 reference; // (Needed for const_reverse_iterator) 85 86 // Iterators are default-constructible. 87 const_iterator(); 88 89 // It's safe to make multiple passes over a UnicodeText. 90 const_iterator& operator=(const const_iterator& other); 91 92 char32 operator*() const; // Dereference 93 94 const_iterator& operator++(); // Advance (++iter) 95 const_iterator operator++(int) { // (iter++) 96 const_iterator result(*this); 97 ++*this; 98 return result; 99 } 100 101 const_iterator& operator--(); // Retreat (--iter) 102 const_iterator operator--(int) { // (iter--) 103 const_iterator result(*this); 104 --*this; 105 return result; 106 } 107 108 friend bool operator==(const CI& lhs, const CI& rhs) { 109 return lhs.it_ == rhs.it_; 110 } 111 friend bool operator!=(const CI& lhs, const CI& rhs) { 112 return !(lhs == rhs); 113 } 114 friend bool operator<(const CI& lhs, const CI& rhs); 115 friend bool operator>(const CI& lhs, const CI& rhs) { return rhs < lhs; } 116 friend bool operator<=(const CI& lhs, const CI& rhs) { 117 return !(rhs < lhs); 118 } 119 friend bool operator>=(const CI& lhs, const CI& rhs) { 120 return !(lhs < rhs); 121 } 122 utf8_length()123 int utf8_length() const { 124 const unsigned char byte = static_cast<unsigned char>(it_[0]); 125 if (byte < 0x80) { 126 return 1; 127 } else if (byte < 0xE0) { 128 return 2; 129 } else if (byte < 0xF0) { 130 return 3; 131 } else { 132 return 4; 133 } 134 } utf8_data()135 const char* utf8_data() const { return it_; } 136 137 private: 138 friend class UnicodeText; const_iterator(const char * it)139 explicit const_iterator(const char* it) : it_(it) {} 140 141 const char* it_; 142 }; 143 144 const_iterator begin() const; 145 const_iterator end() const; 146 147 // Gets pointer to the underlying utf8 data. 148 const char* data() const; 149 150 // Gets length (in bytes) of the underlying utf8 data. 151 int size_bytes() const; 152 153 // Computes length (in number of Unicode codepoints) of the underlying utf8 154 // data. 155 // NOTE: Complexity O(n). 156 int size_codepoints() const; 157 158 bool empty() const; 159 160 // Checks whether the underlying data is valid utf8 data. 161 bool is_valid() const; 162 163 bool operator==(const UnicodeText& other) const; 164 165 // x.PointToUTF8(buf,len) changes x so that it points to buf 166 // ("becomes an alias"). It does not take ownership or copy buf. 167 // This function assumes that the input is interchange valid UTF8. 168 UnicodeText& Copy(const UnicodeText& src); 169 UnicodeText& PointToUTF8(const char* utf8_buffer, int byte_length); 170 UnicodeText& CopyUTF8(const char* utf8_buffer, int byte_length); 171 172 // Calling this may invalidate pointers to underlying data. 173 UnicodeText& AppendUTF8(const char* utf8, int len); 174 UnicodeText& push_back(char32 ch); 175 void clear(); 176 177 std::string ToUTF8String() const; 178 std::string UTF8Substring(int begin_codepoint, int end_codepoint) const; 179 static std::string UTF8Substring(const const_iterator& it_begin, 180 const const_iterator& it_end); 181 static UnicodeText Substring(const UnicodeText& text, int begin_codepoint, 182 int end_codepoint, bool do_copy = true); 183 static UnicodeText Substring(const const_iterator& it_begin, 184 const const_iterator& it_end, 185 bool do_copy = true); 186 187 private: 188 friend class const_iterator; 189 190 class Repr { // A byte-string. 191 public: 192 char* data_; 193 int size_; 194 int capacity_; 195 bool ours_; // Do we own data_? 196 Repr()197 Repr() : data_(nullptr), size_(0), capacity_(0), ours_(true) {} 198 Repr& operator=(Repr&& src); ~Repr()199 ~Repr() { 200 if (ours_) delete[] data_; 201 } 202 203 void clear(); 204 void reserve(int capacity); 205 void resize(int size); 206 207 void append(const char* bytes, int byte_length); 208 void Copy(const char* data, int size); 209 void PointTo(const char* data, int size); 210 211 private: 212 Repr& operator=(const Repr&); 213 Repr(const Repr& other); 214 }; 215 216 Repr repr_; 217 }; 218 219 typedef std::pair<UnicodeText::const_iterator, UnicodeText::const_iterator> 220 UnicodeTextRange; 221 222 // NOTE: The following are needed to avoid implicit conversion from char* to 223 // std::string, or from ::string to std::string, because if this happens it 224 // often results in invalid memory access to a temporary object created during 225 // such conversion (if do_copy == false). 226 // NOTE: These methods don't check if the input string is UTF8 well formed, for 227 // efficiency reasons. Use UnicodeText::is_valid() when explicitly needed. 228 UnicodeText UTF8ToUnicodeText(const char* utf8_buf, int len, 229 bool do_copy = true); 230 UnicodeText UTF8ToUnicodeText(const char* utf8_buf, bool do_copy = true); 231 UnicodeText UTF8ToUnicodeText(const std::string& str, bool do_copy = true); 232 UnicodeText UTF8ToUnicodeText(StringPiece str, bool do_copy = true); 233 234 inline logging::LoggingStringStream& operator<<( 235 logging::LoggingStringStream& stream, const UnicodeText& message) { 236 stream.message.append(message.data(), message.size_bytes()); 237 return stream; 238 } 239 240 } // namespace libtextclassifier3 241 242 #endif // LIBTEXTCLASSIFIER_UTILS_UTF8_UNICODETEXT_H_ 243