1 // Copyright 2007-2010 the V8 project authors. All rights reserved.
2 // Redistribution and use in source and binary forms, with or without
3 // modification, are permitted provided that the following conditions are
4 // met:
5 //
6 // * Redistributions of source code must retain the above copyright
7 // notice, this list of conditions and the following disclaimer.
8 // * Redistributions in binary form must reproduce the above
9 // copyright notice, this list of conditions and the following
10 // disclaimer in the documentation and/or other materials provided
11 // with the distribution.
12 // * Neither the name of Google Inc. nor the names of its
13 // contributors may be used to endorse or promote products derived
14 // from this software without specific prior written permission.
15 //
16 // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
17 // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
18 // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
19 // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
20 // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
21 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
22 // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
23 // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
24 // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
25 // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
26 // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
27
28 #ifndef V8_UNICODE_INL_H_
29 #define V8_UNICODE_INL_H_
30
31 #include "unicode.h"
32
33 namespace unibrow {
34
get(uchar code_point)35 template <class T, int s> bool Predicate<T, s>::get(uchar code_point) {
36 CacheEntry entry = entries_[code_point & kMask];
37 if (entry.code_point_ == code_point) return entry.value_;
38 return CalculateValue(code_point);
39 }
40
CalculateValue(uchar code_point)41 template <class T, int s> bool Predicate<T, s>::CalculateValue(
42 uchar code_point) {
43 bool result = T::Is(code_point);
44 entries_[code_point & kMask] = CacheEntry(code_point, result);
45 return result;
46 }
47
get(uchar c,uchar n,uchar * result)48 template <class T, int s> int Mapping<T, s>::get(uchar c, uchar n,
49 uchar* result) {
50 CacheEntry entry = entries_[c & kMask];
51 if (entry.code_point_ == c) {
52 if (entry.offset_ == 0) {
53 return 0;
54 } else {
55 result[0] = c + entry.offset_;
56 return 1;
57 }
58 } else {
59 return CalculateValue(c, n, result);
60 }
61 }
62
CalculateValue(uchar c,uchar n,uchar * result)63 template <class T, int s> int Mapping<T, s>::CalculateValue(uchar c, uchar n,
64 uchar* result) {
65 bool allow_caching = true;
66 int length = T::Convert(c, n, result, &allow_caching);
67 if (allow_caching) {
68 if (length == 1) {
69 entries_[c & kMask] = CacheEntry(c, result[0] - c);
70 return 1;
71 } else {
72 entries_[c & kMask] = CacheEntry(c, 0);
73 return 0;
74 }
75 } else {
76 return length;
77 }
78 }
79
80
Encode(char * str,uchar c,int previous)81 unsigned Utf8::Encode(char* str, uchar c, int previous) {
82 static const int kMask = ~(1 << 6);
83 if (c <= kMaxOneByteChar) {
84 str[0] = c;
85 return 1;
86 } else if (c <= kMaxTwoByteChar) {
87 str[0] = 0xC0 | (c >> 6);
88 str[1] = 0x80 | (c & kMask);
89 return 2;
90 } else if (c <= kMaxThreeByteChar) {
91 if (Utf16::IsTrailSurrogate(c) &&
92 Utf16::IsLeadSurrogate(previous)) {
93 const int kUnmatchedSize = kSizeOfUnmatchedSurrogate;
94 return Encode(str - kUnmatchedSize,
95 Utf16::CombineSurrogatePair(previous, c),
96 Utf16::kNoPreviousCharacter) - kUnmatchedSize;
97 }
98 str[0] = 0xE0 | (c >> 12);
99 str[1] = 0x80 | ((c >> 6) & kMask);
100 str[2] = 0x80 | (c & kMask);
101 return 3;
102 } else {
103 str[0] = 0xF0 | (c >> 18);
104 str[1] = 0x80 | ((c >> 12) & kMask);
105 str[2] = 0x80 | ((c >> 6) & kMask);
106 str[3] = 0x80 | (c & kMask);
107 return 4;
108 }
109 }
110
111
ValueOf(const byte * bytes,unsigned length,unsigned * cursor)112 uchar Utf8::ValueOf(const byte* bytes, unsigned length, unsigned* cursor) {
113 if (length <= 0) return kBadChar;
114 byte first = bytes[0];
115 // Characters between 0000 and 0007F are encoded as a single character
116 if (first <= kMaxOneByteChar) {
117 *cursor += 1;
118 return first;
119 }
120 return CalculateValue(bytes, length, cursor);
121 }
122
Length(uchar c,int previous)123 unsigned Utf8::Length(uchar c, int previous) {
124 if (c <= kMaxOneByteChar) {
125 return 1;
126 } else if (c <= kMaxTwoByteChar) {
127 return 2;
128 } else if (c <= kMaxThreeByteChar) {
129 if (Utf16::IsTrailSurrogate(c) &&
130 Utf16::IsLeadSurrogate(previous)) {
131 return kSizeOfUnmatchedSurrogate - kBytesSavedByCombiningSurrogates;
132 }
133 return 3;
134 } else {
135 return 4;
136 }
137 }
138
GetNext()139 uchar CharacterStream::GetNext() {
140 uchar result = DecodeCharacter(buffer_, &cursor_);
141 if (remaining_ == 1) {
142 cursor_ = 0;
143 FillBuffer();
144 } else {
145 remaining_--;
146 }
147 return result;
148 }
149
150 #if __BYTE_ORDER == __LITTLE_ENDIAN
151 #define IF_LITTLE(expr) expr
152 #define IF_BIG(expr) ((void) 0)
153 #elif __BYTE_ORDER == __BIG_ENDIAN
154 #define IF_LITTLE(expr) ((void) 0)
155 #define IF_BIG(expr) expr
156 #else
157 #warning Unknown byte ordering
158 #endif
159
EncodeAsciiCharacter(uchar c,byte * buffer,unsigned capacity,unsigned & offset)160 bool CharacterStream::EncodeAsciiCharacter(uchar c, byte* buffer,
161 unsigned capacity, unsigned& offset) {
162 if (offset >= capacity) return false;
163 buffer[offset] = c;
164 offset += 1;
165 return true;
166 }
167
EncodeNonAsciiCharacter(uchar c,byte * buffer,unsigned capacity,unsigned & offset)168 bool CharacterStream::EncodeNonAsciiCharacter(uchar c, byte* buffer,
169 unsigned capacity, unsigned& offset) {
170 unsigned aligned = (offset + 0x3) & ~0x3;
171 if ((aligned + sizeof(uchar)) > capacity)
172 return false;
173 if (offset == aligned) {
174 IF_LITTLE(*reinterpret_cast<uchar*>(buffer + aligned) = (c << 8) | 0x80);
175 IF_BIG(*reinterpret_cast<uchar*>(buffer + aligned) = c | (1 << 31));
176 } else {
177 buffer[offset] = 0x80;
178 IF_LITTLE(*reinterpret_cast<uchar*>(buffer + aligned) = c << 8);
179 IF_BIG(*reinterpret_cast<uchar*>(buffer + aligned) = c);
180 }
181 offset = aligned + sizeof(uchar);
182 return true;
183 }
184
EncodeCharacter(uchar c,byte * buffer,unsigned capacity,unsigned & offset)185 bool CharacterStream::EncodeCharacter(uchar c, byte* buffer, unsigned capacity,
186 unsigned& offset) {
187 if (c <= Utf8::kMaxOneByteChar) {
188 return EncodeAsciiCharacter(c, buffer, capacity, offset);
189 } else {
190 return EncodeNonAsciiCharacter(c, buffer, capacity, offset);
191 }
192 }
193
DecodeCharacter(const byte * buffer,unsigned * offset)194 uchar CharacterStream::DecodeCharacter(const byte* buffer, unsigned* offset) {
195 byte b = buffer[*offset];
196 if (b <= Utf8::kMaxOneByteChar) {
197 (*offset)++;
198 return b;
199 } else {
200 unsigned aligned = (*offset + 0x3) & ~0x3;
201 *offset = aligned + sizeof(uchar);
202 IF_LITTLE(return *reinterpret_cast<const uchar*>(buffer + aligned) >> 8);
203 IF_BIG(return *reinterpret_cast<const uchar*>(buffer + aligned) &
204 ~(1 << 31));
205 }
206 }
207
208 #undef IF_LITTLE
209 #undef IF_BIG
210
211 template <class R, class I, unsigned s>
FillBuffer()212 void InputBuffer<R, I, s>::FillBuffer() {
213 buffer_ = R::ReadBlock(input_, util_buffer_, s, &remaining_, &offset_);
214 }
215
216 template <class R, class I, unsigned s>
Rewind()217 void InputBuffer<R, I, s>::Rewind() {
218 Reset(input_);
219 }
220
221 template <class R, class I, unsigned s>
Reset(unsigned position,I input)222 void InputBuffer<R, I, s>::Reset(unsigned position, I input) {
223 input_ = input;
224 remaining_ = 0;
225 cursor_ = 0;
226 offset_ = position;
227 buffer_ = R::ReadBlock(input_, util_buffer_, s, &remaining_, &offset_);
228 }
229
230 template <class R, class I, unsigned s>
Reset(I input)231 void InputBuffer<R, I, s>::Reset(I input) {
232 Reset(0, input);
233 }
234
235 template <class R, class I, unsigned s>
Seek(unsigned position)236 void InputBuffer<R, I, s>::Seek(unsigned position) {
237 offset_ = position;
238 buffer_ = R::ReadBlock(input_, util_buffer_, s, &remaining_, &offset_);
239 }
240
241 template <unsigned s>
Utf8InputBuffer(const char * data,unsigned length)242 Utf8InputBuffer<s>::Utf8InputBuffer(const char* data, unsigned length)
243 : InputBuffer<Utf8, Buffer<const char*>, s>(Buffer<const char*>(data,
244 length)) {
245 }
246
247 } // namespace unibrow
248
249 #endif // V8_UNICODE_INL_H_
250