Lines Matching refs:new_token
139 Token new_token("", 0, 0); in InternalTokenize() local
152 if (!new_token.value.empty()) { in InternalTokenize()
153 result.push_back(new_token); in InternalTokenize()
155 new_token = Token("", codepoint_index, codepoint_index); in InternalTokenize()
158 new_token.end += 1; in InternalTokenize()
159 AppendCodepointToToken(it, &new_token); in InternalTokenize()
162 if (!new_token.value.empty()) { in InternalTokenize()
163 result.push_back(new_token); in InternalTokenize()
165 new_token = Token("", codepoint_index + 1, codepoint_index + 1); in InternalTokenize()
170 if (!new_token.value.empty()) { in InternalTokenize()
171 result.push_back(new_token); in InternalTokenize()
282 Token new_token("", 0, 0); in NumberTokenize() local
286 auto PushToken = [&new_token, result]() { in NumberTokenize()
287 if (!new_token.value.empty()) { in NumberTokenize()
288 result->push_back(new_token); in NumberTokenize()
293 [&new_token, PushToken, ¤t_token_type]( in NumberTokenize()
298 new_token = Token("", codepoint_index, codepoint_index, in NumberTokenize()
301 new_token.end += 1; in NumberTokenize()
302 AppendCodepointToToken(it, &new_token); in NumberTokenize()
307 [&new_token, result, ¤t_token_type, PushToken]( in NumberTokenize()
314 new_token = Token("", codepoint_index + 1, codepoint_index + 1); in NumberTokenize()
331 new_token.end += 1; in NumberTokenize()
332 AppendCodepointToToken(it, &new_token); in NumberTokenize()