Home
last modified time | relevance | path

Searched refs:tokens (Results 1 – 25 of 1636) sorted by relevance

12345678910>>...66

/external/antlr/runtime/Python3/tests/
Dt048rewrite.py25 tokens = self._parse("abc")
26 tokens.insertBefore(0, "0")
28 result = tokens.toString()
34 tokens = self._parse("abc")
35 tokens.insertAfter(2, "x")
37 result = tokens.toString()
43 tokens = self._parse("abc")
44 tokens.insertBefore(1, "x")
45 tokens.insertAfter(1, "x")
47 result = tokens.toString()
[all …]
/external/antlr/runtime/Python/tests/
Dt048rewrite.py25 tokens = self._parse("abc")
26 tokens.insertBefore(0, "0")
28 result = tokens.toString()
34 tokens = self._parse("abc")
35 tokens.insertAfter(2, "x")
37 result = tokens.toString()
43 tokens = self._parse("abc")
44 tokens.insertBefore(1, "x")
45 tokens.insertAfter(1, "x")
47 result = tokens.toString()
[all …]
/external/libtextclassifier/native/utils/
Dtokenizer-utils_test.cc26 std::vector<Token> tokens = in TEST() local
29 EXPECT_EQ(tokens.size(), 9); in TEST()
31 EXPECT_EQ(tokens[0].value, "Where"); in TEST()
32 EXPECT_EQ(tokens[0].start, 0); in TEST()
33 EXPECT_EQ(tokens[0].end, 5); in TEST()
35 EXPECT_EQ(tokens[1].value, "is"); in TEST()
36 EXPECT_EQ(tokens[1].start, 6); in TEST()
37 EXPECT_EQ(tokens[1].end, 8); in TEST()
39 EXPECT_EQ(tokens[2].value, "Jörg"); in TEST()
40 EXPECT_EQ(tokens[2].start, 9); in TEST()
[all …]
/external/rust/crates/quote/src/
Dlib.rs673 ($tokens:ident $($tts:tt)*) => {
674 $crate::quote_tokens_with_context!($tokens
689 ($tokens:ident $span:ident $($tts:tt)*) => {
690 $crate::quote_tokens_with_context_spanned!($tokens $span
705 ($tokens:ident
711 $crate::quote_token_with_context!($tokens $b3 $b2 $b1 $curr $a1 $a2 $a3);
719 ($tokens:ident $span:ident
725 $crate::quote_token_with_context_spanned!($tokens $span $b3 $b2 $b1 $curr $a1 $a2 $a3);
733 ($tokens:ident $b3:tt $b2:tt $b1:tt @ $a1:tt $a2:tt $a3:tt) => {};
735 ($tokens:ident $b3:tt $b2:tt $b1:tt (#) ( $($inner:tt)* ) * $a3:tt) => {{
[all …]
Dto_tokens.rs55 fn to_tokens(&self, tokens: &mut TokenStream); in to_tokens()
62 let mut tokens = TokenStream::new(); in to_token_stream() localVariable
63 self.to_tokens(&mut tokens); in to_token_stream()
64 tokens in to_token_stream()
80 fn to_tokens(&self, tokens: &mut TokenStream) { in to_tokens()
81 (**self).to_tokens(tokens); in to_tokens()
86 fn to_tokens(&self, tokens: &mut TokenStream) { in to_tokens()
87 (**self).to_tokens(tokens); in to_tokens()
92 fn to_tokens(&self, tokens: &mut TokenStream) { in to_tokens()
93 (**self).to_tokens(tokens); in to_tokens()
[all …]
/external/rust/cxx/syntax/
Dtokens.rs11 fn to_tokens(&self, tokens: &mut TokenStream) { in to_tokens()
16 tokens.extend(quote_spanned!(span=> ::std::os::raw::)); in to_tokens()
19 tokens.extend(quote_spanned!(span=> ::cxx::)); in to_tokens()
21 ident.to_tokens(tokens); in to_tokens()
28 | Type::RustVec(ty) => ty.to_tokens(tokens), in to_tokens()
29 Type::Ref(r) | Type::Str(r) => r.to_tokens(tokens), in to_tokens()
30 Type::Ptr(p) => p.to_tokens(tokens), in to_tokens()
31 Type::Array(a) => a.to_tokens(tokens), in to_tokens()
32 Type::Fn(f) => f.to_tokens(tokens), in to_tokens()
33 Type::Void(span) => tokens.extend(quote_spanned!(*span=> ())), in to_tokens()
[all …]
/external/antlr/tool/src/test/java/org/antlr/test/
DTestTokenRewriteStream.java53 TokenRewriteStream tokens = new TokenRewriteStream(lexEngine); in testInsertBeforeIndex0() local
54 tokens.insertBefore(0, "0"); in testInsertBeforeIndex0()
55 String result = tokens.toString(); in testInsertBeforeIndex0()
68 TokenRewriteStream tokens = new TokenRewriteStream(lexEngine); in testInsertAfterLastIndex() local
69 tokens.insertAfter(2, "x"); in testInsertAfterLastIndex()
70 String result = tokens.toString(); in testInsertAfterLastIndex()
83 TokenRewriteStream tokens = new TokenRewriteStream(lexEngine); in test2InsertBeforeAfterMiddleIndex() local
84 tokens.fill(); in test2InsertBeforeAfterMiddleIndex()
85 tokens.insertBefore(1, "x"); in test2InsertBeforeAfterMiddleIndex()
86 tokens.insertAfter(1, "x"); in test2InsertBeforeAfterMiddleIndex()
[all …]
DTestCommonTokenStream.java54 BufferedTokenStream tokens = new BufferedTokenStream(lexEngine); in testFirstToken() local
56 String result = tokens.LT(1).getText(); in testFirstToken()
75 BufferedTokenStream tokens = new BufferedTokenStream(lexEngine); in test2ndToken() local
77 String result = tokens.LT(2).getText(); in test2ndToken()
96 BufferedTokenStream tokens = new BufferedTokenStream(lexEngine); in testCompleteBuffer() local
99 Token t = tokens.LT(i); in testCompleteBuffer()
102 t = tokens.LT(i); in testCompleteBuffer()
104 tokens.LT(i++); // push it past end in testCompleteBuffer()
105 tokens.LT(i++); in testCompleteBuffer()
107 String result = tokens.toString(); in testCompleteBuffer()
[all …]
/external/minijail/tools/
Dparser.py135 tokens = []
157 tokens.append(
171 if tokens:
174 yield tokens[::]
175 tokens.clear()
263 def _parse_constant(self, tokens): argument
265 if tokens[0].type == 'BITWISE_COMPLEMENT':
267 tokens.pop(0)
268 if not tokens:
270 if tokens[0].type == 'BITWISE_COMPLEMENT':
[all …]
/external/pdfium/samples/
Dpdfium_test_event_helper.cc19 const std::vector<std::string>& tokens) { in SendCharCodeEvent() argument
20 if (tokens.size() != 2) { in SendCharCodeEvent()
25 int keycode = atoi(tokens[1].c_str()); in SendCharCodeEvent()
31 const std::vector<std::string>& tokens) { in SendKeyCodeEvent() argument
32 if (tokens.size() != 2) { in SendKeyCodeEvent()
37 int keycode = atoi(tokens[1].c_str()); in SendKeyCodeEvent()
56 const std::vector<std::string>& tokens) { in SendMouseDownEvent() argument
57 if (tokens.size() != 4 && tokens.size() != 5) { in SendMouseDownEvent()
62 int x = atoi(tokens[2].c_str()); in SendMouseDownEvent()
63 int y = atoi(tokens[3].c_str()); in SendMouseDownEvent()
[all …]
/external/antlr/runtime/Ruby/test/functional/ast-output/
Dtree-rewrite.rb1006 tokens = ANTLR3::CommonTokenStream.new( lexer )
1007 parser = FlatList::Parser.new( tokens )
1011 nodes.token_stream = tokens
1020 tokens = ANTLR3::CommonTokenStream.new( lexer )
1021 parser = SimpleTree::Parser.new( tokens )
1025 nodes.token_stream = tokens
1034 tokens = ANTLR3::CommonTokenStream.new( lexer )
1035 parser = CombinedRewriteAndAuto::Parser.new( tokens )
1039 nodes.token_stream = tokens
1044 tokens = ANTLR3::CommonTokenStream.new( lexer )
[all …]
/external/rust/crates/quote/tests/
Dtest.rs12 fn to_tokens(&self, tokens: &mut TokenStream) { in to_tokens()
13 tokens.append(Ident::new("X", Span::call_site())); in to_tokens()
19 let tokens = quote! { in test_quote_impl() localVariable
21 fn to_tokens(&self, tokens: &mut TokenStream) { in test_quote_impl()
22 (**self).to_tokens(tokens) in test_quote_impl()
35 assert_eq!(expected, tokens.to_string()); in test_quote_impl()
41 let tokens = quote_spanned! {span=> in test_quote_spanned_impl() localVariable
43 fn to_tokens(&self, tokens: &mut TokenStream) { in test_quote_spanned_impl()
44 (**self).to_tokens(tokens) in test_quote_spanned_impl()
57 assert_eq!(expected, tokens.to_string()); in test_quote_spanned_impl()
[all …]
/external/rust/crates/proc-macro-hack/src/
Dparse.rs6 pub(crate) fn parse_input(tokens: Iter) -> Result<Input, Error> { in parse_input()
7 let attrs = parse_attributes(tokens)?; in parse_input()
8 let vis = parse_visibility(tokens)?; in parse_input()
9 let kw = parse_ident(tokens)?; in parse_input()
11 parse_export(attrs, vis, tokens).map(Input::Export) in parse_input()
13 parse_define(attrs, vis, kw.span(), tokens).map(Input::Define) in parse_input()
22 fn parse_export(attrs: TokenStream, vis: Visibility, tokens: Iter) -> Result<Export, Error> { in parse_export()
23 let _ = parse_punct(tokens, ':'); in parse_export()
24 let _ = parse_punct(tokens, ':'); in parse_export()
25 let from = parse_ident(tokens)?; in parse_export()
[all …]
Dquote.rs9 let mut tokens = ::proc_macro::TokenStream::new(); localVariable
10 quote_each_token!(tokens $($tt)*);
11 tokens
16 ($tokens:ident # $var:ident $($rest:tt)*) => {
17 $crate::quote::Tokens::extend(&mut $tokens, &$var);
18 quote_each_token!($tokens $($rest)*);
20 ($tokens:ident $ident:ident $($rest:tt)*) => {
22 &mut $tokens,
32 quote_each_token!($tokens $($rest)*);
34 ($tokens:ident ( $($inner:tt)* ) $($rest:tt)*) => {
[all …]
/external/python/cpython2/Lib/plat-irix5/
Dpanelparser.py21 tokens = []
36 tokens.append(s[:i])
39 tokens.append(c)
47 tokens.append(s[:i])
49 return tokens
56 tokens = []
60 tokens = tokens + tokenize_string(line)
61 return tokens
76 def parse_expr(tokens): argument
77 if (not tokens) or tokens[0] != '(':
[all …]
/external/python/cpython2/Lib/plat-irix6/
Dpanelparser.py21 tokens = []
36 tokens.append(s[:i])
39 tokens.append(c)
47 tokens.append(s[:i])
49 return tokens
56 tokens = []
60 tokens = tokens + tokenize_string(line)
61 return tokens
76 def parse_expr(tokens): argument
77 if (not tokens) or tokens[0] != '(':
[all …]
/external/libtextclassifier/native/annotator/number/
Dnumber.cc77 bool NumberAnnotator::TokensAreValidStart(const std::vector<Token>& tokens, in TokensAreValidStart() argument
79 if (start_index < 0 || tokens[start_index].is_whitespace) { in TokensAreValidStart()
86 const std::vector<Token>& tokens, const int prefix_end_index) const { in TokensAreValidNumberPrefix() argument
87 if (TokensAreValidStart(tokens, prefix_end_index)) { in TokensAreValidNumberPrefix()
92 UTF8ToUnicodeText(tokens[prefix_end_index].value, /*do_copy=*/false) in TokensAreValidNumberPrefix()
95 tokens[prefix_end_index].end - tokens[prefix_end_index].start; in TokensAreValidNumberPrefix()
97 TokensAreValidStart(tokens, prefix_end_index - 1)) { in TokensAreValidNumberPrefix()
101 TokensAreValidStart(tokens, prefix_end_index - 1)) { in TokensAreValidNumberPrefix()
106 TokensAreValidStart(tokens, prefix_end_index - 2)) { in TokensAreValidNumberPrefix()
109 return TryParseNumber(UTF8ToUnicodeText(tokens[prefix_end_index - 1].value, in TokensAreValidNumberPrefix()
[all …]
/external/antlr/runtime/Ruby/test/functional/tree-parser/
Dbasic.rb33 tokens = ANTLR3::CommonTokenStream.new( lexer )
34 parser = FlatList::Parser.new( tokens )
38 nodes.token_stream = tokens
69 tokens = ANTLR3::CommonTokenStream.new( lexer )
70 parser = SimpleTree::Parser.new( tokens )
74 nodes.token_stream = tokens
108 tokens = ANTLR3::CommonTokenStream.new( lexer )
109 parser = FlatVsTreeDecision::Parser.new( tokens )
113 nodes.token_stream = tokens
146 tokens = ANTLR3::CommonTokenStream.new( lexer )
[all …]
/external/mesa3d/src/gallium/drivers/r300/compiler/tests/
Drc_test_helpers.c141 struct src_tokens tokens; in init_rc_normal_src() local
152 tokens.Negate.String = src_str + matches[1].rm_so; in init_rc_normal_src()
153 tokens.Negate.Length = match_length(matches, 1); in init_rc_normal_src()
154 tokens.Abs.String = src_str + matches[2].rm_so; in init_rc_normal_src()
155 tokens.Abs.Length = match_length(matches, 2); in init_rc_normal_src()
156 tokens.File.String = src_str + matches[3].rm_so; in init_rc_normal_src()
157 tokens.File.Length = match_length(matches, 3); in init_rc_normal_src()
158 tokens.Index.String = src_str + matches[4].rm_so; in init_rc_normal_src()
159 tokens.Index.Length = match_length(matches, 4); in init_rc_normal_src()
160 tokens.Swizzle.String = src_str + matches[5].rm_so; in init_rc_normal_src()
[all …]
/external/jackson-databind/src/main/java/com/fasterxml/jackson/databind/type/
DTypeParser.java32 MyTokenizer tokens = new MyTokenizer(canonical.trim()); in parse() local
33 JavaType type = parseType(tokens); in parse()
35 if (tokens.hasMoreTokens()) { in parse()
36 throw _problem(tokens, "Unexpected tokens after complete type"); in parse()
41 protected JavaType parseType(MyTokenizer tokens) in parseType() argument
44 if (!tokens.hasMoreTokens()) { in parseType()
45 throw _problem(tokens, "Unexpected end-of-string"); in parseType()
47 Class<?> base = findClass(tokens.nextToken(), tokens); in parseType()
50 if (tokens.hasMoreTokens()) { in parseType()
51 String token = tokens.nextToken(); in parseType()
[all …]
/external/rust/crates/syn/tests/
Dtest_item.rs12 let tokens = TokenStream::from_iter(vec![ in test_macro_variable_attr() localVariable
20 snapshot!(tokens as Item, @r###" in test_macro_variable_attr()
53 let tokens = quote! { in test_negative_impl() localVariable
56 snapshot!(tokens as Item, @r###" in test_negative_impl()
66 let tokens = quote! { in test_negative_impl() localVariable
69 snapshot!(tokens as Item, @r###" in test_negative_impl()
78 let tokens = quote! { in test_negative_impl() localVariable
81 snapshot!(tokens as Item, @r###" in test_negative_impl()
111 let tokens = quote! { in test_negative_impl() localVariable
114 snapshot!(tokens as Item, @r###" in test_negative_impl()
[all …]
/external/rust/crates/syn-mid/src/
Dfunc.rs90 if let Type::Verbatim(tokens) = &**ty { in parse()
91 if let Ok(dots) = parse2(tokens.clone()) { in parse()
195 fn to_tokens(&self, tokens: &mut TokenStream) { in to_tokens()
196 self.brace_token.surround(tokens, |tokens| { in to_tokens()
197 tokens.append_all(self.stmts.clone()); in to_tokens()
213 let tokens = match pat.ty.as_ref() { in has_variadic() localVariable
214 Type::Verbatim(tokens) => tokens, in has_variadic()
218 tokens.to_string() == "..." in has_variadic()
222 fn to_tokens(&self, tokens: &mut TokenStream) { in to_tokens()
223 self.constness.to_tokens(tokens); in to_tokens()
[all …]
/external/snakeyaml/src/test/java/org/pyyaml/
DCanonicalScanner.java26 import org.yaml.snakeyaml.tokens.AliasToken;
27 import org.yaml.snakeyaml.tokens.AnchorToken;
28 import org.yaml.snakeyaml.tokens.DirectiveToken;
29 import org.yaml.snakeyaml.tokens.DocumentStartToken;
30 import org.yaml.snakeyaml.tokens.FlowEntryToken;
31 import org.yaml.snakeyaml.tokens.FlowMappingEndToken;
32 import org.yaml.snakeyaml.tokens.FlowMappingStartToken;
33 import org.yaml.snakeyaml.tokens.FlowSequenceEndToken;
34 import org.yaml.snakeyaml.tokens.FlowSequenceStartToken;
35 import org.yaml.snakeyaml.tokens.KeyToken;
[all …]
/external/pigweed/pw_tokenizer/py/
Dtokens_test.py25 from pw_tokenizer import tokens
26 from pw_tokenizer.tokens import default_hash, _LOG
93 def read_db_from_csv(csv_str: str) -> tokens.Database:
95 return tokens.Database(tokens.parse_csv(csv_db))
98 def _entries(*strings: str) -> Iterator[tokens.TokenizedStringEntry]:
100 yield tokens.TokenizedStringEntry(default_hash(string), string)
161 hash_1 = tokens.pw_tokenizer_65599_fixed_length_hash('o000', 96)
162 hash_2 = tokens.pw_tokenizer_65599_fixed_length_hash('0Q1Q', 96)
165 db = tokens.Database.from_strings(['o000', '0Q1Q'])
198 db = tokens.Database()
[all …]
/external/chromium-trace/catapult/third_party/polymer/components/web-animations-js/src/
Dposition-handler.js40 var tokens = result[0];
41 tokens[0] = tokens[0] || 'center';
42 tokens[1] = tokens[1] || 'center';
44 tokens[2] = tokens[2] || {px: 0};
46 if (tokens.length != slots) {
50 if (/top|bottom/.test(tokens[0]) || /left|right/.test(tokens[1])) {
51 var tmp = tokens[0];
52 tokens[0] = tokens[1];
53 tokens[1] = tmp;
56 if (!/left|right|center|Object/.test(tokens[0]))
[all …]

12345678910>>...66