| /external/icing/icing/tokenization/ |
| D | raw-query-tokenizer_test.cc | 63 IsOkAndHolds(ElementsAre(EqualsToken(Token::REGULAR, "Hello"), in TEST_F()
64 EqualsToken(Token::REGULAR, "World")))); in TEST_F()
67 IsOkAndHolds(ElementsAre(EqualsToken(Token::REGULAR, "hElLo"), in TEST_F()
68 EqualsToken(Token::REGULAR, "WORLD")))); in TEST_F()
83 EqualsToken(Token::QUERY_LEFT_PARENTHESES, ""), in TEST_F()
84 EqualsToken(Token::QUERY_RIGHT_PARENTHESES, "")))); in TEST_F()
88 EqualsToken(Token::QUERY_LEFT_PARENTHESES, ""), in TEST_F()
89 EqualsToken(Token::QUERY_RIGHT_PARENTHESES, "")))); in TEST_F()
93 EqualsToken(Token::QUERY_LEFT_PARENTHESES, ""), in TEST_F()
94 EqualsToken(Token::REGULAR, "term1"), in TEST_F()
[all …]
|
| D | plain-tokenizer_test.cc | 72 IsOkAndHolds(ElementsAre(EqualsToken(Token::REGULAR, "Hello"), in TEST_F()
73 EqualsToken(Token::REGULAR, "World")))); in TEST_F()
79 IsOkAndHolds(ElementsAre(EqualsToken(Token::REGULAR, "Lorem"), in TEST_F()
80 EqualsToken(Token::REGULAR, "ipsum"), in TEST_F()
81 EqualsToken(Token::REGULAR, "dolor"), in TEST_F()
82 EqualsToken(Token::REGULAR, "sit"), in TEST_F()
83 EqualsToken(Token::REGULAR, "amet"), in TEST_F()
84 EqualsToken(Token::REGULAR, "consectetur"), in TEST_F()
85 EqualsToken(Token::REGULAR, "adipiscing"), in TEST_F()
86 EqualsToken(Token::REGULAR, "elit"), in TEST_F()
[all …]
|
| /external/turbine/java/com/google/turbine/parse/ |
| D | StreamLexer.java | 97 public Token next() { in next()
124 return Token.EOF; in next()
180 return Token.DIVEQ; in next()
182 return Token.DIV; in next()
246 return Token.EOF; in next()
265 return Token.LPAREN; in next()
268 return Token.RPAREN; in next()
271 return Token.LBRACE; in next()
274 return Token.RBRACE; in next()
277 return Token.LBRACK; in next()
[all …]
|
| /external/libtextclassifier/native/utils/ |
| D | tokenizer_test.cc | 94 std::vector<Token> Tokenize(const std::string& utf8_text) const { in Tokenize()
178 std::vector<Token> tokens = tokenizer.Tokenize("Hello world!"); in TEST()
181 ElementsAreArray({Token("Hello", 0, 5), Token("world!", 6, 12)})); in TEST()
214 std::vector<Token>({Token("앨라배마", 0, 4), Token("주", 5, 6), in TEST()
215 Token("전화", 7, 10), Token("(123)", 10, 15), in TEST()
216 Token("456-789", 16, 23), in TEST()
217 Token("웹사이트", 23, 28)})); in TEST()
349 std::vector<Token> tokens; in TEST()
359 ElementsAreArray({Token("問", 0, 1), in TEST()
360 Token("少", 1, 2), in TEST()
[all …]
|
| D | tokenizer-utils_test.cc | 26 std::vector<Token> tokens = in TEST()
69 std::vector<Token> tokens = TokenizeOnDelimiters( in TEST()
100 std::vector<Token> tokens = TokenizeOnDelimiters( in TEST()
146 std::vector<Token> tokens = in TEST()
150 {Token{"I", 0, 1}, Token{"am", 2, 4}, in TEST()
151 Token{"fine", 5, 9}, Token{",", 9, 10}, in TEST()
152 Token{"thanks", 11, 17}, Token{"!", 17, 18}})); in TEST()
158 std::vector<Token> tokens = in TEST()
162 testing::ElementsAreArray({Token{"Good", 0, 4}, Token{"!", 4, 5}, in TEST()
163 Token{"Cool", 6, 10}})); in TEST()
[all …]
|
| /external/libtextclassifier/native/annotator/ |
| D | feature-processor_test.cc | 87 std::vector<Token> tokens{Token("Hělló", 0, 5), in TEST_F()
88 Token("fěěbař@google.com", 6, 23), in TEST_F()
89 Token("heře!", 24, 29)}; in TEST_F()
95 {Token("Hělló", 0, 5), in TEST_F()
96 Token("fěě", 6, 9), in TEST_F()
97 Token("bař", 9, 12), in TEST_F()
98 Token("@google.com", 12, 23), in TEST_F()
99 Token("heře!", 24, 29)})); in TEST_F()
104 std::vector<Token> tokens{Token("Hělló", 0, 5), in TEST_F()
105 Token("fěěbař@google.com", 6, 23), in TEST_F()
[all …]
|
| /external/python/cpython3/Doc/library/ |
| D | token-list.inc | 18 Token value for ``"("``.
22 Token value for ``")"``.
26 Token value for ``"["``.
30 Token value for ``"]"``.
34 Token value for ``":"``.
38 Token value for ``","``.
42 Token value for ``";"``.
46 Token value for ``"+"``.
50 Token value for ``"-"``.
54 Token value for ``"*"``.
[all …]
|
| /external/rust/crates/serde_test/src/ |
| D | de.rs | 8 use token::Token;
12 tokens: &'de [Token],
44 pub fn new(tokens: &'de [Token]) -> Self { in new()
48 fn peek_token_opt(&self) -> Option<Token> { in peek_token_opt() argument
52 fn peek_token(&self) -> Token { in peek_token() argument
59 pub fn next_token_opt(&mut self) -> Option<Token> { in next_token_opt() argument
69 fn next_token(&mut self) -> Token { in next_token() argument
86 end: Token, in visit_seq() argument
104 end: Token, in visit_map() argument
134 Token::Bool(v) => visitor.visit_bool(v), in deserialize_any()
[all …]
|
| /external/skqp/src/sksl/ |
| D | SkSLParser.cpp | 133 , fPushback(Token::INVALID, -1, -1) in Parser()
146 case Token::END_OF_FILE: in file()
148 case Token::DIRECTIVE: { in file()
155 case Token::SECTION: { in file()
173 Token Parser::nextRawToken() { in nextRawToken()
174 if (fPushback.fKind != Token::INVALID) { in nextRawToken()
175 Token result = fPushback; in nextRawToken()
176 fPushback.fKind = Token::INVALID; in nextRawToken()
179 Token result = fLexer.next(); in nextRawToken()
183 Token Parser::nextToken() { in nextToken()
[all …]
|
| /external/llvm-project/mlir/lib/Parser/ |
| D | TypeParser.cpp | 24 case Token::l_paren: in parseOptionalType()
25 case Token::kw_memref: in parseOptionalType()
26 case Token::kw_tensor: in parseOptionalType()
27 case Token::kw_complex: in parseOptionalType()
28 case Token::kw_tuple: in parseOptionalType()
29 case Token::kw_vector: in parseOptionalType()
30 case Token::inttype: in parseOptionalType()
31 case Token::kw_bf16: in parseOptionalType()
32 case Token::kw_f16: in parseOptionalType()
33 case Token::kw_f32: in parseOptionalType()
[all …]
|
| D | Lexer.cpp | 52 Token Lexer::emitError(const char *loc, const Twine &message) { in emitError()
55 return formToken(Token::error, loc); in emitError()
58 Token Lexer::lexToken() { in lexToken()
85 return formToken(Token::eof, tokStart); in lexToken()
89 return formToken(Token::colon, tokStart); in lexToken()
91 return formToken(Token::comma, tokStart); in lexToken()
95 return formToken(Token::l_paren, tokStart); in lexToken()
97 return formToken(Token::r_paren, tokStart); in lexToken()
99 return formToken(Token::l_brace, tokStart); in lexToken()
101 return formToken(Token::r_brace, tokStart); in lexToken()
[all …]
|
| D | AttributeParser.cpp | 46 case Token::kw_affine_map: { in parseAttribute()
47 consumeToken(Token::kw_affine_map); in parseAttribute()
50 if (parseToken(Token::less, "expected '<' in affine map") || in parseAttribute()
52 parseToken(Token::greater, "expected '>' in affine map")) in parseAttribute()
56 case Token::kw_affine_set: { in parseAttribute()
57 consumeToken(Token::kw_affine_set); in parseAttribute()
60 if (parseToken(Token::less, "expected '<' in integer set") || in parseAttribute()
62 parseToken(Token::greater, "expected '>' in integer set")) in parseAttribute()
68 case Token::l_square: { in parseAttribute()
69 consumeToken(Token::l_square); in parseAttribute()
[all …]
|
| /external/rust/crates/syn/src/ |
| D | op.rs | 9 Add(Token![+]),
11 Sub(Token![-]),
13 Mul(Token![*]),
15 Div(Token![/]),
17 Rem(Token![%]),
19 And(Token![&&]),
21 Or(Token![||]),
23 BitXor(Token![^]),
25 BitAnd(Token![&]),
27 BitOr(Token![|]),
[all …]
|
| /external/deqp/framework/randomshaders/ |
| D | rsgToken.hpp | 33 class Token class
127 Token (void);
128 Token (Type type);
129 Token (const char* identifier);
130 Token (float value);
131 Token (int value);
132 Token (bool value);
133 Token (const Token& other);
135 ~Token (void);
140 bool operator== (const Token& other) const;
[all …]
|
| D | rsgVariable.cpp | 50 case STORAGE_CONST: str << Token::CONST; break; in tokenizeDeclaration()
51 case STORAGE_PARAMETER_IN: str << Token::IN; break; in tokenizeDeclaration()
52 case STORAGE_PARAMETER_OUT: str << Token::OUT; break; in tokenizeDeclaration()
53 case STORAGE_PARAMETER_INOUT: str << Token::INOUT; break; in tokenizeDeclaration()
57 str << Token::UNIFORM; in tokenizeDeclaration()
59 str << Token::MEDIUM_PRECISION; in tokenizeDeclaration()
68 …str << Token::LAYOUT << Token::LEFT_PAREN << Token::LOCATION << Token::EQUAL << m_layoutLocation <… in tokenizeDeclaration()
70 str << Token::IN; in tokenizeDeclaration()
73 str << Token::MEDIUM_PRECISION; in tokenizeDeclaration()
81 case Shader::TYPE_VERTEX: str << Token::ATTRIBUTE; break; in tokenizeDeclaration()
[all …]
|
| /external/skia/src/sksl/ |
| D | SkSLParser.cpp | 43 static int parse_modifier_token(Token::Kind token) { in parse_modifier_token()
45 case Token::Kind::TK_UNIFORM: return Modifiers::kUniform_Flag; in parse_modifier_token()
46 case Token::Kind::TK_CONST: return Modifiers::kConst_Flag; in parse_modifier_token()
47 case Token::Kind::TK_IN: return Modifiers::kIn_Flag; in parse_modifier_token()
48 case Token::Kind::TK_OUT: return Modifiers::kOut_Flag; in parse_modifier_token()
49 case Token::Kind::TK_INOUT: return Modifiers::kIn_Flag | Modifiers::kOut_Flag; in parse_modifier_token()
50 case Token::Kind::TK_FLAT: return Modifiers::kFlat_Flag; in parse_modifier_token()
51 case Token::Kind::TK_NOPERSPECTIVE: return Modifiers::kNoPerspective_Flag; in parse_modifier_token()
52 case Token::Kind::TK_HASSIDEEFFECTS: return Modifiers::kHasSideEffects_Flag; in parse_modifier_token()
53 case Token::Kind::TK_INLINE: return Modifiers::kInline_Flag; in parse_modifier_token()
[all …]
|
| /external/angle/src/tests/preprocessor_tests/ |
| D | operator_test.cpp | 27 pp::Token token; in TEST_P()
54 {"++", pp::Token::OP_INC},
55 {"--", pp::Token::OP_DEC},
56 {"<<", pp::Token::OP_LEFT},
57 {">>", pp::Token::OP_RIGHT},
58 {"<=", pp::Token::OP_LE},
59 {">=", pp::Token::OP_GE},
60 {"==", pp::Token::OP_EQ},
61 {"!=", pp::Token::OP_NE},
62 {"&&", pp::Token::OP_AND},
[all …]
|
| /external/swiftshader/third_party/llvm-10.0/llvm/lib/CodeGen/MIRParser/ |
| D | MIParser.cpp | 401 MIToken Token; member in __anon5e931b060111::MIParser
565 CurrentSource.data() + SkipChar, Token, in lex()
569 bool MIParser::error(const Twine &Msg) { return error(Token.location(), Msg); } in error()
609 if (Token.isNot(TokenKind)) in expectAndConsume()
616 if (Token.isNot(TokenKind)) in consumeIfPresent()
624 assert(Token.is(MIToken::MachineBasicBlockLabel)); in parseBasicBlockDefinition()
628 auto Loc = Token.location(); in parseBasicBlockDefinition()
629 auto Name = Token.stringValue(); in parseBasicBlockDefinition()
638 switch (Token.kind()) { in parseBasicBlockDefinition()
693 while (Token.is(MIToken::Newline)) in parseBasicBlockDefinitions()
[all …]
|
| /external/rust/cxx/syntax/ |
| D | mod.rs | 40 use syn::{Expr, Generics, Lifetime, LitInt, Token, Type as RustType};
82 pub visibility: Token![pub],
83 pub type_token: Token![type],
86 pub colon_token: Option<Token![:]>,
88 pub semi_token: Token![;],
96 pub visibility: Token![pub],
97 pub struct_token: Token![struct],
108 pub visibility: Token![pub],
109 pub enum_token: Token![enum],
123 pub visibility: Token![pub],
[all …]
|
| /external/llvm-project/llvm/lib/CodeGen/MIRParser/ |
| D | MIParser.cpp | 401 MIToken Token; member in __anon882c9e920111::MIParser
566 CurrentSource.slice(SkipChar, StringRef::npos), Token, in lex()
570 bool MIParser::error(const Twine &Msg) { return error(Token.location(), Msg); } in error()
610 if (Token.isNot(TokenKind)) in expectAndConsume()
617 if (Token.isNot(TokenKind)) in consumeIfPresent()
625 assert(Token.is(MIToken::kw_bbsections)); in parseSectionID()
627 if (Token.is(MIToken::IntegerLiteral)) { in parseSectionID()
633 const StringRef &S = Token.stringValue(); in parseSectionID()
647 assert(Token.is(MIToken::MachineBasicBlockLabel)); in parseBasicBlockDefinition()
651 auto Loc = Token.location(); in parseBasicBlockDefinition()
[all …]
|
| /external/antlr/runtime/CSharp3/Sources/Antlr3.Runtime/Tree/ |
| D | CommonTree.cs | 75 this.Token = node.Token; in CommonTree()
82 this.Token = t; in CommonTree()
91 if ( Token == null || Token.CharPositionInLine == -1 )
98 return Token.CharPositionInLine;
124 return Token == null;
132 if ( Token == null || Token.Line == 0 )
140 return Token.Line;
166 if ( Token == null )
169 return Token.Text;
177 public IToken Token property in Antlr.Runtime.Tree.CommonTree
[all …]
|
| /external/rust/crates/hashlink/tests/ |
| D | serde.rs | 4 use serde_test::{assert_tokens, Token};
10 assert_tokens(&map, &[Token::Map { len: Some(0) }, Token::MapEnd]); in map_serde_tokens_empty()
23 Token::Map { len: Some(3) }, in map_serde_tokens()
24 Token::Char('a'), in map_serde_tokens()
25 Token::I32(10), in map_serde_tokens()
26 Token::Char('b'), in map_serde_tokens()
27 Token::I32(20), in map_serde_tokens()
28 Token::Char('c'), in map_serde_tokens()
29 Token::I32(30), in map_serde_tokens()
30 Token::MapEnd, in map_serde_tokens()
[all …]
|
| /external/google-java-format/core/src/main/java/com/google/googlejavaformat/java/javadoc/ |
| D | JavadocLexer.java | 21 import static com.google.googlejavaformat.java.javadoc.Token.Type.BEGIN_JAVADOC;
22 import static com.google.googlejavaformat.java.javadoc.Token.Type.BLOCKQUOTE_CLOSE_TAG;
23 import static com.google.googlejavaformat.java.javadoc.Token.Type.BLOCKQUOTE_OPEN_TAG;
24 import static com.google.googlejavaformat.java.javadoc.Token.Type.BR_TAG;
25 import static com.google.googlejavaformat.java.javadoc.Token.Type.CODE_CLOSE_TAG;
26 import static com.google.googlejavaformat.java.javadoc.Token.Type.CODE_OPEN_TAG;
27 import static com.google.googlejavaformat.java.javadoc.Token.Type.END_JAVADOC;
28 import static com.google.googlejavaformat.java.javadoc.Token.Type.FOOTER_JAVADOC_TAG_START;
29 import static com.google.googlejavaformat.java.javadoc.Token.Type.FORCED_NEWLINE;
30 import static com.google.googlejavaformat.java.javadoc.Token.Type.HEADER_CLOSE_TAG;
[all …]
|
| /external/jsilver/src/com/google/clearsilver/jsilver/syntax/lexer/ |
| D | Lexer.java | 11 protected Token token;
32 public Token peek() throws LexerException, IOException in peek()
43 public Token next() throws LexerException, IOException in next()
51 Token result = this.token; in next()
56 protected Token getToken() throws IOException, LexerException in getToken()
160 @SuppressWarnings("hiding") Token token = new0( in getToken()
175 @SuppressWarnings("hiding") Token token = new1( in getToken()
190 @SuppressWarnings("hiding") Token token = new2( in getToken()
204 @SuppressWarnings("hiding") Token token = new3( in getToken()
218 @SuppressWarnings("hiding") Token token = new4( in getToken()
[all …]
|
| /external/rust/crates/uuid/src/adapter/ |
| D | compact.rs | 54 serde_test::Token::Struct { in test_serialize_compact()
58 serde_test::Token::Str("u"), in test_serialize_compact()
59 serde_test::Token::Tuple { len: 16 }, in test_serialize_compact()
60 serde_test::Token::U8(uuid_bytes[0]), in test_serialize_compact()
61 serde_test::Token::U8(uuid_bytes[1]), in test_serialize_compact()
62 serde_test::Token::U8(uuid_bytes[2]), in test_serialize_compact()
63 serde_test::Token::U8(uuid_bytes[3]), in test_serialize_compact()
64 serde_test::Token::U8(uuid_bytes[4]), in test_serialize_compact()
65 serde_test::Token::U8(uuid_bytes[5]), in test_serialize_compact()
66 serde_test::Token::U8(uuid_bytes[6]), in test_serialize_compact()
[all …]
|