Home
last modified time | relevance | path

Searched refs:tokenizer (Results 1 – 25 of 584) sorted by relevance

12345678910>>...24

/external/cronet/tot/third_party/boringssl/src/pki/
Dpem_unittest.cc20 PEMTokenizer tokenizer(string_piece, accepted_types); in TEST() local
21 EXPECT_TRUE(tokenizer.GetNext()); in TEST()
23 EXPECT_EQ("EXPECTED-BLOCK", tokenizer.block_type()); in TEST()
24 EXPECT_EQ("MatchesAcceptedBlockType", tokenizer.data()); in TEST()
26 EXPECT_FALSE(tokenizer.GetNext()); in TEST()
38 PEMTokenizer tokenizer(string_piece, accepted_types); in TEST() local
39 EXPECT_TRUE(tokenizer.GetNext()); in TEST()
41 EXPECT_EQ("EXPECTED-BLOCK", tokenizer.block_type()); in TEST()
42 EXPECT_EQ("MatchesAcceptedBlockType", tokenizer.data()); in TEST()
44 EXPECT_FALSE(tokenizer.GetNext()); in TEST()
[all …]
/external/cronet/stable/third_party/boringssl/src/pki/
Dpem_unittest.cc20 PEMTokenizer tokenizer(string_piece, accepted_types); in TEST() local
21 EXPECT_TRUE(tokenizer.GetNext()); in TEST()
23 EXPECT_EQ("EXPECTED-BLOCK", tokenizer.block_type()); in TEST()
24 EXPECT_EQ("MatchesAcceptedBlockType", tokenizer.data()); in TEST()
26 EXPECT_FALSE(tokenizer.GetNext()); in TEST()
38 PEMTokenizer tokenizer(string_piece, accepted_types); in TEST() local
39 EXPECT_TRUE(tokenizer.GetNext()); in TEST()
41 EXPECT_EQ("EXPECTED-BLOCK", tokenizer.block_type()); in TEST()
42 EXPECT_EQ("MatchesAcceptedBlockType", tokenizer.data()); in TEST()
44 EXPECT_FALSE(tokenizer.GetNext()); in TEST()
[all …]
/external/cronet/stable/third_party/protobuf/csharp/src/Google.Protobuf.Test/
DJsonTokenizerTest.cs88 var tokenizer = JsonTokenizer.FromTextReader(new StringReader(json)); in ObjectDepth()
90 Assert.AreEqual(0, tokenizer.ObjectDepth); in ObjectDepth()
91 Assert.AreEqual(JsonToken.StartObject, tokenizer.Next()); in ObjectDepth()
92 Assert.AreEqual(1, tokenizer.ObjectDepth); in ObjectDepth()
93 Assert.AreEqual(JsonToken.Name("foo"), tokenizer.Next()); in ObjectDepth()
94 Assert.AreEqual(1, tokenizer.ObjectDepth); in ObjectDepth()
95 Assert.AreEqual(JsonToken.StartObject, tokenizer.Next()); in ObjectDepth()
96 Assert.AreEqual(2, tokenizer.ObjectDepth); in ObjectDepth()
97 Assert.AreEqual(JsonToken.Name("x"), tokenizer.Next()); in ObjectDepth()
98 Assert.AreEqual(2, tokenizer.ObjectDepth); in ObjectDepth()
[all …]
/external/protobuf/csharp/src/Google.Protobuf.Test/
DJsonTokenizerTest.cs88 var tokenizer = JsonTokenizer.FromTextReader(new StringReader(json)); in ObjectDepth()
90 Assert.AreEqual(0, tokenizer.ObjectDepth); in ObjectDepth()
91 Assert.AreEqual(JsonToken.StartObject, tokenizer.Next()); in ObjectDepth()
92 Assert.AreEqual(1, tokenizer.ObjectDepth); in ObjectDepth()
93 Assert.AreEqual(JsonToken.Name("foo"), tokenizer.Next()); in ObjectDepth()
94 Assert.AreEqual(1, tokenizer.ObjectDepth); in ObjectDepth()
95 Assert.AreEqual(JsonToken.StartObject, tokenizer.Next()); in ObjectDepth()
96 Assert.AreEqual(2, tokenizer.ObjectDepth); in ObjectDepth()
97 Assert.AreEqual(JsonToken.Name("x"), tokenizer.Next()); in ObjectDepth()
98 Assert.AreEqual(2, tokenizer.ObjectDepth); in ObjectDepth()
[all …]
/external/cronet/tot/third_party/protobuf/csharp/src/Google.Protobuf.Test/
DJsonTokenizerTest.cs88 var tokenizer = JsonTokenizer.FromTextReader(new StringReader(json)); in ObjectDepth()
90 Assert.AreEqual(0, tokenizer.ObjectDepth); in ObjectDepth()
91 Assert.AreEqual(JsonToken.StartObject, tokenizer.Next()); in ObjectDepth()
92 Assert.AreEqual(1, tokenizer.ObjectDepth); in ObjectDepth()
93 Assert.AreEqual(JsonToken.Name("foo"), tokenizer.Next()); in ObjectDepth()
94 Assert.AreEqual(1, tokenizer.ObjectDepth); in ObjectDepth()
95 Assert.AreEqual(JsonToken.StartObject, tokenizer.Next()); in ObjectDepth()
96 Assert.AreEqual(2, tokenizer.ObjectDepth); in ObjectDepth()
97 Assert.AreEqual(JsonToken.Name("x"), tokenizer.Next()); in ObjectDepth()
98 Assert.AreEqual(2, tokenizer.ObjectDepth); in ObjectDepth()
[all …]
/external/doclava/src/com/google/doclava/apicheck/
DApiFile.java70 final Tokenizer tokenizer = new Tokenizer(filename, (new String(buf, 0, size)).toCharArray()); in parseApi() local
74 String token = tokenizer.getToken(); in parseApi()
79 parsePackage(api, tokenizer); in parseApi()
81 throw new ApiParseException("expected package got " + token, tokenizer.getLine()); in parseApi()
91 private static void parsePackage(ApiInfo api, Tokenizer tokenizer) in parsePackage() argument
97 token = tokenizer.requireToken(); in parsePackage()
98 assertIdent(tokenizer, token); in parsePackage()
100 pkg = new PackageInfo(name, tokenizer.pos()); in parsePackage()
101 token = tokenizer.requireToken(); in parsePackage()
103 throw new ApiParseException("expected '{' got " + token, tokenizer.getLine()); in parsePackage()
[all …]
/external/protobuf/csharp/compatibility_tests/v3.0.0/src/Google.Protobuf.Test/
DJsonTokenizerTest.cs88 var tokenizer = JsonTokenizer.FromTextReader(new StringReader(json)); in ObjectDepth()
90 Assert.AreEqual(0, tokenizer.ObjectDepth); in ObjectDepth()
91 Assert.AreEqual(JsonToken.StartObject, tokenizer.Next()); in ObjectDepth()
92 Assert.AreEqual(1, tokenizer.ObjectDepth); in ObjectDepth()
93 Assert.AreEqual(JsonToken.Name("foo"), tokenizer.Next()); in ObjectDepth()
94 Assert.AreEqual(1, tokenizer.ObjectDepth); in ObjectDepth()
95 Assert.AreEqual(JsonToken.StartObject, tokenizer.Next()); in ObjectDepth()
96 Assert.AreEqual(2, tokenizer.ObjectDepth); in ObjectDepth()
97 Assert.AreEqual(JsonToken.Name("x"), tokenizer.Next()); in ObjectDepth()
98 Assert.AreEqual(2, tokenizer.ObjectDepth); in ObjectDepth()
[all …]
/external/cronet/stable/third_party/protobuf/csharp/compatibility_tests/v3.0.0/src/Google.Protobuf.Test/
DJsonTokenizerTest.cs88 var tokenizer = JsonTokenizer.FromTextReader(new StringReader(json)); in ObjectDepth()
90 Assert.AreEqual(0, tokenizer.ObjectDepth); in ObjectDepth()
91 Assert.AreEqual(JsonToken.StartObject, tokenizer.Next()); in ObjectDepth()
92 Assert.AreEqual(1, tokenizer.ObjectDepth); in ObjectDepth()
93 Assert.AreEqual(JsonToken.Name("foo"), tokenizer.Next()); in ObjectDepth()
94 Assert.AreEqual(1, tokenizer.ObjectDepth); in ObjectDepth()
95 Assert.AreEqual(JsonToken.StartObject, tokenizer.Next()); in ObjectDepth()
96 Assert.AreEqual(2, tokenizer.ObjectDepth); in ObjectDepth()
97 Assert.AreEqual(JsonToken.Name("x"), tokenizer.Next()); in ObjectDepth()
98 Assert.AreEqual(2, tokenizer.ObjectDepth); in ObjectDepth()
[all …]
/external/cronet/tot/third_party/protobuf/csharp/compatibility_tests/v3.0.0/src/Google.Protobuf.Test/
DJsonTokenizerTest.cs88 var tokenizer = JsonTokenizer.FromTextReader(new StringReader(json)); in ObjectDepth()
90 Assert.AreEqual(0, tokenizer.ObjectDepth); in ObjectDepth()
91 Assert.AreEqual(JsonToken.StartObject, tokenizer.Next()); in ObjectDepth()
92 Assert.AreEqual(1, tokenizer.ObjectDepth); in ObjectDepth()
93 Assert.AreEqual(JsonToken.Name("foo"), tokenizer.Next()); in ObjectDepth()
94 Assert.AreEqual(1, tokenizer.ObjectDepth); in ObjectDepth()
95 Assert.AreEqual(JsonToken.StartObject, tokenizer.Next()); in ObjectDepth()
96 Assert.AreEqual(2, tokenizer.ObjectDepth); in ObjectDepth()
97 Assert.AreEqual(JsonToken.Name("x"), tokenizer.Next()); in ObjectDepth()
98 Assert.AreEqual(2, tokenizer.ObjectDepth); in ObjectDepth()
[all …]
/external/libtextclassifier/native/utils/
Dbert_tokenizer_test.cc32 void AssertTokenizerResults(std::unique_ptr<BertTokenizer> tokenizer) { in AssertTokenizerResults() argument
33 auto results = tokenizer->Tokenize("i'm question"); in AssertTokenizerResults()
41 auto tokenizer = in TEST() local
44 AssertTokenizerResults(std::move(tokenizer)); in TEST()
48 auto tokenizer = in TEST() local
51 AssertTokenizerResults(std::move(tokenizer)); in TEST()
60 auto tokenizer = std::make_unique<BertTokenizer>(vocab); in TEST() local
62 AssertTokenizerResults(std::move(tokenizer)); in TEST()
66 auto tokenizer = in TEST() local
69 auto results = tokenizer->Tokenize("i'm questionansweraskask"); in TEST()
[all …]
/external/rust/android-crates-io/crates/protobuf-parse/src/pure/
Dparser.rs9 use protobuf_support::lexer::tokenizer::Tokenizer;
10 use protobuf_support::lexer::tokenizer::TokenizerError;
183 pub tokenizer: Tokenizer<'a>, field
296 tokenizer: Tokenizer::new(input, ParserLanguage::Proto), in new()
307 if self.tokenizer.next_symbol_if_eq('.')? { in next_full_ident()
310 full_ident.push_str(&self.tokenizer.next_ident()?); in next_full_ident()
311 while self.tokenizer.next_symbol_if_eq('.')? { in next_full_ident()
313 full_ident.push_str(&self.tokenizer.next_ident()?); in next_full_ident()
321 full_ident.push_str(&self.tokenizer.next_ident()?); in next_full_ident_rel()
322 while self.tokenizer.next_symbol_if_eq('.')? { in next_full_ident_rel()
[all …]
/external/cronet/stable/third_party/protobuf/src/google/protobuf/io/
Dtokenizer_unittest.cc254 Tokenizer tokenizer(&input, &error_collector); in TEST_2D() local
257 EXPECT_EQ(Tokenizer::TYPE_START, tokenizer.current().type); in TEST_2D()
258 EXPECT_EQ("", tokenizer.current().text); in TEST_2D()
259 EXPECT_EQ(0, tokenizer.current().line); in TEST_2D()
260 EXPECT_EQ(0, tokenizer.current().column); in TEST_2D()
261 EXPECT_EQ(0, tokenizer.current().end_column); in TEST_2D()
264 ASSERT_TRUE(tokenizer.Next()); in TEST_2D()
267 EXPECT_EQ(kSimpleTokenCases_case.type, tokenizer.current().type); in TEST_2D()
269 EXPECT_EQ(kSimpleTokenCases_case.input, tokenizer.current().text); in TEST_2D()
271 EXPECT_EQ(0, tokenizer.current().line); in TEST_2D()
[all …]
/external/cronet/tot/third_party/protobuf/src/google/protobuf/io/
Dtokenizer_unittest.cc254 Tokenizer tokenizer(&input, &error_collector); in TEST_2D() local
257 EXPECT_EQ(Tokenizer::TYPE_START, tokenizer.current().type); in TEST_2D()
258 EXPECT_EQ("", tokenizer.current().text); in TEST_2D()
259 EXPECT_EQ(0, tokenizer.current().line); in TEST_2D()
260 EXPECT_EQ(0, tokenizer.current().column); in TEST_2D()
261 EXPECT_EQ(0, tokenizer.current().end_column); in TEST_2D()
264 ASSERT_TRUE(tokenizer.Next()); in TEST_2D()
267 EXPECT_EQ(kSimpleTokenCases_case.type, tokenizer.current().type); in TEST_2D()
269 EXPECT_EQ(kSimpleTokenCases_case.input, tokenizer.current().text); in TEST_2D()
271 EXPECT_EQ(0, tokenizer.current().line); in TEST_2D()
[all …]
/external/protobuf/src/google/protobuf/io/
Dtokenizer_unittest.cc254 Tokenizer tokenizer(&input, &error_collector); in TEST_2D() local
257 EXPECT_EQ(Tokenizer::TYPE_START, tokenizer.current().type); in TEST_2D()
258 EXPECT_EQ("", tokenizer.current().text); in TEST_2D()
259 EXPECT_EQ(0, tokenizer.current().line); in TEST_2D()
260 EXPECT_EQ(0, tokenizer.current().column); in TEST_2D()
261 EXPECT_EQ(0, tokenizer.current().end_column); in TEST_2D()
264 ASSERT_TRUE(tokenizer.Next()); in TEST_2D()
267 EXPECT_EQ(kSimpleTokenCases_case.type, tokenizer.current().type); in TEST_2D()
269 EXPECT_EQ(kSimpleTokenCases_case.input, tokenizer.current().text); in TEST_2D()
271 EXPECT_EQ(0, tokenizer.current().line); in TEST_2D()
[all …]
/external/perfetto/src/protozero/filtering/
Dmessage_tokenizer_unittest.cc70 MessageTokenizer tokenizer; in TEST() local
78 auto token = tokenizer.Push(octet); in TEST()
86 EXPECT_TRUE(tokenizer.idle()); in TEST()
123 MessageTokenizer tokenizer; in TEST() local
126 auto token = tokenizer.Push(octet); in TEST()
130 EXPECT_TRUE(tokenizer.idle()); in TEST()
147 MessageTokenizer tokenizer; in TEST() local
148 EXPECT_FALSE(tokenizer.Push(0x08).valid()); in TEST()
150 EXPECT_FALSE(tokenizer.Push(0xff).valid()); in TEST()
151 EXPECT_FALSE(tokenizer.Push(0x0).valid()); in TEST()
[all …]
/external/cronet/tot/third_party/protobuf/python/google/protobuf/
Dtext_format.py862 tokenizer = Tokenizer(str_lines)
863 while not tokenizer.AtEnd():
864 self._MergeField(tokenizer, message)
866 def _MergeField(self, tokenizer, message): argument
878 tokenizer.TryConsume('[')):
879 type_url_prefix, packed_type_name = self._ConsumeAnyTypeUrl(tokenizer)
880 tokenizer.Consume(']')
881 tokenizer.TryConsume(':')
882 if tokenizer.TryConsume('<'):
885 tokenizer.Consume('{')
[all …]
/external/protobuf/python/google/protobuf/
Dtext_format.py862 tokenizer = Tokenizer(str_lines)
863 while not tokenizer.AtEnd():
864 self._MergeField(tokenizer, message)
866 def _MergeField(self, tokenizer, message): argument
878 tokenizer.TryConsume('[')):
879 type_url_prefix, packed_type_name = self._ConsumeAnyTypeUrl(tokenizer)
880 tokenizer.Consume(']')
881 tokenizer.TryConsume(':')
882 if tokenizer.TryConsume('<'):
885 tokenizer.Consume('{')
[all …]
/external/cronet/stable/third_party/protobuf/python/google/protobuf/
Dtext_format.py862 tokenizer = Tokenizer(str_lines)
863 while not tokenizer.AtEnd():
864 self._MergeField(tokenizer, message)
866 def _MergeField(self, tokenizer, message): argument
878 tokenizer.TryConsume('[')):
879 type_url_prefix, packed_type_name = self._ConsumeAnyTypeUrl(tokenizer)
880 tokenizer.Consume(']')
881 tokenizer.TryConsume(':')
882 if tokenizer.TryConsume('<'):
885 tokenizer.Consume('{')
[all …]
/external/apache-xml/src/main/java/org/apache/xml/utils/
DStylesheetPIHandler.java152 StringTokenizer tokenizer = new StringTokenizer(data, " \t=\n", true); in processingInstruction() local
157 while (tokenizer.hasMoreTokens()) in processingInstruction()
160 token = tokenizer.nextToken(); in processingInstruction()
163 if (tokenizer.hasMoreTokens() && in processingInstruction()
170 token = tokenizer.nextToken(); in processingInstruction()
171 while (tokenizer.hasMoreTokens() && in processingInstruction()
173 token = tokenizer.nextToken(); in processingInstruction()
179 token = tokenizer.nextToken(); in processingInstruction()
180 while (tokenizer.hasMoreTokens() && in processingInstruction()
182 token = tokenizer.nextToken(); in processingInstruction()
[all …]
/external/parameter-framework/upstream/test/tokenizer/
DTest.cpp49 Tokenizer tokenizer("a bcd ef"); variable
53 CHECK(tokenizer.split() == expected);
58 Tokenizer tokenizer(""); variable
62 CHECK(tokenizer.split() == expected);
67 Tokenizer tokenizer(" a \n\t bc "); variable
71 CHECK(tokenizer.split() == expected);
77 Tokenizer tokenizer("/a/bcd/ef g/h/", "/"); variable
81 CHECK(tokenizer.split() == expected);
88 Tokenizer tokenizer("", Tokenizer::defaultDelimiters, false); variable
92 CHECK(tokenizer.split() == expected);
[all …]
/external/cronet/tot/third_party/protobuf/csharp/src/Google.Protobuf/
DJsonParser.cs76 …{ Timestamp.Descriptor.FullName, (parser, message, tokenizer) => MergeTimestamp(message, tokenizer…
77 …{ Duration.Descriptor.FullName, (parser, message, tokenizer) => MergeDuration(message, tokenizer.N…
78 …{ Value.Descriptor.FullName, (parser, message, tokenizer) => parser.MergeStructValue(message, toke…
79 { ListValue.Descriptor.FullName, (parser, message, tokenizer) =>
80 …r.MergeRepeatedField(message, message.Descriptor.Fields[ListValue.ValuesFieldNumber], tokenizer) },
81 …{ Struct.Descriptor.FullName, (parser, message, tokenizer) => parser.MergeStruct(message, tokenize…
82 … { Any.Descriptor.FullName, (parser, message, tokenizer) => parser.MergeAny(message, tokenizer) },
83 …{ FieldMask.Descriptor.FullName, (parser, message, tokenizer) => MergeFieldMask(message, tokenizer…
97 …private static void MergeWrapperField(JsonParser parser, IMessage message, JsonTokenizer tokenizer) in MergeWrapperField() argument
99 …geField(message, message.Descriptor.Fields[WrappersReflection.WrapperValueFieldNumber], tokenizer); in MergeWrapperField()
[all …]
/external/protobuf/csharp/src/Google.Protobuf/
DJsonParser.cs76 …{ Timestamp.Descriptor.FullName, (parser, message, tokenizer) => MergeTimestamp(message, tokenizer…
77 …{ Duration.Descriptor.FullName, (parser, message, tokenizer) => MergeDuration(message, tokenizer.N…
78 …{ Value.Descriptor.FullName, (parser, message, tokenizer) => parser.MergeStructValue(message, toke…
79 { ListValue.Descriptor.FullName, (parser, message, tokenizer) =>
80 …r.MergeRepeatedField(message, message.Descriptor.Fields[ListValue.ValuesFieldNumber], tokenizer) },
81 …{ Struct.Descriptor.FullName, (parser, message, tokenizer) => parser.MergeStruct(message, tokenize…
82 … { Any.Descriptor.FullName, (parser, message, tokenizer) => parser.MergeAny(message, tokenizer) },
83 …{ FieldMask.Descriptor.FullName, (parser, message, tokenizer) => MergeFieldMask(message, tokenizer…
97 …private static void MergeWrapperField(JsonParser parser, IMessage message, JsonTokenizer tokenizer) in MergeWrapperField() argument
99 …geField(message, message.Descriptor.Fields[WrappersReflection.WrapperValueFieldNumber], tokenizer); in MergeWrapperField()
[all …]
/external/cronet/stable/third_party/protobuf/csharp/src/Google.Protobuf/
DJsonParser.cs76 …{ Timestamp.Descriptor.FullName, (parser, message, tokenizer) => MergeTimestamp(message, tokenizer…
77 …{ Duration.Descriptor.FullName, (parser, message, tokenizer) => MergeDuration(message, tokenizer.N…
78 …{ Value.Descriptor.FullName, (parser, message, tokenizer) => parser.MergeStructValue(message, toke…
79 { ListValue.Descriptor.FullName, (parser, message, tokenizer) =>
80 …r.MergeRepeatedField(message, message.Descriptor.Fields[ListValue.ValuesFieldNumber], tokenizer) },
81 …{ Struct.Descriptor.FullName, (parser, message, tokenizer) => parser.MergeStruct(message, tokenize…
82 … { Any.Descriptor.FullName, (parser, message, tokenizer) => parser.MergeAny(message, tokenizer) },
83 …{ FieldMask.Descriptor.FullName, (parser, message, tokenizer) => MergeFieldMask(message, tokenizer…
97 …private static void MergeWrapperField(JsonParser parser, IMessage message, JsonTokenizer tokenizer) in MergeWrapperField() argument
99 …geField(message, message.Descriptor.Fields[WrappersReflection.WrapperValueFieldNumber], tokenizer); in MergeWrapperField()
[all …]
/external/deqp-deps/amber/src/vkscript/
Dparser.cc43 std::string Parser::make_error(const Tokenizer& tokenizer, in make_error() argument
45 return std::to_string(tokenizer.GetCurrentLine()) + ": " + err; in make_error()
152 Tokenizer tokenizer(section.contents); in ProcessRequireBlock() local
153 tokenizer.SetCurrentLine(section.starting_line_number + 1); in ProcessRequireBlock()
155 for (auto token = tokenizer.NextToken(); !token->IsEOS(); in ProcessRequireBlock()
156 token = tokenizer.NextToken()) { in ProcessRequireBlock()
161 tokenizer, in ProcessRequireBlock()
169 token = tokenizer.NextToken(); in ProcessRequireBlock()
171 return Result(make_error(tokenizer, "Missing framebuffer format")); in ProcessRequireBlock()
177 make_error(tokenizer, "Failed to parse framebuffer format: " + in ProcessRequireBlock()
[all …]
/external/jacoco/org.jacoco.core.test/src/org/jacoco/core/test/validation/
DStatementParser.java67 private final StreamTokenizer tokenizer; field in StatementParser
74 tokenizer = new StreamTokenizer(new StringReader(source)); in StatementParser()
75 tokenizer.resetSyntax(); in StatementParser()
76 tokenizer.whitespaceChars(' ', ' '); in StatementParser()
77 tokenizer.whitespaceChars('\t', '\t'); in StatementParser()
78 tokenizer.wordChars('a', 'z'); in StatementParser()
79 tokenizer.wordChars('A', 'Z'); in StatementParser()
80 tokenizer.quoteChar('"'); in StatementParser()
81 tokenizer.parseNumbers(); in StatementParser()
106 return Integer.valueOf((int) tokenizer.nval); in argument()
[all …]

12345678910>>...24