Home
last modified time | relevance | path

Searched refs:tokenizer (Results 1 – 25 of 191) sorted by relevance

12345678

/third_party/protobuf/csharp/src/Google.Protobuf.Test/
DJsonTokenizerTest.cs88 var tokenizer = JsonTokenizer.FromTextReader(new StringReader(json)); in ObjectDepth()
90 Assert.AreEqual(0, tokenizer.ObjectDepth); in ObjectDepth()
91 Assert.AreEqual(JsonToken.StartObject, tokenizer.Next()); in ObjectDepth()
92 Assert.AreEqual(1, tokenizer.ObjectDepth); in ObjectDepth()
93 Assert.AreEqual(JsonToken.Name("foo"), tokenizer.Next()); in ObjectDepth()
94 Assert.AreEqual(1, tokenizer.ObjectDepth); in ObjectDepth()
95 Assert.AreEqual(JsonToken.StartObject, tokenizer.Next()); in ObjectDepth()
96 Assert.AreEqual(2, tokenizer.ObjectDepth); in ObjectDepth()
97 Assert.AreEqual(JsonToken.Name("x"), tokenizer.Next()); in ObjectDepth()
98 Assert.AreEqual(2, tokenizer.ObjectDepth); in ObjectDepth()
[all …]
/third_party/protobuf/csharp/compatibility_tests/v3.0.0/src/Google.Protobuf.Test/
DJsonTokenizerTest.cs88 var tokenizer = JsonTokenizer.FromTextReader(new StringReader(json)); in ObjectDepth()
90 Assert.AreEqual(0, tokenizer.ObjectDepth); in ObjectDepth()
91 Assert.AreEqual(JsonToken.StartObject, tokenizer.Next()); in ObjectDepth()
92 Assert.AreEqual(1, tokenizer.ObjectDepth); in ObjectDepth()
93 Assert.AreEqual(JsonToken.Name("foo"), tokenizer.Next()); in ObjectDepth()
94 Assert.AreEqual(1, tokenizer.ObjectDepth); in ObjectDepth()
95 Assert.AreEqual(JsonToken.StartObject, tokenizer.Next()); in ObjectDepth()
96 Assert.AreEqual(2, tokenizer.ObjectDepth); in ObjectDepth()
97 Assert.AreEqual(JsonToken.Name("x"), tokenizer.Next()); in ObjectDepth()
98 Assert.AreEqual(2, tokenizer.ObjectDepth); in ObjectDepth()
[all …]
/third_party/protobuf/src/google/protobuf/io/
Dtokenizer_unittest.cc253 Tokenizer tokenizer(&input, &error_collector); in TEST_2D() local
256 EXPECT_EQ(Tokenizer::TYPE_START, tokenizer.current().type); in TEST_2D()
257 EXPECT_EQ("", tokenizer.current().text); in TEST_2D()
258 EXPECT_EQ(0, tokenizer.current().line); in TEST_2D()
259 EXPECT_EQ(0, tokenizer.current().column); in TEST_2D()
260 EXPECT_EQ(0, tokenizer.current().end_column); in TEST_2D()
263 ASSERT_TRUE(tokenizer.Next()); in TEST_2D()
266 EXPECT_EQ(kSimpleTokenCases_case.type, tokenizer.current().type); in TEST_2D()
268 EXPECT_EQ(kSimpleTokenCases_case.input, tokenizer.current().text); in TEST_2D()
270 EXPECT_EQ(0, tokenizer.current().line); in TEST_2D()
[all …]
/third_party/protobuf/python/google/protobuf/
Dtext_format.py847 tokenizer = Tokenizer(str_lines)
848 while not tokenizer.AtEnd():
849 self._MergeField(tokenizer, message)
851 def _MergeField(self, tokenizer, message): argument
863 tokenizer.TryConsume('[')):
864 type_url_prefix, packed_type_name = self._ConsumeAnyTypeUrl(tokenizer)
865 tokenizer.Consume(']')
866 tokenizer.TryConsume(':')
867 if tokenizer.TryConsume('<'):
870 tokenizer.Consume('{')
[all …]
/third_party/protobuf/csharp/src/Google.Protobuf/
DJsonParser.cs76 …{ Timestamp.Descriptor.FullName, (parser, message, tokenizer) => MergeTimestamp(message, tokenizer…
77 …{ Duration.Descriptor.FullName, (parser, message, tokenizer) => MergeDuration(message, tokenizer.N…
78 …{ Value.Descriptor.FullName, (parser, message, tokenizer) => parser.MergeStructValue(message, toke…
79 { ListValue.Descriptor.FullName, (parser, message, tokenizer) =>
80 …r.MergeRepeatedField(message, message.Descriptor.Fields[ListValue.ValuesFieldNumber], tokenizer) },
81 …{ Struct.Descriptor.FullName, (parser, message, tokenizer) => parser.MergeStruct(message, tokenize…
82 … { Any.Descriptor.FullName, (parser, message, tokenizer) => parser.MergeAny(message, tokenizer) },
83 …{ FieldMask.Descriptor.FullName, (parser, message, tokenizer) => MergeFieldMask(message, tokenizer…
97 …private static void MergeWrapperField(JsonParser parser, IMessage message, JsonTokenizer tokenizer) in MergeWrapperField() argument
99 …geField(message, message.Descriptor.Fields[WrappersReflection.WrapperValueFieldNumber], tokenizer); in MergeWrapperField()
[all …]
/third_party/protobuf/python/google/protobuf/internal/
Dtext_format_test.py1892 tokenizer = text_format.Tokenizer(text.splitlines())
1893 methods = [(tokenizer.ConsumeIdentifier, 'identifier1'), ':',
1894 (tokenizer.ConsumeString, 'string1'),
1895 (tokenizer.ConsumeIdentifier, 'identifier2'), ':',
1896 (tokenizer.ConsumeInteger, 123),
1897 (tokenizer.ConsumeIdentifier, 'identifier3'), ':',
1898 (tokenizer.ConsumeString, 'string'),
1899 (tokenizer.ConsumeIdentifier, 'identifiER_4'), ':',
1900 (tokenizer.ConsumeFloat, 1.1e+2),
1901 (tokenizer.ConsumeIdentifier, 'ID5'), ':',
[all …]
/third_party/vk-gl-cts/framework/opengl/
DgluVarTypeUtil.cpp92 VarTokenizer tokenizer(nameWithPath); in parseVariableName() local
93 TCU_CHECK(tokenizer.getToken() == VarTokenizer::TOKEN_IDENTIFIER); in parseVariableName()
94 return tokenizer.getIdentifier(); in parseVariableName()
99 VarTokenizer tokenizer(nameWithPath); in parseTypePath() local
101 if (tokenizer.getToken() == VarTokenizer::TOKEN_IDENTIFIER) in parseTypePath()
102 tokenizer.advance(); in parseTypePath()
105 while (tokenizer.getToken() != VarTokenizer::TOKEN_END) in parseTypePath()
109 if (tokenizer.getToken() == VarTokenizer::TOKEN_PERIOD) in parseTypePath()
111 tokenizer.advance(); in parseTypePath()
112 TCU_CHECK(tokenizer.getToken() == VarTokenizer::TOKEN_IDENTIFIER); in parseTypePath()
[all …]
/third_party/protobuf/java/core/src/main/java/com/google/protobuf/
DTextFormat.java1753 final Tokenizer tokenizer = new Tokenizer(input); in merge() local
1758 while (!tokenizer.atEnd()) { in merge()
1759 mergeField(tokenizer, extensionRegistry, target, unknownFields); in merge()
1768 final Tokenizer tokenizer, in mergeField() argument
1774 tokenizer, in mergeField()
1783 final Tokenizer tokenizer, in mergeField() argument
1790 int startLine = tokenizer.getLine(); in mergeField()
1791 int startColumn = tokenizer.getColumn(); in mergeField()
1795 if ("google.protobuf.Any".equals(type.getFullName()) && tokenizer.tryConsume("[")) { in mergeField()
1796 mergeAnyFieldValue(tokenizer, extensionRegistry, target, parseTreeBuilder, unknownFields, in mergeField()
[all …]
/third_party/icu/ohos_icu4j/src/main/tests/ohos/global/icu/dev/test/util/
DStringTokenizerTest.java161 StringTokenizer tokenizer = new StringTokenizer(str, delimiter); in TestSupplementary() local
162 if (!tokenizer.nextElement().equals(expected[0])) { in TestSupplementary()
165 if (tokenizer.hasMoreElements()) { in TestSupplementary()
171 tokenizer = new StringTokenizer(str, delimiter); in TestSupplementary()
173 while (tokenizer.hasMoreElements()) { in TestSupplementary()
174 if (!tokenizer.nextElement().equals(expected1[i ++])) { in TestSupplementary()
178 if (tokenizer.hasMoreElements()) { in TestSupplementary()
185 tokenizer = new StringTokenizer(str, delimiter); in TestSupplementary()
186 if (!tokenizer.nextElement().equals(expected2[0])) { in TestSupplementary()
189 if (tokenizer.hasMoreElements()) { in TestSupplementary()
[all …]
/third_party/icu/icu4j/main/tests/core/src/com/ibm/icu/dev/test/util/
DStringTokenizerTest.java158 StringTokenizer tokenizer = new StringTokenizer(str, delimiter); in TestSupplementary() local
159 if (!tokenizer.nextElement().equals(expected[0])) { in TestSupplementary()
162 if (tokenizer.hasMoreElements()) { in TestSupplementary()
168 tokenizer = new StringTokenizer(str, delimiter); in TestSupplementary()
170 while (tokenizer.hasMoreElements()) { in TestSupplementary()
171 if (!tokenizer.nextElement().equals(expected1[i ++])) { in TestSupplementary()
175 if (tokenizer.hasMoreElements()) { in TestSupplementary()
182 tokenizer = new StringTokenizer(str, delimiter); in TestSupplementary()
183 if (!tokenizer.nextElement().equals(expected2[0])) { in TestSupplementary()
186 if (tokenizer.hasMoreElements()) { in TestSupplementary()
[all …]
/third_party/boost/libs/tokenizer/
DREADME.md4 # [Boost.Tokenizer](http://boost.org/libs/tokenizer)
23 …tokenizer/tree/master) | [![Build Status](https://travis-ci.org/boostorg/tokenizer.svg?branch=mast…
24 …tokenizer/tree/develop) | [![Build Status](https://travis-ci.org/boostorg/tokenizer.svg?branch=dev…
36 #include <boost/tokenizer.hpp>
41 typedef boost::tokenizer<> Tok;
55 #include <boost/tokenizer.hpp>
60 boost::tokenizer<> tok(s);
69 Documentation can be found at [Boost.Tokenizer](http://boost.org/libs/tokenizer)
72 [Boost.Tokenizer](http://theboostcpplibraries.com/boost.tokenizer) Chapter 10 at theboostcpplibrari…
79 …g/trac/boost/query?status=assigned&status=new&status=reopened&component=tokenizer&col=id&col=summa…
[all …]
/third_party/boost/boost/geometry/io/wkt/
Dread.hpp113 typedef boost::tokenizer<boost::char_separator<char> > tokenizer; typedef
120 static inline void apply(tokenizer::iterator& it, in apply()
121 tokenizer::iterator const& end, in apply()
163 static inline void apply(tokenizer::iterator&, in apply()
164 tokenizer::iterator const&, in apply()
223 static inline void apply(tokenizer::iterator& it, in apply()
224 tokenizer::iterator const& end, in apply()
330 static inline void apply(tokenizer::iterator& it, in apply()
331 tokenizer::iterator const& end, in apply()
367 static inline void apply(tokenizer::iterator& it, in apply()
[all …]
/third_party/mindspore/tests/ut/python/dataset/
Dtest_sentencepiece_tokenizer.py27 tokenizer = text.SentencePieceTokenizer(vocab, out_type=SPieceTokenizerOutType.STRING)
29 assert np.array_equal(tokenizer(data), ['▁', '12', '3'])
34 tokenizer = text.SentencePieceTokenizer(vocab, out_type=SPieceTokenizerOutType.STRING)
36 dataset = dataset.map(operations=tokenizer)
46 tokenizer = text.SentencePieceTokenizer(vocab, out_type=SPieceTokenizerOutType.STRING)
48 dataset = dataset.map(operations=tokenizer)
58 tokenizer = text.SentencePieceTokenizer(vocab, out_type=SPieceTokenizerOutType.STRING)
60 dataset = dataset.map(operations=tokenizer)
71 tokenizer = text.SentencePieceTokenizer(vocab, out_type=SPieceTokenizerOutType.STRING)
73 dataset = dataset.map(operations=tokenizer)
[all …]
Dtest_text_tokenizer.py45 tokenizer = text.UnicodeCharTokenizer()
46 dataset = dataset.map(operations=tokenizer)
61 tokenizer = text.UnicodeCharTokenizer(with_offsets=True)
62 dataset = dataset.map(operations=tokenizer, input_columns=['text'],
90 tokenizer = text.WhitespaceTokenizer()
91 dataset = dataset.map(operations=tokenizer)
109 tokenizer = text.WhitespaceTokenizer(with_offsets=True)
110 dataset = dataset.map(operations=tokenizer, input_columns=['text'],
137 tokenizer = text.UnicodeScriptTokenizer(keep_whitespace=False)
138 dataset = dataset.map(operations=tokenizer)
[all …]
/third_party/node/tools/inspector_protocol/encoding/
Dencoding_test.cc173 CBORTokenizer tokenizer(SpanFrom(encoded)); in TEST() local
174 EXPECT_EQ(CBORTokenTag::INT32, tokenizer.TokenTag()); in TEST()
175 EXPECT_EQ(23, tokenizer.GetInt32()); in TEST()
176 tokenizer.Next(); in TEST()
177 EXPECT_EQ(CBORTokenTag::DONE, tokenizer.TokenTag()); in TEST()
191 CBORTokenizer tokenizer(SpanFrom(encoded)); in TEST() local
192 EXPECT_EQ(CBORTokenTag::INT32, tokenizer.TokenTag()); in TEST()
193 EXPECT_EQ(42, tokenizer.GetInt32()); in TEST()
194 tokenizer.Next(); in TEST()
195 EXPECT_EQ(CBORTokenTag::DONE, tokenizer.TokenTag()); in TEST()
[all …]
Dencoding.cc926 CBORTokenizer* tokenizer,
929 CBORTokenizer* tokenizer,
932 CBORTokenizer* tokenizer,
935 void ParseUTF16String(CBORTokenizer* tokenizer, StreamingParserHandler* out) { in ParseUTF16String() argument
937 span<uint8_t> rep = tokenizer->GetString16WireRep(); in ParseUTF16String()
941 tokenizer->Next(); in ParseUTF16String()
944 bool ParseUTF8String(CBORTokenizer* tokenizer, StreamingParserHandler* out) { in ParseUTF8String() argument
945 assert(tokenizer->TokenTag() == CBORTokenTag::STRING8); in ParseUTF8String()
946 out->HandleString8(tokenizer->GetString8()); in ParseUTF8String()
947 tokenizer->Next(); in ParseUTF8String()
[all …]
/third_party/node/tools/inspector_protocol/lib/
DValues_cpp.template97 std::unique_ptr<DictionaryValue> parseMap(int32_t stack_depth, cbor::CBORTokenizer* tokenizer);
98 std::unique_ptr<ListValue> parseArray(int32_t stack_depth, cbor::CBORTokenizer* tokenizer);
99 std::unique_ptr<Value> parseValue(int32_t stack_depth, cbor::CBORTokenizer* tokenizer);
104 std::unique_ptr<ListValue> parseArray(int32_t stack_depth, cbor::CBORTokenizer* tokenizer) {
105 DCHECK(tokenizer->TokenTag() == cbor::CBORTokenTag::ARRAY_START);
106 tokenizer->Next();
108 while (tokenizer->TokenTag() != cbor::CBORTokenTag::STOP) {
110 if (tokenizer->TokenTag() == cbor::CBORTokenTag::DONE) return nullptr;
111 if (tokenizer->TokenTag() == cbor::CBORTokenTag::ERROR_VALUE) return nullptr;
113 auto value = parseValue(stack_depth, tokenizer);
[all …]
/third_party/parse5/packages/parse5/test/
Dlocation-info-tokenizer.test.js88 const tokenizer = new Tokenizer();
91 Mixin.install(tokenizer, LocationInfoTokenizerMixin);
94 tokenizer.write(testCase.htmlChunks[i], i === lastChunkIdx);
98 tokenizer.preprocessor.bufferWaterline = 8;
99 tokenizer.state = testCase.initialMode;
100 tokenizer.lastStartTagName = testCase.lastStartTagName;
102 for (let token = tokenizer.getNextToken(), j = 0; token.type !== Tokenizer.EOF_TOKEN; ) {
119 token = tokenizer.getNextToken();
/third_party/freetype/docs/reference/assets/javascripts/lunr/
Dlunr.th.js17 …tokenizer=t.th.tokenizer:(t.tokenizer&&(t.tokenizer=t.th.tokenizer),this.tokenizerFn&&(this.tokeni… function
Dlunr.ja.js17 …tokenizer=m.ja.tokenizer:(m.tokenizer&&(m.tokenizer=m.ja.tokenizer),this.tokenizerFn&&(this.tokeni… function
/third_party/parse5/packages/parse5-sax-parser/lib/
Dparser-feedback-simulator.js15 constructor(tokenizer) { argument
16 this.tokenizer = tokenizer;
24 const token = this.tokenizer.getNextToken();
57 this.tokenizer.allowCDATA = this.inForeignContent;
66 this.tokenizer.allowCDATA = this.inForeignContent;
72 this.tokenizer.state = Tokenizer.MODE.RCDATA;
74 this.tokenizer.state = Tokenizer.MODE.PLAINTEXT;
76 this.tokenizer.state = Tokenizer.MODE.SCRIPT_DATA;
85 this.tokenizer.state = Tokenizer.MODE.RAWTEXT;
/third_party/boost/libs/coroutine/example/asymmetric/
Dchaining.cpp143 coro_t::pull_type tokenizer(boost::bind(tokenize, _1, boost::ref(reader))); in main() local
144 coro_t::pull_type tracer(boost::bind(trace, _1, boost::ref(tokenizer))); in main()
155 coro_t::pull_type tokenizer(boost::bind(tokenize, _1, boost::ref(reader))); in main() local
156 coro_t::pull_type filter(boost::bind(only_words, _1, boost::ref(tokenizer))); in main()
168 coro_t::pull_type tokenizer(boost::bind(tokenize, _1, boost::ref(reader))); in main() local
169 coro_t::pull_type filter(boost::bind(only_words, _1, boost::ref(tokenizer))); in main()
181 coro_t::pull_type tokenizer(boost::bind(tokenize, _1, boost::ref(reader))); in main() local
182 coro_t::pull_type filter(boost::bind(only_words, _1, boost::ref(tokenizer))); in main()
192 coro_t::pull_type tokenizer(boost::bind(tokenize, _1, boost::ref(reader))); in main() local
197 BOOST_FOREACH(std::string token, tokenizer) in main()
/third_party/boost/libs/tokenizer/example/
Dchar_sep_example_1.cpp19 typedef boost::tokenizer<boost::char_separator<char> > in main()
20 tokenizer; in main() typedef
22 tokenizer tokens(str, sep); in main()
23 for (tokenizer::iterator tok_iter = tokens.begin(); in main()
Dchar_sep_example_2.cpp20 typedef boost::tokenizer<boost::char_separator<char> > in main()
21 tokenizer; in main() typedef
23 tokenizer tokens(str, sep); in main()
24 for (tokenizer::iterator tok_iter = tokens.begin(); in main()
/third_party/parse5/packages/parse5/lib/extensions/location-info/
Dtokenizer-mixin.js8 constructor(tokenizer) { argument
9 super(tokenizer);
11 this.tokenizer = tokenizer;
12 this.posTracker = Mixin.install(tokenizer.preprocessor, PositionTrackingPreprocessorMixin);
33 const currentToken = this.tokenizer.currentToken;
34 const currentAttr = this.tokenizer.currentAttr;

12345678