Home
last modified time | relevance | path

Searched refs:tokenizer (Results 1 – 25 of 162) sorted by relevance

1234567

/external/doclava/src/com/google/doclava/apicheck/
DApiFile.java69 final Tokenizer tokenizer = new Tokenizer(filename, (new String(buf, 0, size)).toCharArray()); in parseApi() local
73 String token = tokenizer.getToken(); in parseApi()
78 parsePackage(api, tokenizer); in parseApi()
80 throw new ApiParseException("expected package got " + token, tokenizer.getLine()); in parseApi()
90 private static void parsePackage(ApiInfo api, Tokenizer tokenizer) in parsePackage() argument
96 token = tokenizer.requireToken(); in parsePackage()
97 assertIdent(tokenizer, token); in parsePackage()
99 pkg = new PackageInfo(name, tokenizer.pos()); in parsePackage()
100 token = tokenizer.requireToken(); in parsePackage()
102 throw new ApiParseException("expected '{' got " + token, tokenizer.getLine()); in parsePackage()
[all …]
/external/protobuf/src/google/protobuf/io/
Dtokenizer_unittest.cc254 Tokenizer tokenizer(&input, &error_collector); in TEST_2D() local
257 EXPECT_EQ(Tokenizer::TYPE_START, tokenizer.current().type); in TEST_2D()
258 EXPECT_EQ("", tokenizer.current().text); in TEST_2D()
259 EXPECT_EQ(0, tokenizer.current().line); in TEST_2D()
260 EXPECT_EQ(0, tokenizer.current().column); in TEST_2D()
261 EXPECT_EQ(0, tokenizer.current().end_column); in TEST_2D()
264 ASSERT_TRUE(tokenizer.Next()); in TEST_2D()
267 EXPECT_EQ(kSimpleTokenCases_case.type, tokenizer.current().type); in TEST_2D()
269 EXPECT_EQ(kSimpleTokenCases_case.input, tokenizer.current().text); in TEST_2D()
271 EXPECT_EQ(0, tokenizer.current().line); in TEST_2D()
[all …]
/external/apache-xml/src/main/java/org/apache/xml/utils/
DStylesheetPIHandler.java152 StringTokenizer tokenizer = new StringTokenizer(data, " \t=\n", true); in processingInstruction() local
157 while (tokenizer.hasMoreTokens()) in processingInstruction()
160 token = tokenizer.nextToken(); in processingInstruction()
163 if (tokenizer.hasMoreTokens() && in processingInstruction()
170 token = tokenizer.nextToken(); in processingInstruction()
171 while (tokenizer.hasMoreTokens() && in processingInstruction()
173 token = tokenizer.nextToken(); in processingInstruction()
179 token = tokenizer.nextToken(); in processingInstruction()
180 while (tokenizer.hasMoreTokens() && in processingInstruction()
182 token = tokenizer.nextToken(); in processingInstruction()
[all …]
/external/protobuf/python/google/protobuf/internal/
Dtext_format_test.py608 tokenizer = text_format._Tokenizer(text.splitlines())
609 methods = [(tokenizer.ConsumeIdentifier, 'identifier1'),
611 (tokenizer.ConsumeString, 'string1'),
612 (tokenizer.ConsumeIdentifier, 'identifier2'),
614 (tokenizer.ConsumeInt32, 123),
615 (tokenizer.ConsumeIdentifier, 'identifier3'),
617 (tokenizer.ConsumeString, 'string'),
618 (tokenizer.ConsumeIdentifier, 'identifiER_4'),
620 (tokenizer.ConsumeFloat, 1.1e+2),
621 (tokenizer.ConsumeIdentifier, 'ID5'),
[all …]
/external/parameter-framework/upstream/test/tokenizer/
DTest.cpp49 Tokenizer tokenizer("a bcd ef"); variable
53 CHECK(tokenizer.split() == expected);
58 Tokenizer tokenizer(""); variable
62 CHECK(tokenizer.split() == expected);
67 Tokenizer tokenizer(" a \n\t bc "); variable
71 CHECK(tokenizer.split() == expected);
77 Tokenizer tokenizer("/a/bcd/ef g/h/", "/"); variable
81 CHECK(tokenizer.split() == expected);
88 Tokenizer tokenizer("", Tokenizer::defaultDelimiters, false); variable
92 CHECK(tokenizer.split() == expected);
[all …]
/external/antlr/antlr-3.4/runtime/CSharp3/Sources/Antlr3.Runtime/Tree/
DTreePatternParser.cs39 protected TreePatternLexer tokenizer; field in Antlr.Runtime.Tree.TreePatternParser
44 … public TreePatternParser( TreePatternLexer tokenizer, TreeWizard wizard, ITreeAdaptor adaptor ) in TreePatternParser() argument
46 this.tokenizer = tokenizer; in TreePatternParser()
49 ttype = tokenizer.NextToken(); // kickstart in TreePatternParser()
75 ttype = tokenizer.NextToken(); in ParseTree()
105 ttype = tokenizer.NextToken(); in ParseTree()
115 ttype = tokenizer.NextToken(); in ParseNode()
120 label = tokenizer.sval.ToString(); in ParseNode()
121 ttype = tokenizer.NextToken(); in ParseNode()
126 ttype = tokenizer.NextToken(); // move to ID following colon in ParseNode()
[all …]
/external/antlr/antlr-3.4/runtime/CSharp2/Sources/Antlr3.Runtime/Antlr.Runtime.Tree/
DTreePatternParser.cs37 protected TreePatternLexer tokenizer; field in Antlr.Runtime.Tree.TreePatternParser
42 … public TreePatternParser(TreePatternLexer tokenizer, TreeWizard wizard, ITreeAdaptor adaptor) { in TreePatternParser() argument
43 this.tokenizer = tokenizer; in TreePatternParser()
46 ttype = tokenizer.NextToken(); // kickstart in TreePatternParser()
66 ttype = tokenizer.NextToken(); in ParseTree()
90 ttype = tokenizer.NextToken(); in ParseTree()
98 ttype = tokenizer.NextToken(); in ParseNode()
102 label = tokenizer.sval.ToString(); in ParseNode()
103 ttype = tokenizer.NextToken(); in ParseNode()
107 ttype = tokenizer.NextToken(); // move to ID following colon in ParseNode()
[all …]
/external/deqp/framework/opengl/
DgluVarTypeUtil.cpp92 VarTokenizer tokenizer(nameWithPath); in parseVariableName() local
93 TCU_CHECK(tokenizer.getToken() == VarTokenizer::TOKEN_IDENTIFIER); in parseVariableName()
94 return tokenizer.getIdentifier(); in parseVariableName()
99 VarTokenizer tokenizer(nameWithPath); in parseTypePath() local
101 if (tokenizer.getToken() == VarTokenizer::TOKEN_IDENTIFIER) in parseTypePath()
102 tokenizer.advance(); in parseTypePath()
105 while (tokenizer.getToken() != VarTokenizer::TOKEN_END) in parseTypePath()
109 if (tokenizer.getToken() == VarTokenizer::TOKEN_PERIOD) in parseTypePath()
111 tokenizer.advance(); in parseTypePath()
112 TCU_CHECK(tokenizer.getToken() == VarTokenizer::TOKEN_IDENTIFIER); in parseTypePath()
[all …]
/external/jacoco/org.jacoco.examples/build/src/main/java/org/jacoco/examples/parser/
DExpressionParser.java30 private final StreamTokenizer tokenizer; field in ExpressionParser
33 tokenizer = new StreamTokenizer(new StringReader(s)); in ExpressionParser()
34 tokenizer.ordinaryChar('('); in ExpressionParser()
35 tokenizer.ordinaryChar(')'); in ExpressionParser()
36 tokenizer.ordinaryChar('+'); in ExpressionParser()
37 tokenizer.ordinaryChar('-'); in ExpressionParser()
38 tokenizer.ordinaryChar('*'); in ExpressionParser()
39 tokenizer.ordinaryChar('/'); in ExpressionParser()
43 tokenizer.nextToken(); in parse()
82 e = new Const(tokenizer.nval); in factor()
[all …]
/external/antlr/antlr-3.4/runtime/Java/src/main/java/org/antlr/runtime/tree/
DTreePatternParser.java34 protected TreePatternLexer tokenizer; field in TreePatternParser
39 public TreePatternParser(TreePatternLexer tokenizer, TreeWizard wizard, TreeAdaptor adaptor) { in TreePatternParser() argument
40 this.tokenizer = tokenizer; in TreePatternParser()
43 ttype = tokenizer.nextToken(); // kickstart in TreePatternParser()
64 ttype = tokenizer.nextToken(); in parseTree()
89 ttype = tokenizer.nextToken(); in parseTree()
97 ttype = tokenizer.nextToken(); in parseNode()
101 label = tokenizer.sval.toString(); in parseNode()
102 ttype = tokenizer.nextToken(); in parseNode()
106 ttype = tokenizer.nextToken(); // move to ID following colon in parseNode()
[all …]
/external/chromium-trace/catapult/third_party/vinn/third_party/parse5/lib/tokenization/
Dlocation_info_mixin.js3 exports.assign = function (tokenizer) { argument
5 var tokenizerProto = Object.getPrototypeOf(tokenizer);
7 tokenizer.tokenStartLoc = -1;
10 tokenizer._attachLocationInfo = function (token) {
18 tokenizer._createStartTagToken = function (tagNameFirstCh) {
23 tokenizer._createEndTagToken = function (tagNameFirstCh) {
28 tokenizer._createCommentToken = function () {
33 tokenizer._createDoctypeToken = function (doctypeNameFirstCh) {
38 tokenizer._createCharacterToken = function (type, ch) {
44 tokenizer._emitCurrentToken = function () {
[all …]
/external/protobuf/java/src/main/java/com/google/protobuf/
DTextFormat.java1296 final Tokenizer tokenizer = new Tokenizer(input); in merge() local
1300 while (!tokenizer.atEnd()) { in merge()
1301 mergeField(tokenizer, extensionRegistry, target); in merge()
1310 private void mergeField(final Tokenizer tokenizer, in mergeField() argument
1318 if (tokenizer.tryConsume("[")) { in mergeField()
1321 new StringBuilder(tokenizer.consumeIdentifier()); in mergeField()
1322 while (tokenizer.tryConsume(".")) { in mergeField()
1324 name.append(tokenizer.consumeIdentifier()); in mergeField()
1332 throw tokenizer.parseExceptionPreviousToken( in mergeField()
1340 throw tokenizer.parseExceptionPreviousToken( in mergeField()
[all …]
/external/icu/icu4j/main/tests/core/src/com/ibm/icu/dev/test/util/
DStringTokenizerTest.java149 StringTokenizer tokenizer = new StringTokenizer(str, delimiter); in TestSupplementary() local
150 if (!tokenizer.nextElement().equals(expected[0])) { in TestSupplementary()
153 if (tokenizer.hasMoreElements()) { in TestSupplementary()
159 tokenizer = new StringTokenizer(str, delimiter); in TestSupplementary()
161 while (tokenizer.hasMoreElements()) { in TestSupplementary()
162 if (!tokenizer.nextElement().equals(expected1[i ++])) { in TestSupplementary()
166 if (tokenizer.hasMoreElements()) { in TestSupplementary()
173 tokenizer = new StringTokenizer(str, delimiter); in TestSupplementary()
174 if (!tokenizer.nextElement().equals(expected2[0])) { in TestSupplementary()
177 if (tokenizer.hasMoreElements()) { in TestSupplementary()
[all …]
/external/icu/android_icu4j/src/main/tests/android/icu/dev/test/util/
DStringTokenizerTest.java153 StringTokenizer tokenizer = new StringTokenizer(str, delimiter); in TestSupplementary() local
154 if (!tokenizer.nextElement().equals(expected[0])) { in TestSupplementary()
157 if (tokenizer.hasMoreElements()) { in TestSupplementary()
163 tokenizer = new StringTokenizer(str, delimiter); in TestSupplementary()
165 while (tokenizer.hasMoreElements()) { in TestSupplementary()
166 if (!tokenizer.nextElement().equals(expected1[i ++])) { in TestSupplementary()
170 if (tokenizer.hasMoreElements()) { in TestSupplementary()
177 tokenizer = new StringTokenizer(str, delimiter); in TestSupplementary()
178 if (!tokenizer.nextElement().equals(expected2[0])) { in TestSupplementary()
181 if (tokenizer.hasMoreElements()) { in TestSupplementary()
[all …]
/external/protobuf/python/google/protobuf/
Dtext_format.py227 tokenizer = _Tokenizer(lines)
228 while not tokenizer.AtEnd():
229 _MergeField(tokenizer, message, allow_multiple_scalars)
302 def _MergeField(tokenizer, message, allow_multiple_scalars): argument
316 if tokenizer.TryConsume('['):
317 name = [tokenizer.ConsumeIdentifier()]
318 while tokenizer.TryConsume('.'):
319 name.append(tokenizer.ConsumeIdentifier())
323 raise tokenizer.ParseErrorPreviousToken(
330 raise tokenizer.ParseErrorPreviousToken(
[all …]
/external/nanopb-c/generator/google/protobuf/
Dtext_format.py156 tokenizer = _Tokenizer(text)
157 while not tokenizer.AtEnd():
158 _MergeField(tokenizer, message)
161 def _MergeField(tokenizer, message): argument
172 if tokenizer.TryConsume('['):
173 name = [tokenizer.ConsumeIdentifier()]
174 while tokenizer.TryConsume('.'):
175 name.append(tokenizer.ConsumeIdentifier())
179 raise tokenizer.ParseErrorPreviousToken(
184 raise tokenizer.ParseErrorPreviousToken(
[all …]
/external/antlr/antlr-3.4/runtime/ObjC/Framework/
DANTLRTreePatternParser.m47 //tokenizer = aTokenizer;
50 //ttype = [tokenizer nextToken]; // kickstart
62 tokenizer = aTokenizer;
63 if ( tokenizer ) [tokenizer retain];
77 if ( tokenizer ) [tokenizer release];
102 ttype = [tokenizer nextToken];
127 ttype = [tokenizer nextToken];
137 ttype = [tokenizer nextToken];
141 label = [tokenizer toString];
142 ttype = [tokenizer nextToken];
[all …]
/external/antlr/antlr-3.4/runtime/Python/antlr3/
Dtreewizard.py164 def __init__(self, tokenizer, wizard, adaptor): argument
165 self.tokenizer = tokenizer
168 self.ttype = tokenizer.nextToken() # kickstart
189 self.ttype = self.tokenizer.nextToken()
209 self.ttype = self.tokenizer.nextToken()
218 self.ttype = self.tokenizer.nextToken()
222 label = self.tokenizer.sval
223 self.ttype = self.tokenizer.nextToken()
227 self.ttype = self.tokenizer.nextToken() # move to ID following colon
231 self.ttype = self.tokenizer.nextToken()
[all …]
/external/chromium-trace/catapult/third_party/vinn/third_party/parse5/lib/simple_api/
Dtokenizer_proxy.js16 this.tokenizer = new Tokenizer(html, options); property
26 var token = this.tokenizer.getNextToken();
49 this.tokenizer.allowCDATA = this.inForeignContent;
58 this.tokenizer.allowCDATA = this.inForeignContent;
64 this.tokenizer.state = Tokenizer.MODE.RCDATA;
67 this.tokenizer.state = Tokenizer.MODE.PLAINTEXT;
70 this.tokenizer.state = Tokenizer.MODE.SCRIPT_DATA;
74 this.tokenizer.state = Tokenizer.MODE.RAWTEXT;
116 this.tokenizer.state = Tokenizer.MODE.DATA;
/external/libchrome/base/
Dsys_info_chromeos.cc132 StringTokenizer tokenizer(version, "."); in ParseLsbRelease() local
133 if (tokenizer.GetNext()) { in ParseLsbRelease()
134 StringToInt(StringPiece(tokenizer.token_begin(), tokenizer.token_end()), in ParseLsbRelease()
137 if (tokenizer.GetNext()) { in ParseLsbRelease()
138 StringToInt(StringPiece(tokenizer.token_begin(), tokenizer.token_end()), in ParseLsbRelease()
141 if (tokenizer.GetNext()) { in ParseLsbRelease()
142 StringToInt(StringPiece(tokenizer.token_begin(), tokenizer.token_end()), in ParseLsbRelease()
/external/chromium-trace/catapult/third_party/vinn/third_party/parse5/test/fixtures/
Dtokenizer_test.js7 var tokenizer = new Tokenizer(html),
11 tokenizer.state = initialState;
14 tokenizer.lastStartTagName = lastStartTag;
17 nextToken = tokenizer.getNextToken();
237 tokenizer = new Tokenizer(html, {locationInfo: true});
239 tokenizer.state = testCase.initialMode;
240 tokenizer.lastStartTagName = testCase.lastStartTagName;
242 for (var token = tokenizer.getNextToken(), i = 0; token.type !== Tokenizer.EOF_TOKEN;) {
247 token = tokenizer.getNextToken();
/external/apache-xml/src/main/java/org/apache/xalan/templates/
DAVT.java172 StringTokenizer tokenizer = new StringTokenizer(stringedValue, "{}\"\'", in AVT() local
174 int nTokens = tokenizer.countTokens(); in AVT()
199 while (tokenizer.hasMoreTokens()) in AVT()
207 t = tokenizer.nextToken(); in AVT()
228 lookahead = tokenizer.nextToken(); in AVT()
275 lookahead = tokenizer.nextToken(); in AVT()
281 lookahead = tokenizer.nextToken(); in AVT()
286 lookahead = tokenizer.nextToken(); in AVT()
323 lookahead = tokenizer.nextToken(); in AVT()
333 lookahead = tokenizer.nextToken(); in AVT()
[all …]
/external/emma/core/java12/com/vladium/emma/
DAppLoggers.java59 final StringTokenizer tokenizer = new StringTokenizer (_filter, COMMA_DELIMITERS); in create() local
60 if (tokenizer.countTokens () > 0) in create()
62 temp = new HashSet (tokenizer.countTokens ()); in create()
63 while (tokenizer.hasMoreTokens ()) in create()
65 temp.add (tokenizer.nextToken ()); in create()
/external/guava/guava/src/com/google/common/net/
DMediaType.java597 Tokenizer tokenizer = new Tokenizer(input);
599 String type = tokenizer.consumeToken(TOKEN_MATCHER);
600 tokenizer.consumeCharacter('/');
601 String subtype = tokenizer.consumeToken(TOKEN_MATCHER);
603 while (tokenizer.hasMore()) {
604 tokenizer.consumeCharacter(';');
605 tokenizer.consumeTokenIfPresent(LINEAR_WHITE_SPACE);
606 String attribute = tokenizer.consumeToken(TOKEN_MATCHER);
607 tokenizer.consumeCharacter('=');
609 if ('"' == tokenizer.previewChar()) {
[all …]
/external/fonttools/Lib/fontTools/misc/
DpsLib.py153 tokenizer = self.tokenizer = PSTokenizer(data)
154 getnexttoken = tokenizer.getnexttoken
170 tokenizer.close()
171 self.tokenizer = None
173 if self.tokenizer is not None:
176 print(self.tokenizer.buf[self.tokenizer.pos-50:self.tokenizer.pos])
178 print(self.tokenizer.buf[self.tokenizer.pos:self.tokenizer.pos+50])

1234567