1 // Copyright (C) 2015 The Android Open Source Project
2 //
3 // Licensed under the Apache License, Version 2.0 (the "License");
4 // you may not use this file except in compliance with the License.
5 // You may obtain a copy of the License at
6 //
7 // http://www.apache.org/licenses/LICENSE-2.0
8 //
9 // Unless required by applicable law or agreed to in writing, software
10 // distributed under the License is distributed on an "AS IS" BASIS,
11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 // See the License for the specific language governing permissions and
13 // limitations under the License.
14
15 #include "tokenizer.h"
16
17 #include <errno.h>
18 #include <gtest/gtest.h>
19
20 #include <string>
21
22 namespace init {
23
24 #define SETUP_TEST(test_data) \
25 std::string data(test_data); \
26 Tokenizer tokenizer(data); \
27 ASSERT_EQ(Tokenizer::TOK_START, tokenizer.current().type)
28
29 #define ASSERT_TEXT_TOKEN(test_text) \
30 ASSERT_TRUE(tokenizer.Next()); \
31 ASSERT_EQ(test_text, tokenizer.current().text); \
32 ASSERT_EQ(Tokenizer::TOK_TEXT, tokenizer.current().type)
33
34 #define ASSERT_NEWLINE_TOKEN() \
35 ASSERT_TRUE(tokenizer.Next()); \
36 ASSERT_EQ(Tokenizer::TOK_NEWLINE, tokenizer.current().type)
37
TEST(Tokenizer,Empty)38 TEST(Tokenizer, Empty) {
39 SETUP_TEST("");
40 ASSERT_FALSE(tokenizer.Next());
41 }
42
TEST(Tokenizer,Simple)43 TEST(Tokenizer, Simple) {
44 SETUP_TEST("test");
45 ASSERT_TEXT_TOKEN("test");
46 ASSERT_FALSE(tokenizer.Next());
47 }
48
TEST(Tokenizer,LeadingWhiteSpace)49 TEST(Tokenizer, LeadingWhiteSpace) {
50 SETUP_TEST(" \t \r test");
51 ASSERT_TEXT_TOKEN("test");
52 ASSERT_FALSE(tokenizer.Next());
53 }
54
TEST(Tokenizer,TrailingWhiteSpace)55 TEST(Tokenizer, TrailingWhiteSpace) {
56 SETUP_TEST("test \t \r ");
57 ASSERT_TEXT_TOKEN("test");
58 ASSERT_FALSE(tokenizer.Next());
59 }
60
TEST(Tokenizer,WhiteSpace)61 TEST(Tokenizer, WhiteSpace) {
62 SETUP_TEST(" \t \r test \t \r ");
63 ASSERT_TEXT_TOKEN("test");
64
65 ASSERT_FALSE(tokenizer.Next());
66 }
67
TEST(Tokenizer,TwoTokens)68 TEST(Tokenizer, TwoTokens) {
69 SETUP_TEST(" foo bar ");
70 ASSERT_TEXT_TOKEN("foo");
71 ASSERT_TEXT_TOKEN("bar");
72
73 ASSERT_FALSE(tokenizer.Next());
74 }
75
TEST(Tokenizer,MultiToken)76 TEST(Tokenizer, MultiToken) {
77 SETUP_TEST("one two three four five");
78 ASSERT_TEXT_TOKEN("one");
79 ASSERT_TEXT_TOKEN("two");
80 ASSERT_TEXT_TOKEN("three");
81 ASSERT_TEXT_TOKEN("four");
82 ASSERT_TEXT_TOKEN("five");
83
84 ASSERT_FALSE(tokenizer.Next());
85 }
86
TEST(Tokenizer,NewLine)87 TEST(Tokenizer, NewLine) {
88 SETUP_TEST("\n");
89 ASSERT_NEWLINE_TOKEN();
90
91 ASSERT_FALSE(tokenizer.Next());
92 }
93
TEST(Tokenizer,TextNewLine)94 TEST(Tokenizer, TextNewLine) {
95 SETUP_TEST("test\n");
96 ASSERT_TEXT_TOKEN("test");
97 ASSERT_NEWLINE_TOKEN();
98
99 ASSERT_FALSE(tokenizer.Next());
100 }
101
TEST(Tokenizer,MultiTextNewLine)102 TEST(Tokenizer, MultiTextNewLine) {
103 SETUP_TEST("one\ntwo\nthree\n");
104 ASSERT_TEXT_TOKEN("one");
105 ASSERT_NEWLINE_TOKEN();
106 ASSERT_TEXT_TOKEN("two");
107 ASSERT_NEWLINE_TOKEN();
108 ASSERT_TEXT_TOKEN("three");
109 ASSERT_NEWLINE_TOKEN();
110
111 ASSERT_FALSE(tokenizer.Next());
112 }
113
TEST(Tokenizer,MultiTextNewLineNoLineEnding)114 TEST(Tokenizer, MultiTextNewLineNoLineEnding) {
115 SETUP_TEST("one\ntwo\nthree");
116 ASSERT_TEXT_TOKEN("one");
117 ASSERT_NEWLINE_TOKEN();
118 ASSERT_TEXT_TOKEN("two");
119 ASSERT_NEWLINE_TOKEN();
120 ASSERT_TEXT_TOKEN("three");
121
122 ASSERT_FALSE(tokenizer.Next());
123 }
124
TEST(Tokenizer,Comment)125 TEST(Tokenizer, Comment) {
126 SETUP_TEST("#test");
127 ASSERT_FALSE(tokenizer.Next());
128 }
129
TEST(Tokenizer,CommentWhiteSpace)130 TEST(Tokenizer, CommentWhiteSpace) {
131 SETUP_TEST(" \t \r #test \t \r ");
132 ASSERT_FALSE(tokenizer.Next());
133 }
134
TEST(Tokenizer,CommentNewLine)135 TEST(Tokenizer, CommentNewLine) {
136 SETUP_TEST(" #test \n");
137 ASSERT_NEWLINE_TOKEN();
138 ASSERT_FALSE(tokenizer.Next());
139 }
140
TEST(Tokenizer,CommentTwoNewLine)141 TEST(Tokenizer, CommentTwoNewLine) {
142 SETUP_TEST(" #test \n#test");
143 ASSERT_NEWLINE_TOKEN();
144 ASSERT_FALSE(tokenizer.Next());
145 }
146
TEST(Tokenizer,CommentWithText)147 TEST(Tokenizer, CommentWithText) {
148 SETUP_TEST("foo bar #test");
149 ASSERT_TEXT_TOKEN("foo");
150 ASSERT_TEXT_TOKEN("bar");
151 ASSERT_FALSE(tokenizer.Next());
152 }
153
TEST(Tokenizer,CommentWithTextNoSpace)154 TEST(Tokenizer, CommentWithTextNoSpace) {
155 SETUP_TEST("foo bar#test");
156 ASSERT_TEXT_TOKEN("foo");
157 ASSERT_TEXT_TOKEN("bar");
158 ASSERT_FALSE(tokenizer.Next());
159 }
160
TEST(Tokenizer,CommentWithTextLineFeed)161 TEST(Tokenizer, CommentWithTextLineFeed) {
162 SETUP_TEST("foo bar #test\n");
163 ASSERT_TEXT_TOKEN("foo");
164 ASSERT_TEXT_TOKEN("bar");
165 ASSERT_NEWLINE_TOKEN();
166 ASSERT_FALSE(tokenizer.Next());
167 }
168
TEST(Tokenizer,CommentWithMultiTextLineFeed)169 TEST(Tokenizer, CommentWithMultiTextLineFeed) {
170 SETUP_TEST("#blah\nfoo bar #test\n#blah");
171 ASSERT_NEWLINE_TOKEN();
172 ASSERT_TEXT_TOKEN("foo");
173 ASSERT_TEXT_TOKEN("bar");
174 ASSERT_NEWLINE_TOKEN();
175 ASSERT_FALSE(tokenizer.Next());
176 }
177
TEST(Tokenizer,SimpleEscaped)178 TEST(Tokenizer, SimpleEscaped) {
179 SETUP_TEST("fo\\no bar");
180 ASSERT_TEXT_TOKEN("fo\\no");
181 ASSERT_TEXT_TOKEN("bar");
182 ASSERT_FALSE(tokenizer.Next());
183 }
184
TEST(Tokenizer,EscapedLineContNoLineFeed)185 TEST(Tokenizer, EscapedLineContNoLineFeed) {
186 SETUP_TEST("fo\\no bar \\ hello");
187 ASSERT_TEXT_TOKEN("fo\\no");
188 ASSERT_TEXT_TOKEN("bar");
189 ASSERT_FALSE(tokenizer.Next());
190 }
191
TEST(Tokenizer,EscapedLineContLineFeed)192 TEST(Tokenizer, EscapedLineContLineFeed) {
193 SETUP_TEST("fo\\no bar \\ hello\n");
194 ASSERT_TEXT_TOKEN("fo\\no");
195 ASSERT_TEXT_TOKEN("bar");
196 ASSERT_FALSE(tokenizer.Next());
197 }
198
TEST(Tokenizer,EscapedLineCont)199 TEST(Tokenizer, EscapedLineCont) {
200 SETUP_TEST("fo\\no bar \\\ntest");
201 ASSERT_TEXT_TOKEN("fo\\no");
202 ASSERT_TEXT_TOKEN("bar");
203 ASSERT_TEXT_TOKEN("test");
204 ASSERT_FALSE(tokenizer.Next());
205 }
206
TEST(Tokenizer,EscapedLineContWithBadChars)207 TEST(Tokenizer, EscapedLineContWithBadChars) {
208 SETUP_TEST("fo\\no bar \\bad bad bad\ntest");
209 ASSERT_TEXT_TOKEN("fo\\no");
210 ASSERT_TEXT_TOKEN("bar");
211 ASSERT_TEXT_TOKEN("test");
212 ASSERT_FALSE(tokenizer.Next());
213 }
214
TEST(Tokenizer,SimpleQuotes)215 TEST(Tokenizer, SimpleQuotes) {
216 SETUP_TEST("foo \"single token\" bar");
217 ASSERT_TEXT_TOKEN("foo");
218 ASSERT_TEXT_TOKEN("single token");
219 ASSERT_TEXT_TOKEN("bar");
220 ASSERT_FALSE(tokenizer.Next());
221 }
222
TEST(Tokenizer,BadQuotes)223 TEST(Tokenizer, BadQuotes) {
224 SETUP_TEST("foo \"single token");
225 ASSERT_TEXT_TOKEN("foo");
226 ASSERT_TEXT_TOKEN("single token");
227 ASSERT_FALSE(tokenizer.Next());
228 }
229
230 } // namespace init
231