1 // Copyright (c) 2006-2008 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
4
5 #include "base/string_tokenizer.h"
6 #include "testing/gtest/include/gtest/gtest.h"
7
8 using std::string;
9
10 namespace {
11 class StringTokenizerTest : public testing::Test {};
12 }
13
TEST(StringTokenizerTest,Simple)14 TEST(StringTokenizerTest, Simple) {
15 string input = "this is a test";
16 StringTokenizer t(input, " ");
17
18 EXPECT_TRUE(t.GetNext());
19 EXPECT_EQ(string("this"), t.token());
20
21 EXPECT_TRUE(t.GetNext());
22 EXPECT_EQ(string("is"), t.token());
23
24 EXPECT_TRUE(t.GetNext());
25 EXPECT_EQ(string("a"), t.token());
26
27 EXPECT_TRUE(t.GetNext());
28 EXPECT_EQ(string("test"), t.token());
29
30 EXPECT_FALSE(t.GetNext());
31 }
32
TEST(StringTokenizerTest,Reset)33 TEST(StringTokenizerTest, Reset) {
34 string input = "this is a test";
35 StringTokenizer t(input, " ");
36
37 for (int i = 0; i < 2; ++i) {
38 EXPECT_TRUE(t.GetNext());
39 EXPECT_EQ(string("this"), t.token());
40
41 EXPECT_TRUE(t.GetNext());
42 EXPECT_EQ(string("is"), t.token());
43
44 EXPECT_TRUE(t.GetNext());
45 EXPECT_EQ(string("a"), t.token());
46
47 EXPECT_TRUE(t.GetNext());
48 EXPECT_EQ(string("test"), t.token());
49
50 EXPECT_FALSE(t.GetNext());
51 t.Reset();
52 }
53 }
54
TEST(StringTokenizerTest,RetDelims)55 TEST(StringTokenizerTest, RetDelims) {
56 string input = "this is a test";
57 StringTokenizer t(input, " ");
58 t.set_options(StringTokenizer::RETURN_DELIMS);
59
60 EXPECT_TRUE(t.GetNext());
61 EXPECT_EQ(string("this"), t.token());
62
63 EXPECT_TRUE(t.GetNext());
64 EXPECT_EQ(string(" "), t.token());
65
66 EXPECT_TRUE(t.GetNext());
67 EXPECT_EQ(string("is"), t.token());
68
69 EXPECT_TRUE(t.GetNext());
70 EXPECT_EQ(string(" "), t.token());
71
72 EXPECT_TRUE(t.GetNext());
73 EXPECT_EQ(string("a"), t.token());
74
75 EXPECT_TRUE(t.GetNext());
76 EXPECT_EQ(string(" "), t.token());
77
78 EXPECT_TRUE(t.GetNext());
79 EXPECT_EQ(string("test"), t.token());
80
81 EXPECT_FALSE(t.GetNext());
82 }
83
TEST(StringTokenizerTest,ManyDelims)84 TEST(StringTokenizerTest, ManyDelims) {
85 string input = "this: is, a-test";
86 StringTokenizer t(input, ": ,-");
87
88 EXPECT_TRUE(t.GetNext());
89 EXPECT_EQ(string("this"), t.token());
90
91 EXPECT_TRUE(t.GetNext());
92 EXPECT_EQ(string("is"), t.token());
93
94 EXPECT_TRUE(t.GetNext());
95 EXPECT_EQ(string("a"), t.token());
96
97 EXPECT_TRUE(t.GetNext());
98 EXPECT_EQ(string("test"), t.token());
99
100 EXPECT_FALSE(t.GetNext());
101 }
102
TEST(StringTokenizerTest,ParseHeader)103 TEST(StringTokenizerTest, ParseHeader) {
104 string input = "Content-Type: text/html ; charset=UTF-8";
105 StringTokenizer t(input, ": ;=");
106 t.set_options(StringTokenizer::RETURN_DELIMS);
107
108 EXPECT_TRUE(t.GetNext());
109 EXPECT_FALSE(t.token_is_delim());
110 EXPECT_EQ(string("Content-Type"), t.token());
111
112 EXPECT_TRUE(t.GetNext());
113 EXPECT_TRUE(t.token_is_delim());
114 EXPECT_EQ(string(":"), t.token());
115
116 EXPECT_TRUE(t.GetNext());
117 EXPECT_TRUE(t.token_is_delim());
118 EXPECT_EQ(string(" "), t.token());
119
120 EXPECT_TRUE(t.GetNext());
121 EXPECT_FALSE(t.token_is_delim());
122 EXPECT_EQ(string("text/html"), t.token());
123
124 EXPECT_TRUE(t.GetNext());
125 EXPECT_TRUE(t.token_is_delim());
126 EXPECT_EQ(string(" "), t.token());
127
128 EXPECT_TRUE(t.GetNext());
129 EXPECT_TRUE(t.token_is_delim());
130 EXPECT_EQ(string(";"), t.token());
131
132 EXPECT_TRUE(t.GetNext());
133 EXPECT_TRUE(t.token_is_delim());
134 EXPECT_EQ(string(" "), t.token());
135
136 EXPECT_TRUE(t.GetNext());
137 EXPECT_FALSE(t.token_is_delim());
138 EXPECT_EQ(string("charset"), t.token());
139
140 EXPECT_TRUE(t.GetNext());
141 EXPECT_TRUE(t.token_is_delim());
142 EXPECT_EQ(string("="), t.token());
143
144 EXPECT_TRUE(t.GetNext());
145 EXPECT_FALSE(t.token_is_delim());
146 EXPECT_EQ(string("UTF-8"), t.token());
147
148 EXPECT_FALSE(t.GetNext());
149 EXPECT_FALSE(t.token_is_delim());
150 }
151
TEST(StringTokenizerTest,ParseQuotedString)152 TEST(StringTokenizerTest, ParseQuotedString) {
153 string input = "foo bar 'hello world' baz";
154 StringTokenizer t(input, " ");
155 t.set_quote_chars("'");
156
157 EXPECT_TRUE(t.GetNext());
158 EXPECT_EQ(string("foo"), t.token());
159
160 EXPECT_TRUE(t.GetNext());
161 EXPECT_EQ(string("bar"), t.token());
162
163 EXPECT_TRUE(t.GetNext());
164 EXPECT_EQ(string("'hello world'"), t.token());
165
166 EXPECT_TRUE(t.GetNext());
167 EXPECT_EQ(string("baz"), t.token());
168
169 EXPECT_FALSE(t.GetNext());
170 }
171
TEST(StringTokenizerTest,ParseQuotedString_Malformed)172 TEST(StringTokenizerTest, ParseQuotedString_Malformed) {
173 string input = "bar 'hello wo";
174 StringTokenizer t(input, " ");
175 t.set_quote_chars("'");
176
177 EXPECT_TRUE(t.GetNext());
178 EXPECT_EQ(string("bar"), t.token());
179
180 EXPECT_TRUE(t.GetNext());
181 EXPECT_EQ(string("'hello wo"), t.token());
182
183 EXPECT_FALSE(t.GetNext());
184 }
185
TEST(StringTokenizerTest,ParseQuotedString_Multiple)186 TEST(StringTokenizerTest, ParseQuotedString_Multiple) {
187 string input = "bar 'hel\"lo\" wo' baz\"";
188 StringTokenizer t(input, " ");
189 t.set_quote_chars("'\"");
190
191 EXPECT_TRUE(t.GetNext());
192 EXPECT_EQ(string("bar"), t.token());
193
194 EXPECT_TRUE(t.GetNext());
195 EXPECT_EQ(string("'hel\"lo\" wo'"), t.token());
196
197 EXPECT_TRUE(t.GetNext());
198 EXPECT_EQ(string("baz\""), t.token());
199
200 EXPECT_FALSE(t.GetNext());
201 }
202
TEST(StringTokenizerTest,ParseQuotedString_EscapedQuotes)203 TEST(StringTokenizerTest, ParseQuotedString_EscapedQuotes) {
204 string input = "foo 'don\\'t do that'";
205 StringTokenizer t(input, " ");
206 t.set_quote_chars("'");
207
208 EXPECT_TRUE(t.GetNext());
209 EXPECT_EQ(string("foo"), t.token());
210
211 EXPECT_TRUE(t.GetNext());
212 EXPECT_EQ(string("'don\\'t do that'"), t.token());
213
214 EXPECT_FALSE(t.GetNext());
215 }
216
TEST(StringTokenizerTest,ParseQuotedString_EscapedQuotes2)217 TEST(StringTokenizerTest, ParseQuotedString_EscapedQuotes2) {
218 string input = "foo='a, b', bar";
219 StringTokenizer t(input, ", ");
220 t.set_quote_chars("'");
221
222 EXPECT_TRUE(t.GetNext());
223 EXPECT_EQ(string("foo='a, b'"), t.token());
224
225 EXPECT_TRUE(t.GetNext());
226 EXPECT_EQ(string("bar"), t.token());
227
228 EXPECT_FALSE(t.GetNext());
229 }
230