1 /*
2 * Copyright (C) 2018 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17 #include "utils/test-utils.h"
18
19 #include "gmock/gmock.h"
20 #include "gtest/gtest.h"
21
22 namespace libtextclassifier3 {
23 namespace {
24
TEST(TestUtilTest,TokenizeOnSpace)25 TEST(TestUtilTest, TokenizeOnSpace) {
26 std::vector<Token> tokens =
27 TokenizeOnSpace("Where is Jörg Borg located? Maybe in Zürich ...");
28
29 EXPECT_EQ(tokens.size(), 9);
30
31 EXPECT_EQ(tokens[0].value, "Where");
32 EXPECT_EQ(tokens[0].start, 0);
33 EXPECT_EQ(tokens[0].end, 5);
34
35 EXPECT_EQ(tokens[1].value, "is");
36 EXPECT_EQ(tokens[1].start, 6);
37 EXPECT_EQ(tokens[1].end, 8);
38
39 EXPECT_EQ(tokens[2].value, "Jörg");
40 EXPECT_EQ(tokens[2].start, 9);
41 EXPECT_EQ(tokens[2].end, 13);
42
43 EXPECT_EQ(tokens[3].value, "Borg");
44 EXPECT_EQ(tokens[3].start, 14);
45 EXPECT_EQ(tokens[3].end, 18);
46
47 EXPECT_EQ(tokens[4].value, "located?");
48 EXPECT_EQ(tokens[4].start, 19);
49 EXPECT_EQ(tokens[4].end, 27);
50
51 EXPECT_EQ(tokens[5].value, "Maybe");
52 EXPECT_EQ(tokens[5].start, 28);
53 EXPECT_EQ(tokens[5].end, 33);
54
55 EXPECT_EQ(tokens[6].value, "in");
56 EXPECT_EQ(tokens[6].start, 34);
57 EXPECT_EQ(tokens[6].end, 36);
58
59 EXPECT_EQ(tokens[7].value, "Zürich");
60 EXPECT_EQ(tokens[7].start, 37);
61 EXPECT_EQ(tokens[7].end, 43);
62
63 EXPECT_EQ(tokens[8].value, "...");
64 EXPECT_EQ(tokens[8].start, 44);
65 EXPECT_EQ(tokens[8].end, 47);
66 }
67
TEST(TestUtilTest,TokenizeOnDelimiters)68 TEST(TestUtilTest, TokenizeOnDelimiters) {
69 std::vector<Token> tokens = TokenizeOnDelimiters(
70 "This might be čomplíčateď?!: Oder?", {' ', '?', '!'});
71
72 EXPECT_EQ(tokens.size(), 6);
73
74 EXPECT_EQ(tokens[0].value, "This");
75 EXPECT_EQ(tokens[0].start, 0);
76 EXPECT_EQ(tokens[0].end, 4);
77
78 EXPECT_EQ(tokens[1].value, "might");
79 EXPECT_EQ(tokens[1].start, 7);
80 EXPECT_EQ(tokens[1].end, 12);
81
82 EXPECT_EQ(tokens[2].value, "be");
83 EXPECT_EQ(tokens[2].start, 13);
84 EXPECT_EQ(tokens[2].end, 15);
85
86 EXPECT_EQ(tokens[3].value, "čomplíčateď");
87 EXPECT_EQ(tokens[3].start, 16);
88 EXPECT_EQ(tokens[3].end, 27);
89
90 EXPECT_EQ(tokens[4].value, ":");
91 EXPECT_EQ(tokens[4].start, 29);
92 EXPECT_EQ(tokens[4].end, 30);
93
94 EXPECT_EQ(tokens[5].value, "Oder");
95 EXPECT_EQ(tokens[5].start, 31);
96 EXPECT_EQ(tokens[5].end, 35);
97 }
98
99 } // namespace
100 } // namespace libtextclassifier3
101