/* * Copyright (C) 2018 The Android Open Source Project * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #include "utils/utf8/unicodetext.h" #include "gtest/gtest.h" namespace libtextclassifier3 { namespace { class UnicodeTextTest : public testing::Test { protected: UnicodeTextTest() : empty_text_() { text_.push_back(0x1C0); text_.push_back(0x4E8C); text_.push_back(0xD7DB); text_.push_back(0x34); text_.push_back(0x1D11E); } UnicodeText empty_text_; UnicodeText text_; }; // Tests for our modifications of UnicodeText. TEST(UnicodeTextTest, Custom) { UnicodeText text = UTF8ToUnicodeText("1234๐Ÿ˜‹hello", /*do_copy=*/false); EXPECT_EQ(text.ToUTF8String(), "1234๐Ÿ˜‹hello"); EXPECT_EQ(text.size_codepoints(), 10); EXPECT_EQ(text.size_bytes(), 13); auto it_begin = text.begin(); std::advance(it_begin, 4); auto it_end = text.begin(); std::advance(it_end, 6); EXPECT_EQ(text.UTF8Substring(it_begin, it_end), "๐Ÿ˜‹h"); } TEST(UnicodeTextTest, Substring) { UnicodeText text = UTF8ToUnicodeText("1234๐Ÿ˜‹hello", /*do_copy=*/false); EXPECT_EQ(UnicodeText::Substring(text, 4, 6, /*do_copy=*/true), UTF8ToUnicodeText("๐Ÿ˜‹h")); EXPECT_EQ(UnicodeText::Substring(text, 4, 6, /*do_copy=*/false), UTF8ToUnicodeText("๐Ÿ˜‹h")); } TEST(UnicodeTextTest, Ownership) { const std::string src = "\u304A\u00B0\u106B"; UnicodeText alias; alias.PointToUTF8(src.data(), src.size()); EXPECT_EQ(alias.data(), src.data()); UnicodeText::const_iterator it = alias.begin(); EXPECT_EQ(*it++, 0x304A); EXPECT_EQ(*it++, 0x00B0); EXPECT_EQ(*it++, 0x106B); EXPECT_EQ(it, alias.end()); UnicodeText t = alias; // Copy initialization copies the data. EXPECT_NE(t.data(), alias.data()); } TEST(UnicodeTextTest, Validation) { EXPECT_TRUE(UTF8ToUnicodeText("1234๐Ÿ˜‹hello", /*do_copy=*/false).is_valid()); EXPECT_TRUE( UTF8ToUnicodeText("\u304A\u00B0\u106B", /*do_copy=*/false).is_valid()); EXPECT_TRUE( UTF8ToUnicodeText("this is a test๐Ÿ˜‹๐Ÿ˜‹๐Ÿ˜‹", /*do_copy=*/false).is_valid()); EXPECT_TRUE( UTF8ToUnicodeText("\xf0\x9f\x98\x8b", /*do_copy=*/false).is_valid()); // Too short (string is too short). EXPECT_FALSE(UTF8ToUnicodeText("\xf0\x9f", /*do_copy=*/false).is_valid()); // Too long (too many trailing bytes). EXPECT_FALSE( UTF8ToUnicodeText("\xf0\x9f\x98\x8b\x8b", /*do_copy=*/false).is_valid()); // Too short (too few trailing bytes). EXPECT_FALSE( UTF8ToUnicodeText("\xf0\x9f\x98\x61\x61", /*do_copy=*/false).is_valid()); // Invalid with context. EXPECT_FALSE( UTF8ToUnicodeText("hello \xf0\x9f\x98\x61\x61 world1", /*do_copy=*/false) .is_valid()); } class IteratorTest : public UnicodeTextTest {}; TEST_F(IteratorTest, Iterates) { UnicodeText::const_iterator iter = text_.begin(); EXPECT_EQ(0x1C0, *iter); EXPECT_EQ(&iter, &++iter); // operator++ returns *this. EXPECT_EQ(0x4E8C, *iter++); EXPECT_EQ(0xD7DB, *iter); // Make sure you can dereference more than once. EXPECT_EQ(0xD7DB, *iter); EXPECT_EQ(0x34, *++iter); EXPECT_EQ(0x1D11E, *++iter); ASSERT_TRUE(iter != text_.end()); iter++; EXPECT_TRUE(iter == text_.end()); } TEST_F(IteratorTest, MultiPass) { // Also tests Default Constructible and Assignable. UnicodeText::const_iterator i1, i2; i1 = text_.begin(); i2 = i1; EXPECT_EQ(0x4E8C, *++i1); EXPECT_TRUE(i1 != i2); EXPECT_EQ(0x1C0, *i2); ++i2; EXPECT_TRUE(i1 == i2); EXPECT_EQ(0x4E8C, *i2); } TEST_F(IteratorTest, ReverseIterates) { UnicodeText::const_iterator iter = text_.end(); EXPECT_TRUE(iter == text_.end()); iter--; ASSERT_TRUE(iter != text_.end()); EXPECT_EQ(0x1D11E, *iter--); EXPECT_EQ(0x34, *iter); EXPECT_EQ(0xD7DB, *--iter); // Make sure you can dereference more than once. EXPECT_EQ(0xD7DB, *iter); --iter; EXPECT_EQ(0x4E8C, *iter--); EXPECT_EQ(0x1C0, *iter); EXPECT_TRUE(iter == text_.begin()); } TEST_F(IteratorTest, Comparable) { UnicodeText::const_iterator i1, i2; i1 = text_.begin(); i2 = i1; ++i2; EXPECT_TRUE(i1 < i2); EXPECT_TRUE(text_.begin() <= i1); EXPECT_FALSE(i1 >= i2); EXPECT_FALSE(i1 > text_.end()); } TEST_F(IteratorTest, Advance) { UnicodeText::const_iterator iter = text_.begin(); EXPECT_EQ(0x1C0, *iter); std::advance(iter, 4); EXPECT_EQ(0x1D11E, *iter); ++iter; EXPECT_TRUE(iter == text_.end()); } TEST_F(IteratorTest, Distance) { UnicodeText::const_iterator iter = text_.begin(); EXPECT_EQ(0, std::distance(text_.begin(), iter)); EXPECT_EQ(5, std::distance(iter, text_.end())); ++iter; ++iter; EXPECT_EQ(2, std::distance(text_.begin(), iter)); EXPECT_EQ(3, std::distance(iter, text_.end())); ++iter; ++iter; EXPECT_EQ(4, std::distance(text_.begin(), iter)); ++iter; EXPECT_EQ(0, std::distance(iter, text_.end())); } class OperatorTest : public UnicodeTextTest {}; TEST_F(OperatorTest, Clear) { UnicodeText empty_text(UTF8ToUnicodeText("", /*do_copy=*/false)); EXPECT_FALSE(text_ == empty_text); text_.clear(); EXPECT_TRUE(text_ == empty_text); } TEST_F(OperatorTest, Empty) { EXPECT_TRUE(empty_text_.empty()); EXPECT_FALSE(text_.empty()); text_.clear(); EXPECT_TRUE(text_.empty()); } } // namespace } // namespace libtextclassifier3