1 // Protocol Buffers - Google's data interchange format 2 // Copyright 2008 Google Inc. All rights reserved. 3 // 4 // Use of this source code is governed by a BSD-style 5 // license that can be found in the LICENSE file or at 6 // https://developers.google.com/open-source/licenses/bsd 7 8 package com.google.protobuf; 9 10 import static com.google.common.truth.Truth.assertThat; 11 import static com.google.protobuf.IsValidUtf8TestUtil.DIRECT_NIO_FACTORY; 12 import static com.google.protobuf.IsValidUtf8TestUtil.EXPECTED_ONE_BYTE_ROUNDTRIPPABLE_COUNT; 13 import static com.google.protobuf.IsValidUtf8TestUtil.EXPECTED_THREE_BYTE_ROUNDTRIPPABLE_COUNT; 14 import static com.google.protobuf.IsValidUtf8TestUtil.HEAP_NIO_FACTORY; 15 import static com.google.protobuf.IsValidUtf8TestUtil.LITERAL_FACTORY; 16 import static com.google.protobuf.IsValidUtf8TestUtil.ROPE_FACTORY; 17 import static com.google.protobuf.IsValidUtf8TestUtil.testBytes; 18 19 import com.google.protobuf.IsValidUtf8TestUtil.ByteStringFactory; 20 import com.google.protobuf.IsValidUtf8TestUtil.Shard; 21 import org.junit.Test; 22 import org.junit.runner.RunWith; 23 import org.junit.runners.JUnit4; 24 25 /** 26 * Tests cases for {@link ByteString#isValidUtf8()}. This includes three brute force tests that 27 * actually test every permutation of one byte, two byte, and three byte sequences to ensure that 28 * the method produces the right result for every possible byte encoding where "right" means it's 29 * consistent with java's UTF-8 string encoding/decoding such that the method returns true for any 30 * sequence that will round trip when converted to a String and then back to bytes and will return 31 * false for any sequence that will not round trip. See also {@link IsValidUtf8FourByteTest}. It 32 * also includes some other more targeted tests. 33 */ 34 @RunWith(JUnit4.class) 35 public class IsValidUtf8Test { 36 /** Tests that round tripping of all two byte permutations work. */ 37 @Test testIsValidUtf8_1Byte()38 public void testIsValidUtf8_1Byte() { 39 testBytes(LITERAL_FACTORY, 1, EXPECTED_ONE_BYTE_ROUNDTRIPPABLE_COUNT); 40 testBytes(HEAP_NIO_FACTORY, 1, EXPECTED_ONE_BYTE_ROUNDTRIPPABLE_COUNT); 41 testBytes(DIRECT_NIO_FACTORY, 1, EXPECTED_ONE_BYTE_ROUNDTRIPPABLE_COUNT); 42 testBytes(ROPE_FACTORY, 1, EXPECTED_ONE_BYTE_ROUNDTRIPPABLE_COUNT); 43 } 44 45 /** Tests that round tripping of all two byte permutations work. */ 46 @Test testIsValidUtf8_2Bytes()47 public void testIsValidUtf8_2Bytes() { 48 testBytes(LITERAL_FACTORY, 2, IsValidUtf8TestUtil.EXPECTED_TWO_BYTE_ROUNDTRIPPABLE_COUNT); 49 testBytes(HEAP_NIO_FACTORY, 2, IsValidUtf8TestUtil.EXPECTED_TWO_BYTE_ROUNDTRIPPABLE_COUNT); 50 testBytes(DIRECT_NIO_FACTORY, 2, IsValidUtf8TestUtil.EXPECTED_TWO_BYTE_ROUNDTRIPPABLE_COUNT); 51 testBytes(ROPE_FACTORY, 2, IsValidUtf8TestUtil.EXPECTED_TWO_BYTE_ROUNDTRIPPABLE_COUNT); 52 } 53 54 /** Tests that round tripping of all three byte permutations work. */ 55 @Test testIsValidUtf8_3Bytes()56 public void testIsValidUtf8_3Bytes() { 57 testBytes(LITERAL_FACTORY, 3, EXPECTED_THREE_BYTE_ROUNDTRIPPABLE_COUNT); 58 testBytes(HEAP_NIO_FACTORY, 3, EXPECTED_THREE_BYTE_ROUNDTRIPPABLE_COUNT); 59 testBytes(DIRECT_NIO_FACTORY, 3, EXPECTED_THREE_BYTE_ROUNDTRIPPABLE_COUNT); 60 testBytes(ROPE_FACTORY, 3, EXPECTED_THREE_BYTE_ROUNDTRIPPABLE_COUNT); 61 } 62 63 /** 64 * Tests that round tripping of a sample of four byte permutations work. All permutations are 65 * prohibitively expensive to test for automated runs; {@link IsValidUtf8FourByteTest} is used for 66 * full coverage. This method tests specific four-byte cases. 67 */ 68 @Test testIsValidUtf8_4BytesSamples()69 public void testIsValidUtf8_4BytesSamples() { 70 // Valid 4 byte. 71 assertValidUtf8(0xF0, 0xA4, 0xAD, 0xA2); 72 73 // Bad trailing bytes 74 assertInvalidUtf8(0xF0, 0xA4, 0xAD, 0x7F); 75 assertInvalidUtf8(0xF0, 0xA4, 0xAD, 0xC0); 76 77 // Special cases for byte2 78 assertInvalidUtf8(0xF0, 0x8F, 0xAD, 0xA2); 79 assertInvalidUtf8(0xF4, 0x90, 0xAD, 0xA2); 80 } 81 82 /** Tests some hard-coded test cases. */ 83 @Test testSomeSequences()84 public void testSomeSequences() { 85 // Empty 86 assertThat(asBytes("").isValidUtf8()).isTrue(); 87 88 // One-byte characters, including control characters 89 assertThat(asBytes("\u0000abc\u007f").isValidUtf8()).isTrue(); 90 91 // Two-byte characters 92 assertThat(asBytes("\u00a2\u00a2").isValidUtf8()).isTrue(); 93 94 // Three-byte characters 95 assertThat(asBytes("\u020ac\u020ac").isValidUtf8()).isTrue(); 96 97 // Four-byte characters 98 assertThat(asBytes("\u024B62\u024B62").isValidUtf8()).isTrue(); 99 100 // Mixed string 101 assertThat(asBytes("a\u020ac\u00a2b\\u024B62u020acc\u00a2de\u024B62").isValidUtf8()).isTrue(); 102 103 // Not a valid string 104 assertInvalidUtf8(-1, 0, -1, 0); 105 } 106 107 @Test testShardsHaveExpectedRoundTrippables()108 public void testShardsHaveExpectedRoundTrippables() { 109 // A sanity check. 110 int actual = 0; 111 for (Shard shard : IsValidUtf8TestUtil.FOUR_BYTE_SHARDS) { 112 actual = (int) (actual + shard.expected); 113 } 114 assertThat(actual).isEqualTo(IsValidUtf8TestUtil.EXPECTED_FOUR_BYTE_ROUNDTRIPPABLE_COUNT); 115 } 116 toByteArray(int... bytes)117 private byte[] toByteArray(int... bytes) { 118 byte[] realBytes = new byte[bytes.length]; 119 for (int i = 0; i < bytes.length; i++) { 120 realBytes[i] = (byte) bytes[i]; 121 } 122 return realBytes; 123 } 124 assertValidUtf8(ByteStringFactory factory, int[] bytes, boolean not)125 private void assertValidUtf8(ByteStringFactory factory, int[] bytes, boolean not) { 126 byte[] realBytes = toByteArray(bytes); 127 assertThat(not ^ Utf8.isValidUtf8(realBytes)).isTrue(); 128 assertThat(not ^ Utf8.isValidUtf8(realBytes, 0, bytes.length)).isTrue(); 129 ByteString leaf = factory.newByteString(realBytes); 130 ByteString sub = leaf.substring(0, bytes.length); 131 assertThat(not ^ leaf.isValidUtf8()).isTrue(); 132 assertThat(not ^ sub.isValidUtf8()).isTrue(); 133 ByteString[] ropes = { 134 RopeByteString.newInstanceForTest(ByteString.EMPTY, leaf), 135 RopeByteString.newInstanceForTest(ByteString.EMPTY, sub), 136 RopeByteString.newInstanceForTest(leaf, ByteString.EMPTY), 137 RopeByteString.newInstanceForTest(sub, ByteString.EMPTY), 138 RopeByteString.newInstanceForTest(sub, leaf) 139 }; 140 for (ByteString rope : ropes) { 141 assertThat(not ^ rope.isValidUtf8()).isTrue(); 142 } 143 } 144 assertValidUtf8(int... bytes)145 private void assertValidUtf8(int... bytes) { 146 assertValidUtf8(LITERAL_FACTORY, bytes, false); 147 assertValidUtf8(HEAP_NIO_FACTORY, bytes, false); 148 assertValidUtf8(DIRECT_NIO_FACTORY, bytes, false); 149 assertValidUtf8(ROPE_FACTORY, bytes, false); 150 } 151 assertInvalidUtf8(int... bytes)152 private void assertInvalidUtf8(int... bytes) { 153 assertValidUtf8(LITERAL_FACTORY, bytes, true); 154 assertValidUtf8(HEAP_NIO_FACTORY, bytes, true); 155 assertValidUtf8(DIRECT_NIO_FACTORY, bytes, true); 156 assertValidUtf8(ROPE_FACTORY, bytes, true); 157 } 158 asBytes(String s)159 private static ByteString asBytes(String s) { 160 return ByteString.copyFromUtf8(s); 161 } 162 } 163