• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 // Tencent is pleased to support the open source community by making RapidJSON available.
2 //
3 // Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved.
4 //
5 // Licensed under the MIT License (the "License"); you may not use this file except
6 // in compliance with the License. You may obtain a copy of the License at
7 //
8 // http://opensource.org/licenses/MIT
9 //
10 // Unless required by applicable law or agreed to in writing, software distributed
11 // under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
12 // CONDITIONS OF ANY KIND, either express or implied. See the License for the
13 // specific language governing permissions and limitations under the License.
14 
15 #include "unittest.h"
16 #include "rapidjson/filereadstream.h"
17 #include "rapidjson/filewritestream.h"
18 #include "rapidjson/encodedstream.h"
19 #include "rapidjson/stringbuffer.h"
20 #include "rapidjson/memorystream.h"
21 #include "rapidjson/memorybuffer.h"
22 
23 using namespace rapidjson;
24 
25 class EncodedStreamTest : public ::testing::Test {
26 public:
EncodedStreamTest()27     EncodedStreamTest() : json_(), length_() {}
28 
SetUp()29     virtual void SetUp() {
30         json_ = ReadFile("utf8.json", true, &length_);
31     }
32 
TearDown()33     virtual void TearDown() {
34         free(json_);
35         json_ = 0;
36     }
37 
38 private:
39     EncodedStreamTest(const EncodedStreamTest&);
40     EncodedStreamTest& operator=(const EncodedStreamTest&);
41 
42 protected:
Open(const char * filename)43     static FILE* Open(const char* filename) {
44         const char *paths[] = {
45             "encodings/%s",
46             "bin/encodings/%s",
47             "../bin/encodings/%s",
48             "../../bin/encodings/%s",
49             "../../../bin/encodings/%s"
50         };
51         char buffer[1024];
52         for (size_t i = 0; i < sizeof(paths) / sizeof(paths[0]); i++) {
53             sprintf(buffer, paths[i], filename);
54             FILE *fp = fopen(buffer, "rb");
55             if (fp)
56                 return fp;
57         }
58         return 0;
59     }
60 
ReadFile(const char * filename,bool appendPath,size_t * outLength)61     static char *ReadFile(const char* filename, bool appendPath, size_t* outLength) {
62         FILE *fp = appendPath ? Open(filename) : fopen(filename, "rb");
63 
64         if (!fp) {
65             *outLength = 0;
66             return 0;
67         }
68 
69         fseek(fp, 0, SEEK_END);
70         *outLength = (size_t)ftell(fp);
71         fseek(fp, 0, SEEK_SET);
72         char* buffer = (char*)malloc(*outLength + 1);
73         size_t readLength = fread(buffer, 1, *outLength, fp);
74         buffer[readLength] = '\0';
75         fclose(fp);
76         return buffer;
77     }
78 
79     template <typename FileEncoding, typename MemoryEncoding>
TestEncodedInputStream(const char * filename)80     void TestEncodedInputStream(const char* filename) {
81         // Test FileReadStream
82         {
83             char buffer[16];
84             FILE *fp = Open(filename);
85             ASSERT_TRUE(fp != 0);
86             FileReadStream fs(fp, buffer, sizeof(buffer));
87             EncodedInputStream<FileEncoding, FileReadStream> eis(fs);
88             StringStream s(json_);
89 
90             while (eis.Peek() != '\0') {
91                 unsigned expected, actual;
92                 EXPECT_TRUE(UTF8<>::Decode(s, &expected));
93                 EXPECT_TRUE(MemoryEncoding::Decode(eis, &actual));
94                 EXPECT_EQ(expected, actual);
95             }
96             EXPECT_EQ('\0', s.Peek());
97             fclose(fp);
98         }
99 
100         // Test MemoryStream
101         {
102             size_t size;
103             char* data = ReadFile(filename, true, &size);
104             MemoryStream ms(data, size);
105             EncodedInputStream<FileEncoding, MemoryStream> eis(ms);
106             StringStream s(json_);
107 
108             while (eis.Peek() != '\0') {
109                 unsigned expected, actual;
110                 EXPECT_TRUE(UTF8<>::Decode(s, &expected));
111                 EXPECT_TRUE(MemoryEncoding::Decode(eis, &actual));
112                 EXPECT_EQ(expected, actual);
113             }
114             EXPECT_EQ('\0', s.Peek());
115             free(data);
116             EXPECT_EQ(size, eis.Tell());
117         }
118     }
119 
TestAutoUTFInputStream(const char * filename,bool expectHasBOM)120     void TestAutoUTFInputStream(const char *filename, bool expectHasBOM) {
121         // Test FileReadStream
122         {
123             char buffer[16];
124             FILE *fp = Open(filename);
125             ASSERT_TRUE(fp != 0);
126             FileReadStream fs(fp, buffer, sizeof(buffer));
127             AutoUTFInputStream<unsigned, FileReadStream> eis(fs);
128             EXPECT_EQ(expectHasBOM, eis.HasBOM());
129             StringStream s(json_);
130             while (eis.Peek() != '\0') {
131                 unsigned expected, actual;
132                 EXPECT_TRUE(UTF8<>::Decode(s, &expected));
133                 EXPECT_TRUE(AutoUTF<unsigned>::Decode(eis, &actual));
134                 EXPECT_EQ(expected, actual);
135             }
136             EXPECT_EQ('\0', s.Peek());
137             fclose(fp);
138         }
139 
140         // Test MemoryStream
141         {
142             size_t size;
143             char* data = ReadFile(filename, true, &size);
144             MemoryStream ms(data, size);
145             AutoUTFInputStream<unsigned, MemoryStream> eis(ms);
146             EXPECT_EQ(expectHasBOM, eis.HasBOM());
147             StringStream s(json_);
148 
149             while (eis.Peek() != '\0') {
150                 unsigned expected, actual;
151                 EXPECT_TRUE(UTF8<>::Decode(s, &expected));
152                 EXPECT_TRUE(AutoUTF<unsigned>::Decode(eis, &actual));
153                 EXPECT_EQ(expected, actual);
154             }
155             EXPECT_EQ('\0', s.Peek());
156             free(data);
157             EXPECT_EQ(size, eis.Tell());
158         }
159     }
160 
161     template <typename FileEncoding, typename MemoryEncoding>
TestEncodedOutputStream(const char * expectedFilename,bool putBOM)162     void TestEncodedOutputStream(const char* expectedFilename, bool putBOM) {
163         // Test FileWriteStream
164         {
165             char filename[L_tmpnam];
166             FILE* fp = TempFile(filename);
167             char buffer[16];
168             FileWriteStream os(fp, buffer, sizeof(buffer));
169             EncodedOutputStream<FileEncoding, FileWriteStream> eos(os, putBOM);
170             StringStream s(json_);
171             while (s.Peek() != '\0') {
172                 bool success = Transcoder<UTF8<>, MemoryEncoding>::Transcode(s, eos);
173                 EXPECT_TRUE(success);
174             }
175             eos.Flush();
176             fclose(fp);
177             EXPECT_TRUE(CompareFile(filename, expectedFilename));
178             remove(filename);
179         }
180 
181         // Test MemoryBuffer
182         {
183             MemoryBuffer mb;
184             EncodedOutputStream<FileEncoding, MemoryBuffer> eos(mb, putBOM);
185             StringStream s(json_);
186             while (s.Peek() != '\0') {
187                 bool success = Transcoder<UTF8<>, MemoryEncoding>::Transcode(s, eos);
188                 EXPECT_TRUE(success);
189             }
190             eos.Flush();
191             EXPECT_TRUE(CompareBufferFile(mb.GetBuffer(), mb.GetSize(), expectedFilename));
192         }
193     }
194 
TestAutoUTFOutputStream(UTFType type,bool putBOM,const char * expectedFilename)195     void TestAutoUTFOutputStream(UTFType type, bool putBOM, const char *expectedFilename) {
196         // Test FileWriteStream
197         {
198             char filename[L_tmpnam];
199             FILE* fp = TempFile(filename);
200 
201             char buffer[16];
202             FileWriteStream os(fp, buffer, sizeof(buffer));
203             AutoUTFOutputStream<unsigned, FileWriteStream> eos(os, type, putBOM);
204             StringStream s(json_);
205             while (s.Peek() != '\0') {
206                 bool success = Transcoder<UTF8<>, AutoUTF<unsigned> >::Transcode(s, eos);
207                 EXPECT_TRUE(success);
208             }
209             eos.Flush();
210             fclose(fp);
211             EXPECT_TRUE(CompareFile(filename, expectedFilename));
212             remove(filename);
213         }
214 
215         // Test MemoryBuffer
216         {
217             MemoryBuffer mb;
218             AutoUTFOutputStream<unsigned, MemoryBuffer> eos(mb, type, putBOM);
219             StringStream s(json_);
220             while (s.Peek() != '\0') {
221                 bool success = Transcoder<UTF8<>, AutoUTF<unsigned> >::Transcode(s, eos);
222                 EXPECT_TRUE(success);
223             }
224             eos.Flush();
225             EXPECT_TRUE(CompareBufferFile(mb.GetBuffer(), mb.GetSize(), expectedFilename));
226         }
227     }
228 
CompareFile(const char * filename,const char * expectedFilename)229     bool CompareFile(const char* filename, const char* expectedFilename) {
230         size_t actualLength, expectedLength;
231         char* actualBuffer = ReadFile(filename, false, &actualLength);
232         char* expectedBuffer = ReadFile(expectedFilename, true, &expectedLength);
233         bool ret = (expectedLength == actualLength) && memcmp(expectedBuffer, actualBuffer, actualLength) == 0;
234         free(actualBuffer);
235         free(expectedBuffer);
236         return ret;
237     }
238 
CompareBufferFile(const char * actualBuffer,size_t actualLength,const char * expectedFilename)239     bool CompareBufferFile(const char* actualBuffer, size_t actualLength, const char* expectedFilename) {
240         size_t expectedLength;
241         char* expectedBuffer = ReadFile(expectedFilename, true, &expectedLength);
242         bool ret = (expectedLength == actualLength) && memcmp(expectedBuffer, actualBuffer, actualLength) == 0;
243         free(expectedBuffer);
244         return ret;
245     }
246 
247     char *json_;
248     size_t length_;
249 };
250 
TEST_F(EncodedStreamTest,EncodedInputStream)251 TEST_F(EncodedStreamTest, EncodedInputStream) {
252     TestEncodedInputStream<UTF8<>,    UTF8<>  >("utf8.json");
253     TestEncodedInputStream<UTF8<>,    UTF8<>  >("utf8bom.json");
254     TestEncodedInputStream<UTF16LE<>, UTF16<> >("utf16le.json");
255     TestEncodedInputStream<UTF16LE<>, UTF16<> >("utf16lebom.json");
256     TestEncodedInputStream<UTF16BE<>, UTF16<> >("utf16be.json");
257     TestEncodedInputStream<UTF16BE<>, UTF16<> >("utf16bebom.json");
258     TestEncodedInputStream<UTF32LE<>, UTF32<> >("utf32le.json");
259     TestEncodedInputStream<UTF32LE<>, UTF32<> >("utf32lebom.json");
260     TestEncodedInputStream<UTF32BE<>, UTF32<> >("utf32be.json");
261     TestEncodedInputStream<UTF32BE<>, UTF32<> >("utf32bebom.json");
262 }
263 
TEST_F(EncodedStreamTest,AutoUTFInputStream)264 TEST_F(EncodedStreamTest, AutoUTFInputStream) {
265     TestAutoUTFInputStream("utf8.json",      false);
266     TestAutoUTFInputStream("utf8bom.json",   true);
267     TestAutoUTFInputStream("utf16le.json",   false);
268     TestAutoUTFInputStream("utf16lebom.json",true);
269     TestAutoUTFInputStream("utf16be.json",   false);
270     TestAutoUTFInputStream("utf16bebom.json",true);
271     TestAutoUTFInputStream("utf32le.json",   false);
272     TestAutoUTFInputStream("utf32lebom.json",true);
273     TestAutoUTFInputStream("utf32be.json",   false);
274     TestAutoUTFInputStream("utf32bebom.json", true);
275 
276     {
277         // Auto detection fail, use user defined UTF type
278         const char json[] = "{ }";
279         MemoryStream ms(json, sizeof(json));
280         AutoUTFInputStream<unsigned, MemoryStream> eis(ms, kUTF8);
281         EXPECT_FALSE(eis.HasBOM());
282         EXPECT_EQ(kUTF8, eis.GetType());
283     }
284 }
285 
TEST_F(EncodedStreamTest,EncodedOutputStream)286 TEST_F(EncodedStreamTest, EncodedOutputStream) {
287     TestEncodedOutputStream<UTF8<>,     UTF8<>  >("utf8.json",      false);
288     TestEncodedOutputStream<UTF8<>,     UTF8<>  >("utf8bom.json",   true);
289     TestEncodedOutputStream<UTF16LE<>,  UTF16<> >("utf16le.json",   false);
290     TestEncodedOutputStream<UTF16LE<>,  UTF16<> >("utf16lebom.json",true);
291     TestEncodedOutputStream<UTF16BE<>,  UTF16<> >("utf16be.json",   false);
292     TestEncodedOutputStream<UTF16BE<>,  UTF16<> >("utf16bebom.json",true);
293     TestEncodedOutputStream<UTF32LE<>,  UTF32<> >("utf32le.json",   false);
294     TestEncodedOutputStream<UTF32LE<>,  UTF32<> >("utf32lebom.json",true);
295     TestEncodedOutputStream<UTF32BE<>,  UTF32<> >("utf32be.json",   false);
296     TestEncodedOutputStream<UTF32BE<>,  UTF32<> >("utf32bebom.json",true);
297 }
298 
TEST_F(EncodedStreamTest,AutoUTFOutputStream)299 TEST_F(EncodedStreamTest, AutoUTFOutputStream) {
300     TestAutoUTFOutputStream(kUTF8,      false,  "utf8.json");
301     TestAutoUTFOutputStream(kUTF8,      true,   "utf8bom.json");
302     TestAutoUTFOutputStream(kUTF16LE,   false,  "utf16le.json");
303     TestAutoUTFOutputStream(kUTF16LE,   true,   "utf16lebom.json");
304     TestAutoUTFOutputStream(kUTF16BE,   false,  "utf16be.json");
305     TestAutoUTFOutputStream(kUTF16BE,   true,   "utf16bebom.json");
306     TestAutoUTFOutputStream(kUTF32LE,   false,  "utf32le.json");
307     TestAutoUTFOutputStream(kUTF32LE,   true,   "utf32lebom.json");
308     TestAutoUTFOutputStream(kUTF32BE,   false,  "utf32be.json");
309     TestAutoUTFOutputStream(kUTF32BE,   true,   "utf32bebom.json");
310 }
311