1 /*
2 * Copyright (c) Meta Platforms, Inc. and affiliates.
3 * All rights reserved.
4 *
5 * This source code is licensed under both the BSD-style license (found in the
6 * LICENSE file in the root directory of this source tree) and the GPLv2 (found
7 * in the COPYING file in the root directory of this source tree).
8 */
9 #include "Pzstd.h"
10 #include "datagen.h"
11 #include "test/RoundTrip.h"
12 #include "utils/ScopeGuard.h"
13
14 #include <cstddef>
15 #include <cstdio>
16 #include <gtest/gtest.h>
17 #include <memory>
18 #include <random>
19
20 using namespace std;
21 using namespace pzstd;
22
TEST(Pzstd,SmallSizes)23 TEST(Pzstd, SmallSizes) {
24 unsigned seed = std::random_device{}();
25 std::fprintf(stderr, "Pzstd.SmallSizes seed: %u\n", seed);
26 std::mt19937 gen(seed);
27
28 for (unsigned len = 1; len < 256; ++len) {
29 if (len % 16 == 0) {
30 std::fprintf(stderr, "%u / 16\n", len / 16);
31 }
32 std::string inputFile = std::tmpnam(nullptr);
33 auto guard = makeScopeGuard([&] { std::remove(inputFile.c_str()); });
34 {
35 static uint8_t buf[256];
36 RDG_genBuffer(buf, len, 0.5, 0.0, gen());
37 auto fd = std::fopen(inputFile.c_str(), "wb");
38 auto written = std::fwrite(buf, 1, len, fd);
39 std::fclose(fd);
40 ASSERT_EQ(written, len);
41 }
42 for (unsigned numThreads = 1; numThreads <= 2; ++numThreads) {
43 for (unsigned level = 1; level <= 4; level *= 4) {
44 auto errorGuard = makeScopeGuard([&] {
45 std::fprintf(stderr, "# threads: %u\n", numThreads);
46 std::fprintf(stderr, "compression level: %u\n", level);
47 });
48 Options options;
49 options.overwrite = true;
50 options.inputFiles = {inputFile};
51 options.numThreads = numThreads;
52 options.compressionLevel = level;
53 options.verbosity = 1;
54 ASSERT_TRUE(roundTrip(options));
55 errorGuard.dismiss();
56 }
57 }
58 }
59 }
60
TEST(Pzstd,LargeSizes)61 TEST(Pzstd, LargeSizes) {
62 unsigned seed = std::random_device{}();
63 std::fprintf(stderr, "Pzstd.LargeSizes seed: %u\n", seed);
64 std::mt19937 gen(seed);
65
66 for (unsigned len = 1 << 20; len <= (1 << 24); len *= 2) {
67 std::string inputFile = std::tmpnam(nullptr);
68 auto guard = makeScopeGuard([&] { std::remove(inputFile.c_str()); });
69 {
70 std::unique_ptr<uint8_t[]> buf(new uint8_t[len]);
71 RDG_genBuffer(buf.get(), len, 0.5, 0.0, gen());
72 auto fd = std::fopen(inputFile.c_str(), "wb");
73 auto written = std::fwrite(buf.get(), 1, len, fd);
74 std::fclose(fd);
75 ASSERT_EQ(written, len);
76 }
77 for (unsigned numThreads = 1; numThreads <= 16; numThreads *= 4) {
78 for (unsigned level = 1; level <= 4; level *= 4) {
79 auto errorGuard = makeScopeGuard([&] {
80 std::fprintf(stderr, "# threads: %u\n", numThreads);
81 std::fprintf(stderr, "compression level: %u\n", level);
82 });
83 Options options;
84 options.overwrite = true;
85 options.inputFiles = {inputFile};
86 options.numThreads = std::min(numThreads, options.numThreads);
87 options.compressionLevel = level;
88 options.verbosity = 1;
89 ASSERT_TRUE(roundTrip(options));
90 errorGuard.dismiss();
91 }
92 }
93 }
94 }
95
TEST(Pzstd,DISABLED_ExtremelyLargeSize)96 TEST(Pzstd, DISABLED_ExtremelyLargeSize) {
97 unsigned seed = std::random_device{}();
98 std::fprintf(stderr, "Pzstd.ExtremelyLargeSize seed: %u\n", seed);
99 std::mt19937 gen(seed);
100
101 std::string inputFile = std::tmpnam(nullptr);
102 auto guard = makeScopeGuard([&] { std::remove(inputFile.c_str()); });
103
104 {
105 // Write 4GB + 64 MB
106 constexpr size_t kLength = 1 << 26;
107 std::unique_ptr<uint8_t[]> buf(new uint8_t[kLength]);
108 auto fd = std::fopen(inputFile.c_str(), "wb");
109 auto closeGuard = makeScopeGuard([&] { std::fclose(fd); });
110 for (size_t i = 0; i < (1 << 6) + 1; ++i) {
111 RDG_genBuffer(buf.get(), kLength, 0.5, 0.0, gen());
112 auto written = std::fwrite(buf.get(), 1, kLength, fd);
113 if (written != kLength) {
114 std::fprintf(stderr, "Failed to write file, skipping test\n");
115 return;
116 }
117 }
118 }
119
120 Options options;
121 options.overwrite = true;
122 options.inputFiles = {inputFile};
123 options.compressionLevel = 1;
124 if (options.numThreads == 0) {
125 options.numThreads = 1;
126 }
127 ASSERT_TRUE(roundTrip(options));
128 }
129
TEST(Pzstd,ExtremelyCompressible)130 TEST(Pzstd, ExtremelyCompressible) {
131 std::string inputFile = std::tmpnam(nullptr);
132 auto guard = makeScopeGuard([&] { std::remove(inputFile.c_str()); });
133 {
134 std::unique_ptr<uint8_t[]> buf(new uint8_t[10000]);
135 std::memset(buf.get(), 'a', 10000);
136 auto fd = std::fopen(inputFile.c_str(), "wb");
137 auto written = std::fwrite(buf.get(), 1, 10000, fd);
138 std::fclose(fd);
139 ASSERT_EQ(written, 10000);
140 }
141 Options options;
142 options.overwrite = true;
143 options.inputFiles = {inputFile};
144 options.numThreads = 1;
145 options.compressionLevel = 1;
146 ASSERT_TRUE(roundTrip(options));
147 }
148