1 /*
2 * Copyright (C) 2021 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17 #include <stdint.h>
18 #include <stdlib.h>
19 #include <string.h>
20
21 #include <algorithm>
22 #include <atomic>
23 #include <memory>
24 #include <mutex>
25 #include <string>
26 #include <utility>
27
28 #include <7zCrc.h>
29 #include <Xz.h>
30 #include <XzCrc64.h>
31
32 #include <unwindstack/Log.h>
33
34 #include "MemoryXz.h"
35
36 namespace unwindstack {
37
38 // Statistics (used only for optional debug log messages).
39 static constexpr bool kLogMemoryXzUsage = false;
40 std::atomic_size_t MemoryXz::total_used_ = 0;
41 std::atomic_size_t MemoryXz::total_size_ = 0;
42 std::atomic_size_t MemoryXz::total_open_ = 0;
43
MemoryXz(Memory * memory,uint64_t addr,uint64_t size,const std::string & name)44 MemoryXz::MemoryXz(Memory* memory, uint64_t addr, uint64_t size, const std::string& name)
45 : compressed_memory_(memory), compressed_addr_(addr), compressed_size_(size), name_(name) {
46 total_open_ += 1;
47 }
48
Init()49 bool MemoryXz::Init() {
50 static std::once_flag crc_initialized;
51 std::call_once(crc_initialized, []() {
52 CrcGenerateTable();
53 Crc64GenerateTable();
54 });
55 if (compressed_size_ >= kMaxCompressedSize) {
56 return false;
57 }
58 if (!ReadBlocks()) {
59 return false;
60 }
61
62 // All blocks (except the last one) must have the same power-of-2 size.
63 if (blocks_.size() > 1) {
64 size_t block_size_log2 = __builtin_ctz(blocks_.front().decompressed_size);
65 auto correct_size = [=](XzBlock& b) { return b.decompressed_size == (1 << block_size_log2); };
66 if (std::all_of(blocks_.begin(), std::prev(blocks_.end()), correct_size) &&
67 blocks_.back().decompressed_size <= (1 << block_size_log2)) {
68 block_size_log2_ = block_size_log2;
69 } else {
70 // Inconsistent block-sizes. Decompress and merge everything now.
71 std::unique_ptr<uint8_t[]> data(new uint8_t[size_]);
72 size_t offset = 0;
73 for (XzBlock& block : blocks_) {
74 if (!Decompress(&block)) {
75 return false;
76 }
77 memcpy(data.get() + offset, block.decompressed_data.get(), block.decompressed_size);
78 offset += block.decompressed_size;
79 }
80 blocks_.clear();
81 blocks_.push_back(XzBlock{
82 .decompressed_data = std::move(data),
83 .decompressed_size = size_,
84 });
85 block_size_log2_ = 31; // Because 32 bits is too big (shift right by 32 is not allowed).
86 }
87 }
88
89 return true;
90 }
91
~MemoryXz()92 MemoryXz::~MemoryXz() {
93 total_used_ -= used_;
94 total_size_ -= size_;
95 total_open_ -= 1;
96 }
97
Read(uint64_t addr,void * buffer,size_t size)98 size_t MemoryXz::Read(uint64_t addr, void* buffer, size_t size) {
99 if (addr >= size_) {
100 return 0; // Read past the end.
101 }
102 uint8_t* dst = reinterpret_cast<uint8_t*>(buffer); // Position in the output buffer.
103 for (size_t i = addr >> block_size_log2_; i < blocks_.size(); i++) {
104 XzBlock* block = &blocks_[i];
105 if (block->decompressed_data == nullptr) {
106 if (!Decompress(block)) {
107 break;
108 }
109 }
110 size_t offset = (addr - (i << block_size_log2_)); // Start inside the block.
111 size_t copy_bytes = std::min<size_t>(size, block->decompressed_size - offset);
112 memcpy(dst, block->decompressed_data.get() + offset, copy_bytes);
113 dst += copy_bytes;
114 addr += copy_bytes;
115 size -= copy_bytes;
116 if (size == 0) {
117 break;
118 }
119 }
120 return dst - reinterpret_cast<uint8_t*>(buffer);
121 }
122
ReadBlocks()123 bool MemoryXz::ReadBlocks() {
124 static ISzAlloc alloc;
125 alloc.Alloc = [](ISzAllocPtr, size_t size) { return malloc(size); };
126 alloc.Free = [](ISzAllocPtr, void* ptr) { return free(ptr); };
127
128 // Read the compressed data, so we can quickly scan through the headers.
129 std::unique_ptr<uint8_t[]> compressed_data(new (std::nothrow) uint8_t[compressed_size_]);
130 if (compressed_data.get() == nullptr) {
131 return false;
132 }
133 if (!compressed_memory_->ReadFully(compressed_addr_, compressed_data.get(), compressed_size_)) {
134 return false;
135 }
136
137 // Implement the required interface for communication
138 // (written in C so we can not use virtual methods or member functions).
139 struct XzLookInStream : public ILookInStream, public ICompressProgress {
140 static SRes LookImpl(const ILookInStream* p, const void** buf, size_t* size) {
141 auto* ctx = reinterpret_cast<const XzLookInStream*>(p);
142 *buf = ctx->data + ctx->offset;
143 *size = std::min(*size, ctx->size - ctx->offset);
144 return SZ_OK;
145 }
146 static SRes SkipImpl(const ILookInStream* p, size_t len) {
147 auto* ctx = reinterpret_cast<XzLookInStream*>(const_cast<ILookInStream*>(p));
148 ctx->offset += len;
149 return SZ_OK;
150 }
151 static SRes ReadImpl(const ILookInStream* p, void* buf, size_t* size) {
152 auto* ctx = reinterpret_cast<const XzLookInStream*>(p);
153 *size = std::min(*size, ctx->size - ctx->offset);
154 memcpy(buf, ctx->data + ctx->offset, *size);
155 return SZ_OK;
156 }
157 static SRes SeekImpl(const ILookInStream* p, Int64* pos, ESzSeek origin) {
158 auto* ctx = reinterpret_cast<XzLookInStream*>(const_cast<ILookInStream*>(p));
159 switch (origin) {
160 case SZ_SEEK_SET:
161 ctx->offset = *pos;
162 break;
163 case SZ_SEEK_CUR:
164 ctx->offset += *pos;
165 break;
166 case SZ_SEEK_END:
167 ctx->offset = ctx->size + *pos;
168 break;
169 }
170 *pos = ctx->offset;
171 return SZ_OK;
172 }
173 static SRes ProgressImpl(const ICompressProgress*, UInt64, UInt64) { return SZ_OK; }
174 size_t offset;
175 uint8_t* data;
176 size_t size;
177 };
178 XzLookInStream callbacks;
179 callbacks.Look = &XzLookInStream::LookImpl;
180 callbacks.Skip = &XzLookInStream::SkipImpl;
181 callbacks.Read = &XzLookInStream::ReadImpl;
182 callbacks.Seek = &XzLookInStream::SeekImpl;
183 callbacks.Progress = &XzLookInStream::ProgressImpl;
184 callbacks.offset = 0;
185 callbacks.data = compressed_data.get();
186 callbacks.size = compressed_size_;
187
188 // Iterate over the internal XZ blocks without decompressing them.
189 CXzs xzs;
190 Xzs_Construct(&xzs);
191 Int64 end_offset = compressed_size_;
192 if (Xzs_ReadBackward(&xzs, &callbacks, &end_offset, &callbacks, &alloc) == SZ_OK) {
193 blocks_.reserve(Xzs_GetNumBlocks(&xzs));
194 size_t dst_offset = 0;
195 for (int s = xzs.num - 1; s >= 0; s--) {
196 const CXzStream& stream = xzs.streams[s];
197 size_t src_offset = stream.startOffset + XZ_STREAM_HEADER_SIZE;
198 for (size_t b = 0; b < stream.numBlocks; b++) {
199 const CXzBlockSizes& block = stream.blocks[b];
200 blocks_.push_back(XzBlock{
201 .decompressed_data = nullptr, // Lazy allocation and decompression.
202 .decompressed_size = static_cast<uint32_t>(block.unpackSize),
203 .compressed_offset = static_cast<uint32_t>(src_offset),
204 .compressed_size = static_cast<uint32_t>((block.totalSize + 3) & ~3u),
205 .stream_flags = stream.flags,
206 });
207 dst_offset += blocks_.back().decompressed_size;
208 src_offset += blocks_.back().compressed_size;
209 }
210 }
211 size_ = dst_offset;
212 total_size_ += dst_offset;
213 }
214 Xzs_Free(&xzs, &alloc);
215 return !blocks_.empty();
216 }
217
Decompress(XzBlock * block)218 bool MemoryXz::Decompress(XzBlock* block) {
219 static ISzAlloc alloc;
220 alloc.Alloc = [](ISzAllocPtr, size_t size) { return malloc(size); };
221 alloc.Free = [](ISzAllocPtr, void* ptr) { return free(ptr); };
222
223 // Read the compressed data for this block.
224 std::unique_ptr<uint8_t[]> compressed_data(new (std::nothrow) uint8_t[block->compressed_size]);
225 if (compressed_data.get() == nullptr) {
226 return false;
227 }
228 if (!compressed_memory_->ReadFully(compressed_addr_ + block->compressed_offset,
229 compressed_data.get(), block->compressed_size)) {
230 return false;
231 }
232
233 // Allocate decompressed memory.
234 std::unique_ptr<uint8_t[]> decompressed_data(new uint8_t[block->decompressed_size]);
235 if (decompressed_data == nullptr) {
236 return false;
237 }
238
239 // Decompress.
240 CXzUnpacker state{};
241 XzUnpacker_Construct(&state, &alloc);
242 state.streamFlags = block->stream_flags;
243 XzUnpacker_PrepareToRandomBlockDecoding(&state);
244 size_t decompressed_size = block->decompressed_size;
245 size_t compressed_size = block->compressed_size;
246 ECoderStatus status;
247 XzUnpacker_SetOutBuf(&state, decompressed_data.get(), decompressed_size);
248 int return_val =
249 XzUnpacker_Code(&state, /*decompressed_data=*/nullptr, &decompressed_size,
250 compressed_data.get(), &compressed_size, true, CODER_FINISH_END, &status);
251 XzUnpacker_Free(&state);
252 if (return_val != SZ_OK || status != CODER_STATUS_FINISHED_WITH_MARK) {
253 Log::Error("Cannot decompress \"%s\"", name_.c_str());
254 return false;
255 }
256
257 used_ += block->decompressed_size;
258 total_used_ += block->decompressed_size;
259 if (kLogMemoryXzUsage) {
260 Log::Info("decompressed memory: %zi%% of %ziKB (%zi files), %i%% of %iKB (%s)",
261 100 * total_used_ / total_size_, total_size_ / 1024, total_open_.load(),
262 100 * used_ / size_, size_ / 1024, name_.c_str());
263 }
264
265 block->decompressed_data = std::move(decompressed_data);
266 return true;
267 }
268
269 } // namespace unwindstack
270