1 //
2 // Copyright (C) 2012 The Android Open Source Project
3 //
4 // Licensed under the Apache License, Version 2.0 (the "License");
5 // you may not use this file except in compliance with the License.
6 // You may obtain a copy of the License at
7 //
8 // http://www.apache.org/licenses/LICENSE-2.0
9 //
10 // Unless required by applicable law or agreed to in writing, software
11 // distributed under the License is distributed on an "AS IS" BASIS,
12 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 // See the License for the specific language governing permissions and
14 // limitations under the License.
15 //
16
17 #include "update_engine/payload_generator/delta_diff_generator.h"
18
19 #include <errno.h>
20 #include <fcntl.h>
21 #include <inttypes.h>
22 #include <sys/stat.h>
23 #include <sys/types.h>
24
25 #include <algorithm>
26 #include <memory>
27 #include <string>
28 #include <utility>
29 #include <vector>
30
31 #include <base/logging.h>
32 #include <base/threading/simple_thread.h>
33
34 #include "update_engine/common/utils.h"
35 #include "update_engine/payload_consumer/delta_performer.h"
36 #include "update_engine/payload_consumer/file_descriptor.h"
37 #include "update_engine/payload_consumer/payload_constants.h"
38 #include "update_engine/payload_generator/ab_generator.h"
39 #include "update_engine/payload_generator/annotated_operation.h"
40 #include "update_engine/payload_generator/blob_file_writer.h"
41 #include "update_engine/payload_generator/cow_size_estimator.h"
42 #include "update_engine/payload_generator/delta_diff_utils.h"
43 #include "update_engine/payload_generator/full_update_generator.h"
44 #include "update_engine/payload_generator/merge_sequence_generator.h"
45 #include "update_engine/payload_generator/payload_file.h"
46 #include "update_engine/update_metadata.pb.h"
47
48 using std::string;
49 using std::unique_ptr;
50 using std::vector;
51
52 namespace chromeos_update_engine {
53
54 // bytes
55 const size_t kRootFSPartitionSize = static_cast<size_t>(2) * 1024 * 1024 * 1024;
56
57 class PartitionProcessor : public base::DelegateSimpleThread::Delegate {
IsDynamicPartition(const std::string & partition_name)58 bool IsDynamicPartition(const std::string& partition_name) {
59 for (const auto& group :
60 config_.target.dynamic_partition_metadata->groups()) {
61 const auto& names = group.partition_names();
62 if (std::find(names.begin(), names.end(), partition_name) !=
63 names.end()) {
64 return true;
65 }
66 }
67 return false;
68 }
69
70 public:
PartitionProcessor(const PayloadGenerationConfig & config,const PartitionConfig & old_part,const PartitionConfig & new_part,BlobFileWriter * file_writer,std::vector<AnnotatedOperation> * aops,std::vector<CowMergeOperation> * cow_merge_sequence,size_t * cow_size,std::unique_ptr<chromeos_update_engine::OperationsGenerator> strategy)71 explicit PartitionProcessor(
72 const PayloadGenerationConfig& config,
73 const PartitionConfig& old_part,
74 const PartitionConfig& new_part,
75 BlobFileWriter* file_writer,
76 std::vector<AnnotatedOperation>* aops,
77 std::vector<CowMergeOperation>* cow_merge_sequence,
78 size_t* cow_size,
79 std::unique_ptr<chromeos_update_engine::OperationsGenerator> strategy)
80 : config_(config),
81 old_part_(old_part),
82 new_part_(new_part),
83 file_writer_(file_writer),
84 aops_(aops),
85 cow_merge_sequence_(cow_merge_sequence),
86 cow_size_(cow_size),
87 strategy_(std::move(strategy)) {}
88 PartitionProcessor(PartitionProcessor&&) noexcept = default;
89
Run()90 void Run() override {
91 LOG(INFO) << "Started an async task to process partition "
92 << new_part_.name;
93 bool success = strategy_->GenerateOperations(
94 config_, old_part_, new_part_, file_writer_, aops_);
95 if (!success) {
96 // ABORT the entire process, so that developer can look
97 // at recent logs and diagnose what happened
98 LOG(FATAL) << "GenerateOperations(" << old_part_.name << ", "
99 << new_part_.name << ") failed";
100 }
101
102 bool snapshot_enabled =
103 config_.target.dynamic_partition_metadata &&
104 config_.target.dynamic_partition_metadata->snapshot_enabled();
105 if (!snapshot_enabled || !IsDynamicPartition(new_part_.name)) {
106 return;
107 }
108 // Skip cow size estimation if VABC isn't enabled
109 if (!config_.target.dynamic_partition_metadata->vabc_enabled()) {
110 return;
111 }
112 if (!old_part_.path.empty()) {
113 auto generator = MergeSequenceGenerator::Create(*aops_);
114 if (!generator || !generator->Generate(cow_merge_sequence_)) {
115 LOG(FATAL) << "Failed to generate merge sequence";
116 }
117 }
118
119 LOG(INFO) << "Estimating COW size for partition: " << new_part_.name;
120 // Need the contents of source/target image bytes when doing
121 // dry run.
122 auto target_fd = std::make_unique<EintrSafeFileDescriptor>();
123 target_fd->Open(new_part_.path.c_str(), O_RDONLY);
124
125 google::protobuf::RepeatedPtrField<InstallOperation> operations;
126
127 for (const AnnotatedOperation& aop : *aops_) {
128 *operations.Add() = aop.op;
129 }
130
131 FileDescriptorPtr source_fd = nullptr;
132 if (config_.enable_vabc_xor) {
133 source_fd = std::make_shared<EintrSafeFileDescriptor>();
134 source_fd->Open(old_part_.path.c_str(), O_RDONLY);
135 }
136
137 *cow_size_ = EstimateCowSize(
138 std::move(source_fd),
139 std::move(target_fd),
140 std::move(operations),
141 {cow_merge_sequence_->begin(), cow_merge_sequence_->end()},
142 config_.block_size,
143 config_.target.dynamic_partition_metadata->vabc_compression_param(),
144 new_part_.size,
145 config_.enable_vabc_xor);
146 LOG(INFO) << "Estimated COW size for partition: " << new_part_.name << " "
147 << *cow_size_;
148 }
149
150 private:
151 const PayloadGenerationConfig& config_;
152 const PartitionConfig& old_part_;
153 const PartitionConfig& new_part_;
154 BlobFileWriter* file_writer_;
155 std::vector<AnnotatedOperation>* aops_;
156 std::vector<CowMergeOperation>* cow_merge_sequence_;
157 size_t* cow_size_;
158 std::unique_ptr<chromeos_update_engine::OperationsGenerator> strategy_;
159 DISALLOW_COPY_AND_ASSIGN(PartitionProcessor);
160 };
161
GenerateUpdatePayloadFile(const PayloadGenerationConfig & config,const string & output_path,const string & private_key_path,uint64_t * metadata_size)162 bool GenerateUpdatePayloadFile(const PayloadGenerationConfig& config,
163 const string& output_path,
164 const string& private_key_path,
165 uint64_t* metadata_size) {
166 if (!config.version.Validate()) {
167 LOG(ERROR) << "Unsupported major.minor version: " << config.version.major
168 << "." << config.version.minor;
169 return false;
170 }
171
172 // Create empty payload file object.
173 PayloadFile payload;
174 TEST_AND_RETURN_FALSE(payload.Init(config));
175
176 ScopedTempFile data_file("CrAU_temp_data.XXXXXX", true);
177 {
178 off_t data_file_size = 0;
179 BlobFileWriter blob_file(data_file.fd(), &data_file_size);
180 if (config.is_delta) {
181 TEST_AND_RETURN_FALSE(config.source.partitions.size() ==
182 config.target.partitions.size());
183 }
184 PartitionConfig empty_part("");
185 std::vector<std::vector<AnnotatedOperation>> all_aops;
186 all_aops.resize(config.target.partitions.size());
187
188 std::vector<std::vector<CowMergeOperation>> all_merge_sequences;
189 all_merge_sequences.resize(config.target.partitions.size());
190
191 std::vector<size_t> all_cow_sizes(config.target.partitions.size(), 0);
192
193 std::vector<PartitionProcessor> partition_tasks{};
194 auto thread_count = std::min<int>(diff_utils::GetMaxThreads(),
195 config.target.partitions.size());
196 base::DelegateSimpleThreadPool thread_pool{"partition-thread-pool",
197 thread_count};
198 for (size_t i = 0; i < config.target.partitions.size(); i++) {
199 const PartitionConfig& old_part =
200 config.is_delta ? config.source.partitions[i] : empty_part;
201 const PartitionConfig& new_part = config.target.partitions[i];
202 LOG(INFO) << "Partition name: " << new_part.name;
203 LOG(INFO) << "Partition size: " << new_part.size;
204 LOG(INFO) << "Block count: " << new_part.size / config.block_size;
205
206 // Select payload generation strategy based on the config.
207 unique_ptr<OperationsGenerator> strategy;
208 if (!old_part.path.empty()) {
209 // Delta update.
210 LOG(INFO) << "Using generator ABGenerator() for partition "
211 << new_part.name;
212 strategy.reset(new ABGenerator());
213 } else {
214 LOG(INFO) << "Using generator FullUpdateGenerator() for partition "
215 << new_part.name;
216 strategy.reset(new FullUpdateGenerator());
217 }
218
219 // Generate the operations using the strategy we selected above.
220 partition_tasks.push_back(PartitionProcessor(config,
221 old_part,
222 new_part,
223 &blob_file,
224 &all_aops[i],
225 &all_merge_sequences[i],
226 &all_cow_sizes[i],
227 std::move(strategy)));
228 }
229 thread_pool.Start();
230 for (auto& processor : partition_tasks) {
231 thread_pool.AddWork(&processor);
232 }
233 thread_pool.JoinAll();
234
235 for (size_t i = 0; i < config.target.partitions.size(); i++) {
236 const PartitionConfig& old_part =
237 config.is_delta ? config.source.partitions[i] : empty_part;
238 const PartitionConfig& new_part = config.target.partitions[i];
239 TEST_AND_RETURN_FALSE(
240 payload.AddPartition(old_part,
241 new_part,
242 std::move(all_aops[i]),
243 std::move(all_merge_sequences[i]),
244 all_cow_sizes[i]));
245 }
246 }
247 data_file.CloseFd();
248
249 LOG(INFO) << "Writing payload file...";
250 // Write payload file to disk.
251 TEST_AND_RETURN_FALSE(payload.WritePayload(
252 output_path, data_file.path(), private_key_path, metadata_size));
253
254 LOG(INFO) << "All done. Successfully created delta file with "
255 << "metadata size = " << *metadata_size;
256 return true;
257 }
258
259 }; // namespace chromeos_update_engine
260