1 // Copyright (C) 2021 The Android Open Source Project 2 // 3 // Licensed under the Apache License, Version 2.0 (the "License"); 4 // you may not use this file except in compliance with the License. 5 // You may obtain a copy of the License at 6 // 7 // http://www.apache.org/licenses/LICENSE-2.0 8 // 9 // Unless required by applicable law or agreed to in writing, software 10 // distributed under the License is distributed on an "AS IS" BASIS, 11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 // See the License for the specific language governing permissions and 13 // limitations under the License. 14 15 #pragma once 16 17 #include <linux/types.h> 18 #include <stdint.h> 19 #include <stdlib.h> 20 #include <sys/mman.h> 21 #include <sys/resource.h> 22 #include <sys/time.h> 23 #include <unistd.h> 24 25 #include <condition_variable> 26 #include <cstring> 27 #include <iostream> 28 #include <mutex> 29 #include <ostream> 30 #include <string> 31 #include <unordered_map> 32 #include <vector> 33 34 #include <android-base/file.h> 35 #include <android-base/logging.h> 36 #include <android-base/stringprintf.h> 37 #include <android-base/unique_fd.h> 38 #include <ext4_utils/ext4_utils.h> 39 #include <libdm/dm.h> 40 #include <libsnapshot/cow_reader.h> 41 #include <libsnapshot/cow_writer.h> 42 #include <snapuserd/block_server.h> 43 #include <snapuserd/snapuserd_buffer.h> 44 #include <snapuserd/snapuserd_kernel.h> 45 #include <storage_literals/storage_literals.h> 46 #include <system/thread_defs.h> 47 #include <user-space-merge/handler_manager.h> 48 #include "snapuserd_readahead.h" 49 #include "snapuserd_verify.h" 50 51 namespace android { 52 namespace snapshot { 53 54 using android::base::unique_fd; 55 using namespace std::chrono_literals; 56 using namespace android::storage_literals; 57 58 static constexpr size_t PAYLOAD_BUFFER_SZ = (1UL << 20); 59 static_assert(PAYLOAD_BUFFER_SZ >= BLOCK_SZ); 60 61 static constexpr int kNumWorkerThreads = 4; 62 63 #define SNAP_LOG(level) LOG(level) << misc_name_ << ": " 64 #define SNAP_PLOG(level) PLOG(level) << misc_name_ << ": " 65 66 enum class MERGE_IO_TRANSITION { 67 INVALID, 68 MERGE_READY, 69 MERGE_BEGIN, 70 MERGE_FAILED, 71 MERGE_COMPLETE, 72 IO_TERMINATED, 73 READ_AHEAD_FAILURE 74 }; 75 76 class MergeWorker; 77 class ReadWorker; 78 79 enum class MERGE_GROUP_STATE { 80 GROUP_MERGE_PENDING, 81 GROUP_MERGE_RA_READY, 82 GROUP_MERGE_IN_PROGRESS, 83 GROUP_MERGE_COMPLETED, 84 GROUP_MERGE_FAILED, 85 GROUP_INVALID, 86 }; 87 88 struct MergeGroupState { 89 MERGE_GROUP_STATE merge_state_; 90 // Ref count I/O when group state 91 // is in "GROUP_MERGE_PENDING" 92 size_t num_ios_in_progress; 93 std::mutex m_lock; 94 std::condition_variable m_cv; 95 MergeGroupStateMergeGroupState96 MergeGroupState(MERGE_GROUP_STATE state, size_t n_ios) 97 : merge_state_(state), num_ios_in_progress(n_ios) {} 98 }; 99 100 class SnapshotHandler : public std::enable_shared_from_this<SnapshotHandler> { 101 public: 102 SnapshotHandler(std::string misc_name, std::string cow_device, std::string backing_device, 103 std::string base_path_merge, std::shared_ptr<IBlockServerOpener> opener, 104 HandlerOptions options); 105 bool InitCowDevice(); 106 bool Start(); 107 GetControlDevicePath()108 const std::string& GetControlDevicePath() { return control_device_; } GetMiscName()109 const std::string& GetMiscName() { return misc_name_; } 110 uint64_t GetNumSectors() const; IsAttached()111 const bool& IsAttached() const { return attached_; } AttachControlDevice()112 void AttachControlDevice() { attached_ = true; } 113 114 bool CheckMergeCompletionStatus(); 115 bool CommitMerge(int num_merge_ops); 116 CloseFds()117 void CloseFds() { cow_fd_ = {}; } 118 void FreeResources(); 119 120 bool InitializeWorkers(); 121 std::unique_ptr<CowReader> CloneReaderForWorker(); GetSharedPtr()122 std::shared_ptr<SnapshotHandler> GetSharedPtr() { return shared_from_this(); } 123 GetChunkVec()124 std::vector<std::pair<sector_t, const CowOperation*>>& GetChunkVec() { return chunk_vec_; } 125 compare(std::pair<sector_t,const CowOperation * > p1,std::pair<sector_t,const CowOperation * > p2)126 static bool compare(std::pair<sector_t, const CowOperation*> p1, 127 std::pair<sector_t, const CowOperation*> p2) { 128 return p1.first < p2.first; 129 } 130 131 void UnmapBufferRegion(); 132 bool MmapMetadata(); 133 134 // Read-ahead related functions GetMappedAddr()135 void* GetMappedAddr() { return mapped_addr_; } 136 void PrepareReadAhead(); GetReadAheadMap()137 std::unordered_map<uint64_t, void*>& GetReadAheadMap() { return read_ahead_buffer_map_; } 138 139 // State transitions for merge 140 void InitiateMerge(); 141 void MonitorMerge(); 142 void WakeupMonitorMergeThread(); 143 void WaitForMergeComplete(); 144 bool WaitForMergeBegin(); 145 void RaThreadStarted(); 146 void WaitForRaThreadToStart(); 147 void NotifyRAForMergeReady(); 148 bool WaitForMergeReady(); 149 void MergeFailed(); 150 bool IsIOTerminated(); 151 void MergeCompleted(); 152 void NotifyIOTerminated(); 153 bool ReadAheadIOCompleted(bool sync); 154 void ReadAheadIOFailed(); 155 ShouldReconstructDataFromCow()156 bool ShouldReconstructDataFromCow() { return populate_data_from_cow_; } FinishReconstructDataFromCow()157 void FinishReconstructDataFromCow() { populate_data_from_cow_ = false; } 158 void MarkMergeComplete(); 159 // Return the snapshot status 160 std::string GetMergeStatus(); 161 162 // RA related functions 163 uint64_t GetBufferMetadataOffset(); 164 size_t GetBufferMetadataSize(); 165 size_t GetBufferDataOffset(); 166 size_t GetBufferDataSize(); 167 168 // Total number of blocks to be merged in a given read-ahead buffer region SetMergedBlockCountForNextCommit(int x)169 void SetMergedBlockCountForNextCommit(int x) { total_ra_blocks_merged_ = x; } GetTotalBlocksToMerge()170 int GetTotalBlocksToMerge() { return total_ra_blocks_merged_; } MergeInitiated()171 bool MergeInitiated() { return merge_initiated_; } MergeMonitored()172 bool MergeMonitored() { return merge_monitored_; } GetMergePercentage()173 double GetMergePercentage() { return merge_completion_percentage_; } 174 void PauseMergeThreads(); 175 void ResumeMergeThreads(); 176 void PauseMergeIfRequired(); 177 178 // Merge Block State Transitions 179 void SetMergeCompleted(size_t block_index); 180 void SetMergeInProgress(size_t block_index); 181 void SetMergeFailed(size_t block_index); 182 void NotifyIOCompletion(uint64_t new_block); 183 bool GetRABuffer(std::unique_lock<std::mutex>* lock, uint64_t block, void* buffer); 184 MERGE_GROUP_STATE ProcessMergingBlock(uint64_t new_block, void* buffer); 185 186 bool IsIouringSupported(); 187 bool CheckPartitionVerification(); GetBufferLock()188 std::mutex& GetBufferLock() { return buffer_lock_; } 189 190 private: 191 bool ReadMetadata(); ChunkToSector(chunk_t chunk)192 sector_t ChunkToSector(chunk_t chunk) { return chunk << CHUNK_SHIFT; } SectorToChunk(sector_t sector)193 chunk_t SectorToChunk(sector_t sector) { return sector >> CHUNK_SHIFT; } IsBlockAligned(uint64_t read_size)194 bool IsBlockAligned(uint64_t read_size) { return ((read_size & (BLOCK_SZ - 1)) == 0); } 195 struct BufferState* GetBufferState(); 196 void UpdateMergeCompletionPercentage(); 197 198 // COW device 199 std::string cow_device_; 200 // Source device 201 std::string backing_store_device_; 202 // dm-user control device 203 std::string control_device_; 204 std::string misc_name_; 205 // Base device for merging 206 std::string base_path_merge_; 207 208 unique_fd cow_fd_; 209 210 std::unique_ptr<CowReader> reader_; 211 212 // chunk_vec stores the pseudo mapping of sector 213 // to COW operations. 214 std::vector<std::pair<sector_t, const CowOperation*>> chunk_vec_; 215 216 std::mutex lock_; 217 std::condition_variable cv; 218 219 // Lock the buffer used for snapshot-merge 220 std::mutex buffer_lock_; 221 222 void* mapped_addr_; 223 size_t total_mapped_addr_length_; 224 225 std::vector<std::unique_ptr<ReadWorker>> worker_threads_; 226 // Read-ahead related 227 bool populate_data_from_cow_ = false; 228 bool ra_thread_ = false; 229 bool ra_thread_started_ = false; 230 int total_ra_blocks_merged_ = 0; 231 MERGE_IO_TRANSITION io_state_ = MERGE_IO_TRANSITION::INVALID; 232 std::unique_ptr<ReadAhead> read_ahead_thread_; 233 std::unordered_map<uint64_t, void*> read_ahead_buffer_map_; 234 235 // user-space-merging 236 std::unordered_map<uint64_t, int> block_to_ra_index_; 237 238 // Merge Block state 239 std::vector<std::unique_ptr<MergeGroupState>> merge_blk_state_; 240 241 std::unique_ptr<MergeWorker> merge_thread_; 242 double merge_completion_percentage_; 243 244 bool merge_initiated_ = false; 245 bool merge_monitored_ = false; 246 bool attached_ = false; 247 bool scratch_space_ = false; 248 int num_worker_threads_ = kNumWorkerThreads; 249 bool perform_verification_ = true; 250 bool resume_merge_ = false; 251 bool merge_complete_ = false; 252 HandlerOptions handler_options_; 253 std::unique_ptr<UpdateVerify> update_verify_; 254 std::shared_ptr<IBlockServerOpener> block_server_opener_; 255 256 // Pause merge threads 257 bool pause_merge_ = false; 258 std::mutex pause_merge_lock_; 259 std::condition_variable pause_merge_cv_; 260 }; 261 262 std::ostream& operator<<(std::ostream& os, MERGE_IO_TRANSITION value); 263 static_assert(sizeof(off_t) == sizeof(uint64_t)); 264 265 } // namespace snapshot 266 } // namespace android 267