1 /** 2 * Copyright 2019 Huawei Technologies Co., Ltd 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 #ifndef MINDSPORE_CCSRC_MINDDATA_DATASET_UTIL_STORAGE_MANAGER_H_ 17 #define MINDSPORE_CCSRC_MINDDATA_DATASET_UTIL_STORAGE_MANAGER_H_ 18 19 #include <unistd.h> 20 21 #include <memory> 22 #include <string> 23 #include <utility> 24 #include <vector> 25 26 #include "minddata/dataset/engine/cache/storage_container.h" 27 #include "minddata/dataset/util/allocator.h" 28 #include "minddata/dataset/util/auto_index.h" 29 #include "minddata/dataset/util/lock.h" 30 #include "minddata/dataset/util/memory_pool.h" 31 #include "minddata/dataset/util/path.h" 32 #include "minddata/dataset/util/service.h" 33 #include "minddata/dataset/util/slice.h" 34 35 using ListOfContainers = std::vector<std::shared_ptr<mindspore::dataset::StorageContainer>>; 36 37 namespace mindspore { 38 namespace dataset { 39 class StorageManager : public Service { 40 public: 41 // Use these traits for the B+ tree inside the StorageManager 42 struct StorageBPlusTreeTraits { 43 // This determines the limit of number of keys in a node. 44 using slot_type = uint16_t; 45 // Number of slots in each leaf of the tree. 46 static constexpr slot_type kLeafSlots = 512; 47 // Number of slots in each inner node of the tree 48 static constexpr slot_type kInnerSlots = 256; 49 }; 50 using value_type = std::pair<int, std::pair<off_t, size_t>>; 51 using storage_index = AutoIndexObj<value_type, std::allocator<value_type>, StorageBPlusTreeTraits>; 52 using key_type = storage_index::key_type; 53 constexpr static int32_t kMaxNumContainers = 1000; 54 55 explicit StorageManager(const Path &); 56 57 StorageManager(const Path &root, size_t pool_size); 58 59 ~StorageManager() override; 60 61 StorageManager(const StorageManager &) = delete; 62 63 StorageManager &operator=(const StorageManager &) = delete; 64 65 Status Write(key_type *out_key, const std::vector<ReadableSlice> &buf); 66 67 Status Read(key_type key, WritableSlice *dest, size_t *bytesRead) const; 68 69 Status DoServiceStart() override; 70 71 Status DoServiceStop() noexcept override; 72 73 friend std::ostream &operator<<(std::ostream &os, const StorageManager &s); 74 75 private: 76 Path root_; 77 ListOfContainers containers_; 78 int file_id_; 79 RWLock rw_lock_; 80 storage_index index_; 81 std::vector<size_t> writable_containers_pool_; 82 size_t pool_size_; 83 84 static std::string GetBaseName(const std::string &prefix, int32_t file_id); 85 86 static std::string ConstructFileName(const std::string &prefix, int32_t file_id, const std::string &suffix); 87 88 /// \brief Add a new storage container 89 /// The newly-created container is going to be added into a pool of writable containers. 90 /// \param replaced_container_pos If provided, will use the newly created container to replace the corresponding old 91 /// container in the pool. If not provided, will just append the newly created container to the end of the pool. 92 /// \return Status object 93 Status AddOneContainer(int replaced_container_pos = -1); 94 }; 95 } // namespace dataset 96 } // namespace mindspore 97 98 #endif // MINDSPORE_CCSRC_MINDDATA_DATASET_UTIL_STORAGE_MANAGER_H_ 99