1 /** 2 * Copyright 2019 Huawei Technologies Co., Ltd 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 #ifndef MINDSPORE_CCSRC_MINDDATA_DATASET_UTIL_STORAGE_MANAGER_H_ 17 #define MINDSPORE_CCSRC_MINDDATA_DATASET_UTIL_STORAGE_MANAGER_H_ 18 19 #include <unistd.h> 20 #include <memory> 21 #include <string> 22 #include <utility> 23 #include <vector> 24 #include "minddata/dataset/engine/cache/storage_container.h" 25 #include "minddata/dataset/util/allocator.h" 26 #include "minddata/dataset/util/auto_index.h" 27 #include "minddata/dataset/util/lock.h" 28 #include "minddata/dataset/util/memory_pool.h" 29 #include "minddata/dataset/util/path.h" 30 #include "minddata/dataset/util/service.h" 31 #include "minddata/dataset/util/slice.h" 32 33 using ListOfContainers = std::vector<std::shared_ptr<mindspore::dataset::StorageContainer>>; 34 namespace mindspore { 35 namespace dataset { 36 class StorageManager : public Service { 37 public: 38 // Use these traits for the B+ tree inside the StorageManager 39 struct StorageBPlusTreeTraits { 40 // This determines the limit of number of keys in a node. 41 using slot_type = uint16_t; 42 // Number of slots in each leaf of the tree. 43 static constexpr slot_type kLeafSlots = 512; 44 // Number of slots in each inner node of the tree 45 static constexpr slot_type kInnerSlots = 256; 46 }; 47 using value_type = std::pair<int, std::pair<off_t, size_t>>; 48 using storage_index = AutoIndexObj<value_type, std::allocator<value_type>, StorageBPlusTreeTraits>; 49 using key_type = storage_index::key_type; 50 constexpr static int32_t kMaxNumContainers = 1000; 51 52 explicit StorageManager(const Path &); 53 54 StorageManager(const Path &root, int pool_size); 55 56 ~StorageManager() override; 57 58 StorageManager(const StorageManager &) = delete; 59 60 StorageManager &operator=(const StorageManager &) = delete; 61 62 Status Write(key_type *out_key, const std::vector<ReadableSlice> &buf); 63 64 Status Read(key_type key, WritableSlice *dest, size_t *bytesRead) const; 65 66 Status DoServiceStart() override; 67 68 Status DoServiceStop() noexcept override; 69 70 friend std::ostream &operator<<(std::ostream &os, const StorageManager &s); 71 72 private: 73 Path root_; 74 ListOfContainers containers_; 75 int file_id_; 76 RWLock rw_lock_; 77 storage_index index_; 78 std::vector<int> writable_containers_pool_; 79 int pool_size_; 80 81 std::string GetBaseName(const std::string &prefix, int32_t file_id); 82 83 std::string ConstructFileName(const std::string &prefix, int32_t file_id, const std::string &suffix); 84 85 /// \brief Add a new storage container 86 /// The newly-created container is going to be added into a pool of writable containers. 87 /// \param replaced_container_pos If provided, will use the newly created container to replace the corresponding old 88 /// container in the pool. If not provided, will just append the newly created container to the end of the pool. 89 /// \return Status object 90 Status AddOneContainer(int replaced_container_pos = -1); 91 }; 92 } // namespace dataset 93 } // namespace mindspore 94 95 #endif // MINDSPORE_CCSRC_MINDDATA_DATASET_UTIL_STORAGE_MANAGER_H_ 96