1 /** 2 * Copyright 2019 Huawei Technologies Co., Ltd 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 #ifndef MINDSPORE_CCSRC_MINDDATA_DATASET_ENGINE_DATASETOPS_SOURCE_IO_BLOCK_H_ 17 #define MINDSPORE_CCSRC_MINDDATA_DATASET_ENGINE_DATASETOPS_SOURCE_IO_BLOCK_H_ 18 19 #include <string> 20 #include <vector> 21 22 #include "minddata/dataset/util/auto_index.h" 23 #include "minddata/dataset/util/status.h" 24 25 namespace mindspore { 26 namespace dataset { 27 // The IOBlock class is used to describe a "unit of work" that a storage leaf operator worker thread 28 // is responsible for acting on. 29 // The IOBlocks and it's derived classes abstracts a key-store and key-lookup interface where each 30 // block contains 1 to n keys, and the keys are used in conjunction with an index to provide the meta 31 // information for satisfying an IO request. 32 class IOBlock { 33 public: 34 enum IOBlockFlags : uint32_t { 35 kDeIoBlockNone = 0, 36 kDeIoBlockFlagEoe = 1u, // end of IOBlocks for one epoch 37 kDeIoBlockFlagEof = 1u << 1, // end of IOBlocks for entire program 38 kDeIoBlockFlagWait = 1u << 2 // control signal for workers to suspend operations 39 }; 40 41 // Constructor of the IOBlock (1). A simpler one for the case when the block only has 1 key. 42 // @param inKey - A single key to add into the block 43 // @param io_block_flags - The flag setting for the block 44 IOBlock(int64_t inKey, IOBlockFlags io_block_flags); 45 46 // Constructor of the IOBlock (2). 47 // @param in_keys - A vector of keys to add into the block 48 // @param io_block_flags - The flag setting for the block 49 IOBlock(const std::vector<int64_t> &in_keys, IOBlockFlags io_block_flags); 50 51 // Constructor of the IOBlock (3). A special IOBlock that is used for control messaging. 52 // @param io_block_flags - The flag setting for the block 53 explicit IOBlock(IOBlockFlags io_block_flags); 54 55 // Destructor 56 virtual ~IOBlock() = default; 57 58 // Fetches the first key from the block. 59 // @note Only useful if you know the block only has 1 key. 60 // @return A copy of the first key from the block 61 // @return Status The status code returned 62 Status GetKey(int64_t *out_key) const; 63 64 // Fetches the list of keys from this block. 65 // @param out_keys - A copy of the vector of keys from the block. 66 // @return Status The status code returned 67 Status GetKeys(std::vector<int64_t> *out_keys) const; 68 69 // Does this block have the eoe flag turned on? 70 // @return T/F if the IOBlock is eoe eoe()71 bool eoe() const { return static_cast<uint32_t>(io_block_flags_) & static_cast<uint32_t>(kDeIoBlockFlagEoe); } 72 73 // Does this block have the eof flag turned on? 74 // @return T/F if the IOBlock is eof eof()75 bool eof() const { return static_cast<uint32_t>(io_block_flags_) & static_cast<uint32_t>(kDeIoBlockFlagEof); } 76 77 // Does this block have the wait flag turned on? 78 // @return T/F is the IOBlock is wait wait()79 bool wait() const { return static_cast<uint32_t>(io_block_flags_) & static_cast<uint32_t>(kDeIoBlockFlagWait); } 80 81 // Adds a key to this block 82 // @param key - The key to add to this block AddKey(int64_t key)83 void AddKey(int64_t key) { index_keys_.push_back(key); } 84 85 protected: 86 std::vector<int64_t> index_keys_; // keys used for lookups to the meta info for the data 87 IOBlockFlags io_block_flags_; 88 }; // class IOBlock 89 90 const int64_t kInvalidOffset = -1; 91 92 // The Filename block derived class implements a style of IO block where each block contains only a 93 // single key that maps to a filename. 94 class FilenameBlock : public IOBlock { 95 public: 96 // Constructor of the FilenameBlock (1) 97 // @param key - The key identifier that can be used to find the data for this block 98 // @param start_offset - Start offset 99 // @param end_offset - End offset 100 // @param io_block_flags - The flag setting for the block 101 FilenameBlock(int64_t key, int64_t start_offset, int64_t end_offset, IOBlockFlags io_block_flags); 102 103 // Constructor of the FilenameBlock (2). A special IOBlock that is used for control messaging. 104 // @param io_block_flags - The flag setting for the block 105 explicit FilenameBlock(IOBlockFlags io_block_flags); 106 107 // Destructor 108 ~FilenameBlock() = default; 109 110 // Gets the filename from the block using the provided index container 111 // @param out_filename - The filename to add to the block 112 // @param index - The index to perform lookup against 113 // @return Status The status code returned 114 Status GetFilename(std::string *out_filename, const AutoIndexObj<std::string> &index) const; 115 116 // Get the start offset of file 117 // @return int64_t - Start offset GetStartOffset()118 int64_t GetStartOffset() const { return start_offset_; } 119 120 // Get the end offset of the file 121 // @return int64_t - Start offset GetEndOffset()122 int64_t GetEndOffset() const { return end_offset_; } 123 124 private: 125 int64_t start_offset_; 126 int64_t end_offset_; 127 }; // class TFBlock 128 } // namespace dataset 129 } // namespace mindspore 130 #endif // MINDSPORE_CCSRC_MINDDATA_DATASET_ENGINE_DATASETOPS_SOURCE_IO_BLOCK_H_ 131