1 /** 2 * Copyright 2019 Huawei Technologies Co., Ltd 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 #ifndef MINDSPORE_CCSRC_MINDDATA_DATASET_ENGINE_DATASETOPS_SOURCE_IO_BLOCK_H_ 17 #define MINDSPORE_CCSRC_MINDDATA_DATASET_ENGINE_DATASETOPS_SOURCE_IO_BLOCK_H_ 18 19 #include <string> 20 #include <vector> 21 22 #include "minddata/dataset/util/auto_index.h" 23 #include "minddata/dataset/util/status.h" 24 25 namespace mindspore { 26 namespace dataset { 27 // The IOBlock class is used to describe a "unit of work" that a storage leaf operator worker thread 28 // is responsible for acting on. 29 // The IOBlocks and it's derived classes abstracts a key-store and key-lookup interface where each 30 // block contains 1 to n keys, and the keys are used in conjunction with an index to provide the meta 31 // information for satisfying an IO request. 32 class IOBlock { 33 public: 34 enum IOBlockFlags : uint32_t { 35 kFlagNone = 0, 36 kFlagEOE = 1U, // end of IOBlocks for one epoch 37 kFlagEOF = 1U << 1, // end of IOBlocks for entire program 38 kFlagWait = 1U << 2, // control signal for workers to suspend operations 39 kFlagQuit = 1U << 3 // control signal for workers to quit 40 }; 41 42 // Constructor of the IOBlock (1). A simpler one for the case when the block only has 1 key. 43 // @param inKey - A single key to add into the block 44 // @param io_block_flags - The flag setting for the block 45 IOBlock(int64_t inKey, IOBlockFlags io_block_flags); 46 47 // Constructor of the IOBlock (2). 48 // @param in_keys - A vector of keys to add into the block 49 // @param io_block_flags - The flag setting for the block 50 IOBlock(const std::vector<int64_t> &in_keys, IOBlockFlags io_block_flags); 51 52 // Constructor of the IOBlock (3). A special IOBlock that is used for control messaging. 53 // @param io_block_flags - The flag setting for the block 54 explicit IOBlock(IOBlockFlags io_block_flags); 55 56 // Destructor 57 virtual ~IOBlock() = default; 58 59 // Fetches the first key from the block. 60 // @note Only useful if you know the block only has 1 key. 61 // @return A copy of the first key from the block 62 // @return Status The status code returned 63 Status GetKey(int64_t *out_key) const; 64 65 // Fetches the list of keys from this block. 66 // @param out_keys - A copy of the vector of keys from the block. 67 // @return Status The status code returned 68 Status GetKeys(std::vector<int64_t> *out_keys) const; 69 70 // Does this block have the eoe flag turned on? 71 // @return T/F if the IOBlock is eoe eoe()72 bool eoe() const { return static_cast<uint32_t>(io_block_flags_) & static_cast<uint32_t>(kFlagEOE); } 73 74 // Does this block have the eof flag turned on? 75 // @return T/F if the IOBlock is eof eof()76 bool eof() const { return static_cast<uint32_t>(io_block_flags_) & static_cast<uint32_t>(kFlagEOF); } 77 78 // Does this block have the wait flag turned on? 79 // @return T/F is the IOBlock is wait wait()80 bool wait() const { return static_cast<uint32_t>(io_block_flags_) & static_cast<uint32_t>(kFlagWait); } 81 82 // Adds a key to this block 83 // @param key - The key to add to this block AddKey(int64_t key)84 void AddKey(int64_t key) { index_keys_.push_back(key); } 85 FlagName()86 std::string FlagName() const { 87 switch (io_block_flags_) { 88 case IOBlockFlags::kFlagNone: 89 return "Data"; 90 case IOBlockFlags::kFlagEOE: 91 return "EOE"; 92 case IOBlockFlags::kFlagEOF: 93 return "EOF"; 94 case IOBlockFlags::kFlagWait: 95 return "Wait"; 96 case IOBlockFlags::kFlagQuit: 97 return "Quit"; 98 default: 99 return "Unknown"; 100 } 101 } 102 103 protected: 104 std::vector<int64_t> index_keys_; // keys used for lookups to the meta info for the data 105 IOBlockFlags io_block_flags_; 106 }; // class IOBlock 107 108 const int64_t kInvalidOffset = -1; 109 110 // The Filename block derived class implements a style of IO block where each block contains only a 111 // single key that maps to a filename. 112 class FilenameBlock : public IOBlock { 113 public: 114 // Constructor of the FilenameBlock (1) 115 // @param key - The key identifier that can be used to find the data for this block 116 // @param start_offset - Start offset 117 // @param end_offset - End offset 118 // @param io_block_flags - The flag setting for the block 119 FilenameBlock(int64_t key, int64_t start_offset, int64_t end_offset, IOBlockFlags io_block_flags); 120 121 // Constructor of the FilenameBlock (2). A special IOBlock that is used for control messaging. 122 // @param io_block_flags - The flag setting for the block 123 explicit FilenameBlock(IOBlockFlags io_block_flags); 124 125 // Destructor 126 ~FilenameBlock() = default; 127 128 // Gets the filename from the block using the provided index container 129 // @param out_filename - The filename to add to the block 130 // @param index - The index to perform lookup against 131 // @return Status The status code returned 132 Status GetFilename(std::string *out_filename, const AutoIndexObj<std::string> &index) const; 133 134 // Get the start offset of file 135 // @return int64_t - Start offset GetStartOffset()136 int64_t GetStartOffset() const { return start_offset_; } 137 138 // Get the end offset of the file 139 // @return int64_t - Start offset GetEndOffset()140 int64_t GetEndOffset() const { return end_offset_; } 141 142 private: 143 int64_t start_offset_; 144 int64_t end_offset_; 145 }; // class TFBlock 146 } // namespace dataset 147 } // namespace mindspore 148 #endif // MINDSPORE_CCSRC_MINDDATA_DATASET_ENGINE_DATASETOPS_SOURCE_IO_BLOCK_H_ 149