1 /* Copyright 2020 The TensorFlow Authors. All Rights Reserved. 2 3 Licensed under the Apache License, Version 2.0 (the "License"); 4 you may not use this file except in compliance with the License. 5 You may obtain a copy of the License at 6 7 http://www.apache.org/licenses/LICENSE-2.0 8 9 Unless required by applicable law or agreed to in writing, software 10 distributed under the License is distributed on an "AS IS" BASIS, 11 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 See the License for the specific language governing permissions and 13 limitations under the License. 14 ==============================================================================*/ 15 16 #ifndef TENSORFLOW_CORE_KERNELS_DATA_SPLIT_UTILS_H_ 17 #define TENSORFLOW_CORE_KERNELS_DATA_SPLIT_UTILS_H_ 18 19 #include "tensorflow/core/framework/dataset.h" 20 21 namespace tensorflow { 22 namespace data { 23 24 // A class which produces splits for a dataset of size N that can be indexed 25 // into. 26 class IndexSplitProvider : public SplitProvider { 27 public: 28 explicit IndexSplitProvider(int64 n); 29 Status GetNext(Tensor* split, bool* end_of_splits) override; 30 Status Reset() override; 31 Status Save(std::function<std::string(std::string)> full_name, 32 IteratorStateWriter* writer) override; 33 Status Restore(std::function<std::string(std::string)> full_name, 34 IteratorStateReader* reader) override; 35 36 private: 37 mutex mu_; 38 int64 i_ TF_GUARDED_BY(mu_); 39 const int64 n_; 40 }; 41 42 // A SplitProvider which wraps another split provider, but drops all splits 43 // where `index != shard_index % num_shards` 44 class ShardingSplitProvider : public SplitProvider { 45 public: 46 ShardingSplitProvider(int64 num_shards, int64 shard_index, 47 std::shared_ptr<SplitProvider> split_provider); 48 49 Status GetNext(Tensor* split, bool* end_of_splits) override; 50 Status Reset() override; 51 Status Save(std::function<std::string(std::string)> full_name, 52 IteratorStateWriter* writer) override; 53 Status Restore(std::function<std::string(std::string)> full_name, 54 IteratorStateReader* reader) override; 55 56 private: 57 const int64 num_shards_; 58 const int64 shard_index_; 59 mutex mu_; 60 std::shared_ptr<SplitProvider> split_provider_ TF_GUARDED_BY(mu_); 61 int64 num_to_skip_ TF_GUARDED_BY(mu_); 62 }; 63 64 } // namespace data 65 } // namespace tensorflow 66 67 #endif // TENSORFLOW_CORE_KERNELS_DATA_SPLIT_UTILS_H_ 68