1syntax = "proto3"; 2 3package tensorflow.data; 4 5option go_package = "github.com/tensorflow/tensorflow/tensorflow/go/core/protobuf/for_core_protos_go_proto"; 6 7message ProcessingModeDef { 8 // Specifies how data is sharded among tf.data service workers. 9 enum ShardingPolicy { 10 // No sharding will be performed. Each worker produces the entire dataset 11 // without any sharding. With this mode, the best practice is to shuffle the 12 // dataset nondeterministically so that workers process the dataset in 13 // different orders. 14 OFF = 0; 15 16 // The input dataset is dynamically split among workers at runtime. Each 17 // worker gets the next split when it reads data from the dispatcher. There 18 // is no fixed sharding with this mode. 19 DYNAMIC = 1; 20 21 // The following are static sharding policies. The semantics are similar to 22 // `tf.data.experimental.AutoShardPolicy`. These policies require: 23 // * The tf.data service cluster has a fixed size, and you need to specify 24 // the workers in DispatcherConfig. 25 // * Each client only reads from the local tf.data service worker. 26 // 27 // Shards by input files (each worker will get a set of files to process). 28 // When this option is selected, make sure that there is at least as many 29 // files as workers. If there are fewer input files than workers, a runtime 30 // error will be raised. 31 FILE = 2; 32 33 // Shards by elements produced by the dataset. Each worker will process the 34 // whole dataset and discard the portion that is not for itself. Note that 35 // for this mode to correctly partitions the dataset elements, the dataset 36 // needs to produce elements in a deterministic order. 37 DATA = 3; 38 39 // Attempts FILE-based sharding, falling back to DATA-based sharding on 40 // failures. 41 FILE_OR_DATA = 4; 42 43 // Looks for the presence of `shard(SHARD_HINT, ...)` which is treated as a 44 // placeholder to replace with `shard(num_workers, worker_index)`. 45 HINT = 5; 46 } 47 ShardingPolicy sharding_policy = 1; 48} 49