1 /* Copyright 2018 The TensorFlow Authors. All Rights Reserved. 2 3 Licensed under the Apache License, Version 2.0 (the "License"); 4 you may not use this file except in compliance with the License. 5 You may obtain a copy of the License at 6 7 http://www.apache.org/licenses/LICENSE-2.0 8 9 Unless required by applicable law or agreed to in writing, software 10 distributed under the License is distributed on an "AS IS" BASIS, 11 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 See the License for the specific language governing permissions and 13 limitations under the License. 14 ==============================================================================*/ 15 #ifndef TENSORFLOW_CORE_COMMON_RUNTIME_COLLECTIVE_RMA_LOCAL_H_ 16 #define TENSORFLOW_CORE_COMMON_RUNTIME_COLLECTIVE_RMA_LOCAL_H_ 17 #include "tensorflow/core/common_runtime/buf_rendezvous.h" 18 #include "tensorflow/core/common_runtime/device_mgr.h" 19 #include "tensorflow/core/framework/collective.h" 20 #include "tensorflow/core/framework/rendezvous.h" 21 22 namespace tensorflow { 23 24 // Basic implementation of PerStepCollectiveRemoteAccess. 25 class CollectiveRemoteAccessLocal : public PerStepCollectiveRemoteAccess { 26 public: CollectiveRemoteAccessLocal(const DeviceMgr * dev_mgr,DeviceResolverInterface * dev_resolver,int64 step_id)27 CollectiveRemoteAccessLocal(const DeviceMgr* dev_mgr, 28 DeviceResolverInterface* dev_resolver, 29 int64 step_id) 30 : dev_mgr_(dev_mgr), 31 dev_resolver_(dev_resolver), 32 buf_rendezvous_(step_id), 33 step_id_(step_id) {} 34 ~CollectiveRemoteAccessLocal()35 virtual ~CollectiveRemoteAccessLocal() {} 36 37 void StartAbort(const Status& s) override; 38 39 void RecvFromPeer(const string& peer_device, const string& peer_task, 40 bool peer_is_local, const string& key, Device* to_device, 41 DeviceContext* to_device_ctx, 42 const AllocatorAttributes& to_alloc_attr, Tensor* to_tensor, 43 const DeviceLocality& client_locality, 44 int dev_to_dev_stream_index, 45 const StatusCallback& done) override; 46 47 void PostToPeer(const string& peer_device, const string& peer_task, 48 const string& key, Device* from_device, 49 DeviceContext* from_device_ctx, 50 const AllocatorAttributes& from_alloc_attr, 51 const Tensor* from_tensor, 52 const DeviceLocality& client_locality, 53 const StatusCallback& done) override; 54 GetDeviceLocalitiesAsync(const CollInstanceParams & ci_params,std::vector<DeviceLocality> * localities,const StatusCallback & done)55 void GetDeviceLocalitiesAsync(const CollInstanceParams& ci_params, 56 std::vector<DeviceLocality>* localities, 57 const StatusCallback& done) override { 58 dev_resolver_->GetDeviceLocalitiesAsync(ci_params, localities, done); 59 } 60 GetLocalityAsync(const string & device,const string & task,DeviceLocality * locality,const StatusCallback & done)61 void GetLocalityAsync(const string& device, const string& task, 62 DeviceLocality* locality, 63 const StatusCallback& done) override { 64 dev_resolver_->GetLocalityAsync(device, task, locality, done); 65 } 66 ClearTask(const string & task)67 void ClearTask(const string& task) override { 68 dev_resolver_->ClearTask(task); 69 } 70 buf_rendezvous()71 BufRendezvous* buf_rendezvous() override { return &buf_rendezvous_; } 72 73 // Copy utility that always copies bytes from src to dst even if 74 // they are on the same device, unlike CopyTensor::ViaDMA which will 75 // just change the dst buffer pointer in that case. 76 static void MemCpyAsync(DeviceContext* src_dev_ctx, 77 DeviceContext* dst_dev_ctx, Device* src_dev, 78 Device* dst_dev, const AllocatorAttributes& src_attr, 79 const AllocatorAttributes& dst_attr, 80 const Tensor* src, Tensor* dst, 81 int dev_to_dev_stream_index, 82 const StatusCallback& done); 83 84 protected: 85 const DeviceMgr* dev_mgr_; // not owned 86 DeviceResolverInterface* dev_resolver_; // not owned 87 BufRendezvous buf_rendezvous_; 88 int64 step_id_; 89 }; 90 91 } // namespace tensorflow 92 #endif // TENSORFLOW_CORE_COMMON_RUNTIME_COLLECTIVE_RMA_LOCAL_H_ 93