1 /* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
2
3 Licensed under the Apache License, Version 2.0 (the "License");
4 you may not use this file except in compliance with the License.
5 You may obtain a copy of the License at
6
7 http://www.apache.org/licenses/LICENSE-2.0
8
9 Unless required by applicable law or agreed to in writing, software
10 distributed under the License is distributed on an "AS IS" BASIS,
11 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 See the License for the specific language governing permissions and
13 limitations under the License.
14 ==============================================================================*/
15
16 #include "tensorflow/core/distributed_runtime/eager/remote_execute_node.h"
17
18 #include <memory>
19 #include <vector>
20
21 #include "absl/strings/str_cat.h"
22 #include "absl/strings/str_join.h"
23
24 namespace tensorflow {
25 namespace eager {
26
RunAsync(StatusCallback done)27 void RemoteExecuteNode::RunAsync(StatusCallback done) {
28 auto response = std::make_shared<EnqueueResponse>();
29
30 const gtl::InlinedVector<TensorHandle*, 4>& inputs = inputs_;
31 const gtl::InlinedVector<TensorHandle*, 2>& retvals = retvals_;
32 Device* device = device_;
33
34 // Filled and used only when VLOG(3) is on.
35 string rpc_description;
36 if (VLOG_IS_ON(3)) {
37 std::vector<string> ops;
38 ops.reserve(request_->queue_size());
39 for (const QueueItem& item : request_->queue()) {
40 if (item.has_operation()) {
41 ops.push_back(item.operation().name());
42 } else {
43 ops.push_back(absl::StrCat("DeleteHandle(",
44 item.handle_to_decref().op_id(), ":",
45 item.handle_to_decref().output_num(), ")"));
46 }
47 }
48 rpc_description =
49 absl::StrCat("RemoteOperation(", absl::StrJoin(ops, ", "), ")");
50 }
51 VLOG(3) << "Issuing: " << rpc_description;
52
53 CancellationManager* cm = cancellation_manager_;
54 CancellationToken token = 0;
55 auto call_opts = std::make_shared<CallOptions>();
56 call_opts->SetTimeout(
57 eager_context_->session_options().config.operation_timeout_in_ms());
58 if (cm != nullptr) {
59 token = cm->get_cancellation_token();
60 const bool already_cancelled = !cm->RegisterCallback(
61 token, [call_opts, response, done]() { call_opts->StartCancel(); });
62 if (already_cancelled) {
63 Status s = errors::Cancelled("RemoteExecuteNode::RunAsync");
64 for (size_t i = 0; i < retvals.size(); ++i) {
65 retvals[i]->PoisonRemote(s, device, context_view_id_);
66 }
67 done(s);
68 return;
69 }
70 }
71
72 for (auto handle : inputs_) {
73 handle->Ref();
74 }
75 for (auto handle : retvals) {
76 handle->Ref();
77 }
78
79 eager_client_->StreamingEnqueueAsync(
80 eager_context_->Executor().StreamingEnqueue(), call_opts.get(),
81 request_.get(), response.get(),
82 [inputs, retvals, call_opts, response, device,
83 context_view_id = context_view_id_, rpc_description, cm, token,
84 done](const Status& status) {
85 if (cm != nullptr) {
86 cm->TryDeregisterCallback(token);
87 }
88 for (auto handle : inputs) {
89 handle->Unref();
90 }
91 if (status.ok()) {
92 VLOG(3) << "Completed successfully: " << rpc_description;
93 } else {
94 VLOG(3) << "Failed: " << rpc_description << " with status "
95 << status.ToString();
96 }
97 for (size_t i = 0; i < retvals.size(); ++i) {
98 if (status.ok()) {
99 const string output_device =
100 response->queue_response(0).device().empty()
101 ? ""
102 : response->queue_response(0).device(i);
103 Status s = retvals[i]->SetRemoteShapeAndDevice(
104 response->queue_response(0).shape(i), device, context_view_id,
105 output_device);
106
107 if (!s.ok()) {
108 LOG(ERROR) << "Ignoring an error encountered when setting "
109 "remote shape of tensor handle: "
110 << retvals[i]
111 << " with execute status: " << status.ToString()
112 << " and SetRemoteShape status: " << s.ToString()
113 << "\nThis should never happen. "
114 "Please file an issue with the TensorFlow Team.";
115 }
116 } else {
117 retvals[i]->PoisonRemote(status, device, context_view_id);
118 }
119 retvals[i]->Unref();
120 }
121 done(status);
122 });
123 }
124
125 } // namespace eager
126 } // namespace tensorflow
127