• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 /* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
2 
3 Licensed under the Apache License, Version 2.0 (the "License");
4 you may not use this file except in compliance with the License.
5 You may obtain a copy of the License at
6 
7     http://www.apache.org/licenses/LICENSE-2.0
8 
9 Unless required by applicable law or agreed to in writing, software
10 distributed under the License is distributed on an "AS IS" BASIS,
11 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 See the License for the specific language governing permissions and
13 limitations under the License.
14 ==============================================================================*/
15 
16 #include "tensorflow/core/kernels/data/serialization_utils.h"
17 
18 #include "tensorflow/core/graph/graph_def_builder.h"
19 #include "tensorflow/core/kernels/data/captured_function.h"
20 
21 namespace tensorflow {
22 namespace data {
23 
24 namespace {
25 
26 // FindStatefulOps searches `graph_def` for all of its stateful ops storing
27 // their names in `stateful_op_names`.
FindStatefulOps(const GraphDef & graph_def,std::vector<string> * stateful_op_names)28 Status FindStatefulOps(const GraphDef& graph_def,
29                        std::vector<string>* stateful_op_names) {
30   FunctionLibraryDefinition lib_def(OpRegistry::Global(), graph_def.library());
31 
32   // Iterate over all nodes in the graph.
33   for (const auto& node : graph_def.node()) {
34     // Each Dataset graph has a _Retval op in the end which is marked stateful
35     if (node.op() == FunctionLibraryDefinition::kRetOp) continue;
36     if (!IsNodeStateful(lib_def, node).ok()) {
37       stateful_op_names->push_back(node.op());
38     }
39   }
40 
41   // Iterate over all functions.
42   for (const auto& fdef : graph_def.library().function()) {
43     if (!fdef.signature().is_stateful()) continue;
44     for (const auto& node : fdef.node_def()) {
45       if (!IsNodeStateful(lib_def, node).ok()) {
46         stateful_op_names->push_back(
47             absl::StrCat(node.op(), " in function: ", fdef.signature().name()));
48       }
49     }
50   }
51   return Status::OK();
52 }
53 
54 }  // namespace
55 
AsGraphDefMinimal(OpKernelContext * ctx,const DatasetBase * input,std::vector<std::pair<string,Tensor>> * input_list,GraphDef * result,string * dataset_node)56 Status AsGraphDefMinimal(OpKernelContext* ctx, const DatasetBase* input,
57                          std::vector<std::pair<string, Tensor>>* input_list,
58                          GraphDef* result, string* dataset_node) {
59   SerializationContext::Params params;
60   params.input_list = input_list;
61   params.external_state_policy =
62       SerializationContext::ExternalStatePolicy::kIgnore;
63   params.fail_if_unimplemented = false;
64   params.serialize_data_tensors = false;
65   params.preserve_random_seeds = false;
66   SerializationContext serialization_ctx(params);
67   TF_RETURN_IF_ERROR(
68       AsGraphDef(ctx, input, std::move(serialization_ctx), result));
69 
70   // Symbolic `_Retval` node indicates which node corresponds to the dataset.
71   for (const auto& node : result->node()) {
72     if (node.op() == "_Retval") {
73       *dataset_node = node.input(0);
74     }
75   }
76   return Status::OK();
77 }
78 
AsGraphDef(OpKernelContext * ctx,const DatasetBase * dataset,SerializationContext && serialization_ctx,GraphDef * graph_def)79 Status AsGraphDef(OpKernelContext* ctx, const DatasetBase* dataset,
80                   SerializationContext&& serialization_ctx,
81                   GraphDef* graph_def) {
82   if (serialization_ctx.external_state_policy() ==
83       SerializationContext::ExternalStatePolicy::kFail) {
84     TF_RETURN_IF_ERROR(dataset->CheckExternalState());
85   }
86   if (serialization_ctx.external_state_policy() ==
87       SerializationContext::ExternalStatePolicy::kWarn) {
88     std::vector<string> stateful_op_names;
89     TF_RETURN_IF_ERROR(FindStatefulOps(*graph_def, &stateful_op_names));
90     if (!stateful_op_names.empty()) {
91       LOG(WARNING)
92           << "We found the following stateful ops in the dataset "
93              "construction graph whose state would not be serialized and might "
94              "cause subtle bugs: "
95           << absl::StrJoin(stateful_op_names, ", ");
96     }
97   }
98   GraphDefBuilder b;
99   DatasetBase::DatasetGraphDefBuilder db(&b);
100   Node* output_node = nullptr;
101   TF_RETURN_IF_ERROR(
102       db.AddInputDataset(&serialization_ctx, dataset, &output_node));
103   // Insert a purely symbolic _Retval node to indicate to consumers which node
104   // represents `dataset`.
105   ops::UnaryOp("_Retval", output_node,
106                b.opts()
107                    .WithName("dataset")
108                    .WithAttr("T", DT_VARIANT)
109                    .WithAttr("index", 0));
110   TF_RETURN_IF_ERROR(b.ToGraphDef(graph_def));
111   return Status::OK();
112 }
113 
114 }  // namespace data
115 }  // namespace tensorflow
116