• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 /* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
2 
3 Licensed under the Apache License, Version 2.0 (the "License");
4 you may not use this file except in compliance with the License.
5 You may obtain a copy of the License at
6 
7     http://www.apache.org/licenses/LICENSE-2.0
8 
9 Unless required by applicable law or agreed to in writing, software
10 distributed under the License is distributed on an "AS IS" BASIS,
11 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 See the License for the specific language governing permissions and
13 limitations under the License.
14 ==============================================================================*/
15 
16 #include "tensorflow/core/framework/common_shape_fns.h"
17 #include "tensorflow/core/framework/op.h"
18 #include "tensorflow/core/framework/shape_inference.h"
19 #include "tensorflow/core/util/example_proto_helper.h"
20 
21 namespace tensorflow {
22 
23 using shape_inference::InferenceContext;
24 using shape_inference::ShapeHandle;
25 
26 REGISTER_OP("DecodeRaw")
27     .Input("bytes: string")
28     .Output("output: out_type")
29     .Attr(
30         "out_type: "
31         "{half,float,double,int32,uint16,uint8,int16,int8,int64,complex64,"
32         "complex128}")
33     .Attr("little_endian: bool = true")
__anon952516500102(InferenceContext* c) 34     .SetShapeFn([](InferenceContext* c) {
35       // Note: last dimension is data dependent.
36       ShapeHandle out;
37       TF_RETURN_IF_ERROR(c->Concatenate(
38           c->input(0), c->Vector(InferenceContext::kUnknownDim), &out));
39       c->set_output(0, out);
40       return Status::OK();
41     });
42 
43 REGISTER_OP("DecodeCompressed")
44     .Input("bytes: string")
45     .Output("output: string")
46     .Attr("compression_type: string = ''")
47     .SetShapeFn(shape_inference::UnchangedShape);
48 
49 REGISTER_OP("ParseExample")
50     .Input("serialized: string")
51     .Input("names: string")
52     .Input("sparse_keys: Nsparse * string")
53     .Input("dense_keys: Ndense * string")
54     .Input("dense_defaults: Tdense")
55     .Output("sparse_indices: Nsparse * int64")
56     .Output("sparse_values: sparse_types")
57     .Output("sparse_shapes: Nsparse * int64")
58     .Output("dense_values: Tdense")
59     .Attr("Nsparse: int >= 0")  // Inferred from sparse_keys
60     .Attr("Ndense: int >= 0")   // Inferred from dense_keys
61     .Attr("sparse_types: list({float,int64,string}) >= 0")
62     .Attr("Tdense: list({float,int64,string}) >= 0")
63     .Attr("dense_shapes: list(shape) >= 0")
__anon952516500202(InferenceContext* c) 64     .SetShapeFn([](InferenceContext* c) {
65       ParseExampleAttrs attrs;
66       TF_RETURN_IF_ERROR(attrs.Init(c));
67 
68       ShapeHandle input;
69       TF_RETURN_IF_ERROR(c->WithRank(c->input(0), 1, &input));
70       ShapeHandle unused;
71       TF_RETURN_IF_ERROR(c->WithRank(c->input(1), 1, &unused));  // names
72 
73       // Output sparse_indices, sparse_values, and sparse_shapes.
74       int output_idx = 0;
75       for (int i = 0; i < attrs.num_sparse; ++i) {
76         c->set_output(output_idx++, c->Matrix(c->UnknownDim(), 2));
77       }
78       for (int i = 0; i < attrs.num_sparse; ++i) {
79         c->set_output(output_idx++, c->Vector(c->UnknownDim()));
80       }
81       for (int i = 0; i < attrs.num_sparse; ++i) {
82         c->set_output(output_idx++, c->Vector(2));
83       }
84 
85       // Output dense_shapes.
86       for (int i = 0; i < attrs.num_dense; ++i) {
87         ShapeHandle dense;
88         TF_RETURN_IF_ERROR(
89             c->MakeShapeFromPartialTensorShape(attrs.dense_shapes[i], &dense));
90         TF_RETURN_IF_ERROR(c->Concatenate(input, dense, &dense));
91         c->set_output(output_idx++, dense);
92       }
93       return Status::OK();
94     });
95 
96 REGISTER_OP("ParseSingleExample")
97     .Input("serialized: string")
98     .Input("dense_defaults: Tdense")
99     .Output("sparse_indices: num_sparse * int64")
100     .Output("sparse_values: sparse_types")
101     .Output("sparse_shapes: num_sparse * int64")
102     .Output("dense_values: Tdense")
103     .Attr("num_sparse: int >= 0")
104     .Attr("sparse_keys: list(string) >= 0")
105     .Attr("dense_keys: list(string) >= 0")
106     .Attr("sparse_types: list({float,int64,string}) >= 0")
107     .Attr("Tdense: list({float,int64,string}) >= 0")
108     .Attr("dense_shapes: list(shape) >= 0")
__anon952516500302(InferenceContext* c) 109     .SetShapeFn([](InferenceContext* c) {
110       ParseSingleExampleAttrs attrs;
111       TF_RETURN_IF_ERROR(attrs.Init(c));
112 
113       ShapeHandle input;
114       TF_RETURN_IF_ERROR(c->WithRank(c->input(0), 0, &input));
115 
116       // Output sparse_indices, sparse_values, and sparse_shapes.
117       int output_idx = 0;
118       for (int i = 0; i < attrs.sparse_keys.size(); ++i) {
119         c->set_output(output_idx++, c->Matrix(c->UnknownDim(), 1));
120       }
121       for (int i = 0; i < attrs.sparse_keys.size(); ++i) {
122         c->set_output(output_idx++, c->Vector(c->UnknownDim()));
123       }
124       for (int i = 0; i < attrs.sparse_keys.size(); ++i) {
125         c->set_output(output_idx++, c->Vector(1));
126       }
127 
128       // Output dense_shapes.
129       for (int i = 0; i < attrs.dense_keys.size(); ++i) {
130         ShapeHandle dense;
131         TF_RETURN_IF_ERROR(
132             c->MakeShapeFromPartialTensorShape(attrs.dense_shapes[i], &dense));
133         c->set_output(output_idx++, dense);
134       }
135       return Status::OK();
136     });
137 
138 REGISTER_OP("ParseSequenceExample")
139     .Input("serialized: string")
140     .Input("debug_name: string")
141     .Input("context_dense_defaults: Tcontext_dense")
142     .Output("context_sparse_indices: Ncontext_sparse * int64")
143     .Output("context_sparse_values: context_sparse_types")
144     .Output("context_sparse_shapes: Ncontext_sparse * int64")
145     .Output("context_dense_values: Tcontext_dense")
146     .Output("feature_list_sparse_indices: Nfeature_list_sparse * int64")
147     .Output("feature_list_sparse_values: feature_list_sparse_types")
148     .Output("feature_list_sparse_shapes: Nfeature_list_sparse * int64")
149     .Output("feature_list_dense_values: feature_list_dense_types")
150     .Output("feature_list_dense_lengths: Nfeature_list_dense * int64")
151     .Attr("feature_list_dense_missing_assumed_empty: list(string) >= 0")
152     .Attr("context_sparse_keys: list(string) >= 0")
153     .Attr("context_dense_keys: list(string) >= 0")
154     .Attr("feature_list_sparse_keys: list(string) >= 0")
155     .Attr("feature_list_dense_keys: list(string) >= 0")
156     .Attr("Ncontext_sparse: int >= 0 = 0")
157     .Attr("Ncontext_dense: int >= 0 = 0")
158     .Attr("Nfeature_list_sparse: int >= 0 = 0")
159     .Attr("Nfeature_list_dense: int >= 0 = 0")
160     .Attr("context_sparse_types: list({float,int64,string}) >= 0 = []")
161     .Attr("Tcontext_dense: list({float,int64,string}) >= 0 = []")
162     .Attr("feature_list_dense_types: list({float,int64,string}) >= 0 = []")
163     .Attr("context_dense_shapes: list(shape) >= 0 = []")
164     .Attr("feature_list_sparse_types: list({float,int64,string}) >= 0 = []")
165     .Attr("feature_list_dense_shapes: list(shape) >= 0 = []")
__anon952516500402(InferenceContext* c) 166     .SetShapeFn([](InferenceContext* c) {
167       ParseSequenceExampleAttrs attrs;
168       TF_RETURN_IF_ERROR(attrs.Init(c));
169 
170       // Verify that the input is a vector, and carry the shape if known.
171       ShapeHandle input;
172       TF_RETURN_IF_ERROR(c->WithRank(c->input(0), 1, &input));
173       shape_inference::DimensionHandle num_examples = c->Dim(input, 0);
174 
175       ShapeHandle unused;
176       TF_RETURN_IF_ERROR(c->WithRank(c->input(1), 1, &unused));  // debug_name
177 
178       int output_idx = 0;
179 
180       // Output context_sparse_indices, context_sparse_values, and
181       // context_sparse_shapes.
182       for (int i = 0; i < attrs.num_context_sparse; ++i) {
183         c->set_output(output_idx++, c->Matrix(c->UnknownDim(), 2));
184       }
185       for (int i = 0; i < attrs.num_context_sparse; ++i) {
186         c->set_output(output_idx++, c->Vector(c->UnknownDim()));
187       }
188       for (int i = 0; i < attrs.num_context_sparse; ++i) {
189         c->set_output(output_idx++, c->Vector(2));
190       }
191 
192       // Output context_dense_values.
193       for (int i = 0; i < attrs.num_context_dense; ++i) {
194         ShapeHandle s;
195         TF_RETURN_IF_ERROR(c->MakeShapeFromPartialTensorShape(
196             attrs.context_dense_shapes[i], &s));
197         TF_RETURN_IF_ERROR(c->Concatenate(c->Vector(num_examples), s, &s));
198         c->set_output(output_idx++, s);
199       }
200 
201       // Output feature_list_sparse_indices, feature_list_sparse_values,
202       // feature_list_sparse_shapes.
203       for (int i = 0; i < attrs.num_feature_list_sparse; ++i) {
204         c->set_output(output_idx++, c->Matrix(c->UnknownDim(), 3));
205       }
206       for (int i = 0; i < attrs.num_feature_list_sparse; ++i) {
207         c->set_output(output_idx++, c->Vector(c->UnknownDim()));
208       }
209       for (int i = 0; i < attrs.num_feature_list_sparse; ++i) {
210         c->set_output(output_idx++, c->Vector(3));
211       }
212 
213       // Output feature_list_dense_shapes.
214       for (int i = 0; i < attrs.num_feature_list_dense; ++i) {
215         ShapeHandle s;
216         TF_RETURN_IF_ERROR(c->MakeShapeFromPartialTensorShape(
217             attrs.feature_list_dense_shapes[i], &s));
218         TF_RETURN_IF_ERROR(
219             c->Concatenate(c->Matrix(num_examples, c->UnknownDim()), s, &s));
220         c->set_output(output_idx++, s);
221       }
222 
223       // Output feature_list_dense_lengths.
224       for (int i = 0; i < attrs.num_feature_list_dense; ++i) {
225         c->set_output(output_idx++, c->Vector(num_examples));
226       }
227 
228       return Status::OK();
229     });
230 
231 REGISTER_OP("ParseSingleSequenceExample")
232     .Input("serialized: string")
233     .Input("feature_list_dense_missing_assumed_empty: string")
234     .Input("context_sparse_keys: Ncontext_sparse * string")
235     .Input("context_dense_keys: Ncontext_dense * string")
236     .Input("feature_list_sparse_keys: Nfeature_list_sparse * string")
237     .Input("feature_list_dense_keys: Nfeature_list_dense * string")
238     .Input("context_dense_defaults: Tcontext_dense")
239     .Input("debug_name: string")
240     .Output("context_sparse_indices: Ncontext_sparse * int64")
241     .Output("context_sparse_values: context_sparse_types")
242     .Output("context_sparse_shapes: Ncontext_sparse * int64")
243     .Output("context_dense_values: Tcontext_dense")
244     .Output("feature_list_sparse_indices: Nfeature_list_sparse * int64")
245     .Output("feature_list_sparse_values: feature_list_sparse_types")
246     .Output("feature_list_sparse_shapes: Nfeature_list_sparse * int64")
247     .Output("feature_list_dense_values: feature_list_dense_types")
248     // Infer from context_sparse_keys
249     .Attr("Ncontext_sparse: int >= 0 = 0")
250     // Infer from context_dense_keys
251     .Attr("Ncontext_dense: int >= 0 = 0")
252     // Infer from feature_list_sparse_keys
253     .Attr("Nfeature_list_sparse: int >= 0 = 0")
254     // Infer from feature_list_dense_keys
255     .Attr("Nfeature_list_dense: int >= 0 = 0")
256     .Attr("context_sparse_types: list({float,int64,string}) >= 0 = []")
257     .Attr("Tcontext_dense: list({float,int64,string}) >= 0 = []")
258     .Attr("feature_list_dense_types: list({float,int64,string}) >= 0 = []")
259     .Attr("context_dense_shapes: list(shape) >= 0 = []")
260     .Attr("feature_list_sparse_types: list({float,int64,string}) >= 0 = []")
261     .Attr("feature_list_dense_shapes: list(shape) >= 0 = []")
__anon952516500502(InferenceContext* c) 262     .SetShapeFn([](InferenceContext* c) {
263       ShapeHandle unused;
264       ParseSingleSequenceExampleAttrs attrs;
265       TF_RETURN_IF_ERROR(attrs.Init(c));
266 
267       ShapeHandle input;
268       TF_RETURN_IF_ERROR(c->WithRank(c->input(0), 0, &input));
269 
270       // feature_list_dense_missing_assumed_empty
271       TF_RETURN_IF_ERROR(c->WithRank(c->input(1), 1, &unused));
272 
273       int output_idx = 0;
274 
275       // Output context_sparse_indices, context_sparse_values, and
276       // context_sparse_shapes.
277       for (int i = 0; i < attrs.num_context_sparse; ++i) {
278         c->set_output(output_idx++, c->Matrix(c->UnknownDim(), 1));
279       }
280       for (int i = 0; i < attrs.num_context_sparse; ++i) {
281         c->set_output(output_idx++, c->Vector(c->UnknownDim()));
282       }
283       for (int i = 0; i < attrs.num_context_sparse; ++i) {
284         c->set_output(output_idx++, c->Vector(1));
285       }
286 
287       // Output context_dense_shapes.
288       for (int i = 0; i < attrs.num_context_dense; ++i) {
289         ShapeHandle s;
290         TF_RETURN_IF_ERROR(c->MakeShapeFromPartialTensorShape(
291             attrs.context_dense_shapes[i], &s));
292         c->set_output(output_idx++, s);
293       }
294 
295       // Output feature_list_sparse_indices, feature_list_sparse_values,
296       // feature_list_sparse_shapes.
297       for (int i = 0; i < attrs.num_feature_list_sparse; ++i) {
298         c->set_output(output_idx++, c->Matrix(c->UnknownDim(), 2));
299       }
300       for (int i = 0; i < attrs.num_feature_list_sparse; ++i) {
301         c->set_output(output_idx++, c->Vector(c->UnknownDim()));
302       }
303       for (int i = 0; i < attrs.num_feature_list_sparse; ++i) {
304         c->set_output(output_idx++, c->Vector(2));
305       }
306 
307       // Output feature_list_dense_shapes.
308       for (int i = 0; i < attrs.num_feature_list_dense; ++i) {
309         ShapeHandle s;
310         TF_RETURN_IF_ERROR(c->MakeShapeFromPartialTensorShape(
311             attrs.feature_list_dense_shapes[i], &s));
312         TF_RETURN_IF_ERROR(
313             c->Concatenate(c->Vector(InferenceContext::kUnknownDim), s, &s));
314         c->set_output(output_idx++, s);
315       }
316       return Status::OK();
317     });
318 
319 REGISTER_OP("ParseTensor")
320     .Input("serialized: string")
321     .Output("output: out_type")
322     .Attr("out_type: type")
323     .SetShapeFn(shape_inference::UnknownShape);
324 
325 REGISTER_OP("SerializeTensor")
326     .Input("tensor: T")
327     .Output("serialized: string")
328     .Attr("T: type")
329     .SetShapeFn(shape_inference::ScalarShape);
330 
331 REGISTER_OP("DecodeJSONExample")
332     .Input("json_examples: string")
333     .Output("binary_examples: string")
334     .SetShapeFn(shape_inference::UnchangedShape);
335 
336 REGISTER_OP("DecodeCSV")
337     .Input("records: string")
338     .Input("record_defaults: OUT_TYPE")
339     .Output("output: OUT_TYPE")
340     .Attr("OUT_TYPE: list({float,double,int32,int64,string})")
341     .Attr("field_delim: string = ','")
342     .Attr("use_quote_delim: bool = true")
343     .Attr("na_value: string = ''")
344     .Attr("select_cols: list(int) = []")
__anon952516500602(InferenceContext* c) 345     .SetShapeFn([](InferenceContext* c) {
346       // Validate the record_defaults inputs.
347       for (int i = 1; i < c->num_inputs(); ++i) {
348         ShapeHandle v;
349         TF_RETURN_IF_ERROR(c->WithRankAtMost(c->input(i), 1, &v));
350         if (c->Rank(c->input(i)) == 1 && c->Value(c->Dim(v, 0)) > 1) {
351           return errors::InvalidArgument(
352               "Shape of a default must be a length-0 or length-1 vector, or a "
353               "scalar.");
354         }
355       }
356 
357       // Propagate shape of the records input.
358       for (int i = 0; i < c->num_outputs(); ++i) c->set_output(i, c->input(0));
359       return Status::OK();
360     });
361 
362 REGISTER_OP("StringToNumber")
363     .Input("string_tensor: string")
364     .Output("output: out_type")
365     .Attr("out_type: {float, double, int32, int64} = DT_FLOAT")
366     .SetShapeFn(shape_inference::UnchangedShape);
367 
368 }  // namespace tensorflow
369