1 /* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
2
3 Licensed under the Apache License, Version 2.0 (the "License");
4 you may not use this file except in compliance with the License.
5 You may obtain a copy of the License at
6
7 http://www.apache.org/licenses/LICENSE-2.0
8
9 Unless required by applicable law or agreed to in writing, software
10 distributed under the License is distributed on an "AS IS" BASIS,
11 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 See the License for the specific language governing permissions and
13 limitations under the License.
14 ==============================================================================*/
15
16 #include "tensorflow/cc/client/client_session.h"
17 #include "tensorflow/cc/framework/grad_op_registry.h"
18 #include "tensorflow/cc/framework/gradient_checker.h"
19 #include "tensorflow/cc/framework/testutil.h"
20 #include "tensorflow/cc/gradients/grad_testutil.h"
21 #include "tensorflow/cc/ops/image_ops.h"
22 #include "tensorflow/cc/ops/standard_ops.h"
23 #include "tensorflow/core/framework/tensor_testutil.h"
24 #include "tensorflow/core/lib/core/status_test_util.h"
25
26 namespace tensorflow {
27 namespace {
28
29 using ops::Const;
30 using ops::CropAndResize;
31 using ops::ResizeBicubic;
32 using ops::ResizeBilinear;
33 using ops::ResizeNearestNeighbor;
34 using ops::ScaleAndTranslate;
35
36 class ImageGradTest : public ::testing::Test {
37 protected:
ImageGradTest()38 ImageGradTest() : scope_(Scope::NewRootScope()) {}
39
40 enum OpType { RESIZE_NEAREST, RESIZE_BILINEAR, RESIZE_BICUBIC };
41
42 template <typename T>
MakeData(const TensorShape & data_shape)43 Tensor MakeData(const TensorShape& data_shape) {
44 DataType data_type = DataTypeToEnum<T>::v();
45 Tensor data(data_type, data_shape);
46 auto data_flat = data.flat<T>();
47 for (int i = 0; i < data_flat.size(); ++i) {
48 data_flat(i) = T(i);
49 }
50 return data;
51 }
52
53 template <typename T>
MakeOp(const OpType op_type,const Tensor & x_data,const Input & y_shape,const bool align_corners,const bool half_pixel_centers,Output * x,Output * y)54 void MakeOp(const OpType op_type, const Tensor& x_data, const Input& y_shape,
55 const bool align_corners, const bool half_pixel_centers,
56 Output* x, Output* y) {
57 *x = Const<T>(scope_, x_data);
58 switch (op_type) {
59 case RESIZE_NEAREST:
60 *y = ResizeNearestNeighbor(
61 scope_, *x, y_shape,
62 ResizeNearestNeighbor::AlignCorners(align_corners));
63 return;
64 case RESIZE_BILINEAR:
65 *y = ResizeBilinear(scope_, *x, y_shape,
66 ResizeBilinear::AlignCorners(align_corners)
67 .HalfPixelCenters(half_pixel_centers));
68 return;
69 case RESIZE_BICUBIC:
70 *y = ResizeBicubic(scope_, *x, y_shape,
71 ResizeBicubic::AlignCorners(align_corners)
72 .HalfPixelCenters(half_pixel_centers));
73 return;
74 }
75 assert(false);
76 }
77
78 template <typename T>
TestResizedShapeForType(const OpType op_type,const bool align_corners,const bool half_pixel_centers)79 void TestResizedShapeForType(const OpType op_type, const bool align_corners,
80 const bool half_pixel_centers) {
81 TensorShape x_shape({1, 2, 2, 1});
82 Tensor x_data = MakeData<T>(x_shape);
83 Output x, y;
84 MakeOp<T>(op_type, x_data, {4, 6}, align_corners, half_pixel_centers, &x,
85 &y);
86
87 ClientSession session(scope_);
88 std::vector<Tensor> outputs;
89 TF_ASSERT_OK(session.Run({y}, &outputs));
90 EXPECT_EQ(outputs.size(), 1);
91 EXPECT_EQ(outputs[0].shape(), TensorShape({1, 4, 6, 1}));
92 }
93
TestResizedShape(OpType op_type)94 void TestResizedShape(OpType op_type) {
95 for (const bool half_pixel_centers : {true, false}) {
96 for (const bool align_corners : {true, false}) {
97 if (half_pixel_centers && align_corners) {
98 continue;
99 }
100 TestResizedShapeForType<Eigen::half>(op_type, align_corners,
101 half_pixel_centers);
102 TestResizedShapeForType<float>(op_type, align_corners,
103 half_pixel_centers);
104 TestResizedShapeForType<double>(op_type, align_corners,
105 half_pixel_centers);
106 }
107 }
108 }
109
110 template <typename X_T, typename Y_T, typename JAC_T>
TestResizeToSmallerAndAlign(const OpType op_type,const bool align_corners,const bool half_pixel_centers)111 void TestResizeToSmallerAndAlign(const OpType op_type,
112 const bool align_corners,
113 const bool half_pixel_centers) {
114 TensorShape x_shape({1, 4, 6, 1});
115 Tensor x_data = MakeData<X_T>(x_shape);
116 Output x, y;
117 MakeOp<X_T>(op_type, x_data, {2, 3}, align_corners, half_pixel_centers, &x,
118 &y);
119 JAC_T max_error;
120 TF_ASSERT_OK((ComputeGradientError<X_T, Y_T, JAC_T>(
121 scope_, x, x_data, y, {1, 2, 3, 1}, &max_error)));
122 EXPECT_LT(max_error, 1.5e-3);
123 }
124
125 template <typename X_T, typename Y_T, typename JAC_T>
TestResizeToLargerAndAlign(const OpType op_type,const bool align_corners,const bool half_pixel_centers)126 void TestResizeToLargerAndAlign(const OpType op_type,
127 const bool align_corners,
128 const bool half_pixel_centers) {
129 TensorShape x_shape({1, 2, 3, 1});
130 Tensor x_data = MakeData<X_T>(x_shape);
131 Output x, y;
132 MakeOp<X_T>(op_type, x_data, {4, 6}, align_corners, half_pixel_centers, &x,
133 &y);
134 JAC_T max_error;
135 TF_ASSERT_OK((ComputeGradientError<X_T, Y_T, JAC_T>(
136 scope_, x, x_data, y, {1, 4, 6, 1}, &max_error)));
137 EXPECT_LT(max_error, 1.5e-3);
138 }
139
140 template <typename X_T, typename Y_T, typename JAC_T>
TestResize(OpType op_type)141 void TestResize(OpType op_type) {
142 for (const bool half_pixel_centers : {true, false}) {
143 for (const bool align_corners : {true, false}) {
144 // if (!half_pixel_centers) continue;
145 if (half_pixel_centers && align_corners) {
146 continue;
147 }
148 TestResizeToSmallerAndAlign<X_T, Y_T, JAC_T>(op_type, align_corners,
149 half_pixel_centers);
150 TestResizeToLargerAndAlign<X_T, Y_T, JAC_T>(op_type, align_corners,
151 half_pixel_centers);
152 }
153 }
154 }
155
156 Scope scope_;
157 };
158
TEST_F(ImageGradTest,TestNearestNeighbor)159 TEST_F(ImageGradTest, TestNearestNeighbor) {
160 TestResizedShape(RESIZE_NEAREST);
161 TestResize<float, float, float>(RESIZE_NEAREST);
162 TestResize<double, double, double>(RESIZE_NEAREST);
163 }
164
TEST_F(ImageGradTest,TestBilinear)165 TEST_F(ImageGradTest, TestBilinear) {
166 TestResizedShape(RESIZE_BILINEAR);
167 TestResize<float, float, float>(RESIZE_BILINEAR);
168 // Note that Y_T is always float for this op. We choose
169 // double for the jacobian to capture the higher precision
170 // between X_T and Y_T.
171 TestResize<double, float, double>(RESIZE_BILINEAR);
172 }
173
TEST_F(ImageGradTest,TestBicubic)174 TEST_F(ImageGradTest, TestBicubic) {
175 TestResizedShape(RESIZE_BICUBIC);
176 TestResize<float, float, float>(RESIZE_BICUBIC);
177 // Note that Y_T is always float for this op. We choose
178 // double for the jacobian to capture the higher precision
179 // between X_T and Y_T.
180 TestResize<double, float, double>(RESIZE_BICUBIC);
181 }
182
183 class ScaleAndTranslateGradTest : public ::testing::Test {
184 protected:
ScaleAndTranslateGradTest()185 ScaleAndTranslateGradTest() : scope_(Scope::NewRootScope()) {}
186
187 template <typename T>
MakeData(const TensorShape & data_shape)188 Tensor MakeData(const TensorShape& data_shape) {
189 DataType data_type = DataTypeToEnum<T>::v();
190 Tensor data(data_type, data_shape);
191 auto data_flat = data.flat<T>();
192 for (int i = 0; i < data_flat.size(); ++i) {
193 data_flat(i) = T(i);
194 }
195 return data;
196 }
197
198 template <typename T>
MakeOp(const Tensor & x_data,const Input & y_shape,Input scale,Input translation,const string & kernel_type,bool antialias,Output * x,Output * y)199 void MakeOp(const Tensor& x_data, const Input& y_shape, Input scale,
200 Input translation, const string& kernel_type, bool antialias,
201 Output* x, Output* y) {
202 *x = Const<T>(scope_, x_data);
203 *y = ScaleAndTranslate(scope_, *x, y_shape, scale, translation,
204 ScaleAndTranslate::KernelType(kernel_type)
205 .Antialias(antialias)
206 .Antialias(antialias));
207 TF_ASSERT_OK(scope_.status());
208 }
209
210 template <typename X_T, typename Y_T, typename JAC_T>
TestScaleAndTranslate(const TensorShape x_shape,const int out_height,const int out_width,Input scale,Input translation,const string & kernel_type,bool antialias)211 void TestScaleAndTranslate(const TensorShape x_shape, const int out_height,
212 const int out_width, Input scale,
213 Input translation, const string& kernel_type,
214 bool antialias) {
215 Tensor x_data = MakeData<X_T>(x_shape);
216 Output x, y;
217 MakeOp<X_T>(x_data, {out_height, out_width}, scale, translation,
218 kernel_type, antialias, &x, &y);
219 JAC_T max_error;
220 TF_ASSERT_OK((ComputeGradientError<X_T, Y_T, JAC_T>(
221 scope_, x, x_data, y, {1, out_height, out_width, 1}, &max_error)));
222 EXPECT_LT(max_error, 2e-3);
223 }
224
225 const std::vector<Input> kScales = {Input{1.0f, 1.0f}, Input{0.37f, 0.47f},
226 Input{2.1f, 2.1f}};
227 const std::vector<Input> kTranslations = {
228 Input{0.0f, 0.0f}, Input{3.14f, 1.19f}, Input{2.1f, 3.1f},
229 Input{100.0f, 200.0f}};
230 Scope scope_;
231 };
232
TEST_F(ScaleAndTranslateGradTest,TestGrads)233 TEST_F(ScaleAndTranslateGradTest, TestGrads) {
234 const std::vector<std::string> kKernelTypes = {"lanczos1", "lanczos3",
235 "lanczos5", "gaussian"};
236 constexpr int kOutHeight = 4;
237 constexpr int kOutWidth = 6;
238
239 const TensorShape kXShape = TensorShape({1, 2, 3, 1});
240 for (const Input scale : kScales) {
241 for (const Input translation : kTranslations) {
242 for (const std::string& kernel_type : kKernelTypes) {
243 TestScaleAndTranslate<float, float, float>(
244 kXShape, kOutHeight, kOutWidth, scale, translation, kernel_type,
245 true);
246 }
247 }
248 }
249 }
250
TEST_F(ScaleAndTranslateGradTest,TestGradsWithoutAntialias)251 TEST_F(ScaleAndTranslateGradTest, TestGradsWithoutAntialias) {
252 constexpr int kOutHeight = 4;
253 constexpr int kOutWidth = 6;
254
255 const TensorShape kXShape = TensorShape({1, 2, 3, 1});
256 for (const Input scale : kScales) {
257 for (const Input translation : kTranslations) {
258 TestScaleAndTranslate<float, float, float>(kXShape, kOutHeight, kOutWidth,
259 scale, translation, "lanczos3",
260 false);
261 }
262 }
263 }
264
TEST_F(ScaleAndTranslateGradTest,TestGradsWithSameShape)265 TEST_F(ScaleAndTranslateGradTest, TestGradsWithSameShape) {
266 const std::vector<std::string> kKernelTypes = {"lanczos3", "gaussian"};
267
268 constexpr int kOutHeight = 2;
269 constexpr int kOutWidth = 3;
270
271 const TensorShape kXShape = TensorShape({1, 2, 3, 1});
272 for (const Input scale : kScales) {
273 for (const Input translation : kTranslations) {
274 for (const std::string& kernel_type : kKernelTypes) {
275 TestScaleAndTranslate<float, float, float>(
276 kXShape, kOutHeight, kOutWidth, scale, translation, kernel_type,
277 true);
278 }
279 }
280 }
281 }
282
TEST_F(ScaleAndTranslateGradTest,TestGradsWithSmallerShape)283 TEST_F(ScaleAndTranslateGradTest, TestGradsWithSmallerShape) {
284 const std::vector<std::string> kKernelTypes = {"lanczos3", "gaussian"};
285 constexpr int kOutHeight = 2;
286 constexpr int kOutWidth = 3;
287
288 const TensorShape kXShape = TensorShape({1, 4, 6, 1});
289 for (const Input scale : kScales) {
290 for (const Input translation : kTranslations) {
291 for (const std::string& kernel_type : kKernelTypes) {
292 TestScaleAndTranslate<float, float, float>(
293 kXShape, kOutHeight, kOutWidth, scale, translation, kernel_type,
294 true);
295 }
296 }
297 }
298 }
299
300 class CropAndResizeGradTest : public ::testing::Test {
301 protected:
CropAndResizeGradTest()302 CropAndResizeGradTest() : scope_(Scope::NewRootScope()) {}
303
304 template <typename T>
MakeData(const TensorShape & data_shape)305 Tensor MakeData(const TensorShape& data_shape) {
306 DataType data_type = DataTypeToEnum<T>::v();
307 Tensor data(data_type, data_shape);
308 auto data_flat = data.flat<T>();
309 for (int i = 0; i < data_flat.size(); ++i) {
310 data_flat(i) = T(i);
311 }
312 return data;
313 }
314
315 template <typename T>
MakeOp(const Tensor & x_data,const Input & boxes,const Input & box_ind,const Input & crop_size,Output * x,Output * y)316 void MakeOp(const Tensor& x_data, const Input& boxes, const Input& box_ind,
317 const Input& crop_size, Output* x, Output* y) {
318 *x = Const<T>(scope_, x_data);
319 *y = CropAndResize(scope_, *x, boxes, box_ind, crop_size,
320 CropAndResize::Method("bilinear"));
321 TF_ASSERT_OK(scope_.status());
322 }
323
324 template <typename X_T, typename Y_T, typename JAC_T>
TestCropAndResize()325 void TestCropAndResize() {
326 TensorShape x_shape({1, 4, 2, 1});
327 Tensor x_data = MakeData<X_T>(x_shape);
328 TensorShape box_shape({1, 4});
329 Tensor boxes = MakeData<X_T>(box_shape);
330 Output x, y;
331 MakeOp<X_T>(x_data, boxes, {0}, {1, 1}, &x, &y);
332 JAC_T max_error;
333 TF_ASSERT_OK((ComputeGradientError<X_T, Y_T, JAC_T>(
334 scope_, x, x_data, y, {1, 1, 1, 1}, &max_error)));
335 EXPECT_LT(max_error, 1e-3);
336 }
337
338 Scope scope_;
339 };
340
TEST_F(CropAndResizeGradTest,TestCrop)341 TEST_F(CropAndResizeGradTest, TestCrop) {
342 TestCropAndResize<float, float, float>();
343 }
344
345 } // namespace
346 } // namespace tensorflow
347