1 /* Copyright 2016 The TensorFlow Authors. All Rights Reserved. 2 3 Licensed under the Apache License, Version 2.0 (the "License"); 4 you may not use this file except in compliance with the License. 5 You may obtain a copy of the License at 6 7 http://www.apache.org/licenses/LICENSE-2.0 8 9 Unless required by applicable law or agreed to in writing, software 10 distributed under the License is distributed on an "AS IS" BASIS, 11 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 See the License for the specific language governing permissions and 13 limitations under the License. 14 ==============================================================================*/ 15 16 // This is the common header for the input and filter backprop kernels. 17 // 18 // The operation to compute Conv2D gradients. 19 // 20 // To compute the gradients for Conv2D, we need three input tensors: 21 // input, filter, and backprop for output. 22 // And we need to compute two backprops: one for input and one for filter. We 23 // compute them in two different kernels. 24 // 25 // Both backprops can be computed as straightforward conv2d. 26 // 27 // Consider a case where the input is 3x3 and the filter is 2x1: 28 // 29 // INPUT = [ A B C ] 30 // [ D E F ] 31 // [ G H I ] 32 // 33 // where each "A", "B", etc is batch x in_depth 34 // 35 // FILTER = [ X Y ] 36 // 37 // where both "X" and "Y" are in_depth x out_depth 38 // 39 // With VALID padding, the output is 3x2: 40 // 41 // OUTPUT = [ a b ] 42 // [ c d ] 43 // [ e f ] 44 // 45 // where each "a", "b", etc is batch x out_depth 46 // 47 // So we have: 48 // 49 // a = A * X + B * Y 50 // b = B * X + C * Y 51 // c = D * X + E * Y 52 // d = E * X + F * Y 53 // e = G * X + H * Y 54 // f = H * X + I * Y 55 // 56 // So when we have backprops for the outputs (we denote them by 57 // a', b', ... ): 58 // 59 // The backprops for the input are: 60 // 61 // A' = a' * X^t 62 // B' = a' * Y^t + b' * X^t 63 // C' = b' * Y^t 64 // ... 65 // 66 // This is essentially computing a 2d conv of 67 // 68 // INPUT = [ 0 a' b' 0 ] 69 // [ 0 c' d' 0 ] 70 // [ 0 e' f' 0 ] 71 // and 72 // 73 // FILTER = [ Y^t X^t ] 74 // 75 // The backprops for the filter are: 76 // 77 // X' = A^t * a' + B^t * b' + D^t * c' + E^t * d' + G^t * e' + H^t * f' 78 // Y' = B^t * a' + C^t * b' + E^t + c' + F^t * d' + H^t * e' + I^t * f' 79 // 80 // This is essentially computing a 2d conv of 81 // 82 // INPUT = [ A^t B^t C^t ] 83 // [ D^t E^t F^t ] 84 // [ G^t H^t I^t ] 85 // 86 // and 87 // 88 // FILTER = [ a' b' ] 89 // [ c' d' ] 90 // [ e' f' ] 91 // 92 // 93 ////////////////////////////////////////////////////////// 94 // 95 // With stride more than one, it's a bit more complicated (we will need to 96 // create holes to the backprop). 97 // 98 // Consider the case where 99 // 100 // INPUT = [ A B C D E ] 101 // [ F G H I J ] 102 // [ K L M N O ] 103 // and 104 // 105 // FILTER = [ X Y Z ] 106 // 107 // with stride 2. 108 // 109 // The output will be 110 // 111 // OUTPUT = [ a b ] 112 // [ c d ] 113 // 114 // where: 115 // 116 // a = A * X + B * Y + C * Z 117 // b = C * X + D * Y + E * Z 118 // c = K * X + L * Y + M * Z 119 // d = M * X + N * Y + O * Z 120 // 121 // 122 // To compute the backprop for INPUT, we need to convolve 123 // 124 // INPUT = [ 0 0 a' 0 b' 0 0 ] 125 // [ 0 0 0 0 0 0 0 ] 126 // [ 0 0 c' 0 d' 0 0 ] 127 // 128 // (notice the holes in INPUT) 129 // 130 // and 131 // 132 // FILTER = [ Z^t Y^t X^t ] 133 // 134 // with stride 1. 135 // 136 // To compute the backprop for FILTER, we need to convolve 137 138 // 139 // INPUT = [ A^t B^t C^t D^t E^t ] 140 // [ F^t G^t H^t I^t J^t ] 141 // [ K^t L^t M^t N^t O^t ] 142 // and 143 // 144 // FILTER = [ a' 0 b' ] 145 // [ 0 0 0 ] 146 // [ c' 0 d' ] 147 // 148 // (notice the holes in FILTER) 149 // 150 // 151 // with stride 1 152 // 153 ////////////////////////////////////////////////////////// 154 // 155 // 156 // The case for SAME padding is in fact very similar to VALID -- we just 157 // need to pad the input tensor a bit when computing the filter_backprop. 158 159 #ifndef TENSORFLOW_CORE_KERNELS_CONV_GRAD_OPS_H_ 160 #define TENSORFLOW_CORE_KERNELS_CONV_GRAD_OPS_H_ 161 162 #include <vector> 163 164 #include "tensorflow/core/framework/tensor_shape.h" 165 #include "tensorflow/core/lib/core/stringpiece.h" 166 #include "tensorflow/core/util/padding.h" 167 #include "tensorflow/core/util/tensor_format.h" 168 169 namespace tensorflow { 170 171 // Forward declaration. 172 class OpKernelContext; 173 174 template <typename Device, typename T> 175 struct LaunchConv2DBackpropInputOp { 176 void operator()(OpKernelContext* ctx, bool use_cudnn, bool cudnn_use_autotune, 177 const Tensor& out_backprop, const Tensor& filter, 178 int row_dilation, int col_dilation, int row_stride, 179 int col_stride, const Padding& padding, 180 const std::vector<int64>& explicit_paddings, 181 Tensor* in_backprop, TensorFormat data_format); 182 }; 183 184 template <typename Device, typename T> 185 struct LaunchConv2DBackpropFilterOp { 186 void operator()(OpKernelContext* ctx, bool use_cudnn, bool cudnn_use_autotune, 187 const Tensor& out_backprop, const Tensor& input, 188 int row_dilation, int col_dilation, int row_stride, 189 int col_stride, const Padding& padding, 190 const std::vector<int64>& explicit_paddings, 191 Tensor* filter_backprop, TensorFormat data_format); 192 }; 193 194 #ifdef GOOGLE_CUDA 195 template <typename T> 196 struct LaunchConv2DBackpropInputOp<Eigen::GpuDevice, T> { 197 void operator()(OpKernelContext* ctx, bool use_cudnn, bool cudnn_use_autotune, 198 const Tensor& input, const Tensor& filter, int row_dilation, 199 int col_dilation, int row_stride, int col_stride, 200 const Padding& padding, 201 const std::vector<int64>& explicit_paddings, Tensor* output, 202 TensorFormat data_format); 203 }; 204 205 template <typename T> 206 struct LaunchConv2DBackpropFilterOp<Eigen::GpuDevice, T> { 207 void operator()(OpKernelContext* ctx, bool use_cudnn, bool cudnn_use_autotune, 208 const Tensor& out_backprop, const Tensor& input, 209 int row_dilation, int col_dilation, int row_stride, 210 int col_stride, const Padding& padding, 211 const std::vector<int64>& explicit_paddings, 212 Tensor* filter_backprop, TensorFormat data_format); 213 }; 214 #endif // GOOGLE_CUDA 215 216 // Information about a single spatial dimension for a convolution 217 // backpropagation. 218 struct ConvBackpropSpatialDimension { 219 int64 input_size; 220 int64 filter_size; 221 int64 output_size; 222 int64 stride; 223 int64 dilation; 224 225 // Output size after scaling by the stride. 226 int64 expanded_output_size; 227 228 // Number of padding elements to be added before/after this dimension of 229 // the input when computing Conv?DBackpropInput. 230 int64 pad_before, pad_after; 231 }; 232 233 // Computed dimensions for a backwards convolution. 234 struct ConvBackpropDimensions { 235 // Information about each spatial dimension. 236 gtl::InlinedVector<ConvBackpropSpatialDimension, 3> spatial_dims; 237 238 // Batch size. 239 int64 batch_size; 240 241 // Input and output feature depth. 242 int64 in_depth, out_depth; 243 244 // Convenience access methods for spatial dimensions properties. 245 int64 input_size(int dim) const { return spatial_dims[dim].input_size; } 246 int64 filter_size(int dim) const { return spatial_dims[dim].filter_size; } 247 int64 output_size(int dim) const { return spatial_dims[dim].output_size; } 248 int64 stride(int dim) const { return spatial_dims[dim].stride; } 249 int64 dilation(int dim) const { return spatial_dims[dim].dilation; } 250 251 // Compute padding for the given spatial dimension. 252 int SpatialPadding(const Padding& padding, int dim) const; 253 }; 254 255 // Common code between implementations of Conv?DBackpropInput and 256 // Conv?DBackpropFilter. Verifies that the dimensions all match, and computes 257 // sizes/padding for the spatial dimensions. Does not support explicit padding. 258 Status ConvBackpropComputeDimensions(StringPiece label, int num_spatial_dims, 259 const TensorShape& input_shape, 260 const TensorShape& filter_shape, 261 const TensorShape& out_backprop_shape, 262 const std::vector<int32>& strides, 263 Padding padding, TensorFormat data_format, 264 ConvBackpropDimensions* dims); 265 266 // The V2 version computes the same outputs with arbitrary dilation rate and 267 // supports explicit padding. 268 // TODO(b/67112639): Merge V2 versions and the original versions eventually. 269 Status ConvBackpropComputeDimensionsV2( 270 StringPiece label, int num_spatial_dims, const TensorShape& input_shape, 271 const TensorShape& filter_shape, const TensorShape& out_backprop_shape, 272 const gtl::ArraySlice<int32>& dilations, const std::vector<int32>& strides, 273 Padding padding, absl::Span<const int64> explicit_paddings, 274 TensorFormat data_format, ConvBackpropDimensions* dims); 275 } // namespace tensorflow 276 277 #endif // TENSORFLOW_CORE_KERNELS_CONV_GRAD_OPS_H_ 278