1 /* Copyright 2015 The TensorFlow Authors. All Rights Reserved. 2 3 Licensed under the Apache License, Version 2.0 (the "License"); 4 you may not use this file except in compliance with the License. 5 You may obtain a copy of the License at 6 7 http://www.apache.org/licenses/LICENSE-2.0 8 9 Unless required by applicable law or agreed to in writing, software 10 distributed under the License is distributed on an "AS IS" BASIS, 11 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 See the License for the specific language governing permissions and 13 limitations under the License. 14 ==============================================================================*/ 15 16 #ifndef TENSORFLOW_CORE_KERNELS_EIGEN_BACKWARD_SPATIAL_CONVOLUTIONS_H_ 17 #define TENSORFLOW_CORE_KERNELS_EIGEN_BACKWARD_SPATIAL_CONVOLUTIONS_H_ 18 19 #include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor" 20 21 namespace Eigen { 22 23 /** SpatialConvolutionBackwardInput 24 * \ingroup CXX11_NeuralNetworks_Module 25 * 26 * \brief Computes the backprop for the input of a 2D convolution. 27 * 28 * The output_backward parameter is expected to be a tensor with a rank of 3 or 29 * more (channels, height, width, and optionally others) 30 * The kernel parameter is expected to be a 4D tensor (filters, channels, 31 * kernel_height, kernel_width) 32 * The output_backward and the kernel must both be in col-major layout. The 33 * result will also be in col-major layout. 34 * 35 * If row_in_stride, col_in_stride > 1, then applies convolution with holes 36 * (aka atrous convolution), sampling every row_in_stride, col_in_stride input 37 * pixels. 38 * 39 * The result can be assigned to a tensor of rank equal to the rank of the 40 * output_backward. The dimensions of the result will be filters, height, width 41 * (and others if applicable). 42 * 43 * It is possible to swap the order of the width and height dimensions provided 44 * that the same order is used in the input, the kernel, and the output. 45 * 46 */ 47 #ifdef EIGEN_HAS_INDEX_LIST 48 typedef IndexList<type2index<0>, type2index<0>, type2index<1>, type2index<1> > 49 ReverseColMajor; 50 typedef IndexList<type2index<1>, type2index<1>, type2index<0>, type2index<0> > 51 ReverseRowMajor; 52 #else 53 typedef array<bool, 4> ReverseColMajor; 54 typedef array<bool, 4> ReverseRowMajor; 55 #endif 56 57 template <typename OutputBackward, typename Kernel> 58 EIGEN_ALWAYS_INLINE static const typename internal::conditional< 59 internal::traits<OutputBackward>::Layout == ColMajor, 60 TensorReshapingOp< 61 const DSizes<typename internal::traits<OutputBackward>::Index, 62 internal::traits<OutputBackward>::NumDimensions>, 63 const TensorContractionOp< 64 const array< 65 IndexPair<typename internal::traits<OutputBackward>::Index>, 1>, 66 const Eigen::TensorForcedEvalOp<const TensorReshapingOp< 67 const DSizes<typename internal::traits<OutputBackward>::Index, 68 2>, 69 const TensorShufflingOp< 70 const array< 71 typename internal::traits<OutputBackward>::Index, 4>, 72 const TensorReverseOp<const ReverseColMajor, 73 const Kernel> > > >, 74 const TensorReshapingOp< 75 const DSizes<typename internal::traits<OutputBackward>::Index, 76 2>, 77 const TensorImagePatchOp<Dynamic, Dynamic, 78 const OutputBackward> > > >, 79 TensorReshapingOp< 80 const DSizes<typename internal::traits<OutputBackward>::Index, 81 internal::traits<OutputBackward>::NumDimensions>, 82 const TensorContractionOp< 83 const array< 84 IndexPair<typename internal::traits<OutputBackward>::Index>, 1>, 85 const TensorReshapingOp< 86 const DSizes<typename internal::traits<OutputBackward>::Index, 87 2>, 88 const TensorImagePatchOp<Dynamic, Dynamic, 89 const OutputBackward> >, 90 const Eigen::TensorForcedEvalOp<const TensorReshapingOp< 91 const DSizes<typename internal::traits<OutputBackward>::Index, 92 2>, 93 const TensorShufflingOp< 94 const array< 95 typename internal::traits<OutputBackward>::Index, 4>, 96 const TensorReverseOp<const ReverseRowMajor, 97 const Kernel> > > > > > >::type 98 SpatialConvolutionBackwardInput( 99 const Kernel& kernel, const OutputBackward& output_backward, 100 typename internal::traits<OutputBackward>::Index inputRows, 101 typename internal::traits<OutputBackward>::Index inputCols, 102 const DenseIndex row_stride = 1, const DenseIndex col_stride = 1, 103 const DenseIndex row_in_stride = 1, const DenseIndex col_in_stride = 1) { 104 typedef typename internal::traits<OutputBackward>::Index TensorIndex; 105 typedef typename internal::traits<OutputBackward>::Scalar OutScalar; 106 TensorRef<Tensor<typename internal::traits<Kernel>::Scalar, 107 internal::traits<Kernel>::NumDimensions, 108 internal::traits<Kernel>::Layout, TensorIndex> > 109 kern(kernel); 110 TensorRef<Tensor<OutScalar, internal::traits<OutputBackward>::NumDimensions, 111 internal::traits<OutputBackward>::Layout, TensorIndex> > 112 out(output_backward); 113 114 EIGEN_STATIC_ASSERT(internal::traits<Kernel>::Layout == 115 internal::traits<OutputBackward>::Layout, 116 YOU_MADE_A_PROGRAMMING_MISTAKE); 117 118 static const bool isColMajor = 119 (internal::traits<OutputBackward>::Layout == ColMajor); 120 121 static const int NumDims = internal::traits<OutputBackward>::NumDimensions; 122 123 // Number of filters to apply. This is the same as the output depth of the 124 // result 125 const TensorIndex kernelFilters = 126 isColMajor ? kern.dimensions()[0] : kern.dimensions()[3]; 127 // Number of channels. This is the same as the input depth. 128 const TensorIndex kernelChannels = 129 isColMajor ? kern.dimensions()[1] : kern.dimensions()[2]; 130 const TensorIndex kernelRows = 131 isColMajor ? kern.dimensions()[2] : kern.dimensions()[1]; 132 const TensorIndex kernelCols = 133 isColMajor ? kern.dimensions()[3] : kern.dimensions()[0]; 134 135 // This is the effective kernel size, taking into account the (*_in_stride - 136 // 1) zero-values 137 // inserted between consecutive kernel elements in atrous convolution 138 const TensorIndex kernelRowsEff = 139 kernelRows + (kernelRows - 1) * (row_in_stride - 1); 140 const TensorIndex kernelColsEff = 141 kernelCols + (kernelCols - 1) * (col_in_stride - 1); 142 143 const TensorIndex outputRows = isColMajor 144 ? output_backward.dimension(1) 145 : output_backward.dimension(NumDims - 2); 146 const TensorIndex outputCols = isColMajor 147 ? output_backward.dimension(2) 148 : output_backward.dimension(NumDims - 3); 149 150 // Computing the forward padding 151 const TensorIndex forward_pad_top = numext::maxi<Index>( 152 0, ((outputRows - 1) * row_stride + kernelRowsEff - inputRows) / 2); 153 const TensorIndex forward_pad_left = numext::maxi<Index>( 154 0, ((outputCols - 1) * col_stride + kernelColsEff - inputCols) / 2); 155 const TensorIndex padding_top = kernelRowsEff - 1 - forward_pad_top; 156 const TensorIndex padding_left = kernelColsEff - 1 - forward_pad_left; 157 158 const TensorIndex padding_bottom = inputRows - (outputRows - 1) * row_stride - 159 2 - padding_top + kernelRowsEff; 160 const TensorIndex padding_right = inputCols - (outputCols - 1) * col_stride - 161 2 - padding_left + kernelColsEff; 162 163 eigen_assert(padding_top >= 0); 164 eigen_assert(padding_left >= 0); 165 eigen_assert(padding_bottom >= 0); 166 eigen_assert(padding_right >= 0); 167 168 // The kernel has dimensions filters X channels X patch_rows X patch_cols 169 // We need to reverse the kernel along dimensions corresponding to rows and 170 // cols. 171 // TODO(yangke): we can make things slightly faster by collapsing the 172 // dimensions 173 // where we don't reverse. Try that once we have a faster compiler. 174 typedef typename internal::conditional<isColMajor, ReverseColMajor, 175 ReverseRowMajor>::type Reverse; 176 Reverse kernel_reverse; 177 178 #ifndef EIGEN_HAS_INDEX_LIST 179 if (isColMajor) { 180 kernel_reverse[0] = false; 181 kernel_reverse[1] = false; 182 kernel_reverse[2] = true; 183 kernel_reverse[3] = true; 184 } else { 185 kernel_reverse[0] = true; 186 kernel_reverse[1] = true; 187 kernel_reverse[2] = false; 188 kernel_reverse[3] = false; 189 } 190 #endif 191 192 // Reorder the dimensions to: 193 // filters x patch_rows x patch_cols x channels 194 array<TensorIndex, 4> kernel_shuffle; 195 if (isColMajor) { 196 // From: filters x channels x rows x cols 197 // To: filters x rows x cols x channels 198 kernel_shuffle[0] = 0; 199 kernel_shuffle[1] = 2; 200 kernel_shuffle[2] = 3; 201 kernel_shuffle[3] = 1; 202 } else { 203 // From: cols x rows x channels x filters 204 // To: channels x cols x rows x filters 205 kernel_shuffle[0] = 2; 206 kernel_shuffle[1] = 0; 207 kernel_shuffle[2] = 1; 208 kernel_shuffle[3] = 3; 209 } 210 211 // Collapse the dims 212 DSizes<TensorIndex, 2> kernel_dims; 213 if (isColMajor) { 214 kernel_dims[0] = kernelFilters * kernelRows * kernelCols; 215 kernel_dims[1] = kernelChannels; 216 } else { 217 kernel_dims[1] = kernelFilters * kernelRows * kernelCols; 218 kernel_dims[0] = kernelChannels; 219 } 220 221 // The output_backward has dimensions out_depth X out_rows X out_cols X OTHERS 222 // When we extract the image patches from output_backward, it will have 223 // dimensions 224 // out_depth X (patch_rows * patch_cols) X (input_rows * input_cols * 225 // OTHERS) 226 DSizes<TensorIndex, 2> pre_contract_dims; 227 if (isColMajor) { 228 pre_contract_dims[0] = kernelFilters * kernelRows * kernelCols; 229 pre_contract_dims[1] = inputRows * inputCols; 230 for (int i = 3; i < NumDims; ++i) { 231 pre_contract_dims[1] *= out.dimension(i); 232 } 233 } else { 234 pre_contract_dims[1] = kernelFilters * kernelRows * kernelCols; 235 pre_contract_dims[0] = inputRows * inputCols; 236 for (int i = 0; i < NumDims - 3; ++i) { 237 pre_contract_dims[0] *= out.dimension(i); 238 } 239 } 240 241 // We will contract along the collapsed dimension that contains the 242 // kernelFilters, the kernelRows and the kernelCols. 243 array<IndexPair<TensorIndex>, 1> contract_dims; 244 if (isColMajor) { 245 // col-major: kernel.contract(output.patches) 246 contract_dims[0] = IndexPair<TensorIndex>(0, 0); 247 } else { 248 // row-major: output.patches.contract(kernel) 249 contract_dims[0] = IndexPair<TensorIndex>(1, 1); 250 } 251 252 // Post contraction, the dimensions of the input_backprop is 253 // channels X input_rows X input_cols X OTHERS 254 DSizes<TensorIndex, NumDims> post_contract_dims; 255 if (isColMajor) { 256 post_contract_dims[0] = kernelChannels; 257 post_contract_dims[1] = inputRows; 258 post_contract_dims[2] = inputCols; 259 for (int i = 3; i < NumDims; ++i) { 260 post_contract_dims[i] = out.dimension(i); 261 } 262 } else { 263 post_contract_dims[NumDims - 1] = kernelChannels; 264 post_contract_dims[NumDims - 2] = inputRows; 265 post_contract_dims[NumDims - 3] = inputCols; 266 for (int i = 0; i < NumDims - 3; ++i) { 267 post_contract_dims[i] = out.dimension(i); 268 } 269 } 270 271 return choose( 272 Cond<internal::traits<OutputBackward>::Layout == ColMajor>(), 273 kernel.reverse(kernel_reverse) 274 .shuffle(kernel_shuffle) 275 .reshape(kernel_dims) 276 .eval() 277 .contract( 278 output_backward 279 .extract_image_patches( 280 kernelRows, kernelCols, 1, 1, row_in_stride, 281 col_in_stride, row_stride, col_stride, padding_top, 282 padding_bottom, padding_left, padding_right, OutScalar(0)) 283 .reshape(pre_contract_dims), 284 contract_dims) 285 .reshape(post_contract_dims), 286 output_backward 287 .extract_image_patches(kernelRows, kernelCols, 1, 1, row_in_stride, 288 col_in_stride, row_stride, col_stride, 289 padding_top, padding_bottom, padding_left, 290 padding_right, OutScalar(0)) 291 .reshape(pre_contract_dims) 292 .contract(kernel.reverse(kernel_reverse) 293 .shuffle(kernel_shuffle) 294 .reshape(kernel_dims) 295 .eval(), 296 contract_dims) 297 .reshape(post_contract_dims)); 298 } 299 300 /** SpatialConvolutionBackwardKernel 301 * \ingroup CXX11_NeuralNetworks_Module 302 * 303 * \brief Computes the backprop for the filter of a 2D convolution. 304 * 305 * The output_backward parameter is expected to be a tensor with a rank of 3 or 306 * more (channels, height, width, and optionally others) 307 * The kernel parameter is expected to be a 4D tensor (filters, channels, 308 * kernel_height, kernel_width) 309 * The output_backward and the kernel must both be in col-major layout. The 310 * result will also be in col-major layout. 311 * 312 * If row_in_stride, col_stride > 1, then applies convolution with holes (aka 313 * atrous convolution), sampling every row_in_stride, col_in_stride input 314 * pixels. 315 * 316 * The result can be assigned to a tensor of rank equal to the rank of the 317 * output_backward. The dimensions of the result will be filters, height, width 318 * (and others if applicable). 319 * 320 * It is possible to swap the order of the width and height dimensions provided 321 * that the same order is used in the input, the kernel, and the output. 322 * 323 */ 324 325 template <typename OutputBackward, typename Input> 326 EIGEN_ALWAYS_INLINE static const typename internal::conditional< 327 internal::traits<OutputBackward>::Layout == ColMajor, 328 TensorReshapingOp< 329 const DSizes<typename internal::traits<Input>::Index, 4>, 330 const TensorContractionOp< 331 const array<IndexPair<typename internal::traits<Input>::Index>, 1>, 332 const TensorReshapingOp< 333 const DSizes<typename internal::traits<Input>::Index, 2>, 334 const OutputBackward>, 335 const TensorReshapingOp< 336 const DSizes<typename internal::traits<Input>::Index, 2>, 337 const TensorImagePatchOp<Dynamic, Dynamic, const Input> > > >, 338 TensorReshapingOp< 339 const DSizes<typename internal::traits<Input>::Index, 4>, 340 const TensorContractionOp< 341 const array<IndexPair<typename internal::traits<Input>::Index>, 1>, 342 const TensorReshapingOp< 343 const DSizes<typename internal::traits<Input>::Index, 2>, 344 const TensorImagePatchOp<Dynamic, Dynamic, const Input> >, 345 const TensorReshapingOp< 346 const DSizes<typename internal::traits<Input>::Index, 2>, 347 const OutputBackward> > > >::type 348 SpatialConvolutionBackwardKernel( 349 const Input& input, const OutputBackward& output_backward, 350 typename internal::traits<Input>::Index kernelRows, 351 typename internal::traits<Input>::Index kernelCols, 352 const DenseIndex row_stride = 1, const DenseIndex col_stride = 1, 353 const DenseIndex row_in_stride = 1, const DenseIndex col_in_stride = 1) { 354 typedef typename internal::traits<Input>::Index TensorIndex; 355 typedef typename internal::traits<OutputBackward>::Scalar OutScalar; 356 TensorRef<Tensor<typename internal::traits<Input>::Scalar, 357 internal::traits<Input>::NumDimensions, 358 internal::traits<Input>::Layout, TensorIndex> > 359 in(input); 360 TensorRef<Tensor<OutScalar, internal::traits<OutputBackward>::NumDimensions, 361 internal::traits<OutputBackward>::Layout, TensorIndex> > 362 out(output_backward); 363 364 EIGEN_STATIC_ASSERT(internal::traits<Input>::Layout == 365 internal::traits<OutputBackward>::Layout, 366 YOU_MADE_A_PROGRAMMING_MISTAKE); 367 368 // stride and in_stride cannot both be larger than 1 369 eigen_assert(!(row_stride > 1 && row_in_stride > 1) && 370 !(col_stride > 1 && col_in_stride > 1)); 371 372 static const bool isColMajor = (internal::traits<Input>::Layout == ColMajor); 373 374 static const int NumDims = internal::traits<Input>::NumDimensions; 375 EIGEN_STATIC_ASSERT(internal::traits<Input>::NumDimensions == 376 internal::traits<OutputBackward>::NumDimensions, 377 YOU_MADE_A_PROGRAMMING_MISTAKE); 378 379 const TensorIndex inputRows = 380 isColMajor ? in.dimension(1) : in.dimension(NumDims - 2); 381 const TensorIndex inputCols = 382 isColMajor ? in.dimension(2) : in.dimension(NumDims - 3); 383 384 const TensorIndex outputRows = isColMajor 385 ? output_backward.dimension(1) 386 : output_backward.dimension(NumDims - 2); 387 const TensorIndex outputCols = isColMajor 388 ? output_backward.dimension(2) 389 : output_backward.dimension(NumDims - 3); 390 391 // Number of filters to apply. This is the same as the output depth of the 392 // result 393 const TensorIndex kernelFilters = 394 isColMajor ? out.dimensions()[0] : out.dimensions()[NumDims - 1]; 395 396 // Number of channels. This is the same as the input depth. 397 const TensorIndex kernelChannels = 398 isColMajor ? in.dimensions()[0] : in.dimensions()[NumDims - 1]; 399 400 // This is the effective kernel size, taking into account the (*_in_stride - 401 // 1) zero-values 402 // inserted between consecutive kernel elements in atrous convolution 403 const TensorIndex kernelRowsEff = 404 kernelRows + (kernelRows - 1) * (row_in_stride - 1); 405 const TensorIndex kernelColsEff = 406 kernelCols + (kernelCols - 1) * (col_in_stride - 1); 407 408 // Computing the forward padding 409 const TensorIndex padRows = numext::maxi<Index>( 410 0, (outputRows - 1) * row_stride + kernelRowsEff - inputRows); 411 const TensorIndex padCols = numext::maxi<Index>( 412 0, (outputCols - 1) * col_stride + kernelColsEff - inputCols); 413 const TensorIndex padding_top = padRows / 2; 414 const TensorIndex padding_bottom = padRows - padding_top; 415 const TensorIndex padding_left = padCols / 2; 416 const TensorIndex padding_right = padCols - padding_left; 417 418 // Reshaped out 419 DSizes<TensorIndex, 2> output_dims; 420 if (isColMajor) { 421 output_dims[0] = kernelFilters; 422 output_dims[1] = outputRows * outputCols; 423 for (int i = 3; i < NumDims; ++i) { 424 output_dims[1] *= out.dimension(i); 425 } 426 } else { 427 output_dims[1] = kernelFilters; 428 output_dims[0] = outputCols * outputRows; 429 for (int i = 0; i < NumDims - 3; ++i) { 430 output_dims[0] *= out.dimension(i); 431 } 432 } 433 434 // Reshaped extract_image_patches(in) 435 DSizes<TensorIndex, 2> pre_contract_dims; 436 if (isColMajor) { 437 pre_contract_dims[0] = kernelChannels * kernelRows * kernelCols; 438 pre_contract_dims[1] = outputRows * outputCols; 439 for (int i = 3; i < NumDims; ++i) { 440 pre_contract_dims[1] *= in.dimension(i); 441 } 442 eigen_assert(output_dims[1] == pre_contract_dims[1]); 443 } else { 444 pre_contract_dims[1] = kernelCols * kernelRows * kernelChannels; 445 pre_contract_dims[0] = outputRows * outputCols; 446 for (int i = 0; i < NumDims - 3; ++i) { 447 pre_contract_dims[0] *= in.dimension(i); 448 } 449 eigen_assert(output_dims[0] == pre_contract_dims[0]); 450 } 451 452 // We will contract along the collapsed dimension that contains the 453 // outputCols, outputRows and OTHERS. 454 array<IndexPair<TensorIndex>, 1> contract_dims; 455 if (isColMajor) { 456 // col-major: output_backward.contract(input.patches) 457 contract_dims[0] = IndexPair<TensorIndex>(1, 1); 458 } else { 459 // row-major: input.patches.contract(output_backward) 460 contract_dims[0] = IndexPair<TensorIndex>(0, 0); 461 } 462 463 // After the contraction, the kernel will have the desired shape 464 // out_depth X in_shape X kernel_rows X kernel_cols 465 DSizes<TensorIndex, 4> kernel_dims; 466 if (isColMajor) { 467 kernel_dims[0] = kernelFilters; 468 kernel_dims[1] = kernelChannels; 469 kernel_dims[2] = kernelRows; 470 kernel_dims[3] = kernelCols; 471 } else { 472 kernel_dims[3] = kernelFilters; 473 kernel_dims[2] = kernelChannels; 474 kernel_dims[1] = kernelRows; 475 kernel_dims[0] = kernelCols; 476 } 477 478 return choose( 479 Cond<internal::traits<Input>::Layout == ColMajor>(), 480 output_backward.reshape(output_dims) 481 .contract( 482 input 483 .extract_image_patches( 484 kernelRows, kernelCols, row_stride, col_stride, 485 row_in_stride, col_in_stride, 1, 1, padding_top, 486 padding_bottom, padding_left, padding_right, OutScalar(0)) 487 .reshape(pre_contract_dims), 488 contract_dims) 489 .reshape(kernel_dims), 490 input 491 .extract_image_patches(kernelRows, kernelCols, row_stride, col_stride, 492 row_in_stride, col_in_stride, 1, 1, 493 padding_top, padding_bottom, padding_left, 494 padding_right, OutScalar(0)) 495 .reshape(pre_contract_dims) 496 .contract(output_backward.reshape(output_dims), contract_dims) 497 .reshape(kernel_dims)); 498 } 499 500 } // end namespace Eigen 501 502 #endif // TENSORFLOW_CORE_KERNELS_EIGEN_BACKWARD_SPATIAL_CONVOLUTIONS_H_ 503