1 /* Copyright 2015 The TensorFlow Authors. All Rights Reserved. 2 3 Licensed under the Apache License, Version 2.0 (the "License"); 4 you may not use this file except in compliance with the License. 5 You may obtain a copy of the License at 6 7 http://www.apache.org/licenses/LICENSE-2.0 8 9 Unless required by applicable law or agreed to in writing, software 10 distributed under the License is distributed on an "AS IS" BASIS, 11 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 See the License for the specific language governing permissions and 13 limitations under the License. 14 ==============================================================================*/ 15 16 #ifndef TENSORFLOW_CORE_KERNELS_RELU_OP_FUNCTOR_H_ 17 #define TENSORFLOW_CORE_KERNELS_RELU_OP_FUNCTOR_H_ 18 // Functor definition for ReluOp and ReluGradOp, must be compilable by nvcc. 19 20 #include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor" 21 #include "tensorflow/core/framework/tensor_types.h" 22 23 namespace tensorflow { 24 namespace functor { 25 26 // Functor used by ReluOp to do the computations. 27 template <typename Device, typename T> 28 struct Relu { 29 // Computes Relu activation. 30 // 31 // features: any shape. 32 // activations: same shape as "features". operatorRelu33 void operator()(const Device& d, typename TTypes<T>::ConstTensor features, 34 typename TTypes<T>::Tensor activations) { 35 activations.device(d) = features.cwiseMax(static_cast<T>(0)); 36 } 37 }; 38 39 // Functor used by ReluGradOp to do the computations. 40 template <typename Device, typename T> 41 struct ReluGrad { 42 // Computes ReluGrad backprops. 43 // 44 // gradients: gradients backpropagated to the Relu op. 45 // features: either the inputs that were passed to the Relu or, or its 46 // outputs (using either one yields the same result here). 47 // backprops: gradients to backpropagate to the Relu inputs. operatorReluGrad48 void operator()(const Device& d, typename TTypes<T>::ConstTensor gradients, 49 typename TTypes<T>::ConstTensor features, 50 typename TTypes<T>::Tensor backprops) { 51 // NOTE: When the activation is exactly zero, we do not propagate the 52 // associated gradient value. This allows the output of the Relu to be used, 53 // as well as its input. 54 backprops.device(d) = 55 gradients * (features > static_cast<T>(0)).template cast<T>(); 56 } 57 }; 58 59 // Functor used by Relu6Op to do the computations. 60 template <typename Device, typename T> 61 struct Relu6 { 62 // Computes Relu6 activation. 63 // 64 // features: any shape. 65 // activations: same shape as "features". operatorRelu666 void operator()(const Device& d, typename TTypes<T>::ConstTensor features, 67 typename TTypes<T>::Tensor activations) { 68 activations.device(d) = 69 features.cwiseMax(static_cast<T>(0)).cwiseMin(static_cast<T>(6)); 70 } 71 }; 72 73 // Functor used by ReluGradOp to do the computations. 74 template <typename Device, typename T> 75 struct Relu6Grad { 76 // Computes Relu6Grad backprops. 77 // 78 // gradients: gradients backpropagated to the Relu6 op. 79 // features: inputs that where passed to the Relu6 op, or its outputs. 80 // backprops: gradients to backpropagate to the Relu6 inputs. operatorRelu6Grad81 void operator()(const Device& d, typename TTypes<T>::ConstTensor gradients, 82 typename TTypes<T>::ConstTensor features, 83 typename TTypes<T>::Tensor backprops) { 84 // NOTE: When the activation is exactly zero or six, we 85 // make sure not to propagate the associated gradient 86 // value. This allows "features" to be either the input or the output of 87 // the relu6. 88 backprops.device(d) = gradients * ((features > static_cast<T>(0)) * 89 (features < static_cast<T>(6))) 90 .template cast<T>(); 91 } 92 }; 93 94 // Functor used by LeakyReluOp to do the computations. 95 template <typename Device, typename T> 96 struct LeakyRelu { 97 // Computes LeakyRelu activation. 98 // 99 // features: any shape. 100 // activations: same shape as "features". operatorLeakyRelu101 void operator()(const Device& d, typename TTypes<T>::ConstTensor features, 102 T alpha, typename TTypes<T>::Tensor activations) { 103 activations.device(d) = features.cwiseMax(features * alpha); 104 } 105 }; 106 107 // Functor used by LeakyReluGradOp to do the computations. 108 template <typename Device, typename T> 109 struct LeakyReluGrad { 110 // Computes LeakyReluGrad backprops. 111 // 112 // gradients: gradients backpropagated to the LeakyRelu op. 113 // features: either the inputs that were passed to the LeakyRelu or, or its 114 // outputs (using either one yields the same result here). 115 // backprops: gradients to backpropagate to the LeakyRelu inputs. operatorLeakyReluGrad116 void operator()(const Device& d, typename TTypes<T>::ConstTensor gradients, 117 typename TTypes<T>::ConstTensor features, T alpha, 118 typename TTypes<T>::Tensor backprops) { 119 backprops.device(d) = 120 (features > static_cast<T>(0)).select(gradients, gradients * alpha); 121 } 122 }; 123 124 // Functor used by EluOp to do the computations. 125 template <typename Device, typename T> 126 struct Elu { 127 // Computes Elu activation. 128 // 129 // features: any shape. 130 // activations: same shape as "features". operatorElu131 void operator()(const Device& d, typename TTypes<T>::ConstTensor features, 132 typename TTypes<T>::Tensor activations) { 133 // features.constant(?) 134 activations.device(d) = 135 (features < static_cast<T>(0)) 136 .select(features.exp() - features.constant(static_cast<T>(1)), 137 features); 138 } 139 }; 140 141 // Functor used by EluGradOp to do the computations. 142 template <typename Device, typename T> 143 struct EluGrad { 144 // Computes EluGrad backprops. 145 // 146 // gradients: gradients backpropagated to the Elu op. 147 // activations: outputs of the Elu op. 148 // backprops: gradients to backpropagate to the Elu inputs. operatorEluGrad149 void operator()(const Device& d, typename TTypes<T>::ConstTensor gradients, 150 typename TTypes<T>::ConstTensor activations, 151 typename TTypes<T>::Tensor backprops) { 152 backprops.device(d) = 153 (activations < static_cast<T>(0)) 154 .select((activations + static_cast<T>(1)) * gradients, gradients); 155 } 156 }; 157 158 // Functor used by SeluOp to do the computations. 159 template <typename Device, typename T> 160 struct Selu { 161 // Computes Selu activation. 162 // 163 // features: any shape. 164 // activations: same shape as "features". operatorSelu165 void operator()(const Device& d, typename TTypes<T>::ConstTensor features, 166 typename TTypes<T>::Tensor activations) { 167 // features.constant(?) 168 const auto scale = static_cast<T>(1.0507009873554804934193349852946); 169 const auto scale_alpha = static_cast<T>(1.7580993408473768599402175208123); 170 const auto one = static_cast<T>(1); 171 const auto zero = static_cast<T>(0); 172 activations.device(d) = 173 (features < zero) 174 .select(scale_alpha * (features.exp() - features.constant(one)), 175 scale * features); 176 } 177 }; 178 179 // Functor used by SeluGradOp to do the computations. 180 template <typename Device, typename T> 181 struct SeluGrad { 182 // Computes SeluGrad backprops. 183 // 184 // gradients: gradients backpropagated to the Selu op. 185 // activations: outputs of the Selu op. 186 // backprops: gradients to backpropagate to the Selu inputs. operatorSeluGrad187 void operator()(const Device& d, typename TTypes<T>::ConstTensor gradients, 188 typename TTypes<T>::ConstTensor activations, 189 typename TTypes<T>::Tensor backprops) { 190 const auto scale = static_cast<T>(1.0507009873554804934193349852946); 191 const auto scale_alpha = static_cast<T>(1.7580993408473768599402175208123); 192 backprops.device(d) = 193 (activations < static_cast<T>(0)) 194 .select(gradients * (activations + scale_alpha), gradients * scale); 195 } 196 }; 197 198 } // namespace functor 199 } // namespace tensorflow 200 201 #endif // TENSORFLOW_CORE_KERNELS_RELU_OP_FUNCTOR_H_ 202