1 /* Copyright 2015 The TensorFlow Authors. All Rights Reserved. 2 3 Licensed under the Apache License, Version 2.0 (the "License"); 4 you may not use this file except in compliance with the License. 5 You may obtain a copy of the License at 6 7 http://www.apache.org/licenses/LICENSE-2.0 8 9 Unless required by applicable law or agreed to in writing, software 10 distributed under the License is distributed on an "AS IS" BASIS, 11 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 See the License for the specific language governing permissions and 13 limitations under the License. 14 ==============================================================================*/ 15 16 #ifndef TENSORFLOW_CORE_KERNELS_RELU_OP_FUNCTOR_H_ 17 #define TENSORFLOW_CORE_KERNELS_RELU_OP_FUNCTOR_H_ 18 // Functor definition for ReluOp and ReluGradOp, must be compilable by nvcc. 19 20 #include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor" 21 #include "tensorflow/core/framework/tensor_types.h" 22 23 namespace tensorflow { 24 namespace functor { 25 26 // Functor used by ReluOp to do the computations. 27 template <typename Device, typename T> 28 struct Relu { 29 // Computes Relu activation. 30 // 31 // features: any shape. 32 // activations: same shape as "features". operatorRelu33 void operator()(const Device& d, typename TTypes<T>::ConstTensor features, 34 typename TTypes<T>::Tensor activations) { 35 activations.device(d) = features.cwiseMax(static_cast<T>(0)); 36 } 37 }; 38 39 // Functor used by ReluGradOp to do the computations. 40 template <typename Device, typename T> 41 struct ReluGrad { 42 // Computes ReluGrad backprops. 43 // 44 // gradients: gradients backpropagated to the Relu op. 45 // features: either the inputs that were passed to the Relu or, or its 46 // outputs (using either one yields the same result here). 47 // backprops: gradients to backpropagate to the Relu inputs. operatorReluGrad48 void operator()(const Device& d, typename TTypes<T>::ConstTensor gradients, 49 typename TTypes<T>::ConstTensor features, 50 typename TTypes<T>::Tensor backprops) { 51 // NOTE: When the activation is exactly zero, we do not propagate the 52 // associated gradient value. This allows the output of the Relu to be used, 53 // as well as its input. 54 backprops.device(d) = 55 gradients * (features > static_cast<T>(0)).template cast<T>(); 56 } 57 }; 58 59 // Functor used by Relu6Op to do the computations. 60 template <typename Device, typename T> 61 struct Relu6 { 62 // Computes Relu6 activation. 63 // 64 // features: any shape. 65 // activations: same shape as "features". operatorRelu666 void operator()(const Device& d, typename TTypes<T>::ConstTensor features, 67 typename TTypes<T>::Tensor activations) { 68 activations.device(d) = 69 features.cwiseMax(static_cast<T>(0)).cwiseMin(static_cast<T>(6)); 70 } 71 }; 72 73 // Functor used by ReluGradOp to do the computations. 74 template <typename Device, typename T> 75 struct Relu6Grad { 76 // Computes Relu6Grad backprops. 77 // 78 // gradients: gradients backpropagated to the Relu6 op. 79 // features: inputs that where passed to the Relu6 op, or its outputs. 80 // backprops: gradients to backpropagate to the Relu6 inputs. operatorRelu6Grad81 void operator()(const Device& d, typename TTypes<T>::ConstTensor gradients, 82 typename TTypes<T>::ConstTensor features, 83 typename TTypes<T>::Tensor backprops) { 84 // NOTE: When the activation is exactly zero or six, we 85 // make sure not to propagate the associated gradient 86 // value. This allows "features" to be either the input or the output of 87 // the relu6. 88 backprops.device(d) = gradients * ((features > static_cast<T>(0)) * 89 (features < static_cast<T>(6))) 90 .template cast<T>(); 91 } 92 }; 93 94 // Functor used by LeakyReluOp to do the computations. 95 template <typename Device, typename T> 96 struct LeakyRelu { 97 // Computes LeakyRelu activation. 98 // 99 // features: any shape. 100 // activations: same shape as "features". operatorLeakyRelu101 void operator()(const Device& d, typename TTypes<T>::ConstTensor features, 102 T alpha, typename TTypes<T>::Tensor activations) { 103 // Note that alpha might be > 1 or < 0, so we don't use cwiseMax here. 104 activations.device(d) = 105 (features > static_cast<T>(0)).select(features, features * alpha); 106 } 107 }; 108 109 // Functor used by LeakyReluGradOp to do the computations. 110 template <typename Device, typename T> 111 struct LeakyReluGrad { 112 // Computes LeakyReluGrad backprops. 113 // 114 // gradients: gradients backpropagated to the LeakyRelu op. 115 // features: either the inputs that were passed to the LeakyRelu or, or its 116 // outputs (using either one yields the same result here). 117 // backprops: gradients to backpropagate to the LeakyRelu inputs. operatorLeakyReluGrad118 void operator()(const Device& d, typename TTypes<T>::ConstTensor gradients, 119 typename TTypes<T>::ConstTensor features, T alpha, 120 typename TTypes<T>::Tensor backprops) { 121 backprops.device(d) = 122 (features > static_cast<T>(0)).select(gradients, gradients * alpha); 123 } 124 }; 125 126 // Functor used by EluOp to do the computations. 127 template <typename Device, typename T> 128 struct Elu { 129 // Computes Elu activation. 130 // 131 // features: any shape. 132 // activations: same shape as "features". operatorElu133 void operator()(const Device& d, typename TTypes<T>::ConstTensor features, 134 typename TTypes<T>::Tensor activations) { 135 // features.constant(?) 136 activations.device(d) = 137 (features < static_cast<T>(0)) 138 .select(features.exp() - features.constant(static_cast<T>(1)), 139 features); 140 } 141 }; 142 143 // Functor used by EluGradOp to do the computations. 144 template <typename Device, typename T> 145 struct EluGrad { 146 // Computes EluGrad backprops. 147 // 148 // gradients: gradients backpropagated to the Elu op. 149 // activations: outputs of the Elu op. 150 // backprops: gradients to backpropagate to the Elu inputs. operatorEluGrad151 void operator()(const Device& d, typename TTypes<T>::ConstTensor gradients, 152 typename TTypes<T>::ConstTensor activations, 153 typename TTypes<T>::Tensor backprops) { 154 backprops.device(d) = 155 (activations < static_cast<T>(0)) 156 .select((activations + static_cast<T>(1)) * gradients, gradients); 157 } 158 }; 159 160 // Functor used by SeluOp to do the computations. 161 template <typename Device, typename T> 162 struct Selu { 163 // Computes Selu activation. 164 // 165 // features: any shape. 166 // activations: same shape as "features". operatorSelu167 void operator()(const Device& d, typename TTypes<T>::ConstTensor features, 168 typename TTypes<T>::Tensor activations) { 169 // features.constant(?) 170 const auto scale = static_cast<T>(1.0507009873554804934193349852946); 171 const auto scale_alpha = static_cast<T>(1.7580993408473768599402175208123); 172 const auto one = static_cast<T>(1); 173 const auto zero = static_cast<T>(0); 174 activations.device(d) = 175 (features < zero) 176 .select(scale_alpha * (features.exp() - features.constant(one)), 177 scale * features); 178 } 179 }; 180 181 // Functor used by SeluGradOp to do the computations. 182 template <typename Device, typename T> 183 struct SeluGrad { 184 // Computes SeluGrad backprops. 185 // 186 // gradients: gradients backpropagated to the Selu op. 187 // activations: outputs of the Selu op. 188 // backprops: gradients to backpropagate to the Selu inputs. operatorSeluGrad189 void operator()(const Device& d, typename TTypes<T>::ConstTensor gradients, 190 typename TTypes<T>::ConstTensor activations, 191 typename TTypes<T>::Tensor backprops) { 192 const auto scale = static_cast<T>(1.0507009873554804934193349852946); 193 const auto scale_alpha = static_cast<T>(1.7580993408473768599402175208123); 194 backprops.device(d) = 195 (activations < static_cast<T>(0)) 196 .select(gradients * (activations + scale_alpha), gradients * scale); 197 } 198 }; 199 200 } // namespace functor 201 } // namespace tensorflow 202 203 #endif // TENSORFLOW_CORE_KERNELS_RELU_OP_FUNCTOR_H_ 204