• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1# Copyright 2015 The TensorFlow Authors. All Rights Reserved.
2#
3# Licensed under the Apache License, Version 2.0 (the "License");
4# you may not use this file except in compliance with the License.
5# You may obtain a copy of the License at
6#
7#     http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS,
11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12# See the License for the specific language governing permissions and
13# limitations under the License.
14# ==============================================================================
15"""Built-in activation functions.
16"""
17from __future__ import absolute_import
18from __future__ import division
19from __future__ import print_function
20
21import six
22
23from tensorflow.python.keras import backend as K
24from tensorflow.python.keras.utils.generic_utils import deserialize_keras_object
25from tensorflow.python.ops import math_ops
26from tensorflow.python.ops import nn
27from tensorflow.python.util.tf_export import keras_export
28
29# b/123041942
30# In TF 2.x, if the `tf.nn.softmax` is used as an activation function in Keras
31# layers, it gets serialized as 'softmax_v2' instead of 'softmax' as the
32# internal method name is returned in serialization. This results in errors in
33# model exporting and loading as Keras can't find any activation function with
34# the name of `softmax_v2`.
35
36# This dict maps the activation function name from its v2 version to its
37# canonical name.
38_TF_ACTIVATIONS_V2 = {
39    'softmax_v2': 'softmax',
40}
41
42
43@keras_export('keras.activations.softmax')
44def softmax(x, axis=-1):
45  """Softmax activation function.
46
47  Arguments:
48      x : Input tensor.
49      axis: Integer, axis along which the softmax normalization is applied.
50
51  Returns:
52      Tensor, output of softmax transformation.
53
54  Raises:
55      ValueError: In case `dim(x) == 1`.
56  """
57  ndim = K.ndim(x)
58  if ndim == 2:
59    return nn.softmax(x)
60  elif ndim > 2:
61    e = math_ops.exp(x - math_ops.reduce_max(x, axis=axis, keepdims=True))
62    s = math_ops.reduce_sum(e, axis=axis, keepdims=True)
63    return e / s
64  else:
65    raise ValueError('Cannot apply softmax to a tensor that is 1D. '
66                     'Received input: %s' % (x,))
67
68
69@keras_export('keras.activations.elu')
70def elu(x, alpha=1.0):
71  """Exponential linear unit.
72
73  Arguments:
74      x: Input tensor.
75      alpha: A scalar, slope of negative section.
76
77  Returns:
78      The exponential linear activation: `x` if `x > 0` and
79        `alpha * (exp(x)-1)` if `x < 0`.
80
81  Reference:
82      - [Fast and Accurate Deep Network Learning by Exponential
83        Linear Units (ELUs)](https://arxiv.org/abs/1511.07289)
84  """
85  return K.elu(x, alpha)
86
87
88@keras_export('keras.activations.selu')
89def selu(x):
90  """Scaled Exponential Linear Unit (SELU).
91
92  SELU is equal to: `scale * elu(x, alpha)`, where alpha and scale
93  are pre-defined constants. The values of `alpha` and `scale` are
94  chosen so that the mean and variance of the inputs are preserved
95  between two consecutive layers as long as the weights are initialized
96  correctly (see `lecun_normal` initialization) and the number of inputs
97  is "large enough" (see references for more information).
98
99  Arguments:
100      x: A tensor or variable to compute the activation function for.
101
102  Returns:
103      The scaled exponential unit activation: `scale * elu(x, alpha)`.
104
105  # Note
106      - To be used together with the initialization "lecun_normal".
107      - To be used together with the dropout variant "AlphaDropout".
108
109  References:
110      - [Self-Normalizing Neural Networks](https://arxiv.org/abs/1706.02515)
111  """
112  alpha = 1.6732632423543772848170429916717
113  scale = 1.0507009873554804934193349852946
114  return scale * K.elu(x, alpha)
115
116
117@keras_export('keras.activations.softplus')
118def softplus(x):
119  """Softplus activation function.
120
121  Arguments:
122      x: Input tensor.
123
124  Returns:
125      The softplus activation: `log(exp(x) + 1)`.
126  """
127  return nn.softplus(x)
128
129
130@keras_export('keras.activations.softsign')
131def softsign(x):
132  """Softsign activation function.
133
134  Arguments:
135      x: Input tensor.
136
137  Returns:
138      The softplus activation: `x / (abs(x) + 1)`.
139  """
140  return nn.softsign(x)
141
142
143@keras_export('keras.activations.relu')
144def relu(x, alpha=0., max_value=None, threshold=0):
145  """Rectified Linear Unit.
146
147  With default values, it returns element-wise `max(x, 0)`.
148
149  Otherwise, it follows:
150  `f(x) = max_value` for `x >= max_value`,
151  `f(x) = x` for `threshold <= x < max_value`,
152  `f(x) = alpha * (x - threshold)` otherwise.
153
154  Arguments:
155      x: A tensor or variable.
156      alpha: A scalar, slope of negative section (default=`0.`).
157      max_value: float. Saturation threshold.
158      threshold: float. Threshold value for thresholded activation.
159
160  Returns:
161      A tensor.
162  """
163  return K.relu(x, alpha=alpha, max_value=max_value, threshold=threshold)
164
165
166@keras_export('keras.activations.tanh')
167def tanh(x):
168  """Hyperbolic Tangent activation function.
169
170  Arguments:
171      x: Input tensor.
172
173  Returns:
174      The tanh activation: `tanh(x) = sinh(x)/cosh(x) = ((exp(x) -
175      exp(-x))/(exp(x) + exp(-x)))`.
176  """
177  return nn.tanh(x)
178
179
180@keras_export('keras.activations.sigmoid')
181def sigmoid(x):
182  """Sigmoid activation function.
183
184  Arguments:
185      x: Input tensor.
186
187  Returns:
188      The sigmoid activation: `(1.0 / (1.0 + exp(-x)))`.
189  """
190  return nn.sigmoid(x)
191
192
193@keras_export('keras.activations.exponential')
194def exponential(x):
195  """Exponential activation function.
196
197  Arguments:
198      x: Input tensor.
199
200  Returns:
201      The exponential activation: `exp(x)`.
202  """
203  return math_ops.exp(x)
204
205
206@keras_export('keras.activations.hard_sigmoid')
207def hard_sigmoid(x):
208  """Hard sigmoid activation function.
209
210  Faster to compute than sigmoid activation.
211
212  Arguments:
213      x: Input tensor.
214
215  Returns:
216      Hard sigmoid activation:
217      - `0` if `x < -2.5`
218      - `1` if `x > 2.5`
219      - `0.2 * x + 0.5` if `-2.5 <= x <= 2.5`.
220  """
221  return K.hard_sigmoid(x)
222
223
224@keras_export('keras.activations.linear')
225def linear(x):
226  """Linear activation function.
227
228  Arguments:
229      x: Input tensor.
230
231  Returns:
232      The linear activation: `x`.
233  """
234  return x
235
236
237@keras_export('keras.activations.serialize')
238def serialize(activation):
239  if activation.__name__ in _TF_ACTIVATIONS_V2:
240    return _TF_ACTIVATIONS_V2[activation.__name__]
241  return activation.__name__
242
243
244@keras_export('keras.activations.deserialize')
245def deserialize(name, custom_objects=None):
246  return deserialize_keras_object(
247      name,
248      module_objects=globals(),
249      custom_objects=custom_objects,
250      printable_module_name='activation function')
251
252
253@keras_export('keras.activations.get')
254def get(identifier):
255  if identifier is None:
256    return linear
257  if isinstance(identifier, six.string_types):
258    identifier = str(identifier)
259    return deserialize(identifier)
260  elif callable(identifier):
261    return identifier
262  else:
263    raise ValueError('Could not interpret '
264                     'activation function identifier:', identifier)
265