1# Copyright 2015 The TensorFlow Authors. All Rights Reserved. 2# 3# Licensed under the Apache License, Version 2.0 (the "License"); 4# you may not use this file except in compliance with the License. 5# You may obtain a copy of the License at 6# 7# http://www.apache.org/licenses/LICENSE-2.0 8# 9# Unless required by applicable law or agreed to in writing, software 10# distributed under the License is distributed on an "AS IS" BASIS, 11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12# See the License for the specific language governing permissions and 13# limitations under the License. 14# ============================================================================== 15"""Layers that operate regularization via the addition of noise. 16""" 17from __future__ import absolute_import 18from __future__ import division 19from __future__ import print_function 20 21import numpy as np 22 23from tensorflow.python.keras import backend as K 24from tensorflow.python.keras.engine.base_layer import Layer 25from tensorflow.python.keras.utils import tf_utils 26from tensorflow.python.ops import array_ops 27from tensorflow.python.ops import math_ops 28from tensorflow.python.util.tf_export import keras_export 29 30 31@keras_export('keras.layers.GaussianNoise') 32class GaussianNoise(Layer): 33 """Apply additive zero-centered Gaussian noise. 34 35 This is useful to mitigate overfitting 36 (you could see it as a form of random data augmentation). 37 Gaussian Noise (GS) is a natural choice as corruption process 38 for real valued inputs. 39 40 As it is a regularization layer, it is only active at training time. 41 42 Arguments: 43 stddev: Float, standard deviation of the noise distribution. 44 45 Call arguments: 46 inputs: Input tensor (of any rank). 47 training: Python boolean indicating whether the layer should behave in 48 training mode (adding noise) or in inference mode (doing nothing). 49 50 Input shape: 51 Arbitrary. Use the keyword argument `input_shape` 52 (tuple of integers, does not include the samples axis) 53 when using this layer as the first layer in a model. 54 55 Output shape: 56 Same shape as input. 57 """ 58 59 def __init__(self, stddev, **kwargs): 60 super(GaussianNoise, self).__init__(**kwargs) 61 self.supports_masking = True 62 self.stddev = stddev 63 64 def call(self, inputs, training=None): 65 66 def noised(): 67 return inputs + K.random_normal( 68 shape=array_ops.shape(inputs), mean=0., stddev=self.stddev) 69 70 return K.in_train_phase(noised, inputs, training=training) 71 72 def get_config(self): 73 config = {'stddev': self.stddev} 74 base_config = super(GaussianNoise, self).get_config() 75 return dict(list(base_config.items()) + list(config.items())) 76 77 @tf_utils.shape_type_conversion 78 def compute_output_shape(self, input_shape): 79 return input_shape 80 81 82@keras_export('keras.layers.GaussianDropout') 83class GaussianDropout(Layer): 84 """Apply multiplicative 1-centered Gaussian noise. 85 86 As it is a regularization layer, it is only active at training time. 87 88 Arguments: 89 rate: Float, drop probability (as with `Dropout`). 90 The multiplicative noise will have 91 standard deviation `sqrt(rate / (1 - rate))`. 92 93 Call arguments: 94 inputs: Input tensor (of any rank). 95 training: Python boolean indicating whether the layer should behave in 96 training mode (adding dropout) or in inference mode (doing nothing). 97 98 Input shape: 99 Arbitrary. Use the keyword argument `input_shape` 100 (tuple of integers, does not include the samples axis) 101 when using this layer as the first layer in a model. 102 103 Output shape: 104 Same shape as input. 105 """ 106 107 def __init__(self, rate, **kwargs): 108 super(GaussianDropout, self).__init__(**kwargs) 109 self.supports_masking = True 110 self.rate = rate 111 112 def call(self, inputs, training=None): 113 if 0 < self.rate < 1: 114 115 def noised(): 116 stddev = np.sqrt(self.rate / (1.0 - self.rate)) 117 return inputs * K.random_normal( 118 shape=array_ops.shape(inputs), mean=1.0, stddev=stddev) 119 120 return K.in_train_phase(noised, inputs, training=training) 121 return inputs 122 123 def get_config(self): 124 config = {'rate': self.rate} 125 base_config = super(GaussianDropout, self).get_config() 126 return dict(list(base_config.items()) + list(config.items())) 127 128 @tf_utils.shape_type_conversion 129 def compute_output_shape(self, input_shape): 130 return input_shape 131 132 133@keras_export('keras.layers.AlphaDropout') 134class AlphaDropout(Layer): 135 """Applies Alpha Dropout to the input. 136 137 Alpha Dropout is a `Dropout` that keeps mean and variance of inputs 138 to their original values, in order to ensure the self-normalizing property 139 even after this dropout. 140 Alpha Dropout fits well to Scaled Exponential Linear Units 141 by randomly setting activations to the negative saturation value. 142 143 Arguments: 144 rate: float, drop probability (as with `Dropout`). 145 The multiplicative noise will have 146 standard deviation `sqrt(rate / (1 - rate))`. 147 seed: A Python integer to use as random seed. 148 149 Call arguments: 150 inputs: Input tensor (of any rank). 151 training: Python boolean indicating whether the layer should behave in 152 training mode (adding dropout) or in inference mode (doing nothing). 153 154 Input shape: 155 Arbitrary. Use the keyword argument `input_shape` 156 (tuple of integers, does not include the samples axis) 157 when using this layer as the first layer in a model. 158 159 Output shape: 160 Same shape as input. 161 """ 162 163 def __init__(self, rate, noise_shape=None, seed=None, **kwargs): 164 super(AlphaDropout, self).__init__(**kwargs) 165 self.rate = rate 166 self.noise_shape = noise_shape 167 self.seed = seed 168 self.supports_masking = True 169 170 def _get_noise_shape(self, inputs): 171 return self.noise_shape if self.noise_shape else array_ops.shape(inputs) 172 173 def call(self, inputs, training=None): 174 if 0. < self.rate < 1.: 175 noise_shape = self._get_noise_shape(inputs) 176 177 def dropped_inputs(inputs=inputs, rate=self.rate, seed=self.seed): # pylint: disable=missing-docstring 178 alpha = 1.6732632423543772848170429916717 179 scale = 1.0507009873554804934193349852946 180 alpha_p = -alpha * scale 181 182 kept_idx = math_ops.greater_equal( 183 K.random_uniform(noise_shape, seed=seed), rate) 184 kept_idx = math_ops.cast(kept_idx, K.floatx()) 185 186 # Get affine transformation params 187 a = ((1 - rate) * (1 + rate * alpha_p**2))**-0.5 188 b = -a * alpha_p * rate 189 190 # Apply mask 191 x = inputs * kept_idx + alpha_p * (1 - kept_idx) 192 193 # Do affine transformation 194 return a * x + b 195 196 return K.in_train_phase(dropped_inputs, inputs, training=training) 197 return inputs 198 199 def get_config(self): 200 config = {'rate': self.rate} 201 base_config = super(AlphaDropout, self).get_config() 202 return dict(list(base_config.items()) + list(config.items())) 203 204 @tf_utils.shape_type_conversion 205 def compute_output_shape(self, input_shape): 206 return input_shape 207