1# Copyright 2015 The TensorFlow Authors. All Rights Reserved. 2# 3# Licensed under the Apache License, Version 2.0 (the "License"); 4# you may not use this file except in compliance with the License. 5# You may obtain a copy of the License at 6# 7# http://www.apache.org/licenses/LICENSE-2.0 8# 9# Unless required by applicable law or agreed to in writing, software 10# distributed under the License is distributed on an "AS IS" BASIS, 11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12# See the License for the specific language governing permissions and 13# limitations under the License. 14# ============================================================================== 15"""Functional test for GradientDescent.""" 16 17from tensorflow.python.eager import backprop 18from tensorflow.python.eager import context 19from tensorflow.python.eager import function 20from tensorflow.python.framework import constant_op 21from tensorflow.python.framework import dtypes 22from tensorflow.python.framework import indexed_slices 23from tensorflow.python.framework import ops 24from tensorflow.python.ops import embedding_ops 25from tensorflow.python.ops import math_ops 26from tensorflow.python.ops import resource_variable_ops 27from tensorflow.python.ops import resources 28from tensorflow.python.ops import variables 29from tensorflow.python.platform import test 30from tensorflow.python.training import gradient_descent 31 32 33class GradientDescentOptimizerTest(test.TestCase): 34 35 def testBasic(self): 36 for dtype in [dtypes.half, dtypes.float32, dtypes.float64]: 37 # train.GradientDescentOptimizer is V1 only API. 38 with ops.Graph().as_default(), self.cached_session(): 39 var0 = variables.Variable([1.0, 2.0], dtype=dtype) 40 var1 = variables.Variable([3.0, 4.0], dtype=dtype) 41 grads0 = constant_op.constant([0.1, 0.1], dtype=dtype) 42 grads1 = constant_op.constant([0.01, 0.01], dtype=dtype) 43 optimizer = gradient_descent.GradientDescentOptimizer(3.0) 44 sgd_op = optimizer.apply_gradients( 45 zip([grads0, grads1], [var0, var1])) 46 self.evaluate(variables.global_variables_initializer()) 47 # Fetch params to validate initial values 48 self.assertAllCloseAccordingToType([1.0, 2.0], self.evaluate(var0)) 49 self.assertAllCloseAccordingToType([3.0, 4.0], self.evaluate(var1)) 50 # Run 1 step of sgd 51 sgd_op.run() 52 # Validate updated params 53 self.assertAllCloseAccordingToType([1.0 - 3.0 * 0.1, 2.0 - 3.0 * 0.1], 54 self.evaluate(var0)) 55 self.assertAllCloseAccordingToType([3.0 - 3.0 * 0.01, 4.0 - 3.0 * 0.01], 56 self.evaluate(var1)) 57 self.assertEqual(0, len(optimizer.variables())) 58 59 def testBasicResourceVariable(self): 60 for dtype in [dtypes.half, dtypes.float32, dtypes.float64]: 61 # train.GradientDescentOptimizer is V1 only API. 62 with ops.Graph().as_default(), self.cached_session(): 63 var0 = resource_variable_ops.ResourceVariable([1.0, 2.0], dtype=dtype) 64 var1 = resource_variable_ops.ResourceVariable([3.0, 4.0], dtype=dtype) 65 grads0 = constant_op.constant([0.1, 0.1], dtype=dtype) 66 grads1 = constant_op.constant([0.01, 0.01], dtype=dtype) 67 sgd_op = gradient_descent.GradientDescentOptimizer(3.0).apply_gradients( 68 zip([grads0, grads1], [var0, var1])) 69 # TODO(apassos) calling initialize_resources on all resources here 70 # doesn't work because the sessions and graph are reused across unit 71 # tests and this would mean trying to reinitialize variables. Figure out 72 # a long-term solution for this. 73 resources.initialize_resources([var0, var1]).run() 74 # Fetch params to validate initial values 75 self.assertAllCloseAccordingToType([1.0, 2.0], self.evaluate(var0)) 76 self.assertAllCloseAccordingToType([3.0, 4.0], self.evaluate(var1)) 77 # Run 1 step of sgd 78 sgd_op.run() 79 # Validate updated params 80 self.assertAllCloseAccordingToType([1.0 - 3.0 * 0.1, 2.0 - 3.0 * 0.1], 81 self.evaluate(var0)) 82 self.assertAllCloseAccordingToType([3.0 - 3.0 * 0.01, 4.0 - 3.0 * 0.01], 83 self.evaluate(var1)) 84 85 def testBasicCallableParams(self): 86 for dtype in [dtypes.half, dtypes.float32, dtypes.float64]: 87 # train.GradientDescentOptimizer is V1 only API. 88 with ops.Graph().as_default(), self.cached_session(): 89 var0 = resource_variable_ops.ResourceVariable([1.0, 2.0], dtype=dtype) 90 var1 = resource_variable_ops.ResourceVariable([3.0, 4.0], dtype=dtype) 91 grads0 = constant_op.constant([0.1, 0.1], dtype=dtype) 92 grads1 = constant_op.constant([0.01, 0.01], dtype=dtype) 93 lr = lambda: 3.0 94 sgd_op = gradient_descent.GradientDescentOptimizer(lr).apply_gradients( 95 zip([grads0, grads1], [var0, var1])) 96 # TODO(apassos) calling initialize_resources on all resources here 97 # doesn't work because the sessions and graph are reused across unit 98 # tests and this would mean trying to reinitialize variables. Figure out 99 # a long-term solution for this. 100 resources.initialize_resources([var0, var1]).run() 101 # Fetch params to validate initial values 102 self.assertAllCloseAccordingToType([1.0, 2.0], self.evaluate(var0)) 103 self.assertAllCloseAccordingToType([3.0, 4.0], self.evaluate(var1)) 104 # Run 1 step of sgd 105 sgd_op.run() 106 # Validate updated params 107 self.assertAllCloseAccordingToType([1.0 - 3.0 * 0.1, 2.0 - 3.0 * 0.1], 108 self.evaluate(var0)) 109 self.assertAllCloseAccordingToType([3.0 - 3.0 * 0.01, 4.0 - 3.0 * 0.01], 110 self.evaluate(var1)) 111 112 def testMinimizeResourceVariable(self): 113 for dtype in [dtypes.half, dtypes.float32, dtypes.float64]: 114 # train.GradientDescentOptimizer is V1 only API. 115 with ops.Graph().as_default(), self.cached_session(): 116 var0 = resource_variable_ops.ResourceVariable([[1.0, 2.0]], dtype=dtype) 117 var1 = resource_variable_ops.ResourceVariable([3.0], dtype=dtype) 118 x = constant_op.constant([[4.0], [5.0]], dtype=dtype) 119 pred = math_ops.matmul(var0, x) + var1 120 loss = pred * pred 121 sgd_op = gradient_descent.GradientDescentOptimizer(1.0).minimize(loss) 122 # TODO(apassos) calling initialize_resources on all resources here 123 # doesn't work because the sessions and graph are reused across unit 124 # tests and this would mean trying to reinitialize variables. Figure out 125 # a long-term solution for this. 126 resources.initialize_resources([var0, var1]).run() 127 # Fetch params to validate initial values 128 self.assertAllCloseAccordingToType([[1.0, 2.0]], self.evaluate(var0)) 129 self.assertAllCloseAccordingToType([3.0], self.evaluate(var1)) 130 # Run 1 step of sgd 131 sgd_op.run() 132 # Validate updated params 133 np_pred = 1.0 * 4.0 + 2.0 * 5.0 + 3.0 134 np_grad = 2 * np_pred 135 self.assertAllCloseAccordingToType( 136 [[1.0 - np_grad * 4.0, 2.0 - np_grad * 5.0]], self.evaluate(var0)) 137 self.assertAllCloseAccordingToType([3.0 - np_grad], self.evaluate(var1)) 138 139 def testMinimizeSparseResourceVariable(self): 140 for dtype in [dtypes.half, dtypes.float32, dtypes.float64]: 141 # train.GradientDescentOptimizer is V1 only API. 142 with ops.Graph().as_default(), self.cached_session(): 143 var0 = resource_variable_ops.ResourceVariable([[1.0, 2.0]], dtype=dtype) 144 var1 = resource_variable_ops.ResourceVariable([3.0], dtype=dtype) 145 x = constant_op.constant([[4.0], [5.0]], dtype=dtype) 146 pred = math_ops.matmul(embedding_ops.embedding_lookup([var0], [0]), x) 147 pred += var1 148 loss = pred * pred 149 sgd_op = gradient_descent.GradientDescentOptimizer(1.0).minimize(loss) 150 # TODO(apassos) calling initialize_resources on all resources here 151 # doesn't work because the sessions and graph are reused across unit 152 # tests and this would mean trying to reinitialize variables. Figure out 153 # a long-term solution for this. 154 self.evaluate(variables.global_variables_initializer()) 155 # Fetch params to validate initial values 156 self.assertAllCloseAccordingToType([[1.0, 2.0]], self.evaluate(var0)) 157 self.assertAllCloseAccordingToType([3.0], self.evaluate(var1)) 158 # Run 1 step of sgd 159 sgd_op.run() 160 # Validate updated params 161 np_pred = 1.0 * 4.0 + 2.0 * 5.0 + 3.0 162 np_grad = 2 * np_pred 163 self.assertAllCloseAccordingToType( 164 [[1.0 - np_grad * 4.0, 2.0 - np_grad * 5.0]], self.evaluate(var0)) 165 self.assertAllCloseAccordingToType([3.0 - np_grad], self.evaluate(var1)) 166 167 def testTensorLearningRate(self): 168 for dtype in [dtypes.half, dtypes.float32, dtypes.float64]: 169 # train.GradientDescentOptimizer is V1 only API. 170 with ops.Graph().as_default(), self.cached_session(): 171 var0 = variables.Variable([1.0, 2.0], dtype=dtype) 172 var1 = variables.Variable([3.0, 4.0], dtype=dtype) 173 grads0 = constant_op.constant([0.1, 0.1], dtype=dtype) 174 grads1 = constant_op.constant([0.01, 0.01], dtype=dtype) 175 lrate = constant_op.constant(3.0) 176 sgd_op = gradient_descent.GradientDescentOptimizer( 177 lrate).apply_gradients(zip([grads0, grads1], [var0, var1])) 178 self.evaluate(variables.global_variables_initializer()) 179 # Fetch params to validate initial values 180 self.assertAllCloseAccordingToType([1.0, 2.0], self.evaluate(var0)) 181 self.assertAllCloseAccordingToType([3.0, 4.0], self.evaluate(var1)) 182 # Run 1 step of sgd 183 sgd_op.run() 184 # Validate updated params 185 self.assertAllCloseAccordingToType([1.0 - 3.0 * 0.1, 2.0 - 3.0 * 0.1], 186 self.evaluate(var0)) 187 self.assertAllCloseAccordingToType([3.0 - 3.0 * 0.01, 4.0 - 3.0 * 0.01], 188 self.evaluate(var1)) 189 190 def testGradWrtRef(self): 191 for dtype in [dtypes.half, dtypes.float32, dtypes.float64]: 192 # train.GradientDescentOptimizer is V1 only API. 193 with ops.Graph().as_default(), self.cached_session(): 194 opt = gradient_descent.GradientDescentOptimizer(3.0) 195 values = [1.0, 3.0] 196 vars_ = [variables.Variable([v], dtype=dtype) for v in values] 197 grads_and_vars = opt.compute_gradients(vars_[0] + vars_[1], vars_) 198 self.evaluate(variables.global_variables_initializer()) 199 for grad, _ in grads_and_vars: 200 self.assertAllCloseAccordingToType([1.0], self.evaluate(grad)) 201 202 def testWithGlobalStep(self): 203 for dtype in [dtypes.half, dtypes.float32, dtypes.float64]: 204 # train.GradientDescentOptimizer is V1 only API. 205 with ops.Graph().as_default(), self.cached_session(): 206 global_step = variables.Variable(0, trainable=False) 207 var0 = variables.Variable([1.0, 2.0], dtype=dtype) 208 var1 = variables.Variable([3.0, 4.0], dtype=dtype) 209 grads0 = constant_op.constant([0.1, 0.1], dtype=dtype) 210 grads1 = constant_op.constant([0.01, 0.01], dtype=dtype) 211 sgd_op = gradient_descent.GradientDescentOptimizer(3.0).apply_gradients( 212 zip([grads0, grads1], [var0, var1]), global_step=global_step) 213 self.evaluate(variables.global_variables_initializer()) 214 # Fetch params to validate initial values 215 self.assertAllCloseAccordingToType([1.0, 2.0], self.evaluate(var0)) 216 self.assertAllCloseAccordingToType([3.0, 4.0], self.evaluate(var1)) 217 # Run 1 step of sgd 218 sgd_op.run() 219 # Validate updated params and global_step 220 self.assertAllCloseAccordingToType([1.0 - 3.0 * 0.1, 2.0 - 3.0 * 0.1], 221 self.evaluate(var0)) 222 self.assertAllCloseAccordingToType([3.0 - 3.0 * 0.01, 4.0 - 3.0 * 0.01], 223 self.evaluate(var1)) 224 self.assertAllCloseAccordingToType(1, self.evaluate(global_step)) 225 226 def testSparseBasic(self): 227 for dtype in [dtypes.half, dtypes.float32, dtypes.float64]: 228 # train.GradientDescentOptimizer is V1 only API. 229 with ops.Graph().as_default(), self.cached_session(): 230 var0 = variables.Variable([[1.0], [2.0]], dtype=dtype) 231 var1 = variables.Variable([[3.0], [4.0]], dtype=dtype) 232 grads0 = indexed_slices.IndexedSlices( 233 constant_op.constant( 234 [0.1], shape=[1, 1], dtype=dtype), 235 constant_op.constant([0]), 236 constant_op.constant([2, 1])) 237 grads1 = indexed_slices.IndexedSlices( 238 constant_op.constant( 239 [0.01], shape=[1, 1], dtype=dtype), 240 constant_op.constant([1]), 241 constant_op.constant([2, 1])) 242 sgd_op = gradient_descent.GradientDescentOptimizer(3.0).apply_gradients( 243 zip([grads0, grads1], [var0, var1])) 244 self.evaluate(variables.global_variables_initializer()) 245 # Fetch params to validate initial values 246 self.assertAllCloseAccordingToType([[1.0], [2.0]], self.evaluate(var0)) 247 self.assertAllCloseAccordingToType([[3.0], [4.0]], self.evaluate(var1)) 248 # Run 1 step of sgd 249 sgd_op.run() 250 # Validate updated params 251 self.assertAllCloseAccordingToType([[1.0 - 3.0 * 0.1], [2.0]], 252 self.evaluate(var0)) 253 self.assertAllCloseAccordingToType([[3.0], [4.0 - 3.0 * 0.01]], 254 self.evaluate(var1)) 255 256 def testCapturingInDefunWhileExecutingEagerly(self): 257 with context.eager_mode(): 258 optimizer = gradient_descent.GradientDescentOptimizer(1.0) 259 260 def step(): 261 self.v = resource_variable_ops.ResourceVariable(1.0) 262 with backprop.GradientTape() as tape: 263 loss = self.v ** 2 264 grad = tape.gradient(loss, self.v) 265 optimizer.apply_gradients([(grad, self.v)]) 266 return self.v.read_value() 267 268 compiled_step = function.defun(step) 269 270 self.assertEqual(float(step()), -1.0) 271 self.assertEqual(float(compiled_step()), -1.0) 272 # This shouldn't fail; in particular, the learning rate tensor should 273 # be an EagerTensor once again, not a graph Tensor. 274 self.assertEqual(float(step()), -1.0) 275 276 277if __name__ == "__main__": 278 test.main() 279