• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
2#
3# Licensed under the Apache License, Version 2.0 (the "License");
4# you may not use this file except in compliance with the License.
5# You may obtain a copy of the License at
6#
7#     http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS,
11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12# See the License for the specific language governing permissions and
13# limitations under the License.
14# =============================================================================
15
16"""Helper context for running models with bfloat16."""
17
18from __future__ import absolute_import
19from __future__ import division
20from __future__ import print_function
21
22from tensorflow.python.framework import dtypes
23from tensorflow.python.ops import math_ops
24from tensorflow.python.ops import variable_scope
25from tensorflow.python.util import tf_contextlib
26from tensorflow.python.util.tf_export import tf_export
27
28
29def _get_custom_getter():
30  """Returns a custom getter that this class's methods must be called under.
31
32  All methods of this class must be called under a variable scope that was
33  passed this custom getter. Example:
34
35  ```python
36  network = ConvNetBuilder(...)
37  with tf.compat.v1.variable_scope('cg',
38                                   custom_getter=network.get_custom_getter()):
39    network.conv(...)
40    # Call more methods of network here
41  ```
42
43  Currently, this custom getter only does anything if self.use_tf_layers is
44  True. In that case, it causes variables to be stored as dtype
45  self.variable_type, then casted to the requested dtype, instead of directly
46  storing the variable as the requested dtype.
47  """
48
49  def inner_custom_getter(getter, *args, **kwargs):
50    """Custom getter that forces variables to have type self.variable_type."""
51    cast_to_bfloat16 = False
52    requested_dtype = kwargs['dtype']
53    if requested_dtype == dtypes.bfloat16:
54      # Only change the variable dtype if doing so does not decrease variable
55      # precision.
56      kwargs['dtype'] = dtypes.float32
57      cast_to_bfloat16 = True
58    var = getter(*args, **kwargs)
59    # This if statement is needed to guard the cast, because batch norm
60    # assigns directly to the return value of this custom getter. The cast
61    # makes the return value not a variable so it cannot be assigned. Batch
62    # norm variables are always in fp32 so this if statement is never
63    # triggered for them.
64    if cast_to_bfloat16:
65      var = math_ops.cast(var, dtypes.bfloat16)
66    return var
67
68  return inner_custom_getter
69
70
71@tf_export(v1=['tpu.bfloat16_scope'])
72@tf_contextlib.contextmanager
73def bfloat16_scope():
74  """Scope class for bfloat16 variables so that the model uses custom getter.
75
76  This enables variables to be read as bfloat16 type when using get_variable.
77  """
78  with variable_scope.variable_scope(
79      '', custom_getter=_get_custom_getter()) as varscope:
80    yield varscope
81