1# Copyright 2018 The TensorFlow Authors. All Rights Reserved. 2# 3# Licensed under the Apache License, Version 2.0 (the "License"); 4# you may not use this file except in compliance with the License. 5# You may obtain a copy of the License at 6# 7# http://www.apache.org/licenses/LICENSE-2.0 8# 9# Unless required by applicable law or agreed to in writing, software 10# distributed under the License is distributed on an "AS IS" BASIS, 11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12# See the License for the specific language governing permissions and 13# limitations under the License. 14# ============================================================================== 15"""Utility to re-use variables created on first device on subsequent devices.""" 16 17from __future__ import absolute_import 18from __future__ import division 19from __future__ import print_function 20 21import re 22 23_VARIABLE_UNIQUIFYING_REGEX = re.compile(r"_\d/") 24_VARIABLE_UNIQUIFYING_REGEX_AT_END = re.compile(r"_\d$") 25 26 27def _canonicalize_variable_name(name): 28 # If no name is specified, uses default name "Variable". 29 if name is None: 30 return "Variable" 31 # Replace all instances of "_<num>/" with "/" 32 name = _VARIABLE_UNIQUIFYING_REGEX.sub("/", name) 33 # Replace any instances of "_<num>" at the end of the string with "" 34 name = _VARIABLE_UNIQUIFYING_REGEX_AT_END.sub("", name) 35 return name 36 37 38def make_fn(shared_variable_store, device_id): 39 """Construct the variable creator function for device `device_id`. 40 41 Constructs custom variable creator functions for the given device. 42 On first device (device_id == 0), it creates the variable using the 43 `next_creator`, and stores it in the provided `shared_variable_store`. 44 On all other devices (device_id > 0), it tries to re-use the variable 45 already created with the same name. If no such variable exists, it throws an 46 error. 47 Additionally, we de-uniquify variable names before checking for matches. This 48 helps re-use variables which are intended to be the same but have different 49 names due to variable uniquification happening upstream. Since this might 50 mean we may have multiple variables with the same canonical name, we store 51 them in a list per canonical name and return them in the same order as well. 52 53 Args: 54 shared_variable_store: A dictionary that we will use to store variables 55 created on the first device, and re-used by creators for other devices. 56 device_id: Integer index of the device whose creator should be 57 constructed. 58 59 Returns: 60 An appropriate creator function based on device_id. 61 62 """ 63 variable_scope_access_index = {} 64 assert isinstance(device_id, int) 65 66 def create_new_variable(next_creator, **kwargs): 67 """Create the variable using `next_creator` and store it.""" 68 canonical_name = _canonicalize_variable_name(kwargs.get("name")) 69 v = next_creator(**kwargs) 70 71 if canonical_name not in shared_variable_store: 72 shared_variable_store[canonical_name] = [] 73 shared_variable_store[canonical_name].append(v) 74 return v 75 76 def reuse_variable(next_creator, **kwargs): 77 """Re-use existing variable from store with same name (in order).""" 78 del next_creator 79 name = kwargs.get("name") 80 canonical_name = _canonicalize_variable_name(name) 81 82 try: 83 variable_index = variable_scope_access_index.get(canonical_name, 0) 84 v = shared_variable_store[canonical_name][variable_index] 85 # TODO(priyag): Make this variable re-use more robust by adding checks 86 # that the requested shape and dtype match the existing variable. 87 variable_scope_access_index[canonical_name] = variable_index + 1 88 return v 89 except (KeyError, IndexError): 90 raise RuntimeError( 91 "Tried to create variable {} with mismatching name on device {}". 92 format(name, device_id)) 93 94 if device_id == 0: 95 return create_new_variable 96 else: 97 return reuse_variable 98