1# Copyright 2017 The TensorFlow Authors. All Rights Reserved. 2# 3# Licensed under the Apache License, Version 2.0 (the "License"); 4# you may not use this file except in compliance with the License. 5# You may obtain a copy of the License at 6# 7# http://www.apache.org/licenses/LICENSE-2.0 8# 9# Unless required by applicable law or agreed to in writing, software 10# distributed under the License is distributed on an "AS IS" BASIS, 11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12# See the License for the specific language governing permissions and 13# limitations under the License. 14# ============================================================================== 15"""Functions to parse RF-related parameters from TF layers.""" 16 17from __future__ import absolute_import 18from __future__ import division 19from __future__ import print_function 20 21import math 22from tensorflow.contrib.util import make_ndarray 23from tensorflow.python.platform import tf_logging as logging 24 25# White-listed layer operations, which do not affect the receptive field 26# computation. 27_UNCHANGED_RF_LAYER_OPS = [ 28 "Add", "BiasAdd", "Cast", "Ceil", "ConcatV2", "Const", "Floor", 29 "FusedBatchNorm", "Identity", "Log", "Mul", "Pow", "RealDiv", "Relu", 30 "Relu6", "Round", "Rsqrt", "Softplus", "Sub", "VariableV2", "LRN", 31 "GreaterEqual" 32] 33 34# Different ways in which padding modes may be spelled. 35_VALID_PADDING = ["VALID", b"VALID"] 36_SAME_PADDING = ["SAME", b"SAME"] 37 38 39def _stride_size(node, name_to_node): 40 """Computes stride size given a TF node. 41 42 Args: 43 node: Tensorflow node (NodeDef proto). 44 name_to_node: For MaxPoolV2, mapping from variable name Tensorflow node. 45 46 Returns: 47 stride_x: Stride size for horizontal direction (integer). 48 stride_y: Stride size for vertical direction (integer). 49 50 Raises: 51 ValueError: If stride input cannot be found in `name_to_node`. 52 """ 53 if node.op == "MaxPoolV2": 54 strides_input_name = node.input[2] 55 if not strides_input_name.endswith("/strides"): 56 raise ValueError("Strides name does not end with '/strides'") 57 strides_node = name_to_node[strides_input_name] 58 value = strides_node.attr["value"] 59 t = make_ndarray(value.tensor) 60 stride_y = t[1] 61 stride_x = t[2] 62 else: 63 strides_attr = node.attr["strides"] 64 logging.vlog(4, "strides_attr = %s", strides_attr) 65 stride_y = strides_attr.list.i[1] 66 stride_x = strides_attr.list.i[2] 67 return stride_x, stride_y 68 69 70def _conv_kernel_size(node, name_to_node): 71 """Computes kernel size given a TF convolution or pooling node. 72 73 Args: 74 node: Tensorflow node (NodeDef proto). 75 name_to_node: Dict keyed by node name, each entry containing the node's 76 NodeDef. 77 78 Returns: 79 kernel_size_x: Kernel size for horizontal direction (integer). 80 kernel_size_y: Kernel size for vertical direction (integer). 81 82 Raises: 83 ValueError: If the weight layer node is invalid. 84 """ 85 weights_layer_read_name = node.input[1] 86 if not weights_layer_read_name.endswith("/read"): 87 raise ValueError( 88 "Weight layer's name input to conv layer does not end with '/read'") 89 weights_layer_param_name = weights_layer_read_name[:-5] 90 weights_node = name_to_node[weights_layer_param_name] 91 if weights_node.op != "VariableV2": 92 raise ValueError("Weight layer is not of type VariableV2") 93 shape = weights_node.attr["shape"] 94 logging.vlog(4, "weight shape = %s", shape) 95 kernel_size_y = shape.shape.dim[0].size 96 kernel_size_x = shape.shape.dim[1].size 97 return kernel_size_x, kernel_size_y 98 99 100def _padding_size_conv_pool(node, kernel_size, stride, input_resolution=None): 101 """Computes padding size given a TF convolution or pooling node. 102 103 Args: 104 node: Tensorflow node (NodeDef proto). 105 kernel_size: Kernel size of node (integer). 106 stride: Stride size of node (integer). 107 input_resolution: Input resolution to assume, if not None (integer). 108 109 Returns: 110 total_padding: Total padding size (integer). 111 padding: Padding size, applied to the left or top (integer). 112 113 Raises: 114 ValueError: If padding is invalid. 115 """ 116 # In this case, we need to carefully consider the different TF padding modes. 117 # The padding depends on kernel size, and may depend on input size. If it 118 # depends on input size and input_resolution is None, we raise an exception. 119 padding_attr = node.attr["padding"] 120 logging.vlog(4, "padding_attr = %s", padding_attr) 121 if padding_attr.s in _VALID_PADDING: 122 total_padding = 0 123 padding = 0 124 elif padding_attr.s in _SAME_PADDING: 125 if input_resolution is None: 126 # In this case, we do not know the input resolution, so we can only know 127 # the padding in some special cases. 128 if kernel_size == 1: 129 total_padding = 0 130 padding = 0 131 elif stride == 1: 132 total_padding = kernel_size - 1 133 padding = int(math.floor(float(total_padding) / 2)) 134 elif stride == 2 and kernel_size % 2 == 0: 135 # In this case, we can be sure of the left/top padding, but not of the 136 # total padding. 137 total_padding = None 138 padding = int(math.floor((float(kernel_size) - 1) / 2)) 139 else: 140 total_padding = None 141 padding = None 142 logging.warning( 143 "Padding depends on input size, which means that the effective " 144 "padding may be different depending on the input image " 145 "dimensionality. In this case, alignment check will be skipped. If" 146 " you know the input resolution, please set it.") 147 else: 148 # First, compute total_padding based on documentation. 149 if input_resolution % stride == 0: 150 total_padding = int(max(float(kernel_size - stride), 0.0)) 151 else: 152 total_padding = int( 153 max(float(kernel_size - (input_resolution % stride)), 0.0)) 154 # Then, compute left/top padding. 155 padding = int(math.floor(float(total_padding) / 2)) 156 157 else: 158 raise ValueError("Invalid padding operation %s" % padding_attr.s) 159 return total_padding, padding 160 161 162def _pool_kernel_size(node, name_to_node): 163 """Computes kernel size given a TF pooling node. 164 165 Args: 166 node: Tensorflow node (NodeDef proto). 167 name_to_node: For MaxPoolV2, mapping from node name to NodeDef. 168 169 Returns: 170 kernel_size_x: Kernel size for horizontal direction (integer). 171 kernel_size_y: Kernel size for vertical direction (integer). 172 173 Raises: 174 ValueError: If pooling is invalid. 175 """ 176 if node.op == "MaxPoolV2": 177 ksize_input_name = node.input[1] 178 if not ksize_input_name.endswith("/ksize"): 179 raise ValueError("Kernel size name does not end with '/ksize'") 180 ksize_node = name_to_node[ksize_input_name] 181 value = ksize_node.attr["value"] 182 t = make_ndarray(value.tensor) 183 kernel_size_y = t[1] 184 kernel_size_x = t[2] 185 if t[0] != 1: 186 raise ValueError("pool ksize for first dim is not 1") 187 if t[3] != 1: 188 raise ValueError("pool ksize for last dim is not 1") 189 else: 190 ksize = node.attr["ksize"] 191 kernel_size_y = ksize.list.i[1] 192 kernel_size_x = ksize.list.i[2] 193 if ksize.list.i[0] != 1: 194 raise ValueError("pool ksize for first dim is not 1") 195 if ksize.list.i[3] != 1: 196 raise ValueError("pool ksize for last dim is not 1") 197 return kernel_size_x, kernel_size_y 198 199 200def _padding_size_pad_layer(node, name_to_node): 201 """Computes padding size given a TF padding node. 202 203 Args: 204 node: Tensorflow node (NodeDef proto). 205 name_to_node: Dict keyed by node name, each entry containing the node's 206 NodeDef. 207 208 Returns: 209 total_padding_x: Total padding size for horizontal direction (integer). 210 padding_x: Padding size for horizontal direction, left side (integer). 211 total_padding_y: Total padding size for vertical direction (integer). 212 padding_y: Padding size for vertical direction, top side (integer). 213 214 Raises: 215 ValueError: If padding layer is invalid. 216 """ 217 paddings_layer_name = node.input[1] 218 if not paddings_layer_name.endswith("/paddings"): 219 raise ValueError("Padding layer name does not end with '/paddings'") 220 paddings_node = name_to_node[paddings_layer_name] 221 if paddings_node.op != "Const": 222 raise ValueError("Padding op is not Const") 223 value = paddings_node.attr["value"] 224 t = make_ndarray(value.tensor) 225 padding_y = t[1][0] 226 padding_x = t[2][0] 227 total_padding_y = padding_y + t[1][1] 228 total_padding_x = padding_x + t[2][1] 229 if (t[0][0] != 0) or (t[0][1] != 0): 230 raise ValueError("padding is not zero for first tensor dim") 231 if (t[3][0] != 0) or (t[3][1] != 0): 232 raise ValueError("padding is not zero for last tensor dim") 233 return total_padding_x, padding_x, total_padding_y, padding_y 234 235 236def get_layer_params(node, name_to_node, input_resolution=None, force=False): 237 """Gets layer parameters relevant for RF computation. 238 239 Currently, only these nodes are supported: 240 - Conv2D 241 - DepthwiseConv2dNative 242 - Pad 243 - MaxPool 244 - AvgPool 245 - all nodes listed in _UNCHANGED_RF_LAYER_OPS 246 247 Args: 248 node: Tensorflow node (NodeDef proto). 249 name_to_node: Dict keyed by node name, each entry containing the node's 250 NodeDef. 251 input_resolution: List with 2 dimensions, denoting the height/width of the 252 input feature map to this layer. If set to None, then the padding may be 253 undefined (in tensorflow, SAME padding depends on input spatial 254 resolution). 255 force: If True, the function does not raise a ValueError if the layer op is 256 unknown. Instead, in this case it sets each of the returned parameters to 257 None. 258 259 Returns: 260 kernel_size_x: Kernel size for horizontal direction (integer). 261 kernel_size_y: Kernel size for vertical direction (integer). 262 stride_x: Stride size for horizontal direction (integer). 263 stride_y: Stride size for vertical direction (integer). 264 padding_x: Padding size for horizontal direction, left side (integer). 265 padding_y: Padding size for vertical direction, top side (integer). 266 total_padding_x: Total padding size for horizontal direction (integer). 267 total_padding_y: Total padding size for vertical direction (integer). 268 269 Raises: 270 ValueError: If layer op is unknown and force is False. 271 """ 272 logging.vlog(3, "node.name = %s", node.name) 273 logging.vlog(3, "node.op = %s", node.op) 274 logging.vlog(4, "node = %s", node) 275 if node.op == "Conv2D" or node.op == "DepthwiseConv2dNative": 276 stride_x, stride_y = _stride_size(node, name_to_node) 277 kernel_size_x, kernel_size_y = _conv_kernel_size(node, name_to_node) 278 # Compute the padding for this node separately for each direction. 279 total_padding_x, padding_x = _padding_size_conv_pool( 280 node, kernel_size_x, stride_x, 281 input_resolution[1] if input_resolution is not None else None) 282 total_padding_y, padding_y = _padding_size_conv_pool( 283 node, kernel_size_y, stride_y, 284 input_resolution[0] if input_resolution is not None else None) 285 elif node.op == "Pad": 286 # Kernel and stride are simply 1 in this case. 287 kernel_size_x = 1 288 kernel_size_y = 1 289 stride_x = 1 290 stride_y = 1 291 total_padding_x, padding_x, total_padding_y, padding_y = ( 292 _padding_size_pad_layer(node, name_to_node)) 293 elif node.op == "MaxPool" or node.op == "MaxPoolV2" or node.op == "AvgPool": 294 stride_x, stride_y = _stride_size(node, name_to_node) 295 kernel_size_x, kernel_size_y = _pool_kernel_size(node, name_to_node) 296 # Compute the padding for this node separately for each direction. 297 total_padding_x, padding_x = _padding_size_conv_pool( 298 node, kernel_size_x, stride_x, 299 input_resolution[1] if input_resolution is not None else None) 300 total_padding_y, padding_y = _padding_size_conv_pool( 301 node, kernel_size_y, stride_y, 302 input_resolution[0] if input_resolution is not None else None) 303 elif node.op in _UNCHANGED_RF_LAYER_OPS: 304 # These nodes do not modify the RF parameters. 305 kernel_size_x = 1 306 kernel_size_y = 1 307 stride_x = 1 308 stride_y = 1 309 total_padding_x = 0 310 padding_x = 0 311 total_padding_y = 0 312 padding_y = 0 313 else: 314 if force: 315 kernel_size_x = None 316 kernel_size_y = None 317 stride_x = None 318 stride_y = None 319 total_padding_x = None 320 padding_x = None 321 total_padding_y = None 322 padding_y = None 323 else: 324 raise ValueError( 325 "Unknown layer for operation '%s': %s" % (node.name, node.op)) 326 return (kernel_size_x, kernel_size_y, stride_x, stride_y, padding_x, 327 padding_y, total_padding_x, total_padding_y) 328