1# Copyright 2020-2021 Huawei Technologies Co., Ltd 2# 3# Licensed under the Apache License, Version 2.0 (the "License"); 4# you may not use this file except in compliance with the License. 5# You may obtain a copy of the License at 6# 7# http://www.apache.org/licenses/LICENSE-2.0 8# 9# Unless required by applicable law or agreed to in writing, software 10# distributed under the License is distributed on an "AS IS" BASIS, 11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12# See the License for the specific language governing permissions and 13# limitations under the License. 14# ============================================================================ 15"""Initializer for cell parameters.""" 16import numbers 17import math 18 19from functools import reduce 20import numpy as np 21from scipy.stats import truncnorm 22from .seed import get_seed, _get_graph_seed 23from . import dtype as mstype 24from .tensor import Tensor 25from .._c_expression import random_normal 26 27_INITIALIZER_ALIAS = dict() 28 29 30class Initializer: 31 """ 32 The base class of the initializer. 33 Initialization of tensor basic attributes and model weight values. 34 35 Args: 36 kwargs (dict): Keyword arguments for Initializer. 37 """ 38 def __init__(self, **kwargs): 39 self._kwargs = kwargs 40 self._seed = None 41 42 @property 43 def seed(self): 44 if self._seed is None: 45 seed, seed2 = _get_graph_seed(get_seed(), "init") 46 else: 47 seed, seed2 = self._seed + 1, 0 48 return seed, seed2 49 50 @seed.setter 51 def seed(self, value): 52 self._seed = value 53 54 def _initialize(self, *kwargs): 55 raise NotImplementedError('Must be overridden!') 56 57 def __call__(self, arr): 58 return self._initialize(arr) 59 60def _register(*aliases): 61 """Return the alias register.""" 62 def alias_reg(cls): 63 name = cls.__name__ 64 name = name.lower() 65 if name not in _INITIALIZER_ALIAS: 66 _INITIALIZER_ALIAS[name] = cls 67 68 for alias in aliases: 69 if alias not in _INITIALIZER_ALIAS: 70 _INITIALIZER_ALIAS[alias] = cls 71 72 return cls 73 74 return alias_reg 75 76 77def _assignment(arr, num): 78 """Assign the value of `num` to `arr`.""" 79 if arr.shape == (): 80 arr = arr.reshape(1) 81 arr[:] = num 82 arr = arr.reshape(()) 83 else: 84 if isinstance(num, np.ndarray): 85 arr[:] = num[:] 86 else: 87 arr[:] = num 88 return arr 89 90 91@_register('zeros') 92class Zero(Initializer): 93 """ 94 Fills the input array with the values zero. 95 96 Args: 97 arr (Array): The array to be assigned. 98 99 100 Examples: 101 >>> import mindspore 102 >>> from mindspore.common.initializer import initializer, Zero 103 >>> tensor1 = initializer(Zero(), [1, 2, 3], mindspore.float32) 104 >>> tensor2 = initializer('zeros', [1, 2, 3], mindspore.float32) 105 """ 106 def _initialize(self, arr): 107 _assignment(arr, 0) 108 109 110@_register('ones') 111class One(Initializer): 112 """ 113 Fills the input array with the values one. 114 115 Args: 116 arr (Array): The array to be assigned. 117 118 119 Examples: 120 >>> import mindspore 121 >>> from mindspore.common.initializer import initializer, One 122 >>> tensor1 = initializer(One(), [1, 2, 3], mindspore.float32) 123 >>> tensor2 = initializer('ones', [1, 2, 3], mindspore.float32) 124 """ 125 def _initialize(self, arr): 126 _assignment(arr, 1) 127 128 129def _calculate_fan_in_and_fan_out(shape): 130 """ 131 calculate fan_in and fan_out 132 133 Args: 134 shape (tuple): input shape. 135 136 Returns: 137 Tuple, a tuple with two elements, the first element is `n_in` and the second element is `n_out`. 138 """ 139 dimensions = len(shape) 140 if dimensions < 2: 141 raise ValueError("'fan_in' and 'fan_out' can not be computed for tensor with fewer than" 142 " 2 dimensions, but got dimensions {}.".format(dimensions)) 143 if dimensions == 2: # Linear 144 fan_in = shape[1] 145 fan_out = shape[0] 146 else: 147 num_input_fmaps = shape[1] 148 num_output_fmaps = shape[0] 149 receptive_field_size = 1 150 for i in range(2, dimensions): 151 receptive_field_size *= shape[i] 152 fan_in = num_input_fmaps * receptive_field_size 153 fan_out = num_output_fmaps * receptive_field_size 154 return fan_in, fan_out 155 156 157def _calculate_correct_fan(shape, mode): 158 """ 159 Calculate fan. 160 161 Args: 162 shape (tuple): input shape. 163 mode (str): only support fan_in and fan_out. 164 165 Returns: 166 fan_in or fan_out. 167 """ 168 mode = mode.lower() 169 valid_modes = ['fan_in', 'fan_out'] 170 if mode not in valid_modes: 171 raise ValueError("Mode {} not supported, please use one of {}".format(mode, valid_modes)) 172 fan_in, fan_out = _calculate_fan_in_and_fan_out(shape) 173 return fan_in if mode == 'fan_in' else fan_out 174 175 176def _calculate_gain(nonlinearity, param=None): 177 """ 178 Calculate gain. 179 180 Args: 181 nonlinearity (str): nonlinearity function. 182 param (str): used to calculate negative_slope. 183 184 Returns: 185 number. 186 """ 187 linear_fns = ['linear', 'conv1d', 'conv2d', 'conv3d', 'conv_transpose1d', 'conv_transpose2d', 'conv_transpose3d'] 188 if nonlinearity in linear_fns or nonlinearity == 'sigmoid': 189 res = 1 190 elif nonlinearity == 'tanh': 191 res = 5.0 / 3 192 elif nonlinearity == 'relu': 193 res = math.sqrt(2.0) 194 elif nonlinearity == 'leaky_relu': 195 if param is None: 196 negative_slope = 0.01 197 elif not isinstance(param, bool) and isinstance(param, int) or isinstance(param, float): 198 # True/False are instances of int, hence check above 199 negative_slope = param 200 else: 201 raise ValueError("negative_slope {} is not a valid number. " 202 "It should be bool, int, or float type.".format(param)) 203 res = math.sqrt(2.0 / (1 + negative_slope ** 2)) 204 else: 205 raise ValueError("Unsupported nonlinearity {}, the argument 'nonlinearity' should be one of " 206 "'sigmoid', 'tanh', 'relu' or 'leaky_relu'.".format(nonlinearity)) 207 return res 208 209 210def _calculate_in_and_out(arr): 211 """ 212 Calculate n_in and n_out. 213 214 Args: 215 arr (Array): Input array. 216 217 Returns: 218 Tuple, a tuple with two elements, the first element is `n_in` and the second element is `n_out`. 219 """ 220 dim = len(arr.shape) 221 if dim < 2: 222 raise ValueError("If initialize data with xavier uniform, the dimension of data must be greater than 1.") 223 224 n_in = arr.shape[1] 225 n_out = arr.shape[0] 226 227 if dim > 2: 228 counter = reduce(lambda x, y: x * y, arr.shape[2:]) 229 n_in *= counter 230 n_out *= counter 231 return n_in, n_out 232 233 234@_register('xavier_uniform') 235class XavierUniform(Initializer): 236 r""" 237 Initialize the array with xavier uniform algorithm, and from a uniform distribution collect samples within 238 :math:`{U}(-\text{boundary}, \text{boundary})` where: 239 240 .. math:: 241 boundary = gain * \sqrt{\frac{6}{n_{in} + n_{out}}} 242 243 - where :math:`gain` is an optional scaling factor. 244 - where :math:`n_{in}` is the number of input units in the weight tensor. 245 - where :math:`n_{out}` is the number of output units in the weight tensor. 246 247 For details of XavierUniform algorithm, please check 248 `<http://proceedings.mlr.press/v9/glorot10a.html>`_. 249 250 Args: 251 gain (float): An optional scaling factor. Default: 1. 252 253 254 Examples: 255 >>> import mindspore 256 >>> from mindspore.common.initializer import initializer, XavierUniform 257 >>> tensor1 = initializer(XavierUniform(), [1, 2, 3], mindspore.float32) 258 >>> tensor2 = initializer('xavier_uniform', [1, 2, 3], mindspore.float32) 259 """ 260 def __init__(self, gain=1): 261 super(XavierUniform, self).__init__(gain=gain) 262 self.gain = gain 263 264 def _initialize(self, arr): 265 n_in, n_out = _calculate_fan_in_and_fan_out(arr.shape) 266 267 boundary = self.gain * math.sqrt(6.0 / (n_in + n_out)) 268 data = np.random.uniform(-boundary, boundary, arr.shape) 269 270 _assignment(arr, data) 271 272 273@_register('he_uniform') 274class HeUniform(Initializer): 275 r""" 276 Initialize the array with HeKaiming Uniform algorithm, and from a uniform distribution collect samples within 277 :math:`{U}(-\text{boundary}, \text{boundary})` where 278 279 .. math:: 280 boundary = \sqrt{\frac{6}{(1 + a^2) \times \text{fan_in}}} 281 282 - where :math:`-boundary` the lower bound of the HeUniform distribution. 283 - where :math:`boundary` the upper bound of the HeUniform distribution. 284 285 For details of HeUniform algorithm, please check 286 `<https://arxiv.org/abs/1502.01852>`_. 287 288 Args: 289 negative_slope (int, float, bool): The negative slope of the rectifier used after this layer 290 (only used when `nonlinearity` is 'leaky_relu'). Default: 0. 291 mode (str): Either 'fan_in' or 'fan_out'. Choosing 'fan_in' preserves the magnitude of the 292 variance of the weights in the forward pass. Choosing 'fan_out' preserves the magnitudes 293 in the backwards pass. Default: fan_in. 294 nonlinearity (str): The non-linear function, recommended to use only with 'relu' or 'leaky_relu'. 295 Default: leaky_relu. 296 297 298 Examples: 299 >>> import mindspore 300 >>> from mindspore.common.initializer import initializer, HeUniform 301 >>> tensor1 = initializer(HeUniform(), [1, 2, 3], mindspore.float32) 302 >>> tensor2 = initializer('he_uniform', [1, 2, 3], mindspore.float32) 303 """ 304 def __init__(self, negative_slope=0, mode='fan_in', nonlinearity='leaky_relu'): 305 super(HeUniform, self).__init__(negative_slope=negative_slope, mode=mode, nonlinearity=nonlinearity) 306 self.negative_slope = negative_slope 307 self.mode = mode 308 self.nonlinearity = nonlinearity 309 310 def _initialize(self, arr): 311 fan = _calculate_correct_fan(arr.shape, self.mode) 312 gain = _calculate_gain(self.nonlinearity, self.negative_slope) 313 std = gain / math.sqrt(fan) 314 boundary = math.sqrt(3.0) * std 315 data = np.random.uniform(-boundary, boundary, arr.shape) 316 317 _assignment(arr, data) 318 319 320@_register('he_normal') 321class HeNormal(Initializer): 322 r""" 323 Initialize the array with HeKaiming Normal algorithm, and from a normal distribution collect samples within 324 :math:`{N}(0, \text{sigma}^2)` where 325 326 .. math:: 327 sigma = \frac{gain} {\sqrt{mode}} 328 329 - where :math:`gain` is an optional scaling factor. 330 - where :math:`mode` is the number of input units or output units in the weight tensor. 331 332 For details of HeUniform algorithm, please check 333 `<https://arxiv.org/abs/1502.01852>`_. 334 335 Args: 336 negative_slope (int, float, bool): The negative slope of the rectifier used after this layer 337 (only used when `nonlinearity` is 'leaky_relu'). Default: 0. 338 mode (str): Either 'fan_in' or 'fan_out'. Choosing 'fan_in' preserves the magnitude of the 339 variance of the weights in the forward pass. Choosing 'fan_out' preserves the magnitudes 340 in the backwards pass. Default: fan_in. 341 nonlinearity (str): The non-linear function, recommended to use only with 'relu' or 'leaky_relu'. 342 Default: leaky_relu. 343 344 345 Examples: 346 >>> import mindspore 347 >>> from mindspore.common.initializer import initializer, HeNormal 348 >>> tensor1 = initializer(HeNormal(), [1, 2, 3], mindspore.float32) 349 >>> tensor2 = initializer('he_normal', [1, 2, 3], mindspore.float32) 350 """ 351 def __init__(self, negative_slope=0, mode='fan_in', nonlinearity='leaky_relu'): 352 super(HeNormal, self).__init__(negative_slope=negative_slope, mode=mode, nonlinearity=nonlinearity) 353 self.negative_slope = negative_slope 354 self.mode = mode 355 self.nonlinearity = nonlinearity 356 357 def _initialize(self, arr): 358 fan = _calculate_correct_fan(arr.shape, self.mode) 359 gain = _calculate_gain(self.nonlinearity, self.negative_slope) 360 std = gain / math.sqrt(fan) 361 data = np.random.normal(0, std, arr.shape) 362 363 _assignment(arr, data) 364 365 366class Constant(Initializer): 367 """ 368 Initialize a constant. 369 370 Args: 371 value (Union[int, numpy.ndarray]): The value to initialize. 372 373 374 Examples: 375 >>> import mindspore 376 >>> from mindspore.common.initializer import initializer 377 >>> tensor1 = initializer(0, [1, 2, 3], mindspore.float32) 378 >>> tensor2 = initializer(5, [1, 2, 3], mindspore.float32) 379 """ 380 def __init__(self, value): 381 super(Constant, self).__init__(value=value) 382 self.value = value 383 384 def _initialize(self, arr): 385 _assignment(arr, self.value) 386 387 388@_register() 389class Uniform(Initializer): 390 r""" 391 Initialize a uniform array, and obtain values :math:`{U}(-\text{scale}, \text{scale})` from the uniform distribution 392 to fill the input tensor. 393 394 Args: 395 scale (float): The scale of the array. Default: 0.07. 396 397 398 Examples: 399 >>> import mindspore 400 >>> from mindspore.common.initializer import initializer, Uniform 401 >>> tensor1 = initializer(Uniform(), [1, 2, 3], mindspore.float32) 402 >>> tensor2 = initializer('uniform', [1, 2, 3], mindspore.float32) 403 """ 404 def __init__(self, scale=0.07): 405 super(Uniform, self).__init__(scale=scale) 406 self.scale = scale 407 408 def _initialize(self, arr): 409 tmp = np.random.uniform(-self.scale, self.scale, arr.shape) 410 _assignment(arr, tmp) 411 412 413@_register() 414class Normal(Initializer): 415 r""" 416 Initialize a normal array, and obtain values :math:`{N}(\text{sigma}, \text{mean})` from the normal distribution 417 to fill the input tensor. 418 419 .. math:: 420 f(x) = \frac{1} {\sqrt{2*π} * sigma}exp(-\frac{(x - mean)^2} {2*{sigma}^2}) 421 422 Args: 423 sigma (float): The sigma of the array. Default: 0.01. 424 mean (float): The mean of the array. Default: 0.0. 425 426 427 Examples: 428 >>> import mindspore 429 >>> from mindspore.common.initializer import initializer, Normal 430 >>> tensor1 = initializer(Normal(), [1, 2, 3], mindspore.float32) 431 >>> tensor2 = initializer('normal', [1, 2, 3], mindspore.float32) 432 """ 433 def __init__(self, sigma=0.01, mean=0.0): 434 super(Normal, self).__init__(sigma=sigma, mean=mean) 435 self.sigma = sigma 436 self.mean = mean 437 438 def _initialize(self, arr): 439 seed, seed2 = self.seed 440 output_tensor = Tensor(np.zeros(arr.shape, dtype=np.float32)) 441 random_normal(arr.shape, seed, seed2, output_tensor) 442 output_data = output_tensor.asnumpy() 443 output_data = output_data * self.sigma + self.mean 444 _assignment(arr, output_data) 445 446@_register() 447class TruncatedNormal(Initializer): 448 r""" 449 Initialize a truncated normal distribution which is a bounded normal distribution 450 within :math:`{N}(\text{low}, \text{high})`. 451 452 Args: 453 sigma (float): The sigma of the array. Default: 0.01. 454 455 456 Examples: 457 >>> import mindspore 458 >>> from mindspore.common.initializer import initializer, TruncatedNormal 459 >>> tensor1 = initializer(TruncatedNormal(), [1, 2, 3], mindspore.float32) 460 >>> tensor2 = initializer('truncatedNormal', [1, 2, 3], mindspore.float32) 461 """ 462 def __init__(self, sigma=0.01): 463 super(TruncatedNormal, self).__init__(sigma=sigma) 464 self.sigma = sigma 465 466 def _initialize(self, arr): 467 tmp = truncnorm.rvs(-2, 2, loc=0, scale=self.sigma, size=arr.shape, random_state=None) 468 _assignment(arr, tmp) 469 470 471def initializer(init, shape=None, dtype=mstype.float32): 472 """ 473 Create and initialize a tensor. 474 475 Args: 476 init (Union[Tensor, str, Initializer, numbers.Number]): Initialize value. 477 478 - `str`: The `init` should be the alias of the class inheriting from `Initializer` and the corresponding 479 class will be called. The value of 'init' can be "normal", "ones" or "zeros", etc. 480 481 - `Initializer`: The `init` should be the class inheriting from `Initializer` to initialize tensor. 482 483 - `numbers.Number`: The `Constant` will be called to initialize tensor. 484 485 shape (Union[tuple, list, int]): A list of integers, a tuple of integers or an integer as the shape of 486 output. Default: None. 487 dtype (:class:`mindspore.dtype`): The type of data in initialized tensor. Default: mindspore.float32. 488 489 Returns: 490 Union[Tensor], return is Tensor object. 491 492 493 Examples: 494 >>> import mindspore 495 >>> from mindspore.common.initializer import initializer, One 496 >>> tensor1 = initializer('ones', [1, 2, 3], mindspore.float32) 497 >>> tensor2 = initializer(One(), [1, 2, 3], mindspore.float32) 498 >>> tensor3 = initializer(0, [1, 2, 3], mindspore.float32) 499 """ 500 if not isinstance(init, (Tensor, numbers.Number, str, Initializer)): 501 raise TypeError("The type of the 'init' argument should be 'Tensor', 'number', 'string' " 502 "or 'initializer', but got {}.".format(type(init))) 503 504 if isinstance(init, Tensor): 505 init_shape = init.shape 506 shape = shape if isinstance(shape, (tuple, list)) else [shape] 507 if shape is not None and init_shape != tuple(shape): 508 raise ValueError("The shape of the 'init' argument should be same as the argument 'shape', but got the " 509 "'init' shape {} and the 'shape' {}.".format(list(init.shape), shape)) 510 return init 511 512 if isinstance(shape, list): 513 shape = tuple(shape) 514 elif isinstance(shape, numbers.Number): 515 shape = (shape,) 516 517 for value in shape if shape is not None else (): 518 if not isinstance(value, int) or value <= 0: 519 raise ValueError(f"The argument 'shape' is invalid, the value of 'shape' must be positive integer, " 520 f"but got {shape}") 521 522 if isinstance(init, str): 523 init = _INITIALIZER_ALIAS[init.lower()]() 524 if init is None: 525 raise ValueError("The class corresponding to '{}' was not found.".format(init)) 526 elif isinstance(init, numbers.Number): 527 init = Constant(init) 528 shape = shape if shape is not None else init.shape 529 init_obj = Tensor(dtype=dtype, shape=shape, init=init) 530 return init_obj 531 532__all__ = [ 533 'Initializer', 534 'initializer', 535 'TruncatedNormal', 536 'Normal', 537 'Uniform', 538 'HeUniform', 539 'HeNormal', 540 'XavierUniform', 541 'One', 542 'Zero', 543 'Constant'] 544