1# Copyright 2020-2022 Huawei Technologies Co., Ltd 2# 3# Licensed under the Apache License, Version 2.0 (the "License"); 4# you may not use this file except in compliance with the License. 5# You may obtain a copy of the License at 6# 7# http://www.apache.org/licenses/LICENSE-2.0 8# 9# Unless required by applicable law or agreed to in writing, software 10# distributed under the License is distributed on an "AS IS" BASIS, 11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12# See the License for the specific language governing permissions and 13# limitations under the License. 14# ============================================================================ 15"""Initializer for cell parameters.""" 16 17from __future__ import absolute_import 18 19import numbers 20import math 21 22from functools import reduce 23import numpy as np 24from mindspore.common.seed import get_seed, _get_graph_seed 25from mindspore.common import dtype as mstype 26from mindspore.common.tensor import Tensor 27from mindspore._c_expression import _random_normal, _random_uniform, _truncated_normal 28 29_INITIALIZER_ALIAS = dict() 30 31 32class Initializer: 33 """ 34 The abstract base class of the initializer. 35 36 Note: 37 Initializers are intended to be used for delayed initialization in parallel mode rather than Tensor 38 initialization. If you have to use Initializers to create a Tensor, :func:`mindspore.Tensor.init_data` should be 39 followed in most of the cases. For more information, please refer to `mindspore.Tensor.init_data 40 <https://www.mindspore.cn/docs/en/master/api_python/mindspore/Tensor/mindspore.Tensor.init_data.html# 41 mindspore-tensor-init-data>`_ . 42 43 Args: 44 kwargs (dict): Keyword arguments for Initializer. 45 """ 46 47 def __init__(self, **kwargs): 48 self._kwargs = kwargs 49 self._seed = None 50 51 @property 52 def seed(self): 53 if self._seed is None: 54 seed, seed2 = _get_graph_seed(get_seed(), "init") 55 else: 56 seed, seed2 = self._seed + 1, 0 57 return seed, seed2 58 59 @seed.setter 60 def seed(self, value): 61 self._seed = value 62 63 def _initialize(self, *kwargs): 64 raise NotImplementedError('Must be overridden!') 65 66 def __call__(self, arr): 67 return self._initialize(arr) 68 69 70def _register(*aliases): 71 """Return the alias register.""" 72 def alias_reg(cls): 73 name = cls.__name__ 74 name = name.lower() 75 if name not in _INITIALIZER_ALIAS: 76 _INITIALIZER_ALIAS[name] = cls 77 78 for alias in aliases: 79 if alias not in _INITIALIZER_ALIAS: 80 _INITIALIZER_ALIAS[alias] = cls 81 82 return cls 83 84 return alias_reg 85 86 87def _assignment(arr, num): 88 """Assign the value of `num` to `arr`.""" 89 if arr.shape == (): 90 arr = arr.reshape(1) 91 arr[:] = num 92 arr = arr.reshape(()) 93 else: 94 if isinstance(num, np.ndarray): 95 arr[:] = num[:] 96 else: 97 arr[:] = num 98 return arr 99 100 101def _numpy_seed(): 102 # This will produce same value after call numpy.random.seed with same seed. 103 return np.random.randint(low=1, high=(1 << 63), dtype=np.int64) 104 105 106def _init_random_normal(mean, sigma, shape): 107 if sigma < 0: 108 raise ValueError("sigma < 0") 109 data = np.ndarray(shape=shape, dtype=np.float32) 110 _random_normal(_numpy_seed(), data, mean, sigma) 111 return data 112 113 114def _init_random_uniform(a, b, shape): 115 data = np.ndarray(shape=shape, dtype=np.float32) 116 _random_uniform(_numpy_seed(), data, a, b) 117 return data 118 119 120def _init_truncated_normal(a, b, mean, sigma, shape): 121 if sigma < 0: 122 raise ValueError("sigma < 0") 123 data = np.ndarray(shape=shape, dtype=np.float32) 124 _truncated_normal(_numpy_seed(), data, a, b, mean, sigma) 125 return data 126 127 128@_register('zeros') 129class Zero(Initializer): 130 """ 131 Generates an array with constant value of zero in order to initialize a tensor. 132 133 Examples: 134 >>> import mindspore 135 >>> from mindspore.common.initializer import initializer, Zero 136 >>> from mindspore import Parameter 137 >>> w1 = Parameter(initializer(Zero(), [1, 2, 3], mindspore.float32)) 138 >>> w2 = Parameter(initializer('zeros', [1, 2, 3], mindspore.float32)) 139 """ 140 141 def _initialize(self, arr): 142 arr.fill(0) 143 144 145@_register('ones') 146class One(Initializer): 147 """ 148 Generates an array with constant value of one in order to initialize a tensor. 149 150 Examples: 151 >>> import mindspore 152 >>> from mindspore.common.initializer import initializer, One 153 >>> from mindspore import Parameter 154 >>> w1 = Parameter(initializer(One(), [1, 2, 3], mindspore.float32)) 155 >>> w2 = Parameter(initializer('ones', [1, 2, 3], mindspore.float32)) 156 """ 157 158 def _initialize(self, arr): 159 arr.fill(1) 160 161 162def _calculate_fan_in_and_fan_out(shape): 163 """ 164 calculate fan_in and fan_out 165 166 Args: 167 shape (tuple): input shape. 168 169 Returns: 170 Tuple, a tuple with two elements, the first element is `n_in` and the second element is `n_out`. 171 """ 172 dimensions = len(shape) 173 if dimensions < 2: 174 raise ValueError("'fan_in' and 'fan_out' can not be computed for tensor with fewer than" 175 " 2 dimensions, but got dimensions {}.".format(dimensions)) 176 if dimensions == 2: # Linear 177 fan_in = shape[1] 178 fan_out = shape[0] 179 else: 180 num_input_fmaps = shape[1] 181 num_output_fmaps = shape[0] 182 receptive_field_size = 1 183 for i in range(2, dimensions): 184 receptive_field_size *= shape[i] 185 fan_in = num_input_fmaps * receptive_field_size 186 fan_out = num_output_fmaps * receptive_field_size 187 return fan_in, fan_out 188 189 190def _calculate_correct_fan(shape, mode): 191 """ 192 Calculate fan. 193 194 Args: 195 shape (tuple): input shape. 196 mode (str): only support fan_in and fan_out. 197 198 Returns: 199 fan_in or fan_out. 200 """ 201 mode = mode.lower() 202 valid_modes = ['fan_in', 'fan_out'] 203 if mode not in valid_modes: 204 raise ValueError("'mode' {} not supported, please use one of {}".format(mode, valid_modes)) 205 fan_in, fan_out = _calculate_fan_in_and_fan_out(shape) 206 return fan_in if mode == 'fan_in' else fan_out 207 208 209def _calculate_gain(nonlinearity, param=None): 210 """ 211 Calculate gain. 212 213 Args: 214 nonlinearity (str): nonlinearity function. 215 param (str): used to calculate negative_slope. 216 217 Returns: 218 number. 219 """ 220 linear_fns = ['linear', 'conv1d', 'conv2d', 'conv3d', 'conv_transpose1d', 'conv_transpose2d', 'conv_transpose3d'] 221 if nonlinearity in linear_fns or nonlinearity == 'sigmoid': 222 res = 1 223 elif nonlinearity == 'tanh': 224 res = 5.0 / 3 225 elif nonlinearity == 'relu': 226 res = math.sqrt(2.0) 227 elif nonlinearity == 'leaky_relu': 228 if param is None: 229 negative_slope = 0.01 230 elif not isinstance(param, bool) and isinstance(param, int) or isinstance(param, float): 231 # True/False are instances of int, hence check above 232 negative_slope = param 233 else: 234 raise ValueError("For 'HeUniform', 'negative_slope' {} is not a valid number." 235 "When 'nonlinearity' has been set to " 236 "'leaky_relu', 'negative_slope' should be int or float type, but got " 237 "{}.".format(param, type(param))) 238 res = math.sqrt(2.0 / (1 + negative_slope ** 2)) 239 else: 240 raise ValueError("For 'HeUniform', the argument 'nonlinearity' should be one of " 241 "['sigmoid', 'tanh', 'relu' or 'leaky_relu'], " 242 "but got {}.".format(nonlinearity)) 243 return res 244 245 246def _calculate_in_and_out(arr): 247 """ 248 Calculate n_in and n_out. 249 250 Args: 251 arr (Array): Input array. 252 253 Returns: 254 Tuple, a tuple with two elements, the first element is `n_in` and the second element is `n_out`. 255 """ 256 dim = len(arr.shape) 257 if dim < 2: 258 raise ValueError("If initialize data with xavier uniform, the dimension of data must be greater than 1, " 259 "but got {}.".format(dim)) 260 261 n_in = arr.shape[1] 262 n_out = arr.shape[0] 263 264 if dim > 2: 265 counter = reduce(lambda x, y: x * y, arr.shape[2:]) 266 n_in *= counter 267 n_out *= counter 268 return n_in, n_out 269 270 271@_register('xavier_normal') 272class XavierNormal(Initializer): 273 r""" 274 Generates an array with values sampled from Xavier normal distribution 275 :math:`{N}(0, \text{sigma}^2)` in order to initialize a tensor, where 276 277 .. math:: 278 sigma = gain * \sqrt{\frac{2}{n_{in} + n_{out}}} 279 280 where :math:`gain` is an optional scaling factor, :math:`n_{in}` is the number of input units in the weight tensor, 281 :math:`n_{out}` is the number of output units in the weight tensor. 282 283 Args: 284 gain (float): An optional scaling factor. Default: ``1`` . 285 286 Examples: 287 >>> import mindspore 288 >>> from mindspore.common.initializer import initializer, XavierNormal 289 >>> from mindspore import Parameter 290 >>> w1 = Parameter(initializer(XavierNormal(), [1, 2, 3], mindspore.float32)) 291 >>> w2 = Parameter(initializer('xavier_normal', [1, 2, 3], mindspore.float32)) 292 """ 293 def __init__(self, gain=1): 294 super().__init__(gain=gain) 295 self.gain = gain 296 297 def _initialize(self, arr): 298 fan_in, fan_out = _calculate_fan_in_and_fan_out(arr.shape) 299 300 std = self.gain * math.sqrt(2.0 / float(fan_in + fan_out)) 301 data = _init_random_normal(0, std, arr.shape) 302 303 _assignment(arr, data) 304 305 306@_register('xavier_uniform') 307class XavierUniform(Initializer): 308 r""" 309 Generates an array with values sampled from Xavier uniform distribution 310 :math:`{U}(-\text{boundary}, \text{boundary})` in order to initialize a tensor, where 311 312 .. math:: 313 boundary = gain * \sqrt{\frac{6}{n_{in} + n_{out}}} 314 315 where :math:`gain` is an optional scaling factor. :math:`n_{in}` is the number of input units in the weight tensor, 316 :math:`n_{out}` is the number of output units in the weight tensor. 317 318 For details of XavierUniform algorithm, please check 319 `<http://proceedings.mlr.press/v9/glorot10a.html>`_. 320 321 Args: 322 gain (float): An optional scaling factor. Default: ``1`` . 323 324 325 Examples: 326 >>> import mindspore 327 >>> from mindspore.common.initializer import initializer, XavierUniform 328 >>> from mindspore import Parameter 329 >>> w1 = Parameter(initializer(XavierUniform(), [1, 2, 3], mindspore.float32)) 330 >>> w2 = Parameter(initializer('xavier_uniform', [1, 2, 3], mindspore.float32)) 331 """ 332 333 def __init__(self, gain=1): 334 super(XavierUniform, self).__init__(gain=gain) 335 self.gain = gain 336 337 def _initialize(self, arr): 338 n_in, n_out = _calculate_fan_in_and_fan_out(arr.shape) 339 boundary = self.gain * math.sqrt(6.0 / (n_in + n_out)) 340 data = _init_random_uniform(-boundary, boundary, arr.shape) 341 _assignment(arr, data) 342 343 344@_register('he_uniform') 345class HeUniform(Initializer): 346 r""" 347 Generates an array with values sampled from HeKaiming Uniform distribution 348 :math:`{U}(-\text{boundary}, \text{boundary})` in order to initialize a tensor, where 349 350 .. math:: 351 boundary = \text{gain} \times \sqrt{\frac{3}{fan\_mode}} 352 353 where :math:`gain` is an optional scaling factor. If :math:`fan\_mode` is ``'fan_in'``, 354 it is the number of input units of the weight tensor. If :math:`fan\_mode` is ``'fan_out'``, 355 it is the number of output units of the weight tensor. 356 357 For details of HeUniform algorithm, please check 358 `<https://arxiv.org/abs/1502.01852>`_. 359 360 Args: 361 negative_slope (int, float, bool): The negative slope of the rectifier used after this layer 362 (only used when `nonlinearity` is 'leaky_relu'). Default: ``0`` . 363 mode (str): Either ``'fan_in'`` or ``'fan_out'`` . Choosing ``'fan_in'`` preserves the magnitude of the 364 variance of the weights in the forward pass. Choosing ``'fan_out'`` preserves the magnitudes 365 in the backwards pass. Default: ``'fan_in'`` . 366 nonlinearity (str): The non-linear function, recommended to use only with ``'relu'`` or ``'leaky_relu'`` . 367 Default: ``'leaky_relu'`` . 368 369 370 Examples: 371 >>> import mindspore 372 >>> from mindspore.common.initializer import initializer, HeUniform 373 >>> from mindspore import Parameter 374 >>> w1 = Parameter(initializer(HeUniform(), [1, 2, 3], mindspore.float32)) 375 >>> w2 = Parameter(initializer('he_uniform', [1, 2, 3], mindspore.float32)) 376 """ 377 378 def __init__(self, negative_slope=0, mode='fan_in', nonlinearity='leaky_relu'): 379 super(HeUniform, self).__init__(negative_slope=negative_slope, mode=mode, nonlinearity=nonlinearity) 380 self.negative_slope = negative_slope 381 self.mode = mode 382 self.nonlinearity = nonlinearity 383 384 def _initialize(self, arr): 385 fan = _calculate_correct_fan(arr.shape, self.mode) 386 gain = _calculate_gain(self.nonlinearity, self.negative_slope) 387 std = gain / math.sqrt(fan) 388 boundary = math.sqrt(3.0) * std 389 data = _init_random_uniform(-boundary, boundary, arr.shape) 390 _assignment(arr, data) 391 392 393@_register('he_normal') 394class HeNormal(Initializer): 395 r""" 396 Generates an array with values sampled from HeKaiming Normal distribution 397 :math:`{N}(0, \text{sigma}^2)` in order to initialize a tensor, where 398 399 .. math:: 400 sigma = \frac{gain} {\sqrt{fan\_mode}} 401 402 where :math:`gain` is an optional scaling factor. :math:`fan\_mode` is the number of input or output units of 403 the weight tensor, depending on the `mode` is 'fan_in' or 'fan_out'. 404 405 For details of HeNormal algorithm, please check `<https://arxiv.org/abs/1502.01852>`_. 406 407 Args: 408 negative_slope (int, float): The negative slope of the rectifier used after this layer 409 (only used when `nonlinearity` is 'leaky_relu'). Default: ``0`` . 410 mode (str): Either ``'fan_in'`` or ``'fan_out'`` . Choosing ``'fan_in'`` preserves the magnitude of the 411 variance of the weights in the forward pass. Choosing ``'fan_out'`` preserves the magnitudes 412 in the backwards pass. Default: ``'fan_in'`` . 413 nonlinearity (str): The non-linear function, recommended to use only with ``'relu'`` or ``'leaky_relu'`` . 414 Default: ``'leaky_relu'`` . 415 416 417 Examples: 418 >>> import mindspore 419 >>> from mindspore.common.initializer import initializer, HeNormal 420 >>> from mindspore import Parameter 421 >>> w1 = Parameter(initializer(HeNormal(), [1, 2, 3], mindspore.float32)) 422 >>> w2 = Parameter(initializer('he_normal', [1, 2, 3], mindspore.float32)) 423 """ 424 425 def __init__(self, negative_slope=0, mode='fan_in', nonlinearity='leaky_relu'): 426 super(HeNormal, self).__init__(negative_slope=negative_slope, mode=mode, nonlinearity=nonlinearity) 427 self.negative_slope = negative_slope 428 self.mode = mode 429 self.nonlinearity = nonlinearity 430 431 def _initialize(self, arr): 432 fan = _calculate_correct_fan(arr.shape, self.mode) 433 gain = _calculate_gain(self.nonlinearity, self.negative_slope) 434 std = gain / math.sqrt(fan) 435 data = _init_random_normal(0, std, arr.shape) 436 _assignment(arr, data) 437 438 439class Constant(Initializer): 440 """ 441 Generates an array with constant value in order to initialize a tensor. 442 443 Args: 444 value (Union[int, numpy.ndarray]): The value to initialize. 445 446 447 Examples: 448 >>> import mindspore 449 >>> from mindspore.common.initializer import initializer, Constant 450 >>> from mindspore import Parameter 451 >>> w1 = Parameter(initializer(Constant(3), [1, 2, 3], mindspore.float32)) 452 """ 453 454 def __init__(self, value): 455 super(Constant, self).__init__(value=value) 456 self.value = value 457 458 def _initialize(self, arr): 459 arr.fill(self.value) 460 461 462@_register() 463class Identity(Initializer): 464 """ 465 Generates a 2 dimension identity matrix array in order to initialize a tensor. 466 467 Raises: 468 ValueError: If the dimension of input tensor is not equal to 2. 469 470 Examples: 471 >>> import mindspore 472 >>> from mindspore.common.initializer import initializer, Identity 473 >>> from mindspore import Parameter 474 >>> w1 = initializer(Identity(), [2, 3], mindspore.float32) 475 >>> w2 = initializer('identity', [2, 3], mindspore.float32) 476 """ 477 478 def _initialize(self, arr): 479 if len(arr.shape) != 2: 480 raise ValueError('For Identity initializer, the dimension of the initialized tensor should be 2, ' 481 'but got {}.'.format(len(arr.shape))) 482 value = np.eye(arr.shape[0], arr.shape[1]) 483 _assignment(arr, value) 484 485 486@_register() 487class Sparse(Initializer): 488 """ 489 Generates a 2 dimension sparse matrix array in order to initialize a tensor. The non-zero positions 490 will be filled with the value sampled from the normal distribution :math:`{N}(0, sigma)`. 491 492 Args: 493 sparsity (float): The fraction of elements being set to zero in each column. 494 sigma (float): The standard deviation of the normal distribution. Default: ``0.01`` . 495 496 Raises: 497 ValueError: If the dimension of input tensor is not equal to 2. 498 499 Examples: 500 >>> import mindspore 501 >>> from mindspore.common.initializer import initializer, Sparse 502 >>> from mindspore import Parameter 503 >>> w1 = Parameter(initializer(Sparse(sparsity=0.1, sigma=0.01), [5, 8], mindspore.float32)) 504 """ 505 506 def __init__(self, sparsity, sigma=0.01): 507 super(Sparse, self).__init__() 508 self.sparsity = sparsity 509 self.sigma = sigma 510 511 def _initialize(self, arr): 512 if len(arr.shape) != 2: 513 raise ValueError('For Sparse initializer, the dimension of the initialized tensor should be 2, ' 514 'but got {}.'.format(len(arr.shape))) 515 rows, cols = arr.shape 516 zero_num = int(np.ceil(self.sparsity * rows)) 517 data = _init_random_normal(0, self.sigma, arr.shape) 518 for col_idx in range(cols): 519 row_idx = np.random.permutation(list(range(rows)))[: zero_num] 520 data[row_idx, col_idx] = 0. 521 _assignment(arr, data) 522 523 524@_register() 525class Dirac(Initializer): 526 """ 527 Generates an array with the Dirac delta function in order to initialize a tensor. 528 It's usually used in convolution layers, preserves as many identities of the inputs as possible. 529 530 Args: 531 groups (int): The number of groups in convolution layer. Each group applies the same initialization. 532 Default: ``1`` . 533 534 Raises: 535 ValueError: If the dimension of the initialized tensor is not in [3, 4, 5]. 536 ValueError: The first dimension of the initialized tensor cannot be divisible by group. 537 538 Examples: 539 >>> import mindspore 540 >>> from mindspore.common.initializer import initializer, Dirac 541 >>> from mindspore import Parameter 542 >>> w1 = Parameter(initializer(Dirac(groups=2), [6, 4, 3, 3], mindspore.float32)) 543 >>> w2 = Parameter(initializer("dirac", [6, 4, 3, 3], mindspore.float32)) 544 """ 545 546 def __init__(self, groups=1): 547 super(Dirac, self).__init__() 548 self.groups = groups 549 550 def _initialize(self, arr): 551 dimension = len(arr.shape) 552 data = np.zeros(arr.shape) 553 if dimension not in [3, 4, 5]: 554 raise ValueError("For Dirac initializer, only support " 555 "to initialize tensor with dimension of 3, 4 or 5, but got {}.".format(dimension)) 556 557 shapes = arr.shape 558 if shapes[0] % self.groups != 0: 559 raise ValueError("For Dirac initializer, the first dimension of" 560 "the initialized tensor must be divisible by groups, " 561 "but got first dimension{}, groups{}.".format(shapes[0], self.groups)) 562 563 out_channel_per_group = shapes[0] // self.groups 564 min_dim = min(out_channel_per_group, shapes[1]) 565 566 for group in range(self.groups): 567 for dim in range(min_dim): 568 if dimension == 3: 569 data[group * out_channel_per_group + dim, dim, shapes[2] // 2] = 1 570 elif dimension == 4: 571 data[group * out_channel_per_group + dim, dim, shapes[2] // 2, shapes[3] // 2] = 1 572 else: 573 data[group * out_channel_per_group + dim, dim, shapes[2] // 2, shapes[3] // 2, shapes[4] // 2] = 1 574 _assignment(arr, data) 575 576 577@_register() 578class Orthogonal(Initializer): 579 r""" 580 Generates a (semi) orthogonal matrix array in order to initialize a tensor. 581 The dimension of input tensor must have at least 2 dimensions. 582 If the dimension is greater than 2, the trailing dimensions will be flattened. 583 584 Args: 585 gain (float): An optional scaling factor. Default: ``1.0`` . 586 587 Raises: 588 ValueError: If the dimension of input tensor is less than 2. 589 590 Examples: 591 >>> import mindspore 592 >>> from mindspore.common.initializer import initializer, Orthogonal 593 >>> from mindspore import Parameter 594 >>> w1 = Parameter(initializer(Orthogonal(gain=2.), [2, 3, 4], mindspore.float32)) 595 >>> w2 = Parameter(initializer('orthogonal', [2, 3, 4], mindspore.float32)) 596 """ 597 598 def __init__(self, gain=1.): 599 super(Orthogonal, self).__init__(gain=gain) 600 self.gain = gain 601 602 def _initialize(self, arr): 603 if len(arr.shape) < 2: 604 raise ValueError('For Orthogonal initializer, the dimension of the initialized tensor should' 605 ' be no less than 2, but got {}.'.format(len(arr.shape))) 606 rows = arr.shape[0] 607 608 cols = np.prod(arr.shape) // rows 609 data = _init_random_normal(0, 1, (rows, cols)) 610 611 if rows < cols: 612 data = data.T 613 614 q, r = np.linalg.qr(data) 615 d = np.diag(r) 616 ph = np.sign(d) 617 q *= ph 618 619 if rows < cols: 620 q = q.T 621 q = q * self.gain 622 _assignment(arr, q.reshape(arr.shape)) 623 624 625@_register() 626class VarianceScaling(Initializer): 627 r""" 628 Generates an random array with scaling in order to initialize a tensor. 629 When `distribution` is 'truncated_normal' or 'untruncated_normal', the value will be sampled from truncated or 630 untruncated normal distribution with a mean of 0 and a scaled standard deviation 631 :math:`stddev = \sqrt{\frac{scale}{n}}`. :math:`n` will be the number of input units if `mode` is ``'fan_in'``, 632 while :math:`n` will be 633 the number of output units if `mode` is ``'fan_out'``. :math:`n` will be the average of ``'fan_in'`` 634 and ``'fan_out'`` if `mode` is ``'fan_avg'``. 635 When `distribution` is ``'uniform'``, the value will be sampled from a uniform distribution within the limit of 636 :math:`[-\sqrt{\frac{3*scale}{n}}, \sqrt{\frac{3*scale}{n}}]`. 637 638 Args: 639 scale (float): The scaling factor. Default: ``1.0`` . 640 mode (str): Should be ``'fan_in'`` , ``'fan_out'`` or ``'fan_avg'`` . Default: ``'fan_in'`` . 641 distribution(str): The type of distribution chose to sample values. It should be 642 ``'uniform'`` , ``'truncated_normal'`` or ``'untruncated_normal'`` . Default: ``'truncated_normal'`` . 643 644 Raises: 645 ValueError: If `scale` is not greater than 0. 646 ValueError: If `mode` is not ``'fan_in'``, ``'fan_out'`` or ``'fan_avg'``. 647 ValueError: If `distribution` is not ``'uniform'``, ``'truncated_normal'`` or ``'untruncated_normal'``. 648 649 Examples: 650 >>> import mindspore 651 >>> from mindspore.common.initializer import initializer, VarianceScaling 652 >>> from mindspore import Parameter 653 >>> w1 = Parameter(initializer(VarianceScaling(scale=1.0, mode='fan_out', 654 ... distribution='untruncated_normal'), [2, 3], mindspore.float32)) 655 >>> w2 = Parameter(initializer('varianceScaling', [2, 3], mindspore.float32)) 656 """ 657 658 def __init__(self, scale=1.0, mode='fan_in', distribution='truncated_normal'): 659 super(VarianceScaling, self).__init__(scale=scale, mode=mode, distribution=distribution) 660 if scale <= 0.: 661 raise ValueError("For VarianceScaling initializer, " 662 "the argument 'scale' must be greater than 0, but got {}.".format(scale)) 663 664 if mode not in ['fan_in', 'fan_out', 'fan_avg']: 665 raise ValueError("For VarianceScaling initializer, the argument 'mode' must be fan_in, " 666 "fan_out or fan_avg, but got {}.".format(mode)) 667 668 if distribution not in ['uniform', 'truncated_normal', 'untruncated_normal']: 669 raise ValueError("For VarianceScaling initializer, the argument 'distribution' must be uniform, " 670 "truncated_norm or untruncated_norm, but got {}.".format(distribution)) 671 672 self.scale = scale 673 self.mode = mode 674 self.distribution = distribution 675 676 def _initialize(self, arr): 677 scale = self.scale 678 fan_in, fan_out = _calculate_fan_in_and_fan_out(arr.shape) 679 if self.mode == 'fan_in': 680 scale /= max(1., fan_in) 681 elif self.mode == 'fan_out': 682 scale /= max(1., fan_out) 683 else: 684 scale /= max(1., (fan_in + fan_out) / 2.) 685 686 if self.distribution == 'truncated_norm': 687 stddev = np.sqrt(scale) / 0.87962566103423978 688 data = _init_truncated_normal(-2, 2, 0, stddev, arr.shape) 689 elif self.distribution == 'untruncated_normal': 690 stddev = np.sqrt(scale) 691 data = _init_random_normal(0, stddev, arr.shape) 692 else: 693 limit = np.sqrt(3.0 * scale) 694 data = _init_random_uniform(-limit, limit, arr.shape) 695 _assignment(arr, data) 696 697 698@_register() 699class Uniform(Initializer): 700 r""" 701 Generates an array with values sampled from Uniform distribution :math:`{U}(-\text{scale}, \text{scale})` in order 702 to initialize a tensor. 703 704 Args: 705 scale (float): The bound of the Uniform distribution. Default: ``0.07`` . 706 707 708 Examples: 709 >>> import mindspore 710 >>> from mindspore.common.initializer import initializer, Uniform 711 >>> from mindspore import Parameter 712 >>> w1 = Parameter(initializer(Uniform(), [1, 2, 3], mindspore.float32)) 713 >>> w2 = Parameter(initializer('uniform', [1, 2, 3], mindspore.float32)) 714 """ 715 716 def __init__(self, scale=0.07): 717 super(Uniform, self).__init__(scale=scale) 718 self.scale = scale 719 720 def _initialize(self, arr): 721 tmp = _init_random_uniform(-self.scale, self.scale, arr.shape) 722 _assignment(arr, tmp) 723 724 725@_register() 726class Normal(Initializer): 727 r""" 728 Generates an array with values sampled from Normal distribution :math:`{N}(\text{sigma}, \text{mean})` in order to 729 initialize a tensor. 730 731 .. math:: 732 f(x) = \frac{1} {\sqrt{2*π} * sigma}exp(-\frac{(x - mean)^2} {2*{sigma}^2}) 733 734 Args: 735 sigma (float): The standard deviation of Normal distribution. Default: ``0.01`` . 736 mean (float): The mean of Normal distribution. Default: ``0.0`` . 737 738 Examples: 739 >>> import mindspore 740 >>> from mindspore.common.initializer import initializer, Normal 741 >>> from mindspore import Parameter 742 >>> w1 = Parameter(initializer(Normal(), [1, 2, 3], mindspore.float32)) 743 >>> w2 = Parameter(initializer('normal', [1, 2, 3], mindspore.float32)) 744 """ 745 746 def __init__(self, sigma=0.01, mean=0.0): 747 super(Normal, self).__init__(sigma=sigma, mean=mean) 748 self.sigma = sigma 749 self.mean = mean 750 751 def _initialize(self, arr): 752 data = _init_random_normal(self.mean, self.sigma, arr.shape) 753 _assignment(arr, data) 754 755 756@_register() 757class TruncatedNormal(Initializer): 758 r""" 759 Generates an array with values sampled from Truncated Normal distribution in order to initialize a tensor. 760 761 Args: 762 sigma (float): The standard deviation of Truncated Normal distribution. Default: ``0.01`` . 763 mean (float): The mean of Truncated Normal distribution. Default: ``0.0`` . 764 a (float): The lower bound of the truncated interval. Default: ``-2.0`` . 765 b (float): The upper bound of the truncated interval. Default: ``2.0`` . 766 767 Examples: 768 >>> import mindspore 769 >>> from mindspore.common.initializer import initializer, TruncatedNormal 770 >>> from mindspore import Parameter 771 >>> w1 = Parameter(initializer(TruncatedNormal(), [1, 2, 3], mindspore.float32)) 772 >>> w2 = Parameter(initializer('truncatedNormal', [1, 2, 3], mindspore.float32)) 773 """ 774 775 def __init__(self, sigma=0.01, mean=0.0, a=-2.0, b=2.0): 776 super(TruncatedNormal, self).__init__(sigma=sigma, mean=mean, a=a, b=b) 777 self.sigma = sigma 778 self.mean = mean 779 self.a = a 780 self.b = b 781 782 def _initialize(self, arr): 783 tmp = _init_truncated_normal(self.a, self.b, self.mean, self.sigma, arr.shape) 784 _assignment(arr, tmp) 785 786 787def initializer(init, shape=None, dtype=mstype.float32): 788 """ 789 Create and initialize a tensor. 790 791 Args: 792 init (Union[Tensor, str, Initializer, numbers.Number]): Initialize value. 793 794 - `str`: The `init` should be the alias of the class inheriting from `Initializer` and the corresponding 795 class will be called in practice. The value of `init` can be ``"normal"``, ``"ones"`` or 796 ``"zeros"``, etc. 797 798 - `Initializer`: The `init` should be the class inheriting from `Initializer` to initialize tensor. 799 800 - `numbers.Number`: The `Constant` will be called to initialize tensor. 801 802 - `Tensor`: The tensor will be called to initialize tensor. 803 804 shape (Union[tuple, list, int]): The shape of the initialized tensor. Default: ``None`` . 805 dtype (:class:`mindspore.dtype`): The type of data in initialized tensor. Default: ``mstype.float32`` . 806 807 Returns: 808 Tensor, return is Tensor object. 809 810 Raises: 811 TypeError: The type of the argument 'init' is not correct. 812 ValueError: The shape of the tensor which is passed through 'init' is not the same as that passed by 'shape'. 813 814 815 Examples: 816 >>> import numpy as np 817 >>> import mindspore 818 >>> from mindspore import Tensor 819 >>> from mindspore.common.initializer import initializer, One 820 >>> from mindspore import Parameter 821 >>> data = Tensor(np.zeros([1, 2, 3]), mindspore.float32) 822 >>> w1 = Parameter(initializer(data, [1, 2, 3], mindspore.float32)) 823 >>> w2 = Parameter(initializer('ones', [1, 2, 3], mindspore.float32)) 824 >>> w3 = Parameter(initializer(One(), [1, 2, 3], mindspore.float32)) 825 >>> w4 = Parameter(initializer(0, [1, 2, 3], mindspore.float32)) 826 """ 827 if not isinstance(init, (Tensor, numbers.Number, str, Initializer)): 828 raise TypeError("For 'initializer', the type of the 'init' argument should be 'Tensor', 'number', 'string' " 829 "or 'initializer', but got {}.".format(type(init))) 830 831 if isinstance(init, Tensor): 832 init_shape = init.shape 833 shape = shape if isinstance(shape, (tuple, list)) else [shape] 834 if shape is not None and init_shape != tuple(shape): 835 raise ValueError("For 'initializer', the shape of the 'init' argument should be same as " 836 "the argument 'shape', but got the " 837 "'init' shape {} and the 'shape' {}.".format(list(init.shape), shape)) 838 return init 839 840 if isinstance(shape, list): 841 shape = tuple(shape) 842 elif isinstance(shape, numbers.Number): 843 shape = (shape,) 844 845 for value in shape if shape is not None else (): 846 if not isinstance(value, int) or value <= 0: 847 raise ValueError(f"For 'initializer', the argument 'shape' is invalid, the value of 'shape' " 848 f"must be positive integer, " 849 f"but got {shape}") 850 851 if isinstance(init, str): 852 class_name = _INITIALIZER_ALIAS.get(init.lower()) 853 if class_name is None: 854 raise ValueError(f"For 'initializer', the class corresponding to '{init}' was not found.") 855 init = class_name() 856 elif isinstance(init, numbers.Number): 857 init = Constant(init) 858 shape = shape if shape is not None else init.shape 859 init_obj = Tensor(dtype=dtype, shape=shape, init=init) 860 return init_obj 861 862 863__all__ = [ 864 'Initializer', 865 'initializer', 866 'TruncatedNormal', 867 'Normal', 868 'Uniform', 869 'HeUniform', 870 'HeNormal', 871 'XavierUniform', 872 'XavierNormal', 873 'One', 874 'Zero', 875 'Constant', 876 'Identity', 877 'Sparse', 878 'Dirac', 879 'Orthogonal', 880 'VarianceScaling'] 881