1# Copyright 2021 Huawei Technologies Co., Ltd 2# 3# Licensed under the Apache License, Version 2.0 (the "License"); 4# you may not use this file except in compliance with the License. 5# You may obtain a copy of the License at 6# 7# http://www.apache.org/licenses/LICENSE-2.0 8# 9# Unless required by applicable law or agreed to in writing, software 10# distributed under the License is distributed on an "AS IS" BASIS, 11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12# See the License for the specific language governing permissions and 13# limitations under the License. 14# ============================================================================== 15 16import math 17import pytest 18import numpy as np 19from mindspore import context 20from mindspore import nn 21from mindspore import Tensor 22from mindspore.common.parameter import ParameterTuple 23from mindspore.common.parameter import Parameter 24from mindspore.ops import composite as c 25 26 27class GradOfAllInputsAndParams(nn.Cell): 28 def __init__(self, network, sens_param): 29 super(GradOfAllInputsAndParams, self).__init__() 30 self.grad = c.GradOperation(get_all=True, get_by_list=True, sens_param=sens_param) 31 self.network = network 32 self.params = ParameterTuple(self.network.trainable_params()) 33 34 def construct(self, *inputs): 35 gout = self.grad(self.network, self.params)(*inputs) 36 return gout 37 38 39class GRU(nn.Cell): 40 def __init__(self, input_size, hidden_size, num_layers, has_bias, batch_first, bidirectional, dropout): 41 super(GRU, self).__init__() 42 self.gru = nn.GRU(input_size=input_size, hidden_size=hidden_size, num_layers=num_layers, has_bias=has_bias, 43 batch_first=batch_first, bidirectional=bidirectional, dropout=dropout) 44 45 def construct(self, inp, h0): 46 return self.gru(inp, h0) 47 48 49class GRUWeightBias(): 50 def __init__(self, num_layers, has_bias, input_size, num_directions, hidden_size, bidirectional): 51 self.num_layers = num_layers 52 self.has_bias = has_bias 53 self.input_size = input_size 54 self.num_directions = num_directions 55 self.hidden_size = hidden_size 56 self.bidirectional = bidirectional 57 58 def get_weight_bias(self): 59 gate_size = 3 * self.hidden_size 60 61 w_ih_list = [] 62 w_hh_list = [] 63 b_ih_list = [] 64 b_hh_list = [] 65 stdv = 1 / math.sqrt(self.hidden_size) 66 for layer in range(self.num_layers): 67 for direction in range(self.num_directions): 68 layer_input_size = self.input_size if layer == 0 else self.hidden_size * self.num_directions 69 suffix = '_reverse' if direction == 1 else '' 70 71 w_ih_list.append(Parameter( 72 Tensor(np.random.uniform(-stdv, stdv, (gate_size, layer_input_size)).astype(np.float32)), 73 name='weight_ih_l{}{}'.format(layer, suffix))) 74 w_hh_list.append(Parameter( 75 Tensor(np.random.uniform(-stdv, stdv, (gate_size, self.hidden_size)).astype(np.float32)), 76 name='weight_hh_l{}{}'.format(layer, suffix))) 77 if self.has_bias: 78 b_ih_list.append(Parameter( 79 Tensor(np.random.uniform(-stdv, stdv, (gate_size)).astype(np.float32)), 80 name='bias_ih_l{}{}'.format(layer, suffix))) 81 b_hh_list.append(Parameter( 82 Tensor(np.random.uniform(-stdv, stdv, (gate_size)).astype(np.float32)), 83 name='bias_hh_l{}{}'.format(layer, suffix))) 84 w_ih_list = ParameterTuple(w_ih_list) 85 w_hh_list = ParameterTuple(w_hh_list) 86 b_ih_list = ParameterTuple(b_ih_list) 87 b_hh_list = ParameterTuple(b_hh_list) 88 return w_ih_list, w_hh_list, b_ih_list, b_hh_list 89 90 91@pytest.mark.level0 92@pytest.mark.platform_arm_ascend_training 93@pytest.mark.platform_x86_ascend_training 94@pytest.mark.env_onecard 95def test_sit_gru_forward_input_3_32_32_is_32_hs_16(): 96 input_size = 32 97 hidden_size = 16 98 has_bias = True 99 bidirectional = False 100 num_layers = 1 101 num_directions = 1 102 103 fact = GRUWeightBias(num_layers, has_bias, input_size, num_directions, hidden_size, bidirectional) 104 w_ih_list, w_hh_list, b_ih_list, b_hh_list = fact.get_weight_bias() 105 106 h0 = Tensor(np.random.randn(num_layers * num_directions, 32, 16).astype(np.float32)) 107 input_ms = Tensor(np.random.randn(3, 32, 32).astype(np.float32)) 108 109 # graph mode 110 context.set_context(mode=context.GRAPH_MODE) 111 net = GRU(input_size=input_size, hidden_size=16, num_layers=num_layers, has_bias=has_bias, batch_first=False, 112 bidirectional=bidirectional, dropout=0.0) 113 net.gru.w_ih_list = w_ih_list 114 net.gru.w_hh_list = w_hh_list 115 net.gru.b_ih_list = b_ih_list 116 net.gru.b_hh_list = b_hh_list 117 out, hy = net(input_ms, h0) 118 119 # pynative mode 120 context.set_context(mode=context.PYNATIVE_MODE) 121 net_pynative = GRU(input_size=input_size, hidden_size=16, num_layers=num_layers, has_bias=has_bias, 122 batch_first=False, bidirectional=bidirectional, dropout=0.0) 123 net_pynative.gru.w_ih_list = w_ih_list 124 net_pynative.gru.w_hh_list = w_hh_list 125 net_pynative.gru.b_ih_list = b_ih_list 126 net_pynative.gru.b_hh_list = b_hh_list 127 out_pynative, hy_pynative = net_pynative(input_ms, h0) 128 129 assert np.allclose(out.asnumpy(), out_pynative.asnumpy(), 0.0001, 0.0001) 130 assert np.allclose(hy.asnumpy(), hy_pynative.asnumpy(), 0.0001, 0.0001) 131 132@pytest.mark.level0 133@pytest.mark.platform_arm_ascend_training 134@pytest.mark.platform_x86_ascend_training 135@pytest.mark.env_onecard 136def test_sit_gru_grad_input_3_32_32_is_32_hs_16(): 137 input_size = 32 138 hidden_size = 16 139 has_bias = True 140 bidirectional = False 141 num_layers = 1 142 num_directions = 1 143 144 fact = GRUWeightBias(num_layers, has_bias, input_size, num_directions, hidden_size, bidirectional) 145 w_ih_list, w_hh_list, b_ih_list, b_hh_list = fact.get_weight_bias() 146 147 h0 = Tensor(np.random.randn(num_layers * num_directions, 32, 16).astype(np.float32)) 148 input_ms = Tensor(np.random.randn(3, 32, 32).astype(np.float32)) 149 150 # graph mode 151 context.set_context(mode=context.GRAPH_MODE) 152 net = GRU(input_size=input_size, hidden_size=16, num_layers=num_layers, has_bias=has_bias, batch_first=False, 153 bidirectional=bidirectional, dropout=0.0) 154 net.gru.w_ih_list = w_ih_list 155 net.gru.w_hh_list = w_hh_list 156 net.gru.b_ih_list = b_ih_list 157 net.gru.b_hh_list = b_hh_list 158 159 grad_net_inp = GradOfAllInputsAndParams(net, sens_param=False) 160 grad_net_inp.set_train() 161 out_grad, _ = grad_net_inp(input_ms, h0) 162 x_grad = out_grad[0].asnumpy() 163 h_grad = out_grad[1].asnumpy() 164 165 # pynative mode 166 context.set_context(mode=context.PYNATIVE_MODE) 167 net_pynative = GRU(input_size=input_size, hidden_size=16, num_layers=num_layers, has_bias=has_bias, 168 batch_first=False, bidirectional=bidirectional, dropout=0.0) 169 net_pynative.gru.w_ih_list = w_ih_list 170 net_pynative.gru.w_hh_list = w_hh_list 171 net_pynative.gru.b_ih_list = b_ih_list 172 net_pynative.gru.b_hh_list = b_hh_list 173 174 grad_net_inp_pynative = GradOfAllInputsAndParams(net_pynative, sens_param=False) 175 grad_net_inp_pynative.set_train() 176 out_grad_pynative, _ = grad_net_inp_pynative(input_ms, h0) 177 x_grad_pynative = out_grad_pynative[0].asnumpy() 178 h_grad_pynative = out_grad_pynative[1].asnumpy() 179 180 assert np.allclose(x_grad, x_grad_pynative, 0.0001, 0.0001) 181 assert np.allclose(h_grad, h_grad_pynative, 0.0001, 0.0001) 182