1 /*
2 * Copyright 2014 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 *
22 */
23
24 #include "amdgpu.h"
25 #include "vega10/soc15ip.h"
26 #include "vega10/NBIO/nbio_6_1_offset.h"
27 #include "vega10/NBIO/nbio_6_1_sh_mask.h"
28 #include "vega10/GC/gc_9_0_offset.h"
29 #include "vega10/GC/gc_9_0_sh_mask.h"
30 #include "soc15.h"
31 #include "vega10_ih.h"
32 #include "soc15_common.h"
33 #include "mxgpu_ai.h"
34
xgpu_ai_mailbox_send_ack(struct amdgpu_device * adev)35 static void xgpu_ai_mailbox_send_ack(struct amdgpu_device *adev)
36 {
37 u32 reg;
38 int timeout = AI_MAILBOX_TIMEDOUT;
39 u32 mask = REG_FIELD_MASK(BIF_BX_PF0_MAILBOX_CONTROL, RCV_MSG_VALID);
40
41 reg = RREG32_NO_KIQ(SOC15_REG_OFFSET(NBIO, 0,
42 mmBIF_BX_PF0_MAILBOX_CONTROL));
43 reg = REG_SET_FIELD(reg, BIF_BX_PF0_MAILBOX_CONTROL, RCV_MSG_ACK, 1);
44 WREG32_NO_KIQ(SOC15_REG_OFFSET(NBIO, 0,
45 mmBIF_BX_PF0_MAILBOX_CONTROL), reg);
46
47 /*Wait for RCV_MSG_VALID to be 0*/
48 reg = RREG32_NO_KIQ(SOC15_REG_OFFSET(NBIO, 0,
49 mmBIF_BX_PF0_MAILBOX_CONTROL));
50 while (reg & mask) {
51 if (timeout <= 0) {
52 pr_err("RCV_MSG_VALID is not cleared\n");
53 break;
54 }
55 mdelay(1);
56 timeout -=1;
57
58 reg = RREG32_NO_KIQ(SOC15_REG_OFFSET(NBIO, 0,
59 mmBIF_BX_PF0_MAILBOX_CONTROL));
60 }
61 }
62
xgpu_ai_mailbox_set_valid(struct amdgpu_device * adev,bool val)63 static void xgpu_ai_mailbox_set_valid(struct amdgpu_device *adev, bool val)
64 {
65 u32 reg;
66
67 reg = RREG32_NO_KIQ(SOC15_REG_OFFSET(NBIO, 0,
68 mmBIF_BX_PF0_MAILBOX_CONTROL));
69 reg = REG_SET_FIELD(reg, BIF_BX_PF0_MAILBOX_CONTROL,
70 TRN_MSG_VALID, val ? 1 : 0);
71 WREG32_NO_KIQ(SOC15_REG_OFFSET(NBIO, 0, mmBIF_BX_PF0_MAILBOX_CONTROL),
72 reg);
73 }
74
xgpu_ai_mailbox_rcv_msg(struct amdgpu_device * adev,enum idh_event event)75 static int xgpu_ai_mailbox_rcv_msg(struct amdgpu_device *adev,
76 enum idh_event event)
77 {
78 u32 reg;
79 u32 mask = REG_FIELD_MASK(BIF_BX_PF0_MAILBOX_CONTROL, RCV_MSG_VALID);
80
81 if (event != IDH_FLR_NOTIFICATION_CMPL) {
82 reg = RREG32_NO_KIQ(SOC15_REG_OFFSET(NBIO, 0,
83 mmBIF_BX_PF0_MAILBOX_CONTROL));
84 if (!(reg & mask))
85 return -ENOENT;
86 }
87
88 reg = RREG32_NO_KIQ(SOC15_REG_OFFSET(NBIO, 0,
89 mmBIF_BX_PF0_MAILBOX_MSGBUF_RCV_DW0));
90 if (reg != event)
91 return -ENOENT;
92
93 xgpu_ai_mailbox_send_ack(adev);
94
95 return 0;
96 }
97
xgpu_ai_poll_ack(struct amdgpu_device * adev)98 static int xgpu_ai_poll_ack(struct amdgpu_device *adev)
99 {
100 int r = 0, timeout = AI_MAILBOX_TIMEDOUT;
101 u32 mask = REG_FIELD_MASK(BIF_BX_PF0_MAILBOX_CONTROL, TRN_MSG_ACK);
102 u32 reg;
103
104 reg = RREG32_NO_KIQ(SOC15_REG_OFFSET(NBIO, 0,
105 mmBIF_BX_PF0_MAILBOX_CONTROL));
106 while (!(reg & mask)) {
107 if (timeout <= 0) {
108 pr_err("Doesn't get ack from pf.\n");
109 r = -ETIME;
110 break;
111 }
112 mdelay(5);
113 timeout -= 5;
114
115 reg = RREG32_NO_KIQ(SOC15_REG_OFFSET(NBIO, 0,
116 mmBIF_BX_PF0_MAILBOX_CONTROL));
117 }
118
119 return r;
120 }
121
xgpu_ai_poll_msg(struct amdgpu_device * adev,enum idh_event event)122 static int xgpu_ai_poll_msg(struct amdgpu_device *adev, enum idh_event event)
123 {
124 int r = 0, timeout = AI_MAILBOX_TIMEDOUT;
125
126 r = xgpu_ai_mailbox_rcv_msg(adev, event);
127 while (r) {
128 if (timeout <= 0) {
129 pr_err("Doesn't get msg:%d from pf.\n", event);
130 r = -ETIME;
131 break;
132 }
133 mdelay(5);
134 timeout -= 5;
135
136 r = xgpu_ai_mailbox_rcv_msg(adev, event);
137 }
138
139 return r;
140 }
141
xgpu_ai_mailbox_trans_msg(struct amdgpu_device * adev,enum idh_request req,u32 data1,u32 data2,u32 data3)142 static void xgpu_ai_mailbox_trans_msg (struct amdgpu_device *adev,
143 enum idh_request req, u32 data1, u32 data2, u32 data3) {
144 u32 reg;
145 int r;
146
147 reg = RREG32_NO_KIQ(SOC15_REG_OFFSET(NBIO, 0,
148 mmBIF_BX_PF0_MAILBOX_MSGBUF_TRN_DW0));
149 reg = REG_SET_FIELD(reg, BIF_BX_PF0_MAILBOX_MSGBUF_TRN_DW0,
150 MSGBUF_DATA, req);
151 WREG32_NO_KIQ(SOC15_REG_OFFSET(NBIO, 0, mmBIF_BX_PF0_MAILBOX_MSGBUF_TRN_DW0),
152 reg);
153 WREG32_NO_KIQ(SOC15_REG_OFFSET(NBIO, 0, mmBIF_BX_PF0_MAILBOX_MSGBUF_TRN_DW1),
154 data1);
155 WREG32_NO_KIQ(SOC15_REG_OFFSET(NBIO, 0, mmBIF_BX_PF0_MAILBOX_MSGBUF_TRN_DW2),
156 data2);
157 WREG32_NO_KIQ(SOC15_REG_OFFSET(NBIO, 0, mmBIF_BX_PF0_MAILBOX_MSGBUF_TRN_DW3),
158 data3);
159
160 xgpu_ai_mailbox_set_valid(adev, true);
161
162 /* start to poll ack */
163 r = xgpu_ai_poll_ack(adev);
164 if (r)
165 pr_err("Doesn't get ack from pf, continue\n");
166
167 xgpu_ai_mailbox_set_valid(adev, false);
168 }
169
xgpu_ai_send_access_requests(struct amdgpu_device * adev,enum idh_request req)170 static int xgpu_ai_send_access_requests(struct amdgpu_device *adev,
171 enum idh_request req)
172 {
173 int r;
174
175 xgpu_ai_mailbox_trans_msg(adev, req, 0, 0, 0);
176
177 /* start to check msg if request is idh_req_gpu_init_access */
178 if (req == IDH_REQ_GPU_INIT_ACCESS ||
179 req == IDH_REQ_GPU_FINI_ACCESS ||
180 req == IDH_REQ_GPU_RESET_ACCESS) {
181 r = xgpu_ai_poll_msg(adev, IDH_READY_TO_ACCESS_GPU);
182 if (r) {
183 pr_err("Doesn't get READY_TO_ACCESS_GPU from pf, give up\n");
184 return r;
185 }
186 }
187
188 return 0;
189 }
190
xgpu_ai_request_reset(struct amdgpu_device * adev)191 static int xgpu_ai_request_reset(struct amdgpu_device *adev)
192 {
193 return xgpu_ai_send_access_requests(adev, IDH_REQ_GPU_RESET_ACCESS);
194 }
195
xgpu_ai_request_full_gpu_access(struct amdgpu_device * adev,bool init)196 static int xgpu_ai_request_full_gpu_access(struct amdgpu_device *adev,
197 bool init)
198 {
199 enum idh_request req;
200
201 req = init ? IDH_REQ_GPU_INIT_ACCESS : IDH_REQ_GPU_FINI_ACCESS;
202 return xgpu_ai_send_access_requests(adev, req);
203 }
204
xgpu_ai_release_full_gpu_access(struct amdgpu_device * adev,bool init)205 static int xgpu_ai_release_full_gpu_access(struct amdgpu_device *adev,
206 bool init)
207 {
208 enum idh_request req;
209 int r = 0;
210
211 req = init ? IDH_REL_GPU_INIT_ACCESS : IDH_REL_GPU_FINI_ACCESS;
212 r = xgpu_ai_send_access_requests(adev, req);
213
214 return r;
215 }
216
xgpu_ai_mailbox_ack_irq(struct amdgpu_device * adev,struct amdgpu_irq_src * source,struct amdgpu_iv_entry * entry)217 static int xgpu_ai_mailbox_ack_irq(struct amdgpu_device *adev,
218 struct amdgpu_irq_src *source,
219 struct amdgpu_iv_entry *entry)
220 {
221 DRM_DEBUG("get ack intr and do nothing.\n");
222 return 0;
223 }
224
xgpu_ai_set_mailbox_ack_irq(struct amdgpu_device * adev,struct amdgpu_irq_src * source,unsigned type,enum amdgpu_interrupt_state state)225 static int xgpu_ai_set_mailbox_ack_irq(struct amdgpu_device *adev,
226 struct amdgpu_irq_src *source,
227 unsigned type,
228 enum amdgpu_interrupt_state state)
229 {
230 u32 tmp = RREG32_NO_KIQ(SOC15_REG_OFFSET(NBIO, 0, mmBIF_BX_PF0_MAILBOX_INT_CNTL));
231
232 tmp = REG_SET_FIELD(tmp, BIF_BX_PF0_MAILBOX_INT_CNTL, ACK_INT_EN,
233 (state == AMDGPU_IRQ_STATE_ENABLE) ? 1 : 0);
234 WREG32_NO_KIQ(SOC15_REG_OFFSET(NBIO, 0, mmBIF_BX_PF0_MAILBOX_INT_CNTL), tmp);
235
236 return 0;
237 }
238
xgpu_ai_mailbox_flr_work(struct work_struct * work)239 static void xgpu_ai_mailbox_flr_work(struct work_struct *work)
240 {
241 struct amdgpu_virt *virt = container_of(work, struct amdgpu_virt, flr_work);
242 struct amdgpu_device *adev = container_of(virt, struct amdgpu_device, virt);
243
244 /* wait until RCV_MSG become 3 */
245 if (xgpu_ai_poll_msg(adev, IDH_FLR_NOTIFICATION_CMPL)) {
246 pr_err("failed to recieve FLR_CMPL\n");
247 return;
248 }
249
250 /* Trigger recovery due to world switch failure */
251 amdgpu_sriov_gpu_reset(adev, NULL);
252 }
253
xgpu_ai_set_mailbox_rcv_irq(struct amdgpu_device * adev,struct amdgpu_irq_src * src,unsigned type,enum amdgpu_interrupt_state state)254 static int xgpu_ai_set_mailbox_rcv_irq(struct amdgpu_device *adev,
255 struct amdgpu_irq_src *src,
256 unsigned type,
257 enum amdgpu_interrupt_state state)
258 {
259 u32 tmp = RREG32_NO_KIQ(SOC15_REG_OFFSET(NBIO, 0, mmBIF_BX_PF0_MAILBOX_INT_CNTL));
260
261 tmp = REG_SET_FIELD(tmp, BIF_BX_PF0_MAILBOX_INT_CNTL, VALID_INT_EN,
262 (state == AMDGPU_IRQ_STATE_ENABLE) ? 1 : 0);
263 WREG32_NO_KIQ(SOC15_REG_OFFSET(NBIO, 0, mmBIF_BX_PF0_MAILBOX_INT_CNTL), tmp);
264
265 return 0;
266 }
267
xgpu_ai_mailbox_rcv_irq(struct amdgpu_device * adev,struct amdgpu_irq_src * source,struct amdgpu_iv_entry * entry)268 static int xgpu_ai_mailbox_rcv_irq(struct amdgpu_device *adev,
269 struct amdgpu_irq_src *source,
270 struct amdgpu_iv_entry *entry)
271 {
272 int r;
273
274 /* trigger gpu-reset by hypervisor only if TDR disbaled */
275 if (amdgpu_lockup_timeout == 0) {
276 /* see what event we get */
277 r = xgpu_ai_mailbox_rcv_msg(adev, IDH_FLR_NOTIFICATION);
278
279 /* sometimes the interrupt is delayed to inject to VM, so under such case
280 * the IDH_FLR_NOTIFICATION is overwritten by VF FLR from GIM side, thus
281 * above recieve message could be failed, we should schedule the flr_work
282 * anyway
283 */
284 if (r) {
285 DRM_ERROR("FLR_NOTIFICATION is missed\n");
286 xgpu_ai_mailbox_send_ack(adev);
287 }
288
289 schedule_work(&adev->virt.flr_work);
290 }
291
292 return 0;
293 }
294
295 static const struct amdgpu_irq_src_funcs xgpu_ai_mailbox_ack_irq_funcs = {
296 .set = xgpu_ai_set_mailbox_ack_irq,
297 .process = xgpu_ai_mailbox_ack_irq,
298 };
299
300 static const struct amdgpu_irq_src_funcs xgpu_ai_mailbox_rcv_irq_funcs = {
301 .set = xgpu_ai_set_mailbox_rcv_irq,
302 .process = xgpu_ai_mailbox_rcv_irq,
303 };
304
xgpu_ai_mailbox_set_irq_funcs(struct amdgpu_device * adev)305 void xgpu_ai_mailbox_set_irq_funcs(struct amdgpu_device *adev)
306 {
307 adev->virt.ack_irq.num_types = 1;
308 adev->virt.ack_irq.funcs = &xgpu_ai_mailbox_ack_irq_funcs;
309 adev->virt.rcv_irq.num_types = 1;
310 adev->virt.rcv_irq.funcs = &xgpu_ai_mailbox_rcv_irq_funcs;
311 }
312
xgpu_ai_mailbox_add_irq_id(struct amdgpu_device * adev)313 int xgpu_ai_mailbox_add_irq_id(struct amdgpu_device *adev)
314 {
315 int r;
316
317 r = amdgpu_irq_add_id(adev, AMDGPU_IH_CLIENTID_BIF, 135, &adev->virt.rcv_irq);
318 if (r)
319 return r;
320
321 r = amdgpu_irq_add_id(adev, AMDGPU_IH_CLIENTID_BIF, 138, &adev->virt.ack_irq);
322 if (r) {
323 amdgpu_irq_put(adev, &adev->virt.rcv_irq, 0);
324 return r;
325 }
326
327 return 0;
328 }
329
xgpu_ai_mailbox_get_irq(struct amdgpu_device * adev)330 int xgpu_ai_mailbox_get_irq(struct amdgpu_device *adev)
331 {
332 int r;
333
334 r = amdgpu_irq_get(adev, &adev->virt.rcv_irq, 0);
335 if (r)
336 return r;
337 r = amdgpu_irq_get(adev, &adev->virt.ack_irq, 0);
338 if (r) {
339 amdgpu_irq_put(adev, &adev->virt.rcv_irq, 0);
340 return r;
341 }
342
343 INIT_WORK(&adev->virt.flr_work, xgpu_ai_mailbox_flr_work);
344
345 return 0;
346 }
347
xgpu_ai_mailbox_put_irq(struct amdgpu_device * adev)348 void xgpu_ai_mailbox_put_irq(struct amdgpu_device *adev)
349 {
350 amdgpu_irq_put(adev, &adev->virt.ack_irq, 0);
351 amdgpu_irq_put(adev, &adev->virt.rcv_irq, 0);
352 }
353
354 const struct amdgpu_virt_ops xgpu_ai_virt_ops = {
355 .req_full_gpu = xgpu_ai_request_full_gpu_access,
356 .rel_full_gpu = xgpu_ai_release_full_gpu_access,
357 .reset_gpu = xgpu_ai_request_reset,
358 .trans_msg = xgpu_ai_mailbox_trans_msg,
359 };
360