1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
2 /* Copyright (c) 2020, Mellanox Technologies inc. All rights reserved. */
3
4 #include "fw_reset.h"
5 #include "diag/fw_tracer.h"
6
7 enum {
8 MLX5_FW_RESET_FLAGS_RESET_REQUESTED,
9 MLX5_FW_RESET_FLAGS_NACK_RESET_REQUEST,
10 MLX5_FW_RESET_FLAGS_PENDING_COMP
11 };
12
13 struct mlx5_fw_reset {
14 struct mlx5_core_dev *dev;
15 struct mlx5_nb nb;
16 struct workqueue_struct *wq;
17 struct work_struct fw_live_patch_work;
18 struct work_struct reset_request_work;
19 struct work_struct reset_reload_work;
20 struct work_struct reset_now_work;
21 struct work_struct reset_abort_work;
22 unsigned long reset_flags;
23 struct timer_list timer;
24 struct completion done;
25 int ret;
26 };
27
mlx5_fw_reset_enable_remote_dev_reset_set(struct mlx5_core_dev * dev,bool enable)28 void mlx5_fw_reset_enable_remote_dev_reset_set(struct mlx5_core_dev *dev, bool enable)
29 {
30 struct mlx5_fw_reset *fw_reset = dev->priv.fw_reset;
31
32 if (enable)
33 clear_bit(MLX5_FW_RESET_FLAGS_NACK_RESET_REQUEST, &fw_reset->reset_flags);
34 else
35 set_bit(MLX5_FW_RESET_FLAGS_NACK_RESET_REQUEST, &fw_reset->reset_flags);
36 }
37
mlx5_fw_reset_enable_remote_dev_reset_get(struct mlx5_core_dev * dev)38 bool mlx5_fw_reset_enable_remote_dev_reset_get(struct mlx5_core_dev *dev)
39 {
40 struct mlx5_fw_reset *fw_reset = dev->priv.fw_reset;
41
42 return !test_bit(MLX5_FW_RESET_FLAGS_NACK_RESET_REQUEST, &fw_reset->reset_flags);
43 }
44
mlx5_reg_mfrl_set(struct mlx5_core_dev * dev,u8 reset_level,u8 reset_type_sel,u8 sync_resp,bool sync_start)45 static int mlx5_reg_mfrl_set(struct mlx5_core_dev *dev, u8 reset_level,
46 u8 reset_type_sel, u8 sync_resp, bool sync_start)
47 {
48 u32 out[MLX5_ST_SZ_DW(mfrl_reg)] = {};
49 u32 in[MLX5_ST_SZ_DW(mfrl_reg)] = {};
50
51 MLX5_SET(mfrl_reg, in, reset_level, reset_level);
52 MLX5_SET(mfrl_reg, in, rst_type_sel, reset_type_sel);
53 MLX5_SET(mfrl_reg, in, pci_sync_for_fw_update_resp, sync_resp);
54 MLX5_SET(mfrl_reg, in, pci_sync_for_fw_update_start, sync_start);
55
56 return mlx5_core_access_reg(dev, in, sizeof(in), out, sizeof(out), MLX5_REG_MFRL, 0, 1);
57 }
58
mlx5_reg_mfrl_query(struct mlx5_core_dev * dev,u8 * reset_level,u8 * reset_type)59 static int mlx5_reg_mfrl_query(struct mlx5_core_dev *dev, u8 *reset_level, u8 *reset_type)
60 {
61 u32 out[MLX5_ST_SZ_DW(mfrl_reg)] = {};
62 u32 in[MLX5_ST_SZ_DW(mfrl_reg)] = {};
63 int err;
64
65 err = mlx5_core_access_reg(dev, in, sizeof(in), out, sizeof(out), MLX5_REG_MFRL, 0, 0);
66 if (err)
67 return err;
68
69 if (reset_level)
70 *reset_level = MLX5_GET(mfrl_reg, out, reset_level);
71 if (reset_type)
72 *reset_type = MLX5_GET(mfrl_reg, out, reset_type);
73
74 return 0;
75 }
76
mlx5_fw_reset_query(struct mlx5_core_dev * dev,u8 * reset_level,u8 * reset_type)77 int mlx5_fw_reset_query(struct mlx5_core_dev *dev, u8 *reset_level, u8 *reset_type)
78 {
79 return mlx5_reg_mfrl_query(dev, reset_level, reset_type);
80 }
81
mlx5_fw_reset_set_reset_sync(struct mlx5_core_dev * dev,u8 reset_type_sel)82 int mlx5_fw_reset_set_reset_sync(struct mlx5_core_dev *dev, u8 reset_type_sel)
83 {
84 struct mlx5_fw_reset *fw_reset = dev->priv.fw_reset;
85 int err;
86
87 set_bit(MLX5_FW_RESET_FLAGS_PENDING_COMP, &fw_reset->reset_flags);
88 err = mlx5_reg_mfrl_set(dev, MLX5_MFRL_REG_RESET_LEVEL3, reset_type_sel, 0, true);
89 if (err)
90 clear_bit(MLX5_FW_RESET_FLAGS_PENDING_COMP, &fw_reset->reset_flags);
91 return err;
92 }
93
mlx5_fw_reset_set_live_patch(struct mlx5_core_dev * dev)94 int mlx5_fw_reset_set_live_patch(struct mlx5_core_dev *dev)
95 {
96 return mlx5_reg_mfrl_set(dev, MLX5_MFRL_REG_RESET_LEVEL0, 0, 0, false);
97 }
98
mlx5_fw_reset_complete_reload(struct mlx5_core_dev * dev)99 static void mlx5_fw_reset_complete_reload(struct mlx5_core_dev *dev)
100 {
101 struct mlx5_fw_reset *fw_reset = dev->priv.fw_reset;
102
103 /* if this is the driver that initiated the fw reset, devlink completed the reload */
104 if (test_bit(MLX5_FW_RESET_FLAGS_PENDING_COMP, &fw_reset->reset_flags)) {
105 complete(&fw_reset->done);
106 } else {
107 mlx5_load_one(dev, false);
108 devlink_remote_reload_actions_performed(priv_to_devlink(dev), 0,
109 BIT(DEVLINK_RELOAD_ACTION_DRIVER_REINIT) |
110 BIT(DEVLINK_RELOAD_ACTION_FW_ACTIVATE));
111 }
112 }
113
mlx5_sync_reset_reload_work(struct work_struct * work)114 static void mlx5_sync_reset_reload_work(struct work_struct *work)
115 {
116 struct mlx5_fw_reset *fw_reset = container_of(work, struct mlx5_fw_reset,
117 reset_reload_work);
118 struct mlx5_core_dev *dev = fw_reset->dev;
119 int err;
120
121 mlx5_enter_error_state(dev, true);
122 mlx5_unload_one(dev, false);
123 err = mlx5_health_wait_pci_up(dev);
124 if (err)
125 mlx5_core_err(dev, "reset reload flow aborted, PCI reads still not working\n");
126 fw_reset->ret = err;
127 mlx5_fw_reset_complete_reload(dev);
128 }
129
mlx5_stop_sync_reset_poll(struct mlx5_core_dev * dev)130 static void mlx5_stop_sync_reset_poll(struct mlx5_core_dev *dev)
131 {
132 struct mlx5_fw_reset *fw_reset = dev->priv.fw_reset;
133
134 del_timer_sync(&fw_reset->timer);
135 }
136
mlx5_sync_reset_clear_reset_requested(struct mlx5_core_dev * dev,bool poll_health)137 static int mlx5_sync_reset_clear_reset_requested(struct mlx5_core_dev *dev, bool poll_health)
138 {
139 struct mlx5_fw_reset *fw_reset = dev->priv.fw_reset;
140
141 if (!test_and_clear_bit(MLX5_FW_RESET_FLAGS_RESET_REQUESTED, &fw_reset->reset_flags)) {
142 mlx5_core_warn(dev, "Reset request was already cleared\n");
143 return -EALREADY;
144 }
145
146 mlx5_stop_sync_reset_poll(dev);
147 if (poll_health)
148 mlx5_start_health_poll(dev);
149 return 0;
150 }
151
152 #define MLX5_RESET_POLL_INTERVAL (HZ / 10)
poll_sync_reset(struct timer_list * t)153 static void poll_sync_reset(struct timer_list *t)
154 {
155 struct mlx5_fw_reset *fw_reset = from_timer(fw_reset, t, timer);
156 struct mlx5_core_dev *dev = fw_reset->dev;
157 u32 fatal_error;
158
159 if (!test_bit(MLX5_FW_RESET_FLAGS_RESET_REQUESTED, &fw_reset->reset_flags))
160 return;
161
162 fatal_error = mlx5_health_check_fatal_sensors(dev);
163
164 if (fatal_error) {
165 mlx5_core_warn(dev, "Got Device Reset\n");
166 mlx5_sync_reset_clear_reset_requested(dev, false);
167 queue_work(fw_reset->wq, &fw_reset->reset_reload_work);
168 return;
169 }
170
171 mod_timer(&fw_reset->timer, round_jiffies(jiffies + MLX5_RESET_POLL_INTERVAL));
172 }
173
mlx5_start_sync_reset_poll(struct mlx5_core_dev * dev)174 static void mlx5_start_sync_reset_poll(struct mlx5_core_dev *dev)
175 {
176 struct mlx5_fw_reset *fw_reset = dev->priv.fw_reset;
177
178 timer_setup(&fw_reset->timer, poll_sync_reset, 0);
179 fw_reset->timer.expires = round_jiffies(jiffies + MLX5_RESET_POLL_INTERVAL);
180 add_timer(&fw_reset->timer);
181 }
182
mlx5_fw_reset_set_reset_sync_ack(struct mlx5_core_dev * dev)183 static int mlx5_fw_reset_set_reset_sync_ack(struct mlx5_core_dev *dev)
184 {
185 return mlx5_reg_mfrl_set(dev, MLX5_MFRL_REG_RESET_LEVEL3, 0, 1, false);
186 }
187
mlx5_fw_reset_set_reset_sync_nack(struct mlx5_core_dev * dev)188 static int mlx5_fw_reset_set_reset_sync_nack(struct mlx5_core_dev *dev)
189 {
190 return mlx5_reg_mfrl_set(dev, MLX5_MFRL_REG_RESET_LEVEL3, 0, 2, false);
191 }
192
mlx5_sync_reset_set_reset_requested(struct mlx5_core_dev * dev)193 static int mlx5_sync_reset_set_reset_requested(struct mlx5_core_dev *dev)
194 {
195 struct mlx5_fw_reset *fw_reset = dev->priv.fw_reset;
196
197 if (test_and_set_bit(MLX5_FW_RESET_FLAGS_RESET_REQUESTED, &fw_reset->reset_flags)) {
198 mlx5_core_warn(dev, "Reset request was already set\n");
199 return -EALREADY;
200 }
201 mlx5_stop_health_poll(dev, true);
202 mlx5_start_sync_reset_poll(dev);
203 return 0;
204 }
205
mlx5_fw_live_patch_event(struct work_struct * work)206 static void mlx5_fw_live_patch_event(struct work_struct *work)
207 {
208 struct mlx5_fw_reset *fw_reset = container_of(work, struct mlx5_fw_reset,
209 fw_live_patch_work);
210 struct mlx5_core_dev *dev = fw_reset->dev;
211 struct mlx5_fw_tracer *tracer;
212
213 mlx5_core_info(dev, "Live patch updated firmware version: %d.%d.%d\n", fw_rev_maj(dev),
214 fw_rev_min(dev), fw_rev_sub(dev));
215
216 tracer = dev->tracer;
217 if (IS_ERR_OR_NULL(tracer))
218 return;
219
220 if (mlx5_fw_tracer_reload(tracer))
221 mlx5_core_err(dev, "Failed to reload FW tracer\n");
222 }
223
mlx5_sync_reset_request_event(struct work_struct * work)224 static void mlx5_sync_reset_request_event(struct work_struct *work)
225 {
226 struct mlx5_fw_reset *fw_reset = container_of(work, struct mlx5_fw_reset,
227 reset_request_work);
228 struct mlx5_core_dev *dev = fw_reset->dev;
229 int err;
230
231 if (test_bit(MLX5_FW_RESET_FLAGS_NACK_RESET_REQUEST, &fw_reset->reset_flags)) {
232 err = mlx5_fw_reset_set_reset_sync_nack(dev);
233 mlx5_core_warn(dev, "PCI Sync FW Update Reset Nack %s",
234 err ? "Failed" : "Sent");
235 return;
236 }
237 if (mlx5_sync_reset_set_reset_requested(dev))
238 return;
239
240 err = mlx5_fw_reset_set_reset_sync_ack(dev);
241 if (err)
242 mlx5_core_warn(dev, "PCI Sync FW Update Reset Ack Failed. Error code: %d\n", err);
243 else
244 mlx5_core_warn(dev, "PCI Sync FW Update Reset Ack. Device reset is expected.\n");
245 }
246
247 #define MLX5_PCI_LINK_UP_TIMEOUT 2000
248
mlx5_pci_link_toggle(struct mlx5_core_dev * dev)249 static int mlx5_pci_link_toggle(struct mlx5_core_dev *dev)
250 {
251 struct pci_bus *bridge_bus = dev->pdev->bus;
252 struct pci_dev *bridge = bridge_bus->self;
253 u16 reg16, dev_id, sdev_id;
254 unsigned long timeout;
255 struct pci_dev *sdev;
256 int cap, err;
257 u32 reg32;
258
259 /* Check that all functions under the pci bridge are PFs of
260 * this device otherwise fail this function.
261 */
262 err = pci_read_config_word(dev->pdev, PCI_DEVICE_ID, &dev_id);
263 if (err)
264 return err;
265 list_for_each_entry(sdev, &bridge_bus->devices, bus_list) {
266 err = pci_read_config_word(sdev, PCI_DEVICE_ID, &sdev_id);
267 if (err)
268 return err;
269 if (sdev_id != dev_id)
270 return -EPERM;
271 }
272
273 cap = pci_find_capability(bridge, PCI_CAP_ID_EXP);
274 if (!cap)
275 return -EOPNOTSUPP;
276
277 list_for_each_entry(sdev, &bridge_bus->devices, bus_list) {
278 pci_save_state(sdev);
279 pci_cfg_access_lock(sdev);
280 }
281 /* PCI link toggle */
282 err = pci_read_config_word(bridge, cap + PCI_EXP_LNKCTL, ®16);
283 if (err)
284 return err;
285 reg16 |= PCI_EXP_LNKCTL_LD;
286 err = pci_write_config_word(bridge, cap + PCI_EXP_LNKCTL, reg16);
287 if (err)
288 return err;
289 msleep(500);
290 reg16 &= ~PCI_EXP_LNKCTL_LD;
291 err = pci_write_config_word(bridge, cap + PCI_EXP_LNKCTL, reg16);
292 if (err)
293 return err;
294
295 /* Check link */
296 err = pci_read_config_dword(bridge, cap + PCI_EXP_LNKCAP, ®32);
297 if (err)
298 return err;
299 if (!(reg32 & PCI_EXP_LNKCAP_DLLLARC)) {
300 mlx5_core_warn(dev, "No PCI link reporting capability (0x%08x)\n", reg32);
301 msleep(1000);
302 goto restore;
303 }
304
305 timeout = jiffies + msecs_to_jiffies(MLX5_PCI_LINK_UP_TIMEOUT);
306 do {
307 err = pci_read_config_word(bridge, cap + PCI_EXP_LNKSTA, ®16);
308 if (err)
309 return err;
310 if (reg16 & PCI_EXP_LNKSTA_DLLLA)
311 break;
312 msleep(20);
313 } while (!time_after(jiffies, timeout));
314
315 if (reg16 & PCI_EXP_LNKSTA_DLLLA) {
316 mlx5_core_info(dev, "PCI Link up\n");
317 } else {
318 mlx5_core_err(dev, "PCI link not ready (0x%04x) after %d ms\n",
319 reg16, MLX5_PCI_LINK_UP_TIMEOUT);
320 err = -ETIMEDOUT;
321 }
322
323 restore:
324 list_for_each_entry(sdev, &bridge_bus->devices, bus_list) {
325 pci_cfg_access_unlock(sdev);
326 pci_restore_state(sdev);
327 }
328
329 return err;
330 }
331
mlx5_sync_reset_now_event(struct work_struct * work)332 static void mlx5_sync_reset_now_event(struct work_struct *work)
333 {
334 struct mlx5_fw_reset *fw_reset = container_of(work, struct mlx5_fw_reset,
335 reset_now_work);
336 struct mlx5_core_dev *dev = fw_reset->dev;
337 int err;
338
339 if (mlx5_sync_reset_clear_reset_requested(dev, false))
340 return;
341
342 mlx5_core_warn(dev, "Sync Reset now. Device is going to reset.\n");
343
344 err = mlx5_cmd_fast_teardown_hca(dev);
345 if (err) {
346 mlx5_core_warn(dev, "Fast teardown failed, no reset done, err %d\n", err);
347 goto done;
348 }
349
350 err = mlx5_pci_link_toggle(dev);
351 if (err) {
352 mlx5_core_warn(dev, "mlx5_pci_link_toggle failed, no reset done, err %d\n", err);
353 goto done;
354 }
355
356 mlx5_enter_error_state(dev, true);
357 mlx5_unload_one(dev, false);
358 done:
359 fw_reset->ret = err;
360 mlx5_fw_reset_complete_reload(dev);
361 }
362
mlx5_sync_reset_abort_event(struct work_struct * work)363 static void mlx5_sync_reset_abort_event(struct work_struct *work)
364 {
365 struct mlx5_fw_reset *fw_reset = container_of(work, struct mlx5_fw_reset,
366 reset_abort_work);
367 struct mlx5_core_dev *dev = fw_reset->dev;
368
369 if (mlx5_sync_reset_clear_reset_requested(dev, true))
370 return;
371 mlx5_core_warn(dev, "PCI Sync FW Update Reset Aborted.\n");
372 }
373
mlx5_sync_reset_events_handle(struct mlx5_fw_reset * fw_reset,struct mlx5_eqe * eqe)374 static void mlx5_sync_reset_events_handle(struct mlx5_fw_reset *fw_reset, struct mlx5_eqe *eqe)
375 {
376 struct mlx5_eqe_sync_fw_update *sync_fw_update_eqe;
377 u8 sync_event_rst_type;
378
379 sync_fw_update_eqe = &eqe->data.sync_fw_update;
380 sync_event_rst_type = sync_fw_update_eqe->sync_rst_state & SYNC_RST_STATE_MASK;
381 switch (sync_event_rst_type) {
382 case MLX5_SYNC_RST_STATE_RESET_REQUEST:
383 queue_work(fw_reset->wq, &fw_reset->reset_request_work);
384 break;
385 case MLX5_SYNC_RST_STATE_RESET_NOW:
386 queue_work(fw_reset->wq, &fw_reset->reset_now_work);
387 break;
388 case MLX5_SYNC_RST_STATE_RESET_ABORT:
389 queue_work(fw_reset->wq, &fw_reset->reset_abort_work);
390 break;
391 }
392 }
393
fw_reset_event_notifier(struct notifier_block * nb,unsigned long action,void * data)394 static int fw_reset_event_notifier(struct notifier_block *nb, unsigned long action, void *data)
395 {
396 struct mlx5_fw_reset *fw_reset = mlx5_nb_cof(nb, struct mlx5_fw_reset, nb);
397 struct mlx5_eqe *eqe = data;
398
399 switch (eqe->sub_type) {
400 case MLX5_GENERAL_SUBTYPE_FW_LIVE_PATCH_EVENT:
401 queue_work(fw_reset->wq, &fw_reset->fw_live_patch_work);
402 break;
403 case MLX5_GENERAL_SUBTYPE_PCI_SYNC_FOR_FW_UPDATE_EVENT:
404 mlx5_sync_reset_events_handle(fw_reset, eqe);
405 break;
406 default:
407 return NOTIFY_DONE;
408 }
409
410 return NOTIFY_OK;
411 }
412
413 #define MLX5_FW_RESET_TIMEOUT_MSEC 5000
mlx5_fw_reset_wait_reset_done(struct mlx5_core_dev * dev)414 int mlx5_fw_reset_wait_reset_done(struct mlx5_core_dev *dev)
415 {
416 unsigned long timeout = msecs_to_jiffies(MLX5_FW_RESET_TIMEOUT_MSEC);
417 struct mlx5_fw_reset *fw_reset = dev->priv.fw_reset;
418 int err;
419
420 if (!wait_for_completion_timeout(&fw_reset->done, timeout)) {
421 mlx5_core_warn(dev, "FW sync reset timeout after %d seconds\n",
422 MLX5_FW_RESET_TIMEOUT_MSEC / 1000);
423 err = -ETIMEDOUT;
424 goto out;
425 }
426 err = fw_reset->ret;
427 out:
428 clear_bit(MLX5_FW_RESET_FLAGS_PENDING_COMP, &fw_reset->reset_flags);
429 return err;
430 }
431
mlx5_fw_reset_events_start(struct mlx5_core_dev * dev)432 void mlx5_fw_reset_events_start(struct mlx5_core_dev *dev)
433 {
434 struct mlx5_fw_reset *fw_reset = dev->priv.fw_reset;
435
436 MLX5_NB_INIT(&fw_reset->nb, fw_reset_event_notifier, GENERAL_EVENT);
437 mlx5_eq_notifier_register(dev, &fw_reset->nb);
438 }
439
mlx5_fw_reset_events_stop(struct mlx5_core_dev * dev)440 void mlx5_fw_reset_events_stop(struct mlx5_core_dev *dev)
441 {
442 mlx5_eq_notifier_unregister(dev, &dev->priv.fw_reset->nb);
443 }
444
mlx5_fw_reset_init(struct mlx5_core_dev * dev)445 int mlx5_fw_reset_init(struct mlx5_core_dev *dev)
446 {
447 struct mlx5_fw_reset *fw_reset = kzalloc(sizeof(*fw_reset), GFP_KERNEL);
448
449 if (!fw_reset)
450 return -ENOMEM;
451 fw_reset->wq = create_singlethread_workqueue("mlx5_fw_reset_events");
452 if (!fw_reset->wq) {
453 kfree(fw_reset);
454 return -ENOMEM;
455 }
456
457 fw_reset->dev = dev;
458 dev->priv.fw_reset = fw_reset;
459
460 INIT_WORK(&fw_reset->fw_live_patch_work, mlx5_fw_live_patch_event);
461 INIT_WORK(&fw_reset->reset_request_work, mlx5_sync_reset_request_event);
462 INIT_WORK(&fw_reset->reset_reload_work, mlx5_sync_reset_reload_work);
463 INIT_WORK(&fw_reset->reset_now_work, mlx5_sync_reset_now_event);
464 INIT_WORK(&fw_reset->reset_abort_work, mlx5_sync_reset_abort_event);
465
466 init_completion(&fw_reset->done);
467 return 0;
468 }
469
mlx5_fw_reset_cleanup(struct mlx5_core_dev * dev)470 void mlx5_fw_reset_cleanup(struct mlx5_core_dev *dev)
471 {
472 struct mlx5_fw_reset *fw_reset = dev->priv.fw_reset;
473
474 destroy_workqueue(fw_reset->wq);
475 kfree(dev->priv.fw_reset);
476 }
477