1 /* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */
2 /*
3 * Copyright (C) 2003-2015, 2018-2022 Intel Corporation
4 * Copyright (C) 2013-2015 Intel Mobile Communications GmbH
5 * Copyright (C) 2016-2017 Intel Deutschland GmbH
6 */
7 #ifndef __iwl_trans_int_pcie_h__
8 #define __iwl_trans_int_pcie_h__
9
10 #include <linux/spinlock.h>
11 #include <linux/interrupt.h>
12 #include <linux/skbuff.h>
13 #include <linux/wait.h>
14 #include <linux/pci.h>
15 #include <linux/timer.h>
16 #include <linux/cpu.h>
17
18 #include "iwl-fh.h"
19 #include "iwl-csr.h"
20 #include "iwl-trans.h"
21 #include "iwl-debug.h"
22 #include "iwl-io.h"
23 #include "iwl-op-mode.h"
24 #include "iwl-drv.h"
25 #include "queue/tx.h"
26
27 /*
28 * RX related structures and functions
29 */
30 #define RX_NUM_QUEUES 1
31 #define RX_POST_REQ_ALLOC 2
32 #define RX_CLAIM_REQ_ALLOC 8
33 #define RX_PENDING_WATERMARK 16
34 #define FIRST_RX_QUEUE 512
35
36 struct iwl_host_cmd;
37
38 /*This file includes the declaration that are internal to the
39 * trans_pcie layer */
40
41 /**
42 * struct iwl_rx_mem_buffer
43 * @page_dma: bus address of rxb page
44 * @page: driver's pointer to the rxb page
45 * @list: list entry for the membuffer
46 * @invalid: rxb is in driver ownership - not owned by HW
47 * @vid: index of this rxb in the global table
48 * @offset: indicates which offset of the page (in bytes)
49 * this buffer uses (if multiple RBs fit into one page)
50 */
51 struct iwl_rx_mem_buffer {
52 dma_addr_t page_dma;
53 struct page *page;
54 struct list_head list;
55 u32 offset;
56 u16 vid;
57 bool invalid;
58 };
59
60 /**
61 * struct isr_statistics - interrupt statistics
62 *
63 */
64 struct isr_statistics {
65 u32 hw;
66 u32 sw;
67 u32 err_code;
68 u32 sch;
69 u32 alive;
70 u32 rfkill;
71 u32 ctkill;
72 u32 wakeup;
73 u32 rx;
74 u32 tx;
75 u32 unhandled;
76 };
77
78 /**
79 * struct iwl_rx_transfer_desc - transfer descriptor
80 * @addr: ptr to free buffer start address
81 * @rbid: unique tag of the buffer
82 * @reserved: reserved
83 */
84 struct iwl_rx_transfer_desc {
85 __le16 rbid;
86 __le16 reserved[3];
87 __le64 addr;
88 } __packed;
89
90 #define IWL_RX_CD_FLAGS_FRAGMENTED BIT(0)
91
92 /**
93 * struct iwl_rx_completion_desc - completion descriptor
94 * @reserved1: reserved
95 * @rbid: unique tag of the received buffer
96 * @flags: flags (0: fragmented, all others: reserved)
97 * @reserved2: reserved
98 */
99 struct iwl_rx_completion_desc {
100 __le32 reserved1;
101 __le16 rbid;
102 u8 flags;
103 u8 reserved2[25];
104 } __packed;
105
106 /**
107 * struct iwl_rx_completion_desc_bz - Bz completion descriptor
108 * @rbid: unique tag of the received buffer
109 * @flags: flags (0: fragmented, all others: reserved)
110 * @reserved: reserved
111 */
112 struct iwl_rx_completion_desc_bz {
113 __le16 rbid;
114 u8 flags;
115 u8 reserved[1];
116 } __packed;
117
118 /**
119 * struct iwl_rxq - Rx queue
120 * @id: queue index
121 * @bd: driver's pointer to buffer of receive buffer descriptors (rbd).
122 * Address size is 32 bit in pre-9000 devices and 64 bit in 9000 devices.
123 * In AX210 devices it is a pointer to a list of iwl_rx_transfer_desc's
124 * @bd_dma: bus address of buffer of receive buffer descriptors (rbd)
125 * @used_bd: driver's pointer to buffer of used receive buffer descriptors (rbd)
126 * @used_bd_dma: physical address of buffer of used receive buffer descriptors (rbd)
127 * @read: Shared index to newest available Rx buffer
128 * @write: Shared index to oldest written Rx packet
129 * @free_count: Number of pre-allocated buffers in rx_free
130 * @used_count: Number of RBDs handled to allocator to use for allocation
131 * @write_actual:
132 * @rx_free: list of RBDs with allocated RB ready for use
133 * @rx_used: list of RBDs with no RB attached
134 * @need_update: flag to indicate we need to update read/write index
135 * @rb_stts: driver's pointer to receive buffer status
136 * @rb_stts_dma: bus address of receive buffer status
137 * @lock:
138 * @queue: actual rx queue. Not used for multi-rx queue.
139 * @next_rb_is_fragment: indicates that the previous RB that we handled set
140 * the fragmented flag, so the next one is still another fragment
141 *
142 * NOTE: rx_free and rx_used are used as a FIFO for iwl_rx_mem_buffers
143 */
144 struct iwl_rxq {
145 int id;
146 void *bd;
147 dma_addr_t bd_dma;
148 void *used_bd;
149 dma_addr_t used_bd_dma;
150 u32 read;
151 u32 write;
152 u32 free_count;
153 u32 used_count;
154 u32 write_actual;
155 u32 queue_size;
156 struct list_head rx_free;
157 struct list_head rx_used;
158 bool need_update, next_rb_is_fragment;
159 void *rb_stts;
160 dma_addr_t rb_stts_dma;
161 spinlock_t lock;
162 struct napi_struct napi;
163 struct iwl_rx_mem_buffer *queue[RX_QUEUE_SIZE];
164 };
165
166 /**
167 * struct iwl_rb_allocator - Rx allocator
168 * @req_pending: number of requests the allcator had not processed yet
169 * @req_ready: number of requests honored and ready for claiming
170 * @rbd_allocated: RBDs with pages allocated and ready to be handled to
171 * the queue. This is a list of &struct iwl_rx_mem_buffer
172 * @rbd_empty: RBDs with no page attached for allocator use. This is a list
173 * of &struct iwl_rx_mem_buffer
174 * @lock: protects the rbd_allocated and rbd_empty lists
175 * @alloc_wq: work queue for background calls
176 * @rx_alloc: work struct for background calls
177 */
178 struct iwl_rb_allocator {
179 atomic_t req_pending;
180 atomic_t req_ready;
181 struct list_head rbd_allocated;
182 struct list_head rbd_empty;
183 spinlock_t lock;
184 struct workqueue_struct *alloc_wq;
185 struct work_struct rx_alloc;
186 };
187
188 /**
189 * iwl_get_closed_rb_stts - get closed rb stts from different structs
190 * @rxq - the rxq to get the rb stts from
191 */
iwl_get_closed_rb_stts(struct iwl_trans * trans,struct iwl_rxq * rxq)192 static inline __le16 iwl_get_closed_rb_stts(struct iwl_trans *trans,
193 struct iwl_rxq *rxq)
194 {
195 if (trans->trans_cfg->device_family >= IWL_DEVICE_FAMILY_AX210) {
196 __le16 *rb_stts = rxq->rb_stts;
197
198 return READ_ONCE(*rb_stts);
199 } else {
200 struct iwl_rb_status *rb_stts = rxq->rb_stts;
201
202 return READ_ONCE(rb_stts->closed_rb_num);
203 }
204 }
205
206 #ifdef CONFIG_IWLWIFI_DEBUGFS
207 /**
208 * enum iwl_fw_mon_dbgfs_state - the different states of the monitor_data
209 * debugfs file
210 *
211 * @IWL_FW_MON_DBGFS_STATE_CLOSED: the file is closed.
212 * @IWL_FW_MON_DBGFS_STATE_OPEN: the file is open.
213 * @IWL_FW_MON_DBGFS_STATE_DISABLED: the file is disabled, once this state is
214 * set the file can no longer be used.
215 */
216 enum iwl_fw_mon_dbgfs_state {
217 IWL_FW_MON_DBGFS_STATE_CLOSED,
218 IWL_FW_MON_DBGFS_STATE_OPEN,
219 IWL_FW_MON_DBGFS_STATE_DISABLED,
220 };
221 #endif
222
223 /**
224 * enum iwl_shared_irq_flags - level of sharing for irq
225 * @IWL_SHARED_IRQ_NON_RX: interrupt vector serves non rx causes.
226 * @IWL_SHARED_IRQ_FIRST_RSS: interrupt vector serves first RSS queue.
227 */
228 enum iwl_shared_irq_flags {
229 IWL_SHARED_IRQ_NON_RX = BIT(0),
230 IWL_SHARED_IRQ_FIRST_RSS = BIT(1),
231 };
232
233 /**
234 * enum iwl_image_response_code - image response values
235 * @IWL_IMAGE_RESP_DEF: the default value of the register
236 * @IWL_IMAGE_RESP_SUCCESS: iml was read successfully
237 * @IWL_IMAGE_RESP_FAIL: iml reading failed
238 */
239 enum iwl_image_response_code {
240 IWL_IMAGE_RESP_DEF = 0,
241 IWL_IMAGE_RESP_SUCCESS = 1,
242 IWL_IMAGE_RESP_FAIL = 2,
243 };
244
245 /**
246 * struct cont_rec: continuous recording data structure
247 * @prev_wr_ptr: the last address that was read in monitor_data
248 * debugfs file
249 * @prev_wrap_cnt: the wrap count that was used during the last read in
250 * monitor_data debugfs file
251 * @state: the state of monitor_data debugfs file as described
252 * in &iwl_fw_mon_dbgfs_state enum
253 * @mutex: locked while reading from monitor_data debugfs file
254 */
255 #ifdef CONFIG_IWLWIFI_DEBUGFS
256 struct cont_rec {
257 u32 prev_wr_ptr;
258 u32 prev_wrap_cnt;
259 u8 state;
260 /* Used to sync monitor_data debugfs file with driver unload flow */
261 struct mutex mutex;
262 };
263 #endif
264
265 enum iwl_pcie_fw_reset_state {
266 FW_RESET_IDLE,
267 FW_RESET_REQUESTED,
268 FW_RESET_OK,
269 FW_RESET_ERROR,
270 };
271
272 /**
273 * struct iwl_trans_pcie - PCIe transport specific data
274 * @rxq: all the RX queue data
275 * @rx_pool: initial pool of iwl_rx_mem_buffer for all the queues
276 * @global_table: table mapping received VID from hw to rxb
277 * @rba: allocator for RX replenishing
278 * @ctxt_info: context information for FW self init
279 * @ctxt_info_gen3: context information for gen3 devices
280 * @prph_info: prph info for self init
281 * @prph_scratch: prph scratch for self init
282 * @ctxt_info_dma_addr: dma addr of context information
283 * @prph_info_dma_addr: dma addr of prph info
284 * @prph_scratch_dma_addr: dma addr of prph scratch
285 * @ctxt_info_dma_addr: dma addr of context information
286 * @init_dram: DRAM data of firmware image (including paging).
287 * Context information addresses will be taken from here.
288 * This is driver's local copy for keeping track of size and
289 * count for allocating and freeing the memory.
290 * @iml: image loader image virtual address
291 * @iml_dma_addr: image loader image DMA address
292 * @trans: pointer to the generic transport area
293 * @scd_base_addr: scheduler sram base address in SRAM
294 * @kw: keep warm address
295 * @pnvm_dram: DRAM area that contains the PNVM data
296 * @pci_dev: basic pci-network driver stuff
297 * @hw_base: pci hardware address support
298 * @ucode_write_complete: indicates that the ucode has been copied.
299 * @ucode_write_waitq: wait queue for uCode load
300 * @cmd_queue - command queue number
301 * @def_rx_queue - default rx queue number
302 * @rx_buf_size: Rx buffer size
303 * @scd_set_active: should the transport configure the SCD for HCMD queue
304 * @rx_page_order: page order for receive buffer size
305 * @rx_buf_bytes: RX buffer (RB) size in bytes
306 * @reg_lock: protect hw register access
307 * @mutex: to protect stop_device / start_fw / start_hw
308 * @cmd_in_flight: true when we have a host command in flight
309 #ifdef CONFIG_IWLWIFI_DEBUGFS
310 * @fw_mon_data: fw continuous recording data
311 #endif
312 * @msix_entries: array of MSI-X entries
313 * @msix_enabled: true if managed to enable MSI-X
314 * @shared_vec_mask: the type of causes the shared vector handles
315 * (see iwl_shared_irq_flags).
316 * @alloc_vecs: the number of interrupt vectors allocated by the OS
317 * @def_irq: default irq for non rx causes
318 * @fh_init_mask: initial unmasked fh causes
319 * @hw_init_mask: initial unmasked hw causes
320 * @fh_mask: current unmasked fh causes
321 * @hw_mask: current unmasked hw causes
322 * @in_rescan: true if we have triggered a device rescan
323 * @base_rb_stts: base virtual address of receive buffer status for all queues
324 * @base_rb_stts_dma: base physical address of receive buffer status
325 * @supported_dma_mask: DMA mask to validate the actual address against,
326 * will be DMA_BIT_MASK(11) or DMA_BIT_MASK(12) depending on the device
327 * @alloc_page_lock: spinlock for the page allocator
328 * @alloc_page: allocated page to still use parts of
329 * @alloc_page_used: how much of the allocated page was already used (bytes)
330 * @rf_name: name/version of the CRF, if any
331 */
332 struct iwl_trans_pcie {
333 struct iwl_rxq *rxq;
334 struct iwl_rx_mem_buffer *rx_pool;
335 struct iwl_rx_mem_buffer **global_table;
336 struct iwl_rb_allocator rba;
337 union {
338 struct iwl_context_info *ctxt_info;
339 struct iwl_context_info_gen3 *ctxt_info_gen3;
340 };
341 struct iwl_prph_info *prph_info;
342 struct iwl_prph_scratch *prph_scratch;
343 void *iml;
344 dma_addr_t ctxt_info_dma_addr;
345 dma_addr_t prph_info_dma_addr;
346 dma_addr_t prph_scratch_dma_addr;
347 dma_addr_t iml_dma_addr;
348 struct iwl_trans *trans;
349
350 struct net_device napi_dev;
351
352 /* INT ICT Table */
353 __le32 *ict_tbl;
354 dma_addr_t ict_tbl_dma;
355 int ict_index;
356 bool use_ict;
357 bool is_down, opmode_down;
358 s8 debug_rfkill;
359 struct isr_statistics isr_stats;
360
361 spinlock_t irq_lock;
362 struct mutex mutex;
363 u32 inta_mask;
364 u32 scd_base_addr;
365 struct iwl_dma_ptr kw;
366
367 struct iwl_dram_data pnvm_dram;
368 struct iwl_dram_data reduce_power_dram;
369
370 struct iwl_txq *txq_memory;
371
372 /* PCI bus related data */
373 struct pci_dev *pci_dev;
374 u8 __iomem *hw_base;
375
376 bool ucode_write_complete;
377 bool sx_complete;
378 wait_queue_head_t ucode_write_waitq;
379 wait_queue_head_t sx_waitq;
380
381 u8 def_rx_queue;
382 u8 n_no_reclaim_cmds;
383 u8 no_reclaim_cmds[MAX_NO_RECLAIM_CMDS];
384 u16 num_rx_bufs;
385
386 enum iwl_amsdu_size rx_buf_size;
387 bool scd_set_active;
388 bool pcie_dbg_dumped_once;
389 u32 rx_page_order;
390 u32 rx_buf_bytes;
391 u32 supported_dma_mask;
392
393 /* allocator lock for the two values below */
394 spinlock_t alloc_page_lock;
395 struct page *alloc_page;
396 u32 alloc_page_used;
397
398 /*protect hw register */
399 spinlock_t reg_lock;
400 bool cmd_hold_nic_awake;
401
402 #ifdef CONFIG_IWLWIFI_DEBUGFS
403 struct cont_rec fw_mon_data;
404 #endif
405
406 struct msix_entry msix_entries[IWL_MAX_RX_HW_QUEUES];
407 bool msix_enabled;
408 u8 shared_vec_mask;
409 u32 alloc_vecs;
410 u32 def_irq;
411 u32 fh_init_mask;
412 u32 hw_init_mask;
413 u32 fh_mask;
414 u32 hw_mask;
415 cpumask_t affinity_mask[IWL_MAX_RX_HW_QUEUES];
416 u16 tx_cmd_queue_size;
417 bool in_rescan;
418
419 void *base_rb_stts;
420 dma_addr_t base_rb_stts_dma;
421
422 bool fw_reset_handshake;
423 enum iwl_pcie_fw_reset_state fw_reset_state;
424 wait_queue_head_t fw_reset_waitq;
425
426 char rf_name[32];
427 };
428
429 static inline struct iwl_trans_pcie *
IWL_TRANS_GET_PCIE_TRANS(struct iwl_trans * trans)430 IWL_TRANS_GET_PCIE_TRANS(struct iwl_trans *trans)
431 {
432 return (void *)trans->trans_specific;
433 }
434
iwl_pcie_clear_irq(struct iwl_trans * trans,int queue)435 static inline void iwl_pcie_clear_irq(struct iwl_trans *trans, int queue)
436 {
437 /*
438 * Before sending the interrupt the HW disables it to prevent
439 * a nested interrupt. This is done by writing 1 to the corresponding
440 * bit in the mask register. After handling the interrupt, it should be
441 * re-enabled by clearing this bit. This register is defined as
442 * write 1 clear (W1C) register, meaning that it's being clear
443 * by writing 1 to the bit.
444 */
445 iwl_write32(trans, CSR_MSIX_AUTOMASK_ST_AD, BIT(queue));
446 }
447
448 static inline struct iwl_trans *
iwl_trans_pcie_get_trans(struct iwl_trans_pcie * trans_pcie)449 iwl_trans_pcie_get_trans(struct iwl_trans_pcie *trans_pcie)
450 {
451 return container_of((void *)trans_pcie, struct iwl_trans,
452 trans_specific);
453 }
454
455 /*
456 * Convention: trans API functions: iwl_trans_pcie_XXX
457 * Other functions: iwl_pcie_XXX
458 */
459 struct iwl_trans
460 *iwl_trans_pcie_alloc(struct pci_dev *pdev,
461 const struct pci_device_id *ent,
462 const struct iwl_cfg_trans_params *cfg_trans);
463 void iwl_trans_pcie_free(struct iwl_trans *trans);
464
465 bool __iwl_trans_pcie_grab_nic_access(struct iwl_trans *trans);
466 #define _iwl_trans_pcie_grab_nic_access(trans) \
467 __cond_lock(nic_access_nobh, \
468 likely(__iwl_trans_pcie_grab_nic_access(trans)))
469
470 /*****************************************************
471 * RX
472 ******************************************************/
473 int iwl_pcie_rx_init(struct iwl_trans *trans);
474 int iwl_pcie_gen2_rx_init(struct iwl_trans *trans);
475 irqreturn_t iwl_pcie_msix_isr(int irq, void *data);
476 irqreturn_t iwl_pcie_irq_handler(int irq, void *dev_id);
477 irqreturn_t iwl_pcie_irq_msix_handler(int irq, void *dev_id);
478 irqreturn_t iwl_pcie_irq_rx_msix_handler(int irq, void *dev_id);
479 int iwl_pcie_rx_stop(struct iwl_trans *trans);
480 void iwl_pcie_rx_free(struct iwl_trans *trans);
481 void iwl_pcie_free_rbs_pool(struct iwl_trans *trans);
482 void iwl_pcie_rx_init_rxb_lists(struct iwl_rxq *rxq);
483 void iwl_pcie_rx_napi_sync(struct iwl_trans *trans);
484 void iwl_pcie_rxq_alloc_rbs(struct iwl_trans *trans, gfp_t priority,
485 struct iwl_rxq *rxq);
486
487 /*****************************************************
488 * ICT - interrupt handling
489 ******************************************************/
490 irqreturn_t iwl_pcie_isr(int irq, void *data);
491 int iwl_pcie_alloc_ict(struct iwl_trans *trans);
492 void iwl_pcie_free_ict(struct iwl_trans *trans);
493 void iwl_pcie_reset_ict(struct iwl_trans *trans);
494 void iwl_pcie_disable_ict(struct iwl_trans *trans);
495
496 /*****************************************************
497 * TX / HCMD
498 ******************************************************/
499 int iwl_pcie_tx_init(struct iwl_trans *trans);
500 void iwl_pcie_tx_start(struct iwl_trans *trans, u32 scd_base_addr);
501 int iwl_pcie_tx_stop(struct iwl_trans *trans);
502 void iwl_pcie_tx_free(struct iwl_trans *trans);
503 bool iwl_trans_pcie_txq_enable(struct iwl_trans *trans, int queue, u16 ssn,
504 const struct iwl_trans_txq_scd_cfg *cfg,
505 unsigned int wdg_timeout);
506 void iwl_trans_pcie_txq_disable(struct iwl_trans *trans, int queue,
507 bool configure_scd);
508 void iwl_trans_pcie_txq_set_shared_mode(struct iwl_trans *trans, u32 txq_id,
509 bool shared_mode);
510 int iwl_trans_pcie_tx(struct iwl_trans *trans, struct sk_buff *skb,
511 struct iwl_device_tx_cmd *dev_cmd, int txq_id);
512 void iwl_pcie_txq_check_wrptrs(struct iwl_trans *trans);
513 int iwl_trans_pcie_send_hcmd(struct iwl_trans *trans, struct iwl_host_cmd *cmd);
514 void iwl_pcie_hcmd_complete(struct iwl_trans *trans,
515 struct iwl_rx_cmd_buffer *rxb);
516 void iwl_trans_pcie_tx_reset(struct iwl_trans *trans);
517
518 /*****************************************************
519 * Error handling
520 ******************************************************/
521 void iwl_pcie_dump_csr(struct iwl_trans *trans);
522
523 /*****************************************************
524 * Helpers
525 ******************************************************/
_iwl_disable_interrupts(struct iwl_trans * trans)526 static inline void _iwl_disable_interrupts(struct iwl_trans *trans)
527 {
528 struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
529
530 clear_bit(STATUS_INT_ENABLED, &trans->status);
531 if (!trans_pcie->msix_enabled) {
532 /* disable interrupts from uCode/NIC to host */
533 iwl_write32(trans, CSR_INT_MASK, 0x00000000);
534
535 /* acknowledge/clear/reset any interrupts still pending
536 * from uCode or flow handler (Rx/Tx DMA) */
537 iwl_write32(trans, CSR_INT, 0xffffffff);
538 iwl_write32(trans, CSR_FH_INT_STATUS, 0xffffffff);
539 } else {
540 /* disable all the interrupt we might use */
541 iwl_write32(trans, CSR_MSIX_FH_INT_MASK_AD,
542 trans_pcie->fh_init_mask);
543 iwl_write32(trans, CSR_MSIX_HW_INT_MASK_AD,
544 trans_pcie->hw_init_mask);
545 }
546 IWL_DEBUG_ISR(trans, "Disabled interrupts\n");
547 }
548
iwl_pcie_get_num_sections(const struct fw_img * fw,int start)549 static inline int iwl_pcie_get_num_sections(const struct fw_img *fw,
550 int start)
551 {
552 int i = 0;
553
554 while (start < fw->num_sec &&
555 fw->sec[start].offset != CPU1_CPU2_SEPARATOR_SECTION &&
556 fw->sec[start].offset != PAGING_SEPARATOR_SECTION) {
557 start++;
558 i++;
559 }
560
561 return i;
562 }
563
iwl_pcie_ctxt_info_free_fw_img(struct iwl_trans * trans)564 static inline void iwl_pcie_ctxt_info_free_fw_img(struct iwl_trans *trans)
565 {
566 struct iwl_self_init_dram *dram = &trans->init_dram;
567 int i;
568
569 if (!dram->fw) {
570 WARN_ON(dram->fw_cnt);
571 return;
572 }
573
574 for (i = 0; i < dram->fw_cnt; i++)
575 dma_free_coherent(trans->dev, dram->fw[i].size,
576 dram->fw[i].block, dram->fw[i].physical);
577
578 kfree(dram->fw);
579 dram->fw_cnt = 0;
580 dram->fw = NULL;
581 }
582
iwl_disable_interrupts(struct iwl_trans * trans)583 static inline void iwl_disable_interrupts(struct iwl_trans *trans)
584 {
585 struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
586
587 spin_lock_bh(&trans_pcie->irq_lock);
588 _iwl_disable_interrupts(trans);
589 spin_unlock_bh(&trans_pcie->irq_lock);
590 }
591
_iwl_enable_interrupts(struct iwl_trans * trans)592 static inline void _iwl_enable_interrupts(struct iwl_trans *trans)
593 {
594 struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
595
596 IWL_DEBUG_ISR(trans, "Enabling interrupts\n");
597 set_bit(STATUS_INT_ENABLED, &trans->status);
598 if (!trans_pcie->msix_enabled) {
599 trans_pcie->inta_mask = CSR_INI_SET_MASK;
600 iwl_write32(trans, CSR_INT_MASK, trans_pcie->inta_mask);
601 } else {
602 /*
603 * fh/hw_mask keeps all the unmasked causes.
604 * Unlike msi, in msix cause is enabled when it is unset.
605 */
606 trans_pcie->hw_mask = trans_pcie->hw_init_mask;
607 trans_pcie->fh_mask = trans_pcie->fh_init_mask;
608 iwl_write32(trans, CSR_MSIX_FH_INT_MASK_AD,
609 ~trans_pcie->fh_mask);
610 iwl_write32(trans, CSR_MSIX_HW_INT_MASK_AD,
611 ~trans_pcie->hw_mask);
612 }
613 }
614
iwl_enable_interrupts(struct iwl_trans * trans)615 static inline void iwl_enable_interrupts(struct iwl_trans *trans)
616 {
617 struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
618
619 spin_lock_bh(&trans_pcie->irq_lock);
620 _iwl_enable_interrupts(trans);
621 spin_unlock_bh(&trans_pcie->irq_lock);
622 }
iwl_enable_hw_int_msk_msix(struct iwl_trans * trans,u32 msk)623 static inline void iwl_enable_hw_int_msk_msix(struct iwl_trans *trans, u32 msk)
624 {
625 struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
626
627 iwl_write32(trans, CSR_MSIX_HW_INT_MASK_AD, ~msk);
628 trans_pcie->hw_mask = msk;
629 }
630
iwl_enable_fh_int_msk_msix(struct iwl_trans * trans,u32 msk)631 static inline void iwl_enable_fh_int_msk_msix(struct iwl_trans *trans, u32 msk)
632 {
633 struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
634
635 iwl_write32(trans, CSR_MSIX_FH_INT_MASK_AD, ~msk);
636 trans_pcie->fh_mask = msk;
637 }
638
iwl_enable_fw_load_int(struct iwl_trans * trans)639 static inline void iwl_enable_fw_load_int(struct iwl_trans *trans)
640 {
641 struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
642
643 IWL_DEBUG_ISR(trans, "Enabling FW load interrupt\n");
644 if (!trans_pcie->msix_enabled) {
645 trans_pcie->inta_mask = CSR_INT_BIT_FH_TX;
646 iwl_write32(trans, CSR_INT_MASK, trans_pcie->inta_mask);
647 } else {
648 iwl_write32(trans, CSR_MSIX_HW_INT_MASK_AD,
649 trans_pcie->hw_init_mask);
650 iwl_enable_fh_int_msk_msix(trans,
651 MSIX_FH_INT_CAUSES_D2S_CH0_NUM);
652 }
653 }
654
iwl_enable_fw_load_int_ctx_info(struct iwl_trans * trans)655 static inline void iwl_enable_fw_load_int_ctx_info(struct iwl_trans *trans)
656 {
657 struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
658
659 IWL_DEBUG_ISR(trans, "Enabling ALIVE interrupt only\n");
660
661 if (!trans_pcie->msix_enabled) {
662 /*
663 * When we'll receive the ALIVE interrupt, the ISR will call
664 * iwl_enable_fw_load_int_ctx_info again to set the ALIVE
665 * interrupt (which is not really needed anymore) but also the
666 * RX interrupt which will allow us to receive the ALIVE
667 * notification (which is Rx) and continue the flow.
668 */
669 trans_pcie->inta_mask = CSR_INT_BIT_ALIVE | CSR_INT_BIT_FH_RX;
670 iwl_write32(trans, CSR_INT_MASK, trans_pcie->inta_mask);
671 } else {
672 iwl_enable_hw_int_msk_msix(trans,
673 MSIX_HW_INT_CAUSES_REG_ALIVE);
674 /*
675 * Leave all the FH causes enabled to get the ALIVE
676 * notification.
677 */
678 iwl_enable_fh_int_msk_msix(trans, trans_pcie->fh_init_mask);
679 }
680 }
681
queue_name(struct device * dev,struct iwl_trans_pcie * trans_p,int i)682 static inline const char *queue_name(struct device *dev,
683 struct iwl_trans_pcie *trans_p, int i)
684 {
685 if (trans_p->shared_vec_mask) {
686 int vec = trans_p->shared_vec_mask &
687 IWL_SHARED_IRQ_FIRST_RSS ? 1 : 0;
688
689 if (i == 0)
690 return DRV_NAME ":shared_IRQ";
691
692 return devm_kasprintf(dev, GFP_KERNEL,
693 DRV_NAME ":queue_%d", i + vec);
694 }
695 if (i == 0)
696 return DRV_NAME ":default_queue";
697
698 if (i == trans_p->alloc_vecs - 1)
699 return DRV_NAME ":exception";
700
701 return devm_kasprintf(dev, GFP_KERNEL,
702 DRV_NAME ":queue_%d", i);
703 }
704
iwl_enable_rfkill_int(struct iwl_trans * trans)705 static inline void iwl_enable_rfkill_int(struct iwl_trans *trans)
706 {
707 struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
708
709 IWL_DEBUG_ISR(trans, "Enabling rfkill interrupt\n");
710 if (!trans_pcie->msix_enabled) {
711 trans_pcie->inta_mask = CSR_INT_BIT_RF_KILL;
712 iwl_write32(trans, CSR_INT_MASK, trans_pcie->inta_mask);
713 } else {
714 iwl_write32(trans, CSR_MSIX_FH_INT_MASK_AD,
715 trans_pcie->fh_init_mask);
716 iwl_enable_hw_int_msk_msix(trans,
717 MSIX_HW_INT_CAUSES_REG_RF_KILL);
718 }
719
720 if (trans->trans_cfg->device_family >= IWL_DEVICE_FAMILY_9000) {
721 /*
722 * On 9000-series devices this bit isn't enabled by default, so
723 * when we power down the device we need set the bit to allow it
724 * to wake up the PCI-E bus for RF-kill interrupts.
725 */
726 iwl_set_bit(trans, CSR_GP_CNTRL,
727 CSR_GP_CNTRL_REG_FLAG_RFKILL_WAKE_L1A_EN);
728 }
729 }
730
731 void iwl_pcie_handle_rfkill_irq(struct iwl_trans *trans, bool from_irq);
732
iwl_is_rfkill_set(struct iwl_trans * trans)733 static inline bool iwl_is_rfkill_set(struct iwl_trans *trans)
734 {
735 struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
736
737 lockdep_assert_held(&trans_pcie->mutex);
738
739 if (trans_pcie->debug_rfkill == 1)
740 return true;
741
742 return !(iwl_read32(trans, CSR_GP_CNTRL) &
743 CSR_GP_CNTRL_REG_FLAG_HW_RF_KILL_SW);
744 }
745
__iwl_trans_pcie_set_bits_mask(struct iwl_trans * trans,u32 reg,u32 mask,u32 value)746 static inline void __iwl_trans_pcie_set_bits_mask(struct iwl_trans *trans,
747 u32 reg, u32 mask, u32 value)
748 {
749 u32 v;
750
751 #ifdef CONFIG_IWLWIFI_DEBUG
752 WARN_ON_ONCE(value & ~mask);
753 #endif
754
755 v = iwl_read32(trans, reg);
756 v &= ~mask;
757 v |= value;
758 iwl_write32(trans, reg, v);
759 }
760
__iwl_trans_pcie_clear_bit(struct iwl_trans * trans,u32 reg,u32 mask)761 static inline void __iwl_trans_pcie_clear_bit(struct iwl_trans *trans,
762 u32 reg, u32 mask)
763 {
764 __iwl_trans_pcie_set_bits_mask(trans, reg, mask, 0);
765 }
766
__iwl_trans_pcie_set_bit(struct iwl_trans * trans,u32 reg,u32 mask)767 static inline void __iwl_trans_pcie_set_bit(struct iwl_trans *trans,
768 u32 reg, u32 mask)
769 {
770 __iwl_trans_pcie_set_bits_mask(trans, reg, mask, mask);
771 }
772
iwl_pcie_dbg_on(struct iwl_trans * trans)773 static inline bool iwl_pcie_dbg_on(struct iwl_trans *trans)
774 {
775 return (trans->dbg.dest_tlv || iwl_trans_dbg_ini_valid(trans));
776 }
777
778 void iwl_trans_pcie_rf_kill(struct iwl_trans *trans, bool state, bool from_irq);
779 void iwl_trans_pcie_dump_regs(struct iwl_trans *trans);
780
781 #ifdef CONFIG_IWLWIFI_DEBUGFS
782 void iwl_trans_pcie_dbgfs_register(struct iwl_trans *trans);
783 #else
iwl_trans_pcie_dbgfs_register(struct iwl_trans * trans)784 static inline void iwl_trans_pcie_dbgfs_register(struct iwl_trans *trans) { }
785 #endif
786
787 void iwl_pcie_rx_allocator_work(struct work_struct *data);
788
789 /* common functions that are used by gen2 transport */
790 int iwl_pcie_gen2_apm_init(struct iwl_trans *trans);
791 void iwl_pcie_apm_config(struct iwl_trans *trans);
792 int iwl_pcie_prepare_card_hw(struct iwl_trans *trans);
793 void iwl_pcie_synchronize_irqs(struct iwl_trans *trans);
794 bool iwl_pcie_check_hw_rf_kill(struct iwl_trans *trans);
795 void iwl_trans_pcie_handle_stop_rfkill(struct iwl_trans *trans,
796 bool was_in_rfkill);
797 void iwl_pcie_apm_stop_master(struct iwl_trans *trans);
798 void iwl_pcie_conf_msix_hw(struct iwl_trans_pcie *trans_pcie);
799 int iwl_pcie_alloc_dma_ptr(struct iwl_trans *trans,
800 struct iwl_dma_ptr *ptr, size_t size);
801 void iwl_pcie_free_dma_ptr(struct iwl_trans *trans, struct iwl_dma_ptr *ptr);
802 void iwl_pcie_apply_destination(struct iwl_trans *trans);
803
804 /* common functions that are used by gen3 transport */
805 void iwl_pcie_alloc_fw_monitor(struct iwl_trans *trans, u8 max_power);
806
807 /* transport gen 2 exported functions */
808 int iwl_trans_pcie_gen2_start_fw(struct iwl_trans *trans,
809 const struct fw_img *fw, bool run_in_rfkill);
810 void iwl_trans_pcie_gen2_fw_alive(struct iwl_trans *trans, u32 scd_addr);
811 int iwl_trans_pcie_gen2_send_hcmd(struct iwl_trans *trans,
812 struct iwl_host_cmd *cmd);
813 void iwl_trans_pcie_gen2_stop_device(struct iwl_trans *trans);
814 void _iwl_trans_pcie_gen2_stop_device(struct iwl_trans *trans);
815 void iwl_pcie_d3_complete_suspend(struct iwl_trans *trans,
816 bool test, bool reset);
817 int iwl_pcie_gen2_enqueue_hcmd(struct iwl_trans *trans,
818 struct iwl_host_cmd *cmd);
819 int iwl_pcie_enqueue_hcmd(struct iwl_trans *trans,
820 struct iwl_host_cmd *cmd);
821 #endif /* __iwl_trans_int_pcie_h__ */
822