1 // SPDX-License-Identifier: GPL-2.0
2
3 /*
4 * Copyright 2016-2019 HabanaLabs, Ltd.
5 * All Rights Reserved.
6 */
7
8 #include "habanalabs.h"
9 #include "include/hw_ip/pci/pci_general.h"
10
11 #include <linux/pci.h>
12
13 #define HL_PLDM_PCI_ELBI_TIMEOUT_MSEC (HL_PCI_ELBI_TIMEOUT_MSEC * 10)
14
15 /**
16 * hl_pci_bars_map() - Map PCI BARs.
17 * @hdev: Pointer to hl_device structure.
18 * @bar_name: Array of BAR names.
19 * @is_wc: Array with flag per BAR whether a write-combined mapping is needed.
20 *
21 * Request PCI regions and map them to kernel virtual addresses.
22 *
23 * Return: 0 on success, non-zero for failure.
24 */
hl_pci_bars_map(struct hl_device * hdev,const char * const name[3],bool is_wc[3])25 int hl_pci_bars_map(struct hl_device *hdev, const char * const name[3],
26 bool is_wc[3])
27 {
28 struct pci_dev *pdev = hdev->pdev;
29 int rc, i, bar;
30
31 rc = pci_request_regions(pdev, HL_NAME);
32 if (rc) {
33 dev_err(hdev->dev, "Cannot obtain PCI resources\n");
34 return rc;
35 }
36
37 for (i = 0 ; i < 3 ; i++) {
38 bar = i * 2; /* 64-bit BARs */
39 hdev->pcie_bar[bar] = is_wc[i] ?
40 pci_ioremap_wc_bar(pdev, bar) :
41 pci_ioremap_bar(pdev, bar);
42 if (!hdev->pcie_bar[bar]) {
43 dev_err(hdev->dev, "pci_ioremap%s_bar failed for %s\n",
44 is_wc[i] ? "_wc" : "", name[i]);
45 rc = -ENODEV;
46 goto err;
47 }
48 }
49
50 return 0;
51
52 err:
53 for (i = 2 ; i >= 0 ; i--) {
54 bar = i * 2; /* 64-bit BARs */
55 if (hdev->pcie_bar[bar])
56 iounmap(hdev->pcie_bar[bar]);
57 }
58
59 pci_release_regions(pdev);
60
61 return rc;
62 }
63
64 /*
65 * hl_pci_bars_unmap() - Unmap PCI BARS.
66 * @hdev: Pointer to hl_device structure.
67 *
68 * Release all PCI BARs and unmap their virtual addresses.
69 */
hl_pci_bars_unmap(struct hl_device * hdev)70 static void hl_pci_bars_unmap(struct hl_device *hdev)
71 {
72 struct pci_dev *pdev = hdev->pdev;
73 int i, bar;
74
75 for (i = 2 ; i >= 0 ; i--) {
76 bar = i * 2; /* 64-bit BARs */
77 iounmap(hdev->pcie_bar[bar]);
78 }
79
80 pci_release_regions(pdev);
81 }
82
83 /*
84 * hl_pci_elbi_write() - Write through the ELBI interface.
85 * @hdev: Pointer to hl_device structure.
86 *
87 * Return: 0 on success, negative value for failure.
88 */
hl_pci_elbi_write(struct hl_device * hdev,u64 addr,u32 data)89 static int hl_pci_elbi_write(struct hl_device *hdev, u64 addr, u32 data)
90 {
91 struct pci_dev *pdev = hdev->pdev;
92 ktime_t timeout;
93 u64 msec;
94 u32 val;
95
96 if (hdev->pldm)
97 msec = HL_PLDM_PCI_ELBI_TIMEOUT_MSEC;
98 else
99 msec = HL_PCI_ELBI_TIMEOUT_MSEC;
100
101 /* Clear previous status */
102 pci_write_config_dword(pdev, mmPCI_CONFIG_ELBI_STS, 0);
103
104 pci_write_config_dword(pdev, mmPCI_CONFIG_ELBI_ADDR, (u32) addr);
105 pci_write_config_dword(pdev, mmPCI_CONFIG_ELBI_DATA, data);
106 pci_write_config_dword(pdev, mmPCI_CONFIG_ELBI_CTRL,
107 PCI_CONFIG_ELBI_CTRL_WRITE);
108
109 timeout = ktime_add_ms(ktime_get(), msec);
110 for (;;) {
111 pci_read_config_dword(pdev, mmPCI_CONFIG_ELBI_STS, &val);
112 if (val & PCI_CONFIG_ELBI_STS_MASK)
113 break;
114 if (ktime_compare(ktime_get(), timeout) > 0) {
115 pci_read_config_dword(pdev, mmPCI_CONFIG_ELBI_STS,
116 &val);
117 break;
118 }
119
120 usleep_range(300, 500);
121 }
122
123 if ((val & PCI_CONFIG_ELBI_STS_MASK) == PCI_CONFIG_ELBI_STS_DONE)
124 return 0;
125
126 if (val & PCI_CONFIG_ELBI_STS_ERR) {
127 dev_err(hdev->dev, "Error writing to ELBI\n");
128 return -EIO;
129 }
130
131 if (!(val & PCI_CONFIG_ELBI_STS_MASK)) {
132 dev_err(hdev->dev, "ELBI write didn't finish in time\n");
133 return -EIO;
134 }
135
136 dev_err(hdev->dev, "ELBI write has undefined bits in status\n");
137 return -EIO;
138 }
139
140 /**
141 * hl_pci_iatu_write() - iatu write routine.
142 * @hdev: Pointer to hl_device structure.
143 *
144 * Return: 0 on success, negative value for failure.
145 */
hl_pci_iatu_write(struct hl_device * hdev,u32 addr,u32 data)146 int hl_pci_iatu_write(struct hl_device *hdev, u32 addr, u32 data)
147 {
148 struct asic_fixed_properties *prop = &hdev->asic_prop;
149 u32 dbi_offset;
150 int rc;
151
152 dbi_offset = addr & 0xFFF;
153
154 rc = hl_pci_elbi_write(hdev, prop->pcie_aux_dbi_reg_addr, 0x00300000);
155 rc |= hl_pci_elbi_write(hdev, prop->pcie_dbi_base_address + dbi_offset,
156 data);
157
158 if (rc)
159 return -EIO;
160
161 return 0;
162 }
163
164 /*
165 * hl_pci_reset_link_through_bridge() - Reset PCI link.
166 * @hdev: Pointer to hl_device structure.
167 */
hl_pci_reset_link_through_bridge(struct hl_device * hdev)168 static void hl_pci_reset_link_through_bridge(struct hl_device *hdev)
169 {
170 struct pci_dev *pdev = hdev->pdev;
171 struct pci_dev *parent_port;
172 u16 val;
173
174 parent_port = pdev->bus->self;
175 pci_read_config_word(parent_port, PCI_BRIDGE_CONTROL, &val);
176 val |= PCI_BRIDGE_CTL_BUS_RESET;
177 pci_write_config_word(parent_port, PCI_BRIDGE_CONTROL, val);
178 ssleep(1);
179
180 val &= ~(PCI_BRIDGE_CTL_BUS_RESET);
181 pci_write_config_word(parent_port, PCI_BRIDGE_CONTROL, val);
182 ssleep(3);
183 }
184
185 /**
186 * hl_pci_set_dram_bar_base() - Set DDR BAR to map specific device address.
187 * @hdev: Pointer to hl_device structure.
188 * @inbound_region: Inbound region number.
189 * @bar: PCI BAR number.
190 * @addr: Address in DRAM. Must be aligned to DRAM bar size.
191 *
192 * Configure the iATU so that the DRAM bar will start at the specified address.
193 *
194 * Return: 0 on success, negative value for failure.
195 */
hl_pci_set_dram_bar_base(struct hl_device * hdev,u8 inbound_region,u8 bar,u64 addr)196 int hl_pci_set_dram_bar_base(struct hl_device *hdev, u8 inbound_region, u8 bar,
197 u64 addr)
198 {
199 struct asic_fixed_properties *prop = &hdev->asic_prop;
200 u32 offset;
201 int rc;
202
203 switch (inbound_region) {
204 case 0:
205 offset = 0x100;
206 break;
207 case 1:
208 offset = 0x300;
209 break;
210 case 2:
211 offset = 0x500;
212 break;
213 default:
214 dev_err(hdev->dev, "Invalid inbound region %d\n",
215 inbound_region);
216 return -EINVAL;
217 }
218
219 if (bar != 0 && bar != 2 && bar != 4) {
220 dev_err(hdev->dev, "Invalid PCI BAR %d\n", bar);
221 return -EINVAL;
222 }
223
224 /* Point to the specified address */
225 rc = hl_pci_iatu_write(hdev, offset + 0x14, lower_32_bits(addr));
226 rc |= hl_pci_iatu_write(hdev, offset + 0x18, upper_32_bits(addr));
227 rc |= hl_pci_iatu_write(hdev, offset + 0x0, 0);
228 /* Enable + BAR match + match enable + BAR number */
229 rc |= hl_pci_iatu_write(hdev, offset + 0x4, 0xC0080000 | (bar << 8));
230
231 /* Return the DBI window to the default location */
232 rc |= hl_pci_elbi_write(hdev, prop->pcie_aux_dbi_reg_addr, 0);
233 rc |= hl_pci_elbi_write(hdev, prop->pcie_aux_dbi_reg_addr + 4, 0);
234
235 if (rc)
236 dev_err(hdev->dev, "failed to map DRAM bar to 0x%08llx\n",
237 addr);
238
239 return rc;
240 }
241
242 /**
243 * hl_pci_init_iatu() - Initialize the iATU unit inside the PCI controller.
244 * @hdev: Pointer to hl_device structure.
245 * @sram_base_address: SRAM base address.
246 * @dram_base_address: DRAM base address.
247 * @host_phys_base_address: Base physical address of host memory for device
248 * transactions.
249 * @host_phys_size: Size of host memory for device transactions.
250 *
251 * This is needed in case the firmware doesn't initialize the iATU.
252 *
253 * Return: 0 on success, negative value for failure.
254 */
hl_pci_init_iatu(struct hl_device * hdev,u64 sram_base_address,u64 dram_base_address,u64 host_phys_base_address,u64 host_phys_size)255 int hl_pci_init_iatu(struct hl_device *hdev, u64 sram_base_address,
256 u64 dram_base_address, u64 host_phys_base_address,
257 u64 host_phys_size)
258 {
259 struct asic_fixed_properties *prop = &hdev->asic_prop;
260 u64 host_phys_end_addr;
261 int rc = 0;
262
263 /* Inbound Region 0 - Bar 0 - Point to SRAM base address */
264 rc = hl_pci_iatu_write(hdev, 0x114, lower_32_bits(sram_base_address));
265 rc |= hl_pci_iatu_write(hdev, 0x118, upper_32_bits(sram_base_address));
266 rc |= hl_pci_iatu_write(hdev, 0x100, 0);
267 /* Enable + Bar match + match enable */
268 rc |= hl_pci_iatu_write(hdev, 0x104, 0xC0080000);
269
270 /* Point to DRAM */
271 if (!hdev->asic_funcs->set_dram_bar_base)
272 return -EINVAL;
273 if (hdev->asic_funcs->set_dram_bar_base(hdev, dram_base_address) ==
274 U64_MAX)
275 return -EIO;
276
277
278 /* Outbound Region 0 - Point to Host */
279 host_phys_end_addr = host_phys_base_address + host_phys_size - 1;
280 rc |= hl_pci_iatu_write(hdev, 0x008,
281 lower_32_bits(host_phys_base_address));
282 rc |= hl_pci_iatu_write(hdev, 0x00C,
283 upper_32_bits(host_phys_base_address));
284 rc |= hl_pci_iatu_write(hdev, 0x010, lower_32_bits(host_phys_end_addr));
285 rc |= hl_pci_iatu_write(hdev, 0x014, 0);
286 rc |= hl_pci_iatu_write(hdev, 0x018, 0);
287 rc |= hl_pci_iatu_write(hdev, 0x020, upper_32_bits(host_phys_end_addr));
288 /* Increase region size */
289 rc |= hl_pci_iatu_write(hdev, 0x000, 0x00002000);
290 /* Enable */
291 rc |= hl_pci_iatu_write(hdev, 0x004, 0x80000000);
292
293 /* Return the DBI window to the default location */
294 rc |= hl_pci_elbi_write(hdev, prop->pcie_aux_dbi_reg_addr, 0);
295 rc |= hl_pci_elbi_write(hdev, prop->pcie_aux_dbi_reg_addr + 4, 0);
296
297 if (rc)
298 return -EIO;
299
300 return 0;
301 }
302
303 /**
304 * hl_pci_set_dma_mask() - Set DMA masks for the device.
305 * @hdev: Pointer to hl_device structure.
306 * @dma_mask: number of bits for the requested dma mask.
307 *
308 * This function sets the DMA masks (regular and consistent) for a specified
309 * value. If it doesn't succeed, it tries to set it to a fall-back value
310 *
311 * Return: 0 on success, non-zero for failure.
312 */
hl_pci_set_dma_mask(struct hl_device * hdev,u8 dma_mask)313 int hl_pci_set_dma_mask(struct hl_device *hdev, u8 dma_mask)
314 {
315 struct pci_dev *pdev = hdev->pdev;
316 int rc;
317
318 /* set DMA mask */
319 rc = pci_set_dma_mask(pdev, DMA_BIT_MASK(dma_mask));
320 if (rc) {
321 dev_warn(hdev->dev,
322 "Failed to set pci dma mask to %d bits, error %d\n",
323 dma_mask, rc);
324
325 dma_mask = hdev->dma_mask;
326
327 rc = pci_set_dma_mask(pdev, DMA_BIT_MASK(dma_mask));
328 if (rc) {
329 dev_err(hdev->dev,
330 "Failed to set pci dma mask to %d bits, error %d\n",
331 dma_mask, rc);
332 return rc;
333 }
334 }
335
336 /*
337 * We managed to set the dma mask, so update the dma mask field. If
338 * the set to the coherent mask will fail with that mask, we will
339 * fail the entire function
340 */
341 hdev->dma_mask = dma_mask;
342
343 rc = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(dma_mask));
344 if (rc) {
345 dev_err(hdev->dev,
346 "Failed to set pci consistent dma mask to %d bits, error %d\n",
347 dma_mask, rc);
348 return rc;
349 }
350
351 return 0;
352 }
353
354 /**
355 * hl_pci_init() - PCI initialization code.
356 * @hdev: Pointer to hl_device structure.
357 * @dma_mask: number of bits for the requested dma mask.
358 *
359 * Set DMA masks, initialize the PCI controller and map the PCI BARs.
360 *
361 * Return: 0 on success, non-zero for failure.
362 */
hl_pci_init(struct hl_device * hdev,u8 dma_mask)363 int hl_pci_init(struct hl_device *hdev, u8 dma_mask)
364 {
365 struct pci_dev *pdev = hdev->pdev;
366 int rc;
367
368 rc = hl_pci_set_dma_mask(hdev, dma_mask);
369 if (rc)
370 return rc;
371
372 if (hdev->reset_pcilink)
373 hl_pci_reset_link_through_bridge(hdev);
374
375 rc = pci_enable_device_mem(pdev);
376 if (rc) {
377 dev_err(hdev->dev, "can't enable PCI device\n");
378 return rc;
379 }
380
381 pci_set_master(pdev);
382
383 rc = hdev->asic_funcs->init_iatu(hdev);
384 if (rc) {
385 dev_err(hdev->dev, "Failed to initialize iATU\n");
386 goto disable_device;
387 }
388
389 rc = hdev->asic_funcs->pci_bars_map(hdev);
390 if (rc) {
391 dev_err(hdev->dev, "Failed to initialize PCI BARs\n");
392 goto disable_device;
393 }
394
395 return 0;
396
397 disable_device:
398 pci_clear_master(pdev);
399 pci_disable_device(pdev);
400
401 return rc;
402 }
403
404 /**
405 * hl_fw_fini() - PCI finalization code.
406 * @hdev: Pointer to hl_device structure
407 *
408 * Unmap PCI bars and disable PCI device.
409 */
hl_pci_fini(struct hl_device * hdev)410 void hl_pci_fini(struct hl_device *hdev)
411 {
412 hl_pci_bars_unmap(hdev);
413
414 pci_clear_master(hdev->pdev);
415 pci_disable_device(hdev->pdev);
416 }
417