1 // SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only)
2 /* Copyright(c) 2014 - 2020 Intel Corporation */
3 #include <linux/types.h>
4 #include <linux/mutex.h>
5 #include <linux/slab.h>
6 #include <linux/iopoll.h>
7 #include <linux/pci.h>
8 #include <linux/dma-mapping.h>
9 #include "adf_accel_devices.h"
10 #include "adf_common_drv.h"
11 #include "icp_qat_fw_init_admin.h"
12
13 /* Admin Messages Registers */
14 #define ADF_DH895XCC_ADMINMSGUR_OFFSET (0x3A000 + 0x574)
15 #define ADF_DH895XCC_ADMINMSGLR_OFFSET (0x3A000 + 0x578)
16 #define ADF_DH895XCC_MAILBOX_BASE_OFFSET 0x20970
17 #define ADF_DH895XCC_MAILBOX_STRIDE 0x1000
18 #define ADF_ADMINMSG_LEN 32
19 #define ADF_CONST_TABLE_SIZE 1024
20 #define ADF_ADMIN_POLL_DELAY_US 20
21 #define ADF_ADMIN_POLL_TIMEOUT_US (5 * USEC_PER_SEC)
22
23 static const u8 const_tab[1024] __aligned(1024) = {
24 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
25 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
26 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
27 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
28 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
29 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00,
30 0x00, 0x00, 0x01, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x11, 0x00, 0x00,
31 0x00, 0x00, 0x00, 0x00, 0x00, 0x11, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
32 0x21, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 0x00,
33 0x00, 0x00, 0x00, 0x03, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, 0x01,
34 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00,
35 0x00, 0x13, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x13, 0x02, 0x00, 0x00,
36 0x00, 0x00, 0x00, 0x00, 0x13, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x13,
37 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x23, 0x00, 0x00, 0x00, 0x00, 0x00,
38 0x00, 0x00, 0x33, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x06, 0x00, 0x00,
39 0x00, 0x00, 0x00, 0x00, 0x00, 0x06, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
40 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
41 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
42 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
43 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
44 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
45 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
46 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00,
47 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
48 0x01, 0x23, 0x45, 0x67, 0x89, 0xab, 0xcd, 0xef, 0xfe, 0xdc, 0xba, 0x98, 0x76,
49 0x54, 0x32, 0x10, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
50 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x67, 0x45, 0x23, 0x01, 0xef, 0xcd, 0xab,
51 0x89, 0x98, 0xba, 0xdc, 0xfe, 0x10, 0x32, 0x54, 0x76, 0xc3, 0xd2, 0xe1, 0xf0,
52 0x00, 0x00, 0x00, 0x00, 0x11, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
53 0x00, 0x00, 0x40, 0x00, 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00,
54 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc1, 0x05, 0x9e,
55 0xd8, 0x36, 0x7c, 0xd5, 0x07, 0x30, 0x70, 0xdd, 0x17, 0xf7, 0x0e, 0x59, 0x39,
56 0xff, 0xc0, 0x0b, 0x31, 0x68, 0x58, 0x15, 0x11, 0x64, 0xf9, 0x8f, 0xa7, 0xbe,
57 0xfa, 0x4f, 0xa4, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
58 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x6a, 0x09, 0xe6, 0x67, 0xbb, 0x67, 0xae,
59 0x85, 0x3c, 0x6e, 0xf3, 0x72, 0xa5, 0x4f, 0xf5, 0x3a, 0x51, 0x0e, 0x52, 0x7f,
60 0x9b, 0x05, 0x68, 0x8c, 0x1f, 0x83, 0xd9, 0xab, 0x5b, 0xe0, 0xcd, 0x19, 0x05,
61 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
62 0x00, 0x00, 0xcb, 0xbb, 0x9d, 0x5d, 0xc1, 0x05, 0x9e, 0xd8, 0x62, 0x9a, 0x29,
63 0x2a, 0x36, 0x7c, 0xd5, 0x07, 0x91, 0x59, 0x01, 0x5a, 0x30, 0x70, 0xdd, 0x17,
64 0x15, 0x2f, 0xec, 0xd8, 0xf7, 0x0e, 0x59, 0x39, 0x67, 0x33, 0x26, 0x67, 0xff,
65 0xc0, 0x0b, 0x31, 0x8e, 0xb4, 0x4a, 0x87, 0x68, 0x58, 0x15, 0x11, 0xdb, 0x0c,
66 0x2e, 0x0d, 0x64, 0xf9, 0x8f, 0xa7, 0x47, 0xb5, 0x48, 0x1d, 0xbe, 0xfa, 0x4f,
67 0xa4, 0x06, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
68 0x00, 0x00, 0x00, 0x00, 0x6a, 0x09, 0xe6, 0x67, 0xf3, 0xbc, 0xc9, 0x08, 0xbb,
69 0x67, 0xae, 0x85, 0x84, 0xca, 0xa7, 0x3b, 0x3c, 0x6e, 0xf3, 0x72, 0xfe, 0x94,
70 0xf8, 0x2b, 0xa5, 0x4f, 0xf5, 0x3a, 0x5f, 0x1d, 0x36, 0xf1, 0x51, 0x0e, 0x52,
71 0x7f, 0xad, 0xe6, 0x82, 0xd1, 0x9b, 0x05, 0x68, 0x8c, 0x2b, 0x3e, 0x6c, 0x1f,
72 0x1f, 0x83, 0xd9, 0xab, 0xfb, 0x41, 0xbd, 0x6b, 0x5b, 0xe0, 0xcd, 0x19, 0x13,
73 0x7e, 0x21, 0x79, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
74 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
75 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
76 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
77 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
78 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
79 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
80 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
81 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
82 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
83 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
84 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
85 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
86 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
87 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
88 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
89 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
90 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
91 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
92 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
93 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
94 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
95 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
96 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
97 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
98 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
99 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
100 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
101 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
102 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00};
103
104 struct adf_admin_comms {
105 dma_addr_t phy_addr;
106 dma_addr_t const_tbl_addr;
107 void *virt_addr;
108 void *virt_tbl_addr;
109 void __iomem *mailbox_addr;
110 struct mutex lock; /* protects adf_admin_comms struct */
111 };
112
adf_put_admin_msg_sync(struct adf_accel_dev * accel_dev,u32 ae,void * in,void * out)113 static int adf_put_admin_msg_sync(struct adf_accel_dev *accel_dev, u32 ae,
114 void *in, void *out)
115 {
116 int ret;
117 u32 status;
118 struct adf_admin_comms *admin = accel_dev->admin;
119 int offset = ae * ADF_ADMINMSG_LEN * 2;
120 void __iomem *mailbox = admin->mailbox_addr;
121 int mb_offset = ae * ADF_DH895XCC_MAILBOX_STRIDE;
122 struct icp_qat_fw_init_admin_req *request = in;
123
124 mutex_lock(&admin->lock);
125
126 if (ADF_CSR_RD(mailbox, mb_offset) == 1) {
127 mutex_unlock(&admin->lock);
128 return -EAGAIN;
129 }
130
131 memcpy(admin->virt_addr + offset, in, ADF_ADMINMSG_LEN);
132 ADF_CSR_WR(mailbox, mb_offset, 1);
133
134 ret = read_poll_timeout(ADF_CSR_RD, status, status == 0,
135 ADF_ADMIN_POLL_DELAY_US,
136 ADF_ADMIN_POLL_TIMEOUT_US, true,
137 mailbox, mb_offset);
138 if (ret < 0) {
139 /* Response timeout */
140 dev_err(&GET_DEV(accel_dev),
141 "Failed to send admin msg %d to accelerator %d\n",
142 request->cmd_id, ae);
143 } else {
144 /* Response received from admin message, we can now
145 * make response data available in "out" parameter.
146 */
147 memcpy(out, admin->virt_addr + offset +
148 ADF_ADMINMSG_LEN, ADF_ADMINMSG_LEN);
149 }
150
151 mutex_unlock(&admin->lock);
152 return ret;
153 }
154
adf_send_admin(struct adf_accel_dev * accel_dev,struct icp_qat_fw_init_admin_req * req,struct icp_qat_fw_init_admin_resp * resp,const unsigned long ae_mask)155 static int adf_send_admin(struct adf_accel_dev *accel_dev,
156 struct icp_qat_fw_init_admin_req *req,
157 struct icp_qat_fw_init_admin_resp *resp,
158 const unsigned long ae_mask)
159 {
160 u32 ae;
161
162 for_each_set_bit(ae, &ae_mask, ICP_QAT_HW_AE_DELIMITER)
163 if (adf_put_admin_msg_sync(accel_dev, ae, req, resp) ||
164 resp->status)
165 return -EFAULT;
166
167 return 0;
168 }
169
adf_init_me(struct adf_accel_dev * accel_dev)170 static int adf_init_me(struct adf_accel_dev *accel_dev)
171 {
172 struct icp_qat_fw_init_admin_req req;
173 struct icp_qat_fw_init_admin_resp resp;
174 struct adf_hw_device_data *hw_device = accel_dev->hw_device;
175 u32 ae_mask = hw_device->ae_mask;
176
177 memset(&req, 0, sizeof(req));
178 memset(&resp, 0, sizeof(resp));
179 req.cmd_id = ICP_QAT_FW_INIT_ME;
180
181 return adf_send_admin(accel_dev, &req, &resp, ae_mask);
182 }
183
adf_set_fw_constants(struct adf_accel_dev * accel_dev)184 static int adf_set_fw_constants(struct adf_accel_dev *accel_dev)
185 {
186 struct icp_qat_fw_init_admin_req req;
187 struct icp_qat_fw_init_admin_resp resp;
188 struct adf_hw_device_data *hw_device = accel_dev->hw_device;
189 u32 ae_mask = hw_device->ae_mask;
190
191 memset(&req, 0, sizeof(req));
192 memset(&resp, 0, sizeof(resp));
193 req.cmd_id = ICP_QAT_FW_CONSTANTS_CFG;
194
195 req.init_cfg_sz = ADF_CONST_TABLE_SIZE;
196 req.init_cfg_ptr = accel_dev->admin->const_tbl_addr;
197
198 return adf_send_admin(accel_dev, &req, &resp, ae_mask);
199 }
200
201 /**
202 * adf_send_admin_init() - Function sends init message to FW
203 * @accel_dev: Pointer to acceleration device.
204 *
205 * Function sends admin init message to the FW
206 *
207 * Return: 0 on success, error code otherwise.
208 */
adf_send_admin_init(struct adf_accel_dev * accel_dev)209 int adf_send_admin_init(struct adf_accel_dev *accel_dev)
210 {
211 int ret;
212
213 ret = adf_init_me(accel_dev);
214 if (ret)
215 return ret;
216
217 return adf_set_fw_constants(accel_dev);
218 }
219 EXPORT_SYMBOL_GPL(adf_send_admin_init);
220
adf_init_admin_comms(struct adf_accel_dev * accel_dev)221 int adf_init_admin_comms(struct adf_accel_dev *accel_dev)
222 {
223 struct adf_admin_comms *admin;
224 struct adf_hw_device_data *hw_data = accel_dev->hw_device;
225 struct adf_bar *pmisc =
226 &GET_BARS(accel_dev)[hw_data->get_misc_bar_id(hw_data)];
227 void __iomem *csr = pmisc->virt_addr;
228 void __iomem *mailbox = (void __iomem *)((uintptr_t)csr +
229 ADF_DH895XCC_MAILBOX_BASE_OFFSET);
230 u64 reg_val;
231
232 admin = kzalloc_node(sizeof(*accel_dev->admin), GFP_KERNEL,
233 dev_to_node(&GET_DEV(accel_dev)));
234 if (!admin)
235 return -ENOMEM;
236 admin->virt_addr = dma_alloc_coherent(&GET_DEV(accel_dev), PAGE_SIZE,
237 &admin->phy_addr, GFP_KERNEL);
238 if (!admin->virt_addr) {
239 dev_err(&GET_DEV(accel_dev), "Failed to allocate dma buff\n");
240 kfree(admin);
241 return -ENOMEM;
242 }
243
244 admin->virt_tbl_addr = dma_alloc_coherent(&GET_DEV(accel_dev),
245 PAGE_SIZE,
246 &admin->const_tbl_addr,
247 GFP_KERNEL);
248 if (!admin->virt_tbl_addr) {
249 dev_err(&GET_DEV(accel_dev), "Failed to allocate const_tbl\n");
250 dma_free_coherent(&GET_DEV(accel_dev), PAGE_SIZE,
251 admin->virt_addr, admin->phy_addr);
252 kfree(admin);
253 return -ENOMEM;
254 }
255
256 memcpy(admin->virt_tbl_addr, const_tab, sizeof(const_tab));
257 reg_val = (u64)admin->phy_addr;
258 ADF_CSR_WR(csr, ADF_DH895XCC_ADMINMSGUR_OFFSET, reg_val >> 32);
259 ADF_CSR_WR(csr, ADF_DH895XCC_ADMINMSGLR_OFFSET, reg_val);
260 mutex_init(&admin->lock);
261 admin->mailbox_addr = mailbox;
262 accel_dev->admin = admin;
263 return 0;
264 }
265 EXPORT_SYMBOL_GPL(adf_init_admin_comms);
266
adf_exit_admin_comms(struct adf_accel_dev * accel_dev)267 void adf_exit_admin_comms(struct adf_accel_dev *accel_dev)
268 {
269 struct adf_admin_comms *admin = accel_dev->admin;
270
271 if (!admin)
272 return;
273
274 if (admin->virt_addr)
275 dma_free_coherent(&GET_DEV(accel_dev), PAGE_SIZE,
276 admin->virt_addr, admin->phy_addr);
277 if (admin->virt_tbl_addr)
278 dma_free_coherent(&GET_DEV(accel_dev), PAGE_SIZE,
279 admin->virt_tbl_addr, admin->const_tbl_addr);
280
281 mutex_destroy(&admin->lock);
282 kfree(admin);
283 accel_dev->admin = NULL;
284 }
285 EXPORT_SYMBOL_GPL(adf_exit_admin_comms);
286