1 // SPDX-License-Identifier: GPL-2.0
2 /* Copyright (c) 2019 HiSilicon Limited. */
3 #include <linux/dma-mapping.h>
4 #include <linux/module.h>
5 #include <linux/slab.h>
6 #include "qm.h"
7
8 #define HISI_ACC_SGL_SGE_NR_MIN 1
9 #define HISI_ACC_SGL_NR_MAX 256
10 #define HISI_ACC_SGL_ALIGN_SIZE 64
11 #define HISI_ACC_MEM_BLOCK_NR 5
12
13 struct acc_hw_sge {
14 dma_addr_t buf;
15 void *page_ctrl;
16 __le32 len;
17 __le32 pad;
18 __le32 pad0;
19 __le32 pad1;
20 };
21
22 /* use default sgl head size 64B */
23 struct hisi_acc_hw_sgl {
24 dma_addr_t next_dma;
25 __le16 entry_sum_in_chain;
26 __le16 entry_sum_in_sgl;
27 __le16 entry_length_in_sgl;
28 __le16 pad0;
29 __le64 pad1[5];
30 struct hisi_acc_hw_sgl *next;
31 struct acc_hw_sge sge_entries[];
32 } __aligned(1);
33
34 struct hisi_acc_sgl_pool {
35 struct mem_block {
36 struct hisi_acc_hw_sgl *sgl;
37 dma_addr_t sgl_dma;
38 size_t size;
39 } mem_block[HISI_ACC_MEM_BLOCK_NR];
40 u32 sgl_num_per_block;
41 u32 block_num;
42 u32 count;
43 u32 sge_nr;
44 size_t sgl_size;
45 };
46
47 /**
48 * hisi_acc_create_sgl_pool() - Create a hw sgl pool.
49 * @dev: The device which hw sgl pool belongs to.
50 * @count: Count of hisi_acc_hw_sgl in pool.
51 * @sge_nr: The count of sge in hw_sgl
52 *
53 * This function creates a hw sgl pool, after this user can get hw sgl memory
54 * from it.
55 */
hisi_acc_create_sgl_pool(struct device * dev,u32 count,u32 sge_nr)56 struct hisi_acc_sgl_pool *hisi_acc_create_sgl_pool(struct device *dev,
57 u32 count, u32 sge_nr)
58 {
59 u32 sgl_size, block_size, sgl_num_per_block, block_num, remain_sgl = 0;
60 struct hisi_acc_sgl_pool *pool;
61 struct mem_block *block;
62 u32 i, j;
63
64 if (!dev || !count || !sge_nr || sge_nr > HISI_ACC_SGL_SGE_NR_MAX)
65 return ERR_PTR(-EINVAL);
66
67 sgl_size = sizeof(struct acc_hw_sge) * sge_nr +
68 sizeof(struct hisi_acc_hw_sgl);
69 block_size = 1 << (PAGE_SHIFT + MAX_ORDER <= 32 ?
70 PAGE_SHIFT + MAX_ORDER - 1 : 31);
71 sgl_num_per_block = block_size / sgl_size;
72 block_num = count / sgl_num_per_block;
73 remain_sgl = count % sgl_num_per_block;
74
75 if ((!remain_sgl && block_num > HISI_ACC_MEM_BLOCK_NR) ||
76 (remain_sgl > 0 && block_num > HISI_ACC_MEM_BLOCK_NR - 1))
77 return ERR_PTR(-EINVAL);
78
79 pool = kzalloc(sizeof(*pool), GFP_KERNEL);
80 if (!pool)
81 return ERR_PTR(-ENOMEM);
82 block = pool->mem_block;
83
84 for (i = 0; i < block_num; i++) {
85 block[i].sgl = dma_alloc_coherent(dev, block_size,
86 &block[i].sgl_dma,
87 GFP_KERNEL);
88 if (!block[i].sgl)
89 goto err_free_mem;
90
91 block[i].size = block_size;
92 }
93
94 if (remain_sgl > 0) {
95 block[i].sgl = dma_alloc_coherent(dev, remain_sgl * sgl_size,
96 &block[i].sgl_dma,
97 GFP_KERNEL);
98 if (!block[i].sgl)
99 goto err_free_mem;
100
101 block[i].size = remain_sgl * sgl_size;
102 }
103
104 pool->sgl_num_per_block = sgl_num_per_block;
105 pool->block_num = remain_sgl ? block_num + 1 : block_num;
106 pool->count = count;
107 pool->sgl_size = sgl_size;
108 pool->sge_nr = sge_nr;
109
110 return pool;
111
112 err_free_mem:
113 for (j = 0; j < i; j++) {
114 dma_free_coherent(dev, block_size, block[j].sgl,
115 block[j].sgl_dma);
116 memset(block + j, 0, sizeof(*block));
117 }
118 kfree(pool);
119 return ERR_PTR(-ENOMEM);
120 }
121 EXPORT_SYMBOL_GPL(hisi_acc_create_sgl_pool);
122
123 /**
124 * hisi_acc_free_sgl_pool() - Free a hw sgl pool.
125 * @dev: The device which hw sgl pool belongs to.
126 * @pool: Pointer of pool.
127 *
128 * This function frees memory of a hw sgl pool.
129 */
hisi_acc_free_sgl_pool(struct device * dev,struct hisi_acc_sgl_pool * pool)130 void hisi_acc_free_sgl_pool(struct device *dev, struct hisi_acc_sgl_pool *pool)
131 {
132 struct mem_block *block;
133 int i;
134
135 if (!dev || !pool)
136 return;
137
138 block = pool->mem_block;
139
140 for (i = 0; i < pool->block_num; i++)
141 dma_free_coherent(dev, block[i].size, block[i].sgl,
142 block[i].sgl_dma);
143
144 kfree(pool);
145 }
146 EXPORT_SYMBOL_GPL(hisi_acc_free_sgl_pool);
147
acc_get_sgl(struct hisi_acc_sgl_pool * pool,u32 index,dma_addr_t * hw_sgl_dma)148 static struct hisi_acc_hw_sgl *acc_get_sgl(struct hisi_acc_sgl_pool *pool,
149 u32 index, dma_addr_t *hw_sgl_dma)
150 {
151 struct mem_block *block;
152 u32 block_index, offset;
153
154 if (!pool || !hw_sgl_dma || index >= pool->count)
155 return ERR_PTR(-EINVAL);
156
157 block = pool->mem_block;
158 block_index = index / pool->sgl_num_per_block;
159 offset = index % pool->sgl_num_per_block;
160
161 *hw_sgl_dma = block[block_index].sgl_dma + pool->sgl_size * offset;
162 return (void *)block[block_index].sgl + pool->sgl_size * offset;
163 }
164
sg_map_to_hw_sg(struct scatterlist * sgl,struct acc_hw_sge * hw_sge)165 static void sg_map_to_hw_sg(struct scatterlist *sgl,
166 struct acc_hw_sge *hw_sge)
167 {
168 hw_sge->buf = sg_dma_address(sgl);
169 hw_sge->len = cpu_to_le32(sg_dma_len(sgl));
170 }
171
inc_hw_sgl_sge(struct hisi_acc_hw_sgl * hw_sgl)172 static void inc_hw_sgl_sge(struct hisi_acc_hw_sgl *hw_sgl)
173 {
174 u16 var = le16_to_cpu(hw_sgl->entry_sum_in_sgl);
175
176 var++;
177 hw_sgl->entry_sum_in_sgl = cpu_to_le16(var);
178 }
179
update_hw_sgl_sum_sge(struct hisi_acc_hw_sgl * hw_sgl,u16 sum)180 static void update_hw_sgl_sum_sge(struct hisi_acc_hw_sgl *hw_sgl, u16 sum)
181 {
182 hw_sgl->entry_sum_in_chain = cpu_to_le16(sum);
183 }
184
185 /**
186 * hisi_acc_sg_buf_map_to_hw_sgl - Map a scatterlist to a hw sgl.
187 * @dev: The device which hw sgl belongs to.
188 * @sgl: Scatterlist which will be mapped to hw sgl.
189 * @pool: Pool which hw sgl memory will be allocated in.
190 * @index: Index of hisi_acc_hw_sgl in pool.
191 * @hw_sgl_dma: The dma address of allocated hw sgl.
192 *
193 * This function builds hw sgl according input sgl, user can use hw_sgl_dma
194 * as src/dst in its BD. Only support single hw sgl currently.
195 */
196 struct hisi_acc_hw_sgl *
hisi_acc_sg_buf_map_to_hw_sgl(struct device * dev,struct scatterlist * sgl,struct hisi_acc_sgl_pool * pool,u32 index,dma_addr_t * hw_sgl_dma)197 hisi_acc_sg_buf_map_to_hw_sgl(struct device *dev,
198 struct scatterlist *sgl,
199 struct hisi_acc_sgl_pool *pool,
200 u32 index, dma_addr_t *hw_sgl_dma)
201 {
202 struct hisi_acc_hw_sgl *curr_hw_sgl;
203 dma_addr_t curr_sgl_dma = 0;
204 struct acc_hw_sge *curr_hw_sge;
205 struct scatterlist *sg;
206 int i, sg_n, sg_n_mapped;
207
208 if (!dev || !sgl || !pool || !hw_sgl_dma)
209 return ERR_PTR(-EINVAL);
210
211 sg_n = sg_nents(sgl);
212
213 sg_n_mapped = dma_map_sg(dev, sgl, sg_n, DMA_BIDIRECTIONAL);
214 if (!sg_n_mapped)
215 return ERR_PTR(-EINVAL);
216
217 if (sg_n_mapped > pool->sge_nr) {
218 dma_unmap_sg(dev, sgl, sg_n, DMA_BIDIRECTIONAL);
219 return ERR_PTR(-EINVAL);
220 }
221
222 curr_hw_sgl = acc_get_sgl(pool, index, &curr_sgl_dma);
223 if (IS_ERR(curr_hw_sgl)) {
224 dma_unmap_sg(dev, sgl, sg_n, DMA_BIDIRECTIONAL);
225 return ERR_PTR(-ENOMEM);
226
227 }
228 curr_hw_sgl->entry_length_in_sgl = cpu_to_le16(pool->sge_nr);
229 curr_hw_sge = curr_hw_sgl->sge_entries;
230
231 for_each_sg(sgl, sg, sg_n_mapped, i) {
232 sg_map_to_hw_sg(sg, curr_hw_sge);
233 inc_hw_sgl_sge(curr_hw_sgl);
234 curr_hw_sge++;
235 }
236
237 update_hw_sgl_sum_sge(curr_hw_sgl, pool->sge_nr);
238 *hw_sgl_dma = curr_sgl_dma;
239
240 return curr_hw_sgl;
241 }
242 EXPORT_SYMBOL_GPL(hisi_acc_sg_buf_map_to_hw_sgl);
243
244 /**
245 * hisi_acc_sg_buf_unmap() - Unmap allocated hw sgl.
246 * @dev: The device which hw sgl belongs to.
247 * @sgl: Related scatterlist.
248 * @hw_sgl: Virtual address of hw sgl.
249 * @hw_sgl_dma: DMA address of hw sgl.
250 * @pool: Pool which hw sgl is allocated in.
251 *
252 * This function unmaps allocated hw sgl.
253 */
hisi_acc_sg_buf_unmap(struct device * dev,struct scatterlist * sgl,struct hisi_acc_hw_sgl * hw_sgl)254 void hisi_acc_sg_buf_unmap(struct device *dev, struct scatterlist *sgl,
255 struct hisi_acc_hw_sgl *hw_sgl)
256 {
257 if (!dev || !sgl || !hw_sgl)
258 return;
259
260 dma_unmap_sg(dev, sgl, sg_nents(sgl), DMA_BIDIRECTIONAL);
261
262 hw_sgl->entry_sum_in_chain = 0;
263 hw_sgl->entry_sum_in_sgl = 0;
264 hw_sgl->entry_length_in_sgl = 0;
265 }
266 EXPORT_SYMBOL_GPL(hisi_acc_sg_buf_unmap);
267