1 /**************************************************************************
2 *
3 * Copyright 2009 VMware, Inc.
4 * All Rights Reserved.
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a
7 * copy of this software and associated documentation files (the
8 * "Software"), to deal in the Software without restriction, including
9 * without limitation the rights to use, copy, modify, merge, publish,
10 * distribute, sub license, and/or sell copies of the Software, and to
11 * permit persons to whom the Software is furnished to do so, subject to
12 * the following conditions:
13 *
14 * The above copyright notice and this permission notice (including the
15 * next paragraph) shall be included in all copies or substantial portions
16 * of the Software.
17 *
18 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
19 * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
20 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT.
21 * IN NO EVENT SHALL VMWARE AND/OR ITS SUPPLIERS BE LIABLE FOR
22 * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
23 * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
24 * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
25 *
26 **************************************************************************/
27
28 /* Helper utility for uploading user buffers & other data, and
29 * coalescing small buffers into larger ones.
30 */
31
32 #include "pipe/p_defines.h"
33 #include "util/u_inlines.h"
34 #include "pipe/p_context.h"
35 #include "util/u_memory.h"
36 #include "util/u_math.h"
37
38 #include "u_upload_mgr.h"
39
40
41 struct u_upload_mgr {
42 struct pipe_context *pipe;
43
44 unsigned default_size; /* Minimum size of the upload buffer, in bytes. */
45 unsigned bind; /* Bitmask of PIPE_BIND_* flags. */
46 enum pipe_resource_usage usage;
47 unsigned flags;
48 unsigned map_flags; /* Bitmask of PIPE_MAP_* flags. */
49 bool map_persistent; /* If persistent mappings are supported. */
50
51 struct pipe_resource *buffer; /* Upload buffer. */
52 struct pipe_transfer *transfer; /* Transfer object for the upload buffer. */
53 uint8_t *map; /* Pointer to the mapped upload buffer. */
54 unsigned buffer_size; /* Same as buffer->width0. */
55 unsigned offset; /* Aligned offset to the upload buffer, pointing
56 * at the first unused byte. */
57 int buffer_private_refcount;
58 };
59
60
61 struct u_upload_mgr *
u_upload_create(struct pipe_context * pipe,unsigned default_size,unsigned bind,enum pipe_resource_usage usage,unsigned flags)62 u_upload_create(struct pipe_context *pipe, unsigned default_size,
63 unsigned bind, enum pipe_resource_usage usage, unsigned flags)
64 {
65 struct u_upload_mgr *upload = CALLOC_STRUCT(u_upload_mgr);
66 if (!upload)
67 return NULL;
68
69 upload->pipe = pipe;
70 upload->default_size = default_size;
71 upload->bind = bind;
72 upload->usage = usage;
73 upload->flags = flags;
74
75 upload->map_persistent =
76 pipe->screen->caps.buffer_map_persistent_coherent;
77
78 if (upload->map_persistent) {
79 upload->map_flags = PIPE_MAP_WRITE |
80 PIPE_MAP_UNSYNCHRONIZED |
81 PIPE_MAP_PERSISTENT |
82 PIPE_MAP_COHERENT;
83 }
84 else {
85 upload->map_flags = PIPE_MAP_WRITE |
86 PIPE_MAP_UNSYNCHRONIZED |
87 PIPE_MAP_FLUSH_EXPLICIT;
88 }
89
90 return upload;
91 }
92
93 struct u_upload_mgr *
u_upload_create_default(struct pipe_context * pipe)94 u_upload_create_default(struct pipe_context *pipe)
95 {
96 return u_upload_create(pipe, 1024 * 1024,
97 PIPE_BIND_VERTEX_BUFFER |
98 PIPE_BIND_INDEX_BUFFER |
99 PIPE_BIND_CONSTANT_BUFFER,
100 PIPE_USAGE_STREAM, 0);
101 }
102
103 struct u_upload_mgr *
u_upload_clone(struct pipe_context * pipe,struct u_upload_mgr * upload)104 u_upload_clone(struct pipe_context *pipe, struct u_upload_mgr *upload)
105 {
106 struct u_upload_mgr *result = u_upload_create(pipe, upload->default_size,
107 upload->bind, upload->usage,
108 upload->flags);
109 if (!upload->map_persistent && result->map_persistent)
110 u_upload_disable_persistent(result);
111
112 return result;
113 }
114
115 void
u_upload_disable_persistent(struct u_upload_mgr * upload)116 u_upload_disable_persistent(struct u_upload_mgr *upload)
117 {
118 upload->map_persistent = false;
119 upload->map_flags &= ~(PIPE_MAP_COHERENT | PIPE_MAP_PERSISTENT);
120 upload->map_flags |= PIPE_MAP_FLUSH_EXPLICIT;
121 }
122
123 static void
upload_unmap_internal(struct u_upload_mgr * upload,bool destroying)124 upload_unmap_internal(struct u_upload_mgr *upload, bool destroying)
125 {
126 if ((!destroying && upload->map_persistent) || !upload->transfer)
127 return;
128
129 struct pipe_box *box = &upload->transfer->box;
130
131 if (!upload->map_persistent && (int) upload->offset > box->x) {
132 pipe_buffer_flush_mapped_range(upload->pipe, upload->transfer,
133 box->x, upload->offset - box->x);
134 }
135
136 pipe_buffer_unmap(upload->pipe, upload->transfer);
137 upload->transfer = NULL;
138 upload->map = NULL;
139 }
140
141
142 void
u_upload_unmap(struct u_upload_mgr * upload)143 u_upload_unmap(struct u_upload_mgr *upload)
144 {
145 upload_unmap_internal(upload, false);
146 }
147
148
149 static void
u_upload_release_buffer(struct u_upload_mgr * upload)150 u_upload_release_buffer(struct u_upload_mgr *upload)
151 {
152 /* Unmap and unreference the upload buffer. */
153 upload_unmap_internal(upload, true);
154 if (upload->buffer_private_refcount) {
155 /* Subtract the remaining private references before unreferencing
156 * the buffer. The mega comment below explains it.
157 */
158 assert(upload->buffer_private_refcount > 0);
159 p_atomic_add(&upload->buffer->reference.count,
160 -upload->buffer_private_refcount);
161 upload->buffer_private_refcount = 0;
162 }
163 pipe_resource_reference(&upload->buffer, NULL);
164 upload->buffer_size = 0;
165 }
166
167
168 void
u_upload_destroy(struct u_upload_mgr * upload)169 u_upload_destroy(struct u_upload_mgr *upload)
170 {
171 u_upload_release_buffer(upload);
172 FREE(upload);
173 }
174
175 /* Return the allocated buffer size or 0 if it failed. */
176 static unsigned
u_upload_alloc_buffer(struct u_upload_mgr * upload,unsigned min_size)177 u_upload_alloc_buffer(struct u_upload_mgr *upload, unsigned min_size)
178 {
179 struct pipe_screen *screen = upload->pipe->screen;
180 struct pipe_resource buffer;
181 unsigned size;
182
183 /* Release the old buffer, if present:
184 */
185 u_upload_release_buffer(upload);
186
187 /* Allocate a new one:
188 */
189 size = align(MAX2(upload->default_size, min_size), 4096);
190
191 memset(&buffer, 0, sizeof buffer);
192 buffer.target = PIPE_BUFFER;
193 buffer.format = PIPE_FORMAT_R8_UNORM; /* want TYPELESS or similar */
194 buffer.bind = upload->bind;
195 buffer.usage = upload->usage;
196 buffer.flags = upload->flags | PIPE_RESOURCE_FLAG_SINGLE_THREAD_USE;
197 buffer.width0 = size;
198 buffer.height0 = 1;
199 buffer.depth0 = 1;
200 buffer.array_size = 1;
201
202 if (upload->map_persistent) {
203 buffer.flags |= PIPE_RESOURCE_FLAG_MAP_PERSISTENT |
204 PIPE_RESOURCE_FLAG_MAP_COHERENT;
205 }
206
207 upload->buffer = screen->resource_create(screen, &buffer);
208 if (upload->buffer == NULL)
209 return 0;
210
211 /* Since atomic operations are very very slow when 2 threads are not
212 * sharing the same L3 cache (which happens on AMD Zen), eliminate all
213 * atomics in u_upload_alloc as follows:
214 *
215 * u_upload_alloc has to return a buffer reference to the caller.
216 * Instead of atomic_inc for every call, it does all possible future
217 * increments in advance here. The maximum number of times u_upload_alloc
218 * can be called per upload buffer is "size", because the minimum
219 * allocation size is 1, thus u_upload_alloc can only return "size" number
220 * of suballocations at most, so we will never need more. This is
221 * the number that is added to reference.count here.
222 *
223 * buffer_private_refcount tracks how many buffer references we can return
224 * without using atomics. If the buffer is full and there are still
225 * references left, they are atomically subtracted from reference.count
226 * before the buffer is unreferenced.
227 *
228 * This technique can increase CPU performance by 10%.
229 *
230 * The caller of u_upload_alloc_buffer will consume min_size bytes,
231 * so init the buffer_private_refcount to 1 + size - min_size, instead
232 * of size to avoid overflowing reference.count when size is huge.
233 */
234 upload->buffer_private_refcount = 1 + (size - min_size);
235 assert(upload->buffer_private_refcount < INT32_MAX / 2);
236 p_atomic_add(&upload->buffer->reference.count, upload->buffer_private_refcount);
237
238 /* Map the new buffer. */
239 upload->map = pipe_buffer_map_range(upload->pipe, upload->buffer,
240 0, size, upload->map_flags,
241 &upload->transfer);
242 if (upload->map == NULL) {
243 u_upload_release_buffer(upload);
244 return 0;
245 }
246
247 upload->buffer_size = size;
248 upload->offset = 0;
249 return size;
250 }
251
252 void
u_upload_alloc(struct u_upload_mgr * upload,unsigned min_out_offset,unsigned size,unsigned alignment,unsigned * out_offset,struct pipe_resource ** outbuf,void ** ptr)253 u_upload_alloc(struct u_upload_mgr *upload,
254 unsigned min_out_offset,
255 unsigned size,
256 unsigned alignment,
257 unsigned *out_offset,
258 struct pipe_resource **outbuf,
259 void **ptr)
260 {
261 unsigned buffer_size = upload->buffer_size;
262 unsigned offset = MAX2(min_out_offset, upload->offset);
263
264 offset = align(offset, alignment);
265
266 /* Make sure we have enough space in the upload buffer
267 * for the sub-allocation.
268 */
269 if (unlikely(offset + size > buffer_size)) {
270 /* Allocate a new buffer and set the offset to the smallest one. */
271 offset = align(min_out_offset, alignment);
272 buffer_size = u_upload_alloc_buffer(upload, offset + size);
273
274 if (unlikely(!buffer_size)) {
275 *out_offset = ~0;
276 pipe_resource_reference(outbuf, NULL);
277 *ptr = NULL;
278 return;
279 }
280 }
281
282 if (unlikely(!upload->map)) {
283 upload->map = pipe_buffer_map_range(upload->pipe, upload->buffer,
284 offset,
285 buffer_size - offset,
286 upload->map_flags,
287 &upload->transfer);
288 if (unlikely(!upload->map)) {
289 upload->transfer = NULL;
290 *out_offset = ~0;
291 pipe_resource_reference(outbuf, NULL);
292 *ptr = NULL;
293 return;
294 }
295
296 upload->map -= offset;
297 }
298
299 assert(offset < buffer_size);
300 assert(offset + size <= buffer_size);
301 assert(size);
302
303 /* Emit the return values: */
304 *ptr = upload->map + offset;
305 *out_offset = offset;
306
307 if (*outbuf != upload->buffer) {
308 pipe_resource_reference(outbuf, NULL);
309 *outbuf = upload->buffer;
310 assert (upload->buffer_private_refcount > 0);
311 upload->buffer_private_refcount--;
312 }
313
314 upload->offset = offset + size;
315 }
316
317 void
u_upload_data(struct u_upload_mgr * upload,unsigned min_out_offset,unsigned size,unsigned alignment,const void * data,unsigned * out_offset,struct pipe_resource ** outbuf)318 u_upload_data(struct u_upload_mgr *upload,
319 unsigned min_out_offset,
320 unsigned size,
321 unsigned alignment,
322 const void *data,
323 unsigned *out_offset,
324 struct pipe_resource **outbuf)
325 {
326 uint8_t *ptr;
327
328 u_upload_alloc(upload, min_out_offset, size, alignment,
329 out_offset, outbuf,
330 (void**)&ptr);
331 if (ptr)
332 memcpy(ptr, data, size);
333 }
334