1 /*
2 * Copyright 2021 Google LLC
3 * SPDX-License-Identifier: MIT
4 */
5 #include "CommandBufferStagingStream.h"
6
7 #include <errno.h>
8 #include <stdio.h>
9 #include <stdlib.h>
10 #include <string.h>
11 #include <unistd.h>
12
13 #include <atomic>
14 #include <vector>
15
16 #include "util/log.h"
17
18 static const size_t kReadSize = 512 * 1024;
19 static const size_t kWriteOffset = kReadSize;
20
21 namespace gfxstream {
22 namespace vk {
23
CommandBufferStagingStream()24 CommandBufferStagingStream::CommandBufferStagingStream()
25 : IOStream(1048576), m_size(0), m_writePos(0) {
26 // use default allocators
27 m_alloc = [](size_t size) -> Memory {
28 return {
29 .deviceMemory = VK_NULL_HANDLE, // no device memory for malloc
30 .ptr = malloc(size),
31 };
32 };
33 m_free = [](const Memory& mem) { free(mem.ptr); };
34 m_realloc = [](const Memory& mem, size_t size) -> Memory {
35 return {.deviceMemory = VK_NULL_HANDLE, .ptr = realloc(mem.ptr, size)};
36 };
37 }
38
CommandBufferStagingStream(const Alloc & allocFn,const Free & freeFn)39 CommandBufferStagingStream::CommandBufferStagingStream(const Alloc& allocFn, const Free& freeFn)
40 : CommandBufferStagingStream() {
41 m_usingCustomAlloc = true;
42 // for custom allocation, allocate metadata memory at the beginning.
43 // m_alloc, m_free and m_realloc wraps sync data logic
44
45 // \param size to allocate
46 // \return ptr starting at data
47 m_alloc = [&allocFn](size_t size) -> Memory {
48 // allocation requested size + sync data size
49
50 // <---sync bytes--><----Data--->
51 // |———————————————|————————————|
52 // |0|1|2|3|4|5|6|7|............|
53 // |———————————————|————————————|
54 // ꜛ ꜛ
55 // allocated ptr ptr to data [dataPtr]
56
57 Memory memory;
58 if (!allocFn) {
59 mesa_loge("Custom allocation (%zu bytes) failed\n", size);
60 return memory;
61 }
62
63 // custom allocation/free requires metadata for sync between host/guest
64 const size_t totalSize = size + kSyncDataSize;
65 memory = allocFn(totalSize);
66 if (!memory.ptr) {
67 mesa_loge("Custom allocation (%zu bytes) failed\n", size);
68 return memory;
69 }
70
71 // set sync data to read complete
72 uint32_t* syncDWordPtr = reinterpret_cast<uint32_t*>(memory.ptr);
73 __atomic_store_n(syncDWordPtr, kSyncDataReadComplete, __ATOMIC_RELEASE);
74 return memory;
75 };
76
77 m_free = [&freeFn](const Memory& mem) {
78 if (!freeFn) {
79 mesa_loge("Custom free for memory(%p) failed\n", mem.ptr);
80 return;
81 }
82 freeFn(mem);
83 };
84
85 // \param ptr is the data pointer currently allocated
86 // \return dataPtr starting at data
87 m_realloc = [this](const Memory& mem, size_t size) -> Memory {
88 // realloc requires freeing previously allocated memory
89 // read sync DWORD to ensure host is done reading this memory
90 // before releasing it.
91
92 size_t hostWaits = 0;
93
94 uint32_t* syncDWordPtr = reinterpret_cast<uint32_t*>(mem.ptr);
95 while (__atomic_load_n(syncDWordPtr, __ATOMIC_ACQUIRE) != kSyncDataReadComplete) {
96 hostWaits++;
97 usleep(10);
98 if (hostWaits > 1000) {
99 mesa_logd("%s: warning, stalled on host decoding on this command buffer stream\n",
100 __func__);
101 }
102 }
103
104 // for custom allocation/free, memory holding metadata must be copied
105 // along with stream data
106 // <---sync bytes--><----Data--->
107 // |———————————————|————————————|
108 // |0|1|2|3|4|5|6|7|............|
109 // |———————————————|————————————|
110 // ꜛ ꜛ
111 // [copyLocation] ptr to data [ptr]
112
113 const size_t toCopySize = m_writePos + kSyncDataSize;
114 unsigned char* copyLocation = static_cast<unsigned char*>(mem.ptr);
115 std::vector<uint8_t> tmp(copyLocation, copyLocation + toCopySize);
116 m_free(mem);
117
118 // get new buffer and copy previous stream data to it
119 Memory newMemory = m_alloc(size);
120 unsigned char* newBuf = static_cast<unsigned char*>(newMemory.ptr);
121 if (!newBuf) {
122 mesa_loge("Custom allocation (%zu bytes) failed\n", size);
123 return newMemory;
124 }
125 // copy previous data
126 memcpy(newBuf, tmp.data(), toCopySize);
127
128 return newMemory;
129 };
130 }
131
~CommandBufferStagingStream()132 CommandBufferStagingStream::~CommandBufferStagingStream() {
133 flush();
134 if (m_mem.ptr) m_free(m_mem);
135 }
136
getDataPtr()137 unsigned char* CommandBufferStagingStream::getDataPtr() {
138 if (!m_mem.ptr) return nullptr;
139 const size_t metadataSize = m_usingCustomAlloc ? kSyncDataSize : 0;
140 return static_cast<unsigned char*>(m_mem.ptr) + metadataSize;
141 }
142
markFlushing()143 void CommandBufferStagingStream::markFlushing() {
144 if (!m_usingCustomAlloc) {
145 return;
146 }
147 uint32_t* syncDWordPtr = reinterpret_cast<uint32_t*>(m_mem.ptr);
148 __atomic_store_n(syncDWordPtr, kSyncDataReadPending, __ATOMIC_RELEASE);
149 }
150
idealAllocSize(size_t len)151 size_t CommandBufferStagingStream::idealAllocSize(size_t len) {
152 if (len > 1048576) return len;
153 return 1048576;
154 }
155
allocBuffer(size_t minSize)156 void* CommandBufferStagingStream::allocBuffer(size_t minSize) {
157 size_t allocSize = (1048576 < minSize ? minSize : 1048576);
158 // Initial case: blank
159 if (!m_mem.ptr) {
160 m_mem = m_alloc(allocSize);
161 m_size = allocSize;
162 return getDataPtr();
163 }
164
165 // Calculate remaining
166 size_t remaining = m_size - m_writePos;
167 // check if there is at least minSize bytes left in buffer
168 // if not, reallocate a buffer of big enough size
169 if (remaining < minSize) {
170 size_t newAllocSize = m_size * 2 + allocSize;
171 m_mem = m_realloc(m_mem, newAllocSize);
172 m_size = newAllocSize;
173
174 return (void*)(getDataPtr() + m_writePos);
175 }
176
177 // for custom allocations, host should have finished reading
178 // data from command buffer since command buffers are flushed
179 // on queue submit.
180 // allocBuffer should not be called on command buffers that are currently
181 // being read by the host
182 if (m_usingCustomAlloc) {
183 uint32_t* syncDWordPtr = reinterpret_cast<uint32_t*>(m_mem.ptr);
184 if (__atomic_load_n(syncDWordPtr, __ATOMIC_ACQUIRE) != kSyncDataReadComplete) {
185 mesa_loge("FATAL: allocBuffer() called but previous read not complete");
186 abort();
187 }
188 }
189
190 return (void*)(getDataPtr() + m_writePos);
191 }
192
commitBuffer(size_t size)193 int CommandBufferStagingStream::commitBuffer(size_t size) {
194 m_writePos += size;
195 return 0;
196 }
197
readFully(void *,size_t)198 const unsigned char* CommandBufferStagingStream::readFully(void*, size_t) {
199 // Not supported
200 mesa_loge("CommandBufferStagingStream::%s: Fatal: not supported\n", __func__);
201 abort();
202 return nullptr;
203 }
204
read(void *,size_t *)205 const unsigned char* CommandBufferStagingStream::read(void*, size_t*) {
206 // Not supported
207 mesa_loge("CommandBufferStagingStream::%s: Fatal: not supported\n", __func__);
208 abort();
209 return nullptr;
210 }
211
writeFully(const void *,size_t)212 int CommandBufferStagingStream::writeFully(const void*, size_t) {
213 // Not supported
214 mesa_loge("CommandBufferStagingStream::%s: Fatal: not supported\n", __func__);
215 abort();
216 return 0;
217 }
218
commitBufferAndReadFully(size_t,void *,size_t)219 const unsigned char* CommandBufferStagingStream::commitBufferAndReadFully(size_t, void*, size_t) {
220 // Not supported
221 mesa_loge("CommandBufferStagingStream::%s: Fatal: not supported\n", __func__);
222 abort();
223 return nullptr;
224 }
225
getWritten(unsigned char ** bufOut,size_t * sizeOut)226 void CommandBufferStagingStream::getWritten(unsigned char** bufOut, size_t* sizeOut) {
227 *bufOut = getDataPtr();
228 *sizeOut = m_writePos;
229 }
230
reset()231 void CommandBufferStagingStream::reset() {
232 m_writePos = 0;
233 IOStream::rewind();
234 }
235
getDeviceMemory()236 VkDeviceMemory CommandBufferStagingStream::getDeviceMemory() { return m_mem.deviceMemory; }
237
238 } // namespace vk
239 } // namespace gfxstream
240