1 /*
2 * Copyright © Microsoft Corporation
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 */
23
24 #include "d3d12_batch.h"
25 #include "d3d12_bufmgr.h"
26 #include "d3d12_residency.h"
27 #include "d3d12_resource.h"
28 #include "d3d12_screen.h"
29
30 #include "util/os_time.h"
31
32 #include <dxguids/dxguids.h>
33
34 static constexpr unsigned residency_batch_size = 128;
35
36 static void
log_eviction_info(struct d3d12_screen * screen,struct d3d12_bo * bo)37 log_eviction_info(struct d3d12_screen *screen, struct d3d12_bo *bo)
38 {
39 screen->total_bytes_evicted += bo->estimated_size;
40 screen->num_evictions++;
41 }
42
43 static void
evict_aged_allocations(struct d3d12_screen * screen,uint64_t completed_fence,int64_t time,int64_t grace_period)44 evict_aged_allocations(struct d3d12_screen *screen, uint64_t completed_fence, int64_t time, int64_t grace_period)
45 {
46 ID3D12Pageable *to_evict[residency_batch_size];
47 unsigned num_pending_evictions = 0;
48
49 list_for_each_entry_safe(struct d3d12_bo, bo, &screen->residency_list, residency_list_entry) {
50 /* This residency list should all be base bos, not suballocated ones */
51 assert(bo->res);
52
53 if (bo->last_used_fence > completed_fence ||
54 time - bo->last_used_timestamp <= grace_period) {
55 /* List is LRU-sorted, this bo is still in use, so we're done */
56 break;
57 }
58
59 assert(bo->residency_status == d3d12_resident);
60
61 to_evict[num_pending_evictions++] = bo->res;
62 log_eviction_info(screen, bo);
63 bo->residency_status = d3d12_evicted;
64 list_del(&bo->residency_list_entry);
65
66 if (num_pending_evictions == residency_batch_size) {
67 screen->dev->Evict(num_pending_evictions, to_evict);
68 num_pending_evictions = 0;
69 }
70 }
71
72 if (num_pending_evictions)
73 screen->dev->Evict(num_pending_evictions, to_evict);
74 }
75
76 static void
evict_to_fence_or_budget(struct d3d12_screen * screen,uint64_t target_fence,uint64_t current_usage,uint64_t target_budget)77 evict_to_fence_or_budget(struct d3d12_screen *screen, uint64_t target_fence, uint64_t current_usage, uint64_t target_budget)
78 {
79 screen->fence->SetEventOnCompletion(target_fence, nullptr);
80
81 ID3D12Pageable *to_evict[residency_batch_size];
82 unsigned num_pending_evictions = 0;
83
84 list_for_each_entry_safe(struct d3d12_bo, bo, &screen->residency_list, residency_list_entry) {
85 /* This residency list should all be base bos, not suballocated ones */
86 assert(bo->res);
87
88 if (bo->last_used_fence > target_fence || current_usage < target_budget) {
89 break;
90 }
91
92 assert(bo->residency_status == d3d12_resident);
93
94 to_evict[num_pending_evictions++] = bo->res;
95 log_eviction_info(screen, bo);
96 bo->residency_status = d3d12_evicted;
97 list_del(&bo->residency_list_entry);
98
99 current_usage -= bo->estimated_size;
100
101 if (num_pending_evictions == residency_batch_size) {
102 screen->dev->Evict(num_pending_evictions, to_evict);
103 num_pending_evictions = 0;
104 }
105 }
106
107 if (num_pending_evictions)
108 screen->dev->Evict(num_pending_evictions, to_evict);
109 }
110
111 static constexpr int64_t eviction_grace_period_seconds_min = 1;
112 static constexpr int64_t eviction_grace_period_seconds_max = 60;
113 static constexpr int64_t microseconds_per_second = 1000000;
114 static constexpr int64_t eviction_grace_period_microseconds_min =
115 eviction_grace_period_seconds_min * microseconds_per_second;
116 static constexpr int64_t eviction_grace_period_microseconds_max =
117 eviction_grace_period_seconds_max * microseconds_per_second;
118 static constexpr double trim_percentage_usage_threshold = 0.7;
119
120 static int64_t
get_eviction_grace_period(struct d3d12_memory_info * mem_info)121 get_eviction_grace_period(struct d3d12_memory_info *mem_info)
122 {
123 double pressure = double(mem_info->usage) / double(mem_info->budget);
124 pressure = MIN2(pressure, 1.0);
125
126 if (pressure > trim_percentage_usage_threshold) {
127 /* Normalize pressure for the range [0, threshold] */
128 pressure = (pressure - trim_percentage_usage_threshold) / (1.0 - trim_percentage_usage_threshold);
129 /* Linearly interpolate between min and max period based on pressure */
130 return (int64_t)((eviction_grace_period_microseconds_max - eviction_grace_period_microseconds_min) *
131 (1.0 - pressure)) + eviction_grace_period_microseconds_min;
132 }
133
134 /* Unlimited grace period, essentially don't trim at all */
135 return INT64_MAX;
136 }
137
138 static void
gather_base_bos(struct d3d12_screen * screen,set * base_bo_set,struct d3d12_bo * bo,uint64_t & size_to_make_resident,uint64_t pending_fence_value,int64_t current_time)139 gather_base_bos(struct d3d12_screen *screen, set *base_bo_set, struct d3d12_bo *bo, uint64_t &size_to_make_resident, uint64_t pending_fence_value, int64_t current_time)
140 {
141 uint64_t offset;
142 struct d3d12_bo *base_bo = d3d12_bo_get_base(bo, &offset);
143
144 if (base_bo->residency_status == d3d12_evicted) {
145 bool added = false;
146 _mesa_set_search_or_add(base_bo_set, base_bo, &added);
147 assert(!added);
148
149 base_bo->residency_status = d3d12_resident;
150 size_to_make_resident += base_bo->estimated_size;
151 list_addtail(&base_bo->residency_list_entry, &screen->residency_list);
152 } else if (base_bo->last_used_fence != pending_fence_value &&
153 base_bo->residency_status == d3d12_resident) {
154 /* First time seeing this already-resident base bo in this batch */
155 list_del(&base_bo->residency_list_entry);
156 list_addtail(&base_bo->residency_list_entry, &screen->residency_list);
157 }
158
159 base_bo->last_used_fence = pending_fence_value;
160 base_bo->last_used_timestamp = current_time;
161 }
162
163 void
d3d12_process_batch_residency(struct d3d12_screen * screen,struct d3d12_batch * batch)164 d3d12_process_batch_residency(struct d3d12_screen *screen, struct d3d12_batch *batch)
165 {
166 d3d12_memory_info mem_info;
167 screen->get_memory_info(screen, &mem_info);
168
169 uint64_t completed_fence_value = screen->fence->GetCompletedValue();
170 uint64_t pending_fence_value = screen->fence_value + 1;
171 int64_t current_time = os_time_get();
172 int64_t grace_period = get_eviction_grace_period(&mem_info);
173
174 /* Gather base bos for the batch */
175 uint64_t size_to_make_resident = 0;
176 set *base_bo_set = _mesa_pointer_set_create(nullptr);
177
178 util_dynarray_foreach(&batch->local_bos, d3d12_bo*, bo)
179 gather_base_bos(screen, base_bo_set, *bo, size_to_make_resident, pending_fence_value, current_time);
180 hash_table_foreach(batch->bos, entry)
181 gather_base_bos(screen, base_bo_set, (struct d3d12_bo *)entry->key, size_to_make_resident, pending_fence_value, current_time);
182
183 /* Now that bos referenced by this batch are moved to the end of the LRU, trim it */
184 evict_aged_allocations(screen, completed_fence_value, current_time, grace_period);
185
186 /* If there's nothing needing to be made newly resident, we're done once we've trimmed */
187 if (base_bo_set->entries == 0) {
188 _mesa_set_destroy(base_bo_set, nullptr);
189 return;
190 }
191
192 uint64_t residency_fence_value_snapshot = screen->residency_fence_value;
193
194 struct set_entry *entry = _mesa_set_next_entry(base_bo_set, nullptr);
195 uint64_t batch_memory_size = 0;
196 unsigned batch_count = 0;
197 ID3D12Pageable *to_make_resident[residency_batch_size];
198 while (true) {
199 /* Refresh memory stats */
200 screen->get_memory_info(screen, &mem_info);
201
202 int64_t available_memory = (int64_t)mem_info.budget - (int64_t)mem_info.usage;
203
204 assert(!list_is_empty(&screen->residency_list));
205 struct d3d12_bo *oldest_resident_bo =
206 list_first_entry(&screen->residency_list, struct d3d12_bo, residency_list_entry);
207 bool anything_to_wait_for = oldest_resident_bo->last_used_fence < pending_fence_value;
208
209 /* We've got some room, or we can't free up any more room, make some resources resident */
210 HRESULT hr = S_OK;
211 if ((available_memory || !anything_to_wait_for) && batch_count < residency_batch_size) {
212 for (; entry; entry = _mesa_set_next_entry(base_bo_set, entry)) {
213 struct d3d12_bo *bo = (struct d3d12_bo *)entry->key;
214 if (anything_to_wait_for &&
215 (int64_t)(batch_memory_size + bo->estimated_size) > available_memory)
216 break;
217
218 batch_memory_size += bo->estimated_size;
219 to_make_resident[batch_count++] = bo->res;
220 if (batch_count == residency_batch_size)
221 break;
222 }
223
224 if (batch_count) {
225 hr = screen->dev->EnqueueMakeResident(D3D12_RESIDENCY_FLAG_NONE, batch_count, to_make_resident,
226 screen->residency_fence, screen->residency_fence_value + 1);
227 if (SUCCEEDED(hr))
228 ++screen->residency_fence_value;
229 }
230
231 if (SUCCEEDED(hr) && batch_count == residency_batch_size) {
232 batch_count = 0;
233 size_to_make_resident -= batch_memory_size;
234 continue;
235 }
236 }
237
238 /* We need to free up some space, either we broke early from the resource loop,
239 * or the MakeResident call itself failed.
240 */
241 if (FAILED(hr) || entry) {
242 if (!anything_to_wait_for) {
243 assert(false);
244 break;
245 }
246
247 evict_to_fence_or_budget(screen, oldest_resident_bo->last_used_fence, mem_info.usage + size_to_make_resident, mem_info.budget);
248 continue;
249 }
250
251 /* Made it to the end without explicitly needing to loop, so we're done */
252 break;
253 }
254 _mesa_set_destroy(base_bo_set, nullptr);
255
256 /* The GPU needs to wait for these resources to be made resident */
257 if (residency_fence_value_snapshot != screen->residency_fence_value)
258 screen->cmdqueue->Wait(screen->residency_fence, screen->residency_fence_value);
259 }
260
261 bool
d3d12_init_residency(struct d3d12_screen * screen)262 d3d12_init_residency(struct d3d12_screen *screen)
263 {
264 list_inithead(&screen->residency_list);
265 if (FAILED(screen->dev->CreateFence(0, D3D12_FENCE_FLAG_NONE, IID_PPV_ARGS(&screen->residency_fence))))
266 return false;
267
268 return true;
269 }
270
271 void
d3d12_deinit_residency(struct d3d12_screen * screen)272 d3d12_deinit_residency(struct d3d12_screen *screen)
273 {
274 if (screen->residency_fence) {
275 screen->residency_fence->Release();
276 screen->residency_fence = nullptr;
277 }
278 }
279
280 void
d3d12_promote_to_permanent_residency(struct d3d12_screen * screen,struct d3d12_resource * resource)281 d3d12_promote_to_permanent_residency(struct d3d12_screen *screen, struct d3d12_resource* resource)
282 {
283 mtx_lock(&screen->submit_mutex);
284 uint64_t offset;
285 struct d3d12_bo *base_bo = d3d12_bo_get_base(resource->bo, &offset);
286
287 /* Promote non-permanent resident resources to permanent residency*/
288 if(base_bo->residency_status != d3d12_permanently_resident) {
289
290 /* Mark as permanently resident*/
291 base_bo->residency_status = d3d12_permanently_resident;
292
293 /* If it wasn't made resident before, make it*/
294 bool was_made_resident = (base_bo->residency_status == d3d12_resident);
295 if(!was_made_resident) {
296 ID3D12Pageable *pageable = base_bo->res;
297 ASSERTED HRESULT hr = screen->dev->MakeResident(1, &pageable);
298 assert(SUCCEEDED(hr));
299 }
300 }
301 mtx_unlock(&screen->submit_mutex);
302 }
303