1 /*
2 * Copyright 2021 Alyssa Rosenzweig
3 * Copyright 2019 Collabora, Ltd.
4 * SPDX-License-Identifier: MIT
5 */
6
7 #include "agx_bo.h"
8 #include <inttypes.h>
9 #include "agx_device.h"
10 #include "decode.h"
11
12 /* Helper to calculate the bucket index of a BO */
13 static unsigned
agx_bucket_index(unsigned size)14 agx_bucket_index(unsigned size)
15 {
16 /* Round down to POT to compute a bucket index */
17 unsigned bucket_index = util_logbase2(size);
18
19 /* Clamp to supported buckets. Huge allocations use the largest bucket */
20 bucket_index = CLAMP(bucket_index, MIN_BO_CACHE_BUCKET, MAX_BO_CACHE_BUCKET);
21
22 /* Reindex from 0 */
23 return (bucket_index - MIN_BO_CACHE_BUCKET);
24 }
25
26 static struct list_head *
agx_bucket(struct agx_device * dev,unsigned size)27 agx_bucket(struct agx_device *dev, unsigned size)
28 {
29 return &dev->bo_cache.buckets[agx_bucket_index(size)];
30 }
31
32 static bool
agx_bo_wait(struct agx_bo * bo,int64_t timeout_ns)33 agx_bo_wait(struct agx_bo *bo, int64_t timeout_ns)
34 {
35 /* TODO: When we allow parallelism we'll need to implement this for real */
36 return true;
37 }
38
39 static void
agx_bo_cache_remove_locked(struct agx_device * dev,struct agx_bo * bo)40 agx_bo_cache_remove_locked(struct agx_device *dev, struct agx_bo *bo)
41 {
42 simple_mtx_assert_locked(&dev->bo_cache.lock);
43 list_del(&bo->bucket_link);
44 list_del(&bo->lru_link);
45 dev->bo_cache.size -= bo->size;
46 }
47
48 /* Tries to fetch a BO of sufficient size with the appropriate flags from the
49 * BO cache. If it succeeds, it returns that BO and removes the BO from the
50 * cache. If it fails, it returns NULL signaling the caller to allocate a new
51 * BO. */
52
53 struct agx_bo *
agx_bo_cache_fetch(struct agx_device * dev,size_t size,size_t align,uint32_t flags,const bool dontwait)54 agx_bo_cache_fetch(struct agx_device *dev, size_t size, size_t align,
55 uint32_t flags, const bool dontwait)
56 {
57 simple_mtx_lock(&dev->bo_cache.lock);
58 struct list_head *bucket = agx_bucket(dev, size);
59 struct agx_bo *bo = NULL;
60
61 /* Iterate the bucket looking for something suitable */
62 list_for_each_entry_safe(struct agx_bo, entry, bucket, bucket_link) {
63 if (entry->size < size || entry->flags != flags)
64 continue;
65
66 /* Do not return more than 2x oversized BOs. */
67 if (entry->size > 2 * size)
68 continue;
69
70 if (align > entry->align)
71 continue;
72
73 /* If the oldest BO in the cache is busy, likely so is
74 * everything newer, so bail. */
75 if (!agx_bo_wait(entry, dontwait ? 0 : INT64_MAX))
76 break;
77
78 /* This one works, use it */
79 agx_bo_cache_remove_locked(dev, entry);
80 bo = entry;
81 break;
82 }
83 simple_mtx_unlock(&dev->bo_cache.lock);
84
85 return bo;
86 }
87
88 static void
agx_bo_cache_evict_stale_bos(struct agx_device * dev,unsigned tv_sec)89 agx_bo_cache_evict_stale_bos(struct agx_device *dev, unsigned tv_sec)
90 {
91 struct timespec time;
92
93 clock_gettime(CLOCK_MONOTONIC, &time);
94 list_for_each_entry_safe(struct agx_bo, entry, &dev->bo_cache.lru,
95 lru_link) {
96 /* We want all entries that have been used more than 1 sec ago to be
97 * dropped, others can be kept. Note the <= 2 check and not <= 1. It's
98 * here to account for the fact that we're only testing ->tv_sec, not
99 * ->tv_nsec. That means we might keep entries that are between 1 and 2
100 * seconds old, but we don't really care, as long as unused BOs are
101 * dropped at some point.
102 */
103 if (time.tv_sec - entry->last_used <= 2)
104 break;
105
106 agx_bo_cache_remove_locked(dev, entry);
107 agx_bo_free(dev, entry);
108 }
109 }
110
111 static void
agx_bo_cache_put_locked(struct agx_bo * bo)112 agx_bo_cache_put_locked(struct agx_bo *bo)
113 {
114 struct agx_device *dev = bo->dev;
115 struct list_head *bucket = agx_bucket(dev, bo->size);
116 struct timespec time;
117
118 /* Add us to the bucket */
119 list_addtail(&bo->bucket_link, bucket);
120
121 /* Add us to the LRU list and update the last_used field. */
122 list_addtail(&bo->lru_link, &dev->bo_cache.lru);
123 clock_gettime(CLOCK_MONOTONIC, &time);
124 bo->last_used = time.tv_sec;
125
126 /* Update statistics */
127 dev->bo_cache.size += bo->size;
128
129 if (0) {
130 printf("BO cache: %zu KiB (+%zu KiB from %s, hit/miss %" PRIu64
131 "/%" PRIu64 ")\n",
132 DIV_ROUND_UP(dev->bo_cache.size, 1024),
133 DIV_ROUND_UP(bo->size, 1024), bo->label,
134 p_atomic_read(&dev->bo_cache.hits),
135 p_atomic_read(&dev->bo_cache.misses));
136 }
137
138 /* Update label for debug */
139 bo->label = "Unused (BO cache)";
140
141 /* Let's do some cleanup in the BO cache while we hold the lock. */
142 agx_bo_cache_evict_stale_bos(dev, time.tv_sec);
143 }
144
145 /* Tries to add a BO to the cache. Returns if it was successful */
146 static bool
agx_bo_cache_put(struct agx_bo * bo)147 agx_bo_cache_put(struct agx_bo *bo)
148 {
149 struct agx_device *dev = bo->dev;
150
151 if (bo->flags & AGX_BO_SHARED) {
152 return false;
153 } else {
154 simple_mtx_lock(&dev->bo_cache.lock);
155 agx_bo_cache_put_locked(bo);
156 simple_mtx_unlock(&dev->bo_cache.lock);
157
158 return true;
159 }
160 }
161
162 void
agx_bo_cache_evict_all(struct agx_device * dev)163 agx_bo_cache_evict_all(struct agx_device *dev)
164 {
165 simple_mtx_lock(&dev->bo_cache.lock);
166 for (unsigned i = 0; i < ARRAY_SIZE(dev->bo_cache.buckets); ++i) {
167 struct list_head *bucket = &dev->bo_cache.buckets[i];
168
169 list_for_each_entry_safe(struct agx_bo, entry, bucket, bucket_link) {
170 agx_bo_cache_remove_locked(dev, entry);
171 agx_bo_free(dev, entry);
172 }
173 }
174 simple_mtx_unlock(&dev->bo_cache.lock);
175 }
176
177 void
agx_bo_reference(struct agx_bo * bo)178 agx_bo_reference(struct agx_bo *bo)
179 {
180 if (bo) {
181 ASSERTED int count = p_atomic_inc_return(&bo->refcnt);
182 assert(count != 1);
183 }
184 }
185
186 void
agx_bo_unreference(struct agx_bo * bo)187 agx_bo_unreference(struct agx_bo *bo)
188 {
189 if (!bo)
190 return;
191
192 /* Don't return to cache if there are still references */
193 if (p_atomic_dec_return(&bo->refcnt))
194 return;
195
196 struct agx_device *dev = bo->dev;
197
198 pthread_mutex_lock(&dev->bo_map_lock);
199
200 /* Someone might have imported this BO while we were waiting for the
201 * lock, let's make sure it's still not referenced before freeing it.
202 */
203 if (p_atomic_read(&bo->refcnt) == 0) {
204 assert(!p_atomic_read_relaxed(&bo->writer_syncobj));
205
206 if (dev->debug & AGX_DBG_TRACE)
207 agxdecode_track_free(bo);
208
209 if (!agx_bo_cache_put(bo))
210 agx_bo_free(dev, bo);
211 }
212
213 pthread_mutex_unlock(&dev->bo_map_lock);
214 }
215
216 struct agx_bo *
agx_bo_create_aligned(struct agx_device * dev,unsigned size,unsigned align,enum agx_bo_flags flags,const char * label)217 agx_bo_create_aligned(struct agx_device *dev, unsigned size, unsigned align,
218 enum agx_bo_flags flags, const char *label)
219 {
220 struct agx_bo *bo;
221 assert(size > 0);
222
223 /* To maximize BO cache usage, don't allocate tiny BOs */
224 size = ALIGN_POT(size, 16384);
225
226 /* See if we have a BO already in the cache */
227 bo = agx_bo_cache_fetch(dev, size, align, flags, true);
228
229 /* Update stats based on the first attempt to fetch */
230 if (bo != NULL)
231 p_atomic_inc(&dev->bo_cache.hits);
232 else
233 p_atomic_inc(&dev->bo_cache.misses);
234
235 /* Otherwise, allocate a fresh BO. If allocation fails, we can try waiting
236 * for something in the cache. But if there's no nothing suitable, we should
237 * flush the cache to make space for the new allocation.
238 */
239 if (!bo)
240 bo = agx_bo_alloc(dev, size, align, flags);
241 if (!bo)
242 bo = agx_bo_cache_fetch(dev, size, align, flags, false);
243 if (!bo) {
244 agx_bo_cache_evict_all(dev);
245 bo = agx_bo_alloc(dev, size, align, flags);
246 }
247
248 if (!bo) {
249 fprintf(stderr, "BO creation failed\n");
250 return NULL;
251 }
252
253 bo->label = label;
254 p_atomic_set(&bo->refcnt, 1);
255
256 if (dev->debug & AGX_DBG_TRACE)
257 agxdecode_track_alloc(bo);
258
259 return bo;
260 }
261