1 #define JEMALLOC_QUARANTINE_C_
2 #include "jemalloc/internal/jemalloc_internal.h"
3
4 /*
5 * Quarantine pointers close to NULL are used to encode state information that
6 * is used for cleaning up during thread shutdown.
7 */
8 #define QUARANTINE_STATE_REINCARNATED ((quarantine_t *)(uintptr_t)1)
9 #define QUARANTINE_STATE_PURGATORY ((quarantine_t *)(uintptr_t)2)
10 #define QUARANTINE_STATE_MAX QUARANTINE_STATE_PURGATORY
11
12 /******************************************************************************/
13 /* Function prototypes for non-inline static functions. */
14
15 static quarantine_t *quarantine_grow(tsd_t *tsd, quarantine_t *quarantine);
16 static void quarantine_drain_one(tsdn_t *tsdn, quarantine_t *quarantine);
17 static void quarantine_drain(tsdn_t *tsdn, quarantine_t *quarantine,
18 size_t upper_bound);
19
20 /******************************************************************************/
21
22 static quarantine_t *
quarantine_init(tsdn_t * tsdn,size_t lg_maxobjs)23 quarantine_init(tsdn_t *tsdn, size_t lg_maxobjs)
24 {
25 quarantine_t *quarantine;
26 size_t size;
27
28 size = offsetof(quarantine_t, objs) + ((ZU(1) << lg_maxobjs) *
29 sizeof(quarantine_obj_t));
30 quarantine = (quarantine_t *)iallocztm(tsdn, size, size2index(size),
31 false, NULL, true, arena_get(TSDN_NULL, 0, true), true);
32 if (quarantine == NULL)
33 return (NULL);
34 quarantine->curbytes = 0;
35 quarantine->curobjs = 0;
36 quarantine->first = 0;
37 quarantine->lg_maxobjs = lg_maxobjs;
38
39 return (quarantine);
40 }
41
42 void
quarantine_alloc_hook_work(tsd_t * tsd)43 quarantine_alloc_hook_work(tsd_t *tsd)
44 {
45 quarantine_t *quarantine;
46
47 if (!tsd_nominal(tsd))
48 return;
49
50 quarantine = quarantine_init(tsd_tsdn(tsd), LG_MAXOBJS_INIT);
51 /*
52 * Check again whether quarantine has been initialized, because
53 * quarantine_init() may have triggered recursive initialization.
54 */
55 if (tsd_quarantine_get(tsd) == NULL)
56 tsd_quarantine_set(tsd, quarantine);
57 else
58 idalloctm(tsd_tsdn(tsd), quarantine, NULL, true, true);
59 }
60
61 static quarantine_t *
quarantine_grow(tsd_t * tsd,quarantine_t * quarantine)62 quarantine_grow(tsd_t *tsd, quarantine_t *quarantine)
63 {
64 quarantine_t *ret;
65
66 ret = quarantine_init(tsd_tsdn(tsd), quarantine->lg_maxobjs + 1);
67 if (ret == NULL) {
68 quarantine_drain_one(tsd_tsdn(tsd), quarantine);
69 return (quarantine);
70 }
71
72 ret->curbytes = quarantine->curbytes;
73 ret->curobjs = quarantine->curobjs;
74 if (quarantine->first + quarantine->curobjs <= (ZU(1) <<
75 quarantine->lg_maxobjs)) {
76 /* objs ring buffer data are contiguous. */
77 memcpy(ret->objs, &quarantine->objs[quarantine->first],
78 quarantine->curobjs * sizeof(quarantine_obj_t));
79 } else {
80 /* objs ring buffer data wrap around. */
81 size_t ncopy_a = (ZU(1) << quarantine->lg_maxobjs) -
82 quarantine->first;
83 size_t ncopy_b = quarantine->curobjs - ncopy_a;
84
85 memcpy(ret->objs, &quarantine->objs[quarantine->first], ncopy_a
86 * sizeof(quarantine_obj_t));
87 memcpy(&ret->objs[ncopy_a], quarantine->objs, ncopy_b *
88 sizeof(quarantine_obj_t));
89 }
90 idalloctm(tsd_tsdn(tsd), quarantine, NULL, true, true);
91
92 tsd_quarantine_set(tsd, ret);
93 return (ret);
94 }
95
96 static void
quarantine_drain_one(tsdn_t * tsdn,quarantine_t * quarantine)97 quarantine_drain_one(tsdn_t *tsdn, quarantine_t *quarantine)
98 {
99 quarantine_obj_t *obj = &quarantine->objs[quarantine->first];
100 assert(obj->usize == isalloc(tsdn, obj->ptr, config_prof));
101 idalloctm(tsdn, obj->ptr, NULL, false, true);
102 quarantine->curbytes -= obj->usize;
103 quarantine->curobjs--;
104 quarantine->first = (quarantine->first + 1) & ((ZU(1) <<
105 quarantine->lg_maxobjs) - 1);
106 }
107
108 static void
quarantine_drain(tsdn_t * tsdn,quarantine_t * quarantine,size_t upper_bound)109 quarantine_drain(tsdn_t *tsdn, quarantine_t *quarantine, size_t upper_bound)
110 {
111
112 while (quarantine->curbytes > upper_bound && quarantine->curobjs > 0)
113 quarantine_drain_one(tsdn, quarantine);
114 }
115
116 void
quarantine(tsd_t * tsd,void * ptr)117 quarantine(tsd_t *tsd, void *ptr)
118 {
119 quarantine_t *quarantine;
120 size_t usize = isalloc(tsd_tsdn(tsd), ptr, config_prof);
121
122 cassert(config_fill);
123 assert(opt_quarantine);
124
125 if ((quarantine = tsd_quarantine_get(tsd)) == NULL) {
126 idalloctm(tsd_tsdn(tsd), ptr, NULL, false, true);
127 return;
128 }
129 /*
130 * Drain one or more objects if the quarantine size limit would be
131 * exceeded by appending ptr.
132 */
133 if (quarantine->curbytes + usize > opt_quarantine) {
134 size_t upper_bound = (opt_quarantine >= usize) ? opt_quarantine
135 - usize : 0;
136 quarantine_drain(tsd_tsdn(tsd), quarantine, upper_bound);
137 }
138 /* Grow the quarantine ring buffer if it's full. */
139 if (quarantine->curobjs == (ZU(1) << quarantine->lg_maxobjs))
140 quarantine = quarantine_grow(tsd, quarantine);
141 /* quarantine_grow() must free a slot if it fails to grow. */
142 assert(quarantine->curobjs < (ZU(1) << quarantine->lg_maxobjs));
143 /* Append ptr if its size doesn't exceed the quarantine size. */
144 if (quarantine->curbytes + usize <= opt_quarantine) {
145 size_t offset = (quarantine->first + quarantine->curobjs) &
146 ((ZU(1) << quarantine->lg_maxobjs) - 1);
147 quarantine_obj_t *obj = &quarantine->objs[offset];
148 obj->ptr = ptr;
149 obj->usize = usize;
150 quarantine->curbytes += usize;
151 quarantine->curobjs++;
152 if (config_fill && unlikely(opt_junk_free)) {
153 /*
154 * Only do redzone validation if Valgrind isn't in
155 * operation.
156 */
157 if ((!config_valgrind || likely(!in_valgrind))
158 && usize <= SMALL_MAXCLASS)
159 arena_quarantine_junk_small(ptr, usize);
160 else
161 memset(ptr, JEMALLOC_FREE_JUNK, usize);
162 }
163 } else {
164 assert(quarantine->curbytes == 0);
165 idalloctm(tsd_tsdn(tsd), ptr, NULL, false, true);
166 }
167 }
168
169 void
quarantine_cleanup(tsd_t * tsd)170 quarantine_cleanup(tsd_t *tsd)
171 {
172 quarantine_t *quarantine;
173
174 if (!config_fill)
175 return;
176
177 quarantine = tsd_quarantine_get(tsd);
178 if (quarantine != NULL) {
179 quarantine_drain(tsd_tsdn(tsd), quarantine, 0);
180 idalloctm(tsd_tsdn(tsd), quarantine, NULL, true, true);
181 tsd_quarantine_set(tsd, NULL);
182 }
183 }
184