1 // SPDX-License-Identifier: GPL-2.0
2 #include <linux/export.h>
3 #include <linux/lockref.h>
4
5 #if USE_CMPXCHG_LOCKREF
6
7 /*
8 * Note that the "cmpxchg()" reloads the "old" value for the
9 * failure case.
10 */
11 #define CMPXCHG_LOOP(CODE, SUCCESS) do { \
12 int retry = 100; \
13 struct lockref old; \
14 BUILD_BUG_ON(sizeof(old) != 8); \
15 old.lock_count = READ_ONCE(lockref->lock_count); \
16 while (likely(arch_spin_value_unlocked(old.lock.rlock.raw_lock))) { \
17 struct lockref new = old, prev = old; \
18 CODE \
19 old.lock_count = cmpxchg64_relaxed(&lockref->lock_count, \
20 old.lock_count, \
21 new.lock_count); \
22 if (likely(old.lock_count == prev.lock_count)) { \
23 SUCCESS; \
24 } \
25 if (!--retry) \
26 break; \
27 } \
28 } while (0)
29
30 #else
31
32 #define CMPXCHG_LOOP(CODE, SUCCESS) do { } while (0)
33
34 #endif
35
36 /**
37 * lockref_get - Increments reference count unconditionally
38 * @lockref: pointer to lockref structure
39 *
40 * This operation is only valid if you already hold a reference
41 * to the object, so you know the count cannot be zero.
42 */
lockref_get(struct lockref * lockref)43 void lockref_get(struct lockref *lockref)
44 {
45 CMPXCHG_LOOP(
46 new.count++;
47 ,
48 return;
49 );
50
51 spin_lock(&lockref->lock);
52 lockref->count++;
53 spin_unlock(&lockref->lock);
54 }
55 EXPORT_SYMBOL(lockref_get);
56
57 /**
58 * lockref_get_not_zero - Increments count unless the count is 0 or dead
59 * @lockref: pointer to lockref structure
60 * Return: 1 if count updated successfully or 0 if count was zero
61 */
lockref_get_not_zero(struct lockref * lockref)62 int lockref_get_not_zero(struct lockref *lockref)
63 {
64 int retval;
65
66 CMPXCHG_LOOP(
67 new.count++;
68 if (old.count <= 0)
69 return 0;
70 ,
71 return 1;
72 );
73
74 spin_lock(&lockref->lock);
75 retval = 0;
76 if (lockref->count > 0) {
77 lockref->count++;
78 retval = 1;
79 }
80 spin_unlock(&lockref->lock);
81 return retval;
82 }
83 EXPORT_SYMBOL(lockref_get_not_zero);
84
85 /**
86 * lockref_put_not_zero - Decrements count unless count <= 1 before decrement
87 * @lockref: pointer to lockref structure
88 * Return: 1 if count updated successfully or 0 if count would become zero
89 */
lockref_put_not_zero(struct lockref * lockref)90 int lockref_put_not_zero(struct lockref *lockref)
91 {
92 int retval;
93
94 CMPXCHG_LOOP(
95 new.count--;
96 if (old.count <= 1)
97 return 0;
98 ,
99 return 1;
100 );
101
102 spin_lock(&lockref->lock);
103 retval = 0;
104 if (lockref->count > 1) {
105 lockref->count--;
106 retval = 1;
107 }
108 spin_unlock(&lockref->lock);
109 return retval;
110 }
111 EXPORT_SYMBOL(lockref_put_not_zero);
112
113 /**
114 * lockref_get_or_lock - Increments count unless the count is 0 or dead
115 * @lockref: pointer to lockref structure
116 * Return: 1 if count updated successfully or 0 if count was zero
117 * and we got the lock instead.
118 */
lockref_get_or_lock(struct lockref * lockref)119 int lockref_get_or_lock(struct lockref *lockref)
120 {
121 CMPXCHG_LOOP(
122 new.count++;
123 if (old.count <= 0)
124 break;
125 ,
126 return 1;
127 );
128
129 spin_lock(&lockref->lock);
130 if (lockref->count <= 0)
131 return 0;
132 lockref->count++;
133 spin_unlock(&lockref->lock);
134 return 1;
135 }
136 EXPORT_SYMBOL(lockref_get_or_lock);
137
138 /**
139 * lockref_put_return - Decrement reference count if possible
140 * @lockref: pointer to lockref structure
141 *
142 * Decrement the reference count and return the new value.
143 * If the lockref was dead or locked, return an error.
144 */
lockref_put_return(struct lockref * lockref)145 int lockref_put_return(struct lockref *lockref)
146 {
147 CMPXCHG_LOOP(
148 new.count--;
149 if (old.count <= 0)
150 return -1;
151 ,
152 return new.count;
153 );
154 return -1;
155 }
156 EXPORT_SYMBOL(lockref_put_return);
157
158 /**
159 * lockref_put_or_lock - decrements count unless count <= 1 before decrement
160 * @lockref: pointer to lockref structure
161 * Return: 1 if count updated successfully or 0 if count <= 1 and lock taken
162 */
lockref_put_or_lock(struct lockref * lockref)163 int lockref_put_or_lock(struct lockref *lockref)
164 {
165 CMPXCHG_LOOP(
166 new.count--;
167 if (old.count <= 1)
168 break;
169 ,
170 return 1;
171 );
172
173 spin_lock(&lockref->lock);
174 if (lockref->count <= 1)
175 return 0;
176 lockref->count--;
177 spin_unlock(&lockref->lock);
178 return 1;
179 }
180 EXPORT_SYMBOL(lockref_put_or_lock);
181
182 /**
183 * lockref_mark_dead - mark lockref dead
184 * @lockref: pointer to lockref structure
185 */
lockref_mark_dead(struct lockref * lockref)186 void lockref_mark_dead(struct lockref *lockref)
187 {
188 assert_spin_locked(&lockref->lock);
189 lockref->count = -128;
190 }
191 EXPORT_SYMBOL(lockref_mark_dead);
192
193 /**
194 * lockref_get_not_dead - Increments count unless the ref is dead
195 * @lockref: pointer to lockref structure
196 * Return: 1 if count updated successfully or 0 if lockref was dead
197 */
lockref_get_not_dead(struct lockref * lockref)198 int lockref_get_not_dead(struct lockref *lockref)
199 {
200 int retval;
201
202 CMPXCHG_LOOP(
203 new.count++;
204 if (old.count < 0)
205 return 0;
206 ,
207 return 1;
208 );
209
210 spin_lock(&lockref->lock);
211 retval = 0;
212 if (lockref->count >= 0) {
213 lockref->count++;
214 retval = 1;
215 }
216 spin_unlock(&lockref->lock);
217 return retval;
218 }
219 EXPORT_SYMBOL(lockref_get_not_dead);
220