• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 // SPDX-License-Identifier: GPL-2.0
2 #include <linux/export.h>
3 #include <linux/lockref.h>
4 
5 #if USE_CMPXCHG_LOCKREF
6 
7 /*
8  * Note that the "cmpxchg()" reloads the "old" value for the
9  * failure case.
10  */
11 #define CMPXCHG_LOOP(CODE, SUCCESS) do {					\
12 	int retry = 100;							\
13 	struct lockref old;							\
14 	BUILD_BUG_ON(sizeof(old) != 8);						\
15 	old.lock_count = READ_ONCE(lockref->lock_count);			\
16 	while (likely(arch_spin_value_unlocked(old.lock.rlock.raw_lock))) {  	\
17 		struct lockref new = old, prev = old;				\
18 		CODE								\
19 		old.lock_count = cmpxchg64_relaxed(&lockref->lock_count,	\
20 						   old.lock_count,		\
21 						   new.lock_count);		\
22 		if (likely(old.lock_count == prev.lock_count)) {		\
23 			SUCCESS;						\
24 		}								\
25 		if (!--retry)							\
26 			break;							\
27 	}									\
28 } while (0)
29 
30 #else
31 
32 #define CMPXCHG_LOOP(CODE, SUCCESS) do { } while (0)
33 
34 #endif
35 
36 /**
37  * lockref_get - Increments reference count unconditionally
38  * @lockref: pointer to lockref structure
39  *
40  * This operation is only valid if you already hold a reference
41  * to the object, so you know the count cannot be zero.
42  */
lockref_get(struct lockref * lockref)43 void lockref_get(struct lockref *lockref)
44 {
45 	CMPXCHG_LOOP(
46 		new.count++;
47 	,
48 		return;
49 	);
50 
51 	spin_lock(&lockref->lock);
52 	lockref->count++;
53 	spin_unlock(&lockref->lock);
54 }
55 EXPORT_SYMBOL(lockref_get);
56 
57 /**
58  * lockref_get_not_zero - Increments count unless the count is 0 or dead
59  * @lockref: pointer to lockref structure
60  * Return: 1 if count updated successfully or 0 if count was zero
61  */
lockref_get_not_zero(struct lockref * lockref)62 int lockref_get_not_zero(struct lockref *lockref)
63 {
64 	int retval;
65 
66 	CMPXCHG_LOOP(
67 		new.count++;
68 		if (old.count <= 0)
69 			return 0;
70 	,
71 		return 1;
72 	);
73 
74 	spin_lock(&lockref->lock);
75 	retval = 0;
76 	if (lockref->count > 0) {
77 		lockref->count++;
78 		retval = 1;
79 	}
80 	spin_unlock(&lockref->lock);
81 	return retval;
82 }
83 EXPORT_SYMBOL(lockref_get_not_zero);
84 
85 /**
86  * lockref_put_not_zero - Decrements count unless count <= 1 before decrement
87  * @lockref: pointer to lockref structure
88  * Return: 1 if count updated successfully or 0 if count would become zero
89  */
lockref_put_not_zero(struct lockref * lockref)90 int lockref_put_not_zero(struct lockref *lockref)
91 {
92 	int retval;
93 
94 	CMPXCHG_LOOP(
95 		new.count--;
96 		if (old.count <= 1)
97 			return 0;
98 	,
99 		return 1;
100 	);
101 
102 	spin_lock(&lockref->lock);
103 	retval = 0;
104 	if (lockref->count > 1) {
105 		lockref->count--;
106 		retval = 1;
107 	}
108 	spin_unlock(&lockref->lock);
109 	return retval;
110 }
111 EXPORT_SYMBOL(lockref_put_not_zero);
112 
113 /**
114  * lockref_get_or_lock - Increments count unless the count is 0 or dead
115  * @lockref: pointer to lockref structure
116  * Return: 1 if count updated successfully or 0 if count was zero
117  * and we got the lock instead.
118  */
lockref_get_or_lock(struct lockref * lockref)119 int lockref_get_or_lock(struct lockref *lockref)
120 {
121 	CMPXCHG_LOOP(
122 		new.count++;
123 		if (old.count <= 0)
124 			break;
125 	,
126 		return 1;
127 	);
128 
129 	spin_lock(&lockref->lock);
130 	if (lockref->count <= 0)
131 		return 0;
132 	lockref->count++;
133 	spin_unlock(&lockref->lock);
134 	return 1;
135 }
136 EXPORT_SYMBOL(lockref_get_or_lock);
137 
138 /**
139  * lockref_put_return - Decrement reference count if possible
140  * @lockref: pointer to lockref structure
141  *
142  * Decrement the reference count and return the new value.
143  * If the lockref was dead or locked, return an error.
144  */
lockref_put_return(struct lockref * lockref)145 int lockref_put_return(struct lockref *lockref)
146 {
147 	CMPXCHG_LOOP(
148 		new.count--;
149 		if (old.count <= 0)
150 			return -1;
151 	,
152 		return new.count;
153 	);
154 	return -1;
155 }
156 EXPORT_SYMBOL(lockref_put_return);
157 
158 /**
159  * lockref_put_or_lock - decrements count unless count <= 1 before decrement
160  * @lockref: pointer to lockref structure
161  * Return: 1 if count updated successfully or 0 if count <= 1 and lock taken
162  */
lockref_put_or_lock(struct lockref * lockref)163 int lockref_put_or_lock(struct lockref *lockref)
164 {
165 	CMPXCHG_LOOP(
166 		new.count--;
167 		if (old.count <= 1)
168 			break;
169 	,
170 		return 1;
171 	);
172 
173 	spin_lock(&lockref->lock);
174 	if (lockref->count <= 1)
175 		return 0;
176 	lockref->count--;
177 	spin_unlock(&lockref->lock);
178 	return 1;
179 }
180 EXPORT_SYMBOL(lockref_put_or_lock);
181 
182 /**
183  * lockref_mark_dead - mark lockref dead
184  * @lockref: pointer to lockref structure
185  */
lockref_mark_dead(struct lockref * lockref)186 void lockref_mark_dead(struct lockref *lockref)
187 {
188 	assert_spin_locked(&lockref->lock);
189 	lockref->count = -128;
190 }
191 EXPORT_SYMBOL(lockref_mark_dead);
192 
193 /**
194  * lockref_get_not_dead - Increments count unless the ref is dead
195  * @lockref: pointer to lockref structure
196  * Return: 1 if count updated successfully or 0 if lockref was dead
197  */
lockref_get_not_dead(struct lockref * lockref)198 int lockref_get_not_dead(struct lockref *lockref)
199 {
200 	int retval;
201 
202 	CMPXCHG_LOOP(
203 		new.count++;
204 		if (old.count < 0)
205 			return 0;
206 	,
207 		return 1;
208 	);
209 
210 	spin_lock(&lockref->lock);
211 	retval = 0;
212 	if (lockref->count >= 0) {
213 		lockref->count++;
214 		retval = 1;
215 	}
216 	spin_unlock(&lockref->lock);
217 	return retval;
218 }
219 EXPORT_SYMBOL(lockref_get_not_dead);
220