1 #include <linux/export.h>
2 #include <linux/lockref.h>
3
4 #if USE_CMPXCHG_LOCKREF
5
6 /*
7 * Allow weakly-ordered memory architectures to provide barrier-less
8 * cmpxchg semantics for lockref updates.
9 */
10 #ifndef cmpxchg64_relaxed
11 # define cmpxchg64_relaxed cmpxchg64
12 #endif
13
14 /*
15 * Note that the "cmpxchg()" reloads the "old" value for the
16 * failure case.
17 */
18 #define CMPXCHG_LOOP(CODE, SUCCESS) do { \
19 struct lockref old; \
20 BUILD_BUG_ON(sizeof(old) != 8); \
21 old.lock_count = ACCESS_ONCE(lockref->lock_count); \
22 while (likely(arch_spin_value_unlocked(old.lock.rlock.raw_lock))) { \
23 struct lockref new = old, prev = old; \
24 CODE \
25 old.lock_count = cmpxchg64_relaxed(&lockref->lock_count, \
26 old.lock_count, \
27 new.lock_count); \
28 if (likely(old.lock_count == prev.lock_count)) { \
29 SUCCESS; \
30 } \
31 cpu_relax_lowlatency(); \
32 } \
33 } while (0)
34
35 #else
36
37 #define CMPXCHG_LOOP(CODE, SUCCESS) do { } while (0)
38
39 #endif
40
41 /**
42 * lockref_get - Increments reference count unconditionally
43 * @lockref: pointer to lockref structure
44 *
45 * This operation is only valid if you already hold a reference
46 * to the object, so you know the count cannot be zero.
47 */
lockref_get(struct lockref * lockref)48 void lockref_get(struct lockref *lockref)
49 {
50 CMPXCHG_LOOP(
51 new.count++;
52 ,
53 return;
54 );
55
56 spin_lock(&lockref->lock);
57 lockref->count++;
58 spin_unlock(&lockref->lock);
59 }
60 EXPORT_SYMBOL(lockref_get);
61
62 /**
63 * lockref_get_not_zero - Increments count unless the count is 0 or dead
64 * @lockref: pointer to lockref structure
65 * Return: 1 if count updated successfully or 0 if count was zero
66 */
lockref_get_not_zero(struct lockref * lockref)67 int lockref_get_not_zero(struct lockref *lockref)
68 {
69 int retval;
70
71 CMPXCHG_LOOP(
72 new.count++;
73 if (old.count <= 0)
74 return 0;
75 ,
76 return 1;
77 );
78
79 spin_lock(&lockref->lock);
80 retval = 0;
81 if (lockref->count > 0) {
82 lockref->count++;
83 retval = 1;
84 }
85 spin_unlock(&lockref->lock);
86 return retval;
87 }
88 EXPORT_SYMBOL(lockref_get_not_zero);
89
90 /**
91 * lockref_get_or_lock - Increments count unless the count is 0 or dead
92 * @lockref: pointer to lockref structure
93 * Return: 1 if count updated successfully or 0 if count was zero
94 * and we got the lock instead.
95 */
lockref_get_or_lock(struct lockref * lockref)96 int lockref_get_or_lock(struct lockref *lockref)
97 {
98 CMPXCHG_LOOP(
99 new.count++;
100 if (old.count <= 0)
101 break;
102 ,
103 return 1;
104 );
105
106 spin_lock(&lockref->lock);
107 if (lockref->count <= 0)
108 return 0;
109 lockref->count++;
110 spin_unlock(&lockref->lock);
111 return 1;
112 }
113 EXPORT_SYMBOL(lockref_get_or_lock);
114
115 /**
116 * lockref_put_return - Decrement reference count if possible
117 * @lockref: pointer to lockref structure
118 *
119 * Decrement the reference count and return the new value.
120 * If the lockref was dead or locked, return an error.
121 */
lockref_put_return(struct lockref * lockref)122 int lockref_put_return(struct lockref *lockref)
123 {
124 CMPXCHG_LOOP(
125 new.count--;
126 if (old.count <= 0)
127 return -1;
128 ,
129 return new.count;
130 );
131 return -1;
132 }
133 EXPORT_SYMBOL(lockref_put_return);
134
135 /**
136 * lockref_put_or_lock - decrements count unless count <= 1 before decrement
137 * @lockref: pointer to lockref structure
138 * Return: 1 if count updated successfully or 0 if count <= 1 and lock taken
139 */
lockref_put_or_lock(struct lockref * lockref)140 int lockref_put_or_lock(struct lockref *lockref)
141 {
142 CMPXCHG_LOOP(
143 new.count--;
144 if (old.count <= 1)
145 break;
146 ,
147 return 1;
148 );
149
150 spin_lock(&lockref->lock);
151 if (lockref->count <= 1)
152 return 0;
153 lockref->count--;
154 spin_unlock(&lockref->lock);
155 return 1;
156 }
157 EXPORT_SYMBOL(lockref_put_or_lock);
158
159 /**
160 * lockref_mark_dead - mark lockref dead
161 * @lockref: pointer to lockref structure
162 */
lockref_mark_dead(struct lockref * lockref)163 void lockref_mark_dead(struct lockref *lockref)
164 {
165 assert_spin_locked(&lockref->lock);
166 lockref->count = -128;
167 }
168 EXPORT_SYMBOL(lockref_mark_dead);
169
170 /**
171 * lockref_get_not_dead - Increments count unless the ref is dead
172 * @lockref: pointer to lockref structure
173 * Return: 1 if count updated successfully or 0 if lockref was dead
174 */
lockref_get_not_dead(struct lockref * lockref)175 int lockref_get_not_dead(struct lockref *lockref)
176 {
177 int retval;
178
179 CMPXCHG_LOOP(
180 new.count++;
181 if (old.count < 0)
182 return 0;
183 ,
184 return 1;
185 );
186
187 spin_lock(&lockref->lock);
188 retval = 0;
189 if (lockref->count >= 0) {
190 lockref->count++;
191 retval = 1;
192 }
193 spin_unlock(&lockref->lock);
194 return retval;
195 }
196 EXPORT_SYMBOL(lockref_get_not_dead);
197