1 /*
2 * Copyright (C) 2015-2017 Netronome Systems, Inc.
3 *
4 * This software is dual licensed under the GNU General License Version 2,
5 * June 1991 as shown in the file COPYING in the top-level directory of this
6 * source tree or the BSD 2-Clause License provided below. You have the
7 * option to license this software under the complete terms of either license.
8 *
9 * The BSD 2-Clause License:
10 *
11 * Redistribution and use in source and binary forms, with or
12 * without modification, are permitted provided that the following
13 * conditions are met:
14 *
15 * 1. Redistributions of source code must retain the above
16 * copyright notice, this list of conditions and the following
17 * disclaimer.
18 *
19 * 2. Redistributions in binary form must reproduce the above
20 * copyright notice, this list of conditions and the following
21 * disclaimer in the documentation and/or other materials
22 * provided with the distribution.
23 *
24 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
25 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
26 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
27 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
28 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
29 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
30 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
31 * SOFTWARE.
32 */
33
34 #include <linux/delay.h>
35 #include <linux/device.h>
36 #include <linux/jiffies.h>
37 #include <linux/types.h>
38 #include <linux/slab.h>
39 #include <linux/wait.h>
40
41 #include "nfp_cpp.h"
42 #include "nfp6000/nfp6000.h"
43
44 struct nfp_cpp_mutex {
45 struct nfp_cpp *cpp;
46 int target;
47 u16 depth;
48 unsigned long long address;
49 u32 key;
50 };
51
nfp_mutex_locked(u16 interface)52 static u32 nfp_mutex_locked(u16 interface)
53 {
54 return (u32)interface << 16 | 0x000f;
55 }
56
nfp_mutex_unlocked(u16 interface)57 static u32 nfp_mutex_unlocked(u16 interface)
58 {
59 return (u32)interface << 16 | 0x0000;
60 }
61
nfp_mutex_is_locked(u32 val)62 static bool nfp_mutex_is_locked(u32 val)
63 {
64 return (val & 0xffff) == 0x000f;
65 }
66
nfp_mutex_is_unlocked(u32 val)67 static bool nfp_mutex_is_unlocked(u32 val)
68 {
69 return (val & 0xffff) == 0000;
70 }
71
72 /* If you need more than 65536 recursive locks, please rethink your code. */
73 #define NFP_MUTEX_DEPTH_MAX 0xffff
74
75 static int
nfp_cpp_mutex_validate(u16 interface,int * target,unsigned long long address)76 nfp_cpp_mutex_validate(u16 interface, int *target, unsigned long long address)
77 {
78 /* Not permitted on invalid interfaces */
79 if (NFP_CPP_INTERFACE_TYPE_of(interface) ==
80 NFP_CPP_INTERFACE_TYPE_INVALID)
81 return -EINVAL;
82
83 /* Address must be 64-bit aligned */
84 if (address & 7)
85 return -EINVAL;
86
87 if (*target != NFP_CPP_TARGET_MU)
88 return -EINVAL;
89
90 return 0;
91 }
92
93 /**
94 * nfp_cpp_mutex_init() - Initialize a mutex location
95 * @cpp: NFP CPP handle
96 * @target: NFP CPP target ID (ie NFP_CPP_TARGET_CLS or NFP_CPP_TARGET_MU)
97 * @address: Offset into the address space of the NFP CPP target ID
98 * @key: Unique 32-bit value for this mutex
99 *
100 * The CPP target:address must point to a 64-bit aligned location, and
101 * will initialize 64 bits of data at the location.
102 *
103 * This creates the initial mutex state, as locked by this
104 * nfp_cpp_interface().
105 *
106 * This function should only be called when setting up
107 * the initial lock state upon boot-up of the system.
108 *
109 * Return: 0 on success, or -errno on failure
110 */
nfp_cpp_mutex_init(struct nfp_cpp * cpp,int target,unsigned long long address,u32 key)111 int nfp_cpp_mutex_init(struct nfp_cpp *cpp,
112 int target, unsigned long long address, u32 key)
113 {
114 const u32 muw = NFP_CPP_ID(target, 4, 0); /* atomic_write */
115 u16 interface = nfp_cpp_interface(cpp);
116 int err;
117
118 err = nfp_cpp_mutex_validate(interface, &target, address);
119 if (err)
120 return err;
121
122 err = nfp_cpp_writel(cpp, muw, address + 4, key);
123 if (err)
124 return err;
125
126 err = nfp_cpp_writel(cpp, muw, address, nfp_mutex_locked(interface));
127 if (err)
128 return err;
129
130 return 0;
131 }
132
133 /**
134 * nfp_cpp_mutex_alloc() - Create a mutex handle
135 * @cpp: NFP CPP handle
136 * @target: NFP CPP target ID (ie NFP_CPP_TARGET_CLS or NFP_CPP_TARGET_MU)
137 * @address: Offset into the address space of the NFP CPP target ID
138 * @key: 32-bit unique key (must match the key at this location)
139 *
140 * The CPP target:address must point to a 64-bit aligned location, and
141 * reserve 64 bits of data at the location for use by the handle.
142 *
143 * Only target/address pairs that point to entities that support the
144 * MU Atomic Engine's CmpAndSwap32 command are supported.
145 *
146 * Return: A non-NULL struct nfp_cpp_mutex * on success, NULL on failure.
147 */
nfp_cpp_mutex_alloc(struct nfp_cpp * cpp,int target,unsigned long long address,u32 key)148 struct nfp_cpp_mutex *nfp_cpp_mutex_alloc(struct nfp_cpp *cpp, int target,
149 unsigned long long address, u32 key)
150 {
151 const u32 mur = NFP_CPP_ID(target, 3, 0); /* atomic_read */
152 u16 interface = nfp_cpp_interface(cpp);
153 struct nfp_cpp_mutex *mutex;
154 int err;
155 u32 tmp;
156
157 err = nfp_cpp_mutex_validate(interface, &target, address);
158 if (err)
159 return NULL;
160
161 err = nfp_cpp_readl(cpp, mur, address + 4, &tmp);
162 if (err < 0)
163 return NULL;
164
165 if (tmp != key)
166 return NULL;
167
168 mutex = kzalloc(sizeof(*mutex), GFP_KERNEL);
169 if (!mutex)
170 return NULL;
171
172 mutex->cpp = cpp;
173 mutex->target = target;
174 mutex->address = address;
175 mutex->key = key;
176 mutex->depth = 0;
177
178 return mutex;
179 }
180
181 /**
182 * nfp_cpp_mutex_free() - Free a mutex handle - does not alter the lock state
183 * @mutex: NFP CPP Mutex handle
184 */
nfp_cpp_mutex_free(struct nfp_cpp_mutex * mutex)185 void nfp_cpp_mutex_free(struct nfp_cpp_mutex *mutex)
186 {
187 kfree(mutex);
188 }
189
190 /**
191 * nfp_cpp_mutex_lock() - Lock a mutex handle, using the NFP MU Atomic Engine
192 * @mutex: NFP CPP Mutex handle
193 *
194 * Return: 0 on success, or -errno on failure
195 */
nfp_cpp_mutex_lock(struct nfp_cpp_mutex * mutex)196 int nfp_cpp_mutex_lock(struct nfp_cpp_mutex *mutex)
197 {
198 unsigned long warn_at = jiffies + NFP_MUTEX_WAIT_FIRST_WARN * HZ;
199 unsigned long err_at = jiffies + NFP_MUTEX_WAIT_ERROR * HZ;
200 unsigned int timeout_ms = 1;
201 int err;
202
203 /* We can't use a waitqueue here, because the unlocker
204 * might be on a separate CPU.
205 *
206 * So just wait for now.
207 */
208 for (;;) {
209 err = nfp_cpp_mutex_trylock(mutex);
210 if (err != -EBUSY)
211 break;
212
213 err = msleep_interruptible(timeout_ms);
214 if (err != 0)
215 return -ERESTARTSYS;
216
217 if (time_is_before_eq_jiffies(warn_at)) {
218 warn_at = jiffies + NFP_MUTEX_WAIT_NEXT_WARN * HZ;
219 nfp_warn(mutex->cpp,
220 "Warning: waiting for NFP mutex [depth:%hd target:%d addr:%llx key:%08x]\n",
221 mutex->depth,
222 mutex->target, mutex->address, mutex->key);
223 }
224 if (time_is_before_eq_jiffies(err_at)) {
225 nfp_err(mutex->cpp, "Error: mutex wait timed out\n");
226 return -EBUSY;
227 }
228 }
229
230 return err;
231 }
232
233 /**
234 * nfp_cpp_mutex_unlock() - Unlock a mutex handle, using the MU Atomic Engine
235 * @mutex: NFP CPP Mutex handle
236 *
237 * Return: 0 on success, or -errno on failure
238 */
nfp_cpp_mutex_unlock(struct nfp_cpp_mutex * mutex)239 int nfp_cpp_mutex_unlock(struct nfp_cpp_mutex *mutex)
240 {
241 const u32 muw = NFP_CPP_ID(mutex->target, 4, 0); /* atomic_write */
242 const u32 mur = NFP_CPP_ID(mutex->target, 3, 0); /* atomic_read */
243 struct nfp_cpp *cpp = mutex->cpp;
244 u32 key, value;
245 u16 interface;
246 int err;
247
248 interface = nfp_cpp_interface(cpp);
249
250 if (mutex->depth > 1) {
251 mutex->depth--;
252 return 0;
253 }
254
255 err = nfp_cpp_readl(mutex->cpp, mur, mutex->address + 4, &key);
256 if (err < 0)
257 return err;
258
259 if (key != mutex->key)
260 return -EPERM;
261
262 err = nfp_cpp_readl(mutex->cpp, mur, mutex->address, &value);
263 if (err < 0)
264 return err;
265
266 if (value != nfp_mutex_locked(interface))
267 return -EACCES;
268
269 err = nfp_cpp_writel(cpp, muw, mutex->address,
270 nfp_mutex_unlocked(interface));
271 if (err < 0)
272 return err;
273
274 mutex->depth = 0;
275 return 0;
276 }
277
278 /**
279 * nfp_cpp_mutex_trylock() - Attempt to lock a mutex handle
280 * @mutex: NFP CPP Mutex handle
281 *
282 * Return: 0 if the lock succeeded, -errno on failure
283 */
nfp_cpp_mutex_trylock(struct nfp_cpp_mutex * mutex)284 int nfp_cpp_mutex_trylock(struct nfp_cpp_mutex *mutex)
285 {
286 const u32 muw = NFP_CPP_ID(mutex->target, 4, 0); /* atomic_write */
287 const u32 mus = NFP_CPP_ID(mutex->target, 5, 3); /* test_set_imm */
288 const u32 mur = NFP_CPP_ID(mutex->target, 3, 0); /* atomic_read */
289 struct nfp_cpp *cpp = mutex->cpp;
290 u32 key, value, tmp;
291 int err;
292
293 if (mutex->depth > 0) {
294 if (mutex->depth == NFP_MUTEX_DEPTH_MAX)
295 return -E2BIG;
296 mutex->depth++;
297 return 0;
298 }
299
300 /* Verify that the lock marker is not damaged */
301 err = nfp_cpp_readl(cpp, mur, mutex->address + 4, &key);
302 if (err < 0)
303 return err;
304
305 if (key != mutex->key)
306 return -EPERM;
307
308 /* Compare against the unlocked state, and if true,
309 * write the interface id into the top 16 bits, and
310 * mark as locked.
311 */
312 value = nfp_mutex_locked(nfp_cpp_interface(cpp));
313
314 /* We use test_set_imm here, as it implies a read
315 * of the current state, and sets the bits in the
316 * bytemask of the command to 1s. Since the mutex
317 * is guaranteed to be 64-bit aligned, the bytemask
318 * of this 32-bit command is ensured to be 8'b00001111,
319 * which implies that the lower 4 bits will be set to
320 * ones regardless of the initial state.
321 *
322 * Since this is a 'Readback' operation, with no Pull
323 * data, we can treat this as a normal Push (read)
324 * atomic, which returns the original value.
325 */
326 err = nfp_cpp_readl(cpp, mus, mutex->address, &tmp);
327 if (err < 0)
328 return err;
329
330 /* Was it unlocked? */
331 if (nfp_mutex_is_unlocked(tmp)) {
332 /* The read value can only be 0x....0000 in the unlocked state.
333 * If there was another contending for this lock, then
334 * the lock state would be 0x....000f
335 */
336
337 /* Write our owner ID into the lock
338 * While not strictly necessary, this helps with
339 * debug and bookkeeping.
340 */
341 err = nfp_cpp_writel(cpp, muw, mutex->address, value);
342 if (err < 0)
343 return err;
344
345 mutex->depth = 1;
346 return 0;
347 }
348
349 return nfp_mutex_is_locked(tmp) ? -EBUSY : -EINVAL;
350 }
351