• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * Copyright (C) Sistina Software, Inc.  1997-2003 All rights reserved.
4  * Copyright (C) 2004-2008 Red Hat, Inc.  All rights reserved.
5  */
6 
7 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
8 
9 #include <linux/sched.h>
10 #include <linux/slab.h>
11 #include <linux/spinlock.h>
12 #include <linux/buffer_head.h>
13 #include <linux/delay.h>
14 #include <linux/sort.h>
15 #include <linux/hash.h>
16 #include <linux/jhash.h>
17 #include <linux/kallsyms.h>
18 #include <linux/gfs2_ondisk.h>
19 #include <linux/list.h>
20 #include <linux/wait.h>
21 #include <linux/module.h>
22 #include <linux/uaccess.h>
23 #include <linux/seq_file.h>
24 #include <linux/debugfs.h>
25 #include <linux/kthread.h>
26 #include <linux/freezer.h>
27 #include <linux/workqueue.h>
28 #include <linux/jiffies.h>
29 #include <linux/rcupdate.h>
30 #include <linux/rculist_bl.h>
31 #include <linux/bit_spinlock.h>
32 #include <linux/percpu.h>
33 #include <linux/list_sort.h>
34 #include <linux/lockref.h>
35 #include <linux/rhashtable.h>
36 
37 #include "gfs2.h"
38 #include "incore.h"
39 #include "glock.h"
40 #include "glops.h"
41 #include "inode.h"
42 #include "lops.h"
43 #include "meta_io.h"
44 #include "quota.h"
45 #include "super.h"
46 #include "util.h"
47 #include "bmap.h"
48 #define CREATE_TRACE_POINTS
49 #include "trace_gfs2.h"
50 
51 struct gfs2_glock_iter {
52 	struct gfs2_sbd *sdp;		/* incore superblock           */
53 	struct rhashtable_iter hti;	/* rhashtable iterator         */
54 	struct gfs2_glock *gl;		/* current glock struct        */
55 	loff_t last_pos;		/* last position               */
56 };
57 
58 typedef void (*glock_examiner) (struct gfs2_glock * gl);
59 
60 static void do_xmote(struct gfs2_glock *gl, struct gfs2_holder *gh, unsigned int target);
61 
62 static struct dentry *gfs2_root;
63 static struct workqueue_struct *glock_workqueue;
64 struct workqueue_struct *gfs2_delete_workqueue;
65 static LIST_HEAD(lru_list);
66 static atomic_t lru_count = ATOMIC_INIT(0);
67 static DEFINE_SPINLOCK(lru_lock);
68 
69 #define GFS2_GL_HASH_SHIFT      15
70 #define GFS2_GL_HASH_SIZE       BIT(GFS2_GL_HASH_SHIFT)
71 
72 static const struct rhashtable_params ht_parms = {
73 	.nelem_hint = GFS2_GL_HASH_SIZE * 3 / 4,
74 	.key_len = offsetofend(struct lm_lockname, ln_type),
75 	.key_offset = offsetof(struct gfs2_glock, gl_name),
76 	.head_offset = offsetof(struct gfs2_glock, gl_node),
77 };
78 
79 static struct rhashtable gl_hash_table;
80 
81 #define GLOCK_WAIT_TABLE_BITS 12
82 #define GLOCK_WAIT_TABLE_SIZE (1 << GLOCK_WAIT_TABLE_BITS)
83 static wait_queue_head_t glock_wait_table[GLOCK_WAIT_TABLE_SIZE] __cacheline_aligned;
84 
85 struct wait_glock_queue {
86 	struct lm_lockname *name;
87 	wait_queue_entry_t wait;
88 };
89 
glock_wake_function(wait_queue_entry_t * wait,unsigned int mode,int sync,void * key)90 static int glock_wake_function(wait_queue_entry_t *wait, unsigned int mode,
91 			       int sync, void *key)
92 {
93 	struct wait_glock_queue *wait_glock =
94 		container_of(wait, struct wait_glock_queue, wait);
95 	struct lm_lockname *wait_name = wait_glock->name;
96 	struct lm_lockname *wake_name = key;
97 
98 	if (wake_name->ln_sbd != wait_name->ln_sbd ||
99 	    wake_name->ln_number != wait_name->ln_number ||
100 	    wake_name->ln_type != wait_name->ln_type)
101 		return 0;
102 	return autoremove_wake_function(wait, mode, sync, key);
103 }
104 
glock_waitqueue(struct lm_lockname * name)105 static wait_queue_head_t *glock_waitqueue(struct lm_lockname *name)
106 {
107 	u32 hash = jhash2((u32 *)name, ht_parms.key_len / 4, 0);
108 
109 	return glock_wait_table + hash_32(hash, GLOCK_WAIT_TABLE_BITS);
110 }
111 
112 /**
113  * wake_up_glock  -  Wake up waiters on a glock
114  * @gl: the glock
115  */
wake_up_glock(struct gfs2_glock * gl)116 static void wake_up_glock(struct gfs2_glock *gl)
117 {
118 	wait_queue_head_t *wq = glock_waitqueue(&gl->gl_name);
119 
120 	if (waitqueue_active(wq))
121 		__wake_up(wq, TASK_NORMAL, 1, &gl->gl_name);
122 }
123 
gfs2_glock_dealloc(struct rcu_head * rcu)124 static void gfs2_glock_dealloc(struct rcu_head *rcu)
125 {
126 	struct gfs2_glock *gl = container_of(rcu, struct gfs2_glock, gl_rcu);
127 
128 	if (gl->gl_ops->go_flags & GLOF_ASPACE) {
129 		kmem_cache_free(gfs2_glock_aspace_cachep, gl);
130 	} else {
131 		kfree(gl->gl_lksb.sb_lvbptr);
132 		kmem_cache_free(gfs2_glock_cachep, gl);
133 	}
134 }
135 
gfs2_glock_free(struct gfs2_glock * gl)136 void gfs2_glock_free(struct gfs2_glock *gl)
137 {
138 	struct gfs2_sbd *sdp = gl->gl_name.ln_sbd;
139 
140 	BUG_ON(atomic_read(&gl->gl_revokes));
141 	rhashtable_remove_fast(&gl_hash_table, &gl->gl_node, ht_parms);
142 	smp_mb();
143 	wake_up_glock(gl);
144 	call_rcu(&gl->gl_rcu, gfs2_glock_dealloc);
145 	if (atomic_dec_and_test(&sdp->sd_glock_disposal))
146 		wake_up(&sdp->sd_glock_wait);
147 }
148 
149 /**
150  * gfs2_glock_hold() - increment reference count on glock
151  * @gl: The glock to hold
152  *
153  */
154 
gfs2_glock_hold(struct gfs2_glock * gl)155 void gfs2_glock_hold(struct gfs2_glock *gl)
156 {
157 	GLOCK_BUG_ON(gl, __lockref_is_dead(&gl->gl_lockref));
158 	lockref_get(&gl->gl_lockref);
159 }
160 
161 /**
162  * demote_ok - Check to see if it's ok to unlock a glock
163  * @gl: the glock
164  *
165  * Returns: 1 if it's ok
166  */
167 
demote_ok(const struct gfs2_glock * gl)168 static int demote_ok(const struct gfs2_glock *gl)
169 {
170 	const struct gfs2_glock_operations *glops = gl->gl_ops;
171 
172 	if (gl->gl_state == LM_ST_UNLOCKED)
173 		return 0;
174 	if (!list_empty(&gl->gl_holders))
175 		return 0;
176 	if (glops->go_demote_ok)
177 		return glops->go_demote_ok(gl);
178 	return 1;
179 }
180 
181 
gfs2_glock_add_to_lru(struct gfs2_glock * gl)182 void gfs2_glock_add_to_lru(struct gfs2_glock *gl)
183 {
184 	if (!(gl->gl_ops->go_flags & GLOF_LRU))
185 		return;
186 
187 	spin_lock(&lru_lock);
188 
189 	list_del(&gl->gl_lru);
190 	list_add_tail(&gl->gl_lru, &lru_list);
191 
192 	if (!test_bit(GLF_LRU, &gl->gl_flags)) {
193 		set_bit(GLF_LRU, &gl->gl_flags);
194 		atomic_inc(&lru_count);
195 	}
196 
197 	spin_unlock(&lru_lock);
198 }
199 
gfs2_glock_remove_from_lru(struct gfs2_glock * gl)200 static void gfs2_glock_remove_from_lru(struct gfs2_glock *gl)
201 {
202 	if (!(gl->gl_ops->go_flags & GLOF_LRU))
203 		return;
204 
205 	spin_lock(&lru_lock);
206 	if (test_bit(GLF_LRU, &gl->gl_flags)) {
207 		list_del_init(&gl->gl_lru);
208 		atomic_dec(&lru_count);
209 		clear_bit(GLF_LRU, &gl->gl_flags);
210 	}
211 	spin_unlock(&lru_lock);
212 }
213 
214 /*
215  * Enqueue the glock on the work queue.  Passes one glock reference on to the
216  * work queue.
217  */
__gfs2_glock_queue_work(struct gfs2_glock * gl,unsigned long delay)218 static void __gfs2_glock_queue_work(struct gfs2_glock *gl, unsigned long delay) {
219 	if (!queue_delayed_work(glock_workqueue, &gl->gl_work, delay)) {
220 		/*
221 		 * We are holding the lockref spinlock, and the work was still
222 		 * queued above.  The queued work (glock_work_func) takes that
223 		 * spinlock before dropping its glock reference(s), so it
224 		 * cannot have dropped them in the meantime.
225 		 */
226 		GLOCK_BUG_ON(gl, gl->gl_lockref.count < 2);
227 		gl->gl_lockref.count--;
228 	}
229 }
230 
gfs2_glock_queue_work(struct gfs2_glock * gl,unsigned long delay)231 static void gfs2_glock_queue_work(struct gfs2_glock *gl, unsigned long delay) {
232 	spin_lock(&gl->gl_lockref.lock);
233 	__gfs2_glock_queue_work(gl, delay);
234 	spin_unlock(&gl->gl_lockref.lock);
235 }
236 
__gfs2_glock_put(struct gfs2_glock * gl)237 static void __gfs2_glock_put(struct gfs2_glock *gl)
238 {
239 	struct gfs2_sbd *sdp = gl->gl_name.ln_sbd;
240 	struct address_space *mapping = gfs2_glock2aspace(gl);
241 
242 	lockref_mark_dead(&gl->gl_lockref);
243 
244 	gfs2_glock_remove_from_lru(gl);
245 	spin_unlock(&gl->gl_lockref.lock);
246 	GLOCK_BUG_ON(gl, !list_empty(&gl->gl_holders));
247 	GLOCK_BUG_ON(gl, mapping && mapping->nrpages);
248 	trace_gfs2_glock_put(gl);
249 	sdp->sd_lockstruct.ls_ops->lm_put_lock(gl);
250 }
251 
252 /*
253  * Cause the glock to be put in work queue context.
254  */
gfs2_glock_queue_put(struct gfs2_glock * gl)255 void gfs2_glock_queue_put(struct gfs2_glock *gl)
256 {
257 	gfs2_glock_queue_work(gl, 0);
258 }
259 
260 /**
261  * gfs2_glock_put() - Decrement reference count on glock
262  * @gl: The glock to put
263  *
264  */
265 
gfs2_glock_put(struct gfs2_glock * gl)266 void gfs2_glock_put(struct gfs2_glock *gl)
267 {
268 	if (lockref_put_or_lock(&gl->gl_lockref))
269 		return;
270 
271 	__gfs2_glock_put(gl);
272 }
273 
274 /**
275  * may_grant - check if its ok to grant a new lock
276  * @gl: The glock
277  * @gh: The lock request which we wish to grant
278  *
279  * Returns: true if its ok to grant the lock
280  */
281 
may_grant(const struct gfs2_glock * gl,const struct gfs2_holder * gh)282 static inline int may_grant(const struct gfs2_glock *gl, const struct gfs2_holder *gh)
283 {
284 	const struct gfs2_holder *gh_head = list_entry(gl->gl_holders.next, const struct gfs2_holder, gh_list);
285 	if ((gh->gh_state == LM_ST_EXCLUSIVE ||
286 	     gh_head->gh_state == LM_ST_EXCLUSIVE) && gh != gh_head)
287 		return 0;
288 	if (gl->gl_state == gh->gh_state)
289 		return 1;
290 	if (gh->gh_flags & GL_EXACT)
291 		return 0;
292 	if (gl->gl_state == LM_ST_EXCLUSIVE) {
293 		if (gh->gh_state == LM_ST_SHARED && gh_head->gh_state == LM_ST_SHARED)
294 			return 1;
295 		if (gh->gh_state == LM_ST_DEFERRED && gh_head->gh_state == LM_ST_DEFERRED)
296 			return 1;
297 	}
298 	if (gl->gl_state != LM_ST_UNLOCKED && (gh->gh_flags & LM_FLAG_ANY))
299 		return 1;
300 	return 0;
301 }
302 
gfs2_holder_wake(struct gfs2_holder * gh)303 static void gfs2_holder_wake(struct gfs2_holder *gh)
304 {
305 	clear_bit(HIF_WAIT, &gh->gh_iflags);
306 	smp_mb__after_atomic();
307 	wake_up_bit(&gh->gh_iflags, HIF_WAIT);
308 	if (gh->gh_flags & GL_ASYNC) {
309 		struct gfs2_sbd *sdp = gh->gh_gl->gl_name.ln_sbd;
310 
311 		wake_up(&sdp->sd_async_glock_wait);
312 	}
313 }
314 
315 /**
316  * do_error - Something unexpected has happened during a lock request
317  *
318  */
319 
do_error(struct gfs2_glock * gl,const int ret)320 static void do_error(struct gfs2_glock *gl, const int ret)
321 {
322 	struct gfs2_holder *gh, *tmp;
323 
324 	list_for_each_entry_safe(gh, tmp, &gl->gl_holders, gh_list) {
325 		if (test_bit(HIF_HOLDER, &gh->gh_iflags))
326 			continue;
327 		if (ret & LM_OUT_ERROR)
328 			gh->gh_error = -EIO;
329 		else if (gh->gh_flags & (LM_FLAG_TRY | LM_FLAG_TRY_1CB))
330 			gh->gh_error = GLR_TRYFAILED;
331 		else
332 			continue;
333 		list_del_init(&gh->gh_list);
334 		trace_gfs2_glock_queue(gh, 0);
335 		gfs2_holder_wake(gh);
336 	}
337 }
338 
339 /**
340  * do_promote - promote as many requests as possible on the current queue
341  * @gl: The glock
342  *
343  * Returns: 1 if there is a blocked holder at the head of the list, or 2
344  *          if a type specific operation is underway.
345  */
346 
do_promote(struct gfs2_glock * gl)347 static int do_promote(struct gfs2_glock *gl)
348 __releases(&gl->gl_lockref.lock)
349 __acquires(&gl->gl_lockref.lock)
350 {
351 	const struct gfs2_glock_operations *glops = gl->gl_ops;
352 	struct gfs2_holder *gh, *tmp;
353 	int ret;
354 
355 restart:
356 	list_for_each_entry_safe(gh, tmp, &gl->gl_holders, gh_list) {
357 		if (test_bit(HIF_HOLDER, &gh->gh_iflags))
358 			continue;
359 		if (may_grant(gl, gh)) {
360 			if (gh->gh_list.prev == &gl->gl_holders &&
361 			    glops->go_lock) {
362 				spin_unlock(&gl->gl_lockref.lock);
363 				/* FIXME: eliminate this eventually */
364 				ret = glops->go_lock(gh);
365 				spin_lock(&gl->gl_lockref.lock);
366 				if (ret) {
367 					if (ret == 1)
368 						return 2;
369 					gh->gh_error = ret;
370 					list_del_init(&gh->gh_list);
371 					trace_gfs2_glock_queue(gh, 0);
372 					gfs2_holder_wake(gh);
373 					goto restart;
374 				}
375 				set_bit(HIF_HOLDER, &gh->gh_iflags);
376 				trace_gfs2_promote(gh, 1);
377 				gfs2_holder_wake(gh);
378 				goto restart;
379 			}
380 			set_bit(HIF_HOLDER, &gh->gh_iflags);
381 			trace_gfs2_promote(gh, 0);
382 			gfs2_holder_wake(gh);
383 			continue;
384 		}
385 		if (gh->gh_list.prev == &gl->gl_holders)
386 			return 1;
387 		do_error(gl, 0);
388 		break;
389 	}
390 	return 0;
391 }
392 
393 /**
394  * find_first_waiter - find the first gh that's waiting for the glock
395  * @gl: the glock
396  */
397 
find_first_waiter(const struct gfs2_glock * gl)398 static inline struct gfs2_holder *find_first_waiter(const struct gfs2_glock *gl)
399 {
400 	struct gfs2_holder *gh;
401 
402 	list_for_each_entry(gh, &gl->gl_holders, gh_list) {
403 		if (!test_bit(HIF_HOLDER, &gh->gh_iflags))
404 			return gh;
405 	}
406 	return NULL;
407 }
408 
409 /**
410  * state_change - record that the glock is now in a different state
411  * @gl: the glock
412  * @new_state the new state
413  *
414  */
415 
state_change(struct gfs2_glock * gl,unsigned int new_state)416 static void state_change(struct gfs2_glock *gl, unsigned int new_state)
417 {
418 	int held1, held2;
419 
420 	held1 = (gl->gl_state != LM_ST_UNLOCKED);
421 	held2 = (new_state != LM_ST_UNLOCKED);
422 
423 	if (held1 != held2) {
424 		GLOCK_BUG_ON(gl, __lockref_is_dead(&gl->gl_lockref));
425 		if (held2)
426 			gl->gl_lockref.count++;
427 		else
428 			gl->gl_lockref.count--;
429 	}
430 	if (held1 && held2 && list_empty(&gl->gl_holders))
431 		clear_bit(GLF_QUEUED, &gl->gl_flags);
432 
433 	if (new_state != gl->gl_target)
434 		/* shorten our minimum hold time */
435 		gl->gl_hold_time = max(gl->gl_hold_time - GL_GLOCK_HOLD_DECR,
436 				       GL_GLOCK_MIN_HOLD);
437 	gl->gl_state = new_state;
438 	gl->gl_tchange = jiffies;
439 }
440 
gfs2_demote_wake(struct gfs2_glock * gl)441 static void gfs2_demote_wake(struct gfs2_glock *gl)
442 {
443 	gl->gl_demote_state = LM_ST_EXCLUSIVE;
444 	clear_bit(GLF_DEMOTE, &gl->gl_flags);
445 	smp_mb__after_atomic();
446 	wake_up_bit(&gl->gl_flags, GLF_DEMOTE);
447 }
448 
449 /**
450  * finish_xmote - The DLM has replied to one of our lock requests
451  * @gl: The glock
452  * @ret: The status from the DLM
453  *
454  */
455 
finish_xmote(struct gfs2_glock * gl,unsigned int ret)456 static void finish_xmote(struct gfs2_glock *gl, unsigned int ret)
457 {
458 	const struct gfs2_glock_operations *glops = gl->gl_ops;
459 	struct gfs2_holder *gh;
460 	unsigned state = ret & LM_OUT_ST_MASK;
461 	int rv;
462 
463 	spin_lock(&gl->gl_lockref.lock);
464 	trace_gfs2_glock_state_change(gl, state);
465 	state_change(gl, state);
466 	gh = find_first_waiter(gl);
467 
468 	/* Demote to UN request arrived during demote to SH or DF */
469 	if (test_bit(GLF_DEMOTE_IN_PROGRESS, &gl->gl_flags) &&
470 	    state != LM_ST_UNLOCKED && gl->gl_demote_state == LM_ST_UNLOCKED)
471 		gl->gl_target = LM_ST_UNLOCKED;
472 
473 	/* Check for state != intended state */
474 	if (unlikely(state != gl->gl_target)) {
475 		if (gh && !test_bit(GLF_DEMOTE_IN_PROGRESS, &gl->gl_flags)) {
476 			/* move to back of queue and try next entry */
477 			if (ret & LM_OUT_CANCELED) {
478 				if ((gh->gh_flags & LM_FLAG_PRIORITY) == 0)
479 					list_move_tail(&gh->gh_list, &gl->gl_holders);
480 				gh = find_first_waiter(gl);
481 				gl->gl_target = gh->gh_state;
482 				goto retry;
483 			}
484 			/* Some error or failed "try lock" - report it */
485 			if ((ret & LM_OUT_ERROR) ||
486 			    (gh->gh_flags & (LM_FLAG_TRY | LM_FLAG_TRY_1CB))) {
487 				gl->gl_target = gl->gl_state;
488 				do_error(gl, ret);
489 				goto out;
490 			}
491 		}
492 		switch(state) {
493 		/* Unlocked due to conversion deadlock, try again */
494 		case LM_ST_UNLOCKED:
495 retry:
496 			do_xmote(gl, gh, gl->gl_target);
497 			break;
498 		/* Conversion fails, unlock and try again */
499 		case LM_ST_SHARED:
500 		case LM_ST_DEFERRED:
501 			do_xmote(gl, gh, LM_ST_UNLOCKED);
502 			break;
503 		default: /* Everything else */
504 			fs_err(gl->gl_name.ln_sbd, "wanted %u got %u\n",
505 			       gl->gl_target, state);
506 			GLOCK_BUG_ON(gl, 1);
507 		}
508 		spin_unlock(&gl->gl_lockref.lock);
509 		return;
510 	}
511 
512 	/* Fast path - we got what we asked for */
513 	if (test_and_clear_bit(GLF_DEMOTE_IN_PROGRESS, &gl->gl_flags))
514 		gfs2_demote_wake(gl);
515 	if (state != LM_ST_UNLOCKED) {
516 		if (glops->go_xmote_bh) {
517 			spin_unlock(&gl->gl_lockref.lock);
518 			rv = glops->go_xmote_bh(gl, gh);
519 			spin_lock(&gl->gl_lockref.lock);
520 			if (rv) {
521 				do_error(gl, rv);
522 				goto out;
523 			}
524 		}
525 		rv = do_promote(gl);
526 		if (rv == 2)
527 			goto out_locked;
528 	}
529 out:
530 	clear_bit(GLF_LOCK, &gl->gl_flags);
531 out_locked:
532 	spin_unlock(&gl->gl_lockref.lock);
533 }
534 
535 /**
536  * do_xmote - Calls the DLM to change the state of a lock
537  * @gl: The lock state
538  * @gh: The holder (only for promotes)
539  * @target: The target lock state
540  *
541  */
542 
do_xmote(struct gfs2_glock * gl,struct gfs2_holder * gh,unsigned int target)543 static void do_xmote(struct gfs2_glock *gl, struct gfs2_holder *gh, unsigned int target)
544 __releases(&gl->gl_lockref.lock)
545 __acquires(&gl->gl_lockref.lock)
546 {
547 	const struct gfs2_glock_operations *glops = gl->gl_ops;
548 	struct gfs2_sbd *sdp = gl->gl_name.ln_sbd;
549 	unsigned int lck_flags = (unsigned int)(gh ? gh->gh_flags : 0);
550 	int ret;
551 
552 	if (unlikely(test_bit(SDF_WITHDRAWN, &sdp->sd_flags)) &&
553 	    target != LM_ST_UNLOCKED)
554 		return;
555 	lck_flags &= (LM_FLAG_TRY | LM_FLAG_TRY_1CB | LM_FLAG_NOEXP |
556 		      LM_FLAG_PRIORITY);
557 	GLOCK_BUG_ON(gl, gl->gl_state == target);
558 	GLOCK_BUG_ON(gl, gl->gl_state == gl->gl_target);
559 	if ((target == LM_ST_UNLOCKED || target == LM_ST_DEFERRED) &&
560 	    glops->go_inval) {
561 		set_bit(GLF_INVALIDATE_IN_PROGRESS, &gl->gl_flags);
562 		do_error(gl, 0); /* Fail queued try locks */
563 	}
564 	gl->gl_req = target;
565 	set_bit(GLF_BLOCKING, &gl->gl_flags);
566 	if ((gl->gl_req == LM_ST_UNLOCKED) ||
567 	    (gl->gl_state == LM_ST_EXCLUSIVE) ||
568 	    (lck_flags & (LM_FLAG_TRY|LM_FLAG_TRY_1CB)))
569 		clear_bit(GLF_BLOCKING, &gl->gl_flags);
570 	spin_unlock(&gl->gl_lockref.lock);
571 	if (glops->go_sync)
572 		glops->go_sync(gl);
573 	if (test_bit(GLF_INVALIDATE_IN_PROGRESS, &gl->gl_flags))
574 		glops->go_inval(gl, target == LM_ST_DEFERRED ? 0 : DIO_METADATA);
575 	clear_bit(GLF_INVALIDATE_IN_PROGRESS, &gl->gl_flags);
576 
577 	gfs2_glock_hold(gl);
578 	if (sdp->sd_lockstruct.ls_ops->lm_lock)	{
579 		/* lock_dlm */
580 		ret = sdp->sd_lockstruct.ls_ops->lm_lock(gl, target, lck_flags);
581 		if (ret == -EINVAL && gl->gl_target == LM_ST_UNLOCKED &&
582 		    target == LM_ST_UNLOCKED &&
583 		    test_bit(SDF_SKIP_DLM_UNLOCK, &sdp->sd_flags)) {
584 			finish_xmote(gl, target);
585 			gfs2_glock_queue_work(gl, 0);
586 		}
587 		else if (ret) {
588 			fs_err(sdp, "lm_lock ret %d\n", ret);
589 			GLOCK_BUG_ON(gl, !test_bit(SDF_WITHDRAWN,
590 						   &sdp->sd_flags));
591 		}
592 	} else { /* lock_nolock */
593 		finish_xmote(gl, target);
594 		gfs2_glock_queue_work(gl, 0);
595 	}
596 
597 	spin_lock(&gl->gl_lockref.lock);
598 }
599 
600 /**
601  * find_first_holder - find the first "holder" gh
602  * @gl: the glock
603  */
604 
find_first_holder(const struct gfs2_glock * gl)605 static inline struct gfs2_holder *find_first_holder(const struct gfs2_glock *gl)
606 {
607 	struct gfs2_holder *gh;
608 
609 	if (!list_empty(&gl->gl_holders)) {
610 		gh = list_entry(gl->gl_holders.next, struct gfs2_holder, gh_list);
611 		if (test_bit(HIF_HOLDER, &gh->gh_iflags))
612 			return gh;
613 	}
614 	return NULL;
615 }
616 
617 /**
618  * run_queue - do all outstanding tasks related to a glock
619  * @gl: The glock in question
620  * @nonblock: True if we must not block in run_queue
621  *
622  */
623 
run_queue(struct gfs2_glock * gl,const int nonblock)624 static void run_queue(struct gfs2_glock *gl, const int nonblock)
625 __releases(&gl->gl_lockref.lock)
626 __acquires(&gl->gl_lockref.lock)
627 {
628 	struct gfs2_holder *gh = NULL;
629 	int ret;
630 
631 	if (test_and_set_bit(GLF_LOCK, &gl->gl_flags))
632 		return;
633 
634 	GLOCK_BUG_ON(gl, test_bit(GLF_DEMOTE_IN_PROGRESS, &gl->gl_flags));
635 
636 	if (test_bit(GLF_DEMOTE, &gl->gl_flags) &&
637 	    gl->gl_demote_state != gl->gl_state) {
638 		if (find_first_holder(gl))
639 			goto out_unlock;
640 		if (nonblock)
641 			goto out_sched;
642 		set_bit(GLF_DEMOTE_IN_PROGRESS, &gl->gl_flags);
643 		GLOCK_BUG_ON(gl, gl->gl_demote_state == LM_ST_EXCLUSIVE);
644 		gl->gl_target = gl->gl_demote_state;
645 	} else {
646 		if (test_bit(GLF_DEMOTE, &gl->gl_flags))
647 			gfs2_demote_wake(gl);
648 		ret = do_promote(gl);
649 		if (ret == 0)
650 			goto out_unlock;
651 		if (ret == 2)
652 			goto out;
653 		gh = find_first_waiter(gl);
654 		gl->gl_target = gh->gh_state;
655 		if (!(gh->gh_flags & (LM_FLAG_TRY | LM_FLAG_TRY_1CB)))
656 			do_error(gl, 0); /* Fail queued try locks */
657 	}
658 	do_xmote(gl, gh, gl->gl_target);
659 out:
660 	return;
661 
662 out_sched:
663 	clear_bit(GLF_LOCK, &gl->gl_flags);
664 	smp_mb__after_atomic();
665 	gl->gl_lockref.count++;
666 	__gfs2_glock_queue_work(gl, 0);
667 	return;
668 
669 out_unlock:
670 	clear_bit(GLF_LOCK, &gl->gl_flags);
671 	smp_mb__after_atomic();
672 	return;
673 }
674 
delete_work_func(struct work_struct * work)675 static void delete_work_func(struct work_struct *work)
676 {
677 	struct gfs2_glock *gl = container_of(work, struct gfs2_glock, gl_delete);
678 	struct gfs2_sbd *sdp = gl->gl_name.ln_sbd;
679 	struct inode *inode;
680 	u64 no_addr = gl->gl_name.ln_number;
681 
682 	/* If someone's using this glock to create a new dinode, the block must
683 	   have been freed by another node, then re-used, in which case our
684 	   iopen callback is too late after the fact. Ignore it. */
685 	if (test_bit(GLF_INODE_CREATING, &gl->gl_flags))
686 		goto out;
687 
688 	inode = gfs2_lookup_by_inum(sdp, no_addr, NULL, GFS2_BLKST_UNLINKED);
689 	if (!IS_ERR_OR_NULL(inode)) {
690 		d_prune_aliases(inode);
691 		iput(inode);
692 	}
693 out:
694 	gfs2_glock_put(gl);
695 }
696 
glock_work_func(struct work_struct * work)697 static void glock_work_func(struct work_struct *work)
698 {
699 	unsigned long delay = 0;
700 	struct gfs2_glock *gl = container_of(work, struct gfs2_glock, gl_work.work);
701 	unsigned int drop_refs = 1;
702 
703 	if (test_and_clear_bit(GLF_REPLY_PENDING, &gl->gl_flags)) {
704 		finish_xmote(gl, gl->gl_reply);
705 		drop_refs++;
706 	}
707 	spin_lock(&gl->gl_lockref.lock);
708 	if (test_bit(GLF_PENDING_DEMOTE, &gl->gl_flags) &&
709 	    gl->gl_state != LM_ST_UNLOCKED &&
710 	    gl->gl_demote_state != LM_ST_EXCLUSIVE) {
711 		unsigned long holdtime, now = jiffies;
712 
713 		holdtime = gl->gl_tchange + gl->gl_hold_time;
714 		if (time_before(now, holdtime))
715 			delay = holdtime - now;
716 
717 		if (!delay) {
718 			clear_bit(GLF_PENDING_DEMOTE, &gl->gl_flags);
719 			set_bit(GLF_DEMOTE, &gl->gl_flags);
720 		}
721 	}
722 	run_queue(gl, 0);
723 	if (delay) {
724 		/* Keep one glock reference for the work we requeue. */
725 		drop_refs--;
726 		if (gl->gl_name.ln_type != LM_TYPE_INODE)
727 			delay = 0;
728 		__gfs2_glock_queue_work(gl, delay);
729 	}
730 
731 	/*
732 	 * Drop the remaining glock references manually here. (Mind that
733 	 * __gfs2_glock_queue_work depends on the lockref spinlock begin held
734 	 * here as well.)
735 	 */
736 	gl->gl_lockref.count -= drop_refs;
737 	if (!gl->gl_lockref.count) {
738 		__gfs2_glock_put(gl);
739 		return;
740 	}
741 	spin_unlock(&gl->gl_lockref.lock);
742 }
743 
find_insert_glock(struct lm_lockname * name,struct gfs2_glock * new)744 static struct gfs2_glock *find_insert_glock(struct lm_lockname *name,
745 					    struct gfs2_glock *new)
746 {
747 	struct wait_glock_queue wait;
748 	wait_queue_head_t *wq = glock_waitqueue(name);
749 	struct gfs2_glock *gl;
750 
751 	wait.name = name;
752 	init_wait(&wait.wait);
753 	wait.wait.func = glock_wake_function;
754 
755 again:
756 	prepare_to_wait(wq, &wait.wait, TASK_UNINTERRUPTIBLE);
757 	rcu_read_lock();
758 	if (new) {
759 		gl = rhashtable_lookup_get_insert_fast(&gl_hash_table,
760 			&new->gl_node, ht_parms);
761 		if (IS_ERR(gl))
762 			goto out;
763 	} else {
764 		gl = rhashtable_lookup_fast(&gl_hash_table,
765 			name, ht_parms);
766 	}
767 	if (gl && !lockref_get_not_dead(&gl->gl_lockref)) {
768 		rcu_read_unlock();
769 		schedule();
770 		goto again;
771 	}
772 out:
773 	rcu_read_unlock();
774 	finish_wait(wq, &wait.wait);
775 	return gl;
776 }
777 
778 /**
779  * gfs2_glock_get() - Get a glock, or create one if one doesn't exist
780  * @sdp: The GFS2 superblock
781  * @number: the lock number
782  * @glops: The glock_operations to use
783  * @create: If 0, don't create the glock if it doesn't exist
784  * @glp: the glock is returned here
785  *
786  * This does not lock a glock, just finds/creates structures for one.
787  *
788  * Returns: errno
789  */
790 
gfs2_glock_get(struct gfs2_sbd * sdp,u64 number,const struct gfs2_glock_operations * glops,int create,struct gfs2_glock ** glp)791 int gfs2_glock_get(struct gfs2_sbd *sdp, u64 number,
792 		   const struct gfs2_glock_operations *glops, int create,
793 		   struct gfs2_glock **glp)
794 {
795 	struct super_block *s = sdp->sd_vfs;
796 	struct lm_lockname name = { .ln_number = number,
797 				    .ln_type = glops->go_type,
798 				    .ln_sbd = sdp };
799 	struct gfs2_glock *gl, *tmp;
800 	struct address_space *mapping;
801 	struct kmem_cache *cachep;
802 	int ret = 0;
803 
804 	gl = find_insert_glock(&name, NULL);
805 	if (gl) {
806 		*glp = gl;
807 		return 0;
808 	}
809 	if (!create)
810 		return -ENOENT;
811 
812 	if (glops->go_flags & GLOF_ASPACE)
813 		cachep = gfs2_glock_aspace_cachep;
814 	else
815 		cachep = gfs2_glock_cachep;
816 	gl = kmem_cache_alloc(cachep, GFP_NOFS);
817 	if (!gl)
818 		return -ENOMEM;
819 
820 	memset(&gl->gl_lksb, 0, sizeof(struct dlm_lksb));
821 
822 	if (glops->go_flags & GLOF_LVB) {
823 		gl->gl_lksb.sb_lvbptr = kzalloc(GFS2_MIN_LVB_SIZE, GFP_NOFS);
824 		if (!gl->gl_lksb.sb_lvbptr) {
825 			kmem_cache_free(cachep, gl);
826 			return -ENOMEM;
827 		}
828 	}
829 
830 	atomic_inc(&sdp->sd_glock_disposal);
831 	gl->gl_node.next = NULL;
832 	gl->gl_flags = 0;
833 	gl->gl_name = name;
834 	gl->gl_lockref.count = 1;
835 	gl->gl_state = LM_ST_UNLOCKED;
836 	gl->gl_target = LM_ST_UNLOCKED;
837 	gl->gl_demote_state = LM_ST_EXCLUSIVE;
838 	gl->gl_ops = glops;
839 	gl->gl_dstamp = 0;
840 	preempt_disable();
841 	/* We use the global stats to estimate the initial per-glock stats */
842 	gl->gl_stats = this_cpu_ptr(sdp->sd_lkstats)->lkstats[glops->go_type];
843 	preempt_enable();
844 	gl->gl_stats.stats[GFS2_LKS_DCOUNT] = 0;
845 	gl->gl_stats.stats[GFS2_LKS_QCOUNT] = 0;
846 	gl->gl_tchange = jiffies;
847 	gl->gl_object = NULL;
848 	gl->gl_hold_time = GL_GLOCK_DFT_HOLD;
849 	INIT_DELAYED_WORK(&gl->gl_work, glock_work_func);
850 	INIT_WORK(&gl->gl_delete, delete_work_func);
851 
852 	mapping = gfs2_glock2aspace(gl);
853 	if (mapping) {
854                 mapping->a_ops = &gfs2_meta_aops;
855 		mapping->host = s->s_bdev->bd_inode;
856 		mapping->flags = 0;
857 		mapping_set_gfp_mask(mapping, GFP_NOFS);
858 		mapping->private_data = NULL;
859 		mapping->writeback_index = 0;
860 	}
861 
862 	tmp = find_insert_glock(&name, gl);
863 	if (!tmp) {
864 		*glp = gl;
865 		goto out;
866 	}
867 	if (IS_ERR(tmp)) {
868 		ret = PTR_ERR(tmp);
869 		goto out_free;
870 	}
871 	*glp = tmp;
872 
873 out_free:
874 	kfree(gl->gl_lksb.sb_lvbptr);
875 	kmem_cache_free(cachep, gl);
876 	if (atomic_dec_and_test(&sdp->sd_glock_disposal))
877 		wake_up(&sdp->sd_glock_wait);
878 
879 out:
880 	return ret;
881 }
882 
883 /**
884  * gfs2_holder_init - initialize a struct gfs2_holder in the default way
885  * @gl: the glock
886  * @state: the state we're requesting
887  * @flags: the modifier flags
888  * @gh: the holder structure
889  *
890  */
891 
gfs2_holder_init(struct gfs2_glock * gl,unsigned int state,u16 flags,struct gfs2_holder * gh)892 void gfs2_holder_init(struct gfs2_glock *gl, unsigned int state, u16 flags,
893 		      struct gfs2_holder *gh)
894 {
895 	INIT_LIST_HEAD(&gh->gh_list);
896 	gh->gh_gl = gl;
897 	gh->gh_ip = _RET_IP_;
898 	gh->gh_owner_pid = get_pid(task_pid(current));
899 	gh->gh_state = state;
900 	gh->gh_flags = flags;
901 	gh->gh_error = 0;
902 	gh->gh_iflags = 0;
903 	gfs2_glock_hold(gl);
904 }
905 
906 /**
907  * gfs2_holder_reinit - reinitialize a struct gfs2_holder so we can requeue it
908  * @state: the state we're requesting
909  * @flags: the modifier flags
910  * @gh: the holder structure
911  *
912  * Don't mess with the glock.
913  *
914  */
915 
gfs2_holder_reinit(unsigned int state,u16 flags,struct gfs2_holder * gh)916 void gfs2_holder_reinit(unsigned int state, u16 flags, struct gfs2_holder *gh)
917 {
918 	gh->gh_state = state;
919 	gh->gh_flags = flags;
920 	gh->gh_iflags = 0;
921 	gh->gh_ip = _RET_IP_;
922 	put_pid(gh->gh_owner_pid);
923 	gh->gh_owner_pid = get_pid(task_pid(current));
924 }
925 
926 /**
927  * gfs2_holder_uninit - uninitialize a holder structure (drop glock reference)
928  * @gh: the holder structure
929  *
930  */
931 
gfs2_holder_uninit(struct gfs2_holder * gh)932 void gfs2_holder_uninit(struct gfs2_holder *gh)
933 {
934 	put_pid(gh->gh_owner_pid);
935 	gfs2_glock_put(gh->gh_gl);
936 	gfs2_holder_mark_uninitialized(gh);
937 	gh->gh_ip = 0;
938 }
939 
gfs2_glock_update_hold_time(struct gfs2_glock * gl,unsigned long start_time)940 static void gfs2_glock_update_hold_time(struct gfs2_glock *gl,
941 					unsigned long start_time)
942 {
943 	/* Have we waited longer that a second? */
944 	if (time_after(jiffies, start_time + HZ)) {
945 		/* Lengthen the minimum hold time. */
946 		gl->gl_hold_time = min(gl->gl_hold_time + GL_GLOCK_HOLD_INCR,
947 				       GL_GLOCK_MAX_HOLD);
948 	}
949 }
950 
951 /**
952  * gfs2_glock_wait - wait on a glock acquisition
953  * @gh: the glock holder
954  *
955  * Returns: 0 on success
956  */
957 
gfs2_glock_wait(struct gfs2_holder * gh)958 int gfs2_glock_wait(struct gfs2_holder *gh)
959 {
960 	unsigned long start_time = jiffies;
961 
962 	might_sleep();
963 	wait_on_bit(&gh->gh_iflags, HIF_WAIT, TASK_UNINTERRUPTIBLE);
964 	gfs2_glock_update_hold_time(gh->gh_gl, start_time);
965 	return gh->gh_error;
966 }
967 
glocks_pending(unsigned int num_gh,struct gfs2_holder * ghs)968 static int glocks_pending(unsigned int num_gh, struct gfs2_holder *ghs)
969 {
970 	int i;
971 
972 	for (i = 0; i < num_gh; i++)
973 		if (test_bit(HIF_WAIT, &ghs[i].gh_iflags))
974 			return 1;
975 	return 0;
976 }
977 
978 /**
979  * gfs2_glock_async_wait - wait on multiple asynchronous glock acquisitions
980  * @num_gh: the number of holders in the array
981  * @ghs: the glock holder array
982  *
983  * Returns: 0 on success, meaning all glocks have been granted and are held.
984  *          -ESTALE if the request timed out, meaning all glocks were released,
985  *          and the caller should retry the operation.
986  */
987 
gfs2_glock_async_wait(unsigned int num_gh,struct gfs2_holder * ghs)988 int gfs2_glock_async_wait(unsigned int num_gh, struct gfs2_holder *ghs)
989 {
990 	struct gfs2_sbd *sdp = ghs[0].gh_gl->gl_name.ln_sbd;
991 	int i, ret = 0, timeout = 0;
992 	unsigned long start_time = jiffies;
993 	bool keep_waiting;
994 
995 	might_sleep();
996 	/*
997 	 * Total up the (minimum hold time * 2) of all glocks and use that to
998 	 * determine the max amount of time we should wait.
999 	 */
1000 	for (i = 0; i < num_gh; i++)
1001 		timeout += ghs[i].gh_gl->gl_hold_time << 1;
1002 
1003 wait_for_dlm:
1004 	if (!wait_event_timeout(sdp->sd_async_glock_wait,
1005 				!glocks_pending(num_gh, ghs), timeout))
1006 		ret = -ESTALE; /* request timed out. */
1007 
1008 	/*
1009 	 * If dlm granted all our requests, we need to adjust the glock
1010 	 * minimum hold time values according to how long we waited.
1011 	 *
1012 	 * If our request timed out, we need to repeatedly release any held
1013 	 * glocks we acquired thus far to allow dlm to acquire the remaining
1014 	 * glocks without deadlocking.  We cannot currently cancel outstanding
1015 	 * glock acquisitions.
1016 	 *
1017 	 * The HIF_WAIT bit tells us which requests still need a response from
1018 	 * dlm.
1019 	 *
1020 	 * If dlm sent us any errors, we return the first error we find.
1021 	 */
1022 	keep_waiting = false;
1023 	for (i = 0; i < num_gh; i++) {
1024 		/* Skip holders we have already dequeued below. */
1025 		if (!gfs2_holder_queued(&ghs[i]))
1026 			continue;
1027 		/* Skip holders with a pending DLM response. */
1028 		if (test_bit(HIF_WAIT, &ghs[i].gh_iflags)) {
1029 			keep_waiting = true;
1030 			continue;
1031 		}
1032 
1033 		if (test_bit(HIF_HOLDER, &ghs[i].gh_iflags)) {
1034 			if (ret == -ESTALE)
1035 				gfs2_glock_dq(&ghs[i]);
1036 			else
1037 				gfs2_glock_update_hold_time(ghs[i].gh_gl,
1038 							    start_time);
1039 		}
1040 		if (!ret)
1041 			ret = ghs[i].gh_error;
1042 	}
1043 
1044 	if (keep_waiting)
1045 		goto wait_for_dlm;
1046 
1047 	/*
1048 	 * At this point, we've either acquired all locks or released them all.
1049 	 */
1050 	return ret;
1051 }
1052 
1053 /**
1054  * handle_callback - process a demote request
1055  * @gl: the glock
1056  * @state: the state the caller wants us to change to
1057  *
1058  * There are only two requests that we are going to see in actual
1059  * practise: LM_ST_SHARED and LM_ST_UNLOCKED
1060  */
1061 
handle_callback(struct gfs2_glock * gl,unsigned int state,unsigned long delay,bool remote)1062 static void handle_callback(struct gfs2_glock *gl, unsigned int state,
1063 			    unsigned long delay, bool remote)
1064 {
1065 	int bit = delay ? GLF_PENDING_DEMOTE : GLF_DEMOTE;
1066 
1067 	set_bit(bit, &gl->gl_flags);
1068 	if (gl->gl_demote_state == LM_ST_EXCLUSIVE) {
1069 		gl->gl_demote_state = state;
1070 		gl->gl_demote_time = jiffies;
1071 	} else if (gl->gl_demote_state != LM_ST_UNLOCKED &&
1072 			gl->gl_demote_state != state) {
1073 		gl->gl_demote_state = LM_ST_UNLOCKED;
1074 	}
1075 	if (gl->gl_ops->go_callback)
1076 		gl->gl_ops->go_callback(gl, remote);
1077 	trace_gfs2_demote_rq(gl, remote);
1078 }
1079 
gfs2_print_dbg(struct seq_file * seq,const char * fmt,...)1080 void gfs2_print_dbg(struct seq_file *seq, const char *fmt, ...)
1081 {
1082 	struct va_format vaf;
1083 	va_list args;
1084 
1085 	va_start(args, fmt);
1086 
1087 	if (seq) {
1088 		seq_vprintf(seq, fmt, args);
1089 	} else {
1090 		vaf.fmt = fmt;
1091 		vaf.va = &args;
1092 
1093 		pr_err("%pV", &vaf);
1094 	}
1095 
1096 	va_end(args);
1097 }
1098 
1099 /**
1100  * add_to_queue - Add a holder to the wait queue (but look for recursion)
1101  * @gh: the holder structure to add
1102  *
1103  * Eventually we should move the recursive locking trap to a
1104  * debugging option or something like that. This is the fast
1105  * path and needs to have the minimum number of distractions.
1106  *
1107  */
1108 
add_to_queue(struct gfs2_holder * gh)1109 static inline void add_to_queue(struct gfs2_holder *gh)
1110 __releases(&gl->gl_lockref.lock)
1111 __acquires(&gl->gl_lockref.lock)
1112 {
1113 	struct gfs2_glock *gl = gh->gh_gl;
1114 	struct gfs2_sbd *sdp = gl->gl_name.ln_sbd;
1115 	struct list_head *insert_pt = NULL;
1116 	struct gfs2_holder *gh2;
1117 	int try_futile = 0;
1118 
1119 	GLOCK_BUG_ON(gl, gh->gh_owner_pid == NULL);
1120 	if (test_and_set_bit(HIF_WAIT, &gh->gh_iflags))
1121 		GLOCK_BUG_ON(gl, true);
1122 
1123 	if (gh->gh_flags & (LM_FLAG_TRY | LM_FLAG_TRY_1CB)) {
1124 		if (test_bit(GLF_LOCK, &gl->gl_flags))
1125 			try_futile = !may_grant(gl, gh);
1126 		if (test_bit(GLF_INVALIDATE_IN_PROGRESS, &gl->gl_flags))
1127 			goto fail;
1128 	}
1129 
1130 	list_for_each_entry(gh2, &gl->gl_holders, gh_list) {
1131 		if (unlikely(gh2->gh_owner_pid == gh->gh_owner_pid &&
1132 		    (gh->gh_gl->gl_ops->go_type != LM_TYPE_FLOCK)))
1133 			goto trap_recursive;
1134 		if (try_futile &&
1135 		    !(gh2->gh_flags & (LM_FLAG_TRY | LM_FLAG_TRY_1CB))) {
1136 fail:
1137 			gh->gh_error = GLR_TRYFAILED;
1138 			gfs2_holder_wake(gh);
1139 			return;
1140 		}
1141 		if (test_bit(HIF_HOLDER, &gh2->gh_iflags))
1142 			continue;
1143 		if (unlikely((gh->gh_flags & LM_FLAG_PRIORITY) && !insert_pt))
1144 			insert_pt = &gh2->gh_list;
1145 	}
1146 	set_bit(GLF_QUEUED, &gl->gl_flags);
1147 	trace_gfs2_glock_queue(gh, 1);
1148 	gfs2_glstats_inc(gl, GFS2_LKS_QCOUNT);
1149 	gfs2_sbstats_inc(gl, GFS2_LKS_QCOUNT);
1150 	if (likely(insert_pt == NULL)) {
1151 		list_add_tail(&gh->gh_list, &gl->gl_holders);
1152 		if (unlikely(gh->gh_flags & LM_FLAG_PRIORITY))
1153 			goto do_cancel;
1154 		return;
1155 	}
1156 	list_add_tail(&gh->gh_list, insert_pt);
1157 do_cancel:
1158 	gh = list_entry(gl->gl_holders.next, struct gfs2_holder, gh_list);
1159 	if (!(gh->gh_flags & LM_FLAG_PRIORITY)) {
1160 		spin_unlock(&gl->gl_lockref.lock);
1161 		if (sdp->sd_lockstruct.ls_ops->lm_cancel)
1162 			sdp->sd_lockstruct.ls_ops->lm_cancel(gl);
1163 		spin_lock(&gl->gl_lockref.lock);
1164 	}
1165 	return;
1166 
1167 trap_recursive:
1168 	fs_err(sdp, "original: %pSR\n", (void *)gh2->gh_ip);
1169 	fs_err(sdp, "pid: %d\n", pid_nr(gh2->gh_owner_pid));
1170 	fs_err(sdp, "lock type: %d req lock state : %d\n",
1171 	       gh2->gh_gl->gl_name.ln_type, gh2->gh_state);
1172 	fs_err(sdp, "new: %pSR\n", (void *)gh->gh_ip);
1173 	fs_err(sdp, "pid: %d\n", pid_nr(gh->gh_owner_pid));
1174 	fs_err(sdp, "lock type: %d req lock state : %d\n",
1175 	       gh->gh_gl->gl_name.ln_type, gh->gh_state);
1176 	gfs2_dump_glock(NULL, gl, true);
1177 	BUG();
1178 }
1179 
1180 /**
1181  * gfs2_glock_nq - enqueue a struct gfs2_holder onto a glock (acquire a glock)
1182  * @gh: the holder structure
1183  *
1184  * if (gh->gh_flags & GL_ASYNC), this never returns an error
1185  *
1186  * Returns: 0, GLR_TRYFAILED, or errno on failure
1187  */
1188 
gfs2_glock_nq(struct gfs2_holder * gh)1189 int gfs2_glock_nq(struct gfs2_holder *gh)
1190 {
1191 	struct gfs2_glock *gl = gh->gh_gl;
1192 	struct gfs2_sbd *sdp = gl->gl_name.ln_sbd;
1193 	int error = 0;
1194 
1195 	if (unlikely(test_bit(SDF_WITHDRAWN, &sdp->sd_flags)))
1196 		return -EIO;
1197 
1198 	if (test_bit(GLF_LRU, &gl->gl_flags))
1199 		gfs2_glock_remove_from_lru(gl);
1200 
1201 	spin_lock(&gl->gl_lockref.lock);
1202 	add_to_queue(gh);
1203 	if (unlikely((LM_FLAG_NOEXP & gh->gh_flags) &&
1204 		     test_and_clear_bit(GLF_FROZEN, &gl->gl_flags))) {
1205 		set_bit(GLF_REPLY_PENDING, &gl->gl_flags);
1206 		gl->gl_lockref.count++;
1207 		__gfs2_glock_queue_work(gl, 0);
1208 	}
1209 	run_queue(gl, 1);
1210 	spin_unlock(&gl->gl_lockref.lock);
1211 
1212 	if (!(gh->gh_flags & GL_ASYNC))
1213 		error = gfs2_glock_wait(gh);
1214 
1215 	return error;
1216 }
1217 
1218 /**
1219  * gfs2_glock_poll - poll to see if an async request has been completed
1220  * @gh: the holder
1221  *
1222  * Returns: 1 if the request is ready to be gfs2_glock_wait()ed on
1223  */
1224 
gfs2_glock_poll(struct gfs2_holder * gh)1225 int gfs2_glock_poll(struct gfs2_holder *gh)
1226 {
1227 	return test_bit(HIF_WAIT, &gh->gh_iflags) ? 0 : 1;
1228 }
1229 
1230 /**
1231  * gfs2_glock_dq - dequeue a struct gfs2_holder from a glock (release a glock)
1232  * @gh: the glock holder
1233  *
1234  */
1235 
gfs2_glock_dq(struct gfs2_holder * gh)1236 void gfs2_glock_dq(struct gfs2_holder *gh)
1237 {
1238 	struct gfs2_glock *gl = gh->gh_gl;
1239 	const struct gfs2_glock_operations *glops = gl->gl_ops;
1240 	unsigned delay = 0;
1241 	int fast_path = 0;
1242 
1243 	spin_lock(&gl->gl_lockref.lock);
1244 	if (gh->gh_flags & GL_NOCACHE)
1245 		handle_callback(gl, LM_ST_UNLOCKED, 0, false);
1246 
1247 	list_del_init(&gh->gh_list);
1248 	clear_bit(HIF_HOLDER, &gh->gh_iflags);
1249 	if (find_first_holder(gl) == NULL) {
1250 		if (glops->go_unlock) {
1251 			GLOCK_BUG_ON(gl, test_and_set_bit(GLF_LOCK, &gl->gl_flags));
1252 			spin_unlock(&gl->gl_lockref.lock);
1253 			glops->go_unlock(gh);
1254 			spin_lock(&gl->gl_lockref.lock);
1255 			clear_bit(GLF_LOCK, &gl->gl_flags);
1256 		}
1257 		if (list_empty(&gl->gl_holders) &&
1258 		    !test_bit(GLF_PENDING_DEMOTE, &gl->gl_flags) &&
1259 		    !test_bit(GLF_DEMOTE, &gl->gl_flags))
1260 			fast_path = 1;
1261 	}
1262 	if (!test_bit(GLF_LFLUSH, &gl->gl_flags) && demote_ok(gl))
1263 		gfs2_glock_add_to_lru(gl);
1264 
1265 	trace_gfs2_glock_queue(gh, 0);
1266 	if (unlikely(!fast_path)) {
1267 		gl->gl_lockref.count++;
1268 		if (test_bit(GLF_PENDING_DEMOTE, &gl->gl_flags) &&
1269 		    !test_bit(GLF_DEMOTE, &gl->gl_flags) &&
1270 		    gl->gl_name.ln_type == LM_TYPE_INODE)
1271 			delay = gl->gl_hold_time;
1272 		__gfs2_glock_queue_work(gl, delay);
1273 	}
1274 	spin_unlock(&gl->gl_lockref.lock);
1275 }
1276 
gfs2_glock_dq_wait(struct gfs2_holder * gh)1277 void gfs2_glock_dq_wait(struct gfs2_holder *gh)
1278 {
1279 	struct gfs2_glock *gl = gh->gh_gl;
1280 	gfs2_glock_dq(gh);
1281 	might_sleep();
1282 	wait_on_bit(&gl->gl_flags, GLF_DEMOTE, TASK_UNINTERRUPTIBLE);
1283 }
1284 
1285 /**
1286  * gfs2_glock_dq_uninit - dequeue a holder from a glock and initialize it
1287  * @gh: the holder structure
1288  *
1289  */
1290 
gfs2_glock_dq_uninit(struct gfs2_holder * gh)1291 void gfs2_glock_dq_uninit(struct gfs2_holder *gh)
1292 {
1293 	gfs2_glock_dq(gh);
1294 	gfs2_holder_uninit(gh);
1295 }
1296 
1297 /**
1298  * gfs2_glock_nq_num - acquire a glock based on lock number
1299  * @sdp: the filesystem
1300  * @number: the lock number
1301  * @glops: the glock operations for the type of glock
1302  * @state: the state to acquire the glock in
1303  * @flags: modifier flags for the acquisition
1304  * @gh: the struct gfs2_holder
1305  *
1306  * Returns: errno
1307  */
1308 
gfs2_glock_nq_num(struct gfs2_sbd * sdp,u64 number,const struct gfs2_glock_operations * glops,unsigned int state,u16 flags,struct gfs2_holder * gh)1309 int gfs2_glock_nq_num(struct gfs2_sbd *sdp, u64 number,
1310 		      const struct gfs2_glock_operations *glops,
1311 		      unsigned int state, u16 flags, struct gfs2_holder *gh)
1312 {
1313 	struct gfs2_glock *gl;
1314 	int error;
1315 
1316 	error = gfs2_glock_get(sdp, number, glops, CREATE, &gl);
1317 	if (!error) {
1318 		error = gfs2_glock_nq_init(gl, state, flags, gh);
1319 		gfs2_glock_put(gl);
1320 	}
1321 
1322 	return error;
1323 }
1324 
1325 /**
1326  * glock_compare - Compare two struct gfs2_glock structures for sorting
1327  * @arg_a: the first structure
1328  * @arg_b: the second structure
1329  *
1330  */
1331 
glock_compare(const void * arg_a,const void * arg_b)1332 static int glock_compare(const void *arg_a, const void *arg_b)
1333 {
1334 	const struct gfs2_holder *gh_a = *(const struct gfs2_holder **)arg_a;
1335 	const struct gfs2_holder *gh_b = *(const struct gfs2_holder **)arg_b;
1336 	const struct lm_lockname *a = &gh_a->gh_gl->gl_name;
1337 	const struct lm_lockname *b = &gh_b->gh_gl->gl_name;
1338 
1339 	if (a->ln_number > b->ln_number)
1340 		return 1;
1341 	if (a->ln_number < b->ln_number)
1342 		return -1;
1343 	BUG_ON(gh_a->gh_gl->gl_ops->go_type == gh_b->gh_gl->gl_ops->go_type);
1344 	return 0;
1345 }
1346 
1347 /**
1348  * nq_m_sync - synchonously acquire more than one glock in deadlock free order
1349  * @num_gh: the number of structures
1350  * @ghs: an array of struct gfs2_holder structures
1351  *
1352  * Returns: 0 on success (all glocks acquired),
1353  *          errno on failure (no glocks acquired)
1354  */
1355 
nq_m_sync(unsigned int num_gh,struct gfs2_holder * ghs,struct gfs2_holder ** p)1356 static int nq_m_sync(unsigned int num_gh, struct gfs2_holder *ghs,
1357 		     struct gfs2_holder **p)
1358 {
1359 	unsigned int x;
1360 	int error = 0;
1361 
1362 	for (x = 0; x < num_gh; x++)
1363 		p[x] = &ghs[x];
1364 
1365 	sort(p, num_gh, sizeof(struct gfs2_holder *), glock_compare, NULL);
1366 
1367 	for (x = 0; x < num_gh; x++) {
1368 		p[x]->gh_flags &= ~(LM_FLAG_TRY | GL_ASYNC);
1369 
1370 		error = gfs2_glock_nq(p[x]);
1371 		if (error) {
1372 			while (x--)
1373 				gfs2_glock_dq(p[x]);
1374 			break;
1375 		}
1376 	}
1377 
1378 	return error;
1379 }
1380 
1381 /**
1382  * gfs2_glock_nq_m - acquire multiple glocks
1383  * @num_gh: the number of structures
1384  * @ghs: an array of struct gfs2_holder structures
1385  *
1386  *
1387  * Returns: 0 on success (all glocks acquired),
1388  *          errno on failure (no glocks acquired)
1389  */
1390 
gfs2_glock_nq_m(unsigned int num_gh,struct gfs2_holder * ghs)1391 int gfs2_glock_nq_m(unsigned int num_gh, struct gfs2_holder *ghs)
1392 {
1393 	struct gfs2_holder *tmp[4];
1394 	struct gfs2_holder **pph = tmp;
1395 	int error = 0;
1396 
1397 	switch(num_gh) {
1398 	case 0:
1399 		return 0;
1400 	case 1:
1401 		ghs->gh_flags &= ~(LM_FLAG_TRY | GL_ASYNC);
1402 		return gfs2_glock_nq(ghs);
1403 	default:
1404 		if (num_gh <= 4)
1405 			break;
1406 		pph = kmalloc_array(num_gh, sizeof(struct gfs2_holder *),
1407 				    GFP_NOFS);
1408 		if (!pph)
1409 			return -ENOMEM;
1410 	}
1411 
1412 	error = nq_m_sync(num_gh, ghs, pph);
1413 
1414 	if (pph != tmp)
1415 		kfree(pph);
1416 
1417 	return error;
1418 }
1419 
1420 /**
1421  * gfs2_glock_dq_m - release multiple glocks
1422  * @num_gh: the number of structures
1423  * @ghs: an array of struct gfs2_holder structures
1424  *
1425  */
1426 
gfs2_glock_dq_m(unsigned int num_gh,struct gfs2_holder * ghs)1427 void gfs2_glock_dq_m(unsigned int num_gh, struct gfs2_holder *ghs)
1428 {
1429 	while (num_gh--)
1430 		gfs2_glock_dq(&ghs[num_gh]);
1431 }
1432 
gfs2_glock_cb(struct gfs2_glock * gl,unsigned int state)1433 void gfs2_glock_cb(struct gfs2_glock *gl, unsigned int state)
1434 {
1435 	unsigned long delay = 0;
1436 	unsigned long holdtime;
1437 	unsigned long now = jiffies;
1438 
1439 	gfs2_glock_hold(gl);
1440 	holdtime = gl->gl_tchange + gl->gl_hold_time;
1441 	if (test_bit(GLF_QUEUED, &gl->gl_flags) &&
1442 	    gl->gl_name.ln_type == LM_TYPE_INODE) {
1443 		if (time_before(now, holdtime))
1444 			delay = holdtime - now;
1445 		if (test_bit(GLF_REPLY_PENDING, &gl->gl_flags))
1446 			delay = gl->gl_hold_time;
1447 	}
1448 
1449 	spin_lock(&gl->gl_lockref.lock);
1450 	handle_callback(gl, state, delay, true);
1451 	__gfs2_glock_queue_work(gl, delay);
1452 	spin_unlock(&gl->gl_lockref.lock);
1453 }
1454 
1455 /**
1456  * gfs2_should_freeze - Figure out if glock should be frozen
1457  * @gl: The glock in question
1458  *
1459  * Glocks are not frozen if (a) the result of the dlm operation is
1460  * an error, (b) the locking operation was an unlock operation or
1461  * (c) if there is a "noexp" flagged request anywhere in the queue
1462  *
1463  * Returns: 1 if freezing should occur, 0 otherwise
1464  */
1465 
gfs2_should_freeze(const struct gfs2_glock * gl)1466 static int gfs2_should_freeze(const struct gfs2_glock *gl)
1467 {
1468 	const struct gfs2_holder *gh;
1469 
1470 	if (gl->gl_reply & ~LM_OUT_ST_MASK)
1471 		return 0;
1472 	if (gl->gl_target == LM_ST_UNLOCKED)
1473 		return 0;
1474 
1475 	list_for_each_entry(gh, &gl->gl_holders, gh_list) {
1476 		if (test_bit(HIF_HOLDER, &gh->gh_iflags))
1477 			continue;
1478 		if (LM_FLAG_NOEXP & gh->gh_flags)
1479 			return 0;
1480 	}
1481 
1482 	return 1;
1483 }
1484 
1485 /**
1486  * gfs2_glock_complete - Callback used by locking
1487  * @gl: Pointer to the glock
1488  * @ret: The return value from the dlm
1489  *
1490  * The gl_reply field is under the gl_lockref.lock lock so that it is ok
1491  * to use a bitfield shared with other glock state fields.
1492  */
1493 
gfs2_glock_complete(struct gfs2_glock * gl,int ret)1494 void gfs2_glock_complete(struct gfs2_glock *gl, int ret)
1495 {
1496 	struct lm_lockstruct *ls = &gl->gl_name.ln_sbd->sd_lockstruct;
1497 
1498 	spin_lock(&gl->gl_lockref.lock);
1499 	gl->gl_reply = ret;
1500 
1501 	if (unlikely(test_bit(DFL_BLOCK_LOCKS, &ls->ls_recover_flags))) {
1502 		if (gfs2_should_freeze(gl)) {
1503 			set_bit(GLF_FROZEN, &gl->gl_flags);
1504 			spin_unlock(&gl->gl_lockref.lock);
1505 			return;
1506 		}
1507 	}
1508 
1509 	gl->gl_lockref.count++;
1510 	set_bit(GLF_REPLY_PENDING, &gl->gl_flags);
1511 	__gfs2_glock_queue_work(gl, 0);
1512 	spin_unlock(&gl->gl_lockref.lock);
1513 }
1514 
glock_cmp(void * priv,struct list_head * a,struct list_head * b)1515 static int glock_cmp(void *priv, struct list_head *a, struct list_head *b)
1516 {
1517 	struct gfs2_glock *gla, *glb;
1518 
1519 	gla = list_entry(a, struct gfs2_glock, gl_lru);
1520 	glb = list_entry(b, struct gfs2_glock, gl_lru);
1521 
1522 	if (gla->gl_name.ln_number > glb->gl_name.ln_number)
1523 		return 1;
1524 	if (gla->gl_name.ln_number < glb->gl_name.ln_number)
1525 		return -1;
1526 
1527 	return 0;
1528 }
1529 
1530 /**
1531  * gfs2_dispose_glock_lru - Demote a list of glocks
1532  * @list: The list to dispose of
1533  *
1534  * Disposing of glocks may involve disk accesses, so that here we sort
1535  * the glocks by number (i.e. disk location of the inodes) so that if
1536  * there are any such accesses, they'll be sent in order (mostly).
1537  *
1538  * Must be called under the lru_lock, but may drop and retake this
1539  * lock. While the lru_lock is dropped, entries may vanish from the
1540  * list, but no new entries will appear on the list (since it is
1541  * private)
1542  */
1543 
gfs2_dispose_glock_lru(struct list_head * list)1544 static void gfs2_dispose_glock_lru(struct list_head *list)
1545 __releases(&lru_lock)
1546 __acquires(&lru_lock)
1547 {
1548 	struct gfs2_glock *gl;
1549 
1550 	list_sort(NULL, list, glock_cmp);
1551 
1552 	while(!list_empty(list)) {
1553 		gl = list_entry(list->next, struct gfs2_glock, gl_lru);
1554 		list_del_init(&gl->gl_lru);
1555 		clear_bit(GLF_LRU, &gl->gl_flags);
1556 		if (!spin_trylock(&gl->gl_lockref.lock)) {
1557 add_back_to_lru:
1558 			list_add(&gl->gl_lru, &lru_list);
1559 			set_bit(GLF_LRU, &gl->gl_flags);
1560 			atomic_inc(&lru_count);
1561 			continue;
1562 		}
1563 		if (test_and_set_bit(GLF_LOCK, &gl->gl_flags)) {
1564 			spin_unlock(&gl->gl_lockref.lock);
1565 			goto add_back_to_lru;
1566 		}
1567 		gl->gl_lockref.count++;
1568 		if (demote_ok(gl))
1569 			handle_callback(gl, LM_ST_UNLOCKED, 0, false);
1570 		WARN_ON(!test_and_clear_bit(GLF_LOCK, &gl->gl_flags));
1571 		__gfs2_glock_queue_work(gl, 0);
1572 		spin_unlock(&gl->gl_lockref.lock);
1573 		cond_resched_lock(&lru_lock);
1574 	}
1575 }
1576 
1577 /**
1578  * gfs2_scan_glock_lru - Scan the LRU looking for locks to demote
1579  * @nr: The number of entries to scan
1580  *
1581  * This function selects the entries on the LRU which are able to
1582  * be demoted, and then kicks off the process by calling
1583  * gfs2_dispose_glock_lru() above.
1584  */
1585 
gfs2_scan_glock_lru(int nr)1586 static long gfs2_scan_glock_lru(int nr)
1587 {
1588 	struct gfs2_glock *gl;
1589 	LIST_HEAD(skipped);
1590 	LIST_HEAD(dispose);
1591 	long freed = 0;
1592 
1593 	spin_lock(&lru_lock);
1594 	while ((nr-- >= 0) && !list_empty(&lru_list)) {
1595 		gl = list_entry(lru_list.next, struct gfs2_glock, gl_lru);
1596 
1597 		/* Test for being demotable */
1598 		if (!test_bit(GLF_LOCK, &gl->gl_flags)) {
1599 			list_move(&gl->gl_lru, &dispose);
1600 			atomic_dec(&lru_count);
1601 			freed++;
1602 			continue;
1603 		}
1604 
1605 		list_move(&gl->gl_lru, &skipped);
1606 	}
1607 	list_splice(&skipped, &lru_list);
1608 	if (!list_empty(&dispose))
1609 		gfs2_dispose_glock_lru(&dispose);
1610 	spin_unlock(&lru_lock);
1611 
1612 	return freed;
1613 }
1614 
gfs2_glock_shrink_scan(struct shrinker * shrink,struct shrink_control * sc)1615 static unsigned long gfs2_glock_shrink_scan(struct shrinker *shrink,
1616 					    struct shrink_control *sc)
1617 {
1618 	if (!(sc->gfp_mask & __GFP_FS))
1619 		return SHRINK_STOP;
1620 	return gfs2_scan_glock_lru(sc->nr_to_scan);
1621 }
1622 
gfs2_glock_shrink_count(struct shrinker * shrink,struct shrink_control * sc)1623 static unsigned long gfs2_glock_shrink_count(struct shrinker *shrink,
1624 					     struct shrink_control *sc)
1625 {
1626 	return vfs_pressure_ratio(atomic_read(&lru_count));
1627 }
1628 
1629 static struct shrinker glock_shrinker = {
1630 	.seeks = DEFAULT_SEEKS,
1631 	.count_objects = gfs2_glock_shrink_count,
1632 	.scan_objects = gfs2_glock_shrink_scan,
1633 };
1634 
1635 /**
1636  * examine_bucket - Call a function for glock in a hash bucket
1637  * @examiner: the function
1638  * @sdp: the filesystem
1639  * @bucket: the bucket
1640  *
1641  * Note that the function can be called multiple times on the same
1642  * object.  So the user must ensure that the function can cope with
1643  * that.
1644  */
1645 
glock_hash_walk(glock_examiner examiner,const struct gfs2_sbd * sdp)1646 static void glock_hash_walk(glock_examiner examiner, const struct gfs2_sbd *sdp)
1647 {
1648 	struct gfs2_glock *gl;
1649 	struct rhashtable_iter iter;
1650 
1651 	rhashtable_walk_enter(&gl_hash_table, &iter);
1652 
1653 	do {
1654 		rhashtable_walk_start(&iter);
1655 
1656 		while ((gl = rhashtable_walk_next(&iter)) && !IS_ERR(gl))
1657 			if (gl->gl_name.ln_sbd == sdp &&
1658 			    lockref_get_not_dead(&gl->gl_lockref))
1659 				examiner(gl);
1660 
1661 		rhashtable_walk_stop(&iter);
1662 	} while (cond_resched(), gl == ERR_PTR(-EAGAIN));
1663 
1664 	rhashtable_walk_exit(&iter);
1665 }
1666 
1667 /**
1668  * thaw_glock - thaw out a glock which has an unprocessed reply waiting
1669  * @gl: The glock to thaw
1670  *
1671  */
1672 
thaw_glock(struct gfs2_glock * gl)1673 static void thaw_glock(struct gfs2_glock *gl)
1674 {
1675 	if (!test_and_clear_bit(GLF_FROZEN, &gl->gl_flags)) {
1676 		gfs2_glock_put(gl);
1677 		return;
1678 	}
1679 	set_bit(GLF_REPLY_PENDING, &gl->gl_flags);
1680 	gfs2_glock_queue_work(gl, 0);
1681 }
1682 
1683 /**
1684  * clear_glock - look at a glock and see if we can free it from glock cache
1685  * @gl: the glock to look at
1686  *
1687  */
1688 
clear_glock(struct gfs2_glock * gl)1689 static void clear_glock(struct gfs2_glock *gl)
1690 {
1691 	gfs2_glock_remove_from_lru(gl);
1692 
1693 	spin_lock(&gl->gl_lockref.lock);
1694 	if (gl->gl_state != LM_ST_UNLOCKED)
1695 		handle_callback(gl, LM_ST_UNLOCKED, 0, false);
1696 	__gfs2_glock_queue_work(gl, 0);
1697 	spin_unlock(&gl->gl_lockref.lock);
1698 }
1699 
1700 /**
1701  * gfs2_glock_thaw - Thaw any frozen glocks
1702  * @sdp: The super block
1703  *
1704  */
1705 
gfs2_glock_thaw(struct gfs2_sbd * sdp)1706 void gfs2_glock_thaw(struct gfs2_sbd *sdp)
1707 {
1708 	glock_hash_walk(thaw_glock, sdp);
1709 }
1710 
dump_glock(struct seq_file * seq,struct gfs2_glock * gl,bool fsid)1711 static void dump_glock(struct seq_file *seq, struct gfs2_glock *gl, bool fsid)
1712 {
1713 	spin_lock(&gl->gl_lockref.lock);
1714 	gfs2_dump_glock(seq, gl, fsid);
1715 	spin_unlock(&gl->gl_lockref.lock);
1716 }
1717 
dump_glock_func(struct gfs2_glock * gl)1718 static void dump_glock_func(struct gfs2_glock *gl)
1719 {
1720 	dump_glock(NULL, gl, true);
1721 }
1722 
1723 /**
1724  * gfs2_gl_hash_clear - Empty out the glock hash table
1725  * @sdp: the filesystem
1726  * @wait: wait until it's all gone
1727  *
1728  * Called when unmounting the filesystem.
1729  */
1730 
gfs2_gl_hash_clear(struct gfs2_sbd * sdp)1731 void gfs2_gl_hash_clear(struct gfs2_sbd *sdp)
1732 {
1733 	set_bit(SDF_SKIP_DLM_UNLOCK, &sdp->sd_flags);
1734 	flush_workqueue(glock_workqueue);
1735 	glock_hash_walk(clear_glock, sdp);
1736 	flush_workqueue(glock_workqueue);
1737 	wait_event_timeout(sdp->sd_glock_wait,
1738 			   atomic_read(&sdp->sd_glock_disposal) == 0,
1739 			   HZ * 600);
1740 	glock_hash_walk(dump_glock_func, sdp);
1741 }
1742 
gfs2_glock_finish_truncate(struct gfs2_inode * ip)1743 void gfs2_glock_finish_truncate(struct gfs2_inode *ip)
1744 {
1745 	struct gfs2_glock *gl = ip->i_gl;
1746 	int ret;
1747 
1748 	ret = gfs2_truncatei_resume(ip);
1749 	gfs2_assert_withdraw(gl->gl_name.ln_sbd, ret == 0);
1750 
1751 	spin_lock(&gl->gl_lockref.lock);
1752 	clear_bit(GLF_LOCK, &gl->gl_flags);
1753 	run_queue(gl, 1);
1754 	spin_unlock(&gl->gl_lockref.lock);
1755 }
1756 
state2str(unsigned state)1757 static const char *state2str(unsigned state)
1758 {
1759 	switch(state) {
1760 	case LM_ST_UNLOCKED:
1761 		return "UN";
1762 	case LM_ST_SHARED:
1763 		return "SH";
1764 	case LM_ST_DEFERRED:
1765 		return "DF";
1766 	case LM_ST_EXCLUSIVE:
1767 		return "EX";
1768 	}
1769 	return "??";
1770 }
1771 
hflags2str(char * buf,u16 flags,unsigned long iflags)1772 static const char *hflags2str(char *buf, u16 flags, unsigned long iflags)
1773 {
1774 	char *p = buf;
1775 	if (flags & LM_FLAG_TRY)
1776 		*p++ = 't';
1777 	if (flags & LM_FLAG_TRY_1CB)
1778 		*p++ = 'T';
1779 	if (flags & LM_FLAG_NOEXP)
1780 		*p++ = 'e';
1781 	if (flags & LM_FLAG_ANY)
1782 		*p++ = 'A';
1783 	if (flags & LM_FLAG_PRIORITY)
1784 		*p++ = 'p';
1785 	if (flags & GL_ASYNC)
1786 		*p++ = 'a';
1787 	if (flags & GL_EXACT)
1788 		*p++ = 'E';
1789 	if (flags & GL_NOCACHE)
1790 		*p++ = 'c';
1791 	if (test_bit(HIF_HOLDER, &iflags))
1792 		*p++ = 'H';
1793 	if (test_bit(HIF_WAIT, &iflags))
1794 		*p++ = 'W';
1795 	if (test_bit(HIF_FIRST, &iflags))
1796 		*p++ = 'F';
1797 	*p = 0;
1798 	return buf;
1799 }
1800 
1801 /**
1802  * dump_holder - print information about a glock holder
1803  * @seq: the seq_file struct
1804  * @gh: the glock holder
1805  * @fs_id_buf: pointer to file system id (if requested)
1806  *
1807  */
1808 
dump_holder(struct seq_file * seq,const struct gfs2_holder * gh,const char * fs_id_buf)1809 static void dump_holder(struct seq_file *seq, const struct gfs2_holder *gh,
1810 			const char *fs_id_buf)
1811 {
1812 	struct task_struct *gh_owner = NULL;
1813 	char flags_buf[32];
1814 
1815 	rcu_read_lock();
1816 	if (gh->gh_owner_pid)
1817 		gh_owner = pid_task(gh->gh_owner_pid, PIDTYPE_PID);
1818 	gfs2_print_dbg(seq, "%s H: s:%s f:%s e:%d p:%ld [%s] %pS\n",
1819 		       fs_id_buf, state2str(gh->gh_state),
1820 		       hflags2str(flags_buf, gh->gh_flags, gh->gh_iflags),
1821 		       gh->gh_error,
1822 		       gh->gh_owner_pid ? (long)pid_nr(gh->gh_owner_pid) : -1,
1823 		       gh_owner ? gh_owner->comm : "(ended)",
1824 		       (void *)gh->gh_ip);
1825 	rcu_read_unlock();
1826 }
1827 
gflags2str(char * buf,const struct gfs2_glock * gl)1828 static const char *gflags2str(char *buf, const struct gfs2_glock *gl)
1829 {
1830 	const unsigned long *gflags = &gl->gl_flags;
1831 	char *p = buf;
1832 
1833 	if (test_bit(GLF_LOCK, gflags))
1834 		*p++ = 'l';
1835 	if (test_bit(GLF_DEMOTE, gflags))
1836 		*p++ = 'D';
1837 	if (test_bit(GLF_PENDING_DEMOTE, gflags))
1838 		*p++ = 'd';
1839 	if (test_bit(GLF_DEMOTE_IN_PROGRESS, gflags))
1840 		*p++ = 'p';
1841 	if (test_bit(GLF_DIRTY, gflags))
1842 		*p++ = 'y';
1843 	if (test_bit(GLF_LFLUSH, gflags))
1844 		*p++ = 'f';
1845 	if (test_bit(GLF_INVALIDATE_IN_PROGRESS, gflags))
1846 		*p++ = 'i';
1847 	if (test_bit(GLF_REPLY_PENDING, gflags))
1848 		*p++ = 'r';
1849 	if (test_bit(GLF_INITIAL, gflags))
1850 		*p++ = 'I';
1851 	if (test_bit(GLF_FROZEN, gflags))
1852 		*p++ = 'F';
1853 	if (test_bit(GLF_QUEUED, gflags))
1854 		*p++ = 'q';
1855 	if (test_bit(GLF_LRU, gflags))
1856 		*p++ = 'L';
1857 	if (gl->gl_object)
1858 		*p++ = 'o';
1859 	if (test_bit(GLF_BLOCKING, gflags))
1860 		*p++ = 'b';
1861 	*p = 0;
1862 	return buf;
1863 }
1864 
1865 /**
1866  * gfs2_dump_glock - print information about a glock
1867  * @seq: The seq_file struct
1868  * @gl: the glock
1869  * @fsid: If true, also dump the file system id
1870  *
1871  * The file format is as follows:
1872  * One line per object, capital letters are used to indicate objects
1873  * G = glock, I = Inode, R = rgrp, H = holder. Glocks are not indented,
1874  * other objects are indented by a single space and follow the glock to
1875  * which they are related. Fields are indicated by lower case letters
1876  * followed by a colon and the field value, except for strings which are in
1877  * [] so that its possible to see if they are composed of spaces for
1878  * example. The field's are n = number (id of the object), f = flags,
1879  * t = type, s = state, r = refcount, e = error, p = pid.
1880  *
1881  */
1882 
gfs2_dump_glock(struct seq_file * seq,struct gfs2_glock * gl,bool fsid)1883 void gfs2_dump_glock(struct seq_file *seq, struct gfs2_glock *gl, bool fsid)
1884 {
1885 	const struct gfs2_glock_operations *glops = gl->gl_ops;
1886 	unsigned long long dtime;
1887 	const struct gfs2_holder *gh;
1888 	char gflags_buf[32];
1889 	struct gfs2_sbd *sdp = gl->gl_name.ln_sbd;
1890 	char fs_id_buf[sizeof(sdp->sd_fsname) + 7];
1891 
1892 	memset(fs_id_buf, 0, sizeof(fs_id_buf));
1893 	if (fsid && sdp) /* safety precaution */
1894 		sprintf(fs_id_buf, "fsid=%s: ", sdp->sd_fsname);
1895 	dtime = jiffies - gl->gl_demote_time;
1896 	dtime *= 1000000/HZ; /* demote time in uSec */
1897 	if (!test_bit(GLF_DEMOTE, &gl->gl_flags))
1898 		dtime = 0;
1899 	gfs2_print_dbg(seq, "%sG:  s:%s n:%u/%llx f:%s t:%s d:%s/%llu a:%d "
1900 		       "v:%d r:%d m:%ld\n", fs_id_buf, state2str(gl->gl_state),
1901 		  gl->gl_name.ln_type,
1902 		  (unsigned long long)gl->gl_name.ln_number,
1903 		  gflags2str(gflags_buf, gl),
1904 		  state2str(gl->gl_target),
1905 		  state2str(gl->gl_demote_state), dtime,
1906 		  atomic_read(&gl->gl_ail_count),
1907 		  atomic_read(&gl->gl_revokes),
1908 		  (int)gl->gl_lockref.count, gl->gl_hold_time);
1909 
1910 	list_for_each_entry(gh, &gl->gl_holders, gh_list)
1911 		dump_holder(seq, gh, fs_id_buf);
1912 
1913 	if (gl->gl_state != LM_ST_UNLOCKED && glops->go_dump)
1914 		glops->go_dump(seq, gl, fs_id_buf);
1915 }
1916 
gfs2_glstats_seq_show(struct seq_file * seq,void * iter_ptr)1917 static int gfs2_glstats_seq_show(struct seq_file *seq, void *iter_ptr)
1918 {
1919 	struct gfs2_glock *gl = iter_ptr;
1920 
1921 	seq_printf(seq, "G: n:%u/%llx rtt:%llu/%llu rttb:%llu/%llu irt:%llu/%llu dcnt: %llu qcnt: %llu\n",
1922 		   gl->gl_name.ln_type,
1923 		   (unsigned long long)gl->gl_name.ln_number,
1924 		   (unsigned long long)gl->gl_stats.stats[GFS2_LKS_SRTT],
1925 		   (unsigned long long)gl->gl_stats.stats[GFS2_LKS_SRTTVAR],
1926 		   (unsigned long long)gl->gl_stats.stats[GFS2_LKS_SRTTB],
1927 		   (unsigned long long)gl->gl_stats.stats[GFS2_LKS_SRTTVARB],
1928 		   (unsigned long long)gl->gl_stats.stats[GFS2_LKS_SIRT],
1929 		   (unsigned long long)gl->gl_stats.stats[GFS2_LKS_SIRTVAR],
1930 		   (unsigned long long)gl->gl_stats.stats[GFS2_LKS_DCOUNT],
1931 		   (unsigned long long)gl->gl_stats.stats[GFS2_LKS_QCOUNT]);
1932 	return 0;
1933 }
1934 
1935 static const char *gfs2_gltype[] = {
1936 	"type",
1937 	"reserved",
1938 	"nondisk",
1939 	"inode",
1940 	"rgrp",
1941 	"meta",
1942 	"iopen",
1943 	"flock",
1944 	"plock",
1945 	"quota",
1946 	"journal",
1947 };
1948 
1949 static const char *gfs2_stype[] = {
1950 	[GFS2_LKS_SRTT]		= "srtt",
1951 	[GFS2_LKS_SRTTVAR]	= "srttvar",
1952 	[GFS2_LKS_SRTTB]	= "srttb",
1953 	[GFS2_LKS_SRTTVARB]	= "srttvarb",
1954 	[GFS2_LKS_SIRT]		= "sirt",
1955 	[GFS2_LKS_SIRTVAR]	= "sirtvar",
1956 	[GFS2_LKS_DCOUNT]	= "dlm",
1957 	[GFS2_LKS_QCOUNT]	= "queue",
1958 };
1959 
1960 #define GFS2_NR_SBSTATS (ARRAY_SIZE(gfs2_gltype) * ARRAY_SIZE(gfs2_stype))
1961 
gfs2_sbstats_seq_show(struct seq_file * seq,void * iter_ptr)1962 static int gfs2_sbstats_seq_show(struct seq_file *seq, void *iter_ptr)
1963 {
1964 	struct gfs2_sbd *sdp = seq->private;
1965 	loff_t pos = *(loff_t *)iter_ptr;
1966 	unsigned index = pos >> 3;
1967 	unsigned subindex = pos & 0x07;
1968 	int i;
1969 
1970 	if (index == 0 && subindex != 0)
1971 		return 0;
1972 
1973 	seq_printf(seq, "%-10s %8s:", gfs2_gltype[index],
1974 		   (index == 0) ? "cpu": gfs2_stype[subindex]);
1975 
1976 	for_each_possible_cpu(i) {
1977                 const struct gfs2_pcpu_lkstats *lkstats = per_cpu_ptr(sdp->sd_lkstats, i);
1978 
1979 		if (index == 0)
1980 			seq_printf(seq, " %15u", i);
1981 		else
1982 			seq_printf(seq, " %15llu", (unsigned long long)lkstats->
1983 				   lkstats[index - 1].stats[subindex]);
1984 	}
1985 	seq_putc(seq, '\n');
1986 	return 0;
1987 }
1988 
gfs2_glock_init(void)1989 int __init gfs2_glock_init(void)
1990 {
1991 	int i, ret;
1992 
1993 	ret = rhashtable_init(&gl_hash_table, &ht_parms);
1994 	if (ret < 0)
1995 		return ret;
1996 
1997 	glock_workqueue = alloc_workqueue("glock_workqueue", WQ_MEM_RECLAIM |
1998 					  WQ_HIGHPRI | WQ_FREEZABLE, 0);
1999 	if (!glock_workqueue) {
2000 		rhashtable_destroy(&gl_hash_table);
2001 		return -ENOMEM;
2002 	}
2003 	gfs2_delete_workqueue = alloc_workqueue("delete_workqueue",
2004 						WQ_MEM_RECLAIM | WQ_FREEZABLE,
2005 						0);
2006 	if (!gfs2_delete_workqueue) {
2007 		destroy_workqueue(glock_workqueue);
2008 		rhashtable_destroy(&gl_hash_table);
2009 		return -ENOMEM;
2010 	}
2011 
2012 	ret = register_shrinker(&glock_shrinker);
2013 	if (ret) {
2014 		destroy_workqueue(gfs2_delete_workqueue);
2015 		destroy_workqueue(glock_workqueue);
2016 		rhashtable_destroy(&gl_hash_table);
2017 		return ret;
2018 	}
2019 
2020 	for (i = 0; i < GLOCK_WAIT_TABLE_SIZE; i++)
2021 		init_waitqueue_head(glock_wait_table + i);
2022 
2023 	return 0;
2024 }
2025 
gfs2_glock_exit(void)2026 void gfs2_glock_exit(void)
2027 {
2028 	unregister_shrinker(&glock_shrinker);
2029 	rhashtable_destroy(&gl_hash_table);
2030 	destroy_workqueue(glock_workqueue);
2031 	destroy_workqueue(gfs2_delete_workqueue);
2032 }
2033 
gfs2_glock_iter_next(struct gfs2_glock_iter * gi,loff_t n)2034 static void gfs2_glock_iter_next(struct gfs2_glock_iter *gi, loff_t n)
2035 {
2036 	struct gfs2_glock *gl = gi->gl;
2037 
2038 	if (gl) {
2039 		if (n == 0)
2040 			return;
2041 		if (!lockref_put_not_zero(&gl->gl_lockref))
2042 			gfs2_glock_queue_put(gl);
2043 	}
2044 	for (;;) {
2045 		gl = rhashtable_walk_next(&gi->hti);
2046 		if (IS_ERR_OR_NULL(gl)) {
2047 			if (gl == ERR_PTR(-EAGAIN)) {
2048 				n = 1;
2049 				continue;
2050 			}
2051 			gl = NULL;
2052 			break;
2053 		}
2054 		if (gl->gl_name.ln_sbd != gi->sdp)
2055 			continue;
2056 		if (n <= 1) {
2057 			if (!lockref_get_not_dead(&gl->gl_lockref))
2058 				continue;
2059 			break;
2060 		} else {
2061 			if (__lockref_is_dead(&gl->gl_lockref))
2062 				continue;
2063 			n--;
2064 		}
2065 	}
2066 	gi->gl = gl;
2067 }
2068 
gfs2_glock_seq_start(struct seq_file * seq,loff_t * pos)2069 static void *gfs2_glock_seq_start(struct seq_file *seq, loff_t *pos)
2070 	__acquires(RCU)
2071 {
2072 	struct gfs2_glock_iter *gi = seq->private;
2073 	loff_t n;
2074 
2075 	/*
2076 	 * We can either stay where we are, skip to the next hash table
2077 	 * entry, or start from the beginning.
2078 	 */
2079 	if (*pos < gi->last_pos) {
2080 		rhashtable_walk_exit(&gi->hti);
2081 		rhashtable_walk_enter(&gl_hash_table, &gi->hti);
2082 		n = *pos + 1;
2083 	} else {
2084 		n = *pos - gi->last_pos;
2085 	}
2086 
2087 	rhashtable_walk_start(&gi->hti);
2088 
2089 	gfs2_glock_iter_next(gi, n);
2090 	gi->last_pos = *pos;
2091 	return gi->gl;
2092 }
2093 
gfs2_glock_seq_next(struct seq_file * seq,void * iter_ptr,loff_t * pos)2094 static void *gfs2_glock_seq_next(struct seq_file *seq, void *iter_ptr,
2095 				 loff_t *pos)
2096 {
2097 	struct gfs2_glock_iter *gi = seq->private;
2098 
2099 	(*pos)++;
2100 	gi->last_pos = *pos;
2101 	gfs2_glock_iter_next(gi, 1);
2102 	return gi->gl;
2103 }
2104 
gfs2_glock_seq_stop(struct seq_file * seq,void * iter_ptr)2105 static void gfs2_glock_seq_stop(struct seq_file *seq, void *iter_ptr)
2106 	__releases(RCU)
2107 {
2108 	struct gfs2_glock_iter *gi = seq->private;
2109 
2110 	rhashtable_walk_stop(&gi->hti);
2111 }
2112 
gfs2_glock_seq_show(struct seq_file * seq,void * iter_ptr)2113 static int gfs2_glock_seq_show(struct seq_file *seq, void *iter_ptr)
2114 {
2115 	dump_glock(seq, iter_ptr, false);
2116 	return 0;
2117 }
2118 
gfs2_sbstats_seq_start(struct seq_file * seq,loff_t * pos)2119 static void *gfs2_sbstats_seq_start(struct seq_file *seq, loff_t *pos)
2120 {
2121 	preempt_disable();
2122 	if (*pos >= GFS2_NR_SBSTATS)
2123 		return NULL;
2124 	return pos;
2125 }
2126 
gfs2_sbstats_seq_next(struct seq_file * seq,void * iter_ptr,loff_t * pos)2127 static void *gfs2_sbstats_seq_next(struct seq_file *seq, void *iter_ptr,
2128 				   loff_t *pos)
2129 {
2130 	(*pos)++;
2131 	if (*pos >= GFS2_NR_SBSTATS)
2132 		return NULL;
2133 	return pos;
2134 }
2135 
gfs2_sbstats_seq_stop(struct seq_file * seq,void * iter_ptr)2136 static void gfs2_sbstats_seq_stop(struct seq_file *seq, void *iter_ptr)
2137 {
2138 	preempt_enable();
2139 }
2140 
2141 static const struct seq_operations gfs2_glock_seq_ops = {
2142 	.start = gfs2_glock_seq_start,
2143 	.next  = gfs2_glock_seq_next,
2144 	.stop  = gfs2_glock_seq_stop,
2145 	.show  = gfs2_glock_seq_show,
2146 };
2147 
2148 static const struct seq_operations gfs2_glstats_seq_ops = {
2149 	.start = gfs2_glock_seq_start,
2150 	.next  = gfs2_glock_seq_next,
2151 	.stop  = gfs2_glock_seq_stop,
2152 	.show  = gfs2_glstats_seq_show,
2153 };
2154 
2155 static const struct seq_operations gfs2_sbstats_seq_ops = {
2156 	.start = gfs2_sbstats_seq_start,
2157 	.next  = gfs2_sbstats_seq_next,
2158 	.stop  = gfs2_sbstats_seq_stop,
2159 	.show  = gfs2_sbstats_seq_show,
2160 };
2161 
2162 #define GFS2_SEQ_GOODSIZE min(PAGE_SIZE << PAGE_ALLOC_COSTLY_ORDER, 65536UL)
2163 
__gfs2_glocks_open(struct inode * inode,struct file * file,const struct seq_operations * ops)2164 static int __gfs2_glocks_open(struct inode *inode, struct file *file,
2165 			      const struct seq_operations *ops)
2166 {
2167 	int ret = seq_open_private(file, ops, sizeof(struct gfs2_glock_iter));
2168 	if (ret == 0) {
2169 		struct seq_file *seq = file->private_data;
2170 		struct gfs2_glock_iter *gi = seq->private;
2171 
2172 		gi->sdp = inode->i_private;
2173 		seq->buf = kmalloc(GFS2_SEQ_GOODSIZE, GFP_KERNEL | __GFP_NOWARN);
2174 		if (seq->buf)
2175 			seq->size = GFS2_SEQ_GOODSIZE;
2176 		/*
2177 		 * Initially, we are "before" the first hash table entry; the
2178 		 * first call to rhashtable_walk_next gets us the first entry.
2179 		 */
2180 		gi->last_pos = -1;
2181 		gi->gl = NULL;
2182 		rhashtable_walk_enter(&gl_hash_table, &gi->hti);
2183 	}
2184 	return ret;
2185 }
2186 
gfs2_glocks_open(struct inode * inode,struct file * file)2187 static int gfs2_glocks_open(struct inode *inode, struct file *file)
2188 {
2189 	return __gfs2_glocks_open(inode, file, &gfs2_glock_seq_ops);
2190 }
2191 
gfs2_glocks_release(struct inode * inode,struct file * file)2192 static int gfs2_glocks_release(struct inode *inode, struct file *file)
2193 {
2194 	struct seq_file *seq = file->private_data;
2195 	struct gfs2_glock_iter *gi = seq->private;
2196 
2197 	if (gi->gl)
2198 		gfs2_glock_put(gi->gl);
2199 	rhashtable_walk_exit(&gi->hti);
2200 	return seq_release_private(inode, file);
2201 }
2202 
gfs2_glstats_open(struct inode * inode,struct file * file)2203 static int gfs2_glstats_open(struct inode *inode, struct file *file)
2204 {
2205 	return __gfs2_glocks_open(inode, file, &gfs2_glstats_seq_ops);
2206 }
2207 
gfs2_sbstats_open(struct inode * inode,struct file * file)2208 static int gfs2_sbstats_open(struct inode *inode, struct file *file)
2209 {
2210 	int ret = seq_open(file, &gfs2_sbstats_seq_ops);
2211 	if (ret == 0) {
2212 		struct seq_file *seq = file->private_data;
2213 		seq->private = inode->i_private;  /* sdp */
2214 	}
2215 	return ret;
2216 }
2217 
2218 static const struct file_operations gfs2_glocks_fops = {
2219 	.owner   = THIS_MODULE,
2220 	.open    = gfs2_glocks_open,
2221 	.read    = seq_read,
2222 	.llseek  = seq_lseek,
2223 	.release = gfs2_glocks_release,
2224 };
2225 
2226 static const struct file_operations gfs2_glstats_fops = {
2227 	.owner   = THIS_MODULE,
2228 	.open    = gfs2_glstats_open,
2229 	.read    = seq_read,
2230 	.llseek  = seq_lseek,
2231 	.release = gfs2_glocks_release,
2232 };
2233 
2234 static const struct file_operations gfs2_sbstats_fops = {
2235 	.owner   = THIS_MODULE,
2236 	.open	 = gfs2_sbstats_open,
2237 	.read    = seq_read,
2238 	.llseek  = seq_lseek,
2239 	.release = seq_release,
2240 };
2241 
gfs2_create_debugfs_file(struct gfs2_sbd * sdp)2242 void gfs2_create_debugfs_file(struct gfs2_sbd *sdp)
2243 {
2244 	sdp->debugfs_dir = debugfs_create_dir(sdp->sd_table_name, gfs2_root);
2245 
2246 	debugfs_create_file("glocks", S_IFREG | S_IRUGO, sdp->debugfs_dir, sdp,
2247 			    &gfs2_glocks_fops);
2248 
2249 	debugfs_create_file("glstats", S_IFREG | S_IRUGO, sdp->debugfs_dir, sdp,
2250 			    &gfs2_glstats_fops);
2251 
2252 	debugfs_create_file("sbstats", S_IFREG | S_IRUGO, sdp->debugfs_dir, sdp,
2253 			    &gfs2_sbstats_fops);
2254 }
2255 
gfs2_delete_debugfs_file(struct gfs2_sbd * sdp)2256 void gfs2_delete_debugfs_file(struct gfs2_sbd *sdp)
2257 {
2258 	debugfs_remove_recursive(sdp->debugfs_dir);
2259 	sdp->debugfs_dir = NULL;
2260 }
2261 
gfs2_register_debugfs(void)2262 void gfs2_register_debugfs(void)
2263 {
2264 	gfs2_root = debugfs_create_dir("gfs2", NULL);
2265 }
2266 
gfs2_unregister_debugfs(void)2267 void gfs2_unregister_debugfs(void)
2268 {
2269 	debugfs_remove(gfs2_root);
2270 	gfs2_root = NULL;
2271 }
2272