Lines Matching +full:wait +full:- +full:pin
1 // SPDX-License-Identifier: GPL-2.0
20 * entries (same as garbage collection would), then we replay them - reinserting
30 struct closure *cl = bio->bi_private; in journal_read_endio()
38 struct journal_device *ja = &ca->journal; in journal_read_bucket()
39 struct bio *bio = &ja->bio; in journal_read_bucket()
42 struct jset *j, *data = ca->set->journal.w[0].data; in journal_read_bucket()
46 sector_t bucket = bucket_to_sector(ca->set, ca->sb.d[bucket_index]); in journal_read_bucket()
52 while (offset < ca->sb.bucket_size) { in journal_read_bucket()
53 reread: left = ca->sb.bucket_size - offset; in journal_read_bucket()
57 bio->bi_iter.bi_sector = bucket + offset; in journal_read_bucket()
58 bio_set_dev(bio, ca->bdev); in journal_read_bucket()
59 bio->bi_iter.bi_size = len << 9; in journal_read_bucket()
61 bio->bi_end_io = journal_read_endio; in journal_read_bucket()
62 bio->bi_private = &cl; in journal_read_bucket()
66 closure_bio_submit(ca->set, bio, &cl); in journal_read_bucket()
80 if (j->magic != jset_magic(&ca->sb)) { in journal_read_bucket()
95 if (j->csum != csum_set(j)) { in journal_read_bucket()
105 * i->j.seq, the node on head has the smallest (oldest) in journal_read_bucket()
112 * i->j.seq < j->last_seq, it means the oldest jset in journal_read_bucket()
120 if (i->j.seq >= j->last_seq) in journal_read_bucket()
122 list_del(&i->list); in journal_read_bucket()
128 if (j->seq == i->j.seq) in journal_read_bucket()
132 * if j->seq is less than any i->j.last_seq in journal_read_bucket()
135 if (j->seq < i->j.last_seq) in journal_read_bucket()
142 if (j->seq > i->j.seq) { in journal_read_bucket()
143 where = &i->list; in journal_read_bucket()
153 return -ENOMEM; in journal_read_bucket()
154 memcpy(&i->j, j, bytes); in journal_read_bucket()
156 list_add(&i->list, where); in journal_read_bucket()
159 if (j->seq > ja->seq[bucket_index]) in journal_read_bucket()
160 ja->seq[bucket_index] = j->seq; in journal_read_bucket()
162 offset += blocks * ca->sb.block_size; in journal_read_bucket()
163 len -= blocks * ca->sb.block_size; in journal_read_bucket()
182 struct cache *ca = c->cache; in bch_journal_read()
184 struct journal_device *ja = &ca->journal; in bch_journal_read()
190 pr_debug("%u journal buckets\n", ca->sb.njournal_buckets); in bch_journal_read()
196 for (i = 0; i < ca->sb.njournal_buckets; i++) { in bch_journal_read()
202 l = (i * 2654435769U) % ca->sb.njournal_buckets; in bch_journal_read()
217 for_each_clear_bit(l, bitmap, ca->sb.njournal_buckets) in bch_journal_read()
222 if (l == ca->sb.njournal_buckets) in bch_journal_read()
229 r = find_next_bit(bitmap, ca->sb.njournal_buckets, l + 1); in bch_journal_read()
233 seq = list_entry(list->prev, struct journal_replay, in bch_journal_read()
234 list)->j.seq; in bch_journal_read()
239 if (seq != list_entry(list->prev, struct journal_replay, in bch_journal_read()
240 list)->j.seq) in bch_journal_read()
251 m, ca->sb.njournal_buckets); in bch_journal_read()
255 if (!l--) in bch_journal_read()
256 l = ca->sb.njournal_buckets - 1; in bch_journal_read()
270 for (i = 0; i < ca->sb.njournal_buckets; i++) in bch_journal_read()
271 if (ja->seq[i] > seq) { in bch_journal_read()
272 seq = ja->seq[i]; in bch_journal_read()
276 * ja->cur_idx in bch_journal_read()
278 ja->cur_idx = i; in bch_journal_read()
279 ja->last_idx = ja->discard_idx = (i + 1) % in bch_journal_read()
280 ca->sb.njournal_buckets; in bch_journal_read()
286 c->journal.seq = list_entry(list->prev, in bch_journal_read()
288 list)->j.seq; in bch_journal_read()
299 struct journal *j = &c->journal; in bch_journal_mark()
300 uint64_t last = j->seq; in bch_journal_mark()
303 * journal.pin should never fill up - we never write a journal in bch_journal_mark()
310 BUG_ON(last < i->j.seq); in bch_journal_mark()
311 i->pin = NULL; in bch_journal_mark()
313 while (last-- != i->j.seq) in bch_journal_mark()
314 if (fifo_free(&j->pin) > 1) { in bch_journal_mark()
315 fifo_push_front(&j->pin, p); in bch_journal_mark()
316 atomic_set(&fifo_front(&j->pin), 0); in bch_journal_mark()
319 if (fifo_free(&j->pin) > 1) { in bch_journal_mark()
320 fifo_push_front(&j->pin, p); in bch_journal_mark()
321 i->pin = &fifo_front(&j->pin); in bch_journal_mark()
322 atomic_set(i->pin, 1); in bch_journal_mark()
325 for (k = i->j.start; in bch_journal_mark()
326 k < bset_bkey_last(&i->j); in bch_journal_mark()
333 atomic_inc(&PTR_BUCKET(c, k, j)->pin); in bch_journal_mark()
342 struct cache *ca = s->cache; in is_discard_enabled()
344 if (ca->discard) in is_discard_enabled()
355 list_entry(list->prev, struct journal_replay, list); in bch_journal_replay()
357 uint64_t start = i->j.last_seq, end = i->j.seq, n = start; in bch_journal_replay()
361 BUG_ON(i->pin && atomic_read(i->pin) != 1); in bch_journal_replay()
363 if (n != i->j.seq) { in bch_journal_replay()
365 pr_info("journal entries %llu-%llu may be discarded! (replaying %llu-%llu)\n", in bch_journal_replay()
366 n, i->j.seq - 1, start, end); in bch_journal_replay()
368 pr_err("journal entries %llu-%llu missing! (replaying %llu-%llu)\n", in bch_journal_replay()
369 n, i->j.seq - 1, start, end); in bch_journal_replay()
370 ret = -EIO; in bch_journal_replay()
375 for (k = i->j.start; in bch_journal_replay()
376 k < bset_bkey_last(&i->j); in bch_journal_replay()
382 ret = bch_btree_insert(s, &keylist, i->pin, NULL); in bch_journal_replay()
392 if (i->pin) in bch_journal_replay()
393 atomic_dec(i->pin); in bch_journal_replay()
394 n = i->j.seq + 1; in bch_journal_replay()
403 list_del(&i->list); in bch_journal_replay()
412 j->do_reserve = true; in bch_journal_space_reserve()
425 if (c->journal.btree_flushing) in btree_flush_write()
428 spin_lock(&c->journal.flush_write_lock); in btree_flush_write()
429 if (c->journal.btree_flushing) { in btree_flush_write()
430 spin_unlock(&c->journal.flush_write_lock); in btree_flush_write()
433 c->journal.btree_flushing = true; in btree_flush_write()
434 spin_unlock(&c->journal.flush_write_lock); in btree_flush_write()
437 spin_lock(&c->journal.lock); in btree_flush_write()
438 fifo_front_p = &fifo_front(&c->journal.pin); in btree_flush_write()
445 spin_unlock(&c->journal.lock); in btree_flush_write()
448 spin_unlock(&c->journal.lock); in btree_flush_write()
450 mask = c->journal.pin.mask; in btree_flush_write()
452 atomic_long_inc(&c->flush_write); in btree_flush_write()
455 mutex_lock(&c->bucket_lock); in btree_flush_write()
456 list_for_each_entry_safe_reverse(b, t, &c->btree_cache, list) { in btree_flush_write()
459 * c->journal.lock here, because we don't need to know in btree_flush_write()
461 * front pointer of c->journal.pin is changed. in btree_flush_write()
463 now_fifo_front_p = &fifo_front(&c->journal.pin); in btree_flush_write()
466 * pointer of c->journal.pin changes, it is unnecessary in btree_flush_write()
467 * to scan c->btree_cache anymore, just quit the loop and in btree_flush_write()
483 mutex_lock(&b->write_lock); in btree_flush_write()
486 mutex_unlock(&b->write_lock); in btree_flush_write()
490 if (!btree_current_write(b)->journal) { in btree_flush_write()
491 mutex_unlock(&b->write_lock); in btree_flush_write()
501 * - the list_for_each_xxx loop will quit when checking in btree_flush_write()
503 * - If there are matched nodes recorded in btree_nodes[], in btree_flush_write()
506 * will be ignored and skipped in the folowing for-loop. in btree_flush_write()
508 if (((btree_current_write(b)->journal - fifo_front_p) & in btree_flush_write()
510 mutex_unlock(&b->write_lock); in btree_flush_write()
516 mutex_unlock(&b->write_lock); in btree_flush_write()
520 * To avoid holding c->bucket_lock too long time, in btree_flush_write()
529 mutex_unlock(&c->bucket_lock); in btree_flush_write()
538 /* safe to check without holding b->write_lock */ in btree_flush_write()
544 mutex_lock(&b->write_lock); in btree_flush_write()
545 if (!btree_current_write(b)->journal) { in btree_flush_write()
546 clear_bit(BTREE_NODE_journal_flush, &b->flags); in btree_flush_write()
547 mutex_unlock(&b->write_lock); in btree_flush_write()
553 clear_bit(BTREE_NODE_journal_flush, &b->flags); in btree_flush_write()
554 mutex_unlock(&b->write_lock); in btree_flush_write()
560 clear_bit(BTREE_NODE_journal_flush, &b->flags); in btree_flush_write()
561 mutex_unlock(&b->write_lock); in btree_flush_write()
565 spin_lock(&c->journal.flush_write_lock); in btree_flush_write()
566 c->journal.btree_flushing = false; in btree_flush_write()
567 spin_unlock(&c->journal.flush_write_lock); in btree_flush_write()
570 #define last_seq(j) ((j)->seq - fifo_used(&(j)->pin) + 1)
578 atomic_set(&ja->discard_in_flight, DISCARD_DONE); in journal_discard_endio()
580 closure_wake_up(&ca->set->journal.wait); in journal_discard_endio()
581 closure_put(&ca->set->cl); in journal_discard_endio()
589 submit_bio(&ja->discard_bio); in journal_discard_work()
594 struct journal_device *ja = &ca->journal; in do_journal_discard()
595 struct bio *bio = &ja->discard_bio; in do_journal_discard()
597 if (!ca->discard) { in do_journal_discard()
598 ja->discard_idx = ja->last_idx; in do_journal_discard()
602 switch (atomic_read(&ja->discard_in_flight)) { in do_journal_discard()
607 ja->discard_idx = (ja->discard_idx + 1) % in do_journal_discard()
608 ca->sb.njournal_buckets; in do_journal_discard()
610 atomic_set(&ja->discard_in_flight, DISCARD_READY); in do_journal_discard()
614 if (ja->discard_idx == ja->last_idx) in do_journal_discard()
617 atomic_set(&ja->discard_in_flight, DISCARD_IN_FLIGHT); in do_journal_discard()
619 bio_init(bio, bio->bi_inline_vecs, 1); in do_journal_discard()
621 bio->bi_iter.bi_sector = bucket_to_sector(ca->set, in do_journal_discard()
622 ca->sb.d[ja->discard_idx]); in do_journal_discard()
623 bio_set_dev(bio, ca->bdev); in do_journal_discard()
624 bio->bi_iter.bi_size = bucket_bytes(ca); in do_journal_discard()
625 bio->bi_end_io = journal_discard_endio; in do_journal_discard()
627 closure_get(&ca->set->cl); in do_journal_discard()
628 INIT_WORK(&ja->discard_work, journal_discard_work); in do_journal_discard()
629 queue_work(bch_journal_wq, &ja->discard_work); in do_journal_discard()
635 struct journal *j = &c->journal; in free_journal_buckets()
636 struct cache *ca = c->cache; in free_journal_buckets()
637 struct journal_device *ja = &c->cache->journal; in free_journal_buckets()
641 if (ja->cur_idx >= ja->discard_idx) in free_journal_buckets()
642 n = ca->sb.njournal_buckets + ja->discard_idx - ja->cur_idx; in free_journal_buckets()
644 n = ja->discard_idx - ja->cur_idx; in free_journal_buckets()
646 if (n > (1 + j->do_reserve)) in free_journal_buckets()
647 return n - (1 + j->do_reserve); in free_journal_buckets()
654 struct bkey *k = &c->journal.key; in journal_reclaim()
655 struct cache *ca = c->cache; in journal_reclaim()
657 struct journal_device *ja = &ca->journal; in journal_reclaim()
660 atomic_long_inc(&c->reclaim); in journal_reclaim()
662 while (!atomic_read(&fifo_front(&c->journal.pin))) in journal_reclaim()
663 fifo_pop(&c->journal.pin, p); in journal_reclaim()
665 last_seq = last_seq(&c->journal); in journal_reclaim()
669 while (ja->last_idx != ja->cur_idx && in journal_reclaim()
670 ja->seq[ja->last_idx] < last_seq) in journal_reclaim()
671 ja->last_idx = (ja->last_idx + 1) % in journal_reclaim()
672 ca->sb.njournal_buckets; in journal_reclaim()
676 if (c->journal.blocks_free) in journal_reclaim()
682 ja->cur_idx = (ja->cur_idx + 1) % ca->sb.njournal_buckets; in journal_reclaim()
683 k->ptr[0] = MAKE_PTR(0, in journal_reclaim()
684 bucket_to_sector(c, ca->sb.d[ja->cur_idx]), in journal_reclaim()
685 ca->sb.nr_this_dev); in journal_reclaim()
686 atomic_long_inc(&c->reclaimed_journal_buckets); in journal_reclaim()
690 c->journal.blocks_free = ca->sb.bucket_size >> c->block_bits; in journal_reclaim()
693 if (!journal_full(&c->journal)) in journal_reclaim()
694 __closure_wake_up(&c->journal.wait); in journal_reclaim()
701 j->cur = (j->cur == j->w) in bch_journal_next()
702 ? &j->w[1] in bch_journal_next()
703 : &j->w[0]; in bch_journal_next()
706 * The fifo_push() needs to happen at the same time as j->seq is in bch_journal_next()
709 BUG_ON(!fifo_push(&j->pin, p)); in bch_journal_next()
710 atomic_set(&fifo_back(&j->pin), 1); in bch_journal_next()
712 j->cur->data->seq = ++j->seq; in bch_journal_next()
713 j->cur->dirty = false; in bch_journal_next()
714 j->cur->need_write = false; in bch_journal_next()
715 j->cur->data->keys = 0; in bch_journal_next()
717 if (fifo_full(&j->pin)) in bch_journal_next()
718 pr_debug("journal_pin full (%zu)\n", fifo_used(&j->pin)); in bch_journal_next()
723 struct journal_write *w = bio->bi_private; in journal_write_endio()
725 cache_set_err_on(bio->bi_status, w->c, "journal io error"); in journal_write_endio()
726 closure_put(&w->c->journal.io); in journal_write_endio()
734 struct journal_write *w = (j->cur == j->w) in journal_write_done()
735 ? &j->w[1] in journal_write_done()
736 : &j->w[0]; in journal_write_done()
738 __closure_wake_up(&w->wait); in journal_write_done()
743 __releases(&c->journal.lock) in journal_write_unlock()
747 c->journal.io_in_flight = 0; in journal_write_unlock()
748 spin_unlock(&c->journal.lock); in journal_write_unlock()
752 __releases(c->journal.lock) in journal_write_unlocked()
755 struct cache *ca = c->cache; in journal_write_unlocked()
756 struct journal_write *w = c->journal.cur; in journal_write_unlocked()
757 struct bkey *k = &c->journal.key; in journal_write_unlocked()
758 unsigned int i, sectors = set_blocks(w->data, block_bytes(ca)) * in journal_write_unlocked()
759 ca->sb.block_size; in journal_write_unlocked()
766 if (!w->need_write) { in journal_write_unlocked()
769 } else if (journal_full(&c->journal)) { in journal_write_unlocked()
771 spin_unlock(&c->journal.lock); in journal_write_unlocked()
778 c->journal.blocks_free -= set_blocks(w->data, block_bytes(ca)); in journal_write_unlocked()
780 w->data->btree_level = c->root->level; in journal_write_unlocked()
782 bkey_copy(&w->data->btree_root, &c->root->key); in journal_write_unlocked()
783 bkey_copy(&w->data->uuid_bucket, &c->uuid_bucket); in journal_write_unlocked()
785 w->data->prio_bucket[ca->sb.nr_this_dev] = ca->prio_buckets[0]; in journal_write_unlocked()
786 w->data->magic = jset_magic(&ca->sb); in journal_write_unlocked()
787 w->data->version = BCACHE_JSET_VERSION; in journal_write_unlocked()
788 w->data->last_seq = last_seq(&c->journal); in journal_write_unlocked()
789 w->data->csum = csum_set(w->data); in journal_write_unlocked()
793 bio = &ca->journal.bio; in journal_write_unlocked()
795 atomic_long_add(sectors, &ca->meta_sectors_written); in journal_write_unlocked()
798 bio->bi_iter.bi_sector = PTR_OFFSET(k, i); in journal_write_unlocked()
799 bio_set_dev(bio, ca->bdev); in journal_write_unlocked()
800 bio->bi_iter.bi_size = sectors << 9; in journal_write_unlocked()
802 bio->bi_end_io = journal_write_endio; in journal_write_unlocked()
803 bio->bi_private = w; in journal_write_unlocked()
806 bch_bio_map(bio, w->data); in journal_write_unlocked()
808 trace_bcache_journal_write(bio, w->data->keys); in journal_write_unlocked()
813 ca->journal.seq[ca->journal.cur_idx] = w->data->seq; in journal_write_unlocked()
819 atomic_dec_bug(&fifo_back(&c->journal.pin)); in journal_write_unlocked()
820 bch_journal_next(&c->journal); in journal_write_unlocked()
823 spin_unlock(&c->journal.lock); in journal_write_unlocked()
835 spin_lock(&c->journal.lock); in journal_write()
840 __releases(c->journal.lock) in journal_try_write()
842 struct closure *cl = &c->journal.io; in journal_try_write()
843 struct journal_write *w = c->journal.cur; in journal_try_write()
845 w->need_write = true; in journal_try_write()
847 if (!c->journal.io_in_flight) { in journal_try_write()
848 c->journal.io_in_flight = 1; in journal_try_write()
849 closure_call(cl, journal_write_unlocked, NULL, &c->cl); in journal_try_write()
851 spin_unlock(&c->journal.lock); in journal_try_write()
857 __acquires(&c->journal.lock) in journal_wait_for_write()
861 bool wait = false; in journal_wait_for_write() local
862 struct cache *ca = c->cache; in journal_wait_for_write()
866 spin_lock(&c->journal.lock); in journal_wait_for_write()
869 struct journal_write *w = c->journal.cur; in journal_wait_for_write()
871 sectors = __set_blocks(w->data, w->data->keys + nkeys, in journal_wait_for_write()
872 block_bytes(ca)) * ca->sb.block_size; in journal_wait_for_write()
875 c->journal.blocks_free * ca->sb.block_size, in journal_wait_for_write()
879 if (wait) in journal_wait_for_write()
880 closure_wait(&c->journal.wait, &cl); in journal_wait_for_write()
882 if (!journal_full(&c->journal)) { in journal_wait_for_write()
883 if (wait) in journal_wait_for_write()
890 * bch_keylist_realloc() - but something to think about. in journal_wait_for_write()
892 BUG_ON(!w->data->keys); in journal_wait_for_write()
896 if (wait) in journal_wait_for_write()
900 spin_unlock(&c->journal.lock); in journal_wait_for_write()
906 spin_lock(&c->journal.lock); in journal_wait_for_write()
907 wait = true; in journal_wait_for_write()
916 spin_lock(&c->journal.lock); in journal_write_work()
917 if (c->journal.cur->dirty) in journal_write_work()
920 spin_unlock(&c->journal.lock); in journal_write_work()
924 * Entry point to the journalling code - bio_insert() and btree_invalidate()
937 if (unlikely(test_bit(CACHE_SET_IO_DISABLE, &c->flags))) in bch_journal()
940 if (!CACHE_SYNC(&c->cache->sb)) in bch_journal()
945 memcpy(bset_bkey_last(w->data), keys->keys, bch_keylist_bytes(keys)); in bch_journal()
946 w->data->keys += bch_keylist_nkeys(keys); in bch_journal()
948 ret = &fifo_back(&c->journal.pin); in bch_journal()
952 closure_wait(&w->wait, parent); in bch_journal()
954 } else if (!w->dirty) { in bch_journal()
955 w->dirty = true; in bch_journal()
956 queue_delayed_work(bch_flush_wq, &c->journal.work, in bch_journal()
957 msecs_to_jiffies(c->journal_delay_ms)); in bch_journal()
958 spin_unlock(&c->journal.lock); in bch_journal()
960 spin_unlock(&c->journal.lock); in bch_journal()
981 free_pages((unsigned long) c->journal.w[1].data, JSET_BITS); in bch_journal_free()
982 free_pages((unsigned long) c->journal.w[0].data, JSET_BITS); in bch_journal_free()
983 free_fifo(&c->journal.pin); in bch_journal_free()
988 struct journal *j = &c->journal; in bch_journal_alloc()
990 spin_lock_init(&j->lock); in bch_journal_alloc()
991 spin_lock_init(&j->flush_write_lock); in bch_journal_alloc()
992 INIT_DELAYED_WORK(&j->work, journal_write_work); in bch_journal_alloc()
994 c->journal_delay_ms = 100; in bch_journal_alloc()
996 j->w[0].c = c; in bch_journal_alloc()
997 j->w[1].c = c; in bch_journal_alloc()
999 if (!(init_fifo(&j->pin, JOURNAL_PIN, GFP_KERNEL)) || in bch_journal_alloc()
1000 !(j->w[0].data = (void *) __get_free_pages(GFP_KERNEL|__GFP_COMP, JSET_BITS)) || in bch_journal_alloc()
1001 !(j->w[1].data = (void *) __get_free_pages(GFP_KERNEL|__GFP_COMP, JSET_BITS))) in bch_journal_alloc()
1002 return -ENOMEM; in bch_journal_alloc()