Lines Matching refs:index
54 static void zram_free_page(struct zram *zram, size_t index);
56 u32 index, int offset, struct bio *bio);
59 static int zram_slot_trylock(struct zram *zram, u32 index) in zram_slot_trylock() argument
61 return bit_spin_trylock(ZRAM_LOCK, &zram->table[index].flags); in zram_slot_trylock()
64 static void zram_slot_lock(struct zram *zram, u32 index) in zram_slot_lock() argument
66 bit_spin_lock(ZRAM_LOCK, &zram->table[index].flags); in zram_slot_lock()
69 static void zram_slot_unlock(struct zram *zram, u32 index) in zram_slot_unlock() argument
71 bit_spin_unlock(ZRAM_LOCK, &zram->table[index].flags); in zram_slot_unlock()
84 static unsigned long zram_get_handle(struct zram *zram, u32 index) in zram_get_handle() argument
86 return zram->table[index].handle; in zram_get_handle()
89 static void zram_set_handle(struct zram *zram, u32 index, unsigned long handle) in zram_set_handle() argument
91 zram->table[index].handle = handle; in zram_set_handle()
95 static bool zram_test_flag(struct zram *zram, u32 index, in zram_test_flag() argument
98 return zram->table[index].flags & BIT(flag); in zram_test_flag()
101 static void zram_set_flag(struct zram *zram, u32 index, in zram_set_flag() argument
104 zram->table[index].flags |= BIT(flag); in zram_set_flag()
107 static void zram_clear_flag(struct zram *zram, u32 index, in zram_clear_flag() argument
110 zram->table[index].flags &= ~BIT(flag); in zram_clear_flag()
113 static inline void zram_set_element(struct zram *zram, u32 index, in zram_set_element() argument
116 zram->table[index].element = element; in zram_set_element()
119 static unsigned long zram_get_element(struct zram *zram, u32 index) in zram_get_element() argument
121 return zram->table[index].element; in zram_get_element()
124 static size_t zram_get_obj_size(struct zram *zram, u32 index) in zram_get_obj_size() argument
126 return zram->table[index].flags & (BIT(ZRAM_FLAG_SHIFT) - 1); in zram_get_obj_size()
130 u32 index, size_t size) in zram_set_obj_size() argument
132 unsigned long flags = zram->table[index].flags >> ZRAM_FLAG_SHIFT; in zram_set_obj_size()
134 zram->table[index].flags = (flags << ZRAM_FLAG_SHIFT) | size; in zram_set_obj_size()
137 static inline bool zram_allocated(struct zram *zram, u32 index) in zram_allocated() argument
139 return zram_get_obj_size(zram, index) || in zram_allocated()
140 zram_test_flag(zram, index, ZRAM_SAME) || in zram_allocated()
141 zram_test_flag(zram, index, ZRAM_WB); in zram_allocated()
180 static void update_position(u32 *index, int *offset, struct bio_vec *bvec) in update_position() argument
182 *index += (*offset + bvec->bv_len) / PAGE_SIZE; in update_position()
292 int index; in idle_store() local
313 for (index = 0; index < nr_pages; index++) { in idle_store()
318 zram_slot_lock(zram, index); in idle_store()
319 if (zram_allocated(zram, index) && in idle_store()
320 !zram_test_flag(zram, index, ZRAM_UNDER_WB)) in idle_store()
321 zram_set_flag(zram, index, ZRAM_IDLE); in idle_store()
322 zram_slot_unlock(zram, index); in idle_store()
622 unsigned long index; in writeback_store() local
664 for (index = 0; index < nr_pages; index++) { in writeback_store()
687 zram_slot_lock(zram, index); in writeback_store()
688 if (!zram_allocated(zram, index)) in writeback_store()
691 if (zram_test_flag(zram, index, ZRAM_WB) || in writeback_store()
692 zram_test_flag(zram, index, ZRAM_SAME) || in writeback_store()
693 zram_test_flag(zram, index, ZRAM_UNDER_WB)) in writeback_store()
697 !zram_test_flag(zram, index, ZRAM_IDLE)) in writeback_store()
700 !zram_test_flag(zram, index, ZRAM_HUGE)) in writeback_store()
706 zram_set_flag(zram, index, ZRAM_UNDER_WB); in writeback_store()
708 zram_set_flag(zram, index, ZRAM_IDLE); in writeback_store()
709 zram_slot_unlock(zram, index); in writeback_store()
710 if (zram_bvec_read(zram, &bvec, index, 0, NULL)) { in writeback_store()
711 zram_slot_lock(zram, index); in writeback_store()
712 zram_clear_flag(zram, index, ZRAM_UNDER_WB); in writeback_store()
713 zram_clear_flag(zram, index, ZRAM_IDLE); in writeback_store()
714 zram_slot_unlock(zram, index); in writeback_store()
731 zram_slot_lock(zram, index); in writeback_store()
732 zram_clear_flag(zram, index, ZRAM_UNDER_WB); in writeback_store()
733 zram_clear_flag(zram, index, ZRAM_IDLE); in writeback_store()
734 zram_slot_unlock(zram, index); in writeback_store()
748 zram_slot_lock(zram, index); in writeback_store()
749 if (!zram_allocated(zram, index) || in writeback_store()
750 !zram_test_flag(zram, index, ZRAM_IDLE)) { in writeback_store()
751 zram_clear_flag(zram, index, ZRAM_UNDER_WB); in writeback_store()
752 zram_clear_flag(zram, index, ZRAM_IDLE); in writeback_store()
756 zram_free_page(zram, index); in writeback_store()
757 zram_clear_flag(zram, index, ZRAM_UNDER_WB); in writeback_store()
758 zram_set_flag(zram, index, ZRAM_WB); in writeback_store()
759 zram_set_element(zram, index, blk_idx); in writeback_store()
767 zram_slot_unlock(zram, index); in writeback_store()
864 static void zram_accessed(struct zram *zram, u32 index) in zram_accessed() argument
866 zram_clear_flag(zram, index, ZRAM_IDLE); in zram_accessed()
867 zram->table[index].ac_time = ktime_get_boottime(); in zram_accessed()
874 ssize_t index, written = 0; in read_block_state() local
890 for (index = *ppos; index < nr_pages; index++) { in read_block_state()
893 zram_slot_lock(zram, index); in read_block_state()
894 if (!zram_allocated(zram, index)) in read_block_state()
897 ts = ktime_to_timespec64(zram->table[index].ac_time); in read_block_state()
900 index, (s64)ts.tv_sec, in read_block_state()
902 zram_test_flag(zram, index, ZRAM_SAME) ? 's' : '.', in read_block_state()
903 zram_test_flag(zram, index, ZRAM_WB) ? 'w' : '.', in read_block_state()
904 zram_test_flag(zram, index, ZRAM_HUGE) ? 'h' : '.', in read_block_state()
905 zram_test_flag(zram, index, ZRAM_IDLE) ? 'i' : '.'); in read_block_state()
908 zram_slot_unlock(zram, index); in read_block_state()
914 zram_slot_unlock(zram, index); in read_block_state()
950 static void zram_accessed(struct zram *zram, u32 index) in zram_accessed() argument
952 zram_clear_flag(zram, index, ZRAM_IDLE); in zram_accessed()
1138 size_t index; in zram_meta_free() local
1141 for (index = 0; index < num_pages; index++) in zram_meta_free()
1142 zram_free_page(zram, index); in zram_meta_free()
1173 static void zram_free_page(struct zram *zram, size_t index) in zram_free_page() argument
1178 zram->table[index].ac_time = 0; in zram_free_page()
1180 if (zram_test_flag(zram, index, ZRAM_IDLE)) in zram_free_page()
1181 zram_clear_flag(zram, index, ZRAM_IDLE); in zram_free_page()
1183 if (zram_test_flag(zram, index, ZRAM_HUGE)) { in zram_free_page()
1184 zram_clear_flag(zram, index, ZRAM_HUGE); in zram_free_page()
1188 if (zram_test_flag(zram, index, ZRAM_WB)) { in zram_free_page()
1189 zram_clear_flag(zram, index, ZRAM_WB); in zram_free_page()
1190 free_block_bdev(zram, zram_get_element(zram, index)); in zram_free_page()
1198 if (zram_test_flag(zram, index, ZRAM_SAME)) { in zram_free_page()
1199 zram_clear_flag(zram, index, ZRAM_SAME); in zram_free_page()
1204 handle = zram_get_handle(zram, index); in zram_free_page()
1210 atomic64_sub(zram_get_obj_size(zram, index), in zram_free_page()
1214 zram_set_handle(zram, index, 0); in zram_free_page()
1215 zram_set_obj_size(zram, index, 0); in zram_free_page()
1216 WARN_ON_ONCE(zram->table[index].flags & in zram_free_page()
1220 static int __zram_bvec_read(struct zram *zram, struct page *page, u32 index, in __zram_bvec_read() argument
1228 zram_slot_lock(zram, index); in __zram_bvec_read()
1229 if (zram_test_flag(zram, index, ZRAM_WB)) { in __zram_bvec_read()
1232 zram_slot_unlock(zram, index); in __zram_bvec_read()
1238 zram_get_element(zram, index), in __zram_bvec_read()
1242 handle = zram_get_handle(zram, index); in __zram_bvec_read()
1243 if (!handle || zram_test_flag(zram, index, ZRAM_SAME)) { in __zram_bvec_read()
1247 value = handle ? zram_get_element(zram, index) : 0; in __zram_bvec_read()
1251 zram_slot_unlock(zram, index); in __zram_bvec_read()
1255 size = zram_get_obj_size(zram, index); in __zram_bvec_read()
1272 zram_slot_unlock(zram, index); in __zram_bvec_read()
1276 pr_err("Decompression failed! err=%d, page=%u\n", ret, index); in __zram_bvec_read()
1282 u32 index, int offset, struct bio *bio) in zram_bvec_read() argument
1295 ret = __zram_bvec_read(zram, page, index, bio, is_partial_io(bvec)); in zram_bvec_read()
1315 u32 index, struct bio *bio) in __zram_bvec_write() argument
1408 zram_slot_lock(zram, index); in __zram_bvec_write()
1409 zram_free_page(zram, index); in __zram_bvec_write()
1412 zram_set_flag(zram, index, ZRAM_HUGE); in __zram_bvec_write()
1417 zram_set_flag(zram, index, flags); in __zram_bvec_write()
1418 zram_set_element(zram, index, element); in __zram_bvec_write()
1420 zram_set_handle(zram, index, handle); in __zram_bvec_write()
1421 zram_set_obj_size(zram, index, comp_len); in __zram_bvec_write()
1423 zram_slot_unlock(zram, index); in __zram_bvec_write()
1431 u32 index, int offset, struct bio *bio) in zram_bvec_write() argument
1449 ret = __zram_bvec_read(zram, page, index, bio, true); in zram_bvec_write()
1464 ret = __zram_bvec_write(zram, &vec, index, bio); in zram_bvec_write()
1476 static void zram_bio_discard(struct zram *zram, u32 index, in zram_bio_discard() argument
1496 index++; in zram_bio_discard()
1500 zram_slot_lock(zram, index); in zram_bio_discard()
1501 zram_free_page(zram, index); in zram_bio_discard()
1502 zram_slot_unlock(zram, index); in zram_bio_discard()
1504 index++; in zram_bio_discard()
1514 static int zram_bvec_rw(struct zram *zram, struct bio_vec *bvec, u32 index, in zram_bvec_rw() argument
1527 ret = zram_bvec_read(zram, bvec, index, offset, bio); in zram_bvec_rw()
1531 ret = zram_bvec_write(zram, bvec, index, offset, bio); in zram_bvec_rw()
1536 zram_slot_lock(zram, index); in zram_bvec_rw()
1537 zram_accessed(zram, index); in zram_bvec_rw()
1538 zram_slot_unlock(zram, index); in zram_bvec_rw()
1553 u32 index; in __zram_make_request() local
1557 index = bio->bi_iter.bi_sector >> SECTORS_PER_PAGE_SHIFT; in __zram_make_request()
1564 zram_bio_discard(zram, index, offset, bio); in __zram_make_request()
1578 if (zram_bvec_rw(zram, &bv, index, offset, in __zram_make_request()
1585 update_position(&index, &offset, &bv); in __zram_make_request()
1618 unsigned long index) in zram_slot_free_notify() argument
1625 if (!zram_slot_trylock(zram, index)) { in zram_slot_free_notify()
1630 zram_free_page(zram, index); in zram_slot_free_notify()
1631 zram_slot_unlock(zram, index); in zram_slot_free_notify()
1638 u32 index; in zram_rw_page() local
1652 index = sector >> SECTORS_PER_PAGE_SHIFT; in zram_rw_page()
1659 ret = zram_bvec_rw(zram, &bv, index, offset, is_write, NULL); in zram_rw_page()