Lines Matching refs:pool
75 int (*evict)(struct z3fold_pool *pool, unsigned long handle);
92 unsigned long pool; /* back link + flags */ member
119 struct z3fold_pool *pool; member
196 static inline struct z3fold_buddy_slots *alloc_slots(struct z3fold_pool *pool, in alloc_slots() argument
201 slots = kmem_cache_alloc(pool->c_handle, in alloc_slots()
206 slots->pool = (unsigned long)pool; in alloc_slots()
214 return (struct z3fold_pool *)(s->pool & ~HANDLE_FLAG_MASK); in slots_to_pool()
243 struct z3fold_pool *pool = slots_to_pool(slots); in free_handle() local
245 kmem_cache_free(pool->c_handle, slots); in free_handle()
278 static int z3fold_register_migration(struct z3fold_pool *pool) in z3fold_register_migration() argument
280 pool->inode = alloc_anon_inode(z3fold_mnt->mnt_sb); in z3fold_register_migration()
281 if (IS_ERR(pool->inode)) { in z3fold_register_migration()
282 pool->inode = NULL; in z3fold_register_migration()
286 pool->inode->i_mapping->private_data = pool; in z3fold_register_migration()
287 pool->inode->i_mapping->a_ops = &z3fold_aops; in z3fold_register_migration()
291 static void z3fold_unregister_migration(struct z3fold_pool *pool) in z3fold_unregister_migration() argument
293 if (pool->inode) in z3fold_unregister_migration()
294 iput(pool->inode); in z3fold_unregister_migration()
299 struct z3fold_pool *pool, gfp_t gfp) in init_z3fold_page() argument
313 slots = alloc_slots(pool, gfp); in init_z3fold_page()
326 zhdr->pool = pool; in init_z3fold_page()
438 return zhdr->pool; in zhdr_to_pool()
444 struct z3fold_pool *pool = zhdr_to_pool(zhdr); in __release_z3fold_page() local
449 spin_lock(&pool->lock); in __release_z3fold_page()
452 spin_unlock(&pool->lock); in __release_z3fold_page()
455 spin_lock(&pool->stale_lock); in __release_z3fold_page()
456 list_add(&zhdr->buddy, &pool->stale); in __release_z3fold_page()
457 queue_work(pool->release_wq, &pool->work); in __release_z3fold_page()
458 spin_unlock(&pool->stale_lock); in __release_z3fold_page()
481 struct z3fold_pool *pool = zhdr_to_pool(zhdr); in release_z3fold_page_locked_list() local
482 spin_lock(&pool->lock); in release_z3fold_page_locked_list()
484 spin_unlock(&pool->lock); in release_z3fold_page_locked_list()
492 struct z3fold_pool *pool = container_of(w, struct z3fold_pool, work); in free_pages_work() local
494 spin_lock(&pool->stale_lock); in free_pages_work()
495 while (!list_empty(&pool->stale)) { in free_pages_work()
496 struct z3fold_header *zhdr = list_first_entry(&pool->stale, in free_pages_work()
503 spin_unlock(&pool->stale_lock); in free_pages_work()
507 spin_lock(&pool->stale_lock); in free_pages_work()
509 spin_unlock(&pool->stale_lock); in free_pages_work()
537 static inline void add_to_unbuddied(struct z3fold_pool *pool, in add_to_unbuddied() argument
542 struct list_head *unbuddied = get_cpu_ptr(pool->unbuddied); in add_to_unbuddied()
545 spin_lock(&pool->lock); in add_to_unbuddied()
547 spin_unlock(&pool->lock); in add_to_unbuddied()
549 put_cpu_ptr(pool->unbuddied); in add_to_unbuddied()
613 struct z3fold_pool *pool = zhdr_to_pool(zhdr); in do_compact_page() local
625 spin_lock(&pool->lock); in do_compact_page()
627 spin_unlock(&pool->lock); in do_compact_page()
630 atomic64_dec(&pool->pages_nr); in do_compact_page()
642 add_to_unbuddied(pool, zhdr); in do_compact_page()
655 static inline struct z3fold_header *__z3fold_alloc(struct z3fold_pool *pool, in __z3fold_alloc() argument
665 unbuddied = get_cpu_ptr(pool->unbuddied); in __z3fold_alloc()
676 spin_lock(&pool->lock); in __z3fold_alloc()
681 spin_unlock(&pool->lock); in __z3fold_alloc()
683 put_cpu_ptr(pool->unbuddied); in __z3fold_alloc()
690 spin_unlock(&pool->lock); in __z3fold_alloc()
696 put_cpu_ptr(pool->unbuddied); in __z3fold_alloc()
711 put_cpu_ptr(pool->unbuddied); in __z3fold_alloc()
720 unbuddied = per_cpu_ptr(pool->unbuddied, cpu); in __z3fold_alloc()
721 spin_lock(&pool->lock); in __z3fold_alloc()
728 spin_unlock(&pool->lock); in __z3fold_alloc()
734 spin_unlock(&pool->lock); in __z3fold_alloc()
768 struct z3fold_pool *pool = NULL; in z3fold_create_pool() local
771 pool = kzalloc(sizeof(struct z3fold_pool), gfp); in z3fold_create_pool()
772 if (!pool) in z3fold_create_pool()
774 pool->c_handle = kmem_cache_create("z3fold_handle", in z3fold_create_pool()
777 if (!pool->c_handle) in z3fold_create_pool()
779 spin_lock_init(&pool->lock); in z3fold_create_pool()
780 spin_lock_init(&pool->stale_lock); in z3fold_create_pool()
781 pool->unbuddied = __alloc_percpu(sizeof(struct list_head)*NCHUNKS, 2); in z3fold_create_pool()
782 if (!pool->unbuddied) in z3fold_create_pool()
786 per_cpu_ptr(pool->unbuddied, cpu); in z3fold_create_pool()
790 INIT_LIST_HEAD(&pool->lru); in z3fold_create_pool()
791 INIT_LIST_HEAD(&pool->stale); in z3fold_create_pool()
792 atomic64_set(&pool->pages_nr, 0); in z3fold_create_pool()
793 pool->name = name; in z3fold_create_pool()
794 pool->compact_wq = create_singlethread_workqueue(pool->name); in z3fold_create_pool()
795 if (!pool->compact_wq) in z3fold_create_pool()
797 pool->release_wq = create_singlethread_workqueue(pool->name); in z3fold_create_pool()
798 if (!pool->release_wq) in z3fold_create_pool()
800 if (z3fold_register_migration(pool)) in z3fold_create_pool()
802 INIT_WORK(&pool->work, free_pages_work); in z3fold_create_pool()
803 pool->ops = ops; in z3fold_create_pool()
804 return pool; in z3fold_create_pool()
807 destroy_workqueue(pool->release_wq); in z3fold_create_pool()
809 destroy_workqueue(pool->compact_wq); in z3fold_create_pool()
811 free_percpu(pool->unbuddied); in z3fold_create_pool()
813 kmem_cache_destroy(pool->c_handle); in z3fold_create_pool()
815 kfree(pool); in z3fold_create_pool()
826 static void z3fold_destroy_pool(struct z3fold_pool *pool) in z3fold_destroy_pool() argument
828 kmem_cache_destroy(pool->c_handle); in z3fold_destroy_pool()
839 destroy_workqueue(pool->compact_wq); in z3fold_destroy_pool()
840 destroy_workqueue(pool->release_wq); in z3fold_destroy_pool()
841 z3fold_unregister_migration(pool); in z3fold_destroy_pool()
842 kfree(pool); in z3fold_destroy_pool()
864 static int z3fold_alloc(struct z3fold_pool *pool, size_t size, gfp_t gfp, in z3fold_alloc() argument
883 zhdr = __z3fold_alloc(pool, size, can_sleep); in z3fold_alloc()
898 atomic64_dec(&pool->pages_nr); in z3fold_alloc()
913 spin_lock(&pool->stale_lock); in z3fold_alloc()
914 zhdr = list_first_entry_or_null(&pool->stale, in z3fold_alloc()
923 spin_unlock(&pool->stale_lock); in z3fold_alloc()
927 spin_unlock(&pool->stale_lock); in z3fold_alloc()
936 zhdr = init_z3fold_page(page, bud == HEADLESS, pool, gfp); in z3fold_alloc()
941 atomic64_inc(&pool->pages_nr); in z3fold_alloc()
949 __SetPageMovable(page, pool->inode->i_mapping); in z3fold_alloc()
953 __SetPageMovable(page, pool->inode->i_mapping); in z3fold_alloc()
968 add_to_unbuddied(pool, zhdr); in z3fold_alloc()
971 spin_lock(&pool->lock); in z3fold_alloc()
976 list_add(&page->lru, &pool->lru); in z3fold_alloc()
979 spin_unlock(&pool->lock); in z3fold_alloc()
996 static void z3fold_free(struct z3fold_pool *pool, unsigned long handle) in z3fold_free() argument
1014 spin_lock(&pool->lock); in z3fold_free()
1016 spin_unlock(&pool->lock); in z3fold_free()
1018 atomic64_dec(&pool->pages_nr); in z3fold_free()
1046 atomic64_dec(&pool->pages_nr); in z3fold_free()
1061 spin_lock(&pool->lock); in z3fold_free()
1063 spin_unlock(&pool->lock); in z3fold_free()
1071 queue_work_on(zhdr->cpu, pool->compact_wq, &zhdr->work); in z3fold_free()
1112 static int z3fold_reclaim_page(struct z3fold_pool *pool, unsigned int retries) in z3fold_reclaim_page() argument
1121 spin_lock(&pool->lock); in z3fold_reclaim_page()
1122 if (!pool->ops || !pool->ops->evict || retries == 0) { in z3fold_reclaim_page()
1123 spin_unlock(&pool->lock); in z3fold_reclaim_page()
1127 if (list_empty(&pool->lru)) { in z3fold_reclaim_page()
1128 spin_unlock(&pool->lock); in z3fold_reclaim_page()
1131 list_for_each_prev(pos, &pool->lru) { in z3fold_reclaim_page()
1166 spin_unlock(&pool->lock); in z3fold_reclaim_page()
1199 ret = pool->ops->evict(pool, middle_handle); in z3fold_reclaim_page()
1204 ret = pool->ops->evict(pool, first_handle); in z3fold_reclaim_page()
1209 ret = pool->ops->evict(pool, last_handle); in z3fold_reclaim_page()
1217 atomic64_dec(&pool->pages_nr); in z3fold_reclaim_page()
1220 spin_lock(&pool->lock); in z3fold_reclaim_page()
1221 list_add(&page->lru, &pool->lru); in z3fold_reclaim_page()
1222 spin_unlock(&pool->lock); in z3fold_reclaim_page()
1228 atomic64_dec(&pool->pages_nr); in z3fold_reclaim_page()
1236 spin_lock(&pool->lock); in z3fold_reclaim_page()
1237 list_add(&page->lru, &pool->lru); in z3fold_reclaim_page()
1238 spin_unlock(&pool->lock); in z3fold_reclaim_page()
1244 spin_lock(&pool->lock); in z3fold_reclaim_page()
1246 spin_unlock(&pool->lock); in z3fold_reclaim_page()
1260 static void *z3fold_map(struct z3fold_pool *pool, unsigned long handle) in z3fold_map() argument
1306 static void z3fold_unmap(struct z3fold_pool *pool, unsigned long handle) in z3fold_unmap() argument
1332 static u64 z3fold_get_pool_size(struct z3fold_pool *pool) in z3fold_get_pool_size() argument
1334 return atomic64_read(&pool->pages_nr); in z3fold_get_pool_size()
1340 struct z3fold_pool *pool; in z3fold_page_isolate() local
1355 pool = zhdr_to_pool(zhdr); in z3fold_page_isolate()
1361 spin_lock(&pool->lock); in z3fold_page_isolate()
1364 spin_unlock(&pool->lock); in z3fold_page_isolate()
1377 struct z3fold_pool *pool; in z3fold_page_migrate() local
1385 pool = zhdr_to_pool(zhdr); in z3fold_page_migrate()
1424 spin_lock(&pool->lock); in z3fold_page_migrate()
1425 list_add(&newpage->lru, &pool->lru); in z3fold_page_migrate()
1426 spin_unlock(&pool->lock); in z3fold_page_migrate()
1430 queue_work_on(new_zhdr->cpu, pool->compact_wq, &new_zhdr->work); in z3fold_page_migrate()
1440 struct z3fold_pool *pool; in z3fold_page_putback() local
1443 pool = zhdr_to_pool(zhdr); in z3fold_page_putback()
1450 atomic64_dec(&pool->pages_nr); in z3fold_page_putback()
1453 spin_lock(&pool->lock); in z3fold_page_putback()
1454 list_add(&page->lru, &pool->lru); in z3fold_page_putback()
1455 spin_unlock(&pool->lock); in z3fold_page_putback()
1469 static int z3fold_zpool_evict(struct z3fold_pool *pool, unsigned long handle) in z3fold_zpool_evict() argument
1471 if (pool->zpool && pool->zpool_ops && pool->zpool_ops->evict) in z3fold_zpool_evict()
1472 return pool->zpool_ops->evict(pool->zpool, handle); in z3fold_zpool_evict()
1485 struct z3fold_pool *pool; in z3fold_zpool_create() local
1487 pool = z3fold_create_pool(name, gfp, in z3fold_zpool_create()
1489 if (pool) { in z3fold_zpool_create()
1490 pool->zpool = zpool; in z3fold_zpool_create()
1491 pool->zpool_ops = zpool_ops; in z3fold_zpool_create()
1493 return pool; in z3fold_zpool_create()
1496 static void z3fold_zpool_destroy(void *pool) in z3fold_zpool_destroy() argument
1498 z3fold_destroy_pool(pool); in z3fold_zpool_destroy()
1501 static int z3fold_zpool_malloc(void *pool, size_t size, gfp_t gfp, in z3fold_zpool_malloc() argument
1504 return z3fold_alloc(pool, size, gfp, handle); in z3fold_zpool_malloc()
1506 static void z3fold_zpool_free(void *pool, unsigned long handle) in z3fold_zpool_free() argument
1508 z3fold_free(pool, handle); in z3fold_zpool_free()
1511 static int z3fold_zpool_shrink(void *pool, unsigned int pages, in z3fold_zpool_shrink() argument
1518 ret = z3fold_reclaim_page(pool, 8); in z3fold_zpool_shrink()
1530 static void *z3fold_zpool_map(void *pool, unsigned long handle, in z3fold_zpool_map() argument
1533 return z3fold_map(pool, handle); in z3fold_zpool_map()
1535 static void z3fold_zpool_unmap(void *pool, unsigned long handle) in z3fold_zpool_unmap() argument
1537 z3fold_unmap(pool, handle); in z3fold_zpool_unmap()
1540 static u64 z3fold_zpool_total_size(void *pool) in z3fold_zpool_total_size() argument
1542 return z3fold_get_pool_size(pool) * PAGE_SIZE; in z3fold_zpool_total_size()