Lines Matching refs:pool
302 static int zs_register_migration(struct zs_pool *pool);
303 static void zs_unregister_migration(struct zs_pool *pool);
307 static void kick_deferred_free(struct zs_pool *pool);
308 static void init_deferred_free(struct zs_pool *pool);
309 static void SetZsPageMovable(struct zs_pool *pool, struct zspage *zspage);
313 static int zs_register_migration(struct zs_pool *pool) { return 0; } in zs_register_migration() argument
314 static void zs_unregister_migration(struct zs_pool *pool) {} in zs_unregister_migration() argument
318 static void kick_deferred_free(struct zs_pool *pool) {} in kick_deferred_free() argument
319 static void init_deferred_free(struct zs_pool *pool) {} in init_deferred_free() argument
320 static void SetZsPageMovable(struct zs_pool *pool, struct zspage *zspage) {} in SetZsPageMovable() argument
323 static int create_cache(struct zs_pool *pool) in create_cache() argument
325 pool->handle_cachep = kmem_cache_create("zs_handle", ZS_HANDLE_SIZE, in create_cache()
327 if (!pool->handle_cachep) in create_cache()
330 pool->zspage_cachep = kmem_cache_create("zspage", sizeof(struct zspage), in create_cache()
332 if (!pool->zspage_cachep) { in create_cache()
333 kmem_cache_destroy(pool->handle_cachep); in create_cache()
334 pool->handle_cachep = NULL; in create_cache()
341 static void destroy_cache(struct zs_pool *pool) in destroy_cache() argument
343 kmem_cache_destroy(pool->handle_cachep); in destroy_cache()
344 kmem_cache_destroy(pool->zspage_cachep); in destroy_cache()
347 static unsigned long cache_alloc_handle(struct zs_pool *pool, gfp_t gfp) in cache_alloc_handle() argument
349 return (unsigned long)kmem_cache_alloc(pool->handle_cachep, in cache_alloc_handle()
353 static void cache_free_handle(struct zs_pool *pool, unsigned long handle) in cache_free_handle() argument
355 kmem_cache_free(pool->handle_cachep, (void *)handle); in cache_free_handle()
358 static struct zspage *cache_alloc_zspage(struct zs_pool *pool, gfp_t flags) in cache_alloc_zspage() argument
360 return kmem_cache_zalloc(pool->zspage_cachep, in cache_alloc_zspage()
364 static void cache_free_zspage(struct zs_pool *pool, struct zspage *zspage) in cache_free_zspage() argument
366 kmem_cache_free(pool->zspage_cachep, zspage); in cache_free_zspage()
395 static void zs_zpool_destroy(void *pool) in zs_zpool_destroy() argument
397 zs_destroy_pool(pool); in zs_zpool_destroy()
400 static int zs_zpool_malloc(void *pool, size_t size, gfp_t gfp, in zs_zpool_malloc() argument
403 *handle = zs_malloc(pool, size, gfp); in zs_zpool_malloc()
406 static void zs_zpool_free(void *pool, unsigned long handle) in zs_zpool_free() argument
408 zs_free(pool, handle); in zs_zpool_free()
411 static void *zs_zpool_map(void *pool, unsigned long handle, in zs_zpool_map() argument
429 return zs_map_object(pool, handle, zs_mm); in zs_zpool_map()
431 static void zs_zpool_unmap(void *pool, unsigned long handle) in zs_zpool_unmap() argument
433 zs_unmap_object(pool, handle); in zs_zpool_unmap()
436 static u64 zs_zpool_total_size(void *pool) in zs_zpool_total_size() argument
438 return zs_get_total_pages(pool) << PAGE_SHIFT; in zs_zpool_total_size()
589 struct zs_pool *pool = s->private; in zs_stats_size_show() local
604 class = pool->size_class[i]; in zs_stats_size_show()
645 static void zs_pool_stat_create(struct zs_pool *pool, const char *name) in zs_pool_stat_create() argument
652 pool->stat_dentry = debugfs_create_dir(name, zs_stat_root); in zs_pool_stat_create()
654 debugfs_create_file("classes", S_IFREG | 0444, pool->stat_dentry, pool, in zs_pool_stat_create()
658 static void zs_pool_stat_destroy(struct zs_pool *pool) in zs_pool_stat_destroy() argument
660 debugfs_remove_recursive(pool->stat_dentry); in zs_pool_stat_destroy()
672 static inline void zs_pool_stat_create(struct zs_pool *pool, const char *name) in zs_pool_stat_create() argument
676 static inline void zs_pool_stat_destroy(struct zs_pool *pool) in zs_pool_stat_destroy() argument
928 static void __free_zspage(struct zs_pool *pool, struct size_class *class, in __free_zspage() argument
953 cache_free_zspage(pool, zspage); in __free_zspage()
957 &pool->pages_allocated); in __free_zspage()
960 static void free_zspage(struct zs_pool *pool, struct size_class *class, in free_zspage() argument
967 kick_deferred_free(pool); in free_zspage()
972 __free_zspage(pool, class, zspage); in free_zspage()
1056 static struct zspage *alloc_zspage(struct zs_pool *pool, in alloc_zspage() argument
1062 struct zspage *zspage = cache_alloc_zspage(pool, gfp); in alloc_zspage()
1079 cache_free_zspage(pool, zspage); in alloc_zspage()
1220 unsigned long zs_get_total_pages(struct zs_pool *pool) in zs_get_total_pages() argument
1222 return atomic_long_read(&pool->pages_allocated); in zs_get_total_pages()
1241 void *zs_map_object(struct zs_pool *pool, unsigned long handle, in zs_map_object() argument
1274 class = pool->size_class[class_idx]; in zs_map_object()
1300 void zs_unmap_object(struct zs_pool *pool, unsigned long handle) in zs_unmap_object() argument
1316 class = pool->size_class[class_idx]; in zs_unmap_object()
1351 size_t zs_huge_class_size(struct zs_pool *pool) in zs_huge_class_size() argument
1409 unsigned long zs_malloc(struct zs_pool *pool, size_t size, gfp_t gfp) in zs_malloc() argument
1419 handle = cache_alloc_handle(pool, gfp); in zs_malloc()
1425 class = pool->size_class[get_size_class_index(size)]; in zs_malloc()
1441 zspage = alloc_zspage(pool, class, gfp); in zs_malloc()
1443 cache_free_handle(pool, handle); in zs_malloc()
1454 &pool->pages_allocated); in zs_malloc()
1458 SetZsPageMovable(pool, zspage); in zs_malloc()
1489 void zs_free(struct zs_pool *pool, unsigned long handle) in zs_free() argument
1511 class = pool->size_class[class_idx]; in zs_free()
1525 free_zspage(pool, class, zspage); in zs_free()
1530 cache_free_handle(pool, handle); in zs_free()
1644 static int migrate_zspage(struct zs_pool *pool, struct size_class *class, in migrate_zspage() argument
1845 static void putback_zspage_deferred(struct zs_pool *pool, in putback_zspage_deferred() argument
1853 schedule_work(&pool->free_work); in putback_zspage_deferred()
1857 static inline void zs_pool_dec_isolated(struct zs_pool *pool) in zs_pool_dec_isolated() argument
1859 VM_BUG_ON(atomic_long_read(&pool->isolated_pages) <= 0); in zs_pool_dec_isolated()
1860 atomic_long_dec(&pool->isolated_pages); in zs_pool_dec_isolated()
1867 if (atomic_long_read(&pool->isolated_pages) == 0 && pool->destroying) in zs_pool_dec_isolated()
1868 wake_up_all(&pool->migration_wait); in zs_pool_dec_isolated()
1896 struct zs_pool *pool; in zs_page_isolate() local
1919 pool = mapping->private_data; in zs_page_isolate()
1920 class = pool->size_class[class_idx]; in zs_page_isolate()
1940 atomic_long_inc(&pool->isolated_pages); in zs_page_isolate()
1953 struct zs_pool *pool; in zs_page_migrate() local
1982 pool = mapping->private_data; in zs_page_migrate()
1983 class = pool->size_class[class_idx]; in zs_page_migrate()
2046 putback_zspage_deferred(pool, class, zspage); in zs_page_migrate()
2047 zs_pool_dec_isolated(pool); in zs_page_migrate()
2079 struct zs_pool *pool; in zs_page_putback() local
2092 pool = mapping->private_data; in zs_page_putback()
2093 class = pool->size_class[class_idx]; in zs_page_putback()
2102 putback_zspage_deferred(pool, class, zspage); in zs_page_putback()
2103 zs_pool_dec_isolated(pool); in zs_page_putback()
2114 static int zs_register_migration(struct zs_pool *pool) in zs_register_migration() argument
2116 pool->inode = alloc_anon_inode(zsmalloc_mnt->mnt_sb); in zs_register_migration()
2117 if (IS_ERR(pool->inode)) { in zs_register_migration()
2118 pool->inode = NULL; in zs_register_migration()
2122 pool->inode->i_mapping->private_data = pool; in zs_register_migration()
2123 pool->inode->i_mapping->a_ops = &zsmalloc_aops; in zs_register_migration()
2127 static bool pool_isolated_are_drained(struct zs_pool *pool) in pool_isolated_are_drained() argument
2129 return atomic_long_read(&pool->isolated_pages) == 0; in pool_isolated_are_drained()
2133 static void wait_for_isolated_drain(struct zs_pool *pool) in wait_for_isolated_drain() argument
2142 wait_event(pool->migration_wait, in wait_for_isolated_drain()
2143 pool_isolated_are_drained(pool)); in wait_for_isolated_drain()
2146 static void zs_unregister_migration(struct zs_pool *pool) in zs_unregister_migration() argument
2148 pool->destroying = true; in zs_unregister_migration()
2156 wait_for_isolated_drain(pool); /* This can block */ in zs_unregister_migration()
2157 flush_work(&pool->free_work); in zs_unregister_migration()
2158 iput(pool->inode); in zs_unregister_migration()
2173 struct zs_pool *pool = container_of(work, struct zs_pool, in async_free_zspage() local
2177 class = pool->size_class[i]; in async_free_zspage()
2193 class = pool->size_class[class_idx]; in async_free_zspage()
2195 __free_zspage(pool, class, zspage); in async_free_zspage()
2200 static void kick_deferred_free(struct zs_pool *pool) in kick_deferred_free() argument
2202 schedule_work(&pool->free_work); in kick_deferred_free()
2205 static void init_deferred_free(struct zs_pool *pool) in init_deferred_free() argument
2207 INIT_WORK(&pool->free_work, async_free_zspage); in init_deferred_free()
2210 static void SetZsPageMovable(struct zs_pool *pool, struct zspage *zspage) in SetZsPageMovable() argument
2216 __SetPageMovable(page, pool->inode->i_mapping); in SetZsPageMovable()
2242 static unsigned long __zs_compact(struct zs_pool *pool, in __zs_compact() argument
2265 if (!migrate_zspage(pool, class, &cc)) in __zs_compact()
2277 free_zspage(pool, class, src_zspage); in __zs_compact()
2293 unsigned long zs_compact(struct zs_pool *pool) in zs_compact() argument
2300 class = pool->size_class[i]; in zs_compact()
2305 pages_freed += __zs_compact(pool, class); in zs_compact()
2307 atomic_long_add(pages_freed, &pool->stats.pages_compacted); in zs_compact()
2313 void zs_pool_stats(struct zs_pool *pool, struct zs_pool_stats *stats) in zs_pool_stats() argument
2315 memcpy(stats, &pool->stats, sizeof(struct zs_pool_stats)); in zs_pool_stats()
2323 struct zs_pool *pool = container_of(shrinker, struct zs_pool, in zs_shrinker_scan() local
2331 pages_freed = zs_compact(pool); in zs_shrinker_scan()
2342 struct zs_pool *pool = container_of(shrinker, struct zs_pool, in zs_shrinker_count() local
2346 class = pool->size_class[i]; in zs_shrinker_count()
2358 static void zs_unregister_shrinker(struct zs_pool *pool) in zs_unregister_shrinker() argument
2360 unregister_shrinker(&pool->shrinker); in zs_unregister_shrinker()
2363 static int zs_register_shrinker(struct zs_pool *pool) in zs_register_shrinker() argument
2365 pool->shrinker.scan_objects = zs_shrinker_scan; in zs_register_shrinker()
2366 pool->shrinker.count_objects = zs_shrinker_count; in zs_register_shrinker()
2367 pool->shrinker.batch = 0; in zs_register_shrinker()
2368 pool->shrinker.seeks = DEFAULT_SEEKS; in zs_register_shrinker()
2370 return register_shrinker(&pool->shrinker); in zs_register_shrinker()
2386 struct zs_pool *pool; in zs_create_pool() local
2389 pool = kzalloc(sizeof(*pool), GFP_KERNEL); in zs_create_pool()
2390 if (!pool) in zs_create_pool()
2393 init_deferred_free(pool); in zs_create_pool()
2395 pool->name = kstrdup(name, GFP_KERNEL); in zs_create_pool()
2396 if (!pool->name) in zs_create_pool()
2400 init_waitqueue_head(&pool->migration_wait); in zs_create_pool()
2403 if (create_cache(pool)) in zs_create_pool()
2455 pool->size_class[i] = prev_class; in zs_create_pool()
2469 pool->size_class[i] = class; in zs_create_pool()
2478 zs_pool_stat_create(pool, name); in zs_create_pool()
2480 if (zs_register_migration(pool)) in zs_create_pool()
2489 zs_register_shrinker(pool); in zs_create_pool()
2491 return pool; in zs_create_pool()
2494 zs_destroy_pool(pool); in zs_create_pool()
2499 void zs_destroy_pool(struct zs_pool *pool) in zs_destroy_pool() argument
2503 zs_unregister_shrinker(pool); in zs_destroy_pool()
2504 zs_unregister_migration(pool); in zs_destroy_pool()
2505 zs_pool_stat_destroy(pool); in zs_destroy_pool()
2509 struct size_class *class = pool->size_class[i]; in zs_destroy_pool()
2526 destroy_cache(pool); in zs_destroy_pool()
2527 kfree(pool->name); in zs_destroy_pool()
2528 kfree(pool); in zs_destroy_pool()