126 int slab_unmergeable(struct kmem_cache *s);
213 void get_slabinfo(struct kmem_cache *s, struct slabinfo *sinfo);
214 void slabinfo_show_stats(struct seq_file *m, struct kmem_cache *s);
227 static inline int cache_vmstat_idx(struct kmem_cache *s)  in cache_vmstat_idx()  argument
229 	return (s->flags & SLAB_RECLAIM_ACCOUNT) ?  in cache_vmstat_idx()
239 extern void print_tracking(struct kmem_cache *s, void *object);
240 extern unsigned long get_each_object_track(struct kmem_cache *s,
250 static inline void print_tracking(struct kmem_cache *s, void *object)  in print_tracking()  argument
258 static inline unsigned long get_each_object_track(struct kmem_cache *s,  in get_each_object_track()  argument
273 static inline bool kmem_cache_debug_flags(struct kmem_cache *s, slab_flags_t flags)  in kmem_cache_debug_flags()  argument
278 		return s->flags & flags;  in kmem_cache_debug_flags()
300 int memcg_alloc_page_obj_cgroups(struct page *page, struct kmem_cache *s,
309 static inline size_t obj_full_size(struct kmem_cache *s)  in obj_full_size()  argument
315 	return s->size + sizeof(struct obj_cgroup *);  in obj_full_size()
321 static inline bool memcg_slab_pre_alloc_hook(struct kmem_cache *s,  in memcg_slab_pre_alloc_hook()  argument
330 	if (!(flags & __GFP_ACCOUNT) && !(s->flags & SLAB_ACCOUNT))  in memcg_slab_pre_alloc_hook()
337 	if (obj_cgroup_charge(objcg, flags, objects * obj_full_size(s))) {  in memcg_slab_pre_alloc_hook()
360 static inline void memcg_slab_post_alloc_hook(struct kmem_cache *s,  in memcg_slab_post_alloc_hook()  argument
377 			    memcg_alloc_page_obj_cgroups(page, s, flags)) {  in memcg_slab_post_alloc_hook()
378 				obj_cgroup_uncharge(objcg, obj_full_size(s));  in memcg_slab_post_alloc_hook()
382 			off = obj_to_index(s, page, p[i]);  in memcg_slab_post_alloc_hook()
386 					cache_vmstat_idx(s), obj_full_size(s));  in memcg_slab_post_alloc_hook()
388 			obj_cgroup_uncharge(objcg, obj_full_size(s));  in memcg_slab_post_alloc_hook()
397 	struct kmem_cache *s;  in memcg_slab_free_hook()  local
415 			s = page->slab_cache;  in memcg_slab_free_hook()
417 			s = s_orig;  in memcg_slab_free_hook()
419 		off = obj_to_index(s, page, p[i]);  in memcg_slab_free_hook()
425 		obj_cgroup_uncharge(objcg, obj_full_size(s));  in memcg_slab_free_hook()
426 		mod_objcg_state(objcg, page_pgdat(page), cache_vmstat_idx(s),  in memcg_slab_free_hook()
427 				-obj_full_size(s));  in memcg_slab_free_hook()
444 					       struct kmem_cache *s, gfp_t gfp)  in memcg_alloc_page_obj_cgroups()  argument
453 static inline bool memcg_slab_pre_alloc_hook(struct kmem_cache *s,  in memcg_slab_pre_alloc_hook()  argument
460 static inline void memcg_slab_post_alloc_hook(struct kmem_cache *s,  in memcg_slab_post_alloc_hook()  argument
467 static inline void memcg_slab_free_hook(struct kmem_cache *s,  in memcg_slab_free_hook()  argument
485 					      struct kmem_cache *s)  in account_slab_page()  argument
487 	mod_node_page_state(page_pgdat(page), cache_vmstat_idx(s),  in account_slab_page()
492 						struct kmem_cache *s)  in unaccount_slab_page()  argument
497 	mod_node_page_state(page_pgdat(page), cache_vmstat_idx(s),  in unaccount_slab_page()
501 static inline struct kmem_cache *cache_from_obj(struct kmem_cache *s, void *x)  in cache_from_obj()  argument
506 	    !kmem_cache_debug_flags(s, SLAB_CONSISTENCY_CHECKS))  in cache_from_obj()
507 		return s;  in cache_from_obj()
510 	if (WARN(cachep && cachep != s,  in cache_from_obj()
512 		  __func__, s->name, cachep->name))  in cache_from_obj()
517 static inline size_t slab_ksize(const struct kmem_cache *s)  in slab_ksize()  argument
520 	return s->object_size;  in slab_ksize()
528 	if (s->flags & (SLAB_RED_ZONE | SLAB_POISON))  in slab_ksize()
529 		return s->object_size;  in slab_ksize()
531 	if (s->flags & SLAB_KASAN)  in slab_ksize()
532 		return s->object_size;  in slab_ksize()
538 	if (s->flags & (SLAB_TYPESAFE_BY_RCU | SLAB_STORE_USER))  in slab_ksize()
539 		return s->inuse;  in slab_ksize()
543 	return s->size;  in slab_ksize()
547 static inline struct kmem_cache *slab_pre_alloc_hook(struct kmem_cache *s,  in slab_pre_alloc_hook()  argument
558 	if (should_failslab(s, flags))  in slab_pre_alloc_hook()
561 	if (!memcg_slab_pre_alloc_hook(s, objcgp, size, flags))  in slab_pre_alloc_hook()
564 	return s;  in slab_pre_alloc_hook()
567 static inline void slab_post_alloc_hook(struct kmem_cache *s,  in slab_post_alloc_hook()  argument
583 		p[i] = kasan_slab_alloc(s, p[i], flags, init);  in slab_post_alloc_hook()
585 			memset(p[i], 0, s->object_size);  in slab_post_alloc_hook()
586 		kmemleak_alloc_recursive(p[i], s->object_size, 1,  in slab_post_alloc_hook()
587 					 s->flags, flags);  in slab_post_alloc_hook()
590 	memcg_slab_post_alloc_hook(s, objcg, flags, size, p);  in slab_post_alloc_hook()
627 static inline struct kmem_cache_node *get_node(struct kmem_cache *s, int node)  in get_node()  argument
629 	return s->node[node];  in get_node()
693 static inline void debugfs_slab_release(struct kmem_cache *s) { }  in debugfs_slab_release()  argument