1 // SPDX-License-Identifier: GPL-2.0-only
9  * Documentation/dev-tools/kmemleak.rst.
12  * ----------------
16  * - kmemleak_lock (raw_spinlock_t): protects the object_list modifications and
19  *   blocks. The object_tree_root is a red black tree used to look-up
25  * - kmemleak_object.lock (raw_spinlock_t): protects a kmemleak_object.
26  *   Accesses to the metadata (e.g. count) are protected by this lock. Note
32  * - scan_mutex (mutex): ensures that only one thread may scan the memory for
36  *   scan_mutex is held. At the end of a scan, the gray_list is always empty.
45  *   scan_mutex [-> object->lock] -> kmemleak_lock -> other_object->lock (SINGLE_DEPTH_NESTING)
47  * No kmemleak_lock and object->lock nesting is allowed outside scan_mutex
52  * 0, this count can no longer be incremented and put_object() schedules the
108 #define SECS_FIRST_SCAN		60	/* delay before the first scan */
127 #define KMEMLEAK_BLACK	-1
132  * object->lock. Insertions or deletions from object_list, gray_list or
134  * the notes on locking above). These objects are reference-counted
144 	/* object usage count; object freed when use_count == 0 */
153 	int count;  member
169 /* flag set to not scan the object */
171 /* flag set to fully scan the object when scan_area allocation failed */
186 /* the list of gray-colored objects (see color_gray comment below) */
278  * with the object->lock held.
283 	const u8 *ptr = (const u8 *)object->pointer;  in hex_dump_object()
287 	len = min_t(size_t, object->size, HEX_MAX_LINES * HEX_ROW_SIZE);  in hex_dump_object()
297  * Object colors, encoded with count and min_count:
298  * - white - orphan object, not enough references to it (count < min_count)
299  * - gray  - not orphan, not marked as false positive (min_count == 0) or
300  *		sufficient references to it (count >= min_count)
301  * - black - ignore, it doesn't contain references (e.g. text section)
302  *		(min_count == -1). No function defined for this color.
303  * Newly created objects don't have any color assigned (object->count == -1)
304  * before the next memory scan when they become white.
308 	return object->count != KMEMLEAK_BLACK &&  in color_white()
309 		object->count < object->min_count;  in color_white()
314 	return object->min_count != KMEMLEAK_BLACK &&  in color_gray()
315 		object->count >= object->min_count;  in color_gray()
325 	return (color_white(object) && object->flags & OBJECT_ALLOCATED) &&  in unreferenced_object()
326 		time_before_eq(object->jiffies + jiffies_min_age,  in unreferenced_object()
332  * print_unreferenced function must be called with the object->lock held.
338 	unsigned int msecs_age = jiffies_to_msecs(jiffies - object->jiffies);  in print_unreferenced()
341 		   object->pointer, object->size);  in print_unreferenced()
343 		   object->comm, object->pid, object->jiffies,  in print_unreferenced()
348 	for (i = 0; i < object->trace_len; i++) {  in print_unreferenced()
349 		void *ptr = (void *)object->trace[i];  in print_unreferenced()
357  * the object->lock held.
362 		  object->pointer, object->size);  in dump_object_info()
364 		  object->comm, object->pid, object->jiffies);  in dump_object_info()
365 	pr_notice("  min_count = %d\n", object->min_count);  in dump_object_info()
366 	pr_notice("  count = %d\n", object->count);  in dump_object_info()
367 	pr_notice("  flags = 0x%x\n", object->flags);  in dump_object_info()
368 	pr_notice("  checksum = %u\n", object->checksum);  in dump_object_info()
370 	stack_trace_print(object->trace, object->trace_len, 4);  in dump_object_info()
374  * Look-up a memory block metadata (kmemleak_object) in the object search
386 		if (ptr < object->pointer)  in lookup_object()
387 			rb = object->rb_node.rb_left;  in lookup_object()
388 		else if (object->pointer + object->size <= ptr)  in lookup_object()
389 			rb = object->rb_node.rb_right;  in lookup_object()
390 		else if (object->pointer == ptr || alias)  in lookup_object()
410 	return atomic_inc_not_zero(&object->use_count);  in get_object()
433 		list_del(&object->object_list);  in mem_pool_alloc()
435 		object = &mem_pool[--mem_pool_free_count];  in mem_pool_alloc()
457 	list_add(&object->object_list, &mem_pool_free_list);  in mem_pool_free()
475 	hlist_for_each_entry_safe(area, tmp, &object->area_list, node) {  in free_object_rcu()
476 		hlist_del(&area->node);  in free_object_rcu()
483  * Decrement the object use_count. Once the count is 0, free the object using
484  * an RCU callback. Since put_object() may be called via the kmemleak_free() ->
486  * recursive call to the kernel allocator. Lock-less RCU object_list traversal
491 	if (!atomic_dec_and_test(&object->use_count))  in put_object()
495 	WARN_ON(object->flags & OBJECT_ALLOCATED);  in put_object()
503 		call_rcu(&object->rcu, free_object_rcu);  in put_object()
505 		free_object_rcu(&object->rcu);  in put_object()
535 	rb_erase(&object->rb_node, &object_tree_root);  in __remove_object()
536 	list_del_rcu(&object->object_list);  in __remove_object()
585 	INIT_LIST_HEAD(&object->object_list);  in create_object()
586 	INIT_LIST_HEAD(&object->gray_list);  in create_object()
587 	INIT_HLIST_HEAD(&object->area_list);  in create_object()
588 	raw_spin_lock_init(&object->lock);  in create_object()
589 	atomic_set(&object->use_count, 1);  in create_object()
590 	object->flags = OBJECT_ALLOCATED;  in create_object()
591 	object->pointer = ptr;  in create_object()
592 	object->size = size;  in create_object()
593 	object->excess_ref = 0;  in create_object()
594 	object->min_count = min_count;  in create_object()
595 	object->count = 0;			/* white color initially */  in create_object()
596 	object->jiffies = jiffies;  in create_object()
597 	object->checksum = 0;  in create_object()
601 		object->pid = 0;  in create_object()
602 		strncpy(object->comm, "hardirq", sizeof(object->comm));  in create_object()
604 		object->pid = 0;  in create_object()
605 		strncpy(object->comm, "softirq", sizeof(object->comm));  in create_object()
607 		object->pid = current->pid;  in create_object()
611 		 * dependency issues with current->alloc_lock. In the worst  in create_object()
614 		strncpy(object->comm, current->comm, sizeof(object->comm));  in create_object()
618 	object->trace_len = __save_stack_trace(object->trace);  in create_object()
630 		if (ptr + size <= parent->pointer)  in create_object()
631 			link = &parent->rb_node.rb_left;  in create_object()
632 		else if (parent->pointer + parent->size <= ptr)  in create_object()
633 			link = &parent->rb_node.rb_right;  in create_object()
638 			 * No need for parent->lock here since "parent" cannot  in create_object()
647 	rb_link_node(&object->rb_node, rb_parent, link);  in create_object()
648 	rb_insert_color(&object->rb_node, &object_tree_root);  in create_object()
650 	list_add_tail_rcu(&object->object_list, &object_list);  in create_object()
663 	WARN_ON(!(object->flags & OBJECT_ALLOCATED));  in __delete_object()
664 	WARN_ON(atomic_read(&object->use_count) < 1);  in __delete_object()
670 	raw_spin_lock_irqsave(&object->lock, flags);  in __delete_object()
671 	object->flags &= ~OBJECT_ALLOCATED;  in __delete_object()
672 	raw_spin_unlock_irqrestore(&object->lock, flags);  in __delete_object()
719 	start = object->pointer;  in delete_object_part()
720 	end = object->pointer + object->size;  in delete_object_part()
722 		create_object(start, ptr - start, object->min_count,  in delete_object_part()
725 		create_object(ptr + size, end - ptr - size, object->min_count,  in delete_object_part()
733 	object->min_count = color;  in __paint_it()
735 		object->flags |= OBJECT_NO_SCAN;  in __paint_it()
742 	raw_spin_lock_irqsave(&object->lock, flags);  in paint_it()
744 	raw_spin_unlock_irqrestore(&object->lock, flags);  in paint_it()
764  * Mark an object permanently as gray-colored so that it can no longer be
773  * Mark the object as black-colored so that it is ignored from scans and
783  * kmemleak will only scan these ranges rather than the whole memory block.
795 		kmemleak_warn("Adding scan area to unknown object at 0x%08lx\n",  in add_scan_area()
801 	untagged_objp = (unsigned long)kasan_reset_tag((void *)object->pointer);  in add_scan_area()
806 	raw_spin_lock_irqsave(&object->lock, flags);  in add_scan_area()
808 		pr_warn_once("Cannot allocate a scan area, scanning the full object\n");  in add_scan_area()
809 		/* mark the object for full scan to avoid false positives */  in add_scan_area()
810 		object->flags |= OBJECT_FULL_SCAN;  in add_scan_area()
814 		size = untagged_objp + object->size - untagged_ptr;  in add_scan_area()
815 	} else if (untagged_ptr + size > untagged_objp + object->size) {  in add_scan_area()
816 		kmemleak_warn("Scan area larger than object 0x%08lx\n", ptr);  in add_scan_area()
822 	INIT_HLIST_NODE(&area->node);  in add_scan_area()
823 	area->start = ptr;  in add_scan_area()
824 	area->size = size;  in add_scan_area()
826 	hlist_add_head(&area->node, &object->area_list);  in add_scan_area()
828 	raw_spin_unlock_irqrestore(&object->lock, flags);  in add_scan_area()
850 	raw_spin_lock_irqsave(&object->lock, flags);  in object_set_excess_ref()
851 	object->excess_ref = excess_ref;  in object_set_excess_ref()
852 	raw_spin_unlock_irqrestore(&object->lock, flags);  in object_set_excess_ref()
872 	raw_spin_lock_irqsave(&object->lock, flags);  in object_no_scan()
873 	object->flags |= OBJECT_NO_SCAN;  in object_no_scan()
874 	raw_spin_unlock_irqrestore(&object->lock, flags);  in object_no_scan()
879  * kmemleak_alloc - register a newly allocated object
885  *		the object is never reported as a leak. If @min_count is -1,
903  * kmemleak_alloc_percpu - register a newly allocated __percpu object
930  * kmemleak_vmalloc - register a newly vmalloc'ed object
947 		create_object((unsigned long)area->addr, size, 2, gfp);  in kmemleak_vmalloc()
949 				      (unsigned long)area->addr);  in kmemleak_vmalloc()
955  * kmemleak_free - unregister a previously registered object
971  * kmemleak_free_part - partially unregister a previously registered object
989  * kmemleak_free_percpu - unregister a previously registered __percpu object
1009  * kmemleak_update_trace - update object allocation stack trace
1034 	raw_spin_lock_irqsave(&object->lock, flags);  in kmemleak_update_trace()
1035 	object->trace_len = __save_stack_trace(object->trace);  in kmemleak_update_trace()
1036 	raw_spin_unlock_irqrestore(&object->lock, flags);  in kmemleak_update_trace()
1043  * kmemleak_not_leak - mark an allocated object as false positive
1059  * kmemleak_ignore - ignore an allocated object
1077  * kmemleak_scan_area - limit the range to be scanned in an allocated object
1079  *		represents the start of the scan area
1080  * @size:	size of the scan area
1084  * contain references to other objects. Kmemleak will only scan these areas
1097  * kmemleak_no_scan - do not scan an allocated object
1100  * This function notifies kmemleak not to scan the given memory block. Useful
1102  * references to other objects. Kmemleak will not scan such objects reducing
1115  * kmemleak_alloc_phys - similar to kmemleak_alloc but taking a physical
1132  * kmemleak_free_part_phys - similar to kmemleak_free_part but taking a
1146  * kmemleak_not_leak_phys - similar to kmemleak_not_leak but taking a physical
1158  * kmemleak_ignore_phys - similar to kmemleak_ignore but taking a physical
1174 	u32 old_csum = object->checksum;  in update_checksum()
1178 	object->checksum = crc32(0, (void *)object->pointer, object->size);  in update_checksum()
1182 	return object->checksum != old_csum;  in update_checksum()
1186  * Update an object's references. object->lock must be held by the caller.
1191 		/* non-orphan, ignored or new */  in update_refs()
1196 	 * Increase the object's reference count (number of pointers to the  in update_refs()
1197 	 * memory block). If this count reaches the required minimum, the  in update_refs()
1201 	object->count++;  in update_refs()
1205 		list_add_tail(&object->gray_list, &gray_list);  in update_refs()
1222 	if (current->mm)  in scan_should_stop()
1231  * Scan a memory block (exclusive range) for valid pointers and add those
1239 	unsigned long *end = _end - (BYTES_PER_POINTER - 1);  in scan_block()
1262 		 * object->use_count cannot be dropped to 0 while the object  in scan_block()
1274 		 * Avoid the lockdep recursive warning on object->lock being  in scan_block()
1278 		raw_spin_lock_nested(&object->lock, SINGLE_DEPTH_NESTING);  in scan_block()
1281 			excess_ref = object->excess_ref;  in scan_block()
1287 		raw_spin_unlock(&object->lock);  in scan_block()
1296 			raw_spin_lock_nested(&object->lock, SINGLE_DEPTH_NESTING);  in scan_block()
1298 			raw_spin_unlock(&object->lock);  in scan_block()
1305  * Scan a large memory block in MAX_SCAN_SIZE chunks to reduce the latency.
1322  * Scan a memory block corresponding to a kmemleak_object. A condition is
1323  * that object->use_count >= 1.
1331 	 * Once the object->lock is acquired, the corresponding memory block  in scan_object()
1334 	raw_spin_lock_irqsave(&object->lock, flags);  in scan_object()
1335 	if (object->flags & OBJECT_NO_SCAN)  in scan_object()
1337 	if (!(object->flags & OBJECT_ALLOCATED))  in scan_object()
1340 	if (hlist_empty(&object->area_list) ||  in scan_object()
1341 	    object->flags & OBJECT_FULL_SCAN) {  in scan_object()
1342 		void *start = (void *)object->pointer;  in scan_object()
1343 		void *end = (void *)(object->pointer + object->size);  in scan_object()
1354 			raw_spin_unlock_irqrestore(&object->lock, flags);  in scan_object()
1356 			raw_spin_lock_irqsave(&object->lock, flags);  in scan_object()
1357 		} while (object->flags & OBJECT_ALLOCATED);  in scan_object()
1359 		hlist_for_each_entry(area, &object->area_list, node)  in scan_object()
1360 			scan_block((void *)area->start,  in scan_object()
1361 				   (void *)(area->start + area->size),  in scan_object()
1364 	raw_spin_unlock_irqrestore(&object->lock, flags);  in scan_object()
1368  * Scan the objects already referenced (gray objects). More objects will be
1381 	while (&object->gray_list != &gray_list) {  in scan_gray_list()
1388 		tmp = list_entry(object->gray_list.next, typeof(*object),  in scan_gray_list()
1392 		list_del(&object->gray_list);  in scan_gray_list()
1401  * Scan data sections and all the referenced memory blocks allocated via the
1418 		raw_spin_lock_irqsave(&object->lock, flags);  in kmemleak_scan()
1424 		if (atomic_read(&object->use_count) > 1) {  in kmemleak_scan()
1425 			pr_debug("object->use_count = %d\n",  in kmemleak_scan()
1426 				 atomic_read(&object->use_count));  in kmemleak_scan()
1430 		/* reset the reference count (whiten the object) */  in kmemleak_scan()
1431 		object->count = 0;  in kmemleak_scan()
1433 			list_add_tail(&object->gray_list, &gray_list);  in kmemleak_scan()
1435 		raw_spin_unlock_irqrestore(&object->lock, flags);  in kmemleak_scan()
1440 	/* per-cpu sections scanning */  in kmemleak_scan()
1451 		unsigned long start_pfn = zone->zone_start_pfn;  in kmemleak_scan()
1461 			/* only scan pages belonging to this zone */  in kmemleak_scan()
1464 			/* only scan if page is in use */  in kmemleak_scan()
1492 	 * Scan the objects already referenced from the sections scanned  in kmemleak_scan()
1499 	 * scan and color them gray until the next scan.  in kmemleak_scan()
1503 		raw_spin_lock_irqsave(&object->lock, flags);  in kmemleak_scan()
1504 		if (color_white(object) && (object->flags & OBJECT_ALLOCATED)  in kmemleak_scan()
1507 			object->count = object->min_count;  in kmemleak_scan()
1508 			list_add_tail(&object->gray_list, &gray_list);  in kmemleak_scan()
1510 		raw_spin_unlock_irqrestore(&object->lock, flags);  in kmemleak_scan()
1515 	 * Re-scan the gray list for modified unreferenced objects.  in kmemleak_scan()
1530 		raw_spin_lock_irqsave(&object->lock, flags);  in kmemleak_scan()
1532 		    !(object->flags & OBJECT_REPORTED)) {  in kmemleak_scan()
1533 			object->flags |= OBJECT_REPORTED;  in kmemleak_scan()
1540 		raw_spin_unlock_irqrestore(&object->lock, flags);  in kmemleak_scan()
1555  * at the end of a memory scan are reported but only the first time.
1565 	 * Wait before the first scan to allow the system to fully initialize.  in kmemleak_scan_thread()
1581 		/* wait before the next scan */  in kmemleak_scan_thread()
1601 		pr_warn("Failed to create the scan thread\n");  in start_scan_thread()
1634 		if (n-- > 0)  in kmemleak_seq_start()
1692 	raw_spin_lock_irqsave(&object->lock, flags);  in kmemleak_seq_show()
1693 	if ((object->flags & OBJECT_REPORTED) && unreferenced_object(object))  in kmemleak_seq_show()
1695 	raw_spin_unlock_irqrestore(&object->lock, flags);  in kmemleak_seq_show()
1718 		return -EINVAL;  in dump_str_object_info()
1722 		return -EINVAL;  in dump_str_object_info()
1725 	raw_spin_lock_irqsave(&object->lock, flags);  in dump_str_object_info()
1727 	raw_spin_unlock_irqrestore(&object->lock, flags);  in dump_str_object_info()
1746 		raw_spin_lock_irqsave(&object->lock, flags);  in kmemleak_clear()
1747 		if ((object->flags & OBJECT_REPORTED) &&  in kmemleak_clear()
1750 		raw_spin_unlock_irqrestore(&object->lock, flags);  in kmemleak_clear()
1760  * File write operation to configure kmemleak at run-time. The following
1762  *   off	- disable kmemleak (irreversible)
1763  *   stack=on	- enable the task stacks scanning
1764  *   stack=off	- disable the tasks stacks scanning
1765  *   scan=on	- start the automatic memory scanning thread
1766  *   scan=off	- stop the automatic memory scanning thread
1767  *   scan=...	- set the automatic memory scanning period in seconds (0 to
1769  *   scan	- trigger a memory scan
1770  *   clear	- mark all current reported unreferenced kmemleak objects as
1773  *   dump=...	- dump information about the object found at the given address
1782 	buf_size = min(size, (sizeof(buf) - 1));  in kmemleak_write()
1784 		return -EFAULT;  in kmemleak_write()
1800 		ret = -EPERM;  in kmemleak_write()
1810 	else if (strncmp(buf, "scan=on", 7) == 0)  in kmemleak_write()
1812 	else if (strncmp(buf, "scan=off", 8) == 0)  in kmemleak_write()
1814 	else if (strncmp(buf, "scan=", 5) == 0) {  in kmemleak_write()
1825 	} else if (strncmp(buf, "scan", 4) == 0)  in kmemleak_write()
1830 		ret = -EINVAL;  in kmemleak_write()
1867  * no previous scan thread (otherwise, kmemleak may still have some useful
1877 	 * longer track object freeing. Ordering of the scan thread stopping and  in kmemleak_do_cleanup()
1915  * Allow boot-time kmemleak disabling (enabled by default).
1920 		return -EINVAL;  in kmemleak_boot_config()
1926 		return -EINVAL;  in kmemleak_boot_config()
1953 	create_object((unsigned long)_sdata, _edata - _sdata,  in kmemleak_init()
1955 	create_object((unsigned long)__bss_start, __bss_stop - __bss_start,  in kmemleak_init()
1960 			      __end_ro_after_init - __start_ro_after_init,  in kmemleak_init()
1978 		 * two clean-up threads but serialized by scan_mutex.  in kmemleak_late_init()
1981 		return -ENOMEM;  in kmemleak_late_init()