Lines Matching +full:scan +full:- +full:count
1 // SPDX-License-Identifier: GPL-2.0-only
9 * Documentation/dev-tools/kmemleak.rst.
12 * ----------------
16 * - kmemleak_lock (raw_spinlock_t): protects the object_list as well as
21 * black trees used to look-up metadata based on a pointer to the
28 * - kmemleak_object.lock (raw_spinlock_t): protects a kmemleak_object.
29 * Accesses to the metadata (e.g. count) are protected by this lock. Note
35 * - scan_mutex (mutex): ensures that only one thread may scan the memory for
39 * scan_mutex is held. At the end of a scan, the gray_list is always empty.
48 * scan_mutex [-> object->lock] -> kmemleak_lock -> other_object->lock (SINGLE_DEPTH_NESTING)
50 * No kmemleak_lock and object->lock nesting is allowed outside scan_mutex
55 * 0, this count can no longer be incremented and put_object() schedules the
113 #define SECS_FIRST_SCAN 60 /* delay before the first scan */
133 #define KMEMLEAK_BLACK -1
138 * object->lock. Insertions or deletions from object_list, gray_list or
140 * the notes on locking above). These objects are reference-counted
150 /* object usage count; object freed when use_count == 0 */
160 int count; member
175 /* flag set to not scan the object */
177 /* flag set to fully scan the object when scan_area allocation failed */
199 /* the list of gray-colored objects (see color_gray comment below) */
293 * with the object->lock held.
298 const u8 *ptr = (const u8 *)object->pointer; in hex_dump_object()
301 if (WARN_ON_ONCE(object->flags & OBJECT_PHYS)) in hex_dump_object()
305 len = min_t(size_t, object->size, HEX_MAX_LINES * HEX_ROW_SIZE); in hex_dump_object()
315 * Object colors, encoded with count and min_count:
316 * - white - orphan object, not enough references to it (count < min_count)
317 * - gray - not orphan, not marked as false positive (min_count == 0) or
318 * sufficient references to it (count >= min_count)
319 * - black - ignore, it doesn't contain references (e.g. text section)
320 * (min_count == -1). No function defined for this color.
321 * Newly created objects don't have any color assigned (object->count == -1)
322 * before the next memory scan when they become white.
326 return object->count != KMEMLEAK_BLACK && in color_white()
327 object->count < object->min_count; in color_white()
332 return object->min_count != KMEMLEAK_BLACK && in color_gray()
333 object->count >= object->min_count; in color_gray()
343 return (color_white(object) && object->flags & OBJECT_ALLOCATED) && in unreferenced_object()
344 time_before_eq(object->jiffies + jiffies_min_age, in unreferenced_object()
350 * print_unreferenced function must be called with the object->lock held.
358 unsigned int msecs_age = jiffies_to_msecs(jiffies - object->jiffies); in print_unreferenced()
360 nr_entries = stack_depot_fetch(object->trace_handle, &entries); in print_unreferenced()
362 object->pointer, object->size); in print_unreferenced()
364 object->comm, object->pid, object->jiffies, in print_unreferenced()
378 * the object->lock held.
383 object->pointer, object->size); in dump_object_info()
385 object->comm, object->pid, object->jiffies); in dump_object_info()
386 pr_notice(" min_count = %d\n", object->min_count); in dump_object_info()
387 pr_notice(" count = %d\n", object->count); in dump_object_info()
388 pr_notice(" flags = 0x%x\n", object->flags); in dump_object_info()
389 pr_notice(" checksum = %u\n", object->checksum); in dump_object_info()
391 if (object->trace_handle) in dump_object_info()
392 stack_depot_print(object->trace_handle); in dump_object_info()
396 * Look-up a memory block metadata (kmemleak_object) in the object search
413 untagged_objp = (unsigned long)kasan_reset_tag((void *)object->pointer); in __lookup_object()
416 rb = object->rb_node.rb_left; in __lookup_object()
417 else if (untagged_objp + object->size <= untagged_ptr) in __lookup_object()
418 rb = object->rb_node.rb_right; in __lookup_object()
431 /* Look-up a kmemleak object which allocated with virtual address. */
445 return atomic_inc_not_zero(&object->use_count); in get_object()
468 list_del(&object->object_list); in mem_pool_alloc()
470 object = &mem_pool[--mem_pool_free_count]; in mem_pool_alloc()
492 list_add(&object->object_list, &mem_pool_free_list); in mem_pool_free()
510 hlist_for_each_entry_safe(area, tmp, &object->area_list, node) { in free_object_rcu()
511 hlist_del(&area->node); in free_object_rcu()
518 * Decrement the object use_count. Once the count is 0, free the object using
519 * an RCU callback. Since put_object() may be called via the kmemleak_free() ->
521 * recursive call to the kernel allocator. Lock-less RCU object_list traversal
526 if (!atomic_dec_and_test(&object->use_count)) in put_object()
530 WARN_ON(object->flags & OBJECT_ALLOCATED); in put_object()
538 call_rcu(&object->rcu, free_object_rcu); in put_object()
540 free_object_rcu(&object->rcu); in put_object()
578 rb_erase(&object->rb_node, object->flags & OBJECT_PHYS ? in __remove_object()
581 if (!(object->del_state & DELSTATE_NO_DELETE)) in __remove_object()
582 list_del_rcu(&object->object_list); in __remove_object()
583 object->del_state |= DELSTATE_REMOVED; in __remove_object()
647 INIT_LIST_HEAD(&object->object_list); in __create_object()
648 INIT_LIST_HEAD(&object->gray_list); in __create_object()
649 INIT_HLIST_HEAD(&object->area_list); in __create_object()
650 raw_spin_lock_init(&object->lock); in __create_object()
651 atomic_set(&object->use_count, 1); in __create_object()
652 object->flags = OBJECT_ALLOCATED | (is_phys ? OBJECT_PHYS : 0); in __create_object()
653 object->pointer = ptr; in __create_object()
654 object->size = kfence_ksize((void *)ptr) ?: size; in __create_object()
655 object->excess_ref = 0; in __create_object()
656 object->min_count = min_count; in __create_object()
657 object->count = 0; /* white color initially */ in __create_object()
658 object->jiffies = jiffies; in __create_object()
659 object->checksum = 0; in __create_object()
660 object->del_state = 0; in __create_object()
664 object->pid = 0; in __create_object()
665 strncpy(object->comm, "hardirq", sizeof(object->comm)); in __create_object()
667 object->pid = 0; in __create_object()
668 strncpy(object->comm, "softirq", sizeof(object->comm)); in __create_object()
670 object->pid = current->pid; in __create_object()
674 * dependency issues with current->alloc_lock. In the worst in __create_object()
677 strncpy(object->comm, current->comm, sizeof(object->comm)); in __create_object()
681 object->trace_handle = set_track_prepare(); in __create_object()
700 untagged_objp = (unsigned long)kasan_reset_tag((void *)parent->pointer); in __create_object()
702 link = &parent->rb_node.rb_left; in __create_object()
703 else if (untagged_objp + parent->size <= untagged_ptr) in __create_object()
704 link = &parent->rb_node.rb_right; in __create_object()
709 * No need for parent->lock here since "parent" cannot in __create_object()
717 rb_link_node(&object->rb_node, rb_parent, link); in __create_object()
718 rb_insert_color(&object->rb_node, is_phys ? &object_phys_tree_root : in __create_object()
720 list_add_tail_rcu(&object->object_list, &object_list); in __create_object()
746 WARN_ON(!(object->flags & OBJECT_ALLOCATED)); in __delete_object()
747 WARN_ON(atomic_read(&object->use_count) < 1); in __delete_object()
753 raw_spin_lock_irqsave(&object->lock, flags); in __delete_object()
754 object->flags &= ~OBJECT_ALLOCATED; in __delete_object()
755 raw_spin_unlock_irqrestore(&object->lock, flags); in __delete_object()
802 start = object->pointer; in delete_object_part()
803 end = object->pointer + object->size; in delete_object_part()
805 __create_object(start, ptr - start, object->min_count, in delete_object_part()
808 __create_object(ptr + size, end - ptr - size, object->min_count, in delete_object_part()
816 object->min_count = color; in __paint_it()
818 object->flags |= OBJECT_NO_SCAN; in __paint_it()
825 raw_spin_lock_irqsave(&object->lock, flags); in paint_it()
827 raw_spin_unlock_irqrestore(&object->lock, flags); in paint_it()
847 * Mark an object permanently as gray-colored so that it can no longer be
856 * Mark the object as black-colored so that it is ignored from scans and
866 * kmemleak will only scan these ranges rather than the whole memory block.
878 kmemleak_warn("Adding scan area to unknown object at 0x%08lx\n", in add_scan_area()
884 untagged_objp = (unsigned long)kasan_reset_tag((void *)object->pointer); in add_scan_area()
889 raw_spin_lock_irqsave(&object->lock, flags); in add_scan_area()
891 pr_warn_once("Cannot allocate a scan area, scanning the full object\n"); in add_scan_area()
892 /* mark the object for full scan to avoid false positives */ in add_scan_area()
893 object->flags |= OBJECT_FULL_SCAN; in add_scan_area()
897 size = untagged_objp + object->size - untagged_ptr; in add_scan_area()
898 } else if (untagged_ptr + size > untagged_objp + object->size) { in add_scan_area()
899 kmemleak_warn("Scan area larger than object 0x%08lx\n", ptr); in add_scan_area()
905 INIT_HLIST_NODE(&area->node); in add_scan_area()
906 area->start = ptr; in add_scan_area()
907 area->size = size; in add_scan_area()
909 hlist_add_head(&area->node, &object->area_list); in add_scan_area()
911 raw_spin_unlock_irqrestore(&object->lock, flags); in add_scan_area()
933 raw_spin_lock_irqsave(&object->lock, flags); in object_set_excess_ref()
934 object->excess_ref = excess_ref; in object_set_excess_ref()
935 raw_spin_unlock_irqrestore(&object->lock, flags); in object_set_excess_ref()
955 raw_spin_lock_irqsave(&object->lock, flags); in object_no_scan()
956 object->flags |= OBJECT_NO_SCAN; in object_no_scan()
957 raw_spin_unlock_irqrestore(&object->lock, flags); in object_no_scan()
962 * kmemleak_alloc - register a newly allocated object
968 * the object is never reported as a leak. If @min_count is -1,
986 * kmemleak_alloc_percpu - register a newly allocated __percpu object
1013 * kmemleak_vmalloc - register a newly vmalloc'ed object
1030 create_object((unsigned long)area->addr, size, 2, gfp); in kmemleak_vmalloc()
1032 (unsigned long)area->addr); in kmemleak_vmalloc()
1038 * kmemleak_free - unregister a previously registered object
1054 * kmemleak_free_part - partially unregister a previously registered object
1072 * kmemleak_free_percpu - unregister a previously registered __percpu object
1092 * kmemleak_update_trace - update object allocation stack trace
1117 raw_spin_lock_irqsave(&object->lock, flags); in kmemleak_update_trace()
1118 object->trace_handle = set_track_prepare(); in kmemleak_update_trace()
1119 raw_spin_unlock_irqrestore(&object->lock, flags); in kmemleak_update_trace()
1126 * kmemleak_not_leak - mark an allocated object as false positive
1142 * kmemleak_ignore - ignore an allocated object
1160 * kmemleak_scan_area - limit the range to be scanned in an allocated object
1162 * represents the start of the scan area
1163 * @size: size of the scan area
1167 * contain references to other objects. Kmemleak will only scan these areas
1180 * kmemleak_no_scan - do not scan an allocated object
1183 * This function notifies kmemleak not to scan the given memory block. Useful
1185 * references to other objects. Kmemleak will not scan such objects reducing
1198 * kmemleak_alloc_phys - similar to kmemleak_alloc but taking a physical
1218 * kmemleak_free_part_phys - similar to kmemleak_free_part but taking a
1234 * kmemleak_ignore_phys - similar to kmemleak_ignore but taking a physical
1252 u32 old_csum = object->checksum; in update_checksum()
1254 if (WARN_ON_ONCE(object->flags & OBJECT_PHYS)) in update_checksum()
1259 object->checksum = crc32(0, kasan_reset_tag((void *)object->pointer), object->size); in update_checksum()
1263 return object->checksum != old_csum; in update_checksum()
1267 * Update an object's references. object->lock must be held by the caller.
1272 /* non-orphan, ignored or new */ in update_refs()
1277 * Increase the object's reference count (number of pointers to the in update_refs()
1278 * memory block). If this count reaches the required minimum, the in update_refs()
1282 object->count++; in update_refs()
1286 list_add_tail(&object->gray_list, &gray_list); in update_refs()
1303 if (current->mm) in scan_should_stop()
1312 * Scan a memory block (exclusive range) for valid pointers and add those
1320 unsigned long *end = _end - (BYTES_PER_POINTER - 1); in scan_block()
1343 * object->use_count cannot be dropped to 0 while the object in scan_block()
1355 * Avoid the lockdep recursive warning on object->lock being in scan_block()
1359 raw_spin_lock_nested(&object->lock, SINGLE_DEPTH_NESTING); in scan_block()
1362 excess_ref = object->excess_ref; in scan_block()
1368 raw_spin_unlock(&object->lock); in scan_block()
1377 raw_spin_lock_nested(&object->lock, SINGLE_DEPTH_NESTING); in scan_block()
1379 raw_spin_unlock(&object->lock); in scan_block()
1386 * Scan a large memory block in MAX_SCAN_SIZE chunks to reduce the latency.
1403 * Scan a memory block corresponding to a kmemleak_object. A condition is
1404 * that object->use_count >= 1.
1413 * Once the object->lock is acquired, the corresponding memory block in scan_object()
1416 raw_spin_lock_irqsave(&object->lock, flags); in scan_object()
1417 if (object->flags & OBJECT_NO_SCAN) in scan_object()
1419 if (!(object->flags & OBJECT_ALLOCATED)) in scan_object()
1423 obj_ptr = object->flags & OBJECT_PHYS ? in scan_object()
1424 __va((phys_addr_t)object->pointer) : in scan_object()
1425 (void *)object->pointer; in scan_object()
1427 if (hlist_empty(&object->area_list) || in scan_object()
1428 object->flags & OBJECT_FULL_SCAN) { in scan_object()
1430 void *end = obj_ptr + object->size; in scan_object()
1441 raw_spin_unlock_irqrestore(&object->lock, flags); in scan_object()
1443 raw_spin_lock_irqsave(&object->lock, flags); in scan_object()
1444 } while (object->flags & OBJECT_ALLOCATED); in scan_object()
1446 hlist_for_each_entry(area, &object->area_list, node) in scan_object()
1447 scan_block((void *)area->start, in scan_object()
1448 (void *)(area->start + area->size), in scan_object()
1451 raw_spin_unlock_irqrestore(&object->lock, flags); in scan_object()
1455 * Scan the objects already referenced (gray objects). More objects will be
1468 while (&object->gray_list != &gray_list) { in scan_gray_list()
1475 tmp = list_entry(object->gray_list.next, typeof(*object), in scan_gray_list()
1479 list_del(&object->gray_list); in scan_gray_list()
1498 if (object->del_state & DELSTATE_REMOVED) in kmemleak_cond_resched()
1500 object->del_state |= DELSTATE_NO_DELETE; in kmemleak_cond_resched()
1508 if (object->del_state & DELSTATE_REMOVED) in kmemleak_cond_resched()
1509 list_del_rcu(&object->object_list); in kmemleak_cond_resched()
1510 object->del_state &= ~DELSTATE_NO_DELETE; in kmemleak_cond_resched()
1517 * Scan data sections and all the referenced memory blocks allocated via the
1533 raw_spin_lock_irq(&object->lock); in kmemleak_scan()
1539 if (atomic_read(&object->use_count) > 1) { in kmemleak_scan()
1540 pr_debug("object->use_count = %d\n", in kmemleak_scan()
1541 atomic_read(&object->use_count)); in kmemleak_scan()
1547 if ((object->flags & OBJECT_PHYS) && in kmemleak_scan()
1548 !(object->flags & OBJECT_NO_SCAN)) { in kmemleak_scan()
1549 unsigned long phys = object->pointer; in kmemleak_scan()
1552 PHYS_PFN(phys + object->size) > max_low_pfn) in kmemleak_scan()
1556 /* reset the reference count (whiten the object) */ in kmemleak_scan()
1557 object->count = 0; in kmemleak_scan()
1559 list_add_tail(&object->gray_list, &gray_list); in kmemleak_scan()
1561 raw_spin_unlock_irq(&object->lock); in kmemleak_scan()
1569 /* per-cpu sections scanning */ in kmemleak_scan()
1580 unsigned long start_pfn = zone->zone_start_pfn; in kmemleak_scan()
1593 /* only scan pages belonging to this zone */ in kmemleak_scan()
1596 /* only scan if page is in use */ in kmemleak_scan()
1622 * Scan the objects already referenced from the sections scanned in kmemleak_scan()
1629 * scan and color them gray until the next scan. in kmemleak_scan()
1639 * the next scan. in kmemleak_scan()
1643 raw_spin_lock_irq(&object->lock); in kmemleak_scan()
1644 if (color_white(object) && (object->flags & OBJECT_ALLOCATED) in kmemleak_scan()
1647 object->count = object->min_count; in kmemleak_scan()
1648 list_add_tail(&object->gray_list, &gray_list); in kmemleak_scan()
1650 raw_spin_unlock_irq(&object->lock); in kmemleak_scan()
1655 * Re-scan the gray list for modified unreferenced objects. in kmemleak_scan()
1676 * the next scan. in kmemleak_scan()
1680 raw_spin_lock_irq(&object->lock); in kmemleak_scan()
1682 !(object->flags & OBJECT_REPORTED)) { in kmemleak_scan()
1683 object->flags |= OBJECT_REPORTED; in kmemleak_scan()
1690 raw_spin_unlock_irq(&object->lock); in kmemleak_scan()
1705 * at the end of a memory scan are reported but only the first time.
1715 * Wait before the first scan to allow the system to fully initialize. in kmemleak_scan_thread()
1731 /* wait before the next scan */ in kmemleak_scan_thread()
1751 pr_warn("Failed to create the scan thread\n"); in start_scan_thread()
1784 if (n-- > 0) in kmemleak_seq_start()
1842 raw_spin_lock_irqsave(&object->lock, flags); in kmemleak_seq_show()
1843 if ((object->flags & OBJECT_REPORTED) && unreferenced_object(object)) in kmemleak_seq_show()
1845 raw_spin_unlock_irqrestore(&object->lock, flags); in kmemleak_seq_show()
1868 return -EINVAL; in dump_str_object_info()
1872 return -EINVAL; in dump_str_object_info()
1875 raw_spin_lock_irqsave(&object->lock, flags); in dump_str_object_info()
1877 raw_spin_unlock_irqrestore(&object->lock, flags); in dump_str_object_info()
1895 raw_spin_lock_irq(&object->lock); in kmemleak_clear()
1896 if ((object->flags & OBJECT_REPORTED) && in kmemleak_clear()
1899 raw_spin_unlock_irq(&object->lock); in kmemleak_clear()
1909 * File write operation to configure kmemleak at run-time. The following
1911 * off - disable kmemleak (irreversible)
1912 * stack=on - enable the task stacks scanning
1913 * stack=off - disable the tasks stacks scanning
1914 * scan=on - start the automatic memory scanning thread
1915 * scan=off - stop the automatic memory scanning thread
1916 * scan=... - set the automatic memory scanning period in seconds (0 to
1918 * scan - trigger a memory scan
1919 * clear - mark all current reported unreferenced kmemleak objects as
1922 * dump=... - dump information about the object found at the given address
1931 buf_size = min(size, (sizeof(buf) - 1)); in kmemleak_write()
1933 return -EFAULT; in kmemleak_write()
1949 ret = -EPERM; in kmemleak_write()
1959 else if (strncmp(buf, "scan=on", 7) == 0) in kmemleak_write()
1961 else if (strncmp(buf, "scan=off", 8) == 0) in kmemleak_write()
1963 else if (strncmp(buf, "scan=", 5) == 0) { in kmemleak_write()
1980 } else if (strncmp(buf, "scan", 4) == 0) in kmemleak_write()
1985 ret = -EINVAL; in kmemleak_write()
2022 * no previous scan thread (otherwise, kmemleak may still have some useful
2032 * longer track object freeing. Ordering of the scan thread stopping and in kmemleak_do_cleanup()
2070 * Allow boot-time kmemleak disabling (enabled by default).
2075 return -EINVAL; in kmemleak_boot_config()
2083 return -EINVAL; in kmemleak_boot_config()
2110 create_object((unsigned long)_sdata, _edata - _sdata, in kmemleak_init()
2112 create_object((unsigned long)__bss_start, __bss_stop - __bss_start, in kmemleak_init()
2117 __end_ro_after_init - __start_ro_after_init, in kmemleak_init()
2135 * two clean-up threads but serialized by scan_mutex. in kmemleak_late_init()
2138 return -ENOMEM; in kmemleak_late_init()