Lines Matching refs:uprobe
65 struct uprobe { struct
79 struct uprobe *uprobe; argument
335 static int match_uprobe(struct uprobe *l, struct uprobe *r) in match_uprobe()
352 static struct uprobe *__find_uprobe(struct inode *inode, loff_t offset) in __find_uprobe()
354 struct uprobe u = { .inode = inode, .offset = offset }; in __find_uprobe()
356 struct uprobe *uprobe; in __find_uprobe() local
360 uprobe = rb_entry(n, struct uprobe, rb_node); in __find_uprobe()
361 match = match_uprobe(&u, uprobe); in __find_uprobe()
363 atomic_inc(&uprobe->ref); in __find_uprobe()
364 return uprobe; in __find_uprobe()
379 static struct uprobe *find_uprobe(struct inode *inode, loff_t offset) in find_uprobe()
381 struct uprobe *uprobe; in find_uprobe() local
384 uprobe = __find_uprobe(inode, offset); in find_uprobe()
387 return uprobe; in find_uprobe()
390 static struct uprobe *__insert_uprobe(struct uprobe *uprobe) in __insert_uprobe() argument
394 struct uprobe *u; in __insert_uprobe()
399 u = rb_entry(parent, struct uprobe, rb_node); in __insert_uprobe()
400 match = match_uprobe(uprobe, u); in __insert_uprobe()
414 rb_link_node(&uprobe->rb_node, parent, p); in __insert_uprobe()
415 rb_insert_color(&uprobe->rb_node, &uprobes_tree); in __insert_uprobe()
417 atomic_set(&uprobe->ref, 2); in __insert_uprobe()
430 static struct uprobe *insert_uprobe(struct uprobe *uprobe) in insert_uprobe() argument
432 struct uprobe *u; in insert_uprobe()
435 u = __insert_uprobe(uprobe); in insert_uprobe()
441 static void put_uprobe(struct uprobe *uprobe) in put_uprobe() argument
443 if (atomic_dec_and_test(&uprobe->ref)) in put_uprobe()
444 kfree(uprobe); in put_uprobe()
447 static struct uprobe *alloc_uprobe(struct inode *inode, loff_t offset) in alloc_uprobe()
449 struct uprobe *uprobe, *cur_uprobe; in alloc_uprobe() local
451 uprobe = kzalloc(sizeof(struct uprobe), GFP_KERNEL); in alloc_uprobe()
452 if (!uprobe) in alloc_uprobe()
455 uprobe->inode = igrab(inode); in alloc_uprobe()
456 uprobe->offset = offset; in alloc_uprobe()
457 init_rwsem(&uprobe->register_rwsem); in alloc_uprobe()
458 init_rwsem(&uprobe->consumer_rwsem); in alloc_uprobe()
460 __set_bit(UPROBE_SKIP_SSTEP, &uprobe->flags); in alloc_uprobe()
463 cur_uprobe = insert_uprobe(uprobe); in alloc_uprobe()
467 kfree(uprobe); in alloc_uprobe()
468 uprobe = cur_uprobe; in alloc_uprobe()
472 return uprobe; in alloc_uprobe()
475 static void consumer_add(struct uprobe *uprobe, struct uprobe_consumer *uc) in consumer_add() argument
477 down_write(&uprobe->consumer_rwsem); in consumer_add()
478 uc->next = uprobe->consumers; in consumer_add()
479 uprobe->consumers = uc; in consumer_add()
480 up_write(&uprobe->consumer_rwsem); in consumer_add()
488 static bool consumer_del(struct uprobe *uprobe, struct uprobe_consumer *uc) in consumer_del() argument
493 down_write(&uprobe->consumer_rwsem); in consumer_del()
494 for (con = &uprobe->consumers; *con; con = &(*con)->next) { in consumer_del()
501 up_write(&uprobe->consumer_rwsem); in consumer_del()
528 static int copy_insn(struct uprobe *uprobe, struct file *filp) in copy_insn() argument
534 nbytes = PAGE_SIZE - (uprobe->offset & ~PAGE_MASK); in copy_insn()
535 mapping = uprobe->inode->i_mapping; in copy_insn()
538 if (uprobe->offset + MAX_UINSN_BYTES > uprobe->inode->i_size) in copy_insn()
539 bytes = uprobe->inode->i_size - uprobe->offset; in copy_insn()
545 int err = __copy_insn(mapping, filp, uprobe->arch.insn + nbytes, in copy_insn()
546 bytes - nbytes, uprobe->offset + nbytes); in copy_insn()
551 return __copy_insn(mapping, filp, uprobe->arch.insn, bytes, uprobe->offset); in copy_insn()
554 static int prepare_uprobe(struct uprobe *uprobe, struct file *file, in prepare_uprobe() argument
559 if (test_bit(UPROBE_COPY_INSN, &uprobe->flags)) in prepare_uprobe()
563 down_write(&uprobe->consumer_rwsem); in prepare_uprobe()
564 if (test_bit(UPROBE_COPY_INSN, &uprobe->flags)) in prepare_uprobe()
567 ret = copy_insn(uprobe, file); in prepare_uprobe()
572 if (is_trap_insn((uprobe_opcode_t *)uprobe->arch.insn)) in prepare_uprobe()
575 ret = arch_uprobe_analyze_insn(&uprobe->arch, mm, vaddr); in prepare_uprobe()
580 BUG_ON((uprobe->offset & ~PAGE_MASK) + in prepare_uprobe()
584 set_bit(UPROBE_COPY_INSN, &uprobe->flags); in prepare_uprobe()
587 up_write(&uprobe->consumer_rwsem); in prepare_uprobe()
598 static bool filter_chain(struct uprobe *uprobe, in filter_chain() argument
604 down_read(&uprobe->consumer_rwsem); in filter_chain()
605 for (uc = uprobe->consumers; uc; uc = uc->next) { in filter_chain()
610 up_read(&uprobe->consumer_rwsem); in filter_chain()
616 install_breakpoint(struct uprobe *uprobe, struct mm_struct *mm, in install_breakpoint() argument
622 ret = prepare_uprobe(uprobe, vma->vm_file, mm, vaddr); in install_breakpoint()
634 ret = set_swbp(&uprobe->arch, mm, vaddr); in install_breakpoint()
644 remove_breakpoint(struct uprobe *uprobe, struct mm_struct *mm, unsigned long vaddr) in remove_breakpoint() argument
647 return set_orig_insn(&uprobe->arch, mm, vaddr); in remove_breakpoint()
650 static inline bool uprobe_is_active(struct uprobe *uprobe) in uprobe_is_active() argument
652 return !RB_EMPTY_NODE(&uprobe->rb_node); in uprobe_is_active()
659 static void delete_uprobe(struct uprobe *uprobe) in delete_uprobe() argument
661 if (WARN_ON(!uprobe_is_active(uprobe))) in delete_uprobe()
665 rb_erase(&uprobe->rb_node, &uprobes_tree); in delete_uprobe()
667 RB_CLEAR_NODE(&uprobe->rb_node); /* for uprobe_is_active() */ in delete_uprobe()
668 iput(uprobe->inode); in delete_uprobe()
669 put_uprobe(uprobe); in delete_uprobe()
756 register_for_each_vma(struct uprobe *uprobe, struct uprobe_consumer *new) in register_for_each_vma() argument
763 info = build_map_info(uprobe->inode->i_mapping, in register_for_each_vma()
764 uprobe->offset, is_register); in register_for_each_vma()
780 file_inode(vma->vm_file) != uprobe->inode) in register_for_each_vma()
784 vaddr_to_offset(vma, info->vaddr) != uprobe->offset) in register_for_each_vma()
791 err = install_breakpoint(uprobe, mm, vma, info->vaddr); in register_for_each_vma()
793 if (!filter_chain(uprobe, in register_for_each_vma()
795 err |= remove_breakpoint(uprobe, mm, info->vaddr); in register_for_each_vma()
809 static int __uprobe_register(struct uprobe *uprobe, struct uprobe_consumer *uc) in __uprobe_register() argument
811 consumer_add(uprobe, uc); in __uprobe_register()
812 return register_for_each_vma(uprobe, uc); in __uprobe_register()
815 static void __uprobe_unregister(struct uprobe *uprobe, struct uprobe_consumer *uc) in __uprobe_unregister() argument
819 if (!consumer_del(uprobe, uc)) /* WARN? */ in __uprobe_unregister()
822 err = register_for_each_vma(uprobe, NULL); in __uprobe_unregister()
824 if (!uprobe->consumers && !err) in __uprobe_unregister()
825 delete_uprobe(uprobe); in __uprobe_unregister()
847 struct uprobe *uprobe; in uprobe_register() local
859 uprobe = alloc_uprobe(inode, offset); in uprobe_register()
860 if (!uprobe) in uprobe_register()
866 down_write(&uprobe->register_rwsem); in uprobe_register()
868 if (likely(uprobe_is_active(uprobe))) { in uprobe_register()
869 ret = __uprobe_register(uprobe, uc); in uprobe_register()
871 __uprobe_unregister(uprobe, uc); in uprobe_register()
873 up_write(&uprobe->register_rwsem); in uprobe_register()
874 put_uprobe(uprobe); in uprobe_register()
892 struct uprobe *uprobe; in uprobe_apply() local
896 uprobe = find_uprobe(inode, offset); in uprobe_apply()
897 if (!uprobe) in uprobe_apply()
900 down_write(&uprobe->register_rwsem); in uprobe_apply()
901 for (con = uprobe->consumers; con && con != uc ; con = con->next) in uprobe_apply()
904 ret = register_for_each_vma(uprobe, add ? uc : NULL); in uprobe_apply()
905 up_write(&uprobe->register_rwsem); in uprobe_apply()
906 put_uprobe(uprobe); in uprobe_apply()
919 struct uprobe *uprobe; in uprobe_unregister() local
921 uprobe = find_uprobe(inode, offset); in uprobe_unregister()
922 if (!uprobe) in uprobe_unregister()
925 down_write(&uprobe->register_rwsem); in uprobe_unregister()
926 __uprobe_unregister(uprobe, uc); in uprobe_unregister()
927 up_write(&uprobe->register_rwsem); in uprobe_unregister()
928 put_uprobe(uprobe); in uprobe_unregister()
932 static int unapply_uprobe(struct uprobe *uprobe, struct mm_struct *mm) in unapply_uprobe() argument
943 file_inode(vma->vm_file) != uprobe->inode) in unapply_uprobe()
947 if (uprobe->offset < offset || in unapply_uprobe()
948 uprobe->offset >= offset + vma->vm_end - vma->vm_start) in unapply_uprobe()
951 vaddr = offset_to_vaddr(vma, uprobe->offset); in unapply_uprobe()
952 err |= remove_breakpoint(uprobe, mm, vaddr); in unapply_uprobe()
965 struct uprobe *u = rb_entry(n, struct uprobe, rb_node); in find_node_in_range()
994 struct uprobe *u; in build_probe_list()
1004 u = rb_entry(t, struct uprobe, rb_node); in build_probe_list()
1011 u = rb_entry(t, struct uprobe, rb_node); in build_probe_list()
1030 struct uprobe *uprobe, *u; in uprobe_mmap() local
1047 list_for_each_entry_safe(uprobe, u, &tmp_list, pending_list) { in uprobe_mmap()
1049 filter_chain(uprobe, UPROBE_FILTER_MMAP, vma->vm_mm)) { in uprobe_mmap()
1050 unsigned long vaddr = offset_to_vaddr(vma, uprobe->offset); in uprobe_mmap()
1051 install_breakpoint(uprobe, vma->vm_mm, vma, vaddr); in uprobe_mmap()
1053 put_uprobe(uprobe); in uprobe_mmap()
1245 static unsigned long xol_get_insn_slot(struct uprobe *uprobe) in xol_get_insn_slot() argument
1259 copy_to_page(area->page, xol_vaddr, uprobe->arch.insn, MAX_UINSN_BYTES); in xol_get_insn_slot()
1338 put_uprobe(tmp->uprobe); in uprobe_free_utask()
1389 static void prepare_uretprobe(struct uprobe *uprobe, struct pt_regs *regs) in prepare_uretprobe() argument
1439 atomic_inc(&uprobe->ref); in prepare_uretprobe()
1440 ri->uprobe = uprobe; in prepare_uretprobe()
1459 pre_ssout(struct uprobe *uprobe, struct pt_regs *regs, unsigned long bp_vaddr) in pre_ssout() argument
1469 xol_vaddr = xol_get_insn_slot(uprobe); in pre_ssout()
1476 err = arch_uprobe_pre_xol(&uprobe->arch, regs); in pre_ssout()
1482 utask->active_uprobe = uprobe; in pre_ssout()
1525 static bool can_skip_sstep(struct uprobe *uprobe, struct pt_regs *regs) in can_skip_sstep() argument
1527 if (test_bit(UPROBE_SKIP_SSTEP, &uprobe->flags)) { in can_skip_sstep()
1528 if (arch_uprobe_skip_sstep(&uprobe->arch, regs)) in can_skip_sstep()
1530 clear_bit(UPROBE_SKIP_SSTEP, &uprobe->flags); in can_skip_sstep()
1580 static struct uprobe *find_active_uprobe(unsigned long bp_vaddr, int *is_swbp) in find_active_uprobe()
1583 struct uprobe *uprobe = NULL; in find_active_uprobe() local
1593 uprobe = find_uprobe(inode, offset); in find_active_uprobe()
1596 if (!uprobe) in find_active_uprobe()
1602 if (!uprobe && test_and_clear_bit(MMF_RECALC_UPROBES, &mm->flags)) in find_active_uprobe()
1606 return uprobe; in find_active_uprobe()
1609 static void handler_chain(struct uprobe *uprobe, struct pt_regs *regs) in handler_chain() argument
1615 down_read(&uprobe->register_rwsem); in handler_chain()
1616 for (uc = uprobe->consumers; uc; uc = uc->next) { in handler_chain()
1632 prepare_uretprobe(uprobe, regs); /* put bp at return */ in handler_chain()
1634 if (remove && uprobe->consumers) { in handler_chain()
1635 WARN_ON(!uprobe_is_active(uprobe)); in handler_chain()
1636 unapply_uprobe(uprobe, current->mm); in handler_chain()
1638 up_read(&uprobe->register_rwsem); in handler_chain()
1644 struct uprobe *uprobe = ri->uprobe; in handle_uretprobe_chain() local
1647 down_read(&uprobe->register_rwsem); in handle_uretprobe_chain()
1648 for (uc = uprobe->consumers; uc; uc = uc->next) { in handle_uretprobe_chain()
1652 up_read(&uprobe->register_rwsem); in handle_uretprobe_chain()
1680 put_uprobe(ri->uprobe); in handle_trampoline()
1705 struct uprobe *uprobe; in handle_swbp() local
1718 uprobe = find_active_uprobe(bp_vaddr, &is_swbp); in handle_swbp()
1719 if (!uprobe) { in handle_swbp()
1746 if (unlikely(!test_bit(UPROBE_COPY_INSN, &uprobe->flags))) in handle_swbp()
1749 handler_chain(uprobe, regs); in handle_swbp()
1750 if (can_skip_sstep(uprobe, regs)) in handle_swbp()
1753 if (!pre_ssout(uprobe, regs, bp_vaddr)) in handle_swbp()
1758 put_uprobe(uprobe); in handle_swbp()
1767 struct uprobe *uprobe; in handle_singlestep() local
1769 uprobe = utask->active_uprobe; in handle_singlestep()
1771 arch_uprobe_post_xol(&uprobe->arch, regs); in handle_singlestep()
1773 arch_uprobe_abort_xol(&uprobe->arch, regs); in handle_singlestep()
1777 put_uprobe(uprobe); in handle_singlestep()