Lines Matching refs:base
626 trigger_dyntick_cpu(struct timer_base *base, struct timer_list *timer) in trigger_dyntick_cpu() argument
646 if (base->is_idle) { in trigger_dyntick_cpu()
648 tick_nohz_full_cpu(base->cpu))); in trigger_dyntick_cpu()
649 wake_up_nohz_cpu(base->cpu); in trigger_dyntick_cpu()
658 static void enqueue_timer(struct timer_base *base, struct timer_list *timer, in enqueue_timer() argument
662 hlist_add_head(&timer->entry, base->vectors + idx); in enqueue_timer()
663 __set_bit(idx, base->pending_map); in enqueue_timer()
673 if (time_before(bucket_expiry, base->next_expiry)) { in enqueue_timer()
678 WRITE_ONCE(base->next_expiry, bucket_expiry); in enqueue_timer()
679 base->timers_pending = true; in enqueue_timer()
680 base->next_expiry_recalc = false; in enqueue_timer()
681 trigger_dyntick_cpu(base, timer); in enqueue_timer()
685 static void internal_add_timer(struct timer_base *base, struct timer_list *timer) in internal_add_timer() argument
690 idx = calc_wheel_index(timer->expires, base->clk, &bucket_expiry); in internal_add_timer()
691 enqueue_timer(base, timer, idx, bucket_expiry); in internal_add_timer()
943 static int detach_if_pending(struct timer_list *timer, struct timer_base *base, in detach_if_pending() argument
951 if (hlist_is_singular_node(&timer->entry, base->vectors + idx)) { in detach_if_pending()
952 __clear_bit(idx, base->pending_map); in detach_if_pending()
953 base->next_expiry_recalc = true; in detach_if_pending()
963 struct timer_base *base; in get_timer_cpu_base() local
965 base = per_cpu_ptr(&timer_bases[index], cpu); in get_timer_cpu_base()
972 base = per_cpu_ptr(&timer_bases[BASE_DEF], cpu); in get_timer_cpu_base()
973 return base; in get_timer_cpu_base()
979 struct timer_base *base; in get_timer_this_cpu_base() local
981 base = this_cpu_ptr(&timer_bases[index]); in get_timer_this_cpu_base()
988 base = this_cpu_ptr(&timer_bases[BASE_DEF]); in get_timer_this_cpu_base()
989 return base; in get_timer_this_cpu_base()
997 static inline void __forward_timer_base(struct timer_base *base, in __forward_timer_base() argument
1004 if (time_before_eq(basej, base->clk)) in __forward_timer_base()
1011 if (time_after(base->next_expiry, basej)) { in __forward_timer_base()
1012 base->clk = basej; in __forward_timer_base()
1014 if (WARN_ON_ONCE(time_before(base->next_expiry, base->clk))) in __forward_timer_base()
1016 base->clk = base->next_expiry; in __forward_timer_base()
1021 static inline void forward_timer_base(struct timer_base *base) in forward_timer_base() argument
1023 __forward_timer_base(base, READ_ONCE(jiffies)); in forward_timer_base()
1039 __acquires(timer->base->lock) in lock_timer_base()
1042 struct timer_base *base; in lock_timer_base() local
1053 base = get_timer_base(tf); in lock_timer_base()
1054 raw_spin_lock_irqsave(&base->lock, *flags); in lock_timer_base()
1056 return base; in lock_timer_base()
1057 raw_spin_unlock_irqrestore(&base->lock, *flags); in lock_timer_base()
1071 struct timer_base *base, *new_base; in __mod_timer() local
1101 base = lock_timer_base(timer, &flags); in __mod_timer()
1110 forward_timer_base(base); in __mod_timer()
1118 clk = base->clk; in __mod_timer()
1135 base = lock_timer_base(timer, &flags); in __mod_timer()
1144 forward_timer_base(base); in __mod_timer()
1147 ret = detach_if_pending(timer, base, false); in __mod_timer()
1153 if (base != new_base) { in __mod_timer()
1161 if (likely(base->running_timer != timer)) { in __mod_timer()
1165 raw_spin_unlock(&base->lock); in __mod_timer()
1166 base = new_base; in __mod_timer()
1167 raw_spin_lock(&base->lock); in __mod_timer()
1169 (timer->flags & ~TIMER_BASEMASK) | base->cpu); in __mod_timer()
1170 forward_timer_base(base); in __mod_timer()
1183 if (idx != UINT_MAX && clk == base->clk) in __mod_timer()
1184 enqueue_timer(base, timer, idx, bucket_expiry); in __mod_timer()
1186 internal_add_timer(base, timer); in __mod_timer()
1189 raw_spin_unlock_irqrestore(&base->lock, flags); in __mod_timer()
1351 struct timer_base *new_base, *base; in add_timer_on() local
1369 base = lock_timer_base(timer, &flags); in add_timer_on()
1377 if (base != new_base) { in add_timer_on()
1380 raw_spin_unlock(&base->lock); in add_timer_on()
1381 base = new_base; in add_timer_on()
1382 raw_spin_lock(&base->lock); in add_timer_on()
1386 forward_timer_base(base); in add_timer_on()
1389 internal_add_timer(base, timer); in add_timer_on()
1391 raw_spin_unlock_irqrestore(&base->lock, flags); in add_timer_on()
1412 struct timer_base *base; in __timer_delete() local
1430 base = lock_timer_base(timer, &flags); in __timer_delete()
1431 ret = detach_if_pending(timer, base, true); in __timer_delete()
1434 raw_spin_unlock_irqrestore(&base->lock, flags); in __timer_delete()
1503 struct timer_base *base; in __try_to_del_timer_sync() local
1509 base = lock_timer_base(timer, &flags); in __try_to_del_timer_sync()
1511 if (base->running_timer != timer) in __try_to_del_timer_sync()
1512 ret = detach_if_pending(timer, base, true); in __try_to_del_timer_sync()
1516 raw_spin_unlock_irqrestore(&base->lock, flags); in __try_to_del_timer_sync()
1544 static __init void timer_base_init_expiry_lock(struct timer_base *base) in timer_base_init_expiry_lock() argument
1546 spin_lock_init(&base->expiry_lock); in timer_base_init_expiry_lock()
1549 static inline void timer_base_lock_expiry(struct timer_base *base) in timer_base_lock_expiry() argument
1551 spin_lock(&base->expiry_lock); in timer_base_lock_expiry()
1554 static inline void timer_base_unlock_expiry(struct timer_base *base) in timer_base_unlock_expiry() argument
1556 spin_unlock(&base->expiry_lock); in timer_base_unlock_expiry()
1566 static void timer_sync_wait_running(struct timer_base *base) in timer_sync_wait_running() argument
1567 __releases(&base->lock) __releases(&base->expiry_lock) in timer_sync_wait_running()
1568 __acquires(&base->expiry_lock) __acquires(&base->lock) in timer_sync_wait_running()
1570 if (atomic_read(&base->timer_waiters)) { in timer_sync_wait_running()
1571 raw_spin_unlock_irq(&base->lock); in timer_sync_wait_running()
1572 spin_unlock(&base->expiry_lock); in timer_sync_wait_running()
1573 spin_lock(&base->expiry_lock); in timer_sync_wait_running()
1574 raw_spin_lock_irq(&base->lock); in timer_sync_wait_running()
1594 struct timer_base *base = get_timer_base(tf); in del_timer_wait_running() local
1604 atomic_inc(&base->timer_waiters); in del_timer_wait_running()
1605 spin_lock_bh(&base->expiry_lock); in del_timer_wait_running()
1606 atomic_dec(&base->timer_waiters); in del_timer_wait_running()
1607 spin_unlock_bh(&base->expiry_lock); in del_timer_wait_running()
1611 static inline void timer_base_init_expiry_lock(struct timer_base *base) { } in timer_base_init_expiry_lock() argument
1612 static inline void timer_base_lock_expiry(struct timer_base *base) { } in timer_base_lock_expiry() argument
1613 static inline void timer_base_unlock_expiry(struct timer_base *base) { } in timer_base_unlock_expiry() argument
1614 static inline void timer_sync_wait_running(struct timer_base *base) { } in timer_sync_wait_running() argument
1815 static void expire_timers(struct timer_base *base, struct hlist_head *head) in expire_timers() argument
1822 unsigned long baseclk = base->clk - 1; in expire_timers()
1830 base->running_timer = timer; in expire_timers()
1837 base->running_timer = NULL; in expire_timers()
1842 raw_spin_unlock(&base->lock); in expire_timers()
1844 raw_spin_lock(&base->lock); in expire_timers()
1845 base->running_timer = NULL; in expire_timers()
1847 raw_spin_unlock_irq(&base->lock); in expire_timers()
1849 raw_spin_lock_irq(&base->lock); in expire_timers()
1850 base->running_timer = NULL; in expire_timers()
1851 timer_sync_wait_running(base); in expire_timers()
1856 static int collect_expired_timers(struct timer_base *base, in collect_expired_timers() argument
1859 unsigned long clk = base->clk = base->next_expiry; in collect_expired_timers()
1867 if (__test_and_clear_bit(idx, base->pending_map)) { in collect_expired_timers()
1868 vec = base->vectors + idx; in collect_expired_timers()
1886 static int next_pending_bucket(struct timer_base *base, unsigned offset, in next_pending_bucket() argument
1892 pos = find_next_bit(base->pending_map, end, start); in next_pending_bucket()
1896 pos = find_next_bit(base->pending_map, start, offset); in next_pending_bucket()
1906 static void timer_recalc_next_expiry(struct timer_base *base) in timer_recalc_next_expiry() argument
1911 next = base->clk + NEXT_TIMER_MAX_DELTA; in timer_recalc_next_expiry()
1912 clk = base->clk; in timer_recalc_next_expiry()
1914 int pos = next_pending_bucket(base, offset, clk & LVL_MASK); in timer_recalc_next_expiry()
1972 WRITE_ONCE(base->next_expiry, next); in timer_recalc_next_expiry()
1973 base->next_expiry_recalc = false; in timer_recalc_next_expiry()
1974 base->timers_pending = !(next == base->clk + NEXT_TIMER_MAX_DELTA); in timer_recalc_next_expiry()
2011 static unsigned long next_timer_interrupt(struct timer_base *base, in next_timer_interrupt() argument
2014 if (base->next_expiry_recalc) in next_timer_interrupt()
2015 timer_recalc_next_expiry(base); in next_timer_interrupt()
2025 if (!base->timers_pending) in next_timer_interrupt()
2026 WRITE_ONCE(base->next_expiry, basej + NEXT_TIMER_MAX_DELTA); in next_timer_interrupt()
2028 return base->next_expiry; in next_timer_interrupt()
2173 static void __run_timer_base(struct timer_base *base);
2183 struct timer_base *base = per_cpu_ptr(&timer_bases[BASE_GLOBAL], cpu); in timer_expire_remote() local
2185 __run_timer_base(base); in timer_expire_remote()
2392 static inline void __run_timers(struct timer_base *base) in __run_timers() argument
2397 lockdep_assert_held(&base->lock); in __run_timers()
2399 if (base->running_timer) in __run_timers()
2402 while (time_after_eq(jiffies, base->clk) && in __run_timers()
2403 time_after_eq(jiffies, base->next_expiry)) { in __run_timers()
2404 levels = collect_expired_timers(base, heads); in __run_timers()
2412 WARN_ON_ONCE(!levels && !base->next_expiry_recalc in __run_timers()
2413 && base->timers_pending); in __run_timers()
2418 base->clk++; in __run_timers()
2419 timer_recalc_next_expiry(base); in __run_timers()
2422 expire_timers(base, heads + levels); in __run_timers()
2426 static void __run_timer_base(struct timer_base *base) in __run_timer_base() argument
2429 if (time_before(jiffies, READ_ONCE(base->next_expiry))) in __run_timer_base()
2432 timer_base_lock_expiry(base); in __run_timer_base()
2433 raw_spin_lock_irq(&base->lock); in __run_timer_base()
2434 __run_timers(base); in __run_timer_base()
2435 raw_spin_unlock_irq(&base->lock); in __run_timer_base()
2436 timer_base_unlock_expiry(base); in __run_timer_base()
2441 struct timer_base *base = this_cpu_ptr(&timer_bases[index]); in run_timer_base() local
2443 __run_timer_base(base); in run_timer_base()
2466 struct timer_base *base = this_cpu_ptr(&timer_bases[BASE_LOCAL]); in run_local_timers() local
2470 for (int i = 0; i < NR_BASES; i++, base++) { in run_local_timers()
2504 if (time_after_eq(jiffies, READ_ONCE(base->next_expiry)) || in run_local_timers()
2684 struct timer_base *base; in timers_prepare_cpu() local
2688 base = per_cpu_ptr(&timer_bases[b], cpu); in timers_prepare_cpu()
2689 base->clk = jiffies; in timers_prepare_cpu()
2690 base->next_expiry = base->clk + NEXT_TIMER_MAX_DELTA; in timers_prepare_cpu()
2691 base->next_expiry_recalc = false; in timers_prepare_cpu()
2692 base->timers_pending = false; in timers_prepare_cpu()
2693 base->is_idle = false; in timers_prepare_cpu()
2737 struct timer_base *base; in init_timer_cpu() local
2741 base = per_cpu_ptr(&timer_bases[i], cpu); in init_timer_cpu()
2742 base->cpu = cpu; in init_timer_cpu()
2743 raw_spin_lock_init(&base->lock); in init_timer_cpu()
2744 base->clk = jiffies; in init_timer_cpu()
2745 base->next_expiry = base->clk + NEXT_TIMER_MAX_DELTA; in init_timer_cpu()
2746 timer_base_init_expiry_lock(base); in init_timer_cpu()