1 #ifndef _LINUX_PSI_TYPES_H 2 #define _LINUX_PSI_TYPES_H 3 4 #include <linux/kthread.h> 5 #include <linux/seqlock.h> 6 #include <linux/types.h> 7 #include <linux/kref.h> 8 #include <linux/wait.h> 9 10 #ifdef CONFIG_PSI 11 12 /* Tracked task states */ 13 enum psi_task_count { 14 NR_IOWAIT, 15 NR_MEMSTALL, 16 NR_RUNNING, 17 /* 18 * This can't have values other than 0 or 1 and could be 19 * implemented as a bit flag. But for now we still have room 20 * in the first cacheline of psi_group_cpu, and this way we 21 * don't have to special case any state tracking for it. 22 */ 23 NR_ONCPU, 24 NR_PSI_TASK_COUNTS = 4, 25 }; 26 27 /* Task state bitmasks */ 28 #define TSK_IOWAIT (1 << NR_IOWAIT) 29 #define TSK_MEMSTALL (1 << NR_MEMSTALL) 30 #define TSK_RUNNING (1 << NR_RUNNING) 31 #define TSK_ONCPU (1 << NR_ONCPU) 32 33 /* Resources that workloads could be stalled on */ 34 enum psi_res { 35 PSI_IO, 36 PSI_MEM, 37 PSI_CPU, 38 NR_PSI_RESOURCES = 3, 39 }; 40 41 /* 42 * Pressure states for each resource: 43 * 44 * SOME: Stalled tasks & working tasks 45 * FULL: Stalled tasks & no working tasks 46 */ 47 enum psi_states { 48 PSI_IO_SOME, 49 PSI_IO_FULL, 50 PSI_MEM_SOME, 51 PSI_MEM_FULL, 52 PSI_CPU_SOME, 53 /* Only per-CPU, to weigh the CPU in the global average: */ 54 PSI_NONIDLE, 55 NR_PSI_STATES = 6, 56 }; 57 58 enum psi_aggregators { 59 PSI_AVGS = 0, 60 PSI_POLL, 61 NR_PSI_AGGREGATORS, 62 }; 63 64 struct psi_group_cpu { 65 /* 1st cacheline updated by the scheduler */ 66 67 /* Aggregator needs to know of concurrent changes */ 68 seqcount_t seq ____cacheline_aligned_in_smp; 69 70 /* States of the tasks belonging to this group */ 71 unsigned int tasks[NR_PSI_TASK_COUNTS]; 72 73 /* Aggregate pressure state derived from the tasks */ 74 u32 state_mask; 75 76 /* Period time sampling buckets for each state of interest (ns) */ 77 u32 times[NR_PSI_STATES]; 78 79 /* Time of last task change in this group (rq_clock) */ 80 u64 state_start; 81 82 /* 2nd cacheline updated by the aggregator */ 83 84 /* Delta detection against the sampling buckets */ 85 u32 times_prev[NR_PSI_AGGREGATORS][NR_PSI_STATES] 86 ____cacheline_aligned_in_smp; 87 }; 88 89 /* PSI growth tracking window */ 90 struct psi_window { 91 /* Window size in ns */ 92 u64 size; 93 94 /* Start time of the current window in ns */ 95 u64 start_time; 96 97 /* Value at the start of the window */ 98 u64 start_value; 99 100 /* Value growth in the previous window */ 101 u64 prev_growth; 102 }; 103 104 struct psi_trigger { 105 /* PSI state being monitored by the trigger */ 106 enum psi_states state; 107 108 /* User-spacified threshold in ns */ 109 u64 threshold; 110 111 /* List node inside triggers list */ 112 struct list_head node; 113 114 /* Backpointer needed during trigger destruction */ 115 struct psi_group *group; 116 117 /* Wait queue for polling */ 118 wait_queue_head_t event_wait; 119 120 /* Pending event flag */ 121 int event; 122 123 /* Tracking window */ 124 struct psi_window win; 125 126 /* 127 * Time last event was generated. Used for rate-limiting 128 * events to one per window 129 */ 130 u64 last_event_time; 131 }; 132 133 struct psi_group { 134 /* Protects data used by the aggregator */ 135 struct mutex avgs_lock; 136 137 /* Per-cpu task state & time tracking */ 138 struct psi_group_cpu __percpu *pcpu; 139 140 /* Running pressure averages */ 141 u64 avg_total[NR_PSI_STATES - 1]; 142 u64 avg_last_update; 143 u64 avg_next_update; 144 145 /* Aggregator work control */ 146 struct delayed_work avgs_work; 147 148 /* Total stall times and sampled pressure averages */ 149 u64 total[NR_PSI_AGGREGATORS][NR_PSI_STATES - 1]; 150 unsigned long avg[NR_PSI_STATES - 1][3]; 151 152 /* Monitor work control */ 153 struct task_struct __rcu *poll_task; 154 struct timer_list poll_timer; 155 wait_queue_head_t poll_wait; 156 atomic_t poll_wakeup; 157 158 /* Protects data used by the monitor */ 159 struct mutex trigger_lock; 160 161 /* Configured polling triggers */ 162 struct list_head triggers; 163 u32 nr_triggers[NR_PSI_STATES - 1]; 164 u32 poll_states; 165 u64 poll_min_period; 166 167 /* Total stall times at the start of monitor activation */ 168 u64 polling_total[NR_PSI_STATES - 1]; 169 u64 polling_next_update; 170 u64 polling_until; 171 }; 172 173 #else /* CONFIG_PSI */ 174 175 struct psi_group { }; 176 177 #endif /* CONFIG_PSI */ 178 179 #endif /* _LINUX_PSI_TYPES_H */ 180