1 /* SPDX-License-Identifier: GPL-2.0-only */ 2 /* 3 * padata.h - header for the padata parallelization interface 4 * 5 * Copyright (C) 2008, 2009 secunet Security Networks AG 6 * Copyright (C) 2008, 2009 Steffen Klassert <steffen.klassert@secunet.com> 7 */ 8 9 #ifndef PADATA_H 10 #define PADATA_H 11 12 #include <linux/compiler_types.h> 13 #include <linux/workqueue.h> 14 #include <linux/spinlock.h> 15 #include <linux/list.h> 16 #include <linux/notifier.h> 17 #include <linux/kobject.h> 18 19 #define PADATA_CPU_SERIAL 0x01 20 #define PADATA_CPU_PARALLEL 0x02 21 22 /** 23 * struct padata_priv - Embedded to the users data structure. 24 * 25 * @list: List entry, to attach to the padata lists. 26 * @pd: Pointer to the internal control structure. 27 * @cb_cpu: Callback cpu for serializatioon. 28 * @cpu: Cpu for parallelization. 29 * @seq_nr: Sequence number of the parallelized data object. 30 * @info: Used to pass information from the parallel to the serial function. 31 * @parallel: Parallel execution function. 32 * @serial: Serial complete function. 33 */ 34 struct padata_priv { 35 struct list_head list; 36 struct parallel_data *pd; 37 int cb_cpu; 38 int cpu; 39 unsigned int seq_nr; 40 int info; 41 void (*parallel)(struct padata_priv *padata); 42 void (*serial)(struct padata_priv *padata); 43 }; 44 45 /** 46 * struct padata_list 47 * 48 * @list: List head. 49 * @lock: List lock. 50 */ 51 struct padata_list { 52 struct list_head list; 53 spinlock_t lock; 54 }; 55 56 /** 57 * struct padata_serial_queue - The percpu padata serial queue 58 * 59 * @serial: List to wait for serialization after reordering. 60 * @work: work struct for serialization. 61 * @pd: Backpointer to the internal control structure. 62 */ 63 struct padata_serial_queue { 64 struct padata_list serial; 65 struct work_struct work; 66 struct parallel_data *pd; 67 }; 68 69 /** 70 * struct padata_parallel_queue - The percpu padata parallel queue 71 * 72 * @parallel: List to wait for parallelization. 73 * @reorder: List to wait for reordering after parallel processing. 74 * @serial: List to wait for serialization after reordering. 75 * @pwork: work struct for parallelization. 76 * @swork: work struct for serialization. 77 * @work: work struct for parallelization. 78 * @num_obj: Number of objects that are processed by this cpu. 79 */ 80 struct padata_parallel_queue { 81 struct padata_list parallel; 82 struct padata_list reorder; 83 struct work_struct work; 84 atomic_t num_obj; 85 }; 86 87 /** 88 * struct padata_cpumask - The cpumasks for the parallel/serial workers 89 * 90 * @pcpu: cpumask for the parallel workers. 91 * @cbcpu: cpumask for the serial (callback) workers. 92 */ 93 struct padata_cpumask { 94 cpumask_var_t pcpu; 95 cpumask_var_t cbcpu; 96 }; 97 98 /** 99 * struct parallel_data - Internal control structure, covers everything 100 * that depends on the cpumask in use. 101 * 102 * @sh: padata_shell object. 103 * @pqueue: percpu padata queues used for parallelization. 104 * @squeue: percpu padata queues used for serialuzation. 105 * @reorder_objects: Number of objects waiting in the reorder queues. 106 * @refcnt: Number of objects holding a reference on this parallel_data. 107 * @max_seq_nr: Maximal used sequence number. 108 * @processed: Number of already processed objects. 109 * @cpu: Next CPU to be processed. 110 * @cpumask: The cpumasks in use for parallel and serial workers. 111 * @reorder_work: work struct for reordering. 112 * @lock: Reorder lock. 113 */ 114 struct parallel_data { 115 struct padata_shell *ps; 116 struct padata_parallel_queue __percpu *pqueue; 117 struct padata_serial_queue __percpu *squeue; 118 atomic_t reorder_objects; 119 atomic_t refcnt; 120 atomic_t seq_nr; 121 unsigned int processed; 122 int cpu; 123 struct padata_cpumask cpumask; 124 struct work_struct reorder_work; 125 spinlock_t lock ____cacheline_aligned; 126 }; 127 128 /** 129 * struct padata_shell - Wrapper around struct parallel_data, its 130 * purpose is to allow the underlying control structure to be replaced 131 * on the fly using RCU. 132 * 133 * @pinst: padat instance. 134 * @pd: Actual parallel_data structure which may be substituted on the fly. 135 * @opd: Pointer to old pd to be freed by padata_replace. 136 * @list: List entry in padata_instance list. 137 */ 138 struct padata_shell { 139 struct padata_instance *pinst; 140 struct parallel_data __rcu *pd; 141 struct parallel_data *opd; 142 struct list_head list; 143 }; 144 145 /** 146 * struct padata_instance - The overall control structure. 147 * 148 * @cpu_online_node: Linkage for CPU online callback. 149 * @cpu_dead_node: Linkage for CPU offline callback. 150 * @parallel_wq: The workqueue used for parallel work. 151 * @serial_wq: The workqueue used for serial work. 152 * @pslist: List of padata_shell objects attached to this instance. 153 * @cpumask: User supplied cpumasks for parallel and serial works. 154 * @rcpumask: Actual cpumasks based on user cpumask and cpu_online_mask. 155 * @omask: Temporary storage used to compute the notification mask. 156 * @cpumask_change_notifier: Notifiers chain for user-defined notify 157 * callbacks that will be called when either @pcpu or @cbcpu 158 * or both cpumasks change. 159 * @kobj: padata instance kernel object. 160 * @lock: padata instance lock. 161 * @flags: padata flags. 162 */ 163 struct padata_instance { 164 struct hlist_node cpu_online_node; 165 struct hlist_node cpu_dead_node; 166 struct workqueue_struct *parallel_wq; 167 struct workqueue_struct *serial_wq; 168 struct list_head pslist; 169 struct padata_cpumask cpumask; 170 struct padata_cpumask rcpumask; 171 cpumask_var_t omask; 172 struct blocking_notifier_head cpumask_change_notifier; 173 struct kobject kobj; 174 struct mutex lock; 175 u8 flags; 176 #define PADATA_INIT 1 177 #define PADATA_RESET 2 178 #define PADATA_INVALID 4 179 }; 180 181 extern struct padata_instance *padata_alloc_possible(const char *name); 182 extern void padata_free(struct padata_instance *pinst); 183 extern struct padata_shell *padata_alloc_shell(struct padata_instance *pinst); 184 extern void padata_free_shell(struct padata_shell *ps); 185 extern int padata_do_parallel(struct padata_shell *ps, 186 struct padata_priv *padata, int *cb_cpu); 187 extern void padata_do_serial(struct padata_priv *padata); 188 extern int padata_set_cpumask(struct padata_instance *pinst, int cpumask_type, 189 cpumask_var_t cpumask); 190 extern int padata_start(struct padata_instance *pinst); 191 extern void padata_stop(struct padata_instance *pinst); 192 extern int padata_register_cpumask_notifier(struct padata_instance *pinst, 193 struct notifier_block *nblock); 194 extern int padata_unregister_cpumask_notifier(struct padata_instance *pinst, 195 struct notifier_block *nblock); 196 #endif 197