1 // SPDX-License-Identifier: MIT or GPL-2.0-only
2
3 #if !defined(UBLKSRV_INTERNAL_H_)
4 #error "Never include <ublksrv_priv.h> directly; use <ublksrv.h> instead."
5 #endif
6
7 #ifndef UBLKSRV_PRIVATE_INC_H
8 #define UBLKSRV_PRIVATE_INC_H
9
10 #include <unistd.h>
11 #include <stdlib.h>
12 #include <stddef.h>
13 #include <signal.h>
14 #include <limits.h>
15 #include <pthread.h>
16 #include <string.h>
17 #include <sys/types.h>
18 #include <sys/eventfd.h>
19 #include <sys/epoll.h>
20 #include <sys/poll.h>
21
22 #include "ublk_cmd.h"
23 #include "ublksrv_utils.h"
24 #include "ublksrv.h"
25 #include "ublksrv_aio.h"
26
27
28 /* todo: relace the hardcode name with /dev/char/maj:min */
29 #ifdef UBLKC_PREFIX
30 #define UBLKC_DEV UBLKC_PREFIX "/ublkc"
31 #else
32 #define UBLKC_DEV "/dev/ublkc"
33 #endif
34 #define UBLKC_PATH_MAX 32
35
36 #ifdef __cplusplus
37 extern "C" {
38 #endif
39
40 struct ublksrv_ctrl_dev {
41 struct io_uring ring;
42
43 int ctrl_fd;
44 unsigned bs_shift;
45 struct ublksrv_ctrl_dev_info dev_info;
46
47 const char *tgt_type;
48 const struct ublksrv_tgt_type *tgt_ops;
49
50 /*
51 * default is UBLKSRV_RUN_DIR but can be specified via command line,
52 * pid file will be saved there
53 */
54 const char *run_dir;
55
56 union {
57 /* used by ->init_tgt() */
58 struct {
59 int tgt_argc;
60 char **tgt_argv;
61 };
62 /* used by ->recovery_tgt(), tgt_argc == -1 */
63 struct {
64 int padding;
65 const char *recovery_jbuf;
66 };
67 };
68
69 cpu_set_t *queues_cpuset;
70
71 unsigned long reserved[4];
72 };
73
74 struct ublk_io {
75 char *buf_addr;
76
77 #define UBLKSRV_NEED_FETCH_RQ (1UL << 0)
78 #define UBLKSRV_NEED_COMMIT_RQ_COMP (1UL << 1)
79 #define UBLKSRV_IO_FREE (1UL << 2)
80 #define UBLKSRV_NEED_GET_DATA (1UL << 3)
81 unsigned int flags;
82
83 /* result is updated after all target ios are done */
84 unsigned int result;
85
86 struct ublk_io_data data;
87 };
88
89 struct _ublksrv_queue {
90 /********** part of API, can't change ************/
91 int q_id;
92 int q_depth;
93
94 struct io_uring *ring_ptr;
95 struct _ublksrv_dev *dev;
96 void *private_data;
97 /*************************************************/
98
99 /*
100 * Read only by ublksrv daemon, setup via mmap on /dev/ublkcN.
101 *
102 * ublksrv_io_desc(iod) is stored in this buffer, so iod
103 * can be retrieved by request's tag directly.
104 *
105 * ublksrv writes the iod into this array, and notify ublksrv daemon
106 * by issued io_uring command beforehand.
107 * */
108 char *io_cmd_buf;
109 char *io_buf;
110
111 unsigned cmd_inflight, tgt_io_inflight; //obsolete
112 unsigned state;
113
114 /* eventfd */
115 int efd;
116
117 /* cache tgt ops */
118 const struct ublksrv_tgt_type *tgt_ops;
119
120 /*
121 * ring for submit io command to ublk driver, can only be issued
122 * from ublksrv daemon.
123 *
124 * ring depth == dev_info->queue_depth.
125 */
126 struct io_uring ring;
127
128 unsigned tid;
129
130 #define UBLKSRV_NR_CTX_BATCH 4
131 int nr_ctxs;
132 struct ublksrv_aio_ctx *ctxs[UBLKSRV_NR_CTX_BATCH];
133
134 unsigned long reserved[8];
135
136 struct ublk_io ios[0];
137 };
138
139 struct _ublksrv_dev {
140 //keep same with ublksrv_dev
141 /********** part of API, can't change ************/
142 struct ublksrv_tgt_info tgt;
143 /************************************************/
144
145 struct _ublksrv_queue *__queues[MAX_NR_HW_QUEUES];
146 char *io_buf_start;
147 pthread_t *thread;
148 int cdev_fd;
149 int pid_file_fd;
150
151 const struct ublksrv_ctrl_dev *ctrl_dev;
152 void *target_data;
153 int cq_depth;
154 int pad;
155
156 /* reserved isn't necessary any more */
157 unsigned long reserved[3];
158 };
159
160 #define local_to_tq(q) ((struct ublksrv_queue *)(q))
161 #define tq_to_local(q) ((struct _ublksrv_queue *)(q))
162
163 #define local_to_tdev(d) ((struct ublksrv_dev *)(d))
164 #define tdev_to_local(d) ((struct _ublksrv_dev *)(d))
165
ublk_is_unprivileged(const struct ublksrv_ctrl_dev * ctrl_dev)166 static inline bool ublk_is_unprivileged(const struct ublksrv_ctrl_dev *ctrl_dev)
167 {
168 return !!(ctrl_dev->dev_info.flags & UBLK_F_UNPRIVILEGED_DEV);
169 }
170
ublksrv_get_queue_affinity(const struct ublksrv_ctrl_dev * dev,int qid)171 static inline cpu_set_t *ublksrv_get_queue_affinity(
172 const struct ublksrv_ctrl_dev *dev, int qid)
173 {
174 unsigned char *buf = (unsigned char *)&dev->queues_cpuset[qid];
175
176 if (ublk_is_unprivileged(dev))
177 return (cpu_set_t *)&buf[UBLKC_PATH_MAX];
178
179 return &dev->queues_cpuset[qid];
180 }
181
ublksrv_mark_io_done(struct ublk_io * io,int res)182 static inline void ublksrv_mark_io_done(struct ublk_io *io, int res)
183 {
184 /*
185 * mark io done by target, so that ->ubq_daemon can commit its
186 * result and fetch new request via io_uring command.
187 */
188 io->flags |= (UBLKSRV_NEED_COMMIT_RQ_COMP | UBLKSRV_IO_FREE);
189
190 io->result = res;
191 }
192
ublksrv_io_done(struct ublk_io * io)193 static inline bool ublksrv_io_done(struct ublk_io *io)
194 {
195 return io->flags & UBLKSRV_IO_FREE;
196 }
197
198 int create_pid_file(const char *pid_file, int *pid_fd);
199
200 extern void ublksrv_build_cpu_str(char *buf, int len, const cpu_set_t *cpuset);
201
202 /* bit63: target io, bit62: eventfd data */
build_eventfd_data()203 static inline __u64 build_eventfd_data()
204 {
205 return 0x3ULL << 62;
206 }
207
is_eventfd_io(__u64 user_data)208 static inline int is_eventfd_io(__u64 user_data)
209 {
210 return (user_data & (1ULL << 62)) != 0;
211 }
212
is_target_io(__u64 user_data)213 static inline int is_target_io(__u64 user_data)
214 {
215 return (user_data & (1ULL << 63)) != 0;
216 }
217
ublksrv_setup_ring_params(struct io_uring_params * p,int cq_depth,unsigned flags)218 static inline void ublksrv_setup_ring_params(struct io_uring_params *p,
219 int cq_depth, unsigned flags)
220 {
221 memset(p, 0, sizeof(*p));
222 p->flags = flags | IORING_SETUP_CQSIZE;
223 p->cq_entries = cq_depth;
224 }
225
ublksrv_uring_get_sqe(struct io_uring * r,int idx,bool is_sqe128)226 static inline struct io_uring_sqe *ublksrv_uring_get_sqe(struct io_uring *r,
227 int idx, bool is_sqe128)
228 {
229 if (is_sqe128)
230 return &r->sq.sqes[idx << 1];
231 return &r->sq.sqes[idx];
232 }
233
ublksrv_get_sqe_cmd(struct io_uring_sqe * sqe)234 static inline void *ublksrv_get_sqe_cmd(struct io_uring_sqe *sqe)
235 {
236 return (void *)&sqe->addr3;
237 }
238
ublksrv_set_sqe_cmd_op(struct io_uring_sqe * sqe,__u32 cmd_op)239 static inline void ublksrv_set_sqe_cmd_op(struct io_uring_sqe *sqe, __u32 cmd_op)
240 {
241 __u32 *addr = (__u32 *)&sqe->off;
242
243 addr[0] = cmd_op;
244 addr[1] = 0;
245 }
246
247 /*
248 * ublksrv_aio_ctx is used to offload IO handling from ublksrv io_uring
249 * context.
250 *
251 * ublksrv_aio_ctx is bound with one single pthread which has to belong
252 * to same process of the io_uring where IO is originated, so we can
253 * support to handle IO from multiple queues of the same device. At
254 * default, ublksrv_aio_ctx supports to handle device wide aio or io
255 * offloading except for UBLKSRV_AIO_QUEUE_WIDE.
256 *
257 * Meantime ublksrv_aio_ctx can be created per each queue, and only handle
258 * IOs from this queue.
259 *
260 * The final io handling in the aio context depends on user's implementation,
261 * either sync or async IO submitting is supported.
262 */
263 struct ublksrv_aio_ctx {
264 struct ublksrv_aio_list submit;
265
266 /* per-queue completion list */
267 struct ublksrv_aio_list *complete;
268
269 int efd; //for wakeup us
270
271 #define UBLKSRV_AIO_QUEUE_WIDE (1U << 0)
272 unsigned int flags;
273 bool dead;
274
275 const struct ublksrv_dev *dev;
276
277 void *ctx_data;
278
279 unsigned long reserved[8];
280 };
281
282 #ifdef __cplusplus
283 }
284 #endif
285
286 #endif
287