1 /*
2 * GPL HEADER START
3 *
4 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
5 *
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License version 2 only,
8 * as published by the Free Software Foundation.
9 *
10 * This program is distributed in the hope that it will be useful, but
11 * WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 * General Public License version 2 for more details (a copy is included
14 * in the LICENSE file that accompanied this code).
15 *
16 * You should have received a copy of the GNU General Public License
17 * version 2 along with this program; If not, see
18 * http://www.sun.com/software/products/lustre/docs/GPLv2.pdf
19 *
20 * Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa Clara,
21 * CA 95054 USA or visit www.sun.com if you need additional information or
22 * have any questions.
23 *
24 * GPL HEADER END
25 */
26 /*
27 * Copyright (c) 2002, 2010, Oracle and/or its affiliates. All rights reserved.
28 * Use is subject to license terms.
29 *
30 * Copyright (c) 2011, 2012, Intel Corporation.
31 */
32 /*
33 * This file is part of Lustre, http://www.lustre.org/
34 * Lustre is a trademark of Sun Microsystems, Inc.
35 */
36 /** \defgroup obd_import PtlRPC import definitions
37 * Imports are client-side representation of remote obd target.
38 *
39 * @{
40 */
41
42 #ifndef __IMPORT_H
43 #define __IMPORT_H
44
45 /** \defgroup export export
46 *
47 * @{
48 */
49
50 #include "lustre_handles.h"
51 #include "lustre/lustre_idl.h"
52
53 /**
54 * Adaptive Timeout stuff
55 *
56 * @{
57 */
58 #define D_ADAPTTO D_OTHER
59 #define AT_BINS 4 /* "bin" means "N seconds of history" */
60 #define AT_FLG_NOHIST 0x1 /* use last reported value only */
61
62 struct adaptive_timeout {
63 time64_t at_binstart; /* bin start time */
64 unsigned int at_hist[AT_BINS]; /* timeout history bins */
65 unsigned int at_flags;
66 unsigned int at_current; /* current timeout value */
67 unsigned int at_worst_ever; /* worst-ever timeout value */
68 time64_t at_worst_time; /* worst-ever timeout timestamp */
69 spinlock_t at_lock;
70 };
71
72 struct ptlrpc_at_array {
73 struct list_head *paa_reqs_array; /** array to hold requests */
74 __u32 paa_size; /** the size of array */
75 __u32 paa_count; /** the total count of reqs */
76 time64_t paa_deadline; /** the earliest deadline of reqs */
77 __u32 *paa_reqs_count; /** the count of reqs in each entry */
78 };
79
80 #define IMP_AT_MAX_PORTALS 8
81 struct imp_at {
82 int iat_portal[IMP_AT_MAX_PORTALS];
83 struct adaptive_timeout iat_net_latency;
84 struct adaptive_timeout iat_service_estimate[IMP_AT_MAX_PORTALS];
85 };
86
87 /** @} */
88
89 /** Possible import states */
90 enum lustre_imp_state {
91 LUSTRE_IMP_CLOSED = 1,
92 LUSTRE_IMP_NEW = 2,
93 LUSTRE_IMP_DISCON = 3,
94 LUSTRE_IMP_CONNECTING = 4,
95 LUSTRE_IMP_REPLAY = 5,
96 LUSTRE_IMP_REPLAY_LOCKS = 6,
97 LUSTRE_IMP_REPLAY_WAIT = 7,
98 LUSTRE_IMP_RECOVER = 8,
99 LUSTRE_IMP_FULL = 9,
100 LUSTRE_IMP_EVICTED = 10,
101 };
102
103 /** Returns test string representation of numeric import state \a state */
ptlrpc_import_state_name(enum lustre_imp_state state)104 static inline char *ptlrpc_import_state_name(enum lustre_imp_state state)
105 {
106 static char *import_state_names[] = {
107 "<UNKNOWN>", "CLOSED", "NEW", "DISCONN",
108 "CONNECTING", "REPLAY", "REPLAY_LOCKS", "REPLAY_WAIT",
109 "RECOVER", "FULL", "EVICTED",
110 };
111
112 LASSERT (state <= LUSTRE_IMP_EVICTED);
113 return import_state_names[state];
114 }
115
116 /**
117 * List of import event types
118 */
119 enum obd_import_event {
120 IMP_EVENT_DISCON = 0x808001,
121 IMP_EVENT_INACTIVE = 0x808002,
122 IMP_EVENT_INVALIDATE = 0x808003,
123 IMP_EVENT_ACTIVE = 0x808004,
124 IMP_EVENT_OCD = 0x808005,
125 IMP_EVENT_DEACTIVATE = 0x808006,
126 IMP_EVENT_ACTIVATE = 0x808007,
127 };
128
129 /**
130 * Definition of import connection structure
131 */
132 struct obd_import_conn {
133 /** Item for linking connections together */
134 struct list_head oic_item;
135 /** Pointer to actual PortalRPC connection */
136 struct ptlrpc_connection *oic_conn;
137 /** uuid of remote side */
138 struct obd_uuid oic_uuid;
139 /**
140 * Time (64 bit jiffies) of last connection attempt on this connection
141 */
142 __u64 oic_last_attempt;
143 };
144
145 /* state history */
146 #define IMP_STATE_HIST_LEN 16
147 struct import_state_hist {
148 enum lustre_imp_state ish_state;
149 time64_t ish_time;
150 };
151
152 /**
153 * Definition of PortalRPC import structure.
154 * Imports are representing client-side view to remote target.
155 */
156 struct obd_import {
157 /** Local handle (== id) for this import. */
158 struct portals_handle imp_handle;
159 /** Reference counter */
160 atomic_t imp_refcount;
161 struct lustre_handle imp_dlm_handle; /* client's ldlm export */
162 /** Currently active connection */
163 struct ptlrpc_connection *imp_connection;
164 /** PortalRPC client structure for this import */
165 struct ptlrpc_client *imp_client;
166 /** List element for linking into pinger chain */
167 struct list_head imp_pinger_chain;
168 /** List element for linking into chain for destruction */
169 struct list_head imp_zombie_chain;
170
171 /**
172 * Lists of requests that are retained for replay, waiting for a reply,
173 * or waiting for recovery to complete, respectively.
174 * @{
175 */
176 struct list_head imp_replay_list;
177 struct list_head imp_sending_list;
178 struct list_head imp_delayed_list;
179 /** @} */
180
181 /**
182 * List of requests that are retained for committed open replay. Once
183 * open is committed, open replay request will be moved from the
184 * imp_replay_list into the imp_committed_list.
185 * The imp_replay_cursor is for accelerating searching during replay.
186 * @{
187 */
188 struct list_head imp_committed_list;
189 struct list_head *imp_replay_cursor;
190 /** @} */
191
192 /** obd device for this import */
193 struct obd_device *imp_obd;
194
195 /**
196 * some seciruty-related fields
197 * @{
198 */
199 struct ptlrpc_sec *imp_sec;
200 struct mutex imp_sec_mutex;
201 time64_t imp_sec_expire;
202 /** @} */
203
204 /** Wait queue for those who need to wait for recovery completion */
205 wait_queue_head_t imp_recovery_waitq;
206
207 /** Number of requests currently in-flight */
208 atomic_t imp_inflight;
209 /** Number of requests currently unregistering */
210 atomic_t imp_unregistering;
211 /** Number of replay requests inflight */
212 atomic_t imp_replay_inflight;
213 /** Number of currently happening import invalidations */
214 atomic_t imp_inval_count;
215 /** Numbner of request timeouts */
216 atomic_t imp_timeouts;
217 /** Current import state */
218 enum lustre_imp_state imp_state;
219 /** Last replay state */
220 enum lustre_imp_state imp_replay_state;
221 /** History of import states */
222 struct import_state_hist imp_state_hist[IMP_STATE_HIST_LEN];
223 int imp_state_hist_idx;
224 /** Current import generation. Incremented on every reconnect */
225 int imp_generation;
226 /** Incremented every time we send reconnection request */
227 __u32 imp_conn_cnt;
228 /**
229 * \see ptlrpc_free_committed remembers imp_generation value here
230 * after a check to save on unnecessary replay list iterations
231 */
232 int imp_last_generation_checked;
233 /** Last transno we replayed */
234 __u64 imp_last_replay_transno;
235 /** Last transno committed on remote side */
236 __u64 imp_peer_committed_transno;
237 /**
238 * \see ptlrpc_free_committed remembers last_transno since its last
239 * check here and if last_transno did not change since last run of
240 * ptlrpc_free_committed and import generation is the same, we can
241 * skip looking for requests to remove from replay list as optimisation
242 */
243 __u64 imp_last_transno_checked;
244 /**
245 * Remote export handle. This is how remote side knows what export
246 * we are talking to. Filled from response to connect request
247 */
248 struct lustre_handle imp_remote_handle;
249 /** When to perform next ping. time in jiffies. */
250 unsigned long imp_next_ping;
251 /** When we last successfully connected. time in 64bit jiffies */
252 __u64 imp_last_success_conn;
253
254 /** List of all possible connection for import. */
255 struct list_head imp_conn_list;
256 /**
257 * Current connection. \a imp_connection is imp_conn_current->oic_conn
258 */
259 struct obd_import_conn *imp_conn_current;
260
261 /** Protects flags, level, generation, conn_cnt, *_list */
262 spinlock_t imp_lock;
263
264 /* flags */
265 unsigned long imp_no_timeout:1, /* timeouts are disabled */
266 imp_invalid:1, /* evicted */
267 /* administratively disabled */
268 imp_deactive:1,
269 /* try to recover the import */
270 imp_replayable:1,
271 /* don't run recovery (timeout instead) */
272 imp_dlm_fake:1,
273 /* use 1/2 timeout on MDS' OSCs */
274 imp_server_timeout:1,
275 /* VBR: imp in delayed recovery */
276 imp_delayed_recovery:1,
277 /* VBR: if gap was found then no lock replays
278 */
279 imp_no_lock_replay:1,
280 /* recovery by versions was failed */
281 imp_vbr_failed:1,
282 /* force an immediate ping */
283 imp_force_verify:1,
284 /* force a scheduled ping */
285 imp_force_next_verify:1,
286 /* pingable */
287 imp_pingable:1,
288 /* resend for replay */
289 imp_resend_replay:1,
290 /* disable normal recovery, for test only. */
291 imp_no_pinger_recover:1,
292 /* need IR MNE swab */
293 imp_need_mne_swab:1,
294 /* import must be reconnected instead of
295 * chose new connection */
296 imp_force_reconnect:1,
297 /* import has tried to connect with server */
298 imp_connect_tried:1;
299 __u32 imp_connect_op;
300 struct obd_connect_data imp_connect_data;
301 __u64 imp_connect_flags_orig;
302 int imp_connect_error;
303
304 __u32 imp_msg_magic;
305 __u32 imp_msghdr_flags; /* adjusted based on server capability */
306
307 struct imp_at imp_at; /* adaptive timeout data */
308 time64_t imp_last_reply_time; /* for health check */
309 };
310
311 typedef void (*obd_import_callback)(struct obd_import *imp, void *closure,
312 int event, void *event_arg, void *cb_data);
313
314 /**
315 * Structure for import observer.
316 * It is possible to register "observer" on an import and every time
317 * something happens to an import (like connect/evict/disconnect)
318 * obderver will get its callback called with event type
319 */
320 struct obd_import_observer {
321 struct list_head oio_chain;
322 obd_import_callback oio_cb;
323 void *oio_cb_data;
324 };
325
326 void class_observe_import(struct obd_import *imp, obd_import_callback cb,
327 void *cb_data);
328 void class_unobserve_import(struct obd_import *imp, obd_import_callback cb,
329 void *cb_data);
330 void class_notify_import_observers(struct obd_import *imp, int event,
331 void *event_arg);
332
333 /* import.c */
at_est2timeout(unsigned int val)334 static inline unsigned int at_est2timeout(unsigned int val)
335 {
336 /* add an arbitrary minimum: 125% +5 sec */
337 return (val + (val >> 2) + 5);
338 }
339
at_timeout2est(unsigned int val)340 static inline unsigned int at_timeout2est(unsigned int val)
341 {
342 /* restore estimate value from timeout: e=4/5(t-5) */
343 LASSERT(val);
344 return (max((val << 2) / 5, 5U) - 4);
345 }
346
at_reset(struct adaptive_timeout * at,int val)347 static inline void at_reset(struct adaptive_timeout *at, int val)
348 {
349 spin_lock(&at->at_lock);
350 at->at_current = val;
351 at->at_worst_ever = val;
352 at->at_worst_time = ktime_get_real_seconds();
353 spin_unlock(&at->at_lock);
354 }
355
at_init(struct adaptive_timeout * at,int val,int flags)356 static inline void at_init(struct adaptive_timeout *at, int val, int flags)
357 {
358 memset(at, 0, sizeof(*at));
359 spin_lock_init(&at->at_lock);
360 at->at_flags = flags;
361 at_reset(at, val);
362 }
363
364 extern unsigned int at_min;
at_get(struct adaptive_timeout * at)365 static inline int at_get(struct adaptive_timeout *at)
366 {
367 return (at->at_current > at_min) ? at->at_current : at_min;
368 }
369
370 int at_measured(struct adaptive_timeout *at, unsigned int val);
371 int import_at_get_index(struct obd_import *imp, int portal);
372 extern unsigned int at_max;
373 #define AT_OFF (at_max == 0)
374
375 /* genops.c */
376 struct obd_export;
377 struct obd_import *class_exp2cliimp(struct obd_export *);
378
379 /** @} import */
380
381 #endif /* __IMPORT_H */
382
383 /** @} obd_import */
384