• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 /*
2  * GPL HEADER START
3  *
4  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
5  *
6  * This program is free software; you can redistribute it and/or modify
7  * it under the terms of the GNU General Public License version 2 only,
8  * as published by the Free Software Foundation.
9  *
10  * This program is distributed in the hope that it will be useful, but
11  * WITHOUT ANY WARRANTY; without even the implied warranty of
12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
13  * General Public License version 2 for more details (a copy is included
14  * in the LICENSE file that accompanied this code).
15  *
16  * You should have received a copy of the GNU General Public License
17  * version 2 along with this program; If not, see
18  * http://www.sun.com/software/products/lustre/docs/GPLv2.pdf
19  *
20  * Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa Clara,
21  * CA 95054 USA or visit www.sun.com if you need additional information or
22  * have any questions.
23  *
24  * GPL HEADER END
25  */
26 /*
27  * Copyright (c) 2007, 2010, Oracle and/or its affiliates. All rights reserved.
28  * Use is subject to license terms.
29  *
30  * Copyright (c) 2011, Intel Corporation.
31  */
32 /*
33  * This file is part of Lustre, http://www.lustre.org/
34  * Lustre is a trademark of Sun Microsystems, Inc.
35  *
36  * lnet/include/lnet/lnetst.h
37  *
38  * Author: Liang Zhen <liangzhen@clusterfs.com>
39  */
40 
41 #ifndef __LNET_ST_H__
42 #define __LNET_ST_H__
43 
44 #include "../libcfs/libcfs.h"
45 #include "lnet.h"
46 #include "lib-types.h"
47 
48 #define LST_FEAT_NONE		(0)
49 #define LST_FEAT_BULK_LEN	(1 << 0)	/* enable variable page size */
50 
51 #define LST_FEATS_EMPTY		(LST_FEAT_NONE)
52 #define LST_FEATS_MASK		(LST_FEAT_NONE | LST_FEAT_BULK_LEN)
53 
54 #define LST_NAME_SIZE	   32	      /* max name buffer length */
55 
56 #define LSTIO_DEBUG	     0xC00	   /* debug */
57 #define LSTIO_SESSION_NEW       0xC01	   /* create session */
58 #define LSTIO_SESSION_END       0xC02	   /* end session */
59 #define LSTIO_SESSION_INFO      0xC03	   /* query session */
60 #define LSTIO_GROUP_ADD	 0xC10	   /* add group */
61 #define LSTIO_GROUP_LIST	0xC11	   /* list all groups in session */
62 #define LSTIO_GROUP_INFO	0xC12	   /* query default information of specified group */
63 #define LSTIO_GROUP_DEL	 0xC13	   /* delete group */
64 #define LSTIO_NODES_ADD	 0xC14	   /* add nodes to specified group */
65 #define LSTIO_GROUP_UPDATE      0xC15	   /* update group */
66 #define LSTIO_BATCH_ADD	 0xC20	   /* add batch */
67 #define LSTIO_BATCH_START       0xC21	   /* start batch */
68 #define LSTIO_BATCH_STOP	0xC22	   /* stop batch */
69 #define LSTIO_BATCH_DEL	 0xC23	   /* delete batch */
70 #define LSTIO_BATCH_LIST	0xC24	   /* show all batches in the session */
71 #define LSTIO_BATCH_INFO	0xC25	   /* show defail of specified batch */
72 #define LSTIO_TEST_ADD	  0xC26	   /* add test (to batch) */
73 #define LSTIO_BATCH_QUERY       0xC27	   /* query batch status */
74 #define LSTIO_STAT_QUERY	0xC30	   /* get stats */
75 
76 typedef struct {
77 	lnet_nid_t	      ses_nid;		/* nid of console node */
78 	__u64		   ses_stamp;	      /* time stamp */
79 } lst_sid_t;					    /*** session id */
80 
81 extern lst_sid_t LST_INVALID_SID;
82 
83 typedef struct {
84 	__u64		   bat_id;		 /* unique id in session */
85 } lst_bid_t;					    /*** batch id (group of tests) */
86 
87 /* Status of test node */
88 #define LST_NODE_ACTIVE	 0x1		     /* node in this session */
89 #define LST_NODE_BUSY	   0x2		     /* node is taken by other session */
90 #define LST_NODE_DOWN	   0x4		     /* node is down */
91 #define LST_NODE_UNKNOWN	0x8		     /* node not in session */
92 
93 typedef struct {
94 	lnet_process_id_t       nde_id;		 /* id of node */
95 	int		     nde_state;	      /* state of node */
96 } lstcon_node_ent_t;				    /*** node entry, for list_group command */
97 
98 typedef struct {
99 	int		     nle_nnode;	      /* # of nodes */
100 	int		     nle_nactive;	    /* # of active nodes */
101 	int		     nle_nbusy;	      /* # of busy nodes */
102 	int		     nle_ndown;	      /* # of down nodes */
103 	int		     nle_nunknown;	   /* # of unknown nodes */
104 } lstcon_ndlist_ent_t;				  /*** node_list entry, for list_batch command */
105 
106 typedef struct {
107 	int		     tse_type;	       /* test type */
108 	int		     tse_loop;	       /* loop count */
109 	int		     tse_concur;	     /* concurrency of test */
110 } lstcon_test_ent_t;				    /*** test summary entry, for list_batch command */
111 
112 typedef struct {
113 	int		     bae_state;	      /* batch status */
114 	int		     bae_timeout;	    /* batch timeout */
115 	int		     bae_ntest;	      /* # of tests in the batch */
116 } lstcon_batch_ent_t;				   /*** batch summary entry, for list_batch command */
117 
118 typedef struct {
119 	lstcon_ndlist_ent_t     tbe_cli_nle;	    /* client (group) node_list entry */
120 	lstcon_ndlist_ent_t     tbe_srv_nle;	    /* server (group) node_list entry */
121 	union {
122 		lstcon_test_ent_t  tbe_test;	    /* test entry */
123 		lstcon_batch_ent_t tbe_batch;	   /* batch entry */
124 	} u;
125 } lstcon_test_batch_ent_t;			      /*** test/batch verbose information entry,
126 							 *** for list_batch command */
127 
128 typedef struct {
129 	struct list_head	      rpe_link;	       /* link chain */
130 	lnet_process_id_t       rpe_peer;	       /* peer's id */
131 	struct timeval	  rpe_stamp;	      /* time stamp of RPC */
132 	int		     rpe_state;	      /* peer's state */
133 	int		     rpe_rpc_errno;	  /* RPC errno */
134 
135 	lst_sid_t	       rpe_sid;		/* peer's session id */
136 	int		     rpe_fwk_errno;	  /* framework errno */
137 	int		     rpe_priv[4];	    /* private data */
138 	char		    rpe_payload[0];	 /* private reply payload */
139 } lstcon_rpc_ent_t;
140 
141 typedef struct {
142 	int		     trs_rpc_stat[4];	/* RPCs stat (0: total, 1: failed, 2: finished, 4: reserved */
143 	int		     trs_rpc_errno;	  /* RPC errno */
144 	int		     trs_fwk_stat[8];	/* framework stat */
145 	int		     trs_fwk_errno;	  /* errno of the first remote error */
146 	void		   *trs_fwk_private;	/* private framework stat */
147 } lstcon_trans_stat_t;
148 
149 static inline int
lstcon_rpc_stat_total(lstcon_trans_stat_t * stat,int inc)150 lstcon_rpc_stat_total(lstcon_trans_stat_t *stat, int inc)
151 {
152 	return inc ? ++stat->trs_rpc_stat[0] : stat->trs_rpc_stat[0];
153 }
154 
155 static inline int
lstcon_rpc_stat_success(lstcon_trans_stat_t * stat,int inc)156 lstcon_rpc_stat_success(lstcon_trans_stat_t *stat, int inc)
157 {
158 	return inc ? ++stat->trs_rpc_stat[1] : stat->trs_rpc_stat[1];
159 }
160 
161 static inline int
lstcon_rpc_stat_failure(lstcon_trans_stat_t * stat,int inc)162 lstcon_rpc_stat_failure(lstcon_trans_stat_t *stat, int inc)
163 {
164 	return inc ? ++stat->trs_rpc_stat[2] : stat->trs_rpc_stat[2];
165 }
166 
167 static inline int
lstcon_sesop_stat_success(lstcon_trans_stat_t * stat,int inc)168 lstcon_sesop_stat_success(lstcon_trans_stat_t *stat, int inc)
169 {
170 	return inc ? ++stat->trs_fwk_stat[0] : stat->trs_fwk_stat[0];
171 }
172 
173 static inline int
lstcon_sesop_stat_failure(lstcon_trans_stat_t * stat,int inc)174 lstcon_sesop_stat_failure(lstcon_trans_stat_t *stat, int inc)
175 {
176 	return inc ? ++stat->trs_fwk_stat[1] : stat->trs_fwk_stat[1];
177 }
178 
179 static inline int
lstcon_sesqry_stat_active(lstcon_trans_stat_t * stat,int inc)180 lstcon_sesqry_stat_active(lstcon_trans_stat_t *stat, int inc)
181 {
182 	return inc ? ++stat->trs_fwk_stat[0] : stat->trs_fwk_stat[0];
183 }
184 
185 static inline int
lstcon_sesqry_stat_busy(lstcon_trans_stat_t * stat,int inc)186 lstcon_sesqry_stat_busy(lstcon_trans_stat_t *stat, int inc)
187 {
188 	return inc ? ++stat->trs_fwk_stat[1] : stat->trs_fwk_stat[1];
189 }
190 
191 static inline int
lstcon_sesqry_stat_unknown(lstcon_trans_stat_t * stat,int inc)192 lstcon_sesqry_stat_unknown(lstcon_trans_stat_t *stat, int inc)
193 {
194 	return inc ? ++stat->trs_fwk_stat[2] : stat->trs_fwk_stat[2];
195 }
196 
197 static inline int
lstcon_tsbop_stat_success(lstcon_trans_stat_t * stat,int inc)198 lstcon_tsbop_stat_success(lstcon_trans_stat_t *stat, int inc)
199 {
200 	return inc ? ++stat->trs_fwk_stat[0] : stat->trs_fwk_stat[0];
201 }
202 
203 static inline int
lstcon_tsbop_stat_failure(lstcon_trans_stat_t * stat,int inc)204 lstcon_tsbop_stat_failure(lstcon_trans_stat_t *stat, int inc)
205 {
206 	return inc ? ++stat->trs_fwk_stat[1] : stat->trs_fwk_stat[1];
207 }
208 
209 static inline int
lstcon_tsbqry_stat_idle(lstcon_trans_stat_t * stat,int inc)210 lstcon_tsbqry_stat_idle(lstcon_trans_stat_t *stat, int inc)
211 {
212 	return inc ? ++stat->trs_fwk_stat[0] : stat->trs_fwk_stat[0];
213 }
214 
215 static inline int
lstcon_tsbqry_stat_run(lstcon_trans_stat_t * stat,int inc)216 lstcon_tsbqry_stat_run(lstcon_trans_stat_t *stat, int inc)
217 {
218 	return inc ? ++stat->trs_fwk_stat[1] : stat->trs_fwk_stat[1];
219 }
220 
221 static inline int
lstcon_tsbqry_stat_failure(lstcon_trans_stat_t * stat,int inc)222 lstcon_tsbqry_stat_failure(lstcon_trans_stat_t *stat, int inc)
223 {
224 	return inc ? ++stat->trs_fwk_stat[2] : stat->trs_fwk_stat[2];
225 }
226 
227 static inline int
lstcon_statqry_stat_success(lstcon_trans_stat_t * stat,int inc)228 lstcon_statqry_stat_success(lstcon_trans_stat_t *stat, int inc)
229 {
230 	return inc ? ++stat->trs_fwk_stat[0] : stat->trs_fwk_stat[0];
231 }
232 
233 static inline int
lstcon_statqry_stat_failure(lstcon_trans_stat_t * stat,int inc)234 lstcon_statqry_stat_failure(lstcon_trans_stat_t *stat, int inc)
235 {
236 	return inc ? ++stat->trs_fwk_stat[1] : stat->trs_fwk_stat[1];
237 }
238 
239 /* create a session */
240 typedef struct {
241 	int		     lstio_ses_key;	  /* IN: local key */
242 	int		     lstio_ses_timeout;      /* IN: session timeout */
243 	int		     lstio_ses_force;	/* IN: force create ? */
244 	/** IN: session features */
245 	unsigned		lstio_ses_feats;
246 	lst_sid_t	      *lstio_ses_idp;	  /* OUT: session id */
247 	int		     lstio_ses_nmlen;	/* IN: name length */
248 	char		   *lstio_ses_namep;	/* IN: session name */
249 } lstio_session_new_args_t;
250 
251 /* query current session */
252 typedef struct {
253 	lst_sid_t	      *lstio_ses_idp;	  /* OUT: session id */
254 	int		    *lstio_ses_keyp;	 /* OUT: local key */
255 	/** OUT: session features */
256 	unsigned	       *lstio_ses_featp;
257 	lstcon_ndlist_ent_t    *lstio_ses_ndinfo;       /* OUT: */
258 	int		     lstio_ses_nmlen;	/* IN: name length */
259 	char		   *lstio_ses_namep;	/* OUT: session name */
260 } lstio_session_info_args_t;
261 
262 /* delete a session */
263 typedef struct {
264 	int		     lstio_ses_key;	  /* IN: session key */
265 } lstio_session_end_args_t;
266 
267 #define LST_OPC_SESSION	 1
268 #define LST_OPC_GROUP	   2
269 #define LST_OPC_NODES	   3
270 #define LST_OPC_BATCHCLI	4
271 #define LST_OPC_BATCHSRV	5
272 
273 typedef struct {
274 	int		     lstio_dbg_key;	  /* IN: session key */
275 	int		     lstio_dbg_type;	 /* IN: debug sessin|batch|group|nodes list */
276 	int		     lstio_dbg_flags;	/* IN: reserved debug flags */
277 	int		     lstio_dbg_timeout;      /* IN: timeout of debug */
278 
279 	int		     lstio_dbg_nmlen;	/* IN: len of name */
280 	char		   *lstio_dbg_namep;	/* IN: name of group|batch */
281 	int		     lstio_dbg_count;	/* IN: # of test nodes to debug */
282 	lnet_process_id_t      *lstio_dbg_idsp;	 /* IN: id of test nodes */
283 	struct list_head	     *lstio_dbg_resultp;      /* OUT: list head of result buffer */
284 } lstio_debug_args_t;
285 
286 typedef struct {
287 	int		     lstio_grp_key;	  /* IN: session key */
288 	int		     lstio_grp_nmlen;	/* IN: name length */
289 	char		   *lstio_grp_namep;	/* IN: group name */
290 } lstio_group_add_args_t;
291 
292 typedef struct {
293 	int		     lstio_grp_key;	  /* IN: session key */
294 	int		     lstio_grp_nmlen;	/* IN: name length */
295 	char		   *lstio_grp_namep;	/* IN: group name */
296 } lstio_group_del_args_t;
297 
298 #define LST_GROUP_CLEAN	 1		       /* remove inactive nodes in the group */
299 #define LST_GROUP_REFRESH       2		       /* refresh inactive nodes in the group */
300 #define LST_GROUP_RMND	  3		       /* delete nodes from the group */
301 
302 typedef struct {
303 	int		     lstio_grp_key;	  /* IN: session key */
304 	int		     lstio_grp_opc;	  /* IN: OPC */
305 	int		     lstio_grp_args;	 /* IN: arguments */
306 	int		     lstio_grp_nmlen;	/* IN: name length */
307 	char		   *lstio_grp_namep;	/* IN: group name */
308 	int		     lstio_grp_count;	/* IN: # of nodes id */
309 	lnet_process_id_t      *lstio_grp_idsp;	 /* IN: array of nodes */
310 	struct list_head	     *lstio_grp_resultp;      /* OUT: list head of result buffer */
311 } lstio_group_update_args_t;
312 
313 typedef struct {
314 	int		     lstio_grp_key;	  /* IN: session key */
315 	int		     lstio_grp_nmlen;	/* IN: name length */
316 	char		   *lstio_grp_namep;	/* IN: group name */
317 	int		     lstio_grp_count;	/* IN: # of nodes */
318 	/** OUT: session features */
319 	unsigned	       *lstio_grp_featp;
320 	lnet_process_id_t      *lstio_grp_idsp;	 /* IN: nodes */
321 	struct list_head	     *lstio_grp_resultp;      /* OUT: list head of result buffer */
322 } lstio_group_nodes_args_t;
323 
324 typedef struct {
325 	int		     lstio_grp_key;	  /* IN: session key */
326 	int		     lstio_grp_idx;	  /* IN: group idx */
327 	int		     lstio_grp_nmlen;	/* IN: name len */
328 	char		   *lstio_grp_namep;	/* OUT: name */
329 } lstio_group_list_args_t;
330 
331 typedef struct {
332 	int		     lstio_grp_key;	  /* IN: session key */
333 	int		     lstio_grp_nmlen;	/* IN: name len */
334 	char		   *lstio_grp_namep;	/* IN: name */
335 	lstcon_ndlist_ent_t    *lstio_grp_entp;	 /* OUT: description of group */
336 
337 	int		    *lstio_grp_idxp;	 /* IN/OUT: node index */
338 	int		    *lstio_grp_ndentp;       /* IN/OUT: # of nodent */
339 	lstcon_node_ent_t      *lstio_grp_dentsp;       /* OUT: nodent array */
340 } lstio_group_info_args_t;
341 
342 #define LST_DEFAULT_BATCH       "batch"		 /* default batch name */
343 
344 typedef struct {
345 	int		     lstio_bat_key;	  /* IN: session key */
346 	int		     lstio_bat_nmlen;	/* IN: name length */
347 	char		   *lstio_bat_namep;	/* IN: batch name */
348 } lstio_batch_add_args_t;
349 
350 typedef struct {
351 	int		     lstio_bat_key;	  /* IN: session key */
352 	int		     lstio_bat_nmlen;	/* IN: name length */
353 	char		   *lstio_bat_namep;	/* IN: batch name */
354 } lstio_batch_del_args_t;
355 
356 typedef struct {
357 	int		     lstio_bat_key;	  /* IN: session key */
358 	int		     lstio_bat_timeout;      /* IN: timeout for the batch */
359 	int		     lstio_bat_nmlen;	/* IN: name length */
360 	char		   *lstio_bat_namep;	/* IN: batch name */
361 	struct list_head	     *lstio_bat_resultp;      /* OUT: list head of result buffer */
362 } lstio_batch_run_args_t;
363 
364 typedef struct {
365 	int		     lstio_bat_key;	  /* IN: session key */
366 	int		     lstio_bat_force;	/* IN: abort unfinished test RPC */
367 	int		     lstio_bat_nmlen;	/* IN: name length */
368 	char		   *lstio_bat_namep;	/* IN: batch name */
369 	struct list_head	     *lstio_bat_resultp;      /* OUT: list head of result buffer */
370 } lstio_batch_stop_args_t;
371 
372 typedef struct {
373 	int		     lstio_bat_key;	  /* IN: session key */
374 	int		     lstio_bat_testidx;      /* IN: test index */
375 	int		     lstio_bat_client;       /* IN: is test client? */
376 	int		     lstio_bat_timeout;      /* IN: timeout for waiting */
377 	int		     lstio_bat_nmlen;	/* IN: name length */
378 	char		   *lstio_bat_namep;	/* IN: batch name */
379 	struct list_head	     *lstio_bat_resultp;      /* OUT: list head of result buffer */
380 } lstio_batch_query_args_t;
381 
382 typedef struct {
383 	int		     lstio_bat_key;	  /* IN: session key */
384 	int		     lstio_bat_idx;	  /* IN: index */
385 	int		     lstio_bat_nmlen;	/* IN: name length */
386 	char		   *lstio_bat_namep;	/* IN: batch name */
387 } lstio_batch_list_args_t;
388 
389 typedef struct {
390 	int		     lstio_bat_key;	  /* IN: session key */
391 	int		     lstio_bat_nmlen;	/* IN: name length */
392 	char		   *lstio_bat_namep;	/* IN: name */
393 	int		     lstio_bat_server;       /* IN: query server or not */
394 	int		     lstio_bat_testidx;      /* IN: test index */
395 	lstcon_test_batch_ent_t *lstio_bat_entp;	/* OUT: batch ent */
396 
397 	int		    *lstio_bat_idxp;	 /* IN/OUT: index of node */
398 	int		    *lstio_bat_ndentp;       /* IN/OUT: # of nodent */
399 	lstcon_node_ent_t      *lstio_bat_dentsp;       /* array of nodent */
400 } lstio_batch_info_args_t;
401 
402 /* add stat in session */
403 typedef struct {
404 	int		     lstio_sta_key;	  /* IN: session key */
405 	int		     lstio_sta_timeout;      /* IN: timeout for stat request */
406 	int		     lstio_sta_nmlen;	/* IN: group name length */
407 	char		   *lstio_sta_namep;	/* IN: group name */
408 	int		     lstio_sta_count;	/* IN: # of pid */
409 	lnet_process_id_t      *lstio_sta_idsp;	 /* IN: pid */
410 	struct list_head	     *lstio_sta_resultp;      /* OUT: list head of result buffer */
411 } lstio_stat_args_t;
412 
413 typedef enum {
414 	LST_TEST_BULK   = 1,
415 	LST_TEST_PING   = 2
416 } lst_test_type_t;
417 
418 /* create a test in a batch */
419 #define LST_MAX_CONCUR	  1024		    /* Max concurrency of test */
420 
421 typedef struct {
422 	int		     lstio_tes_key;	  /* IN: session key */
423 	int		     lstio_tes_bat_nmlen;    /* IN: batch name len */
424 	char		   *lstio_tes_bat_name;     /* IN: batch name */
425 	int		     lstio_tes_type;	 /* IN: test type */
426 	int		     lstio_tes_oneside;      /* IN: one sided test */
427 	int		     lstio_tes_loop;	 /* IN: loop count */
428 	int		     lstio_tes_concur;       /* IN: concurrency */
429 
430 	int		     lstio_tes_dist;	 /* IN: node distribution in destination groups */
431 	int		     lstio_tes_span;	 /* IN: node span in destination groups */
432 	int		     lstio_tes_sgrp_nmlen;   /* IN: source group name length */
433 	char		   *lstio_tes_sgrp_name;    /* IN: group name */
434 	int		     lstio_tes_dgrp_nmlen;   /* IN: destination group name length */
435 	char		   *lstio_tes_dgrp_name;    /* IN: group name */
436 
437 	int		     lstio_tes_param_len;    /* IN: param buffer len */
438 	void		   *lstio_tes_param;	/* IN: parameter for specified test:
439 							       lstio_bulk_param_t,
440 							       lstio_ping_param_t,
441 							       ... more */
442 	int		    *lstio_tes_retp;	 /* OUT: private returned value */
443 	struct list_head	     *lstio_tes_resultp;      /* OUT: list head of result buffer */
444 } lstio_test_args_t;
445 
446 typedef enum {
447 	LST_BRW_READ    = 1,
448 	LST_BRW_WRITE   = 2
449 } lst_brw_type_t;
450 
451 typedef enum {
452 	LST_BRW_CHECK_NONE   = 1,
453 	LST_BRW_CHECK_SIMPLE = 2,
454 	LST_BRW_CHECK_FULL   = 3
455 } lst_brw_flags_t;
456 
457 typedef struct {
458 	int		     blk_opc;		/* bulk operation code */
459 	int		     blk_size;	       /* size (bytes) */
460 	int		     blk_time;	       /* time of running the test*/
461 	int		     blk_flags;	      /* reserved flags */
462 } lst_test_bulk_param_t;
463 
464 typedef struct {
465 	int		     png_size;	       /* size of ping message */
466 	int		     png_time;	       /* time */
467 	int		     png_loop;	       /* loop */
468 	int		     png_flags;	      /* reserved flags */
469 } lst_test_ping_param_t;
470 
471 /* more tests */
472 typedef struct {
473 	__u32 errors;
474 	__u32 rpcs_sent;
475 	__u32 rpcs_rcvd;
476 	__u32 rpcs_dropped;
477 	__u32 rpcs_expired;
478 	__u64 bulk_get;
479 	__u64 bulk_put;
480 } WIRE_ATTR srpc_counters_t;
481 
482 typedef struct {
483 	/** milliseconds since current session started */
484 	__u32 running_ms;
485 	__u32 active_batches;
486 	__u32 zombie_sessions;
487 	__u32 brw_errors;
488 	__u32 ping_errors;
489 } WIRE_ATTR sfw_counters_t;
490 
491 #endif
492