Lines Matching +full:dout +full:- +full:default
1 // SPDX-License-Identifier: GPL-2.0
71 WARN_ON(!rwsem_is_locked(&osdc->lock)); in verify_osdc_locked()
75 WARN_ON(!rwsem_is_wrlocked(&osdc->lock)); in verify_osdc_wrlocked()
79 struct ceph_osd_client *osdc = osd->o_osdc; in verify_osd_locked()
81 WARN_ON(!(mutex_is_locked(&osd->lock) && in verify_osd_locked()
82 rwsem_is_locked(&osdc->lock)) && in verify_osd_locked()
83 !rwsem_is_wrlocked(&osdc->lock)); in verify_osd_locked()
87 WARN_ON(!mutex_is_locked(&lreq->lock)); in verify_lreq_locked()
115 dout(" skipping last %llu, final file extent %llu~%llu\n", in calc_layout()
116 orig_len - *plen, off, *plen); in calc_layout()
119 dout("calc_layout objnum=%llx %llu~%llu\n", *objnum, *objoff, *objlen); in calc_layout()
126 osd_data->type = CEPH_OSD_DATA_TYPE_NONE; in ceph_osd_data_init()
136 osd_data->type = CEPH_OSD_DATA_TYPE_PAGES; in ceph_osd_data_pages_init()
137 osd_data->pages = pages; in ceph_osd_data_pages_init()
138 osd_data->length = length; in ceph_osd_data_pages_init()
139 osd_data->alignment = alignment; in ceph_osd_data_pages_init()
140 osd_data->pages_from_pool = pages_from_pool; in ceph_osd_data_pages_init()
141 osd_data->own_pages = own_pages; in ceph_osd_data_pages_init()
150 osd_data->type = CEPH_OSD_DATA_TYPE_PAGELIST; in ceph_osd_data_pagelist_init()
151 osd_data->pagelist = pagelist; in ceph_osd_data_pagelist_init()
159 osd_data->type = CEPH_OSD_DATA_TYPE_BIO; in ceph_osd_data_bio_init()
160 osd_data->bio_pos = *bio_pos; in ceph_osd_data_bio_init()
161 osd_data->bio_length = bio_length; in ceph_osd_data_bio_init()
169 osd_data->type = CEPH_OSD_DATA_TYPE_BVECS; in ceph_osd_data_bvecs_init()
170 osd_data->bvec_pos = *bvec_pos; in ceph_osd_data_bvecs_init()
171 osd_data->num_bvecs = num_bvecs; in ceph_osd_data_bvecs_init()
177 BUG_ON(which >= osd_req->r_num_ops); in osd_req_op_raw_data_in()
179 return &osd_req->r_ops[which].raw_data_in; in osd_req_op_raw_data_in()
285 osd_req->r_ops[which].cls.indata_len += pagelist->length; in osd_req_op_cls_request_data_pagelist()
286 osd_req->r_ops[which].indata_len += pagelist->length; in osd_req_op_cls_request_data_pagelist()
299 osd_req->r_ops[which].cls.indata_len += length; in osd_req_op_cls_request_data_pages()
300 osd_req->r_ops[which].indata_len += length; in osd_req_op_cls_request_data_pages()
317 osd_req->r_ops[which].cls.indata_len += bytes; in osd_req_op_cls_request_data_bvecs()
318 osd_req->r_ops[which].indata_len += bytes; in osd_req_op_cls_request_data_bvecs()
336 switch (osd_data->type) { in ceph_osd_data_length()
340 return osd_data->length; in ceph_osd_data_length()
342 return (u64)osd_data->pagelist->length; in ceph_osd_data_length()
345 return (u64)osd_data->bio_length; in ceph_osd_data_length()
348 return osd_data->bvec_pos.iter.bi_size; in ceph_osd_data_length()
349 default: in ceph_osd_data_length()
350 WARN(true, "unrecognized data type %d\n", (int)osd_data->type); in ceph_osd_data_length()
357 if (osd_data->type == CEPH_OSD_DATA_TYPE_PAGES && osd_data->own_pages) { in ceph_osd_data_release()
360 num_pages = calc_pages_for((u64)osd_data->alignment, in ceph_osd_data_release()
361 (u64)osd_data->length); in ceph_osd_data_release()
362 ceph_release_page_vector(osd_data->pages, num_pages); in ceph_osd_data_release()
363 } else if (osd_data->type == CEPH_OSD_DATA_TYPE_PAGELIST) { in ceph_osd_data_release()
364 ceph_pagelist_release(osd_data->pagelist); in ceph_osd_data_release()
374 BUG_ON(which >= osd_req->r_num_ops); in osd_req_op_data_release()
375 op = &osd_req->r_ops[which]; in osd_req_op_data_release()
377 switch (op->op) { in osd_req_op_data_release()
381 ceph_osd_data_release(&op->extent.osd_data); in osd_req_op_data_release()
384 ceph_osd_data_release(&op->cls.request_info); in osd_req_op_data_release()
385 ceph_osd_data_release(&op->cls.request_data); in osd_req_op_data_release()
386 ceph_osd_data_release(&op->cls.response_data); in osd_req_op_data_release()
390 ceph_osd_data_release(&op->xattr.osd_data); in osd_req_op_data_release()
393 ceph_osd_data_release(&op->raw_data_in); in osd_req_op_data_release()
396 ceph_osd_data_release(&op->notify_ack.request_data); in osd_req_op_data_release()
399 ceph_osd_data_release(&op->notify.request_data); in osd_req_op_data_release()
400 ceph_osd_data_release(&op->notify.response_data); in osd_req_op_data_release()
403 ceph_osd_data_release(&op->list_watchers.response_data); in osd_req_op_data_release()
406 ceph_osd_data_release(&op->copy_from.osd_data); in osd_req_op_data_release()
408 default: in osd_req_op_data_release()
414 * Assumes @t is zero-initialized.
418 ceph_oid_init(&t->base_oid); in target_init()
419 ceph_oloc_init(&t->base_oloc); in target_init()
420 ceph_oid_init(&t->target_oid); in target_init()
421 ceph_oloc_init(&t->target_oloc); in target_init()
423 ceph_osds_init(&t->acting); in target_init()
424 ceph_osds_init(&t->up); in target_init()
425 t->size = -1; in target_init()
426 t->min_size = -1; in target_init()
428 t->osd = CEPH_HOMELESS_OSD; in target_init()
434 ceph_oid_copy(&dest->base_oid, &src->base_oid); in target_copy()
435 ceph_oloc_copy(&dest->base_oloc, &src->base_oloc); in target_copy()
436 ceph_oid_copy(&dest->target_oid, &src->target_oid); in target_copy()
437 ceph_oloc_copy(&dest->target_oloc, &src->target_oloc); in target_copy()
439 dest->pgid = src->pgid; /* struct */ in target_copy()
440 dest->spgid = src->spgid; /* struct */ in target_copy()
441 dest->pg_num = src->pg_num; in target_copy()
442 dest->pg_num_mask = src->pg_num_mask; in target_copy()
443 ceph_osds_copy(&dest->acting, &src->acting); in target_copy()
444 ceph_osds_copy(&dest->up, &src->up); in target_copy()
445 dest->size = src->size; in target_copy()
446 dest->min_size = src->min_size; in target_copy()
447 dest->sort_bitwise = src->sort_bitwise; in target_copy()
448 dest->recovery_deletes = src->recovery_deletes; in target_copy()
450 dest->flags = src->flags; in target_copy()
451 dest->used_replica = src->used_replica; in target_copy()
452 dest->paused = src->paused; in target_copy()
454 dest->epoch = src->epoch; in target_copy()
455 dest->last_force_resend = src->last_force_resend; in target_copy()
457 dest->osd = src->osd; in target_copy()
462 ceph_oid_destroy(&t->base_oid); in target_destroy()
463 ceph_oloc_destroy(&t->base_oloc); in target_destroy()
464 ceph_oid_destroy(&t->target_oid); in target_destroy()
465 ceph_oloc_destroy(&t->target_oloc); in target_destroy()
473 WARN_ON(!RB_EMPTY_NODE(&req->r_node)); in request_release_checks()
474 WARN_ON(!RB_EMPTY_NODE(&req->r_mc_node)); in request_release_checks()
475 WARN_ON(!list_empty(&req->r_private_item)); in request_release_checks()
476 WARN_ON(req->r_osd); in request_release_checks()
485 dout("%s %p (r_request %p r_reply %p)\n", __func__, req, in ceph_osdc_release_request()
486 req->r_request, req->r_reply); in ceph_osdc_release_request()
489 if (req->r_request) in ceph_osdc_release_request()
490 ceph_msg_put(req->r_request); in ceph_osdc_release_request()
491 if (req->r_reply) in ceph_osdc_release_request()
492 ceph_msg_put(req->r_reply); in ceph_osdc_release_request()
494 for (which = 0; which < req->r_num_ops; which++) in ceph_osdc_release_request()
497 target_destroy(&req->r_t); in ceph_osdc_release_request()
498 ceph_put_snap_context(req->r_snapc); in ceph_osdc_release_request()
500 if (req->r_mempool) in ceph_osdc_release_request()
501 mempool_free(req, req->r_osdc->req_mempool); in ceph_osdc_release_request()
502 else if (req->r_num_ops <= CEPH_OSD_SLAB_OPS) in ceph_osdc_release_request()
510 dout("%s %p (was %d)\n", __func__, req, in ceph_osdc_get_request()
511 kref_read(&req->r_kref)); in ceph_osdc_get_request()
512 kref_get(&req->r_kref); in ceph_osdc_get_request()
519 dout("%s %p (was %d)\n", __func__, req, in ceph_osdc_put_request()
520 kref_read(&req->r_kref)); in ceph_osdc_put_request()
521 kref_put(&req->r_kref, ceph_osdc_release_request); in ceph_osdc_put_request()
531 kref_init(&req->r_kref); in request_init()
532 init_completion(&req->r_completion); in request_init()
533 RB_CLEAR_NODE(&req->r_node); in request_init()
534 RB_CLEAR_NODE(&req->r_mc_node); in request_init()
535 INIT_LIST_HEAD(&req->r_private_item); in request_init()
537 target_init(&req->r_t); in request_init()
550 req = mempool_alloc(osdc->req_mempool, gfp_flags); in ceph_osdc_alloc_request()
561 req->r_osdc = osdc; in ceph_osdc_alloc_request()
562 req->r_mempool = use_mempool; in ceph_osdc_alloc_request()
563 req->r_num_ops = num_ops; in ceph_osdc_alloc_request()
564 req->r_snapid = CEPH_NOSNAP; in ceph_osdc_alloc_request()
565 req->r_snapc = ceph_get_snap_context(snapc); in ceph_osdc_alloc_request()
567 dout("%s req %p\n", __func__, req); in ceph_osdc_alloc_request()
574 return 8 + 4 + 4 + 4 + (oloc->pool_ns ? oloc->pool_ns->len : 0); in ceph_oloc_encoding_size()
581 struct ceph_osd_client *osdc = req->r_osdc; in __ceph_osdc_alloc_messages()
585 WARN_ON(req->r_request || req->r_reply); in __ceph_osdc_alloc_messages()
586 WARN_ON(ceph_oid_empty(&req->r_base_oid)); in __ceph_osdc_alloc_messages()
587 WARN_ON(ceph_oloc_empty(&req->r_base_oloc)); in __ceph_osdc_alloc_messages()
598 ceph_oloc_encoding_size(&req->r_base_oloc); /* oloc */ in __ceph_osdc_alloc_messages()
599 msg_size += 4 + req->r_base_oid.name_len; /* oid */ in __ceph_osdc_alloc_messages()
600 msg_size += 2 + req->r_num_ops * sizeof(struct ceph_osd_op); in __ceph_osdc_alloc_messages()
603 msg_size += 4 + 8 * (req->r_snapc ? req->r_snapc->num_snaps : 0); in __ceph_osdc_alloc_messages()
606 if (req->r_mempool) in __ceph_osdc_alloc_messages()
607 msg = ceph_msgpool_get(&osdc->msgpool_op, msg_size, in __ceph_osdc_alloc_messages()
613 return -ENOMEM; in __ceph_osdc_alloc_messages()
615 memset(msg->front.iov_base, 0, msg->front.iov_len); in __ceph_osdc_alloc_messages()
616 req->r_request = msg; in __ceph_osdc_alloc_messages()
620 msg_size += req->r_base_oid.name_len; in __ceph_osdc_alloc_messages()
621 msg_size += req->r_num_ops * sizeof(struct ceph_osd_op); in __ceph_osdc_alloc_messages()
623 if (req->r_mempool) in __ceph_osdc_alloc_messages()
624 msg = ceph_msgpool_get(&osdc->msgpool_op_reply, msg_size, in __ceph_osdc_alloc_messages()
630 return -ENOMEM; in __ceph_osdc_alloc_messages()
632 req->r_reply = msg; in __ceph_osdc_alloc_messages()
643 default: in osd_req_opcode_valid()
657 for (op = req->r_ops; op != &req->r_ops[req->r_num_ops]; op++) { in get_num_data_items()
658 switch (op->op) { in get_num_data_items()
686 default: in get_num_data_items()
687 WARN_ON(!osd_req_opcode_valid(op->op)); in get_num_data_items()
718 BUG_ON(which >= osd_req->r_num_ops); in osd_req_op_init()
721 op = &osd_req->r_ops[which]; in osd_req_op_init()
723 op->op = opcode; in osd_req_op_init()
724 op->flags = flags; in osd_req_op_init()
743 op->extent.offset = offset; in osd_req_op_extent_init()
744 op->extent.length = length; in osd_req_op_extent_init()
745 op->extent.truncate_size = truncate_size; in osd_req_op_extent_init()
746 op->extent.truncate_seq = truncate_seq; in osd_req_op_extent_init()
750 op->indata_len = payload_len; in osd_req_op_extent_init()
760 BUG_ON(which >= osd_req->r_num_ops); in osd_req_op_extent_update()
761 op = &osd_req->r_ops[which]; in osd_req_op_extent_update()
762 previous = op->extent.length; in osd_req_op_extent_update()
768 op->extent.length = length; in osd_req_op_extent_update()
769 if (op->op == CEPH_OSD_OP_WRITE || op->op == CEPH_OSD_OP_WRITEFULL) in osd_req_op_extent_update()
770 op->indata_len -= previous - length; in osd_req_op_extent_update()
779 BUG_ON(which + 1 >= osd_req->r_num_ops); in osd_req_op_extent_dup_last()
781 prev_op = &osd_req->r_ops[which]; in osd_req_op_extent_dup_last()
782 op = osd_req_op_init(osd_req, which + 1, prev_op->op, prev_op->flags); in osd_req_op_extent_dup_last()
784 op->indata_len = prev_op->indata_len; in osd_req_op_extent_dup_last()
785 op->outdata_len = prev_op->outdata_len; in osd_req_op_extent_dup_last()
786 op->extent = prev_op->extent; in osd_req_op_extent_dup_last()
788 op->extent.offset += offset_inc; in osd_req_op_extent_dup_last()
789 op->extent.length -= offset_inc; in osd_req_op_extent_dup_last()
791 if (op->op == CEPH_OSD_OP_WRITE || op->op == CEPH_OSD_OP_WRITEFULL) in osd_req_op_extent_dup_last()
792 op->indata_len -= offset_inc; in osd_req_op_extent_dup_last()
809 return -ENOMEM; in osd_req_op_cls_init()
811 op->cls.class_name = class; in osd_req_op_cls_init()
814 op->cls.class_len = size; in osd_req_op_cls_init()
820 op->cls.method_name = method; in osd_req_op_cls_init()
823 op->cls.method_len = size; in osd_req_op_cls_init()
830 op->indata_len = payload_len; in osd_req_op_cls_init()
853 return -ENOMEM; in osd_req_op_xattr_init()
856 op->xattr.name_len = payload_len; in osd_req_op_xattr_init()
861 op->xattr.value_len = size; in osd_req_op_xattr_init()
867 op->xattr.cmp_op = cmp_op; in osd_req_op_xattr_init()
868 op->xattr.cmp_mode = cmp_mode; in osd_req_op_xattr_init()
870 ceph_osd_data_pagelist_init(&op->xattr.osd_data, pagelist); in osd_req_op_xattr_init()
871 op->indata_len = payload_len; in osd_req_op_xattr_init()
889 op->watch.cookie = cookie; in osd_req_op_watch_init()
890 op->watch.op = watch_opcode; in osd_req_op_watch_init()
891 op->watch.gen = gen; in osd_req_op_watch_init()
904 op->notify.cookie = cookie; in osd_req_op_notify_init()
906 ceph_osd_data_pagelist_init(&op->notify.request_data, request_pl); in osd_req_op_notify_init()
907 op->indata_len = request_pl->length; in osd_req_op_notify_init()
922 op->alloc_hint.expected_object_size = expected_object_size; in osd_req_op_alloc_hint_init()
923 op->alloc_hint.expected_write_size = expected_write_size; in osd_req_op_alloc_hint_init()
924 op->alloc_hint.flags = flags; in osd_req_op_alloc_hint_init()
928 * not worth a feature bit. Set FAILOK per-op flag to make in osd_req_op_alloc_hint_init()
931 op->flags |= CEPH_OSD_OP_FLAG_FAILOK; in osd_req_op_alloc_hint_init()
940 if (osd_data->type == CEPH_OSD_DATA_TYPE_PAGES) { in ceph_osdc_msg_data_add()
943 ceph_msg_data_add_pages(msg, osd_data->pages, in ceph_osdc_msg_data_add()
944 length, osd_data->alignment, false); in ceph_osdc_msg_data_add()
945 } else if (osd_data->type == CEPH_OSD_DATA_TYPE_PAGELIST) { in ceph_osdc_msg_data_add()
947 ceph_msg_data_add_pagelist(msg, osd_data->pagelist); in ceph_osdc_msg_data_add()
949 } else if (osd_data->type == CEPH_OSD_DATA_TYPE_BIO) { in ceph_osdc_msg_data_add()
950 ceph_msg_data_add_bio(msg, &osd_data->bio_pos, length); in ceph_osdc_msg_data_add()
952 } else if (osd_data->type == CEPH_OSD_DATA_TYPE_BVECS) { in ceph_osdc_msg_data_add()
953 ceph_msg_data_add_bvecs(msg, &osd_data->bvec_pos); in ceph_osdc_msg_data_add()
955 BUG_ON(osd_data->type != CEPH_OSD_DATA_TYPE_NONE); in ceph_osdc_msg_data_add()
962 switch (src->op) { in osd_req_encode_op()
970 dst->extent.offset = cpu_to_le64(src->extent.offset); in osd_req_encode_op()
971 dst->extent.length = cpu_to_le64(src->extent.length); in osd_req_encode_op()
972 dst->extent.truncate_size = in osd_req_encode_op()
973 cpu_to_le64(src->extent.truncate_size); in osd_req_encode_op()
974 dst->extent.truncate_seq = in osd_req_encode_op()
975 cpu_to_le32(src->extent.truncate_seq); in osd_req_encode_op()
978 dst->cls.class_len = src->cls.class_len; in osd_req_encode_op()
979 dst->cls.method_len = src->cls.method_len; in osd_req_encode_op()
980 dst->cls.indata_len = cpu_to_le32(src->cls.indata_len); in osd_req_encode_op()
983 dst->watch.cookie = cpu_to_le64(src->watch.cookie); in osd_req_encode_op()
984 dst->watch.ver = cpu_to_le64(0); in osd_req_encode_op()
985 dst->watch.op = src->watch.op; in osd_req_encode_op()
986 dst->watch.gen = cpu_to_le32(src->watch.gen); in osd_req_encode_op()
991 dst->notify.cookie = cpu_to_le64(src->notify.cookie); in osd_req_encode_op()
996 dst->alloc_hint.expected_object_size = in osd_req_encode_op()
997 cpu_to_le64(src->alloc_hint.expected_object_size); in osd_req_encode_op()
998 dst->alloc_hint.expected_write_size = in osd_req_encode_op()
999 cpu_to_le64(src->alloc_hint.expected_write_size); in osd_req_encode_op()
1000 dst->alloc_hint.flags = cpu_to_le32(src->alloc_hint.flags); in osd_req_encode_op()
1004 dst->xattr.name_len = cpu_to_le32(src->xattr.name_len); in osd_req_encode_op()
1005 dst->xattr.value_len = cpu_to_le32(src->xattr.value_len); in osd_req_encode_op()
1006 dst->xattr.cmp_op = src->xattr.cmp_op; in osd_req_encode_op()
1007 dst->xattr.cmp_mode = src->xattr.cmp_mode; in osd_req_encode_op()
1013 dst->copy_from.snapid = cpu_to_le64(src->copy_from.snapid); in osd_req_encode_op()
1014 dst->copy_from.src_version = in osd_req_encode_op()
1015 cpu_to_le64(src->copy_from.src_version); in osd_req_encode_op()
1016 dst->copy_from.flags = src->copy_from.flags; in osd_req_encode_op()
1017 dst->copy_from.src_fadvise_flags = in osd_req_encode_op()
1018 cpu_to_le32(src->copy_from.src_fadvise_flags); in osd_req_encode_op()
1020 default: in osd_req_encode_op()
1022 ceph_osd_op_name(src->op)); in osd_req_encode_op()
1028 dst->op = cpu_to_le16(src->op); in osd_req_encode_op()
1029 dst->flags = cpu_to_le32(src->flags); in osd_req_encode_op()
1030 dst->payload_len = cpu_to_le32(src->indata_len); in osd_req_encode_op()
1032 return src->indata_len; in osd_req_encode_op()
1067 r = -ENOMEM; in ceph_osdc_new_request()
1079 u32 object_size = layout->object_size; in ceph_osdc_new_request()
1080 u32 object_base = off - objoff; in ceph_osdc_new_request()
1081 if (!(truncate_seq == 1 && truncate_size == -1ULL)) { in ceph_osdc_new_request()
1085 truncate_size -= object_base; in ceph_osdc_new_request()
1094 req->r_base_oloc.pool = layout->pool_id; in ceph_osdc_new_request()
1095 req->r_base_oloc.pool_ns = ceph_try_get_string(layout->pool_ns); in ceph_osdc_new_request()
1096 ceph_oid_printf(&req->r_base_oid, "%llx.%08llx", vino.ino, objnum); in ceph_osdc_new_request()
1097 req->r_flags = flags | osdc->client->options->read_from_replica; in ceph_osdc_new_request()
1099 req->r_snapid = vino.snap; in ceph_osdc_new_request()
1101 req->r_data_offset = off; in ceph_osdc_new_request()
1106 * also covers ceph_uninline_data(). If more multi-op request in ceph_osdc_new_request()
1124 * We keep osd requests in an rbtree, sorted by ->r_tid.
1138 for (n = rb_first(&osdc->osds); n; n = rb_next(n)) { in DEFINE_RB_FUNCS()
1141 for (p = rb_first(&osd->o_requests); p; ) { in DEFINE_RB_FUNCS()
1151 for (p = rb_first(&osdc->homeless_osd.o_requests); p; ) { in DEFINE_RB_FUNCS()
1163 return osd->o_osd == CEPH_HOMELESS_OSD; in osd_homeless()
1168 verify_osdc_locked(osd->o_osdc); in osd_registered()
1170 return !RB_EMPTY_NODE(&osd->o_node); in osd_registered()
1174 * Assumes @osd is zero-initialized.
1178 refcount_set(&osd->o_ref, 1); in osd_init()
1179 RB_CLEAR_NODE(&osd->o_node); in osd_init()
1180 osd->o_requests = RB_ROOT; in osd_init()
1181 osd->o_linger_requests = RB_ROOT; in osd_init()
1182 osd->o_backoff_mappings = RB_ROOT; in osd_init()
1183 osd->o_backoffs_by_id = RB_ROOT; in osd_init()
1184 INIT_LIST_HEAD(&osd->o_osd_lru); in osd_init()
1185 INIT_LIST_HEAD(&osd->o_keepalive_item); in osd_init()
1186 osd->o_incarnation = 1; in osd_init()
1187 mutex_init(&osd->lock); in osd_init()
1192 WARN_ON(!RB_EMPTY_NODE(&osd->o_node)); in osd_cleanup()
1193 WARN_ON(!RB_EMPTY_ROOT(&osd->o_requests)); in osd_cleanup()
1194 WARN_ON(!RB_EMPTY_ROOT(&osd->o_linger_requests)); in osd_cleanup()
1195 WARN_ON(!RB_EMPTY_ROOT(&osd->o_backoff_mappings)); in osd_cleanup()
1196 WARN_ON(!RB_EMPTY_ROOT(&osd->o_backoffs_by_id)); in osd_cleanup()
1197 WARN_ON(!list_empty(&osd->o_osd_lru)); in osd_cleanup()
1198 WARN_ON(!list_empty(&osd->o_keepalive_item)); in osd_cleanup()
1200 if (osd->o_auth.authorizer) { in osd_cleanup()
1202 ceph_auth_destroy_authorizer(osd->o_auth.authorizer); in osd_cleanup()
1217 osd->o_osdc = osdc; in create_osd()
1218 osd->o_osd = onum; in create_osd()
1220 ceph_con_init(&osd->o_con, osd, &osd_con_ops, &osdc->client->msgr); in create_osd()
1227 if (refcount_inc_not_zero(&osd->o_ref)) { in get_osd()
1228 dout("get_osd %p %d -> %d\n", osd, refcount_read(&osd->o_ref)-1, in get_osd()
1229 refcount_read(&osd->o_ref)); in get_osd()
1232 dout("get_osd %p FAIL\n", osd); in get_osd()
1239 dout("put_osd %p %d -> %d\n", osd, refcount_read(&osd->o_ref), in put_osd()
1240 refcount_read(&osd->o_ref) - 1); in put_osd()
1241 if (refcount_dec_and_test(&osd->o_ref)) { in put_osd()
1251 struct ceph_osd_client *osdc = osd->o_osdc; in DEFINE_RB_FUNCS()
1253 dout("%s osd %p osd%d\n", __func__, osd, osd->o_osd); in DEFINE_RB_FUNCS()
1254 BUG_ON(!list_empty(&osd->o_osd_lru)); in DEFINE_RB_FUNCS()
1256 spin_lock(&osdc->osd_lru_lock); in DEFINE_RB_FUNCS()
1257 list_add_tail(&osd->o_osd_lru, &osdc->osd_lru); in DEFINE_RB_FUNCS()
1258 spin_unlock(&osdc->osd_lru_lock); in DEFINE_RB_FUNCS()
1260 osd->lru_ttl = jiffies + osdc->client->options->osd_idle_ttl; in DEFINE_RB_FUNCS()
1265 if (RB_EMPTY_ROOT(&osd->o_requests) && in maybe_move_osd_to_lru()
1266 RB_EMPTY_ROOT(&osd->o_linger_requests)) in maybe_move_osd_to_lru()
1272 struct ceph_osd_client *osdc = osd->o_osdc; in __remove_osd_from_lru()
1274 dout("%s osd %p osd%d\n", __func__, osd, osd->o_osd); in __remove_osd_from_lru()
1276 spin_lock(&osdc->osd_lru_lock); in __remove_osd_from_lru()
1277 if (!list_empty(&osd->o_osd_lru)) in __remove_osd_from_lru()
1278 list_del_init(&osd->o_osd_lru); in __remove_osd_from_lru()
1279 spin_unlock(&osdc->osd_lru_lock); in __remove_osd_from_lru()
1288 struct ceph_osd_client *osdc = osd->o_osdc; in close_osd()
1292 dout("%s osd %p osd%d\n", __func__, osd, osd->o_osd); in close_osd()
1294 ceph_con_close(&osd->o_con); in close_osd()
1296 for (n = rb_first(&osd->o_requests); n; ) { in close_osd()
1302 dout(" reassigning req %p tid %llu\n", req, req->r_tid); in close_osd()
1304 link_request(&osdc->homeless_osd, req); in close_osd()
1306 for (n = rb_first(&osd->o_linger_requests); n; ) { in close_osd()
1312 dout(" reassigning lreq %p linger_id %llu\n", lreq, in close_osd()
1313 lreq->linger_id); in close_osd()
1315 link_linger(&osdc->homeless_osd, lreq); in close_osd()
1320 erase_osd(&osdc->osds, osd); in close_osd()
1331 dout("%s osd %p osd%d\n", __func__, osd, osd->o_osd); in reopen_osd()
1333 if (RB_EMPTY_ROOT(&osd->o_requests) && in reopen_osd()
1334 RB_EMPTY_ROOT(&osd->o_linger_requests)) { in reopen_osd()
1336 return -ENODEV; in reopen_osd()
1339 peer_addr = &osd->o_osdc->osdmap->osd_addr[osd->o_osd]; in reopen_osd()
1340 if (!memcmp(peer_addr, &osd->o_con.peer_addr, sizeof (*peer_addr)) && in reopen_osd()
1341 !ceph_con_opened(&osd->o_con)) { in reopen_osd()
1344 dout("osd addr hasn't changed and connection never opened, " in reopen_osd()
1347 for (n = rb_first(&osd->o_requests); n; n = rb_next(n)) { in reopen_osd()
1350 req->r_stamp = jiffies; in reopen_osd()
1353 return -EAGAIN; in reopen_osd()
1356 ceph_con_close(&osd->o_con); in reopen_osd()
1357 ceph_con_open(&osd->o_con, CEPH_ENTITY_TYPE_OSD, osd->o_osd, peer_addr); in reopen_osd()
1358 osd->o_incarnation++; in reopen_osd()
1374 osd = lookup_osd(&osdc->osds, o); in lookup_create_osd()
1376 osd = &osdc->homeless_osd; in lookup_create_osd()
1379 return ERR_PTR(-EAGAIN); in lookup_create_osd()
1382 insert_osd(&osdc->osds, osd); in lookup_create_osd()
1383 ceph_con_open(&osd->o_con, CEPH_ENTITY_TYPE_OSD, osd->o_osd, in lookup_create_osd()
1384 &osdc->osdmap->osd_addr[osd->o_osd]); in lookup_create_osd()
1387 dout("%s osdc %p osd%d -> osd %p\n", __func__, osdc, o, osd); in lookup_create_osd()
1392 * Create request <-> OSD session relation.
1399 WARN_ON(!req->r_tid || req->r_osd); in link_request()
1400 dout("%s osd %p osd%d req %p tid %llu\n", __func__, osd, osd->o_osd, in link_request()
1401 req, req->r_tid); in link_request()
1406 atomic_inc(&osd->o_osdc->num_homeless); in link_request()
1409 insert_request(&osd->o_requests, req); in link_request()
1410 req->r_osd = osd; in link_request()
1416 WARN_ON(req->r_osd != osd); in unlink_request()
1417 dout("%s osd %p osd%d req %p tid %llu\n", __func__, osd, osd->o_osd, in unlink_request()
1418 req, req->r_tid); in unlink_request()
1420 req->r_osd = NULL; in unlink_request()
1421 erase_request(&osd->o_requests, req); in unlink_request()
1427 atomic_dec(&osd->o_osdc->num_homeless); in unlink_request()
1432 return pi->flags & CEPH_POOL_FLAG_FULL; in __pool_full()
1439 for (n = rb_first(&osdc->osdmap->pg_pools); n; n = rb_next(n)) { in have_pool_full()
1454 pi = ceph_pg_pool_by_id(osdc->osdmap, pool_id); in pool_full()
1474 WARN_ON(pi->id != t->target_oloc.pool); in target_should_be_paused()
1475 return ((t->flags & CEPH_OSD_FLAG_READ) && pauserd) || in target_should_be_paused()
1476 ((t->flags & CEPH_OSD_FLAG_WRITE) && pausewr) || in target_should_be_paused()
1477 (osdc->osdmap->epoch < osdc->epoch_barrier); in target_should_be_paused()
1482 int i = prandom_u32() % acting->size; in pick_random_replica()
1484 dout("%s picked osd%d, primary osd%d\n", __func__, in pick_random_replica()
1485 acting->osds[i], acting->primary); in pick_random_replica()
1497 struct ceph_options *opt = osdc->client->options; in pick_closest_replica()
1502 locality = ceph_get_crush_locality(osdc->osdmap, in pick_closest_replica()
1503 acting->osds[i], in pick_closest_replica()
1504 &opt->crush_locs); in pick_closest_replica()
1512 } while (++i < acting->size); in pick_closest_replica()
1514 dout("%s picked osd%d with locality %d, primary osd%d\n", __func__, in pick_closest_replica()
1515 acting->osds[best_i], best_locality, acting->primary); in pick_closest_replica()
1532 bool is_read = t->flags & CEPH_OSD_FLAG_READ; in calc_target()
1533 bool is_write = t->flags & CEPH_OSD_FLAG_WRITE; in calc_target()
1543 t->epoch = osdc->osdmap->epoch; in calc_target()
1544 pi = ceph_pg_pool_by_id(osdc->osdmap, t->base_oloc.pool); in calc_target()
1546 t->osd = CEPH_HOMELESS_OSD; in calc_target()
1551 if (osdc->osdmap->epoch == pi->last_force_request_resend) { in calc_target()
1552 if (t->last_force_resend < pi->last_force_request_resend) { in calc_target()
1553 t->last_force_resend = pi->last_force_request_resend; in calc_target()
1555 } else if (t->last_force_resend == 0) { in calc_target()
1561 ceph_oid_copy(&t->target_oid, &t->base_oid); in calc_target()
1562 ceph_oloc_copy(&t->target_oloc, &t->base_oloc); in calc_target()
1563 if ((t->flags & CEPH_OSD_FLAG_IGNORE_OVERLAY) == 0) { in calc_target()
1564 if (is_read && pi->read_tier >= 0) in calc_target()
1565 t->target_oloc.pool = pi->read_tier; in calc_target()
1566 if (is_write && pi->write_tier >= 0) in calc_target()
1567 t->target_oloc.pool = pi->write_tier; in calc_target()
1569 pi = ceph_pg_pool_by_id(osdc->osdmap, t->target_oloc.pool); in calc_target()
1571 t->osd = CEPH_HOMELESS_OSD; in calc_target()
1577 __ceph_object_locator_to_pg(pi, &t->target_oid, &t->target_oloc, &pgid); in calc_target()
1579 last_pgid.seed = ceph_stable_mod(pgid.seed, t->pg_num, t->pg_num_mask); in calc_target()
1581 ceph_pg_to_up_acting_osds(osdc->osdmap, pi, &pgid, &up, &acting); in calc_target()
1583 ceph_is_new_interval(&t->acting, in calc_target()
1585 &t->up, in calc_target()
1587 t->size, in calc_target()
1588 pi->size, in calc_target()
1589 t->min_size, in calc_target()
1590 pi->min_size, in calc_target()
1591 t->pg_num, in calc_target()
1592 pi->pg_num, in calc_target()
1593 t->sort_bitwise, in calc_target()
1595 t->recovery_deletes, in calc_target()
1600 if (t->paused && !target_should_be_paused(osdc, t, pi)) { in calc_target()
1601 t->paused = false; in calc_target()
1604 legacy_change = ceph_pg_compare(&t->pgid, &pgid) || in calc_target()
1605 ceph_osds_changed(&t->acting, &acting, in calc_target()
1606 t->used_replica || any_change); in calc_target()
1607 if (t->pg_num) in calc_target()
1608 split = ceph_pg_is_split(&last_pgid, t->pg_num, pi->pg_num); in calc_target()
1611 t->pgid = pgid; /* struct */ in calc_target()
1612 ceph_pg_to_primary_shard(osdc->osdmap, pi, &pgid, &t->spgid); in calc_target()
1613 ceph_osds_copy(&t->acting, &acting); in calc_target()
1614 ceph_osds_copy(&t->up, &up); in calc_target()
1615 t->size = pi->size; in calc_target()
1616 t->min_size = pi->min_size; in calc_target()
1617 t->pg_num = pi->pg_num; in calc_target()
1618 t->pg_num_mask = pi->pg_num_mask; in calc_target()
1619 t->sort_bitwise = sort_bitwise; in calc_target()
1620 t->recovery_deletes = recovery_deletes; in calc_target()
1622 if ((t->flags & (CEPH_OSD_FLAG_BALANCE_READS | in calc_target()
1624 !is_write && pi->type == CEPH_POOL_TYPE_REP && in calc_target()
1629 if (t->flags & CEPH_OSD_FLAG_BALANCE_READS) { in calc_target()
1634 t->osd = acting.osds[pos]; in calc_target()
1635 t->used_replica = pos > 0; in calc_target()
1637 t->osd = acting.primary; in calc_target()
1638 t->used_replica = false; in calc_target()
1648 dout("%s t %p -> %d%d%d%d ct_res %d osd%d\n", __func__, t, unpaused, in calc_target()
1649 legacy_change, force_resend, split, ct_res, t->osd); in calc_target()
1661 RB_CLEAR_NODE(&spg->node); in alloc_spg_mapping()
1662 spg->backoffs = RB_ROOT; in alloc_spg_mapping()
1668 WARN_ON(!RB_EMPTY_NODE(&spg->node)); in free_spg_mapping()
1669 WARN_ON(!RB_EMPTY_ROOT(&spg->backoffs)); in free_spg_mapping()
1676 * ceph_pg_mapping. Used to track OSD backoffs -- a backoff [range] is
1685 return hoid->is_max ? 0x100000000ull : hoid->hash_reverse_bits; in DEFINE_RB_FUNCS2()
1691 if (hoid->key_len) { in hoid_get_effective_key()
1692 *pkey = hoid->key; in hoid_get_effective_key()
1693 *pkey_len = hoid->key_len; in hoid_get_effective_key()
1695 *pkey = hoid->oid; in hoid_get_effective_key()
1696 *pkey_len = hoid->oid_len; in hoid_get_effective_key()
1708 ret = -1; in compare_names()
1722 if (lhs->is_max < rhs->is_max) in hoid_compare()
1723 return -1; in hoid_compare()
1724 if (lhs->is_max > rhs->is_max) in hoid_compare()
1727 if (lhs->pool < rhs->pool) in hoid_compare()
1728 return -1; in hoid_compare()
1729 if (lhs->pool > rhs->pool) in hoid_compare()
1733 return -1; in hoid_compare()
1737 ret = compare_names(lhs->nspace, lhs->nspace_len, in hoid_compare()
1738 rhs->nspace, rhs->nspace_len); in hoid_compare()
1749 ret = compare_names(lhs->oid, lhs->oid_len, rhs->oid, rhs->oid_len); in hoid_compare()
1753 if (lhs->snapid < rhs->snapid) in hoid_compare()
1754 return -1; in hoid_compare()
1755 if (lhs->snapid > rhs->snapid) in hoid_compare()
1762 * For decoding ->begin and ->end of MOSDBackoff only -- no MIN/MAX
1765 * Assumes @hoid is zero-initialized.
1783 hoid->key = ceph_extract_encoded_string(p, end, &hoid->key_len, in decode_hoid()
1785 if (IS_ERR(hoid->key)) { in decode_hoid()
1786 ret = PTR_ERR(hoid->key); in decode_hoid()
1787 hoid->key = NULL; in decode_hoid()
1791 hoid->oid = ceph_extract_encoded_string(p, end, &hoid->oid_len, in decode_hoid()
1793 if (IS_ERR(hoid->oid)) { in decode_hoid()
1794 ret = PTR_ERR(hoid->oid); in decode_hoid()
1795 hoid->oid = NULL; in decode_hoid()
1799 ceph_decode_64_safe(p, end, hoid->snapid, e_inval); in decode_hoid()
1800 ceph_decode_32_safe(p, end, hoid->hash, e_inval); in decode_hoid()
1801 ceph_decode_8_safe(p, end, hoid->is_max, e_inval); in decode_hoid()
1803 hoid->nspace = ceph_extract_encoded_string(p, end, &hoid->nspace_len, in decode_hoid()
1805 if (IS_ERR(hoid->nspace)) { in decode_hoid()
1806 ret = PTR_ERR(hoid->nspace); in decode_hoid()
1807 hoid->nspace = NULL; in decode_hoid()
1811 ceph_decode_64_safe(p, end, hoid->pool, e_inval); in decode_hoid()
1817 return -EINVAL; in decode_hoid()
1823 4 + hoid->key_len + 4 + hoid->oid_len + 4 + hoid->nspace_len; in hoid_encoding_size()
1829 ceph_encode_string(p, end, hoid->key, hoid->key_len); in encode_hoid()
1830 ceph_encode_string(p, end, hoid->oid, hoid->oid_len); in encode_hoid()
1831 ceph_encode_64(p, hoid->snapid); in encode_hoid()
1832 ceph_encode_32(p, hoid->hash); in encode_hoid()
1833 ceph_encode_8(p, hoid->is_max); in encode_hoid()
1834 ceph_encode_string(p, end, hoid->nspace, hoid->nspace_len); in encode_hoid()
1835 ceph_encode_64(p, hoid->pool); in encode_hoid()
1841 kfree(hoid->key); in free_hoid()
1842 kfree(hoid->oid); in free_hoid()
1843 kfree(hoid->nspace); in free_hoid()
1856 RB_CLEAR_NODE(&backoff->spg_node); in alloc_backoff()
1857 RB_CLEAR_NODE(&backoff->id_node); in alloc_backoff()
1863 WARN_ON(!RB_EMPTY_NODE(&backoff->spg_node)); in free_backoff()
1864 WARN_ON(!RB_EMPTY_NODE(&backoff->id_node)); in free_backoff()
1866 free_hoid(backoff->begin); in free_backoff()
1867 free_hoid(backoff->end); in free_backoff()
1872 * Within a specific spgid, backoffs are managed by ->begin hoid.
1880 struct rb_node *n = root->rb_node; in lookup_containing_backoff()
1887 cmp = hoid_compare(hoid, cur->begin); in lookup_containing_backoff()
1889 n = n->rb_left; in lookup_containing_backoff()
1891 if (hoid_compare(hoid, cur->end) < 0) in lookup_containing_backoff()
1894 n = n->rb_right; in lookup_containing_backoff()
1910 while (!RB_EMPTY_ROOT(&osd->o_backoff_mappings)) { in DEFINE_RB_FUNCS()
1912 rb_entry(rb_first(&osd->o_backoff_mappings), in DEFINE_RB_FUNCS()
1915 while (!RB_EMPTY_ROOT(&spg->backoffs)) { in DEFINE_RB_FUNCS()
1917 rb_entry(rb_first(&spg->backoffs), in DEFINE_RB_FUNCS()
1920 erase_backoff(&spg->backoffs, backoff); in DEFINE_RB_FUNCS()
1921 erase_backoff_by_id(&osd->o_backoffs_by_id, backoff); in DEFINE_RB_FUNCS()
1924 erase_spg_mapping(&osd->o_backoff_mappings, spg); in DEFINE_RB_FUNCS()
1930 * Set up a temporary, non-owning view into @t.
1935 hoid->key = NULL; in hoid_fill_from_target()
1936 hoid->key_len = 0; in hoid_fill_from_target()
1937 hoid->oid = t->target_oid.name; in hoid_fill_from_target()
1938 hoid->oid_len = t->target_oid.name_len; in hoid_fill_from_target()
1939 hoid->snapid = CEPH_NOSNAP; in hoid_fill_from_target()
1940 hoid->hash = t->pgid.seed; in hoid_fill_from_target()
1941 hoid->is_max = false; in hoid_fill_from_target()
1942 if (t->target_oloc.pool_ns) { in hoid_fill_from_target()
1943 hoid->nspace = t->target_oloc.pool_ns->str; in hoid_fill_from_target()
1944 hoid->nspace_len = t->target_oloc.pool_ns->len; in hoid_fill_from_target()
1946 hoid->nspace = NULL; in hoid_fill_from_target()
1947 hoid->nspace_len = 0; in hoid_fill_from_target()
1949 hoid->pool = t->target_oloc.pool; in hoid_fill_from_target()
1955 struct ceph_osd *osd = req->r_osd; in should_plug_request()
1960 spg = lookup_spg_mapping(&osd->o_backoff_mappings, &req->r_t.spgid); in should_plug_request()
1964 hoid_fill_from_target(&hoid, &req->r_t); in should_plug_request()
1965 backoff = lookup_containing_backoff(&spg->backoffs, &hoid); in should_plug_request()
1969 dout("%s req %p tid %llu backoff osd%d spgid %llu.%xs%d id %llu\n", in should_plug_request()
1970 __func__, req, req->r_tid, osd->o_osd, backoff->spgid.pgid.pool, in should_plug_request()
1971 backoff->spgid.pgid.seed, backoff->spgid.shard, backoff->id); in should_plug_request()
1980 struct ceph_msg *request_msg = req->r_request; in setup_request_data()
1981 struct ceph_msg *reply_msg = req->r_reply; in setup_request_data()
1984 if (req->r_request->num_data_items || req->r_reply->num_data_items) in setup_request_data()
1987 WARN_ON(request_msg->data_length || reply_msg->data_length); in setup_request_data()
1988 for (op = req->r_ops; op != &req->r_ops[req->r_num_ops]; op++) { in setup_request_data()
1989 switch (op->op) { in setup_request_data()
1993 WARN_ON(op->indata_len != op->extent.length); in setup_request_data()
1995 &op->extent.osd_data); in setup_request_data()
1999 WARN_ON(op->indata_len != op->xattr.name_len + in setup_request_data()
2000 op->xattr.value_len); in setup_request_data()
2002 &op->xattr.osd_data); in setup_request_data()
2006 &op->notify_ack.request_data); in setup_request_data()
2010 &op->copy_from.osd_data); in setup_request_data()
2016 &op->raw_data_in); in setup_request_data()
2020 &op->extent.osd_data); in setup_request_data()
2024 &op->list_watchers.response_data); in setup_request_data()
2029 WARN_ON(op->indata_len != op->cls.class_len + in setup_request_data()
2030 op->cls.method_len + in setup_request_data()
2031 op->cls.indata_len); in setup_request_data()
2033 &op->cls.request_info); in setup_request_data()
2036 &op->cls.request_data); in setup_request_data()
2039 &op->cls.response_data); in setup_request_data()
2043 &op->notify.request_data); in setup_request_data()
2045 &op->notify.response_data); in setup_request_data()
2054 ceph_encode_64(p, pgid->pool); in encode_pgid()
2055 ceph_encode_32(p, pgid->seed); in encode_pgid()
2056 ceph_encode_32(p, -1); /* preferred */ in encode_pgid()
2062 encode_pgid(p, &spgid->pgid); in encode_spgid()
2063 ceph_encode_8(p, spgid->shard); in encode_spgid()
2070 ceph_encode_64(p, oloc->pool); in encode_oloc()
2071 ceph_encode_32(p, -1); /* preferred */ in encode_oloc()
2073 if (oloc->pool_ns) in encode_oloc()
2074 ceph_encode_string(p, end, oloc->pool_ns->str, in encode_oloc()
2075 oloc->pool_ns->len); in encode_oloc()
2083 void *p = msg->front.iov_base; in encode_request_partial()
2084 void *const end = p + msg->front_alloc_len; in encode_request_partial()
2088 if (req->r_flags & CEPH_OSD_FLAG_WRITE) { in encode_request_partial()
2090 WARN_ON(req->r_snapid != CEPH_NOSNAP); in encode_request_partial()
2092 WARN_ON(req->r_mtime.tv_sec || req->r_mtime.tv_nsec || in encode_request_partial()
2093 req->r_data_offset || req->r_snapc); in encode_request_partial()
2098 encode_spgid(&p, &req->r_t.spgid); /* actual spg */ in encode_request_partial()
2099 ceph_encode_32(&p, req->r_t.pgid.seed); /* raw hash */ in encode_request_partial()
2100 ceph_encode_32(&p, req->r_osdc->osdmap->epoch); in encode_request_partial()
2101 ceph_encode_32(&p, req->r_flags); in encode_request_partial()
2113 ceph_encode_timespec64(p, &req->r_mtime); in encode_request_partial()
2116 encode_oloc(&p, end, &req->r_t.target_oloc); in encode_request_partial()
2117 ceph_encode_string(&p, end, req->r_t.target_oid.name, in encode_request_partial()
2118 req->r_t.target_oid.name_len); in encode_request_partial()
2121 ceph_encode_16(&p, req->r_num_ops); in encode_request_partial()
2122 for (i = 0; i < req->r_num_ops; i++) { in encode_request_partial()
2123 data_len += osd_req_encode_op(p, &req->r_ops[i]); in encode_request_partial()
2127 ceph_encode_64(&p, req->r_snapid); /* snapid */ in encode_request_partial()
2128 if (req->r_snapc) { in encode_request_partial()
2129 ceph_encode_64(&p, req->r_snapc->seq); in encode_request_partial()
2130 ceph_encode_32(&p, req->r_snapc->num_snaps); in encode_request_partial()
2131 for (i = 0; i < req->r_snapc->num_snaps; i++) in encode_request_partial()
2132 ceph_encode_64(&p, req->r_snapc->snaps[i]); in encode_request_partial()
2138 ceph_encode_32(&p, req->r_attempts); /* retry_attempt */ in encode_request_partial()
2139 BUG_ON(p > end - 8); /* space for features */ in encode_request_partial()
2141 msg->hdr.version = cpu_to_le16(8); /* MOSDOp v8 */ in encode_request_partial()
2143 msg->front.iov_len = p - msg->front.iov_base; in encode_request_partial()
2144 msg->hdr.front_len = cpu_to_le32(msg->front.iov_len); in encode_request_partial()
2145 msg->hdr.data_len = cpu_to_le32(data_len); in encode_request_partial()
2149 * need to re-copy it before writing it to disk (direct I/O). in encode_request_partial()
2151 msg->hdr.data_off = cpu_to_le16(req->r_data_offset); in encode_request_partial()
2153 dout("%s req %p msg %p oid %s oid_len %d\n", __func__, req, msg, in encode_request_partial()
2154 req->r_t.target_oid.name, req->r_t.target_oid.name_len); in encode_request_partial()
2159 void *p = msg->front.iov_base; in encode_request_finish()
2160 void *const partial_end = p + msg->front.iov_len; in encode_request_finish()
2161 void *const end = p + msg->front_alloc_len; in encode_request_finish()
2163 if (CEPH_HAVE_FEATURE(msg->con->peer_features, RESEND_ON_SPLIT)) { in encode_request_finish()
2164 /* luminous OSD -- encode features and be done */ in encode_request_finish()
2166 ceph_encode_64(&p, msg->con->peer_features); in encode_request_finish()
2186 * Pre-luminous OSD -- reencode v8 into v4 using @head in encode_request_finish()
2200 oloc_len = p - oloc; in encode_request_finish()
2205 oid_len = p - oid; in encode_request_finish()
2208 tail_len = partial_end - p; in encode_request_finish()
2210 p = msg->front.iov_base; in encode_request_finish()
2231 /* tail -- ops, snapid, snapc, retry_attempt */ in encode_request_finish()
2236 msg->hdr.version = cpu_to_le16(4); /* MOSDOp v4 */ in encode_request_finish()
2240 msg->front.iov_len = p - msg->front.iov_base; in encode_request_finish()
2241 msg->hdr.front_len = cpu_to_le32(msg->front.iov_len); in encode_request_finish()
2243 dout("%s msg %p tid %llu %u+%u+%u v%d\n", __func__, msg, in encode_request_finish()
2244 le64_to_cpu(msg->hdr.tid), le32_to_cpu(msg->hdr.front_len), in encode_request_finish()
2245 le32_to_cpu(msg->hdr.middle_len), le32_to_cpu(msg->hdr.data_len), in encode_request_finish()
2246 le16_to_cpu(msg->hdr.version)); in encode_request_finish()
2254 struct ceph_osd *osd = req->r_osd; in send_request()
2257 WARN_ON(osd->o_osd != req->r_t.osd); in send_request()
2267 if (req->r_sent) in send_request()
2268 ceph_msg_revoke(req->r_request); in send_request()
2270 req->r_flags |= CEPH_OSD_FLAG_KNOWN_REDIR; in send_request()
2271 if (req->r_attempts) in send_request()
2272 req->r_flags |= CEPH_OSD_FLAG_RETRY; in send_request()
2274 WARN_ON(req->r_flags & CEPH_OSD_FLAG_RETRY); in send_request()
2276 encode_request_partial(req, req->r_request); in send_request()
2278 dout("%s req %p tid %llu to pgid %llu.%x spgid %llu.%xs%d osd%d e%u flags 0x%x attempt %d\n", in send_request()
2279 __func__, req, req->r_tid, req->r_t.pgid.pool, req->r_t.pgid.seed, in send_request()
2280 req->r_t.spgid.pgid.pool, req->r_t.spgid.pgid.seed, in send_request()
2281 req->r_t.spgid.shard, osd->o_osd, req->r_t.epoch, req->r_flags, in send_request()
2282 req->r_attempts); in send_request()
2284 req->r_t.paused = false; in send_request()
2285 req->r_stamp = jiffies; in send_request()
2286 req->r_attempts++; in send_request()
2288 req->r_sent = osd->o_incarnation; in send_request()
2289 req->r_request->hdr.tid = cpu_to_le64(req->r_tid); in send_request()
2290 ceph_con_send(&osd->o_con, ceph_msg_get(req->r_request)); in send_request()
2298 WARN_ON(!osdc->osdmap->epoch); in maybe_request_map()
2303 dout("%s osdc %p continuous\n", __func__, osdc); in maybe_request_map()
2306 dout("%s osdc %p onetime\n", __func__, osdc); in maybe_request_map()
2309 if (ceph_monc_want_map(&osdc->client->monc, CEPH_SUB_OSDMAP, in maybe_request_map()
2310 osdc->osdmap->epoch + 1, continuous)) in maybe_request_map()
2311 ceph_monc_renew_subs(&osdc->client->monc); in maybe_request_map()
2319 struct ceph_osd_client *osdc = req->r_osdc; in __submit_request()
2326 WARN_ON(req->r_tid); in __submit_request()
2327 dout("%s req %p wrlocked %d\n", __func__, req, wrlocked); in __submit_request()
2330 ct_res = calc_target(osdc, &req->r_t, false); in __submit_request()
2334 osd = lookup_create_osd(osdc, req->r_t.osd, wrlocked); in __submit_request()
2336 WARN_ON(PTR_ERR(osd) != -EAGAIN || wrlocked); in __submit_request()
2340 if (osdc->abort_err) { in __submit_request()
2341 dout("req %p abort_err %d\n", req, osdc->abort_err); in __submit_request()
2342 err = osdc->abort_err; in __submit_request()
2343 } else if (osdc->osdmap->epoch < osdc->epoch_barrier) { in __submit_request()
2344 dout("req %p epoch %u barrier %u\n", req, osdc->osdmap->epoch, in __submit_request()
2345 osdc->epoch_barrier); in __submit_request()
2346 req->r_t.paused = true; in __submit_request()
2348 } else if ((req->r_flags & CEPH_OSD_FLAG_WRITE) && in __submit_request()
2350 dout("req %p pausewr\n", req); in __submit_request()
2351 req->r_t.paused = true; in __submit_request()
2353 } else if ((req->r_flags & CEPH_OSD_FLAG_READ) && in __submit_request()
2355 dout("req %p pauserd\n", req); in __submit_request()
2356 req->r_t.paused = true; in __submit_request()
2358 } else if ((req->r_flags & CEPH_OSD_FLAG_WRITE) && in __submit_request()
2359 !(req->r_flags & (CEPH_OSD_FLAG_FULL_TRY | in __submit_request()
2362 pool_full(osdc, req->r_t.base_oloc.pool))) { in __submit_request()
2363 dout("req %p full/pool_full\n", req); in __submit_request()
2364 if (ceph_test_opt(osdc->client, ABORT_ON_FULL)) { in __submit_request()
2365 err = -ENOSPC; in __submit_request()
2368 req->r_t.paused = true; in __submit_request()
2377 mutex_lock(&osd->lock); in __submit_request()
2383 req->r_tid = atomic64_inc_return(&osdc->last_tid); in __submit_request()
2389 mutex_unlock(&osd->lock); in __submit_request()
2395 downgrade_write(&osdc->lock); in __submit_request()
2399 up_read(&osdc->lock); in __submit_request()
2400 down_write(&osdc->lock); in __submit_request()
2408 WARN_ON(req->r_flags & (CEPH_OSD_FLAG_ACK | CEPH_OSD_FLAG_ONDISK)); in account_request()
2409 WARN_ON(!(req->r_flags & (CEPH_OSD_FLAG_READ | CEPH_OSD_FLAG_WRITE))); in account_request()
2411 req->r_flags |= CEPH_OSD_FLAG_ONDISK; in account_request()
2412 atomic_inc(&req->r_osdc->num_requests); in account_request()
2414 req->r_start_stamp = jiffies; in account_request()
2415 req->r_start_latency = ktime_get(); in account_request()
2427 struct ceph_osd_client *osdc = req->r_osdc; in finish_request()
2429 WARN_ON(lookup_request_mc(&osdc->map_checks, req->r_tid)); in finish_request()
2430 dout("%s req %p tid %llu\n", __func__, req, req->r_tid); in finish_request()
2432 req->r_end_latency = ktime_get(); in finish_request()
2434 if (req->r_osd) in finish_request()
2435 unlink_request(req->r_osd, req); in finish_request()
2436 atomic_dec(&osdc->num_requests); in finish_request()
2444 ceph_msg_revoke(req->r_request); in finish_request()
2445 ceph_msg_revoke_incoming(req->r_reply); in finish_request()
2450 dout("%s req %p tid %llu cb %ps result %d\n", __func__, req, in __complete_request()
2451 req->r_tid, req->r_callback, req->r_result); in __complete_request()
2453 if (req->r_callback) in __complete_request()
2454 req->r_callback(req); in __complete_request()
2455 complete_all(&req->r_completion); in __complete_request()
2468 * This is open-coded in handle_reply().
2472 dout("%s req %p tid %llu err %d\n", __func__, req, req->r_tid, err); in complete_request()
2474 req->r_result = err; in complete_request()
2477 INIT_WORK(&req->r_complete_work, complete_request_workfn); in complete_request()
2478 queue_work(req->r_osdc->completion_wq, &req->r_complete_work); in complete_request()
2483 struct ceph_osd_client *osdc = req->r_osdc; in cancel_map_check()
2488 lookup_req = lookup_request_mc(&osdc->map_checks, req->r_tid); in cancel_map_check()
2493 erase_request_mc(&osdc->map_checks, req); in cancel_map_check()
2499 dout("%s req %p tid %llu\n", __func__, req, req->r_tid); in cancel_request()
2503 complete_all(&req->r_completion); in cancel_request()
2509 dout("%s req %p tid %llu err %d\n", __func__, req, req->r_tid, err); in abort_request()
2524 * Abort all in-flight requests with @err and arrange for all future
2529 dout("%s osdc %p err %d\n", __func__, osdc, err); in ceph_osdc_abort_requests()
2530 down_write(&osdc->lock); in ceph_osdc_abort_requests()
2532 osdc->abort_err = err; in ceph_osdc_abort_requests()
2533 up_write(&osdc->lock); in ceph_osdc_abort_requests()
2539 down_write(&osdc->lock); in ceph_osdc_clear_abort_err()
2540 osdc->abort_err = 0; in ceph_osdc_clear_abort_err()
2541 up_write(&osdc->lock); in ceph_osdc_clear_abort_err()
2547 if (likely(eb > osdc->epoch_barrier)) { in update_epoch_barrier()
2548 dout("updating epoch_barrier from %u to %u\n", in update_epoch_barrier()
2549 osdc->epoch_barrier, eb); in update_epoch_barrier()
2550 osdc->epoch_barrier = eb; in update_epoch_barrier()
2552 if (eb > osdc->osdmap->epoch) in update_epoch_barrier()
2559 down_read(&osdc->lock); in ceph_osdc_update_epoch_barrier()
2560 if (unlikely(eb > osdc->epoch_barrier)) { in ceph_osdc_update_epoch_barrier()
2561 up_read(&osdc->lock); in ceph_osdc_update_epoch_barrier()
2562 down_write(&osdc->lock); in ceph_osdc_update_epoch_barrier()
2564 up_write(&osdc->lock); in ceph_osdc_update_epoch_barrier()
2566 up_read(&osdc->lock); in ceph_osdc_update_epoch_barrier()
2579 struct ceph_osd_client *osdc = req->r_osdc; in abort_on_full_fn()
2582 if ((req->r_flags & CEPH_OSD_FLAG_WRITE) && in abort_on_full_fn()
2584 pool_full(osdc, req->r_t.base_oloc.pool))) { in abort_on_full_fn()
2586 update_epoch_barrier(osdc, osdc->osdmap->epoch); in abort_on_full_fn()
2589 abort_request(req, -ENOSPC); in abort_on_full_fn()
2598 * osdc->epoch_barrier to the latest map epoch that we've seen if any were
2605 if (ceph_test_opt(osdc->client, ABORT_ON_FULL) && in ceph_osdc_abort_on_full()
2612 struct ceph_osd_client *osdc = req->r_osdc; in check_pool_dne()
2613 struct ceph_osdmap *map = osdc->osdmap; in check_pool_dne()
2616 WARN_ON(!map->epoch); in check_pool_dne()
2618 if (req->r_attempts) { in check_pool_dne()
2624 req->r_map_dne_bound = map->epoch; in check_pool_dne()
2625 dout("%s req %p tid %llu pool disappeared\n", __func__, req, in check_pool_dne()
2626 req->r_tid); in check_pool_dne()
2628 dout("%s req %p tid %llu map_dne_bound %u have %u\n", __func__, in check_pool_dne()
2629 req, req->r_tid, req->r_map_dne_bound, map->epoch); in check_pool_dne()
2632 if (req->r_map_dne_bound) { in check_pool_dne()
2633 if (map->epoch >= req->r_map_dne_bound) { in check_pool_dne()
2636 req->r_tid); in check_pool_dne()
2637 complete_request(req, -ENOENT); in check_pool_dne()
2646 struct ceph_osd_client *osdc = &greq->monc->client->osdc; in map_check_cb()
2648 u64 tid = greq->private_data; in map_check_cb()
2650 WARN_ON(greq->result || !greq->u.newest); in map_check_cb()
2652 down_write(&osdc->lock); in map_check_cb()
2653 req = lookup_request_mc(&osdc->map_checks, tid); in map_check_cb()
2655 dout("%s tid %llu dne\n", __func__, tid); in map_check_cb()
2659 dout("%s req %p tid %llu map_dne_bound %u newest %llu\n", __func__, in map_check_cb()
2660 req, req->r_tid, req->r_map_dne_bound, greq->u.newest); in map_check_cb()
2661 if (!req->r_map_dne_bound) in map_check_cb()
2662 req->r_map_dne_bound = greq->u.newest; in map_check_cb()
2663 erase_request_mc(&osdc->map_checks, req); in map_check_cb()
2668 up_write(&osdc->lock); in map_check_cb()
2673 struct ceph_osd_client *osdc = req->r_osdc; in send_map_check()
2679 lookup_req = lookup_request_mc(&osdc->map_checks, req->r_tid); in send_map_check()
2686 insert_request_mc(&osdc->map_checks, req); in send_map_check()
2687 ret = ceph_monc_get_version_async(&osdc->client->monc, "osdmap", in send_map_check()
2688 map_check_cb, req->r_tid); in send_map_check()
2700 dout("%s lreq %p reg_req %p ping_req %p\n", __func__, lreq, in linger_release()
2701 lreq->reg_req, lreq->ping_req); in linger_release()
2702 WARN_ON(!RB_EMPTY_NODE(&lreq->node)); in linger_release()
2703 WARN_ON(!RB_EMPTY_NODE(&lreq->osdc_node)); in linger_release()
2704 WARN_ON(!RB_EMPTY_NODE(&lreq->mc_node)); in linger_release()
2705 WARN_ON(!list_empty(&lreq->scan_item)); in linger_release()
2706 WARN_ON(!list_empty(&lreq->pending_lworks)); in linger_release()
2707 WARN_ON(lreq->osd); in linger_release()
2709 if (lreq->request_pl) in linger_release()
2710 ceph_pagelist_release(lreq->request_pl); in linger_release()
2711 if (lreq->notify_id_pages) in linger_release()
2712 ceph_release_page_vector(lreq->notify_id_pages, 1); in linger_release()
2714 ceph_osdc_put_request(lreq->reg_req); in linger_release()
2715 ceph_osdc_put_request(lreq->ping_req); in linger_release()
2716 target_destroy(&lreq->t); in linger_release()
2723 kref_put(&lreq->kref, linger_release); in linger_put()
2729 kref_get(&lreq->kref); in linger_get()
2742 kref_init(&lreq->kref); in linger_alloc()
2743 mutex_init(&lreq->lock); in linger_alloc()
2744 RB_CLEAR_NODE(&lreq->node); in linger_alloc()
2745 RB_CLEAR_NODE(&lreq->osdc_node); in linger_alloc()
2746 RB_CLEAR_NODE(&lreq->mc_node); in linger_alloc()
2747 INIT_LIST_HEAD(&lreq->scan_item); in linger_alloc()
2748 INIT_LIST_HEAD(&lreq->pending_lworks); in linger_alloc()
2749 init_completion(&lreq->reg_commit_wait); in linger_alloc()
2750 init_completion(&lreq->notify_finish_wait); in linger_alloc()
2752 lreq->osdc = osdc; in linger_alloc()
2753 target_init(&lreq->t); in linger_alloc()
2755 dout("%s lreq %p\n", __func__, lreq); in linger_alloc()
2764 * Create linger request <-> OSD session relation. in DEFINE_RB_INSDEL_FUNCS()
2772 WARN_ON(!lreq->linger_id || lreq->osd); in DEFINE_RB_INSDEL_FUNCS()
2773 dout("%s osd %p osd%d lreq %p linger_id %llu\n", __func__, osd, in DEFINE_RB_INSDEL_FUNCS()
2774 osd->o_osd, lreq, lreq->linger_id); in DEFINE_RB_INSDEL_FUNCS()
2779 atomic_inc(&osd->o_osdc->num_homeless); in DEFINE_RB_INSDEL_FUNCS()
2782 insert_linger(&osd->o_linger_requests, lreq); in DEFINE_RB_INSDEL_FUNCS()
2783 lreq->osd = osd; in DEFINE_RB_INSDEL_FUNCS()
2790 WARN_ON(lreq->osd != osd); in unlink_linger()
2791 dout("%s osd %p osd%d lreq %p linger_id %llu\n", __func__, osd, in unlink_linger()
2792 osd->o_osd, lreq, lreq->linger_id); in unlink_linger()
2794 lreq->osd = NULL; in unlink_linger()
2795 erase_linger(&osd->o_linger_requests, lreq); in unlink_linger()
2801 atomic_dec(&osd->o_osdc->num_homeless); in unlink_linger()
2806 verify_osdc_locked(lreq->osdc); in __linger_registered()
2808 return !RB_EMPTY_NODE(&lreq->osdc_node); in __linger_registered()
2813 struct ceph_osd_client *osdc = lreq->osdc; in linger_registered()
2816 down_read(&osdc->lock); in linger_registered()
2818 up_read(&osdc->lock); in linger_registered()
2825 struct ceph_osd_client *osdc = lreq->osdc; in linger_register()
2828 WARN_ON(lreq->linger_id); in linger_register()
2831 lreq->linger_id = ++osdc->last_linger_id; in linger_register()
2832 insert_linger_osdc(&osdc->linger_requests, lreq); in linger_register()
2837 struct ceph_osd_client *osdc = lreq->osdc; in linger_unregister()
2841 erase_linger_osdc(&osdc->linger_requests, lreq); in linger_unregister()
2847 struct ceph_osd_linger_request *lreq = req->r_priv; in cancel_linger_request()
2849 WARN_ON(!req->r_linger); in cancel_linger_request()
2884 INIT_WORK(&lwork->work, workfn); in lwork_alloc()
2885 INIT_LIST_HEAD(&lwork->pending_item); in lwork_alloc()
2886 lwork->lreq = linger_get(lreq); in lwork_alloc()
2893 struct ceph_osd_linger_request *lreq = lwork->lreq; in lwork_free()
2895 mutex_lock(&lreq->lock); in lwork_free()
2896 list_del(&lwork->pending_item); in lwork_free()
2897 mutex_unlock(&lreq->lock); in lwork_free()
2905 struct ceph_osd_linger_request *lreq = lwork->lreq; in lwork_queue()
2906 struct ceph_osd_client *osdc = lreq->osdc; in lwork_queue()
2909 WARN_ON(!list_empty(&lwork->pending_item)); in lwork_queue()
2911 lwork->queued_stamp = jiffies; in lwork_queue()
2912 list_add_tail(&lwork->pending_item, &lreq->pending_lworks); in lwork_queue()
2913 queue_work(osdc->notify_wq, &lwork->work); in lwork_queue()
2919 struct ceph_osd_linger_request *lreq = lwork->lreq; in do_watch_notify()
2922 dout("%s lreq %p not registered\n", __func__, lreq); in do_watch_notify()
2926 WARN_ON(!lreq->is_watch); in do_watch_notify()
2927 dout("%s lreq %p notify_id %llu notifier_id %llu payload_len %zu\n", in do_watch_notify()
2928 __func__, lreq, lwork->notify.notify_id, lwork->notify.notifier_id, in do_watch_notify()
2929 lwork->notify.payload_len); in do_watch_notify()
2930 lreq->wcb(lreq->data, lwork->notify.notify_id, lreq->linger_id, in do_watch_notify()
2931 lwork->notify.notifier_id, lwork->notify.payload, in do_watch_notify()
2932 lwork->notify.payload_len); in do_watch_notify()
2935 ceph_msg_put(lwork->notify.msg); in do_watch_notify()
2942 struct ceph_osd_linger_request *lreq = lwork->lreq; in do_watch_error()
2945 dout("%s lreq %p not registered\n", __func__, lreq); in do_watch_error()
2949 dout("%s lreq %p err %d\n", __func__, lreq, lwork->error.err); in do_watch_error()
2950 lreq->errcb(lreq->data, lreq->linger_id, lwork->error.err); in do_watch_error()
2962 pr_err("failed to allocate error-lwork\n"); in queue_watch_error()
2966 lwork->error.err = lreq->last_error; in queue_watch_error()
2973 if (!completion_done(&lreq->reg_commit_wait)) { in linger_reg_commit_complete()
2974 lreq->reg_commit_error = (result <= 0 ? result : 0); in linger_reg_commit_complete()
2975 complete_all(&lreq->reg_commit_wait); in linger_reg_commit_complete()
2981 struct ceph_osd_linger_request *lreq = req->r_priv; in linger_commit_cb()
2983 mutex_lock(&lreq->lock); in linger_commit_cb()
2984 if (req != lreq->reg_req) { in linger_commit_cb()
2985 dout("%s lreq %p linger_id %llu unknown req (%p != %p)\n", in linger_commit_cb()
2986 __func__, lreq, lreq->linger_id, req, lreq->reg_req); in linger_commit_cb()
2990 dout("%s lreq %p linger_id %llu result %d\n", __func__, lreq, in linger_commit_cb()
2991 lreq->linger_id, req->r_result); in linger_commit_cb()
2992 linger_reg_commit_complete(lreq, req->r_result); in linger_commit_cb()
2993 lreq->committed = true; in linger_commit_cb()
2995 if (!lreq->is_watch) { in linger_commit_cb()
2998 void *p = page_address(osd_data->pages[0]); in linger_commit_cb()
3000 WARN_ON(req->r_ops[0].op != CEPH_OSD_OP_NOTIFY || in linger_commit_cb()
3001 osd_data->type != CEPH_OSD_DATA_TYPE_PAGES); in linger_commit_cb()
3004 if (req->r_ops[0].outdata_len >= sizeof(u64)) { in linger_commit_cb()
3005 lreq->notify_id = ceph_decode_64(&p); in linger_commit_cb()
3006 dout("lreq %p notify_id %llu\n", lreq, in linger_commit_cb()
3007 lreq->notify_id); in linger_commit_cb()
3009 dout("lreq %p no notify_id\n", lreq); in linger_commit_cb()
3014 mutex_unlock(&lreq->lock); in linger_commit_cb()
3021 * Translate ENOENT -> ENOTCONN so that a delete->disconnection in normalize_watch_error()
3025 if (err == -ENOENT) in normalize_watch_error()
3026 err = -ENOTCONN; in normalize_watch_error()
3033 struct ceph_osd_linger_request *lreq = req->r_priv; in linger_reconnect_cb()
3035 mutex_lock(&lreq->lock); in linger_reconnect_cb()
3036 if (req != lreq->reg_req) { in linger_reconnect_cb()
3037 dout("%s lreq %p linger_id %llu unknown req (%p != %p)\n", in linger_reconnect_cb()
3038 __func__, lreq, lreq->linger_id, req, lreq->reg_req); in linger_reconnect_cb()
3042 dout("%s lreq %p linger_id %llu result %d last_error %d\n", __func__, in linger_reconnect_cb()
3043 lreq, lreq->linger_id, req->r_result, lreq->last_error); in linger_reconnect_cb()
3044 if (req->r_result < 0) { in linger_reconnect_cb()
3045 if (!lreq->last_error) { in linger_reconnect_cb()
3046 lreq->last_error = normalize_watch_error(req->r_result); in linger_reconnect_cb()
3052 mutex_unlock(&lreq->lock); in linger_reconnect_cb()
3058 struct ceph_osd_client *osdc = lreq->osdc; in send_linger()
3063 mutex_lock(&lreq->lock); in send_linger()
3064 dout("%s lreq %p linger_id %llu\n", __func__, lreq, lreq->linger_id); in send_linger()
3066 if (lreq->reg_req) { in send_linger()
3067 if (lreq->reg_req->r_osd) in send_linger()
3068 cancel_linger_request(lreq->reg_req); in send_linger()
3069 ceph_osdc_put_request(lreq->reg_req); in send_linger()
3075 target_copy(&req->r_t, &lreq->t); in send_linger()
3076 req->r_mtime = lreq->mtime; in send_linger()
3078 if (lreq->is_watch && lreq->committed) { in send_linger()
3080 lreq->linger_id, ++lreq->register_gen); in send_linger()
3081 dout("lreq %p reconnect register_gen %u\n", lreq, in send_linger()
3082 req->r_ops[0].watch.gen); in send_linger()
3083 req->r_callback = linger_reconnect_cb; in send_linger()
3085 if (lreq->is_watch) { in send_linger()
3087 lreq->linger_id, 0); in send_linger()
3089 lreq->notify_id = 0; in send_linger()
3091 refcount_inc(&lreq->request_pl->refcnt); in send_linger()
3092 osd_req_op_notify_init(req, 0, lreq->linger_id, in send_linger()
3093 lreq->request_pl); in send_linger()
3096 lreq->notify_id_pages, PAGE_SIZE, 0, false, false); in send_linger()
3098 dout("lreq %p register\n", lreq); in send_linger()
3099 req->r_callback = linger_commit_cb; in send_linger()
3105 req->r_priv = linger_get(lreq); in send_linger()
3106 req->r_linger = true; in send_linger()
3107 lreq->reg_req = req; in send_linger()
3108 mutex_unlock(&lreq->lock); in send_linger()
3115 struct ceph_osd_linger_request *lreq = req->r_priv; in linger_ping_cb()
3117 mutex_lock(&lreq->lock); in linger_ping_cb()
3118 if (req != lreq->ping_req) { in linger_ping_cb()
3119 dout("%s lreq %p linger_id %llu unknown req (%p != %p)\n", in linger_ping_cb()
3120 __func__, lreq, lreq->linger_id, req, lreq->ping_req); in linger_ping_cb()
3124 dout("%s lreq %p linger_id %llu result %d ping_sent %lu last_error %d\n", in linger_ping_cb()
3125 __func__, lreq, lreq->linger_id, req->r_result, lreq->ping_sent, in linger_ping_cb()
3126 lreq->last_error); in linger_ping_cb()
3127 if (lreq->register_gen == req->r_ops[0].watch.gen) { in linger_ping_cb()
3128 if (!req->r_result) { in linger_ping_cb()
3129 lreq->watch_valid_thru = lreq->ping_sent; in linger_ping_cb()
3130 } else if (!lreq->last_error) { in linger_ping_cb()
3131 lreq->last_error = normalize_watch_error(req->r_result); in linger_ping_cb()
3135 dout("lreq %p register_gen %u ignoring old pong %u\n", lreq, in linger_ping_cb()
3136 lreq->register_gen, req->r_ops[0].watch.gen); in linger_ping_cb()
3140 mutex_unlock(&lreq->lock); in linger_ping_cb()
3146 struct ceph_osd_client *osdc = lreq->osdc; in send_linger_ping()
3151 dout("%s PAUSERD\n", __func__); in send_linger_ping()
3155 lreq->ping_sent = jiffies; in send_linger_ping()
3156 dout("%s lreq %p linger_id %llu ping_sent %lu register_gen %u\n", in send_linger_ping()
3157 __func__, lreq, lreq->linger_id, lreq->ping_sent, in send_linger_ping()
3158 lreq->register_gen); in send_linger_ping()
3160 if (lreq->ping_req) { in send_linger_ping()
3161 if (lreq->ping_req->r_osd) in send_linger_ping()
3162 cancel_linger_request(lreq->ping_req); in send_linger_ping()
3163 ceph_osdc_put_request(lreq->ping_req); in send_linger_ping()
3169 target_copy(&req->r_t, &lreq->t); in send_linger_ping()
3170 osd_req_op_watch_init(req, 0, CEPH_OSD_WATCH_OP_PING, lreq->linger_id, in send_linger_ping()
3171 lreq->register_gen); in send_linger_ping()
3172 req->r_callback = linger_ping_cb; in send_linger_ping()
3177 req->r_priv = linger_get(lreq); in send_linger_ping()
3178 req->r_linger = true; in send_linger_ping()
3179 lreq->ping_req = req; in send_linger_ping()
3183 req->r_tid = atomic64_inc_return(&osdc->last_tid); in send_linger_ping()
3184 link_request(lreq->osd, req); in send_linger_ping()
3190 struct ceph_osd_client *osdc = lreq->osdc; in linger_submit()
3193 down_write(&osdc->lock); in linger_submit()
3196 calc_target(osdc, &lreq->t, false); in linger_submit()
3197 osd = lookup_create_osd(osdc, lreq->t.osd, true); in linger_submit()
3201 up_write(&osdc->lock); in linger_submit()
3206 struct ceph_osd_client *osdc = lreq->osdc; in cancel_linger_map_check()
3211 lookup_lreq = lookup_linger_mc(&osdc->linger_map_checks, in cancel_linger_map_check()
3212 lreq->linger_id); in cancel_linger_map_check()
3217 erase_linger_mc(&osdc->linger_map_checks, lreq); in cancel_linger_map_check()
3226 if (lreq->ping_req && lreq->ping_req->r_osd) in __linger_cancel()
3227 cancel_linger_request(lreq->ping_req); in __linger_cancel()
3228 if (lreq->reg_req && lreq->reg_req->r_osd) in __linger_cancel()
3229 cancel_linger_request(lreq->reg_req); in __linger_cancel()
3231 unlink_linger(lreq->osd, lreq); in __linger_cancel()
3237 struct ceph_osd_client *osdc = lreq->osdc; in linger_cancel()
3239 down_write(&osdc->lock); in linger_cancel()
3242 up_write(&osdc->lock); in linger_cancel()
3249 struct ceph_osd_client *osdc = lreq->osdc; in check_linger_pool_dne()
3250 struct ceph_osdmap *map = osdc->osdmap; in check_linger_pool_dne()
3253 WARN_ON(!map->epoch); in check_linger_pool_dne()
3255 if (lreq->register_gen) { in check_linger_pool_dne()
3256 lreq->map_dne_bound = map->epoch; in check_linger_pool_dne()
3257 dout("%s lreq %p linger_id %llu pool disappeared\n", __func__, in check_linger_pool_dne()
3258 lreq, lreq->linger_id); in check_linger_pool_dne()
3260 dout("%s lreq %p linger_id %llu map_dne_bound %u have %u\n", in check_linger_pool_dne()
3261 __func__, lreq, lreq->linger_id, lreq->map_dne_bound, in check_linger_pool_dne()
3262 map->epoch); in check_linger_pool_dne()
3265 if (lreq->map_dne_bound) { in check_linger_pool_dne()
3266 if (map->epoch >= lreq->map_dne_bound) { in check_linger_pool_dne()
3269 lreq->linger_id); in check_linger_pool_dne()
3270 linger_reg_commit_complete(lreq, -ENOENT); in check_linger_pool_dne()
3280 struct ceph_osd_client *osdc = &greq->monc->client->osdc; in linger_map_check_cb()
3282 u64 linger_id = greq->private_data; in linger_map_check_cb()
3284 WARN_ON(greq->result || !greq->u.newest); in linger_map_check_cb()
3286 down_write(&osdc->lock); in linger_map_check_cb()
3287 lreq = lookup_linger_mc(&osdc->linger_map_checks, linger_id); in linger_map_check_cb()
3289 dout("%s linger_id %llu dne\n", __func__, linger_id); in linger_map_check_cb()
3293 dout("%s lreq %p linger_id %llu map_dne_bound %u newest %llu\n", in linger_map_check_cb()
3294 __func__, lreq, lreq->linger_id, lreq->map_dne_bound, in linger_map_check_cb()
3295 greq->u.newest); in linger_map_check_cb()
3296 if (!lreq->map_dne_bound) in linger_map_check_cb()
3297 lreq->map_dne_bound = greq->u.newest; in linger_map_check_cb()
3298 erase_linger_mc(&osdc->linger_map_checks, lreq); in linger_map_check_cb()
3303 up_write(&osdc->lock); in linger_map_check_cb()
3308 struct ceph_osd_client *osdc = lreq->osdc; in send_linger_map_check()
3314 lookup_lreq = lookup_linger_mc(&osdc->linger_map_checks, in send_linger_map_check()
3315 lreq->linger_id); in send_linger_map_check()
3322 insert_linger_mc(&osdc->linger_map_checks, lreq); in send_linger_map_check()
3323 ret = ceph_monc_get_version_async(&osdc->client->monc, "osdmap", in send_linger_map_check()
3324 linger_map_check_cb, lreq->linger_id); in send_linger_map_check()
3332 dout("%s lreq %p linger_id %llu\n", __func__, lreq, lreq->linger_id); in linger_reg_commit_wait()
3333 ret = wait_for_completion_killable(&lreq->reg_commit_wait); in linger_reg_commit_wait()
3334 return ret ?: lreq->reg_commit_error; in linger_reg_commit_wait()
3342 dout("%s lreq %p linger_id %llu\n", __func__, lreq, lreq->linger_id); in linger_notify_finish_wait()
3343 left = wait_for_completion_killable_timeout(&lreq->notify_finish_wait, in linger_notify_finish_wait()
3346 left = left ?: -ETIMEDOUT; in linger_notify_finish_wait()
3348 left = lreq->notify_finish_error; /* completed */ in linger_notify_finish_wait()
3363 struct ceph_options *opts = osdc->client->options; in handle_timeout()
3364 unsigned long cutoff = jiffies - opts->osd_keepalive_timeout; in handle_timeout()
3365 unsigned long expiry_cutoff = jiffies - opts->osd_request_timeout; in handle_timeout()
3369 dout("%s osdc %p\n", __func__, osdc); in handle_timeout()
3370 down_write(&osdc->lock); in handle_timeout()
3377 for (n = rb_first(&osdc->osds); n; n = rb_next(n)) { in handle_timeout()
3381 for (p = rb_first(&osd->o_requests); p; ) { in handle_timeout()
3387 if (time_before(req->r_stamp, cutoff)) { in handle_timeout()
3388 dout(" req %p tid %llu on osd%d is laggy\n", in handle_timeout()
3389 req, req->r_tid, osd->o_osd); in handle_timeout()
3392 if (opts->osd_request_timeout && in handle_timeout()
3393 time_before(req->r_start_stamp, expiry_cutoff)) { in handle_timeout()
3395 req->r_tid, osd->o_osd); in handle_timeout()
3396 abort_request(req, -ETIMEDOUT); in handle_timeout()
3399 for (p = rb_first(&osd->o_linger_requests); p; p = rb_next(p)) { in handle_timeout()
3403 dout(" lreq %p linger_id %llu is served by osd%d\n", in handle_timeout()
3404 lreq, lreq->linger_id, osd->o_osd); in handle_timeout()
3407 mutex_lock(&lreq->lock); in handle_timeout()
3408 if (lreq->is_watch && lreq->committed && !lreq->last_error) in handle_timeout()
3410 mutex_unlock(&lreq->lock); in handle_timeout()
3414 list_move_tail(&osd->o_keepalive_item, &slow_osds); in handle_timeout()
3417 if (opts->osd_request_timeout) { in handle_timeout()
3418 for (p = rb_first(&osdc->homeless_osd.o_requests); p; ) { in handle_timeout()
3424 if (time_before(req->r_start_stamp, expiry_cutoff)) { in handle_timeout()
3426 req->r_tid, osdc->homeless_osd.o_osd); in handle_timeout()
3427 abort_request(req, -ETIMEDOUT); in handle_timeout()
3432 if (atomic_read(&osdc->num_homeless) || !list_empty(&slow_osds)) in handle_timeout()
3439 list_del_init(&osd->o_keepalive_item); in handle_timeout()
3440 ceph_con_keepalive(&osd->o_con); in handle_timeout()
3443 up_write(&osdc->lock); in handle_timeout()
3444 schedule_delayed_work(&osdc->timeout_work, in handle_timeout()
3445 osdc->client->options->osd_keepalive_timeout); in handle_timeout()
3453 unsigned long delay = osdc->client->options->osd_idle_ttl / 4; in handle_osds_timeout()
3456 dout("%s osdc %p\n", __func__, osdc); in handle_osds_timeout()
3457 down_write(&osdc->lock); in handle_osds_timeout()
3458 list_for_each_entry_safe(osd, nosd, &osdc->osd_lru, o_osd_lru) { in handle_osds_timeout()
3459 if (time_before(jiffies, osd->lru_ttl)) in handle_osds_timeout()
3462 WARN_ON(!RB_EMPTY_ROOT(&osd->o_requests)); in handle_osds_timeout()
3463 WARN_ON(!RB_EMPTY_ROOT(&osd->o_linger_requests)); in handle_osds_timeout()
3467 up_write(&osdc->lock); in handle_osds_timeout()
3468 schedule_delayed_work(&osdc->osds_timeout_work, in handle_osds_timeout()
3497 oloc->pool = ceph_decode_64(p); in ceph_oloc_decode()
3512 if (!oloc->pool_ns || in ceph_oloc_decode()
3513 ceph_compare_string(oloc->pool_ns, *p, len)) in ceph_oloc_decode()
3517 if (oloc->pool_ns) in ceph_oloc_decode()
3529 if (hash != -1) { in ceph_oloc_decode()
3541 ret = -EINVAL; in ceph_oloc_decode()
3565 ret = ceph_oloc_decode(p, end, &redir->oloc); in ceph_redirect_decode()
3581 ret = -EINVAL; in ceph_redirect_decode()
3601 void *p = msg->front.iov_base; in decode_MOSDOpReply()
3602 void *const end = p + msg->front.iov_len; in decode_MOSDOpReply()
3603 u16 version = le16_to_cpu(msg->hdr.version); in decode_MOSDOpReply()
3614 ret = ceph_decode_pgid(&p, end, &m->pgid); in decode_MOSDOpReply()
3618 ceph_decode_64_safe(&p, end, m->flags, e_inval); in decode_MOSDOpReply()
3619 ceph_decode_32_safe(&p, end, m->result, e_inval); in decode_MOSDOpReply()
3623 ceph_decode_32_safe(&p, end, m->epoch, e_inval); in decode_MOSDOpReply()
3625 ceph_decode_32_safe(&p, end, m->num_ops, e_inval); in decode_MOSDOpReply()
3626 if (m->num_ops > ARRAY_SIZE(m->outdata_len)) in decode_MOSDOpReply()
3629 ceph_decode_need(&p, end, m->num_ops * sizeof(struct ceph_osd_op), in decode_MOSDOpReply()
3631 for (i = 0; i < m->num_ops; i++) { in decode_MOSDOpReply()
3634 m->outdata_len[i] = le32_to_cpu(op->payload_len); in decode_MOSDOpReply()
3638 ceph_decode_32_safe(&p, end, m->retry_attempt, e_inval); in decode_MOSDOpReply()
3639 for (i = 0; i < m->num_ops; i++) in decode_MOSDOpReply()
3640 ceph_decode_32_safe(&p, end, m->rval[i], e_inval); in decode_MOSDOpReply()
3643 ceph_decode_need(&p, end, sizeof(m->replay_version), e_inval); in decode_MOSDOpReply()
3644 memcpy(&m->replay_version, p, sizeof(m->replay_version)); in decode_MOSDOpReply()
3645 p += sizeof(m->replay_version); in decode_MOSDOpReply()
3646 ceph_decode_64_safe(&p, end, m->user_version, e_inval); in decode_MOSDOpReply()
3648 m->replay_version = bad_replay_version; /* struct */ in decode_MOSDOpReply()
3649 m->user_version = le64_to_cpu(m->replay_version.version); in decode_MOSDOpReply()
3662 ret = ceph_redirect_decode(&p, end, &m->redirect); in decode_MOSDOpReply()
3666 ceph_oloc_init(&m->redirect.oloc); in decode_MOSDOpReply()
3672 return -EINVAL; in decode_MOSDOpReply()
3676 * Handle MOSDOpReply. Set ->r_result and call the callback if it is
3681 struct ceph_osd_client *osdc = osd->o_osdc; in handle_reply()
3684 u64 tid = le64_to_cpu(msg->hdr.tid); in handle_reply()
3689 dout("%s msg %p tid %llu\n", __func__, msg, tid); in handle_reply()
3691 down_read(&osdc->lock); in handle_reply()
3693 dout("%s osd%d unknown\n", __func__, osd->o_osd); in handle_reply()
3696 WARN_ON(osd->o_osd != le64_to_cpu(msg->hdr.src.num)); in handle_reply()
3698 mutex_lock(&osd->lock); in handle_reply()
3699 req = lookup_request(&osd->o_requests, tid); in handle_reply()
3701 dout("%s osd%d tid %llu unknown\n", __func__, osd->o_osd, tid); in handle_reply()
3705 m.redirect.oloc.pool_ns = req->r_t.target_oloc.pool_ns; in handle_reply()
3710 req->r_tid, ret); in handle_reply()
3714 dout("%s req %p tid %llu flags 0x%llx pgid %llu.%x epoch %u attempt %d v %u'%llu uv %llu\n", in handle_reply()
3715 __func__, req, req->r_tid, m.flags, m.pgid.pool, m.pgid.seed, in handle_reply()
3720 if (m.retry_attempt != req->r_attempts - 1) { in handle_reply()
3721 dout("req %p tid %llu retry_attempt %d != %d, ignoring\n", in handle_reply()
3722 req, req->r_tid, m.retry_attempt, in handle_reply()
3723 req->r_attempts - 1); in handle_reply()
3731 dout("req %p tid %llu redirect pool %lld\n", req, req->r_tid, in handle_reply()
3734 mutex_unlock(&osd->lock); in handle_reply()
3737 * Not ceph_oloc_copy() - changing pool_ns is not in handle_reply()
3740 req->r_t.target_oloc.pool = m.redirect.oloc.pool; in handle_reply()
3741 req->r_flags |= CEPH_OSD_FLAG_REDIRECTED | in handle_reply()
3744 req->r_tid = 0; in handle_reply()
3749 if (m.result == -EAGAIN) { in handle_reply()
3750 dout("req %p tid %llu EAGAIN\n", req, req->r_tid); in handle_reply()
3752 mutex_unlock(&osd->lock); in handle_reply()
3759 req->r_t.pgid.pool = 0; in handle_reply()
3760 req->r_t.pgid.seed = 0; in handle_reply()
3761 WARN_ON(!req->r_t.used_replica); in handle_reply()
3762 req->r_flags &= ~(CEPH_OSD_FLAG_BALANCE_READS | in handle_reply()
3764 req->r_tid = 0; in handle_reply()
3769 if (m.num_ops != req->r_num_ops) { in handle_reply()
3771 req->r_num_ops, req->r_tid); in handle_reply()
3774 for (i = 0; i < req->r_num_ops; i++) { in handle_reply()
3775 dout(" req %p tid %llu op %d rval %d len %u\n", req, in handle_reply()
3776 req->r_tid, i, m.rval[i], m.outdata_len[i]); in handle_reply()
3777 req->r_ops[i].rval = m.rval[i]; in handle_reply()
3778 req->r_ops[i].outdata_len = m.outdata_len[i]; in handle_reply()
3781 if (data_len != le32_to_cpu(msg->hdr.data_len)) { in handle_reply()
3783 le32_to_cpu(msg->hdr.data_len), req->r_tid); in handle_reply()
3786 dout("%s req %p tid %llu result %d data_len %u\n", __func__, in handle_reply()
3787 req, req->r_tid, m.result, data_len); in handle_reply()
3794 req->r_result = m.result ?: data_len; in handle_reply()
3796 mutex_unlock(&osd->lock); in handle_reply()
3797 up_read(&osdc->lock); in handle_reply()
3803 complete_request(req, -EIO); in handle_reply()
3805 mutex_unlock(&osd->lock); in handle_reply()
3807 up_read(&osdc->lock); in handle_reply()
3814 for (n = rb_first(&osdc->osdmap->pg_pools); n; n = rb_next(n)) { in set_pool_was_full()
3818 pi->was_full = __pool_full(pi); in set_pool_was_full()
3826 pi = ceph_pg_pool_by_id(osdc->osdmap, pool_id); in pool_cleared_full()
3830 return pi->was_full && !__pool_full(pi); in pool_cleared_full()
3836 struct ceph_osd_client *osdc = lreq->osdc; in recalc_linger_target()
3839 ct_res = calc_target(osdc, &lreq->t, true); in recalc_linger_target()
3843 osd = lookup_create_osd(osdc, lreq->t.osd, true); in recalc_linger_target()
3844 if (osd != lreq->osd) { in recalc_linger_target()
3845 unlink_linger(lreq->osd, lreq); in recalc_linger_target()
3863 struct ceph_osd_client *osdc = osd->o_osdc; in scan_requests()
3867 for (n = rb_first(&osd->o_linger_requests); n; ) { in scan_requests()
3874 dout("%s lreq %p linger_id %llu\n", __func__, lreq, in scan_requests()
3875 lreq->linger_id); in scan_requests()
3881 pool_cleared_full(osdc, lreq->t.base_oloc.pool)); in scan_requests()
3893 if (list_empty(&lreq->scan_item)) in scan_requests()
3894 list_add_tail(&lreq->scan_item, need_resend_linger); in scan_requests()
3897 list_del_init(&lreq->scan_item); in scan_requests()
3903 for (n = rb_first(&osd->o_requests); n; ) { in scan_requests()
3910 dout("%s req %p tid %llu\n", __func__, req, req->r_tid); in scan_requests()
3911 ct_res = calc_target(osdc, &req->r_t, false); in scan_requests()
3916 pool_cleared_full(osdc, req->r_t.base_oloc.pool)); in scan_requests()
3918 (!(req->r_flags & CEPH_OSD_FLAG_WRITE) || in scan_requests()
3949 newmap = osdmap_apply_incremental(&p, end, osdc->osdmap); in handle_one_map()
3955 if (newmap != osdc->osdmap) { in handle_one_map()
3957 * Preserve ->was_full before destroying the old map. in handle_one_map()
3958 * For pools that weren't in the old map, ->was_full in handle_one_map()
3961 for (n = rb_first(&newmap->pg_pools); n; n = rb_next(n)) { in handle_one_map()
3966 old_pi = ceph_pg_pool_by_id(osdc->osdmap, pi->id); in handle_one_map()
3968 pi->was_full = old_pi->was_full; in handle_one_map()
3970 WARN_ON(pi->was_full); in handle_one_map()
3973 if (osdc->osdmap->epoch && in handle_one_map()
3974 osdc->osdmap->epoch + 1 < newmap->epoch) { in handle_one_map()
3979 ceph_osdmap_destroy(osdc->osdmap); in handle_one_map()
3980 osdc->osdmap = newmap; in handle_one_map()
3984 scan_requests(&osdc->homeless_osd, skipped_map, was_full, true, in handle_one_map()
3987 for (n = rb_first(&osdc->osds); n; ) { in handle_one_map()
3994 if (!ceph_osd_is_up(osdc->osdmap, osd->o_osd) || in handle_one_map()
3995 memcmp(&osd->o_con.peer_addr, in handle_one_map()
3996 ceph_osd_addr(osdc->osdmap, osd->o_osd), in handle_one_map()
4019 if (req->r_t.epoch < osdc->osdmap->epoch) { in kick_requests()
4020 ct_res = calc_target(osdc, &req->r_t, false); in kick_requests()
4036 osd = lookup_create_osd(osdc, req->r_t.osd, true); in kick_requests()
4038 if (!req->r_linger) { in kick_requests()
4039 if (!osd_homeless(osd) && !req->r_t.paused) in kick_requests()
4047 if (!osd_homeless(lreq->osd)) in kick_requests()
4050 list_del_init(&lreq->scan_item); in kick_requests()
4063 void *p = msg->front.iov_base; in ceph_osdc_handle_map()
4064 void *const end = p + msg->front.iov_len; in ceph_osdc_handle_map()
4075 dout("%s have %u\n", __func__, osdc->osdmap->epoch); in ceph_osdc_handle_map()
4076 down_write(&osdc->lock); in ceph_osdc_handle_map()
4081 if (ceph_check_fsid(osdc->client, &fsid) < 0) in ceph_osdc_handle_map()
4091 dout(" %d inc maps\n", nr_maps); in ceph_osdc_handle_map()
4097 if (osdc->osdmap->epoch && in ceph_osdc_handle_map()
4098 osdc->osdmap->epoch + 1 == epoch) { in ceph_osdc_handle_map()
4099 dout("applying incremental map %u len %d\n", in ceph_osdc_handle_map()
4107 dout("ignoring incremental map %u len %d\n", in ceph_osdc_handle_map()
4111 nr_maps--; in ceph_osdc_handle_map()
4118 dout(" %d full maps\n", nr_maps); in ceph_osdc_handle_map()
4125 dout("skipping non-latest full map %u len %d\n", in ceph_osdc_handle_map()
4127 } else if (osdc->osdmap->epoch >= epoch) { in ceph_osdc_handle_map()
4128 dout("skipping full map %u len %d, " in ceph_osdc_handle_map()
4130 osdc->osdmap->epoch); in ceph_osdc_handle_map()
4132 dout("taking full map %u len %d\n", epoch, maplen); in ceph_osdc_handle_map()
4139 nr_maps--; in ceph_osdc_handle_map()
4153 osdc->osdmap->epoch < osdc->epoch_barrier) in ceph_osdc_handle_map()
4159 ceph_monc_got_map(&osdc->client->monc, CEPH_SUB_OSDMAP, in ceph_osdc_handle_map()
4160 osdc->osdmap->epoch); in ceph_osdc_handle_map()
4161 up_write(&osdc->lock); in ceph_osdc_handle_map()
4162 wake_up_all(&osdc->client->auth_wq); in ceph_osdc_handle_map()
4168 up_write(&osdc->lock); in ceph_osdc_handle_map()
4180 for (n = rb_first(&osd->o_requests); n; ) { in kick_osd_requests()
4186 if (!req->r_linger) { in kick_osd_requests()
4187 if (!req->r_t.paused) in kick_osd_requests()
4193 for (n = rb_first(&osd->o_linger_requests); n; n = rb_next(n)) { in kick_osd_requests()
4206 struct ceph_osd *osd = con->private; in osd_fault()
4207 struct ceph_osd_client *osdc = osd->o_osdc; in osd_fault()
4209 dout("%s osd %p osd%d\n", __func__, osd, osd->o_osd); in osd_fault()
4211 down_write(&osdc->lock); in osd_fault()
4213 dout("%s osd%d unknown\n", __func__, osd->o_osd); in osd_fault()
4222 up_write(&osdc->lock); in osd_fault()
4236 void *p = msg->front.iov_base; in decode_MOSDBackoff()
4237 void *const end = p + msg->front.iov_len; in decode_MOSDBackoff()
4246 ret = ceph_decode_pgid(&p, end, &m->spgid.pgid); in decode_MOSDBackoff()
4250 ceph_decode_8_safe(&p, end, m->spgid.shard, e_inval); in decode_MOSDBackoff()
4251 ceph_decode_32_safe(&p, end, m->map_epoch, e_inval); in decode_MOSDBackoff()
4252 ceph_decode_8_safe(&p, end, m->op, e_inval); in decode_MOSDBackoff()
4253 ceph_decode_64_safe(&p, end, m->id, e_inval); in decode_MOSDBackoff()
4255 m->begin = kzalloc(sizeof(*m->begin), GFP_NOIO); in decode_MOSDBackoff()
4256 if (!m->begin) in decode_MOSDBackoff()
4257 return -ENOMEM; in decode_MOSDBackoff()
4259 ret = decode_hoid(&p, end, m->begin); in decode_MOSDBackoff()
4261 free_hoid(m->begin); in decode_MOSDBackoff()
4265 m->end = kzalloc(sizeof(*m->end), GFP_NOIO); in decode_MOSDBackoff()
4266 if (!m->end) { in decode_MOSDBackoff()
4267 free_hoid(m->begin); in decode_MOSDBackoff()
4268 return -ENOMEM; in decode_MOSDBackoff()
4271 ret = decode_hoid(&p, end, m->end); in decode_MOSDBackoff()
4273 free_hoid(m->begin); in decode_MOSDBackoff()
4274 free_hoid(m->end); in decode_MOSDBackoff()
4281 return -EINVAL; in decode_MOSDBackoff()
4296 hoid_encoding_size(backoff->begin); in create_backoff_message()
4298 hoid_encoding_size(backoff->end); in create_backoff_message()
4304 p = msg->front.iov_base; in create_backoff_message()
4305 end = p + msg->front_alloc_len; in create_backoff_message()
4307 encode_spgid(&p, &backoff->spgid); in create_backoff_message()
4310 ceph_encode_64(&p, backoff->id); in create_backoff_message()
4311 encode_hoid(&p, end, backoff->begin); in create_backoff_message()
4312 encode_hoid(&p, end, backoff->end); in create_backoff_message()
4315 msg->front.iov_len = p - msg->front.iov_base; in create_backoff_message()
4316 msg->hdr.version = cpu_to_le16(1); /* MOSDBackoff v1 */ in create_backoff_message()
4317 msg->hdr.front_len = cpu_to_le32(msg->front.iov_len); in create_backoff_message()
4328 dout("%s osd%d spgid %llu.%xs%d id %llu\n", __func__, osd->o_osd, in handle_backoff_block()
4329 m->spgid.pgid.pool, m->spgid.pgid.seed, m->spgid.shard, m->id); in handle_backoff_block()
4331 spg = lookup_spg_mapping(&osd->o_backoff_mappings, &m->spgid); in handle_backoff_block()
4338 spg->spgid = m->spgid; /* struct */ in handle_backoff_block()
4339 insert_spg_mapping(&osd->o_backoff_mappings, spg); in handle_backoff_block()
4347 backoff->spgid = m->spgid; /* struct */ in handle_backoff_block()
4348 backoff->id = m->id; in handle_backoff_block()
4349 backoff->begin = m->begin; in handle_backoff_block()
4350 m->begin = NULL; /* backoff now owns this */ in handle_backoff_block()
4351 backoff->end = m->end; in handle_backoff_block()
4352 m->end = NULL; /* ditto */ in handle_backoff_block()
4354 insert_backoff(&spg->backoffs, backoff); in handle_backoff_block()
4355 insert_backoff_by_id(&osd->o_backoffs_by_id, backoff); in handle_backoff_block()
4361 msg = create_backoff_message(backoff, m->map_epoch); in handle_backoff_block()
4366 ceph_con_send(&osd->o_con, msg); in handle_backoff_block()
4388 dout("%s osd%d spgid %llu.%xs%d id %llu\n", __func__, osd->o_osd, in handle_backoff_unblock()
4389 m->spgid.pgid.pool, m->spgid.pgid.seed, m->spgid.shard, m->id); in handle_backoff_unblock()
4391 backoff = lookup_backoff_by_id(&osd->o_backoffs_by_id, m->id); in handle_backoff_unblock()
4394 __func__, osd->o_osd, m->spgid.pgid.pool, in handle_backoff_unblock()
4395 m->spgid.pgid.seed, m->spgid.shard, m->id); in handle_backoff_unblock()
4399 if (hoid_compare(backoff->begin, m->begin) && in handle_backoff_unblock()
4400 hoid_compare(backoff->end, m->end)) { in handle_backoff_unblock()
4402 __func__, osd->o_osd, m->spgid.pgid.pool, in handle_backoff_unblock()
4403 m->spgid.pgid.seed, m->spgid.shard, m->id); in handle_backoff_unblock()
4407 spg = lookup_spg_mapping(&osd->o_backoff_mappings, &backoff->spgid); in handle_backoff_unblock()
4410 erase_backoff(&spg->backoffs, backoff); in handle_backoff_unblock()
4411 erase_backoff_by_id(&osd->o_backoffs_by_id, backoff); in handle_backoff_unblock()
4414 if (RB_EMPTY_ROOT(&spg->backoffs)) { in handle_backoff_unblock()
4415 erase_spg_mapping(&osd->o_backoff_mappings, spg); in handle_backoff_unblock()
4419 for (n = rb_first(&osd->o_requests); n; n = rb_next(n)) { in handle_backoff_unblock()
4423 if (!ceph_spg_compare(&req->r_t.spgid, &m->spgid)) { in handle_backoff_unblock()
4425 * Match against @m, not @backoff -- the PG may in handle_backoff_unblock()
4428 if (target_contained_by(&req->r_t, m->begin, m->end)) { in handle_backoff_unblock()
4441 struct ceph_osd_client *osdc = osd->o_osdc; in handle_backoff()
4445 down_read(&osdc->lock); in handle_backoff()
4447 dout("%s osd%d unknown\n", __func__, osd->o_osd); in handle_backoff()
4448 up_read(&osdc->lock); in handle_backoff()
4451 WARN_ON(osd->o_osd != le64_to_cpu(msg->hdr.src.num)); in handle_backoff()
4453 mutex_lock(&osd->lock); in handle_backoff()
4468 default: in handle_backoff()
4469 pr_err("%s osd%d unknown op %d\n", __func__, osd->o_osd, m.op); in handle_backoff()
4476 mutex_unlock(&osd->lock); in handle_backoff()
4477 up_read(&osdc->lock); in handle_backoff()
4486 void *p = msg->front.iov_base; in handle_watch_notify()
4487 void *const end = p + msg->front.iov_len; in handle_watch_notify()
4510 if (le16_to_cpu(msg->hdr.version) >= 2) in handle_watch_notify()
4513 if (le16_to_cpu(msg->hdr.version) >= 3) in handle_watch_notify()
4516 down_read(&osdc->lock); in handle_watch_notify()
4517 lreq = lookup_linger_osdc(&osdc->linger_requests, cookie); in handle_watch_notify()
4519 dout("%s opcode %d cookie %llu dne\n", __func__, opcode, in handle_watch_notify()
4524 mutex_lock(&lreq->lock); in handle_watch_notify()
4525 dout("%s opcode %d cookie %llu lreq %p is_watch %d\n", __func__, in handle_watch_notify()
4526 opcode, cookie, lreq, lreq->is_watch); in handle_watch_notify()
4528 if (!lreq->last_error) { in handle_watch_notify()
4529 lreq->last_error = -ENOTCONN; in handle_watch_notify()
4532 } else if (!lreq->is_watch) { in handle_watch_notify()
4534 if (lreq->notify_id && lreq->notify_id != notify_id) { in handle_watch_notify()
4535 dout("lreq %p notify_id %llu != %llu, ignoring\n", lreq, in handle_watch_notify()
4536 lreq->notify_id, notify_id); in handle_watch_notify()
4537 } else if (!completion_done(&lreq->notify_finish_wait)) { in handle_watch_notify()
4539 msg->num_data_items ? &msg->data[0] : NULL; in handle_watch_notify()
4542 if (lreq->preply_pages) { in handle_watch_notify()
4543 WARN_ON(data->type != in handle_watch_notify()
4545 *lreq->preply_pages = data->pages; in handle_watch_notify()
4546 *lreq->preply_len = data->length; in handle_watch_notify()
4547 data->own_pages = false; in handle_watch_notify()
4550 lreq->notify_finish_error = return_code; in handle_watch_notify()
4551 complete_all(&lreq->notify_finish_wait); in handle_watch_notify()
4557 pr_err("failed to allocate notify-lwork\n"); in handle_watch_notify()
4561 lwork->notify.notify_id = notify_id; in handle_watch_notify()
4562 lwork->notify.notifier_id = notifier_id; in handle_watch_notify()
4563 lwork->notify.payload = payload; in handle_watch_notify()
4564 lwork->notify.payload_len = payload_len; in handle_watch_notify()
4565 lwork->notify.msg = ceph_msg_get(msg); in handle_watch_notify()
4570 mutex_unlock(&lreq->lock); in handle_watch_notify()
4572 up_read(&osdc->lock); in handle_watch_notify()
4586 down_read(&osdc->lock); in ceph_osdc_start_request()
4588 up_read(&osdc->lock); in ceph_osdc_start_request()
4596 * ->r_result isn't set and __complete_request() isn't called.
4600 struct ceph_osd_client *osdc = req->r_osdc; in ceph_osdc_cancel_request()
4602 down_write(&osdc->lock); in ceph_osdc_cancel_request()
4603 if (req->r_osd) in ceph_osdc_cancel_request()
4605 up_write(&osdc->lock); in ceph_osdc_cancel_request()
4617 dout("%s req %p tid %llu\n", __func__, req, req->r_tid); in wait_request_timeout()
4618 left = wait_for_completion_killable_timeout(&req->r_completion, in wait_request_timeout()
4621 left = left ?: -ETIMEDOUT; in wait_request_timeout()
4624 left = req->r_result; /* completed */ in wait_request_timeout()
4641 * sync - wait for all in-flight requests to flush. avoid starvation.
4646 u64 last_tid = atomic64_read(&osdc->last_tid); in ceph_osdc_sync()
4649 down_read(&osdc->lock); in ceph_osdc_sync()
4650 for (n = rb_first(&osdc->osds); n; n = rb_next(n)) { in ceph_osdc_sync()
4653 mutex_lock(&osd->lock); in ceph_osdc_sync()
4654 for (p = rb_first(&osd->o_requests); p; p = rb_next(p)) { in ceph_osdc_sync()
4658 if (req->r_tid > last_tid) in ceph_osdc_sync()
4661 if (!(req->r_flags & CEPH_OSD_FLAG_WRITE)) in ceph_osdc_sync()
4665 mutex_unlock(&osd->lock); in ceph_osdc_sync()
4666 up_read(&osdc->lock); in ceph_osdc_sync()
4667 dout("%s waiting on req %p tid %llu last_tid %llu\n", in ceph_osdc_sync()
4668 __func__, req, req->r_tid, last_tid); in ceph_osdc_sync()
4669 wait_for_completion(&req->r_completion); in ceph_osdc_sync()
4674 mutex_unlock(&osd->lock); in ceph_osdc_sync()
4677 up_read(&osdc->lock); in ceph_osdc_sync()
4678 dout("%s done last_tid %llu\n", __func__, last_tid); in ceph_osdc_sync()
4698 return ERR_PTR(-ENOMEM); in ceph_osdc_watch()
4700 lreq->is_watch = true; in ceph_osdc_watch()
4701 lreq->wcb = wcb; in ceph_osdc_watch()
4702 lreq->errcb = errcb; in ceph_osdc_watch()
4703 lreq->data = data; in ceph_osdc_watch()
4704 lreq->watch_valid_thru = jiffies; in ceph_osdc_watch()
4706 ceph_oid_copy(&lreq->t.base_oid, oid); in ceph_osdc_watch()
4707 ceph_oloc_copy(&lreq->t.base_oloc, oloc); in ceph_osdc_watch()
4708 lreq->t.flags = CEPH_OSD_FLAG_WRITE; in ceph_osdc_watch()
4709 ktime_get_real_ts64(&lreq->mtime); in ceph_osdc_watch()
4736 struct ceph_options *opts = osdc->client->options; in ceph_osdc_unwatch()
4742 return -ENOMEM; in ceph_osdc_unwatch()
4744 ceph_oid_copy(&req->r_base_oid, &lreq->t.base_oid); in ceph_osdc_unwatch()
4745 ceph_oloc_copy(&req->r_base_oloc, &lreq->t.base_oloc); in ceph_osdc_unwatch()
4746 req->r_flags = CEPH_OSD_FLAG_WRITE; in ceph_osdc_unwatch()
4747 ktime_get_real_ts64(&req->r_mtime); in ceph_osdc_unwatch()
4749 lreq->linger_id, 0); in ceph_osdc_unwatch()
4758 ret = wait_request_timeout(req, opts->mount_timeout); in ceph_osdc_unwatch()
4778 return -ENOMEM; in osd_req_op_notify_ack_init()
4790 return -ENOMEM; in osd_req_op_notify_ack_init()
4793 ceph_osd_data_pagelist_init(&op->notify_ack.request_data, pl); in osd_req_op_notify_ack_init()
4794 op->indata_len = pl->length; in osd_req_op_notify_ack_init()
4811 return -ENOMEM; in ceph_osdc_notify_ack()
4813 ceph_oid_copy(&req->r_base_oid, oid); in ceph_osdc_notify_ack()
4814 ceph_oloc_copy(&req->r_base_oloc, oloc); in ceph_osdc_notify_ack()
4815 req->r_flags = CEPH_OSD_FLAG_READ; in ceph_osdc_notify_ack()
4863 return -ENOMEM; in ceph_osdc_notify()
4865 lreq->request_pl = ceph_pagelist_alloc(GFP_NOIO); in ceph_osdc_notify()
4866 if (!lreq->request_pl) { in ceph_osdc_notify()
4867 ret = -ENOMEM; in ceph_osdc_notify()
4871 ret = ceph_pagelist_encode_32(lreq->request_pl, 1); /* prot_ver */ in ceph_osdc_notify()
4872 ret |= ceph_pagelist_encode_32(lreq->request_pl, timeout); in ceph_osdc_notify()
4873 ret |= ceph_pagelist_encode_32(lreq->request_pl, payload_len); in ceph_osdc_notify()
4874 ret |= ceph_pagelist_append(lreq->request_pl, payload, payload_len); in ceph_osdc_notify()
4876 ret = -ENOMEM; in ceph_osdc_notify()
4881 lreq->notify_id_pages = ceph_alloc_page_vector(1, GFP_NOIO); in ceph_osdc_notify()
4882 if (IS_ERR(lreq->notify_id_pages)) { in ceph_osdc_notify()
4883 ret = PTR_ERR(lreq->notify_id_pages); in ceph_osdc_notify()
4884 lreq->notify_id_pages = NULL; in ceph_osdc_notify()
4888 lreq->preply_pages = preply_pages; in ceph_osdc_notify()
4889 lreq->preply_len = preply_len; in ceph_osdc_notify()
4891 ceph_oid_copy(&lreq->t.base_oid, oid); in ceph_osdc_notify()
4892 ceph_oloc_copy(&lreq->t.base_oloc, oloc); in ceph_osdc_notify()
4893 lreq->t.flags = CEPH_OSD_FLAG_READ; in ceph_osdc_notify()
4901 dout("lreq %p failed to initiate notify %d\n", lreq, ret); in ceph_osdc_notify()
4921 down_read(&osdc->lock); in ceph_osdc_watch_check()
4922 mutex_lock(&lreq->lock); in ceph_osdc_watch_check()
4923 stamp = lreq->watch_valid_thru; in ceph_osdc_watch_check()
4924 if (!list_empty(&lreq->pending_lworks)) { in ceph_osdc_watch_check()
4926 list_first_entry(&lreq->pending_lworks, in ceph_osdc_watch_check()
4930 if (time_before(lwork->queued_stamp, stamp)) in ceph_osdc_watch_check()
4931 stamp = lwork->queued_stamp; in ceph_osdc_watch_check()
4933 age = jiffies - stamp; in ceph_osdc_watch_check()
4934 dout("%s lreq %p linger_id %llu age %lu last_error %d\n", __func__, in ceph_osdc_watch_check()
4935 lreq, lreq->linger_id, age, lreq->last_error); in ceph_osdc_watch_check()
4937 ret = lreq->last_error ?: 1 + jiffies_to_msecs(age); in ceph_osdc_watch_check()
4939 mutex_unlock(&lreq->lock); in ceph_osdc_watch_check()
4940 up_read(&osdc->lock); in ceph_osdc_watch_check()
4955 ret = -EINVAL; in decode_watcher()
4956 ceph_decode_copy_safe(p, end, &item->name, sizeof(item->name), bad); in decode_watcher()
4957 ceph_decode_64_safe(p, end, item->cookie, bad); in decode_watcher()
4961 ret = ceph_decode_entity_addr(p, end, &item->addr); in decode_watcher()
4968 dout("%s %s%llu cookie %llu addr %s\n", __func__, in decode_watcher()
4969 ENTITY_NAME(item->name), item->cookie, in decode_watcher()
4970 ceph_pr_addr(&item->addr)); in decode_watcher()
4992 return -ENOMEM; in decode_watchers()
5022 return -ENOMEM; in ceph_osdc_list_watchers()
5024 ceph_oid_copy(&req->r_base_oid, oid); in ceph_osdc_list_watchers()
5025 ceph_oloc_copy(&req->r_base_oloc, oloc); in ceph_osdc_list_watchers()
5026 req->r_flags = CEPH_OSD_FLAG_READ; in ceph_osdc_list_watchers()
5047 void *const end = p + req->r_ops[0].outdata_len; in ceph_osdc_list_watchers()
5059 * Call all pending notify callbacks - for use after a watch is
5064 dout("%s osdc %p\n", __func__, osdc); in ceph_osdc_flush_notifies()
5065 flush_workqueue(osdc->notify_wq); in ceph_osdc_flush_notifies()
5071 down_read(&osdc->lock); in ceph_osdc_maybe_request_map()
5073 up_read(&osdc->lock); in ceph_osdc_maybe_request_map()
5095 return -E2BIG; in ceph_osdc_call()
5099 return -ENOMEM; in ceph_osdc_call()
5101 ceph_oid_copy(&req->r_base_oid, oid); in ceph_osdc_call()
5102 ceph_oloc_copy(&req->r_base_oloc, oloc); in ceph_osdc_call()
5103 req->r_flags = flags; in ceph_osdc_call()
5123 ret = req->r_ops[0].rval; in ceph_osdc_call()
5125 *resp_len = req->r_ops[0].outdata_len; in ceph_osdc_call()
5141 down_write(&osdc->lock); in ceph_osdc_reopen_osds()
5142 for (n = rb_first(&osdc->osds); n; ) { in ceph_osdc_reopen_osds()
5149 up_write(&osdc->lock); in ceph_osdc_reopen_osds()
5159 dout("init\n"); in ceph_osdc_init()
5160 osdc->client = client; in ceph_osdc_init()
5161 init_rwsem(&osdc->lock); in ceph_osdc_init()
5162 osdc->osds = RB_ROOT; in ceph_osdc_init()
5163 INIT_LIST_HEAD(&osdc->osd_lru); in ceph_osdc_init()
5164 spin_lock_init(&osdc->osd_lru_lock); in ceph_osdc_init()
5165 osd_init(&osdc->homeless_osd); in ceph_osdc_init()
5166 osdc->homeless_osd.o_osdc = osdc; in ceph_osdc_init()
5167 osdc->homeless_osd.o_osd = CEPH_HOMELESS_OSD; in ceph_osdc_init()
5168 osdc->last_linger_id = CEPH_LINGER_ID_START; in ceph_osdc_init()
5169 osdc->linger_requests = RB_ROOT; in ceph_osdc_init()
5170 osdc->map_checks = RB_ROOT; in ceph_osdc_init()
5171 osdc->linger_map_checks = RB_ROOT; in ceph_osdc_init()
5172 INIT_DELAYED_WORK(&osdc->timeout_work, handle_timeout); in ceph_osdc_init()
5173 INIT_DELAYED_WORK(&osdc->osds_timeout_work, handle_osds_timeout); in ceph_osdc_init()
5175 err = -ENOMEM; in ceph_osdc_init()
5176 osdc->osdmap = ceph_osdmap_alloc(); in ceph_osdc_init()
5177 if (!osdc->osdmap) in ceph_osdc_init()
5180 osdc->req_mempool = mempool_create_slab_pool(10, in ceph_osdc_init()
5182 if (!osdc->req_mempool) in ceph_osdc_init()
5185 err = ceph_msgpool_init(&osdc->msgpool_op, CEPH_MSG_OSD_OP, in ceph_osdc_init()
5189 err = ceph_msgpool_init(&osdc->msgpool_op_reply, CEPH_MSG_OSD_OPREPLY, in ceph_osdc_init()
5195 err = -ENOMEM; in ceph_osdc_init()
5196 osdc->notify_wq = create_singlethread_workqueue("ceph-watch-notify"); in ceph_osdc_init()
5197 if (!osdc->notify_wq) in ceph_osdc_init()
5200 osdc->completion_wq = create_singlethread_workqueue("ceph-completion"); in ceph_osdc_init()
5201 if (!osdc->completion_wq) in ceph_osdc_init()
5204 schedule_delayed_work(&osdc->timeout_work, in ceph_osdc_init()
5205 osdc->client->options->osd_keepalive_timeout); in ceph_osdc_init()
5206 schedule_delayed_work(&osdc->osds_timeout_work, in ceph_osdc_init()
5207 round_jiffies_relative(osdc->client->options->osd_idle_ttl)); in ceph_osdc_init()
5212 destroy_workqueue(osdc->notify_wq); in ceph_osdc_init()
5214 ceph_msgpool_destroy(&osdc->msgpool_op_reply); in ceph_osdc_init()
5216 ceph_msgpool_destroy(&osdc->msgpool_op); in ceph_osdc_init()
5218 mempool_destroy(osdc->req_mempool); in ceph_osdc_init()
5220 ceph_osdmap_destroy(osdc->osdmap); in ceph_osdc_init()
5227 destroy_workqueue(osdc->completion_wq); in ceph_osdc_stop()
5228 destroy_workqueue(osdc->notify_wq); in ceph_osdc_stop()
5229 cancel_delayed_work_sync(&osdc->timeout_work); in ceph_osdc_stop()
5230 cancel_delayed_work_sync(&osdc->osds_timeout_work); in ceph_osdc_stop()
5232 down_write(&osdc->lock); in ceph_osdc_stop()
5233 while (!RB_EMPTY_ROOT(&osdc->osds)) { in ceph_osdc_stop()
5234 struct ceph_osd *osd = rb_entry(rb_first(&osdc->osds), in ceph_osdc_stop()
5238 up_write(&osdc->lock); in ceph_osdc_stop()
5239 WARN_ON(refcount_read(&osdc->homeless_osd.o_ref) != 1); in ceph_osdc_stop()
5240 osd_cleanup(&osdc->homeless_osd); in ceph_osdc_stop()
5242 WARN_ON(!list_empty(&osdc->osd_lru)); in ceph_osdc_stop()
5243 WARN_ON(!RB_EMPTY_ROOT(&osdc->linger_requests)); in ceph_osdc_stop()
5244 WARN_ON(!RB_EMPTY_ROOT(&osdc->map_checks)); in ceph_osdc_stop()
5245 WARN_ON(!RB_EMPTY_ROOT(&osdc->linger_map_checks)); in ceph_osdc_stop()
5246 WARN_ON(atomic_read(&osdc->num_requests)); in ceph_osdc_stop()
5247 WARN_ON(atomic_read(&osdc->num_homeless)); in ceph_osdc_stop()
5249 ceph_osdmap_destroy(osdc->osdmap); in ceph_osdc_stop()
5250 mempool_destroy(osdc->req_mempool); in ceph_osdc_stop()
5251 ceph_msgpool_destroy(&osdc->msgpool_op); in ceph_osdc_stop()
5252 ceph_msgpool_destroy(&osdc->msgpool_op_reply); in ceph_osdc_stop()
5274 op->copy_from.snapid = src_snapid; in osd_req_op_copy_from_init()
5275 op->copy_from.src_version = src_version; in osd_req_op_copy_from_init()
5276 op->copy_from.flags = copy_from_flags; in osd_req_op_copy_from_init()
5277 op->copy_from.src_fadvise_flags = src_fadvise_flags; in osd_req_op_copy_from_init()
5281 ceph_encode_string(&p, end, src_oid->name, src_oid->name_len); in osd_req_op_copy_from_init()
5285 op->indata_len = PAGE_SIZE - (end - p); in osd_req_op_copy_from_init()
5287 ceph_osd_data_pages_init(&op->copy_from.osd_data, pages, in osd_req_op_copy_from_init()
5288 op->indata_len, 0, false, true); in osd_req_op_copy_from_init()
5308 return -ENOMEM; in ceph_osdc_copy_from()
5310 req->r_flags = CEPH_OSD_FLAG_WRITE; in ceph_osdc_copy_from()
5312 ceph_oloc_copy(&req->r_t.base_oloc, dst_oloc); in ceph_osdc_copy_from()
5313 ceph_oid_copy(&req->r_t.base_oid, dst_oid); in ceph_osdc_copy_from()
5344 return ceph_osd_request_cache ? 0 : -ENOMEM; in ceph_osdc_setup()
5359 struct ceph_osd *osd = con->private; in dispatch()
5360 struct ceph_osd_client *osdc = osd->o_osdc; in dispatch()
5361 int type = le16_to_cpu(msg->hdr.type); in dispatch()
5377 default: in dispatch()
5388 * message at the moment - for now, just skip the message.
5394 struct ceph_osd *osd = con->private; in get_reply()
5395 struct ceph_osd_client *osdc = osd->o_osdc; in get_reply()
5398 int front_len = le32_to_cpu(hdr->front_len); in get_reply()
5399 int data_len = le32_to_cpu(hdr->data_len); in get_reply()
5400 u64 tid = le64_to_cpu(hdr->tid); in get_reply()
5402 down_read(&osdc->lock); in get_reply()
5404 dout("%s osd%d unknown, skipping\n", __func__, osd->o_osd); in get_reply()
5408 WARN_ON(osd->o_osd != le64_to_cpu(hdr->src.num)); in get_reply()
5410 mutex_lock(&osd->lock); in get_reply()
5411 req = lookup_request(&osd->o_requests, tid); in get_reply()
5413 dout("%s osd%d tid %llu unknown, skipping\n", __func__, in get_reply()
5414 osd->o_osd, tid); in get_reply()
5419 ceph_msg_revoke_incoming(req->r_reply); in get_reply()
5421 if (front_len > req->r_reply->front_alloc_len) { in get_reply()
5423 __func__, osd->o_osd, req->r_tid, front_len, in get_reply()
5424 req->r_reply->front_alloc_len); in get_reply()
5429 ceph_msg_put(req->r_reply); in get_reply()
5430 req->r_reply = m; in get_reply()
5433 if (data_len > req->r_reply->data_length) { in get_reply()
5435 __func__, osd->o_osd, req->r_tid, data_len, in get_reply()
5436 req->r_reply->data_length); in get_reply()
5442 m = ceph_msg_get(req->r_reply); in get_reply()
5443 dout("get_reply tid %lld %p\n", tid, m); in get_reply()
5446 mutex_unlock(&osd->lock); in get_reply()
5448 up_read(&osdc->lock); in get_reply()
5455 int type = le16_to_cpu(hdr->type); in alloc_msg_with_page_vector()
5456 u32 front_len = le32_to_cpu(hdr->front_len); in alloc_msg_with_page_vector()
5457 u32 data_len = le32_to_cpu(hdr->data_len); in alloc_msg_with_page_vector()
5483 struct ceph_osd *osd = con->private; in alloc_msg()
5484 int type = le16_to_cpu(hdr->type); in alloc_msg()
5494 default: in alloc_msg()
5496 osd->o_osd, type); in alloc_msg()
5507 struct ceph_osd *osd = con->private; in get_osd_con()
5515 struct ceph_osd *osd = con->private; in put_osd_con()
5529 struct ceph_osd *o = con->private; in get_authorizer()
5530 struct ceph_osd_client *osdc = o->o_osdc; in get_authorizer()
5531 struct ceph_auth_client *ac = osdc->client->monc.auth; in get_authorizer()
5532 struct ceph_auth_handshake *auth = &o->o_auth; in get_authorizer()
5534 if (force_new && auth->authorizer) { in get_authorizer()
5535 ceph_auth_destroy_authorizer(auth->authorizer); in get_authorizer()
5536 auth->authorizer = NULL; in get_authorizer()
5538 if (!auth->authorizer) { in get_authorizer()
5549 *proto = ac->protocol; in get_authorizer()
5557 struct ceph_osd *o = con->private; in add_authorizer_challenge()
5558 struct ceph_osd_client *osdc = o->o_osdc; in add_authorizer_challenge()
5559 struct ceph_auth_client *ac = osdc->client->monc.auth; in add_authorizer_challenge()
5561 return ceph_auth_add_authorizer_challenge(ac, o->o_auth.authorizer, in add_authorizer_challenge()
5567 struct ceph_osd *o = con->private; in verify_authorizer_reply()
5568 struct ceph_osd_client *osdc = o->o_osdc; in verify_authorizer_reply()
5569 struct ceph_auth_client *ac = osdc->client->monc.auth; in verify_authorizer_reply()
5571 return ceph_auth_verify_authorizer_reply(ac, o->o_auth.authorizer); in verify_authorizer_reply()
5576 struct ceph_osd *o = con->private; in invalidate_authorizer()
5577 struct ceph_osd_client *osdc = o->o_osdc; in invalidate_authorizer()
5578 struct ceph_auth_client *ac = osdc->client->monc.auth; in invalidate_authorizer()
5581 return ceph_monc_validate_auth(&osdc->client->monc); in invalidate_authorizer()
5586 int type = le16_to_cpu(msg->hdr.type); in osd_reencode_message()
5594 struct ceph_osd *o = msg->con->private; in osd_sign_message()
5595 struct ceph_auth_handshake *auth = &o->o_auth; in osd_sign_message()
5602 struct ceph_osd *o = msg->con->private; in osd_check_message_signature()
5603 struct ceph_auth_handshake *auth = &o->o_auth; in osd_check_message_signature()