Lines Matching +full:locality +full:- +full:specific
1 // SPDX-License-Identifier: GPL-2.0
71 WARN_ON(!rwsem_is_locked(&osdc->lock)); in verify_osdc_locked()
75 WARN_ON(!rwsem_is_wrlocked(&osdc->lock)); in verify_osdc_wrlocked()
79 struct ceph_osd_client *osdc = osd->o_osdc; in verify_osd_locked()
81 WARN_ON(!(mutex_is_locked(&osd->lock) && in verify_osd_locked()
82 rwsem_is_locked(&osdc->lock)) && in verify_osd_locked()
83 !rwsem_is_wrlocked(&osdc->lock)); in verify_osd_locked()
87 WARN_ON(!mutex_is_locked(&lreq->lock)); in verify_lreq_locked()
116 orig_len - *plen, off, *plen); in calc_layout()
126 osd_data->type = CEPH_OSD_DATA_TYPE_NONE; in ceph_osd_data_init()
136 osd_data->type = CEPH_OSD_DATA_TYPE_PAGES; in ceph_osd_data_pages_init()
137 osd_data->pages = pages; in ceph_osd_data_pages_init()
138 osd_data->length = length; in ceph_osd_data_pages_init()
139 osd_data->alignment = alignment; in ceph_osd_data_pages_init()
140 osd_data->pages_from_pool = pages_from_pool; in ceph_osd_data_pages_init()
141 osd_data->own_pages = own_pages; in ceph_osd_data_pages_init()
150 osd_data->type = CEPH_OSD_DATA_TYPE_PAGELIST; in ceph_osd_data_pagelist_init()
151 osd_data->pagelist = pagelist; in ceph_osd_data_pagelist_init()
159 osd_data->type = CEPH_OSD_DATA_TYPE_BIO; in ceph_osd_data_bio_init()
160 osd_data->bio_pos = *bio_pos; in ceph_osd_data_bio_init()
161 osd_data->bio_length = bio_length; in ceph_osd_data_bio_init()
169 osd_data->type = CEPH_OSD_DATA_TYPE_BVECS; in ceph_osd_data_bvecs_init()
170 osd_data->bvec_pos = *bvec_pos; in ceph_osd_data_bvecs_init()
171 osd_data->num_bvecs = num_bvecs; in ceph_osd_data_bvecs_init()
177 BUG_ON(which >= osd_req->r_num_ops); in osd_req_op_raw_data_in()
179 return &osd_req->r_ops[which].raw_data_in; in osd_req_op_raw_data_in()
285 osd_req->r_ops[which].cls.indata_len += pagelist->length; in osd_req_op_cls_request_data_pagelist()
286 osd_req->r_ops[which].indata_len += pagelist->length; in osd_req_op_cls_request_data_pagelist()
299 osd_req->r_ops[which].cls.indata_len += length; in osd_req_op_cls_request_data_pages()
300 osd_req->r_ops[which].indata_len += length; in osd_req_op_cls_request_data_pages()
317 osd_req->r_ops[which].cls.indata_len += bytes; in osd_req_op_cls_request_data_bvecs()
318 osd_req->r_ops[which].indata_len += bytes; in osd_req_op_cls_request_data_bvecs()
336 switch (osd_data->type) { in ceph_osd_data_length()
340 return osd_data->length; in ceph_osd_data_length()
342 return (u64)osd_data->pagelist->length; in ceph_osd_data_length()
345 return (u64)osd_data->bio_length; in ceph_osd_data_length()
348 return osd_data->bvec_pos.iter.bi_size; in ceph_osd_data_length()
350 WARN(true, "unrecognized data type %d\n", (int)osd_data->type); in ceph_osd_data_length()
357 if (osd_data->type == CEPH_OSD_DATA_TYPE_PAGES && osd_data->own_pages) { in ceph_osd_data_release()
360 num_pages = calc_pages_for((u64)osd_data->alignment, in ceph_osd_data_release()
361 (u64)osd_data->length); in ceph_osd_data_release()
362 ceph_release_page_vector(osd_data->pages, num_pages); in ceph_osd_data_release()
363 } else if (osd_data->type == CEPH_OSD_DATA_TYPE_PAGELIST) { in ceph_osd_data_release()
364 ceph_pagelist_release(osd_data->pagelist); in ceph_osd_data_release()
374 BUG_ON(which >= osd_req->r_num_ops); in osd_req_op_data_release()
375 op = &osd_req->r_ops[which]; in osd_req_op_data_release()
377 switch (op->op) { in osd_req_op_data_release()
381 ceph_osd_data_release(&op->extent.osd_data); in osd_req_op_data_release()
384 ceph_osd_data_release(&op->cls.request_info); in osd_req_op_data_release()
385 ceph_osd_data_release(&op->cls.request_data); in osd_req_op_data_release()
386 ceph_osd_data_release(&op->cls.response_data); in osd_req_op_data_release()
390 ceph_osd_data_release(&op->xattr.osd_data); in osd_req_op_data_release()
393 ceph_osd_data_release(&op->raw_data_in); in osd_req_op_data_release()
396 ceph_osd_data_release(&op->notify_ack.request_data); in osd_req_op_data_release()
399 ceph_osd_data_release(&op->notify.request_data); in osd_req_op_data_release()
400 ceph_osd_data_release(&op->notify.response_data); in osd_req_op_data_release()
403 ceph_osd_data_release(&op->list_watchers.response_data); in osd_req_op_data_release()
406 ceph_osd_data_release(&op->copy_from.osd_data); in osd_req_op_data_release()
414 * Assumes @t is zero-initialized.
418 ceph_oid_init(&t->base_oid); in target_init()
419 ceph_oloc_init(&t->base_oloc); in target_init()
420 ceph_oid_init(&t->target_oid); in target_init()
421 ceph_oloc_init(&t->target_oloc); in target_init()
423 ceph_osds_init(&t->acting); in target_init()
424 ceph_osds_init(&t->up); in target_init()
425 t->size = -1; in target_init()
426 t->min_size = -1; in target_init()
428 t->osd = CEPH_HOMELESS_OSD; in target_init()
434 ceph_oid_copy(&dest->base_oid, &src->base_oid); in target_copy()
435 ceph_oloc_copy(&dest->base_oloc, &src->base_oloc); in target_copy()
436 ceph_oid_copy(&dest->target_oid, &src->target_oid); in target_copy()
437 ceph_oloc_copy(&dest->target_oloc, &src->target_oloc); in target_copy()
439 dest->pgid = src->pgid; /* struct */ in target_copy()
440 dest->spgid = src->spgid; /* struct */ in target_copy()
441 dest->pg_num = src->pg_num; in target_copy()
442 dest->pg_num_mask = src->pg_num_mask; in target_copy()
443 ceph_osds_copy(&dest->acting, &src->acting); in target_copy()
444 ceph_osds_copy(&dest->up, &src->up); in target_copy()
445 dest->size = src->size; in target_copy()
446 dest->min_size = src->min_size; in target_copy()
447 dest->sort_bitwise = src->sort_bitwise; in target_copy()
448 dest->recovery_deletes = src->recovery_deletes; in target_copy()
450 dest->flags = src->flags; in target_copy()
451 dest->used_replica = src->used_replica; in target_copy()
452 dest->paused = src->paused; in target_copy()
454 dest->epoch = src->epoch; in target_copy()
455 dest->last_force_resend = src->last_force_resend; in target_copy()
457 dest->osd = src->osd; in target_copy()
462 ceph_oid_destroy(&t->base_oid); in target_destroy()
463 ceph_oloc_destroy(&t->base_oloc); in target_destroy()
464 ceph_oid_destroy(&t->target_oid); in target_destroy()
465 ceph_oloc_destroy(&t->target_oloc); in target_destroy()
473 WARN_ON(!RB_EMPTY_NODE(&req->r_node)); in request_release_checks()
474 WARN_ON(!RB_EMPTY_NODE(&req->r_mc_node)); in request_release_checks()
475 WARN_ON(!list_empty(&req->r_private_item)); in request_release_checks()
476 WARN_ON(req->r_osd); in request_release_checks()
486 req->r_request, req->r_reply); in ceph_osdc_release_request()
489 if (req->r_request) in ceph_osdc_release_request()
490 ceph_msg_put(req->r_request); in ceph_osdc_release_request()
491 if (req->r_reply) in ceph_osdc_release_request()
492 ceph_msg_put(req->r_reply); in ceph_osdc_release_request()
494 for (which = 0; which < req->r_num_ops; which++) in ceph_osdc_release_request()
497 target_destroy(&req->r_t); in ceph_osdc_release_request()
498 ceph_put_snap_context(req->r_snapc); in ceph_osdc_release_request()
500 if (req->r_mempool) in ceph_osdc_release_request()
501 mempool_free(req, req->r_osdc->req_mempool); in ceph_osdc_release_request()
502 else if (req->r_num_ops <= CEPH_OSD_SLAB_OPS) in ceph_osdc_release_request()
511 kref_read(&req->r_kref)); in ceph_osdc_get_request()
512 kref_get(&req->r_kref); in ceph_osdc_get_request()
520 kref_read(&req->r_kref)); in ceph_osdc_put_request()
521 kref_put(&req->r_kref, ceph_osdc_release_request); in ceph_osdc_put_request()
531 kref_init(&req->r_kref); in request_init()
532 init_completion(&req->r_completion); in request_init()
533 RB_CLEAR_NODE(&req->r_node); in request_init()
534 RB_CLEAR_NODE(&req->r_mc_node); in request_init()
535 INIT_LIST_HEAD(&req->r_private_item); in request_init()
537 target_init(&req->r_t); in request_init()
550 req = mempool_alloc(osdc->req_mempool, gfp_flags); in ceph_osdc_alloc_request()
561 req->r_osdc = osdc; in ceph_osdc_alloc_request()
562 req->r_mempool = use_mempool; in ceph_osdc_alloc_request()
563 req->r_num_ops = num_ops; in ceph_osdc_alloc_request()
564 req->r_snapid = CEPH_NOSNAP; in ceph_osdc_alloc_request()
565 req->r_snapc = ceph_get_snap_context(snapc); in ceph_osdc_alloc_request()
574 return 8 + 4 + 4 + 4 + (oloc->pool_ns ? oloc->pool_ns->len : 0); in ceph_oloc_encoding_size()
581 struct ceph_osd_client *osdc = req->r_osdc; in __ceph_osdc_alloc_messages()
585 WARN_ON(req->r_request || req->r_reply); in __ceph_osdc_alloc_messages()
586 WARN_ON(ceph_oid_empty(&req->r_base_oid)); in __ceph_osdc_alloc_messages()
587 WARN_ON(ceph_oloc_empty(&req->r_base_oloc)); in __ceph_osdc_alloc_messages()
598 ceph_oloc_encoding_size(&req->r_base_oloc); /* oloc */ in __ceph_osdc_alloc_messages()
599 msg_size += 4 + req->r_base_oid.name_len; /* oid */ in __ceph_osdc_alloc_messages()
600 msg_size += 2 + req->r_num_ops * sizeof(struct ceph_osd_op); in __ceph_osdc_alloc_messages()
603 msg_size += 4 + 8 * (req->r_snapc ? req->r_snapc->num_snaps : 0); in __ceph_osdc_alloc_messages()
606 if (req->r_mempool) in __ceph_osdc_alloc_messages()
607 msg = ceph_msgpool_get(&osdc->msgpool_op, msg_size, in __ceph_osdc_alloc_messages()
613 return -ENOMEM; in __ceph_osdc_alloc_messages()
615 memset(msg->front.iov_base, 0, msg->front.iov_len); in __ceph_osdc_alloc_messages()
616 req->r_request = msg; in __ceph_osdc_alloc_messages()
620 msg_size += req->r_base_oid.name_len; in __ceph_osdc_alloc_messages()
621 msg_size += req->r_num_ops * sizeof(struct ceph_osd_op); in __ceph_osdc_alloc_messages()
623 if (req->r_mempool) in __ceph_osdc_alloc_messages()
624 msg = ceph_msgpool_get(&osdc->msgpool_op_reply, msg_size, in __ceph_osdc_alloc_messages()
630 return -ENOMEM; in __ceph_osdc_alloc_messages()
632 req->r_reply = msg; in __ceph_osdc_alloc_messages()
657 for (op = req->r_ops; op != &req->r_ops[req->r_num_ops]; op++) { in get_num_data_items()
658 switch (op->op) { in get_num_data_items()
687 WARN_ON(!osd_req_opcode_valid(op->op)); in get_num_data_items()
718 BUG_ON(which >= osd_req->r_num_ops); in osd_req_op_init()
721 op = &osd_req->r_ops[which]; in osd_req_op_init()
723 op->op = opcode; in osd_req_op_init()
724 op->flags = flags; in osd_req_op_init()
743 op->extent.offset = offset; in osd_req_op_extent_init()
744 op->extent.length = length; in osd_req_op_extent_init()
745 op->extent.truncate_size = truncate_size; in osd_req_op_extent_init()
746 op->extent.truncate_seq = truncate_seq; in osd_req_op_extent_init()
750 op->indata_len = payload_len; in osd_req_op_extent_init()
760 BUG_ON(which >= osd_req->r_num_ops); in osd_req_op_extent_update()
761 op = &osd_req->r_ops[which]; in osd_req_op_extent_update()
762 previous = op->extent.length; in osd_req_op_extent_update()
768 op->extent.length = length; in osd_req_op_extent_update()
769 if (op->op == CEPH_OSD_OP_WRITE || op->op == CEPH_OSD_OP_WRITEFULL) in osd_req_op_extent_update()
770 op->indata_len -= previous - length; in osd_req_op_extent_update()
779 BUG_ON(which + 1 >= osd_req->r_num_ops); in osd_req_op_extent_dup_last()
781 prev_op = &osd_req->r_ops[which]; in osd_req_op_extent_dup_last()
782 op = osd_req_op_init(osd_req, which + 1, prev_op->op, prev_op->flags); in osd_req_op_extent_dup_last()
784 op->indata_len = prev_op->indata_len; in osd_req_op_extent_dup_last()
785 op->outdata_len = prev_op->outdata_len; in osd_req_op_extent_dup_last()
786 op->extent = prev_op->extent; in osd_req_op_extent_dup_last()
788 op->extent.offset += offset_inc; in osd_req_op_extent_dup_last()
789 op->extent.length -= offset_inc; in osd_req_op_extent_dup_last()
791 if (op->op == CEPH_OSD_OP_WRITE || op->op == CEPH_OSD_OP_WRITEFULL) in osd_req_op_extent_dup_last()
792 op->indata_len -= offset_inc; in osd_req_op_extent_dup_last()
809 return -ENOMEM; in osd_req_op_cls_init()
811 op->cls.class_name = class; in osd_req_op_cls_init()
814 op->cls.class_len = size; in osd_req_op_cls_init()
820 op->cls.method_name = method; in osd_req_op_cls_init()
823 op->cls.method_len = size; in osd_req_op_cls_init()
830 op->indata_len = payload_len; in osd_req_op_cls_init()
853 return -ENOMEM; in osd_req_op_xattr_init()
856 op->xattr.name_len = payload_len; in osd_req_op_xattr_init()
861 op->xattr.value_len = size; in osd_req_op_xattr_init()
867 op->xattr.cmp_op = cmp_op; in osd_req_op_xattr_init()
868 op->xattr.cmp_mode = cmp_mode; in osd_req_op_xattr_init()
870 ceph_osd_data_pagelist_init(&op->xattr.osd_data, pagelist); in osd_req_op_xattr_init()
871 op->indata_len = payload_len; in osd_req_op_xattr_init()
889 op->watch.cookie = cookie; in osd_req_op_watch_init()
890 op->watch.op = watch_opcode; in osd_req_op_watch_init()
891 op->watch.gen = gen; in osd_req_op_watch_init()
904 op->notify.cookie = cookie; in osd_req_op_notify_init()
906 ceph_osd_data_pagelist_init(&op->notify.request_data, request_pl); in osd_req_op_notify_init()
907 op->indata_len = request_pl->length; in osd_req_op_notify_init()
922 op->alloc_hint.expected_object_size = expected_object_size; in osd_req_op_alloc_hint_init()
923 op->alloc_hint.expected_write_size = expected_write_size; in osd_req_op_alloc_hint_init()
924 op->alloc_hint.flags = flags; in osd_req_op_alloc_hint_init()
928 * not worth a feature bit. Set FAILOK per-op flag to make in osd_req_op_alloc_hint_init()
931 op->flags |= CEPH_OSD_OP_FLAG_FAILOK; in osd_req_op_alloc_hint_init()
940 if (osd_data->type == CEPH_OSD_DATA_TYPE_PAGES) { in ceph_osdc_msg_data_add()
943 ceph_msg_data_add_pages(msg, osd_data->pages, in ceph_osdc_msg_data_add()
944 length, osd_data->alignment, false); in ceph_osdc_msg_data_add()
945 } else if (osd_data->type == CEPH_OSD_DATA_TYPE_PAGELIST) { in ceph_osdc_msg_data_add()
947 ceph_msg_data_add_pagelist(msg, osd_data->pagelist); in ceph_osdc_msg_data_add()
949 } else if (osd_data->type == CEPH_OSD_DATA_TYPE_BIO) { in ceph_osdc_msg_data_add()
950 ceph_msg_data_add_bio(msg, &osd_data->bio_pos, length); in ceph_osdc_msg_data_add()
952 } else if (osd_data->type == CEPH_OSD_DATA_TYPE_BVECS) { in ceph_osdc_msg_data_add()
953 ceph_msg_data_add_bvecs(msg, &osd_data->bvec_pos); in ceph_osdc_msg_data_add()
955 BUG_ON(osd_data->type != CEPH_OSD_DATA_TYPE_NONE); in ceph_osdc_msg_data_add()
962 switch (src->op) { in osd_req_encode_op()
970 dst->extent.offset = cpu_to_le64(src->extent.offset); in osd_req_encode_op()
971 dst->extent.length = cpu_to_le64(src->extent.length); in osd_req_encode_op()
972 dst->extent.truncate_size = in osd_req_encode_op()
973 cpu_to_le64(src->extent.truncate_size); in osd_req_encode_op()
974 dst->extent.truncate_seq = in osd_req_encode_op()
975 cpu_to_le32(src->extent.truncate_seq); in osd_req_encode_op()
978 dst->cls.class_len = src->cls.class_len; in osd_req_encode_op()
979 dst->cls.method_len = src->cls.method_len; in osd_req_encode_op()
980 dst->cls.indata_len = cpu_to_le32(src->cls.indata_len); in osd_req_encode_op()
983 dst->watch.cookie = cpu_to_le64(src->watch.cookie); in osd_req_encode_op()
984 dst->watch.ver = cpu_to_le64(0); in osd_req_encode_op()
985 dst->watch.op = src->watch.op; in osd_req_encode_op()
986 dst->watch.gen = cpu_to_le32(src->watch.gen); in osd_req_encode_op()
991 dst->notify.cookie = cpu_to_le64(src->notify.cookie); in osd_req_encode_op()
996 dst->alloc_hint.expected_object_size = in osd_req_encode_op()
997 cpu_to_le64(src->alloc_hint.expected_object_size); in osd_req_encode_op()
998 dst->alloc_hint.expected_write_size = in osd_req_encode_op()
999 cpu_to_le64(src->alloc_hint.expected_write_size); in osd_req_encode_op()
1000 dst->alloc_hint.flags = cpu_to_le32(src->alloc_hint.flags); in osd_req_encode_op()
1004 dst->xattr.name_len = cpu_to_le32(src->xattr.name_len); in osd_req_encode_op()
1005 dst->xattr.value_len = cpu_to_le32(src->xattr.value_len); in osd_req_encode_op()
1006 dst->xattr.cmp_op = src->xattr.cmp_op; in osd_req_encode_op()
1007 dst->xattr.cmp_mode = src->xattr.cmp_mode; in osd_req_encode_op()
1013 dst->copy_from.snapid = cpu_to_le64(src->copy_from.snapid); in osd_req_encode_op()
1014 dst->copy_from.src_version = in osd_req_encode_op()
1015 cpu_to_le64(src->copy_from.src_version); in osd_req_encode_op()
1016 dst->copy_from.flags = src->copy_from.flags; in osd_req_encode_op()
1017 dst->copy_from.src_fadvise_flags = in osd_req_encode_op()
1018 cpu_to_le32(src->copy_from.src_fadvise_flags); in osd_req_encode_op()
1022 ceph_osd_op_name(src->op)); in osd_req_encode_op()
1028 dst->op = cpu_to_le16(src->op); in osd_req_encode_op()
1029 dst->flags = cpu_to_le32(src->flags); in osd_req_encode_op()
1030 dst->payload_len = cpu_to_le32(src->indata_len); in osd_req_encode_op()
1032 return src->indata_len; in osd_req_encode_op()
1067 r = -ENOMEM; in ceph_osdc_new_request()
1079 u32 object_size = layout->object_size; in ceph_osdc_new_request()
1080 u32 object_base = off - objoff; in ceph_osdc_new_request()
1081 if (!(truncate_seq == 1 && truncate_size == -1ULL)) { in ceph_osdc_new_request()
1085 truncate_size -= object_base; in ceph_osdc_new_request()
1094 req->r_base_oloc.pool = layout->pool_id; in ceph_osdc_new_request()
1095 req->r_base_oloc.pool_ns = ceph_try_get_string(layout->pool_ns); in ceph_osdc_new_request()
1096 ceph_oid_printf(&req->r_base_oid, "%llx.%08llx", vino.ino, objnum); in ceph_osdc_new_request()
1097 req->r_flags = flags | osdc->client->options->read_from_replica; in ceph_osdc_new_request()
1099 req->r_snapid = vino.snap; in ceph_osdc_new_request()
1101 req->r_data_offset = off; in ceph_osdc_new_request()
1106 * also covers ceph_uninline_data(). If more multi-op request in ceph_osdc_new_request()
1124 * We keep osd requests in an rbtree, sorted by ->r_tid.
1138 for (n = rb_first(&osdc->osds); n; n = rb_next(n)) { in DEFINE_RB_FUNCS()
1141 for (p = rb_first(&osd->o_requests); p; ) { in DEFINE_RB_FUNCS()
1151 for (p = rb_first(&osdc->homeless_osd.o_requests); p; ) { in DEFINE_RB_FUNCS()
1163 return osd->o_osd == CEPH_HOMELESS_OSD; in osd_homeless()
1168 verify_osdc_locked(osd->o_osdc); in osd_registered()
1170 return !RB_EMPTY_NODE(&osd->o_node); in osd_registered()
1174 * Assumes @osd is zero-initialized.
1178 refcount_set(&osd->o_ref, 1); in osd_init()
1179 RB_CLEAR_NODE(&osd->o_node); in osd_init()
1180 osd->o_requests = RB_ROOT; in osd_init()
1181 osd->o_linger_requests = RB_ROOT; in osd_init()
1182 osd->o_backoff_mappings = RB_ROOT; in osd_init()
1183 osd->o_backoffs_by_id = RB_ROOT; in osd_init()
1184 INIT_LIST_HEAD(&osd->o_osd_lru); in osd_init()
1185 INIT_LIST_HEAD(&osd->o_keepalive_item); in osd_init()
1186 osd->o_incarnation = 1; in osd_init()
1187 mutex_init(&osd->lock); in osd_init()
1192 WARN_ON(!RB_EMPTY_NODE(&osd->o_node)); in osd_cleanup()
1193 WARN_ON(!RB_EMPTY_ROOT(&osd->o_requests)); in osd_cleanup()
1194 WARN_ON(!RB_EMPTY_ROOT(&osd->o_linger_requests)); in osd_cleanup()
1195 WARN_ON(!RB_EMPTY_ROOT(&osd->o_backoff_mappings)); in osd_cleanup()
1196 WARN_ON(!RB_EMPTY_ROOT(&osd->o_backoffs_by_id)); in osd_cleanup()
1197 WARN_ON(!list_empty(&osd->o_osd_lru)); in osd_cleanup()
1198 WARN_ON(!list_empty(&osd->o_keepalive_item)); in osd_cleanup()
1200 if (osd->o_auth.authorizer) { in osd_cleanup()
1202 ceph_auth_destroy_authorizer(osd->o_auth.authorizer); in osd_cleanup()
1217 osd->o_osdc = osdc; in create_osd()
1218 osd->o_osd = onum; in create_osd()
1220 ceph_con_init(&osd->o_con, osd, &osd_con_ops, &osdc->client->msgr); in create_osd()
1227 if (refcount_inc_not_zero(&osd->o_ref)) { in get_osd()
1228 dout("get_osd %p %d -> %d\n", osd, refcount_read(&osd->o_ref)-1, in get_osd()
1229 refcount_read(&osd->o_ref)); in get_osd()
1239 dout("put_osd %p %d -> %d\n", osd, refcount_read(&osd->o_ref), in put_osd()
1240 refcount_read(&osd->o_ref) - 1); in put_osd()
1241 if (refcount_dec_and_test(&osd->o_ref)) { in put_osd()
1251 struct ceph_osd_client *osdc = osd->o_osdc; in DEFINE_RB_FUNCS()
1253 dout("%s osd %p osd%d\n", __func__, osd, osd->o_osd); in DEFINE_RB_FUNCS()
1254 BUG_ON(!list_empty(&osd->o_osd_lru)); in DEFINE_RB_FUNCS()
1256 spin_lock(&osdc->osd_lru_lock); in DEFINE_RB_FUNCS()
1257 list_add_tail(&osd->o_osd_lru, &osdc->osd_lru); in DEFINE_RB_FUNCS()
1258 spin_unlock(&osdc->osd_lru_lock); in DEFINE_RB_FUNCS()
1260 osd->lru_ttl = jiffies + osdc->client->options->osd_idle_ttl; in DEFINE_RB_FUNCS()
1265 if (RB_EMPTY_ROOT(&osd->o_requests) && in maybe_move_osd_to_lru()
1266 RB_EMPTY_ROOT(&osd->o_linger_requests)) in maybe_move_osd_to_lru()
1272 struct ceph_osd_client *osdc = osd->o_osdc; in __remove_osd_from_lru()
1274 dout("%s osd %p osd%d\n", __func__, osd, osd->o_osd); in __remove_osd_from_lru()
1276 spin_lock(&osdc->osd_lru_lock); in __remove_osd_from_lru()
1277 if (!list_empty(&osd->o_osd_lru)) in __remove_osd_from_lru()
1278 list_del_init(&osd->o_osd_lru); in __remove_osd_from_lru()
1279 spin_unlock(&osdc->osd_lru_lock); in __remove_osd_from_lru()
1288 struct ceph_osd_client *osdc = osd->o_osdc; in close_osd()
1292 dout("%s osd %p osd%d\n", __func__, osd, osd->o_osd); in close_osd()
1294 ceph_con_close(&osd->o_con); in close_osd()
1296 for (n = rb_first(&osd->o_requests); n; ) { in close_osd()
1302 dout(" reassigning req %p tid %llu\n", req, req->r_tid); in close_osd()
1304 link_request(&osdc->homeless_osd, req); in close_osd()
1306 for (n = rb_first(&osd->o_linger_requests); n; ) { in close_osd()
1313 lreq->linger_id); in close_osd()
1315 link_linger(&osdc->homeless_osd, lreq); in close_osd()
1320 erase_osd(&osdc->osds, osd); in close_osd()
1331 dout("%s osd %p osd%d\n", __func__, osd, osd->o_osd); in reopen_osd()
1333 if (RB_EMPTY_ROOT(&osd->o_requests) && in reopen_osd()
1334 RB_EMPTY_ROOT(&osd->o_linger_requests)) { in reopen_osd()
1336 return -ENODEV; in reopen_osd()
1339 peer_addr = &osd->o_osdc->osdmap->osd_addr[osd->o_osd]; in reopen_osd()
1340 if (!memcmp(peer_addr, &osd->o_con.peer_addr, sizeof (*peer_addr)) && in reopen_osd()
1341 !ceph_con_opened(&osd->o_con)) { in reopen_osd()
1347 for (n = rb_first(&osd->o_requests); n; n = rb_next(n)) { in reopen_osd()
1350 req->r_stamp = jiffies; in reopen_osd()
1353 return -EAGAIN; in reopen_osd()
1356 ceph_con_close(&osd->o_con); in reopen_osd()
1357 ceph_con_open(&osd->o_con, CEPH_ENTITY_TYPE_OSD, osd->o_osd, peer_addr); in reopen_osd()
1358 osd->o_incarnation++; in reopen_osd()
1374 osd = lookup_osd(&osdc->osds, o); in lookup_create_osd()
1376 osd = &osdc->homeless_osd; in lookup_create_osd()
1379 return ERR_PTR(-EAGAIN); in lookup_create_osd()
1382 insert_osd(&osdc->osds, osd); in lookup_create_osd()
1383 ceph_con_open(&osd->o_con, CEPH_ENTITY_TYPE_OSD, osd->o_osd, in lookup_create_osd()
1384 &osdc->osdmap->osd_addr[osd->o_osd]); in lookup_create_osd()
1387 dout("%s osdc %p osd%d -> osd %p\n", __func__, osdc, o, osd); in lookup_create_osd()
1392 * Create request <-> OSD session relation.
1399 WARN_ON(!req->r_tid || req->r_osd); in link_request()
1400 dout("%s osd %p osd%d req %p tid %llu\n", __func__, osd, osd->o_osd, in link_request()
1401 req, req->r_tid); in link_request()
1406 atomic_inc(&osd->o_osdc->num_homeless); in link_request()
1409 insert_request(&osd->o_requests, req); in link_request()
1410 req->r_osd = osd; in link_request()
1416 WARN_ON(req->r_osd != osd); in unlink_request()
1417 dout("%s osd %p osd%d req %p tid %llu\n", __func__, osd, osd->o_osd, in unlink_request()
1418 req, req->r_tid); in unlink_request()
1420 req->r_osd = NULL; in unlink_request()
1421 erase_request(&osd->o_requests, req); in unlink_request()
1427 atomic_dec(&osd->o_osdc->num_homeless); in unlink_request()
1432 return pi->flags & CEPH_POOL_FLAG_FULL; in __pool_full()
1439 for (n = rb_first(&osdc->osdmap->pg_pools); n; n = rb_next(n)) { in have_pool_full()
1454 pi = ceph_pg_pool_by_id(osdc->osdmap, pool_id); in pool_full()
1474 WARN_ON(pi->id != t->target_oloc.pool); in target_should_be_paused()
1475 return ((t->flags & CEPH_OSD_FLAG_READ) && pauserd) || in target_should_be_paused()
1476 ((t->flags & CEPH_OSD_FLAG_WRITE) && pausewr) || in target_should_be_paused()
1477 (osdc->osdmap->epoch < osdc->epoch_barrier); in target_should_be_paused()
1482 int i = prandom_u32() % acting->size; in pick_random_replica()
1485 acting->osds[i], acting->primary); in pick_random_replica()
1491 * crush_location option. Prefers the primary if the locality is
1497 struct ceph_options *opt = osdc->client->options; in pick_closest_replica()
1499 int i = 0, locality; in pick_closest_replica() local
1502 locality = ceph_get_crush_locality(osdc->osdmap, in pick_closest_replica()
1503 acting->osds[i], in pick_closest_replica()
1504 &opt->crush_locs); in pick_closest_replica()
1506 (locality >= 0 && best_locality < 0) || in pick_closest_replica()
1507 (locality >= 0 && best_locality >= 0 && in pick_closest_replica()
1508 locality < best_locality)) { in pick_closest_replica()
1510 best_locality = locality; in pick_closest_replica()
1512 } while (++i < acting->size); in pick_closest_replica()
1514 dout("%s picked osd%d with locality %d, primary osd%d\n", __func__, in pick_closest_replica()
1515 acting->osds[best_i], best_locality, acting->primary); in pick_closest_replica()
1532 bool is_read = t->flags & CEPH_OSD_FLAG_READ; in calc_target()
1533 bool is_write = t->flags & CEPH_OSD_FLAG_WRITE; in calc_target()
1543 t->epoch = osdc->osdmap->epoch; in calc_target()
1544 pi = ceph_pg_pool_by_id(osdc->osdmap, t->base_oloc.pool); in calc_target()
1546 t->osd = CEPH_HOMELESS_OSD; in calc_target()
1551 if (osdc->osdmap->epoch == pi->last_force_request_resend) { in calc_target()
1552 if (t->last_force_resend < pi->last_force_request_resend) { in calc_target()
1553 t->last_force_resend = pi->last_force_request_resend; in calc_target()
1555 } else if (t->last_force_resend == 0) { in calc_target()
1561 ceph_oid_copy(&t->target_oid, &t->base_oid); in calc_target()
1562 ceph_oloc_copy(&t->target_oloc, &t->base_oloc); in calc_target()
1563 if ((t->flags & CEPH_OSD_FLAG_IGNORE_OVERLAY) == 0) { in calc_target()
1564 if (is_read && pi->read_tier >= 0) in calc_target()
1565 t->target_oloc.pool = pi->read_tier; in calc_target()
1566 if (is_write && pi->write_tier >= 0) in calc_target()
1567 t->target_oloc.pool = pi->write_tier; in calc_target()
1569 pi = ceph_pg_pool_by_id(osdc->osdmap, t->target_oloc.pool); in calc_target()
1571 t->osd = CEPH_HOMELESS_OSD; in calc_target()
1577 __ceph_object_locator_to_pg(pi, &t->target_oid, &t->target_oloc, &pgid); in calc_target()
1579 last_pgid.seed = ceph_stable_mod(pgid.seed, t->pg_num, t->pg_num_mask); in calc_target()
1581 ceph_pg_to_up_acting_osds(osdc->osdmap, pi, &pgid, &up, &acting); in calc_target()
1583 ceph_is_new_interval(&t->acting, in calc_target()
1585 &t->up, in calc_target()
1587 t->size, in calc_target()
1588 pi->size, in calc_target()
1589 t->min_size, in calc_target()
1590 pi->min_size, in calc_target()
1591 t->pg_num, in calc_target()
1592 pi->pg_num, in calc_target()
1593 t->sort_bitwise, in calc_target()
1595 t->recovery_deletes, in calc_target()
1600 if (t->paused && !target_should_be_paused(osdc, t, pi)) { in calc_target()
1601 t->paused = false; in calc_target()
1604 legacy_change = ceph_pg_compare(&t->pgid, &pgid) || in calc_target()
1605 ceph_osds_changed(&t->acting, &acting, in calc_target()
1606 t->used_replica || any_change); in calc_target()
1607 if (t->pg_num) in calc_target()
1608 split = ceph_pg_is_split(&last_pgid, t->pg_num, pi->pg_num); in calc_target()
1611 t->pgid = pgid; /* struct */ in calc_target()
1612 ceph_pg_to_primary_shard(osdc->osdmap, pi, &pgid, &t->spgid); in calc_target()
1613 ceph_osds_copy(&t->acting, &acting); in calc_target()
1614 ceph_osds_copy(&t->up, &up); in calc_target()
1615 t->size = pi->size; in calc_target()
1616 t->min_size = pi->min_size; in calc_target()
1617 t->pg_num = pi->pg_num; in calc_target()
1618 t->pg_num_mask = pi->pg_num_mask; in calc_target()
1619 t->sort_bitwise = sort_bitwise; in calc_target()
1620 t->recovery_deletes = recovery_deletes; in calc_target()
1622 if ((t->flags & (CEPH_OSD_FLAG_BALANCE_READS | in calc_target()
1624 !is_write && pi->type == CEPH_POOL_TYPE_REP && in calc_target()
1629 if (t->flags & CEPH_OSD_FLAG_BALANCE_READS) { in calc_target()
1634 t->osd = acting.osds[pos]; in calc_target()
1635 t->used_replica = pos > 0; in calc_target()
1637 t->osd = acting.primary; in calc_target()
1638 t->used_replica = false; in calc_target()
1648 dout("%s t %p -> %d%d%d%d ct_res %d osd%d\n", __func__, t, unpaused, in calc_target()
1649 legacy_change, force_resend, split, ct_res, t->osd); in calc_target()
1661 RB_CLEAR_NODE(&spg->node); in alloc_spg_mapping()
1662 spg->backoffs = RB_ROOT; in alloc_spg_mapping()
1668 WARN_ON(!RB_EMPTY_NODE(&spg->node)); in free_spg_mapping()
1669 WARN_ON(!RB_EMPTY_ROOT(&spg->backoffs)); in free_spg_mapping()
1676 * ceph_pg_mapping. Used to track OSD backoffs -- a backoff [range] is
1677 * defined only within a specific spgid; it does not pass anything to
1685 return hoid->is_max ? 0x100000000ull : hoid->hash_reverse_bits; in DEFINE_RB_FUNCS2()
1691 if (hoid->key_len) { in hoid_get_effective_key()
1692 *pkey = hoid->key; in hoid_get_effective_key()
1693 *pkey_len = hoid->key_len; in hoid_get_effective_key()
1695 *pkey = hoid->oid; in hoid_get_effective_key()
1696 *pkey_len = hoid->oid_len; in hoid_get_effective_key()
1708 ret = -1; in compare_names()
1722 if (lhs->is_max < rhs->is_max) in hoid_compare()
1723 return -1; in hoid_compare()
1724 if (lhs->is_max > rhs->is_max) in hoid_compare()
1727 if (lhs->pool < rhs->pool) in hoid_compare()
1728 return -1; in hoid_compare()
1729 if (lhs->pool > rhs->pool) in hoid_compare()
1733 return -1; in hoid_compare()
1737 ret = compare_names(lhs->nspace, lhs->nspace_len, in hoid_compare()
1738 rhs->nspace, rhs->nspace_len); in hoid_compare()
1749 ret = compare_names(lhs->oid, lhs->oid_len, rhs->oid, rhs->oid_len); in hoid_compare()
1753 if (lhs->snapid < rhs->snapid) in hoid_compare()
1754 return -1; in hoid_compare()
1755 if (lhs->snapid > rhs->snapid) in hoid_compare()
1762 * For decoding ->begin and ->end of MOSDBackoff only -- no MIN/MAX
1765 * Assumes @hoid is zero-initialized.
1783 hoid->key = ceph_extract_encoded_string(p, end, &hoid->key_len, in decode_hoid()
1785 if (IS_ERR(hoid->key)) { in decode_hoid()
1786 ret = PTR_ERR(hoid->key); in decode_hoid()
1787 hoid->key = NULL; in decode_hoid()
1791 hoid->oid = ceph_extract_encoded_string(p, end, &hoid->oid_len, in decode_hoid()
1793 if (IS_ERR(hoid->oid)) { in decode_hoid()
1794 ret = PTR_ERR(hoid->oid); in decode_hoid()
1795 hoid->oid = NULL; in decode_hoid()
1799 ceph_decode_64_safe(p, end, hoid->snapid, e_inval); in decode_hoid()
1800 ceph_decode_32_safe(p, end, hoid->hash, e_inval); in decode_hoid()
1801 ceph_decode_8_safe(p, end, hoid->is_max, e_inval); in decode_hoid()
1803 hoid->nspace = ceph_extract_encoded_string(p, end, &hoid->nspace_len, in decode_hoid()
1805 if (IS_ERR(hoid->nspace)) { in decode_hoid()
1806 ret = PTR_ERR(hoid->nspace); in decode_hoid()
1807 hoid->nspace = NULL; in decode_hoid()
1811 ceph_decode_64_safe(p, end, hoid->pool, e_inval); in decode_hoid()
1817 return -EINVAL; in decode_hoid()
1823 4 + hoid->key_len + 4 + hoid->oid_len + 4 + hoid->nspace_len; in hoid_encoding_size()
1829 ceph_encode_string(p, end, hoid->key, hoid->key_len); in encode_hoid()
1830 ceph_encode_string(p, end, hoid->oid, hoid->oid_len); in encode_hoid()
1831 ceph_encode_64(p, hoid->snapid); in encode_hoid()
1832 ceph_encode_32(p, hoid->hash); in encode_hoid()
1833 ceph_encode_8(p, hoid->is_max); in encode_hoid()
1834 ceph_encode_string(p, end, hoid->nspace, hoid->nspace_len); in encode_hoid()
1835 ceph_encode_64(p, hoid->pool); in encode_hoid()
1841 kfree(hoid->key); in free_hoid()
1842 kfree(hoid->oid); in free_hoid()
1843 kfree(hoid->nspace); in free_hoid()
1856 RB_CLEAR_NODE(&backoff->spg_node); in alloc_backoff()
1857 RB_CLEAR_NODE(&backoff->id_node); in alloc_backoff()
1863 WARN_ON(!RB_EMPTY_NODE(&backoff->spg_node)); in free_backoff()
1864 WARN_ON(!RB_EMPTY_NODE(&backoff->id_node)); in free_backoff()
1866 free_hoid(backoff->begin); in free_backoff()
1867 free_hoid(backoff->end); in free_backoff()
1872 * Within a specific spgid, backoffs are managed by ->begin hoid.
1880 struct rb_node *n = root->rb_node; in lookup_containing_backoff()
1887 cmp = hoid_compare(hoid, cur->begin); in lookup_containing_backoff()
1889 n = n->rb_left; in lookup_containing_backoff()
1891 if (hoid_compare(hoid, cur->end) < 0) in lookup_containing_backoff()
1894 n = n->rb_right; in lookup_containing_backoff()
1910 while (!RB_EMPTY_ROOT(&osd->o_backoff_mappings)) { in DEFINE_RB_FUNCS()
1912 rb_entry(rb_first(&osd->o_backoff_mappings), in DEFINE_RB_FUNCS()
1915 while (!RB_EMPTY_ROOT(&spg->backoffs)) { in DEFINE_RB_FUNCS()
1917 rb_entry(rb_first(&spg->backoffs), in DEFINE_RB_FUNCS()
1920 erase_backoff(&spg->backoffs, backoff); in DEFINE_RB_FUNCS()
1921 erase_backoff_by_id(&osd->o_backoffs_by_id, backoff); in DEFINE_RB_FUNCS()
1924 erase_spg_mapping(&osd->o_backoff_mappings, spg); in DEFINE_RB_FUNCS()
1930 * Set up a temporary, non-owning view into @t.
1935 hoid->key = NULL; in hoid_fill_from_target()
1936 hoid->key_len = 0; in hoid_fill_from_target()
1937 hoid->oid = t->target_oid.name; in hoid_fill_from_target()
1938 hoid->oid_len = t->target_oid.name_len; in hoid_fill_from_target()
1939 hoid->snapid = CEPH_NOSNAP; in hoid_fill_from_target()
1940 hoid->hash = t->pgid.seed; in hoid_fill_from_target()
1941 hoid->is_max = false; in hoid_fill_from_target()
1942 if (t->target_oloc.pool_ns) { in hoid_fill_from_target()
1943 hoid->nspace = t->target_oloc.pool_ns->str; in hoid_fill_from_target()
1944 hoid->nspace_len = t->target_oloc.pool_ns->len; in hoid_fill_from_target()
1946 hoid->nspace = NULL; in hoid_fill_from_target()
1947 hoid->nspace_len = 0; in hoid_fill_from_target()
1949 hoid->pool = t->target_oloc.pool; in hoid_fill_from_target()
1955 struct ceph_osd *osd = req->r_osd; in should_plug_request()
1960 spg = lookup_spg_mapping(&osd->o_backoff_mappings, &req->r_t.spgid); in should_plug_request()
1964 hoid_fill_from_target(&hoid, &req->r_t); in should_plug_request()
1965 backoff = lookup_containing_backoff(&spg->backoffs, &hoid); in should_plug_request()
1970 __func__, req, req->r_tid, osd->o_osd, backoff->spgid.pgid.pool, in should_plug_request()
1971 backoff->spgid.pgid.seed, backoff->spgid.shard, backoff->id); in should_plug_request()
1980 struct ceph_msg *request_msg = req->r_request; in setup_request_data()
1981 struct ceph_msg *reply_msg = req->r_reply; in setup_request_data()
1984 if (req->r_request->num_data_items || req->r_reply->num_data_items) in setup_request_data()
1987 WARN_ON(request_msg->data_length || reply_msg->data_length); in setup_request_data()
1988 for (op = req->r_ops; op != &req->r_ops[req->r_num_ops]; op++) { in setup_request_data()
1989 switch (op->op) { in setup_request_data()
1993 WARN_ON(op->indata_len != op->extent.length); in setup_request_data()
1995 &op->extent.osd_data); in setup_request_data()
1999 WARN_ON(op->indata_len != op->xattr.name_len + in setup_request_data()
2000 op->xattr.value_len); in setup_request_data()
2002 &op->xattr.osd_data); in setup_request_data()
2006 &op->notify_ack.request_data); in setup_request_data()
2010 &op->copy_from.osd_data); in setup_request_data()
2016 &op->raw_data_in); in setup_request_data()
2020 &op->extent.osd_data); in setup_request_data()
2024 &op->list_watchers.response_data); in setup_request_data()
2029 WARN_ON(op->indata_len != op->cls.class_len + in setup_request_data()
2030 op->cls.method_len + in setup_request_data()
2031 op->cls.indata_len); in setup_request_data()
2033 &op->cls.request_info); in setup_request_data()
2036 &op->cls.request_data); in setup_request_data()
2039 &op->cls.response_data); in setup_request_data()
2043 &op->notify.request_data); in setup_request_data()
2045 &op->notify.response_data); in setup_request_data()
2054 ceph_encode_64(p, pgid->pool); in encode_pgid()
2055 ceph_encode_32(p, pgid->seed); in encode_pgid()
2056 ceph_encode_32(p, -1); /* preferred */ in encode_pgid()
2062 encode_pgid(p, &spgid->pgid); in encode_spgid()
2063 ceph_encode_8(p, spgid->shard); in encode_spgid()
2070 ceph_encode_64(p, oloc->pool); in encode_oloc()
2071 ceph_encode_32(p, -1); /* preferred */ in encode_oloc()
2073 if (oloc->pool_ns) in encode_oloc()
2074 ceph_encode_string(p, end, oloc->pool_ns->str, in encode_oloc()
2075 oloc->pool_ns->len); in encode_oloc()
2083 void *p = msg->front.iov_base; in encode_request_partial()
2084 void *const end = p + msg->front_alloc_len; in encode_request_partial()
2088 if (req->r_flags & CEPH_OSD_FLAG_WRITE) { in encode_request_partial()
2090 WARN_ON(req->r_snapid != CEPH_NOSNAP); in encode_request_partial()
2092 WARN_ON(req->r_mtime.tv_sec || req->r_mtime.tv_nsec || in encode_request_partial()
2093 req->r_data_offset || req->r_snapc); in encode_request_partial()
2098 encode_spgid(&p, &req->r_t.spgid); /* actual spg */ in encode_request_partial()
2099 ceph_encode_32(&p, req->r_t.pgid.seed); /* raw hash */ in encode_request_partial()
2100 ceph_encode_32(&p, req->r_osdc->osdmap->epoch); in encode_request_partial()
2101 ceph_encode_32(&p, req->r_flags); in encode_request_partial()
2113 ceph_encode_timespec64(p, &req->r_mtime); in encode_request_partial()
2116 encode_oloc(&p, end, &req->r_t.target_oloc); in encode_request_partial()
2117 ceph_encode_string(&p, end, req->r_t.target_oid.name, in encode_request_partial()
2118 req->r_t.target_oid.name_len); in encode_request_partial()
2121 ceph_encode_16(&p, req->r_num_ops); in encode_request_partial()
2122 for (i = 0; i < req->r_num_ops; i++) { in encode_request_partial()
2123 data_len += osd_req_encode_op(p, &req->r_ops[i]); in encode_request_partial()
2127 ceph_encode_64(&p, req->r_snapid); /* snapid */ in encode_request_partial()
2128 if (req->r_snapc) { in encode_request_partial()
2129 ceph_encode_64(&p, req->r_snapc->seq); in encode_request_partial()
2130 ceph_encode_32(&p, req->r_snapc->num_snaps); in encode_request_partial()
2131 for (i = 0; i < req->r_snapc->num_snaps; i++) in encode_request_partial()
2132 ceph_encode_64(&p, req->r_snapc->snaps[i]); in encode_request_partial()
2138 ceph_encode_32(&p, req->r_attempts); /* retry_attempt */ in encode_request_partial()
2139 BUG_ON(p > end - 8); /* space for features */ in encode_request_partial()
2141 msg->hdr.version = cpu_to_le16(8); /* MOSDOp v8 */ in encode_request_partial()
2143 msg->front.iov_len = p - msg->front.iov_base; in encode_request_partial()
2144 msg->hdr.front_len = cpu_to_le32(msg->front.iov_len); in encode_request_partial()
2145 msg->hdr.data_len = cpu_to_le32(data_len); in encode_request_partial()
2149 * need to re-copy it before writing it to disk (direct I/O). in encode_request_partial()
2151 msg->hdr.data_off = cpu_to_le16(req->r_data_offset); in encode_request_partial()
2154 req->r_t.target_oid.name, req->r_t.target_oid.name_len); in encode_request_partial()
2159 void *p = msg->front.iov_base; in encode_request_finish()
2160 void *const partial_end = p + msg->front.iov_len; in encode_request_finish()
2161 void *const end = p + msg->front_alloc_len; in encode_request_finish()
2163 if (CEPH_HAVE_FEATURE(msg->con->peer_features, RESEND_ON_SPLIT)) { in encode_request_finish()
2164 /* luminous OSD -- encode features and be done */ in encode_request_finish()
2166 ceph_encode_64(&p, msg->con->peer_features); in encode_request_finish()
2186 * Pre-luminous OSD -- reencode v8 into v4 using @head in encode_request_finish()
2200 oloc_len = p - oloc; in encode_request_finish()
2205 oid_len = p - oid; in encode_request_finish()
2208 tail_len = partial_end - p; in encode_request_finish()
2210 p = msg->front.iov_base; in encode_request_finish()
2231 /* tail -- ops, snapid, snapc, retry_attempt */ in encode_request_finish()
2236 msg->hdr.version = cpu_to_le16(4); /* MOSDOp v4 */ in encode_request_finish()
2240 msg->front.iov_len = p - msg->front.iov_base; in encode_request_finish()
2241 msg->hdr.front_len = cpu_to_le32(msg->front.iov_len); in encode_request_finish()
2244 le64_to_cpu(msg->hdr.tid), le32_to_cpu(msg->hdr.front_len), in encode_request_finish()
2245 le32_to_cpu(msg->hdr.middle_len), le32_to_cpu(msg->hdr.data_len), in encode_request_finish()
2246 le16_to_cpu(msg->hdr.version)); in encode_request_finish()
2254 struct ceph_osd *osd = req->r_osd; in send_request()
2257 WARN_ON(osd->o_osd != req->r_t.osd); in send_request()
2267 if (req->r_sent) in send_request()
2268 ceph_msg_revoke(req->r_request); in send_request()
2270 req->r_flags |= CEPH_OSD_FLAG_KNOWN_REDIR; in send_request()
2271 if (req->r_attempts) in send_request()
2272 req->r_flags |= CEPH_OSD_FLAG_RETRY; in send_request()
2274 WARN_ON(req->r_flags & CEPH_OSD_FLAG_RETRY); in send_request()
2276 encode_request_partial(req, req->r_request); in send_request()
2279 __func__, req, req->r_tid, req->r_t.pgid.pool, req->r_t.pgid.seed, in send_request()
2280 req->r_t.spgid.pgid.pool, req->r_t.spgid.pgid.seed, in send_request()
2281 req->r_t.spgid.shard, osd->o_osd, req->r_t.epoch, req->r_flags, in send_request()
2282 req->r_attempts); in send_request()
2284 req->r_t.paused = false; in send_request()
2285 req->r_stamp = jiffies; in send_request()
2286 req->r_attempts++; in send_request()
2288 req->r_sent = osd->o_incarnation; in send_request()
2289 req->r_request->hdr.tid = cpu_to_le64(req->r_tid); in send_request()
2290 ceph_con_send(&osd->o_con, ceph_msg_get(req->r_request)); in send_request()
2298 WARN_ON(!osdc->osdmap->epoch); in maybe_request_map()
2309 if (ceph_monc_want_map(&osdc->client->monc, CEPH_SUB_OSDMAP, in maybe_request_map()
2310 osdc->osdmap->epoch + 1, continuous)) in maybe_request_map()
2311 ceph_monc_renew_subs(&osdc->client->monc); in maybe_request_map()
2319 struct ceph_osd_client *osdc = req->r_osdc; in __submit_request()
2326 WARN_ON(req->r_tid); in __submit_request()
2330 ct_res = calc_target(osdc, &req->r_t, false); in __submit_request()
2334 osd = lookup_create_osd(osdc, req->r_t.osd, wrlocked); in __submit_request()
2336 WARN_ON(PTR_ERR(osd) != -EAGAIN || wrlocked); in __submit_request()
2340 if (osdc->abort_err) { in __submit_request()
2341 dout("req %p abort_err %d\n", req, osdc->abort_err); in __submit_request()
2342 err = osdc->abort_err; in __submit_request()
2343 } else if (osdc->osdmap->epoch < osdc->epoch_barrier) { in __submit_request()
2344 dout("req %p epoch %u barrier %u\n", req, osdc->osdmap->epoch, in __submit_request()
2345 osdc->epoch_barrier); in __submit_request()
2346 req->r_t.paused = true; in __submit_request()
2348 } else if ((req->r_flags & CEPH_OSD_FLAG_WRITE) && in __submit_request()
2351 req->r_t.paused = true; in __submit_request()
2353 } else if ((req->r_flags & CEPH_OSD_FLAG_READ) && in __submit_request()
2356 req->r_t.paused = true; in __submit_request()
2358 } else if ((req->r_flags & CEPH_OSD_FLAG_WRITE) && in __submit_request()
2359 !(req->r_flags & (CEPH_OSD_FLAG_FULL_TRY | in __submit_request()
2362 pool_full(osdc, req->r_t.base_oloc.pool))) { in __submit_request()
2364 if (ceph_test_opt(osdc->client, ABORT_ON_FULL)) { in __submit_request()
2365 err = -ENOSPC; in __submit_request()
2368 req->r_t.paused = true; in __submit_request()
2377 mutex_lock(&osd->lock); in __submit_request()
2383 req->r_tid = atomic64_inc_return(&osdc->last_tid); in __submit_request()
2389 mutex_unlock(&osd->lock); in __submit_request()
2395 downgrade_write(&osdc->lock); in __submit_request()
2399 up_read(&osdc->lock); in __submit_request()
2400 down_write(&osdc->lock); in __submit_request()
2408 WARN_ON(req->r_flags & (CEPH_OSD_FLAG_ACK | CEPH_OSD_FLAG_ONDISK)); in account_request()
2409 WARN_ON(!(req->r_flags & (CEPH_OSD_FLAG_READ | CEPH_OSD_FLAG_WRITE))); in account_request()
2411 req->r_flags |= CEPH_OSD_FLAG_ONDISK; in account_request()
2412 atomic_inc(&req->r_osdc->num_requests); in account_request()
2414 req->r_start_stamp = jiffies; in account_request()
2415 req->r_start_latency = ktime_get(); in account_request()
2427 struct ceph_osd_client *osdc = req->r_osdc; in finish_request()
2429 WARN_ON(lookup_request_mc(&osdc->map_checks, req->r_tid)); in finish_request()
2430 dout("%s req %p tid %llu\n", __func__, req, req->r_tid); in finish_request()
2432 req->r_end_latency = ktime_get(); in finish_request()
2434 if (req->r_osd) in finish_request()
2435 unlink_request(req->r_osd, req); in finish_request()
2436 atomic_dec(&osdc->num_requests); in finish_request()
2444 ceph_msg_revoke(req->r_request); in finish_request()
2445 ceph_msg_revoke_incoming(req->r_reply); in finish_request()
2451 req->r_tid, req->r_callback, req->r_result); in __complete_request()
2453 if (req->r_callback) in __complete_request()
2454 req->r_callback(req); in __complete_request()
2455 complete_all(&req->r_completion); in __complete_request()
2468 * This is open-coded in handle_reply().
2472 dout("%s req %p tid %llu err %d\n", __func__, req, req->r_tid, err); in complete_request()
2474 req->r_result = err; in complete_request()
2477 INIT_WORK(&req->r_complete_work, complete_request_workfn); in complete_request()
2478 queue_work(req->r_osdc->completion_wq, &req->r_complete_work); in complete_request()
2483 struct ceph_osd_client *osdc = req->r_osdc; in cancel_map_check()
2488 lookup_req = lookup_request_mc(&osdc->map_checks, req->r_tid); in cancel_map_check()
2493 erase_request_mc(&osdc->map_checks, req); in cancel_map_check()
2499 dout("%s req %p tid %llu\n", __func__, req, req->r_tid); in cancel_request()
2503 complete_all(&req->r_completion); in cancel_request()
2509 dout("%s req %p tid %llu err %d\n", __func__, req, req->r_tid, err); in abort_request()
2524 * Abort all in-flight requests with @err and arrange for all future
2530 down_write(&osdc->lock); in ceph_osdc_abort_requests()
2532 osdc->abort_err = err; in ceph_osdc_abort_requests()
2533 up_write(&osdc->lock); in ceph_osdc_abort_requests()
2539 down_write(&osdc->lock); in ceph_osdc_clear_abort_err()
2540 osdc->abort_err = 0; in ceph_osdc_clear_abort_err()
2541 up_write(&osdc->lock); in ceph_osdc_clear_abort_err()
2547 if (likely(eb > osdc->epoch_barrier)) { in update_epoch_barrier()
2549 osdc->epoch_barrier, eb); in update_epoch_barrier()
2550 osdc->epoch_barrier = eb; in update_epoch_barrier()
2552 if (eb > osdc->osdmap->epoch) in update_epoch_barrier()
2559 down_read(&osdc->lock); in ceph_osdc_update_epoch_barrier()
2560 if (unlikely(eb > osdc->epoch_barrier)) { in ceph_osdc_update_epoch_barrier()
2561 up_read(&osdc->lock); in ceph_osdc_update_epoch_barrier()
2562 down_write(&osdc->lock); in ceph_osdc_update_epoch_barrier()
2564 up_write(&osdc->lock); in ceph_osdc_update_epoch_barrier()
2566 up_read(&osdc->lock); in ceph_osdc_update_epoch_barrier()
2579 struct ceph_osd_client *osdc = req->r_osdc; in abort_on_full_fn()
2582 if ((req->r_flags & CEPH_OSD_FLAG_WRITE) && in abort_on_full_fn()
2584 pool_full(osdc, req->r_t.base_oloc.pool))) { in abort_on_full_fn()
2586 update_epoch_barrier(osdc, osdc->osdmap->epoch); in abort_on_full_fn()
2589 abort_request(req, -ENOSPC); in abort_on_full_fn()
2598 * osdc->epoch_barrier to the latest map epoch that we've seen if any were
2605 if (ceph_test_opt(osdc->client, ABORT_ON_FULL) && in ceph_osdc_abort_on_full()
2612 struct ceph_osd_client *osdc = req->r_osdc; in check_pool_dne()
2613 struct ceph_osdmap *map = osdc->osdmap; in check_pool_dne()
2616 WARN_ON(!map->epoch); in check_pool_dne()
2618 if (req->r_attempts) { in check_pool_dne()
2624 req->r_map_dne_bound = map->epoch; in check_pool_dne()
2626 req->r_tid); in check_pool_dne()
2629 req, req->r_tid, req->r_map_dne_bound, map->epoch); in check_pool_dne()
2632 if (req->r_map_dne_bound) { in check_pool_dne()
2633 if (map->epoch >= req->r_map_dne_bound) { in check_pool_dne()
2636 req->r_tid); in check_pool_dne()
2637 complete_request(req, -ENOENT); in check_pool_dne()
2646 struct ceph_osd_client *osdc = &greq->monc->client->osdc; in map_check_cb()
2648 u64 tid = greq->private_data; in map_check_cb()
2650 WARN_ON(greq->result || !greq->u.newest); in map_check_cb()
2652 down_write(&osdc->lock); in map_check_cb()
2653 req = lookup_request_mc(&osdc->map_checks, tid); in map_check_cb()
2660 req, req->r_tid, req->r_map_dne_bound, greq->u.newest); in map_check_cb()
2661 if (!req->r_map_dne_bound) in map_check_cb()
2662 req->r_map_dne_bound = greq->u.newest; in map_check_cb()
2663 erase_request_mc(&osdc->map_checks, req); in map_check_cb()
2668 up_write(&osdc->lock); in map_check_cb()
2673 struct ceph_osd_client *osdc = req->r_osdc; in send_map_check()
2679 lookup_req = lookup_request_mc(&osdc->map_checks, req->r_tid); in send_map_check()
2686 insert_request_mc(&osdc->map_checks, req); in send_map_check()
2687 ret = ceph_monc_get_version_async(&osdc->client->monc, "osdmap", in send_map_check()
2688 map_check_cb, req->r_tid); in send_map_check()
2701 lreq->reg_req, lreq->ping_req); in linger_release()
2702 WARN_ON(!RB_EMPTY_NODE(&lreq->node)); in linger_release()
2703 WARN_ON(!RB_EMPTY_NODE(&lreq->osdc_node)); in linger_release()
2704 WARN_ON(!RB_EMPTY_NODE(&lreq->mc_node)); in linger_release()
2705 WARN_ON(!list_empty(&lreq->scan_item)); in linger_release()
2706 WARN_ON(!list_empty(&lreq->pending_lworks)); in linger_release()
2707 WARN_ON(lreq->osd); in linger_release()
2709 if (lreq->request_pl) in linger_release()
2710 ceph_pagelist_release(lreq->request_pl); in linger_release()
2711 if (lreq->notify_id_pages) in linger_release()
2712 ceph_release_page_vector(lreq->notify_id_pages, 1); in linger_release()
2714 ceph_osdc_put_request(lreq->reg_req); in linger_release()
2715 ceph_osdc_put_request(lreq->ping_req); in linger_release()
2716 target_destroy(&lreq->t); in linger_release()
2723 kref_put(&lreq->kref, linger_release); in linger_put()
2729 kref_get(&lreq->kref); in linger_get()
2742 kref_init(&lreq->kref); in linger_alloc()
2743 mutex_init(&lreq->lock); in linger_alloc()
2744 RB_CLEAR_NODE(&lreq->node); in linger_alloc()
2745 RB_CLEAR_NODE(&lreq->osdc_node); in linger_alloc()
2746 RB_CLEAR_NODE(&lreq->mc_node); in linger_alloc()
2747 INIT_LIST_HEAD(&lreq->scan_item); in linger_alloc()
2748 INIT_LIST_HEAD(&lreq->pending_lworks); in linger_alloc()
2749 init_completion(&lreq->reg_commit_wait); in linger_alloc()
2750 init_completion(&lreq->notify_finish_wait); in linger_alloc()
2752 lreq->osdc = osdc; in linger_alloc()
2753 target_init(&lreq->t); in linger_alloc()
2764 * Create linger request <-> OSD session relation. in DEFINE_RB_INSDEL_FUNCS()
2772 WARN_ON(!lreq->linger_id || lreq->osd); in DEFINE_RB_INSDEL_FUNCS()
2774 osd->o_osd, lreq, lreq->linger_id); in DEFINE_RB_INSDEL_FUNCS()
2779 atomic_inc(&osd->o_osdc->num_homeless); in DEFINE_RB_INSDEL_FUNCS()
2782 insert_linger(&osd->o_linger_requests, lreq); in DEFINE_RB_INSDEL_FUNCS()
2783 lreq->osd = osd; in DEFINE_RB_INSDEL_FUNCS()
2790 WARN_ON(lreq->osd != osd); in unlink_linger()
2792 osd->o_osd, lreq, lreq->linger_id); in unlink_linger()
2794 lreq->osd = NULL; in unlink_linger()
2795 erase_linger(&osd->o_linger_requests, lreq); in unlink_linger()
2801 atomic_dec(&osd->o_osdc->num_homeless); in unlink_linger()
2806 verify_osdc_locked(lreq->osdc); in __linger_registered()
2808 return !RB_EMPTY_NODE(&lreq->osdc_node); in __linger_registered()
2813 struct ceph_osd_client *osdc = lreq->osdc; in linger_registered()
2816 down_read(&osdc->lock); in linger_registered()
2818 up_read(&osdc->lock); in linger_registered()
2825 struct ceph_osd_client *osdc = lreq->osdc; in linger_register()
2828 WARN_ON(lreq->linger_id); in linger_register()
2831 lreq->linger_id = ++osdc->last_linger_id; in linger_register()
2832 insert_linger_osdc(&osdc->linger_requests, lreq); in linger_register()
2837 struct ceph_osd_client *osdc = lreq->osdc; in linger_unregister()
2841 erase_linger_osdc(&osdc->linger_requests, lreq); in linger_unregister()
2847 struct ceph_osd_linger_request *lreq = req->r_priv; in cancel_linger_request()
2849 WARN_ON(!req->r_linger); in cancel_linger_request()
2884 INIT_WORK(&lwork->work, workfn); in lwork_alloc()
2885 INIT_LIST_HEAD(&lwork->pending_item); in lwork_alloc()
2886 lwork->lreq = linger_get(lreq); in lwork_alloc()
2893 struct ceph_osd_linger_request *lreq = lwork->lreq; in lwork_free()
2895 mutex_lock(&lreq->lock); in lwork_free()
2896 list_del(&lwork->pending_item); in lwork_free()
2897 mutex_unlock(&lreq->lock); in lwork_free()
2905 struct ceph_osd_linger_request *lreq = lwork->lreq; in lwork_queue()
2906 struct ceph_osd_client *osdc = lreq->osdc; in lwork_queue()
2909 WARN_ON(!list_empty(&lwork->pending_item)); in lwork_queue()
2911 lwork->queued_stamp = jiffies; in lwork_queue()
2912 list_add_tail(&lwork->pending_item, &lreq->pending_lworks); in lwork_queue()
2913 queue_work(osdc->notify_wq, &lwork->work); in lwork_queue()
2919 struct ceph_osd_linger_request *lreq = lwork->lreq; in do_watch_notify()
2926 WARN_ON(!lreq->is_watch); in do_watch_notify()
2928 __func__, lreq, lwork->notify.notify_id, lwork->notify.notifier_id, in do_watch_notify()
2929 lwork->notify.payload_len); in do_watch_notify()
2930 lreq->wcb(lreq->data, lwork->notify.notify_id, lreq->linger_id, in do_watch_notify()
2931 lwork->notify.notifier_id, lwork->notify.payload, in do_watch_notify()
2932 lwork->notify.payload_len); in do_watch_notify()
2935 ceph_msg_put(lwork->notify.msg); in do_watch_notify()
2942 struct ceph_osd_linger_request *lreq = lwork->lreq; in do_watch_error()
2949 dout("%s lreq %p err %d\n", __func__, lreq, lwork->error.err); in do_watch_error()
2950 lreq->errcb(lreq->data, lreq->linger_id, lwork->error.err); in do_watch_error()
2962 pr_err("failed to allocate error-lwork\n"); in queue_watch_error()
2966 lwork->error.err = lreq->last_error; in queue_watch_error()
2973 if (!completion_done(&lreq->reg_commit_wait)) { in linger_reg_commit_complete()
2974 lreq->reg_commit_error = (result <= 0 ? result : 0); in linger_reg_commit_complete()
2975 complete_all(&lreq->reg_commit_wait); in linger_reg_commit_complete()
2981 struct ceph_osd_linger_request *lreq = req->r_priv; in linger_commit_cb()
2983 mutex_lock(&lreq->lock); in linger_commit_cb()
2984 if (req != lreq->reg_req) { in linger_commit_cb()
2986 __func__, lreq, lreq->linger_id, req, lreq->reg_req); in linger_commit_cb()
2991 lreq->linger_id, req->r_result); in linger_commit_cb()
2992 linger_reg_commit_complete(lreq, req->r_result); in linger_commit_cb()
2993 lreq->committed = true; in linger_commit_cb()
2995 if (!lreq->is_watch) { in linger_commit_cb()
2998 void *p = page_address(osd_data->pages[0]); in linger_commit_cb()
3000 WARN_ON(req->r_ops[0].op != CEPH_OSD_OP_NOTIFY || in linger_commit_cb()
3001 osd_data->type != CEPH_OSD_DATA_TYPE_PAGES); in linger_commit_cb()
3004 if (req->r_ops[0].outdata_len >= sizeof(u64)) { in linger_commit_cb()
3005 lreq->notify_id = ceph_decode_64(&p); in linger_commit_cb()
3007 lreq->notify_id); in linger_commit_cb()
3014 mutex_unlock(&lreq->lock); in linger_commit_cb()
3021 * Translate ENOENT -> ENOTCONN so that a delete->disconnection in normalize_watch_error()
3025 if (err == -ENOENT) in normalize_watch_error()
3026 err = -ENOTCONN; in normalize_watch_error()
3033 struct ceph_osd_linger_request *lreq = req->r_priv; in linger_reconnect_cb()
3035 mutex_lock(&lreq->lock); in linger_reconnect_cb()
3036 if (req != lreq->reg_req) { in linger_reconnect_cb()
3038 __func__, lreq, lreq->linger_id, req, lreq->reg_req); in linger_reconnect_cb()
3043 lreq, lreq->linger_id, req->r_result, lreq->last_error); in linger_reconnect_cb()
3044 if (req->r_result < 0) { in linger_reconnect_cb()
3045 if (!lreq->last_error) { in linger_reconnect_cb()
3046 lreq->last_error = normalize_watch_error(req->r_result); in linger_reconnect_cb()
3052 mutex_unlock(&lreq->lock); in linger_reconnect_cb()
3058 struct ceph_osd_client *osdc = lreq->osdc; in send_linger()
3063 mutex_lock(&lreq->lock); in send_linger()
3064 dout("%s lreq %p linger_id %llu\n", __func__, lreq, lreq->linger_id); in send_linger()
3066 if (lreq->reg_req) { in send_linger()
3067 if (lreq->reg_req->r_osd) in send_linger()
3068 cancel_linger_request(lreq->reg_req); in send_linger()
3069 ceph_osdc_put_request(lreq->reg_req); in send_linger()
3075 target_copy(&req->r_t, &lreq->t); in send_linger()
3076 req->r_mtime = lreq->mtime; in send_linger()
3078 if (lreq->is_watch && lreq->committed) { in send_linger()
3080 lreq->linger_id, ++lreq->register_gen); in send_linger()
3082 req->r_ops[0].watch.gen); in send_linger()
3083 req->r_callback = linger_reconnect_cb; in send_linger()
3085 if (lreq->is_watch) { in send_linger()
3087 lreq->linger_id, 0); in send_linger()
3089 lreq->notify_id = 0; in send_linger()
3091 refcount_inc(&lreq->request_pl->refcnt); in send_linger()
3092 osd_req_op_notify_init(req, 0, lreq->linger_id, in send_linger()
3093 lreq->request_pl); in send_linger()
3096 lreq->notify_id_pages, PAGE_SIZE, 0, false, false); in send_linger()
3099 req->r_callback = linger_commit_cb; in send_linger()
3105 req->r_priv = linger_get(lreq); in send_linger()
3106 req->r_linger = true; in send_linger()
3107 lreq->reg_req = req; in send_linger()
3108 mutex_unlock(&lreq->lock); in send_linger()
3115 struct ceph_osd_linger_request *lreq = req->r_priv; in linger_ping_cb()
3117 mutex_lock(&lreq->lock); in linger_ping_cb()
3118 if (req != lreq->ping_req) { in linger_ping_cb()
3120 __func__, lreq, lreq->linger_id, req, lreq->ping_req); in linger_ping_cb()
3125 __func__, lreq, lreq->linger_id, req->r_result, lreq->ping_sent, in linger_ping_cb()
3126 lreq->last_error); in linger_ping_cb()
3127 if (lreq->register_gen == req->r_ops[0].watch.gen) { in linger_ping_cb()
3128 if (!req->r_result) { in linger_ping_cb()
3129 lreq->watch_valid_thru = lreq->ping_sent; in linger_ping_cb()
3130 } else if (!lreq->last_error) { in linger_ping_cb()
3131 lreq->last_error = normalize_watch_error(req->r_result); in linger_ping_cb()
3136 lreq->register_gen, req->r_ops[0].watch.gen); in linger_ping_cb()
3140 mutex_unlock(&lreq->lock); in linger_ping_cb()
3146 struct ceph_osd_client *osdc = lreq->osdc; in send_linger_ping()
3155 lreq->ping_sent = jiffies; in send_linger_ping()
3157 __func__, lreq, lreq->linger_id, lreq->ping_sent, in send_linger_ping()
3158 lreq->register_gen); in send_linger_ping()
3160 if (lreq->ping_req) { in send_linger_ping()
3161 if (lreq->ping_req->r_osd) in send_linger_ping()
3162 cancel_linger_request(lreq->ping_req); in send_linger_ping()
3163 ceph_osdc_put_request(lreq->ping_req); in send_linger_ping()
3169 target_copy(&req->r_t, &lreq->t); in send_linger_ping()
3170 osd_req_op_watch_init(req, 0, CEPH_OSD_WATCH_OP_PING, lreq->linger_id, in send_linger_ping()
3171 lreq->register_gen); in send_linger_ping()
3172 req->r_callback = linger_ping_cb; in send_linger_ping()
3177 req->r_priv = linger_get(lreq); in send_linger_ping()
3178 req->r_linger = true; in send_linger_ping()
3179 lreq->ping_req = req; in send_linger_ping()
3183 req->r_tid = atomic64_inc_return(&osdc->last_tid); in send_linger_ping()
3184 link_request(lreq->osd, req); in send_linger_ping()
3190 struct ceph_osd_client *osdc = lreq->osdc; in linger_submit()
3193 down_write(&osdc->lock); in linger_submit()
3196 calc_target(osdc, &lreq->t, false); in linger_submit()
3197 osd = lookup_create_osd(osdc, lreq->t.osd, true); in linger_submit()
3201 up_write(&osdc->lock); in linger_submit()
3206 struct ceph_osd_client *osdc = lreq->osdc; in cancel_linger_map_check()
3211 lookup_lreq = lookup_linger_mc(&osdc->linger_map_checks, in cancel_linger_map_check()
3212 lreq->linger_id); in cancel_linger_map_check()
3217 erase_linger_mc(&osdc->linger_map_checks, lreq); in cancel_linger_map_check()
3226 if (lreq->ping_req && lreq->ping_req->r_osd) in __linger_cancel()
3227 cancel_linger_request(lreq->ping_req); in __linger_cancel()
3228 if (lreq->reg_req && lreq->reg_req->r_osd) in __linger_cancel()
3229 cancel_linger_request(lreq->reg_req); in __linger_cancel()
3231 unlink_linger(lreq->osd, lreq); in __linger_cancel()
3237 struct ceph_osd_client *osdc = lreq->osdc; in linger_cancel()
3239 down_write(&osdc->lock); in linger_cancel()
3242 up_write(&osdc->lock); in linger_cancel()
3249 struct ceph_osd_client *osdc = lreq->osdc; in check_linger_pool_dne()
3250 struct ceph_osdmap *map = osdc->osdmap; in check_linger_pool_dne()
3253 WARN_ON(!map->epoch); in check_linger_pool_dne()
3255 if (lreq->register_gen) { in check_linger_pool_dne()
3256 lreq->map_dne_bound = map->epoch; in check_linger_pool_dne()
3258 lreq, lreq->linger_id); in check_linger_pool_dne()
3261 __func__, lreq, lreq->linger_id, lreq->map_dne_bound, in check_linger_pool_dne()
3262 map->epoch); in check_linger_pool_dne()
3265 if (lreq->map_dne_bound) { in check_linger_pool_dne()
3266 if (map->epoch >= lreq->map_dne_bound) { in check_linger_pool_dne()
3269 lreq->linger_id); in check_linger_pool_dne()
3270 linger_reg_commit_complete(lreq, -ENOENT); in check_linger_pool_dne()
3280 struct ceph_osd_client *osdc = &greq->monc->client->osdc; in linger_map_check_cb()
3282 u64 linger_id = greq->private_data; in linger_map_check_cb()
3284 WARN_ON(greq->result || !greq->u.newest); in linger_map_check_cb()
3286 down_write(&osdc->lock); in linger_map_check_cb()
3287 lreq = lookup_linger_mc(&osdc->linger_map_checks, linger_id); in linger_map_check_cb()
3294 __func__, lreq, lreq->linger_id, lreq->map_dne_bound, in linger_map_check_cb()
3295 greq->u.newest); in linger_map_check_cb()
3296 if (!lreq->map_dne_bound) in linger_map_check_cb()
3297 lreq->map_dne_bound = greq->u.newest; in linger_map_check_cb()
3298 erase_linger_mc(&osdc->linger_map_checks, lreq); in linger_map_check_cb()
3303 up_write(&osdc->lock); in linger_map_check_cb()
3308 struct ceph_osd_client *osdc = lreq->osdc; in send_linger_map_check()
3314 lookup_lreq = lookup_linger_mc(&osdc->linger_map_checks, in send_linger_map_check()
3315 lreq->linger_id); in send_linger_map_check()
3322 insert_linger_mc(&osdc->linger_map_checks, lreq); in send_linger_map_check()
3323 ret = ceph_monc_get_version_async(&osdc->client->monc, "osdmap", in send_linger_map_check()
3324 linger_map_check_cb, lreq->linger_id); in send_linger_map_check()
3332 dout("%s lreq %p linger_id %llu\n", __func__, lreq, lreq->linger_id); in linger_reg_commit_wait()
3333 ret = wait_for_completion_interruptible(&lreq->reg_commit_wait); in linger_reg_commit_wait()
3334 return ret ?: lreq->reg_commit_error; in linger_reg_commit_wait()
3341 dout("%s lreq %p linger_id %llu\n", __func__, lreq, lreq->linger_id); in linger_notify_finish_wait()
3342 ret = wait_for_completion_interruptible(&lreq->notify_finish_wait); in linger_notify_finish_wait()
3343 return ret ?: lreq->notify_finish_error; in linger_notify_finish_wait()
3356 struct ceph_options *opts = osdc->client->options; in handle_timeout()
3357 unsigned long cutoff = jiffies - opts->osd_keepalive_timeout; in handle_timeout()
3358 unsigned long expiry_cutoff = jiffies - opts->osd_request_timeout; in handle_timeout()
3363 down_write(&osdc->lock); in handle_timeout()
3370 for (n = rb_first(&osdc->osds); n; n = rb_next(n)) { in handle_timeout()
3374 for (p = rb_first(&osd->o_requests); p; ) { in handle_timeout()
3380 if (time_before(req->r_stamp, cutoff)) { in handle_timeout()
3382 req, req->r_tid, osd->o_osd); in handle_timeout()
3385 if (opts->osd_request_timeout && in handle_timeout()
3386 time_before(req->r_start_stamp, expiry_cutoff)) { in handle_timeout()
3388 req->r_tid, osd->o_osd); in handle_timeout()
3389 abort_request(req, -ETIMEDOUT); in handle_timeout()
3392 for (p = rb_first(&osd->o_linger_requests); p; p = rb_next(p)) { in handle_timeout()
3397 lreq, lreq->linger_id, osd->o_osd); in handle_timeout()
3400 mutex_lock(&lreq->lock); in handle_timeout()
3401 if (lreq->is_watch && lreq->committed && !lreq->last_error) in handle_timeout()
3403 mutex_unlock(&lreq->lock); in handle_timeout()
3407 list_move_tail(&osd->o_keepalive_item, &slow_osds); in handle_timeout()
3410 if (opts->osd_request_timeout) { in handle_timeout()
3411 for (p = rb_first(&osdc->homeless_osd.o_requests); p; ) { in handle_timeout()
3417 if (time_before(req->r_start_stamp, expiry_cutoff)) { in handle_timeout()
3419 req->r_tid, osdc->homeless_osd.o_osd); in handle_timeout()
3420 abort_request(req, -ETIMEDOUT); in handle_timeout()
3425 if (atomic_read(&osdc->num_homeless) || !list_empty(&slow_osds)) in handle_timeout()
3432 list_del_init(&osd->o_keepalive_item); in handle_timeout()
3433 ceph_con_keepalive(&osd->o_con); in handle_timeout()
3436 up_write(&osdc->lock); in handle_timeout()
3437 schedule_delayed_work(&osdc->timeout_work, in handle_timeout()
3438 osdc->client->options->osd_keepalive_timeout); in handle_timeout()
3446 unsigned long delay = osdc->client->options->osd_idle_ttl / 4; in handle_osds_timeout()
3450 down_write(&osdc->lock); in handle_osds_timeout()
3451 list_for_each_entry_safe(osd, nosd, &osdc->osd_lru, o_osd_lru) { in handle_osds_timeout()
3452 if (time_before(jiffies, osd->lru_ttl)) in handle_osds_timeout()
3455 WARN_ON(!RB_EMPTY_ROOT(&osd->o_requests)); in handle_osds_timeout()
3456 WARN_ON(!RB_EMPTY_ROOT(&osd->o_linger_requests)); in handle_osds_timeout()
3460 up_write(&osdc->lock); in handle_osds_timeout()
3461 schedule_delayed_work(&osdc->osds_timeout_work, in handle_osds_timeout()
3490 oloc->pool = ceph_decode_64(p); in ceph_oloc_decode()
3505 if (!oloc->pool_ns || in ceph_oloc_decode()
3506 ceph_compare_string(oloc->pool_ns, *p, len)) in ceph_oloc_decode()
3510 if (oloc->pool_ns) in ceph_oloc_decode()
3522 if (hash != -1) { in ceph_oloc_decode()
3534 ret = -EINVAL; in ceph_oloc_decode()
3558 ret = ceph_oloc_decode(p, end, &redir->oloc); in ceph_redirect_decode()
3574 ret = -EINVAL; in ceph_redirect_decode()
3594 void *p = msg->front.iov_base; in decode_MOSDOpReply()
3595 void *const end = p + msg->front.iov_len; in decode_MOSDOpReply()
3596 u16 version = le16_to_cpu(msg->hdr.version); in decode_MOSDOpReply()
3607 ret = ceph_decode_pgid(&p, end, &m->pgid); in decode_MOSDOpReply()
3611 ceph_decode_64_safe(&p, end, m->flags, e_inval); in decode_MOSDOpReply()
3612 ceph_decode_32_safe(&p, end, m->result, e_inval); in decode_MOSDOpReply()
3616 ceph_decode_32_safe(&p, end, m->epoch, e_inval); in decode_MOSDOpReply()
3618 ceph_decode_32_safe(&p, end, m->num_ops, e_inval); in decode_MOSDOpReply()
3619 if (m->num_ops > ARRAY_SIZE(m->outdata_len)) in decode_MOSDOpReply()
3622 ceph_decode_need(&p, end, m->num_ops * sizeof(struct ceph_osd_op), in decode_MOSDOpReply()
3624 for (i = 0; i < m->num_ops; i++) { in decode_MOSDOpReply()
3627 m->outdata_len[i] = le32_to_cpu(op->payload_len); in decode_MOSDOpReply()
3631 ceph_decode_32_safe(&p, end, m->retry_attempt, e_inval); in decode_MOSDOpReply()
3632 for (i = 0; i < m->num_ops; i++) in decode_MOSDOpReply()
3633 ceph_decode_32_safe(&p, end, m->rval[i], e_inval); in decode_MOSDOpReply()
3636 ceph_decode_need(&p, end, sizeof(m->replay_version), e_inval); in decode_MOSDOpReply()
3637 memcpy(&m->replay_version, p, sizeof(m->replay_version)); in decode_MOSDOpReply()
3638 p += sizeof(m->replay_version); in decode_MOSDOpReply()
3639 ceph_decode_64_safe(&p, end, m->user_version, e_inval); in decode_MOSDOpReply()
3641 m->replay_version = bad_replay_version; /* struct */ in decode_MOSDOpReply()
3642 m->user_version = le64_to_cpu(m->replay_version.version); in decode_MOSDOpReply()
3655 ret = ceph_redirect_decode(&p, end, &m->redirect); in decode_MOSDOpReply()
3659 ceph_oloc_init(&m->redirect.oloc); in decode_MOSDOpReply()
3665 return -EINVAL; in decode_MOSDOpReply()
3669 * Handle MOSDOpReply. Set ->r_result and call the callback if it is
3674 struct ceph_osd_client *osdc = osd->o_osdc; in handle_reply()
3677 u64 tid = le64_to_cpu(msg->hdr.tid); in handle_reply()
3684 down_read(&osdc->lock); in handle_reply()
3686 dout("%s osd%d unknown\n", __func__, osd->o_osd); in handle_reply()
3689 WARN_ON(osd->o_osd != le64_to_cpu(msg->hdr.src.num)); in handle_reply()
3691 mutex_lock(&osd->lock); in handle_reply()
3692 req = lookup_request(&osd->o_requests, tid); in handle_reply()
3694 dout("%s osd%d tid %llu unknown\n", __func__, osd->o_osd, tid); in handle_reply()
3698 m.redirect.oloc.pool_ns = req->r_t.target_oloc.pool_ns; in handle_reply()
3703 req->r_tid, ret); in handle_reply()
3708 __func__, req, req->r_tid, m.flags, m.pgid.pool, m.pgid.seed, in handle_reply()
3713 if (m.retry_attempt != req->r_attempts - 1) { in handle_reply()
3715 req, req->r_tid, m.retry_attempt, in handle_reply()
3716 req->r_attempts - 1); in handle_reply()
3724 dout("req %p tid %llu redirect pool %lld\n", req, req->r_tid, in handle_reply()
3727 mutex_unlock(&osd->lock); in handle_reply()
3730 * Not ceph_oloc_copy() - changing pool_ns is not in handle_reply()
3733 req->r_t.target_oloc.pool = m.redirect.oloc.pool; in handle_reply()
3734 req->r_flags |= CEPH_OSD_FLAG_REDIRECTED | in handle_reply()
3737 req->r_tid = 0; in handle_reply()
3742 if (m.result == -EAGAIN) { in handle_reply()
3743 dout("req %p tid %llu EAGAIN\n", req, req->r_tid); in handle_reply()
3745 mutex_unlock(&osd->lock); in handle_reply()
3752 req->r_t.pgid.pool = 0; in handle_reply()
3753 req->r_t.pgid.seed = 0; in handle_reply()
3754 WARN_ON(!req->r_t.used_replica); in handle_reply()
3755 req->r_flags &= ~(CEPH_OSD_FLAG_BALANCE_READS | in handle_reply()
3757 req->r_tid = 0; in handle_reply()
3762 if (m.num_ops != req->r_num_ops) { in handle_reply()
3764 req->r_num_ops, req->r_tid); in handle_reply()
3767 for (i = 0; i < req->r_num_ops; i++) { in handle_reply()
3769 req->r_tid, i, m.rval[i], m.outdata_len[i]); in handle_reply()
3770 req->r_ops[i].rval = m.rval[i]; in handle_reply()
3771 req->r_ops[i].outdata_len = m.outdata_len[i]; in handle_reply()
3774 if (data_len != le32_to_cpu(msg->hdr.data_len)) { in handle_reply()
3776 le32_to_cpu(msg->hdr.data_len), req->r_tid); in handle_reply()
3780 req, req->r_tid, m.result, data_len); in handle_reply()
3787 req->r_result = m.result ?: data_len; in handle_reply()
3789 mutex_unlock(&osd->lock); in handle_reply()
3790 up_read(&osdc->lock); in handle_reply()
3796 complete_request(req, -EIO); in handle_reply()
3798 mutex_unlock(&osd->lock); in handle_reply()
3800 up_read(&osdc->lock); in handle_reply()
3807 for (n = rb_first(&osdc->osdmap->pg_pools); n; n = rb_next(n)) { in set_pool_was_full()
3811 pi->was_full = __pool_full(pi); in set_pool_was_full()
3819 pi = ceph_pg_pool_by_id(osdc->osdmap, pool_id); in pool_cleared_full()
3823 return pi->was_full && !__pool_full(pi); in pool_cleared_full()
3829 struct ceph_osd_client *osdc = lreq->osdc; in recalc_linger_target()
3832 ct_res = calc_target(osdc, &lreq->t, true); in recalc_linger_target()
3836 osd = lookup_create_osd(osdc, lreq->t.osd, true); in recalc_linger_target()
3837 if (osd != lreq->osd) { in recalc_linger_target()
3838 unlink_linger(lreq->osd, lreq); in recalc_linger_target()
3856 struct ceph_osd_client *osdc = osd->o_osdc; in scan_requests()
3860 for (n = rb_first(&osd->o_linger_requests); n; ) { in scan_requests()
3868 lreq->linger_id); in scan_requests()
3874 pool_cleared_full(osdc, lreq->t.base_oloc.pool)); in scan_requests()
3886 if (list_empty(&lreq->scan_item)) in scan_requests()
3887 list_add_tail(&lreq->scan_item, need_resend_linger); in scan_requests()
3890 list_del_init(&lreq->scan_item); in scan_requests()
3896 for (n = rb_first(&osd->o_requests); n; ) { in scan_requests()
3903 dout("%s req %p tid %llu\n", __func__, req, req->r_tid); in scan_requests()
3904 ct_res = calc_target(osdc, &req->r_t, false); in scan_requests()
3909 pool_cleared_full(osdc, req->r_t.base_oloc.pool)); in scan_requests()
3911 (!(req->r_flags & CEPH_OSD_FLAG_WRITE) || in scan_requests()
3942 newmap = osdmap_apply_incremental(&p, end, osdc->osdmap); in handle_one_map()
3948 if (newmap != osdc->osdmap) { in handle_one_map()
3950 * Preserve ->was_full before destroying the old map. in handle_one_map()
3951 * For pools that weren't in the old map, ->was_full in handle_one_map()
3954 for (n = rb_first(&newmap->pg_pools); n; n = rb_next(n)) { in handle_one_map()
3959 old_pi = ceph_pg_pool_by_id(osdc->osdmap, pi->id); in handle_one_map()
3961 pi->was_full = old_pi->was_full; in handle_one_map()
3963 WARN_ON(pi->was_full); in handle_one_map()
3966 if (osdc->osdmap->epoch && in handle_one_map()
3967 osdc->osdmap->epoch + 1 < newmap->epoch) { in handle_one_map()
3972 ceph_osdmap_destroy(osdc->osdmap); in handle_one_map()
3973 osdc->osdmap = newmap; in handle_one_map()
3977 scan_requests(&osdc->homeless_osd, skipped_map, was_full, true, in handle_one_map()
3980 for (n = rb_first(&osdc->osds); n; ) { in handle_one_map()
3987 if (!ceph_osd_is_up(osdc->osdmap, osd->o_osd) || in handle_one_map()
3988 memcmp(&osd->o_con.peer_addr, in handle_one_map()
3989 ceph_osd_addr(osdc->osdmap, osd->o_osd), in handle_one_map()
4012 if (req->r_t.epoch < osdc->osdmap->epoch) { in kick_requests()
4013 ct_res = calc_target(osdc, &req->r_t, false); in kick_requests()
4029 osd = lookup_create_osd(osdc, req->r_t.osd, true); in kick_requests()
4031 if (!req->r_linger) { in kick_requests()
4032 if (!osd_homeless(osd) && !req->r_t.paused) in kick_requests()
4040 if (!osd_homeless(lreq->osd)) in kick_requests()
4043 list_del_init(&lreq->scan_item); in kick_requests()
4056 void *p = msg->front.iov_base; in ceph_osdc_handle_map()
4057 void *const end = p + msg->front.iov_len; in ceph_osdc_handle_map()
4068 dout("%s have %u\n", __func__, osdc->osdmap->epoch); in ceph_osdc_handle_map()
4069 down_write(&osdc->lock); in ceph_osdc_handle_map()
4074 if (ceph_check_fsid(osdc->client, &fsid) < 0) in ceph_osdc_handle_map()
4090 if (osdc->osdmap->epoch && in ceph_osdc_handle_map()
4091 osdc->osdmap->epoch + 1 == epoch) { in ceph_osdc_handle_map()
4104 nr_maps--; in ceph_osdc_handle_map()
4118 dout("skipping non-latest full map %u len %d\n", in ceph_osdc_handle_map()
4120 } else if (osdc->osdmap->epoch >= epoch) { in ceph_osdc_handle_map()
4123 osdc->osdmap->epoch); in ceph_osdc_handle_map()
4132 nr_maps--; in ceph_osdc_handle_map()
4146 osdc->osdmap->epoch < osdc->epoch_barrier) in ceph_osdc_handle_map()
4152 ceph_monc_got_map(&osdc->client->monc, CEPH_SUB_OSDMAP, in ceph_osdc_handle_map()
4153 osdc->osdmap->epoch); in ceph_osdc_handle_map()
4154 up_write(&osdc->lock); in ceph_osdc_handle_map()
4155 wake_up_all(&osdc->client->auth_wq); in ceph_osdc_handle_map()
4161 up_write(&osdc->lock); in ceph_osdc_handle_map()
4173 for (n = rb_first(&osd->o_requests); n; ) { in kick_osd_requests()
4179 if (!req->r_linger) { in kick_osd_requests()
4180 if (!req->r_t.paused) in kick_osd_requests()
4186 for (n = rb_first(&osd->o_linger_requests); n; n = rb_next(n)) { in kick_osd_requests()
4199 struct ceph_osd *osd = con->private; in osd_fault()
4200 struct ceph_osd_client *osdc = osd->o_osdc; in osd_fault()
4202 dout("%s osd %p osd%d\n", __func__, osd, osd->o_osd); in osd_fault()
4204 down_write(&osdc->lock); in osd_fault()
4206 dout("%s osd%d unknown\n", __func__, osd->o_osd); in osd_fault()
4215 up_write(&osdc->lock); in osd_fault()
4229 void *p = msg->front.iov_base; in decode_MOSDBackoff()
4230 void *const end = p + msg->front.iov_len; in decode_MOSDBackoff()
4239 ret = ceph_decode_pgid(&p, end, &m->spgid.pgid); in decode_MOSDBackoff()
4243 ceph_decode_8_safe(&p, end, m->spgid.shard, e_inval); in decode_MOSDBackoff()
4244 ceph_decode_32_safe(&p, end, m->map_epoch, e_inval); in decode_MOSDBackoff()
4245 ceph_decode_8_safe(&p, end, m->op, e_inval); in decode_MOSDBackoff()
4246 ceph_decode_64_safe(&p, end, m->id, e_inval); in decode_MOSDBackoff()
4248 m->begin = kzalloc(sizeof(*m->begin), GFP_NOIO); in decode_MOSDBackoff()
4249 if (!m->begin) in decode_MOSDBackoff()
4250 return -ENOMEM; in decode_MOSDBackoff()
4252 ret = decode_hoid(&p, end, m->begin); in decode_MOSDBackoff()
4254 free_hoid(m->begin); in decode_MOSDBackoff()
4258 m->end = kzalloc(sizeof(*m->end), GFP_NOIO); in decode_MOSDBackoff()
4259 if (!m->end) { in decode_MOSDBackoff()
4260 free_hoid(m->begin); in decode_MOSDBackoff()
4261 return -ENOMEM; in decode_MOSDBackoff()
4264 ret = decode_hoid(&p, end, m->end); in decode_MOSDBackoff()
4266 free_hoid(m->begin); in decode_MOSDBackoff()
4267 free_hoid(m->end); in decode_MOSDBackoff()
4274 return -EINVAL; in decode_MOSDBackoff()
4289 hoid_encoding_size(backoff->begin); in create_backoff_message()
4291 hoid_encoding_size(backoff->end); in create_backoff_message()
4297 p = msg->front.iov_base; in create_backoff_message()
4298 end = p + msg->front_alloc_len; in create_backoff_message()
4300 encode_spgid(&p, &backoff->spgid); in create_backoff_message()
4303 ceph_encode_64(&p, backoff->id); in create_backoff_message()
4304 encode_hoid(&p, end, backoff->begin); in create_backoff_message()
4305 encode_hoid(&p, end, backoff->end); in create_backoff_message()
4308 msg->front.iov_len = p - msg->front.iov_base; in create_backoff_message()
4309 msg->hdr.version = cpu_to_le16(1); /* MOSDBackoff v1 */ in create_backoff_message()
4310 msg->hdr.front_len = cpu_to_le32(msg->front.iov_len); in create_backoff_message()
4321 dout("%s osd%d spgid %llu.%xs%d id %llu\n", __func__, osd->o_osd, in handle_backoff_block()
4322 m->spgid.pgid.pool, m->spgid.pgid.seed, m->spgid.shard, m->id); in handle_backoff_block()
4324 spg = lookup_spg_mapping(&osd->o_backoff_mappings, &m->spgid); in handle_backoff_block()
4331 spg->spgid = m->spgid; /* struct */ in handle_backoff_block()
4332 insert_spg_mapping(&osd->o_backoff_mappings, spg); in handle_backoff_block()
4340 backoff->spgid = m->spgid; /* struct */ in handle_backoff_block()
4341 backoff->id = m->id; in handle_backoff_block()
4342 backoff->begin = m->begin; in handle_backoff_block()
4343 m->begin = NULL; /* backoff now owns this */ in handle_backoff_block()
4344 backoff->end = m->end; in handle_backoff_block()
4345 m->end = NULL; /* ditto */ in handle_backoff_block()
4347 insert_backoff(&spg->backoffs, backoff); in handle_backoff_block()
4348 insert_backoff_by_id(&osd->o_backoffs_by_id, backoff); in handle_backoff_block()
4354 msg = create_backoff_message(backoff, m->map_epoch); in handle_backoff_block()
4359 ceph_con_send(&osd->o_con, msg); in handle_backoff_block()
4381 dout("%s osd%d spgid %llu.%xs%d id %llu\n", __func__, osd->o_osd, in handle_backoff_unblock()
4382 m->spgid.pgid.pool, m->spgid.pgid.seed, m->spgid.shard, m->id); in handle_backoff_unblock()
4384 backoff = lookup_backoff_by_id(&osd->o_backoffs_by_id, m->id); in handle_backoff_unblock()
4387 __func__, osd->o_osd, m->spgid.pgid.pool, in handle_backoff_unblock()
4388 m->spgid.pgid.seed, m->spgid.shard, m->id); in handle_backoff_unblock()
4392 if (hoid_compare(backoff->begin, m->begin) && in handle_backoff_unblock()
4393 hoid_compare(backoff->end, m->end)) { in handle_backoff_unblock()
4395 __func__, osd->o_osd, m->spgid.pgid.pool, in handle_backoff_unblock()
4396 m->spgid.pgid.seed, m->spgid.shard, m->id); in handle_backoff_unblock()
4400 spg = lookup_spg_mapping(&osd->o_backoff_mappings, &backoff->spgid); in handle_backoff_unblock()
4403 erase_backoff(&spg->backoffs, backoff); in handle_backoff_unblock()
4404 erase_backoff_by_id(&osd->o_backoffs_by_id, backoff); in handle_backoff_unblock()
4407 if (RB_EMPTY_ROOT(&spg->backoffs)) { in handle_backoff_unblock()
4408 erase_spg_mapping(&osd->o_backoff_mappings, spg); in handle_backoff_unblock()
4412 for (n = rb_first(&osd->o_requests); n; n = rb_next(n)) { in handle_backoff_unblock()
4416 if (!ceph_spg_compare(&req->r_t.spgid, &m->spgid)) { in handle_backoff_unblock()
4418 * Match against @m, not @backoff -- the PG may in handle_backoff_unblock()
4421 if (target_contained_by(&req->r_t, m->begin, m->end)) { in handle_backoff_unblock()
4434 struct ceph_osd_client *osdc = osd->o_osdc; in handle_backoff()
4438 down_read(&osdc->lock); in handle_backoff()
4440 dout("%s osd%d unknown\n", __func__, osd->o_osd); in handle_backoff()
4441 up_read(&osdc->lock); in handle_backoff()
4444 WARN_ON(osd->o_osd != le64_to_cpu(msg->hdr.src.num)); in handle_backoff()
4446 mutex_lock(&osd->lock); in handle_backoff()
4462 pr_err("%s osd%d unknown op %d\n", __func__, osd->o_osd, m.op); in handle_backoff()
4469 mutex_unlock(&osd->lock); in handle_backoff()
4470 up_read(&osdc->lock); in handle_backoff()
4479 void *p = msg->front.iov_base; in handle_watch_notify()
4480 void *const end = p + msg->front.iov_len; in handle_watch_notify()
4503 if (le16_to_cpu(msg->hdr.version) >= 2) in handle_watch_notify()
4506 if (le16_to_cpu(msg->hdr.version) >= 3) in handle_watch_notify()
4509 down_read(&osdc->lock); in handle_watch_notify()
4510 lreq = lookup_linger_osdc(&osdc->linger_requests, cookie); in handle_watch_notify()
4517 mutex_lock(&lreq->lock); in handle_watch_notify()
4519 opcode, cookie, lreq, lreq->is_watch); in handle_watch_notify()
4521 if (!lreq->last_error) { in handle_watch_notify()
4522 lreq->last_error = -ENOTCONN; in handle_watch_notify()
4525 } else if (!lreq->is_watch) { in handle_watch_notify()
4527 if (lreq->notify_id && lreq->notify_id != notify_id) { in handle_watch_notify()
4529 lreq->notify_id, notify_id); in handle_watch_notify()
4530 } else if (!completion_done(&lreq->notify_finish_wait)) { in handle_watch_notify()
4532 msg->num_data_items ? &msg->data[0] : NULL; in handle_watch_notify()
4535 if (lreq->preply_pages) { in handle_watch_notify()
4536 WARN_ON(data->type != in handle_watch_notify()
4538 *lreq->preply_pages = data->pages; in handle_watch_notify()
4539 *lreq->preply_len = data->length; in handle_watch_notify()
4540 data->own_pages = false; in handle_watch_notify()
4543 lreq->notify_finish_error = return_code; in handle_watch_notify()
4544 complete_all(&lreq->notify_finish_wait); in handle_watch_notify()
4550 pr_err("failed to allocate notify-lwork\n"); in handle_watch_notify()
4554 lwork->notify.notify_id = notify_id; in handle_watch_notify()
4555 lwork->notify.notifier_id = notifier_id; in handle_watch_notify()
4556 lwork->notify.payload = payload; in handle_watch_notify()
4557 lwork->notify.payload_len = payload_len; in handle_watch_notify()
4558 lwork->notify.msg = ceph_msg_get(msg); in handle_watch_notify()
4563 mutex_unlock(&lreq->lock); in handle_watch_notify()
4565 up_read(&osdc->lock); in handle_watch_notify()
4579 down_read(&osdc->lock); in ceph_osdc_start_request()
4581 up_read(&osdc->lock); in ceph_osdc_start_request()
4589 * ->r_result isn't set and __complete_request() isn't called.
4593 struct ceph_osd_client *osdc = req->r_osdc; in ceph_osdc_cancel_request()
4595 down_write(&osdc->lock); in ceph_osdc_cancel_request()
4596 if (req->r_osd) in ceph_osdc_cancel_request()
4598 up_write(&osdc->lock); in ceph_osdc_cancel_request()
4610 dout("%s req %p tid %llu\n", __func__, req, req->r_tid); in wait_request_timeout()
4611 left = wait_for_completion_killable_timeout(&req->r_completion, in wait_request_timeout()
4614 left = left ?: -ETIMEDOUT; in wait_request_timeout()
4617 left = req->r_result; /* completed */ in wait_request_timeout()
4634 * sync - wait for all in-flight requests to flush. avoid starvation.
4639 u64 last_tid = atomic64_read(&osdc->last_tid); in ceph_osdc_sync()
4642 down_read(&osdc->lock); in ceph_osdc_sync()
4643 for (n = rb_first(&osdc->osds); n; n = rb_next(n)) { in ceph_osdc_sync()
4646 mutex_lock(&osd->lock); in ceph_osdc_sync()
4647 for (p = rb_first(&osd->o_requests); p; p = rb_next(p)) { in ceph_osdc_sync()
4651 if (req->r_tid > last_tid) in ceph_osdc_sync()
4654 if (!(req->r_flags & CEPH_OSD_FLAG_WRITE)) in ceph_osdc_sync()
4658 mutex_unlock(&osd->lock); in ceph_osdc_sync()
4659 up_read(&osdc->lock); in ceph_osdc_sync()
4661 __func__, req, req->r_tid, last_tid); in ceph_osdc_sync()
4662 wait_for_completion(&req->r_completion); in ceph_osdc_sync()
4667 mutex_unlock(&osd->lock); in ceph_osdc_sync()
4670 up_read(&osdc->lock); in ceph_osdc_sync()
4691 return ERR_PTR(-ENOMEM); in ceph_osdc_watch()
4693 lreq->is_watch = true; in ceph_osdc_watch()
4694 lreq->wcb = wcb; in ceph_osdc_watch()
4695 lreq->errcb = errcb; in ceph_osdc_watch()
4696 lreq->data = data; in ceph_osdc_watch()
4697 lreq->watch_valid_thru = jiffies; in ceph_osdc_watch()
4699 ceph_oid_copy(&lreq->t.base_oid, oid); in ceph_osdc_watch()
4700 ceph_oloc_copy(&lreq->t.base_oloc, oloc); in ceph_osdc_watch()
4701 lreq->t.flags = CEPH_OSD_FLAG_WRITE; in ceph_osdc_watch()
4702 ktime_get_real_ts64(&lreq->mtime); in ceph_osdc_watch()
4729 struct ceph_options *opts = osdc->client->options; in ceph_osdc_unwatch()
4735 return -ENOMEM; in ceph_osdc_unwatch()
4737 ceph_oid_copy(&req->r_base_oid, &lreq->t.base_oid); in ceph_osdc_unwatch()
4738 ceph_oloc_copy(&req->r_base_oloc, &lreq->t.base_oloc); in ceph_osdc_unwatch()
4739 req->r_flags = CEPH_OSD_FLAG_WRITE; in ceph_osdc_unwatch()
4740 ktime_get_real_ts64(&req->r_mtime); in ceph_osdc_unwatch()
4742 lreq->linger_id, 0); in ceph_osdc_unwatch()
4751 ret = wait_request_timeout(req, opts->mount_timeout); in ceph_osdc_unwatch()
4771 return -ENOMEM; in osd_req_op_notify_ack_init()
4783 return -ENOMEM; in osd_req_op_notify_ack_init()
4786 ceph_osd_data_pagelist_init(&op->notify_ack.request_data, pl); in osd_req_op_notify_ack_init()
4787 op->indata_len = pl->length; in osd_req_op_notify_ack_init()
4804 return -ENOMEM; in ceph_osdc_notify_ack()
4806 ceph_oid_copy(&req->r_base_oid, oid); in ceph_osdc_notify_ack()
4807 ceph_oloc_copy(&req->r_base_oloc, oloc); in ceph_osdc_notify_ack()
4808 req->r_flags = CEPH_OSD_FLAG_READ; in ceph_osdc_notify_ack()
4856 return -ENOMEM; in ceph_osdc_notify()
4858 lreq->request_pl = ceph_pagelist_alloc(GFP_NOIO); in ceph_osdc_notify()
4859 if (!lreq->request_pl) { in ceph_osdc_notify()
4860 ret = -ENOMEM; in ceph_osdc_notify()
4864 ret = ceph_pagelist_encode_32(lreq->request_pl, 1); /* prot_ver */ in ceph_osdc_notify()
4865 ret |= ceph_pagelist_encode_32(lreq->request_pl, timeout); in ceph_osdc_notify()
4866 ret |= ceph_pagelist_encode_32(lreq->request_pl, payload_len); in ceph_osdc_notify()
4867 ret |= ceph_pagelist_append(lreq->request_pl, payload, payload_len); in ceph_osdc_notify()
4869 ret = -ENOMEM; in ceph_osdc_notify()
4874 lreq->notify_id_pages = ceph_alloc_page_vector(1, GFP_NOIO); in ceph_osdc_notify()
4875 if (IS_ERR(lreq->notify_id_pages)) { in ceph_osdc_notify()
4876 ret = PTR_ERR(lreq->notify_id_pages); in ceph_osdc_notify()
4877 lreq->notify_id_pages = NULL; in ceph_osdc_notify()
4881 lreq->preply_pages = preply_pages; in ceph_osdc_notify()
4882 lreq->preply_len = preply_len; in ceph_osdc_notify()
4884 ceph_oid_copy(&lreq->t.base_oid, oid); in ceph_osdc_notify()
4885 ceph_oloc_copy(&lreq->t.base_oloc, oloc); in ceph_osdc_notify()
4886 lreq->t.flags = CEPH_OSD_FLAG_READ; in ceph_osdc_notify()
4913 down_read(&osdc->lock); in ceph_osdc_watch_check()
4914 mutex_lock(&lreq->lock); in ceph_osdc_watch_check()
4915 stamp = lreq->watch_valid_thru; in ceph_osdc_watch_check()
4916 if (!list_empty(&lreq->pending_lworks)) { in ceph_osdc_watch_check()
4918 list_first_entry(&lreq->pending_lworks, in ceph_osdc_watch_check()
4922 if (time_before(lwork->queued_stamp, stamp)) in ceph_osdc_watch_check()
4923 stamp = lwork->queued_stamp; in ceph_osdc_watch_check()
4925 age = jiffies - stamp; in ceph_osdc_watch_check()
4927 lreq, lreq->linger_id, age, lreq->last_error); in ceph_osdc_watch_check()
4929 ret = lreq->last_error ?: 1 + jiffies_to_msecs(age); in ceph_osdc_watch_check()
4931 mutex_unlock(&lreq->lock); in ceph_osdc_watch_check()
4932 up_read(&osdc->lock); in ceph_osdc_watch_check()
4947 ret = -EINVAL; in decode_watcher()
4948 ceph_decode_copy_safe(p, end, &item->name, sizeof(item->name), bad); in decode_watcher()
4949 ceph_decode_64_safe(p, end, item->cookie, bad); in decode_watcher()
4953 ret = ceph_decode_entity_addr(p, end, &item->addr); in decode_watcher()
4961 ENTITY_NAME(item->name), item->cookie, in decode_watcher()
4962 ceph_pr_addr(&item->addr)); in decode_watcher()
4984 return -ENOMEM; in decode_watchers()
5014 return -ENOMEM; in ceph_osdc_list_watchers()
5016 ceph_oid_copy(&req->r_base_oid, oid); in ceph_osdc_list_watchers()
5017 ceph_oloc_copy(&req->r_base_oloc, oloc); in ceph_osdc_list_watchers()
5018 req->r_flags = CEPH_OSD_FLAG_READ; in ceph_osdc_list_watchers()
5039 void *const end = p + req->r_ops[0].outdata_len; in ceph_osdc_list_watchers()
5051 * Call all pending notify callbacks - for use after a watch is
5057 flush_workqueue(osdc->notify_wq); in ceph_osdc_flush_notifies()
5063 down_read(&osdc->lock); in ceph_osdc_maybe_request_map()
5065 up_read(&osdc->lock); in ceph_osdc_maybe_request_map()
5087 return -E2BIG; in ceph_osdc_call()
5091 return -ENOMEM; in ceph_osdc_call()
5093 ceph_oid_copy(&req->r_base_oid, oid); in ceph_osdc_call()
5094 ceph_oloc_copy(&req->r_base_oloc, oloc); in ceph_osdc_call()
5095 req->r_flags = flags; in ceph_osdc_call()
5115 ret = req->r_ops[0].rval; in ceph_osdc_call()
5117 *resp_len = req->r_ops[0].outdata_len; in ceph_osdc_call()
5133 down_write(&osdc->lock); in ceph_osdc_reopen_osds()
5134 for (n = rb_first(&osdc->osds); n; ) { in ceph_osdc_reopen_osds()
5141 up_write(&osdc->lock); in ceph_osdc_reopen_osds()
5152 osdc->client = client; in ceph_osdc_init()
5153 init_rwsem(&osdc->lock); in ceph_osdc_init()
5154 osdc->osds = RB_ROOT; in ceph_osdc_init()
5155 INIT_LIST_HEAD(&osdc->osd_lru); in ceph_osdc_init()
5156 spin_lock_init(&osdc->osd_lru_lock); in ceph_osdc_init()
5157 osd_init(&osdc->homeless_osd); in ceph_osdc_init()
5158 osdc->homeless_osd.o_osdc = osdc; in ceph_osdc_init()
5159 osdc->homeless_osd.o_osd = CEPH_HOMELESS_OSD; in ceph_osdc_init()
5160 osdc->last_linger_id = CEPH_LINGER_ID_START; in ceph_osdc_init()
5161 osdc->linger_requests = RB_ROOT; in ceph_osdc_init()
5162 osdc->map_checks = RB_ROOT; in ceph_osdc_init()
5163 osdc->linger_map_checks = RB_ROOT; in ceph_osdc_init()
5164 INIT_DELAYED_WORK(&osdc->timeout_work, handle_timeout); in ceph_osdc_init()
5165 INIT_DELAYED_WORK(&osdc->osds_timeout_work, handle_osds_timeout); in ceph_osdc_init()
5167 err = -ENOMEM; in ceph_osdc_init()
5168 osdc->osdmap = ceph_osdmap_alloc(); in ceph_osdc_init()
5169 if (!osdc->osdmap) in ceph_osdc_init()
5172 osdc->req_mempool = mempool_create_slab_pool(10, in ceph_osdc_init()
5174 if (!osdc->req_mempool) in ceph_osdc_init()
5177 err = ceph_msgpool_init(&osdc->msgpool_op, CEPH_MSG_OSD_OP, in ceph_osdc_init()
5181 err = ceph_msgpool_init(&osdc->msgpool_op_reply, CEPH_MSG_OSD_OPREPLY, in ceph_osdc_init()
5187 err = -ENOMEM; in ceph_osdc_init()
5188 osdc->notify_wq = create_singlethread_workqueue("ceph-watch-notify"); in ceph_osdc_init()
5189 if (!osdc->notify_wq) in ceph_osdc_init()
5192 osdc->completion_wq = create_singlethread_workqueue("ceph-completion"); in ceph_osdc_init()
5193 if (!osdc->completion_wq) in ceph_osdc_init()
5196 schedule_delayed_work(&osdc->timeout_work, in ceph_osdc_init()
5197 osdc->client->options->osd_keepalive_timeout); in ceph_osdc_init()
5198 schedule_delayed_work(&osdc->osds_timeout_work, in ceph_osdc_init()
5199 round_jiffies_relative(osdc->client->options->osd_idle_ttl)); in ceph_osdc_init()
5204 destroy_workqueue(osdc->notify_wq); in ceph_osdc_init()
5206 ceph_msgpool_destroy(&osdc->msgpool_op_reply); in ceph_osdc_init()
5208 ceph_msgpool_destroy(&osdc->msgpool_op); in ceph_osdc_init()
5210 mempool_destroy(osdc->req_mempool); in ceph_osdc_init()
5212 ceph_osdmap_destroy(osdc->osdmap); in ceph_osdc_init()
5219 destroy_workqueue(osdc->completion_wq); in ceph_osdc_stop()
5220 destroy_workqueue(osdc->notify_wq); in ceph_osdc_stop()
5221 cancel_delayed_work_sync(&osdc->timeout_work); in ceph_osdc_stop()
5222 cancel_delayed_work_sync(&osdc->osds_timeout_work); in ceph_osdc_stop()
5224 down_write(&osdc->lock); in ceph_osdc_stop()
5225 while (!RB_EMPTY_ROOT(&osdc->osds)) { in ceph_osdc_stop()
5226 struct ceph_osd *osd = rb_entry(rb_first(&osdc->osds), in ceph_osdc_stop()
5230 up_write(&osdc->lock); in ceph_osdc_stop()
5231 WARN_ON(refcount_read(&osdc->homeless_osd.o_ref) != 1); in ceph_osdc_stop()
5232 osd_cleanup(&osdc->homeless_osd); in ceph_osdc_stop()
5234 WARN_ON(!list_empty(&osdc->osd_lru)); in ceph_osdc_stop()
5235 WARN_ON(!RB_EMPTY_ROOT(&osdc->linger_requests)); in ceph_osdc_stop()
5236 WARN_ON(!RB_EMPTY_ROOT(&osdc->map_checks)); in ceph_osdc_stop()
5237 WARN_ON(!RB_EMPTY_ROOT(&osdc->linger_map_checks)); in ceph_osdc_stop()
5238 WARN_ON(atomic_read(&osdc->num_requests)); in ceph_osdc_stop()
5239 WARN_ON(atomic_read(&osdc->num_homeless)); in ceph_osdc_stop()
5241 ceph_osdmap_destroy(osdc->osdmap); in ceph_osdc_stop()
5242 mempool_destroy(osdc->req_mempool); in ceph_osdc_stop()
5243 ceph_msgpool_destroy(&osdc->msgpool_op); in ceph_osdc_stop()
5244 ceph_msgpool_destroy(&osdc->msgpool_op_reply); in ceph_osdc_stop()
5266 op->copy_from.snapid = src_snapid; in osd_req_op_copy_from_init()
5267 op->copy_from.src_version = src_version; in osd_req_op_copy_from_init()
5268 op->copy_from.flags = copy_from_flags; in osd_req_op_copy_from_init()
5269 op->copy_from.src_fadvise_flags = src_fadvise_flags; in osd_req_op_copy_from_init()
5273 ceph_encode_string(&p, end, src_oid->name, src_oid->name_len); in osd_req_op_copy_from_init()
5277 op->indata_len = PAGE_SIZE - (end - p); in osd_req_op_copy_from_init()
5279 ceph_osd_data_pages_init(&op->copy_from.osd_data, pages, in osd_req_op_copy_from_init()
5280 op->indata_len, 0, false, true); in osd_req_op_copy_from_init()
5300 return -ENOMEM; in ceph_osdc_copy_from()
5302 req->r_flags = CEPH_OSD_FLAG_WRITE; in ceph_osdc_copy_from()
5304 ceph_oloc_copy(&req->r_t.base_oloc, dst_oloc); in ceph_osdc_copy_from()
5305 ceph_oid_copy(&req->r_t.base_oid, dst_oid); in ceph_osdc_copy_from()
5336 return ceph_osd_request_cache ? 0 : -ENOMEM; in ceph_osdc_setup()
5351 struct ceph_osd *osd = con->private; in dispatch()
5352 struct ceph_osd_client *osdc = osd->o_osdc; in dispatch()
5353 int type = le16_to_cpu(msg->hdr.type); in dispatch()
5380 * message at the moment - for now, just skip the message.
5386 struct ceph_osd *osd = con->private; in get_reply()
5387 struct ceph_osd_client *osdc = osd->o_osdc; in get_reply()
5390 int front_len = le32_to_cpu(hdr->front_len); in get_reply()
5391 int data_len = le32_to_cpu(hdr->data_len); in get_reply()
5392 u64 tid = le64_to_cpu(hdr->tid); in get_reply()
5394 down_read(&osdc->lock); in get_reply()
5396 dout("%s osd%d unknown, skipping\n", __func__, osd->o_osd); in get_reply()
5400 WARN_ON(osd->o_osd != le64_to_cpu(hdr->src.num)); in get_reply()
5402 mutex_lock(&osd->lock); in get_reply()
5403 req = lookup_request(&osd->o_requests, tid); in get_reply()
5406 osd->o_osd, tid); in get_reply()
5411 ceph_msg_revoke_incoming(req->r_reply); in get_reply()
5413 if (front_len > req->r_reply->front_alloc_len) { in get_reply()
5415 __func__, osd->o_osd, req->r_tid, front_len, in get_reply()
5416 req->r_reply->front_alloc_len); in get_reply()
5421 ceph_msg_put(req->r_reply); in get_reply()
5422 req->r_reply = m; in get_reply()
5425 if (data_len > req->r_reply->data_length) { in get_reply()
5427 __func__, osd->o_osd, req->r_tid, data_len, in get_reply()
5428 req->r_reply->data_length); in get_reply()
5434 m = ceph_msg_get(req->r_reply); in get_reply()
5438 mutex_unlock(&osd->lock); in get_reply()
5440 up_read(&osdc->lock); in get_reply()
5447 int type = le16_to_cpu(hdr->type); in alloc_msg_with_page_vector()
5448 u32 front_len = le32_to_cpu(hdr->front_len); in alloc_msg_with_page_vector()
5449 u32 data_len = le32_to_cpu(hdr->data_len); in alloc_msg_with_page_vector()
5475 struct ceph_osd *osd = con->private; in alloc_msg()
5476 int type = le16_to_cpu(hdr->type); in alloc_msg()
5488 osd->o_osd, type); in alloc_msg()
5499 struct ceph_osd *osd = con->private; in get_osd_con()
5507 struct ceph_osd *osd = con->private; in put_osd_con()
5521 struct ceph_osd *o = con->private; in get_authorizer()
5522 struct ceph_osd_client *osdc = o->o_osdc; in get_authorizer()
5523 struct ceph_auth_client *ac = osdc->client->monc.auth; in get_authorizer()
5524 struct ceph_auth_handshake *auth = &o->o_auth; in get_authorizer()
5526 if (force_new && auth->authorizer) { in get_authorizer()
5527 ceph_auth_destroy_authorizer(auth->authorizer); in get_authorizer()
5528 auth->authorizer = NULL; in get_authorizer()
5530 if (!auth->authorizer) { in get_authorizer()
5541 *proto = ac->protocol; in get_authorizer()
5549 struct ceph_osd *o = con->private; in add_authorizer_challenge()
5550 struct ceph_osd_client *osdc = o->o_osdc; in add_authorizer_challenge()
5551 struct ceph_auth_client *ac = osdc->client->monc.auth; in add_authorizer_challenge()
5553 return ceph_auth_add_authorizer_challenge(ac, o->o_auth.authorizer, in add_authorizer_challenge()
5559 struct ceph_osd *o = con->private; in verify_authorizer_reply()
5560 struct ceph_osd_client *osdc = o->o_osdc; in verify_authorizer_reply()
5561 struct ceph_auth_client *ac = osdc->client->monc.auth; in verify_authorizer_reply()
5563 return ceph_auth_verify_authorizer_reply(ac, o->o_auth.authorizer); in verify_authorizer_reply()
5568 struct ceph_osd *o = con->private; in invalidate_authorizer()
5569 struct ceph_osd_client *osdc = o->o_osdc; in invalidate_authorizer()
5570 struct ceph_auth_client *ac = osdc->client->monc.auth; in invalidate_authorizer()
5573 return ceph_monc_validate_auth(&osdc->client->monc); in invalidate_authorizer()
5578 int type = le16_to_cpu(msg->hdr.type); in osd_reencode_message()
5586 struct ceph_osd *o = msg->con->private; in osd_sign_message()
5587 struct ceph_auth_handshake *auth = &o->o_auth; in osd_sign_message()
5594 struct ceph_osd *o = msg->con->private; in osd_check_message_signature()
5595 struct ceph_auth_handshake *auth = &o->o_auth; in osd_check_message_signature()