1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * Copyright (c) 2009, Microsoft Corporation.
4 *
5 * Authors:
6 * Haiyang Zhang <haiyangz@microsoft.com>
7 * Hank Janssen <hjanssen@microsoft.com>
8 */
9 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
10
11 #include <linux/kernel.h>
12 #include <linux/interrupt.h>
13 #include <linux/sched.h>
14 #include <linux/wait.h>
15 #include <linux/mm.h>
16 #include <linux/slab.h>
17 #include <linux/list.h>
18 #include <linux/module.h>
19 #include <linux/completion.h>
20 #include <linux/delay.h>
21 #include <linux/cpu.h>
22 #include <linux/hyperv.h>
23 #include <asm/mshyperv.h>
24
25 #include "hyperv_vmbus.h"
26
27 static void init_vp_index(struct vmbus_channel *channel);
28
29 const struct vmbus_device vmbus_devs[] = {
30 /* IDE */
31 { .dev_type = HV_IDE,
32 HV_IDE_GUID,
33 .perf_device = true,
34 .allowed_in_isolated = false,
35 },
36
37 /* SCSI */
38 { .dev_type = HV_SCSI,
39 HV_SCSI_GUID,
40 .perf_device = true,
41 .allowed_in_isolated = true,
42 },
43
44 /* Fibre Channel */
45 { .dev_type = HV_FC,
46 HV_SYNTHFC_GUID,
47 .perf_device = true,
48 .allowed_in_isolated = false,
49 },
50
51 /* Synthetic NIC */
52 { .dev_type = HV_NIC,
53 HV_NIC_GUID,
54 .perf_device = true,
55 .allowed_in_isolated = true,
56 },
57
58 /* Network Direct */
59 { .dev_type = HV_ND,
60 HV_ND_GUID,
61 .perf_device = true,
62 .allowed_in_isolated = false,
63 },
64
65 /* PCIE */
66 { .dev_type = HV_PCIE,
67 HV_PCIE_GUID,
68 .perf_device = false,
69 .allowed_in_isolated = false,
70 },
71
72 /* Synthetic Frame Buffer */
73 { .dev_type = HV_FB,
74 HV_SYNTHVID_GUID,
75 .perf_device = false,
76 .allowed_in_isolated = false,
77 },
78
79 /* Synthetic Keyboard */
80 { .dev_type = HV_KBD,
81 HV_KBD_GUID,
82 .perf_device = false,
83 .allowed_in_isolated = false,
84 },
85
86 /* Synthetic MOUSE */
87 { .dev_type = HV_MOUSE,
88 HV_MOUSE_GUID,
89 .perf_device = false,
90 .allowed_in_isolated = false,
91 },
92
93 /* KVP */
94 { .dev_type = HV_KVP,
95 HV_KVP_GUID,
96 .perf_device = false,
97 .allowed_in_isolated = false,
98 },
99
100 /* Time Synch */
101 { .dev_type = HV_TS,
102 HV_TS_GUID,
103 .perf_device = false,
104 .allowed_in_isolated = true,
105 },
106
107 /* Heartbeat */
108 { .dev_type = HV_HB,
109 HV_HEART_BEAT_GUID,
110 .perf_device = false,
111 .allowed_in_isolated = true,
112 },
113
114 /* Shutdown */
115 { .dev_type = HV_SHUTDOWN,
116 HV_SHUTDOWN_GUID,
117 .perf_device = false,
118 .allowed_in_isolated = true,
119 },
120
121 /* File copy */
122 { .dev_type = HV_FCOPY,
123 HV_FCOPY_GUID,
124 .perf_device = false,
125 .allowed_in_isolated = false,
126 },
127
128 /* Backup */
129 { .dev_type = HV_BACKUP,
130 HV_VSS_GUID,
131 .perf_device = false,
132 .allowed_in_isolated = false,
133 },
134
135 /* Dynamic Memory */
136 { .dev_type = HV_DM,
137 HV_DM_GUID,
138 .perf_device = false,
139 .allowed_in_isolated = false,
140 },
141
142 /* Unknown GUID */
143 { .dev_type = HV_UNKNOWN,
144 .perf_device = false,
145 .allowed_in_isolated = false,
146 },
147 };
148
149 static const struct {
150 guid_t guid;
151 } vmbus_unsupported_devs[] = {
152 { HV_AVMA1_GUID },
153 { HV_AVMA2_GUID },
154 { HV_RDV_GUID },
155 };
156
157 /*
158 * The rescinded channel may be blocked waiting for a response from the host;
159 * take care of that.
160 */
vmbus_rescind_cleanup(struct vmbus_channel * channel)161 static void vmbus_rescind_cleanup(struct vmbus_channel *channel)
162 {
163 struct vmbus_channel_msginfo *msginfo;
164 unsigned long flags;
165
166
167 spin_lock_irqsave(&vmbus_connection.channelmsg_lock, flags);
168 channel->rescind = true;
169 list_for_each_entry(msginfo, &vmbus_connection.chn_msg_list,
170 msglistentry) {
171
172 if (msginfo->waiting_channel == channel) {
173 complete(&msginfo->waitevent);
174 break;
175 }
176 }
177 spin_unlock_irqrestore(&vmbus_connection.channelmsg_lock, flags);
178 }
179
is_unsupported_vmbus_devs(const guid_t * guid)180 static bool is_unsupported_vmbus_devs(const guid_t *guid)
181 {
182 int i;
183
184 for (i = 0; i < ARRAY_SIZE(vmbus_unsupported_devs); i++)
185 if (guid_equal(guid, &vmbus_unsupported_devs[i].guid))
186 return true;
187 return false;
188 }
189
hv_get_dev_type(const struct vmbus_channel * channel)190 static u16 hv_get_dev_type(const struct vmbus_channel *channel)
191 {
192 const guid_t *guid = &channel->offermsg.offer.if_type;
193 u16 i;
194
195 if (is_hvsock_channel(channel) || is_unsupported_vmbus_devs(guid))
196 return HV_UNKNOWN;
197
198 for (i = HV_IDE; i < HV_UNKNOWN; i++) {
199 if (guid_equal(guid, &vmbus_devs[i].guid))
200 return i;
201 }
202 pr_info("Unknown GUID: %pUl\n", guid);
203 return i;
204 }
205
206 /**
207 * vmbus_prep_negotiate_resp() - Create default response for Negotiate message
208 * @icmsghdrp: Pointer to msg header structure
209 * @buf: Raw buffer channel data
210 * @buflen: Length of the raw buffer channel data.
211 * @fw_version: The framework versions we can support.
212 * @fw_vercnt: The size of @fw_version.
213 * @srv_version: The service versions we can support.
214 * @srv_vercnt: The size of @srv_version.
215 * @nego_fw_version: The selected framework version.
216 * @nego_srv_version: The selected service version.
217 *
218 * Note: Versions are given in decreasing order.
219 *
220 * Set up and fill in default negotiate response message.
221 * Mainly used by Hyper-V drivers.
222 */
vmbus_prep_negotiate_resp(struct icmsg_hdr * icmsghdrp,u8 * buf,u32 buflen,const int * fw_version,int fw_vercnt,const int * srv_version,int srv_vercnt,int * nego_fw_version,int * nego_srv_version)223 bool vmbus_prep_negotiate_resp(struct icmsg_hdr *icmsghdrp, u8 *buf,
224 u32 buflen, const int *fw_version, int fw_vercnt,
225 const int *srv_version, int srv_vercnt,
226 int *nego_fw_version, int *nego_srv_version)
227 {
228 int icframe_major, icframe_minor;
229 int icmsg_major, icmsg_minor;
230 int fw_major, fw_minor;
231 int srv_major, srv_minor;
232 int i, j;
233 bool found_match = false;
234 struct icmsg_negotiate *negop;
235
236 /* Check that there's enough space for icframe_vercnt, icmsg_vercnt */
237 if (buflen < ICMSG_HDR + offsetof(struct icmsg_negotiate, reserved)) {
238 pr_err_ratelimited("Invalid icmsg negotiate\n");
239 return false;
240 }
241
242 icmsghdrp->icmsgsize = 0x10;
243 negop = (struct icmsg_negotiate *)&buf[ICMSG_HDR];
244
245 icframe_major = negop->icframe_vercnt;
246 icframe_minor = 0;
247
248 icmsg_major = negop->icmsg_vercnt;
249 icmsg_minor = 0;
250
251 /* Validate negop packet */
252 if (icframe_major > IC_VERSION_NEGOTIATION_MAX_VER_COUNT ||
253 icmsg_major > IC_VERSION_NEGOTIATION_MAX_VER_COUNT ||
254 ICMSG_NEGOTIATE_PKT_SIZE(icframe_major, icmsg_major) > buflen) {
255 pr_err_ratelimited("Invalid icmsg negotiate - icframe_major: %u, icmsg_major: %u\n",
256 icframe_major, icmsg_major);
257 goto fw_error;
258 }
259
260 /*
261 * Select the framework version number we will
262 * support.
263 */
264
265 for (i = 0; i < fw_vercnt; i++) {
266 fw_major = (fw_version[i] >> 16);
267 fw_minor = (fw_version[i] & 0xFFFF);
268
269 for (j = 0; j < negop->icframe_vercnt; j++) {
270 if ((negop->icversion_data[j].major == fw_major) &&
271 (negop->icversion_data[j].minor == fw_minor)) {
272 icframe_major = negop->icversion_data[j].major;
273 icframe_minor = negop->icversion_data[j].minor;
274 found_match = true;
275 break;
276 }
277 }
278
279 if (found_match)
280 break;
281 }
282
283 if (!found_match)
284 goto fw_error;
285
286 found_match = false;
287
288 for (i = 0; i < srv_vercnt; i++) {
289 srv_major = (srv_version[i] >> 16);
290 srv_minor = (srv_version[i] & 0xFFFF);
291
292 for (j = negop->icframe_vercnt;
293 (j < negop->icframe_vercnt + negop->icmsg_vercnt);
294 j++) {
295
296 if ((negop->icversion_data[j].major == srv_major) &&
297 (negop->icversion_data[j].minor == srv_minor)) {
298
299 icmsg_major = negop->icversion_data[j].major;
300 icmsg_minor = negop->icversion_data[j].minor;
301 found_match = true;
302 break;
303 }
304 }
305
306 if (found_match)
307 break;
308 }
309
310 /*
311 * Respond with the framework and service
312 * version numbers we can support.
313 */
314
315 fw_error:
316 if (!found_match) {
317 negop->icframe_vercnt = 0;
318 negop->icmsg_vercnt = 0;
319 } else {
320 negop->icframe_vercnt = 1;
321 negop->icmsg_vercnt = 1;
322 }
323
324 if (nego_fw_version)
325 *nego_fw_version = (icframe_major << 16) | icframe_minor;
326
327 if (nego_srv_version)
328 *nego_srv_version = (icmsg_major << 16) | icmsg_minor;
329
330 negop->icversion_data[0].major = icframe_major;
331 negop->icversion_data[0].minor = icframe_minor;
332 negop->icversion_data[1].major = icmsg_major;
333 negop->icversion_data[1].minor = icmsg_minor;
334 return found_match;
335 }
336 EXPORT_SYMBOL_GPL(vmbus_prep_negotiate_resp);
337
338 /*
339 * alloc_channel - Allocate and initialize a vmbus channel object
340 */
alloc_channel(void)341 static struct vmbus_channel *alloc_channel(void)
342 {
343 struct vmbus_channel *channel;
344
345 channel = kzalloc(sizeof(*channel), GFP_ATOMIC);
346 if (!channel)
347 return NULL;
348
349 spin_lock_init(&channel->sched_lock);
350 init_completion(&channel->rescind_event);
351
352 INIT_LIST_HEAD(&channel->sc_list);
353
354 tasklet_init(&channel->callback_event,
355 vmbus_on_event, (unsigned long)channel);
356
357 hv_ringbuffer_pre_init(channel);
358
359 return channel;
360 }
361
362 /*
363 * free_channel - Release the resources used by the vmbus channel object
364 */
free_channel(struct vmbus_channel * channel)365 static void free_channel(struct vmbus_channel *channel)
366 {
367 tasklet_kill(&channel->callback_event);
368 vmbus_remove_channel_attr_group(channel);
369
370 kobject_put(&channel->kobj);
371 }
372
vmbus_channel_map_relid(struct vmbus_channel * channel)373 void vmbus_channel_map_relid(struct vmbus_channel *channel)
374 {
375 if (WARN_ON(channel->offermsg.child_relid >= MAX_CHANNEL_RELIDS))
376 return;
377 /*
378 * The mapping of the channel's relid is visible from the CPUs that
379 * execute vmbus_chan_sched() by the time that vmbus_chan_sched() will
380 * execute:
381 *
382 * (a) In the "normal (i.e., not resuming from hibernation)" path,
383 * the full barrier in virt_store_mb() guarantees that the store
384 * is propagated to all CPUs before the add_channel_work work
385 * is queued. In turn, add_channel_work is queued before the
386 * channel's ring buffer is allocated/initialized and the
387 * OPENCHANNEL message for the channel is sent in vmbus_open().
388 * Hyper-V won't start sending the interrupts for the channel
389 * before the OPENCHANNEL message is acked. The memory barrier
390 * in vmbus_chan_sched() -> sync_test_and_clear_bit() ensures
391 * that vmbus_chan_sched() must find the channel's relid in
392 * recv_int_page before retrieving the channel pointer from the
393 * array of channels.
394 *
395 * (b) In the "resuming from hibernation" path, the virt_store_mb()
396 * guarantees that the store is propagated to all CPUs before
397 * the VMBus connection is marked as ready for the resume event
398 * (cf. check_ready_for_resume_event()). The interrupt handler
399 * of the VMBus driver and vmbus_chan_sched() can not run before
400 * vmbus_bus_resume() has completed execution (cf. resume_noirq).
401 */
402 virt_store_mb(
403 vmbus_connection.channels[channel->offermsg.child_relid],
404 channel);
405 }
406
vmbus_channel_unmap_relid(struct vmbus_channel * channel)407 void vmbus_channel_unmap_relid(struct vmbus_channel *channel)
408 {
409 if (WARN_ON(channel->offermsg.child_relid >= MAX_CHANNEL_RELIDS))
410 return;
411 WRITE_ONCE(
412 vmbus_connection.channels[channel->offermsg.child_relid],
413 NULL);
414 }
415
vmbus_release_relid(u32 relid)416 static void vmbus_release_relid(u32 relid)
417 {
418 struct vmbus_channel_relid_released msg;
419 int ret;
420
421 memset(&msg, 0, sizeof(struct vmbus_channel_relid_released));
422 msg.child_relid = relid;
423 msg.header.msgtype = CHANNELMSG_RELID_RELEASED;
424 ret = vmbus_post_msg(&msg, sizeof(struct vmbus_channel_relid_released),
425 true);
426
427 trace_vmbus_release_relid(&msg, ret);
428 }
429
hv_process_channel_removal(struct vmbus_channel * channel)430 void hv_process_channel_removal(struct vmbus_channel *channel)
431 {
432 lockdep_assert_held(&vmbus_connection.channel_mutex);
433 BUG_ON(!channel->rescind);
434
435 /*
436 * hv_process_channel_removal() could find INVALID_RELID only for
437 * hv_sock channels. See the inline comments in vmbus_onoffer().
438 */
439 WARN_ON(channel->offermsg.child_relid == INVALID_RELID &&
440 !is_hvsock_channel(channel));
441
442 /*
443 * Upon suspend, an in-use hv_sock channel is removed from the array of
444 * channels and the relid is invalidated. After hibernation, when the
445 * user-space appplication destroys the channel, it's unnecessary and
446 * unsafe to remove the channel from the array of channels. See also
447 * the inline comments before the call of vmbus_release_relid() below.
448 */
449 if (channel->offermsg.child_relid != INVALID_RELID)
450 vmbus_channel_unmap_relid(channel);
451
452 if (channel->primary_channel == NULL)
453 list_del(&channel->listentry);
454 else
455 list_del(&channel->sc_list);
456
457 /*
458 * If this is a "perf" channel, updates the hv_numa_map[] masks so that
459 * init_vp_index() can (re-)use the CPU.
460 */
461 if (hv_is_perf_channel(channel))
462 hv_clear_alloced_cpu(channel->target_cpu);
463
464 /*
465 * Upon suspend, an in-use hv_sock channel is marked as "rescinded" and
466 * the relid is invalidated; after hibernation, when the user-space app
467 * destroys the channel, the relid is INVALID_RELID, and in this case
468 * it's unnecessary and unsafe to release the old relid, since the same
469 * relid can refer to a completely different channel now.
470 */
471 if (channel->offermsg.child_relid != INVALID_RELID)
472 vmbus_release_relid(channel->offermsg.child_relid);
473
474 free_channel(channel);
475 }
476
vmbus_free_channels(void)477 void vmbus_free_channels(void)
478 {
479 struct vmbus_channel *channel, *tmp;
480
481 list_for_each_entry_safe(channel, tmp, &vmbus_connection.chn_list,
482 listentry) {
483 /* hv_process_channel_removal() needs this */
484 channel->rescind = true;
485
486 vmbus_device_unregister(channel->device_obj);
487 }
488 }
489
490 /* Note: the function can run concurrently for primary/sub channels. */
vmbus_add_channel_work(struct work_struct * work)491 static void vmbus_add_channel_work(struct work_struct *work)
492 {
493 struct vmbus_channel *newchannel =
494 container_of(work, struct vmbus_channel, add_channel_work);
495 struct vmbus_channel *primary_channel = newchannel->primary_channel;
496 int ret;
497
498 /*
499 * This state is used to indicate a successful open
500 * so that when we do close the channel normally, we
501 * can cleanup properly.
502 */
503 newchannel->state = CHANNEL_OPEN_STATE;
504
505 if (primary_channel != NULL) {
506 /* newchannel is a sub-channel. */
507 struct hv_device *dev = primary_channel->device_obj;
508
509 if (vmbus_add_channel_kobj(dev, newchannel))
510 goto err_deq_chan;
511
512 if (primary_channel->sc_creation_callback != NULL)
513 primary_channel->sc_creation_callback(newchannel);
514
515 newchannel->probe_done = true;
516 return;
517 }
518
519 /*
520 * Start the process of binding the primary channel to the driver
521 */
522 newchannel->device_obj = vmbus_device_create(
523 &newchannel->offermsg.offer.if_type,
524 &newchannel->offermsg.offer.if_instance,
525 newchannel);
526 if (!newchannel->device_obj)
527 goto err_deq_chan;
528
529 newchannel->device_obj->device_id = newchannel->device_id;
530 /*
531 * Add the new device to the bus. This will kick off device-driver
532 * binding which eventually invokes the device driver's AddDevice()
533 * method.
534 *
535 * If vmbus_device_register() fails, the 'device_obj' is freed in
536 * vmbus_device_release() as called by device_unregister() in the
537 * error path of vmbus_device_register(). In the outside error
538 * path, there's no need to free it.
539 */
540 ret = vmbus_device_register(newchannel->device_obj);
541
542 if (ret != 0) {
543 pr_err("unable to add child device object (relid %d)\n",
544 newchannel->offermsg.child_relid);
545 goto err_deq_chan;
546 }
547
548 newchannel->probe_done = true;
549 return;
550
551 err_deq_chan:
552 mutex_lock(&vmbus_connection.channel_mutex);
553
554 /*
555 * We need to set the flag, otherwise
556 * vmbus_onoffer_rescind() can be blocked.
557 */
558 newchannel->probe_done = true;
559
560 if (primary_channel == NULL)
561 list_del(&newchannel->listentry);
562 else
563 list_del(&newchannel->sc_list);
564
565 /* vmbus_process_offer() has mapped the channel. */
566 vmbus_channel_unmap_relid(newchannel);
567
568 mutex_unlock(&vmbus_connection.channel_mutex);
569
570 vmbus_release_relid(newchannel->offermsg.child_relid);
571
572 free_channel(newchannel);
573 }
574
575 /*
576 * vmbus_process_offer - Process the offer by creating a channel/device
577 * associated with this offer
578 */
vmbus_process_offer(struct vmbus_channel * newchannel)579 static void vmbus_process_offer(struct vmbus_channel *newchannel)
580 {
581 struct vmbus_channel *channel;
582 struct workqueue_struct *wq;
583 bool fnew = true;
584
585 /*
586 * Synchronize vmbus_process_offer() and CPU hotplugging:
587 *
588 * CPU1 CPU2
589 *
590 * [vmbus_process_offer()] [Hot removal of the CPU]
591 *
592 * CPU_READ_LOCK CPUS_WRITE_LOCK
593 * LOAD cpu_online_mask SEARCH chn_list
594 * STORE target_cpu LOAD target_cpu
595 * INSERT chn_list STORE cpu_online_mask
596 * CPUS_READ_UNLOCK CPUS_WRITE_UNLOCK
597 *
598 * Forbids: CPU1's LOAD from *not* seing CPU2's STORE &&
599 * CPU2's SEARCH from *not* seeing CPU1's INSERT
600 *
601 * Forbids: CPU2's SEARCH from seeing CPU1's INSERT &&
602 * CPU2's LOAD from *not* seing CPU1's STORE
603 */
604 cpus_read_lock();
605
606 /*
607 * Serializes the modifications of the chn_list list as well as
608 * the accesses to next_numa_node_id in init_vp_index().
609 */
610 mutex_lock(&vmbus_connection.channel_mutex);
611
612 list_for_each_entry(channel, &vmbus_connection.chn_list, listentry) {
613 if (guid_equal(&channel->offermsg.offer.if_type,
614 &newchannel->offermsg.offer.if_type) &&
615 guid_equal(&channel->offermsg.offer.if_instance,
616 &newchannel->offermsg.offer.if_instance)) {
617 fnew = false;
618 newchannel->primary_channel = channel;
619 break;
620 }
621 }
622
623 init_vp_index(newchannel);
624
625 /* Remember the channels that should be cleaned up upon suspend. */
626 if (is_hvsock_channel(newchannel) || is_sub_channel(newchannel))
627 atomic_inc(&vmbus_connection.nr_chan_close_on_suspend);
628
629 /*
630 * Now that we have acquired the channel_mutex,
631 * we can release the potentially racing rescind thread.
632 */
633 atomic_dec(&vmbus_connection.offer_in_progress);
634
635 if (fnew) {
636 list_add_tail(&newchannel->listentry,
637 &vmbus_connection.chn_list);
638 } else {
639 /*
640 * Check to see if this is a valid sub-channel.
641 */
642 if (newchannel->offermsg.offer.sub_channel_index == 0) {
643 mutex_unlock(&vmbus_connection.channel_mutex);
644 cpus_read_unlock();
645 /*
646 * Don't call free_channel(), because newchannel->kobj
647 * is not initialized yet.
648 */
649 kfree(newchannel);
650 WARN_ON_ONCE(1);
651 return;
652 }
653 /*
654 * Process the sub-channel.
655 */
656 list_add_tail(&newchannel->sc_list, &channel->sc_list);
657 }
658
659 vmbus_channel_map_relid(newchannel);
660
661 mutex_unlock(&vmbus_connection.channel_mutex);
662 cpus_read_unlock();
663
664 /*
665 * vmbus_process_offer() mustn't call channel->sc_creation_callback()
666 * directly for sub-channels, because sc_creation_callback() ->
667 * vmbus_open() may never get the host's response to the
668 * OPEN_CHANNEL message (the host may rescind a channel at any time,
669 * e.g. in the case of hot removing a NIC), and vmbus_onoffer_rescind()
670 * may not wake up the vmbus_open() as it's blocked due to a non-zero
671 * vmbus_connection.offer_in_progress, and finally we have a deadlock.
672 *
673 * The above is also true for primary channels, if the related device
674 * drivers use sync probing mode by default.
675 *
676 * And, usually the handling of primary channels and sub-channels can
677 * depend on each other, so we should offload them to different
678 * workqueues to avoid possible deadlock, e.g. in sync-probing mode,
679 * NIC1's netvsc_subchan_work() can race with NIC2's netvsc_probe() ->
680 * rtnl_lock(), and causes deadlock: the former gets the rtnl_lock
681 * and waits for all the sub-channels to appear, but the latter
682 * can't get the rtnl_lock and this blocks the handling of
683 * sub-channels.
684 */
685 INIT_WORK(&newchannel->add_channel_work, vmbus_add_channel_work);
686 wq = fnew ? vmbus_connection.handle_primary_chan_wq :
687 vmbus_connection.handle_sub_chan_wq;
688 queue_work(wq, &newchannel->add_channel_work);
689 }
690
691 /*
692 * Check if CPUs used by other channels of the same device.
693 * It should only be called by init_vp_index().
694 */
hv_cpuself_used(u32 cpu,struct vmbus_channel * chn)695 static bool hv_cpuself_used(u32 cpu, struct vmbus_channel *chn)
696 {
697 struct vmbus_channel *primary = chn->primary_channel;
698 struct vmbus_channel *sc;
699
700 lockdep_assert_held(&vmbus_connection.channel_mutex);
701
702 if (!primary)
703 return false;
704
705 if (primary->target_cpu == cpu)
706 return true;
707
708 list_for_each_entry(sc, &primary->sc_list, sc_list)
709 if (sc != chn && sc->target_cpu == cpu)
710 return true;
711
712 return false;
713 }
714
715 /*
716 * We use this state to statically distribute the channel interrupt load.
717 */
718 static int next_numa_node_id;
719
720 /*
721 * Starting with Win8, we can statically distribute the incoming
722 * channel interrupt load by binding a channel to VCPU.
723 *
724 * For pre-win8 hosts or non-performance critical channels we assign the
725 * VMBUS_CONNECT_CPU.
726 *
727 * Starting with win8, performance critical channels will be distributed
728 * evenly among all the available NUMA nodes. Once the node is assigned,
729 * we will assign the CPU based on a simple round robin scheme.
730 */
init_vp_index(struct vmbus_channel * channel)731 static void init_vp_index(struct vmbus_channel *channel)
732 {
733 bool perf_chn = hv_is_perf_channel(channel);
734 u32 i, ncpu = num_online_cpus();
735 cpumask_var_t available_mask;
736 struct cpumask *alloced_mask;
737 u32 target_cpu;
738 int numa_node;
739
740 if ((vmbus_proto_version == VERSION_WS2008) ||
741 (vmbus_proto_version == VERSION_WIN7) || (!perf_chn) ||
742 !alloc_cpumask_var(&available_mask, GFP_KERNEL)) {
743 /*
744 * Prior to win8, all channel interrupts are
745 * delivered on VMBUS_CONNECT_CPU.
746 * Also if the channel is not a performance critical
747 * channel, bind it to VMBUS_CONNECT_CPU.
748 * In case alloc_cpumask_var() fails, bind it to
749 * VMBUS_CONNECT_CPU.
750 */
751 channel->target_cpu = VMBUS_CONNECT_CPU;
752 if (perf_chn)
753 hv_set_alloced_cpu(VMBUS_CONNECT_CPU);
754 return;
755 }
756
757 for (i = 1; i <= ncpu + 1; i++) {
758 while (true) {
759 numa_node = next_numa_node_id++;
760 if (numa_node == nr_node_ids) {
761 next_numa_node_id = 0;
762 continue;
763 }
764 if (cpumask_empty(cpumask_of_node(numa_node)))
765 continue;
766 break;
767 }
768 alloced_mask = &hv_context.hv_numa_map[numa_node];
769
770 if (cpumask_weight(alloced_mask) ==
771 cpumask_weight(cpumask_of_node(numa_node))) {
772 /*
773 * We have cycled through all the CPUs in the node;
774 * reset the alloced map.
775 */
776 cpumask_clear(alloced_mask);
777 }
778
779 cpumask_xor(available_mask, alloced_mask,
780 cpumask_of_node(numa_node));
781
782 target_cpu = cpumask_first(available_mask);
783 cpumask_set_cpu(target_cpu, alloced_mask);
784
785 if (channel->offermsg.offer.sub_channel_index >= ncpu ||
786 i > ncpu || !hv_cpuself_used(target_cpu, channel))
787 break;
788 }
789
790 channel->target_cpu = target_cpu;
791
792 free_cpumask_var(available_mask);
793 }
794
795 #define UNLOAD_DELAY_UNIT_MS 10 /* 10 milliseconds */
796 #define UNLOAD_WAIT_MS (100*1000) /* 100 seconds */
797 #define UNLOAD_WAIT_LOOPS (UNLOAD_WAIT_MS/UNLOAD_DELAY_UNIT_MS)
798 #define UNLOAD_MSG_MS (5*1000) /* Every 5 seconds */
799 #define UNLOAD_MSG_LOOPS (UNLOAD_MSG_MS/UNLOAD_DELAY_UNIT_MS)
800
vmbus_wait_for_unload(void)801 static void vmbus_wait_for_unload(void)
802 {
803 int cpu;
804 void *page_addr;
805 struct hv_message *msg;
806 struct vmbus_channel_message_header *hdr;
807 u32 message_type, i;
808
809 /*
810 * CHANNELMSG_UNLOAD_RESPONSE is always delivered to the CPU which was
811 * used for initial contact or to CPU0 depending on host version. When
812 * we're crashing on a different CPU let's hope that IRQ handler on
813 * the cpu which receives CHANNELMSG_UNLOAD_RESPONSE is still
814 * functional and vmbus_unload_response() will complete
815 * vmbus_connection.unload_event. If not, the last thing we can do is
816 * read message pages for all CPUs directly.
817 *
818 * Wait up to 100 seconds since an Azure host must writeback any dirty
819 * data in its disk cache before the VMbus UNLOAD request will
820 * complete. This flushing has been empirically observed to take up
821 * to 50 seconds in cases with a lot of dirty data, so allow additional
822 * leeway and for inaccuracies in mdelay(). But eventually time out so
823 * that the panic path can't get hung forever in case the response
824 * message isn't seen.
825 */
826 for (i = 1; i <= UNLOAD_WAIT_LOOPS; i++) {
827 if (completion_done(&vmbus_connection.unload_event))
828 goto completed;
829
830 for_each_present_cpu(cpu) {
831 struct hv_per_cpu_context *hv_cpu
832 = per_cpu_ptr(hv_context.cpu_context, cpu);
833
834 /*
835 * In a CoCo VM the synic_message_page is not allocated
836 * in hv_synic_alloc(). Instead it is set/cleared in
837 * hv_synic_enable_regs() and hv_synic_disable_regs()
838 * such that it is set only when the CPU is online. If
839 * not all present CPUs are online, the message page
840 * might be NULL, so skip such CPUs.
841 */
842 page_addr = hv_cpu->synic_message_page;
843 if (!page_addr)
844 continue;
845
846 msg = (struct hv_message *)page_addr
847 + VMBUS_MESSAGE_SINT;
848
849 message_type = READ_ONCE(msg->header.message_type);
850 if (message_type == HVMSG_NONE)
851 continue;
852
853 hdr = (struct vmbus_channel_message_header *)
854 msg->u.payload;
855
856 if (hdr->msgtype == CHANNELMSG_UNLOAD_RESPONSE)
857 complete(&vmbus_connection.unload_event);
858
859 vmbus_signal_eom(msg, message_type);
860 }
861
862 /*
863 * Give a notice periodically so someone watching the
864 * serial output won't think it is completely hung.
865 */
866 if (!(i % UNLOAD_MSG_LOOPS))
867 pr_notice("Waiting for VMBus UNLOAD to complete\n");
868
869 mdelay(UNLOAD_DELAY_UNIT_MS);
870 }
871 pr_err("Continuing even though VMBus UNLOAD did not complete\n");
872
873 completed:
874 /*
875 * We're crashing and already got the UNLOAD_RESPONSE, cleanup all
876 * maybe-pending messages on all CPUs to be able to receive new
877 * messages after we reconnect.
878 */
879 for_each_present_cpu(cpu) {
880 struct hv_per_cpu_context *hv_cpu
881 = per_cpu_ptr(hv_context.cpu_context, cpu);
882
883 page_addr = hv_cpu->synic_message_page;
884 if (!page_addr)
885 continue;
886
887 msg = (struct hv_message *)page_addr + VMBUS_MESSAGE_SINT;
888 msg->header.message_type = HVMSG_NONE;
889 }
890 }
891
892 /*
893 * vmbus_unload_response - Handler for the unload response.
894 */
vmbus_unload_response(struct vmbus_channel_message_header * hdr)895 static void vmbus_unload_response(struct vmbus_channel_message_header *hdr)
896 {
897 /*
898 * This is a global event; just wakeup the waiting thread.
899 * Once we successfully unload, we can cleanup the monitor state.
900 *
901 * NB. A malicious or compromised Hyper-V could send a spurious
902 * message of type CHANNELMSG_UNLOAD_RESPONSE, and trigger a call
903 * of the complete() below. Make sure that unload_event has been
904 * initialized by the time this complete() is executed.
905 */
906 complete(&vmbus_connection.unload_event);
907 }
908
vmbus_initiate_unload(bool crash)909 void vmbus_initiate_unload(bool crash)
910 {
911 struct vmbus_channel_message_header hdr;
912
913 if (xchg(&vmbus_connection.conn_state, DISCONNECTED) == DISCONNECTED)
914 return;
915
916 /* Pre-Win2012R2 hosts don't support reconnect */
917 if (vmbus_proto_version < VERSION_WIN8_1)
918 return;
919
920 reinit_completion(&vmbus_connection.unload_event);
921 memset(&hdr, 0, sizeof(struct vmbus_channel_message_header));
922 hdr.msgtype = CHANNELMSG_UNLOAD;
923 vmbus_post_msg(&hdr, sizeof(struct vmbus_channel_message_header),
924 !crash);
925
926 /*
927 * vmbus_initiate_unload() is also called on crash and the crash can be
928 * happening in an interrupt context, where scheduling is impossible.
929 */
930 if (!crash)
931 wait_for_completion(&vmbus_connection.unload_event);
932 else
933 vmbus_wait_for_unload();
934 }
935
check_ready_for_resume_event(void)936 static void check_ready_for_resume_event(void)
937 {
938 /*
939 * If all the old primary channels have been fixed up, then it's safe
940 * to resume.
941 */
942 if (atomic_dec_and_test(&vmbus_connection.nr_chan_fixup_on_resume))
943 complete(&vmbus_connection.ready_for_resume_event);
944 }
945
vmbus_setup_channel_state(struct vmbus_channel * channel,struct vmbus_channel_offer_channel * offer)946 static void vmbus_setup_channel_state(struct vmbus_channel *channel,
947 struct vmbus_channel_offer_channel *offer)
948 {
949 /*
950 * Setup state for signalling the host.
951 */
952 channel->sig_event = VMBUS_EVENT_CONNECTION_ID;
953
954 if (vmbus_proto_version != VERSION_WS2008) {
955 channel->is_dedicated_interrupt =
956 (offer->is_dedicated_interrupt != 0);
957 channel->sig_event = offer->connection_id;
958 }
959
960 memcpy(&channel->offermsg, offer,
961 sizeof(struct vmbus_channel_offer_channel));
962 channel->monitor_grp = (u8)offer->monitorid / 32;
963 channel->monitor_bit = (u8)offer->monitorid % 32;
964 channel->device_id = hv_get_dev_type(channel);
965 }
966
967 /*
968 * find_primary_channel_by_offer - Get the channel object given the new offer.
969 * This is only used in the resume path of hibernation.
970 */
971 static struct vmbus_channel *
find_primary_channel_by_offer(const struct vmbus_channel_offer_channel * offer)972 find_primary_channel_by_offer(const struct vmbus_channel_offer_channel *offer)
973 {
974 struct vmbus_channel *channel = NULL, *iter;
975 const guid_t *inst1, *inst2;
976
977 /* Ignore sub-channel offers. */
978 if (offer->offer.sub_channel_index != 0)
979 return NULL;
980
981 mutex_lock(&vmbus_connection.channel_mutex);
982
983 list_for_each_entry(iter, &vmbus_connection.chn_list, listentry) {
984 inst1 = &iter->offermsg.offer.if_instance;
985 inst2 = &offer->offer.if_instance;
986
987 if (guid_equal(inst1, inst2)) {
988 channel = iter;
989 break;
990 }
991 }
992
993 mutex_unlock(&vmbus_connection.channel_mutex);
994
995 return channel;
996 }
997
vmbus_is_valid_device(const guid_t * guid)998 static bool vmbus_is_valid_device(const guid_t *guid)
999 {
1000 u16 i;
1001
1002 if (!hv_is_isolation_supported())
1003 return true;
1004
1005 for (i = 0; i < ARRAY_SIZE(vmbus_devs); i++) {
1006 if (guid_equal(guid, &vmbus_devs[i].guid))
1007 return vmbus_devs[i].allowed_in_isolated;
1008 }
1009 return false;
1010 }
1011
1012 /*
1013 * vmbus_onoffer - Handler for channel offers from vmbus in parent partition.
1014 *
1015 */
vmbus_onoffer(struct vmbus_channel_message_header * hdr)1016 static void vmbus_onoffer(struct vmbus_channel_message_header *hdr)
1017 {
1018 struct vmbus_channel_offer_channel *offer;
1019 struct vmbus_channel *oldchannel, *newchannel;
1020 size_t offer_sz;
1021
1022 offer = (struct vmbus_channel_offer_channel *)hdr;
1023
1024 trace_vmbus_onoffer(offer);
1025
1026 if (!vmbus_is_valid_device(&offer->offer.if_type)) {
1027 pr_err_ratelimited("Invalid offer %d from the host supporting isolation\n",
1028 offer->child_relid);
1029 atomic_dec(&vmbus_connection.offer_in_progress);
1030 return;
1031 }
1032
1033 oldchannel = find_primary_channel_by_offer(offer);
1034
1035 if (oldchannel != NULL) {
1036 /*
1037 * We're resuming from hibernation: all the sub-channel and
1038 * hv_sock channels we had before the hibernation should have
1039 * been cleaned up, and now we must be seeing a re-offered
1040 * primary channel that we had before the hibernation.
1041 */
1042
1043 /*
1044 * { Initially: channel relid = INVALID_RELID,
1045 * channels[valid_relid] = NULL }
1046 *
1047 * CPU1 CPU2
1048 *
1049 * [vmbus_onoffer()] [vmbus_device_release()]
1050 *
1051 * LOCK channel_mutex LOCK channel_mutex
1052 * STORE channel relid = valid_relid LOAD r1 = channel relid
1053 * MAP_RELID channel if (r1 != INVALID_RELID)
1054 * UNLOCK channel_mutex UNMAP_RELID channel
1055 * UNLOCK channel_mutex
1056 *
1057 * Forbids: r1 == valid_relid &&
1058 * channels[valid_relid] == channel
1059 *
1060 * Note. r1 can be INVALID_RELID only for an hv_sock channel.
1061 * None of the hv_sock channels which were present before the
1062 * suspend are re-offered upon the resume. See the WARN_ON()
1063 * in hv_process_channel_removal().
1064 */
1065 mutex_lock(&vmbus_connection.channel_mutex);
1066
1067 atomic_dec(&vmbus_connection.offer_in_progress);
1068
1069 WARN_ON(oldchannel->offermsg.child_relid != INVALID_RELID);
1070 /* Fix up the relid. */
1071 oldchannel->offermsg.child_relid = offer->child_relid;
1072
1073 offer_sz = sizeof(*offer);
1074 if (memcmp(offer, &oldchannel->offermsg, offer_sz) != 0) {
1075 /*
1076 * This is not an error, since the host can also change
1077 * the other field(s) of the offer, e.g. on WS RS5
1078 * (Build 17763), the offer->connection_id of the
1079 * Mellanox VF vmbus device can change when the host
1080 * reoffers the device upon resume.
1081 */
1082 pr_debug("vmbus offer changed: relid=%d\n",
1083 offer->child_relid);
1084
1085 print_hex_dump_debug("Old vmbus offer: ",
1086 DUMP_PREFIX_OFFSET, 16, 4,
1087 &oldchannel->offermsg, offer_sz,
1088 false);
1089 print_hex_dump_debug("New vmbus offer: ",
1090 DUMP_PREFIX_OFFSET, 16, 4,
1091 offer, offer_sz, false);
1092
1093 /* Fix up the old channel. */
1094 vmbus_setup_channel_state(oldchannel, offer);
1095 }
1096
1097 /* Add the channel back to the array of channels. */
1098 vmbus_channel_map_relid(oldchannel);
1099 check_ready_for_resume_event();
1100
1101 mutex_unlock(&vmbus_connection.channel_mutex);
1102 return;
1103 }
1104
1105 /* Allocate the channel object and save this offer. */
1106 newchannel = alloc_channel();
1107 if (!newchannel) {
1108 vmbus_release_relid(offer->child_relid);
1109 atomic_dec(&vmbus_connection.offer_in_progress);
1110 pr_err("Unable to allocate channel object\n");
1111 return;
1112 }
1113
1114 vmbus_setup_channel_state(newchannel, offer);
1115
1116 vmbus_process_offer(newchannel);
1117 }
1118
check_ready_for_suspend_event(void)1119 static void check_ready_for_suspend_event(void)
1120 {
1121 /*
1122 * If all the sub-channels or hv_sock channels have been cleaned up,
1123 * then it's safe to suspend.
1124 */
1125 if (atomic_dec_and_test(&vmbus_connection.nr_chan_close_on_suspend))
1126 complete(&vmbus_connection.ready_for_suspend_event);
1127 }
1128
1129 /*
1130 * vmbus_onoffer_rescind - Rescind offer handler.
1131 *
1132 * We queue a work item to process this offer synchronously
1133 */
vmbus_onoffer_rescind(struct vmbus_channel_message_header * hdr)1134 static void vmbus_onoffer_rescind(struct vmbus_channel_message_header *hdr)
1135 {
1136 struct vmbus_channel_rescind_offer *rescind;
1137 struct vmbus_channel *channel;
1138 struct device *dev;
1139 bool clean_up_chan_for_suspend;
1140
1141 rescind = (struct vmbus_channel_rescind_offer *)hdr;
1142
1143 trace_vmbus_onoffer_rescind(rescind);
1144
1145 /*
1146 * The offer msg and the corresponding rescind msg
1147 * from the host are guranteed to be ordered -
1148 * offer comes in first and then the rescind.
1149 * Since we process these events in work elements,
1150 * and with preemption, we may end up processing
1151 * the events out of order. We rely on the synchronization
1152 * provided by offer_in_progress and by channel_mutex for
1153 * ordering these events:
1154 *
1155 * { Initially: offer_in_progress = 1 }
1156 *
1157 * CPU1 CPU2
1158 *
1159 * [vmbus_onoffer()] [vmbus_onoffer_rescind()]
1160 *
1161 * LOCK channel_mutex WAIT_ON offer_in_progress == 0
1162 * DECREMENT offer_in_progress LOCK channel_mutex
1163 * STORE channels[] LOAD channels[]
1164 * UNLOCK channel_mutex UNLOCK channel_mutex
1165 *
1166 * Forbids: CPU2's LOAD from *not* seeing CPU1's STORE
1167 */
1168
1169 while (atomic_read(&vmbus_connection.offer_in_progress) != 0) {
1170 /*
1171 * We wait here until any channel offer is currently
1172 * being processed.
1173 */
1174 msleep(1);
1175 }
1176
1177 mutex_lock(&vmbus_connection.channel_mutex);
1178 channel = relid2channel(rescind->child_relid);
1179 if (channel != NULL) {
1180 /*
1181 * Guarantee that no other instance of vmbus_onoffer_rescind()
1182 * has got a reference to the channel object. Synchronize on
1183 * &vmbus_connection.channel_mutex.
1184 */
1185 if (channel->rescind_ref) {
1186 mutex_unlock(&vmbus_connection.channel_mutex);
1187 return;
1188 }
1189 channel->rescind_ref = true;
1190 }
1191 mutex_unlock(&vmbus_connection.channel_mutex);
1192
1193 if (channel == NULL) {
1194 /*
1195 * We failed in processing the offer message;
1196 * we would have cleaned up the relid in that
1197 * failure path.
1198 */
1199 return;
1200 }
1201
1202 clean_up_chan_for_suspend = is_hvsock_channel(channel) ||
1203 is_sub_channel(channel);
1204 /*
1205 * Before setting channel->rescind in vmbus_rescind_cleanup(), we
1206 * should make sure the channel callback is not running any more.
1207 */
1208 vmbus_reset_channel_cb(channel);
1209
1210 /*
1211 * Now wait for offer handling to complete.
1212 */
1213 vmbus_rescind_cleanup(channel);
1214 while (READ_ONCE(channel->probe_done) == false) {
1215 /*
1216 * We wait here until any channel offer is currently
1217 * being processed.
1218 */
1219 msleep(1);
1220 }
1221
1222 /*
1223 * At this point, the rescind handling can proceed safely.
1224 */
1225
1226 if (channel->device_obj) {
1227 if (channel->chn_rescind_callback) {
1228 channel->chn_rescind_callback(channel);
1229
1230 if (clean_up_chan_for_suspend)
1231 check_ready_for_suspend_event();
1232
1233 return;
1234 }
1235 /*
1236 * We will have to unregister this device from the
1237 * driver core.
1238 */
1239 dev = get_device(&channel->device_obj->device);
1240 if (dev) {
1241 vmbus_device_unregister(channel->device_obj);
1242 put_device(dev);
1243 }
1244 } else if (channel->primary_channel != NULL) {
1245 /*
1246 * Sub-channel is being rescinded. Following is the channel
1247 * close sequence when initiated from the driveri (refer to
1248 * vmbus_close() for details):
1249 * 1. Close all sub-channels first
1250 * 2. Then close the primary channel.
1251 */
1252 mutex_lock(&vmbus_connection.channel_mutex);
1253 if (channel->state == CHANNEL_OPEN_STATE) {
1254 /*
1255 * The channel is currently not open;
1256 * it is safe for us to cleanup the channel.
1257 */
1258 hv_process_channel_removal(channel);
1259 } else {
1260 complete(&channel->rescind_event);
1261 }
1262 mutex_unlock(&vmbus_connection.channel_mutex);
1263 }
1264
1265 /* The "channel" may have been freed. Do not access it any longer. */
1266
1267 if (clean_up_chan_for_suspend)
1268 check_ready_for_suspend_event();
1269 }
1270
vmbus_hvsock_device_unregister(struct vmbus_channel * channel)1271 void vmbus_hvsock_device_unregister(struct vmbus_channel *channel)
1272 {
1273 BUG_ON(!is_hvsock_channel(channel));
1274
1275 /* We always get a rescind msg when a connection is closed. */
1276 while (!READ_ONCE(channel->probe_done) || !READ_ONCE(channel->rescind))
1277 msleep(1);
1278
1279 vmbus_device_unregister(channel->device_obj);
1280 }
1281 EXPORT_SYMBOL_GPL(vmbus_hvsock_device_unregister);
1282
1283
1284 /*
1285 * vmbus_onoffers_delivered -
1286 * This is invoked when all offers have been delivered.
1287 *
1288 * Nothing to do here.
1289 */
vmbus_onoffers_delivered(struct vmbus_channel_message_header * hdr)1290 static void vmbus_onoffers_delivered(
1291 struct vmbus_channel_message_header *hdr)
1292 {
1293 }
1294
1295 /*
1296 * vmbus_onopen_result - Open result handler.
1297 *
1298 * This is invoked when we received a response to our channel open request.
1299 * Find the matching request, copy the response and signal the requesting
1300 * thread.
1301 */
vmbus_onopen_result(struct vmbus_channel_message_header * hdr)1302 static void vmbus_onopen_result(struct vmbus_channel_message_header *hdr)
1303 {
1304 struct vmbus_channel_open_result *result;
1305 struct vmbus_channel_msginfo *msginfo;
1306 struct vmbus_channel_message_header *requestheader;
1307 struct vmbus_channel_open_channel *openmsg;
1308 unsigned long flags;
1309
1310 result = (struct vmbus_channel_open_result *)hdr;
1311
1312 trace_vmbus_onopen_result(result);
1313
1314 /*
1315 * Find the open msg, copy the result and signal/unblock the wait event
1316 */
1317 spin_lock_irqsave(&vmbus_connection.channelmsg_lock, flags);
1318
1319 list_for_each_entry(msginfo, &vmbus_connection.chn_msg_list,
1320 msglistentry) {
1321 requestheader =
1322 (struct vmbus_channel_message_header *)msginfo->msg;
1323
1324 if (requestheader->msgtype == CHANNELMSG_OPENCHANNEL) {
1325 openmsg =
1326 (struct vmbus_channel_open_channel *)msginfo->msg;
1327 if (openmsg->child_relid == result->child_relid &&
1328 openmsg->openid == result->openid) {
1329 memcpy(&msginfo->response.open_result,
1330 result,
1331 sizeof(
1332 struct vmbus_channel_open_result));
1333 complete(&msginfo->waitevent);
1334 break;
1335 }
1336 }
1337 }
1338 spin_unlock_irqrestore(&vmbus_connection.channelmsg_lock, flags);
1339 }
1340
1341 /*
1342 * vmbus_ongpadl_created - GPADL created handler.
1343 *
1344 * This is invoked when we received a response to our gpadl create request.
1345 * Find the matching request, copy the response and signal the requesting
1346 * thread.
1347 */
vmbus_ongpadl_created(struct vmbus_channel_message_header * hdr)1348 static void vmbus_ongpadl_created(struct vmbus_channel_message_header *hdr)
1349 {
1350 struct vmbus_channel_gpadl_created *gpadlcreated;
1351 struct vmbus_channel_msginfo *msginfo;
1352 struct vmbus_channel_message_header *requestheader;
1353 struct vmbus_channel_gpadl_header *gpadlheader;
1354 unsigned long flags;
1355
1356 gpadlcreated = (struct vmbus_channel_gpadl_created *)hdr;
1357
1358 trace_vmbus_ongpadl_created(gpadlcreated);
1359
1360 /*
1361 * Find the establish msg, copy the result and signal/unblock the wait
1362 * event
1363 */
1364 spin_lock_irqsave(&vmbus_connection.channelmsg_lock, flags);
1365
1366 list_for_each_entry(msginfo, &vmbus_connection.chn_msg_list,
1367 msglistentry) {
1368 requestheader =
1369 (struct vmbus_channel_message_header *)msginfo->msg;
1370
1371 if (requestheader->msgtype == CHANNELMSG_GPADL_HEADER) {
1372 gpadlheader =
1373 (struct vmbus_channel_gpadl_header *)requestheader;
1374
1375 if ((gpadlcreated->child_relid ==
1376 gpadlheader->child_relid) &&
1377 (gpadlcreated->gpadl == gpadlheader->gpadl)) {
1378 memcpy(&msginfo->response.gpadl_created,
1379 gpadlcreated,
1380 sizeof(
1381 struct vmbus_channel_gpadl_created));
1382 complete(&msginfo->waitevent);
1383 break;
1384 }
1385 }
1386 }
1387 spin_unlock_irqrestore(&vmbus_connection.channelmsg_lock, flags);
1388 }
1389
1390 /*
1391 * vmbus_onmodifychannel_response - Modify Channel response handler.
1392 *
1393 * This is invoked when we received a response to our channel modify request.
1394 * Find the matching request, copy the response and signal the requesting thread.
1395 */
vmbus_onmodifychannel_response(struct vmbus_channel_message_header * hdr)1396 static void vmbus_onmodifychannel_response(struct vmbus_channel_message_header *hdr)
1397 {
1398 struct vmbus_channel_modifychannel_response *response;
1399 struct vmbus_channel_msginfo *msginfo;
1400 unsigned long flags;
1401
1402 response = (struct vmbus_channel_modifychannel_response *)hdr;
1403
1404 trace_vmbus_onmodifychannel_response(response);
1405
1406 /*
1407 * Find the modify msg, copy the response and signal/unblock the wait event.
1408 */
1409 spin_lock_irqsave(&vmbus_connection.channelmsg_lock, flags);
1410
1411 list_for_each_entry(msginfo, &vmbus_connection.chn_msg_list, msglistentry) {
1412 struct vmbus_channel_message_header *responseheader =
1413 (struct vmbus_channel_message_header *)msginfo->msg;
1414
1415 if (responseheader->msgtype == CHANNELMSG_MODIFYCHANNEL) {
1416 struct vmbus_channel_modifychannel *modifymsg;
1417
1418 modifymsg = (struct vmbus_channel_modifychannel *)msginfo->msg;
1419 if (modifymsg->child_relid == response->child_relid) {
1420 memcpy(&msginfo->response.modify_response, response,
1421 sizeof(*response));
1422 complete(&msginfo->waitevent);
1423 break;
1424 }
1425 }
1426 }
1427 spin_unlock_irqrestore(&vmbus_connection.channelmsg_lock, flags);
1428 }
1429
1430 /*
1431 * vmbus_ongpadl_torndown - GPADL torndown handler.
1432 *
1433 * This is invoked when we received a response to our gpadl teardown request.
1434 * Find the matching request, copy the response and signal the requesting
1435 * thread.
1436 */
vmbus_ongpadl_torndown(struct vmbus_channel_message_header * hdr)1437 static void vmbus_ongpadl_torndown(
1438 struct vmbus_channel_message_header *hdr)
1439 {
1440 struct vmbus_channel_gpadl_torndown *gpadl_torndown;
1441 struct vmbus_channel_msginfo *msginfo;
1442 struct vmbus_channel_message_header *requestheader;
1443 struct vmbus_channel_gpadl_teardown *gpadl_teardown;
1444 unsigned long flags;
1445
1446 gpadl_torndown = (struct vmbus_channel_gpadl_torndown *)hdr;
1447
1448 trace_vmbus_ongpadl_torndown(gpadl_torndown);
1449
1450 /*
1451 * Find the open msg, copy the result and signal/unblock the wait event
1452 */
1453 spin_lock_irqsave(&vmbus_connection.channelmsg_lock, flags);
1454
1455 list_for_each_entry(msginfo, &vmbus_connection.chn_msg_list,
1456 msglistentry) {
1457 requestheader =
1458 (struct vmbus_channel_message_header *)msginfo->msg;
1459
1460 if (requestheader->msgtype == CHANNELMSG_GPADL_TEARDOWN) {
1461 gpadl_teardown =
1462 (struct vmbus_channel_gpadl_teardown *)requestheader;
1463
1464 if (gpadl_torndown->gpadl == gpadl_teardown->gpadl) {
1465 memcpy(&msginfo->response.gpadl_torndown,
1466 gpadl_torndown,
1467 sizeof(
1468 struct vmbus_channel_gpadl_torndown));
1469 complete(&msginfo->waitevent);
1470 break;
1471 }
1472 }
1473 }
1474 spin_unlock_irqrestore(&vmbus_connection.channelmsg_lock, flags);
1475 }
1476
1477 /*
1478 * vmbus_onversion_response - Version response handler
1479 *
1480 * This is invoked when we received a response to our initiate contact request.
1481 * Find the matching request, copy the response and signal the requesting
1482 * thread.
1483 */
vmbus_onversion_response(struct vmbus_channel_message_header * hdr)1484 static void vmbus_onversion_response(
1485 struct vmbus_channel_message_header *hdr)
1486 {
1487 struct vmbus_channel_msginfo *msginfo;
1488 struct vmbus_channel_message_header *requestheader;
1489 struct vmbus_channel_version_response *version_response;
1490 unsigned long flags;
1491
1492 version_response = (struct vmbus_channel_version_response *)hdr;
1493
1494 trace_vmbus_onversion_response(version_response);
1495
1496 spin_lock_irqsave(&vmbus_connection.channelmsg_lock, flags);
1497
1498 list_for_each_entry(msginfo, &vmbus_connection.chn_msg_list,
1499 msglistentry) {
1500 requestheader =
1501 (struct vmbus_channel_message_header *)msginfo->msg;
1502
1503 if (requestheader->msgtype ==
1504 CHANNELMSG_INITIATE_CONTACT) {
1505 memcpy(&msginfo->response.version_response,
1506 version_response,
1507 sizeof(struct vmbus_channel_version_response));
1508 complete(&msginfo->waitevent);
1509 }
1510 }
1511 spin_unlock_irqrestore(&vmbus_connection.channelmsg_lock, flags);
1512 }
1513
1514 /* Channel message dispatch table */
1515 const struct vmbus_channel_message_table_entry
1516 channel_message_table[CHANNELMSG_COUNT] = {
1517 { CHANNELMSG_INVALID, 0, NULL, 0},
1518 { CHANNELMSG_OFFERCHANNEL, 0, vmbus_onoffer,
1519 sizeof(struct vmbus_channel_offer_channel)},
1520 { CHANNELMSG_RESCIND_CHANNELOFFER, 0, vmbus_onoffer_rescind,
1521 sizeof(struct vmbus_channel_rescind_offer) },
1522 { CHANNELMSG_REQUESTOFFERS, 0, NULL, 0},
1523 { CHANNELMSG_ALLOFFERS_DELIVERED, 1, vmbus_onoffers_delivered, 0},
1524 { CHANNELMSG_OPENCHANNEL, 0, NULL, 0},
1525 { CHANNELMSG_OPENCHANNEL_RESULT, 1, vmbus_onopen_result,
1526 sizeof(struct vmbus_channel_open_result)},
1527 { CHANNELMSG_CLOSECHANNEL, 0, NULL, 0},
1528 { CHANNELMSG_GPADL_HEADER, 0, NULL, 0},
1529 { CHANNELMSG_GPADL_BODY, 0, NULL, 0},
1530 { CHANNELMSG_GPADL_CREATED, 1, vmbus_ongpadl_created,
1531 sizeof(struct vmbus_channel_gpadl_created)},
1532 { CHANNELMSG_GPADL_TEARDOWN, 0, NULL, 0},
1533 { CHANNELMSG_GPADL_TORNDOWN, 1, vmbus_ongpadl_torndown,
1534 sizeof(struct vmbus_channel_gpadl_torndown) },
1535 { CHANNELMSG_RELID_RELEASED, 0, NULL, 0},
1536 { CHANNELMSG_INITIATE_CONTACT, 0, NULL, 0},
1537 { CHANNELMSG_VERSION_RESPONSE, 1, vmbus_onversion_response,
1538 sizeof(struct vmbus_channel_version_response)},
1539 { CHANNELMSG_UNLOAD, 0, NULL, 0},
1540 { CHANNELMSG_UNLOAD_RESPONSE, 1, vmbus_unload_response, 0},
1541 { CHANNELMSG_18, 0, NULL, 0},
1542 { CHANNELMSG_19, 0, NULL, 0},
1543 { CHANNELMSG_20, 0, NULL, 0},
1544 { CHANNELMSG_TL_CONNECT_REQUEST, 0, NULL, 0},
1545 { CHANNELMSG_MODIFYCHANNEL, 0, NULL, 0},
1546 { CHANNELMSG_TL_CONNECT_RESULT, 0, NULL, 0},
1547 { CHANNELMSG_MODIFYCHANNEL_RESPONSE, 1, vmbus_onmodifychannel_response,
1548 sizeof(struct vmbus_channel_modifychannel_response)},
1549 };
1550
1551 /*
1552 * vmbus_onmessage - Handler for channel protocol messages.
1553 *
1554 * This is invoked in the vmbus worker thread context.
1555 */
vmbus_onmessage(struct vmbus_channel_message_header * hdr)1556 void vmbus_onmessage(struct vmbus_channel_message_header *hdr)
1557 {
1558 trace_vmbus_on_message(hdr);
1559
1560 /*
1561 * vmbus_on_msg_dpc() makes sure the hdr->msgtype here can not go
1562 * out of bound and the message_handler pointer can not be NULL.
1563 */
1564 channel_message_table[hdr->msgtype].message_handler(hdr);
1565 }
1566
1567 /*
1568 * vmbus_request_offers - Send a request to get all our pending offers.
1569 */
vmbus_request_offers(void)1570 int vmbus_request_offers(void)
1571 {
1572 struct vmbus_channel_message_header *msg;
1573 struct vmbus_channel_msginfo *msginfo;
1574 int ret;
1575
1576 msginfo = kmalloc(sizeof(*msginfo) +
1577 sizeof(struct vmbus_channel_message_header),
1578 GFP_KERNEL);
1579 if (!msginfo)
1580 return -ENOMEM;
1581
1582 msg = (struct vmbus_channel_message_header *)msginfo->msg;
1583
1584 msg->msgtype = CHANNELMSG_REQUESTOFFERS;
1585
1586 ret = vmbus_post_msg(msg, sizeof(struct vmbus_channel_message_header),
1587 true);
1588
1589 trace_vmbus_request_offers(ret);
1590
1591 if (ret != 0) {
1592 pr_err("Unable to request offers - %d\n", ret);
1593
1594 goto cleanup;
1595 }
1596
1597 cleanup:
1598 kfree(msginfo);
1599
1600 return ret;
1601 }
1602
invoke_sc_cb(struct vmbus_channel * primary_channel)1603 static void invoke_sc_cb(struct vmbus_channel *primary_channel)
1604 {
1605 struct list_head *cur, *tmp;
1606 struct vmbus_channel *cur_channel;
1607
1608 if (primary_channel->sc_creation_callback == NULL)
1609 return;
1610
1611 list_for_each_safe(cur, tmp, &primary_channel->sc_list) {
1612 cur_channel = list_entry(cur, struct vmbus_channel, sc_list);
1613
1614 primary_channel->sc_creation_callback(cur_channel);
1615 }
1616 }
1617
vmbus_set_sc_create_callback(struct vmbus_channel * primary_channel,void (* sc_cr_cb)(struct vmbus_channel * new_sc))1618 void vmbus_set_sc_create_callback(struct vmbus_channel *primary_channel,
1619 void (*sc_cr_cb)(struct vmbus_channel *new_sc))
1620 {
1621 primary_channel->sc_creation_callback = sc_cr_cb;
1622 }
1623 EXPORT_SYMBOL_GPL(vmbus_set_sc_create_callback);
1624
vmbus_are_subchannels_present(struct vmbus_channel * primary)1625 bool vmbus_are_subchannels_present(struct vmbus_channel *primary)
1626 {
1627 bool ret;
1628
1629 ret = !list_empty(&primary->sc_list);
1630
1631 if (ret) {
1632 /*
1633 * Invoke the callback on sub-channel creation.
1634 * This will present a uniform interface to the
1635 * clients.
1636 */
1637 invoke_sc_cb(primary);
1638 }
1639
1640 return ret;
1641 }
1642 EXPORT_SYMBOL_GPL(vmbus_are_subchannels_present);
1643
vmbus_set_chn_rescind_callback(struct vmbus_channel * channel,void (* chn_rescind_cb)(struct vmbus_channel *))1644 void vmbus_set_chn_rescind_callback(struct vmbus_channel *channel,
1645 void (*chn_rescind_cb)(struct vmbus_channel *))
1646 {
1647 channel->chn_rescind_callback = chn_rescind_cb;
1648 }
1649 EXPORT_SYMBOL_GPL(vmbus_set_chn_rescind_callback);
1650