Lines Matching +full:update +full:- +full:fc +full:- +full:fixup
1 // SPDX-License-Identifier: GPL-2.0-or-later
39 #define VIRTIO_BALLOON_HINT_BLOCK_ORDER (MAX_ORDER - 1)
66 /* Balloon's own wq for cpu-intensive work items */
104 * at vb_dev_info->pages list.
110 /* Synchronize access/update to this struct virtio_balloon elements */
120 /* Shrinker to return free pages - VIRTIO_BALLOON_F_FREE_PAGE_HINT */
123 /* OOM notifier to deflate on OOM - VIRTIO_BALLOON_F_DEFLATE_ON_OOM */
147 struct virtio_balloon *vb = vq->vdev->priv; in balloon_ack()
149 wake_up(&vb->acked); in balloon_ack()
157 sg_init_one(&sg, vb->pfns, sizeof(vb->pfns[0]) * vb->num_pfns); in tell_host()
164 wait_event(vb->acked, virtqueue_get_buf(vq, &len)); in tell_host()
173 struct virtqueue *vq = vb->reporting_vq; in virtballoon_free_page_report()
190 wait_event(vb->acked, virtqueue_get_buf(vq, &unused)); in virtballoon_free_page_report()
207 pfns[i] = cpu_to_virtio32(vb->vdev, in set_page_pfns()
219 num = min(num, ARRAY_SIZE(vb->pfns)); in fill_balloon()
226 dev_info_ratelimited(&vb->vdev->dev, in fill_balloon()
237 mutex_lock(&vb->balloon_lock); in fill_balloon()
239 vb->num_pfns = 0; in fill_balloon()
242 balloon_page_enqueue(&vb->vb_dev_info, page); in fill_balloon()
244 set_page_pfns(vb, vb->pfns + vb->num_pfns, page); in fill_balloon()
245 vb->num_pages += VIRTIO_BALLOON_PAGES_PER_PAGE; in fill_balloon()
246 if (!virtio_has_feature(vb->vdev, in fill_balloon()
248 adjust_managed_page_count(page, -1); in fill_balloon()
249 vb->num_pfns += VIRTIO_BALLOON_PAGES_PER_PAGE; in fill_balloon()
252 num_allocated_pages = vb->num_pfns; in fill_balloon()
254 if (vb->num_pfns != 0) in fill_balloon()
255 tell_host(vb, vb->inflate_vq); in fill_balloon()
256 mutex_unlock(&vb->balloon_lock); in fill_balloon()
267 if (!virtio_has_feature(vb->vdev, in release_pages_balloon()
270 list_del(&page->lru); in release_pages_balloon()
279 struct balloon_dev_info *vb_dev_info = &vb->vb_dev_info; in leak_balloon()
283 num = min(num, ARRAY_SIZE(vb->pfns)); in leak_balloon()
285 mutex_lock(&vb->balloon_lock); in leak_balloon()
287 num = min(num, (size_t)vb->num_pages); in leak_balloon()
288 for (vb->num_pfns = 0; vb->num_pfns < num; in leak_balloon()
289 vb->num_pfns += VIRTIO_BALLOON_PAGES_PER_PAGE) { in leak_balloon()
293 set_page_pfns(vb, vb->pfns + vb->num_pfns, page); in leak_balloon()
294 list_add(&page->lru, &pages); in leak_balloon()
295 vb->num_pages -= VIRTIO_BALLOON_PAGES_PER_PAGE; in leak_balloon()
298 num_freed_pages = vb->num_pfns; in leak_balloon()
304 if (vb->num_pfns != 0) in leak_balloon()
305 tell_host(vb, vb->deflate_vq); in leak_balloon()
307 mutex_unlock(&vb->balloon_lock); in leak_balloon()
315 vb->stats[idx].tag = cpu_to_virtio16(vb->vdev, tag); in update_stat()
316 vb->stats[idx].val = cpu_to_virtio64(vb->vdev, val); in update_stat()
362 * While most virtqueues communicate guest-initiated requests to the hypervisor,
372 struct virtio_balloon *vb = vq->vdev->priv; in stats_request()
374 spin_lock(&vb->stop_update_lock); in stats_request()
375 if (!vb->stop_update) in stats_request()
376 queue_work(system_freezable_wq, &vb->update_balloon_stats_work); in stats_request()
377 spin_unlock(&vb->stop_update_lock); in stats_request()
388 vq = vb->stats_vq; in stats_handle_request()
391 sg_init_one(&sg, vb->stats, sizeof(vb->stats[0]) * num_stats); in stats_handle_request()
402 virtio_cread_le(vb->vdev, struct virtio_balloon_config, num_pages, in towards_target()
410 return target - vb->num_pages; in towards_target()
420 spin_lock_irq(&vb->free_page_list_lock); in return_free_pages_to_mm()
422 page = balloon_page_pop(&vb->free_page_list); in return_free_pages_to_mm()
428 vb->num_free_page_blocks -= num_returned; in return_free_pages_to_mm()
429 spin_unlock_irq(&vb->free_page_list_lock); in return_free_pages_to_mm()
436 if (!virtio_has_feature(vb->vdev, VIRTIO_BALLOON_F_FREE_PAGE_HINT)) in virtio_balloon_queue_free_page_work()
441 &vb->config_read_bitmap)) in virtio_balloon_queue_free_page_work()
444 queue_work(vb->balloon_wq, &vb->report_free_page_work); in virtio_balloon_queue_free_page_work()
449 struct virtio_balloon *vb = vdev->priv; in virtballoon_changed()
452 spin_lock_irqsave(&vb->stop_update_lock, flags); in virtballoon_changed()
453 if (!vb->stop_update) { in virtballoon_changed()
455 &vb->update_balloon_size_work); in virtballoon_changed()
458 spin_unlock_irqrestore(&vb->stop_update_lock, flags); in virtballoon_changed()
463 u32 actual = vb->num_pages; in update_balloon_size()
466 virtio_cwrite_le(vb->vdev, struct virtio_balloon_config, actual, in update_balloon_size()
492 diff -= fill_balloon(vb, diff); in update_balloon_size_func()
494 diff += leak_balloon(vb, -diff); in update_balloon_size_func()
523 if (virtio_has_feature(vb->vdev, VIRTIO_BALLOON_F_STATS_VQ)) { in init_vqs()
528 if (virtio_has_feature(vb->vdev, VIRTIO_BALLOON_F_FREE_PAGE_HINT)) { in init_vqs()
533 if (virtio_has_feature(vb->vdev, VIRTIO_BALLOON_F_REPORTING)) { in init_vqs()
538 err = vb->vdev->config->find_vqs(vb->vdev, VIRTIO_BALLOON_VQ_MAX, in init_vqs()
543 vb->inflate_vq = vqs[VIRTIO_BALLOON_VQ_INFLATE]; in init_vqs()
544 vb->deflate_vq = vqs[VIRTIO_BALLOON_VQ_DEFLATE]; in init_vqs()
545 if (virtio_has_feature(vb->vdev, VIRTIO_BALLOON_F_STATS_VQ)) { in init_vqs()
548 vb->stats_vq = vqs[VIRTIO_BALLOON_VQ_STATS]; in init_vqs()
556 sg_init_one(&sg, vb->stats, sizeof(vb->stats[0]) * num_stats); in init_vqs()
557 err = virtqueue_add_outbuf(vb->stats_vq, &sg, 1, vb, in init_vqs()
560 dev_warn(&vb->vdev->dev, "%s: add stat_vq failed\n", in init_vqs()
564 virtqueue_kick(vb->stats_vq); in init_vqs()
567 if (virtio_has_feature(vb->vdev, VIRTIO_BALLOON_F_FREE_PAGE_HINT)) in init_vqs()
568 vb->free_page_vq = vqs[VIRTIO_BALLOON_VQ_FREE_PAGE]; in init_vqs()
570 if (virtio_has_feature(vb->vdev, VIRTIO_BALLOON_F_REPORTING)) in init_vqs()
571 vb->reporting_vq = vqs[VIRTIO_BALLOON_VQ_REPORTING]; in init_vqs()
579 &vb->config_read_bitmap)) { in virtio_balloon_cmd_id_received()
581 virtio_cread_le(vb->vdev, struct virtio_balloon_config, in virtio_balloon_cmd_id_received()
583 &vb->cmd_id_received_cache); in virtio_balloon_cmd_id_received()
586 return vb->cmd_id_received_cache; in virtio_balloon_cmd_id_received()
592 struct virtqueue *vq = vb->free_page_vq; in send_cmd_id_start()
599 vb->cmd_id_active = cpu_to_virtio32(vb->vdev, in send_cmd_id_start()
601 sg_init_one(&sg, &vb->cmd_id_active, sizeof(vb->cmd_id_active)); in send_cmd_id_start()
602 err = virtqueue_add_outbuf(vq, &sg, 1, &vb->cmd_id_active, GFP_KERNEL); in send_cmd_id_start()
611 struct virtqueue *vq = vb->free_page_vq; in send_cmd_id_stop()
618 sg_init_one(&sg, &vb->cmd_id_stop, sizeof(vb->cmd_id_stop)); in send_cmd_id_stop()
619 err = virtqueue_add_outbuf(vq, &sg, 1, &vb->cmd_id_stop, GFP_KERNEL); in send_cmd_id_stop()
627 struct virtqueue *vq = vb->free_page_vq; in get_free_page_and_send()
641 * the possible free pages, so return -EINTR to stop. in get_free_page_and_send()
644 return -EINTR; in get_free_page_and_send()
649 if (vq->num_free > 1) { in get_free_page_and_send()
657 spin_lock_irq(&vb->free_page_list_lock); in get_free_page_and_send()
658 balloon_page_push(&vb->free_page_list, page); in get_free_page_and_send()
659 vb->num_free_page_blocks++; in get_free_page_and_send()
660 spin_unlock_irq(&vb->free_page_list_lock); in get_free_page_and_send()
682 cmd_id_active = virtio32_to_cpu(vb->vdev, vb->cmd_id_active); in send_free_pages()
692 if (err == -EINTR) in send_free_pages()
704 struct device *dev = &vb->vdev->dev; in virtio_balloon_report_free_page()
733 virtio32_to_cpu(vb->vdev, vb->cmd_id_active)) { in report_free_page_func()
740 * virtballoon_migratepage - perform the balloon page migration on behalf of
745 * @mode : compaction mode -- not used for balloon page migration.
751 * 1) insert newpage into vb->pages list and update the host about it;
752 * 2) update the host about the old page removed from vb->pages list;
755 * Called through balloon_mapping->a_ops->migratepage
772 if (!mutex_trylock(&vb->balloon_lock)) in virtballoon_migratepage()
773 return -EAGAIN; in virtballoon_migratepage()
779 * managed page count when inflating, we have to fixup the count of in virtballoon_migratepage()
782 if (!virtio_has_feature(vb->vdev, VIRTIO_BALLOON_F_DEFLATE_ON_OOM) && in virtballoon_migratepage()
785 adjust_managed_page_count(newpage, -1); in virtballoon_migratepage()
788 /* balloon's page migration 1st step -- inflate "newpage" */ in virtballoon_migratepage()
789 spin_lock_irqsave(&vb_dev_info->pages_lock, flags); in virtballoon_migratepage()
791 vb_dev_info->isolated_pages--; in virtballoon_migratepage()
793 spin_unlock_irqrestore(&vb_dev_info->pages_lock, flags); in virtballoon_migratepage()
794 vb->num_pfns = VIRTIO_BALLOON_PAGES_PER_PAGE; in virtballoon_migratepage()
795 set_page_pfns(vb, vb->pfns, newpage); in virtballoon_migratepage()
796 tell_host(vb, vb->inflate_vq); in virtballoon_migratepage()
798 /* balloon's page migration 2nd step -- deflate "page" */ in virtballoon_migratepage()
799 spin_lock_irqsave(&vb_dev_info->pages_lock, flags); in virtballoon_migratepage()
801 spin_unlock_irqrestore(&vb_dev_info->pages_lock, flags); in virtballoon_migratepage()
802 vb->num_pfns = VIRTIO_BALLOON_PAGES_PER_PAGE; in virtballoon_migratepage()
803 set_page_pfns(vb, vb->pfns, page); in virtballoon_migratepage()
804 tell_host(vb, vb->deflate_vq); in virtballoon_migratepage()
806 mutex_unlock(&vb->balloon_lock); in virtballoon_migratepage()
813 static int balloon_init_fs_context(struct fs_context *fc) in balloon_init_fs_context() argument
815 return init_pseudo(fc, BALLOON_KVM_MAGIC) ? 0 : -ENOMEM; in balloon_init_fs_context()
819 .name = "balloon-kvm",
845 return shrink_free_pages(vb, sc->nr_to_scan); in virtio_balloon_shrinker_scan()
854 return vb->num_free_page_blocks * VIRTIO_BALLOON_HINT_BLOCK_PAGES; in virtio_balloon_shrinker_count()
873 unregister_shrinker(&vb->shrinker); in virtio_balloon_unregister_shrinker()
878 vb->shrinker.scan_objects = virtio_balloon_shrinker_scan; in virtio_balloon_register_shrinker()
879 vb->shrinker.count_objects = virtio_balloon_shrinker_count; in virtio_balloon_register_shrinker()
880 vb->shrinker.seeks = DEFAULT_SEEKS; in virtio_balloon_register_shrinker()
882 return register_shrinker(&vb->shrinker); in virtio_balloon_register_shrinker()
890 if (!vdev->config->get) { in virtballoon_probe()
891 dev_err(&vdev->dev, "%s failure: config access disabled\n", in virtballoon_probe()
893 return -EINVAL; in virtballoon_probe()
896 vdev->priv = vb = kzalloc(sizeof(*vb), GFP_KERNEL); in virtballoon_probe()
898 err = -ENOMEM; in virtballoon_probe()
902 INIT_WORK(&vb->update_balloon_stats_work, update_balloon_stats_func); in virtballoon_probe()
903 INIT_WORK(&vb->update_balloon_size_work, update_balloon_size_func); in virtballoon_probe()
904 spin_lock_init(&vb->stop_update_lock); in virtballoon_probe()
905 mutex_init(&vb->balloon_lock); in virtballoon_probe()
906 init_waitqueue_head(&vb->acked); in virtballoon_probe()
907 vb->vdev = vdev; in virtballoon_probe()
909 balloon_devinfo_init(&vb->vb_dev_info); in virtballoon_probe()
922 vb->vb_dev_info.migratepage = virtballoon_migratepage; in virtballoon_probe()
923 vb->vb_dev_info.inode = alloc_anon_inode(balloon_mnt->mnt_sb); in virtballoon_probe()
924 if (IS_ERR(vb->vb_dev_info.inode)) { in virtballoon_probe()
925 err = PTR_ERR(vb->vb_dev_info.inode); in virtballoon_probe()
928 vb->vb_dev_info.inode->i_mapping->a_ops = &balloon_aops; in virtballoon_probe()
935 if (virtqueue_get_vring_size(vb->free_page_vq) < 2) { in virtballoon_probe()
936 err = -ENOSPC; in virtballoon_probe()
939 vb->balloon_wq = alloc_workqueue("balloon-wq", in virtballoon_probe()
941 if (!vb->balloon_wq) { in virtballoon_probe()
942 err = -ENOMEM; in virtballoon_probe()
945 INIT_WORK(&vb->report_free_page_work, report_free_page_func); in virtballoon_probe()
946 vb->cmd_id_received_cache = VIRTIO_BALLOON_CMD_ID_STOP; in virtballoon_probe()
947 vb->cmd_id_active = cpu_to_virtio32(vb->vdev, in virtballoon_probe()
949 vb->cmd_id_stop = cpu_to_virtio32(vb->vdev, in virtballoon_probe()
951 spin_lock_init(&vb->free_page_list_lock); in virtballoon_probe()
952 INIT_LIST_HEAD(&vb->free_page_list); in virtballoon_probe()
962 if (virtio_has_feature(vb->vdev, VIRTIO_BALLOON_F_DEFLATE_ON_OOM)) { in virtballoon_probe()
963 vb->oom_nb.notifier_call = virtio_balloon_oom_notify; in virtballoon_probe()
964 vb->oom_nb.priority = VIRTIO_BALLOON_OOM_NOTIFY_PRIORITY; in virtballoon_probe()
965 err = register_oom_notifier(&vb->oom_nb); in virtballoon_probe()
980 * little-endian. However for now it is a single byte so we in virtballoon_probe()
981 * can pass it as-is. in virtballoon_probe()
986 virtio_cwrite_le(vb->vdev, struct virtio_balloon_config, in virtballoon_probe()
990 vb->pr_dev_info.report = virtballoon_free_page_report; in virtballoon_probe()
991 if (virtio_has_feature(vb->vdev, VIRTIO_BALLOON_F_REPORTING)) { in virtballoon_probe()
994 capacity = virtqueue_get_vring_size(vb->reporting_vq); in virtballoon_probe()
996 err = -ENOSPC; in virtballoon_probe()
1000 err = page_reporting_register(&vb->pr_dev_info); in virtballoon_probe()
1012 if (virtio_has_feature(vb->vdev, VIRTIO_BALLOON_F_DEFLATE_ON_OOM)) in virtballoon_probe()
1013 unregister_oom_notifier(&vb->oom_nb); in virtballoon_probe()
1015 if (virtio_has_feature(vb->vdev, VIRTIO_BALLOON_F_FREE_PAGE_HINT)) in virtballoon_probe()
1019 destroy_workqueue(vb->balloon_wq); in virtballoon_probe()
1022 iput(vb->vb_dev_info.inode); in virtballoon_probe()
1027 vdev->config->del_vqs(vdev); in virtballoon_probe()
1037 while (vb->num_pages) in remove_common()
1038 leak_balloon(vb, vb->num_pages); in remove_common()
1042 if (virtio_has_feature(vb->vdev, VIRTIO_BALLOON_F_FREE_PAGE_HINT)) in remove_common()
1046 vb->vdev->config->reset(vb->vdev); in remove_common()
1048 vb->vdev->config->del_vqs(vb->vdev); in remove_common()
1053 struct virtio_balloon *vb = vdev->priv; in virtballoon_remove()
1055 if (virtio_has_feature(vb->vdev, VIRTIO_BALLOON_F_REPORTING)) in virtballoon_remove()
1056 page_reporting_unregister(&vb->pr_dev_info); in virtballoon_remove()
1057 if (virtio_has_feature(vb->vdev, VIRTIO_BALLOON_F_DEFLATE_ON_OOM)) in virtballoon_remove()
1058 unregister_oom_notifier(&vb->oom_nb); in virtballoon_remove()
1059 if (virtio_has_feature(vb->vdev, VIRTIO_BALLOON_F_FREE_PAGE_HINT)) in virtballoon_remove()
1061 spin_lock_irq(&vb->stop_update_lock); in virtballoon_remove()
1062 vb->stop_update = true; in virtballoon_remove()
1063 spin_unlock_irq(&vb->stop_update_lock); in virtballoon_remove()
1064 cancel_work_sync(&vb->update_balloon_size_work); in virtballoon_remove()
1065 cancel_work_sync(&vb->update_balloon_stats_work); in virtballoon_remove()
1068 cancel_work_sync(&vb->report_free_page_work); in virtballoon_remove()
1069 destroy_workqueue(vb->balloon_wq); in virtballoon_remove()
1074 if (vb->vb_dev_info.inode) in virtballoon_remove()
1075 iput(vb->vb_dev_info.inode); in virtballoon_remove()
1085 struct virtio_balloon *vb = vdev->priv; in virtballoon_freeze()
1097 struct virtio_balloon *vb = vdev->priv; in virtballoon_restore()
1100 ret = init_vqs(vdev->priv); in virtballoon_restore()