Lines Matching +full:grant +full:- +full:dma
6 * Copyright (c) 2005-2006, Christopher Clark
7 * Copyright (c) 2004-2005, K A Fraser
50 #include <linux/dma-mapping.h>
58 #include <xen/hvc-console.h>
59 #include <xen/swiotlb-xen.h>
64 #include <xen/mem-reservation.h>
79 * linked by grant ref, the last element contains GNTTAB_LIST_END. The number
110 /*This is a structure of function pointers for grant table*/
113 * Version of the grant interface.
117 * Grant refs per grant frame.
121 * Mapping a list of frames for storing grant entries. Frames parameter
122 * is used to store grant table address when grant table being setup,
123 * nr_gframes is the number of frames to map grant table. Returning
128 * Release a list of frames which are mapped in map_frames for grant
133 * Introducing a valid entry into the grant table, granting the frame of
134 * this grant entry to domain for accessing. Ref
135 * parameter is reference of this introduced grant entry, domid is id of
137 * status of the grant entry to be updated.
142 * Stop granting a grant entry to domain for accessing. Ref parameter is
143 * reference of a grant entry whose grant access will be stopped.
144 * If the grant entry is currently mapped for reading or writing, just
145 * return failure(==0) directly and don't tear down the grant access.
146 * Otherwise, stop grant access for this entry and return success(==1).
150 * Read the frame number related to a given grant reference.
162 /* This reflects status of grant entries, so act as a global value. */
176 /* This can be used as an l-value */
188 ((rc = gnttab_expand(count - gnttab_free_count)) < 0)) { in get_free_entries()
194 gnttab_free_count -= count; in get_free_entries()
195 while (count--) { in get_free_entries()
221 return gnttab_last_free - *gnttab_free_tail_ptr + 1; in get_seq_entry_count()
224 /* Rebuilds the free grant list and tries to find count consecutive entries. */
227 int ret = -ENOSPC; in get_free_seq()
239 if (ret < 0 && to - from >= count) { in get_free_seq()
243 gnttab_free_count -= count; in get_free_seq()
260 gnttab_free_tail_ptr = __gnttab_entry(to - 1); in get_free_seq()
264 if (gnttab_last_free != gnttab_size - 1) in get_free_seq()
278 ret = gnttab_expand(count - gnttab_free_count); in get_free_entries_seq()
287 ret = gnttab_expand(count - get_seq_entry_count()); in get_free_entries_seq()
293 *gnttab_free_tail_ptr = gnttab_entry(ret + count - 1); in get_free_entries_seq()
294 gnttab_free_count -= count; in get_free_entries_seq()
313 next = callback->next; in do_free_callbacks()
314 if (gnttab_free_count >= callback->count) { in do_free_callbacks()
315 callback->next = NULL; in do_free_callbacks()
316 callback->fn(callback->arg); in do_free_callbacks()
318 callback->next = gnttab_free_callback_list; in do_free_callbacks()
360 for (i = start; i < start + n - 1; i++) in gnttab_set_free()
378 * Introducing a valid entry into the grant table:
379 * 1. Write ent->domid.
380 * 2. Write ent->frame: Frame to which access is permitted.
382 * 4. Write ent->flags, inc. valid type.
403 * Public grant-issuing interface functions
408 gnttab_interface->update_entry(ref, domid, frame, in gnttab_grant_foreign_access_ref()
420 return -ENOSPC; in gnttab_grant_foreign_access()
469 return gnttab_interface->end_foreign_access_ref(ref); in _gnttab_end_foreign_access_ref()
515 while ((ignore_limit || nr--) && !list_empty(&deferred_list)) { in gnttab_handle_deferred()
522 list_del(&entry->list); in gnttab_handle_deferred()
524 if (_gnttab_end_foreign_access_ref(entry->ref)) { in gnttab_handle_deferred()
527 put_free_entry(entry->ref); in gnttab_handle_deferred()
529 entry->ref, page_to_pfn(entry->page), in gnttab_handle_deferred()
531 put_page(entry->page); in gnttab_handle_deferred()
536 if (!--entry->warn_delay) in gnttab_handle_deferred()
537 pr_info("g.e. %#x still pending\n", entry->ref); in gnttab_handle_deferred()
543 list_add_tail(&entry->list, &deferred_list); in gnttab_handle_deferred()
563 unsigned long gfn = gnttab_interface->read_frame(ref); in gnttab_add_deferred()
572 entry->ref = ref; in gnttab_add_deferred()
573 entry->page = page; in gnttab_add_deferred()
574 entry->warn_delay = 60; in gnttab_add_deferred()
576 list_add_tail(&entry->list, &deferred_list); in gnttab_add_deferred()
585 ref, page ? page_to_pfn(page) : -1, deferred, leaked); in gnttab_add_deferred()
590 ref, page ? page_to_pfn(page) : -1, deferred, leaked); in gnttab_add_deferred()
643 for (i = count; i > 0; i--) in gnttab_free_grant_reference_seq()
644 put_free_entry_locked(head + i - 1); in gnttab_free_grant_reference_seq()
655 return -ENOSPC; in gnttab_alloc_grant_references()
673 return -ENOSPC; in gnttab_alloc_grant_reference_seq()
691 return -ENOSPC; in gnttab_claim_grant_reference()
718 cb = cb->next; in gnttab_request_free_callback()
721 callback->fn = fn; in gnttab_request_free_callback()
722 callback->arg = arg; in gnttab_request_free_callback()
723 callback->count = count; in gnttab_request_free_callback()
724 callback->next = gnttab_free_callback_list; in gnttab_request_free_callback()
738 for (pcb = &gnttab_free_callback_list; *pcb; pcb = &(*pcb)->next) { in gnttab_cancel_free_callback()
740 *pcb = callback->next; in gnttab_cancel_free_callback()
750 return (frames * gnttab_interface->grefs_per_grant_frame + align - 1) / in gnttab_frames()
760 grefs_per_frame = gnttab_interface->grefs_per_grant_frame; in grow_gnttab_list()
786 while (i-- > nr_glist_frames) in grow_gnttab_list()
788 return -ENOMEM; in grow_gnttab_list()
828 return -EINVAL; in gnttab_setup_auto_xlat_frames()
834 return -ENOMEM; in gnttab_setup_auto_xlat_frames()
839 return -ENOMEM; in gnttab_setup_auto_xlat_frames()
875 return -ENOMEM; in gnttab_pages_set_private()
887 * gnttab_alloc_pages - alloc pages suitable for grant mapping into
910 cache->pages = NULL; in cache_init()
915 return !cache->pages; in cache_empty()
922 page = cache->pages; in cache_deq()
923 cache->pages = page->zone_device_data; in cache_deq()
930 page->zone_device_data = cache->pages; in cache_enq()
931 cache->pages = page; in cache_enq()
936 INIT_LIST_HEAD(&cache->pages); in cache_init()
941 return list_empty(&cache->pages); in cache_empty()
948 page = list_first_entry(&cache->pages, struct page, lru); in cache_deq()
949 list_del(&page->lru); in cache_deq()
956 list_add(&page->lru, &cache->pages); in cache_enq()
962 spin_lock_init(&cache->lock); in gnttab_page_cache_init()
964 cache->num_pages = 0; in gnttab_page_cache_init()
972 spin_lock_irqsave(&cache->lock, flags); in gnttab_page_cache_get()
975 spin_unlock_irqrestore(&cache->lock, flags); in gnttab_page_cache_get()
980 cache->num_pages--; in gnttab_page_cache_get()
982 spin_unlock_irqrestore(&cache->lock, flags); in gnttab_page_cache_get()
994 spin_lock_irqsave(&cache->lock, flags); in gnttab_page_cache_put()
998 cache->num_pages += num; in gnttab_page_cache_put()
1000 spin_unlock_irqrestore(&cache->lock, flags); in gnttab_page_cache_put()
1010 spin_lock_irqsave(&cache->lock, flags); in gnttab_page_cache_shrink()
1012 while (cache->num_pages > num) { in gnttab_page_cache_shrink()
1014 cache->num_pages--; in gnttab_page_cache_shrink()
1016 spin_unlock_irqrestore(&cache->lock, flags); in gnttab_page_cache_shrink()
1019 spin_lock_irqsave(&cache->lock, flags); in gnttab_page_cache_shrink()
1023 spin_unlock_irqrestore(&cache->lock, flags); in gnttab_page_cache_shrink()
1046 * gnttab_free_pages - free pages allocated by gnttab_alloc_pages()
1059 * gnttab_dma_alloc_pages - alloc DMAable pages suitable for grant mapping into
1068 if (args->nr_pages < 0 || args->nr_pages > (INT_MAX >> PAGE_SHIFT)) in gnttab_dma_alloc_pages()
1069 return -ENOMEM; in gnttab_dma_alloc_pages()
1071 size = args->nr_pages << PAGE_SHIFT; in gnttab_dma_alloc_pages()
1072 if (args->coherent) in gnttab_dma_alloc_pages()
1073 args->vaddr = dma_alloc_coherent(args->dev, size, in gnttab_dma_alloc_pages()
1074 &args->dev_bus_addr, in gnttab_dma_alloc_pages()
1077 args->vaddr = dma_alloc_wc(args->dev, size, in gnttab_dma_alloc_pages()
1078 &args->dev_bus_addr, in gnttab_dma_alloc_pages()
1080 if (!args->vaddr) { in gnttab_dma_alloc_pages()
1081 pr_debug("Failed to allocate DMA buffer of size %zu\n", size); in gnttab_dma_alloc_pages()
1082 return -ENOMEM; in gnttab_dma_alloc_pages()
1085 start_pfn = __phys_to_pfn(args->dev_bus_addr); in gnttab_dma_alloc_pages()
1086 for (pfn = start_pfn, i = 0; pfn < start_pfn + args->nr_pages; in gnttab_dma_alloc_pages()
1090 args->pages[i] = page; in gnttab_dma_alloc_pages()
1091 args->frames[i] = xen_page_to_gfn(page); in gnttab_dma_alloc_pages()
1095 xenmem_reservation_va_mapping_reset(args->nr_pages, args->pages); in gnttab_dma_alloc_pages()
1097 ret = xenmem_reservation_decrease(args->nr_pages, args->frames); in gnttab_dma_alloc_pages()
1098 if (ret != args->nr_pages) { in gnttab_dma_alloc_pages()
1099 pr_debug("Failed to decrease reservation for DMA buffer\n"); in gnttab_dma_alloc_pages()
1100 ret = -EFAULT; in gnttab_dma_alloc_pages()
1104 ret = gnttab_pages_set_private(args->nr_pages, args->pages); in gnttab_dma_alloc_pages()
1117 * gnttab_dma_free_pages - free DMAable pages
1125 gnttab_pages_clear_private(args->nr_pages, args->pages); in gnttab_dma_free_pages()
1127 for (i = 0; i < args->nr_pages; i++) in gnttab_dma_free_pages()
1128 args->frames[i] = page_to_xen_pfn(args->pages[i]); in gnttab_dma_free_pages()
1130 ret = xenmem_reservation_increase(args->nr_pages, args->frames); in gnttab_dma_free_pages()
1131 if (ret != args->nr_pages) { in gnttab_dma_free_pages()
1132 pr_debug("Failed to increase reservation for DMA buffer\n"); in gnttab_dma_free_pages()
1133 ret = -EFAULT; in gnttab_dma_free_pages()
1138 xenmem_reservation_va_mapping_update(args->nr_pages, args->pages, in gnttab_dma_free_pages()
1139 args->frames); in gnttab_dma_free_pages()
1141 size = args->nr_pages << PAGE_SHIFT; in gnttab_dma_free_pages()
1142 if (args->coherent) in gnttab_dma_free_pages()
1143 dma_free_coherent(args->dev, size, in gnttab_dma_free_pages()
1144 args->vaddr, args->dev_bus_addr); in gnttab_dma_free_pages()
1146 dma_free_wc(args->dev, size, in gnttab_dma_free_pages()
1147 args->vaddr, args->dev_bus_addr); in gnttab_dma_free_pages()
1153 /* Handling of paged out grant targets (GNTST_eagain) */
1168 pr_err("%s: %s eagain grant\n", func, current->comm); in gnttab_retry_eagain_gop()
1180 if (op->status == GNTST_eagain) in gnttab_batch_map()
1182 &op->status, __func__); in gnttab_batch_map()
1193 if (op->status == GNTST_eagain) in gnttab_batch_copy()
1195 &op->status, __func__); in gnttab_batch_copy()
1209 len = min_t(unsigned int, PAGE_SIZE - offset, len); in gnttab_foreach_grant_in_range()
1215 glen = min_t(unsigned int, XEN_PAGE_SIZE - goffset, len); in gnttab_foreach_grant_in_range()
1220 len -= glen; in gnttab_foreach_grant_in_range()
1265 foreign->domid = map_ops[i].dom; in gnttab_map_refs()
1266 foreign->gref = map_ops[i].ref; in gnttab_map_refs()
1280 i--; in gnttab_map_refs()
1320 if (unmap_data->age != UINT_MAX) in gnttab_unmap_work()
1321 unmap_data->age++; in gnttab_unmap_work()
1330 for (pc = 0; pc < item->count; pc++) { in __gnttab_unmap_refs_async()
1331 if (page_count(item->pages[pc]) > 1) { in __gnttab_unmap_refs_async()
1332 unsigned long delay = GNTTAB_UNMAP_REFS_DELAY * (item->age + 1); in __gnttab_unmap_refs_async()
1333 schedule_delayed_work(&item->gnttab_work, in __gnttab_unmap_refs_async()
1339 ret = gnttab_unmap_refs(item->unmap_ops, item->kunmap_ops, in __gnttab_unmap_refs_async()
1340 item->pages, item->count); in __gnttab_unmap_refs_async()
1341 item->done(ret, item); in __gnttab_unmap_refs_async()
1346 INIT_DELAYED_WORK(&item->gnttab_work, gnttab_unmap_work); in gnttab_unmap_refs_async()
1347 item->age = 0; in gnttab_unmap_refs_async()
1356 struct unmap_refs_callback_data *d = data->data; in unmap_refs_callback()
1358 d->result = result; in unmap_refs_callback()
1359 complete(&d->completion); in unmap_refs_callback()
1367 item->data = &data; in gnttab_unmap_refs_sync()
1368 item->done = &unmap_refs_callback; in gnttab_unmap_refs_sync()
1412 return -ENOMEM; in gnttab_map_frames_v2()
1420 if (rc == -ENOSYS) { in gnttab_map_frames_v2()
1422 return -ENOSYS; in gnttab_map_frames_v2()
1470 pr_warn("grant table add_to_physmap failed, err=%d\n", in gnttab_map()
1474 } while (i-- > start_idx); in gnttab_map()
1484 return -ENOMEM; in gnttab_map()
1491 if (rc == -ENOSYS) { in gnttab_map()
1493 return -ENOSYS; in gnttab_map()
1498 rc = gnttab_interface->map_frames(frames, nr_gframes); in gnttab_map()
1563 pr_info("Grant tables using version %d layout\n", in gnttab_request_version()
1564 gnttab_interface->version); in gnttab_request_version()
1573 return -ENOSYS; in gnttab_setup()
1579 return -ENOMEM; in gnttab_setup()
1582 return gnttab_map(0, nr_grant_frames - 1); in gnttab_setup()
1594 gnttab_interface->unmap_frames(); in gnttab_suspend()
1604 extra = ((req_entries + gnttab_interface->grefs_per_grant_frame - 1) / in gnttab_expand()
1605 gnttab_interface->grefs_per_grant_frame); in gnttab_expand()
1607 pr_warn_ratelimited("xen/grant-table: max_grant_frames reached" in gnttab_expand()
1612 return -ENOSPC; in gnttab_expand()
1615 rc = gnttab_map(cur, cur + extra - 1); in gnttab_expand()
1632 gnttab_interface->grefs_per_grant_frame; in gnttab_init()
1636 * grant reference free list on the current hypervisor. in gnttab_init()
1644 return -ENOMEM; in gnttab_init()
1650 ret = -ENOMEM; in gnttab_init()
1657 ret = -ENOMEM; in gnttab_init()
1667 ret = -ENODEV; in gnttab_init()
1671 gnttab_size = nr_grant_frames * gnttab_interface->grefs_per_grant_frame; in gnttab_init()
1674 gnttab_size - GNTTAB_NR_RESERVED_ENTRIES); in gnttab_init()
1676 printk("Grant table initialized\n"); in gnttab_init()
1680 for (i--; i >= 0; i--) in gnttab_init()
1691 return -ENODEV; in __gnttab_init()
1693 /* Delay grant-table initialization in the PV on HVM case */ in __gnttab_init()