Lines Matching +full:diff +full:- +full:channels
1 // SPDX-License-Identifier: GPL-2.0-only
12 #include <linux/dma-mapping.h>
41 "Maximum number of channels to use (default: all)");
51 "dmatest 0-memcpy 1-memset (default: 0)");
66 "Pass -1 for infinite timeout");
80 static int alignment = -1;
82 …M_DESC(alignment, "Custom data address alignment taken as 2^(alignment) (default: not used (-1))");
93 * struct dmatest_params - test parameters.
98 * @max_channels: maximum number of channels to use
102 * @timeout: transfer timeout in msec, -1 for infinite timeout
127 * struct dmatest_info - test information.
129 * @channels: channels under test
130 * @nr_channels: number of channels under test
140 struct list_head channels; member
146 .channels = LIST_HEAD_INIT(test_info.channels),
210 /* poor man's completion - we want to use wait_event_freezable() on it */
250 list_for_each_entry(dtc, &info->channels, node) { in is_threaded_test_run()
253 list_for_each_entry(thread, &dtc->threads, node) { in is_threaded_test_run()
254 if (!thread->done && !thread->pending) in is_threaded_test_run()
266 list_for_each_entry(dtc, &info->channels, node) { in is_threaded_test_pending()
269 list_for_each_entry(thread, &dtc->threads, node) { in is_threaded_test_pending()
270 if (thread->pending) in is_threaded_test_pending()
281 struct dmatest_params *params = &info->params; in dmatest_wait_get()
283 if (params->iterations) in dmatest_wait_get()
299 if (params->channel[0] == '\0') in dmatest_match_channel()
301 return strcmp(dma_chan_name(chan), params->channel) == 0; in dmatest_match_channel()
307 if (params->device[0] == '\0') in dmatest_match_device()
309 return strcmp(dev_name(device->dev), params->device) == 0; in dmatest_match_device()
374 u8 diff = actual ^ pattern; in dmatest_mismatch() local
376 const char *thread_name = current->comm; in dmatest_mismatch()
382 && (diff & (PATTERN_COPY | PATTERN_OVERWRITE))) in dmatest_mismatch()
385 else if (diff & PATTERN_SRC) in dmatest_mismatch()
422 current->comm, error_count - MAX_ERROR_COUNT); in dmatest_verify()
433 if (!thread->done) { in dmatest_callback()
434 done->done = true; in dmatest_callback()
435 wake_up_all(done->wait); in dmatest_callback()
438 * If thread->done, it means that this callback occurred in dmatest_callback()
452 return val % 2 ? val : val - 1; in min_odd()
460 current->comm, n, err, src_off, dst_off, len, data); in result()
463 current->comm, n, err, src_off, dst_off, len, data); in result()
472 current->comm, n, err, src_off, dst_off, len, data); in dbg_result()
489 /* drop precision until runtime is 32-bits */ in dmatest_persec()
512 kfree(d->raw[i]); in __dmatest_free_test_data()
514 kfree(d->aligned); in __dmatest_free_test_data()
515 kfree(d->raw); in __dmatest_free_test_data()
520 __dmatest_free_test_data(d, d->cnt); in dmatest_free_test_data()
528 d->raw = kcalloc(d->cnt + 1, sizeof(u8 *), GFP_KERNEL); in dmatest_alloc_test_data()
529 if (!d->raw) in dmatest_alloc_test_data()
530 return -ENOMEM; in dmatest_alloc_test_data()
532 d->aligned = kcalloc(d->cnt + 1, sizeof(u8 *), GFP_KERNEL); in dmatest_alloc_test_data()
533 if (!d->aligned) in dmatest_alloc_test_data()
536 for (i = 0; i < d->cnt; i++) { in dmatest_alloc_test_data()
537 d->raw[i] = kmalloc(buf_size + align, GFP_KERNEL); in dmatest_alloc_test_data()
538 if (!d->raw[i]) in dmatest_alloc_test_data()
543 d->aligned[i] = PTR_ALIGN(d->raw[i], align); in dmatest_alloc_test_data()
545 d->aligned[i] = d->raw[i]; in dmatest_alloc_test_data()
551 return -ENOMEM; in dmatest_alloc_test_data()
558 * in parallel for a single channel, and there may be multiple channels
571 struct dmatest_done *done = &thread->test_done; in dmatest_func()
588 ktime_t ktime, start, diff; in dmatest_func() local
601 ret = -ENOMEM; in dmatest_func()
604 thread->pending = false; in dmatest_func()
605 info = thread->info; in dmatest_func()
606 params = &info->params; in dmatest_func()
607 chan = thread->chan; in dmatest_func()
608 dev = chan->device; in dmatest_func()
609 src = &thread->src; in dmatest_func()
610 dst = &thread->dst; in dmatest_func()
611 if (thread->type == DMA_MEMCPY) { in dmatest_func()
612 align = params->alignment < 0 ? dev->copy_align : in dmatest_func()
613 params->alignment; in dmatest_func()
614 src->cnt = dst->cnt = 1; in dmatest_func()
615 } else if (thread->type == DMA_MEMSET) { in dmatest_func()
616 align = params->alignment < 0 ? dev->fill_align : in dmatest_func()
617 params->alignment; in dmatest_func()
618 src->cnt = dst->cnt = 1; in dmatest_func()
620 } else if (thread->type == DMA_XOR) { in dmatest_func()
622 src->cnt = min_odd(params->xor_sources | 1, dev->max_xor); in dmatest_func()
623 dst->cnt = 1; in dmatest_func()
624 align = params->alignment < 0 ? dev->xor_align : in dmatest_func()
625 params->alignment; in dmatest_func()
626 } else if (thread->type == DMA_PQ) { in dmatest_func()
628 src->cnt = min_odd(params->pq_sources | 1, dma_maxpq(dev, 0)); in dmatest_func()
629 dst->cnt = 2; in dmatest_func()
630 align = params->alignment < 0 ? dev->pq_align : in dmatest_func()
631 params->alignment; in dmatest_func()
633 pq_coefs = kmalloc(params->pq_sources + 1, GFP_KERNEL); in dmatest_func()
637 for (i = 0; i < src->cnt; i++) in dmatest_func()
643 if ((src->cnt + dst->cnt) >= 255) { in dmatest_func()
645 src->cnt + dst->cnt); in dmatest_func()
649 buf_size = params->buf_size; in dmatest_func()
651 pr_err("%u-byte buffer too small for %d-byte alignment\n", in dmatest_func()
664 srcs = kcalloc(src->cnt, sizeof(dma_addr_t), GFP_KERNEL); in dmatest_func()
668 dma_pq = kcalloc(dst->cnt, sizeof(dma_addr_t), GFP_KERNEL); in dmatest_func()
675 if (params->polled) in dmatest_func()
682 (params->iterations && total_tests >= params->iterations))) { in dmatest_func()
690 if (params->transfer_size) { in dmatest_func()
691 if (params->transfer_size >= buf_size) { in dmatest_func()
692 pr_err("%u-byte transfer size must be lower than %u-buffer size\n", in dmatest_func()
693 params->transfer_size, buf_size); in dmatest_func()
696 len = params->transfer_size; in dmatest_func()
697 } else if (params->norandom) { in dmatest_func()
704 if (!params->transfer_size) { in dmatest_func()
711 if (params->norandom) { in dmatest_func()
712 src->off = 0; in dmatest_func()
713 dst->off = 0; in dmatest_func()
715 src->off = dmatest_random() % (buf_size - len + 1); in dmatest_func()
716 dst->off = dmatest_random() % (buf_size - len + 1); in dmatest_func()
718 src->off = (src->off >> align) << align; in dmatest_func()
719 dst->off = (dst->off >> align) << align; in dmatest_func()
722 if (!params->noverify) { in dmatest_func()
724 dmatest_init_srcs(src->aligned, src->off, len, in dmatest_func()
726 dmatest_init_dsts(dst->aligned, dst->off, len, in dmatest_func()
729 diff = ktime_sub(ktime_get(), start); in dmatest_func()
730 filltime = ktime_add(filltime, diff); in dmatest_func()
733 um = dmaengine_get_unmap_data(dev->dev, src->cnt + dst->cnt, in dmatest_func()
738 src->off, dst->off, len, ret); in dmatest_func()
742 um->len = buf_size; in dmatest_func()
743 for (i = 0; i < src->cnt; i++) { in dmatest_func()
744 void *buf = src->aligned[i]; in dmatest_func()
748 um->addr[i] = dma_map_page(dev->dev, pg, pg_off, in dmatest_func()
749 um->len, DMA_TO_DEVICE); in dmatest_func()
750 srcs[i] = um->addr[i] + src->off; in dmatest_func()
751 ret = dma_mapping_error(dev->dev, um->addr[i]); in dmatest_func()
754 src->off, dst->off, len, ret); in dmatest_func()
757 um->to_cnt++; in dmatest_func()
760 dsts = &um->addr[src->cnt]; in dmatest_func()
761 for (i = 0; i < dst->cnt; i++) { in dmatest_func()
762 void *buf = dst->aligned[i]; in dmatest_func()
766 dsts[i] = dma_map_page(dev->dev, pg, pg_off, um->len, in dmatest_func()
768 ret = dma_mapping_error(dev->dev, dsts[i]); in dmatest_func()
771 src->off, dst->off, len, ret); in dmatest_func()
774 um->bidi_cnt++; in dmatest_func()
777 if (thread->type == DMA_MEMCPY) in dmatest_func()
778 tx = dev->device_prep_dma_memcpy(chan, in dmatest_func()
779 dsts[0] + dst->off, in dmatest_func()
781 else if (thread->type == DMA_MEMSET) in dmatest_func()
782 tx = dev->device_prep_dma_memset(chan, in dmatest_func()
783 dsts[0] + dst->off, in dmatest_func()
784 *(src->aligned[0] + src->off), in dmatest_func()
786 else if (thread->type == DMA_XOR) in dmatest_func()
787 tx = dev->device_prep_dma_xor(chan, in dmatest_func()
788 dsts[0] + dst->off, in dmatest_func()
789 srcs, src->cnt, in dmatest_func()
791 else if (thread->type == DMA_PQ) { in dmatest_func()
792 for (i = 0; i < dst->cnt; i++) in dmatest_func()
793 dma_pq[i] = dsts[i] + dst->off; in dmatest_func()
794 tx = dev->device_prep_dma_pq(chan, dma_pq, srcs, in dmatest_func()
795 src->cnt, pq_coefs, in dmatest_func()
800 result("prep error", total_tests, src->off, in dmatest_func()
801 dst->off, len, ret); in dmatest_func()
806 done->done = false; in dmatest_func()
807 if (!params->polled) { in dmatest_func()
808 tx->callback = dmatest_callback; in dmatest_func()
809 tx->callback_param = done; in dmatest_func()
811 cookie = tx->tx_submit(tx); in dmatest_func()
814 result("submit error", total_tests, src->off, in dmatest_func()
815 dst->off, len, ret); in dmatest_func()
820 if (params->polled) { in dmatest_func()
824 done->done = true; in dmatest_func()
828 wait_event_freezable_timeout(thread->done_wait, in dmatest_func()
829 done->done, in dmatest_func()
830 msecs_to_jiffies(params->timeout)); in dmatest_func()
836 if (!done->done) { in dmatest_func()
837 result("test timed out", total_tests, src->off, dst->off, in dmatest_func()
842 dev->cap_mask) && in dmatest_func()
846 "completion busy status", total_tests, src->off, in dmatest_func()
847 dst->off, len, ret); in dmatest_func()
853 if (params->noverify) { in dmatest_func()
854 verbose_result("test passed", total_tests, src->off, in dmatest_func()
855 dst->off, len, 0); in dmatest_func()
860 pr_debug("%s: verifying source buffer...\n", current->comm); in dmatest_func()
861 error_count = dmatest_verify(src->aligned, 0, src->off, in dmatest_func()
863 error_count += dmatest_verify(src->aligned, src->off, in dmatest_func()
864 src->off + len, src->off, in dmatest_func()
866 error_count += dmatest_verify(src->aligned, src->off + len, in dmatest_func()
867 buf_size, src->off + len, in dmatest_func()
870 pr_debug("%s: verifying dest buffer...\n", current->comm); in dmatest_func()
871 error_count += dmatest_verify(dst->aligned, 0, dst->off, in dmatest_func()
874 error_count += dmatest_verify(dst->aligned, dst->off, in dmatest_func()
875 dst->off + len, src->off, in dmatest_func()
878 error_count += dmatest_verify(dst->aligned, dst->off + len, in dmatest_func()
879 buf_size, dst->off + len, in dmatest_func()
882 diff = ktime_sub(ktime_get(), start); in dmatest_func()
883 comparetime = ktime_add(comparetime, diff); in dmatest_func()
886 result("data error", total_tests, src->off, dst->off, in dmatest_func()
890 verbose_result("test passed", total_tests, src->off, in dmatest_func()
891 dst->off, len, 0); in dmatest_func()
918 current->comm, total_tests, failed_tests, in dmatest_func()
922 /* terminate all transfers on specified channels */ in dmatest_func()
926 thread->done = true; in dmatest_func()
938 list_for_each_entry_safe(thread, _thread, &dtc->threads, node) { in dmatest_cleanup_channel()
939 ret = kthread_stop(thread->task); in dmatest_cleanup_channel()
941 thread->task->comm, ret); in dmatest_cleanup_channel()
942 list_del(&thread->node); in dmatest_cleanup_channel()
943 put_task_struct(thread->task); in dmatest_cleanup_channel()
947 /* terminate all transfers on specified channels */ in dmatest_cleanup_channel()
948 dmaengine_terminate_sync(dtc->chan); in dmatest_cleanup_channel()
956 struct dmatest_params *params = &info->params; in dmatest_add_threads()
958 struct dma_chan *chan = dtc->chan; in dmatest_add_threads()
971 return -EINVAL; in dmatest_add_threads()
973 for (i = 0; i < params->threads_per_chan; i++) { in dmatest_add_threads()
976 pr_warn("No memory for %s-%s%u\n", in dmatest_add_threads()
980 thread->info = info; in dmatest_add_threads()
981 thread->chan = dtc->chan; in dmatest_add_threads()
982 thread->type = type; in dmatest_add_threads()
983 thread->test_done.wait = &thread->done_wait; in dmatest_add_threads()
984 init_waitqueue_head(&thread->done_wait); in dmatest_add_threads()
986 thread->task = kthread_create(dmatest_func, thread, "%s-%s%u", in dmatest_add_threads()
988 if (IS_ERR(thread->task)) { in dmatest_add_threads()
989 pr_warn("Failed to create thread %s-%s%u\n", in dmatest_add_threads()
996 get_task_struct(thread->task); in dmatest_add_threads()
997 list_add_tail(&thread->node, &dtc->threads); in dmatest_add_threads()
998 thread->pending = true; in dmatest_add_threads()
1008 struct dma_device *dma_dev = chan->device; in dmatest_add_channel()
1015 return -ENOMEM; in dmatest_add_channel()
1018 dtc->chan = chan; in dmatest_add_channel()
1019 INIT_LIST_HEAD(&dtc->threads); in dmatest_add_channel()
1021 if (dma_has_cap(DMA_COMPLETION_NO_ORDER, dma_dev->cap_mask) && in dmatest_add_channel()
1022 info->params.polled) { in dmatest_add_channel()
1023 info->params.polled = false; in dmatest_add_channel()
1027 if (dma_has_cap(DMA_MEMCPY, dma_dev->cap_mask)) { in dmatest_add_channel()
1034 if (dma_has_cap(DMA_MEMSET, dma_dev->cap_mask)) { in dmatest_add_channel()
1041 if (dma_has_cap(DMA_XOR, dma_dev->cap_mask)) { in dmatest_add_channel()
1045 if (dma_has_cap(DMA_PQ, dma_dev->cap_mask)) { in dmatest_add_channel()
1053 list_add_tail(&dtc->node, &info->channels); in dmatest_add_channel()
1054 info->nr_channels++; in dmatest_add_channel()
1061 return dmatest_match_channel(param, chan) && dmatest_match_device(param, chan->device); in filter()
1072 struct dmatest_params *params = &info->params; in request_channels()
1082 break; /* no more channels available */ in request_channels()
1083 if (params->max_channels && in request_channels()
1084 info->nr_channels >= params->max_channels) in request_channels()
1091 struct dmatest_params *params = &info->params; in add_threaded_test()
1094 params->buf_size = test_buf_size; in add_threaded_test()
1095 strlcpy(params->channel, strim(test_channel), sizeof(params->channel)); in add_threaded_test()
1096 strlcpy(params->device, strim(test_device), sizeof(params->device)); in add_threaded_test()
1097 params->threads_per_chan = threads_per_chan; in add_threaded_test()
1098 params->max_channels = max_channels; in add_threaded_test()
1099 params->iterations = iterations; in add_threaded_test()
1100 params->xor_sources = xor_sources; in add_threaded_test()
1101 params->pq_sources = pq_sources; in add_threaded_test()
1102 params->timeout = timeout; in add_threaded_test()
1103 params->noverify = noverify; in add_threaded_test()
1104 params->norandom = norandom; in add_threaded_test()
1105 params->alignment = alignment; in add_threaded_test()
1106 params->transfer_size = transfer_size; in add_threaded_test()
1107 params->polled = polled; in add_threaded_test()
1120 list_for_each_entry(dtc, &info->channels, node) { in run_pending_tests()
1124 list_for_each_entry(thread, &dtc->threads, node) { in run_pending_tests()
1125 wake_up_process(thread->task); in run_pending_tests()
1129 thread_count, dma_chan_name(dtc->chan)); in run_pending_tests()
1138 list_for_each_entry_safe(dtc, _dtc, &info->channels, node) { in stop_threaded_test()
1139 list_del(&dtc->node); in stop_threaded_test()
1140 chan = dtc->chan; in stop_threaded_test()
1146 info->nr_channels = 0; in stop_threaded_test()
1154 if (!info->did_init) in start_threaded_tests()
1164 mutex_lock(&info->lock); in dmatest_run_get()
1172 mutex_unlock(&info->lock); in dmatest_run_get()
1182 mutex_lock(&info->lock); in dmatest_run_set()
1185 mutex_unlock(&info->lock); in dmatest_run_set()
1192 ret = info->last_error; in dmatest_run_set()
1196 mutex_unlock(&info->lock); in dmatest_run_set()
1200 pr_info("No channels configured, continue with any\n"); in dmatest_run_set()
1211 mutex_unlock(&info->lock); in dmatest_run_set()
1223 mutex_lock(&info->lock); in dmatest_chan_set()
1226 mutex_unlock(&info->lock); in dmatest_chan_set()
1232 /* Reject channels that are already registered */ in dmatest_chan_set()
1234 list_for_each_entry(dtc, &info->channels, node) { in dmatest_chan_set()
1235 if (strcmp(dma_chan_name(dtc->chan), in dmatest_chan_set()
1237 dtc = list_last_entry(&info->channels, in dmatest_chan_set()
1241 dma_chan_name(dtc->chan), in dmatest_chan_set()
1243 ret = -EBUSY; in dmatest_chan_set()
1252 if (!list_empty(&info->channels)) { in dmatest_chan_set()
1259 dtc = list_last_entry(&info->channels, struct dmatest_chan, node); in dmatest_chan_set()
1260 if ((strcmp(dma_chan_name(dtc->chan), strim(test_channel)) != 0) in dmatest_chan_set()
1262 ret = -EINVAL; in dmatest_chan_set()
1263 strlcpy(chan_reset_val, dma_chan_name(dtc->chan), in dmatest_chan_set()
1269 /* Clear test_channel if no channels were added successfully */ in dmatest_chan_set()
1271 ret = -EBUSY; in dmatest_chan_set()
1275 info->last_error = ret; in dmatest_chan_set()
1276 mutex_unlock(&info->lock); in dmatest_chan_set()
1282 info->last_error = ret; in dmatest_chan_set()
1283 mutex_unlock(&info->lock); in dmatest_chan_set()
1292 mutex_lock(&info->lock); in dmatest_chan_get()
1297 mutex_unlock(&info->lock); in dmatest_chan_get()
1308 list_for_each_entry(dtc, &info->channels, node) { in dmatest_test_list_get()
1312 list_for_each_entry(thread, &dtc->threads, node) { in dmatest_test_list_get()
1316 thread_count, dma_chan_name(dtc->chan)); in dmatest_test_list_get()
1325 struct dmatest_params *params = &info->params; in dmatest_init()
1328 mutex_lock(&info->lock); in dmatest_init()
1331 mutex_unlock(&info->lock); in dmatest_init()
1334 if (params->iterations && wait) in dmatest_init()
1340 info->did_init = true; in dmatest_init()
1344 /* when compiled-in wait for drivers to load first */
1351 mutex_lock(&info->lock); in dmatest_exit()
1353 mutex_unlock(&info->lock); in dmatest_exit()