Lines Matching +full:interrupt +full:- +full:affinity
2 * Copyright(c) 2015 - 2020 Intel Corporation.
24 * - Redistributions of source code must retain the above copyright
26 * - Redistributions in binary form must reproduce the above copyright
30 * - Neither the name of Intel Corporation nor the names of its
50 #include <linux/interrupt.h>
54 #include "affinity.h"
77 cpumask_clear(&set->mask); in init_cpu_mask_set()
78 cpumask_clear(&set->used); in init_cpu_mask_set()
79 set->gen = 0; in init_cpu_mask_set()
85 if (cpumask_equal(&set->mask, &set->used)) { in _cpu_mask_set_gen_inc()
90 set->gen++; in _cpu_mask_set_gen_inc()
91 cpumask_clear(&set->used); in _cpu_mask_set_gen_inc()
97 if (cpumask_empty(&set->used) && set->gen) { in _cpu_mask_set_gen_dec()
98 set->gen--; in _cpu_mask_set_gen_dec()
99 cpumask_copy(&set->used, &set->mask); in _cpu_mask_set_gen_dec()
109 return -EINVAL; in cpu_mask_set_get_first()
114 cpumask_andnot(diff, &set->mask, &set->used); in cpu_mask_set_get_first()
118 cpu = -EINVAL; in cpu_mask_set_get_first()
120 cpumask_set_cpu(cpu, &set->used); in cpu_mask_set_get_first()
130 cpumask_clear_cpu(cpu, &set->used); in cpu_mask_set_put()
134 /* Initialize non-HT cpu cores mask */
187 * The real cpu mask is part of the affinity struct but it has to be in node_affinity_init()
196 return -ENOMEM; in node_affinity_init()
198 while (ids->vendor) { in node_affinity_init()
200 while ((dev = pci_get_device(ids->vendor, ids->device, dev))) { in node_affinity_init()
201 node = pcibus_to_node(dev->bus); in node_affinity_init()
229 free_percpu(entry->comp_vect_affinity); in node_affinity_destroy()
256 entry->node = node; in node_affinity_allocate()
257 entry->comp_vect_affinity = alloc_percpu(u16); in node_affinity_allocate()
258 INIT_LIST_HEAD(&entry->list); in node_affinity_allocate()
269 list_add_tail(&entry->list, &node_affinity.list); in node_affinity_add_tail()
280 if (entry->node == node) in node_affinity_lookup()
296 ret_cpu = -EINVAL; in per_cpu_affinity_get()
301 ret_cpu = -EINVAL; in per_cpu_affinity_get()
307 ret_cpu = -EINVAL; in per_cpu_affinity_get()
336 return -EINVAL; in per_cpu_affinity_put_max()
339 return -EINVAL; in per_cpu_affinity_put_max()
343 return -EINVAL; in per_cpu_affinity_put_max()
355 *per_cpu_ptr(comp_vect_affinity, max_cpu) -= 1; in per_cpu_affinity_put_max()
361 * Non-interrupt CPUs are used first, then interrupt CPUs.
371 struct cpu_mask_set *set = dd->comp_vect; in _dev_comp_vect_cpu_get()
375 cpu = -1; in _dev_comp_vect_cpu_get()
380 cpu = -1; in _dev_comp_vect_cpu_get()
386 cpumask_andnot(available_cpus, &set->mask, &set->used); in _dev_comp_vect_cpu_get()
390 &entry->def_intr.used); in _dev_comp_vect_cpu_get()
392 /* If there are non-interrupt CPUs available, use them first */ in _dev_comp_vect_cpu_get()
395 else /* Otherwise, use interrupt CPUs */ in _dev_comp_vect_cpu_get()
399 cpu = -1; in _dev_comp_vect_cpu_get()
402 cpumask_set_cpu(cpu, &set->used); in _dev_comp_vect_cpu_get()
410 struct cpu_mask_set *set = dd->comp_vect; in _dev_comp_vect_cpu_put()
423 if (!dd->comp_vect_mappings) in _dev_comp_vect_mappings_destroy()
426 for (i = 0; i < dd->comp_vect_possible_cpus; i++) { in _dev_comp_vect_mappings_destroy()
427 cpu = dd->comp_vect_mappings[i]; in _dev_comp_vect_mappings_destroy()
429 dd->comp_vect_mappings[i] = -1; in _dev_comp_vect_mappings_destroy()
430 hfi1_cdbg(AFFINITY, in _dev_comp_vect_mappings_destroy()
432 rvt_get_ibdev_name(&(dd)->verbs_dev.rdi), cpu, i); in _dev_comp_vect_mappings_destroy()
435 kfree(dd->comp_vect_mappings); in _dev_comp_vect_mappings_destroy()
436 dd->comp_vect_mappings = NULL; in _dev_comp_vect_mappings_destroy()
454 return -ENOMEM; in _dev_comp_vect_mappings_create()
458 return -ENOMEM; in _dev_comp_vect_mappings_create()
461 dd->comp_vect_mappings = kcalloc(dd->comp_vect_possible_cpus, in _dev_comp_vect_mappings_create()
462 sizeof(*dd->comp_vect_mappings), in _dev_comp_vect_mappings_create()
464 if (!dd->comp_vect_mappings) { in _dev_comp_vect_mappings_create()
465 ret = -ENOMEM; in _dev_comp_vect_mappings_create()
468 for (i = 0; i < dd->comp_vect_possible_cpus; i++) in _dev_comp_vect_mappings_create()
469 dd->comp_vect_mappings[i] = -1; in _dev_comp_vect_mappings_create()
471 for (i = 0; i < dd->comp_vect_possible_cpus; i++) { in _dev_comp_vect_mappings_create()
475 ret = -EINVAL; in _dev_comp_vect_mappings_create()
479 dd->comp_vect_mappings[i] = cpu; in _dev_comp_vect_mappings_create()
480 hfi1_cdbg(AFFINITY, in _dev_comp_vect_mappings_create()
481 "[%s] Completion Vector %d -> CPU %d", in _dev_comp_vect_mappings_create()
482 rvt_get_ibdev_name(&(dd)->verbs_dev.rdi), i, cpu); in _dev_comp_vect_mappings_create()
503 entry = node_affinity_lookup(dd->node); in hfi1_comp_vectors_set_up()
505 ret = -EINVAL; in hfi1_comp_vectors_set_up()
525 if (!dd->comp_vect_mappings) in hfi1_comp_vect_mappings_lookup()
526 return -EINVAL; in hfi1_comp_vect_mappings_lookup()
527 if (comp_vect >= dd->comp_vect_possible_cpus) in hfi1_comp_vect_mappings_lookup()
528 return -EINVAL; in hfi1_comp_vect_mappings_lookup()
530 return dd->comp_vect_mappings[comp_vect]; in hfi1_comp_vect_mappings_lookup()
534 * It assumes dd->comp_vect_possible_cpus is available.
543 struct cpumask *dev_comp_vect_mask = &dd->comp_vect->mask; in _dev_comp_vect_cpu_mask_init()
553 if (cpumask_weight(&entry->comp_vect_mask) == 1) { in _dev_comp_vect_cpu_mask_init()
556 … "Number of kernel receive queues is too large for completion vector affinity to be effective\n"); in _dev_comp_vect_cpu_mask_init()
559 cpumask_weight(&entry->comp_vect_mask) / in _dev_comp_vect_cpu_mask_init()
560 hfi1_per_node_cntr[dd->node]; in _dev_comp_vect_cpu_mask_init()
568 cpumask_weight(&entry->comp_vect_mask) % in _dev_comp_vect_cpu_mask_init()
569 hfi1_per_node_cntr[dd->node] != 0) in _dev_comp_vect_cpu_mask_init()
573 dd->comp_vect_possible_cpus = possible_cpus_comp_vect; in _dev_comp_vect_cpu_mask_init()
576 for (i = 0; i < dd->comp_vect_possible_cpus; i++) { in _dev_comp_vect_cpu_mask_init()
577 curr_cpu = per_cpu_affinity_get(&entry->comp_vect_mask, in _dev_comp_vect_cpu_mask_init()
578 entry->comp_vect_affinity); in _dev_comp_vect_cpu_mask_init()
585 hfi1_cdbg(AFFINITY, in _dev_comp_vect_cpu_mask_init()
586 "[%s] Completion vector affinity CPU set(s) %*pbl", in _dev_comp_vect_cpu_mask_init()
587 rvt_get_ibdev_name(&(dd)->verbs_dev.rdi), in _dev_comp_vect_cpu_mask_init()
594 per_cpu_affinity_put_max(&entry->comp_vect_mask, in _dev_comp_vect_cpu_mask_init()
595 entry->comp_vect_affinity); in _dev_comp_vect_cpu_mask_init()
601 * It assumes dd->comp_vect_possible_cpus is available.
610 if (!dd->comp_vect_possible_cpus) in _dev_comp_vect_cpu_mask_clean_up()
613 for (i = 0; i < dd->comp_vect_possible_cpus; i++) { in _dev_comp_vect_cpu_mask_clean_up()
614 cpu = per_cpu_affinity_put_max(&dd->comp_vect->mask, in _dev_comp_vect_cpu_mask_clean_up()
615 entry->comp_vect_affinity); in _dev_comp_vect_cpu_mask_clean_up()
618 cpumask_clear_cpu(cpu, &dd->comp_vect->mask); in _dev_comp_vect_cpu_mask_clean_up()
621 dd->comp_vect_possible_cpus = 0; in _dev_comp_vect_cpu_mask_clean_up()
625 * Interrupt affinity.
627 * non-rcv avail gets a default mask that
642 local_mask = cpumask_of_node(dd->node); in hfi1_dev_affinity_init()
647 entry = node_affinity_lookup(dd->node); in hfi1_dev_affinity_init()
650 * If this is the first time this NUMA node's affinity is used, in hfi1_dev_affinity_init()
651 * create an entry in the global affinity structure and initialize it. in hfi1_dev_affinity_init()
654 entry = node_affinity_allocate(dd->node); in hfi1_dev_affinity_init()
657 "Unable to allocate global affinity node\n"); in hfi1_dev_affinity_init()
658 ret = -ENOMEM; in hfi1_dev_affinity_init()
663 init_cpu_mask_set(&entry->def_intr); in hfi1_dev_affinity_init()
664 init_cpu_mask_set(&entry->rcv_intr); in hfi1_dev_affinity_init()
665 cpumask_clear(&entry->comp_vect_mask); in hfi1_dev_affinity_init()
666 cpumask_clear(&entry->general_intr_mask); in hfi1_dev_affinity_init()
668 cpumask_and(&entry->def_intr.mask, &node_affinity.real_cpu_mask, in hfi1_dev_affinity_init()
672 possible = cpumask_weight(&entry->def_intr.mask); in hfi1_dev_affinity_init()
673 curr_cpu = cpumask_first(&entry->def_intr.mask); in hfi1_dev_affinity_init()
677 cpumask_set_cpu(curr_cpu, &entry->rcv_intr.mask); in hfi1_dev_affinity_init()
678 cpumask_set_cpu(curr_cpu, &entry->general_intr_mask); in hfi1_dev_affinity_init()
683 * list and added to the general interrupt list. in hfi1_dev_affinity_init()
685 cpumask_clear_cpu(curr_cpu, &entry->def_intr.mask); in hfi1_dev_affinity_init()
686 cpumask_set_cpu(curr_cpu, &entry->general_intr_mask); in hfi1_dev_affinity_init()
688 &entry->def_intr.mask); in hfi1_dev_affinity_init()
695 i < (dd->n_krcv_queues - 1) * in hfi1_dev_affinity_init()
696 hfi1_per_node_cntr[dd->node]; in hfi1_dev_affinity_init()
699 &entry->def_intr.mask); in hfi1_dev_affinity_init()
701 &entry->rcv_intr.mask); in hfi1_dev_affinity_init()
703 &entry->def_intr.mask); in hfi1_dev_affinity_init()
713 if (cpumask_weight(&entry->def_intr.mask) == 0) in hfi1_dev_affinity_init()
714 cpumask_copy(&entry->def_intr.mask, in hfi1_dev_affinity_init()
715 &entry->general_intr_mask); in hfi1_dev_affinity_init()
719 cpumask_and(&entry->comp_vect_mask, in hfi1_dev_affinity_init()
721 cpumask_andnot(&entry->comp_vect_mask, in hfi1_dev_affinity_init()
722 &entry->comp_vect_mask, in hfi1_dev_affinity_init()
723 &entry->rcv_intr.mask); in hfi1_dev_affinity_init()
724 cpumask_andnot(&entry->comp_vect_mask, in hfi1_dev_affinity_init()
725 &entry->comp_vect_mask, in hfi1_dev_affinity_init()
726 &entry->general_intr_mask); in hfi1_dev_affinity_init()
733 if (cpumask_weight(&entry->comp_vect_mask) == 0) in hfi1_dev_affinity_init()
734 cpumask_copy(&entry->comp_vect_mask, in hfi1_dev_affinity_init()
735 &entry->general_intr_mask); in hfi1_dev_affinity_init()
745 dd->affinity_entry = entry; in hfi1_dev_affinity_init()
762 if (!dd->affinity_entry) in hfi1_dev_affinity_clean_up()
764 entry = node_affinity_lookup(dd->node); in hfi1_dev_affinity_clean_up()
774 dd->affinity_entry = NULL; in hfi1_dev_affinity_clean_up()
779 * Function updates the irq affinity hint for msix after it has been changed
785 struct sdma_engine *sde = msix->arg; in hfi1_update_sdma_affinity()
786 struct hfi1_devdata *dd = sde->dd; in hfi1_update_sdma_affinity()
791 if (cpu > num_online_cpus() || cpu == sde->cpu) in hfi1_update_sdma_affinity()
795 entry = node_affinity_lookup(dd->node); in hfi1_update_sdma_affinity()
799 old_cpu = sde->cpu; in hfi1_update_sdma_affinity()
800 sde->cpu = cpu; in hfi1_update_sdma_affinity()
801 cpumask_clear(&msix->mask); in hfi1_update_sdma_affinity()
802 cpumask_set_cpu(cpu, &msix->mask); in hfi1_update_sdma_affinity()
803 dd_dev_dbg(dd, "IRQ: %u, type %s engine %u -> cpu: %d\n", in hfi1_update_sdma_affinity()
804 msix->irq, irq_type_names[msix->type], in hfi1_update_sdma_affinity()
805 sde->this_idx, cpu); in hfi1_update_sdma_affinity()
806 irq_set_affinity_hint(msix->irq, &msix->mask); in hfi1_update_sdma_affinity()
812 set = &entry->def_intr; in hfi1_update_sdma_affinity()
813 cpumask_set_cpu(cpu, &set->mask); in hfi1_update_sdma_affinity()
814 cpumask_set_cpu(cpu, &set->used); in hfi1_update_sdma_affinity()
815 for (i = 0; i < dd->msix_info.max_requested; i++) { in hfi1_update_sdma_affinity()
818 other_msix = &dd->msix_info.msix_entries[i]; in hfi1_update_sdma_affinity()
819 if (other_msix->type != IRQ_SDMA || other_msix == msix) in hfi1_update_sdma_affinity()
822 if (cpumask_test_cpu(old_cpu, &other_msix->mask)) in hfi1_update_sdma_affinity()
825 cpumask_clear_cpu(old_cpu, &set->mask); in hfi1_update_sdma_affinity()
826 cpumask_clear_cpu(old_cpu, &set->used); in hfi1_update_sdma_affinity()
846 * This is required by affinity notifier. We don't have anything to in hfi1_irq_notifier_release()
853 struct irq_affinity_notify *notify = &msix->notify; in hfi1_setup_sdma_notifier()
855 notify->irq = msix->irq; in hfi1_setup_sdma_notifier()
856 notify->notify = hfi1_irq_notifier_notify; in hfi1_setup_sdma_notifier()
857 notify->release = hfi1_irq_notifier_release; in hfi1_setup_sdma_notifier()
859 if (irq_set_affinity_notifier(notify->irq, notify)) in hfi1_setup_sdma_notifier()
860 pr_err("Failed to register sdma irq affinity notifier for irq %d\n", in hfi1_setup_sdma_notifier()
861 notify->irq); in hfi1_setup_sdma_notifier()
866 struct irq_affinity_notify *notify = &msix->notify; in hfi1_cleanup_sdma_notifier()
868 if (irq_set_affinity_notifier(notify->irq, NULL)) in hfi1_cleanup_sdma_notifier()
869 pr_err("Failed to cleanup sdma irq affinity notifier for irq %d\n", in hfi1_cleanup_sdma_notifier()
870 notify->irq); in hfi1_cleanup_sdma_notifier()
874 * Function sets the irq affinity for msix.
886 int cpu = -1; in get_irq_affinity()
889 cpumask_clear(&msix->mask); in get_irq_affinity()
891 entry = node_affinity_lookup(dd->node); in get_irq_affinity()
893 switch (msix->type) { in get_irq_affinity()
895 sde = (struct sdma_engine *)msix->arg; in get_irq_affinity()
896 scnprintf(extra, 64, "engine %u", sde->this_idx); in get_irq_affinity()
897 set = &entry->def_intr; in get_irq_affinity()
900 cpu = cpumask_first(&entry->general_intr_mask); in get_irq_affinity()
903 rcd = (struct hfi1_ctxtdata *)msix->arg; in get_irq_affinity()
904 if (rcd->ctxt == HFI1_CTRL_CTXT) in get_irq_affinity()
905 cpu = cpumask_first(&entry->general_intr_mask); in get_irq_affinity()
907 set = &entry->rcv_intr; in get_irq_affinity()
908 scnprintf(extra, 64, "ctxt %u", rcd->ctxt); in get_irq_affinity()
911 rcd = (struct hfi1_ctxtdata *)msix->arg; in get_irq_affinity()
912 set = &entry->def_intr; in get_irq_affinity()
913 scnprintf(extra, 64, "ctxt %u", rcd->ctxt); in get_irq_affinity()
916 dd_dev_err(dd, "Invalid IRQ type %d\n", msix->type); in get_irq_affinity()
917 return -EINVAL; in get_irq_affinity()
925 if (cpu == -1 && set) { in get_irq_affinity()
927 return -ENOMEM; in get_irq_affinity()
939 cpumask_set_cpu(cpu, &msix->mask); in get_irq_affinity()
940 dd_dev_info(dd, "IRQ: %u, type %s %s -> cpu: %d\n", in get_irq_affinity()
941 msix->irq, irq_type_names[msix->type], in get_irq_affinity()
943 irq_set_affinity_hint(msix->irq, &msix->mask); in get_irq_affinity()
945 if (msix->type == IRQ_SDMA) { in get_irq_affinity()
946 sde->cpu = cpu; in get_irq_affinity()
971 entry = node_affinity_lookup(dd->node); in hfi1_put_irq_affinity()
973 switch (msix->type) { in hfi1_put_irq_affinity()
975 set = &entry->def_intr; in hfi1_put_irq_affinity()
982 rcd = (struct hfi1_ctxtdata *)msix->arg; in hfi1_put_irq_affinity()
984 if (rcd->ctxt != HFI1_CTRL_CTXT) in hfi1_put_irq_affinity()
985 set = &entry->rcv_intr; in hfi1_put_irq_affinity()
988 rcd = (struct hfi1_ctxtdata *)msix->arg; in hfi1_put_irq_affinity()
989 set = &entry->def_intr; in hfi1_put_irq_affinity()
997 cpumask_andnot(&set->used, &set->used, &msix->mask); in hfi1_put_irq_affinity()
1001 irq_set_affinity_hint(msix->irq, NULL); in hfi1_put_irq_affinity()
1002 cpumask_clear(&msix->mask); in hfi1_put_irq_affinity()
1008 struct hfi1_affinity_node_list *affinity) in find_hw_thread_mask() argument
1012 affinity->num_core_siblings / in find_hw_thread_mask()
1015 cpumask_copy(hw_thread_mask, &affinity->proc.mask); in find_hw_thread_mask()
1016 if (affinity->num_core_siblings > 0) { in find_hw_thread_mask()
1040 int cpu = -1, ret, i; in hfi1_get_proc_affinity()
1044 *proc_mask = current->cpus_ptr; in hfi1_get_proc_affinity()
1045 struct hfi1_affinity_node_list *affinity = &node_affinity; in hfi1_get_proc_affinity() local
1046 struct cpu_mask_set *set = &affinity->proc; in hfi1_get_proc_affinity()
1049 * check whether process/context affinity has already in hfi1_get_proc_affinity()
1052 if (current->nr_cpus_allowed == 1) { in hfi1_get_proc_affinity()
1053 hfi1_cdbg(PROC, "PID %u %s affinity set to CPU %*pbl", in hfi1_get_proc_affinity()
1054 current->pid, current->comm, in hfi1_get_proc_affinity()
1057 * Mark the pre-set CPU as used. This is atomic so we don't in hfi1_get_proc_affinity()
1061 cpumask_set_cpu(cpu, &set->used); in hfi1_get_proc_affinity()
1063 } else if (current->nr_cpus_allowed < cpumask_weight(&set->mask)) { in hfi1_get_proc_affinity()
1064 hfi1_cdbg(PROC, "PID %u %s affinity set to CPU set(s) %*pbl", in hfi1_get_proc_affinity()
1065 current->pid, current->comm, in hfi1_get_proc_affinity()
1071 * The process does not have a preset CPU affinity so find one to in hfi1_get_proc_affinity()
1104 mutex_lock(&affinity->lock); in hfi1_get_proc_affinity()
1112 * If NUMA node has CPUs used by interrupt handlers, include them in the in hfi1_get_proc_affinity()
1113 * interrupt handler mask. in hfi1_get_proc_affinity()
1117 cpumask_copy(intrs_mask, (entry->def_intr.gen ? in hfi1_get_proc_affinity()
1118 &entry->def_intr.mask : in hfi1_get_proc_affinity()
1119 &entry->def_intr.used)); in hfi1_get_proc_affinity()
1120 cpumask_or(intrs_mask, intrs_mask, (entry->rcv_intr.gen ? in hfi1_get_proc_affinity()
1121 &entry->rcv_intr.mask : in hfi1_get_proc_affinity()
1122 &entry->rcv_intr.used)); in hfi1_get_proc_affinity()
1123 cpumask_or(intrs_mask, intrs_mask, &entry->general_intr_mask); in hfi1_get_proc_affinity()
1128 cpumask_copy(hw_thread_mask, &set->mask); in hfi1_get_proc_affinity()
1134 if (affinity->num_core_siblings > 0) { in hfi1_get_proc_affinity()
1135 for (i = 0; i < affinity->num_core_siblings; i++) { in hfi1_get_proc_affinity()
1136 find_hw_thread_mask(i, hw_thread_mask, affinity); in hfi1_get_proc_affinity()
1144 * (set->mask == set->used) before this loop. in hfi1_get_proc_affinity()
1146 cpumask_andnot(diff, hw_thread_mask, &set->used); in hfi1_get_proc_affinity()
1160 cpumask_andnot(available_mask, available_mask, &set->used); in hfi1_get_proc_affinity()
1166 * CPUs as interrupt handlers. Then, CPUs running interrupt in hfi1_get_proc_affinity()
1170 * non-interrupt handlers available, so diff gets copied in hfi1_get_proc_affinity()
1172 * 2) If diff is empty, then all CPUs not running interrupt in hfi1_get_proc_affinity()
1174 * available CPUs running interrupt handlers. in hfi1_get_proc_affinity()
1186 cpumask_andnot(available_mask, hw_thread_mask, &set->used); in hfi1_get_proc_affinity()
1195 * CPUs as interrupt handlers. in hfi1_get_proc_affinity()
1206 cpu = -1; in hfi1_get_proc_affinity()
1208 cpumask_set_cpu(cpu, &set->used); in hfi1_get_proc_affinity()
1210 mutex_unlock(&affinity->lock); in hfi1_get_proc_affinity()
1226 struct hfi1_affinity_node_list *affinity = &node_affinity; in hfi1_put_proc_affinity() local
1227 struct cpu_mask_set *set = &affinity->proc; in hfi1_put_proc_affinity()
1232 mutex_lock(&affinity->lock); in hfi1_put_proc_affinity()
1235 mutex_unlock(&affinity->lock); in hfi1_put_proc_affinity()