Lines Matching +full:interrupt +full:- +full:affinity
2 * Copyright(c) 2015 - 2020 Intel Corporation.
24 * - Redistributions of source code must retain the above copyright
26 * - Redistributions in binary form must reproduce the above copyright
30 * - Neither the name of Intel Corporation nor the names of its
50 #include <linux/interrupt.h>
54 #include "affinity.h"
77 cpumask_clear(&set->mask); in init_cpu_mask_set()
78 cpumask_clear(&set->used); in init_cpu_mask_set()
79 set->gen = 0; in init_cpu_mask_set()
85 if (cpumask_equal(&set->mask, &set->used)) { in _cpu_mask_set_gen_inc()
90 set->gen++; in _cpu_mask_set_gen_inc()
91 cpumask_clear(&set->used); in _cpu_mask_set_gen_inc()
97 if (cpumask_empty(&set->used) && set->gen) { in _cpu_mask_set_gen_dec()
98 set->gen--; in _cpu_mask_set_gen_dec()
99 cpumask_copy(&set->used, &set->mask); in _cpu_mask_set_gen_dec()
109 return -EINVAL; in cpu_mask_set_get_first()
114 cpumask_andnot(diff, &set->mask, &set->used); in cpu_mask_set_get_first()
118 cpu = -EINVAL; in cpu_mask_set_get_first()
120 cpumask_set_cpu(cpu, &set->used); in cpu_mask_set_get_first()
130 cpumask_clear_cpu(cpu, &set->used); in cpu_mask_set_put()
134 /* Initialize non-HT cpu cores mask */
187 * The real cpu mask is part of the affinity struct but it has to be in node_affinity_init()
196 return -ENOMEM; in node_affinity_init()
198 while (ids->vendor) { in node_affinity_init()
200 while ((dev = pci_get_device(ids->vendor, ids->device, dev))) { in node_affinity_init()
201 node = pcibus_to_node(dev->bus); in node_affinity_init()
227 free_percpu(entry->comp_vect_affinity); in node_affinity_destroy()
254 entry->node = node; in node_affinity_allocate()
255 entry->comp_vect_affinity = alloc_percpu(u16); in node_affinity_allocate()
256 INIT_LIST_HEAD(&entry->list); in node_affinity_allocate()
267 list_add_tail(&entry->list, &node_affinity.list); in node_affinity_add_tail()
278 if (entry->node == node) in node_affinity_lookup()
294 ret_cpu = -EINVAL; in per_cpu_affinity_get()
299 ret_cpu = -EINVAL; in per_cpu_affinity_get()
305 ret_cpu = -EINVAL; in per_cpu_affinity_get()
334 return -EINVAL; in per_cpu_affinity_put_max()
337 return -EINVAL; in per_cpu_affinity_put_max()
341 return -EINVAL; in per_cpu_affinity_put_max()
353 *per_cpu_ptr(comp_vect_affinity, max_cpu) -= 1; in per_cpu_affinity_put_max()
359 * Non-interrupt CPUs are used first, then interrupt CPUs.
369 struct cpu_mask_set *set = dd->comp_vect; in _dev_comp_vect_cpu_get()
373 cpu = -1; in _dev_comp_vect_cpu_get()
378 cpu = -1; in _dev_comp_vect_cpu_get()
384 cpumask_andnot(available_cpus, &set->mask, &set->used); in _dev_comp_vect_cpu_get()
388 &entry->def_intr.used); in _dev_comp_vect_cpu_get()
390 /* If there are non-interrupt CPUs available, use them first */ in _dev_comp_vect_cpu_get()
393 else /* Otherwise, use interrupt CPUs */ in _dev_comp_vect_cpu_get()
397 cpu = -1; in _dev_comp_vect_cpu_get()
400 cpumask_set_cpu(cpu, &set->used); in _dev_comp_vect_cpu_get()
408 struct cpu_mask_set *set = dd->comp_vect; in _dev_comp_vect_cpu_put()
421 if (!dd->comp_vect_mappings) in _dev_comp_vect_mappings_destroy()
424 for (i = 0; i < dd->comp_vect_possible_cpus; i++) { in _dev_comp_vect_mappings_destroy()
425 cpu = dd->comp_vect_mappings[i]; in _dev_comp_vect_mappings_destroy()
427 dd->comp_vect_mappings[i] = -1; in _dev_comp_vect_mappings_destroy()
428 hfi1_cdbg(AFFINITY, in _dev_comp_vect_mappings_destroy()
430 rvt_get_ibdev_name(&(dd)->verbs_dev.rdi), cpu, i); in _dev_comp_vect_mappings_destroy()
433 kfree(dd->comp_vect_mappings); in _dev_comp_vect_mappings_destroy()
434 dd->comp_vect_mappings = NULL; in _dev_comp_vect_mappings_destroy()
452 return -ENOMEM; in _dev_comp_vect_mappings_create()
456 return -ENOMEM; in _dev_comp_vect_mappings_create()
459 dd->comp_vect_mappings = kcalloc(dd->comp_vect_possible_cpus, in _dev_comp_vect_mappings_create()
460 sizeof(*dd->comp_vect_mappings), in _dev_comp_vect_mappings_create()
462 if (!dd->comp_vect_mappings) { in _dev_comp_vect_mappings_create()
463 ret = -ENOMEM; in _dev_comp_vect_mappings_create()
466 for (i = 0; i < dd->comp_vect_possible_cpus; i++) in _dev_comp_vect_mappings_create()
467 dd->comp_vect_mappings[i] = -1; in _dev_comp_vect_mappings_create()
469 for (i = 0; i < dd->comp_vect_possible_cpus; i++) { in _dev_comp_vect_mappings_create()
473 ret = -EINVAL; in _dev_comp_vect_mappings_create()
477 dd->comp_vect_mappings[i] = cpu; in _dev_comp_vect_mappings_create()
478 hfi1_cdbg(AFFINITY, in _dev_comp_vect_mappings_create()
479 "[%s] Completion Vector %d -> CPU %d", in _dev_comp_vect_mappings_create()
480 rvt_get_ibdev_name(&(dd)->verbs_dev.rdi), i, cpu); in _dev_comp_vect_mappings_create()
501 entry = node_affinity_lookup(dd->node); in hfi1_comp_vectors_set_up()
503 ret = -EINVAL; in hfi1_comp_vectors_set_up()
523 if (!dd->comp_vect_mappings) in hfi1_comp_vect_mappings_lookup()
524 return -EINVAL; in hfi1_comp_vect_mappings_lookup()
525 if (comp_vect >= dd->comp_vect_possible_cpus) in hfi1_comp_vect_mappings_lookup()
526 return -EINVAL; in hfi1_comp_vect_mappings_lookup()
528 return dd->comp_vect_mappings[comp_vect]; in hfi1_comp_vect_mappings_lookup()
532 * It assumes dd->comp_vect_possible_cpus is available.
541 struct cpumask *dev_comp_vect_mask = &dd->comp_vect->mask; in _dev_comp_vect_cpu_mask_init()
551 if (cpumask_weight(&entry->comp_vect_mask) == 1) { in _dev_comp_vect_cpu_mask_init()
554 … "Number of kernel receive queues is too large for completion vector affinity to be effective\n"); in _dev_comp_vect_cpu_mask_init()
557 cpumask_weight(&entry->comp_vect_mask) / in _dev_comp_vect_cpu_mask_init()
558 hfi1_per_node_cntr[dd->node]; in _dev_comp_vect_cpu_mask_init()
566 cpumask_weight(&entry->comp_vect_mask) % in _dev_comp_vect_cpu_mask_init()
567 hfi1_per_node_cntr[dd->node] != 0) in _dev_comp_vect_cpu_mask_init()
571 dd->comp_vect_possible_cpus = possible_cpus_comp_vect; in _dev_comp_vect_cpu_mask_init()
574 for (i = 0; i < dd->comp_vect_possible_cpus; i++) { in _dev_comp_vect_cpu_mask_init()
575 curr_cpu = per_cpu_affinity_get(&entry->comp_vect_mask, in _dev_comp_vect_cpu_mask_init()
576 entry->comp_vect_affinity); in _dev_comp_vect_cpu_mask_init()
583 hfi1_cdbg(AFFINITY, in _dev_comp_vect_cpu_mask_init()
584 "[%s] Completion vector affinity CPU set(s) %*pbl", in _dev_comp_vect_cpu_mask_init()
585 rvt_get_ibdev_name(&(dd)->verbs_dev.rdi), in _dev_comp_vect_cpu_mask_init()
592 per_cpu_affinity_put_max(&entry->comp_vect_mask, in _dev_comp_vect_cpu_mask_init()
593 entry->comp_vect_affinity); in _dev_comp_vect_cpu_mask_init()
599 * It assumes dd->comp_vect_possible_cpus is available.
608 if (!dd->comp_vect_possible_cpus) in _dev_comp_vect_cpu_mask_clean_up()
611 for (i = 0; i < dd->comp_vect_possible_cpus; i++) { in _dev_comp_vect_cpu_mask_clean_up()
612 cpu = per_cpu_affinity_put_max(&dd->comp_vect->mask, in _dev_comp_vect_cpu_mask_clean_up()
613 entry->comp_vect_affinity); in _dev_comp_vect_cpu_mask_clean_up()
616 cpumask_clear_cpu(cpu, &dd->comp_vect->mask); in _dev_comp_vect_cpu_mask_clean_up()
619 dd->comp_vect_possible_cpus = 0; in _dev_comp_vect_cpu_mask_clean_up()
623 * Interrupt affinity.
625 * non-rcv avail gets a default mask that
640 local_mask = cpumask_of_node(dd->node); in hfi1_dev_affinity_init()
645 entry = node_affinity_lookup(dd->node); in hfi1_dev_affinity_init()
648 * If this is the first time this NUMA node's affinity is used, in hfi1_dev_affinity_init()
649 * create an entry in the global affinity structure and initialize it. in hfi1_dev_affinity_init()
652 entry = node_affinity_allocate(dd->node); in hfi1_dev_affinity_init()
655 "Unable to allocate global affinity node\n"); in hfi1_dev_affinity_init()
656 ret = -ENOMEM; in hfi1_dev_affinity_init()
661 init_cpu_mask_set(&entry->def_intr); in hfi1_dev_affinity_init()
662 init_cpu_mask_set(&entry->rcv_intr); in hfi1_dev_affinity_init()
663 cpumask_clear(&entry->comp_vect_mask); in hfi1_dev_affinity_init()
664 cpumask_clear(&entry->general_intr_mask); in hfi1_dev_affinity_init()
666 cpumask_and(&entry->def_intr.mask, &node_affinity.real_cpu_mask, in hfi1_dev_affinity_init()
670 possible = cpumask_weight(&entry->def_intr.mask); in hfi1_dev_affinity_init()
671 curr_cpu = cpumask_first(&entry->def_intr.mask); in hfi1_dev_affinity_init()
675 cpumask_set_cpu(curr_cpu, &entry->rcv_intr.mask); in hfi1_dev_affinity_init()
676 cpumask_set_cpu(curr_cpu, &entry->general_intr_mask); in hfi1_dev_affinity_init()
681 * list and added to the general interrupt list. in hfi1_dev_affinity_init()
683 cpumask_clear_cpu(curr_cpu, &entry->def_intr.mask); in hfi1_dev_affinity_init()
684 cpumask_set_cpu(curr_cpu, &entry->general_intr_mask); in hfi1_dev_affinity_init()
686 &entry->def_intr.mask); in hfi1_dev_affinity_init()
693 i < (dd->n_krcv_queues - 1) * in hfi1_dev_affinity_init()
694 hfi1_per_node_cntr[dd->node]; in hfi1_dev_affinity_init()
697 &entry->def_intr.mask); in hfi1_dev_affinity_init()
699 &entry->rcv_intr.mask); in hfi1_dev_affinity_init()
701 &entry->def_intr.mask); in hfi1_dev_affinity_init()
711 if (cpumask_weight(&entry->def_intr.mask) == 0) in hfi1_dev_affinity_init()
712 cpumask_copy(&entry->def_intr.mask, in hfi1_dev_affinity_init()
713 &entry->general_intr_mask); in hfi1_dev_affinity_init()
717 cpumask_and(&entry->comp_vect_mask, in hfi1_dev_affinity_init()
719 cpumask_andnot(&entry->comp_vect_mask, in hfi1_dev_affinity_init()
720 &entry->comp_vect_mask, in hfi1_dev_affinity_init()
721 &entry->rcv_intr.mask); in hfi1_dev_affinity_init()
722 cpumask_andnot(&entry->comp_vect_mask, in hfi1_dev_affinity_init()
723 &entry->comp_vect_mask, in hfi1_dev_affinity_init()
724 &entry->general_intr_mask); in hfi1_dev_affinity_init()
731 if (cpumask_weight(&entry->comp_vect_mask) == 0) in hfi1_dev_affinity_init()
732 cpumask_copy(&entry->comp_vect_mask, in hfi1_dev_affinity_init()
733 &entry->general_intr_mask); in hfi1_dev_affinity_init()
743 dd->affinity_entry = entry; in hfi1_dev_affinity_init()
760 if (!dd->affinity_entry) in hfi1_dev_affinity_clean_up()
762 entry = node_affinity_lookup(dd->node); in hfi1_dev_affinity_clean_up()
772 dd->affinity_entry = NULL; in hfi1_dev_affinity_clean_up()
777 * Function updates the irq affinity hint for msix after it has been changed
783 struct sdma_engine *sde = msix->arg; in hfi1_update_sdma_affinity()
784 struct hfi1_devdata *dd = sde->dd; in hfi1_update_sdma_affinity()
789 if (cpu > num_online_cpus() || cpu == sde->cpu) in hfi1_update_sdma_affinity()
793 entry = node_affinity_lookup(dd->node); in hfi1_update_sdma_affinity()
797 old_cpu = sde->cpu; in hfi1_update_sdma_affinity()
798 sde->cpu = cpu; in hfi1_update_sdma_affinity()
799 cpumask_clear(&msix->mask); in hfi1_update_sdma_affinity()
800 cpumask_set_cpu(cpu, &msix->mask); in hfi1_update_sdma_affinity()
801 dd_dev_dbg(dd, "IRQ: %u, type %s engine %u -> cpu: %d\n", in hfi1_update_sdma_affinity()
802 msix->irq, irq_type_names[msix->type], in hfi1_update_sdma_affinity()
803 sde->this_idx, cpu); in hfi1_update_sdma_affinity()
804 irq_set_affinity_hint(msix->irq, &msix->mask); in hfi1_update_sdma_affinity()
810 set = &entry->def_intr; in hfi1_update_sdma_affinity()
811 cpumask_set_cpu(cpu, &set->mask); in hfi1_update_sdma_affinity()
812 cpumask_set_cpu(cpu, &set->used); in hfi1_update_sdma_affinity()
813 for (i = 0; i < dd->msix_info.max_requested; i++) { in hfi1_update_sdma_affinity()
816 other_msix = &dd->msix_info.msix_entries[i]; in hfi1_update_sdma_affinity()
817 if (other_msix->type != IRQ_SDMA || other_msix == msix) in hfi1_update_sdma_affinity()
820 if (cpumask_test_cpu(old_cpu, &other_msix->mask)) in hfi1_update_sdma_affinity()
823 cpumask_clear_cpu(old_cpu, &set->mask); in hfi1_update_sdma_affinity()
824 cpumask_clear_cpu(old_cpu, &set->used); in hfi1_update_sdma_affinity()
844 * This is required by affinity notifier. We don't have anything to in hfi1_irq_notifier_release()
851 struct irq_affinity_notify *notify = &msix->notify; in hfi1_setup_sdma_notifier()
853 notify->irq = msix->irq; in hfi1_setup_sdma_notifier()
854 notify->notify = hfi1_irq_notifier_notify; in hfi1_setup_sdma_notifier()
855 notify->release = hfi1_irq_notifier_release; in hfi1_setup_sdma_notifier()
857 if (irq_set_affinity_notifier(notify->irq, notify)) in hfi1_setup_sdma_notifier()
858 pr_err("Failed to register sdma irq affinity notifier for irq %d\n", in hfi1_setup_sdma_notifier()
859 notify->irq); in hfi1_setup_sdma_notifier()
864 struct irq_affinity_notify *notify = &msix->notify; in hfi1_cleanup_sdma_notifier()
866 if (irq_set_affinity_notifier(notify->irq, NULL)) in hfi1_cleanup_sdma_notifier()
867 pr_err("Failed to cleanup sdma irq affinity notifier for irq %d\n", in hfi1_cleanup_sdma_notifier()
868 notify->irq); in hfi1_cleanup_sdma_notifier()
872 * Function sets the irq affinity for msix.
884 int cpu = -1; in get_irq_affinity()
887 cpumask_clear(&msix->mask); in get_irq_affinity()
889 entry = node_affinity_lookup(dd->node); in get_irq_affinity()
891 switch (msix->type) { in get_irq_affinity()
893 sde = (struct sdma_engine *)msix->arg; in get_irq_affinity()
894 scnprintf(extra, 64, "engine %u", sde->this_idx); in get_irq_affinity()
895 set = &entry->def_intr; in get_irq_affinity()
898 cpu = cpumask_first(&entry->general_intr_mask); in get_irq_affinity()
901 rcd = (struct hfi1_ctxtdata *)msix->arg; in get_irq_affinity()
902 if (rcd->ctxt == HFI1_CTRL_CTXT) in get_irq_affinity()
903 cpu = cpumask_first(&entry->general_intr_mask); in get_irq_affinity()
905 set = &entry->rcv_intr; in get_irq_affinity()
906 scnprintf(extra, 64, "ctxt %u", rcd->ctxt); in get_irq_affinity()
909 rcd = (struct hfi1_ctxtdata *)msix->arg; in get_irq_affinity()
910 set = &entry->def_intr; in get_irq_affinity()
911 scnprintf(extra, 64, "ctxt %u", rcd->ctxt); in get_irq_affinity()
914 dd_dev_err(dd, "Invalid IRQ type %d\n", msix->type); in get_irq_affinity()
915 return -EINVAL; in get_irq_affinity()
923 if (cpu == -1 && set) { in get_irq_affinity()
925 return -ENOMEM; in get_irq_affinity()
937 cpumask_set_cpu(cpu, &msix->mask); in get_irq_affinity()
938 dd_dev_info(dd, "IRQ: %u, type %s %s -> cpu: %d\n", in get_irq_affinity()
939 msix->irq, irq_type_names[msix->type], in get_irq_affinity()
941 irq_set_affinity_hint(msix->irq, &msix->mask); in get_irq_affinity()
943 if (msix->type == IRQ_SDMA) { in get_irq_affinity()
944 sde->cpu = cpu; in get_irq_affinity()
969 entry = node_affinity_lookup(dd->node); in hfi1_put_irq_affinity()
971 switch (msix->type) { in hfi1_put_irq_affinity()
973 set = &entry->def_intr; in hfi1_put_irq_affinity()
980 rcd = (struct hfi1_ctxtdata *)msix->arg; in hfi1_put_irq_affinity()
982 if (rcd->ctxt != HFI1_CTRL_CTXT) in hfi1_put_irq_affinity()
983 set = &entry->rcv_intr; in hfi1_put_irq_affinity()
986 rcd = (struct hfi1_ctxtdata *)msix->arg; in hfi1_put_irq_affinity()
987 set = &entry->def_intr; in hfi1_put_irq_affinity()
995 cpumask_andnot(&set->used, &set->used, &msix->mask); in hfi1_put_irq_affinity()
999 irq_set_affinity_hint(msix->irq, NULL); in hfi1_put_irq_affinity()
1000 cpumask_clear(&msix->mask); in hfi1_put_irq_affinity()
1006 struct hfi1_affinity_node_list *affinity) in find_hw_thread_mask() argument
1010 affinity->num_core_siblings / in find_hw_thread_mask()
1013 cpumask_copy(hw_thread_mask, &affinity->proc.mask); in find_hw_thread_mask()
1014 if (affinity->num_core_siblings > 0) { in find_hw_thread_mask()
1038 int cpu = -1, ret, i; in hfi1_get_proc_affinity()
1042 *proc_mask = current->cpus_ptr; in hfi1_get_proc_affinity()
1043 struct hfi1_affinity_node_list *affinity = &node_affinity; in hfi1_get_proc_affinity() local
1044 struct cpu_mask_set *set = &affinity->proc; in hfi1_get_proc_affinity()
1047 * check whether process/context affinity has already in hfi1_get_proc_affinity()
1050 if (current->nr_cpus_allowed == 1) { in hfi1_get_proc_affinity()
1051 hfi1_cdbg(PROC, "PID %u %s affinity set to CPU %*pbl", in hfi1_get_proc_affinity()
1052 current->pid, current->comm, in hfi1_get_proc_affinity()
1055 * Mark the pre-set CPU as used. This is atomic so we don't in hfi1_get_proc_affinity()
1059 cpumask_set_cpu(cpu, &set->used); in hfi1_get_proc_affinity()
1061 } else if (current->nr_cpus_allowed < cpumask_weight(&set->mask)) { in hfi1_get_proc_affinity()
1062 hfi1_cdbg(PROC, "PID %u %s affinity set to CPU set(s) %*pbl", in hfi1_get_proc_affinity()
1063 current->pid, current->comm, in hfi1_get_proc_affinity()
1069 * The process does not have a preset CPU affinity so find one to in hfi1_get_proc_affinity()
1102 mutex_lock(&affinity->lock); in hfi1_get_proc_affinity()
1110 * If NUMA node has CPUs used by interrupt handlers, include them in the in hfi1_get_proc_affinity()
1111 * interrupt handler mask. in hfi1_get_proc_affinity()
1115 cpumask_copy(intrs_mask, (entry->def_intr.gen ? in hfi1_get_proc_affinity()
1116 &entry->def_intr.mask : in hfi1_get_proc_affinity()
1117 &entry->def_intr.used)); in hfi1_get_proc_affinity()
1118 cpumask_or(intrs_mask, intrs_mask, (entry->rcv_intr.gen ? in hfi1_get_proc_affinity()
1119 &entry->rcv_intr.mask : in hfi1_get_proc_affinity()
1120 &entry->rcv_intr.used)); in hfi1_get_proc_affinity()
1121 cpumask_or(intrs_mask, intrs_mask, &entry->general_intr_mask); in hfi1_get_proc_affinity()
1126 cpumask_copy(hw_thread_mask, &set->mask); in hfi1_get_proc_affinity()
1132 if (affinity->num_core_siblings > 0) { in hfi1_get_proc_affinity()
1133 for (i = 0; i < affinity->num_core_siblings; i++) { in hfi1_get_proc_affinity()
1134 find_hw_thread_mask(i, hw_thread_mask, affinity); in hfi1_get_proc_affinity()
1142 * (set->mask == set->used) before this loop. in hfi1_get_proc_affinity()
1144 cpumask_andnot(diff, hw_thread_mask, &set->used); in hfi1_get_proc_affinity()
1158 cpumask_andnot(available_mask, available_mask, &set->used); in hfi1_get_proc_affinity()
1164 * CPUs as interrupt handlers. Then, CPUs running interrupt in hfi1_get_proc_affinity()
1168 * non-interrupt handlers available, so diff gets copied in hfi1_get_proc_affinity()
1170 * 2) If diff is empty, then all CPUs not running interrupt in hfi1_get_proc_affinity()
1172 * available CPUs running interrupt handlers. in hfi1_get_proc_affinity()
1184 cpumask_andnot(available_mask, hw_thread_mask, &set->used); in hfi1_get_proc_affinity()
1193 * CPUs as interrupt handlers. in hfi1_get_proc_affinity()
1204 cpu = -1; in hfi1_get_proc_affinity()
1206 cpumask_set_cpu(cpu, &set->used); in hfi1_get_proc_affinity()
1208 mutex_unlock(&affinity->lock); in hfi1_get_proc_affinity()
1224 struct hfi1_affinity_node_list *affinity = &node_affinity; in hfi1_put_proc_affinity() local
1225 struct cpu_mask_set *set = &affinity->proc; in hfi1_put_proc_affinity()
1230 mutex_lock(&affinity->lock); in hfi1_put_proc_affinity()
1233 mutex_unlock(&affinity->lock); in hfi1_put_proc_affinity()