Lines Matching refs:gts
181 struct gru_thread_state *gts) in reserve_gru_resources() argument
184 gts->ts_cbr_map = in reserve_gru_resources()
185 gru_reserve_cb_resources(gru, gts->ts_cbr_au_count, in reserve_gru_resources()
186 gts->ts_cbr_idx); in reserve_gru_resources()
187 gts->ts_dsr_map = in reserve_gru_resources()
188 gru_reserve_ds_resources(gru, gts->ts_dsr_au_count, NULL); in reserve_gru_resources()
192 struct gru_thread_state *gts) in free_gru_resources() argument
195 gru->gs_cbr_map |= gts->ts_cbr_map; in free_gru_resources()
196 gru->gs_dsr_map |= gts->ts_dsr_map; in free_gru_resources()
269 void gts_drop(struct gru_thread_state *gts) in gts_drop() argument
271 if (gts && atomic_dec_return(>s->ts_refcnt) == 0) { in gts_drop()
272 gru_drop_mmu_notifier(gts->ts_gms); in gts_drop()
273 kfree(gts); in gts_drop()
284 struct gru_thread_state *gts; in gru_find_current_gts_nolock() local
286 list_for_each_entry(gts, &vdata->vd_head, ts_next) in gru_find_current_gts_nolock()
287 if (gts->ts_tsid == tsid) in gru_find_current_gts_nolock()
288 return gts; in gru_find_current_gts_nolock()
299 struct gru_thread_state *gts; in gru_alloc_gts() local
305 gts = kzalloc(bytes, GFP_KERNEL); in gru_alloc_gts()
306 if (!gts) in gru_alloc_gts()
310 atomic_set(>s->ts_refcnt, 1); in gru_alloc_gts()
311 mutex_init(>s->ts_ctxlock); in gru_alloc_gts()
312 gts->ts_cbr_au_count = vdata->vd_cbr_au_count; in gru_alloc_gts()
313 gts->ts_dsr_au_count = vdata->vd_dsr_au_count; in gru_alloc_gts()
314 gts->ts_user_options = vdata->vd_user_options; in gru_alloc_gts()
315 gts->ts_tsid = tsid; in gru_alloc_gts()
316 gts->ts_user_options = vdata->vd_user_options; in gru_alloc_gts()
317 gts->ts_ctxnum = NULLCTX; in gru_alloc_gts()
318 gts->ts_mm = current->mm; in gru_alloc_gts()
319 gts->ts_vma = vma; in gru_alloc_gts()
320 gts->ts_tlb_int_select = -1; in gru_alloc_gts()
321 gts->ts_gms = gru_register_mmu_notifier(); in gru_alloc_gts()
322 if (!gts->ts_gms) in gru_alloc_gts()
325 gru_dbg(grudev, "alloc vdata %p, new gts %p\n", vdata, gts); in gru_alloc_gts()
326 return gts; in gru_alloc_gts()
329 gts_drop(gts); in gru_alloc_gts()
357 struct gru_thread_state *gts; in gru_find_thread_state() local
360 gts = gru_find_current_gts_nolock(vdata, tsid); in gru_find_thread_state()
362 gru_dbg(grudev, "vma %p, gts %p\n", vma, gts); in gru_find_thread_state()
363 return gts; in gru_find_thread_state()
374 struct gru_thread_state *gts, *ngts; in gru_alloc_thread_state() local
376 gts = gru_alloc_gts(vma, vdata, tsid); in gru_alloc_thread_state()
377 if (!gts) in gru_alloc_thread_state()
383 gts_drop(gts); in gru_alloc_thread_state()
384 gts = ngts; in gru_alloc_thread_state()
387 list_add(>s->ts_next, &vdata->vd_head); in gru_alloc_thread_state()
390 gru_dbg(grudev, "vma %p, gts %p\n", vma, gts); in gru_alloc_thread_state()
391 return gts; in gru_alloc_thread_state()
397 static void gru_free_gru_context(struct gru_thread_state *gts) in gru_free_gru_context() argument
401 gru = gts->ts_gru; in gru_free_gru_context()
402 gru_dbg(grudev, "gts %p, gru %p\n", gts, gru); in gru_free_gru_context()
405 gru->gs_gts[gts->ts_ctxnum] = NULL; in gru_free_gru_context()
406 free_gru_resources(gru, gts); in gru_free_gru_context()
407 BUG_ON(test_bit(gts->ts_ctxnum, &gru->gs_context_map) == 0); in gru_free_gru_context()
408 __clear_bit(gts->ts_ctxnum, &gru->gs_context_map); in gru_free_gru_context()
409 gts->ts_ctxnum = NULLCTX; in gru_free_gru_context()
410 gts->ts_gru = NULL; in gru_free_gru_context()
413 gts_drop(gts); in gru_free_gru_context()
494 void gru_unload_context(struct gru_thread_state *gts, int savestate) in gru_unload_context() argument
496 struct gru_state *gru = gts->ts_gru; in gru_unload_context()
498 int ctxnum = gts->ts_ctxnum; in gru_unload_context()
500 zap_vma_ptes(gts->ts_vma, UGRUADDR(gts), GRU_GSEG_PAGESIZE); in gru_unload_context()
506 gru_dbg(grudev, "gts %p\n", gts); in gru_unload_context()
508 gru_unload_mm_tracker(gru, gts->ts_gms, gts->ts_ctxnum); in gru_unload_context()
510 gru_unload_context_data(gts->ts_gdata, gru->gs_gru_base_vaddr, in gru_unload_context()
511 ctxnum, gts->ts_cbr_map, in gru_unload_context()
512 gts->ts_dsr_map); in gru_unload_context()
516 gts->ts_force_unload = 0; /* ts_force_unload locked by CCH lock */ in gru_unload_context()
519 gru_free_gru_context(gts); in gru_unload_context()
527 static void gru_load_context(struct gru_thread_state *gts) in gru_load_context() argument
529 struct gru_state *gru = gts->ts_gru; in gru_load_context()
531 int err, asid, ctxnum = gts->ts_ctxnum; in gru_load_context()
533 gru_dbg(grudev, "gts %p\n", gts); in gru_load_context()
537 asid = gru_load_mm_tracker(gru, gts->ts_gms, gts->ts_ctxnum); in gru_load_context()
539 (gts->ts_user_options == GRU_OPT_MISS_FMM_POLL in gru_load_context()
540 || gts->ts_user_options == GRU_OPT_MISS_FMM_INTR); in gru_load_context()
541 cch->tlb_int_enable = (gts->ts_user_options == GRU_OPT_MISS_FMM_INTR); in gru_load_context()
543 gts->ts_tlb_int_select = gru_cpu_fault_map_id(); in gru_load_context()
544 cch->tlb_int_select = gts->ts_tlb_int_select; in gru_load_context()
547 err = cch_allocate(cch, asid, gts->ts_cbr_map, gts->ts_dsr_map); in gru_load_context()
551 err, cch, gts, gts->ts_cbr_map, gts->ts_dsr_map); in gru_load_context()
555 gru_load_context_data(gts->ts_gdata, gru->gs_gru_base_vaddr, ctxnum, in gru_load_context()
556 gts->ts_cbr_map, gts->ts_dsr_map); in gru_load_context()
572 static int gru_update_cch(struct gru_thread_state *gts, int int_select) in gru_update_cch() argument
575 struct gru_state *gru = gts->ts_gru; in gru_update_cch()
576 int i, ctxnum = gts->ts_ctxnum, ret = 0; in gru_update_cch()
582 if (gru->gs_gts[gts->ts_ctxnum] != gts) in gru_update_cch()
587 gts->ts_tlb_int_select = int_select; in gru_update_cch()
594 gts->ts_force_unload = 1; in gru_update_cch()
612 static int gru_retarget_intr(struct gru_thread_state *gts) in gru_retarget_intr() argument
614 if (gts->ts_tlb_int_select < 0 in gru_retarget_intr()
615 || gts->ts_tlb_int_select == gru_cpu_fault_map_id()) in gru_retarget_intr()
618 gru_dbg(grudev, "retarget from %d to %d\n", gts->ts_tlb_int_select, in gru_retarget_intr()
620 return gru_update_cch(gts, gru_cpu_fault_map_id()); in gru_retarget_intr()
632 static void gru_steal_context(struct gru_thread_state *gts) in gru_steal_context() argument
639 cbr = gts->ts_cbr_au_count; in gru_steal_context()
640 dsr = gts->ts_dsr_au_count; in gru_steal_context()
700 static struct gru_state *gru_assign_gru_context(struct gru_thread_state *gts) in gru_assign_gru_context() argument
711 if (check_gru_resources(grux, gts->ts_cbr_au_count, in gru_assign_gru_context()
712 gts->ts_dsr_au_count, in gru_assign_gru_context()
723 if (!check_gru_resources(gru, gts->ts_cbr_au_count, in gru_assign_gru_context()
724 gts->ts_dsr_au_count, GRU_NUM_CCH)) { in gru_assign_gru_context()
728 reserve_gru_resources(gru, gts); in gru_assign_gru_context()
729 gts->ts_gru = gru; in gru_assign_gru_context()
730 gts->ts_ctxnum = in gru_assign_gru_context()
732 BUG_ON(gts->ts_ctxnum == GRU_NUM_CCH); in gru_assign_gru_context()
733 atomic_inc(>s->ts_refcnt); in gru_assign_gru_context()
734 gru->gs_gts[gts->ts_ctxnum] = gts; in gru_assign_gru_context()
735 __set_bit(gts->ts_ctxnum, &gru->gs_context_map); in gru_assign_gru_context()
741 gseg_virtual_address(gts->ts_gru, gts->ts_ctxnum), gts, in gru_assign_gru_context()
742 gts->ts_gru->gs_gid, gts->ts_ctxnum, in gru_assign_gru_context()
743 gts->ts_cbr_au_count, gts->ts_dsr_au_count); in gru_assign_gru_context()
762 struct gru_thread_state *gts; in gru_fault() local
771 gts = gru_find_thread_state(vma, TSID(vaddr, vma)); in gru_fault()
772 if (!gts) in gru_fault()
777 mutex_lock(>s->ts_ctxlock); in gru_fault()
778 if (gts->ts_gru) { in gru_fault()
779 if (gts->ts_gru->gs_blade_id != uv_numa_blade_id()) { in gru_fault()
781 gru_unload_context(gts, 1); in gru_fault()
783 if (gru_retarget_intr(gts)) in gru_fault()
788 if (!gts->ts_gru) { in gru_fault()
789 if (!gru_assign_gru_context(gts)) { in gru_fault()
790 mutex_unlock(>s->ts_ctxlock); in gru_fault()
793 if (gts->ts_steal_jiffies + GRU_STEAL_DELAY < jiffies) in gru_fault()
794 gru_steal_context(gts); in gru_fault()
797 gru_load_context(gts); in gru_fault()
798 paddr = gseg_physical_address(gts->ts_gru, gts->ts_ctxnum); in gru_fault()
804 mutex_unlock(>s->ts_ctxlock); in gru_fault()