63 			u8  fault;  member
65 		} **fault;  member
155 	 * page fault) and maybe some other commands.  in nouveau_svmm_bind()
377 /* Issue fault replay for GPU to retry accesses that faulted previously. */
388 /* Cancel a replayable fault that could not be handled.
390  * Cancelling the fault will trigger recovery to reset the engine
410 			       struct nouveau_svm_fault *fault)  in nouveau_svm_fault_cancel_fault()  argument
412 	nouveau_svm_fault_cancel(svm, fault->inst,  in nouveau_svm_fault_cancel_fault()
413 				      fault->hub,  in nouveau_svm_fault_cancel_fault()
414 				      fault->gpc,  in nouveau_svm_fault_cancel_fault()
415 				      fault->client);  in nouveau_svm_fault_cancel_fault()
450 	struct nouveau_svm_fault *fault;  in nouveau_svm_fault_cache()  local
458 	if (!buffer->fault[buffer->fault_nr]) {  in nouveau_svm_fault_cache()
459 		fault = kmalloc(sizeof(*fault), GFP_KERNEL);  in nouveau_svm_fault_cache()
460 		if (WARN_ON(!fault)) {  in nouveau_svm_fault_cache()
464 		buffer->fault[buffer->fault_nr] = fault;  in nouveau_svm_fault_cache()
467 	fault = buffer->fault[buffer->fault_nr++];  in nouveau_svm_fault_cache()
468 	fault->inst   = inst;  in nouveau_svm_fault_cache()
469 	fault->addr   = (u64)addrhi << 32 | addrlo;  in nouveau_svm_fault_cache()
470 	fault->time   = (u64)timehi << 32 | timelo;  in nouveau_svm_fault_cache()
471 	fault->engine = engine;  in nouveau_svm_fault_cache()
472 	fault->gpc    = gpc;  in nouveau_svm_fault_cache()
473 	fault->hub    = hub;  in nouveau_svm_fault_cache()
474 	fault->access = (info & 0x000f0000) >> 16;  in nouveau_svm_fault_cache()
475 	fault->client = client;  in nouveau_svm_fault_cache()
476 	fault->fault  = (info & 0x0000001f);  in nouveau_svm_fault_cache()
478 	SVM_DBG(svm, "fault %016llx %016llx %02x",  in nouveau_svm_fault_cache()
479 		fault->inst, fault->addr, fault->access);  in nouveau_svm_fault_cache()
570 	/* Have HMM fault pages within the fault window to the GPU. */  in nouveau_range_fault()
634 	/* Parse available fault buffer entries into a cache, and update  in nouveau_svm_fault()
637 	SVM_DBG(svm, "fault handler");  in nouveau_svm_fault()
653 	SVM_DBG(svm, "%d fault(s) pending", buffer->fault_nr);  in nouveau_svm_fault()
659 	sort(buffer->fault, buffer->fault_nr, sizeof(*buffer->fault),  in nouveau_svm_fault()
665 		if (!svmm || buffer->fault[fi]->inst != inst) {  in nouveau_svm_fault()
667 				nouveau_ivmm_find(svm, buffer->fault[fi]->inst);  in nouveau_svm_fault()
669 			inst = buffer->fault[fi]->inst;  in nouveau_svm_fault()
672 		buffer->fault[fi]->svmm = svmm;  in nouveau_svm_fault()
688 		if (!(svmm = buffer->fault[fi]->svmm)) {  in nouveau_svm_fault()
689 			nouveau_svm_fault_cancel_fault(svm, buffer->fault[fi]);  in nouveau_svm_fault()
692 		SVMM_DBG(svmm, "addr %016llx", buffer->fault[fi]->addr);  in nouveau_svm_fault()
697 		start = buffer->fault[fi]->addr;  in nouveau_svm_fault()
704 		 * fault window, determining required pages and access  in nouveau_svm_fault()
711 		 * Determine required permissions based on GPU fault  in nouveau_svm_fault()
715 		switch (buffer->fault[fi]->access) {  in nouveau_svm_fault()
729 			nouveau_svm_fault_cancel_fault(svm, buffer->fault[fi]);  in nouveau_svm_fault()
753 			 * fault addresses have sufficient access permission.  in nouveau_svm_fault()
755 			if (buffer->fault[fn]->svmm != svmm ||  in nouveau_svm_fault()
756 			    buffer->fault[fn]->addr >= limit ||  in nouveau_svm_fault()
757 			    (buffer->fault[fi]->access == 0 /* READ. */ &&  in nouveau_svm_fault()
759 			    (buffer->fault[fi]->access != 0 /* READ. */ &&  in nouveau_svm_fault()
760 			     buffer->fault[fi]->access != 3 /* PREFETCH. */ &&  in nouveau_svm_fault()
768 				struct nouveau_svm_fault *fault =  in nouveau_svm_fault()  local
769 					buffer->fault[fi++];  in nouveau_svm_fault()
771 				nouveau_svm_fault_cancel_fault(svm, fault);  in nouveau_svm_fault()
777 	/* Issue fault replay to the GPU. */  in nouveau_svm_fault()
857 	if (buffer->fault) {  in nouveau_svm_fault_buffer_dtor()
858 		for (i = 0; buffer->fault[i] && i < buffer->entries; i++)  in nouveau_svm_fault_buffer_dtor()
859 			kfree(buffer->fault[i]);  in nouveau_svm_fault_buffer_dtor()
860 		kvfree(buffer->fault);  in nouveau_svm_fault_buffer_dtor()
883 		SVM_ERR(svm, "Fault buffer allocation failed: %d", ret);  in nouveau_svm_fault_buffer_ctor()
898 	buffer->fault = kvzalloc(sizeof(*buffer->fault) * buffer->entries, GFP_KERNEL);  in nouveau_svm_fault_buffer_ctor()
899 	if (!buffer->fault)  in nouveau_svm_fault_buffer_ctor()
959 		SVM_DBG(svm, "No supported fault buffer class");  in nouveau_svm_init()