117 	struct vmw_fence_manager *fman = fman_from_fence(fence);  in vmw_fence_obj_destroy()  local
119 	spin_lock(&fman->lock);  in vmw_fence_obj_destroy()
121 	--fman->num_fence_objects;  in vmw_fence_obj_destroy()
122 	spin_unlock(&fman->lock);  in vmw_fence_obj_destroy()
141 	struct vmw_fence_manager *fman = fman_from_fence(fence);  in vmw_fence_enable_signaling()  local
142 	struct vmw_private *dev_priv = fman->dev_priv;  in vmw_fence_enable_signaling()
168 static void __vmw_fences_update(struct vmw_fence_manager *fman);
175 	struct vmw_fence_manager *fman = fman_from_fence(fence);  in vmw_fence_wait()  local
176 	struct vmw_private *dev_priv = fman->dev_priv;  in vmw_fence_wait()
201 		__vmw_fences_update(fman);  in vmw_fence_wait()
262 	struct vmw_fence_manager *fman =  in vmw_fence_work_func()  local
270 		mutex_lock(&fman->goal_irq_mutex);  in vmw_fence_work_func()
272 		spin_lock(&fman->lock);  in vmw_fence_work_func()
273 		list_splice_init(&fman->cleanup_list, &list);  in vmw_fence_work_func()
274 		seqno_valid = fman->seqno_valid;  in vmw_fence_work_func()
275 		spin_unlock(&fman->lock);  in vmw_fence_work_func()
277 		if (!seqno_valid && fman->goal_irq_on) {  in vmw_fence_work_func()
278 			fman->goal_irq_on = false;  in vmw_fence_work_func()
279 			vmw_goal_waiter_remove(fman->dev_priv);  in vmw_fence_work_func()
281 		mutex_unlock(&fman->goal_irq_mutex);  in vmw_fence_work_func()
302 	struct vmw_fence_manager *fman = kzalloc(sizeof(*fman), GFP_KERNEL);  in vmw_fence_manager_init()  local
304 	if (unlikely(!fman))  in vmw_fence_manager_init()
307 	fman->dev_priv = dev_priv;  in vmw_fence_manager_init()
308 	spin_lock_init(&fman->lock);  in vmw_fence_manager_init()
309 	INIT_LIST_HEAD(&fman->fence_list);  in vmw_fence_manager_init()
310 	INIT_LIST_HEAD(&fman->cleanup_list);  in vmw_fence_manager_init()
311 	INIT_WORK(&fman->work, &vmw_fence_work_func);  in vmw_fence_manager_init()
312 	fman->fifo_down = true;  in vmw_fence_manager_init()
313 	fman->user_fence_size = ttm_round_pot(sizeof(struct vmw_user_fence)) +  in vmw_fence_manager_init()
315 	fman->fence_size = ttm_round_pot(sizeof(struct vmw_fence_obj));  in vmw_fence_manager_init()
316 	fman->event_fence_action_size =  in vmw_fence_manager_init()
318 	mutex_init(&fman->goal_irq_mutex);  in vmw_fence_manager_init()
319 	fman->ctx = dma_fence_context_alloc(1);  in vmw_fence_manager_init()
321 	return fman;  in vmw_fence_manager_init()
324 void vmw_fence_manager_takedown(struct vmw_fence_manager *fman)  in vmw_fence_manager_takedown()  argument
328 	(void) cancel_work_sync(&fman->work);  in vmw_fence_manager_takedown()
330 	spin_lock(&fman->lock);  in vmw_fence_manager_takedown()
331 	lists_empty = list_empty(&fman->fence_list) &&  in vmw_fence_manager_takedown()
332 		list_empty(&fman->cleanup_list);  in vmw_fence_manager_takedown()
333 	spin_unlock(&fman->lock);  in vmw_fence_manager_takedown()
336 	kfree(fman);  in vmw_fence_manager_takedown()
339 static int vmw_fence_obj_init(struct vmw_fence_manager *fman,  in vmw_fence_obj_init()  argument
345 	dma_fence_init(&fence->base, &vmw_fence_ops, &fman->lock,  in vmw_fence_obj_init()
346 		       fman->ctx, seqno);  in vmw_fence_obj_init()
350 	spin_lock(&fman->lock);  in vmw_fence_obj_init()
351 	if (unlikely(fman->fifo_down)) {  in vmw_fence_obj_init()
355 	list_add_tail(&fence->head, &fman->fence_list);  in vmw_fence_obj_init()
356 	++fman->num_fence_objects;  in vmw_fence_obj_init()
359 	spin_unlock(&fman->lock);  in vmw_fence_obj_init()
364 static void vmw_fences_perform_actions(struct vmw_fence_manager *fman,  in vmw_fences_perform_actions()  argument
371 		fman->pending_actions[action->type]--;  in vmw_fences_perform_actions()
380 		list_add_tail(&action->head, &fman->cleanup_list);  in vmw_fences_perform_actions()
400 static bool vmw_fence_goal_new_locked(struct vmw_fence_manager *fman,  in vmw_fence_goal_new_locked()  argument
407 	if (likely(!fman->seqno_valid))  in vmw_fence_goal_new_locked()
410 	fifo_mem = fman->dev_priv->mmio_virt;  in vmw_fence_goal_new_locked()
415 	fman->seqno_valid = false;  in vmw_fence_goal_new_locked()
416 	list_for_each_entry(fence, &fman->fence_list, head) {  in vmw_fence_goal_new_locked()
418 			fman->seqno_valid = true;  in vmw_fence_goal_new_locked()
446 	struct vmw_fence_manager *fman = fman_from_fence(fence);  in vmw_fence_goal_check_locked()  local
453 	fifo_mem = fman->dev_priv->mmio_virt;  in vmw_fence_goal_check_locked()
455 	if (likely(fman->seqno_valid &&  in vmw_fence_goal_check_locked()
460 	fman->seqno_valid = true;  in vmw_fence_goal_check_locked()
465 static void __vmw_fences_update(struct vmw_fence_manager *fman)  in __vmw_fences_update()  argument
471 	u32 *fifo_mem = fman->dev_priv->mmio_virt;  in __vmw_fences_update()
475 	list_for_each_entry_safe(fence, next_fence, &fman->fence_list, head) {  in __vmw_fences_update()
482 			vmw_fences_perform_actions(fman, &action_list);  in __vmw_fences_update()
493 	needs_rerun = vmw_fence_goal_new_locked(fman, seqno);  in __vmw_fences_update()
502 	if (!list_empty(&fman->cleanup_list))  in __vmw_fences_update()
503 		(void) schedule_work(&fman->work);  in __vmw_fences_update()
506 void vmw_fences_update(struct vmw_fence_manager *fman)  in vmw_fences_update()  argument
508 	spin_lock(&fman->lock);  in vmw_fences_update()
509 	__vmw_fences_update(fman);  in vmw_fences_update()
510 	spin_unlock(&fman->lock);  in vmw_fences_update()
515 	struct vmw_fence_manager *fman = fman_from_fence(fence);  in vmw_fence_obj_signaled()  local
520 	vmw_fences_update(fman);  in vmw_fence_obj_signaled()
550 int vmw_fence_create(struct vmw_fence_manager *fman,  in vmw_fence_create()  argument
561 	ret = vmw_fence_obj_init(fman, fence, seqno,  in vmw_fence_create()
579 	struct vmw_fence_manager *fman = fman_from_fence(fence);  in vmw_user_fence_destroy()  local
585 	ttm_mem_global_free(vmw_mem_glob(fman->dev_priv),  in vmw_user_fence_destroy()
586 			    fman->user_fence_size);  in vmw_user_fence_destroy()
601 			  struct vmw_fence_manager *fman,  in vmw_user_fence_create()  argument
609 	struct ttm_mem_global *mem_glob = vmw_mem_glob(fman->dev_priv);  in vmw_user_fence_create()
621 	ret = ttm_mem_global_alloc(mem_glob, fman->user_fence_size,  in vmw_user_fence_create()
632 	ret = vmw_fence_obj_init(fman, &ufence->fence, seqno,  in vmw_user_fence_create()
665 	ttm_mem_global_free(mem_glob, fman->user_fence_size);  in vmw_user_fence_create()
679 int vmw_wait_dma_fence(struct vmw_fence_manager *fman,  in vmw_wait_dma_fence()  argument
719 void vmw_fence_fifo_down(struct vmw_fence_manager *fman)  in vmw_fence_fifo_down()  argument
729 	spin_lock(&fman->lock);  in vmw_fence_fifo_down()
730 	fman->fifo_down = true;  in vmw_fence_fifo_down()
731 	while (!list_empty(&fman->fence_list)) {  in vmw_fence_fifo_down()
733 			list_entry(fman->fence_list.prev, struct vmw_fence_obj,  in vmw_fence_fifo_down()
736 		spin_unlock(&fman->lock);  in vmw_fence_fifo_down()
747 			vmw_fences_perform_actions(fman, &action_list);  in vmw_fence_fifo_down()
752 		spin_lock(&fman->lock);  in vmw_fence_fifo_down()
754 	spin_unlock(&fman->lock);  in vmw_fence_fifo_down()
757 void vmw_fence_fifo_up(struct vmw_fence_manager *fman)  in vmw_fence_fifo_up()  argument
759 	spin_lock(&fman->lock);  in vmw_fence_fifo_up()
760 	fman->fifo_down = false;  in vmw_fence_fifo_up()
761 	spin_unlock(&fman->lock);  in vmw_fence_fifo_up()
862 	struct vmw_fence_manager *fman;  in vmw_fence_obj_signaled_ioctl()  local
871 	fman = fman_from_fence(fence);  in vmw_fence_obj_signaled_ioctl()
876 	spin_lock(&fman->lock);  in vmw_fence_obj_signaled_ioctl()
878 	spin_unlock(&fman->lock);  in vmw_fence_obj_signaled_ioctl()
964 	struct vmw_fence_manager *fman = fman_from_fence(fence);  in vmw_fence_obj_add_action()  local
967 	mutex_lock(&fman->goal_irq_mutex);  in vmw_fence_obj_add_action()
968 	spin_lock(&fman->lock);  in vmw_fence_obj_add_action()
970 	fman->pending_actions[action->type]++;  in vmw_fence_obj_add_action()
976 		vmw_fences_perform_actions(fman, &action_list);  in vmw_fence_obj_add_action()
987 	spin_unlock(&fman->lock);  in vmw_fence_obj_add_action()
990 		if (!fman->goal_irq_on) {  in vmw_fence_obj_add_action()
991 			fman->goal_irq_on = true;  in vmw_fence_obj_add_action()
992 			vmw_goal_waiter_add(fman->dev_priv);  in vmw_fence_obj_add_action()
994 		vmw_fences_update(fman);  in vmw_fence_obj_add_action()
996 	mutex_unlock(&fman->goal_irq_mutex);  in vmw_fence_obj_add_action()
1023 	struct vmw_fence_manager *fman = fman_from_fence(fence);  in vmw_event_fence_action_queue()  local
1036 	eaction->dev = fman->dev_priv->dev;  in vmw_event_fence_action_queue()
1057 	struct vmw_fence_manager *fman = fman_from_fence(fence);  in vmw_event_fence_action_create()  local
1058 	struct drm_device *dev = fman->dev_priv->dev;  in vmw_event_fence_action_create()