35 #include "sst-dsp.h"
36 #include "sst-dsp-priv.h"
110 /* copy to DSP */
119 /* copy from DSP */
129 static void block_list_remove(struct sst_dsp *dsp,  in block_list_remove()  argument
141 				dev_err(dsp->dev,  in block_list_remove()
150 		list_move(&block->list, &dsp->free_block_list);  in block_list_remove()
151 		dev_dbg(dsp->dev, "block freed %d:%d at offset 0x%x\n",  in block_list_remove()
157 static int block_list_prepare(struct sst_dsp *dsp,  in block_list_prepare()  argument
169 				dev_err(dsp->dev,  in block_list_prepare()
221 	struct sst_dsp *dsp = (struct sst_dsp *)param;  in dma_chan_filter()  local
223 	return chan->device->dev == dsp->dma_dev;  in dma_chan_filter()
226 int sst_dsp_dma_get_channel(struct sst_dsp *dsp, int chan_id)  in sst_dsp_dma_get_channel()  argument
228 	struct sst_dma *dma = dsp->dma;  in sst_dsp_dma_get_channel()
237 	dma->ch = dma_request_channel(mask, dma_chan_filter, dsp);  in sst_dsp_dma_get_channel()
239 		dev_err(dsp->dev, "error: DMA request channel failed\n");  in sst_dsp_dma_get_channel()
251 		dev_err(dsp->dev, "error: unable to set DMA slave config %d\n",  in sst_dsp_dma_get_channel()
261 void sst_dsp_dma_put_channel(struct sst_dsp *dsp)  in sst_dsp_dma_put_channel()  argument
263 	struct sst_dma *dma = dsp->dma;  in sst_dsp_dma_put_channel()
339 struct sst_fw *sst_fw_new(struct sst_dsp *dsp,   in sst_fw_new()  argument
345 	if (!dsp->ops->parse_fw)  in sst_fw_new()
352 	sst_fw->dsp = dsp;  in sst_fw_new()
357 	sst_fw->dma_buf = dma_alloc_coherent(dsp->dma_dev, sst_fw->size,  in sst_fw_new()
360 		dev_err(dsp->dev, "error: DMA alloc failed\n");  in sst_fw_new()
368 	if (dsp->fw_use_dma) {  in sst_fw_new()
369 		err = sst_dsp_dma_get_channel(dsp, 0);  in sst_fw_new()
374 	/* call core specific FW paser to load FW data into DSP */  in sst_fw_new()
375 	err = dsp->ops->parse_fw(sst_fw);  in sst_fw_new()
377 		dev_err(dsp->dev, "error: parse fw failed %d\n", err);  in sst_fw_new()
381 	if (dsp->fw_use_dma)  in sst_fw_new()
382 		sst_dsp_dma_put_channel(dsp);  in sst_fw_new()
384 	mutex_lock(&dsp->mutex);  in sst_fw_new()
385 	list_add(&sst_fw->list, &dsp->fw_list);  in sst_fw_new()
386 	mutex_unlock(&dsp->mutex);  in sst_fw_new()
391 	if (dsp->fw_use_dma)  in sst_fw_new()
392 		sst_dsp_dma_put_channel(dsp);  in sst_fw_new()
394 	dma_free_coherent(dsp->dma_dev, sst_fw->size,  in sst_fw_new()
405 	struct sst_dsp *dsp = sst_fw->dsp;  in sst_fw_reload()  local
408 	dev_dbg(dsp->dev, "reloading firmware\n");  in sst_fw_reload()
410 	/* call core specific FW paser to load FW data into DSP */  in sst_fw_reload()
411 	ret = dsp->ops->parse_fw(sst_fw);  in sst_fw_reload()
413 		dev_err(dsp->dev, "error: parse fw failed %d\n", ret);  in sst_fw_reload()
421 	struct sst_dsp *dsp = sst_fw->dsp;  in sst_fw_unload()  local
425 	dev_dbg(dsp->dev, "unloading firmware\n");  in sst_fw_unload()
427 	mutex_lock(&dsp->mutex);  in sst_fw_unload()
430 	list_for_each_entry_safe(module, mtmp, &dsp->module_list, list) {  in sst_fw_unload()
436 				block_list_remove(dsp, &runtime->block_list);  in sst_fw_unload()
442 			block_list_remove(dsp, &module->block_list);  in sst_fw_unload()
449 	block_list_remove(dsp, &dsp->scratch_block_list);  in sst_fw_unload()
451 	mutex_unlock(&dsp->mutex);  in sst_fw_unload()
458 	struct sst_dsp *dsp = sst_fw->dsp;  in sst_fw_free()  local
460 	mutex_lock(&dsp->mutex);  in sst_fw_free()
462 	mutex_unlock(&dsp->mutex);  in sst_fw_free()
465 		dma_free_coherent(dsp->dma_dev, sst_fw->size, sst_fw->dma_buf,  in sst_fw_free()
472 void sst_fw_free_all(struct sst_dsp *dsp)  in sst_fw_free_all()  argument
476 	mutex_lock(&dsp->mutex);  in sst_fw_free_all()
477 	list_for_each_entry_safe(sst_fw, t, &dsp->fw_list, list) {  in sst_fw_free_all()
480 		dma_free_coherent(dsp->dev, sst_fw->size, sst_fw->dma_buf,  in sst_fw_free_all()
484 	mutex_unlock(&dsp->mutex);  in sst_fw_free_all()
492 	struct sst_dsp *dsp = sst_fw->dsp;  in sst_module_new()  local
500 	sst_module->dsp = dsp;  in sst_module_new()
510 	mutex_lock(&dsp->mutex);  in sst_module_new()
511 	list_add(&sst_module->list, &dsp->module_list);  in sst_module_new()
512 	mutex_unlock(&dsp->mutex);  in sst_module_new()
521 	struct sst_dsp *dsp = sst_module->dsp;  in sst_module_free()  local
523 	mutex_lock(&dsp->mutex);  in sst_module_free()
525 	mutex_unlock(&dsp->mutex);  in sst_module_free()
534 	struct sst_dsp *dsp = module->dsp;  in sst_module_runtime_new()  local
542 	runtime->dsp = dsp;  in sst_module_runtime_new()
546 	mutex_lock(&dsp->mutex);  in sst_module_runtime_new()
548 	mutex_unlock(&dsp->mutex);  in sst_module_runtime_new()
556 	struct sst_dsp *dsp = runtime->dsp;  in sst_module_runtime_free()  local
558 	mutex_lock(&dsp->mutex);  in sst_module_runtime_free()
560 	mutex_unlock(&dsp->mutex);  in sst_module_runtime_free()
566 static struct sst_mem_block *find_block(struct sst_dsp *dsp,  in find_block()  argument
571 	list_for_each_entry(block, &dsp->free_block_list, list) {  in find_block()
580 static int block_alloc_contiguous(struct sst_dsp *dsp,  in block_alloc_contiguous()  argument
590 		block = find_block(dsp, ba);  in block_alloc_contiguous()
592 			list_splice(&tmp, &dsp->free_block_list);  in block_alloc_contiguous()
613 		dev_dbg(dsp->dev, "block allocated %d:%d at offset 0x%x\n",  in block_alloc_contiguous()
617 	list_splice(&tmp, &dsp->used_block_list);  in block_alloc_contiguous()
621 /* allocate first free DSP blocks for data - callers hold locks */
622 static int block_alloc(struct sst_dsp *dsp, struct sst_block_allocator *ba,  in block_alloc()  argument
632 	list_for_each_entry_safe(block, tmp, &dsp->free_block_list, list) {  in block_alloc()
644 		list_move(&block->list, &dsp->used_block_list);  in block_alloc()
645 		dev_dbg(dsp->dev, "block allocated %d:%d at offset 0x%x\n",  in block_alloc()
651 	list_for_each_entry_safe(block, tmp, &dsp->free_block_list, list) {  in block_alloc()
663 			ret = block_alloc_contiguous(dsp, ba, block_list);  in block_alloc()
674 int sst_alloc_blocks(struct sst_dsp *dsp, struct sst_block_allocator *ba,  in sst_alloc_blocks()  argument
679 	dev_dbg(dsp->dev, "block request 0x%x bytes at offset 0x%x type %d\n",  in sst_alloc_blocks()
682 	mutex_lock(&dsp->mutex);  in sst_alloc_blocks()
684 	ret = block_alloc(dsp, ba, block_list);  in sst_alloc_blocks()
686 		dev_err(dsp->dev, "error: can't alloc blocks %d\n", ret);  in sst_alloc_blocks()
690 	/* prepare DSP blocks for module usage */  in sst_alloc_blocks()
691 	ret = block_list_prepare(dsp, block_list);  in sst_alloc_blocks()
693 		dev_err(dsp->dev, "error: prepare failed\n");  in sst_alloc_blocks()
696 	mutex_unlock(&dsp->mutex);  in sst_alloc_blocks()
701 int sst_free_blocks(struct sst_dsp *dsp, struct list_head *block_list)  in sst_free_blocks()  argument
703 	mutex_lock(&dsp->mutex);  in sst_free_blocks()
704 	block_list_remove(dsp, block_list);  in sst_free_blocks()
705 	mutex_unlock(&dsp->mutex);  in sst_free_blocks()
711 static int block_alloc_fixed(struct sst_dsp *dsp, struct sst_block_allocator *ba,  in block_alloc_fixed()  argument
742 			err = block_alloc_contiguous(dsp, &ba_tmp, block_list);  in block_alloc_fixed()
752 	list_for_each_entry_safe(block, tmp, &dsp->free_block_list, list) {  in block_alloc_fixed()
763 			list_move(&block->list, &dsp->used_block_list);  in block_alloc_fixed()
765 			dev_dbg(dsp->dev, "block allocated %d:%d at offset 0x%x\n",  in block_alloc_fixed()
774 			list_move(&block->list, &dsp->used_block_list);  in block_alloc_fixed()
780 			err = block_alloc_contiguous(dsp, &ba_tmp, block_list);  in block_alloc_fixed()
791 /* Load fixed module data into DSP memory blocks */
794 	struct sst_dsp *dsp = module->dsp;  in sst_module_alloc_blocks()  local
804 	dev_dbg(dsp->dev, "block request 0x%x bytes at offset 0x%x type %d\n",  in sst_module_alloc_blocks()
807 	mutex_lock(&dsp->mutex);  in sst_module_alloc_blocks()
810 	ret = block_alloc_fixed(dsp, &ba, &module->block_list);  in sst_module_alloc_blocks()
812 		dev_err(dsp->dev,  in sst_module_alloc_blocks()
815 		mutex_unlock(&dsp->mutex);  in sst_module_alloc_blocks()
819 	/* prepare DSP blocks for module copy */  in sst_module_alloc_blocks()
820 	ret = block_list_prepare(dsp, &module->block_list);  in sst_module_alloc_blocks()
822 		dev_err(dsp->dev, "error: fw module prepare failed\n");  in sst_module_alloc_blocks()
827 	if (dsp->fw_use_dma) {  in sst_module_alloc_blocks()
828 		ret = sst_dsp_dma_copyto(dsp,  in sst_module_alloc_blocks()
829 			dsp->addr.lpe_base + module->offset,  in sst_module_alloc_blocks()
833 			dev_err(dsp->dev, "error: module copy failed\n");  in sst_module_alloc_blocks()
837 		sst_memcpy32(dsp->addr.lpe + module->offset, module->data,  in sst_module_alloc_blocks()
840 	mutex_unlock(&dsp->mutex);  in sst_module_alloc_blocks()
844 	block_list_remove(dsp, &module->block_list);  in sst_module_alloc_blocks()
845 	mutex_unlock(&dsp->mutex);  in sst_module_alloc_blocks()
850 /* Unload entire module from DSP memory */
853 	struct sst_dsp *dsp = module->dsp;  in sst_module_free_blocks()  local
855 	mutex_lock(&dsp->mutex);  in sst_module_free_blocks()
856 	block_list_remove(dsp, &module->block_list);  in sst_module_free_blocks()
857 	mutex_unlock(&dsp->mutex);  in sst_module_free_blocks()
865 	struct sst_dsp *dsp = runtime->dsp;  in sst_module_runtime_alloc_blocks()  local
877 	mutex_lock(&dsp->mutex);  in sst_module_runtime_alloc_blocks()
884 		dev_dbg(dsp->dev, "persistent fixed block request 0x%x bytes type %d offset 0x%x\n",  in sst_module_runtime_alloc_blocks()
888 		ret = block_alloc_fixed(dsp, &ba, &runtime->block_list);  in sst_module_runtime_alloc_blocks()
891 		dev_dbg(dsp->dev, "persistent block request 0x%x bytes type %d\n",  in sst_module_runtime_alloc_blocks()
895 		ret = block_alloc(dsp, &ba, &runtime->block_list);  in sst_module_runtime_alloc_blocks()
898 		dev_err(dsp->dev,  in sst_module_runtime_alloc_blocks()
901 		mutex_unlock(&dsp->mutex);  in sst_module_runtime_alloc_blocks()
906 	/* prepare DSP blocks for module copy */  in sst_module_runtime_alloc_blocks()
907 	ret = block_list_prepare(dsp, &runtime->block_list);  in sst_module_runtime_alloc_blocks()
909 		dev_err(dsp->dev, "error: runtime block prepare failed\n");  in sst_module_runtime_alloc_blocks()
913 	mutex_unlock(&dsp->mutex);  in sst_module_runtime_alloc_blocks()
917 	block_list_remove(dsp, &module->block_list);  in sst_module_runtime_alloc_blocks()
918 	mutex_unlock(&dsp->mutex);  in sst_module_runtime_alloc_blocks()
925 	struct sst_dsp *dsp = runtime->dsp;  in sst_module_runtime_free_blocks()  local
927 	mutex_lock(&dsp->mutex);  in sst_module_runtime_free_blocks()
928 	block_list_remove(dsp, &runtime->block_list);  in sst_module_runtime_free_blocks()
929 	mutex_unlock(&dsp->mutex);  in sst_module_runtime_free_blocks()
937 	struct sst_dsp *dsp = runtime->dsp;  in sst_module_runtime_save()  local
941 	dev_dbg(dsp->dev, "saving runtime %d memory at 0x%x size 0x%x\n",  in sst_module_runtime_save()
945 	context->buffer = dma_alloc_coherent(dsp->dma_dev,  in sst_module_runtime_save()
949 		dev_err(dsp->dev, "error: DMA context alloc failed\n");  in sst_module_runtime_save()
953 	mutex_lock(&dsp->mutex);  in sst_module_runtime_save()
955 	if (dsp->fw_use_dma) {  in sst_module_runtime_save()
957 		ret = sst_dsp_dma_get_channel(dsp, 0);  in sst_module_runtime_save()
961 		ret = sst_dsp_dma_copyfrom(dsp, context->dma_buffer,  in sst_module_runtime_save()
962 			dsp->addr.lpe_base + runtime->persistent_offset,  in sst_module_runtime_save()
964 		sst_dsp_dma_put_channel(dsp);  in sst_module_runtime_save()
966 			dev_err(dsp->dev, "error: context copy failed\n");  in sst_module_runtime_save()
970 		sst_memcpy32(context->buffer, dsp->addr.lpe +  in sst_module_runtime_save()
975 	mutex_unlock(&dsp->mutex);  in sst_module_runtime_save()
983 	struct sst_dsp *dsp = runtime->dsp;  in sst_module_runtime_restore()  local
987 	dev_dbg(dsp->dev, "restoring runtime %d memory at 0x%x size 0x%x\n",  in sst_module_runtime_restore()
991 	mutex_lock(&dsp->mutex);  in sst_module_runtime_restore()
994 		dev_info(dsp->dev, "no context buffer need to restore!\n");  in sst_module_runtime_restore()
998 	if (dsp->fw_use_dma) {  in sst_module_runtime_restore()
1000 		ret = sst_dsp_dma_get_channel(dsp, 0);  in sst_module_runtime_restore()
1004 		ret = sst_dsp_dma_copyto(dsp,  in sst_module_runtime_restore()
1005 			dsp->addr.lpe_base + runtime->persistent_offset,  in sst_module_runtime_restore()
1007 		sst_dsp_dma_put_channel(dsp);  in sst_module_runtime_restore()
1009 			dev_err(dsp->dev, "error: module copy failed\n");  in sst_module_runtime_restore()
1013 		sst_memcpy32(dsp->addr.lpe + runtime->persistent_offset,  in sst_module_runtime_restore()
1016 	dma_free_coherent(dsp->dma_dev, module->persistent_size,  in sst_module_runtime_restore()
1021 	mutex_unlock(&dsp->mutex);  in sst_module_runtime_restore()
1026 /* register a DSP memory block for use with FW based modules */
1027 struct sst_mem_block *sst_mem_block_register(struct sst_dsp *dsp, u32 offset,  in sst_mem_block_register()  argument
1041 	block->dsp = dsp;  in sst_mem_block_register()
1045 	mutex_lock(&dsp->mutex);  in sst_mem_block_register()
1046 	list_add(&block->list, &dsp->free_block_list);  in sst_mem_block_register()
1047 	mutex_unlock(&dsp->mutex);  in sst_mem_block_register()
1053 /* unregister all DSP memory blocks */
1054 void sst_mem_block_unregister_all(struct sst_dsp *dsp)  in sst_mem_block_unregister_all()  argument
1058 	mutex_lock(&dsp->mutex);  in sst_mem_block_unregister_all()
1061 	list_for_each_entry_safe(block, tmp, &dsp->used_block_list, list) {  in sst_mem_block_unregister_all()
1067 	list_for_each_entry_safe(block, tmp, &dsp->free_block_list, list) {  in sst_mem_block_unregister_all()
1072 	mutex_unlock(&dsp->mutex);  in sst_mem_block_unregister_all()
1077 int sst_block_alloc_scratch(struct sst_dsp *dsp)  in sst_block_alloc_scratch()  argument
1083 	mutex_lock(&dsp->mutex);  in sst_block_alloc_scratch()
1086 	dsp->scratch_size = 0;  in sst_block_alloc_scratch()
1087 	list_for_each_entry(module, &dsp->module_list, list) {  in sst_block_alloc_scratch()
1088 		dev_dbg(dsp->dev, "module %d scratch req 0x%x bytes\n",  in sst_block_alloc_scratch()
1090 		if (dsp->scratch_size < module->scratch_size)  in sst_block_alloc_scratch()
1091 			dsp->scratch_size = module->scratch_size;  in sst_block_alloc_scratch()
1094 	dev_dbg(dsp->dev, "scratch buffer required is 0x%x bytes\n",  in sst_block_alloc_scratch()
1095 		dsp->scratch_size);  in sst_block_alloc_scratch()
1097 	if (dsp->scratch_size == 0) {  in sst_block_alloc_scratch()
1098 		dev_info(dsp->dev, "no modules need scratch buffer\n");  in sst_block_alloc_scratch()
1099 		mutex_unlock(&dsp->mutex);  in sst_block_alloc_scratch()
1104 	dev_dbg(dsp->dev, "allocating scratch blocks\n");  in sst_block_alloc_scratch()
1106 	ba.size = dsp->scratch_size;  in sst_block_alloc_scratch()
1110 	if (dsp->scratch_offset != 0) {  in sst_block_alloc_scratch()
1112 		dev_dbg(dsp->dev, "block request 0x%x bytes type %d at 0x%x\n",  in sst_block_alloc_scratch()
1115 		ba.offset = dsp->scratch_offset;  in sst_block_alloc_scratch()
1118 		ret = block_alloc_fixed(dsp, &ba, &dsp->scratch_block_list);  in sst_block_alloc_scratch()
1121 		dev_dbg(dsp->dev, "block request 0x%x bytes type %d\n",  in sst_block_alloc_scratch()
1125 		ret = block_alloc(dsp, &ba, &dsp->scratch_block_list);  in sst_block_alloc_scratch()
1128 		dev_err(dsp->dev, "error: can't alloc scratch blocks\n");  in sst_block_alloc_scratch()
1129 		mutex_unlock(&dsp->mutex);  in sst_block_alloc_scratch()
1133 	ret = block_list_prepare(dsp, &dsp->scratch_block_list);  in sst_block_alloc_scratch()
1135 		dev_err(dsp->dev, "error: scratch block prepare failed\n");  in sst_block_alloc_scratch()
1136 		mutex_unlock(&dsp->mutex);  in sst_block_alloc_scratch()
1141 	dsp->scratch_offset = ba.offset;  in sst_block_alloc_scratch()
1142 	mutex_unlock(&dsp->mutex);  in sst_block_alloc_scratch()
1143 	return dsp->scratch_size;  in sst_block_alloc_scratch()
1148 void sst_block_free_scratch(struct sst_dsp *dsp)  in sst_block_free_scratch()  argument
1150 	mutex_lock(&dsp->mutex);  in sst_block_free_scratch()
1151 	block_list_remove(dsp, &dsp->scratch_block_list);  in sst_block_free_scratch()
1152 	mutex_unlock(&dsp->mutex);  in sst_block_free_scratch()
1157 struct sst_module *sst_module_get_from_id(struct sst_dsp *dsp, u32 id)  in sst_module_get_from_id()  argument
1161 	mutex_lock(&dsp->mutex);  in sst_module_get_from_id()
1163 	list_for_each_entry(module, &dsp->module_list, list) {  in sst_module_get_from_id()
1165 			mutex_unlock(&dsp->mutex);  in sst_module_get_from_id()
1170 	mutex_unlock(&dsp->mutex);  in sst_module_get_from_id()
1179 	struct sst_dsp *dsp = module->dsp;  in sst_module_runtime_get_from_id()  local
1181 	mutex_lock(&dsp->mutex);  in sst_module_runtime_get_from_id()
1185 			mutex_unlock(&dsp->mutex);  in sst_module_runtime_get_from_id()
1190 	mutex_unlock(&dsp->mutex);  in sst_module_runtime_get_from_id()
1195 /* returns block address in DSP address space */
1196 u32 sst_dsp_get_offset(struct sst_dsp *dsp, u32 offset,  in sst_dsp_get_offset()  argument
1201 		return offset - dsp->addr.iram_offset +  in sst_dsp_get_offset()
1202 			dsp->addr.dsp_iram_offset;  in sst_dsp_get_offset()
1204 		return offset - dsp->addr.dram_offset +  in sst_dsp_get_offset()
1205 			dsp->addr.dsp_dram_offset;  in sst_dsp_get_offset()
1218 	dev_dbg(dev, "initialising audio DSP id 0x%x\n", pdata->id);  in sst_dsp_new()
1240 	/* Initialise SST Audio DSP */  in sst_dsp_new()