• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * Copyright (c) 2018-2020, The Linux Foundation. All rights reserved.
4  *
5  */
6 
7 #include <linux/delay.h>
8 #include <linux/device.h>
9 #include <linux/dma-direction.h>
10 #include <linux/dma-mapping.h>
11 #include <linux/interrupt.h>
12 #include <linux/list.h>
13 #include <linux/mhi.h>
14 #include <linux/module.h>
15 #include <linux/skbuff.h>
16 #include <linux/slab.h>
17 #include "internal.h"
18 
mhi_read_reg(struct mhi_controller * mhi_cntrl,void __iomem * base,u32 offset,u32 * out)19 int __must_check mhi_read_reg(struct mhi_controller *mhi_cntrl,
20 			      void __iomem *base, u32 offset, u32 *out)
21 {
22 	return mhi_cntrl->read_reg(mhi_cntrl, base + offset, out);
23 }
24 
mhi_read_reg_field(struct mhi_controller * mhi_cntrl,void __iomem * base,u32 offset,u32 mask,u32 shift,u32 * out)25 int __must_check mhi_read_reg_field(struct mhi_controller *mhi_cntrl,
26 				    void __iomem *base, u32 offset,
27 				    u32 mask, u32 shift, u32 *out)
28 {
29 	u32 tmp;
30 	int ret;
31 
32 	ret = mhi_read_reg(mhi_cntrl, base, offset, &tmp);
33 	if (ret)
34 		return ret;
35 
36 	*out = (tmp & mask) >> shift;
37 
38 	return 0;
39 }
40 
mhi_poll_reg_field(struct mhi_controller * mhi_cntrl,void __iomem * base,u32 offset,u32 mask,u32 shift,u32 val,u32 delayus)41 int __must_check mhi_poll_reg_field(struct mhi_controller *mhi_cntrl,
42 				    void __iomem *base, u32 offset,
43 				    u32 mask, u32 shift, u32 val, u32 delayus)
44 {
45 	int ret;
46 	u32 out, retry = (mhi_cntrl->timeout_ms * 1000) / delayus;
47 
48 	while (retry--) {
49 		ret = mhi_read_reg_field(mhi_cntrl, base, offset, mask, shift,
50 					 &out);
51 		if (ret)
52 			return ret;
53 
54 		if (out == val)
55 			return 0;
56 
57 		fsleep(delayus);
58 	}
59 
60 	return -ETIMEDOUT;
61 }
62 
mhi_write_reg(struct mhi_controller * mhi_cntrl,void __iomem * base,u32 offset,u32 val)63 void mhi_write_reg(struct mhi_controller *mhi_cntrl, void __iomem *base,
64 		   u32 offset, u32 val)
65 {
66 	mhi_cntrl->write_reg(mhi_cntrl, base + offset, val);
67 }
68 
mhi_write_reg_field(struct mhi_controller * mhi_cntrl,void __iomem * base,u32 offset,u32 mask,u32 shift,u32 val)69 void mhi_write_reg_field(struct mhi_controller *mhi_cntrl, void __iomem *base,
70 			 u32 offset, u32 mask, u32 shift, u32 val)
71 {
72 	int ret;
73 	u32 tmp;
74 
75 	ret = mhi_read_reg(mhi_cntrl, base, offset, &tmp);
76 	if (ret)
77 		return;
78 
79 	tmp &= ~mask;
80 	tmp |= (val << shift);
81 	mhi_write_reg(mhi_cntrl, base, offset, tmp);
82 }
83 
mhi_write_db(struct mhi_controller * mhi_cntrl,void __iomem * db_addr,dma_addr_t db_val)84 void mhi_write_db(struct mhi_controller *mhi_cntrl, void __iomem *db_addr,
85 		  dma_addr_t db_val)
86 {
87 	mhi_write_reg(mhi_cntrl, db_addr, 4, upper_32_bits(db_val));
88 	mhi_write_reg(mhi_cntrl, db_addr, 0, lower_32_bits(db_val));
89 }
90 
mhi_db_brstmode(struct mhi_controller * mhi_cntrl,struct db_cfg * db_cfg,void __iomem * db_addr,dma_addr_t db_val)91 void mhi_db_brstmode(struct mhi_controller *mhi_cntrl,
92 		     struct db_cfg *db_cfg,
93 		     void __iomem *db_addr,
94 		     dma_addr_t db_val)
95 {
96 	if (db_cfg->db_mode) {
97 		db_cfg->db_val = db_val;
98 		mhi_write_db(mhi_cntrl, db_addr, db_val);
99 		db_cfg->db_mode = 0;
100 	}
101 }
102 
mhi_db_brstmode_disable(struct mhi_controller * mhi_cntrl,struct db_cfg * db_cfg,void __iomem * db_addr,dma_addr_t db_val)103 void mhi_db_brstmode_disable(struct mhi_controller *mhi_cntrl,
104 			     struct db_cfg *db_cfg,
105 			     void __iomem *db_addr,
106 			     dma_addr_t db_val)
107 {
108 	db_cfg->db_val = db_val;
109 	mhi_write_db(mhi_cntrl, db_addr, db_val);
110 }
111 
mhi_ring_er_db(struct mhi_event * mhi_event)112 void mhi_ring_er_db(struct mhi_event *mhi_event)
113 {
114 	struct mhi_ring *ring = &mhi_event->ring;
115 
116 	mhi_event->db_cfg.process_db(mhi_event->mhi_cntrl, &mhi_event->db_cfg,
117 				     ring->db_addr, le64_to_cpu(*ring->ctxt_wp));
118 }
119 
mhi_ring_cmd_db(struct mhi_controller * mhi_cntrl,struct mhi_cmd * mhi_cmd)120 void mhi_ring_cmd_db(struct mhi_controller *mhi_cntrl, struct mhi_cmd *mhi_cmd)
121 {
122 	dma_addr_t db;
123 	struct mhi_ring *ring = &mhi_cmd->ring;
124 
125 	db = ring->iommu_base + (ring->wp - ring->base);
126 	*ring->ctxt_wp = cpu_to_le64(db);
127 	mhi_write_db(mhi_cntrl, ring->db_addr, db);
128 }
129 
mhi_ring_chan_db(struct mhi_controller * mhi_cntrl,struct mhi_chan * mhi_chan)130 void mhi_ring_chan_db(struct mhi_controller *mhi_cntrl,
131 		      struct mhi_chan *mhi_chan)
132 {
133 	struct mhi_ring *ring = &mhi_chan->tre_ring;
134 	dma_addr_t db;
135 
136 	db = ring->iommu_base + (ring->wp - ring->base);
137 
138 	/*
139 	 * Writes to the new ring element must be visible to the hardware
140 	 * before letting h/w know there is new element to fetch.
141 	 */
142 	dma_wmb();
143 	*ring->ctxt_wp = cpu_to_le64(db);
144 
145 	mhi_chan->db_cfg.process_db(mhi_cntrl, &mhi_chan->db_cfg,
146 				    ring->db_addr, db);
147 }
148 
mhi_get_exec_env(struct mhi_controller * mhi_cntrl)149 enum mhi_ee_type mhi_get_exec_env(struct mhi_controller *mhi_cntrl)
150 {
151 	u32 exec;
152 	int ret = mhi_read_reg(mhi_cntrl, mhi_cntrl->bhi, BHI_EXECENV, &exec);
153 
154 	return (ret) ? MHI_EE_MAX : exec;
155 }
156 EXPORT_SYMBOL_GPL(mhi_get_exec_env);
157 
mhi_get_mhi_state(struct mhi_controller * mhi_cntrl)158 enum mhi_state mhi_get_mhi_state(struct mhi_controller *mhi_cntrl)
159 {
160 	u32 state;
161 	int ret = mhi_read_reg_field(mhi_cntrl, mhi_cntrl->regs, MHISTATUS,
162 				     MHISTATUS_MHISTATE_MASK,
163 				     MHISTATUS_MHISTATE_SHIFT, &state);
164 	return ret ? MHI_STATE_MAX : state;
165 }
166 EXPORT_SYMBOL_GPL(mhi_get_mhi_state);
167 
mhi_soc_reset(struct mhi_controller * mhi_cntrl)168 void mhi_soc_reset(struct mhi_controller *mhi_cntrl)
169 {
170 	if (mhi_cntrl->reset) {
171 		mhi_cntrl->reset(mhi_cntrl);
172 		return;
173 	}
174 
175 	/* Generic MHI SoC reset */
176 	mhi_write_reg(mhi_cntrl, mhi_cntrl->regs, MHI_SOC_RESET_REQ_OFFSET,
177 		      MHI_SOC_RESET_REQ);
178 }
179 EXPORT_SYMBOL_GPL(mhi_soc_reset);
180 
mhi_map_single_no_bb(struct mhi_controller * mhi_cntrl,struct mhi_buf_info * buf_info)181 int mhi_map_single_no_bb(struct mhi_controller *mhi_cntrl,
182 			 struct mhi_buf_info *buf_info)
183 {
184 	buf_info->p_addr = dma_map_single(mhi_cntrl->cntrl_dev,
185 					  buf_info->v_addr, buf_info->len,
186 					  buf_info->dir);
187 	if (dma_mapping_error(mhi_cntrl->cntrl_dev, buf_info->p_addr))
188 		return -ENOMEM;
189 
190 	return 0;
191 }
192 
mhi_map_single_use_bb(struct mhi_controller * mhi_cntrl,struct mhi_buf_info * buf_info)193 int mhi_map_single_use_bb(struct mhi_controller *mhi_cntrl,
194 			  struct mhi_buf_info *buf_info)
195 {
196 	void *buf = dma_alloc_coherent(mhi_cntrl->cntrl_dev, buf_info->len,
197 				       &buf_info->p_addr, GFP_ATOMIC);
198 
199 	if (!buf)
200 		return -ENOMEM;
201 
202 	if (buf_info->dir == DMA_TO_DEVICE)
203 		memcpy(buf, buf_info->v_addr, buf_info->len);
204 
205 	buf_info->bb_addr = buf;
206 
207 	return 0;
208 }
209 
mhi_unmap_single_no_bb(struct mhi_controller * mhi_cntrl,struct mhi_buf_info * buf_info)210 void mhi_unmap_single_no_bb(struct mhi_controller *mhi_cntrl,
211 			    struct mhi_buf_info *buf_info)
212 {
213 	dma_unmap_single(mhi_cntrl->cntrl_dev, buf_info->p_addr, buf_info->len,
214 			 buf_info->dir);
215 }
216 
mhi_unmap_single_use_bb(struct mhi_controller * mhi_cntrl,struct mhi_buf_info * buf_info)217 void mhi_unmap_single_use_bb(struct mhi_controller *mhi_cntrl,
218 			     struct mhi_buf_info *buf_info)
219 {
220 	if (buf_info->dir == DMA_FROM_DEVICE)
221 		memcpy(buf_info->v_addr, buf_info->bb_addr, buf_info->len);
222 
223 	dma_free_coherent(mhi_cntrl->cntrl_dev, buf_info->len,
224 			  buf_info->bb_addr, buf_info->p_addr);
225 }
226 
get_nr_avail_ring_elements(struct mhi_controller * mhi_cntrl,struct mhi_ring * ring)227 static int get_nr_avail_ring_elements(struct mhi_controller *mhi_cntrl,
228 				      struct mhi_ring *ring)
229 {
230 	int nr_el;
231 
232 	if (ring->wp < ring->rp) {
233 		nr_el = ((ring->rp - ring->wp) / ring->el_size) - 1;
234 	} else {
235 		nr_el = (ring->rp - ring->base) / ring->el_size;
236 		nr_el += ((ring->base + ring->len - ring->wp) /
237 			  ring->el_size) - 1;
238 	}
239 
240 	return nr_el;
241 }
242 
mhi_to_virtual(struct mhi_ring * ring,dma_addr_t addr)243 static void *mhi_to_virtual(struct mhi_ring *ring, dma_addr_t addr)
244 {
245 	return (addr - ring->iommu_base) + ring->base;
246 }
247 
mhi_add_ring_element(struct mhi_controller * mhi_cntrl,struct mhi_ring * ring)248 static void mhi_add_ring_element(struct mhi_controller *mhi_cntrl,
249 				 struct mhi_ring *ring)
250 {
251 	ring->wp += ring->el_size;
252 	if (ring->wp >= (ring->base + ring->len))
253 		ring->wp = ring->base;
254 	/* smp update */
255 	smp_wmb();
256 }
257 
mhi_del_ring_element(struct mhi_controller * mhi_cntrl,struct mhi_ring * ring)258 static void mhi_del_ring_element(struct mhi_controller *mhi_cntrl,
259 				 struct mhi_ring *ring)
260 {
261 	ring->rp += ring->el_size;
262 	if (ring->rp >= (ring->base + ring->len))
263 		ring->rp = ring->base;
264 	/* smp update */
265 	smp_wmb();
266 }
267 
is_valid_ring_ptr(struct mhi_ring * ring,dma_addr_t addr)268 static bool is_valid_ring_ptr(struct mhi_ring *ring, dma_addr_t addr)
269 {
270 	return addr >= ring->iommu_base && addr < ring->iommu_base + ring->len &&
271 			!(addr & (sizeof(struct mhi_ring_element) - 1));
272 }
273 
mhi_destroy_device(struct device * dev,void * data)274 int mhi_destroy_device(struct device *dev, void *data)
275 {
276 	struct mhi_chan *ul_chan, *dl_chan;
277 	struct mhi_device *mhi_dev;
278 	struct mhi_controller *mhi_cntrl;
279 	enum mhi_ee_type ee = MHI_EE_MAX;
280 
281 	if (dev->bus != &mhi_bus_type)
282 		return 0;
283 
284 	mhi_dev = to_mhi_device(dev);
285 	mhi_cntrl = mhi_dev->mhi_cntrl;
286 
287 	/* Only destroy virtual devices thats attached to bus */
288 	if (mhi_dev->dev_type == MHI_DEVICE_CONTROLLER)
289 		return 0;
290 
291 	ul_chan = mhi_dev->ul_chan;
292 	dl_chan = mhi_dev->dl_chan;
293 
294 	/*
295 	 * If execution environment is specified, remove only those devices that
296 	 * started in them based on ee_mask for the channels as we move on to a
297 	 * different execution environment
298 	 */
299 	if (data)
300 		ee = *(enum mhi_ee_type *)data;
301 
302 	/*
303 	 * For the suspend and resume case, this function will get called
304 	 * without mhi_unregister_controller(). Hence, we need to drop the
305 	 * references to mhi_dev created for ul and dl channels. We can
306 	 * be sure that there will be no instances of mhi_dev left after
307 	 * this.
308 	 */
309 	if (ul_chan) {
310 		if (ee != MHI_EE_MAX && !(ul_chan->ee_mask & BIT(ee)))
311 			return 0;
312 
313 		put_device(&ul_chan->mhi_dev->dev);
314 	}
315 
316 	if (dl_chan) {
317 		if (ee != MHI_EE_MAX && !(dl_chan->ee_mask & BIT(ee)))
318 			return 0;
319 
320 		put_device(&dl_chan->mhi_dev->dev);
321 	}
322 
323 	dev_dbg(&mhi_cntrl->mhi_dev->dev, "destroy device for chan:%s\n",
324 		 mhi_dev->name);
325 
326 	/* Notify the client and remove the device from MHI bus */
327 	device_del(dev);
328 	put_device(dev);
329 
330 	return 0;
331 }
332 
mhi_get_free_desc_count(struct mhi_device * mhi_dev,enum dma_data_direction dir)333 int mhi_get_free_desc_count(struct mhi_device *mhi_dev,
334 				enum dma_data_direction dir)
335 {
336 	struct mhi_controller *mhi_cntrl = mhi_dev->mhi_cntrl;
337 	struct mhi_chan *mhi_chan = (dir == DMA_TO_DEVICE) ?
338 		mhi_dev->ul_chan : mhi_dev->dl_chan;
339 	struct mhi_ring *tre_ring = &mhi_chan->tre_ring;
340 
341 	return get_nr_avail_ring_elements(mhi_cntrl, tre_ring);
342 }
343 EXPORT_SYMBOL_GPL(mhi_get_free_desc_count);
344 
mhi_notify(struct mhi_device * mhi_dev,enum mhi_callback cb_reason)345 void mhi_notify(struct mhi_device *mhi_dev, enum mhi_callback cb_reason)
346 {
347 	struct mhi_driver *mhi_drv;
348 
349 	if (!mhi_dev->dev.driver)
350 		return;
351 
352 	mhi_drv = to_mhi_driver(mhi_dev->dev.driver);
353 
354 	if (mhi_drv->status_cb)
355 		mhi_drv->status_cb(mhi_dev, cb_reason);
356 }
357 EXPORT_SYMBOL_GPL(mhi_notify);
358 
359 /* Bind MHI channels to MHI devices */
mhi_create_devices(struct mhi_controller * mhi_cntrl)360 void mhi_create_devices(struct mhi_controller *mhi_cntrl)
361 {
362 	struct mhi_chan *mhi_chan;
363 	struct mhi_device *mhi_dev;
364 	struct device *dev = &mhi_cntrl->mhi_dev->dev;
365 	int i, ret;
366 
367 	mhi_chan = mhi_cntrl->mhi_chan;
368 	for (i = 0; i < mhi_cntrl->max_chan; i++, mhi_chan++) {
369 		if (!mhi_chan->configured || mhi_chan->mhi_dev ||
370 		    !(mhi_chan->ee_mask & BIT(mhi_cntrl->ee)))
371 			continue;
372 		mhi_dev = mhi_alloc_device(mhi_cntrl);
373 		if (IS_ERR(mhi_dev))
374 			return;
375 
376 		mhi_dev->dev_type = MHI_DEVICE_XFER;
377 		switch (mhi_chan->dir) {
378 		case DMA_TO_DEVICE:
379 			mhi_dev->ul_chan = mhi_chan;
380 			mhi_dev->ul_chan_id = mhi_chan->chan;
381 			break;
382 		case DMA_FROM_DEVICE:
383 			/* We use dl_chan as offload channels */
384 			mhi_dev->dl_chan = mhi_chan;
385 			mhi_dev->dl_chan_id = mhi_chan->chan;
386 			break;
387 		default:
388 			dev_err(dev, "Direction not supported\n");
389 			put_device(&mhi_dev->dev);
390 			return;
391 		}
392 
393 		get_device(&mhi_dev->dev);
394 		mhi_chan->mhi_dev = mhi_dev;
395 
396 		/* Check next channel if it matches */
397 		if ((i + 1) < mhi_cntrl->max_chan && mhi_chan[1].configured) {
398 			if (!strcmp(mhi_chan[1].name, mhi_chan->name)) {
399 				i++;
400 				mhi_chan++;
401 				if (mhi_chan->dir == DMA_TO_DEVICE) {
402 					mhi_dev->ul_chan = mhi_chan;
403 					mhi_dev->ul_chan_id = mhi_chan->chan;
404 				} else {
405 					mhi_dev->dl_chan = mhi_chan;
406 					mhi_dev->dl_chan_id = mhi_chan->chan;
407 				}
408 				get_device(&mhi_dev->dev);
409 				mhi_chan->mhi_dev = mhi_dev;
410 			}
411 		}
412 
413 		/* Channel name is same for both UL and DL */
414 		mhi_dev->name = mhi_chan->name;
415 		dev_set_name(&mhi_dev->dev, "%s_%s",
416 			     dev_name(&mhi_cntrl->mhi_dev->dev),
417 			     mhi_dev->name);
418 
419 		/* Init wakeup source if available */
420 		if (mhi_dev->dl_chan && mhi_dev->dl_chan->wake_capable)
421 			device_init_wakeup(&mhi_dev->dev, true);
422 
423 		ret = device_add(&mhi_dev->dev);
424 		if (ret)
425 			put_device(&mhi_dev->dev);
426 	}
427 }
428 
mhi_irq_handler(int irq_number,void * dev)429 irqreturn_t mhi_irq_handler(int irq_number, void *dev)
430 {
431 	struct mhi_event *mhi_event = dev;
432 	struct mhi_controller *mhi_cntrl = mhi_event->mhi_cntrl;
433 	struct mhi_event_ctxt *er_ctxt =
434 		&mhi_cntrl->mhi_ctxt->er_ctxt[mhi_event->er_index];
435 	struct mhi_ring *ev_ring = &mhi_event->ring;
436 	dma_addr_t ptr = le64_to_cpu(er_ctxt->rp);
437 	void *dev_rp;
438 
439 	if (!is_valid_ring_ptr(ev_ring, ptr)) {
440 		dev_err(&mhi_cntrl->mhi_dev->dev,
441 			"Event ring rp points outside of the event ring\n");
442 		return IRQ_HANDLED;
443 	}
444 
445 	dev_rp = mhi_to_virtual(ev_ring, ptr);
446 
447 	/* Only proceed if event ring has pending events */
448 	if (ev_ring->rp == dev_rp)
449 		return IRQ_HANDLED;
450 
451 	/* For client managed event ring, notify pending data */
452 	if (mhi_event->cl_manage) {
453 		struct mhi_chan *mhi_chan = mhi_event->mhi_chan;
454 		struct mhi_device *mhi_dev = mhi_chan->mhi_dev;
455 
456 		if (mhi_dev)
457 			mhi_notify(mhi_dev, MHI_CB_PENDING_DATA);
458 	} else {
459 		tasklet_schedule(&mhi_event->task);
460 	}
461 
462 	return IRQ_HANDLED;
463 }
464 
mhi_intvec_threaded_handler(int irq_number,void * priv)465 irqreturn_t mhi_intvec_threaded_handler(int irq_number, void *priv)
466 {
467 	struct mhi_controller *mhi_cntrl = priv;
468 	struct device *dev = &mhi_cntrl->mhi_dev->dev;
469 	enum mhi_state state;
470 	enum mhi_pm_state pm_state = 0;
471 	enum mhi_ee_type ee;
472 
473 	write_lock_irq(&mhi_cntrl->pm_lock);
474 	if (!MHI_REG_ACCESS_VALID(mhi_cntrl->pm_state)) {
475 		write_unlock_irq(&mhi_cntrl->pm_lock);
476 		goto exit_intvec;
477 	}
478 
479 	state = mhi_get_mhi_state(mhi_cntrl);
480 	ee = mhi_get_exec_env(mhi_cntrl);
481 	dev_dbg(dev, "local ee: %s state: %s device ee: %s state: %s\n",
482 		TO_MHI_EXEC_STR(mhi_cntrl->ee),
483 		TO_MHI_STATE_STR(mhi_cntrl->dev_state),
484 		TO_MHI_EXEC_STR(ee), TO_MHI_STATE_STR(state));
485 
486 	if (state == MHI_STATE_SYS_ERR) {
487 		dev_dbg(dev, "System error detected\n");
488 		pm_state = mhi_tryset_pm_state(mhi_cntrl,
489 					       MHI_PM_SYS_ERR_DETECT);
490 	}
491 	write_unlock_irq(&mhi_cntrl->pm_lock);
492 
493 	if (pm_state != MHI_PM_SYS_ERR_DETECT)
494 		goto exit_intvec;
495 
496 	switch (ee) {
497 	case MHI_EE_RDDM:
498 		/* proceed if power down is not already in progress */
499 		if (mhi_cntrl->rddm_image && mhi_is_active(mhi_cntrl)) {
500 			mhi_cntrl->status_cb(mhi_cntrl, MHI_CB_EE_RDDM);
501 			mhi_cntrl->ee = ee;
502 			wake_up_all(&mhi_cntrl->state_event);
503 		}
504 		break;
505 	case MHI_EE_PBL:
506 	case MHI_EE_EDL:
507 	case MHI_EE_PTHRU:
508 		mhi_cntrl->status_cb(mhi_cntrl, MHI_CB_FATAL_ERROR);
509 		mhi_cntrl->ee = ee;
510 		wake_up_all(&mhi_cntrl->state_event);
511 		mhi_pm_sys_err_handler(mhi_cntrl);
512 		break;
513 	default:
514 		wake_up_all(&mhi_cntrl->state_event);
515 		mhi_pm_sys_err_handler(mhi_cntrl);
516 		break;
517 	}
518 
519 exit_intvec:
520 
521 	return IRQ_HANDLED;
522 }
523 
mhi_intvec_handler(int irq_number,void * dev)524 irqreturn_t mhi_intvec_handler(int irq_number, void *dev)
525 {
526 	struct mhi_controller *mhi_cntrl = dev;
527 
528 	/* Wake up events waiting for state change */
529 	wake_up_all(&mhi_cntrl->state_event);
530 
531 	return IRQ_WAKE_THREAD;
532 }
533 
mhi_recycle_ev_ring_element(struct mhi_controller * mhi_cntrl,struct mhi_ring * ring)534 static void mhi_recycle_ev_ring_element(struct mhi_controller *mhi_cntrl,
535 					struct mhi_ring *ring)
536 {
537 	dma_addr_t ctxt_wp;
538 
539 	/* Update the WP */
540 	ring->wp += ring->el_size;
541 	ctxt_wp = le64_to_cpu(*ring->ctxt_wp) + ring->el_size;
542 
543 	if (ring->wp >= (ring->base + ring->len)) {
544 		ring->wp = ring->base;
545 		ctxt_wp = ring->iommu_base;
546 	}
547 
548 	*ring->ctxt_wp = cpu_to_le64(ctxt_wp);
549 
550 	/* Update the RP */
551 	ring->rp += ring->el_size;
552 	if (ring->rp >= (ring->base + ring->len))
553 		ring->rp = ring->base;
554 
555 	/* Update to all cores */
556 	smp_wmb();
557 }
558 
parse_xfer_event(struct mhi_controller * mhi_cntrl,struct mhi_ring_element * event,struct mhi_chan * mhi_chan)559 static int parse_xfer_event(struct mhi_controller *mhi_cntrl,
560 			    struct mhi_ring_element *event,
561 			    struct mhi_chan *mhi_chan)
562 {
563 	struct mhi_ring *buf_ring, *tre_ring;
564 	struct device *dev = &mhi_cntrl->mhi_dev->dev;
565 	struct mhi_result result;
566 	unsigned long flags = 0;
567 	u32 ev_code;
568 
569 	ev_code = MHI_TRE_GET_EV_CODE(event);
570 	buf_ring = &mhi_chan->buf_ring;
571 	tre_ring = &mhi_chan->tre_ring;
572 
573 	result.transaction_status = (ev_code == MHI_EV_CC_OVERFLOW) ?
574 		-EOVERFLOW : 0;
575 
576 	/*
577 	 * If it's a DB Event then we need to grab the lock
578 	 * with preemption disabled and as a write because we
579 	 * have to update db register and there are chances that
580 	 * another thread could be doing the same.
581 	 */
582 	if (ev_code >= MHI_EV_CC_OOB)
583 		write_lock_irqsave(&mhi_chan->lock, flags);
584 	else
585 		read_lock_bh(&mhi_chan->lock);
586 
587 	if (mhi_chan->ch_state != MHI_CH_STATE_ENABLED)
588 		goto end_process_tx_event;
589 
590 	switch (ev_code) {
591 	case MHI_EV_CC_OVERFLOW:
592 	case MHI_EV_CC_EOB:
593 	case MHI_EV_CC_EOT:
594 	{
595 		dma_addr_t ptr = MHI_TRE_GET_EV_PTR(event);
596 		struct mhi_ring_element *local_rp, *ev_tre;
597 		void *dev_rp;
598 		struct mhi_buf_info *buf_info;
599 		u16 xfer_len;
600 
601 		if (!is_valid_ring_ptr(tre_ring, ptr)) {
602 			dev_err(&mhi_cntrl->mhi_dev->dev,
603 				"Event element points outside of the tre ring\n");
604 			break;
605 		}
606 		/* Get the TRB this event points to */
607 		ev_tre = mhi_to_virtual(tre_ring, ptr);
608 
609 		dev_rp = ev_tre + 1;
610 		if (dev_rp >= (tre_ring->base + tre_ring->len))
611 			dev_rp = tre_ring->base;
612 
613 		result.dir = mhi_chan->dir;
614 
615 		local_rp = tre_ring->rp;
616 		while (local_rp != dev_rp) {
617 			buf_info = buf_ring->rp;
618 			/* If it's the last TRE, get length from the event */
619 			if (local_rp == ev_tre)
620 				xfer_len = MHI_TRE_GET_EV_LEN(event);
621 			else
622 				xfer_len = buf_info->len;
623 
624 			/* Unmap if it's not pre-mapped by client */
625 			if (likely(!buf_info->pre_mapped))
626 				mhi_cntrl->unmap_single(mhi_cntrl, buf_info);
627 
628 			result.buf_addr = buf_info->cb_buf;
629 
630 			/* truncate to buf len if xfer_len is larger */
631 			result.bytes_xferd =
632 				min_t(u16, xfer_len, buf_info->len);
633 			mhi_del_ring_element(mhi_cntrl, buf_ring);
634 			mhi_del_ring_element(mhi_cntrl, tre_ring);
635 			local_rp = tre_ring->rp;
636 
637 			read_unlock_bh(&mhi_chan->lock);
638 
639 			/* notify client */
640 			mhi_chan->xfer_cb(mhi_chan->mhi_dev, &result);
641 
642 			if (mhi_chan->dir == DMA_TO_DEVICE) {
643 				atomic_dec(&mhi_cntrl->pending_pkts);
644 				/* Release the reference got from mhi_queue() */
645 				mhi_cntrl->runtime_put(mhi_cntrl);
646 			}
647 
648 			/*
649 			 * Recycle the buffer if buffer is pre-allocated,
650 			 * if there is an error, not much we can do apart
651 			 * from dropping the packet
652 			 */
653 			if (mhi_chan->pre_alloc) {
654 				if (mhi_queue_buf(mhi_chan->mhi_dev,
655 						  mhi_chan->dir,
656 						  buf_info->cb_buf,
657 						  buf_info->len, MHI_EOT)) {
658 					dev_err(dev,
659 						"Error recycling buffer for chan:%d\n",
660 						mhi_chan->chan);
661 					kfree(buf_info->cb_buf);
662 				}
663 			}
664 
665 			read_lock_bh(&mhi_chan->lock);
666 		}
667 		break;
668 	} /* CC_EOT */
669 	case MHI_EV_CC_OOB:
670 	case MHI_EV_CC_DB_MODE:
671 	{
672 		unsigned long pm_lock_flags;
673 
674 		mhi_chan->db_cfg.db_mode = 1;
675 		read_lock_irqsave(&mhi_cntrl->pm_lock, pm_lock_flags);
676 		if (tre_ring->wp != tre_ring->rp &&
677 		    MHI_DB_ACCESS_VALID(mhi_cntrl)) {
678 			mhi_ring_chan_db(mhi_cntrl, mhi_chan);
679 		}
680 		read_unlock_irqrestore(&mhi_cntrl->pm_lock, pm_lock_flags);
681 		break;
682 	}
683 	case MHI_EV_CC_BAD_TRE:
684 	default:
685 		dev_err(dev, "Unknown event 0x%x\n", ev_code);
686 		break;
687 	} /* switch(MHI_EV_READ_CODE(EV_TRB_CODE,event)) */
688 
689 end_process_tx_event:
690 	if (ev_code >= MHI_EV_CC_OOB)
691 		write_unlock_irqrestore(&mhi_chan->lock, flags);
692 	else
693 		read_unlock_bh(&mhi_chan->lock);
694 
695 	return 0;
696 }
697 
parse_rsc_event(struct mhi_controller * mhi_cntrl,struct mhi_ring_element * event,struct mhi_chan * mhi_chan)698 static int parse_rsc_event(struct mhi_controller *mhi_cntrl,
699 			   struct mhi_ring_element *event,
700 			   struct mhi_chan *mhi_chan)
701 {
702 	struct mhi_ring *buf_ring, *tre_ring;
703 	struct mhi_buf_info *buf_info;
704 	struct mhi_result result;
705 	int ev_code;
706 	u32 cookie; /* offset to local descriptor */
707 	u16 xfer_len;
708 
709 	buf_ring = &mhi_chan->buf_ring;
710 	tre_ring = &mhi_chan->tre_ring;
711 
712 	ev_code = MHI_TRE_GET_EV_CODE(event);
713 	cookie = MHI_TRE_GET_EV_COOKIE(event);
714 	xfer_len = MHI_TRE_GET_EV_LEN(event);
715 
716 	/* Received out of bound cookie */
717 	WARN_ON(cookie >= buf_ring->len);
718 
719 	buf_info = buf_ring->base + cookie;
720 
721 	result.transaction_status = (ev_code == MHI_EV_CC_OVERFLOW) ?
722 		-EOVERFLOW : 0;
723 
724 	/* truncate to buf len if xfer_len is larger */
725 	result.bytes_xferd = min_t(u16, xfer_len, buf_info->len);
726 	result.buf_addr = buf_info->cb_buf;
727 	result.dir = mhi_chan->dir;
728 
729 	read_lock_bh(&mhi_chan->lock);
730 
731 	if (mhi_chan->ch_state != MHI_CH_STATE_ENABLED)
732 		goto end_process_rsc_event;
733 
734 	WARN_ON(!buf_info->used);
735 
736 	/* notify the client */
737 	mhi_chan->xfer_cb(mhi_chan->mhi_dev, &result);
738 
739 	/*
740 	 * Note: We're arbitrarily incrementing RP even though, completion
741 	 * packet we processed might not be the same one, reason we can do this
742 	 * is because device guaranteed to cache descriptors in order it
743 	 * receive, so even though completion event is different we can re-use
744 	 * all descriptors in between.
745 	 * Example:
746 	 * Transfer Ring has descriptors: A, B, C, D
747 	 * Last descriptor host queue is D (WP) and first descriptor
748 	 * host queue is A (RP).
749 	 * The completion event we just serviced is descriptor C.
750 	 * Then we can safely queue descriptors to replace A, B, and C
751 	 * even though host did not receive any completions.
752 	 */
753 	mhi_del_ring_element(mhi_cntrl, tre_ring);
754 	buf_info->used = false;
755 
756 end_process_rsc_event:
757 	read_unlock_bh(&mhi_chan->lock);
758 
759 	return 0;
760 }
761 
mhi_process_cmd_completion(struct mhi_controller * mhi_cntrl,struct mhi_ring_element * tre)762 static void mhi_process_cmd_completion(struct mhi_controller *mhi_cntrl,
763 				       struct mhi_ring_element *tre)
764 {
765 	dma_addr_t ptr = MHI_TRE_GET_EV_PTR(tre);
766 	struct mhi_cmd *cmd_ring = &mhi_cntrl->mhi_cmd[PRIMARY_CMD_RING];
767 	struct mhi_ring *mhi_ring = &cmd_ring->ring;
768 	struct mhi_ring_element *cmd_pkt;
769 	struct mhi_chan *mhi_chan;
770 	u32 chan;
771 
772 	if (!is_valid_ring_ptr(mhi_ring, ptr)) {
773 		dev_err(&mhi_cntrl->mhi_dev->dev,
774 			"Event element points outside of the cmd ring\n");
775 		return;
776 	}
777 
778 	cmd_pkt = mhi_to_virtual(mhi_ring, ptr);
779 
780 	chan = MHI_TRE_GET_CMD_CHID(cmd_pkt);
781 
782 	if (chan < mhi_cntrl->max_chan &&
783 	    mhi_cntrl->mhi_chan[chan].configured) {
784 		mhi_chan = &mhi_cntrl->mhi_chan[chan];
785 		write_lock_bh(&mhi_chan->lock);
786 		mhi_chan->ccs = MHI_TRE_GET_EV_CODE(tre);
787 		complete(&mhi_chan->completion);
788 		write_unlock_bh(&mhi_chan->lock);
789 	} else {
790 		dev_err(&mhi_cntrl->mhi_dev->dev,
791 			"Completion packet for invalid channel ID: %d\n", chan);
792 	}
793 
794 	mhi_del_ring_element(mhi_cntrl, mhi_ring);
795 }
796 
mhi_process_ctrl_ev_ring(struct mhi_controller * mhi_cntrl,struct mhi_event * mhi_event,u32 event_quota)797 int mhi_process_ctrl_ev_ring(struct mhi_controller *mhi_cntrl,
798 			     struct mhi_event *mhi_event,
799 			     u32 event_quota)
800 {
801 	struct mhi_ring_element *dev_rp, *local_rp;
802 	struct mhi_ring *ev_ring = &mhi_event->ring;
803 	struct mhi_event_ctxt *er_ctxt =
804 		&mhi_cntrl->mhi_ctxt->er_ctxt[mhi_event->er_index];
805 	struct mhi_chan *mhi_chan;
806 	struct device *dev = &mhi_cntrl->mhi_dev->dev;
807 	u32 chan;
808 	int count = 0;
809 	dma_addr_t ptr = le64_to_cpu(er_ctxt->rp);
810 
811 	/*
812 	 * This is a quick check to avoid unnecessary event processing
813 	 * in case MHI is already in error state, but it's still possible
814 	 * to transition to error state while processing events
815 	 */
816 	if (unlikely(MHI_EVENT_ACCESS_INVALID(mhi_cntrl->pm_state)))
817 		return -EIO;
818 
819 	if (!is_valid_ring_ptr(ev_ring, ptr)) {
820 		dev_err(&mhi_cntrl->mhi_dev->dev,
821 			"Event ring rp points outside of the event ring\n");
822 		return -EIO;
823 	}
824 
825 	dev_rp = mhi_to_virtual(ev_ring, ptr);
826 	local_rp = ev_ring->rp;
827 
828 	while (dev_rp != local_rp) {
829 		enum mhi_pkt_type type = MHI_TRE_GET_EV_TYPE(local_rp);
830 
831 		switch (type) {
832 		case MHI_PKT_TYPE_BW_REQ_EVENT:
833 		{
834 			struct mhi_link_info *link_info;
835 
836 			link_info = &mhi_cntrl->mhi_link_info;
837 			write_lock_irq(&mhi_cntrl->pm_lock);
838 			link_info->target_link_speed =
839 				MHI_TRE_GET_EV_LINKSPEED(local_rp);
840 			link_info->target_link_width =
841 				MHI_TRE_GET_EV_LINKWIDTH(local_rp);
842 			write_unlock_irq(&mhi_cntrl->pm_lock);
843 			dev_dbg(dev, "Received BW_REQ event\n");
844 			mhi_cntrl->status_cb(mhi_cntrl, MHI_CB_BW_REQ);
845 			break;
846 		}
847 		case MHI_PKT_TYPE_STATE_CHANGE_EVENT:
848 		{
849 			enum mhi_state new_state;
850 
851 			new_state = MHI_TRE_GET_EV_STATE(local_rp);
852 
853 			dev_dbg(dev, "State change event to state: %s\n",
854 				TO_MHI_STATE_STR(new_state));
855 
856 			switch (new_state) {
857 			case MHI_STATE_M0:
858 				mhi_pm_m0_transition(mhi_cntrl);
859 				break;
860 			case MHI_STATE_M1:
861 				mhi_pm_m1_transition(mhi_cntrl);
862 				break;
863 			case MHI_STATE_M3:
864 				mhi_pm_m3_transition(mhi_cntrl);
865 				break;
866 			case MHI_STATE_SYS_ERR:
867 			{
868 				enum mhi_pm_state pm_state;
869 
870 				dev_dbg(dev, "System error detected\n");
871 				write_lock_irq(&mhi_cntrl->pm_lock);
872 				pm_state = mhi_tryset_pm_state(mhi_cntrl,
873 							MHI_PM_SYS_ERR_DETECT);
874 				write_unlock_irq(&mhi_cntrl->pm_lock);
875 				if (pm_state == MHI_PM_SYS_ERR_DETECT)
876 					mhi_pm_sys_err_handler(mhi_cntrl);
877 				break;
878 			}
879 			default:
880 				dev_err(dev, "Invalid state: %s\n",
881 					TO_MHI_STATE_STR(new_state));
882 			}
883 
884 			break;
885 		}
886 		case MHI_PKT_TYPE_CMD_COMPLETION_EVENT:
887 			mhi_process_cmd_completion(mhi_cntrl, local_rp);
888 			break;
889 		case MHI_PKT_TYPE_EE_EVENT:
890 		{
891 			enum dev_st_transition st = DEV_ST_TRANSITION_MAX;
892 			enum mhi_ee_type event = MHI_TRE_GET_EV_EXECENV(local_rp);
893 
894 			dev_dbg(dev, "Received EE event: %s\n",
895 				TO_MHI_EXEC_STR(event));
896 			switch (event) {
897 			case MHI_EE_SBL:
898 				st = DEV_ST_TRANSITION_SBL;
899 				break;
900 			case MHI_EE_WFW:
901 			case MHI_EE_AMSS:
902 				st = DEV_ST_TRANSITION_MISSION_MODE;
903 				break;
904 			case MHI_EE_FP:
905 				st = DEV_ST_TRANSITION_FP;
906 				break;
907 			case MHI_EE_RDDM:
908 				mhi_cntrl->status_cb(mhi_cntrl, MHI_CB_EE_RDDM);
909 				write_lock_irq(&mhi_cntrl->pm_lock);
910 				mhi_cntrl->ee = event;
911 				write_unlock_irq(&mhi_cntrl->pm_lock);
912 				wake_up_all(&mhi_cntrl->state_event);
913 				break;
914 			default:
915 				dev_err(dev,
916 					"Unhandled EE event: 0x%x\n", type);
917 			}
918 			if (st != DEV_ST_TRANSITION_MAX)
919 				mhi_queue_state_transition(mhi_cntrl, st);
920 
921 			break;
922 		}
923 		case MHI_PKT_TYPE_TX_EVENT:
924 			chan = MHI_TRE_GET_EV_CHID(local_rp);
925 
926 			WARN_ON(chan >= mhi_cntrl->max_chan);
927 
928 			/*
929 			 * Only process the event ring elements whose channel
930 			 * ID is within the maximum supported range.
931 			 */
932 			if (chan < mhi_cntrl->max_chan) {
933 				mhi_chan = &mhi_cntrl->mhi_chan[chan];
934 				if (!mhi_chan->configured)
935 					break;
936 				parse_xfer_event(mhi_cntrl, local_rp, mhi_chan);
937 				event_quota--;
938 			}
939 			break;
940 		default:
941 			dev_err(dev, "Unhandled event type: %d\n", type);
942 			break;
943 		}
944 
945 		mhi_recycle_ev_ring_element(mhi_cntrl, ev_ring);
946 		local_rp = ev_ring->rp;
947 
948 		ptr = le64_to_cpu(er_ctxt->rp);
949 		if (!is_valid_ring_ptr(ev_ring, ptr)) {
950 			dev_err(&mhi_cntrl->mhi_dev->dev,
951 				"Event ring rp points outside of the event ring\n");
952 			return -EIO;
953 		}
954 
955 		dev_rp = mhi_to_virtual(ev_ring, ptr);
956 		count++;
957 	}
958 
959 	read_lock_bh(&mhi_cntrl->pm_lock);
960 	if (likely(MHI_DB_ACCESS_VALID(mhi_cntrl)))
961 		mhi_ring_er_db(mhi_event);
962 	read_unlock_bh(&mhi_cntrl->pm_lock);
963 
964 	return count;
965 }
966 
mhi_process_data_event_ring(struct mhi_controller * mhi_cntrl,struct mhi_event * mhi_event,u32 event_quota)967 int mhi_process_data_event_ring(struct mhi_controller *mhi_cntrl,
968 				struct mhi_event *mhi_event,
969 				u32 event_quota)
970 {
971 	struct mhi_ring_element *dev_rp, *local_rp;
972 	struct mhi_ring *ev_ring = &mhi_event->ring;
973 	struct mhi_event_ctxt *er_ctxt =
974 		&mhi_cntrl->mhi_ctxt->er_ctxt[mhi_event->er_index];
975 	int count = 0;
976 	u32 chan;
977 	struct mhi_chan *mhi_chan;
978 	dma_addr_t ptr = le64_to_cpu(er_ctxt->rp);
979 
980 	if (unlikely(MHI_EVENT_ACCESS_INVALID(mhi_cntrl->pm_state)))
981 		return -EIO;
982 
983 	if (!is_valid_ring_ptr(ev_ring, ptr)) {
984 		dev_err(&mhi_cntrl->mhi_dev->dev,
985 			"Event ring rp points outside of the event ring\n");
986 		return -EIO;
987 	}
988 
989 	dev_rp = mhi_to_virtual(ev_ring, ptr);
990 	local_rp = ev_ring->rp;
991 
992 	while (dev_rp != local_rp && event_quota > 0) {
993 		enum mhi_pkt_type type = MHI_TRE_GET_EV_TYPE(local_rp);
994 
995 		chan = MHI_TRE_GET_EV_CHID(local_rp);
996 
997 		WARN_ON(chan >= mhi_cntrl->max_chan);
998 
999 		/*
1000 		 * Only process the event ring elements whose channel
1001 		 * ID is within the maximum supported range.
1002 		 */
1003 		if (chan < mhi_cntrl->max_chan &&
1004 		    mhi_cntrl->mhi_chan[chan].configured) {
1005 			mhi_chan = &mhi_cntrl->mhi_chan[chan];
1006 
1007 			if (likely(type == MHI_PKT_TYPE_TX_EVENT)) {
1008 				parse_xfer_event(mhi_cntrl, local_rp, mhi_chan);
1009 				event_quota--;
1010 			} else if (type == MHI_PKT_TYPE_RSC_TX_EVENT) {
1011 				parse_rsc_event(mhi_cntrl, local_rp, mhi_chan);
1012 				event_quota--;
1013 			}
1014 		}
1015 
1016 		mhi_recycle_ev_ring_element(mhi_cntrl, ev_ring);
1017 		local_rp = ev_ring->rp;
1018 
1019 		ptr = le64_to_cpu(er_ctxt->rp);
1020 		if (!is_valid_ring_ptr(ev_ring, ptr)) {
1021 			dev_err(&mhi_cntrl->mhi_dev->dev,
1022 				"Event ring rp points outside of the event ring\n");
1023 			return -EIO;
1024 		}
1025 
1026 		dev_rp = mhi_to_virtual(ev_ring, ptr);
1027 		count++;
1028 	}
1029 	read_lock_bh(&mhi_cntrl->pm_lock);
1030 	if (likely(MHI_DB_ACCESS_VALID(mhi_cntrl)))
1031 		mhi_ring_er_db(mhi_event);
1032 	read_unlock_bh(&mhi_cntrl->pm_lock);
1033 
1034 	return count;
1035 }
1036 
mhi_ev_task(unsigned long data)1037 void mhi_ev_task(unsigned long data)
1038 {
1039 	struct mhi_event *mhi_event = (struct mhi_event *)data;
1040 	struct mhi_controller *mhi_cntrl = mhi_event->mhi_cntrl;
1041 
1042 	/* process all pending events */
1043 	spin_lock_bh(&mhi_event->lock);
1044 	mhi_event->process_event(mhi_cntrl, mhi_event, U32_MAX);
1045 	spin_unlock_bh(&mhi_event->lock);
1046 }
1047 
mhi_ctrl_ev_task(unsigned long data)1048 void mhi_ctrl_ev_task(unsigned long data)
1049 {
1050 	struct mhi_event *mhi_event = (struct mhi_event *)data;
1051 	struct mhi_controller *mhi_cntrl = mhi_event->mhi_cntrl;
1052 	struct device *dev = &mhi_cntrl->mhi_dev->dev;
1053 	enum mhi_state state;
1054 	enum mhi_pm_state pm_state = 0;
1055 	int ret;
1056 
1057 	/*
1058 	 * We can check PM state w/o a lock here because there is no way
1059 	 * PM state can change from reg access valid to no access while this
1060 	 * thread being executed.
1061 	 */
1062 	if (!MHI_REG_ACCESS_VALID(mhi_cntrl->pm_state)) {
1063 		/*
1064 		 * We may have a pending event but not allowed to
1065 		 * process it since we are probably in a suspended state,
1066 		 * so trigger a resume.
1067 		 */
1068 		mhi_trigger_resume(mhi_cntrl);
1069 
1070 		return;
1071 	}
1072 
1073 	/* Process ctrl events events */
1074 	ret = mhi_event->process_event(mhi_cntrl, mhi_event, U32_MAX);
1075 
1076 	/*
1077 	 * We received an IRQ but no events to process, maybe device went to
1078 	 * SYS_ERR state? Check the state to confirm.
1079 	 */
1080 	if (!ret) {
1081 		write_lock_irq(&mhi_cntrl->pm_lock);
1082 		state = mhi_get_mhi_state(mhi_cntrl);
1083 		if (state == MHI_STATE_SYS_ERR) {
1084 			dev_dbg(dev, "System error detected\n");
1085 			pm_state = mhi_tryset_pm_state(mhi_cntrl,
1086 						       MHI_PM_SYS_ERR_DETECT);
1087 		}
1088 		write_unlock_irq(&mhi_cntrl->pm_lock);
1089 		if (pm_state == MHI_PM_SYS_ERR_DETECT)
1090 			mhi_pm_sys_err_handler(mhi_cntrl);
1091 	}
1092 }
1093 
mhi_is_ring_full(struct mhi_controller * mhi_cntrl,struct mhi_ring * ring)1094 static bool mhi_is_ring_full(struct mhi_controller *mhi_cntrl,
1095 			     struct mhi_ring *ring)
1096 {
1097 	void *tmp = ring->wp + ring->el_size;
1098 
1099 	if (tmp >= (ring->base + ring->len))
1100 		tmp = ring->base;
1101 
1102 	return (tmp == ring->rp);
1103 }
1104 
mhi_queue(struct mhi_device * mhi_dev,struct mhi_buf_info * buf_info,enum dma_data_direction dir,enum mhi_flags mflags)1105 static int mhi_queue(struct mhi_device *mhi_dev, struct mhi_buf_info *buf_info,
1106 		     enum dma_data_direction dir, enum mhi_flags mflags)
1107 {
1108 	struct mhi_controller *mhi_cntrl = mhi_dev->mhi_cntrl;
1109 	struct mhi_chan *mhi_chan = (dir == DMA_TO_DEVICE) ? mhi_dev->ul_chan :
1110 							     mhi_dev->dl_chan;
1111 	struct mhi_ring *tre_ring = &mhi_chan->tre_ring;
1112 	unsigned long flags;
1113 	int ret;
1114 
1115 	if (unlikely(MHI_PM_IN_ERROR_STATE(mhi_cntrl->pm_state)))
1116 		return -EIO;
1117 
1118 	ret = mhi_is_ring_full(mhi_cntrl, tre_ring);
1119 	if (unlikely(ret))
1120 		return -EAGAIN;
1121 
1122 	ret = mhi_gen_tre(mhi_cntrl, mhi_chan, buf_info, mflags);
1123 	if (unlikely(ret))
1124 		return ret;
1125 
1126 	read_lock_irqsave(&mhi_cntrl->pm_lock, flags);
1127 
1128 	/* Packet is queued, take a usage ref to exit M3 if necessary
1129 	 * for host->device buffer, balanced put is done on buffer completion
1130 	 * for device->host buffer, balanced put is after ringing the DB
1131 	 */
1132 	mhi_cntrl->runtime_get(mhi_cntrl);
1133 
1134 	/* Assert dev_wake (to exit/prevent M1/M2)*/
1135 	mhi_cntrl->wake_toggle(mhi_cntrl);
1136 
1137 	if (mhi_chan->dir == DMA_TO_DEVICE)
1138 		atomic_inc(&mhi_cntrl->pending_pkts);
1139 
1140 	if (likely(MHI_DB_ACCESS_VALID(mhi_cntrl)))
1141 		mhi_ring_chan_db(mhi_cntrl, mhi_chan);
1142 
1143 	if (dir == DMA_FROM_DEVICE)
1144 		mhi_cntrl->runtime_put(mhi_cntrl);
1145 
1146 	read_unlock_irqrestore(&mhi_cntrl->pm_lock, flags);
1147 
1148 	return ret;
1149 }
1150 
mhi_queue_skb(struct mhi_device * mhi_dev,enum dma_data_direction dir,struct sk_buff * skb,size_t len,enum mhi_flags mflags)1151 int mhi_queue_skb(struct mhi_device *mhi_dev, enum dma_data_direction dir,
1152 		  struct sk_buff *skb, size_t len, enum mhi_flags mflags)
1153 {
1154 	struct mhi_chan *mhi_chan = (dir == DMA_TO_DEVICE) ? mhi_dev->ul_chan :
1155 							     mhi_dev->dl_chan;
1156 	struct mhi_buf_info buf_info = { };
1157 
1158 	buf_info.v_addr = skb->data;
1159 	buf_info.cb_buf = skb;
1160 	buf_info.len = len;
1161 
1162 	if (unlikely(mhi_chan->pre_alloc))
1163 		return -EINVAL;
1164 
1165 	return mhi_queue(mhi_dev, &buf_info, dir, mflags);
1166 }
1167 EXPORT_SYMBOL_GPL(mhi_queue_skb);
1168 
mhi_queue_dma(struct mhi_device * mhi_dev,enum dma_data_direction dir,struct mhi_buf * mhi_buf,size_t len,enum mhi_flags mflags)1169 int mhi_queue_dma(struct mhi_device *mhi_dev, enum dma_data_direction dir,
1170 		  struct mhi_buf *mhi_buf, size_t len, enum mhi_flags mflags)
1171 {
1172 	struct mhi_chan *mhi_chan = (dir == DMA_TO_DEVICE) ? mhi_dev->ul_chan :
1173 							     mhi_dev->dl_chan;
1174 	struct mhi_buf_info buf_info = { };
1175 
1176 	buf_info.p_addr = mhi_buf->dma_addr;
1177 	buf_info.cb_buf = mhi_buf;
1178 	buf_info.pre_mapped = true;
1179 	buf_info.len = len;
1180 
1181 	if (unlikely(mhi_chan->pre_alloc))
1182 		return -EINVAL;
1183 
1184 	return mhi_queue(mhi_dev, &buf_info, dir, mflags);
1185 }
1186 EXPORT_SYMBOL_GPL(mhi_queue_dma);
1187 
mhi_gen_tre(struct mhi_controller * mhi_cntrl,struct mhi_chan * mhi_chan,struct mhi_buf_info * info,enum mhi_flags flags)1188 int mhi_gen_tre(struct mhi_controller *mhi_cntrl, struct mhi_chan *mhi_chan,
1189 			struct mhi_buf_info *info, enum mhi_flags flags)
1190 {
1191 	struct mhi_ring *buf_ring, *tre_ring;
1192 	struct mhi_ring_element *mhi_tre;
1193 	struct mhi_buf_info *buf_info;
1194 	int eot, eob, chain, bei;
1195 	int ret;
1196 
1197 	/* Protect accesses for reading and incrementing WP */
1198 	write_lock_bh(&mhi_chan->lock);
1199 
1200 	buf_ring = &mhi_chan->buf_ring;
1201 	tre_ring = &mhi_chan->tre_ring;
1202 
1203 	buf_info = buf_ring->wp;
1204 	WARN_ON(buf_info->used);
1205 	buf_info->pre_mapped = info->pre_mapped;
1206 	if (info->pre_mapped)
1207 		buf_info->p_addr = info->p_addr;
1208 	else
1209 		buf_info->v_addr = info->v_addr;
1210 	buf_info->cb_buf = info->cb_buf;
1211 	buf_info->wp = tre_ring->wp;
1212 	buf_info->dir = mhi_chan->dir;
1213 	buf_info->len = info->len;
1214 
1215 	if (!info->pre_mapped) {
1216 		ret = mhi_cntrl->map_single(mhi_cntrl, buf_info);
1217 		if (ret) {
1218 			write_unlock_bh(&mhi_chan->lock);
1219 			return ret;
1220 		}
1221 	}
1222 
1223 	eob = !!(flags & MHI_EOB);
1224 	eot = !!(flags & MHI_EOT);
1225 	chain = !!(flags & MHI_CHAIN);
1226 	bei = !!(mhi_chan->intmod);
1227 
1228 	mhi_tre = tre_ring->wp;
1229 	mhi_tre->ptr = MHI_TRE_DATA_PTR(buf_info->p_addr);
1230 	mhi_tre->dword[0] = MHI_TRE_DATA_DWORD0(info->len);
1231 	mhi_tre->dword[1] = MHI_TRE_DATA_DWORD1(bei, eot, eob, chain);
1232 
1233 	/* increment WP */
1234 	mhi_add_ring_element(mhi_cntrl, tre_ring);
1235 	mhi_add_ring_element(mhi_cntrl, buf_ring);
1236 
1237 	write_unlock_bh(&mhi_chan->lock);
1238 
1239 	return 0;
1240 }
1241 
mhi_queue_buf(struct mhi_device * mhi_dev,enum dma_data_direction dir,void * buf,size_t len,enum mhi_flags mflags)1242 int mhi_queue_buf(struct mhi_device *mhi_dev, enum dma_data_direction dir,
1243 		  void *buf, size_t len, enum mhi_flags mflags)
1244 {
1245 	struct mhi_buf_info buf_info = { };
1246 
1247 	buf_info.v_addr = buf;
1248 	buf_info.cb_buf = buf;
1249 	buf_info.len = len;
1250 
1251 	return mhi_queue(mhi_dev, &buf_info, dir, mflags);
1252 }
1253 EXPORT_SYMBOL_GPL(mhi_queue_buf);
1254 
mhi_queue_is_full(struct mhi_device * mhi_dev,enum dma_data_direction dir)1255 bool mhi_queue_is_full(struct mhi_device *mhi_dev, enum dma_data_direction dir)
1256 {
1257 	struct mhi_controller *mhi_cntrl = mhi_dev->mhi_cntrl;
1258 	struct mhi_chan *mhi_chan = (dir == DMA_TO_DEVICE) ?
1259 					mhi_dev->ul_chan : mhi_dev->dl_chan;
1260 	struct mhi_ring *tre_ring = &mhi_chan->tre_ring;
1261 
1262 	return mhi_is_ring_full(mhi_cntrl, tre_ring);
1263 }
1264 EXPORT_SYMBOL_GPL(mhi_queue_is_full);
1265 
mhi_send_cmd(struct mhi_controller * mhi_cntrl,struct mhi_chan * mhi_chan,enum mhi_cmd_type cmd)1266 int mhi_send_cmd(struct mhi_controller *mhi_cntrl,
1267 		 struct mhi_chan *mhi_chan,
1268 		 enum mhi_cmd_type cmd)
1269 {
1270 	struct mhi_ring_element *cmd_tre = NULL;
1271 	struct mhi_cmd *mhi_cmd = &mhi_cntrl->mhi_cmd[PRIMARY_CMD_RING];
1272 	struct mhi_ring *ring = &mhi_cmd->ring;
1273 	struct device *dev = &mhi_cntrl->mhi_dev->dev;
1274 	int chan = 0;
1275 
1276 	if (mhi_chan)
1277 		chan = mhi_chan->chan;
1278 
1279 	spin_lock_bh(&mhi_cmd->lock);
1280 	if (!get_nr_avail_ring_elements(mhi_cntrl, ring)) {
1281 		spin_unlock_bh(&mhi_cmd->lock);
1282 		return -ENOMEM;
1283 	}
1284 
1285 	/* prepare the cmd tre */
1286 	cmd_tre = ring->wp;
1287 	switch (cmd) {
1288 	case MHI_CMD_RESET_CHAN:
1289 		cmd_tre->ptr = MHI_TRE_CMD_RESET_PTR;
1290 		cmd_tre->dword[0] = MHI_TRE_CMD_RESET_DWORD0;
1291 		cmd_tre->dword[1] = MHI_TRE_CMD_RESET_DWORD1(chan);
1292 		break;
1293 	case MHI_CMD_STOP_CHAN:
1294 		cmd_tre->ptr = MHI_TRE_CMD_STOP_PTR;
1295 		cmd_tre->dword[0] = MHI_TRE_CMD_STOP_DWORD0;
1296 		cmd_tre->dword[1] = MHI_TRE_CMD_STOP_DWORD1(chan);
1297 		break;
1298 	case MHI_CMD_START_CHAN:
1299 		cmd_tre->ptr = MHI_TRE_CMD_START_PTR;
1300 		cmd_tre->dword[0] = MHI_TRE_CMD_START_DWORD0;
1301 		cmd_tre->dword[1] = MHI_TRE_CMD_START_DWORD1(chan);
1302 		break;
1303 	default:
1304 		dev_err(dev, "Command not supported\n");
1305 		break;
1306 	}
1307 
1308 	/* queue to hardware */
1309 	mhi_add_ring_element(mhi_cntrl, ring);
1310 	read_lock_bh(&mhi_cntrl->pm_lock);
1311 	if (likely(MHI_DB_ACCESS_VALID(mhi_cntrl)))
1312 		mhi_ring_cmd_db(mhi_cntrl, mhi_cmd);
1313 	read_unlock_bh(&mhi_cntrl->pm_lock);
1314 	spin_unlock_bh(&mhi_cmd->lock);
1315 
1316 	return 0;
1317 }
1318 
mhi_update_channel_state(struct mhi_controller * mhi_cntrl,struct mhi_chan * mhi_chan,enum mhi_ch_state_type to_state)1319 static int mhi_update_channel_state(struct mhi_controller *mhi_cntrl,
1320 				    struct mhi_chan *mhi_chan,
1321 				    enum mhi_ch_state_type to_state)
1322 {
1323 	struct device *dev = &mhi_chan->mhi_dev->dev;
1324 	enum mhi_cmd_type cmd = MHI_CMD_NOP;
1325 	int ret;
1326 
1327 	dev_dbg(dev, "%d: Updating channel state to: %s\n", mhi_chan->chan,
1328 		TO_CH_STATE_TYPE_STR(to_state));
1329 
1330 	switch (to_state) {
1331 	case MHI_CH_STATE_TYPE_RESET:
1332 		write_lock_irq(&mhi_chan->lock);
1333 		if (mhi_chan->ch_state != MHI_CH_STATE_STOP &&
1334 		    mhi_chan->ch_state != MHI_CH_STATE_ENABLED &&
1335 		    mhi_chan->ch_state != MHI_CH_STATE_SUSPENDED) {
1336 			write_unlock_irq(&mhi_chan->lock);
1337 			return -EINVAL;
1338 		}
1339 		mhi_chan->ch_state = MHI_CH_STATE_DISABLED;
1340 		write_unlock_irq(&mhi_chan->lock);
1341 
1342 		cmd = MHI_CMD_RESET_CHAN;
1343 		break;
1344 	case MHI_CH_STATE_TYPE_STOP:
1345 		if (mhi_chan->ch_state != MHI_CH_STATE_ENABLED)
1346 			return -EINVAL;
1347 
1348 		cmd = MHI_CMD_STOP_CHAN;
1349 		break;
1350 	case MHI_CH_STATE_TYPE_START:
1351 		if (mhi_chan->ch_state != MHI_CH_STATE_STOP &&
1352 		    mhi_chan->ch_state != MHI_CH_STATE_DISABLED)
1353 			return -EINVAL;
1354 
1355 		cmd = MHI_CMD_START_CHAN;
1356 		break;
1357 	default:
1358 		dev_err(dev, "%d: Channel state update to %s not allowed\n",
1359 			mhi_chan->chan, TO_CH_STATE_TYPE_STR(to_state));
1360 		return -EINVAL;
1361 	}
1362 
1363 	/* bring host and device out of suspended states */
1364 	ret = mhi_device_get_sync(mhi_cntrl->mhi_dev);
1365 	if (ret)
1366 		return ret;
1367 	mhi_cntrl->runtime_get(mhi_cntrl);
1368 
1369 	reinit_completion(&mhi_chan->completion);
1370 	ret = mhi_send_cmd(mhi_cntrl, mhi_chan, cmd);
1371 	if (ret) {
1372 		dev_err(dev, "%d: Failed to send %s channel command\n",
1373 			mhi_chan->chan, TO_CH_STATE_TYPE_STR(to_state));
1374 		goto exit_channel_update;
1375 	}
1376 
1377 	ret = wait_for_completion_timeout(&mhi_chan->completion,
1378 				       msecs_to_jiffies(mhi_cntrl->timeout_ms));
1379 	if (!ret || mhi_chan->ccs != MHI_EV_CC_SUCCESS) {
1380 		dev_err(dev,
1381 			"%d: Failed to receive %s channel command completion\n",
1382 			mhi_chan->chan, TO_CH_STATE_TYPE_STR(to_state));
1383 		ret = -EIO;
1384 		goto exit_channel_update;
1385 	}
1386 
1387 	ret = 0;
1388 
1389 	if (to_state != MHI_CH_STATE_TYPE_RESET) {
1390 		write_lock_irq(&mhi_chan->lock);
1391 		mhi_chan->ch_state = (to_state == MHI_CH_STATE_TYPE_START) ?
1392 				      MHI_CH_STATE_ENABLED : MHI_CH_STATE_STOP;
1393 		write_unlock_irq(&mhi_chan->lock);
1394 	}
1395 
1396 	dev_dbg(dev, "%d: Channel state change to %s successful\n",
1397 		mhi_chan->chan, TO_CH_STATE_TYPE_STR(to_state));
1398 
1399 exit_channel_update:
1400 	mhi_cntrl->runtime_put(mhi_cntrl);
1401 	mhi_device_put(mhi_cntrl->mhi_dev);
1402 
1403 	return ret;
1404 }
1405 
mhi_unprepare_channel(struct mhi_controller * mhi_cntrl,struct mhi_chan * mhi_chan)1406 static void mhi_unprepare_channel(struct mhi_controller *mhi_cntrl,
1407 				  struct mhi_chan *mhi_chan)
1408 {
1409 	int ret;
1410 	struct device *dev = &mhi_chan->mhi_dev->dev;
1411 
1412 	mutex_lock(&mhi_chan->mutex);
1413 
1414 	if (!(BIT(mhi_cntrl->ee) & mhi_chan->ee_mask)) {
1415 		dev_dbg(dev, "Current EE: %s Required EE Mask: 0x%x\n",
1416 			TO_MHI_EXEC_STR(mhi_cntrl->ee), mhi_chan->ee_mask);
1417 		goto exit_unprepare_channel;
1418 	}
1419 
1420 	/* no more processing events for this channel */
1421 	ret = mhi_update_channel_state(mhi_cntrl, mhi_chan,
1422 				       MHI_CH_STATE_TYPE_RESET);
1423 	if (ret)
1424 		dev_err(dev, "%d: Failed to reset channel, still resetting\n",
1425 			mhi_chan->chan);
1426 
1427 exit_unprepare_channel:
1428 	write_lock_irq(&mhi_chan->lock);
1429 	mhi_chan->ch_state = MHI_CH_STATE_DISABLED;
1430 	write_unlock_irq(&mhi_chan->lock);
1431 
1432 	if (!mhi_chan->offload_ch) {
1433 		mhi_reset_chan(mhi_cntrl, mhi_chan);
1434 		mhi_deinit_chan_ctxt(mhi_cntrl, mhi_chan);
1435 	}
1436 	dev_dbg(dev, "%d: successfully reset\n", mhi_chan->chan);
1437 
1438 	mutex_unlock(&mhi_chan->mutex);
1439 }
1440 
mhi_prepare_channel(struct mhi_controller * mhi_cntrl,struct mhi_chan * mhi_chan)1441 int mhi_prepare_channel(struct mhi_controller *mhi_cntrl,
1442 			struct mhi_chan *mhi_chan)
1443 {
1444 	int ret = 0;
1445 	struct device *dev = &mhi_chan->mhi_dev->dev;
1446 
1447 	if (!(BIT(mhi_cntrl->ee) & mhi_chan->ee_mask)) {
1448 		dev_err(dev, "Current EE: %s Required EE Mask: 0x%x\n",
1449 			TO_MHI_EXEC_STR(mhi_cntrl->ee), mhi_chan->ee_mask);
1450 		return -ENOTCONN;
1451 	}
1452 
1453 	mutex_lock(&mhi_chan->mutex);
1454 
1455 	/* Check of client manages channel context for offload channels */
1456 	if (!mhi_chan->offload_ch) {
1457 		ret = mhi_init_chan_ctxt(mhi_cntrl, mhi_chan);
1458 		if (ret)
1459 			goto error_init_chan;
1460 	}
1461 
1462 	ret = mhi_update_channel_state(mhi_cntrl, mhi_chan,
1463 				       MHI_CH_STATE_TYPE_START);
1464 	if (ret)
1465 		goto error_pm_state;
1466 
1467 	/* Pre-allocate buffer for xfer ring */
1468 	if (mhi_chan->pre_alloc) {
1469 		int nr_el = get_nr_avail_ring_elements(mhi_cntrl,
1470 						       &mhi_chan->tre_ring);
1471 		size_t len = mhi_cntrl->buffer_len;
1472 
1473 		while (nr_el--) {
1474 			void *buf;
1475 			struct mhi_buf_info info = { };
1476 			buf = kmalloc(len, GFP_KERNEL);
1477 			if (!buf) {
1478 				ret = -ENOMEM;
1479 				goto error_pre_alloc;
1480 			}
1481 
1482 			/* Prepare transfer descriptors */
1483 			info.v_addr = buf;
1484 			info.cb_buf = buf;
1485 			info.len = len;
1486 			ret = mhi_gen_tre(mhi_cntrl, mhi_chan, &info, MHI_EOT);
1487 			if (ret) {
1488 				kfree(buf);
1489 				goto error_pre_alloc;
1490 			}
1491 		}
1492 
1493 		read_lock_bh(&mhi_cntrl->pm_lock);
1494 		if (MHI_DB_ACCESS_VALID(mhi_cntrl)) {
1495 			read_lock_irq(&mhi_chan->lock);
1496 			mhi_ring_chan_db(mhi_cntrl, mhi_chan);
1497 			read_unlock_irq(&mhi_chan->lock);
1498 		}
1499 		read_unlock_bh(&mhi_cntrl->pm_lock);
1500 	}
1501 
1502 	mutex_unlock(&mhi_chan->mutex);
1503 
1504 	return 0;
1505 
1506 error_pm_state:
1507 	if (!mhi_chan->offload_ch)
1508 		mhi_deinit_chan_ctxt(mhi_cntrl, mhi_chan);
1509 
1510 error_init_chan:
1511 	mutex_unlock(&mhi_chan->mutex);
1512 
1513 	return ret;
1514 
1515 error_pre_alloc:
1516 	mutex_unlock(&mhi_chan->mutex);
1517 	mhi_unprepare_channel(mhi_cntrl, mhi_chan);
1518 
1519 	return ret;
1520 }
1521 
mhi_mark_stale_events(struct mhi_controller * mhi_cntrl,struct mhi_event * mhi_event,struct mhi_event_ctxt * er_ctxt,int chan)1522 static void mhi_mark_stale_events(struct mhi_controller *mhi_cntrl,
1523 				  struct mhi_event *mhi_event,
1524 				  struct mhi_event_ctxt *er_ctxt,
1525 				  int chan)
1526 
1527 {
1528 	struct mhi_ring_element *dev_rp, *local_rp;
1529 	struct mhi_ring *ev_ring;
1530 	struct device *dev = &mhi_cntrl->mhi_dev->dev;
1531 	unsigned long flags;
1532 	dma_addr_t ptr;
1533 
1534 	dev_dbg(dev, "Marking all events for chan: %d as stale\n", chan);
1535 
1536 	ev_ring = &mhi_event->ring;
1537 
1538 	/* mark all stale events related to channel as STALE event */
1539 	spin_lock_irqsave(&mhi_event->lock, flags);
1540 
1541 	ptr = le64_to_cpu(er_ctxt->rp);
1542 	if (!is_valid_ring_ptr(ev_ring, ptr)) {
1543 		dev_err(&mhi_cntrl->mhi_dev->dev,
1544 			"Event ring rp points outside of the event ring\n");
1545 		dev_rp = ev_ring->rp;
1546 	} else {
1547 		dev_rp = mhi_to_virtual(ev_ring, ptr);
1548 	}
1549 
1550 	local_rp = ev_ring->rp;
1551 	while (dev_rp != local_rp) {
1552 		if (MHI_TRE_GET_EV_TYPE(local_rp) == MHI_PKT_TYPE_TX_EVENT &&
1553 		    chan == MHI_TRE_GET_EV_CHID(local_rp))
1554 			local_rp->dword[1] = MHI_TRE_EV_DWORD1(chan,
1555 					MHI_PKT_TYPE_STALE_EVENT);
1556 		local_rp++;
1557 		if (local_rp == (ev_ring->base + ev_ring->len))
1558 			local_rp = ev_ring->base;
1559 	}
1560 
1561 	dev_dbg(dev, "Finished marking events as stale events\n");
1562 	spin_unlock_irqrestore(&mhi_event->lock, flags);
1563 }
1564 
mhi_reset_data_chan(struct mhi_controller * mhi_cntrl,struct mhi_chan * mhi_chan)1565 static void mhi_reset_data_chan(struct mhi_controller *mhi_cntrl,
1566 				struct mhi_chan *mhi_chan)
1567 {
1568 	struct mhi_ring *buf_ring, *tre_ring;
1569 	struct mhi_result result;
1570 
1571 	/* Reset any pending buffers */
1572 	buf_ring = &mhi_chan->buf_ring;
1573 	tre_ring = &mhi_chan->tre_ring;
1574 	result.transaction_status = -ENOTCONN;
1575 	result.bytes_xferd = 0;
1576 	while (tre_ring->rp != tre_ring->wp) {
1577 		struct mhi_buf_info *buf_info = buf_ring->rp;
1578 
1579 		if (mhi_chan->dir == DMA_TO_DEVICE) {
1580 			atomic_dec(&mhi_cntrl->pending_pkts);
1581 			/* Release the reference got from mhi_queue() */
1582 			mhi_cntrl->runtime_put(mhi_cntrl);
1583 		}
1584 
1585 		if (!buf_info->pre_mapped)
1586 			mhi_cntrl->unmap_single(mhi_cntrl, buf_info);
1587 
1588 		mhi_del_ring_element(mhi_cntrl, buf_ring);
1589 		mhi_del_ring_element(mhi_cntrl, tre_ring);
1590 
1591 		if (mhi_chan->pre_alloc) {
1592 			kfree(buf_info->cb_buf);
1593 		} else {
1594 			result.buf_addr = buf_info->cb_buf;
1595 			mhi_chan->xfer_cb(mhi_chan->mhi_dev, &result);
1596 		}
1597 	}
1598 }
1599 
mhi_reset_chan(struct mhi_controller * mhi_cntrl,struct mhi_chan * mhi_chan)1600 void mhi_reset_chan(struct mhi_controller *mhi_cntrl, struct mhi_chan *mhi_chan)
1601 {
1602 	struct mhi_event *mhi_event;
1603 	struct mhi_event_ctxt *er_ctxt;
1604 	int chan = mhi_chan->chan;
1605 
1606 	/* Nothing to reset, client doesn't queue buffers */
1607 	if (mhi_chan->offload_ch)
1608 		return;
1609 
1610 	read_lock_bh(&mhi_cntrl->pm_lock);
1611 	mhi_event = &mhi_cntrl->mhi_event[mhi_chan->er_index];
1612 	er_ctxt = &mhi_cntrl->mhi_ctxt->er_ctxt[mhi_chan->er_index];
1613 
1614 	mhi_mark_stale_events(mhi_cntrl, mhi_event, er_ctxt, chan);
1615 
1616 	mhi_reset_data_chan(mhi_cntrl, mhi_chan);
1617 
1618 	read_unlock_bh(&mhi_cntrl->pm_lock);
1619 }
1620 
1621 /* Move channel to start state */
mhi_prepare_for_transfer(struct mhi_device * mhi_dev)1622 int mhi_prepare_for_transfer(struct mhi_device *mhi_dev)
1623 {
1624 	int ret, dir;
1625 	struct mhi_controller *mhi_cntrl = mhi_dev->mhi_cntrl;
1626 	struct mhi_chan *mhi_chan;
1627 
1628 	for (dir = 0; dir < 2; dir++) {
1629 		mhi_chan = dir ? mhi_dev->dl_chan : mhi_dev->ul_chan;
1630 		if (!mhi_chan)
1631 			continue;
1632 
1633 		ret = mhi_prepare_channel(mhi_cntrl, mhi_chan);
1634 		if (ret)
1635 			goto error_open_chan;
1636 	}
1637 
1638 	return 0;
1639 
1640 error_open_chan:
1641 	for (--dir; dir >= 0; dir--) {
1642 		mhi_chan = dir ? mhi_dev->dl_chan : mhi_dev->ul_chan;
1643 		if (!mhi_chan)
1644 			continue;
1645 
1646 		mhi_unprepare_channel(mhi_cntrl, mhi_chan);
1647 	}
1648 
1649 	return ret;
1650 }
1651 EXPORT_SYMBOL_GPL(mhi_prepare_for_transfer);
1652 
mhi_unprepare_from_transfer(struct mhi_device * mhi_dev)1653 void mhi_unprepare_from_transfer(struct mhi_device *mhi_dev)
1654 {
1655 	struct mhi_controller *mhi_cntrl = mhi_dev->mhi_cntrl;
1656 	struct mhi_chan *mhi_chan;
1657 	int dir;
1658 
1659 	for (dir = 0; dir < 2; dir++) {
1660 		mhi_chan = dir ? mhi_dev->ul_chan : mhi_dev->dl_chan;
1661 		if (!mhi_chan)
1662 			continue;
1663 
1664 		mhi_unprepare_channel(mhi_cntrl, mhi_chan);
1665 	}
1666 }
1667 EXPORT_SYMBOL_GPL(mhi_unprepare_from_transfer);
1668 
mhi_poll(struct mhi_device * mhi_dev,u32 budget)1669 int mhi_poll(struct mhi_device *mhi_dev, u32 budget)
1670 {
1671 	struct mhi_controller *mhi_cntrl = mhi_dev->mhi_cntrl;
1672 	struct mhi_chan *mhi_chan = mhi_dev->dl_chan;
1673 	struct mhi_event *mhi_event = &mhi_cntrl->mhi_event[mhi_chan->er_index];
1674 	int ret;
1675 
1676 	spin_lock_bh(&mhi_event->lock);
1677 	ret = mhi_event->process_event(mhi_cntrl, mhi_event, budget);
1678 	spin_unlock_bh(&mhi_event->lock);
1679 
1680 	return ret;
1681 }
1682 EXPORT_SYMBOL_GPL(mhi_poll);
1683