• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 // SPDX-License-Identifier: GPL-2.0
2 /* Copyright (c) 2021 Rockchip Electronics Co., Ltd. */
3 
4 #include <linux/delay.h>
5 #include <linux/pm_runtime.h>
6 #include <media/v4l2-common.h>
7 #include <media/v4l2-event.h>
8 #include <media/v4l2-fh.h>
9 #include <media/v4l2-ioctl.h>
10 #include <media/v4l2-subdev.h>
11 #include <media/videobuf2-dma-contig.h>
12 #include <media/videobuf2-dma-sg.h>
13 #include "dev.h"
14 #include "regs.h"
15 
16 #define CIF_ISP_REQ_BUFS_MIN 0
17 
18 static int mi_frame_end(struct rkisp_stream *stream);
19 
20 static const struct capture_fmt fbc_fmts[] = {
21 	{
22 		.fourcc = V4L2_PIX_FMT_FBC0,
23 		.fmt_type = FMT_FBC,
24 		.bpp = { 8, 16 },
25 		.cplanes = 2,
26 		.mplanes = 1,
27 		.write_format = ISP3X_MPFBC_YUV420,
28 	}, {
29 		.fourcc = V4L2_PIX_FMT_FBC2,
30 		.fmt_type = FMT_FBC,
31 		.bpp = { 8, 16 },
32 		.cplanes = 2,
33 		.mplanes = 1,
34 		.write_format = ISP3X_MPFBC_YUV422,
35 	}
36 };
37 
38 static const struct capture_fmt bp_fmts[] = {
39 	{
40 		.fourcc = V4L2_PIX_FMT_UYVY,
41 		.fmt_type = FMT_YUV,
42 		.bpp = { 16 },
43 		.cplanes = 1,
44 		.mplanes = 1,
45 		.write_format = ISP3X_BP_FORMAT_INT,
46 	}, {
47 		.fourcc = V4L2_PIX_FMT_NV16,
48 		.fmt_type = FMT_YUV,
49 		.bpp = { 8, 16 },
50 		.cplanes = 2,
51 		.mplanes = 1,
52 		.write_format = ISP3X_BP_FORMAT_SPLA,
53 	}, {
54 		.fourcc = V4L2_PIX_FMT_NV12,
55 		.fmt_type = FMT_YUV,
56 		.bpp = { 8, 16 },
57 		.cplanes = 2,
58 		.mplanes = 1,
59 		.write_format = ISP3X_BP_FORMAT_SPLA,
60 	}, {
61 		.fourcc = V4L2_PIX_FMT_NV12M,
62 		.fmt_type = FMT_YUV,
63 		.bpp = { 8, 16 },
64 		.cplanes = 2,
65 		.mplanes = 2,
66 		.write_format = ISP3X_BP_FORMAT_SPLA,
67 	}
68 };
69 
70 struct stream_config rkisp_fbc_stream_config = {
71 	.fmts = fbc_fmts,
72 	.fmt_size = ARRAY_SIZE(fbc_fmts),
73 	.frame_end_id = ISP3X_MI_MPFBC_FRAME,
74 	.dual_crop = {
75 		.ctrl = ISP3X_DUAL_CROP_CTRL,
76 		.yuvmode_mask = ISP3X_DUAL_CROP_FBC_MODE,
77 		.h_offset = ISP3X_DUAL_CROP_FBC_H_OFFS,
78 		.v_offset = ISP3X_DUAL_CROP_FBC_V_OFFS,
79 		.h_size = ISP3X_DUAL_CROP_FBC_H_SIZE,
80 		.v_size = ISP3X_DUAL_CROP_FBC_V_SIZE,
81 	},
82 	.mi = {
83 		.y_base_ad_init = ISP3X_MPFBC_HEAD_PTR,
84 		.cb_base_ad_init = ISP3X_MPFBC_PAYL_PTR,
85 		.y_base_ad_shd = ISP3X_MPFBC_HEAD_PTR,
86 	},
87 };
88 
89 struct stream_config rkisp_bp_stream_config = {
90 	.fmts = bp_fmts,
91 	.fmt_size = ARRAY_SIZE(bp_fmts),
92 	.frame_end_id = ISP3X_MI_BP_FRAME,
93 	.dual_crop = {
94 		.ctrl = ISP3X_DUAL_CROP_CTRL,
95 		.yuvmode_mask = ISP3X_DUAL_CROP_FBC_MODE,
96 		.h_offset = ISP3X_DUAL_CROP_FBC_H_OFFS,
97 		.v_offset = ISP3X_DUAL_CROP_FBC_V_OFFS,
98 		.h_size = ISP3X_DUAL_CROP_FBC_H_SIZE,
99 		.v_size = ISP3X_DUAL_CROP_FBC_V_SIZE,
100 	},
101 	.mi = {
102 		.y_size_init = ISP3X_MI_BP_WR_Y_SIZE,
103 		.cb_size_init = ISP3X_MI_BP_WR_CB_SIZE,
104 		.y_base_ad_init = ISP3X_MI_BP_WR_Y_BASE,
105 		.cb_base_ad_init = ISP3X_MI_BP_WR_CB_BASE,
106 		.y_offs_cnt_init = ISP3X_MI_BP_WR_Y_OFFS_CNT,
107 		.cb_offs_cnt_init = ISP3X_MI_BP_WR_CB_OFFS_CNT,
108 		.y_base_ad_shd = ISP3X_MI_BP_WR_Y_BASE_SHD,
109 	},
110 };
111 
is_fbc_stream_stopped(void __iomem * base)112 static bool is_fbc_stream_stopped(void __iomem *base)
113 {
114 	u32 ret = readl(base + ISP3X_MPFBC_CTRL);
115 
116 	return !(ret & ISP3X_MPFBC_EN_SHD);
117 }
118 
get_stream_irq_mask(struct rkisp_stream * stream)119 static int get_stream_irq_mask(struct rkisp_stream *stream)
120 {
121 	int ret;
122 
123 	switch (stream->id) {
124 	case RKISP_STREAM_SP:
125 		ret = ISP_FRAME_SP;
126 		break;
127 	case RKISP_STREAM_FBC:
128 		ret = ISP_FRAME_MPFBC;
129 		break;
130 	case RKISP_STREAM_BP:
131 		ret = ISP_FRAME_BP;
132 		break;
133 	case RKISP_STREAM_MP:
134 	default:
135 		ret = ISP_FRAME_MP;
136 	}
137 
138 	return ret;
139 }
140 
141 /* configure dual-crop unit */
rkisp_stream_config_dcrop(struct rkisp_stream * stream,bool async)142 static int rkisp_stream_config_dcrop(struct rkisp_stream *stream, bool async)
143 {
144 	struct rkisp_device *dev = stream->ispdev;
145 	struct v4l2_rect *dcrop = &stream->dcrop;
146 	struct v4l2_rect *input_win;
147 
148 	/* dual-crop unit get data from isp */
149 	input_win = rkisp_get_isp_sd_win(&dev->isp_sdev);
150 
151 	if (dcrop->width == input_win->width &&
152 	    dcrop->height == input_win->height &&
153 	    dcrop->left == 0 && dcrop->top == 0 &&
154 	    !dev->hw_dev->is_unite) {
155 		rkisp_disable_dcrop(stream, async);
156 		v4l2_dbg(1, rkisp_debug, &dev->v4l2_dev,
157 			 "stream %d crop disabled\n", stream->id);
158 		return 0;
159 	}
160 
161 	v4l2_dbg(1, rkisp_debug, &dev->v4l2_dev,
162 		 "stream %d crop: %dx%d -> %dx%d\n", stream->id,
163 		 input_win->width, input_win->height,
164 		 dcrop->width, dcrop->height);
165 
166 	rkisp_config_dcrop(stream, dcrop, async);
167 
168 	return 0;
169 }
170 
171 /* configure scale unit */
rkisp_stream_config_rsz(struct rkisp_stream * stream,bool async)172 static int rkisp_stream_config_rsz(struct rkisp_stream *stream, bool async)
173 {
174 	struct rkisp_device *dev = stream->ispdev;
175 	struct v4l2_pix_format_mplane output_fmt = stream->out_fmt;
176 	struct ispsd_out_fmt *input_isp_fmt =
177 			rkisp_get_ispsd_out_fmt(&dev->isp_sdev);
178 	struct v4l2_rect in_y, in_c, out_y, out_c;
179 
180 	if (input_isp_fmt->fmt_type == FMT_BAYER ||
181 	    stream->id == RKISP_STREAM_FBC ||
182 	    stream->id == RKISP_STREAM_BP)
183 		goto disable;
184 
185 	/* set input and output sizes for scale calculation
186 	 * input/output yuv422
187 	 */
188 	in_y.width = stream->dcrop.width;
189 	in_y.height = stream->dcrop.height;
190 	in_c.width = in_y.width / 2;
191 	in_c.height = in_y.height;
192 
193 	out_y.width = output_fmt.width;
194 	out_y.height = output_fmt.height;
195 	out_c.width = out_y.width / 2;
196 	out_c.height = out_y.height;
197 
198 	if (in_c.width == out_c.width && in_c.height == out_c.height)
199 		goto disable;
200 
201 	/* set RSZ input and output */
202 	v4l2_dbg(1, rkisp_debug, &dev->v4l2_dev,
203 		 "stream %d rsz/scale: %dx%d -> %dx%d\n",
204 		 stream->id, stream->dcrop.width, stream->dcrop.height,
205 		 output_fmt.width, output_fmt.height);
206 	v4l2_dbg(1, rkisp_debug, &dev->v4l2_dev,
207 		 "chroma scaling %dx%d -> %dx%d\n",
208 		 in_c.width, in_c.height, out_c.width, out_c.height);
209 
210 	/* calculate and set scale */
211 	rkisp_config_rsz(stream, &in_y, &in_c, &out_y, &out_c, async);
212 
213 	return 0;
214 
215 disable:
216 	rkisp_disable_rsz(stream, async);
217 
218 	return 0;
219 }
220 
221 /***************************** stream operations*******************************/
222 
223 /*
224  * memory base addresses should be with respect
225  * to the burst alignment restriction for AXI.
226  */
calc_burst_len(struct rkisp_stream * stream)227 static u32 calc_burst_len(struct rkisp_stream *stream)
228 {
229 	struct rkisp_device *dev = stream->ispdev;
230 	u32 y_size = stream->out_fmt.plane_fmt[0].bytesperline *
231 		stream->out_fmt.height;
232 	u32 cb_size = stream->out_fmt.plane_fmt[1].sizeimage;
233 	u32 cr_size = stream->out_fmt.plane_fmt[2].sizeimage;
234 	u32 cb_offs, cr_offs;
235 	u32 bus = 16, burst;
236 	int i;
237 
238 	/* y/c base addr: burstN * bus alignment */
239 	cb_offs = y_size;
240 	cr_offs = cr_size ? (cb_size + cb_offs) : 0;
241 
242 	if (!(cb_offs % (bus * 16)) && !(cr_offs % (bus * 16)))
243 		burst = CIF_MI_CTRL_BURST_LEN_LUM_16 |
244 			CIF_MI_CTRL_BURST_LEN_CHROM_16;
245 	else if (!(cb_offs % (bus * 8)) && !(cr_offs % (bus * 8)))
246 		burst = CIF_MI_CTRL_BURST_LEN_LUM_8 |
247 			CIF_MI_CTRL_BURST_LEN_CHROM_8;
248 	else
249 		burst = CIF_MI_CTRL_BURST_LEN_LUM_4 |
250 			CIF_MI_CTRL_BURST_LEN_CHROM_4;
251 
252 	if (cb_offs % (bus * 4) || cr_offs % (bus * 4))
253 		v4l2_warn(&dev->v4l2_dev,
254 			"%dx%d fmt:0x%x not support, should be %d aligned\n",
255 			stream->out_fmt.width,
256 			stream->out_fmt.height,
257 			stream->out_fmt.pixelformat,
258 			(cr_offs == 0) ? bus * 4 : bus * 16);
259 
260 	stream->burst = burst;
261 	for (i = 0; i <= RKISP_STREAM_SP; i++)
262 		if (burst > dev->cap_dev.stream[i].burst)
263 			burst = dev->cap_dev.stream[i].burst;
264 
265 	if (stream->interlaced) {
266 		if (!stream->out_fmt.width % (bus * 16))
267 			stream->burst = CIF_MI_CTRL_BURST_LEN_LUM_16 |
268 				CIF_MI_CTRL_BURST_LEN_CHROM_16;
269 		else if (!stream->out_fmt.width % (bus * 8))
270 			stream->burst = CIF_MI_CTRL_BURST_LEN_LUM_8 |
271 				CIF_MI_CTRL_BURST_LEN_CHROM_8;
272 		else
273 			stream->burst = CIF_MI_CTRL_BURST_LEN_LUM_4 |
274 				CIF_MI_CTRL_BURST_LEN_CHROM_4;
275 		if (stream->out_fmt.width % (bus * 4))
276 			v4l2_warn(&dev->v4l2_dev,
277 				"interlaced: width should be %d aligned\n",
278 				bus * 4);
279 		burst = min(stream->burst, burst);
280 		stream->burst = burst;
281 	}
282 
283 	return burst;
284 }
285 
286 /*
287  * configure memory interface for mainpath
288  * This should only be called when stream-on
289  */
mp_config_mi(struct rkisp_stream * stream)290 static int mp_config_mi(struct rkisp_stream *stream)
291 {
292 	struct rkisp_device *dev = stream->ispdev;
293 	struct v4l2_pix_format_mplane *out_fmt = &stream->out_fmt;
294 	bool is_unite = dev->hw_dev->is_unite;
295 	u32 val, mask;
296 
297        /*
298 	* NOTE: plane_fmt[0].sizeimage is total size of all planes for single
299 	* memory plane formats, so calculate the size explicitly.
300 	*/
301 	val = out_fmt->plane_fmt[0].bytesperline * out_fmt->height;
302 	rkisp_unite_write(dev, stream->config->mi.y_size_init, val, false, is_unite);
303 
304 	val = out_fmt->plane_fmt[1].sizeimage;
305 	rkisp_unite_write(dev, stream->config->mi.cb_size_init, val, false, is_unite);
306 
307 	val = out_fmt->plane_fmt[2].sizeimage;
308 	rkisp_unite_write(dev, stream->config->mi.cr_size_init, val, false, is_unite);
309 
310 	val = is_unite ? out_fmt->width / 2 : out_fmt->width;
311 	rkisp_unite_write(dev, ISP3X_MI_MP_WR_Y_PIC_WIDTH, val, false, is_unite);
312 
313 	val = out_fmt->height;
314 	rkisp_unite_write(dev, ISP3X_MI_MP_WR_Y_PIC_HEIGHT, val, false, is_unite);
315 
316 	val = ALIGN(out_fmt->plane_fmt[0].bytesperline, 16);
317 	rkisp_unite_write(dev, ISP3X_MI_MP_WR_Y_LLENGTH, val, false, is_unite);
318 
319 	val = stream->out_isp_fmt.uv_swap ? ISP3X_MI_XTD_FORMAT_MP_UV_SWAP : 0;
320 	mask = ISP3X_MI_XTD_FORMAT_MP_UV_SWAP;
321 	rkisp_unite_set_bits(dev, ISP3X_MI_WR_XTD_FORMAT_CTRL, mask, val, false, is_unite);
322 
323 	mask = ISP3X_MPFBC_FORCE_UPD | ISP3X_MP_YUV_MODE;
324 	val = rkisp_read_reg_cache(dev, ISP3X_MPFBC_CTRL) & ~mask;
325 	if (stream->out_fmt.pixelformat == V4L2_PIX_FMT_NV21 ||
326 	    stream->out_fmt.pixelformat == V4L2_PIX_FMT_NV12 ||
327 	    stream->out_fmt.pixelformat == V4L2_PIX_FMT_NV21M ||
328 	    stream->out_fmt.pixelformat == V4L2_PIX_FMT_NV12M ||
329 	    stream->out_fmt.pixelformat == V4L2_PIX_FMT_YUV420)
330 		val |= ISP3X_SEPERATE_YUV_CFG;
331 	else
332 		val |= ISP3X_SEPERATE_YUV_CFG | ISP3X_MP_YUV_MODE;
333 	rkisp_unite_write(dev, ISP3X_MPFBC_CTRL, val, false, is_unite);
334 
335 	val = calc_burst_len(stream) | CIF_MI_CTRL_INIT_BASE_EN |
336 		CIF_MI_CTRL_INIT_OFFSET_EN | CIF_MI_MP_AUTOUPDATE_ENABLE |
337 		stream->out_isp_fmt.write_format;
338 	mask = GENMASK(19, 16) | MI_CTRL_MP_FMT_MASK;
339 	rkisp_unite_set_bits(dev, ISP3X_MI_WR_CTRL, mask, val, false, is_unite);
340 
341 	mi_frame_end_int_enable(stream);
342 	/* set up first buffer */
343 	mi_frame_end(stream);
344 	return 0;
345 }
346 
mbus_code_sp_in_fmt(u32 in_mbus_code,u32 out_fourcc,u32 * format)347 static int mbus_code_sp_in_fmt(u32 in_mbus_code, u32 out_fourcc, u32 *format)
348 {
349 	switch (in_mbus_code) {
350 	case MEDIA_BUS_FMT_YUYV8_2X8:
351 		*format = MI_CTRL_SP_INPUT_YUV422;
352 		break;
353 	default:
354 		return -EINVAL;
355 	}
356 
357 	/*
358 	 * Only SP can support output format of YCbCr4:0:0,
359 	 * and the input format of SP must be YCbCr4:0:0
360 	 * when outputting YCbCr4:0:0.
361 	 * The output format of isp is YCbCr4:2:2,
362 	 * so the CbCr data is discarded here.
363 	 */
364 	if (out_fourcc == V4L2_PIX_FMT_GREY)
365 		*format = MI_CTRL_SP_INPUT_YUV400;
366 
367 	return 0;
368 }
369 
370 /*
371  * configure memory interface for selfpath
372  * This should only be called when stream-on
373  */
sp_config_mi(struct rkisp_stream * stream)374 static int sp_config_mi(struct rkisp_stream *stream)
375 {
376 	struct rkisp_device *dev = stream->ispdev;
377 	struct v4l2_pix_format_mplane *out_fmt = &stream->out_fmt;
378 	struct ispsd_out_fmt *input_isp_fmt =
379 			rkisp_get_ispsd_out_fmt(&dev->isp_sdev);
380 	bool is_unite = dev->hw_dev->is_unite;
381 	u32 sp_in_fmt, val, mask;
382 
383 	if (mbus_code_sp_in_fmt(input_isp_fmt->mbus_code,
384 				out_fmt->pixelformat, &sp_in_fmt)) {
385 		v4l2_err(&dev->v4l2_dev, "Can't find the input format\n");
386 		return -EINVAL;
387 	}
388 
389        /*
390 	* NOTE: plane_fmt[0].sizeimage is total size of all planes for single
391 	* memory plane formats, so calculate the size explicitly.
392 	*/
393 	val = out_fmt->plane_fmt[0].bytesperline * out_fmt->height;
394 	rkisp_unite_write(dev, stream->config->mi.y_size_init, val, false, is_unite);
395 
396 	val = out_fmt->plane_fmt[1].sizeimage;
397 	rkisp_unite_write(dev, stream->config->mi.cb_size_init, val, false, is_unite);
398 
399 	val = out_fmt->plane_fmt[2].sizeimage;
400 	rkisp_unite_write(dev, stream->config->mi.cr_size_init, val, false, is_unite);
401 
402 	val = is_unite ? out_fmt->width / 2 : out_fmt->width;
403 	rkisp_unite_write(dev, ISP3X_MI_SP_WR_Y_PIC_WIDTH, val, false, is_unite);
404 
405 	val = out_fmt->height;
406 	rkisp_unite_write(dev, ISP3X_MI_SP_WR_Y_PIC_HEIGHT, val, false, is_unite);
407 
408 	val = ALIGN(out_fmt->plane_fmt[0].bytesperline, 16);
409 	rkisp_unite_write(dev, ISP3X_MI_SP_WR_Y_LLENGTH, val, false, is_unite);
410 
411 	val = stream->out_isp_fmt.uv_swap ? ISP3X_MI_XTD_FORMAT_SP_UV_SWAP : 0;
412 	mask = ISP3X_MI_XTD_FORMAT_SP_UV_SWAP;
413 	rkisp_unite_set_bits(dev, ISP3X_MI_WR_XTD_FORMAT_CTRL, mask, val, false, is_unite);
414 
415 	mask = ISP3X_MPFBC_FORCE_UPD | ISP3X_SP_YUV_MODE;
416 	val = rkisp_read_reg_cache(dev, ISP3X_MPFBC_CTRL) & ~mask;
417 	if (stream->out_fmt.pixelformat == V4L2_PIX_FMT_NV21 ||
418 	    stream->out_fmt.pixelformat == V4L2_PIX_FMT_NV12 ||
419 	    stream->out_fmt.pixelformat == V4L2_PIX_FMT_NV21M ||
420 	    stream->out_fmt.pixelformat == V4L2_PIX_FMT_NV12M ||
421 	    stream->out_fmt.pixelformat == V4L2_PIX_FMT_YUV420)
422 		val |= ISP3X_SEPERATE_YUV_CFG;
423 	else
424 		val |= ISP3X_SEPERATE_YUV_CFG | ISP3X_SP_YUV_MODE;
425 	rkisp_unite_write(dev, ISP3X_MPFBC_CTRL, val, false, is_unite);
426 
427 	val = calc_burst_len(stream) | CIF_MI_CTRL_INIT_BASE_EN |
428 		CIF_MI_CTRL_INIT_OFFSET_EN | stream->out_isp_fmt.write_format |
429 		sp_in_fmt | stream->out_isp_fmt.output_format |
430 		CIF_MI_SP_AUTOUPDATE_ENABLE;
431 	mask = GENMASK(19, 16) | MI_CTRL_SP_FMT_MASK;
432 	rkisp_unite_set_bits(dev, ISP3X_MI_WR_CTRL, mask, val, false, is_unite);
433 
434 	mi_frame_end_int_enable(stream);
435 	/* set up first buffer */
436 	mi_frame_end(stream);
437 	return 0;
438 }
439 
fbc_config_mi(struct rkisp_stream * stream)440 static int fbc_config_mi(struct rkisp_stream *stream)
441 {
442 	/* yuv422 is 16*2, yuv420 is 16*1.5 */
443 	u32 mult = stream->out_isp_fmt.write_format ? 32 : 24;
444 	u32 h = ALIGN(stream->out_fmt.height, 16);
445 	u32 w = ALIGN(stream->out_fmt.width, 16);
446 	u32 offs = ALIGN(w * h / 16, RK_MPP_ALIGN);
447 	bool is_unite = stream->ispdev->hw_dev->is_unite;
448 
449 	rkisp_write(stream->ispdev, ISP3X_MPFBC_HEAD_OFFSET, offs, false);
450 	rkisp_unite_write(stream->ispdev, ISP3X_MPFBC_VIR_WIDTH, w, false, is_unite);
451 	rkisp_unite_write(stream->ispdev, ISP3X_MPFBC_PAYL_WIDTH, w, false, is_unite);
452 	rkisp_unite_write(stream->ispdev, ISP3X_MPFBC_VIR_HEIGHT, h, false, is_unite);
453 	if (is_unite) {
454 		u32 left_w = (stream->out_fmt.width / 2) & ~0xf;
455 
456 		offs += left_w * mult;
457 		rkisp_next_write(stream->ispdev, ISP3X_MPFBC_HEAD_OFFSET, offs, false);
458 	}
459 	rkisp_unite_set_bits(stream->ispdev, ISP3X_MI_WR_CTRL, 0,
460 			     CIF_MI_CTRL_INIT_BASE_EN | CIF_MI_CTRL_INIT_OFFSET_EN,
461 			     false, is_unite);
462 	mi_frame_end_int_enable(stream);
463 	/* set up first buffer */
464 	mi_frame_end(stream);
465 	return 0;
466 }
467 
bp_config_mi(struct rkisp_stream * stream)468 static int bp_config_mi(struct rkisp_stream *stream)
469 {
470 	struct v4l2_pix_format_mplane *out_fmt = &stream->out_fmt;
471 	struct rkisp_device *dev = stream->ispdev;
472 	bool is_unite = dev->hw_dev->is_unite;
473 	u32 val, mask;
474 
475        /*
476 	* NOTE: plane_fmt[0].sizeimage is total size of all planes for single
477 	* memory plane formats, so calculate the size explicitly.
478 	*/
479 	val = out_fmt->plane_fmt[0].bytesperline * out_fmt->height;
480 	rkisp_unite_write(dev, stream->config->mi.y_size_init, val, false, is_unite);
481 
482 	val = out_fmt->plane_fmt[1].sizeimage;
483 	rkisp_unite_write(dev, stream->config->mi.cb_size_init, val, false, is_unite);
484 
485 	val = is_unite ? out_fmt->width / 2 : out_fmt->width;
486 	rkisp_unite_write(dev, ISP3X_MI_BP_WR_Y_PIC_WIDTH, val, false, is_unite);
487 
488 	val = out_fmt->height;
489 	rkisp_unite_write(dev, ISP3X_MI_BP_WR_Y_PIC_HEIGHT, val, false, is_unite);
490 
491 	val = ALIGN(out_fmt->plane_fmt[0].bytesperline, 16);
492 	rkisp_unite_write(dev, ISP3X_MI_BP_WR_Y_LLENGTH, val, false, is_unite);
493 
494 	mask = ISP3X_MPFBC_FORCE_UPD | ISP3X_BP_YUV_MODE;
495 	val = rkisp_read_reg_cache(dev, ISP3X_MPFBC_CTRL) & ~mask;
496 
497 	if (out_fmt->pixelformat == V4L2_PIX_FMT_NV12 ||
498 	    out_fmt->pixelformat == V4L2_PIX_FMT_NV12M)
499 		val |= ISP3X_SEPERATE_YUV_CFG;
500 	else
501 		val |= ISP3X_SEPERATE_YUV_CFG | ISP3X_BP_YUV_MODE;
502 	rkisp_unite_write(dev, ISP3X_MPFBC_CTRL, val, false, is_unite);
503 	val = CIF_MI_CTRL_INIT_BASE_EN | CIF_MI_CTRL_INIT_OFFSET_EN;
504 	rkisp_unite_set_bits(dev, ISP3X_MI_WR_CTRL, 0, val, false, is_unite);
505 	mi_frame_end_int_enable(stream);
506 	/* set up first buffer */
507 	mi_frame_end(stream);
508 	return 0;
509 }
510 
mp_enable_mi(struct rkisp_stream * stream)511 static void mp_enable_mi(struct rkisp_stream *stream)
512 {
513 	struct capture_fmt *isp_fmt = &stream->out_isp_fmt;
514 	u32 mask = CIF_MI_CTRL_MP_ENABLE | CIF_MI_CTRL_RAW_ENABLE;
515 	u32 val = CIF_MI_CTRL_MP_ENABLE;
516 
517 	if (isp_fmt->fmt_type == FMT_BAYER)
518 		val = CIF_MI_CTRL_RAW_ENABLE;
519 	rkisp_unite_set_bits(stream->ispdev, ISP3X_MI_WR_CTRL, mask, val,
520 			     false, stream->ispdev->hw_dev->is_unite);
521 }
522 
sp_enable_mi(struct rkisp_stream * stream)523 static void sp_enable_mi(struct rkisp_stream *stream)
524 {
525 	rkisp_unite_set_bits(stream->ispdev, ISP3X_MI_WR_CTRL, 0,
526 			     CIF_MI_CTRL_SP_ENABLE, false,
527 			     stream->ispdev->hw_dev->is_unite);
528 }
529 
fbc_enable_mi(struct rkisp_stream * stream)530 static void fbc_enable_mi(struct rkisp_stream *stream)
531 {
532 	u32 val, mask = ISP3X_MPFBC_FORCE_UPD | ISP3X_MPFBC_YUV_MASK |
533 			ISP3X_MPFBC_SPARSE_MODE;
534 	bool is_unite = stream->ispdev->hw_dev->is_unite;
535 
536 	/* config no effect immediately, read back is shadow, get config value from cache */
537 	val = rkisp_read_reg_cache(stream->ispdev, ISP3X_MPFBC_CTRL) & ~mask;
538 	val |= stream->out_isp_fmt.write_format | ISP3X_HEAD_OFFSET_EN | ISP3X_MPFBC_EN;
539 	rkisp_unite_write(stream->ispdev, ISP3X_MPFBC_CTRL, val, false, is_unite);
540 }
541 
bp_enable_mi(struct rkisp_stream * stream)542 static void bp_enable_mi(struct rkisp_stream *stream)
543 {
544 	u32 val = stream->out_isp_fmt.write_format |
545 		ISP3X_BP_ENABLE | ISP3X_BP_AUTO_UPD;
546 
547 	rkisp_unite_write(stream->ispdev, ISP3X_MI_BP_WR_CTRL, val, false,
548 			  stream->ispdev->hw_dev->is_unite);
549 }
550 
mp_disable_mi(struct rkisp_stream * stream)551 static void mp_disable_mi(struct rkisp_stream *stream)
552 {
553 	u32 mask = CIF_MI_CTRL_MP_ENABLE | CIF_MI_CTRL_RAW_ENABLE;
554 
555 	rkisp_unite_clear_bits(stream->ispdev, ISP3X_MI_WR_CTRL, mask, false,
556 			       stream->ispdev->hw_dev->is_unite);
557 }
558 
sp_disable_mi(struct rkisp_stream * stream)559 static void sp_disable_mi(struct rkisp_stream *stream)
560 {
561 	rkisp_unite_clear_bits(stream->ispdev, ISP3X_MI_WR_CTRL, CIF_MI_CTRL_SP_ENABLE,
562 			       false, stream->ispdev->hw_dev->is_unite);
563 }
564 
fbc_disable_mi(struct rkisp_stream * stream)565 static void fbc_disable_mi(struct rkisp_stream *stream)
566 {
567 	u32 mask = ISP3X_MPFBC_FORCE_UPD | ISP3X_MPFBC_EN;
568 
569 	rkisp_unite_clear_bits(stream->ispdev, ISP3X_MPFBC_CTRL, mask,
570 			       false, stream->ispdev->hw_dev->is_unite);
571 }
572 
bp_disable_mi(struct rkisp_stream * stream)573 static void bp_disable_mi(struct rkisp_stream *stream)
574 {
575 	rkisp_unite_clear_bits(stream->ispdev, ISP3X_MI_BP_WR_CTRL, ISP3X_BP_ENABLE,
576 			       false, stream->ispdev->hw_dev->is_unite);
577 }
578 
update_mi(struct rkisp_stream * stream)579 static void update_mi(struct rkisp_stream *stream)
580 {
581 	struct rkisp_device *dev = stream->ispdev;
582 	struct rkisp_dummy_buffer *dummy_buf = &dev->hw_dev->dummy_buf;
583 	u32 val, reg;
584 
585 	if (stream->next_buf) {
586 		reg = stream->config->mi.y_base_ad_init;
587 		val = stream->next_buf->buff_addr[RKISP_PLANE_Y];
588 		rkisp_write(dev, reg, val, false);
589 
590 		reg = stream->config->mi.cb_base_ad_init;
591 		val = stream->next_buf->buff_addr[RKISP_PLANE_CB];
592 		rkisp_write(dev, reg, val, false);
593 
594 		if (stream->id != RKISP_STREAM_FBC && stream->id != RKISP_STREAM_BP) {
595 			reg = stream->config->mi.cr_base_ad_init;
596 			val = stream->next_buf->buff_addr[RKISP_PLANE_CR];
597 			rkisp_write(dev, reg, val, false);
598 		}
599 
600 		if (dev->hw_dev->is_unite) {
601 			u32 mult = stream->id != RKISP_STREAM_FBC ? 1 :
602 				   (stream->out_isp_fmt.write_format ? 32 : 24);
603 
604 			reg = stream->config->mi.y_base_ad_init;
605 			val = stream->next_buf->buff_addr[RKISP_PLANE_Y];
606 			val += ((stream->out_fmt.width / 2) & ~0xf);
607 			rkisp_next_write(dev, reg, val, false);
608 
609 			reg = stream->config->mi.cb_base_ad_init;
610 			val = stream->next_buf->buff_addr[RKISP_PLANE_CB];
611 			val += ((stream->out_fmt.width / 2) & ~0xf) * mult;
612 			rkisp_next_write(dev, reg, val, false);
613 
614 			if (stream->id != RKISP_STREAM_FBC && stream->id != RKISP_STREAM_BP) {
615 				reg = stream->config->mi.cr_base_ad_init;
616 				val = stream->next_buf->buff_addr[RKISP_PLANE_CR];
617 				val += ((stream->out_fmt.width / 2) & ~0xf);
618 				rkisp_next_write(dev, reg, val, false);
619 			}
620 		}
621 
622 		/* single buf updated at readback for multidevice */
623 		if (!dev->hw_dev->is_single) {
624 			stream->curr_buf = stream->next_buf;
625 			stream->next_buf = NULL;
626 		}
627 	} else if (dummy_buf->mem_priv) {
628 		stream->dbg.frameloss++;
629 		val = dummy_buf->dma_addr;
630 		reg = stream->config->mi.y_base_ad_init;
631 		rkisp_unite_write(dev, reg, val, false, dev->hw_dev->is_unite);
632 		reg = stream->config->mi.cb_base_ad_init;
633 		rkisp_unite_write(dev, reg, val, false, dev->hw_dev->is_unite);
634 		reg = stream->config->mi.cr_base_ad_init;
635 		if (stream->id != RKISP_STREAM_FBC && stream->id != RKISP_STREAM_BP)
636 			rkisp_unite_write(dev, reg, val, false, dev->hw_dev->is_unite);
637 	}
638 
639 	if (stream->id != RKISP_STREAM_FBC) {
640 		reg = stream->config->mi.y_offs_cnt_init;
641 		rkisp_unite_write(dev, reg, 0, false, dev->hw_dev->is_unite);
642 		reg = stream->config->mi.cb_offs_cnt_init;
643 		rkisp_unite_write(dev, reg, 0, false, dev->hw_dev->is_unite);
644 		reg = stream->config->mi.cr_offs_cnt_init;
645 		if (stream->id != RKISP_STREAM_BP)
646 			rkisp_unite_write(dev, reg, 0, false, dev->hw_dev->is_unite);
647 	}
648 
649 	v4l2_dbg(2, rkisp_debug, &dev->v4l2_dev,
650 		 "%s stream:%d Y:0x%x CB:0x%x | Y_SHD:0x%x\n",
651 		 __func__, stream->id,
652 		 rkisp_read(dev, stream->config->mi.y_base_ad_init, false),
653 		 rkisp_read(dev, stream->config->mi.cb_base_ad_init, false),
654 		 rkisp_read(dev, stream->config->mi.y_base_ad_shd, true));
655 	if (dev->hw_dev->is_unite)
656 		v4l2_dbg(2, rkisp_debug, &dev->v4l2_dev,
657 			 "%s stream:%d Y:0x%x CB:0x%x | Y_SHD:0x%x, right\n",
658 			 __func__, stream->id,
659 			 rkisp_next_read(dev, stream->config->mi.y_base_ad_init, false),
660 			 rkisp_next_read(dev, stream->config->mi.cb_base_ad_init, false),
661 			 rkisp_next_read(dev, stream->config->mi.y_base_ad_shd, true));
662 }
663 
664 static struct streams_ops rkisp_mp_streams_ops = {
665 	.config_mi = mp_config_mi,
666 	.enable_mi = mp_enable_mi,
667 	.disable_mi = mp_disable_mi,
668 	.set_data_path = stream_data_path,
669 	.is_stream_stopped = mp_is_stream_stopped,
670 	.update_mi = update_mi,
671 	.frame_end = mi_frame_end,
672 };
673 
674 static struct streams_ops rkisp_sp_streams_ops = {
675 	.config_mi = sp_config_mi,
676 	.enable_mi = sp_enable_mi,
677 	.disable_mi = sp_disable_mi,
678 	.set_data_path = stream_data_path,
679 	.is_stream_stopped = sp_is_stream_stopped,
680 	.update_mi = update_mi,
681 	.frame_end = mi_frame_end,
682 };
683 
684 static struct streams_ops rkisp_fbc_streams_ops = {
685 	.config_mi = fbc_config_mi,
686 	.enable_mi = fbc_enable_mi,
687 	.disable_mi = fbc_disable_mi,
688 	.is_stream_stopped = is_fbc_stream_stopped,
689 	.update_mi = update_mi,
690 	.frame_end = mi_frame_end,
691 };
692 
693 static struct streams_ops rkisp_bp_streams_ops = {
694 	.config_mi = bp_config_mi,
695 	.enable_mi = bp_enable_mi,
696 	.disable_mi = bp_disable_mi,
697 	.is_stream_stopped = is_bp_stream_stopped,
698 	.update_mi = update_mi,
699 	.frame_end = mi_frame_end,
700 };
701 
702 /*
703  * This function is called when a frame end come. The next frame
704  * is processing and we should set up buffer for next-next frame,
705  * otherwise it will overflow.
706  */
mi_frame_end(struct rkisp_stream * stream)707 static int mi_frame_end(struct rkisp_stream *stream)
708 {
709 	struct rkisp_device *dev = stream->ispdev;
710 	struct capture_fmt *isp_fmt = &stream->out_isp_fmt;
711 	unsigned long lock_flags = 0;
712 	int i = 0;
713 
714 	if (stream->id == RKISP_STREAM_VIR)
715 		return 0;
716 
717 	if (stream->curr_buf) {
718 		struct rkisp_stream *vir = &dev->cap_dev.stream[RKISP_STREAM_VIR];
719 		struct vb2_buffer *vb2_buf = &stream->curr_buf->vb.vb2_buf;
720 		u64 ns = 0;
721 
722 		/* Dequeue a filled buffer */
723 		for (i = 0; i < isp_fmt->mplanes; i++) {
724 			u32 payload_size = stream->out_fmt.plane_fmt[i].sizeimage;
725 
726 			vb2_set_plane_payload(vb2_buf, i, payload_size);
727 		}
728 
729 		rkisp_dmarx_get_frame(dev, &i, NULL, &ns, true);
730 		stream->curr_buf->vb.sequence = i;
731 		if (!ns)
732 			ns = ktime_get_ns();
733 		vb2_buf->timestamp = ns;
734 
735 		ns = ktime_get_ns();
736 		stream->dbg.interval = ns - stream->dbg.timestamp;
737 		stream->dbg.timestamp = ns;
738 		stream->dbg.id = stream->curr_buf->vb.sequence;
739 		stream->dbg.delay = ns - dev->isp_sdev.frm_timestamp;
740 
741 		if (vir->streaming && vir->conn_id == stream->id) {
742 
743 			spin_lock_irqsave(&vir->vbq_lock, lock_flags);
744 			if (vir->streaming)
745 				list_add_tail(&stream->curr_buf->queue,
746 					&dev->cap_dev.vir_cpy.queue);
747 			spin_unlock_irqrestore(&vir->vbq_lock, lock_flags);
748 			if (!completion_done(&dev->cap_dev.vir_cpy.cmpl))
749 				complete(&dev->cap_dev.vir_cpy.cmpl);
750 
751 			if (!vir->streaming)
752 				vb2_buffer_done(vb2_buf, VB2_BUF_STATE_DONE);
753 
754 		} else {
755 			vb2_buffer_done(vb2_buf, VB2_BUF_STATE_DONE);
756 		}
757 
758 		stream->curr_buf = NULL;
759 	}
760 
761 	stream->curr_buf = stream->next_buf;
762 	stream->next_buf = NULL;
763 	spin_lock_irqsave(&stream->vbq_lock, lock_flags);
764 	if (!list_empty(&stream->buf_queue)) {
765 		stream->next_buf = list_first_entry(&stream->buf_queue,
766 						    struct rkisp_buffer, queue);
767 		list_del(&stream->next_buf->queue);
768 	}
769 	spin_unlock_irqrestore(&stream->vbq_lock, lock_flags);
770 
771 	stream->ops->update_mi(stream);
772 
773 	return 0;
774 }
775 
776 /***************************** vb2 operations*******************************/
777 
778 /*
779  * Set flags and wait, it should stop in interrupt.
780  * If it didn't, stop it by force.
781  */
rkisp_stream_stop(struct rkisp_stream * stream)782 static void rkisp_stream_stop(struct rkisp_stream *stream)
783 {
784 	struct rkisp_device *dev = stream->ispdev;
785 	struct v4l2_device *v4l2_dev = &dev->v4l2_dev;
786 	int ret = 0;
787 
788 	stream->stopping = true;
789 	if (dev->hw_dev->is_single)
790 		stream->ops->disable_mi(stream);
791 	if (dev->isp_state & ISP_START &&
792 	    !stream->ops->is_stream_stopped(dev->base_addr)) {
793 		ret = wait_event_timeout(stream->done,
794 					 !stream->streaming,
795 					 msecs_to_jiffies(500));
796 		if (!ret)
797 			v4l2_warn(v4l2_dev, "%s id:%d timeout\n",
798 				  __func__, stream->id);
799 	}
800 
801 	stream->stopping = false;
802 	stream->streaming = false;
803 	stream->ops->disable_mi(stream);
804 	rkisp_disable_dcrop(stream, true);
805 	if (stream->id == RKISP_STREAM_MP || stream->id == RKISP_STREAM_SP)
806 		rkisp_disable_rsz(stream, true);
807 	ret = get_stream_irq_mask(stream);
808 	dev->irq_ends_mask &= ~ret;
809 
810 	stream->burst =
811 		CIF_MI_CTRL_BURST_LEN_LUM_16 |
812 		CIF_MI_CTRL_BURST_LEN_CHROM_16;
813 	stream->interlaced = false;
814 }
815 
816 /*
817  * Most of registers inside rockchip isp1 have shadow register since
818  * they must be not changed during processing a frame.
819  * Usually, each sub-module updates its shadow register after
820  * processing the last pixel of a frame.
821  */
rkisp_start(struct rkisp_stream * stream)822 static int rkisp_start(struct rkisp_stream *stream)
823 {
824 	struct rkisp_device *dev = stream->ispdev;
825 	bool is_update = atomic_read(&dev->cap_dev.refcnt) > 1 ? false : true;
826 	int ret;
827 
828 	if (stream->ops->set_data_path)
829 		stream->ops->set_data_path(stream);
830 	ret = stream->ops->config_mi(stream);
831 	if (ret)
832 		return ret;
833 
834 	stream->ops->enable_mi(stream);
835 	if (is_update)
836 		dev->irq_ends_mask |= get_stream_irq_mask(stream);
837 	stream->streaming = true;
838 
839 	return 0;
840 }
841 
rkisp_queue_setup(struct vb2_queue * queue,unsigned int * num_buffers,unsigned int * num_planes,unsigned int sizes[],struct device * alloc_ctxs[])842 static int rkisp_queue_setup(struct vb2_queue *queue,
843 			      unsigned int *num_buffers,
844 			      unsigned int *num_planes,
845 			      unsigned int sizes[],
846 			      struct device *alloc_ctxs[])
847 {
848 	struct rkisp_stream *stream = queue->drv_priv;
849 	struct rkisp_device *dev = stream->ispdev;
850 	const struct v4l2_pix_format_mplane *pixm = NULL;
851 	const struct capture_fmt *isp_fmt = NULL;
852 	u32 i;
853 
854 	pixm = &stream->out_fmt;
855 	isp_fmt = &stream->out_isp_fmt;
856 	*num_planes = isp_fmt->mplanes;
857 
858 	for (i = 0; i < isp_fmt->mplanes; i++) {
859 		const struct v4l2_plane_pix_format *plane_fmt;
860 
861 		plane_fmt = &pixm->plane_fmt[i];
862 		/* height to align with 16 when allocating memory
863 		 * so that Rockchip encoder can use DMA buffer directly
864 		 */
865 		sizes[i] = (isp_fmt->fmt_type == FMT_YUV) ?
866 			plane_fmt->sizeimage / pixm->height *
867 			ALIGN(pixm->height, 16) :
868 			plane_fmt->sizeimage;
869 	}
870 
871 	rkisp_chk_tb_over(dev);
872 	v4l2_dbg(1, rkisp_debug, &dev->v4l2_dev, "%s %s count %d, size %d\n",
873 		 stream->vnode.vdev.name, v4l2_type_names[queue->type], *num_buffers, sizes[0]);
874 
875 	return 0;
876 }
877 
878 /*
879  * The vb2_buffer are stored in rkisp_buffer, in order to unify
880  * mplane buffer and none-mplane buffer.
881  */
rkisp_buf_queue(struct vb2_buffer * vb)882 static void rkisp_buf_queue(struct vb2_buffer *vb)
883 {
884 	struct vb2_v4l2_buffer *vbuf = to_vb2_v4l2_buffer(vb);
885 	struct rkisp_buffer *ispbuf = to_rkisp_buffer(vbuf);
886 	struct vb2_queue *queue = vb->vb2_queue;
887 	struct rkisp_stream *stream = queue->drv_priv;
888 	unsigned long lock_flags = 0;
889 	struct v4l2_pix_format_mplane *pixm = &stream->out_fmt;
890 	struct capture_fmt *isp_fmt = &stream->out_isp_fmt;
891 	struct sg_table *sgt;
892 	u32 height, size, offset;
893 	int i;
894 
895 	memset(ispbuf->buff_addr, 0, sizeof(ispbuf->buff_addr));
896 	for (i = 0; i < isp_fmt->mplanes; i++) {
897 		vb2_plane_vaddr(vb, i);
898 		if (stream->ispdev->hw_dev->is_mmu) {
899 			sgt = vb2_dma_sg_plane_desc(vb, i);
900 			ispbuf->buff_addr[i] = sg_dma_address(sgt->sgl);
901 		} else {
902 			ispbuf->buff_addr[i] = vb2_dma_contig_plane_dma_addr(vb, i);
903 		}
904 	}
905 	/*
906 	 * NOTE: plane_fmt[0].sizeimage is total size of all planes for single
907 	 * memory plane formats, so calculate the size explicitly.
908 	 */
909 	if (isp_fmt->mplanes == 1) {
910 		for (i = 0; i < isp_fmt->cplanes - 1; i++) {
911 			/* FBC mode calculate payload offset */
912 			height = (isp_fmt->fmt_type == FMT_FBC) ?
913 				ALIGN(pixm->height, 16) >> 4 : pixm->height;
914 			size = (i == 0) ?
915 				pixm->plane_fmt[i].bytesperline * height :
916 				pixm->plane_fmt[i].sizeimage;
917 			offset = (isp_fmt->fmt_type == FMT_FBC) ?
918 				ALIGN(size, RK_MPP_ALIGN) : size;
919 			ispbuf->buff_addr[i + 1] =
920 				ispbuf->buff_addr[i] + offset;
921 		}
922 	}
923 
924 	v4l2_dbg(2, rkisp_debug, &stream->ispdev->v4l2_dev,
925 		 "stream:%d queue buf:0x%x\n",
926 		 stream->id, ispbuf->buff_addr[0]);
927 
928 	spin_lock_irqsave(&stream->vbq_lock, lock_flags);
929 	list_add_tail(&ispbuf->queue, &stream->buf_queue);
930 	spin_unlock_irqrestore(&stream->vbq_lock, lock_flags);
931 }
932 
rkisp_create_dummy_buf(struct rkisp_stream * stream)933 static int rkisp_create_dummy_buf(struct rkisp_stream *stream)
934 {
935 	return rkisp_alloc_common_dummy_buf(stream->ispdev);
936 }
937 
rkisp_destroy_dummy_buf(struct rkisp_stream * stream)938 static void rkisp_destroy_dummy_buf(struct rkisp_stream *stream)
939 {
940 	struct rkisp_device *dev = stream->ispdev;
941 
942 	rkisp_free_common_dummy_buf(dev);
943 }
944 
destroy_buf_queue(struct rkisp_stream * stream,enum vb2_buffer_state state)945 static void destroy_buf_queue(struct rkisp_stream *stream,
946 			      enum vb2_buffer_state state)
947 {
948 	unsigned long lock_flags = 0;
949 	struct rkisp_buffer *buf;
950 
951 	spin_lock_irqsave(&stream->vbq_lock, lock_flags);
952 	if (stream->curr_buf) {
953 		list_add_tail(&stream->curr_buf->queue, &stream->buf_queue);
954 		if (stream->curr_buf == stream->next_buf)
955 			stream->next_buf = NULL;
956 		stream->curr_buf = NULL;
957 	}
958 	if (stream->next_buf) {
959 		list_add_tail(&stream->next_buf->queue, &stream->buf_queue);
960 		stream->next_buf = NULL;
961 	}
962 	while (!list_empty(&stream->buf_queue)) {
963 		buf = list_first_entry(&stream->buf_queue,
964 			struct rkisp_buffer, queue);
965 		list_del(&buf->queue);
966 		vb2_buffer_done(&buf->vb.vb2_buf, state);
967 	}
968 	spin_unlock_irqrestore(&stream->vbq_lock, lock_flags);
969 }
970 
rkisp_stop_streaming(struct vb2_queue * queue)971 static void rkisp_stop_streaming(struct vb2_queue *queue)
972 {
973 	struct rkisp_stream *stream = queue->drv_priv;
974 	struct rkisp_vdev_node *node = &stream->vnode;
975 	struct rkisp_device *dev = stream->ispdev;
976 	struct v4l2_device *v4l2_dev = &dev->v4l2_dev;
977 	int ret;
978 
979 	mutex_lock(&dev->hw_dev->dev_lock);
980 
981 	v4l2_dbg(1, rkisp_debug, v4l2_dev, "%s %s %d\n",
982 		 __func__, node->vdev.name, stream->id);
983 
984 	if (!stream->streaming)
985 		goto end;
986 
987 	if (stream->id == RKISP_STREAM_VIR) {
988 		stream->stopping = true;
989 		wait_event_timeout(stream->done,
990 				   stream->frame_end,
991 				   msecs_to_jiffies(500));
992 		stream->streaming = false;
993 		stream->stopping = false;
994 		destroy_buf_queue(stream, VB2_BUF_STATE_ERROR);
995 
996 		if (!completion_done(&dev->cap_dev.vir_cpy.cmpl))
997 			complete(&dev->cap_dev.vir_cpy.cmpl);
998 		goto end;
999 	}
1000 
1001 	rkisp_stream_stop(stream);
1002 	/* call to the other devices */
1003 	media_pipeline_stop(&node->vdev.entity);
1004 	ret = dev->pipe.set_stream(&dev->pipe, false);
1005 	if (ret < 0)
1006 		v4l2_err(v4l2_dev, "pipeline stream-off failed:%d\n", ret);
1007 
1008 	/* release buffers */
1009 	destroy_buf_queue(stream, VB2_BUF_STATE_ERROR);
1010 
1011 	ret = dev->pipe.close(&dev->pipe);
1012 	if (ret < 0)
1013 		v4l2_err(v4l2_dev, "pipeline close failed error:%d\n", ret);
1014 	rkisp_destroy_dummy_buf(stream);
1015 	atomic_dec(&dev->cap_dev.refcnt);
1016 
1017 end:
1018 	mutex_unlock(&dev->hw_dev->dev_lock);
1019 }
1020 
vir_cpy_image(struct work_struct * work)1021 static void vir_cpy_image(struct work_struct *work)
1022 {
1023 	struct rkisp_vir_cpy *cpy =
1024 	container_of(work, struct rkisp_vir_cpy, work);
1025 	struct rkisp_stream *vir = cpy->stream;
1026 	struct rkisp_buffer *src_buf = NULL;
1027 	unsigned long lock_flags = 0;
1028 	u32 i;
1029 
1030 	v4l2_dbg(1, rkisp_debug, &vir->ispdev->v4l2_dev,
1031 		 "%s enter\n", __func__);
1032 
1033 	vir->streaming = true;
1034 	spin_lock_irqsave(&vir->vbq_lock, lock_flags);
1035 	if (!list_empty(&cpy->queue)) {
1036 		src_buf = list_first_entry(&cpy->queue,
1037 				struct rkisp_buffer, queue);
1038 		list_del(&src_buf->queue);
1039 	}
1040 	spin_unlock_irqrestore(&vir->vbq_lock, lock_flags);
1041 
1042 	while (src_buf || vir->streaming) {
1043 		if (vir->stopping || !vir->streaming)
1044 			goto end;
1045 
1046 		if (!src_buf)
1047 			wait_for_completion(&cpy->cmpl);
1048 
1049 		vir->frame_end = false;
1050 		spin_lock_irqsave(&vir->vbq_lock, lock_flags);
1051 
1052 		if (!src_buf && !list_empty(&cpy->queue)) {
1053 			src_buf = list_first_entry(&cpy->queue,
1054 					struct rkisp_buffer, queue);
1055 			list_del(&src_buf->queue);
1056 		}
1057 
1058 		if (src_buf && !vir->curr_buf && !list_empty(&vir->buf_queue)) {
1059 			vir->curr_buf = list_first_entry(&vir->buf_queue,
1060 					struct rkisp_buffer, queue);
1061 			list_del(&vir->curr_buf->queue);
1062 		}
1063 		spin_unlock_irqrestore(&vir->vbq_lock, lock_flags);
1064 
1065 		if (!vir->curr_buf || !src_buf)
1066 			goto end;
1067 
1068 		for (i = 0; i < vir->out_isp_fmt.mplanes; i++) {
1069 			u32 payload_size = vir->out_fmt.plane_fmt[i].sizeimage;
1070 			void *src = vb2_plane_vaddr(&src_buf->vb.vb2_buf, i);
1071 			void *dst = vb2_plane_vaddr(&vir->curr_buf->vb.vb2_buf, i);
1072 
1073 			if (!src || !dst)
1074 				break;
1075 			vb2_set_plane_payload(&vir->curr_buf->vb.vb2_buf, i, payload_size);
1076 			memcpy(dst, src, payload_size);
1077 		}
1078 
1079 		vir->curr_buf->vb.sequence = src_buf->vb.sequence;
1080 		vir->curr_buf->vb.vb2_buf.timestamp = src_buf->vb.vb2_buf.timestamp;
1081 		vb2_buffer_done(&vir->curr_buf->vb.vb2_buf, VB2_BUF_STATE_DONE);
1082 		vir->curr_buf = NULL;
1083 end:
1084 		if (src_buf)
1085 			vb2_buffer_done(&src_buf->vb.vb2_buf, VB2_BUF_STATE_DONE);
1086 		src_buf = NULL;
1087 		spin_lock_irqsave(&vir->vbq_lock, lock_flags);
1088 
1089 		if (!list_empty(&cpy->queue)) {
1090 			src_buf = list_first_entry(&cpy->queue,
1091 					struct rkisp_buffer, queue);
1092 			list_del(&src_buf->queue);
1093 		} else if (vir->stopping) {
1094 			vir->streaming = false;
1095 		}
1096 
1097 		spin_unlock_irqrestore(&vir->vbq_lock, lock_flags);
1098 	}
1099 
1100 	vir->frame_end = true;
1101 
1102 	if (vir->stopping) {
1103 		vir->stopping = false;
1104 		vir->streaming = false;
1105 		wake_up(&vir->done);
1106 	}
1107 
1108 	v4l2_dbg(1, rkisp_debug, &vir->ispdev->v4l2_dev,
1109 		 "%s exit\n", __func__);
1110 }
1111 
rkisp_stream_start(struct rkisp_stream * stream)1112 static int rkisp_stream_start(struct rkisp_stream *stream)
1113 {
1114 	struct v4l2_device *v4l2_dev = &stream->ispdev->v4l2_dev;
1115 	struct rkisp_device *dev = stream->ispdev;
1116 	bool async = false;
1117 	int ret;
1118 
1119 	async = (stream->id == RKISP_STREAM_MP) ?
1120 		dev->cap_dev.stream[RKISP_STREAM_SP].streaming :
1121 		dev->cap_dev.stream[RKISP_STREAM_MP].streaming;
1122 
1123 	/*
1124 	 * can't be async now, otherwise the latter started stream fails to
1125 	 * produce mi interrupt.
1126 	 */
1127 	ret = rkisp_stream_config_dcrop(stream, false);
1128 	if (ret < 0) {
1129 		v4l2_err(v4l2_dev, "config dcrop failed with error %d\n", ret);
1130 		return ret;
1131 	}
1132 
1133 	if (stream->id == RKISP_STREAM_FBC || stream->id == RKISP_STREAM_BP)
1134 		goto end;
1135 
1136 	ret = rkisp_stream_config_rsz(stream, async);
1137 	if (ret < 0) {
1138 		v4l2_err(v4l2_dev, "config rsz failed with error %d\n", ret);
1139 		return ret;
1140 	}
1141 end:
1142 	return rkisp_start(stream);
1143 }
1144 
1145 static int
rkisp_start_streaming(struct vb2_queue * queue,unsigned int count)1146 rkisp_start_streaming(struct vb2_queue *queue, unsigned int count)
1147 {
1148 	struct rkisp_stream *stream = queue->drv_priv;
1149 	struct rkisp_vdev_node *node = &stream->vnode;
1150 	struct rkisp_device *dev = stream->ispdev;
1151 	struct v4l2_device *v4l2_dev = &dev->v4l2_dev;
1152 	int ret = -1;
1153 
1154 	mutex_lock(&dev->hw_dev->dev_lock);
1155 
1156 	v4l2_dbg(1, rkisp_debug, v4l2_dev, "%s %s id:%d\n",
1157 		 __func__, node->vdev.name, stream->id);
1158 
1159 	if (WARN_ON(stream->streaming)) {
1160 		mutex_unlock(&dev->hw_dev->dev_lock);
1161 		return -EBUSY;
1162 	}
1163 
1164 	if (stream->id == RKISP_STREAM_VIR) {
1165 		struct rkisp_stream *t = &dev->cap_dev.stream[stream->conn_id];
1166 
1167 		if (t->streaming) {
1168 			INIT_WORK(&dev->cap_dev.vir_cpy.work, vir_cpy_image);
1169 			init_completion(&dev->cap_dev.vir_cpy.cmpl);
1170 			INIT_LIST_HEAD(&dev->cap_dev.vir_cpy.queue);
1171 			dev->cap_dev.vir_cpy.stream = stream;
1172 			schedule_work(&dev->cap_dev.vir_cpy.work);
1173 			ret = 0;
1174 		} else {
1175 			v4l2_err(&dev->v4l2_dev,
1176 				 "no stream enable for iqtool\n");
1177 			destroy_buf_queue(stream, VB2_BUF_STATE_QUEUED);
1178 			ret = -EINVAL;
1179 		}
1180 
1181 		mutex_unlock(&dev->hw_dev->dev_lock);
1182 
1183 		return ret;
1184 	}
1185 
1186 	memset(&stream->dbg, 0, sizeof(stream->dbg));
1187 	atomic_inc(&dev->cap_dev.refcnt);
1188 	if (!dev->isp_inp || !stream->linked) {
1189 		v4l2_err(v4l2_dev, "check %s link or isp input\n", node->vdev.name);
1190 		goto buffer_done;
1191 	}
1192 
1193 	if (atomic_read(&dev->cap_dev.refcnt) == 1 &&
1194 	    (dev->isp_inp & INP_CIF)) {
1195 		/* update sensor info when first streaming */
1196 		ret = rkisp_update_sensor_info(dev);
1197 		if (ret < 0) {
1198 			v4l2_err(v4l2_dev, "update sensor info failed %d\n", ret);
1199 			goto buffer_done;
1200 		}
1201 	}
1202 
1203 	ret = rkisp_create_dummy_buf(stream);
1204 	if (ret < 0)
1205 		goto buffer_done;
1206 
1207 	/* enable clocks/power-domains */
1208 	ret = dev->pipe.open(&dev->pipe, &node->vdev.entity, true);
1209 	if (ret < 0) {
1210 		v4l2_err(v4l2_dev, "open isp pipeline failed %d\n", ret);
1211 		goto destroy_dummy_buf;
1212 	}
1213 
1214 	/* configure stream hardware to start */
1215 	ret = rkisp_stream_start(stream);
1216 	if (ret < 0) {
1217 		v4l2_err(v4l2_dev, "start %s failed\n", node->vdev.name);
1218 		goto close_pipe;
1219 	}
1220 
1221 	/* start sub-devices */
1222 	ret = dev->pipe.set_stream(&dev->pipe, true);
1223 	if (ret < 0)
1224 		goto stop_stream;
1225 
1226 	ret = media_pipeline_start(&node->vdev.entity, &dev->pipe.pipe);
1227 	if (ret < 0) {
1228 		v4l2_err(v4l2_dev, "start pipeline failed %d\n", ret);
1229 		goto pipe_stream_off;
1230 	}
1231 
1232 	mutex_unlock(&dev->hw_dev->dev_lock);
1233 	return 0;
1234 
1235 pipe_stream_off:
1236 	dev->pipe.set_stream(&dev->pipe, false);
1237 stop_stream:
1238 	rkisp_stream_stop(stream);
1239 close_pipe:
1240 	dev->pipe.close(&dev->pipe);
1241 destroy_dummy_buf:
1242 	rkisp_destroy_dummy_buf(stream);
1243 buffer_done:
1244 	destroy_buf_queue(stream, VB2_BUF_STATE_QUEUED);
1245 	atomic_dec(&dev->cap_dev.refcnt);
1246 	stream->streaming = false;
1247 	mutex_unlock(&dev->hw_dev->dev_lock);
1248 	return ret;
1249 }
1250 
1251 static struct vb2_ops rkisp_vb2_ops = {
1252 	.queue_setup = rkisp_queue_setup,
1253 	.buf_queue = rkisp_buf_queue,
1254 	.wait_prepare = vb2_ops_wait_prepare,
1255 	.wait_finish = vb2_ops_wait_finish,
1256 	.stop_streaming = rkisp_stop_streaming,
1257 	.start_streaming = rkisp_start_streaming,
1258 };
1259 
rkisp_init_vb2_queue(struct vb2_queue * q,struct rkisp_stream * stream,enum v4l2_buf_type buf_type)1260 static int rkisp_init_vb2_queue(struct vb2_queue *q,
1261 				struct rkisp_stream *stream,
1262 				enum v4l2_buf_type buf_type)
1263 {
1264 	q->type = buf_type;
1265 	q->io_modes = VB2_MMAP | VB2_USERPTR | VB2_DMABUF;
1266 	q->drv_priv = stream;
1267 	q->ops = &rkisp_vb2_ops;
1268 	q->mem_ops = stream->ispdev->hw_dev->mem_ops;
1269 	q->buf_struct_size = sizeof(struct rkisp_buffer);
1270 	q->min_buffers_needed = CIF_ISP_REQ_BUFS_MIN;
1271 	q->timestamp_flags = V4L2_BUF_FLAG_TIMESTAMP_MONOTONIC;
1272 	q->lock = &stream->apilock;
1273 	q->dev = stream->ispdev->hw_dev->dev;
1274 	q->allow_cache_hints = 1;
1275 	q->bidirectional = 1;
1276 	if (stream->ispdev->hw_dev->is_dma_contig)
1277 		q->dma_attrs = DMA_ATTR_FORCE_CONTIGUOUS;
1278 	q->gfp_flags = GFP_DMA32;
1279 	return vb2_queue_init(q);
1280 }
1281 
rkisp_stream_init(struct rkisp_device * dev,u32 id)1282 static int rkisp_stream_init(struct rkisp_device *dev, u32 id)
1283 {
1284 	struct rkisp_capture_device *cap_dev = &dev->cap_dev;
1285 	struct rkisp_stream *stream;
1286 	struct video_device *vdev;
1287 	struct rkisp_vdev_node *node;
1288 	int ret = 0;
1289 
1290 	stream = &cap_dev->stream[id];
1291 	stream->id = id;
1292 	stream->ispdev = dev;
1293 	vdev = &stream->vnode.vdev;
1294 
1295 	INIT_LIST_HEAD(&stream->buf_queue);
1296 	init_waitqueue_head(&stream->done);
1297 	spin_lock_init(&stream->vbq_lock);
1298 	stream->linked = true;
1299 
1300 	switch (id) {
1301 	case RKISP_STREAM_SP:
1302 		strscpy(vdev->name, SP_VDEV_NAME, sizeof(vdev->name));
1303 		stream->ops = &rkisp_sp_streams_ops;
1304 		stream->config = &rkisp_sp_stream_config;
1305 		break;
1306 	case RKISP_STREAM_FBC:
1307 		strscpy(vdev->name, FBC_VDEV_NAME, sizeof(vdev->name));
1308 		stream->ops = &rkisp_fbc_streams_ops;
1309 		stream->config = &rkisp_fbc_stream_config;
1310 		break;
1311 	case RKISP_STREAM_BP:
1312 		strscpy(vdev->name, BP_VDEV_NAME, sizeof(vdev->name));
1313 		stream->ops = &rkisp_bp_streams_ops;
1314 		stream->config = &rkisp_bp_stream_config;
1315 		break;
1316 	case RKISP_STREAM_VIR:
1317 		strscpy(vdev->name, VIR_VDEV_NAME, sizeof(vdev->name));
1318 		stream->ops = NULL;
1319 		stream->config = &rkisp_mp_stream_config;
1320 		break;
1321 	default:
1322 		strscpy(vdev->name, MP_VDEV_NAME, sizeof(vdev->name));
1323 		stream->ops = &rkisp_mp_streams_ops;
1324 		stream->config = &rkisp_mp_stream_config;
1325 		if (dev->br_dev.linked)
1326 			stream->linked = false;
1327 	}
1328 
1329 	node = vdev_to_node(vdev);
1330 	rkisp_init_vb2_queue(&node->buf_queue, stream,
1331 			     V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE);
1332 	ret = rkisp_register_stream_vdev(stream);
1333 	if (ret < 0)
1334 		return ret;
1335 
1336 	stream->streaming = false;
1337 	stream->interlaced = false;
1338 	stream->burst =
1339 		CIF_MI_CTRL_BURST_LEN_LUM_16 |
1340 		CIF_MI_CTRL_BURST_LEN_CHROM_16;
1341 	atomic_set(&stream->sequence, 0);
1342 	return 0;
1343 }
1344 
rkisp_register_stream_v30(struct rkisp_device * dev)1345 int rkisp_register_stream_v30(struct rkisp_device *dev)
1346 {
1347 	struct rkisp_capture_device *cap_dev = &dev->cap_dev;
1348 	int ret;
1349 
1350 	ret = rkisp_stream_init(dev, RKISP_STREAM_MP);
1351 	if (ret < 0)
1352 		goto err;
1353 	ret = rkisp_stream_init(dev, RKISP_STREAM_SP);
1354 	if (ret < 0)
1355 		goto err_free_mp;
1356 	ret = rkisp_stream_init(dev, RKISP_STREAM_FBC);
1357 	if (ret < 0)
1358 		goto err_free_sp;
1359 	ret = rkisp_stream_init(dev, RKISP_STREAM_VIR);
1360 	if (ret < 0)
1361 		goto err_free_fbc;
1362 #ifdef RKISP_STREAM_BP_EN
1363 	ret = rkisp_stream_init(dev, RKISP_STREAM_BP);
1364 	if (ret < 0)
1365 		goto err_free_vir;
1366 #endif
1367 	return 0;
1368 #ifdef RKISP_STREAM_BP_EN
1369 err_free_vir:
1370 	rkisp_unregister_stream_vdev(&cap_dev->stream[RKISP_STREAM_VIR]);
1371 #endif
1372 err_free_fbc:
1373 	rkisp_unregister_stream_vdev(&cap_dev->stream[RKISP_STREAM_FBC]);
1374 err_free_sp:
1375 	rkisp_unregister_stream_vdev(&cap_dev->stream[RKISP_STREAM_SP]);
1376 err_free_mp:
1377 	rkisp_unregister_stream_vdev(&cap_dev->stream[RKISP_STREAM_MP]);
1378 err:
1379 	return ret;
1380 }
1381 
rkisp_unregister_stream_v30(struct rkisp_device * dev)1382 void rkisp_unregister_stream_v30(struct rkisp_device *dev)
1383 {
1384 	struct rkisp_capture_device *cap_dev = &dev->cap_dev;
1385 	struct rkisp_stream *stream;
1386 
1387 	stream = &cap_dev->stream[RKISP_STREAM_MP];
1388 	rkisp_unregister_stream_vdev(stream);
1389 	stream = &cap_dev->stream[RKISP_STREAM_SP];
1390 	rkisp_unregister_stream_vdev(stream);
1391 	stream = &cap_dev->stream[RKISP_STREAM_FBC];
1392 	rkisp_unregister_stream_vdev(stream);
1393 	stream = &cap_dev->stream[RKISP_STREAM_VIR];
1394 	rkisp_unregister_stream_vdev(stream);
1395 #ifdef RKISP_STREAM_BP_EN
1396 	stream = &cap_dev->stream[RKISP_STREAM_BP];
1397 	rkisp_unregister_stream_vdev(stream);
1398 #endif
1399 }
1400 
1401 /****************  Interrupter Handler ****************/
1402 
rkisp_mi_v30_isr(u32 mis_val,struct rkisp_device * dev)1403 void rkisp_mi_v30_isr(u32 mis_val, struct rkisp_device *dev)
1404 {
1405 	struct rkisp_stream *stream;
1406 	unsigned int i;
1407 
1408 	if (dev->hw_dev->is_unite) {
1409 		u32 val = rkisp_read(dev, ISP3X_MI_RIS, true);
1410 
1411 		if (val) {
1412 			rkisp_write(dev, ISP3X_MI_ICR, val, true);
1413 			v4l2_dbg(3, rkisp_debug, &dev->v4l2_dev,
1414 				 "left mi isr:0x%x\n", val);
1415 		}
1416 	}
1417 	v4l2_dbg(3, rkisp_debug, &dev->v4l2_dev,
1418 		 "mi isr:0x%x\n", mis_val);
1419 
1420 	rkisp_bridge_isr(&mis_val, dev);
1421 
1422 	for (i = 0; i < RKISP_MAX_STREAM; ++i) {
1423 		stream = &dev->cap_dev.stream[i];
1424 
1425 		if (!(mis_val & CIF_MI_FRAME(stream)) ||
1426 		    stream->id == RKISP_STREAM_VIR)
1427 			continue;
1428 
1429 		mi_frame_end_int_clear(stream);
1430 
1431 		if (stream->stopping) {
1432 			/*
1433 			 * Make sure stream is actually stopped, whose state
1434 			 * can be read from the shadow register, before
1435 			 * wake_up() thread which would immediately free all
1436 			 * frame buffers. disable_mi() takes effect at the next
1437 			 * frame end that sync the configurations to shadow
1438 			 * regs.
1439 			 */
1440 			if (!dev->hw_dev->is_single) {
1441 				stream->stopping = false;
1442 				stream->streaming = false;
1443 				stream->ops->disable_mi(stream);
1444 				wake_up(&stream->done);
1445 			} else if (stream->ops->is_stream_stopped(dev->base_addr)) {
1446 				stream->stopping = false;
1447 				stream->streaming = false;
1448 				wake_up(&stream->done);
1449 			}
1450 		} else {
1451 			mi_frame_end(stream);
1452 		}
1453 	}
1454 
1455 	if (mis_val & ISP3X_MI_MP_FRAME) {
1456 		stream = &dev->cap_dev.stream[RKISP_STREAM_MP];
1457 		if (!stream->streaming)
1458 			dev->irq_ends_mask &= ~ISP_FRAME_MP;
1459 		else
1460 			dev->irq_ends_mask |= ISP_FRAME_MP;
1461 		rkisp_check_idle(dev, ISP_FRAME_MP);
1462 	}
1463 	if (mis_val & ISP3X_MI_SP_FRAME) {
1464 		stream = &dev->cap_dev.stream[RKISP_STREAM_SP];
1465 		if (!stream->streaming)
1466 			dev->irq_ends_mask &= ~ISP_FRAME_SP;
1467 		else
1468 			dev->irq_ends_mask |= ISP_FRAME_SP;
1469 		rkisp_check_idle(dev, ISP_FRAME_SP);
1470 	}
1471 	if (mis_val & ISP3X_MI_MPFBC_FRAME) {
1472 		stream = &dev->cap_dev.stream[RKISP_STREAM_FBC];
1473 		if (!stream->streaming)
1474 			dev->irq_ends_mask &= ~ISP_FRAME_MPFBC;
1475 		else
1476 			dev->irq_ends_mask |= ISP_FRAME_MPFBC;
1477 		rkisp_check_idle(dev, ISP_FRAME_MPFBC);
1478 	}
1479 	if (mis_val & ISP3X_MI_BP_FRAME) {
1480 		stream = &dev->cap_dev.stream[RKISP_STREAM_BP];
1481 		if (!stream->streaming)
1482 			dev->irq_ends_mask &= ~ISP_FRAME_BP;
1483 		else
1484 			dev->irq_ends_mask |= ISP_FRAME_BP;
1485 		rkisp_check_idle(dev, ISP_FRAME_BP);
1486 	}
1487 }
1488 
rkisp_mipi_v30_isr(unsigned int phy,unsigned int packet,unsigned int overflow,unsigned int state,struct rkisp_device * dev)1489 void rkisp_mipi_v30_isr(unsigned int phy, unsigned int packet,
1490 			unsigned int overflow, unsigned int state,
1491 			struct rkisp_device *dev)
1492 {
1493 	if (state & GENMASK(19, 17))
1494 		v4l2_warn(&dev->v4l2_dev, "RD_SIZE_ERR:0x%08x\n", state);
1495 }
1496