• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 /*
2  * Copyright (c) 2015-2021, Renesas Electronics Corporation.
3  * All rights reserved.
4  *
5  * SPDX-License-Identifier: BSD-3-Clause
6  */
7 
8 #include <stdint.h>
9 #include <string.h>
10 #include <stdio.h>
11 
12 #include <common/debug.h>
13 #include <lib/mmio.h>
14 
15 #include "ddr_regdef.h"
16 #include "init_dram_tbl_h3.h"
17 #include "init_dram_tbl_m3.h"
18 #include "init_dram_tbl_h3ver2.h"
19 #include "init_dram_tbl_m3n.h"
20 #include "boot_init_dram_regdef.h"
21 #include "boot_init_dram.h"
22 #include "dram_sub_func.h"
23 #include "micro_delay.h"
24 #include "rcar_def.h"
25 
26 #define DDR_BACKUPMODE
27 #define FATAL_MSG(x) NOTICE(x)
28 
29 /* variables */
30 #ifdef RCAR_DDR_FIXED_LSI_TYPE
31 #ifndef RCAR_AUTO
32 #define RCAR_AUTO	99
33 #define RCAR_H3		0
34 #define RCAR_M3		1
35 #define RCAR_M3N	2
36 #define RCAR_E3		3	/* NON */
37 #define RCAR_H3N	4
38 
39 #define RZ_G2M		100U
40 #define RZ_G2H		101U
41 #define RZ_G2N		102U
42 
43 #define RCAR_CUT_10	0
44 #define RCAR_CUT_11	1
45 #define RCAR_CUT_20	10
46 #define RCAR_CUT_30	20
47 #endif
48 #ifndef RCAR_LSI
49 #define RCAR_LSI	RCAR_AUTO
50 #endif
51 
52 #if (RCAR_LSI == RCAR_AUTO)
53 static uint32_t prr_product;
54 static uint32_t prr_cut;
55 #else
56 #if (RCAR_LSI == RCAR_H3)
57 static const uint32_t prr_product = PRR_PRODUCT_H3;
58 #elif(RCAR_LSI == RCAR_M3 || RCAR_LSI == RZ_G2M)
59 static const uint32_t prr_product = PRR_PRODUCT_M3;
60 #elif(RCAR_LSI == RCAR_M3N || RCAR_LSI == RZ_G2N)
61 static const uint32_t prr_product = PRR_PRODUCT_M3N;
62 #elif(RCAR_LSI == RCAR_H3N || RCAR_LSI == RZ_G2H)
63 static const uint32_t prr_product = PRR_PRODUCT_H3;
64 #endif /* RCAR_LSI */
65 
66 #ifndef RCAR_LSI_CUT
67 static uint32_t prr_cut;
68 #else /* RCAR_LSI_CUT */
69 #if (RCAR_LSI_CUT == RCAR_CUT_10)
70 static const uint32_t prr_cut = PRR_PRODUCT_10;
71 #elif(RCAR_LSI_CUT == RCAR_CUT_11)
72 static const uint32_t prr_cut = PRR_PRODUCT_11;
73 #elif(RCAR_LSI_CUT == RCAR_CUT_20)
74 static const uint32_t prr_cut = PRR_PRODUCT_20;
75 #elif(RCAR_LSI_CUT == RCAR_CUT_30)
76 static const uint32_t prr_cut = PRR_PRODUCT_30;
77 #endif /* RCAR_LSI_CUT */
78 #endif /* RCAR_LSI_CUT */
79 #endif /* RCAR_AUTO_NON */
80 #else /* RCAR_DDR_FIXED_LSI_TYPE */
81 static uint32_t prr_product;
82 static uint32_t prr_cut;
83 #endif /* RCAR_DDR_FIXED_LSI_TYPE */
84 
85 static const uint32_t *p_ddr_regdef_tbl;
86 static uint32_t brd_clk;
87 static uint32_t brd_clkdiv;
88 static uint32_t brd_clkdiva;
89 static uint32_t ddr_mbps;
90 static uint32_t ddr_mbpsdiv;
91 static uint32_t ddr_tccd;
92 static uint32_t ddr_phycaslice;
93 static const struct _boardcnf *board_cnf;
94 static uint32_t ddr_phyvalid;
95 static uint32_t ddr_density[DRAM_CH_CNT][CS_CNT];
96 static uint32_t ch_have_this_cs[CS_CNT] __aligned(64);
97 static uint32_t rdqdm_dly[DRAM_CH_CNT][CSAB_CNT][SLICE_CNT * 2][9];
98 static uint32_t max_density;
99 static uint32_t ddr0800_mul;
100 static uint32_t ddr_mul;
101 static uint32_t DDR_PHY_SLICE_REGSET_OFS;
102 static uint32_t DDR_PHY_ADR_V_REGSET_OFS;
103 static uint32_t DDR_PHY_ADR_I_REGSET_OFS;
104 static uint32_t DDR_PHY_ADR_G_REGSET_OFS;
105 static uint32_t DDR_PI_REGSET_OFS;
106 static uint32_t DDR_PHY_SLICE_REGSET_SIZE;
107 static uint32_t DDR_PHY_ADR_V_REGSET_SIZE;
108 static uint32_t DDR_PHY_ADR_I_REGSET_SIZE;
109 static uint32_t DDR_PHY_ADR_G_REGSET_SIZE;
110 static uint32_t DDR_PI_REGSET_SIZE;
111 static uint32_t DDR_PHY_SLICE_REGSET_NUM;
112 static uint32_t DDR_PHY_ADR_V_REGSET_NUM;
113 static uint32_t DDR_PHY_ADR_I_REGSET_NUM;
114 static uint32_t DDR_PHY_ADR_G_REGSET_NUM;
115 static uint32_t DDR_PI_REGSET_NUM;
116 static uint32_t DDR_PHY_ADR_I_NUM;
117 #define DDR_PHY_REGSET_MAX 128
118 #define DDR_PI_REGSET_MAX 320
119 static uint32_t _cnf_DDR_PHY_SLICE_REGSET[DDR_PHY_REGSET_MAX];
120 static uint32_t _cnf_DDR_PHY_ADR_V_REGSET[DDR_PHY_REGSET_MAX];
121 static uint32_t _cnf_DDR_PHY_ADR_I_REGSET[DDR_PHY_REGSET_MAX];
122 static uint32_t _cnf_DDR_PHY_ADR_G_REGSET[DDR_PHY_REGSET_MAX];
123 static uint32_t _cnf_DDR_PI_REGSET[DDR_PI_REGSET_MAX];
124 static uint32_t pll3_mode;
125 static uint32_t loop_max;
126 #ifdef DDR_BACKUPMODE
127 uint32_t ddr_backup;
128 /* #define DDR_BACKUPMODE_HALF           //for Half channel(ch0,1 only) */
129 #endif
130 
131 #ifdef ddr_qos_init_setting	/*  only for non qos_init */
132 #define OPERATING_FREQ			(400U)	/* Mhz */
133 #define BASE_SUB_SLOT_NUM		(0x6U)
134 #define SUB_SLOT_CYCLE			(0x7EU)	/* 126 */
135 #define QOSWT_WTSET0_CYCLE		\
136 	((SUB_SLOT_CYCLE * BASE_SUB_SLOT_NUM * 1000U) / \
137 	OPERATING_FREQ)	/* unit:ns */
138 
get_refperiod(void)139 uint32_t get_refperiod(void)
140 {
141 	return QOSWT_WTSET0_CYCLE;
142 }
143 #else /*  ddr_qos_init_setting // only for non qos_init */
144 extern uint32_t get_refperiod(void);
145 #endif /* ddr_qos_init_setting // only for non qos_init */
146 
147 #define _reg_PHY_RX_CAL_X_NUM 11
148 static const uint32_t _reg_PHY_RX_CAL_X[_reg_PHY_RX_CAL_X_NUM] = {
149 	_reg_PHY_RX_CAL_DQ0,
150 	_reg_PHY_RX_CAL_DQ1,
151 	_reg_PHY_RX_CAL_DQ2,
152 	_reg_PHY_RX_CAL_DQ3,
153 	_reg_PHY_RX_CAL_DQ4,
154 	_reg_PHY_RX_CAL_DQ5,
155 	_reg_PHY_RX_CAL_DQ6,
156 	_reg_PHY_RX_CAL_DQ7,
157 	_reg_PHY_RX_CAL_DM,
158 	_reg_PHY_RX_CAL_DQS,
159 	_reg_PHY_RX_CAL_FDBK
160 };
161 
162 #define _reg_PHY_CLK_WRX_SLAVE_DELAY_NUM 10
163 static const uint32_t _reg_PHY_CLK_WRX_SLAVE_DELAY
164 	[_reg_PHY_CLK_WRX_SLAVE_DELAY_NUM] = {
165 	_reg_PHY_CLK_WRDQ0_SLAVE_DELAY,
166 	_reg_PHY_CLK_WRDQ1_SLAVE_DELAY,
167 	_reg_PHY_CLK_WRDQ2_SLAVE_DELAY,
168 	_reg_PHY_CLK_WRDQ3_SLAVE_DELAY,
169 	_reg_PHY_CLK_WRDQ4_SLAVE_DELAY,
170 	_reg_PHY_CLK_WRDQ5_SLAVE_DELAY,
171 	_reg_PHY_CLK_WRDQ6_SLAVE_DELAY,
172 	_reg_PHY_CLK_WRDQ7_SLAVE_DELAY,
173 	_reg_PHY_CLK_WRDM_SLAVE_DELAY,
174 	_reg_PHY_CLK_WRDQS_SLAVE_DELAY
175 };
176 
177 #define _reg_PHY_RDDQS_X_FALL_SLAVE_DELAY_NUM 9
178 static const uint32_t _reg_PHY_RDDQS_X_FALL_SLAVE_DELAY
179 	[_reg_PHY_RDDQS_X_FALL_SLAVE_DELAY_NUM] = {
180 	_reg_PHY_RDDQS_DQ0_FALL_SLAVE_DELAY,
181 	_reg_PHY_RDDQS_DQ1_FALL_SLAVE_DELAY,
182 	_reg_PHY_RDDQS_DQ2_FALL_SLAVE_DELAY,
183 	_reg_PHY_RDDQS_DQ3_FALL_SLAVE_DELAY,
184 	_reg_PHY_RDDQS_DQ4_FALL_SLAVE_DELAY,
185 	_reg_PHY_RDDQS_DQ5_FALL_SLAVE_DELAY,
186 	_reg_PHY_RDDQS_DQ6_FALL_SLAVE_DELAY,
187 	_reg_PHY_RDDQS_DQ7_FALL_SLAVE_DELAY,
188 	_reg_PHY_RDDQS_DM_FALL_SLAVE_DELAY
189 };
190 
191 #define _reg_PHY_RDDQS_X_RISE_SLAVE_DELAY_NUM 9
192 static const uint32_t _reg_PHY_RDDQS_X_RISE_SLAVE_DELAY
193 	[_reg_PHY_RDDQS_X_RISE_SLAVE_DELAY_NUM] = {
194 	_reg_PHY_RDDQS_DQ0_RISE_SLAVE_DELAY,
195 	_reg_PHY_RDDQS_DQ1_RISE_SLAVE_DELAY,
196 	_reg_PHY_RDDQS_DQ2_RISE_SLAVE_DELAY,
197 	_reg_PHY_RDDQS_DQ3_RISE_SLAVE_DELAY,
198 	_reg_PHY_RDDQS_DQ4_RISE_SLAVE_DELAY,
199 	_reg_PHY_RDDQS_DQ5_RISE_SLAVE_DELAY,
200 	_reg_PHY_RDDQS_DQ6_RISE_SLAVE_DELAY,
201 	_reg_PHY_RDDQS_DQ7_RISE_SLAVE_DELAY,
202 	_reg_PHY_RDDQS_DM_RISE_SLAVE_DELAY
203 };
204 
205 #define _reg_PHY_PAD_TERM_X_NUM 8
206 static const uint32_t _reg_PHY_PAD_TERM_X[_reg_PHY_PAD_TERM_X_NUM] = {
207 	_reg_PHY_PAD_FDBK_TERM,
208 	_reg_PHY_PAD_DATA_TERM,
209 	_reg_PHY_PAD_DQS_TERM,
210 	_reg_PHY_PAD_ADDR_TERM,
211 	_reg_PHY_PAD_CLK_TERM,
212 	_reg_PHY_PAD_CKE_TERM,
213 	_reg_PHY_PAD_RST_TERM,
214 	_reg_PHY_PAD_CS_TERM
215 };
216 
217 #define _reg_PHY_CLK_CACS_SLAVE_DELAY_X_NUM 10
218 static const uint32_t _reg_PHY_CLK_CACS_SLAVE_DELAY_X
219 	[_reg_PHY_CLK_CACS_SLAVE_DELAY_X_NUM] = {
220 	_reg_PHY_ADR0_CLK_WR_SLAVE_DELAY,
221 	_reg_PHY_ADR1_CLK_WR_SLAVE_DELAY,
222 	_reg_PHY_ADR2_CLK_WR_SLAVE_DELAY,
223 	_reg_PHY_ADR3_CLK_WR_SLAVE_DELAY,
224 	_reg_PHY_ADR4_CLK_WR_SLAVE_DELAY,
225 	_reg_PHY_ADR5_CLK_WR_SLAVE_DELAY,
226 
227 	_reg_PHY_GRP_SLAVE_DELAY_0,
228 	_reg_PHY_GRP_SLAVE_DELAY_1,
229 	_reg_PHY_GRP_SLAVE_DELAY_2,
230 	_reg_PHY_GRP_SLAVE_DELAY_3
231 };
232 
233 /* Prototypes */
234 static inline uint32_t vch_nxt(uint32_t pos);
235 static void cpg_write_32(uint32_t a, uint32_t v);
236 static void pll3_control(uint32_t high);
237 static inline void dsb_sev(void);
238 static void wait_dbcmd(void);
239 static void send_dbcmd(uint32_t cmd);
240 static uint32_t reg_ddrphy_read(uint32_t phyno, uint32_t regadd);
241 static void reg_ddrphy_write(uint32_t phyno, uint32_t regadd, uint32_t regdata);
242 static void reg_ddrphy_write_a(uint32_t regadd, uint32_t regdata);
243 static inline uint32_t ddr_regdef(uint32_t _regdef);
244 static inline uint32_t ddr_regdef_adr(uint32_t _regdef);
245 static inline uint32_t ddr_regdef_lsb(uint32_t _regdef);
246 static void ddr_setval_s(uint32_t ch, uint32_t slice, uint32_t _regdef,
247 			 uint32_t val);
248 static uint32_t ddr_getval_s(uint32_t ch, uint32_t slice, uint32_t _regdef);
249 static void ddr_setval(uint32_t ch, uint32_t regdef, uint32_t val);
250 static void ddr_setval_ach_s(uint32_t slice, uint32_t regdef, uint32_t val);
251 static void ddr_setval_ach(uint32_t regdef, uint32_t val);
252 static void ddr_setval_ach_as(uint32_t regdef, uint32_t val);
253 static uint32_t ddr_getval(uint32_t ch, uint32_t regdef);
254 static uint32_t ddr_getval_ach(uint32_t regdef, uint32_t *p);
255 static uint32_t ddr_getval_ach_as(uint32_t regdef, uint32_t *p);
256 static void _tblcopy(uint32_t *to, const uint32_t *from, uint32_t size);
257 static void ddrtbl_setval(uint32_t *tbl, uint32_t _regdef, uint32_t val);
258 static uint32_t ddrtbl_getval(uint32_t *tbl, uint32_t _regdef);
259 static uint32_t ddrphy_regif_chk(void);
260 static inline void ddrphy_regif_idle(void);
261 static uint16_t _f_scale(uint32_t _ddr_mbps, uint32_t _ddr_mbpsdiv, uint32_t ps,
262 			 uint16_t cyc);
263 static void _f_scale_js2(uint32_t _ddr_mbps, uint32_t _ddr_mbpsdiv,
264 			 uint16_t *_js2);
265 static int16_t _f_scale_adj(int16_t ps);
266 static void ddrtbl_load(void);
267 static void ddr_config_sub(void);
268 static void get_ca_swizzle(uint32_t ch, uint32_t ddr_csn, uint32_t *p_swz);
269 static void ddr_config_sub_h3v1x(void);
270 static void ddr_config(void);
271 static void dbsc_regset(void);
272 static void dbsc_regset_post(void);
273 static uint32_t dfi_init_start(void);
274 static void change_lpddr4_en(uint32_t mode);
275 static uint32_t set_term_code(void);
276 static void ddr_register_set(void);
277 static inline uint32_t wait_freqchgreq(uint32_t assert);
278 static inline void set_freqchgack(uint32_t assert);
279 static inline void set_dfifrequency(uint32_t freq);
280 static uint32_t pll3_freq(uint32_t on);
281 static void update_dly(void);
282 static uint32_t pi_training_go(void);
283 static uint32_t init_ddr(void);
284 static uint32_t swlvl1(uint32_t ddr_csn, uint32_t reg_cs, uint32_t reg_kick);
285 static uint32_t wdqdm_man1(void);
286 static uint32_t wdqdm_man(void);
287 static uint32_t rdqdm_man1(void);
288 static uint32_t rdqdm_man(void);
289 
290 static int32_t _find_change(uint64_t val, uint32_t dir);
291 static uint32_t _rx_offset_cal_updn(uint32_t code);
292 static uint32_t rx_offset_cal(void);
293 static uint32_t rx_offset_cal_hw(void);
294 static void adjust_rddqs_latency(void);
295 static void adjust_wpath_latency(void);
296 
297 struct ddrt_data {
298 	int32_t init_temp;	/* Initial Temperature (do) */
299 	uint32_t init_cal[4];	/* Initial io-code (4 is for H3) */
300 	uint32_t tcomp_cal[4];	/* Temp. compensated io-code (4 is for H3) */
301 };
302 
303 static struct ddrt_data tcal;
304 
305 static void pvtcode_update(void);
306 static void pvtcode_update2(void);
307 static void ddr_padcal_tcompensate_getinit(uint32_t override);
308 
309 /* load board configuration */
310 #include "boot_init_dram_config.c"
311 
312 #ifndef DDR_FAST_INIT
313 static uint32_t rdqdm_le[DRAM_CH_CNT][CS_CNT][SLICE_CNT * 2][9];
314 static uint32_t rdqdm_te[DRAM_CH_CNT][CS_CNT][SLICE_CNT * 2][9];
315 static uint32_t rdqdm_nw[DRAM_CH_CNT][CS_CNT][SLICE_CNT * 2][9];
316 static uint32_t rdqdm_win[DRAM_CH_CNT][CS_CNT][SLICE_CNT];
317 static uint32_t rdqdm_st[DRAM_CH_CNT][CS_CNT][SLICE_CNT * 2];
318 static void rdqdm_clr1(uint32_t ch, uint32_t ddr_csn);
319 static uint32_t rdqdm_ana1(uint32_t ch, uint32_t ddr_csn);
320 
321 static uint32_t wdqdm_le[DRAM_CH_CNT][CS_CNT][SLICE_CNT][9];
322 static uint32_t wdqdm_te[DRAM_CH_CNT][CS_CNT][SLICE_CNT][9];
323 static uint32_t wdqdm_dly[DRAM_CH_CNT][CS_CNT][SLICE_CNT][9];
324 static uint32_t wdqdm_st[DRAM_CH_CNT][CS_CNT][SLICE_CNT];
325 static uint32_t wdqdm_win[DRAM_CH_CNT][CS_CNT][SLICE_CNT];
326 static void wdqdm_clr1(uint32_t ch, uint32_t ddr_csn);
327 static uint32_t wdqdm_ana1(uint32_t ch, uint32_t ddr_csn);
328 #endif/* DDR_FAST_INIT */
329 
330 /* macro for channel selection loop */
vch_nxt(uint32_t pos)331 static inline uint32_t vch_nxt(uint32_t pos)
332 {
333 	uint32_t posn;
334 
335 	for (posn = pos; posn < DRAM_CH_CNT; posn++) {
336 		if (ddr_phyvalid & (1U << posn))
337 			break;
338 	}
339 	return posn;
340 }
341 
342 #define foreach_vch(ch) \
343 for (ch = vch_nxt(0); ch < DRAM_CH_CNT; ch = vch_nxt(ch + 1))
344 
345 #define foreach_ech(ch) \
346 for (ch = 0; ch < DRAM_CH_CNT; ch++)
347 
348 /* Printing functions */
349 #define MSG_LF(...)
350 
351 /* clock settings, reset control */
cpg_write_32(uint32_t a,uint32_t v)352 static void cpg_write_32(uint32_t a, uint32_t v)
353 {
354 	mmio_write_32(CPG_CPGWPR, ~v);
355 	mmio_write_32(a, v);
356 }
357 
pll3_control(uint32_t high)358 static void pll3_control(uint32_t high)
359 {
360 	uint32_t data_l, data_div, data_mul, tmp_div;
361 
362 	if (high) {
363 		tmp_div = 3999 * brd_clkdiv * (brd_clkdiva + 1) /
364 			(brd_clk * ddr_mul) / 2;
365 		data_mul = ((ddr_mul * tmp_div) - 1) << 24;
366 		pll3_mode = 1;
367 		loop_max = 2;
368 	} else {
369 		tmp_div = 3999 * brd_clkdiv * (brd_clkdiva + 1) /
370 			(brd_clk * ddr0800_mul) / 2;
371 		data_mul = ((ddr0800_mul * tmp_div) - 1) << 24;
372 		pll3_mode = 0;
373 		loop_max = 8;
374 	}
375 
376 	switch (tmp_div) {
377 	case 1:
378 		data_div = 0;
379 		break;
380 	case 2:
381 	case 3:
382 	case 4:
383 		data_div = tmp_div;
384 		break;
385 	default:
386 		data_div = 6;
387 		data_mul = (data_mul * tmp_div) / 3;
388 		break;
389 	}
390 	data_mul = data_mul | (brd_clkdiva << 7);
391 
392 	/* PLL3 disable */
393 	data_l = mmio_read_32(CPG_PLLECR) & ~CPG_PLLECR_PLL3E_BIT;
394 	cpg_write_32(CPG_PLLECR, data_l);
395 	dsb_sev();
396 
397 	if ((prr_product == PRR_PRODUCT_M3) ||
398 	    ((prr_product == PRR_PRODUCT_H3) && (prr_cut <= PRR_PRODUCT_20))) {
399 		/* PLL3 DIV resetting(Lowest value:3) */
400 		data_l = 0x00030003 | (0xFF80FF80 & mmio_read_32(CPG_FRQCRD));
401 		cpg_write_32(CPG_FRQCRD, data_l);
402 		dsb_sev();
403 
404 		/* zb3 clk stop */
405 		data_l = CPG_ZB3CKCR_ZB3ST_BIT | mmio_read_32(CPG_ZB3CKCR);
406 		cpg_write_32(CPG_ZB3CKCR, data_l);
407 		dsb_sev();
408 
409 		/* PLL3 enable */
410 		data_l = CPG_PLLECR_PLL3E_BIT | mmio_read_32(CPG_PLLECR);
411 		cpg_write_32(CPG_PLLECR, data_l);
412 		dsb_sev();
413 
414 		do {
415 			data_l = mmio_read_32(CPG_PLLECR);
416 		} while ((data_l & CPG_PLLECR_PLL3ST_BIT) == 0);
417 		dsb_sev();
418 
419 		/* PLL3 DIV resetting (Highest value:0) */
420 		data_l = (0xFF80FF80 & mmio_read_32(CPG_FRQCRD));
421 		cpg_write_32(CPG_FRQCRD, data_l);
422 		dsb_sev();
423 
424 		/* DIV SET KICK */
425 		data_l = CPG_FRQCRB_KICK_BIT | mmio_read_32(CPG_FRQCRB);
426 		cpg_write_32(CPG_FRQCRB, data_l);
427 		dsb_sev();
428 
429 		/* PLL3 multiplie set */
430 		cpg_write_32(CPG_PLL3CR, data_mul);
431 		dsb_sev();
432 
433 		do {
434 			data_l = mmio_read_32(CPG_PLLECR);
435 		} while ((data_l & CPG_PLLECR_PLL3ST_BIT) == 0);
436 		dsb_sev();
437 
438 		/* PLL3 DIV resetting(Target value) */
439 		data_l = (data_div << 16) | data_div |
440 			 (mmio_read_32(CPG_FRQCRD) & 0xFF80FF80);
441 		cpg_write_32(CPG_FRQCRD, data_l);
442 		dsb_sev();
443 
444 		/* DIV SET KICK */
445 		data_l = CPG_FRQCRB_KICK_BIT | mmio_read_32(CPG_FRQCRB);
446 		cpg_write_32(CPG_FRQCRB, data_l);
447 		dsb_sev();
448 
449 		do {
450 			data_l = mmio_read_32(CPG_PLLECR);
451 		} while ((data_l & CPG_PLLECR_PLL3ST_BIT) == 0);
452 		dsb_sev();
453 
454 		/* zb3 clk start */
455 		data_l = (~CPG_ZB3CKCR_ZB3ST_BIT) & mmio_read_32(CPG_ZB3CKCR);
456 		cpg_write_32(CPG_ZB3CKCR, data_l);
457 		dsb_sev();
458 
459 	} else { /*  H3Ver.3.0/M3N/V3H */
460 
461 		/* PLL3 multiplie set */
462 		cpg_write_32(CPG_PLL3CR, data_mul);
463 		dsb_sev();
464 
465 		/* PLL3 DIV set(Target value) */
466 		data_l = (data_div << 16) | data_div |
467 			 (mmio_read_32(CPG_FRQCRD) & 0xFF80FF80);
468 		cpg_write_32(CPG_FRQCRD, data_l);
469 
470 		/* DIV SET KICK */
471 		data_l = CPG_FRQCRB_KICK_BIT | mmio_read_32(CPG_FRQCRB);
472 		cpg_write_32(CPG_FRQCRB, data_l);
473 		dsb_sev();
474 
475 		/* PLL3 enable */
476 		data_l = CPG_PLLECR_PLL3E_BIT | mmio_read_32(CPG_PLLECR);
477 		cpg_write_32(CPG_PLLECR, data_l);
478 		dsb_sev();
479 
480 		do {
481 			data_l = mmio_read_32(CPG_PLLECR);
482 		} while ((data_l & CPG_PLLECR_PLL3ST_BIT) == 0);
483 		dsb_sev();
484 	}
485 }
486 
487 /* barrier */
dsb_sev(void)488 static inline void dsb_sev(void)
489 {
490 	__asm__ __volatile__("dsb sy");
491 }
492 
493 /* DDR memory register access */
wait_dbcmd(void)494 static void wait_dbcmd(void)
495 {
496 	uint32_t data_l;
497 	/* dummy read */
498 	data_l = mmio_read_32(DBSC_DBCMD);
499 	dsb_sev();
500 	while (1) {
501 		/* wait DBCMD 1=busy, 0=ready */
502 		data_l = mmio_read_32(DBSC_DBWAIT);
503 		dsb_sev();
504 		if ((data_l & 0x00000001) == 0x00)
505 			break;
506 	}
507 }
508 
send_dbcmd(uint32_t cmd)509 static void send_dbcmd(uint32_t cmd)
510 {
511 	/* dummy read */
512 	wait_dbcmd();
513 	mmio_write_32(DBSC_DBCMD, cmd);
514 	dsb_sev();
515 }
516 
dbwait_loop(uint32_t wait_loop)517 static void dbwait_loop(uint32_t wait_loop)
518 {
519 	uint32_t i;
520 
521 	for (i = 0; i < wait_loop; i++)
522 		wait_dbcmd();
523 }
524 
525 /* DDRPHY register access (raw) */
reg_ddrphy_read(uint32_t phyno,uint32_t regadd)526 static uint32_t reg_ddrphy_read(uint32_t phyno, uint32_t regadd)
527 {
528 	uint32_t val;
529 	uint32_t loop;
530 
531 	val = 0;
532 	if ((prr_product != PRR_PRODUCT_M3N) &&
533 	    (prr_product != PRR_PRODUCT_V3H)) {
534 		mmio_write_32(DBSC_DBPDRGA(phyno), regadd);
535 		dsb_sev();
536 
537 		while (mmio_read_32(DBSC_DBPDRGA(phyno)) != regadd) {
538 			dsb_sev();
539 		}
540 		dsb_sev();
541 
542 		for (loop = 0; loop < loop_max; loop++) {
543 			val = mmio_read_32(DBSC_DBPDRGD(phyno));
544 			dsb_sev();
545 		}
546 		(void)val;
547 	} else {
548 		mmio_write_32(DBSC_DBPDRGA(phyno), regadd | 0x00004000);
549 		dsb_sev();
550 		while (mmio_read_32(DBSC_DBPDRGA(phyno)) !=
551 		       (regadd | 0x0000C000)) {
552 			dsb_sev();
553 		};
554 		val = mmio_read_32(DBSC_DBPDRGA(phyno));
555 		mmio_write_32(DBSC_DBPDRGA(phyno), regadd | 0x00008000);
556 		while (mmio_read_32(DBSC_DBPDRGA(phyno)) != regadd) {
557 			dsb_sev();
558 		};
559 		dsb_sev();
560 
561 		mmio_write_32(DBSC_DBPDRGA(phyno), regadd | 0x00008000);
562 		while (mmio_read_32(DBSC_DBPDRGA(phyno)) != regadd) {
563 			dsb_sev();
564 		};
565 
566 		dsb_sev();
567 		val = mmio_read_32(DBSC_DBPDRGD(phyno));
568 		dsb_sev();
569 		(void)val;
570 	}
571 	return val;
572 }
573 
reg_ddrphy_write(uint32_t phyno,uint32_t regadd,uint32_t regdata)574 static void reg_ddrphy_write(uint32_t phyno, uint32_t regadd, uint32_t regdata)
575 {
576 	uint32_t val;
577 	uint32_t loop;
578 
579 	if ((prr_product != PRR_PRODUCT_M3N) &&
580 	    (prr_product != PRR_PRODUCT_V3H)) {
581 		mmio_write_32(DBSC_DBPDRGA(phyno), regadd);
582 		dsb_sev();
583 		for (loop = 0; loop < loop_max; loop++) {
584 			val = mmio_read_32(DBSC_DBPDRGA(phyno));
585 			dsb_sev();
586 		}
587 		mmio_write_32(DBSC_DBPDRGD(phyno), regdata);
588 		dsb_sev();
589 
590 		for (loop = 0; loop < loop_max; loop++) {
591 			val = mmio_read_32(DBSC_DBPDRGD(phyno));
592 			dsb_sev();
593 		}
594 	} else {
595 		mmio_write_32(DBSC_DBPDRGA(phyno), regadd);
596 		dsb_sev();
597 
598 		while (mmio_read_32(DBSC_DBPDRGA(phyno)) != regadd) {
599 			dsb_sev();
600 		};
601 		dsb_sev();
602 
603 		mmio_write_32(DBSC_DBPDRGD(phyno), regdata);
604 		dsb_sev();
605 
606 		while (mmio_read_32(DBSC_DBPDRGA(phyno)) !=
607 		       (regadd | 0x00008000)) {
608 			dsb_sev();
609 		};
610 		mmio_write_32(DBSC_DBPDRGA(phyno), regadd | 0x00008000);
611 
612 		while (mmio_read_32(DBSC_DBPDRGA(phyno)) != regadd) {
613 			dsb_sev();
614 		};
615 		dsb_sev();
616 
617 		mmio_write_32(DBSC_DBPDRGA(phyno), regadd);
618 	}
619 	(void)val;
620 }
621 
reg_ddrphy_write_a(uint32_t regadd,uint32_t regdata)622 static void reg_ddrphy_write_a(uint32_t regadd, uint32_t regdata)
623 {
624 	uint32_t ch;
625 	uint32_t val;
626 	uint32_t loop;
627 
628 	if ((prr_product != PRR_PRODUCT_M3N) &&
629 	    (prr_product != PRR_PRODUCT_V3H)) {
630 		foreach_vch(ch) {
631 			mmio_write_32(DBSC_DBPDRGA(ch), regadd);
632 			dsb_sev();
633 		}
634 
635 		foreach_vch(ch) {
636 			mmio_write_32(DBSC_DBPDRGD(ch), regdata);
637 			dsb_sev();
638 		}
639 
640 		for (loop = 0; loop < loop_max; loop++) {
641 			val = mmio_read_32(DBSC_DBPDRGD(0));
642 			dsb_sev();
643 		}
644 		(void)val;
645 	} else {
646 		foreach_vch(ch) {
647 			reg_ddrphy_write(ch, regadd, regdata);
648 			dsb_sev();
649 		}
650 	}
651 }
652 
ddrphy_regif_idle(void)653 static inline void ddrphy_regif_idle(void)
654 {
655 	uint32_t val;
656 
657 	val = reg_ddrphy_read(0, ddr_regdef_adr(_reg_PI_INT_STATUS));
658 	dsb_sev();
659 	(void)val;
660 }
661 
662 /* DDRPHY register access (field modify) */
ddr_regdef(uint32_t _regdef)663 static inline uint32_t ddr_regdef(uint32_t _regdef)
664 {
665 	return p_ddr_regdef_tbl[_regdef];
666 }
667 
ddr_regdef_adr(uint32_t _regdef)668 static inline uint32_t ddr_regdef_adr(uint32_t _regdef)
669 {
670 	return DDR_REGDEF_ADR(p_ddr_regdef_tbl[_regdef]);
671 }
672 
ddr_regdef_lsb(uint32_t _regdef)673 static inline uint32_t ddr_regdef_lsb(uint32_t _regdef)
674 {
675 	return DDR_REGDEF_LSB(p_ddr_regdef_tbl[_regdef]);
676 }
677 
ddr_setval_s(uint32_t ch,uint32_t slice,uint32_t _regdef,uint32_t val)678 static void ddr_setval_s(uint32_t ch, uint32_t slice, uint32_t _regdef,
679 			 uint32_t val)
680 {
681 	uint32_t adr;
682 	uint32_t lsb;
683 	uint32_t len;
684 	uint32_t msk;
685 	uint32_t tmp;
686 	uint32_t regdef;
687 
688 	regdef = ddr_regdef(_regdef);
689 	adr = DDR_REGDEF_ADR(regdef) + 0x80 * slice;
690 	len = DDR_REGDEF_LEN(regdef);
691 	lsb = DDR_REGDEF_LSB(regdef);
692 	if (len == 0x20)
693 		msk = 0xffffffff;
694 	else
695 		msk = ((1U << len) - 1) << lsb;
696 
697 	tmp = reg_ddrphy_read(ch, adr);
698 	tmp = (tmp & (~msk)) | ((val << lsb) & msk);
699 	reg_ddrphy_write(ch, adr, tmp);
700 }
701 
ddr_getval_s(uint32_t ch,uint32_t slice,uint32_t _regdef)702 static uint32_t ddr_getval_s(uint32_t ch, uint32_t slice, uint32_t _regdef)
703 {
704 	uint32_t adr;
705 	uint32_t lsb;
706 	uint32_t len;
707 	uint32_t msk;
708 	uint32_t tmp;
709 	uint32_t regdef;
710 
711 	regdef = ddr_regdef(_regdef);
712 	adr = DDR_REGDEF_ADR(regdef) + 0x80 * slice;
713 	len = DDR_REGDEF_LEN(regdef);
714 	lsb = DDR_REGDEF_LSB(regdef);
715 	if (len == 0x20)
716 		msk = 0xffffffff;
717 	else
718 		msk = ((1U << len) - 1);
719 
720 	tmp = reg_ddrphy_read(ch, adr);
721 	tmp = (tmp >> lsb) & msk;
722 
723 	return tmp;
724 }
725 
ddr_setval(uint32_t ch,uint32_t regdef,uint32_t val)726 static void ddr_setval(uint32_t ch, uint32_t regdef, uint32_t val)
727 {
728 	ddr_setval_s(ch, 0, regdef, val);
729 }
730 
ddr_setval_ach_s(uint32_t slice,uint32_t regdef,uint32_t val)731 static void ddr_setval_ach_s(uint32_t slice, uint32_t regdef, uint32_t val)
732 {
733 	uint32_t ch;
734 
735 	foreach_vch(ch)
736 	    ddr_setval_s(ch, slice, regdef, val);
737 }
738 
ddr_setval_ach(uint32_t regdef,uint32_t val)739 static void ddr_setval_ach(uint32_t regdef, uint32_t val)
740 {
741 	ddr_setval_ach_s(0, regdef, val);
742 }
743 
ddr_setval_ach_as(uint32_t regdef,uint32_t val)744 static void ddr_setval_ach_as(uint32_t regdef, uint32_t val)
745 {
746 	uint32_t slice;
747 
748 	for (slice = 0; slice < SLICE_CNT; slice++)
749 		ddr_setval_ach_s(slice, regdef, val);
750 }
751 
ddr_getval(uint32_t ch,uint32_t regdef)752 static uint32_t ddr_getval(uint32_t ch, uint32_t regdef)
753 {
754 	return ddr_getval_s(ch, 0, regdef);
755 }
756 
ddr_getval_ach(uint32_t regdef,uint32_t * p)757 static uint32_t ddr_getval_ach(uint32_t regdef, uint32_t *p)
758 {
759 	uint32_t ch;
760 
761 	foreach_vch(ch)
762 	    p[ch] = ddr_getval_s(ch, 0, regdef);
763 	return p[0];
764 }
765 
ddr_getval_ach_as(uint32_t regdef,uint32_t * p)766 static uint32_t ddr_getval_ach_as(uint32_t regdef, uint32_t *p)
767 {
768 	uint32_t ch, slice;
769 	uint32_t *pp;
770 
771 	pp = p;
772 	foreach_vch(ch)
773 		for (slice = 0; slice < SLICE_CNT; slice++)
774 			*pp++ = ddr_getval_s(ch, slice, regdef);
775 	return p[0];
776 }
777 
778 /* handling functions for setteing ddrphy value table */
_tblcopy(uint32_t * to,const uint32_t * from,uint32_t size)779 static void _tblcopy(uint32_t *to, const uint32_t *from, uint32_t size)
780 {
781 	uint32_t i;
782 
783 	for (i = 0; i < size; i++) {
784 		to[i] = from[i];
785 	}
786 }
787 
ddrtbl_setval(uint32_t * tbl,uint32_t _regdef,uint32_t val)788 static void ddrtbl_setval(uint32_t *tbl, uint32_t _regdef, uint32_t val)
789 {
790 	uint32_t adr;
791 	uint32_t lsb;
792 	uint32_t len;
793 	uint32_t msk;
794 	uint32_t tmp;
795 	uint32_t adrmsk;
796 	uint32_t regdef;
797 
798 	regdef = ddr_regdef(_regdef);
799 	adr = DDR_REGDEF_ADR(regdef);
800 	len = DDR_REGDEF_LEN(regdef);
801 	lsb = DDR_REGDEF_LSB(regdef);
802 	if (len == 0x20)
803 		msk = 0xffffffff;
804 	else
805 		msk = ((1U << len) - 1) << lsb;
806 
807 	if (adr < 0x400) {
808 		adrmsk = 0xff;
809 	} else {
810 		adrmsk = 0x7f;
811 	}
812 
813 	tmp = tbl[adr & adrmsk];
814 	tmp = (tmp & (~msk)) | ((val << lsb) & msk);
815 	tbl[adr & adrmsk] = tmp;
816 }
817 
ddrtbl_getval(uint32_t * tbl,uint32_t _regdef)818 static uint32_t ddrtbl_getval(uint32_t *tbl, uint32_t _regdef)
819 {
820 	uint32_t adr;
821 	uint32_t lsb;
822 	uint32_t len;
823 	uint32_t msk;
824 	uint32_t tmp;
825 	uint32_t adrmsk;
826 	uint32_t regdef;
827 
828 	regdef = ddr_regdef(_regdef);
829 	adr = DDR_REGDEF_ADR(regdef);
830 	len = DDR_REGDEF_LEN(regdef);
831 	lsb = DDR_REGDEF_LSB(regdef);
832 	if (len == 0x20)
833 		msk = 0xffffffff;
834 	else
835 		msk = ((1U << len) - 1);
836 
837 	if (adr < 0x400) {
838 		adrmsk = 0xff;
839 	} else {
840 		adrmsk = 0x7f;
841 	}
842 
843 	tmp = tbl[adr & adrmsk];
844 	tmp = (tmp >> lsb) & msk;
845 
846 	return tmp;
847 }
848 
849 /* DDRPHY register access handling */
ddrphy_regif_chk(void)850 static uint32_t ddrphy_regif_chk(void)
851 {
852 	uint32_t tmp_ach[DRAM_CH_CNT];
853 	uint32_t ch;
854 	uint32_t err;
855 	uint32_t PI_VERSION_CODE;
856 
857 	if (((prr_product == PRR_PRODUCT_H3) && (prr_cut <= PRR_PRODUCT_11)) ||
858 	    (prr_product == PRR_PRODUCT_M3)) {
859 		PI_VERSION_CODE = 0x2041; /* H3 Ver.1.x/M3-W */
860 	} else {
861 		PI_VERSION_CODE = 0x2040; /* H3 Ver.2.0 or later/M3-N/V3H */
862 	}
863 
864 	ddr_getval_ach(_reg_PI_VERSION, (uint32_t *)tmp_ach);
865 	err = 0;
866 	foreach_vch(ch) {
867 		if (tmp_ach[ch] != PI_VERSION_CODE)
868 			err = 1;
869 	}
870 	return err;
871 }
872 
873 /* functions and parameters for timing setting */
874 struct _jedec_spec1 {
875 	uint16_t fx3;
876 	uint8_t rlwodbi;
877 	uint8_t rlwdbi;
878 	uint8_t WL;
879 	uint8_t nwr;
880 	uint8_t nrtp;
881 	uint8_t odtlon;
882 	uint8_t MR1;
883 	uint8_t MR2;
884 };
885 
886 #define JS1_USABLEC_SPEC_LO 2
887 #define JS1_USABLEC_SPEC_HI 5
888 #define JS1_FREQ_TBL_NUM 8
889 #define JS1_MR1(f) (0x04 | ((f) << 4))
890 #define JS1_MR2(f) (0x00 | ((f) << 3) | (f))
891 const struct _jedec_spec1 js1[JS1_FREQ_TBL_NUM] = {
892 	/* 533.333Mbps */
893 	{  800,  6,  6,  4,  6,  8, 0, JS1_MR1(0), JS1_MR2(0) | 0x40 },
894 	/* 1066.666Mbps */
895 	{ 1600, 10, 12,  8, 10,  8, 0, JS1_MR1(1), JS1_MR2(1) | 0x40 },
896 	/* 1600.000Mbps */
897 	{ 2400, 14, 16, 12, 16,  8, 6, JS1_MR1(2), JS1_MR2(2) | 0x40 },
898 	/* 2133.333Mbps */
899 	{ 3200, 20, 22, 10, 20,  8, 4, JS1_MR1(3), JS1_MR2(3) },
900 	/* 2666.666Mbps */
901 	{ 4000, 24, 28, 12, 24, 10, 4, JS1_MR1(4), JS1_MR2(4) },
902 	/* 3200.000Mbps */
903 	{ 4800, 28, 32, 14, 30, 12, 6, JS1_MR1(5), JS1_MR2(5) },
904 	/* 3733.333Mbps */
905 	{ 5600, 32, 36, 16, 34, 14, 6, JS1_MR1(6), JS1_MR2(6) },
906 	/* 4266.666Mbps */
907 	{ 6400, 36, 40, 18, 40, 16, 8, JS1_MR1(7), JS1_MR2(7) }
908 };
909 
910 struct _jedec_spec2 {
911 	uint16_t ps;
912 	uint16_t cyc;
913 };
914 
915 #define js2_tsr 0
916 #define js2_txp 1
917 #define js2_trtp 2
918 #define js2_trcd 3
919 #define js2_trppb 4
920 #define js2_trpab 5
921 #define js2_tras 6
922 #define js2_twr 7
923 #define js2_twtr 8
924 #define js2_trrd 9
925 #define js2_tppd 10
926 #define js2_tfaw 11
927 #define js2_tdqsck 12
928 #define js2_tckehcmd 13
929 #define js2_tckelcmd 14
930 #define js2_tckelpd 15
931 #define js2_tmrr 16
932 #define js2_tmrw 17
933 #define js2_tmrd 18
934 #define js2_tzqcalns 19
935 #define js2_tzqlat 20
936 #define js2_tiedly 21
937 #define js2_tODTon_min 22
938 #define JS2_TBLCNT 23
939 
940 #define js2_trcpb (JS2_TBLCNT)
941 #define js2_trcab (JS2_TBLCNT + 1)
942 #define js2_trfcab (JS2_TBLCNT + 2)
943 #define JS2_CNT (JS2_TBLCNT + 3)
944 
945 #ifndef JS2_DERATE
946 #define JS2_DERATE 0
947 #endif
948 const struct _jedec_spec2 jedec_spec2[2][JS2_TBLCNT] = {
949 	{
950 /*tSR   */ {15000, 3},
951 /*tXP   */ {7500, 3},
952 /*tRTP  */ {7500, 8},
953 /*tRCD  */ {18000, 4},
954 /*tRPpb */ {18000, 3},
955 /*tRPab */ {21000, 3},
956 /*tRAS  */ {42000, 3},
957 /*tWR   */ {18000, 4},
958 /*tWTR  */ {10000, 8},
959 /*tRRD  */ {10000, 4},
960 /*tPPD  */ {0, 0},
961 /*tFAW  */ {40000, 0},
962 /*tDQSCK*/ {3500, 0},
963 /*tCKEHCMD*/ {7500, 3},
964 /*tCKELCMD*/ {7500, 3},
965 /*tCKELPD*/ {7500, 3},
966 /*tMRR*/ {0, 8},
967 /*tMRW*/ {10000, 10},
968 /*tMRD*/ {14000, 10},
969 /*tZQCALns*/ {1000 * 10, 0},
970 /*tZQLAT*/ {30000, 10},
971 /*tIEdly*/ {12500, 0},
972 /*tODTon_min*/ {1500, 0}
973 	 }, {
974 /*tSR   */ {15000, 3},
975 /*tXP   */ {7500, 3},
976 /*tRTP  */ {7500, 8},
977 /*tRCD  */ {19875, 4},
978 /*tRPpb */ {19875, 3},
979 /*tRPab */ {22875, 3},
980 /*tRAS  */ {43875, 3},
981 /*tWR   */ {18000, 4},
982 /*tWTR  */ {10000, 8},
983 /*tRRD  */ {11875, 4},
984 /*tPPD  */ {0, 0},
985 /*tFAW  */ {40000, 0},
986 /*tDQSCK*/ {3600, 0},
987 /*tCKEHCMD*/ {7500, 3},
988 /*tCKELCMD*/ {7500, 3},
989 /*tCKELPD*/ {7500, 3},
990 /*tMRR*/ {0, 8},
991 /*tMRW*/ {10000, 10},
992 /*tMRD*/ {14000, 10},
993 /*tZQCALns*/ {1000 * 10, 0},
994 /*tZQLAT*/ {30000, 10},
995 /*tIEdly*/ {12500, 0},
996 /*tODTon_min*/ {1500, 0}
997 	}
998 };
999 
1000 const uint16_t jedec_spec2_trfc_ab[7] = {
1001 /*	4Gb, 6Gb, 8Gb,12Gb, 16Gb, 24Gb(non), 32Gb(non)	*/
1002 	 130, 180, 180, 280, 280, 560, 560
1003 };
1004 
1005 static uint32_t js1_ind;
1006 static uint16_t js2[JS2_CNT];
1007 static uint8_t RL;
1008 static uint8_t WL;
1009 
_f_scale(uint32_t _ddr_mbps,uint32_t _ddr_mbpsdiv,uint32_t ps,uint16_t cyc)1010 static uint16_t _f_scale(uint32_t _ddr_mbps, uint32_t _ddr_mbpsdiv, uint32_t ps,
1011 			 uint16_t cyc)
1012 {
1013 	uint32_t tmp;
1014 	uint32_t div;
1015 
1016 	tmp = (((uint32_t)(ps) + 9) / 10) * _ddr_mbps;
1017 	div = tmp / (200000 * _ddr_mbpsdiv);
1018 	if (tmp != (div * 200000 * _ddr_mbpsdiv))
1019 		div = div + 1;
1020 
1021 	if (div > cyc)
1022 		return (uint16_t)div;
1023 	return cyc;
1024 }
1025 
_f_scale_js2(uint32_t _ddr_mbps,uint32_t _ddr_mbpsdiv,uint16_t * _js2)1026 static void _f_scale_js2(uint32_t _ddr_mbps, uint32_t _ddr_mbpsdiv,
1027 			 uint16_t *_js2)
1028 {
1029 	int i;
1030 
1031 	for (i = 0; i < JS2_TBLCNT; i++) {
1032 		_js2[i] = _f_scale(_ddr_mbps, _ddr_mbpsdiv,
1033 				  1UL * jedec_spec2[JS2_DERATE][i].ps,
1034 				  jedec_spec2[JS2_DERATE][i].cyc);
1035 	}
1036 
1037 	_js2[js2_trcpb] = _js2[js2_tras] + _js2[js2_trppb];
1038 	_js2[js2_trcab] = _js2[js2_tras] + _js2[js2_trpab];
1039 }
1040 
1041 /* scaler for DELAY value */
_f_scale_adj(int16_t ps)1042 static int16_t _f_scale_adj(int16_t ps)
1043 {
1044 	int32_t tmp;
1045 	/*
1046 	 * tmp = (int32_t)512 * ps * ddr_mbps /2 / ddr_mbpsdiv / 1000 / 1000;
1047 	 *     = ps * ddr_mbps /2 / ddr_mbpsdiv *512 / 8 / 8 / 125 / 125
1048 	 *     = ps * ddr_mbps / ddr_mbpsdiv *4 / 125 / 125
1049 	 */
1050 	tmp =
1051 	    (int32_t)4 * (int32_t)ps * (int32_t)ddr_mbps /
1052 	    (int32_t)ddr_mbpsdiv;
1053 	tmp = (int32_t)tmp / (int32_t)15625;
1054 
1055 	return (int16_t)tmp;
1056 }
1057 
1058 static const uint32_t reg_pi_mr1_data_fx_csx[2][CSAB_CNT] = {
1059 	{
1060 	 _reg_PI_MR1_DATA_F0_0,
1061 	 _reg_PI_MR1_DATA_F0_1,
1062 	 _reg_PI_MR1_DATA_F0_2,
1063 	 _reg_PI_MR1_DATA_F0_3},
1064 	{
1065 	 _reg_PI_MR1_DATA_F1_0,
1066 	 _reg_PI_MR1_DATA_F1_1,
1067 	 _reg_PI_MR1_DATA_F1_2,
1068 	 _reg_PI_MR1_DATA_F1_3}
1069 };
1070 
1071 static const uint32_t reg_pi_mr2_data_fx_csx[2][CSAB_CNT] = {
1072 	{
1073 	 _reg_PI_MR2_DATA_F0_0,
1074 	 _reg_PI_MR2_DATA_F0_1,
1075 	 _reg_PI_MR2_DATA_F0_2,
1076 	 _reg_PI_MR2_DATA_F0_3},
1077 	{
1078 	 _reg_PI_MR2_DATA_F1_0,
1079 	 _reg_PI_MR2_DATA_F1_1,
1080 	 _reg_PI_MR2_DATA_F1_2,
1081 	 _reg_PI_MR2_DATA_F1_3}
1082 };
1083 
1084 static const uint32_t reg_pi_mr3_data_fx_csx[2][CSAB_CNT] = {
1085 	{
1086 	 _reg_PI_MR3_DATA_F0_0,
1087 	 _reg_PI_MR3_DATA_F0_1,
1088 	 _reg_PI_MR3_DATA_F0_2,
1089 	 _reg_PI_MR3_DATA_F0_3},
1090 	{
1091 	 _reg_PI_MR3_DATA_F1_0,
1092 	 _reg_PI_MR3_DATA_F1_1,
1093 	 _reg_PI_MR3_DATA_F1_2,
1094 	 _reg_PI_MR3_DATA_F1_3}
1095 };
1096 
1097 const uint32_t reg_pi_mr11_data_fx_csx[2][CSAB_CNT] = {
1098 	{
1099 	 _reg_PI_MR11_DATA_F0_0,
1100 	 _reg_PI_MR11_DATA_F0_1,
1101 	 _reg_PI_MR11_DATA_F0_2,
1102 	 _reg_PI_MR11_DATA_F0_3},
1103 	{
1104 	 _reg_PI_MR11_DATA_F1_0,
1105 	 _reg_PI_MR11_DATA_F1_1,
1106 	 _reg_PI_MR11_DATA_F1_2,
1107 	 _reg_PI_MR11_DATA_F1_3}
1108 };
1109 
1110 const uint32_t reg_pi_mr12_data_fx_csx[2][CSAB_CNT] = {
1111 	{
1112 	 _reg_PI_MR12_DATA_F0_0,
1113 	 _reg_PI_MR12_DATA_F0_1,
1114 	 _reg_PI_MR12_DATA_F0_2,
1115 	 _reg_PI_MR12_DATA_F0_3},
1116 	{
1117 	 _reg_PI_MR12_DATA_F1_0,
1118 	 _reg_PI_MR12_DATA_F1_1,
1119 	 _reg_PI_MR12_DATA_F1_2,
1120 	 _reg_PI_MR12_DATA_F1_3}
1121 };
1122 
1123 const uint32_t reg_pi_mr14_data_fx_csx[2][CSAB_CNT] = {
1124 	{
1125 	 _reg_PI_MR14_DATA_F0_0,
1126 	 _reg_PI_MR14_DATA_F0_1,
1127 	 _reg_PI_MR14_DATA_F0_2,
1128 	 _reg_PI_MR14_DATA_F0_3},
1129 	{
1130 	 _reg_PI_MR14_DATA_F1_0,
1131 	 _reg_PI_MR14_DATA_F1_1,
1132 	 _reg_PI_MR14_DATA_F1_2,
1133 	 _reg_PI_MR14_DATA_F1_3}
1134 };
1135 
1136 /*
1137  * regif pll w/a   ( REGIF H3 Ver.2.0 or later/M3-N/V3H WA )
1138  */
regif_pll_wa(void)1139 static void regif_pll_wa(void)
1140 {
1141 	uint32_t ch;
1142 
1143 	if ((prr_product == PRR_PRODUCT_H3) && (prr_cut <= PRR_PRODUCT_11)) {
1144 		// PLL setting for PHY : H3 Ver.1.x
1145 		reg_ddrphy_write_a(ddr_regdef_adr(_reg_PHY_PLL_WAIT),
1146 				   (0x0064U <<
1147 				    ddr_regdef_lsb(_reg_PHY_PLL_WAIT)));
1148 		reg_ddrphy_write_a(ddr_regdef_adr(_reg_PHY_PLL_CTRL),
1149 				   ddrtbl_getval(_cnf_DDR_PHY_ADR_G_REGSET,
1150 						 _reg_PHY_PLL_CTRL));
1151 
1152 		reg_ddrphy_write_a(ddr_regdef_adr(_reg_PHY_LP4_BOOT_PLL_CTRL),
1153 				   ddrtbl_getval(_cnf_DDR_PHY_ADR_G_REGSET,
1154 						 _reg_PHY_LP4_BOOT_PLL_CTRL));
1155 
1156 	} else {
1157 		/*  PLL setting for PHY : M3-W/M3-N/V3H/H3 Ver.2.0 or later */
1158 		reg_ddrphy_write_a(ddr_regdef_adr(_reg_PHY_PLL_WAIT),
1159 				   (0x5064U <<
1160 				    ddr_regdef_lsb(_reg_PHY_PLL_WAIT)));
1161 
1162 		reg_ddrphy_write_a(ddr_regdef_adr(_reg_PHY_PLL_CTRL),
1163 				   (ddrtbl_getval
1164 				    (_cnf_DDR_PHY_ADR_G_REGSET,
1165 				     _reg_PHY_PLL_CTRL_TOP) << 16) |
1166 				   ddrtbl_getval(_cnf_DDR_PHY_ADR_G_REGSET,
1167 						 _reg_PHY_PLL_CTRL));
1168 		reg_ddrphy_write_a(ddr_regdef_adr(_reg_PHY_PLL_CTRL_CA),
1169 				   ddrtbl_getval(_cnf_DDR_PHY_ADR_G_REGSET,
1170 						 _reg_PHY_PLL_CTRL_CA));
1171 
1172 		reg_ddrphy_write_a(ddr_regdef_adr(_reg_PHY_LP4_BOOT_PLL_CTRL),
1173 				   (ddrtbl_getval
1174 				    (_cnf_DDR_PHY_ADR_G_REGSET,
1175 				     _reg_PHY_LP4_BOOT_PLL_CTRL_CA) << 16) |
1176 				   ddrtbl_getval(_cnf_DDR_PHY_ADR_G_REGSET,
1177 						 _reg_PHY_LP4_BOOT_PLL_CTRL));
1178 		reg_ddrphy_write_a(ddr_regdef_adr
1179 				   (_reg_PHY_LP4_BOOT_TOP_PLL_CTRL),
1180 				   ddrtbl_getval(_cnf_DDR_PHY_ADR_G_REGSET,
1181 						 _reg_PHY_LP4_BOOT_TOP_PLL_CTRL
1182 						 ));
1183 	}
1184 
1185 	reg_ddrphy_write_a(ddr_regdef_adr(_reg_PHY_LPDDR3_CS),
1186 			   _cnf_DDR_PHY_ADR_G_REGSET
1187 			   [ddr_regdef_adr(_reg_PHY_LPDDR3_CS) -
1188 			   DDR_PHY_ADR_G_REGSET_OFS]);
1189 
1190 	/* protect register interface */
1191 	ddrphy_regif_idle();
1192 	pll3_control(0);
1193 
1194 	if ((prr_product == PRR_PRODUCT_H3) && (prr_cut <= PRR_PRODUCT_11)) {
1195 		/*  non */
1196 	} else {
1197 		reg_ddrphy_write_a(ddr_regdef_adr(_reg_PHY_DLL_RST_EN),
1198 				   (0x01U <<
1199 				    ddr_regdef_lsb(_reg_PHY_DLL_RST_EN)));
1200 		ddrphy_regif_idle();
1201 	}
1202 
1203 	/* init start */
1204 	/* dbdficnt0:
1205 	 * dfi_dram_clk_disable=1
1206 	 * dfi_frequency = 0
1207 	 * freq_ratio = 01 (2:1)
1208 	 * init_start =0
1209 	 */
1210 	foreach_vch(ch)
1211 	    mmio_write_32(DBSC_DBDFICNT(ch), 0x00000F10);
1212 	dsb_sev();
1213 
1214 	/* dbdficnt0:
1215 	 * dfi_dram_clk_disable=1
1216 	 * dfi_frequency = 0
1217 	 * freq_ratio = 01 (2:1)
1218 	 * init_start =1
1219 	 */
1220 	foreach_vch(ch)
1221 	    mmio_write_32(DBSC_DBDFICNT(ch), 0x00000F11);
1222 	dsb_sev();
1223 
1224 	foreach_ech(ch)
1225 	if ((board_cnf->phyvalid) & BIT(ch))
1226 		while ((mmio_read_32(DBSC_PLL_LOCK(ch)) & 0x1f) != 0x1f)
1227 			;
1228 	dsb_sev();
1229 }
1230 
1231 /* load table data into DDR registers */
ddrtbl_load(void)1232 static void ddrtbl_load(void)
1233 {
1234 	uint32_t i;
1235 	uint32_t slice;
1236 	uint32_t csab;
1237 	uint32_t adr;
1238 	uint32_t data_l;
1239 	uint32_t tmp[3];
1240 	uint16_t dataS;
1241 
1242 	/* TIMING REGISTERS */
1243 	/* search jedec_spec1 index */
1244 	for (i = JS1_USABLEC_SPEC_LO; i < JS1_FREQ_TBL_NUM - 1; i++) {
1245 		if (js1[i].fx3 * 2U * ddr_mbpsdiv >= ddr_mbps * 3U)
1246 			break;
1247 	}
1248 	if (i > JS1_USABLEC_SPEC_HI)
1249 		js1_ind = JS1_USABLEC_SPEC_HI;
1250 	else
1251 		js1_ind = i;
1252 
1253 	if (board_cnf->dbi_en)
1254 		RL = js1[js1_ind].rlwdbi;
1255 	else
1256 		RL = js1[js1_ind].rlwodbi;
1257 
1258 	WL = js1[js1_ind].WL;
1259 
1260 	/* calculate jedec_spec2 */
1261 	_f_scale_js2(ddr_mbps, ddr_mbpsdiv, js2);
1262 
1263 	/* PREPARE TBL */
1264 	if (prr_product == PRR_PRODUCT_H3) {
1265 		if (prr_cut <= PRR_PRODUCT_11) {
1266 			/*  H3 Ver.1.x */
1267 			_tblcopy(_cnf_DDR_PHY_SLICE_REGSET,
1268 				 DDR_PHY_SLICE_REGSET_H3,
1269 				 DDR_PHY_SLICE_REGSET_NUM_H3);
1270 			_tblcopy(_cnf_DDR_PHY_ADR_V_REGSET,
1271 				 DDR_PHY_ADR_V_REGSET_H3,
1272 				 DDR_PHY_ADR_V_REGSET_NUM_H3);
1273 			_tblcopy(_cnf_DDR_PHY_ADR_I_REGSET,
1274 				 DDR_PHY_ADR_I_REGSET_H3,
1275 				 DDR_PHY_ADR_I_REGSET_NUM_H3);
1276 			_tblcopy(_cnf_DDR_PHY_ADR_G_REGSET,
1277 				 DDR_PHY_ADR_G_REGSET_H3,
1278 				 DDR_PHY_ADR_G_REGSET_NUM_H3);
1279 			_tblcopy(_cnf_DDR_PI_REGSET, DDR_PI_REGSET_H3,
1280 				 DDR_PI_REGSET_NUM_H3);
1281 
1282 			DDR_PHY_SLICE_REGSET_OFS = DDR_PHY_SLICE_REGSET_OFS_H3;
1283 			DDR_PHY_ADR_V_REGSET_OFS = DDR_PHY_ADR_V_REGSET_OFS_H3;
1284 			DDR_PHY_ADR_I_REGSET_OFS = DDR_PHY_ADR_I_REGSET_OFS_H3;
1285 			DDR_PHY_ADR_G_REGSET_OFS = DDR_PHY_ADR_G_REGSET_OFS_H3;
1286 			DDR_PI_REGSET_OFS = DDR_PI_REGSET_OFS_H3;
1287 			DDR_PHY_SLICE_REGSET_SIZE =
1288 			    DDR_PHY_SLICE_REGSET_SIZE_H3;
1289 			DDR_PHY_ADR_V_REGSET_SIZE =
1290 			    DDR_PHY_ADR_V_REGSET_SIZE_H3;
1291 			DDR_PHY_ADR_I_REGSET_SIZE =
1292 			    DDR_PHY_ADR_I_REGSET_SIZE_H3;
1293 			DDR_PHY_ADR_G_REGSET_SIZE =
1294 			    DDR_PHY_ADR_G_REGSET_SIZE_H3;
1295 			DDR_PI_REGSET_SIZE = DDR_PI_REGSET_SIZE_H3;
1296 			DDR_PHY_SLICE_REGSET_NUM = DDR_PHY_SLICE_REGSET_NUM_H3;
1297 			DDR_PHY_ADR_V_REGSET_NUM = DDR_PHY_ADR_V_REGSET_NUM_H3;
1298 			DDR_PHY_ADR_I_REGSET_NUM = DDR_PHY_ADR_I_REGSET_NUM_H3;
1299 			DDR_PHY_ADR_G_REGSET_NUM = DDR_PHY_ADR_G_REGSET_NUM_H3;
1300 			DDR_PI_REGSET_NUM = DDR_PI_REGSET_NUM_H3;
1301 
1302 			DDR_PHY_ADR_I_NUM = 1;
1303 		} else {
1304 			/*  H3 Ver.2.0 or later */
1305 			_tblcopy(_cnf_DDR_PHY_SLICE_REGSET,
1306 				 DDR_PHY_SLICE_REGSET_H3VER2,
1307 				 DDR_PHY_SLICE_REGSET_NUM_H3VER2);
1308 			_tblcopy(_cnf_DDR_PHY_ADR_V_REGSET,
1309 				 DDR_PHY_ADR_V_REGSET_H3VER2,
1310 				 DDR_PHY_ADR_V_REGSET_NUM_H3VER2);
1311 			_tblcopy(_cnf_DDR_PHY_ADR_G_REGSET,
1312 				 DDR_PHY_ADR_G_REGSET_H3VER2,
1313 				 DDR_PHY_ADR_G_REGSET_NUM_H3VER2);
1314 			_tblcopy(_cnf_DDR_PI_REGSET, DDR_PI_REGSET_H3VER2,
1315 				 DDR_PI_REGSET_NUM_H3VER2);
1316 
1317 			DDR_PHY_SLICE_REGSET_OFS =
1318 			    DDR_PHY_SLICE_REGSET_OFS_H3VER2;
1319 			DDR_PHY_ADR_V_REGSET_OFS =
1320 			    DDR_PHY_ADR_V_REGSET_OFS_H3VER2;
1321 			DDR_PHY_ADR_G_REGSET_OFS =
1322 			    DDR_PHY_ADR_G_REGSET_OFS_H3VER2;
1323 			DDR_PI_REGSET_OFS = DDR_PI_REGSET_OFS_H3VER2;
1324 			DDR_PHY_SLICE_REGSET_SIZE =
1325 			    DDR_PHY_SLICE_REGSET_SIZE_H3VER2;
1326 			DDR_PHY_ADR_V_REGSET_SIZE =
1327 			    DDR_PHY_ADR_V_REGSET_SIZE_H3VER2;
1328 			DDR_PHY_ADR_G_REGSET_SIZE =
1329 			    DDR_PHY_ADR_G_REGSET_SIZE_H3VER2;
1330 			DDR_PI_REGSET_SIZE = DDR_PI_REGSET_SIZE_H3VER2;
1331 			DDR_PHY_SLICE_REGSET_NUM =
1332 			    DDR_PHY_SLICE_REGSET_NUM_H3VER2;
1333 			DDR_PHY_ADR_V_REGSET_NUM =
1334 			    DDR_PHY_ADR_V_REGSET_NUM_H3VER2;
1335 			DDR_PHY_ADR_G_REGSET_NUM =
1336 			    DDR_PHY_ADR_G_REGSET_NUM_H3VER2;
1337 			DDR_PI_REGSET_NUM = DDR_PI_REGSET_NUM_H3VER2;
1338 
1339 			DDR_PHY_ADR_I_NUM = 0;
1340 		}
1341 	} else if (prr_product == PRR_PRODUCT_M3) {
1342 		/*  M3-W */
1343 		_tblcopy(_cnf_DDR_PHY_SLICE_REGSET,
1344 			 DDR_PHY_SLICE_REGSET_M3, DDR_PHY_SLICE_REGSET_NUM_M3);
1345 		_tblcopy(_cnf_DDR_PHY_ADR_V_REGSET,
1346 			 DDR_PHY_ADR_V_REGSET_M3, DDR_PHY_ADR_V_REGSET_NUM_M3);
1347 		_tblcopy(_cnf_DDR_PHY_ADR_I_REGSET,
1348 			 DDR_PHY_ADR_I_REGSET_M3, DDR_PHY_ADR_I_REGSET_NUM_M3);
1349 		_tblcopy(_cnf_DDR_PHY_ADR_G_REGSET,
1350 			 DDR_PHY_ADR_G_REGSET_M3, DDR_PHY_ADR_G_REGSET_NUM_M3);
1351 		_tblcopy(_cnf_DDR_PI_REGSET,
1352 			 DDR_PI_REGSET_M3, DDR_PI_REGSET_NUM_M3);
1353 
1354 		DDR_PHY_SLICE_REGSET_OFS = DDR_PHY_SLICE_REGSET_OFS_M3;
1355 		DDR_PHY_ADR_V_REGSET_OFS = DDR_PHY_ADR_V_REGSET_OFS_M3;
1356 		DDR_PHY_ADR_I_REGSET_OFS = DDR_PHY_ADR_I_REGSET_OFS_M3;
1357 		DDR_PHY_ADR_G_REGSET_OFS = DDR_PHY_ADR_G_REGSET_OFS_M3;
1358 		DDR_PI_REGSET_OFS = DDR_PI_REGSET_OFS_M3;
1359 		DDR_PHY_SLICE_REGSET_SIZE = DDR_PHY_SLICE_REGSET_SIZE_M3;
1360 		DDR_PHY_ADR_V_REGSET_SIZE = DDR_PHY_ADR_V_REGSET_SIZE_M3;
1361 		DDR_PHY_ADR_I_REGSET_SIZE = DDR_PHY_ADR_I_REGSET_SIZE_M3;
1362 		DDR_PHY_ADR_G_REGSET_SIZE = DDR_PHY_ADR_G_REGSET_SIZE_M3;
1363 		DDR_PI_REGSET_SIZE = DDR_PI_REGSET_SIZE_M3;
1364 		DDR_PHY_SLICE_REGSET_NUM = DDR_PHY_SLICE_REGSET_NUM_M3;
1365 		DDR_PHY_ADR_V_REGSET_NUM = DDR_PHY_ADR_V_REGSET_NUM_M3;
1366 		DDR_PHY_ADR_I_REGSET_NUM = DDR_PHY_ADR_I_REGSET_NUM_M3;
1367 		DDR_PHY_ADR_G_REGSET_NUM = DDR_PHY_ADR_G_REGSET_NUM_M3;
1368 		DDR_PI_REGSET_NUM = DDR_PI_REGSET_NUM_M3;
1369 
1370 		DDR_PHY_ADR_I_NUM = 2;
1371 	} else {
1372 		/*  M3-N/V3H */
1373 		_tblcopy(_cnf_DDR_PHY_SLICE_REGSET,
1374 			 DDR_PHY_SLICE_REGSET_M3N,
1375 			 DDR_PHY_SLICE_REGSET_NUM_M3N);
1376 		_tblcopy(_cnf_DDR_PHY_ADR_V_REGSET, DDR_PHY_ADR_V_REGSET_M3N,
1377 			 DDR_PHY_ADR_V_REGSET_NUM_M3N);
1378 		_tblcopy(_cnf_DDR_PHY_ADR_I_REGSET, DDR_PHY_ADR_I_REGSET_M3N,
1379 			 DDR_PHY_ADR_I_REGSET_NUM_M3N);
1380 		_tblcopy(_cnf_DDR_PHY_ADR_G_REGSET, DDR_PHY_ADR_G_REGSET_M3N,
1381 			 DDR_PHY_ADR_G_REGSET_NUM_M3N);
1382 		_tblcopy(_cnf_DDR_PI_REGSET, DDR_PI_REGSET_M3N,
1383 			 DDR_PI_REGSET_NUM_M3N);
1384 
1385 		DDR_PHY_SLICE_REGSET_OFS = DDR_PHY_SLICE_REGSET_OFS_M3N;
1386 		DDR_PHY_ADR_V_REGSET_OFS = DDR_PHY_ADR_V_REGSET_OFS_M3N;
1387 		DDR_PHY_ADR_I_REGSET_OFS = DDR_PHY_ADR_I_REGSET_OFS_M3N;
1388 		DDR_PHY_ADR_G_REGSET_OFS = DDR_PHY_ADR_G_REGSET_OFS_M3N;
1389 		DDR_PI_REGSET_OFS = DDR_PI_REGSET_OFS_M3N;
1390 		DDR_PHY_SLICE_REGSET_SIZE = DDR_PHY_SLICE_REGSET_SIZE_M3N;
1391 		DDR_PHY_ADR_V_REGSET_SIZE = DDR_PHY_ADR_V_REGSET_SIZE_M3N;
1392 		DDR_PHY_ADR_I_REGSET_SIZE = DDR_PHY_ADR_I_REGSET_SIZE_M3N;
1393 		DDR_PHY_ADR_G_REGSET_SIZE = DDR_PHY_ADR_G_REGSET_SIZE_M3N;
1394 		DDR_PI_REGSET_SIZE = DDR_PI_REGSET_SIZE_M3N;
1395 		DDR_PHY_SLICE_REGSET_NUM = DDR_PHY_SLICE_REGSET_NUM_M3N;
1396 		DDR_PHY_ADR_V_REGSET_NUM = DDR_PHY_ADR_V_REGSET_NUM_M3N;
1397 		DDR_PHY_ADR_I_REGSET_NUM = DDR_PHY_ADR_I_REGSET_NUM_M3N;
1398 		DDR_PHY_ADR_G_REGSET_NUM = DDR_PHY_ADR_G_REGSET_NUM_M3N;
1399 		DDR_PI_REGSET_NUM = DDR_PI_REGSET_NUM_M3N;
1400 
1401 		DDR_PHY_ADR_I_NUM = 2;
1402 	}
1403 
1404 	/* PLL CODE CHANGE */
1405 	if ((prr_product == PRR_PRODUCT_H3) && (prr_cut == PRR_PRODUCT_11)) {
1406 		ddrtbl_setval(_cnf_DDR_PHY_ADR_G_REGSET, _reg_PHY_PLL_CTRL,
1407 			      0x1142);
1408 		ddrtbl_setval(_cnf_DDR_PHY_ADR_G_REGSET,
1409 			      _reg_PHY_LP4_BOOT_PLL_CTRL, 0x1142);
1410 	}
1411 
1412 	/* on fly gate adjust */
1413 	if ((prr_product == PRR_PRODUCT_M3) && (prr_cut == PRR_PRODUCT_10)) {
1414 		ddrtbl_setval(_cnf_DDR_PHY_SLICE_REGSET,
1415 			      _reg_ON_FLY_GATE_ADJUST_EN, 0x00);
1416 	}
1417 
1418 	/* Adjust PI parameters */
1419 #ifdef _def_LPDDR4_ODT
1420 	for (i = 0; i < 2; i++) {
1421 		for (csab = 0; csab < CSAB_CNT; csab++) {
1422 			ddrtbl_setval(_cnf_DDR_PI_REGSET,
1423 				      reg_pi_mr11_data_fx_csx[i][csab],
1424 				      _def_LPDDR4_ODT);
1425 		}
1426 	}
1427 #endif /* _def_LPDDR4_ODT */
1428 
1429 #ifdef _def_LPDDR4_VREFCA
1430 	for (i = 0; i < 2; i++) {
1431 		for (csab = 0; csab < CSAB_CNT; csab++) {
1432 			ddrtbl_setval(_cnf_DDR_PI_REGSET,
1433 				      reg_pi_mr12_data_fx_csx[i][csab],
1434 				      _def_LPDDR4_VREFCA);
1435 		}
1436 	}
1437 #endif /* _def_LPDDR4_VREFCA */
1438 	if ((prr_product == PRR_PRODUCT_M3N) ||
1439 	    (prr_product == PRR_PRODUCT_V3H)) {
1440 		js2[js2_tiedly] = _f_scale(ddr_mbps, ddr_mbpsdiv, 7000, 0) + 7U;
1441 		if (js2[js2_tiedly] > (RL))
1442 			js2[js2_tiedly] = RL;
1443 	} else if ((prr_product == PRR_PRODUCT_H3) &&
1444 		   (prr_cut > PRR_PRODUCT_11)) {
1445 		js2[js2_tiedly] = _f_scale(ddr_mbps, ddr_mbpsdiv, 9000, 0) + 4U;
1446 	} else if ((prr_product == PRR_PRODUCT_H3) &&
1447 		   (prr_cut <= PRR_PRODUCT_11)) {
1448 		js2[js2_tiedly] = _f_scale(ddr_mbps, ddr_mbpsdiv, 10000, 0);
1449 	}
1450 
1451 	if (((prr_product == PRR_PRODUCT_H3) && (prr_cut > PRR_PRODUCT_11)) ||
1452 	    (prr_product == PRR_PRODUCT_M3N) ||
1453 	    (prr_product == PRR_PRODUCT_V3H)) {
1454 		if ((js2[js2_tiedly]) >= 0x1e)
1455 			dataS = 0x1e;
1456 		else
1457 			dataS = js2[js2_tiedly];
1458 	} else {
1459 		if ((js2[js2_tiedly]) >= 0x0e)
1460 			dataS = 0x0e;
1461 		else
1462 			dataS = js2[js2_tiedly];
1463 	}
1464 
1465 	ddrtbl_setval(_cnf_DDR_PHY_SLICE_REGSET, _reg_PHY_RDDATA_EN_DLY, dataS);
1466 	ddrtbl_setval(_cnf_DDR_PHY_SLICE_REGSET, _reg_PHY_RDDATA_EN_TSEL_DLY,
1467 		      (dataS - 2));
1468 	if ((prr_product == PRR_PRODUCT_M3N) ||
1469 	    (prr_product == PRR_PRODUCT_V3H)) {
1470 		ddrtbl_setval(_cnf_DDR_PHY_SLICE_REGSET,
1471 			      _reg_PHY_RDDATA_EN_OE_DLY, dataS - 2);
1472 	}
1473 	ddrtbl_setval(_cnf_DDR_PI_REGSET, _reg_PI_RDLAT_ADJ_F1, RL - dataS);
1474 
1475 	if (ddrtbl_getval
1476 	    (_cnf_DDR_PHY_SLICE_REGSET, _reg_PHY_WRITE_PATH_LAT_ADD)) {
1477 		data_l = WL - 1;
1478 	} else {
1479 		data_l = WL;
1480 	}
1481 	ddrtbl_setval(_cnf_DDR_PI_REGSET, _reg_PI_WRLAT_ADJ_F1, data_l - 2);
1482 	ddrtbl_setval(_cnf_DDR_PI_REGSET, _reg_PI_WRLAT_F1, data_l);
1483 
1484 	if (board_cnf->dbi_en) {
1485 		ddrtbl_setval(_cnf_DDR_PHY_SLICE_REGSET, _reg_PHY_DBI_MODE,
1486 			      0x01);
1487 		ddrtbl_setval(_cnf_DDR_PHY_SLICE_REGSET,
1488 			      _reg_PHY_WDQLVL_DATADM_MASK, 0x000);
1489 	} else {
1490 		ddrtbl_setval(_cnf_DDR_PHY_SLICE_REGSET, _reg_PHY_DBI_MODE,
1491 			      0x00);
1492 		ddrtbl_setval(_cnf_DDR_PHY_SLICE_REGSET,
1493 			      _reg_PHY_WDQLVL_DATADM_MASK, 0x100);
1494 	}
1495 
1496 	tmp[0] = js1[js1_ind].MR1;
1497 	tmp[1] = js1[js1_ind].MR2;
1498 	data_l = ddrtbl_getval(_cnf_DDR_PI_REGSET, _reg_PI_MR3_DATA_F1_0);
1499 	if (board_cnf->dbi_en)
1500 		tmp[2] = data_l | 0xc0;
1501 	else
1502 		tmp[2] = data_l & (~0xc0);
1503 
1504 	for (i = 0; i < 2; i++) {
1505 		for (csab = 0; csab < CSAB_CNT; csab++) {
1506 			ddrtbl_setval(_cnf_DDR_PI_REGSET,
1507 				      reg_pi_mr1_data_fx_csx[i][csab], tmp[0]);
1508 			ddrtbl_setval(_cnf_DDR_PI_REGSET,
1509 				      reg_pi_mr2_data_fx_csx[i][csab], tmp[1]);
1510 			ddrtbl_setval(_cnf_DDR_PI_REGSET,
1511 				      reg_pi_mr3_data_fx_csx[i][csab], tmp[2]);
1512 		}
1513 	}
1514 
1515 	/* DDRPHY INT START */
1516 	if ((prr_product == PRR_PRODUCT_H3) && (prr_cut <= PRR_PRODUCT_11)) {
1517 		/* non */
1518 	} else {
1519 		regif_pll_wa();
1520 		dbwait_loop(5);
1521 	}
1522 
1523 	/* FREQ_SEL_MULTICAST & PER_CS_TRAINING_MULTICAST SET (for safety) */
1524 	reg_ddrphy_write_a(ddr_regdef_adr(_reg_PHY_FREQ_SEL_MULTICAST_EN),
1525 			   BIT(ddr_regdef_lsb(_reg_PHY_FREQ_SEL_MULTICAST_EN)));
1526 	ddr_setval_ach_as(_reg_PHY_PER_CS_TRAINING_MULTICAST_EN, 0x01);
1527 
1528 	/* SET DATA SLICE TABLE */
1529 	for (slice = 0; slice < SLICE_CNT; slice++) {
1530 		adr =
1531 		    DDR_PHY_SLICE_REGSET_OFS +
1532 		    DDR_PHY_SLICE_REGSET_SIZE * slice;
1533 		for (i = 0; i < DDR_PHY_SLICE_REGSET_NUM; i++) {
1534 			reg_ddrphy_write_a(adr + i,
1535 					   _cnf_DDR_PHY_SLICE_REGSET[i]);
1536 		}
1537 	}
1538 
1539 	/* SET ADR SLICE TABLE */
1540 	adr = DDR_PHY_ADR_V_REGSET_OFS;
1541 	for (i = 0; i < DDR_PHY_ADR_V_REGSET_NUM; i++) {
1542 		reg_ddrphy_write_a(adr + i, _cnf_DDR_PHY_ADR_V_REGSET[i]);
1543 	}
1544 
1545 	if (((prr_product == PRR_PRODUCT_M3) ||
1546 	     (prr_product == PRR_PRODUCT_M3N)) &&
1547 	    ((0x00ffffff & (uint32_t)((board_cnf->ch[0].ca_swap) >> 40))
1548 	    != 0x00)) {
1549 		adr = DDR_PHY_ADR_I_REGSET_OFS + DDR_PHY_ADR_I_REGSET_SIZE;
1550 		for (i = 0; i < DDR_PHY_ADR_V_REGSET_NUM; i++) {
1551 			reg_ddrphy_write_a(adr + i,
1552 					   _cnf_DDR_PHY_ADR_V_REGSET[i]);
1553 		}
1554 		ddrtbl_setval(_cnf_DDR_PHY_ADR_G_REGSET,
1555 			      _reg_PHY_ADR_DISABLE, 0x02);
1556 		DDR_PHY_ADR_I_NUM -= 1;
1557 		ddr_phycaslice = 1;
1558 
1559 #ifndef _def_LPDDR4_ODT
1560 		for (i = 0; i < 2; i++) {
1561 			for (csab = 0; csab < CSAB_CNT; csab++) {
1562 				ddrtbl_setval(_cnf_DDR_PI_REGSET,
1563 					      reg_pi_mr11_data_fx_csx[i][csab],
1564 					      0x66);
1565 			}
1566 		}
1567 #endif/* _def_LPDDR4_ODT */
1568 	} else {
1569 		ddr_phycaslice = 0;
1570 	}
1571 
1572 	if (DDR_PHY_ADR_I_NUM > 0) {
1573 		for (slice = 0; slice < DDR_PHY_ADR_I_NUM; slice++) {
1574 			adr =
1575 			    DDR_PHY_ADR_I_REGSET_OFS +
1576 			    DDR_PHY_ADR_I_REGSET_SIZE * slice;
1577 			for (i = 0; i < DDR_PHY_ADR_I_REGSET_NUM; i++) {
1578 				reg_ddrphy_write_a(adr + i,
1579 						   _cnf_DDR_PHY_ADR_I_REGSET
1580 						   [i]);
1581 			}
1582 		}
1583 	}
1584 
1585 	/* SET ADRCTRL SLICE TABLE */
1586 	adr = DDR_PHY_ADR_G_REGSET_OFS;
1587 	for (i = 0; i < DDR_PHY_ADR_G_REGSET_NUM; i++) {
1588 		reg_ddrphy_write_a(adr + i, _cnf_DDR_PHY_ADR_G_REGSET[i]);
1589 	}
1590 
1591 	/* SET PI REGISTERS */
1592 	adr = DDR_PI_REGSET_OFS;
1593 	for (i = 0; i < DDR_PI_REGSET_NUM; i++) {
1594 		reg_ddrphy_write_a(adr + i, _cnf_DDR_PI_REGSET[i]);
1595 	}
1596 }
1597 
1598 /* CONFIGURE DDR REGISTERS */
ddr_config_sub(void)1599 static void ddr_config_sub(void)
1600 {
1601 	uint32_t i;
1602 	uint32_t ch, slice;
1603 	uint32_t data_l;
1604 	uint32_t tmp;
1605 	uint8_t high_byte[SLICE_CNT];
1606 	const uint32_t _par_CALVL_DEVICE_MAP = 1;
1607 
1608 	foreach_vch(ch) {
1609 	/* BOARD SETTINGS (DQ,DM,VREF_DRIVING) */
1610 		for (slice = 0; slice < SLICE_CNT; slice++) {
1611 			high_byte[slice] =
1612 			    (board_cnf->ch[ch].dqs_swap >> (4 * slice)) % 2;
1613 			ddr_setval_s(ch, slice, _reg_PHY_DQ_DM_SWIZZLE0,
1614 				     board_cnf->ch[ch].dq_swap[slice]);
1615 			ddr_setval_s(ch, slice, _reg_PHY_DQ_DM_SWIZZLE1,
1616 				     board_cnf->ch[ch].dm_swap[slice]);
1617 			if (high_byte[slice]) {
1618 				/* HIGHER 16 BYTE */
1619 				ddr_setval_s(ch, slice,
1620 					     _reg_PHY_CALVL_VREF_DRIVING_SLICE,
1621 					     0x00);
1622 			} else {
1623 				/* LOWER 16 BYTE */
1624 				ddr_setval_s(ch, slice,
1625 					     _reg_PHY_CALVL_VREF_DRIVING_SLICE,
1626 					     0x01);
1627 			}
1628 		}
1629 
1630 	/* BOARD SETTINGS (CA,ADDR_SEL) */
1631 		data_l = (0x00ffffff & (uint32_t)(board_cnf->ch[ch].ca_swap)) |
1632 			0x00888888;
1633 
1634 		/* --- ADR_CALVL_SWIZZLE --- */
1635 		if (prr_product == PRR_PRODUCT_M3) {
1636 			ddr_setval(ch, _reg_PHY_ADR_CALVL_SWIZZLE0_0, data_l);
1637 			ddr_setval(ch, _reg_PHY_ADR_CALVL_SWIZZLE1_0,
1638 				   0x00000000);
1639 			ddr_setval(ch, _reg_PHY_ADR_CALVL_SWIZZLE0_1, data_l);
1640 			ddr_setval(ch, _reg_PHY_ADR_CALVL_SWIZZLE1_1,
1641 				   0x00000000);
1642 			ddr_setval(ch, _reg_PHY_ADR_CALVL_DEVICE_MAP,
1643 				   _par_CALVL_DEVICE_MAP);
1644 		} else {
1645 			ddr_setval(ch, _reg_PHY_ADR_CALVL_SWIZZLE0, data_l);
1646 			ddr_setval(ch, _reg_PHY_ADR_CALVL_SWIZZLE1, 0x00000000);
1647 			ddr_setval(ch, _reg_PHY_CALVL_DEVICE_MAP,
1648 				   _par_CALVL_DEVICE_MAP);
1649 		}
1650 
1651 		/* --- ADR_ADDR_SEL --- */
1652 		if ((prr_product == PRR_PRODUCT_H3) &&
1653 		    (prr_cut > PRR_PRODUCT_11)) {
1654 			data_l = 0x00FFFFFF & board_cnf->ch[ch].ca_swap;
1655 		} else {
1656 			data_l = 0;
1657 			tmp = board_cnf->ch[ch].ca_swap;
1658 			for (i = 0; i < 6; i++) {
1659 				data_l |= ((tmp & 0x0f) << (i * 5));
1660 				tmp = tmp >> 4;
1661 			}
1662 		}
1663 		ddr_setval(ch, _reg_PHY_ADR_ADDR_SEL, data_l);
1664 		if (ddr_phycaslice == 1) {
1665 			/* ----------- adr slice2 swap ----------- */
1666 			tmp  = (uint32_t)((board_cnf->ch[ch].ca_swap) >> 40);
1667 			data_l = (tmp & 0x00ffffff) | 0x00888888;
1668 
1669 			/* --- ADR_CALVL_SWIZZLE --- */
1670 			if (prr_product == PRR_PRODUCT_M3) {
1671 				ddr_setval_s(ch, 2,
1672 					     _reg_PHY_ADR_CALVL_SWIZZLE0_0,
1673 					     data_l);
1674 				ddr_setval_s(ch, 2,
1675 					     _reg_PHY_ADR_CALVL_SWIZZLE1_0,
1676 					     0x00000000);
1677 				ddr_setval_s(ch, 2,
1678 					     _reg_PHY_ADR_CALVL_SWIZZLE0_1,
1679 					     data_l);
1680 				ddr_setval_s(ch, 2,
1681 					     _reg_PHY_ADR_CALVL_SWIZZLE1_1,
1682 					     0x00000000);
1683 				ddr_setval_s(ch, 2,
1684 					     _reg_PHY_ADR_CALVL_DEVICE_MAP,
1685 					     _par_CALVL_DEVICE_MAP);
1686 			} else {
1687 				ddr_setval_s(ch, 2,
1688 					     _reg_PHY_ADR_CALVL_SWIZZLE0,
1689 					     data_l);
1690 				ddr_setval_s(ch, 2,
1691 					     _reg_PHY_ADR_CALVL_SWIZZLE1,
1692 					     0x00000000);
1693 				ddr_setval_s(ch, 2,
1694 					     _reg_PHY_CALVL_DEVICE_MAP,
1695 					     _par_CALVL_DEVICE_MAP);
1696 			}
1697 
1698 			/* --- ADR_ADDR_SEL --- */
1699 			data_l = 0;
1700 			for (i = 0; i < 6; i++) {
1701 				data_l |= ((tmp & 0x0f) << (i * 5));
1702 				tmp = tmp >> 4;
1703 			}
1704 
1705 			ddr_setval_s(ch, 2, _reg_PHY_ADR_ADDR_SEL, data_l);
1706 		}
1707 
1708 	/* BOARD SETTINGS (BYTE_ORDER_SEL) */
1709 		if (prr_product == PRR_PRODUCT_M3) {
1710 			/* --- DATA_BYTE_SWAP --- */
1711 			data_l = 0;
1712 			tmp = board_cnf->ch[ch].dqs_swap;
1713 			for (i = 0; i < 4; i++) {
1714 				data_l |= ((tmp & 0x03) << (i * 2));
1715 				tmp = tmp >> 4;
1716 			}
1717 		} else {
1718 			/* --- DATA_BYTE_SWAP --- */
1719 			data_l = board_cnf->ch[ch].dqs_swap;
1720 			ddr_setval(ch, _reg_PI_DATA_BYTE_SWAP_EN, 0x01);
1721 			ddr_setval(ch, _reg_PI_DATA_BYTE_SWAP_SLICE0,
1722 				   (data_l) & 0x0f);
1723 			ddr_setval(ch, _reg_PI_DATA_BYTE_SWAP_SLICE1,
1724 				   (data_l >> 4 * 1) & 0x0f);
1725 			ddr_setval(ch, _reg_PI_DATA_BYTE_SWAP_SLICE2,
1726 				   (data_l >> 4 * 2) & 0x0f);
1727 			ddr_setval(ch, _reg_PI_DATA_BYTE_SWAP_SLICE3,
1728 				   (data_l >> 4 * 3) & 0x0f);
1729 
1730 			ddr_setval(ch, _reg_PHY_DATA_BYTE_ORDER_SEL_HIGH, 0x00);
1731 		}
1732 		ddr_setval(ch, _reg_PHY_DATA_BYTE_ORDER_SEL, data_l);
1733 	}
1734 }
1735 
get_ca_swizzle(uint32_t ch,uint32_t ddr_csn,uint32_t * p_swz)1736 static void get_ca_swizzle(uint32_t ch, uint32_t ddr_csn, uint32_t *p_swz)
1737 {
1738 	uint32_t slice;
1739 	uint32_t tmp;
1740 	uint32_t tgt;
1741 
1742 	if (ddr_csn / 2) {
1743 		tgt = 3;
1744 	} else {
1745 		tgt = 1;
1746 	}
1747 
1748 	for (slice = 0; slice < SLICE_CNT; slice++) {
1749 		tmp = (board_cnf->ch[ch].dqs_swap >> (4 * slice)) & 0x0f;
1750 		if (tgt == tmp)
1751 			break;
1752 	}
1753 	tmp = 0x00FFFFFF & board_cnf->ch[ch].ca_swap;
1754 	if (slice % 2)
1755 		tmp |= 0x00888888;
1756 	*p_swz = tmp;
1757 }
1758 
ddr_config_sub_h3v1x(void)1759 static void ddr_config_sub_h3v1x(void)
1760 {
1761 	uint32_t ch, slice;
1762 	uint32_t data_l;
1763 	uint32_t tmp;
1764 	uint8_t high_byte[SLICE_CNT];
1765 	uint32_t ca_swizzle;
1766 	uint32_t ca;
1767 	uint32_t csmap;
1768 	uint32_t o_inv;
1769 	uint32_t inv;
1770 	uint32_t bit_soc;
1771 	uint32_t bit_mem;
1772 	uint32_t j;
1773 
1774 	const uint8_t o_mr15 = 0x55;
1775 	const uint8_t o_mr20 = 0x55;
1776 	const uint16_t o_mr32_mr40 = 0x5a3c;
1777 
1778 	foreach_vch(ch) {
1779 	/* BOARD SETTINGS (DQ,DM,VREF_DRIVING) */
1780 		csmap = 0;
1781 		for (slice = 0; slice < SLICE_CNT; slice++) {
1782 			tmp = (board_cnf->ch[ch].dqs_swap >> (4 * slice)) &
1783 			      0x0f;
1784 			high_byte[slice] = tmp % 2;
1785 			if (tmp == 1 && (slice >= 2))
1786 				csmap |= 0x05;
1787 			if (tmp == 3 && (slice >= 2))
1788 				csmap |= 0x50;
1789 			ddr_setval_s(ch, slice, _reg_PHY_DQ_SWIZZLING,
1790 				     board_cnf->ch[ch].dq_swap[slice]);
1791 			if (high_byte[slice]) {
1792 				/* HIGHER 16 BYTE */
1793 				ddr_setval_s(ch, slice,
1794 					     _reg_PHY_CALVL_VREF_DRIVING_SLICE,
1795 					     0x00);
1796 			} else {
1797 				/* LOWER 16 BYTE */
1798 				ddr_setval_s(ch, slice,
1799 					     _reg_PHY_CALVL_VREF_DRIVING_SLICE,
1800 					     0x01);
1801 			}
1802 		}
1803 	/* BOARD SETTINGS (CA,ADDR_SEL) */
1804 		ca = 0x00FFFFFF & board_cnf->ch[ch].ca_swap;
1805 		ddr_setval(ch, _reg_PHY_ADR_ADDR_SEL, ca);
1806 		ddr_setval(ch, _reg_PHY_CALVL_CS_MAP, csmap);
1807 
1808 		get_ca_swizzle(ch, 0, &ca_swizzle);
1809 
1810 		ddr_setval(ch, _reg_PHY_ADR_CALVL_SWIZZLE0_0, ca_swizzle);
1811 		ddr_setval(ch, _reg_PHY_ADR_CALVL_SWIZZLE1_0, 0x00000000);
1812 		ddr_setval(ch, _reg_PHY_ADR_CALVL_SWIZZLE0_1, 0x00000000);
1813 		ddr_setval(ch, _reg_PHY_ADR_CALVL_SWIZZLE1_1, 0x00000000);
1814 		ddr_setval(ch, _reg_PHY_ADR_CALVL_DEVICE_MAP, 0x01);
1815 
1816 		for (slice = 0; slice < SLICE_CNT; slice++) {
1817 			ddr_setval_s(ch, slice, _reg_PI_RDLVL_PATTERN_NUM,
1818 				     0x01);
1819 			ddr_setval_s(ch, slice, _reg_PI_RDLVL_PATTERN_START,
1820 				     0x08);
1821 
1822 			if (high_byte[slice])
1823 				o_inv = o_mr20;
1824 			else
1825 				o_inv = o_mr15;
1826 
1827 			tmp = board_cnf->ch[ch].dq_swap[slice];
1828 			inv = 0;
1829 			j = 0;
1830 			for (bit_soc = 0; bit_soc < 8; bit_soc++) {
1831 				bit_mem = (tmp >> (4 * bit_soc)) & 0x0f;
1832 				j |= (1U << bit_mem);
1833 				if (o_inv & (1U << bit_mem))
1834 					inv |= (1U << bit_soc);
1835 			}
1836 			data_l = o_mr32_mr40;
1837 			if (!high_byte[slice])
1838 				data_l |= (inv << 24);
1839 			if (high_byte[slice])
1840 				data_l |= (inv << 16);
1841 			ddr_setval_s(ch, slice, _reg_PHY_LP4_RDLVL_PATT8,
1842 				     data_l);
1843 		}
1844 	}
1845 }
1846 
ddr_config(void)1847 static void ddr_config(void)
1848 {
1849 	int32_t i;
1850 	uint32_t ch, slice;
1851 	uint32_t data_l;
1852 	uint32_t tmp;
1853 	int8_t _adj;
1854 	int16_t adj;
1855 	uint32_t dq;
1856 	union {
1857 		uint32_t ui32[4];
1858 		uint8_t ui8[16];
1859 	} patt;
1860 	uint16_t patm;
1861 
1862 	/* configure ddrphy registers */
1863 	if ((prr_product == PRR_PRODUCT_H3) && (prr_cut <= PRR_PRODUCT_11)) {
1864 		ddr_config_sub_h3v1x();
1865 	} else {	/*  H3 Ver.2.0 or later/M3-N/V3H is same as M3-W */
1866 		ddr_config_sub();
1867 	}
1868 
1869 	/* WDQ_USER_PATT */
1870 	foreach_vch(ch) {
1871 		for (slice = 0; slice < SLICE_CNT; slice++) {
1872 			patm = 0;
1873 			for (i = 0; i < 16; i++) {
1874 				tmp = board_cnf->ch[ch].wdqlvl_patt[i];
1875 				patt.ui8[i] = tmp & 0xff;
1876 				if (tmp & 0x100)
1877 					patm |= (1U << i);
1878 			}
1879 			ddr_setval_s(ch, slice, _reg_PHY_USER_PATT0,
1880 				     patt.ui32[0]);
1881 			ddr_setval_s(ch, slice, _reg_PHY_USER_PATT1,
1882 				     patt.ui32[1]);
1883 			ddr_setval_s(ch, slice, _reg_PHY_USER_PATT2,
1884 				     patt.ui32[2]);
1885 			ddr_setval_s(ch, slice, _reg_PHY_USER_PATT3,
1886 				     patt.ui32[3]);
1887 			ddr_setval_s(ch, slice, _reg_PHY_USER_PATT4, patm);
1888 		}
1889 	}
1890 
1891 	/* CACS DLY */
1892 	data_l = board_cnf->cacs_dly + _f_scale_adj(board_cnf->cacs_dly_adj);
1893 	reg_ddrphy_write_a(ddr_regdef_adr(_reg_PHY_FREQ_SEL_MULTICAST_EN),
1894 			   0x00U);
1895 	foreach_vch(ch) {
1896 		for (i = 0; i < _reg_PHY_CLK_CACS_SLAVE_DELAY_X_NUM - 4; i++) {
1897 			adj = _f_scale_adj(board_cnf->ch[ch].cacs_adj[i]);
1898 			ddrtbl_setval(_cnf_DDR_PHY_ADR_V_REGSET,
1899 				      _reg_PHY_CLK_CACS_SLAVE_DELAY_X[i],
1900 				      data_l + adj);
1901 			reg_ddrphy_write(ch,
1902 					 ddr_regdef_adr
1903 					 (_reg_PHY_CLK_CACS_SLAVE_DELAY_X[i]),
1904 					_cnf_DDR_PHY_ADR_V_REGSET
1905 					[ddr_regdef_adr
1906 					(_reg_PHY_CLK_CACS_SLAVE_DELAY_X[i]) -
1907 					DDR_PHY_ADR_V_REGSET_OFS]);
1908 		}
1909 
1910 		for (i = (_reg_PHY_CLK_CACS_SLAVE_DELAY_X_NUM - 4);
1911 		     i < _reg_PHY_CLK_CACS_SLAVE_DELAY_X_NUM; i++) {
1912 			adj = _f_scale_adj(board_cnf->ch[ch].cacs_adj[i]);
1913 			ddrtbl_setval(_cnf_DDR_PHY_ADR_G_REGSET,
1914 				      _reg_PHY_CLK_CACS_SLAVE_DELAY_X[i],
1915 				      data_l + adj);
1916 			reg_ddrphy_write(ch,
1917 					 ddr_regdef_adr
1918 					 (_reg_PHY_CLK_CACS_SLAVE_DELAY_X[i]),
1919 					_cnf_DDR_PHY_ADR_G_REGSET
1920 					[ddr_regdef_adr
1921 					(_reg_PHY_CLK_CACS_SLAVE_DELAY_X[i]) -
1922 					DDR_PHY_ADR_G_REGSET_OFS]);
1923 		}
1924 
1925 		if (ddr_phycaslice == 1) {
1926 			for (i = 0; i < 6; i++) {
1927 				adj = _f_scale_adj
1928 					(board_cnf->ch[ch].cacs_adj
1929 					[i +
1930 					_reg_PHY_CLK_CACS_SLAVE_DELAY_X_NUM]);
1931 				ddrtbl_setval(_cnf_DDR_PHY_ADR_V_REGSET,
1932 					      _reg_PHY_CLK_CACS_SLAVE_DELAY_X
1933 					      [i],
1934 					      data_l + adj);
1935 				reg_ddrphy_write(ch,
1936 						 ddr_regdef_adr
1937 					(_reg_PHY_CLK_CACS_SLAVE_DELAY_X[i]) +
1938 					0x0100,
1939 					_cnf_DDR_PHY_ADR_V_REGSET
1940 					[ddr_regdef_adr
1941 					(_reg_PHY_CLK_CACS_SLAVE_DELAY_X[i]) -
1942 					DDR_PHY_ADR_V_REGSET_OFS]);
1943 			}
1944 		}
1945 	}
1946 
1947 	reg_ddrphy_write_a(ddr_regdef_adr(_reg_PHY_FREQ_SEL_MULTICAST_EN),
1948 			   BIT(ddr_regdef_lsb(_reg_PHY_FREQ_SEL_MULTICAST_EN)));
1949 
1950 	/* WDQDM DLY */
1951 	data_l = board_cnf->dqdm_dly_w;
1952 	foreach_vch(ch) {
1953 		for (slice = 0; slice < SLICE_CNT; slice++) {
1954 			for (i = 0; i <= 8; i++) {
1955 				dq = slice * 8 + i;
1956 				if (i == 8)
1957 					_adj = board_cnf->ch[ch].dm_adj_w[slice];
1958 				else
1959 					_adj = board_cnf->ch[ch].dq_adj_w[dq];
1960 				adj = _f_scale_adj(_adj);
1961 				ddr_setval_s(ch, slice,
1962 					     _reg_PHY_CLK_WRX_SLAVE_DELAY[i],
1963 					     data_l + adj);
1964 			}
1965 		}
1966 	}
1967 
1968 	/* RDQDM DLY */
1969 	data_l = board_cnf->dqdm_dly_r;
1970 	foreach_vch(ch) {
1971 		for (slice = 0; slice < SLICE_CNT; slice++) {
1972 			for (i = 0; i <= 8; i++) {
1973 				dq = slice * 8 + i;
1974 				if (i == 8)
1975 					_adj = board_cnf->ch[ch].dm_adj_r[slice];
1976 				else
1977 					_adj = board_cnf->ch[ch].dq_adj_r[dq];
1978 				adj = _f_scale_adj(_adj);
1979 				ddr_setval_s(ch, slice,
1980 					     _reg_PHY_RDDQS_X_FALL_SLAVE_DELAY
1981 					     [i], data_l + adj);
1982 				ddr_setval_s(ch, slice,
1983 					     _reg_PHY_RDDQS_X_RISE_SLAVE_DELAY
1984 					     [i], data_l + adj);
1985 			}
1986 		}
1987 	}
1988 }
1989 
1990 /* DBSC register setting functions */
dbsc_regset_pre(void)1991 static void dbsc_regset_pre(void)
1992 {
1993 	uint32_t ch, csab;
1994 	uint32_t data_l;
1995 
1996 	/* PRIMARY SETTINGS */
1997 	/* LPDDR4, BL=16, DFI interface */
1998 	mmio_write_32(DBSC_DBKIND, 0x0000000a);
1999 	mmio_write_32(DBSC_DBBL, 0x00000002);
2000 	mmio_write_32(DBSC_DBPHYCONF0, 0x00000001);
2001 
2002 	/* FREQRATIO=2 */
2003 	mmio_write_32(DBSC_DBSYSCONF1, 0x00000002);
2004 
2005 	/* Chanel map (H3 Ver.1.x) */
2006 	if ((prr_product == PRR_PRODUCT_H3) && (prr_cut <= PRR_PRODUCT_11))
2007 		mmio_write_32(DBSC_DBSCHCNT1, 0x00001010);
2008 
2009 	/* DRAM SIZE REGISTER:
2010 	 * set all ranks as density=0(4Gb) for PHY initialization
2011 	 */
2012 	foreach_vch(ch) {
2013 		for (csab = 0; csab < 4; csab++) {
2014 			mmio_write_32(DBSC_DBMEMCONF(ch, csab),
2015 				      DBMEMCONF_REGD(0));
2016 		}
2017 	}
2018 
2019 	if (prr_product == PRR_PRODUCT_M3) {
2020 		data_l = 0xe4e4e4e4;
2021 		foreach_ech(ch) {
2022 			if ((ddr_phyvalid & (1U << ch)))
2023 				data_l = (data_l & (~(0x000000FF << (ch * 8))))
2024 				    | (((board_cnf->ch[ch].dqs_swap & 0x0003)
2025 					| ((board_cnf->ch[ch].dqs_swap & 0x0030)
2026 					   >> 2)
2027 					| ((board_cnf->ch[ch].dqs_swap & 0x0300)
2028 					   >> 4)
2029 					| ((board_cnf->ch[ch].dqs_swap & 0x3000)
2030 					   >> 6)) << (ch * 8));
2031 		}
2032 		mmio_write_32(DBSC_DBBSWAP, data_l);
2033 	}
2034 }
2035 
dbsc_regset(void)2036 static void dbsc_regset(void)
2037 {
2038 	int32_t i;
2039 	uint32_t ch;
2040 	uint32_t data_l;
2041 	uint32_t data_l2;
2042 	uint32_t tmp[4];
2043 
2044 	/* RFC */
2045 	if ((prr_product == PRR_PRODUCT_H3) && (prr_cut == PRR_PRODUCT_20) &&
2046 	    (max_density == 0)) {
2047 		js2[js2_trfcab] =
2048 		    _f_scale(ddr_mbps, ddr_mbpsdiv,
2049 			     1UL * jedec_spec2_trfc_ab[1] * 1000, 0);
2050 	} else {
2051 		js2[js2_trfcab] =
2052 		    _f_scale(ddr_mbps, ddr_mbpsdiv,
2053 			     1UL * jedec_spec2_trfc_ab[max_density] *
2054 			     1000, 0);
2055 	}
2056 
2057 	/* DBTR0.CL  : RL */
2058 	mmio_write_32(DBSC_DBTR(0), RL);
2059 
2060 	/* DBTR1.CWL : WL */
2061 	mmio_write_32(DBSC_DBTR(1), WL);
2062 
2063 	/* DBTR2.AL  : 0 */
2064 	mmio_write_32(DBSC_DBTR(2), 0);
2065 
2066 	/* DBTR3.TRCD: tRCD */
2067 	mmio_write_32(DBSC_DBTR(3), js2[js2_trcd]);
2068 
2069 	/* DBTR4.TRPA,TRP: tRPab,tRPpb */
2070 	mmio_write_32(DBSC_DBTR(4), (js2[js2_trpab] << 16) | js2[js2_trppb]);
2071 
2072 	/* DBTR5.TRC : use tRCpb */
2073 	mmio_write_32(DBSC_DBTR(5), js2[js2_trcpb]);
2074 
2075 	/* DBTR6.TRAS : tRAS */
2076 	mmio_write_32(DBSC_DBTR(6), js2[js2_tras]);
2077 
2078 	/* DBTR7.TRRD : tRRD */
2079 	mmio_write_32(DBSC_DBTR(7), (js2[js2_trrd] << 16) | js2[js2_trrd]);
2080 
2081 	/* DBTR8.TFAW : tFAW */
2082 	mmio_write_32(DBSC_DBTR(8), js2[js2_tfaw]);
2083 
2084 	/* DBTR9.TRDPR : tRTP */
2085 	mmio_write_32(DBSC_DBTR(9), js2[js2_trtp]);
2086 
2087 	/* DBTR10.TWR : nWR */
2088 	mmio_write_32(DBSC_DBTR(10), js1[js1_ind].nwr);
2089 
2090 	/*
2091 	 * DBTR11.TRDWR : RL +  BL / 2 + Rounddown(tRPST) + PHY_ODTLoff -
2092 	 * 		  odtlon + tDQSCK - tODTon,min +
2093 	 * 		  PCB delay (out+in) + tPHY_ODToff
2094 	 */
2095 	mmio_write_32(DBSC_DBTR(11),
2096 		      RL + (16 / 2) + 1 + 2 - js1[js1_ind].odtlon +
2097 		      js2[js2_tdqsck] - js2[js2_tODTon_min] +
2098 		      _f_scale(ddr_mbps, ddr_mbpsdiv, 1300, 0));
2099 
2100 	/* DBTR12.TWRRD : WL + 1 + BL/2 + tWTR */
2101 	data_l = WL + 1 + (16 / 2) + js2[js2_twtr];
2102 	mmio_write_32(DBSC_DBTR(12), (data_l << 16) | data_l);
2103 
2104 	/* DBTR13.TRFCAB : tRFCab */
2105 	mmio_write_32(DBSC_DBTR(13), (js2[js2_trfcab]));
2106 
2107 	/* DBTR14.TCKEHDLL,tCKEH : tCKEHCMD,tCKEHCMD */
2108 	mmio_write_32(DBSC_DBTR(14),
2109 		      (js2[js2_tckehcmd] << 16) | (js2[js2_tckehcmd]));
2110 
2111 	/* DBTR15.TCKESR,TCKEL : tSR,tCKELPD */
2112 	mmio_write_32(DBSC_DBTR(15), (js2[js2_tsr] << 16) | (js2[js2_tckelpd]));
2113 
2114 	/* DBTR16 */
2115 	/* WDQL : tphy_wrlat + tphy_wrdata */
2116 	tmp[0] = ddrtbl_getval(_cnf_DDR_PI_REGSET, _reg_PI_WRLAT_F1);
2117 	/* DQENLTNCY : tphy_wrlat = WL-2 : PHY_WRITE_PATH_LAT_ADD == 0
2118 	 *             tphy_wrlat = WL-3 : PHY_WRITE_PATH_LAT_ADD != 0
2119 	 */
2120 	tmp[1] = ddrtbl_getval(_cnf_DDR_PI_REGSET, _reg_PI_WRLAT_ADJ_F1);
2121 	/* DQL : tphy_rdlat + trdata_en */
2122 	/* it is not important for dbsc */
2123 	tmp[2] = RL + 16;
2124 	/* DQIENLTNCY : trdata_en */
2125 	tmp[3] = ddrtbl_getval(_cnf_DDR_PI_REGSET, _reg_PI_RDLAT_ADJ_F1) - 1;
2126 	mmio_write_32(DBSC_DBTR(16),
2127 		      (tmp[3] << 24) | (tmp[2] << 16) | (tmp[1] << 8) | tmp[0]);
2128 
2129 	/* DBTR24 */
2130 	/* WRCSLAT = WRLAT -5 */
2131 	tmp[0] -= 5;
2132 	/* WRCSGAP = 5 */
2133 	tmp[1] = 5;
2134 	/* RDCSLAT = RDLAT_ADJ +2 */
2135 	if (prr_product == PRR_PRODUCT_M3) {
2136 		tmp[2] = tmp[3];
2137 	} else {
2138 		tmp[2] = tmp[3] + 2;
2139 	}
2140 	/* RDCSGAP = 6 */
2141 	if (prr_product == PRR_PRODUCT_M3) {
2142 		tmp[3] = 4;
2143 	} else {
2144 		tmp[3] = 6;
2145 	}
2146 	mmio_write_32(DBSC_DBTR(24),
2147 		      (tmp[3] << 24) | (tmp[2] << 16) | (tmp[1] << 8) | tmp[0]);
2148 
2149 	/* DBTR17.TMODRD,TMOD,TRDMR: tMRR,tMRD,(0) */
2150 	mmio_write_32(DBSC_DBTR(17),
2151 		      (js2[js2_tmrr] << 24) | (js2[js2_tmrd] << 16));
2152 
2153 	/* DBTR18.RODTL, RODTA, WODTL, WODTA : do not use in LPDDR4 */
2154 	mmio_write_32(DBSC_DBTR(18), 0);
2155 
2156 	/* DBTR19.TZQCL, TZQCS : do not use in LPDDR4 */
2157 	mmio_write_32(DBSC_DBTR(19), 0);
2158 
2159 	/* DBTR20.TXSDLL, TXS : tRFCab+tCKEHCMD */
2160 	data_l = js2[js2_trfcab] + js2[js2_tckehcmd];
2161 	mmio_write_32(DBSC_DBTR(20), (data_l << 16) | data_l);
2162 
2163 	/* DBTR21.TCCD */
2164 	/* DBTR23.TCCD */
2165 	/* H3 Ver.1.0 cannot use TBTR23 feature */
2166 	if (ddr_tccd == 8 &&
2167 	    !((prr_product == PRR_PRODUCT_H3) && (prr_cut <= PRR_PRODUCT_10))
2168 	    ) {
2169 		data_l = 8;
2170 		mmio_write_32(DBSC_DBTR(21), (data_l << 16) | data_l);
2171 		mmio_write_32(DBSC_DBTR(23), 0x00000002);
2172 	} else if (ddr_tccd <= 11) {
2173 		data_l = 11;
2174 		mmio_write_32(DBSC_DBTR(21), (data_l << 16) | data_l);
2175 		mmio_write_32(DBSC_DBTR(23), 0x00000000);
2176 	} else {
2177 		data_l = ddr_tccd;
2178 		mmio_write_32(DBSC_DBTR(21), (data_l << 16) | data_l);
2179 		mmio_write_32(DBSC_DBTR(23), 0x00000000);
2180 	}
2181 
2182 	/* DBTR22.ZQLAT : */
2183 	data_l = js2[js2_tzqcalns] * 100;	/*  1000 * 1000 ps */
2184 	data_l = (data_l << 16) | (js2[js2_tzqlat] + 24 + 20);
2185 	mmio_write_32(DBSC_DBTR(22), data_l);
2186 
2187 	/* DBTR25 : do not use in LPDDR4 */
2188 	mmio_write_32(DBSC_DBTR(25), 0);
2189 
2190 	/* DBRNK : */
2191 	/*
2192 	 * DBSC_DBRNK2 rkrr
2193 	 * DBSC_DBRNK3 rkrw
2194 	 * DBSC_DBRNK4 rkwr
2195 	 * DBSC_DBRNK5 rkww
2196 	 */
2197 #define _par_DBRNK_VAL		(0x7007)
2198 
2199 	for (i = 0; i < 4; i++) {
2200 		data_l = (_par_DBRNK_VAL >> (i * 4)) & 0x0f;
2201 		if ((prr_product == PRR_PRODUCT_H3) &&
2202 		    (prr_cut > PRR_PRODUCT_11) && (i == 0)) {
2203 			data_l += 1;
2204 		}
2205 		data_l2 = 0;
2206 		foreach_vch(ch) {
2207 			data_l2 = data_l2 | (data_l << (4 * ch));
2208 		}
2209 		mmio_write_32(DBSC_DBRNK(2 + i), data_l2);
2210 	}
2211 	mmio_write_32(DBSC_DBADJ0, 0x00000000);
2212 
2213 	/* timing registers for Scheduler */
2214 	/* SCFCTST0 */
2215 	/* SCFCTST0 ACT-ACT */
2216 	tmp[3] = 1UL * js2[js2_trcpb] * 800 * ddr_mbpsdiv / ddr_mbps;
2217 	/* SCFCTST0 RDA-ACT */
2218 	tmp[2] =
2219 	    1UL * ((16 / 2) + js2[js2_trtp] - 8 +
2220 		   js2[js2_trppb]) * 800 * ddr_mbpsdiv / ddr_mbps;
2221 	/* SCFCTST0 WRA-ACT */
2222 	tmp[1] =
2223 	    1UL * (WL + 1 + (16 / 2) +
2224 		   js1[js1_ind].nwr) * 800 * ddr_mbpsdiv / ddr_mbps;
2225 	/* SCFCTST0 PRE-ACT */
2226 	tmp[0] = 1UL * js2[js2_trppb];
2227 	mmio_write_32(DBSC_SCFCTST0,
2228 		      (tmp[3] << 24) | (tmp[2] << 16) | (tmp[1] << 8) | tmp[0]);
2229 
2230 	/* SCFCTST1 */
2231 	/* SCFCTST1 RD-WR */
2232 	tmp[3] =
2233 	    1UL * (mmio_read_32(DBSC_DBTR(11)) & 0xff) * 800 * ddr_mbpsdiv /
2234 	    ddr_mbps;
2235 	/* SCFCTST1 WR-RD */
2236 	tmp[2] =
2237 	    1UL * (mmio_read_32(DBSC_DBTR(12)) & 0xff) * 800 * ddr_mbpsdiv /
2238 	    ddr_mbps;
2239 	/* SCFCTST1 ACT-RD/WR */
2240 	tmp[1] = 1UL * js2[js2_trcd] * 800 * ddr_mbpsdiv / ddr_mbps;
2241 	/* SCFCTST1 ASYNCOFS */
2242 	tmp[0] = 12;
2243 	mmio_write_32(DBSC_SCFCTST1,
2244 		      (tmp[3] << 24) | (tmp[2] << 16) | (tmp[1] << 8) | tmp[0]);
2245 
2246 	/* DBSCHRW1 */
2247 	/* DBSCHRW1 SCTRFCAB */
2248 	tmp[0] = 1UL * js2[js2_trfcab] * 800 * ddr_mbpsdiv / ddr_mbps;
2249 	data_l = (((mmio_read_32(DBSC_DBTR(16)) & 0x00FF0000) >> 16)
2250 		 + (mmio_read_32(DBSC_DBTR(22)) & 0x0000FFFF)
2251 		 + (0x28 * 2)) * 400 * 2 * ddr_mbpsdiv / ddr_mbps + 7;
2252 	if (tmp[0] < data_l)
2253 		tmp[0] = data_l;
2254 
2255 	if ((prr_product == PRR_PRODUCT_M3) && (prr_cut < PRR_PRODUCT_30)) {
2256 		mmio_write_32(DBSC_DBSCHRW1, tmp[0]
2257 			+ ((mmio_read_32(DBSC_DBTR(22)) & 0x0000FFFF)
2258 			* 400 * 2 * ddr_mbpsdiv + (ddr_mbps - 1)) /
2259 			ddr_mbps - 3);
2260 	} else {
2261 		mmio_write_32(DBSC_DBSCHRW1, tmp[0]
2262 			+ ((mmio_read_32(DBSC_DBTR(22)) & 0x0000FFFF)
2263 			* 400 * 2 * ddr_mbpsdiv + (ddr_mbps - 1)) /
2264 			ddr_mbps);
2265 	}
2266 
2267 	/* QOS and CAM */
2268 #ifdef ddr_qos_init_setting	/*  only for non qos_init */
2269 	/*wbkwait(0004), wbkmdhi(4,2),wbkmdlo(1,8) */
2270 	mmio_write_32(DBSC_DBCAM0CNF1, 0x00043218);
2271 	/*0(fillunit),8(dirtymax),4(dirtymin) */
2272 	mmio_write_32(DBSC_DBCAM0CNF2, 0x000000F4);
2273 	/*stop_tolerance */
2274 	mmio_write_32(DBSC_DBSCHRW0, 0x22421111);
2275 	/*rd-wr/wr-rd toggle priority */
2276 	mmio_write_32(DBSC_SCFCTST2, 0x012F1123);
2277 	mmio_write_32(DBSC_DBSCHSZ0, 0x00000001);
2278 	mmio_write_32(DBSC_DBSCHCNT0, 0x000F0037);
2279 
2280 	/* QoS Settings */
2281 	mmio_write_32(DBSC_DBSCHQOS00, 0x00000F00U);
2282 	mmio_write_32(DBSC_DBSCHQOS01, 0x00000B00U);
2283 	mmio_write_32(DBSC_DBSCHQOS02, 0x00000000U);
2284 	mmio_write_32(DBSC_DBSCHQOS03, 0x00000000U);
2285 	mmio_write_32(DBSC_DBSCHQOS40, 0x00000300U);
2286 	mmio_write_32(DBSC_DBSCHQOS41, 0x000002F0U);
2287 	mmio_write_32(DBSC_DBSCHQOS42, 0x00000200U);
2288 	mmio_write_32(DBSC_DBSCHQOS43, 0x00000100U);
2289 	mmio_write_32(DBSC_DBSCHQOS90, 0x00000100U);
2290 	mmio_write_32(DBSC_DBSCHQOS91, 0x000000F0U);
2291 	mmio_write_32(DBSC_DBSCHQOS92, 0x000000A0U);
2292 	mmio_write_32(DBSC_DBSCHQOS93, 0x00000040U);
2293 	mmio_write_32(DBSC_DBSCHQOS120, 0x00000040U);
2294 	mmio_write_32(DBSC_DBSCHQOS121, 0x00000030U);
2295 	mmio_write_32(DBSC_DBSCHQOS122, 0x00000020U);
2296 	mmio_write_32(DBSC_DBSCHQOS123, 0x00000010U);
2297 	mmio_write_32(DBSC_DBSCHQOS130, 0x00000100U);
2298 	mmio_write_32(DBSC_DBSCHQOS131, 0x000000F0U);
2299 	mmio_write_32(DBSC_DBSCHQOS132, 0x000000A0U);
2300 	mmio_write_32(DBSC_DBSCHQOS133, 0x00000040U);
2301 	mmio_write_32(DBSC_DBSCHQOS140, 0x000000C0U);
2302 	mmio_write_32(DBSC_DBSCHQOS141, 0x000000B0U);
2303 	mmio_write_32(DBSC_DBSCHQOS142, 0x00000080U);
2304 	mmio_write_32(DBSC_DBSCHQOS143, 0x00000040U);
2305 	mmio_write_32(DBSC_DBSCHQOS150, 0x00000040U);
2306 	mmio_write_32(DBSC_DBSCHQOS151, 0x00000030U);
2307 	mmio_write_32(DBSC_DBSCHQOS152, 0x00000020U);
2308 	mmio_write_32(DBSC_DBSCHQOS153, 0x00000010U);
2309 
2310 	mmio_write_32(QOSCTRL_RAEN, 0x00000001U);
2311 #endif /* ddr_qos_init_setting */
2312 	/* H3 Ver.1.1 need to set monitor function */
2313 	if ((prr_product == PRR_PRODUCT_H3) && (prr_cut == PRR_PRODUCT_11)) {
2314 		mmio_write_32(DBSC_DBMONCONF4, 0x00700000);
2315 	}
2316 
2317 	if (prr_product == PRR_PRODUCT_H3) {
2318 		if (prr_cut == PRR_PRODUCT_10) {
2319 			/* resrdis, simple mode, sc off */
2320 			mmio_write_32(DBSC_DBBCAMDIS, 0x00000007);
2321 		} else if (prr_cut == PRR_PRODUCT_11) {
2322 			/* resrdis, simple mode         */
2323 			mmio_write_32(DBSC_DBBCAMDIS, 0x00000005);
2324 		} else if (prr_cut < PRR_PRODUCT_30) {
2325 			/* H3 Ver.2.0                   */
2326 			/* resrdis                      */
2327 			mmio_write_32(DBSC_DBBCAMDIS, 0x00000001);
2328 		} else {	/* H3 Ver.3.0(include H3N)      */
2329 			/* exprespque                   */
2330 			mmio_write_32(DBSC_DBBCAMDIS, 0x00000010);
2331 		}
2332 	} else {		/* M3-W/M3-N/V3H                */
2333 		/* resrdis                      */
2334 		mmio_write_32(DBSC_DBBCAMDIS, 0x00000001);
2335 	}
2336 }
2337 
dbsc_regset_post(void)2338 static void dbsc_regset_post(void)
2339 {
2340 	uint32_t ch, cs;
2341 	uint32_t data_l;
2342 	uint32_t slice, rdlat_max, rdlat_min;
2343 
2344 	rdlat_max = 0;
2345 	rdlat_min = 0xffff;
2346 	foreach_vch(ch) {
2347 		for (cs = 0; cs < CS_CNT; cs++) {
2348 			if ((ch_have_this_cs[cs] & (1U << ch)) != 0) {
2349 				for (slice = 0; slice < SLICE_CNT; slice++) {
2350 					ddr_setval_s(ch, slice,
2351 						     _reg_PHY_PER_CS_TRAINING_INDEX,
2352 						     cs);
2353 					data_l = ddr_getval_s(ch, slice,
2354 							      _reg_PHY_RDDQS_LATENCY_ADJUST);
2355 					if (data_l > rdlat_max)
2356 						rdlat_max = data_l;
2357 					if (data_l < rdlat_min)
2358 						rdlat_min = data_l;
2359 				}
2360 			}
2361 		}
2362 	}
2363 	if ((prr_product == PRR_PRODUCT_H3) && (prr_cut > PRR_PRODUCT_11)) {
2364 #if RCAR_DRAM_SPLIT == 2
2365 		if (board_cnf->phyvalid == 0x05) {
2366 			mmio_write_32(DBSC_DBTR(24),
2367 				      (rdlat_max << 24) + (rdlat_min << 16) +
2368 				      mmio_read_32(DBSC_DBTR(24)));
2369 		} else {
2370 			mmio_write_32(DBSC_DBTR(24),
2371 				      ((rdlat_max * 2 - rdlat_min + 4) << 24) +
2372 				      ((rdlat_min + 2) << 16) +
2373 				      mmio_read_32(DBSC_DBTR(24)));
2374 		}
2375 #else /*RCAR_DRAM_SPLIT == 2 */
2376 		mmio_write_32(DBSC_DBTR(24),
2377 			      ((rdlat_max * 2 - rdlat_min + 4) << 24) +
2378 			      ((rdlat_min + 2) << 16) +
2379 			      mmio_read_32(DBSC_DBTR(24)));
2380 #endif /*RCAR_DRAM_SPLIT == 2 */
2381 	} else {
2382 		mmio_write_32(DBSC_DBTR(24),
2383 			      ((rdlat_max + 2) << 24) +
2384 			      ((rdlat_max + 2) << 16) +
2385 			      mmio_read_32(DBSC_DBTR(24)));
2386 	}
2387 
2388 	/* set ddr density information */
2389 	foreach_ech(ch) {
2390 		for (cs = 0; cs < CS_CNT; cs++) {
2391 			if (ddr_density[ch][cs] == 0xff) {
2392 				mmio_write_32(DBSC_DBMEMCONF(ch, cs), 0x00);
2393 			} else {
2394 				mmio_write_32(DBSC_DBMEMCONF(ch, cs),
2395 					      DBMEMCONF_REGD(ddr_density[ch]
2396 							     [cs]));
2397 			}
2398 		}
2399 		mmio_write_32(DBSC_DBMEMCONF(ch, 2), 0x00000000);
2400 		mmio_write_32(DBSC_DBMEMCONF(ch, 3), 0x00000000);
2401 	}
2402 
2403 	mmio_write_32(DBSC_DBBUS0CNF1, 0x00000010);
2404 
2405 	/*set DBI */
2406 	if (board_cnf->dbi_en)
2407 		mmio_write_32(DBSC_DBDBICNT, 0x00000003);
2408 
2409 	/* H3 Ver.2.0 or later/M3-N/V3H DBI wa */
2410 	if ((((prr_product == PRR_PRODUCT_H3) &&
2411 	      (prr_cut > PRR_PRODUCT_11)) ||
2412 	     (prr_product == PRR_PRODUCT_M3N) ||
2413 	     (prr_product == PRR_PRODUCT_V3H)) &&
2414 	    board_cnf->dbi_en)
2415 		reg_ddrphy_write_a(0x00001010, 0x01000000);
2416 
2417 	/*set REFCYCLE */
2418 	data_l = (get_refperiod()) * ddr_mbps / 2000 / ddr_mbpsdiv;
2419 	mmio_write_32(DBSC_DBRFCNF1, 0x00080000 | (data_l & 0x0000ffff));
2420 	mmio_write_32(DBSC_DBRFCNF2, 0x00010000 | DBSC_REFINTS);
2421 
2422 #if RCAR_REWT_TRAINING != 0
2423 	/* Periodic-WriteDQ Training seeting */
2424 	if (((prr_product == PRR_PRODUCT_H3) &&
2425 	     (prr_cut <= PRR_PRODUCT_11)) ||
2426 	    ((prr_product == PRR_PRODUCT_M3) &&
2427 	     (prr_cut == PRR_PRODUCT_10))) {
2428 		/* non : H3 Ver.1.x/M3-W Ver.1.0 not support */
2429 	} else {
2430 		/* H3 Ver.2.0 or later/M3-W Ver.1.1 or later/M3-N/V3H */
2431 		mmio_write_32(DBSC_DBDFIPMSTRCNF, 0x00000000);
2432 
2433 		ddr_setval_ach_as(_reg_PHY_WDQLVL_PATT, 0x04);
2434 		ddr_setval_ach_as(_reg_PHY_WDQLVL_QTR_DLY_STEP, 0x0F);
2435 		ddr_setval_ach_as(_reg_PHY_WDQLVL_DLY_STEP, 0x50);
2436 		ddr_setval_ach_as(_reg_PHY_WDQLVL_DQDM_SLV_DLY_START, 0x0300);
2437 
2438 		ddr_setval_ach(_reg_PI_WDQLVL_CS_MAP,
2439 			       ddrtbl_getval(_cnf_DDR_PI_REGSET,
2440 					     _reg_PI_WDQLVL_CS_MAP));
2441 		ddr_setval_ach(_reg_PI_LONG_COUNT_MASK, 0x1f);
2442 		ddr_setval_ach(_reg_PI_WDQLVL_VREF_EN, 0x00);
2443 		ddr_setval_ach(_reg_PI_WDQLVL_ROTATE, 0x01);
2444 		ddr_setval_ach(_reg_PI_TREF_F0, 0x0000);
2445 		ddr_setval_ach(_reg_PI_TREF_F1, 0x0000);
2446 		ddr_setval_ach(_reg_PI_TREF_F2, 0x0000);
2447 
2448 		if (prr_product == PRR_PRODUCT_M3) {
2449 			ddr_setval_ach(_reg_PI_WDQLVL_EN, 0x02);
2450 		} else {
2451 			ddr_setval_ach(_reg_PI_WDQLVL_EN_F1, 0x02);
2452 		}
2453 		ddr_setval_ach(_reg_PI_WDQLVL_PERIODIC, 0x01);
2454 
2455 		/* DFI_PHYMSTR_ACK , WTmode setting */
2456 		/* DFI_PHYMSTR_ACK: WTmode =b'01 */
2457 		mmio_write_32(DBSC_DBDFIPMSTRCNF, 0x00000011);
2458 	}
2459 #endif /* RCAR_REWT_TRAINING */
2460 	/* periodic dram zqcal enable */
2461 	mmio_write_32(DBSC_DBCALCNF, 0x01000010);
2462 
2463 	/* periodic phy ctrl update enable */
2464 	if (((prr_product == PRR_PRODUCT_H3) &&
2465 	     (prr_cut <= PRR_PRODUCT_11)) ||
2466 	    ((prr_product == PRR_PRODUCT_M3) &&
2467 	     (prr_cut < PRR_PRODUCT_30))) {
2468 		/* non : H3 Ver.1.x/M3-W Ver.1.x not support */
2469 	} else {
2470 #if RCAR_DRAM_SPLIT == 2
2471 		if ((prr_product == PRR_PRODUCT_H3) &&
2472 		    (board_cnf->phyvalid == 0x05))
2473 			mmio_write_32(DBSC_DBDFICUPDCNF, 0x2a240001);
2474 		else
2475 			mmio_write_32(DBSC_DBDFICUPDCNF, 0x28240001);
2476 #else /* RCAR_DRAM_SPLIT == 2 */
2477 		mmio_write_32(DBSC_DBDFICUPDCNF, 0x28240001);
2478 #endif /* RCAR_DRAM_SPLIT == 2 */
2479 	}
2480 
2481 #ifdef DDR_BACKUPMODE
2482 	/* SRX */
2483 	if (ddr_backup == DRAM_BOOT_STATUS_WARM) {
2484 #ifdef DDR_BACKUPMODE_HALF		/* for Half channel(ch0, 1 only) */
2485 		NOTICE("BL2: [DEBUG_MESS] DDR_BACKUPMODE_HALF\n");
2486 		send_dbcmd(0x0A040001);
2487 		if (Prr_Product == PRR_PRODUCT_H3)
2488 			send_dbcmd(0x0A140001);
2489 #else /* DDR_BACKUPMODE_HALF */		/* for All channels */
2490 		send_dbcmd(0x0A840001);
2491 #endif /* DDR_BACKUPMODE_HALF */
2492 	}
2493 #endif /* DDR_BACKUPMODE */
2494 
2495 	/* set Auto Refresh */
2496 	mmio_write_32(DBSC_DBRFEN, 0x00000001);
2497 
2498 #if RCAR_REWT_TRAINING != 0
2499 	/* Periodic WriteDQ Traning */
2500 	if (((prr_product == PRR_PRODUCT_H3) &&
2501 	     (prr_cut <= PRR_PRODUCT_11)) ||
2502 	    ((prr_product == PRR_PRODUCT_M3) &&
2503 	     (prr_cut == PRR_PRODUCT_10))) {
2504 		/* non : H3 Ver.1.x/M3-W Ver.1.0 not support */
2505 	} else {
2506 		/* H3 Ver.2.0 or later/M3-W Ver.1.1 or later/M3-N/V3H */
2507 		ddr_setval_ach(_reg_PI_WDQLVL_INTERVAL, 0x0100);
2508 	}
2509 #endif /* RCAR_REWT_TRAINING */
2510 
2511 	/* dram access enable */
2512 	mmio_write_32(DBSC_DBACEN, 0x00000001);
2513 
2514 	MSG_LF(__func__ "(done)");
2515 }
2516 
2517 /* DFI_INIT_START */
dfi_init_start(void)2518 static uint32_t dfi_init_start(void)
2519 {
2520 	uint32_t ch;
2521 	uint32_t phytrainingok;
2522 	uint32_t retry;
2523 	uint32_t data_l;
2524 	const uint32_t RETRY_MAX = 0x10000;
2525 
2526 	if ((prr_product == PRR_PRODUCT_H3) && (prr_cut <= PRR_PRODUCT_11)) {
2527 		/* PLL3 Disable */
2528 		/* protect register interface */
2529 		ddrphy_regif_idle();
2530 
2531 		pll3_control(0);
2532 
2533 		/* init start */
2534 		/* dbdficnt0:
2535 		 * dfi_dram_clk_disable=1
2536 		 * dfi_frequency = 0
2537 		 * freq_ratio = 01 (2:1)
2538 		 * init_start =0
2539 		 */
2540 		foreach_vch(ch)
2541 		    mmio_write_32(DBSC_DBDFICNT(ch), 0x00000F10);
2542 		dsb_sev();
2543 
2544 		/* dbdficnt0:
2545 		 * dfi_dram_clk_disable=1
2546 		 * dfi_frequency = 0
2547 		 * freq_ratio = 01 (2:1)
2548 		 * init_start =1
2549 		 */
2550 		foreach_vch(ch)
2551 		    mmio_write_32(DBSC_DBDFICNT(ch), 0x00000F11);
2552 		dsb_sev();
2553 
2554 	} else {
2555 		ddr_setval_ach_as(_reg_PHY_DLL_RST_EN, 0x02);
2556 		dsb_sev();
2557 		ddrphy_regif_idle();
2558 	}
2559 
2560 	/* dll_rst negate */
2561 	foreach_vch(ch)
2562 	    mmio_write_32(DBSC_DBPDCNT3(ch), 0x0000CF01);
2563 	dsb_sev();
2564 
2565 	/* wait init_complete */
2566 	phytrainingok = 0;
2567 	retry = 0;
2568 	while (retry++ < RETRY_MAX) {
2569 		foreach_vch(ch) {
2570 			data_l = mmio_read_32(DBSC_DBDFISTAT(ch));
2571 			if (data_l & 0x00000001)
2572 				phytrainingok |= (1U << ch);
2573 		}
2574 		dsb_sev();
2575 		if (phytrainingok == ddr_phyvalid)
2576 			break;
2577 		if (retry % 256 == 0)
2578 			ddr_setval_ach_as(_reg_SC_PHY_RX_CAL_START, 0x01);
2579 	}
2580 
2581 	/* all ch ok? */
2582 	if ((phytrainingok & ddr_phyvalid) != ddr_phyvalid)
2583 		return 0xff;
2584 
2585 	/* dbdficnt0:
2586 	 * dfi_dram_clk_disable=0
2587 	 * dfi_frequency = 0
2588 	 * freq_ratio = 01 (2:1)
2589 	 * init_start =0
2590 	 */
2591 	foreach_vch(ch)
2592 	    mmio_write_32(DBSC_DBDFICNT(ch), 0x00000010);
2593 	dsb_sev();
2594 
2595 	return 0;
2596 }
2597 
2598 /* drivablity setting : CMOS MODE ON/OFF */
change_lpddr4_en(uint32_t mode)2599 static void change_lpddr4_en(uint32_t mode)
2600 {
2601 	uint32_t ch;
2602 	uint32_t i;
2603 	uint32_t data_l;
2604 	const uint32_t _reg_PHY_PAD_DRIVE_X[3] = {
2605 		_reg_PHY_PAD_ADDR_DRIVE,
2606 		_reg_PHY_PAD_CLK_DRIVE,
2607 		_reg_PHY_PAD_CS_DRIVE
2608 	};
2609 
2610 	foreach_vch(ch) {
2611 		for (i = 0; i < 3; i++) {
2612 			data_l = ddr_getval(ch, _reg_PHY_PAD_DRIVE_X[i]);
2613 			if (mode) {
2614 				data_l |= (1U << 14);
2615 			} else {
2616 				data_l &= ~(1U << 14);
2617 			}
2618 			ddr_setval(ch, _reg_PHY_PAD_DRIVE_X[i], data_l);
2619 		}
2620 	}
2621 }
2622 
2623 /* drivablity setting */
set_term_code(void)2624 static uint32_t set_term_code(void)
2625 {
2626 	int32_t i;
2627 	uint32_t ch, index;
2628 	uint32_t data_l;
2629 	uint32_t chip_id[2];
2630 	uint32_t term_code;
2631 	uint32_t override;
2632 	uint32_t pvtr;
2633 	uint32_t pvtp;
2634 	uint32_t pvtn;
2635 
2636 	term_code = ddrtbl_getval(_cnf_DDR_PHY_ADR_G_REGSET,
2637 				  _reg_PHY_PAD_DATA_TERM);
2638 	override = 0;
2639 	for (i = 0; i < 2; i++)
2640 		chip_id[i] = mmio_read_32(LIFEC_CHIPID(i));
2641 
2642 	index = 0;
2643 	while (1) {
2644 		if (termcode_by_sample[index][0] == 0xffffffff) {
2645 			break;
2646 		}
2647 		if ((termcode_by_sample[index][0] == chip_id[0]) &&
2648 		    (termcode_by_sample[index][1] == chip_id[1])) {
2649 			term_code = termcode_by_sample[index][2];
2650 			override = 1;
2651 			break;
2652 		}
2653 		index++;
2654 	}
2655 
2656 	if (override) {
2657 		for (index = 0; index < _reg_PHY_PAD_TERM_X_NUM; index++) {
2658 			data_l =
2659 			    ddrtbl_getval(_cnf_DDR_PHY_ADR_G_REGSET,
2660 					  _reg_PHY_PAD_TERM_X[index]);
2661 			data_l = (data_l & 0xfffe0000) | term_code;
2662 			ddr_setval_ach(_reg_PHY_PAD_TERM_X[index], data_l);
2663 		}
2664 	} else if ((prr_product == PRR_PRODUCT_M3) &&
2665 		   (prr_cut == PRR_PRODUCT_10)) {
2666 		/*  non */
2667 	} else {
2668 		ddr_setval_ach(_reg_PHY_PAD_TERM_X[0],
2669 			       (ddrtbl_getval
2670 				(_cnf_DDR_PHY_ADR_G_REGSET,
2671 				 _reg_PHY_PAD_TERM_X[0]) & 0xFFFE0000));
2672 		ddr_setval_ach(_reg_PHY_CAL_CLEAR_0, 0x01);
2673 		ddr_setval_ach(_reg_PHY_CAL_START_0, 0x01);
2674 		foreach_vch(ch) {
2675 			do {
2676 				data_l =
2677 				    ddr_getval(ch, _reg_PHY_CAL_RESULT2_OBS_0);
2678 			} while (!(data_l & 0x00800000));
2679 		}
2680 		if ((prr_product == PRR_PRODUCT_H3) &&
2681 		    (prr_cut <= PRR_PRODUCT_11)) {
2682 			foreach_vch(ch) {
2683 				data_l = ddr_getval(ch, _reg_PHY_PAD_TERM_X[0]);
2684 				pvtr = (data_l >> 12) & 0x1f;
2685 				pvtr += 8;
2686 				if (pvtr > 0x1f)
2687 					pvtr = 0x1f;
2688 				data_l =
2689 				    ddr_getval(ch, _reg_PHY_CAL_RESULT2_OBS_0);
2690 				pvtn = (data_l >> 6) & 0x03f;
2691 				pvtp = (data_l >> 0) & 0x03f;
2692 
2693 				for (index = 0; index < _reg_PHY_PAD_TERM_X_NUM;
2694 				     index++) {
2695 					data_l =
2696 					    ddrtbl_getval
2697 					    (_cnf_DDR_PHY_ADR_G_REGSET,
2698 					     _reg_PHY_PAD_TERM_X[index]);
2699 					data_l = (data_l & 0xfffe0000)
2700 					    | (pvtr << 12)
2701 					    | (pvtn << 6)
2702 					    | (pvtp);
2703 					ddr_setval(ch,
2704 						   _reg_PHY_PAD_TERM_X[index],
2705 						   data_l);
2706 				}
2707 			}
2708 		} else {
2709 			/* M3-W Ver.1.1 or later/H3 Ver.2.0 or later/M3-N/V3H */
2710 			foreach_vch(ch) {
2711 				for (index = 0; index < _reg_PHY_PAD_TERM_X_NUM;
2712 				     index++) {
2713 					data_l =
2714 					    ddr_getval(ch,
2715 						       _reg_PHY_PAD_TERM_X
2716 						       [index]);
2717 					ddr_setval(ch,
2718 						   _reg_PHY_PAD_TERM_X[index],
2719 						   (data_l & 0xFFFE0FFF) |
2720 						   0x00015000);
2721 				}
2722 			}
2723 		}
2724 	}
2725 
2726 	if ((prr_product == PRR_PRODUCT_H3) && (prr_cut <= PRR_PRODUCT_11)) {
2727 		/* non */
2728 	} else {
2729 		ddr_padcal_tcompensate_getinit(override);
2730 	}
2731 
2732 	return 0;
2733 }
2734 
2735 /* DDR mode register setting */
ddr_register_set(void)2736 static void ddr_register_set(void)
2737 {
2738 	int32_t fspwp;
2739 	uint32_t tmp;
2740 
2741 	for (fspwp = 1; fspwp >= 0; fspwp--) {
2742 		/*MR13, fspwp */
2743 		send_dbcmd(0x0e840d08 | ((2 - fspwp) << 6));
2744 
2745 		tmp =
2746 		    ddrtbl_getval(_cnf_DDR_PI_REGSET,
2747 				  reg_pi_mr1_data_fx_csx[fspwp][0]);
2748 		send_dbcmd(0x0e840100 | tmp);
2749 
2750 		tmp =
2751 		    ddrtbl_getval(_cnf_DDR_PI_REGSET,
2752 				  reg_pi_mr2_data_fx_csx[fspwp][0]);
2753 		send_dbcmd(0x0e840200 | tmp);
2754 
2755 		tmp =
2756 		    ddrtbl_getval(_cnf_DDR_PI_REGSET,
2757 				  reg_pi_mr3_data_fx_csx[fspwp][0]);
2758 		send_dbcmd(0x0e840300 | tmp);
2759 
2760 		tmp =
2761 		    ddrtbl_getval(_cnf_DDR_PI_REGSET,
2762 				  reg_pi_mr11_data_fx_csx[fspwp][0]);
2763 		send_dbcmd(0x0e840b00 | tmp);
2764 
2765 		tmp =
2766 		    ddrtbl_getval(_cnf_DDR_PI_REGSET,
2767 				  reg_pi_mr12_data_fx_csx[fspwp][0]);
2768 		send_dbcmd(0x0e840c00 | tmp);
2769 
2770 		tmp =
2771 		    ddrtbl_getval(_cnf_DDR_PI_REGSET,
2772 				  reg_pi_mr14_data_fx_csx[fspwp][0]);
2773 		send_dbcmd(0x0e840e00 | tmp);
2774 		/* MR22 */
2775 		send_dbcmd(0x0e841616);
2776 
2777 		/* ZQCAL start */
2778 		send_dbcmd(0x0d84004F);
2779 
2780 		/* ZQLAT */
2781 		send_dbcmd(0x0d840051);
2782 	}
2783 
2784 	/* MR13, fspwp */
2785 	send_dbcmd(0x0e840d08);
2786 }
2787 
2788 /* Training handshake functions */
wait_freqchgreq(uint32_t assert)2789 static inline uint32_t wait_freqchgreq(uint32_t assert)
2790 {
2791 	uint32_t data_l;
2792 	uint32_t count;
2793 	uint32_t ch;
2794 
2795 	count = 100000;
2796 
2797 	/* H3 Ver.1.x cannot see frqchg_req */
2798 	if ((prr_product == PRR_PRODUCT_H3) && (prr_cut <= PRR_PRODUCT_11)) {
2799 		return 0;
2800 	}
2801 
2802 	if (assert) {
2803 		do {
2804 			data_l = 1;
2805 			foreach_vch(ch) {
2806 				data_l &= mmio_read_32(DBSC_DBPDSTAT(ch));
2807 			}
2808 			count = count - 1;
2809 		} while (((data_l & 0x01) != 0x01) & (count != 0));
2810 	} else {
2811 		do {
2812 			data_l = 0;
2813 			foreach_vch(ch) {
2814 				data_l |= mmio_read_32(DBSC_DBPDSTAT(ch));
2815 			}
2816 			count = count - 1;
2817 		} while (((data_l & 0x01) != 0x00) & (count != 0));
2818 	}
2819 
2820 	return (count == 0);
2821 }
2822 
set_freqchgack(uint32_t assert)2823 static inline void set_freqchgack(uint32_t assert)
2824 {
2825 	uint32_t ch;
2826 	uint32_t data_l;
2827 
2828 	if (assert)
2829 		data_l = 0x0CF20000;
2830 	else
2831 		data_l = 0x00000000;
2832 
2833 	foreach_vch(ch)
2834 	    mmio_write_32(DBSC_DBPDCNT2(ch), data_l);
2835 }
2836 
set_dfifrequency(uint32_t freq)2837 static inline void set_dfifrequency(uint32_t freq)
2838 {
2839 	uint32_t ch;
2840 
2841 	if ((prr_product == PRR_PRODUCT_H3) && (prr_cut <= PRR_PRODUCT_11)) {
2842 		foreach_vch(ch)
2843 		    mmio_clrsetbits_32(DBSC_DBPDCNT1(ch), 0x1fU, freq);
2844 	} else {
2845 		foreach_vch(ch) {
2846 			mmio_clrsetbits_32(DBSC_DBDFICNT(ch), 0x1fU << 24,
2847 					   (freq << 24));
2848 		}
2849 	}
2850 	dsb_sev();
2851 }
2852 
pll3_freq(uint32_t on)2853 static uint32_t pll3_freq(uint32_t on)
2854 {
2855 	uint32_t timeout;
2856 
2857 	timeout = wait_freqchgreq(1);
2858 
2859 	if (timeout) {
2860 		return 1;
2861 	}
2862 
2863 	pll3_control(on);
2864 	set_dfifrequency(on);
2865 
2866 	set_freqchgack(1);
2867 	timeout = wait_freqchgreq(0);
2868 	set_freqchgack(0);
2869 
2870 	if (timeout) {
2871 		FATAL_MSG("BL2: Time out[2]\n");
2872 		return 1;
2873 	}
2874 	return 0;
2875 }
2876 
2877 /* update dly */
update_dly(void)2878 static void update_dly(void)
2879 {
2880 	ddr_setval_ach(_reg_SC_PHY_MANUAL_UPDATE, 0x01);
2881 	ddr_setval_ach(_reg_PHY_ADRCTL_MANUAL_UPDATE, 0x01);
2882 }
2883 
2884 /* training by pi */
pi_training_go(void)2885 static uint32_t pi_training_go(void)
2886 {
2887 	uint32_t flag;
2888 	uint32_t data_l;
2889 	uint32_t retry;
2890 	const uint32_t RETRY_MAX = 4096 * 16;
2891 	uint32_t ch;
2892 
2893 	uint32_t mst_ch;
2894 	uint32_t cur_frq;
2895 	uint32_t complete;
2896 	uint32_t frqchg_req;
2897 
2898 	/* pi_start */
2899 	ddr_setval_ach(_reg_PI_START, 0x01);
2900 	foreach_vch(ch)
2901 	    ddr_getval(ch, _reg_PI_INT_STATUS);
2902 
2903 	/* set dfi_phymstr_ack = 1 */
2904 	mmio_write_32(DBSC_DBDFIPMSTRCNF, 0x00000001);
2905 	dsb_sev();
2906 
2907 	/* wait pi_int_status[0] */
2908 	mst_ch = 0;
2909 	flag = 0;
2910 	complete = 0;
2911 	cur_frq = 0;
2912 	retry = RETRY_MAX;
2913 	do {
2914 		frqchg_req = mmio_read_32(DBSC_DBPDSTAT(mst_ch)) & 0x01;
2915 
2916 		/* H3 Ver.1.x cannot see frqchg_req */
2917 		if ((prr_product == PRR_PRODUCT_H3) &&
2918 		    (prr_cut <= PRR_PRODUCT_11)) {
2919 			if ((retry % 4096) == 1) {
2920 				frqchg_req = 1;
2921 			} else {
2922 				frqchg_req = 0;
2923 			}
2924 		}
2925 
2926 		if (frqchg_req) {
2927 			if (cur_frq) {
2928 				/* Low frequency */
2929 				flag = pll3_freq(0);
2930 				cur_frq = 0;
2931 			} else {
2932 				/* High frequency */
2933 				flag = pll3_freq(1);
2934 				cur_frq = 1;
2935 			}
2936 			if (flag)
2937 				break;
2938 		} else {
2939 			if (cur_frq) {
2940 				foreach_vch(ch) {
2941 					if (complete & (1U << ch))
2942 						continue;
2943 					data_l =
2944 					    ddr_getval(ch, _reg_PI_INT_STATUS);
2945 					if (data_l & 0x01) {
2946 						complete |= (1U << ch);
2947 					}
2948 				}
2949 				if (complete == ddr_phyvalid)
2950 					break;
2951 			}
2952 		}
2953 	} while (--retry);
2954 	foreach_vch(ch) {
2955 		/* dummy read */
2956 		data_l = ddr_getval_s(ch, 0, _reg_PHY_CAL_RESULT2_OBS_0);
2957 		data_l = ddr_getval(ch, _reg_PI_INT_STATUS);
2958 		ddr_setval(ch, _reg_PI_INT_ACK, data_l);
2959 	}
2960 	if (ddrphy_regif_chk()) {
2961 		return 0xfd;
2962 	}
2963 	return complete;
2964 }
2965 
2966 /* Initialize DDR */
init_ddr(void)2967 static uint32_t init_ddr(void)
2968 {
2969 	int32_t i;
2970 	uint32_t data_l;
2971 	uint32_t phytrainingok;
2972 	uint32_t ch, slice;
2973 	uint32_t err;
2974 	int16_t adj;
2975 
2976 	MSG_LF(__func__ ":0\n");
2977 
2978 #ifdef DDR_BACKUPMODE
2979 	rcar_dram_get_boot_status(&ddr_backup);
2980 #endif
2981 
2982 	/* unlock phy */
2983 	/* Unlock DDRPHY register(AGAIN) */
2984 	foreach_vch(ch)
2985 	    mmio_write_32(DBSC_DBPDLK(ch), 0x0000A55A);
2986 	dsb_sev();
2987 
2988 	if ((((prr_product == PRR_PRODUCT_H3) &&
2989 	      (prr_cut > PRR_PRODUCT_11)) ||
2990 	     (prr_product == PRR_PRODUCT_M3N) ||
2991 	     (prr_product == PRR_PRODUCT_V3H)) && board_cnf->dbi_en)
2992 		reg_ddrphy_write_a(0x00001010, 0x01000001);
2993 	else
2994 		reg_ddrphy_write_a(0x00001010, 0x00000001);
2995 	/* DBSC register pre-setting */
2996 	dbsc_regset_pre();
2997 
2998 	/* load ddrphy registers */
2999 
3000 	ddrtbl_load();
3001 
3002 	/* configure ddrphy registers */
3003 	ddr_config();
3004 
3005 	/* dfi_reset assert */
3006 	foreach_vch(ch)
3007 	    mmio_write_32(DBSC_DBPDCNT0(ch), 0x01);
3008 	dsb_sev();
3009 
3010 	/* dbsc register set */
3011 	dbsc_regset();
3012 	MSG_LF(__func__ ":1\n");
3013 
3014 	/* dfi_reset negate */
3015 	foreach_vch(ch)
3016 	    mmio_write_32(DBSC_DBPDCNT0(ch), 0x00);
3017 	dsb_sev();
3018 
3019 	/* dfi_init_start (start ddrphy) */
3020 	err = dfi_init_start();
3021 	if (err) {
3022 		return INITDRAM_ERR_I;
3023 	}
3024 	MSG_LF(__func__ ":2\n");
3025 
3026 	/* ddr backupmode end */
3027 #ifdef DDR_BACKUPMODE
3028 	if (ddr_backup) {
3029 		NOTICE("BL2: [WARM_BOOT]\n");
3030 	} else {
3031 		NOTICE("BL2: [COLD_BOOT]\n");
3032 	}
3033 	err = rcar_dram_update_boot_status(ddr_backup);
3034 	if (err) {
3035 		NOTICE("BL2: [BOOT_STATUS_UPDATE_ERROR]\n");
3036 		return INITDRAM_ERR_I;
3037 	}
3038 #endif
3039 	MSG_LF(__func__ ":3\n");
3040 
3041 	/* override term code after dfi_init_complete */
3042 	err = set_term_code();
3043 	if (err) {
3044 		return INITDRAM_ERR_I;
3045 	}
3046 	MSG_LF(__func__ ":4\n");
3047 
3048 	/* rx offset calibration */
3049 	if ((prr_cut > PRR_PRODUCT_11) || (prr_product == PRR_PRODUCT_M3N) ||
3050 	    (prr_product == PRR_PRODUCT_V3H)) {
3051 		err = rx_offset_cal_hw();
3052 	} else {
3053 		err = rx_offset_cal();
3054 	}
3055 	if (err)
3056 		return INITDRAM_ERR_O;
3057 	MSG_LF(__func__ ":5\n");
3058 
3059 	/* Dummy PDE */
3060 	send_dbcmd(0x08840000);
3061 
3062 	/* PDX */
3063 	send_dbcmd(0x08840001);
3064 
3065 	/* check register i/f is alive */
3066 	err = ddrphy_regif_chk();
3067 	if (err) {
3068 		return INITDRAM_ERR_O;
3069 	}
3070 	MSG_LF(__func__ ":6\n");
3071 
3072 	/* phy initialize end */
3073 
3074 	/* setup DDR mode registers */
3075 	/* CMOS MODE */
3076 	change_lpddr4_en(0);
3077 
3078 	/* MRS */
3079 	ddr_register_set();
3080 
3081 	/* Thermal sensor setting */
3082 	/* THCTR Bit6: PONM=0 , Bit0: THSST=1  */
3083 	data_l = (mmio_read_32(THS1_THCTR) & 0xFFFFFFBF) | 0x00000001;
3084 	mmio_write_32(THS1_THCTR, data_l);
3085 
3086 	/* LPDDR4 MODE */
3087 	change_lpddr4_en(1);
3088 
3089 	MSG_LF(__func__ ":7\n");
3090 
3091 	/* mask CS_MAP if RANKx is not found */
3092 	foreach_vch(ch) {
3093 		data_l = ddr_getval(ch, _reg_PI_CS_MAP);
3094 		if (!(ch_have_this_cs[1] & (1U << ch)))
3095 			data_l = data_l & 0x05;
3096 		ddr_setval(ch, _reg_PI_CS_MAP, data_l);
3097 	}
3098 
3099 	/* exec pi_training */
3100 	reg_ddrphy_write_a(ddr_regdef_adr(_reg_PHY_FREQ_SEL_MULTICAST_EN),
3101 			   BIT(ddr_regdef_lsb(_reg_PHY_FREQ_SEL_MULTICAST_EN)));
3102 	ddr_setval_ach_as(_reg_PHY_PER_CS_TRAINING_MULTICAST_EN, 0x00);
3103 
3104 	if ((prr_product == PRR_PRODUCT_H3) && (prr_cut <= PRR_PRODUCT_11)) {
3105 		ddr_setval_ach_as(_reg_PHY_PER_CS_TRAINING_EN, 0x01);
3106 	} else {
3107 		foreach_vch(ch) {
3108 			for (slice = 0; slice < SLICE_CNT; slice++) {
3109 				ddr_setval_s(ch, slice,
3110 					     _reg_PHY_PER_CS_TRAINING_EN,
3111 					     ((ch_have_this_cs[1]) >> ch)
3112 					     & 0x01);
3113 			}
3114 		}
3115 	}
3116 
3117 	phytrainingok = pi_training_go();
3118 
3119 	if (ddr_phyvalid != (phytrainingok & ddr_phyvalid)) {
3120 		return INITDRAM_ERR_T | phytrainingok;
3121 	}
3122 
3123 	MSG_LF(__func__ ":8\n");
3124 
3125 	/* CACS DLY ADJUST */
3126 	data_l = board_cnf->cacs_dly + _f_scale_adj(board_cnf->cacs_dly_adj);
3127 	foreach_vch(ch) {
3128 		for (i = 0; i < _reg_PHY_CLK_CACS_SLAVE_DELAY_X_NUM; i++) {
3129 			adj = _f_scale_adj(board_cnf->ch[ch].cacs_adj[i]);
3130 			ddr_setval(ch, _reg_PHY_CLK_CACS_SLAVE_DELAY_X[i],
3131 				   data_l + adj);
3132 		}
3133 
3134 		if (ddr_phycaslice == 1) {
3135 			for (i = 0; i < 6; i++) {
3136 				adj = _f_scale_adj(board_cnf->ch[ch].cacs_adj
3137 					[i +
3138 					_reg_PHY_CLK_CACS_SLAVE_DELAY_X_NUM]);
3139 				ddr_setval_s(ch, 2,
3140 					     _reg_PHY_CLK_CACS_SLAVE_DELAY_X
3141 					     [i],
3142 					     data_l + adj
3143 				);
3144 			}
3145 		}
3146 	}
3147 
3148 	update_dly();
3149 	MSG_LF(__func__ ":9\n");
3150 
3151 	/* H3 fix rd latency to avoid bug in elasitic buffer */
3152 	if ((prr_product == PRR_PRODUCT_H3) && (prr_cut <= PRR_PRODUCT_11))
3153 		adjust_rddqs_latency();
3154 
3155 	/* Adjust Write path latency */
3156 	if (ddrtbl_getval
3157 	    (_cnf_DDR_PHY_SLICE_REGSET, _reg_PHY_WRITE_PATH_LAT_ADD))
3158 		adjust_wpath_latency();
3159 
3160 	/* RDQLVL Training */
3161 	if (!ddrtbl_getval(_cnf_DDR_PHY_SLICE_REGSET, _reg_PHY_IE_MODE))
3162 		ddr_setval_ach_as(_reg_PHY_IE_MODE, 0x01);
3163 
3164 	err = rdqdm_man();
3165 
3166 	if (!ddrtbl_getval(_cnf_DDR_PHY_SLICE_REGSET, _reg_PHY_IE_MODE))
3167 		ddr_setval_ach_as(_reg_PHY_IE_MODE, 0x00);
3168 
3169 	if (err) {
3170 		return INITDRAM_ERR_T;
3171 	}
3172 	update_dly();
3173 	MSG_LF(__func__ ":10\n");
3174 
3175 	/* WDQLVL Training */
3176 	err = wdqdm_man();
3177 	if (err) {
3178 		return INITDRAM_ERR_T;
3179 	}
3180 	update_dly();
3181 	MSG_LF(__func__ ":11\n");
3182 
3183 	/* training complete, setup DBSC */
3184 	if (((prr_product == PRR_PRODUCT_H3) && (prr_cut > PRR_PRODUCT_11)) ||
3185 	    (prr_product == PRR_PRODUCT_M3N) ||
3186 	    (prr_product == PRR_PRODUCT_V3H)) {
3187 		ddr_setval_ach_as(_reg_PHY_DFI40_POLARITY, 0x00);
3188 		ddr_setval_ach(_reg_PI_DFI40_POLARITY, 0x00);
3189 	}
3190 
3191 	dbsc_regset_post();
3192 	MSG_LF(__func__ ":12\n");
3193 
3194 	return phytrainingok;
3195 }
3196 
3197 /* SW LEVELING COMMON */
swlvl1(uint32_t ddr_csn,uint32_t reg_cs,uint32_t reg_kick)3198 static uint32_t swlvl1(uint32_t ddr_csn, uint32_t reg_cs, uint32_t reg_kick)
3199 {
3200 	uint32_t ch;
3201 	uint32_t data_l;
3202 	uint32_t retry;
3203 	uint32_t waiting;
3204 	uint32_t err;
3205 
3206 	const uint32_t RETRY_MAX = 0x1000;
3207 
3208 	err = 0;
3209 	/* set EXIT -> OP_DONE is cleared */
3210 	ddr_setval_ach(_reg_PI_SWLVL_EXIT, 0x01);
3211 
3212 	/* kick */
3213 	foreach_vch(ch) {
3214 		if (ch_have_this_cs[ddr_csn % 2] & (1U << ch)) {
3215 			ddr_setval(ch, reg_cs, ddr_csn);
3216 			ddr_setval(ch, reg_kick, 0x01);
3217 		}
3218 	}
3219 	foreach_vch(ch) {
3220 		/*PREPARE ADDR REGISTER (for SWLVL_OP_DONE) */
3221 		ddr_getval(ch, _reg_PI_SWLVL_OP_DONE);
3222 	}
3223 	waiting = ch_have_this_cs[ddr_csn % 2];
3224 	dsb_sev();
3225 	retry = RETRY_MAX;
3226 	do {
3227 		foreach_vch(ch) {
3228 			if (!(waiting & (1U << ch)))
3229 				continue;
3230 			data_l = ddr_getval(ch, _reg_PI_SWLVL_OP_DONE);
3231 			if (data_l & 0x01)
3232 				waiting &= ~(1U << ch);
3233 		}
3234 		retry--;
3235 	} while (waiting && (retry > 0));
3236 	if (retry == 0) {
3237 		err = 1;
3238 	}
3239 
3240 	dsb_sev();
3241 	/* set EXIT -> OP_DONE is cleared */
3242 	ddr_setval_ach(_reg_PI_SWLVL_EXIT, 0x01);
3243 	dsb_sev();
3244 
3245 	return err;
3246 }
3247 
3248 /* WDQ TRAINING */
3249 #ifndef DDR_FAST_INIT
wdqdm_clr1(uint32_t ch,uint32_t ddr_csn)3250 static void wdqdm_clr1(uint32_t ch, uint32_t ddr_csn)
3251 {
3252 	int32_t i, k;
3253 	uint32_t cs, slice;
3254 	uint32_t data_l;
3255 
3256 	/* clr of training results buffer */
3257 	cs = ddr_csn % 2;
3258 	data_l = board_cnf->dqdm_dly_w;
3259 	for (slice = 0; slice < SLICE_CNT; slice++) {
3260 		k = (board_cnf->ch[ch].dqs_swap >> (4 * slice)) & 0x0f;
3261 		if (((k >= 2) && (ddr_csn < 2)) || ((k < 2) && (ddr_csn >= 2)))
3262 			continue;
3263 
3264 		for (i = 0; i <= 8; i++) {
3265 			if (ch_have_this_cs[CS_CNT - 1 - cs] & (1U << ch))
3266 				wdqdm_dly[ch][cs][slice][i] =
3267 				    wdqdm_dly[ch][CS_CNT - 1 - cs][slice][i];
3268 			else
3269 				wdqdm_dly[ch][cs][slice][i] = data_l;
3270 			wdqdm_le[ch][cs][slice][i] = 0;
3271 			wdqdm_te[ch][cs][slice][i] = 0;
3272 		}
3273 		wdqdm_st[ch][cs][slice] = 0;
3274 		wdqdm_win[ch][cs][slice] = 0;
3275 	}
3276 }
3277 
wdqdm_ana1(uint32_t ch,uint32_t ddr_csn)3278 static uint32_t wdqdm_ana1(uint32_t ch, uint32_t ddr_csn)
3279 {
3280 	int32_t i, k;
3281 	uint32_t cs, slice;
3282 	uint32_t data_l;
3283 	uint32_t err;
3284 	const uint32_t _par_WDQLVL_RETRY_THRES = 0x7c0;
3285 
3286 	int32_t min_win;
3287 	int32_t win;
3288 	int8_t _adj;
3289 	int16_t adj;
3290 	uint32_t dq;
3291 
3292 	/* analysis of training results */
3293 	err = 0;
3294 	for (slice = 0; slice < SLICE_CNT; slice += 1) {
3295 		k = (board_cnf->ch[ch].dqs_swap >> (4 * slice)) & 0x0f;
3296 		if (((k >= 2) && (ddr_csn < 2)) || ((k < 2) && (ddr_csn >= 2)))
3297 			continue;
3298 
3299 		cs = ddr_csn % 2;
3300 		ddr_setval_s(ch, slice, _reg_PHY_PER_CS_TRAINING_INDEX, cs);
3301 		for (i = 0; i < 9; i++) {
3302 			dq = slice * 8 + i;
3303 			if (i == 8)
3304 				_adj = board_cnf->ch[ch].dm_adj_w[slice];
3305 			else
3306 				_adj = board_cnf->ch[ch].dq_adj_w[dq];
3307 			adj = _f_scale_adj(_adj);
3308 
3309 			data_l =
3310 			    ddr_getval_s(ch, slice,
3311 					 _reg_PHY_CLK_WRX_SLAVE_DELAY[i]) + adj;
3312 			ddr_setval_s(ch, slice, _reg_PHY_CLK_WRX_SLAVE_DELAY[i],
3313 				     data_l);
3314 			wdqdm_dly[ch][cs][slice][i] = data_l;
3315 		}
3316 		ddr_setval_s(ch, slice, _reg_PHY_PER_CS_TRAINING_EN, 0x00);
3317 		data_l = ddr_getval_s(ch, slice, _reg_PHY_WDQLVL_STATUS_OBS);
3318 		wdqdm_st[ch][cs][slice] = data_l;
3319 		min_win = INT_LEAST32_MAX;
3320 		for (i = 0; i <= 8; i++) {
3321 			ddr_setval_s(ch, slice, _reg_PHY_WDQLVL_DQDM_OBS_SELECT,
3322 				     i);
3323 
3324 			data_l =
3325 			    ddr_getval_s(ch, slice,
3326 					 _reg_PHY_WDQLVL_DQDM_TE_DLY_OBS);
3327 			wdqdm_te[ch][cs][slice][i] = data_l;
3328 			data_l =
3329 			    ddr_getval_s(ch, slice,
3330 					 _reg_PHY_WDQLVL_DQDM_LE_DLY_OBS);
3331 			wdqdm_le[ch][cs][slice][i] = data_l;
3332 			win =
3333 			    (int32_t)wdqdm_te[ch][cs][slice][i] -
3334 			    wdqdm_le[ch][cs][slice][i];
3335 			if (min_win > win)
3336 				min_win = win;
3337 			if (data_l >= _par_WDQLVL_RETRY_THRES)
3338 				err = 2;
3339 		}
3340 		wdqdm_win[ch][cs][slice] = min_win;
3341 		if ((prr_product == PRR_PRODUCT_H3) &&
3342 		    (prr_cut <= PRR_PRODUCT_11)) {
3343 			ddr_setval_s(ch, slice, _reg_PHY_PER_CS_TRAINING_EN,
3344 				     0x01);
3345 		} else {
3346 			ddr_setval_s(ch, slice, _reg_PHY_PER_CS_TRAINING_EN,
3347 				     ((ch_have_this_cs[1]) >> ch) & 0x01);
3348 		}
3349 	}
3350 	return err;
3351 }
3352 #endif/* DDR_FAST_INIT */
3353 
wdqdm_cp(uint32_t ddr_csn,uint32_t restore)3354 static void wdqdm_cp(uint32_t ddr_csn, uint32_t restore)
3355 {
3356 	uint32_t i;
3357 	uint32_t ch, slice;
3358 	uint32_t tgt_cs, src_cs;
3359 	uint32_t tmp_r;
3360 
3361 	/* copy of training results */
3362 	foreach_vch(ch) {
3363 		for (tgt_cs = 0; tgt_cs < CS_CNT; tgt_cs++) {
3364 			for (slice = 0; slice < SLICE_CNT; slice++) {
3365 				ddr_setval_s(ch, slice,
3366 					     _reg_PHY_PER_CS_TRAINING_INDEX,
3367 					     tgt_cs);
3368 				src_cs = ddr_csn % 2;
3369 				if (!(ch_have_this_cs[1] & (1U << ch)))
3370 					src_cs = 0;
3371 				for (i = 0; i <= 4; i += 4) {
3372 					if (restore)
3373 						tmp_r =
3374 						    rdqdm_dly[ch][tgt_cs][slice]
3375 						    [i];
3376 					else
3377 						tmp_r =
3378 						    rdqdm_dly[ch][src_cs][slice]
3379 						    [i];
3380 
3381 					ddr_setval_s(ch, slice,
3382 						     _reg_PHY_RDDQS_X_RISE_SLAVE_DELAY
3383 						     [i], tmp_r);
3384 				}
3385 			}
3386 		}
3387 	}
3388 }
3389 
wdqdm_man1(void)3390 static uint32_t wdqdm_man1(void)
3391 {
3392 	int32_t k;
3393 	uint32_t ch, cs, slice;
3394 	uint32_t ddr_csn;
3395 	uint32_t data_l;
3396 	uint32_t err;
3397 	uint32_t high_dq[DRAM_CH_CNT];
3398 	uint32_t mr14_csab0_bak[DRAM_CH_CNT];
3399 #ifndef DDR_FAST_INIT
3400 	uint32_t err_flg;
3401 #endif/* DDR_FAST_INIT */
3402 
3403 	/* manual execution of training */
3404 	if ((prr_product == PRR_PRODUCT_H3) && (prr_cut <= PRR_PRODUCT_11)) {
3405 		foreach_vch(ch) {
3406 			high_dq[ch] = 0;
3407 			for (slice = 0; slice < SLICE_CNT; slice++) {
3408 				k = (board_cnf->ch[ch].dqs_swap >>
3409 				    (4 * slice)) & 0x0f;
3410 				if (k >= 2)
3411 					high_dq[ch] |= (1U << slice);
3412 			}
3413 			ddr_setval(ch, _reg_PI_16BIT_DRAM_CONNECT, 0x00);
3414 		}
3415 	}
3416 	err = 0;
3417 	/* CLEAR PREV RESULT */
3418 	for (cs = 0; cs < CS_CNT; cs++) {
3419 		ddr_setval_ach_as(_reg_PHY_PER_CS_TRAINING_INDEX, cs);
3420 		if (((prr_product == PRR_PRODUCT_H3) &&
3421 		     (prr_cut > PRR_PRODUCT_11)) ||
3422 		    (prr_product == PRR_PRODUCT_M3N) ||
3423 		    (prr_product == PRR_PRODUCT_V3H)) {
3424 			ddr_setval_ach_as(_reg_SC_PHY_WDQLVL_CLR_PREV_RESULTS,
3425 					  0x01);
3426 		} else {
3427 			ddr_setval_ach_as(_reg_PHY_WDQLVL_CLR_PREV_RESULTS,
3428 					  0x01);
3429 		}
3430 	}
3431 	ddrphy_regif_idle();
3432 
3433 #ifndef DDR_FAST_INIT
3434 	err_flg = 0;
3435 #endif/* DDR_FAST_INIT */
3436 	for (ddr_csn = 0; ddr_csn < CSAB_CNT; ddr_csn++) {
3437 		if ((prr_product == PRR_PRODUCT_H3) &&
3438 		    (prr_cut <= PRR_PRODUCT_11)) {
3439 			foreach_vch(ch) {
3440 				data_l = mmio_read_32(DBSC_DBDFICNT(ch));
3441 				data_l &= ~(0x00ffU << 16);
3442 
3443 				if (ddr_csn >= 2)
3444 					k = (high_dq[ch] ^ 0x0f);
3445 				else
3446 					k = high_dq[ch];
3447 				data_l |= (k << 16);
3448 				mmio_write_32(DBSC_DBDFICNT(ch), data_l);
3449 				ddr_setval(ch, _reg_PI_WDQLVL_RESP_MASK, k);
3450 			}
3451 		}
3452 		if (((prr_product == PRR_PRODUCT_H3) &&
3453 		     (prr_cut <= PRR_PRODUCT_11)) ||
3454 		    ((prr_product == PRR_PRODUCT_M3) &&
3455 		     (prr_cut == PRR_PRODUCT_10))) {
3456 			wdqdm_cp(ddr_csn, 0);
3457 		}
3458 
3459 		foreach_vch(ch) {
3460 			data_l =
3461 			    ddr_getval(ch,
3462 				       reg_pi_mr14_data_fx_csx[1][ddr_csn]);
3463 			ddr_setval(ch, reg_pi_mr14_data_fx_csx[1][0], data_l);
3464 		}
3465 
3466 		/* KICK WDQLVL */
3467 		err = swlvl1(ddr_csn, _reg_PI_WDQLVL_CS, _reg_PI_WDQLVL_REQ);
3468 		if (err)
3469 			goto err_exit;
3470 
3471 		if (ddr_csn == 0)
3472 			foreach_vch(ch) {
3473 			mr14_csab0_bak[ch] =
3474 			    ddr_getval(ch, reg_pi_mr14_data_fx_csx[1][0]);
3475 		} else
3476 			foreach_vch(ch) {
3477 			ddr_setval(ch, reg_pi_mr14_data_fx_csx[1][0],
3478 				   mr14_csab0_bak[ch]);
3479 			}
3480 #ifndef DDR_FAST_INIT
3481 		foreach_vch(ch) {
3482 			if (!(ch_have_this_cs[ddr_csn % 2] & (1U << ch))) {
3483 				wdqdm_clr1(ch, ddr_csn);
3484 				continue;
3485 			}
3486 			err = wdqdm_ana1(ch, ddr_csn);
3487 			if (err)
3488 				err_flg |= (1U << (ddr_csn * 4 + ch));
3489 			ddrphy_regif_idle();
3490 		}
3491 #endif/* DDR_FAST_INIT */
3492 	}
3493 err_exit:
3494 #ifndef DDR_FAST_INIT
3495 	err |= err_flg;
3496 #endif/* DDR_FAST_INIT */
3497 	if ((prr_product == PRR_PRODUCT_H3) && (prr_cut <= PRR_PRODUCT_11)) {
3498 		ddr_setval_ach(_reg_PI_16BIT_DRAM_CONNECT, 0x01);
3499 		foreach_vch(ch) {
3500 			data_l = mmio_read_32(DBSC_DBDFICNT(ch));
3501 			data_l &= ~(0x00ffU << 16);
3502 			mmio_write_32(DBSC_DBDFICNT(ch), data_l);
3503 			ddr_setval(ch, _reg_PI_WDQLVL_RESP_MASK, 0x00);
3504 		}
3505 	}
3506 	return err;
3507 }
3508 
wdqdm_man(void)3509 static uint32_t wdqdm_man(void)
3510 {
3511 	uint32_t err, retry_cnt;
3512 	const uint32_t retry_max = 0x10;
3513 	uint32_t datal, ch, ddr_csn, mr14_bkup[4][4];
3514 
3515 	datal = RL + js2[js2_tdqsck] + (16 / 2) + 1 - WL + 2 + 2 + 19;
3516 	if ((mmio_read_32(DBSC_DBTR(11)) & 0xFF) > datal)
3517 		datal = mmio_read_32(DBSC_DBTR(11)) & 0xFF;
3518 	ddr_setval_ach(_reg_PI_TDFI_WDQLVL_RW, datal);
3519 
3520 	if (((prr_product == PRR_PRODUCT_H3) && (prr_cut > PRR_PRODUCT_11)) ||
3521 	    (prr_product == PRR_PRODUCT_M3N) ||
3522 	    (prr_product == PRR_PRODUCT_V3H)) {
3523 		ddr_setval_ach(_reg_PI_TDFI_WDQLVL_WR_F0,
3524 			       (mmio_read_32(DBSC_DBTR(12)) & 0xFF) + 10);
3525 		ddr_setval_ach(_reg_PI_TDFI_WDQLVL_WR_F1,
3526 			       (mmio_read_32(DBSC_DBTR(12)) & 0xFF) + 10);
3527 	} else {
3528 		ddr_setval_ach(_reg_PI_TDFI_WDQLVL_WR,
3529 			       (mmio_read_32(DBSC_DBTR(12)) & 0xFF) + 10);
3530 	}
3531 	ddr_setval_ach(_reg_PI_TRFC_F0, mmio_read_32(DBSC_DBTR(13)) & 0x1FF);
3532 	ddr_setval_ach(_reg_PI_TRFC_F1, mmio_read_32(DBSC_DBTR(13)) & 0x1FF);
3533 
3534 	retry_cnt = 0;
3535 	err = 0;
3536 	do {
3537 		if ((prr_product == PRR_PRODUCT_H3) &&
3538 		    (prr_cut <= PRR_PRODUCT_11)) {
3539 			err = wdqdm_man1();
3540 		} else {
3541 			ddr_setval_ach(_reg_PI_WDQLVL_VREF_EN, 0x01);
3542 			ddr_setval_ach(_reg_PI_WDQLVL_VREF_NORMAL_STEPSIZE,
3543 				       0x01);
3544 			if ((prr_product == PRR_PRODUCT_M3N) ||
3545 			    (prr_product == PRR_PRODUCT_V3H)) {
3546 				ddr_setval_ach(_reg_PI_WDQLVL_VREF_DELTA_F1,
3547 					       0x0C);
3548 			} else {
3549 				ddr_setval_ach(_reg_PI_WDQLVL_VREF_DELTA, 0x0C);
3550 			}
3551 			dsb_sev();
3552 			err = wdqdm_man1();
3553 			foreach_vch(ch) {
3554 				for (ddr_csn = 0; ddr_csn < CSAB_CNT; ddr_csn++) {
3555 					mr14_bkup[ch][ddr_csn] =
3556 					    ddr_getval(ch,
3557 						       reg_pi_mr14_data_fx_csx
3558 						       [1][ddr_csn]);
3559 					dsb_sev();
3560 				}
3561 			}
3562 
3563 			if ((prr_product == PRR_PRODUCT_M3N) ||
3564 			    (prr_product == PRR_PRODUCT_V3H)) {
3565 				ddr_setval_ach(_reg_PI_WDQLVL_VREF_DELTA_F1,
3566 					       0x04);
3567 			} else {
3568 				ddr_setval_ach(_reg_PI_WDQLVL_VREF_DELTA, 0x04);
3569 			}
3570 			pvtcode_update();
3571 			err = wdqdm_man1();
3572 			foreach_vch(ch) {
3573 				for (ddr_csn = 0; ddr_csn < CSAB_CNT; ddr_csn++) {
3574 					mr14_bkup[ch][ddr_csn] =
3575 					    (mr14_bkup[ch][ddr_csn] +
3576 					     ddr_getval(ch,
3577 							reg_pi_mr14_data_fx_csx
3578 							[1][ddr_csn])) / 2;
3579 					ddr_setval(ch,
3580 						   reg_pi_mr14_data_fx_csx[1]
3581 						   [ddr_csn],
3582 						   mr14_bkup[ch][ddr_csn]);
3583 				}
3584 			}
3585 
3586 			ddr_setval_ach(_reg_PI_WDQLVL_VREF_NORMAL_STEPSIZE,
3587 				       0x00);
3588 			if ((prr_product == PRR_PRODUCT_M3N) ||
3589 			    (prr_product == PRR_PRODUCT_V3H)) {
3590 				ddr_setval_ach(_reg_PI_WDQLVL_VREF_DELTA_F1,
3591 					       0x00);
3592 				ddr_setval_ach
3593 				    (_reg_PI_WDQLVL_VREF_INITIAL_START_POINT_F1,
3594 				     0x00);
3595 				ddr_setval_ach
3596 				    (_reg_PI_WDQLVL_VREF_INITIAL_STOP_POINT_F1,
3597 				     0x00);
3598 			} else {
3599 				ddr_setval_ach(_reg_PI_WDQLVL_VREF_DELTA, 0x00);
3600 				ddr_setval_ach
3601 				    (_reg_PI_WDQLVL_VREF_INITIAL_START_POINT,
3602 				     0x00);
3603 				ddr_setval_ach
3604 				    (_reg_PI_WDQLVL_VREF_INITIAL_STOP_POINT,
3605 				     0x00);
3606 			}
3607 			ddr_setval_ach(_reg_PI_WDQLVL_VREF_INITIAL_STEPSIZE,
3608 				       0x00);
3609 
3610 			pvtcode_update2();
3611 			err = wdqdm_man1();
3612 			ddr_setval_ach(_reg_PI_WDQLVL_VREF_EN, 0x00);
3613 		}
3614 	} while (err && (++retry_cnt < retry_max));
3615 
3616 	if (((prr_product == PRR_PRODUCT_H3) && (prr_cut <= PRR_PRODUCT_11)) ||
3617 	    ((prr_product == PRR_PRODUCT_M3) && (prr_cut <= PRR_PRODUCT_10))) {
3618 		wdqdm_cp(0, 1);
3619 	}
3620 
3621 	return (retry_cnt >= retry_max);
3622 }
3623 
3624 /* RDQ TRAINING */
3625 #ifndef DDR_FAST_INIT
rdqdm_clr1(uint32_t ch,uint32_t ddr_csn)3626 static void rdqdm_clr1(uint32_t ch, uint32_t ddr_csn)
3627 {
3628 	int32_t i, k;
3629 	uint32_t cs, slice;
3630 	uint32_t data_l;
3631 
3632 	/* clr of training results buffer */
3633 	cs = ddr_csn % 2;
3634 	data_l = board_cnf->dqdm_dly_r;
3635 	for (slice = 0; slice < SLICE_CNT; slice++) {
3636 		k = (board_cnf->ch[ch].dqs_swap >> (4 * slice)) & 0x0f;
3637 		if (((k >= 2) && (ddr_csn < 2)) || ((k < 2) && (ddr_csn >= 2)))
3638 			continue;
3639 
3640 		for (i = 0; i <= 8; i++) {
3641 			if (ch_have_this_cs[CS_CNT - 1 - cs] & (1U << ch)) {
3642 				rdqdm_dly[ch][cs][slice][i] =
3643 				    rdqdm_dly[ch][CS_CNT - 1 - cs][slice][i];
3644 				rdqdm_dly[ch][cs][slice + SLICE_CNT][i] =
3645 				    rdqdm_dly[ch][CS_CNT - 1 - cs][slice +
3646 								   SLICE_CNT]
3647 				    [i];
3648 			} else {
3649 				rdqdm_dly[ch][cs][slice][i] = data_l;
3650 				rdqdm_dly[ch][cs][slice + SLICE_CNT][i] =
3651 					data_l;
3652 			}
3653 			rdqdm_le[ch][cs][slice][i] = 0;
3654 			rdqdm_le[ch][cs][slice + SLICE_CNT][i] = 0;
3655 			rdqdm_te[ch][cs][slice][i] = 0;
3656 			rdqdm_te[ch][cs][slice + SLICE_CNT][i] = 0;
3657 			rdqdm_nw[ch][cs][slice][i] = 0;
3658 			rdqdm_nw[ch][cs][slice + SLICE_CNT][i] = 0;
3659 		}
3660 		rdqdm_st[ch][cs][slice] = 0;
3661 		rdqdm_win[ch][cs][slice] = 0;
3662 	}
3663 }
3664 
rdqdm_ana1(uint32_t ch,uint32_t ddr_csn)3665 static uint32_t rdqdm_ana1(uint32_t ch, uint32_t ddr_csn)
3666 {
3667 	int32_t i, k;
3668 	uint32_t cs, slice;
3669 	uint32_t data_l;
3670 	uint32_t err;
3671 	int8_t _adj;
3672 	int16_t adj;
3673 	uint32_t dq;
3674 	int32_t min_win;
3675 	int32_t win;
3676 	uint32_t rdq_status_obs_select;
3677 
3678 	/* analysis of training results */
3679 	err = 0;
3680 	for (slice = 0; slice < SLICE_CNT; slice++) {
3681 		k = (board_cnf->ch[ch].dqs_swap >> (4 * slice)) & 0x0f;
3682 		if (((k >= 2) && (ddr_csn < 2)) || ((k < 2) && (ddr_csn >= 2)))
3683 			continue;
3684 
3685 		cs = ddr_csn % 2;
3686 		ddr_setval_s(ch, slice, _reg_PHY_PER_CS_TRAINING_INDEX, cs);
3687 		ddrphy_regif_idle();
3688 
3689 		ddr_getval_s(ch, slice, _reg_PHY_PER_CS_TRAINING_INDEX);
3690 		ddrphy_regif_idle();
3691 
3692 		for (i = 0; i <= 8; i++) {
3693 			dq = slice * 8 + i;
3694 			if (i == 8)
3695 				_adj = board_cnf->ch[ch].dm_adj_r[slice];
3696 			else
3697 				_adj = board_cnf->ch[ch].dq_adj_r[dq];
3698 
3699 			adj = _f_scale_adj(_adj);
3700 
3701 			data_l =
3702 			    ddr_getval_s(ch, slice,
3703 					 _reg_PHY_RDDQS_X_RISE_SLAVE_DELAY[i]) +
3704 			    adj;
3705 			ddr_setval_s(ch, slice,
3706 				     _reg_PHY_RDDQS_X_RISE_SLAVE_DELAY[i],
3707 				     data_l);
3708 			rdqdm_dly[ch][cs][slice][i] = data_l;
3709 
3710 			data_l =
3711 			    ddr_getval_s(ch, slice,
3712 					 _reg_PHY_RDDQS_X_FALL_SLAVE_DELAY[i]) +
3713 			    adj;
3714 			ddr_setval_s(ch, slice,
3715 				     _reg_PHY_RDDQS_X_FALL_SLAVE_DELAY[i],
3716 				     data_l);
3717 			rdqdm_dly[ch][cs][slice + SLICE_CNT][i] = data_l;
3718 		}
3719 		min_win = INT_LEAST32_MAX;
3720 		for (i = 0; i <= 8; i++) {
3721 			data_l =
3722 			    ddr_getval_s(ch, slice, _reg_PHY_RDLVL_STATUS_OBS);
3723 			rdqdm_st[ch][cs][slice] = data_l;
3724 			rdqdm_st[ch][cs][slice + SLICE_CNT] = data_l;
3725 			/* k : rise/fall */
3726 			for (k = 0; k < 2; k++) {
3727 				if (i == 8) {
3728 					rdq_status_obs_select = 16 + 8 * k;
3729 				} else {
3730 					rdq_status_obs_select = i + k * 8;
3731 				}
3732 				ddr_setval_s(ch, slice,
3733 					     _reg_PHY_RDLVL_RDDQS_DQ_OBS_SELECT,
3734 					     rdq_status_obs_select);
3735 
3736 				data_l =
3737 				    ddr_getval_s(ch, slice,
3738 						 _reg_PHY_RDLVL_RDDQS_DQ_LE_DLY_OBS);
3739 				rdqdm_le[ch][cs][slice + SLICE_CNT * k][i] =
3740 				    data_l;
3741 
3742 				data_l =
3743 				    ddr_getval_s(ch, slice,
3744 						 _reg_PHY_RDLVL_RDDQS_DQ_TE_DLY_OBS);
3745 				rdqdm_te[ch][cs][slice + SLICE_CNT * k][i] =
3746 				    data_l;
3747 
3748 				data_l =
3749 				    ddr_getval_s(ch, slice,
3750 						 _reg_PHY_RDLVL_RDDQS_DQ_NUM_WINDOWS_OBS);
3751 				rdqdm_nw[ch][cs][slice + SLICE_CNT * k][i] =
3752 				    data_l;
3753 
3754 				win =
3755 				    (int32_t)rdqdm_te[ch][cs][slice +
3756 							      SLICE_CNT *
3757 							      k][i] -
3758 				    rdqdm_le[ch][cs][slice + SLICE_CNT * k][i];
3759 				if (i != 8) {
3760 					if (min_win > win)
3761 						min_win = win;
3762 				}
3763 			}
3764 		}
3765 		rdqdm_win[ch][cs][slice] = min_win;
3766 		if (min_win <= 0) {
3767 			err = 2;
3768 		}
3769 	}
3770 	return err;
3771 }
3772 #endif/* DDR_FAST_INIT */
3773 
rdqdm_man1(void)3774 static uint32_t rdqdm_man1(void)
3775 {
3776 	uint32_t ch;
3777 	uint32_t ddr_csn;
3778 #ifdef DDR_FAST_INIT
3779 	uint32_t slice;
3780 	uint32_t i, adj, data_l;
3781 #endif/* DDR_FAST_INIT */
3782 	uint32_t err;
3783 
3784 	/* manual execution of training */
3785 	err = 0;
3786 
3787 	for (ddr_csn = 0; ddr_csn < CSAB_CNT; ddr_csn++) {
3788 		/* KICK RDQLVL */
3789 		err = swlvl1(ddr_csn, _reg_PI_RDLVL_CS, _reg_PI_RDLVL_REQ);
3790 		if (err)
3791 			goto err_exit;
3792 #ifndef DDR_FAST_INIT
3793 		foreach_vch(ch) {
3794 			if (!(ch_have_this_cs[ddr_csn % 2] & (1U << ch))) {
3795 				rdqdm_clr1(ch, ddr_csn);
3796 				ddrphy_regif_idle();
3797 				continue;
3798 			}
3799 			err = rdqdm_ana1(ch, ddr_csn);
3800 			ddrphy_regif_idle();
3801 			if (err)
3802 				goto err_exit;
3803 		}
3804 #else/* DDR_FAST_INIT */
3805 		foreach_vch(ch) {
3806 			if (ch_have_this_cs[ddr_csn] & (1U << ch)) {
3807 				for (slice = 0; slice < SLICE_CNT; slice++) {
3808 					if (ddr_getval_s(ch, slice,
3809 							 _reg_PHY_RDLVL_STATUS_OBS) !=
3810 					    0x0D00FFFF) {
3811 						err = (1U << ch) |
3812 							(0x10U << slice);
3813 						goto err_exit;
3814 					}
3815 				}
3816 			}
3817 			if (((prr_product == PRR_PRODUCT_H3) &&
3818 			     (prr_cut <= PRR_PRODUCT_11)) ||
3819 			    ((prr_product == PRR_PRODUCT_M3) &&
3820 			     (prr_cut <= PRR_PRODUCT_10))) {
3821 				for (slice = 0; slice < SLICE_CNT; slice++) {
3822 					for (i = 0; i <= 8; i++) {
3823 						if (i == 8)
3824 							adj = _f_scale_adj(board_cnf->ch[ch].dm_adj_r[slice]);
3825 						else
3826 							adj = _f_scale_adj(board_cnf->ch[ch].dq_adj_r[slice * 8 + i]);
3827 						ddr_setval_s(ch, slice, _reg_PHY_PER_CS_TRAINING_INDEX, ddr_csn);
3828 						data_l = ddr_getval_s(ch, slice, _reg_PHY_RDDQS_X_RISE_SLAVE_DELAY[i]) + adj;
3829 						ddr_setval_s(ch, slice, _reg_PHY_RDDQS_X_RISE_SLAVE_DELAY[i], data_l);
3830 						rdqdm_dly[ch][ddr_csn][slice][i] = data_l;
3831 						rdqdm_dly[ch][ddr_csn | 1][slice][i] = data_l;
3832 
3833 						data_l = ddr_getval_s(ch, slice, _reg_PHY_RDDQS_X_FALL_SLAVE_DELAY[i]) + adj;
3834 						ddr_setval_s(ch, slice, _reg_PHY_RDDQS_X_FALL_SLAVE_DELAY[i], data_l);
3835 						rdqdm_dly[ch][ddr_csn][slice + SLICE_CNT][i] = data_l;
3836 						rdqdm_dly[ch][ddr_csn | 1][slice + SLICE_CNT][i] = data_l;
3837 					}
3838 				}
3839 			}
3840 		}
3841 		ddrphy_regif_idle();
3842 
3843 #endif/* DDR_FAST_INIT */
3844 	}
3845 
3846 err_exit:
3847 	return err;
3848 }
3849 
rdqdm_man(void)3850 static uint32_t rdqdm_man(void)
3851 {
3852 	uint32_t err, retry_cnt;
3853 	const uint32_t retry_max = 0x01;
3854 
3855 	ddr_setval_ach_as(_reg_PHY_DQ_TSEL_ENABLE,
3856 			  0x00000004 | ddrtbl_getval(_cnf_DDR_PHY_SLICE_REGSET,
3857 						     _reg_PHY_DQ_TSEL_ENABLE));
3858 	ddr_setval_ach_as(_reg_PHY_DQS_TSEL_ENABLE,
3859 			  0x00000004 | ddrtbl_getval(_cnf_DDR_PHY_SLICE_REGSET,
3860 						     _reg_PHY_DQS_TSEL_ENABLE));
3861 	ddr_setval_ach_as(_reg_PHY_DQ_TSEL_SELECT,
3862 			  0xFF0FFFFF & ddrtbl_getval(_cnf_DDR_PHY_SLICE_REGSET,
3863 						     _reg_PHY_DQ_TSEL_SELECT));
3864 	ddr_setval_ach_as(_reg_PHY_DQS_TSEL_SELECT,
3865 			  0xFF0FFFFF & ddrtbl_getval(_cnf_DDR_PHY_SLICE_REGSET,
3866 						     _reg_PHY_DQS_TSEL_SELECT));
3867 
3868 	retry_cnt = 0;
3869 	do {
3870 		err = rdqdm_man1();
3871 		ddrphy_regif_idle();
3872 	} while (err && (++retry_cnt < retry_max));
3873 	ddr_setval_ach_as(_reg_PHY_DQ_TSEL_ENABLE,
3874 			  ddrtbl_getval(_cnf_DDR_PHY_SLICE_REGSET,
3875 					_reg_PHY_DQ_TSEL_ENABLE));
3876 	ddr_setval_ach_as(_reg_PHY_DQS_TSEL_ENABLE,
3877 			  ddrtbl_getval(_cnf_DDR_PHY_SLICE_REGSET,
3878 					_reg_PHY_DQS_TSEL_ENABLE));
3879 	ddr_setval_ach_as(_reg_PHY_DQ_TSEL_SELECT,
3880 			  ddrtbl_getval(_cnf_DDR_PHY_SLICE_REGSET,
3881 					_reg_PHY_DQ_TSEL_SELECT));
3882 	ddr_setval_ach_as(_reg_PHY_DQS_TSEL_SELECT,
3883 			  ddrtbl_getval(_cnf_DDR_PHY_SLICE_REGSET,
3884 					_reg_PHY_DQS_TSEL_SELECT));
3885 
3886 	return (retry_cnt >= retry_max);
3887 }
3888 
3889 /* rx offset calibration */
_find_change(uint64_t val,uint32_t dir)3890 static int32_t _find_change(uint64_t val, uint32_t dir)
3891 {
3892 	int32_t i;
3893 	uint32_t startval;
3894 	uint32_t curval;
3895 	const int32_t VAL_END = 0x3f;
3896 
3897 	if (dir == 0) {
3898 		startval = (val & 0x01);
3899 		for (i = 1; i <= VAL_END; i++) {
3900 			curval = (val >> i) & 0x01;
3901 			if (curval != startval)
3902 				return i;
3903 		}
3904 		return VAL_END;
3905 	}
3906 
3907 	startval = (val >> dir) & 0x01;
3908 	for (i = dir - 1; i >= 0; i--) {
3909 		curval = (val >> i) & 0x01;
3910 		if (curval != startval)
3911 			return i;
3912 	}
3913 	return 0;
3914 }
3915 
_rx_offset_cal_updn(uint32_t code)3916 static uint32_t _rx_offset_cal_updn(uint32_t code)
3917 {
3918 	const uint32_t CODE_MAX = 0x40;
3919 	uint32_t tmp;
3920 
3921 	if ((prr_product == PRR_PRODUCT_H3) && (prr_cut <= PRR_PRODUCT_11)) {
3922 		if (code == 0)
3923 			tmp = (1U << 6) | (CODE_MAX - 1);
3924 		else if (code <= 0x20)
3925 			tmp =
3926 			    ((CODE_MAX - 1 -
3927 			      (0x20 - code) * 2) << 6) | (CODE_MAX - 1);
3928 		else
3929 			tmp =
3930 			    ((CODE_MAX - 1) << 6) | (CODE_MAX - 1 -
3931 						     (code - 0x20) * 2);
3932 	} else {
3933 		if (code == 0)
3934 			tmp = (1U << 6) | (CODE_MAX - 1);
3935 		else
3936 			tmp = (code << 6) | (CODE_MAX - code);
3937 	}
3938 	return tmp;
3939 }
3940 
rx_offset_cal(void)3941 static uint32_t rx_offset_cal(void)
3942 {
3943 	uint32_t index;
3944 	uint32_t code;
3945 	const uint32_t CODE_MAX = 0x40;
3946 	const uint32_t CODE_STEP = 2;
3947 	uint32_t ch, slice;
3948 	uint32_t tmp;
3949 	uint32_t tmp_ach_as[DRAM_CH_CNT][SLICE_CNT];
3950 	uint64_t val[DRAM_CH_CNT][SLICE_CNT][_reg_PHY_RX_CAL_X_NUM];
3951 	uint64_t tmpval;
3952 	int32_t lsb, msb;
3953 
3954 	ddr_setval_ach_as(_reg_PHY_RX_CAL_OVERRIDE, 0x01);
3955 	foreach_vch(ch) {
3956 		for (slice = 0; slice < SLICE_CNT; slice++) {
3957 			for (index = 0; index < _reg_PHY_RX_CAL_X_NUM; index++)
3958 				val[ch][slice][index] = 0;
3959 		}
3960 	}
3961 
3962 	for (code = 0; code < CODE_MAX / CODE_STEP; code++) {
3963 		tmp = _rx_offset_cal_updn(code * CODE_STEP);
3964 		for (index = 0; index < _reg_PHY_RX_CAL_X_NUM; index++) {
3965 			ddr_setval_ach_as(_reg_PHY_RX_CAL_X[index], tmp);
3966 		}
3967 		dsb_sev();
3968 		ddr_getval_ach_as(_reg_PHY_RX_CAL_OBS, (uint32_t *)tmp_ach_as);
3969 
3970 		foreach_vch(ch) {
3971 			for (slice = 0; slice < SLICE_CNT; slice++) {
3972 				tmp = tmp_ach_as[ch][slice];
3973 				for (index = 0; index < _reg_PHY_RX_CAL_X_NUM;
3974 				     index++) {
3975 					if (tmp & (1U << index)) {
3976 						val[ch][slice][index] |=
3977 						    (1ULL << code);
3978 					} else {
3979 						val[ch][slice][index] &=
3980 						    ~(1ULL << code);
3981 					}
3982 				}
3983 			}
3984 		}
3985 	}
3986 	foreach_vch(ch) {
3987 		for (slice = 0; slice < SLICE_CNT; slice++) {
3988 			for (index = 0; index < _reg_PHY_RX_CAL_X_NUM;
3989 			     index++) {
3990 				tmpval = val[ch][slice][index];
3991 				lsb = _find_change(tmpval, 0);
3992 				msb =
3993 				    _find_change(tmpval,
3994 						 (CODE_MAX / CODE_STEP) - 1);
3995 				tmp = (lsb + msb) >> 1;
3996 
3997 				tmp = _rx_offset_cal_updn(tmp * CODE_STEP);
3998 				ddr_setval_s(ch, slice,
3999 					     _reg_PHY_RX_CAL_X[index], tmp);
4000 			}
4001 		}
4002 	}
4003 	ddr_setval_ach_as(_reg_PHY_RX_CAL_OVERRIDE, 0x00);
4004 
4005 	return 0;
4006 }
4007 
rx_offset_cal_hw(void)4008 static uint32_t rx_offset_cal_hw(void)
4009 {
4010 	uint32_t ch, slice;
4011 	uint32_t retry;
4012 	uint32_t complete;
4013 	uint32_t tmp;
4014 	uint32_t tmp_ach_as[DRAM_CH_CNT][SLICE_CNT];
4015 
4016 	ddr_setval_ach_as(_reg_PHY_RX_CAL_X[9], 0x00);
4017 	ddr_setval_ach_as(_reg_PHY_RX_CAL_OVERRIDE, 0x00);
4018 	ddr_setval_ach_as(_reg_PHY_RX_CAL_SAMPLE_WAIT, 0x0f);
4019 
4020 	retry = 0;
4021 	while (retry < 4096) {
4022 		if ((retry & 0xff) == 0) {
4023 			ddr_setval_ach_as(_reg_SC_PHY_RX_CAL_START, 0x01);
4024 		}
4025 		foreach_vch(ch)
4026 		for (slice = 0; slice < SLICE_CNT; slice++)
4027 			tmp_ach_as[ch][slice] =
4028 			    ddr_getval_s(ch, slice, _reg_PHY_RX_CAL_X[9]);
4029 
4030 		complete = 1;
4031 		foreach_vch(ch) {
4032 			for (slice = 0; slice < SLICE_CNT; slice++) {
4033 				tmp = tmp_ach_as[ch][slice];
4034 				tmp = (tmp & 0x3f) + ((tmp >> 6) & 0x3f);
4035 				if (((prr_product == PRR_PRODUCT_H3) &&
4036 				     (prr_cut > PRR_PRODUCT_11)) ||
4037 				    (prr_product == PRR_PRODUCT_M3N) ||
4038 				    (prr_product == PRR_PRODUCT_V3H)) {
4039 					if (tmp != 0x3E)
4040 						complete = 0;
4041 				} else {
4042 					if (tmp != 0x40)
4043 						complete = 0;
4044 				}
4045 			}
4046 		}
4047 		if (complete)
4048 			break;
4049 
4050 		retry++;
4051 	}
4052 
4053 	return (complete == 0);
4054 }
4055 
4056 /* adjust rddqs latency */
adjust_rddqs_latency(void)4057 static void adjust_rddqs_latency(void)
4058 {
4059 	uint32_t ch, slice;
4060 	uint32_t dly;
4061 	uint32_t maxlatx2;
4062 	uint32_t tmp;
4063 	uint32_t rdlat_adjx2[SLICE_CNT];
4064 
4065 	foreach_vch(ch) {
4066 		maxlatx2 = 0;
4067 		for (slice = 0; slice < SLICE_CNT; slice++) {
4068 			ddr_setval_s(ch, slice, _reg_PHY_PER_CS_TRAINING_INDEX,
4069 				     0x00);
4070 
4071 			dly =
4072 			    ddr_getval_s(ch, slice,
4073 					 _reg_PHY_RDDQS_GATE_SLAVE_DELAY);
4074 			tmp =
4075 			    ddr_getval_s(ch, slice,
4076 					 _reg_PHY_RDDQS_LATENCY_ADJUST);
4077 			/* note gate_slave_delay[9] is always 0 */
4078 			tmp = (tmp << 1) + (dly >> 8);
4079 			rdlat_adjx2[slice] = tmp;
4080 			if (maxlatx2 < tmp)
4081 				maxlatx2 = tmp;
4082 		}
4083 		maxlatx2 = ((maxlatx2 + 1) >> 1) << 1;
4084 		for (slice = 0; slice < SLICE_CNT; slice++) {
4085 			tmp = maxlatx2 - rdlat_adjx2[slice];
4086 			tmp = (tmp >> 1);
4087 			if (tmp) {
4088 				ddr_setval_s(ch, slice, _reg_PHY_RPTR_UPDATE,
4089 					     ddr_getval_s(ch, slice,
4090 							  _reg_PHY_RPTR_UPDATE)
4091 					     + 1);
4092 			}
4093 		}
4094 	}
4095 }
4096 
4097 /* adjust wpath latency */
adjust_wpath_latency(void)4098 static void adjust_wpath_latency(void)
4099 {
4100 	uint32_t ch, cs, slice;
4101 	uint32_t dly;
4102 	uint32_t wpath_add;
4103 	const uint32_t _par_EARLY_THRESHOLD_VAL = 0x180;
4104 
4105 	foreach_vch(ch) {
4106 		for (slice = 0; slice < SLICE_CNT; slice += 1) {
4107 			for (cs = 0; cs < CS_CNT; cs++) {
4108 				ddr_setval_s(ch, slice,
4109 					     _reg_PHY_PER_CS_TRAINING_INDEX,
4110 					     cs);
4111 				ddr_getval_s(ch, slice,
4112 					     _reg_PHY_PER_CS_TRAINING_INDEX);
4113 				dly =
4114 				    ddr_getval_s(ch, slice,
4115 						 _reg_PHY_CLK_WRDQS_SLAVE_DELAY);
4116 				if (dly <= _par_EARLY_THRESHOLD_VAL)
4117 					continue;
4118 
4119 				wpath_add =
4120 				    ddr_getval_s(ch, slice,
4121 						 _reg_PHY_WRITE_PATH_LAT_ADD);
4122 				ddr_setval_s(ch, slice,
4123 					     _reg_PHY_WRITE_PATH_LAT_ADD,
4124 					     wpath_add - 1);
4125 			}
4126 		}
4127 	}
4128 }
4129 
4130 /* DDR Initialize entry */
rcar_dram_init(void)4131 int32_t rcar_dram_init(void)
4132 {
4133 	uint32_t ch, cs;
4134 	uint32_t data_l;
4135 	uint32_t bus_mbps, bus_mbpsdiv;
4136 	uint32_t tmp_tccd;
4137 	uint32_t failcount;
4138 	uint32_t cnf_boardtype;
4139 
4140 	/* Thermal sensor setting */
4141 	data_l = mmio_read_32(CPG_MSTPSR5);
4142 	if (data_l & BIT(22)) {	/*  case THS/TSC Standby */
4143 		data_l &= ~BIT(22);
4144 		cpg_write_32(CPG_SMSTPCR5, data_l);
4145 		while (mmio_read_32(CPG_MSTPSR5) & BIT(22))
4146 			;  /*  wait bit=0 */
4147 	}
4148 
4149 	/* THCTR Bit6: PONM=0 , Bit0: THSST=0   */
4150 	data_l = mmio_read_32(THS1_THCTR);
4151 	if (data_l & 0x00000040U) {
4152 		data_l = data_l & 0xFFFFFFBEU;
4153 	} else {
4154 		data_l = data_l | BIT(1);
4155 	}
4156 
4157 	mmio_write_32(THS1_THCTR, data_l);
4158 
4159 	/* Judge product and cut */
4160 #ifdef RCAR_DDR_FIXED_LSI_TYPE
4161 #if (RCAR_LSI == RCAR_AUTO)
4162 	prr_product = mmio_read_32(PRR) & PRR_PRODUCT_MASK;
4163 	prr_cut = mmio_read_32(PRR) & PRR_CUT_MASK;
4164 #else /* RCAR_LSI */
4165 #ifndef RCAR_LSI_CUT
4166 	prr_cut = mmio_read_32(PRR) & PRR_CUT_MASK;
4167 #endif /* RCAR_LSI_CUT */
4168 #endif /* RCAR_LSI */
4169 #else /* RCAR_DDR_FIXED_LSI_TYPE */
4170 	prr_product = mmio_read_32(PRR) & PRR_PRODUCT_MASK;
4171 	prr_cut = mmio_read_32(PRR) & PRR_CUT_MASK;
4172 #endif /* RCAR_DDR_FIXED_LSI_TYPE */
4173 
4174 	if (prr_product == PRR_PRODUCT_H3) {
4175 		if (prr_cut <= PRR_PRODUCT_11) {
4176 			p_ddr_regdef_tbl =
4177 				(const uint32_t *)&DDR_REGDEF_TBL[0][0];
4178 		} else {
4179 			p_ddr_regdef_tbl =
4180 				(const uint32_t *)&DDR_REGDEF_TBL[2][0];
4181 		}
4182 	} else if (prr_product == PRR_PRODUCT_M3) {
4183 		p_ddr_regdef_tbl =
4184 			(const uint32_t *)&DDR_REGDEF_TBL[1][0];
4185 	} else if ((prr_product == PRR_PRODUCT_M3N) ||
4186 		   (prr_product == PRR_PRODUCT_V3H)) {
4187 		p_ddr_regdef_tbl =
4188 			(const uint32_t *)&DDR_REGDEF_TBL[3][0];
4189 	} else {
4190 		FATAL_MSG("BL2: DDR:Unknown Product\n");
4191 		return 0xff;
4192 	}
4193 
4194 	if (((prr_product == PRR_PRODUCT_H3) && (prr_cut <= PRR_PRODUCT_11)) ||
4195 	    ((prr_product == PRR_PRODUCT_M3) && (prr_cut < PRR_PRODUCT_30))) {
4196 		/* non : H3 Ver.1.x/M3-W Ver.1.x not support */
4197 	} else {
4198 		mmio_write_32(DBSC_DBSYSCNT0, 0x00001234);
4199 	}
4200 
4201 	/* Judge board type */
4202 	cnf_boardtype = boardcnf_get_brd_type();
4203 	if (cnf_boardtype >= BOARDNUM) {
4204 		FATAL_MSG("BL2: DDR:Unknown Board\n");
4205 		return 0xff;
4206 	}
4207 	board_cnf = (const struct _boardcnf *)&boardcnfs[cnf_boardtype];
4208 
4209 /* RCAR_DRAM_SPLIT_2CH           (2U) */
4210 #if RCAR_DRAM_SPLIT == 2
4211 	/* H3(Test for future H3-N): Swap ch2 and ch1 for 2ch-split */
4212 	if ((prr_product == PRR_PRODUCT_H3) && (board_cnf->phyvalid == 0x05)) {
4213 		mmio_write_32(DBSC_DBMEMSWAPCONF0, 0x00000006);
4214 		ddr_phyvalid = 0x03;
4215 	} else {
4216 		ddr_phyvalid = board_cnf->phyvalid;
4217 	}
4218 #else /* RCAR_DRAM_SPLIT_2CH */
4219 	ddr_phyvalid = board_cnf->phyvalid;
4220 #endif /* RCAR_DRAM_SPLIT_2CH */
4221 
4222 	max_density = 0;
4223 
4224 	for (cs = 0; cs < CS_CNT; cs++) {
4225 		ch_have_this_cs[cs] = 0;
4226 	}
4227 
4228 	foreach_ech(ch)
4229 	for (cs = 0; cs < CS_CNT; cs++)
4230 		ddr_density[ch][cs] = 0xff;
4231 
4232 	foreach_vch(ch) {
4233 		for (cs = 0; cs < CS_CNT; cs++) {
4234 			data_l = board_cnf->ch[ch].ddr_density[cs];
4235 			ddr_density[ch][cs] = data_l;
4236 
4237 			if (data_l == 0xff)
4238 				continue;
4239 			if (data_l > max_density)
4240 				max_density = data_l;
4241 			if ((cs == 1) && (prr_product == PRR_PRODUCT_H3) &&
4242 			    (prr_cut <= PRR_PRODUCT_11))
4243 				continue;
4244 			ch_have_this_cs[cs] |= (1U << ch);
4245 		}
4246 	}
4247 
4248 	/* Judge board clock frequency (in MHz) */
4249 	boardcnf_get_brd_clk(cnf_boardtype, &brd_clk, &brd_clkdiv);
4250 	if ((brd_clk / brd_clkdiv) > 25) {
4251 		brd_clkdiva = 1;
4252 	} else {
4253 		brd_clkdiva = 0;
4254 	}
4255 
4256 	/* Judge ddr operating frequency clock(in Mbps) */
4257 	boardcnf_get_ddr_mbps(cnf_boardtype, &ddr_mbps, &ddr_mbpsdiv);
4258 
4259 	ddr0800_mul = CLK_DIV(800, 2, brd_clk, brd_clkdiv * (brd_clkdiva + 1));
4260 
4261 	ddr_mul = CLK_DIV(ddr_mbps, ddr_mbpsdiv * 2, brd_clk,
4262 			  brd_clkdiv * (brd_clkdiva + 1));
4263 
4264 	/* Adjust tccd */
4265 	data_l = (0x00006000 & mmio_read_32(RST_MODEMR)) >> 13;
4266 	bus_mbps = 0;
4267 	bus_mbpsdiv = 0;
4268 	switch (data_l) {
4269 	case 0:
4270 		bus_mbps = brd_clk * 0x60 * 2;
4271 		bus_mbpsdiv = brd_clkdiv * 1;
4272 		break;
4273 	case 1:
4274 		bus_mbps = brd_clk * 0x50 * 2;
4275 		bus_mbpsdiv = brd_clkdiv * 1;
4276 		break;
4277 	case 2:
4278 		bus_mbps = brd_clk * 0x40 * 2;
4279 		bus_mbpsdiv = brd_clkdiv * 1;
4280 		break;
4281 	case 3:
4282 		bus_mbps = brd_clk * 0x60 * 2;
4283 		bus_mbpsdiv = brd_clkdiv * 2;
4284 		break;
4285 	default:
4286 		bus_mbps = brd_clk * 0x60 * 2;
4287 		bus_mbpsdiv = brd_clkdiv * 2;
4288 		break;
4289 	}
4290 	tmp_tccd = CLK_DIV(ddr_mbps * 8, ddr_mbpsdiv, bus_mbps, bus_mbpsdiv);
4291 	if (8 * ddr_mbps * bus_mbpsdiv != tmp_tccd * bus_mbps * ddr_mbpsdiv)
4292 		tmp_tccd = tmp_tccd + 1;
4293 
4294 	if (tmp_tccd < 8)
4295 		ddr_tccd = 8;
4296 	else
4297 		ddr_tccd = tmp_tccd;
4298 
4299 	NOTICE("BL2: DDR%d(%s)\n", ddr_mbps / ddr_mbpsdiv, RCAR_DDR_VERSION);
4300 
4301 	MSG_LF("Start\n");
4302 
4303 	/* PLL Setting */
4304 	pll3_control(1);
4305 
4306 	/* initialize DDR */
4307 	data_l = init_ddr();
4308 	if (data_l == ddr_phyvalid) {
4309 		failcount = 0;
4310 	} else {
4311 		failcount = 1;
4312 	}
4313 
4314 	foreach_vch(ch)
4315 	    mmio_write_32(DBSC_DBPDLK(ch), 0x00000000);
4316 	if (((prr_product == PRR_PRODUCT_H3) && (prr_cut <= PRR_PRODUCT_11)) ||
4317 	    ((prr_product == PRR_PRODUCT_M3) && (prr_cut < PRR_PRODUCT_30))) {
4318 		/* non : H3 Ver.1.x/M3-W Ver.1.x not support */
4319 	} else {
4320 		mmio_write_32(DBSC_DBSYSCNT0, 0x00000000);
4321 	}
4322 
4323 	if (failcount == 0) {
4324 		return INITDRAM_OK;
4325 	} else {
4326 		return INITDRAM_NG;
4327 	}
4328 }
4329 
pvtcode_update(void)4330 void pvtcode_update(void)
4331 {
4332 	uint32_t ch;
4333 	uint32_t data_l;
4334 	uint32_t pvtp[4], pvtn[4], pvtp_init, pvtn_init;
4335 	int32_t pvtp_tmp, pvtn_tmp;
4336 
4337 	foreach_vch(ch) {
4338 		pvtn_init = (tcal.tcomp_cal[ch] & 0xFC0) >> 6;
4339 		pvtp_init = (tcal.tcomp_cal[ch] & 0x03F) >> 0;
4340 
4341 		if (8912 * pvtp_init > 44230) {
4342 			pvtp_tmp = (5000 + 8912 * pvtp_init - 44230) / 10000;
4343 		} else {
4344 			pvtp_tmp =
4345 			    -((-(5000 + 8912 * pvtp_init - 44230)) / 10000);
4346 		}
4347 		pvtn_tmp = (5000 + 5776 * pvtn_init + 30280) / 10000;
4348 
4349 		pvtn[ch] = pvtn_tmp + pvtn_init;
4350 		pvtp[ch] = pvtp_tmp + pvtp_init;
4351 
4352 		if (pvtn[ch] > 63) {
4353 			pvtn[ch] = 63;
4354 			pvtp[ch] =
4355 			    (pvtp_tmp) * (63 - 6 * pvtn_tmp -
4356 					  pvtn_init) / (pvtn_tmp) +
4357 			    6 * pvtp_tmp + pvtp_init;
4358 		}
4359 		if ((prr_product == PRR_PRODUCT_H3) &&
4360 		    (prr_cut <= PRR_PRODUCT_11)) {
4361 			data_l = pvtp[ch] | (pvtn[ch] << 6) |
4362 				 (tcal.tcomp_cal[ch] & 0xfffff000);
4363 			reg_ddrphy_write(ch,
4364 					 ddr_regdef_adr(_reg_PHY_PAD_FDBK_TERM),
4365 					 data_l | 0x00020000);
4366 			reg_ddrphy_write(ch,
4367 					 ddr_regdef_adr(_reg_PHY_PAD_DATA_TERM),
4368 					 data_l);
4369 			reg_ddrphy_write(ch,
4370 					 ddr_regdef_adr(_reg_PHY_PAD_DQS_TERM),
4371 					 data_l);
4372 			reg_ddrphy_write(ch,
4373 					 ddr_regdef_adr(_reg_PHY_PAD_ADDR_TERM),
4374 					 data_l);
4375 			reg_ddrphy_write(ch,
4376 					 ddr_regdef_adr(_reg_PHY_PAD_CS_TERM),
4377 					 data_l);
4378 		} else {
4379 			data_l = pvtp[ch] | (pvtn[ch] << 6) | 0x00015000;
4380 			reg_ddrphy_write(ch,
4381 					 ddr_regdef_adr(_reg_PHY_PAD_FDBK_TERM),
4382 					 data_l | 0x00020000);
4383 			reg_ddrphy_write(ch,
4384 					 ddr_regdef_adr(_reg_PHY_PAD_DATA_TERM),
4385 					 data_l);
4386 			reg_ddrphy_write(ch,
4387 					 ddr_regdef_adr(_reg_PHY_PAD_DQS_TERM),
4388 					 data_l);
4389 			reg_ddrphy_write(ch,
4390 					 ddr_regdef_adr(_reg_PHY_PAD_ADDR_TERM),
4391 					 data_l);
4392 			reg_ddrphy_write(ch,
4393 					 ddr_regdef_adr(_reg_PHY_PAD_CS_TERM),
4394 					 data_l);
4395 		}
4396 	}
4397 }
4398 
pvtcode_update2(void)4399 void pvtcode_update2(void)
4400 {
4401 	uint32_t ch;
4402 
4403 	foreach_vch(ch) {
4404 		reg_ddrphy_write(ch, ddr_regdef_adr(_reg_PHY_PAD_FDBK_TERM),
4405 				 tcal.init_cal[ch] | 0x00020000);
4406 		reg_ddrphy_write(ch, ddr_regdef_adr(_reg_PHY_PAD_DATA_TERM),
4407 				 tcal.init_cal[ch]);
4408 		reg_ddrphy_write(ch, ddr_regdef_adr(_reg_PHY_PAD_DQS_TERM),
4409 				 tcal.init_cal[ch]);
4410 		reg_ddrphy_write(ch, ddr_regdef_adr(_reg_PHY_PAD_ADDR_TERM),
4411 				 tcal.init_cal[ch]);
4412 		reg_ddrphy_write(ch, ddr_regdef_adr(_reg_PHY_PAD_CS_TERM),
4413 				 tcal.init_cal[ch]);
4414 	}
4415 }
4416 
ddr_padcal_tcompensate_getinit(uint32_t override)4417 void ddr_padcal_tcompensate_getinit(uint32_t override)
4418 {
4419 	uint32_t ch;
4420 	uint32_t data_l;
4421 	uint32_t pvtp, pvtn;
4422 
4423 	tcal.init_temp = 0;
4424 	for (ch = 0; ch < 4; ch++) {
4425 		tcal.init_cal[ch] = 0;
4426 		tcal.tcomp_cal[ch] = 0;
4427 	}
4428 
4429 	foreach_vch(ch) {
4430 		tcal.init_cal[ch] = ddr_getval(ch, _reg_PHY_PAD_TERM_X[1]);
4431 		tcal.tcomp_cal[ch] = ddr_getval(ch, _reg_PHY_PAD_TERM_X[1]);
4432 	}
4433 
4434 	if (!override) {
4435 		data_l = mmio_read_32(THS1_TEMP);
4436 		if (data_l < 2800) {
4437 			tcal.init_temp =
4438 			    (143 * (int32_t)data_l - 359000) / 1000;
4439 		} else {
4440 			tcal.init_temp =
4441 			    (121 * (int32_t)data_l - 296300) / 1000;
4442 		}
4443 
4444 		foreach_vch(ch) {
4445 			pvtp = (tcal.init_cal[ch] >> 0) & 0x000003F;
4446 			pvtn = (tcal.init_cal[ch] >> 6) & 0x000003F;
4447 			if ((int32_t)pvtp >
4448 			    ((tcal.init_temp * 29 - 3625) / 1000))
4449 				pvtp =
4450 				    (int32_t)pvtp +
4451 				    ((3625 - tcal.init_temp * 29) / 1000);
4452 			else
4453 				pvtp = 0;
4454 
4455 			if ((int32_t)pvtn >
4456 			    ((tcal.init_temp * 54 - 6750) / 1000))
4457 				pvtn =
4458 				    (int32_t)pvtn +
4459 				    ((6750 - tcal.init_temp * 54) / 1000);
4460 			else
4461 				pvtn = 0;
4462 
4463 			if ((prr_product == PRR_PRODUCT_H3) &&
4464 			    (prr_cut <= PRR_PRODUCT_11)) {
4465 				tcal.init_cal[ch] =
4466 				    (tcal.init_cal[ch] & 0xfffff000) |
4467 				    (pvtn << 6) |
4468 				    pvtp;
4469 			} else {
4470 				tcal.init_cal[ch] =
4471 				    0x00015000 | (pvtn << 6) | pvtp;
4472 			}
4473 		}
4474 		tcal.init_temp = 125;
4475 	}
4476 }
4477 
4478 #ifndef ddr_qos_init_setting
4479 /* For QoS init */
get_boardcnf_phyvalid(void)4480 uint8_t get_boardcnf_phyvalid(void)
4481 {
4482 	return ddr_phyvalid;
4483 }
4484 #endif /* ddr_qos_init_setting */
4485