1 /*
2 * Copyright (c) 2020 HiSilicon (Shanghai) Technologies CO., LIMITED.
3 *
4 * This program is free software; you can redistribute it and/or modify it
5 * under the terms of the GNU General Public License as published by the
6 * Free Software Foundation; either version 2 of the License, or (at your
7 * option) any later version.
8 *
9 * This program is distributed in the hope that it will be useful,
10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 * GNU General Public License for more details.
13 *
14 * You should have received a copy of the GNU General Public License
15 * along with this program. If not, see <http://www.gnu.org/licenses/>.
16 *
17
18 */
19
20 #ifndef DDR_TRAINING_IMPL_H
21 #define DDR_TRAINING_IMPL_H
22
23 #ifndef __ASSEMBLY__
24
25 #include "ddr_training_custom.h"
26 #include "ddr_training_internal_config.h"
27 #include "ddr_interface.h"
28
29 /****** special config define*******************************************/
30 #ifdef DDR_DATAEYE_NORMAL_NOT_ADJ_CONFIG
31 /* Adjust dataeye window consume a lot of time, disable it will make boot
32 * faster.
33 * NOTE: The WDQ Phase and RDQS MUST be config a good value in the init table
34 * to avoid window trend to one side.
35 */
36 #define DDR_DATAEYE_NORMAL_ADJUST (DDR_FALSE)
37 #else
38 #define DDR_DATAEYE_NORMAL_ADJUST (DDR_TRUE)
39 #endif
40 /* MUST adjust dataeye window after HW or MPR training */
41 #define DDR_DATAEYE_ABNORMAL_ADJUST (DDR_TRUE)
42
43 /****** ddr training item bypass mask define ****************************/
44 #define DDR_BYPASS_PHY0_MASK 0x1 /* [0]PHY0 training */
45 #define DDR_BYPASS_PHY1_MASK 0x2 /* [1]PHY1 training */
46 #define DDR_BYPASS_WL_MASK 0x10 /* [4]Write leveling */
47 #define DDR_BYPASS_GATE_MASK 0x100 /* [8]Gate training */
48 #define DDR_BYPASS_DATAEYE_MASK 0x10000 /* [16]Dataeye training */
49 #define DDR_BYPASS_PCODE_MASK 0x40000 /* [18]Pcode training */
50 #define DDR_BYPASS_HW_MASK 0x100000 /* [20]Hardware read training */
51 #define DDR_BYPASS_MPR_MASK 0x200000 /* [21]MPR training */
52 #define DDR_BYPASS_AC_MASK 0x400000 /* [22]AC training */
53 #define DDR_BYPASS_LPCA_MASK 0x800000 /* [23]LPDDR CA training */
54 #define DDR_BYPASS_VREF_HOST_MASK 0x1000000 /* [24]Host Vref training */
55 #define DDR_BYPASS_VREF_DRAM_MASK 0x2000000 /* [25]DRAM Vref training */
56 #define DDR_BYPASS_DCC_MASK 0x08000000 /* [27]DCC training */
57 #define DDR_BYPASS_DATAEYE_ADJ_MASK 0x10000000 /* [28]Dataeye adjust */
58 #define DDR_BYPASS_WL_ADJ_MASK 0x20000000 /* [29]WL write adjust */
59 #define DDR_BYPASS_HW_ADJ_MASK 0x40000000 /* [30]HW read adjust */
60 #define DDR_BYPASS_ALL_MASK 0xffffffff /* all bypass */
61
62 /****** ddr read/write define **********************************************/
63 unsigned int ddr_read(unsigned addr);
64 void ddr_write(unsigned val, unsigned addr);
65
66 /****** common define **********************************************/
67 /* special ddrt need special read and write register */
68 #ifdef DDR_DDRT_SPECIAL_CONFIG
69 #define DDRT_REG_READ(addr) ddr_ddrt_read(addr)
70 #define DDRT_REG_WRITE(val, addr) ddr_ddrt_write(val, addr)
71 #else
72 #define DDRT_REG_READ(addr) ddr_read(addr)
73 #define DDRT_REG_WRITE(val, addr) ddr_write(val, addr)
74 #endif
75
76 #define DDR_MODE_READ (1 << 0)
77 #define DDR_MODE_WRITE (1 << 1)
78
79 #define DDR_ENTER_SREF (1 << 0)
80 #define DDR_EXIT_SREF (1 << 1)
81
82 /* DSB to make sure the operation is complete */
83 #ifndef DDR_ASM_DSB
84 #if (__LINUX_ARM_ARCH__ >= 8)
85 #define DDR_ASM_DSB() { __asm__ __volatile__("dsb sy"); }
86 #else
87 #define DDR_ASM_DSB() { __asm__ __volatile__("dsb"); }
88 #endif
89 #endif
90
91 #define DDR_HWR_WAIT_TIMEOUT 0xffffffff
92 #define DDR_SFC_WAIT_TIMEOUT (1000)
93 #define DDR_LPCA_WAIT_TIMEOUT (1000)
94
95 #ifdef CFG_EDA_VERIFY
96 #define DDR_AUTO_TIMING_DELAY (1)
97 #else
98 #define DDR_AUTO_TIMING_DELAY (1000)
99 #endif
100
101 #define DDR_FIND_DQ_BOTH (1 << 0) /* find a valid value*/
102 /* x is valid, (x-1) is invalid*/
103 #define DDR_FIND_DQ_LEFT (1 << 1)
104 /* x is valid, (x+1) is invalid*/
105 #define DDR_FIND_DQ_RIGHT (1 << 2)
106
107 #define DDR_VREF_DRAM_VAL_MAX (0x32) /* 92.50%*VDDIO */
108 #define DDR_VREF_DRAM_VAL_MIN (0x0) /* 60.00%*VDDIO */
109
110 #define DDR_PHY_REG_DQ_NUM 4 /* one register has 4 DQ BDL */
111
112 #define DDR_PHY_CA_MAX 10
113 #define DDR_PHY_CA_REG_MAX (DDR_PHY_CA_MAX >> 1)
114
115 #define DDR_TRUE 1
116 #define DDR_FALSE 0
117
118 #define DDR_WIN_MIDDLE (1 << 0)
119 #define DDR_WIN_LEFT (1 << 1)
120 #define DDR_WIN_RIGHT (1 << 2)
121
122 #define DDR_DELAY_PHASE 1
123 #define DDR_DELAY_BDL 2
124
125 #ifndef DDR_DATAEYE_WIN_NUM
126 /* Dateeye window number. More bigger more slower when Vref training. */
127 #define DDR_DATAEYE_WIN_NUM 8
128 #endif
129 #ifndef DDR_LOOP_TIMES_LMT
130 /* Dataeye DQ deskew times for best result. More bigger more slower. */
131 #define DDR_LOOP_TIMES_LMT 1
132 #endif
133 #ifndef DDR_VREF_COMPARE_TIMES
134 /* Compare times when find best vref value. More bigger more slower. */
135 #define DDR_VREF_COMPARE_TIMES 3
136 #endif
137 #ifndef DDR_MPR_RDQS_FIND_TIMES
138 /* MPR Find first start rdqs times. More bigger, start rdqs more bigger. */
139 #define DDR_MPR_RDQS_FIND_TIMES 3
140 #endif
141 #ifndef DDR_VREF_COMPARE_STEP
142 /* Compare step when begin to find. More bigger, more mistake, more stable. */
143 #define DDR_VREF_COMPARE_STEP 3
144 #endif
145
146 #define DDR_DATAEYE_RESULT_MASK 0xffff
147 #define DDR_DATAEYE_RESULT_BIT 16
148
149 #define DDR_WL_BDL_STEP 2 /* wl bdl step */
150 #define DDR_GATE_BDL_STEP 2 /* gate bdl step */
151 #define DDR_DQS_ADJ_STEP 1 /* WR/RD DQS adjust step */
152
153 #define DDR_DDRT_MODE_GATE (1 << 0)
154 #define DDR_DDRT_MODE_DATAEYE (1 << 1)
155
156 #define DDR_CHECK_TYPE_DDRT (1 << 0)
157 #define DDR_CHECK_TYPE_MPR (1 << 1)
158
159 #define DDR_MPR_BYTE_MASK 0xff
160 #define DDR_MPR_BIT_MASK 0x1
161 #define DDR_MPR_BYTE_BIT 16 /* 16 bit (2 byte) */
162
163 #define DDR_PHY_AC_TEST_VAL0 0x0
164 #define DDR_PHY_AC_TEST_VAL1 0xffffffff
165 #define DDR_PHY_AC_TEST_VAL2 0x55555555
166 #define DDR_PHY_AC_TEST_VAL3 0xaaaaaaaa
167
168 /*******log define ***********************************************/
169 #if defined(DDR_TRAINING_CMD) && defined(DDR_TRAINING_LOG_CONFIG)
170 #define DDR_INFO(fmt...) ddr_training_log(__func__, DDR_LOG_INFO, fmt)
171 #define DDR_DEBUG(fmt...) ddr_training_log(__func__, DDR_LOG_DEBUG, fmt)
172 #define DDR_WARNING(fmt...) ddr_training_log(__func__, DDR_LOG_WARNING, fmt)
173 #define DDR_ERROR(fmt...) ddr_training_log(__func__, DDR_LOG_ERROR, fmt)
174 #define DDR_FATAL(fmt...) ddr_training_log(__func__, DDR_LOG_FATAL, fmt)
175 #else
176 #define DDR_INFO(fmt...)
177 #define DDR_DEBUG(fmt...)
178 #define DDR_WARNING(fmt...)
179 #define DDR_ERROR(fmt...)
180 #define DDR_FATAL(fmt...)
181 #endif /* DDR_TRAINING_CMD && DDR_TRAINING_LOG_CONFIG */
182
183 /* [11:0] Error type */
184 /* 0x00000001 Write Leveling error */
185 #define DDR_ERR_WL (1 << 0)
186 /* 0x00000002 Hardware Gatining error */
187 #define DDR_ERR_HW_GATING (1 << 1)
188 /* 0x00000004 Sofeware Gatining error */
189 #define DDR_ERR_GATING (1 << 2)
190 /* 0x00000008 DDRT test time out */
191 #define DDR_ERR_DDRT_TIME_OUT (1 << 3)
192 /* 0x00000010 Hardware read dataeye error */
193 #define DDR_ERR_HW_RD_DATAEYE (1 << 4)
194 /* 0x00000020 MPR error */
195 #define DDR_ERR_MPR (1 << 5)
196 /* 0x00000040 Dataeye error */
197 #define DDR_ERR_DATAEYE (1 << 6)
198 /* 0x00000080 LPDDR CA error */
199 #define DDR_ERR_LPCA (1 << 7)
200
201 /* [13:12] Error phy */
202 /* 0x00001000 PHY0 training error */
203 #define DDR_ERR_PHY0 (1 << 12)
204 /* 0x00002000 PHY1 training error */
205 #define DDR_ERR_PHY1 (1 << 13)
206
207 #define DDR_ERR_BYTE_BIT 24 /* [28:24] Error DQ0-31 */
208 #define DDR_ERR_DQ_BIT 20 /* [22:20] Error Byte0-3 */
209
210 /*******data define*********************************************/
211 #define GET_BYTE_NUM(cfg) (cfg->phy[cfg->phy_idx].dmc[cfg->dmc_idx].byte_num)
212
213
214 #ifndef DDR_RELATE_REG_DECLARE
215 struct tr_custom_reg {
216 };
217 #endif
218
219 struct dmc_cfg_sref_st {
220 unsigned int val[DDR_DMC_PER_PHY_MAX];
221 };
222
223 struct ddr_bdl_st {
224 unsigned int bdl[DDR_PHY_BYTE_MAX];
225 };
226
227 struct ddr_timing_st {
228 unsigned int val[DDR_DMC_PER_PHY_MAX];
229 };
230
231 struct rdqs_data_st {
232 struct ddr_bdl_st origin;
233 struct ddr_bdl_st rank[DDR_RANK_NUM];
234 };
235
236 struct ddr_delay_st {
237 unsigned int phase[DDR_PHY_BYTE_MAX];
238 unsigned int bdl[DDR_PHY_BYTE_MAX];
239 };
240
241 struct tr_relate_reg {
242 unsigned int auto_ref_timing;
243 unsigned int power_down;
244 unsigned int dmc_scramb;
245 unsigned int dmc_scramb_cfg;
246 unsigned int misc_scramb;
247 unsigned int ac_phy_ctl;
248 unsigned int swapdfibyte_en;
249 struct tr_custom_reg custom;
250 struct ddr_ddrc_data ddrc;
251 };
252
253 struct tr_dq_data {
254 unsigned int dq03[DDR_PHY_BYTE_MAX]; /* DQ0-DQ3 BDL */
255 unsigned int dq47[DDR_PHY_BYTE_MAX]; /* DQ4-DQ7 BDL */
256 unsigned int rdqs[DDR_PHY_BYTE_MAX]; /* RDQS */
257 unsigned int rdm[DDR_PHY_BYTE_MAX]; /* RDM */
258 unsigned int wdm[DDR_PHY_BYTE_MAX]; /* WDM */
259 };
260
261 struct ca_bit_st {
262 unsigned int bit_p;
263 unsigned int bit_n;
264 };
265
266 struct ca_data_st {
267 unsigned int base_dmc;
268 unsigned int base_phy;
269 unsigned int done; /* whether all ca found bdl range */
270 unsigned int min; /* min left bound */
271 unsigned int max; /* max right bound */
272 unsigned def[DDR_PHY_CA_REG_MAX];
273 int left[DDR_PHY_CA_MAX];
274 int right[DDR_PHY_CA_MAX];
275 struct ca_bit_st bits[DDR_PHY_CA_MAX];
276 };
277
278 struct ddr_dmc_st {
279 unsigned int addr;
280 unsigned int byte_num;
281 unsigned int ddrt_pattern; /* ddrt reversed data */
282 };
283
284 struct ddr_rank_st {
285 unsigned int item; /* software training item */
286 unsigned int item_hw; /* hardware training item */
287 };
288
289 struct ddr_phy_st {
290 unsigned int addr;
291 unsigned int dram_type;
292 unsigned int dmc_num;
293 unsigned int rank_num;
294 unsigned int total_byte_num;
295 struct ddr_dmc_st dmc[DDR_DMC_PER_PHY_MAX];
296 struct ddr_rank_st rank[DDR_RANK_NUM];
297 };
298
299 struct ddr_cfg_st {
300 struct ddr_phy_st phy[DDR_PHY_NUM];
301 unsigned int phy_num;
302 unsigned int cur_phy; /* current training phy addr */
303 unsigned int cur_dmc; /* current training dmc addr */
304 unsigned int cur_item; /* current SW or HW training item */
305 unsigned int cur_pattern; /* current ddrt pattern */
306 unsigned int cur_mode; /* read or write */
307 unsigned int cur_byte; /* current training byte index */
308 unsigned int cur_dq; /* current training dq index */
309 unsigned int phy_idx; /* current training phy index */
310 unsigned int rank_idx; /* current training rank index */
311 unsigned int dmc_idx; /* current training dmc index */
312 unsigned int adjust; /* whether need to adjust dataeye window */
313 unsigned int dq_check_type; /* ddrt or mpr */
314 void *cmd_st; /* struct ddr_cmd_st */
315 void *res_st; /* SW: struct ddr_training_result_st, HW: struct rdqs_data_st */
316 };
317
318 struct dcc_ck_st {
319 unsigned int val[DDR_CK_RESULT_MAX];
320 unsigned int win;
321 unsigned int win_min_ctl;
322 unsigned int win_max_ctl;
323 unsigned int win_min_duty;
324 unsigned int win_max_duty;
325 unsigned int def_bp;
326 unsigned int def_ctl;
327 unsigned int def_duty;
328 unsigned int idx_duty;
329 unsigned int idx_duty_ctl;
330 unsigned int idx_ctl;
331 unsigned int BYPASS_CK_BIT;
332 unsigned int ACIOCTL21_CTL_BIT;
333 unsigned int ACIOCTL21_CK_BIT;
334 };
335
336 #ifdef DDR_DCC_TRAINING_CONFIG
337 struct dcc_data_st {
338 struct tr_dq_data rank[DDR_RANK_NUM];
339 struct dcc_ck_st ck[DDR_CK_NUM];
340 unsigned int item[DDR_CK_NUM];
341 unsigned int ioctl21_tmp;
342 };
343 #endif
344 /*******Uart early function ***********************************************/
345 #ifndef DDR_PUTS
346 #define DDR_PUTS uart_early_puts
347 #endif
348 #ifndef DDR_PUT_HEX
349 #define DDR_PUT_HEX uart_early_put_hex
350 #endif
351 #ifndef DDR_PUTC
352 #define DDR_PUTC uart_early_putc
353 #endif
354
355 #if defined(DDR_TRAINING_UART_CONFIG) || defined(DDR_TRAINING_LOG_CONFIG)
356 extern void uart_early_puts(const char *s);
357 extern void uart_early_put_hex(int hex);
358 extern void uart_early_putc(int chr);
359 #else
360 #undef DDR_PUTS
361 #undef DDR_PUT_HEX
362 #undef DDR_PUTC
363 #endif
364 /*******function interface define*********************************************/
365 #ifndef DDR_SW_TRAINING_FUNC
366 #define DDR_SW_TRAINING_FUNC_PUBLIC
367 #define DDR_SW_TRAINING_FUNC ddr_sw_training_func
368 #endif
369
370 #ifndef DDR_HW_TRAINING_FUNC
371 #define DDR_HW_TRAINING_FUNC_PUBLIC
372 #define DDR_HW_TRAINING_FUNC ddr_hw_training_func
373 #endif
374
375 #ifndef DDR_PCODE_TRAINING_FUNC
376 #define DDR_PCODE_TRAINING_FUNC ddr_pcode_training_func
377 #endif
378
379 #ifndef DDR_TRAINING_CONSOLE
380 #define DDR_TRAINING_CONSOLE_PUBLIC
381 #define DDR_TRAINING_CONSOLE ddr_training_console
382 #endif
383 /*******Custom function ***********************************************/
384 #ifndef DDR_TRAINING_DDRT_PREPARE_FUNC
385 #define DDR_TRAINING_DDRT_PREPARE_FUNC()
386 #endif
387 #ifndef DDR_TRAINING_SAVE_REG_FUNC
388 #define DDR_TRAINING_SAVE_REG_FUNC(relate_reg, mask)
389 #endif
390 #ifndef DDR_TRAINING_RESTORE_REG_FUNC
391 #define DDR_TRAINING_RESTORE_REG_FUNC(relate_reg)
392 #endif
393 #ifndef ddr_boot_cmd_save_func
394 #define ddr_boot_cmd_save_func(relate_reg) ((void)(relate_reg))
395 #endif
396 #ifndef ddr_boot_cmd_restore_func
397 #define ddr_boot_cmd_restore_func(relate_reg) ((void)(relate_reg))
398 #endif
399 /*******function define*********************************************/
400 int ddr_sw_training_func(void);
401 int ddr_training_boot_func(struct ddr_cfg_st *cfg);
402 int ddr_training_cmd_func(struct ddr_cfg_st *cfg);
403
404 void* ddrtr_memset(void *b, int c, unsigned int len);
405 void* ddrtr_memcpy(void *dst, const void *src, unsigned int len);
406 void ddr_training_cfg_init(struct ddr_cfg_st *cfg);
407 int ddr_training_by_dmc(struct ddr_cfg_st *cfg);
408 int ddr_training_by_rank(struct ddr_cfg_st *cfg);
409 int ddr_training_by_phy(struct ddr_cfg_st *cfg);
410 int ddr_training_all(struct ddr_cfg_st *cfg);
411 int ddr_dataeye_training_func(struct ddr_cfg_st *cfg);
412 int ddr_vref_training_func(struct ddr_cfg_st *cfg);
413 int ddr_wl_func(struct ddr_cfg_st *cfg);
414 int ddr_gating_func(struct ddr_cfg_st *cfg);
415 int ddr_ac_training_func(struct ddr_cfg_st *cfg);
416 int ddr_lpca_training_func(struct ddr_cfg_st *cfg);
417 int ddr_dcc_training_func(struct ddr_cfg_st *cfg);
418
419 void ddr_phy_cfg_update(unsigned int base_phy);
420 void ddr_phy_set_dq_bdl(struct ddr_cfg_st *cfg, unsigned int value);
421 int ddr_hw_training(struct ddr_cfg_st *cfg);
422 int ddr_pcode_training(struct ddr_cfg_st *cfg);
423
424 int ddr_mpr_training(struct ddr_cfg_st *cfg);
425 int ddr_write_leveling(struct ddr_cfg_st *cfg);
426 int ddr_gate_training(struct ddr_cfg_st *cfg);
427 int ddr_dataeye_training(struct ddr_cfg_st *cfg);
428 int ddr_vref_training(struct ddr_cfg_st *cfg);
429 int ddr_ac_training(struct ddr_cfg_st *cfg);
430 int ddr_lpca_training(struct ddr_cfg_st *cfg);
431 int ddr_dataeye_deskew(struct ddr_cfg_st *cfg, struct training_data *training);
432 void ddr_adjust_dataeye(struct ddr_cfg_st *cfg, struct training_data *training);
433 void ddr_result_data_save(struct ddr_cfg_st *cfg, struct training_data *training);
434 void ddr_lpca_data_save(struct ca_data_st *data);
435 unsigned int ddr_ddrt_get_test_addr(void);
436 int ddr_ddrt_test(unsigned int mask, int byte, int dq);
437 int ddr_dataeye_check_dq(struct ddr_cfg_st *cfg);
438 void ddr_ddrt_init(struct ddr_cfg_st *cfg, unsigned int mode);
439 int ddr_training_check_bypass(struct ddr_cfg_st *cfg, unsigned int mask);
440 int ddr_training_phy_disable(int index);
441 void ddr_training_save_reg(struct ddr_cfg_st *cfg, struct tr_relate_reg *relate_reg,
442 unsigned int mask);
443 void ddr_training_restore_reg(struct ddr_cfg_st *cfg, struct tr_relate_reg *relate_reg);
444 void ddr_training_get_base(int index, unsigned int *base_dmc,
445 unsigned int *base_phy);
446 void ddr_training_switch_axi(struct ddr_cfg_st *cfg);
447 void ddr_training_log(const char *func, int level, const char *fmt, ...);
448 void ddr_training_stat(unsigned int mask, unsigned int phy, int byte, int dq);
449 void ddr_training_error(unsigned int mask, unsigned int phy, int byte, int dq);
450 void ddr_training_start(void);
451 void ddr_training_suc(void);
452 unsigned int ddr_phy_get_byte_num(unsigned int base_dmc);
453 void ddr_training_set_timing(unsigned int base_dmc, unsigned int timing);
454 int ddr_hw_dataeye_read(struct ddr_cfg_st *cfg);
455
456 #ifdef DDR_MPR_TRAINING_CONFIG
457 int ddr_mpr_training_func(struct ddr_cfg_st *cfg);
458 int ddr_mpr_check(struct ddr_cfg_st *cfg);
459 #else
ddr_mpr_training_func(struct ddr_cfg_st * cfg)460 static inline int ddr_mpr_training_func(struct ddr_cfg_st *cfg)
461 {
462 DDR_WARNING("Not support DDR MPR training.");
463 return 0;
464 }
ddr_mpr_check(struct ddr_cfg_st * cfg)465 static inline int ddr_mpr_check(struct ddr_cfg_st *cfg) { return 0;}
466 #endif
467
468 #endif /* __ASSEMBLY__ */
469 #endif /* DDR_TRAINING_IMPL_H */
470