1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * Copyright (C) Marvell International Ltd. and its affiliates
4 */
5
6 #include "ddr3_init.h"
7
8 #define WL_ITERATION_NUM 10
9
10 static u32 pup_mask_table[] = {
11 0x000000ff,
12 0x0000ff00,
13 0x00ff0000,
14 0xff000000
15 };
16
17 static struct write_supp_result wr_supp_res[MAX_INTERFACE_NUM][MAX_BUS_NUM];
18
19 static int ddr3_tip_dynamic_write_leveling_seq(u32 dev_num);
20 static int ddr3_tip_dynamic_read_leveling_seq(u32 dev_num);
21 static int ddr3_tip_dynamic_per_bit_read_leveling_seq(u32 dev_num);
22 static int ddr3_tip_wl_supp_align_phase_shift(u32 dev_num, u32 if_id,
23 u32 bus_id);
24 static int ddr3_tip_xsb_compare_test(u32 dev_num, u32 if_id, u32 bus_id,
25 u32 edge_offset);
26
ddr3_tip_max_cs_get(u32 dev_num)27 u32 ddr3_tip_max_cs_get(u32 dev_num)
28 {
29 u32 c_cs, if_id, bus_id;
30 static u32 max_cs;
31 struct mv_ddr_topology_map *tm = mv_ddr_topology_map_get();
32 u32 octets_per_if_num = ddr3_tip_dev_attr_get(dev_num, MV_ATTR_OCTET_PER_INTERFACE);
33
34 if (!max_cs) {
35 CHECK_STATUS(ddr3_tip_get_first_active_if((u8)dev_num,
36 tm->if_act_mask,
37 &if_id));
38 for (bus_id = 0; bus_id < octets_per_if_num; bus_id++) {
39 VALIDATE_BUS_ACTIVE(tm->bus_act_mask, bus_id);
40 break;
41 }
42
43 for (c_cs = 0; c_cs < NUM_OF_CS; c_cs++) {
44 VALIDATE_ACTIVE(tm->
45 interface_params[if_id].as_bus_params[bus_id].
46 cs_bitmask, c_cs);
47 max_cs++;
48 }
49 }
50
51 return max_cs;
52 }
53
54 enum {
55 PASS,
56 FAIL
57 };
58 /*****************************************************************************
59 Dynamic read leveling
60 ******************************************************************************/
ddr3_tip_dynamic_read_leveling(u32 dev_num,u32 freq)61 int ddr3_tip_dynamic_read_leveling(u32 dev_num, u32 freq)
62 {
63 u32 data, mask;
64 u32 max_cs = ddr3_tip_max_cs_get(dev_num);
65 u32 bus_num, if_id, cl_val;
66 enum hws_speed_bin speed_bin_index;
67 /* save current CS value */
68 u32 cs_enable_reg_val[MAX_INTERFACE_NUM] = { 0 };
69 int is_any_pup_fail = 0;
70 u32 data_read[MAX_INTERFACE_NUM + 1] = { 0 };
71 u8 rl_values[NUM_OF_CS][MAX_BUS_NUM][MAX_INTERFACE_NUM];
72 struct pattern_info *pattern_table = ddr3_tip_get_pattern_table();
73 u16 *mask_results_pup_reg_map = ddr3_tip_get_mask_results_pup_reg_map();
74 u32 octets_per_if_num = ddr3_tip_dev_attr_get(dev_num, MV_ATTR_OCTET_PER_INTERFACE);
75 struct mv_ddr_topology_map *tm = mv_ddr_topology_map_get();
76
77 for (effective_cs = 0; effective_cs < NUM_OF_CS; effective_cs++)
78 for (bus_num = 0; bus_num < MAX_BUS_NUM; bus_num++)
79 for (if_id = 0; if_id <= MAX_INTERFACE_NUM - 1; if_id++)
80 rl_values[effective_cs][bus_num][if_id] = 0;
81
82 for (effective_cs = 0; effective_cs < max_cs; effective_cs++) {
83 for (if_id = 0; if_id <= MAX_INTERFACE_NUM - 1; if_id++) {
84 VALIDATE_IF_ACTIVE(tm->if_act_mask, if_id);
85 training_result[training_stage][if_id] = TEST_SUCCESS;
86
87 /* save current cs enable reg val */
88 CHECK_STATUS(ddr3_tip_if_read
89 (dev_num, ACCESS_TYPE_UNICAST, if_id,
90 DUAL_DUNIT_CFG_REG, cs_enable_reg_val,
91 MASK_ALL_BITS));
92 /* enable single cs */
93 CHECK_STATUS(ddr3_tip_if_write
94 (dev_num, ACCESS_TYPE_UNICAST, if_id,
95 DUAL_DUNIT_CFG_REG, (1 << 3), (1 << 3)));
96 }
97
98 ddr3_tip_reset_fifo_ptr(dev_num);
99
100 /*
101 * Phase 1: Load pattern (using ODPG)
102 *
103 * enter Read Leveling mode
104 * only 27 bits are masked
105 * assuming non multi-CS configuration
106 * write to CS = 0 for the non multi CS configuration, note
107 * that the results shall be read back to the required CS !!!
108 */
109
110 /* BUS count is 0 shifted 26 */
111 CHECK_STATUS(ddr3_tip_if_write
112 (dev_num, ACCESS_TYPE_MULTICAST, PARAM_NOT_CARE,
113 ODPG_DATA_CTRL_REG, 0x3, 0x3));
114 CHECK_STATUS(ddr3_tip_configure_odpg
115 (dev_num, ACCESS_TYPE_MULTICAST, PARAM_NOT_CARE, 0,
116 pattern_table[PATTERN_RL].num_of_phases_tx, 0,
117 pattern_table[PATTERN_RL].num_of_phases_rx, 0, 0,
118 effective_cs, STRESS_NONE, DURATION_SINGLE));
119
120 /* load pattern to ODPG */
121 ddr3_tip_load_pattern_to_odpg(dev_num, ACCESS_TYPE_MULTICAST,
122 PARAM_NOT_CARE, PATTERN_RL,
123 pattern_table[PATTERN_RL].
124 start_addr);
125
126 /*
127 * Phase 2: ODPG to Read Leveling mode
128 */
129
130 /* General Training Opcode register */
131 CHECK_STATUS(ddr3_tip_if_write
132 (dev_num, ACCESS_TYPE_MULTICAST, PARAM_NOT_CARE,
133 ODPG_WR_RD_MODE_ENA_REG, 0,
134 MASK_ALL_BITS));
135
136 CHECK_STATUS(ddr3_tip_if_write
137 (dev_num, ACCESS_TYPE_MULTICAST, PARAM_NOT_CARE,
138 GENERAL_TRAINING_OPCODE_REG,
139 (0x301b01 | effective_cs << 2), 0x3c3fef));
140
141 /* Object1 opcode register 0 & 1 */
142 for (if_id = 0; if_id <= MAX_INTERFACE_NUM - 1; if_id++) {
143 VALIDATE_IF_ACTIVE(tm->if_act_mask, if_id);
144 speed_bin_index =
145 tm->interface_params[if_id].speed_bin_index;
146 cl_val =
147 cas_latency_table[speed_bin_index].cl_val[freq];
148 data = (cl_val << 17) | (0x3 << 25);
149 mask = (0xff << 9) | (0x1f << 17) | (0x3 << 25);
150 CHECK_STATUS(ddr3_tip_if_write
151 (dev_num, ACCESS_TYPE_UNICAST, if_id,
152 OPCODE_REG0_REG(1), data, mask));
153 }
154
155 /* Set iteration count to max value */
156 CHECK_STATUS(ddr3_tip_if_write
157 (dev_num, ACCESS_TYPE_MULTICAST, PARAM_NOT_CARE,
158 OPCODE_REG1_REG(1), 0xd00, 0xd00));
159
160 /*
161 * Phase 2: Mask config
162 */
163
164 ddr3_tip_dynamic_read_leveling_seq(dev_num);
165
166 /*
167 * Phase 3: Read Leveling execution
168 */
169
170 /* temporary jira dunit=14751 */
171 CHECK_STATUS(ddr3_tip_if_write
172 (dev_num, ACCESS_TYPE_MULTICAST, PARAM_NOT_CARE,
173 TRAINING_DBG_1_REG, 0, (u32)(1 << 31)));
174 /* configure phy reset value */
175 CHECK_STATUS(ddr3_tip_if_write
176 (dev_num, ACCESS_TYPE_MULTICAST, PARAM_NOT_CARE,
177 TRAINING_DBG_3_REG, (0x7f << 24),
178 (u32)(0xff << 24)));
179 /* data pup rd reset enable */
180 CHECK_STATUS(ddr3_tip_if_write
181 (dev_num, ACCESS_TYPE_MULTICAST, PARAM_NOT_CARE,
182 SDRAM_CFG_REG, 0, (1 << 30)));
183 /* data pup rd reset disable */
184 CHECK_STATUS(ddr3_tip_if_write
185 (dev_num, ACCESS_TYPE_MULTICAST, PARAM_NOT_CARE,
186 SDRAM_CFG_REG, (1 << 30), (1 << 30)));
187 /* training SW override & training RL mode */
188 CHECK_STATUS(ddr3_tip_if_write
189 (dev_num, ACCESS_TYPE_MULTICAST, PARAM_NOT_CARE,
190 TRAINING_SW_2_REG, 0x1, 0x9));
191 /* training enable */
192 CHECK_STATUS(ddr3_tip_if_write
193 (dev_num, ACCESS_TYPE_MULTICAST, PARAM_NOT_CARE,
194 TRAINING_REG, (1 << 24) | (1 << 20),
195 (1 << 24) | (1 << 20)));
196 CHECK_STATUS(ddr3_tip_if_write
197 (dev_num, ACCESS_TYPE_MULTICAST, PARAM_NOT_CARE,
198 TRAINING_REG, (u32)(1 << 31), (u32)(1 << 31)));
199
200 /* trigger training */
201 mv_ddr_training_enable();
202
203 /* check for training done */
204 if (mv_ddr_is_training_done(MAX_POLLING_ITERATIONS, &data) != MV_OK) {
205 DEBUG_LEVELING(DEBUG_LEVEL_ERROR, ("training done failed\n"));
206 return MV_FAIL;
207 }
208 /* check for training pass */
209 if (data != PASS)
210 DEBUG_LEVELING(DEBUG_LEVEL_INFO, ("training result failed\n"));
211
212 /* disable odpg; switch back to functional mode */
213 mv_ddr_odpg_disable();
214
215 if (mv_ddr_is_odpg_done(MAX_POLLING_ITERATIONS) != MV_OK) {
216 DEBUG_LEVELING(DEBUG_LEVEL_ERROR, ("odpg disable failed\n"));
217 return MV_FAIL;
218 }
219
220 ddr3_tip_if_write(0, ACCESS_TYPE_MULTICAST, PARAM_NOT_CARE,
221 ODPG_DATA_CTRL_REG, 0, MASK_ALL_BITS);
222
223 /* double loop on bus, pup */
224 for (if_id = 0; if_id <= MAX_INTERFACE_NUM - 1; if_id++) {
225 VALIDATE_IF_ACTIVE(tm->if_act_mask, if_id);
226 /* check training done */
227 is_any_pup_fail = 0;
228 for (bus_num = 0;
229 bus_num < octets_per_if_num;
230 bus_num++) {
231 VALIDATE_BUS_ACTIVE(tm->bus_act_mask, bus_num);
232 if (ddr3_tip_if_polling
233 (dev_num, ACCESS_TYPE_UNICAST,
234 if_id, (1 << 25), (1 << 25),
235 mask_results_pup_reg_map[bus_num],
236 MAX_POLLING_ITERATIONS) != MV_OK) {
237 DEBUG_LEVELING(DEBUG_LEVEL_ERROR,
238 ("\n_r_l: DDR3 poll failed(2) for IF %d CS %d bus %d",
239 if_id, effective_cs, bus_num));
240 is_any_pup_fail = 1;
241 } else {
242 /* read result per pup */
243 CHECK_STATUS(ddr3_tip_if_read
244 (dev_num,
245 ACCESS_TYPE_UNICAST,
246 if_id,
247 mask_results_pup_reg_map
248 [bus_num], data_read,
249 0xff));
250 rl_values[effective_cs][bus_num]
251 [if_id] = (u8)data_read[if_id];
252 }
253 }
254
255 if (is_any_pup_fail == 1) {
256 training_result[training_stage][if_id] =
257 TEST_FAILED;
258 if (debug_mode == 0)
259 return MV_FAIL;
260 }
261 }
262
263 DEBUG_LEVELING(DEBUG_LEVEL_INFO, ("RL exit read leveling\n"));
264
265 /*
266 * Phase 3: Exit Read Leveling
267 */
268
269 CHECK_STATUS(ddr3_tip_if_write
270 (dev_num, ACCESS_TYPE_MULTICAST, PARAM_NOT_CARE,
271 TRAINING_SW_2_REG, (1 << 3), (1 << 3)));
272 CHECK_STATUS(ddr3_tip_if_write
273 (dev_num, ACCESS_TYPE_MULTICAST, PARAM_NOT_CARE,
274 TRAINING_SW_1_REG, (1 << 16), (1 << 16)));
275 /* set ODPG to functional */
276 CHECK_STATUS(ddr3_tip_if_write
277 (dev_num, ACCESS_TYPE_MULTICAST, PARAM_NOT_CARE,
278 ODPG_DATA_CTRL_REG, 0x0, MASK_ALL_BITS));
279
280 /*
281 * Copy the result from the effective CS search to the
282 * real Functional CS
283 */
284 /*ddr3_tip_write_cs_result(dev_num, RL_PHY_REG(0); */
285 CHECK_STATUS(ddr3_tip_if_write
286 (dev_num, ACCESS_TYPE_MULTICAST, PARAM_NOT_CARE,
287 ODPG_DATA_CTRL_REG, 0x0, MASK_ALL_BITS));
288 }
289
290 for (effective_cs = 0; effective_cs < max_cs; effective_cs++) {
291 /* double loop on bus, pup */
292 for (if_id = 0; if_id <= MAX_INTERFACE_NUM - 1; if_id++) {
293 VALIDATE_IF_ACTIVE(tm->if_act_mask, if_id);
294 for (bus_num = 0;
295 bus_num < octets_per_if_num;
296 bus_num++) {
297 VALIDATE_BUS_ACTIVE(tm->bus_act_mask, bus_num);
298 /* read result per pup from arry */
299 data = rl_values[effective_cs][bus_num][if_id];
300 data = (data & 0x1f) |
301 (((data & 0xe0) >> 5) << 6);
302 ddr3_tip_bus_write(dev_num,
303 ACCESS_TYPE_UNICAST,
304 if_id,
305 ACCESS_TYPE_UNICAST,
306 bus_num, DDR_PHY_DATA,
307 RL_PHY_REG(effective_cs),
308 data);
309 }
310 }
311 }
312 /* Set to 0 after each loop to avoid illegal value may be used */
313 effective_cs = 0;
314
315 for (if_id = 0; if_id <= MAX_INTERFACE_NUM - 1; if_id++) {
316 VALIDATE_IF_ACTIVE(tm->if_act_mask, if_id);
317 /* restore cs enable value */
318 CHECK_STATUS(ddr3_tip_if_write
319 (dev_num, ACCESS_TYPE_UNICAST, if_id,
320 DUAL_DUNIT_CFG_REG, cs_enable_reg_val[if_id],
321 MASK_ALL_BITS));
322 if (odt_config != 0) {
323 CHECK_STATUS(ddr3_tip_write_additional_odt_setting
324 (dev_num, if_id));
325 }
326 }
327
328 for (if_id = 0; if_id <= MAX_INTERFACE_NUM - 1; if_id++) {
329 VALIDATE_IF_ACTIVE(tm->if_act_mask, if_id);
330 if (training_result[training_stage][if_id] == TEST_FAILED)
331 return MV_FAIL;
332 }
333
334 return MV_OK;
335 }
336
337 /*
338 * Legacy Dynamic write leveling
339 */
ddr3_tip_legacy_dynamic_write_leveling(u32 dev_num)340 int ddr3_tip_legacy_dynamic_write_leveling(u32 dev_num)
341 {
342 u32 c_cs, if_id, cs_mask = 0;
343 u32 max_cs = ddr3_tip_max_cs_get(dev_num);
344 struct mv_ddr_topology_map *tm = mv_ddr_topology_map_get();
345
346 /*
347 * In TRAINIUNG reg (0x15b0) write 0x80000008 | cs_mask:
348 * Trn_start
349 * cs_mask = 0x1 <<20 Trn_CS0 - CS0 is included in the DDR3 training
350 * cs_mask = 0x1 <<21 Trn_CS1 - CS1 is included in the DDR3 training
351 * cs_mask = 0x1 <<22 Trn_CS2 - CS2 is included in the DDR3 training
352 * cs_mask = 0x1 <<23 Trn_CS3 - CS3 is included in the DDR3 training
353 * Trn_auto_seq = write leveling
354 */
355 for (c_cs = 0; c_cs < max_cs; c_cs++)
356 cs_mask = cs_mask | 1 << (20 + c_cs);
357
358 for (if_id = 0; if_id < MAX_INTERFACE_NUM; if_id++) {
359 VALIDATE_IF_ACTIVE(tm->if_act_mask, if_id);
360 CHECK_STATUS(ddr3_tip_if_write
361 (dev_num, ACCESS_TYPE_MULTICAST, 0,
362 TRAINING_REG, (0x80000008 | cs_mask),
363 0xffffffff));
364 mdelay(20);
365 if (ddr3_tip_if_polling
366 (dev_num, ACCESS_TYPE_UNICAST, if_id, 0,
367 (u32)0x80000000, TRAINING_REG,
368 MAX_POLLING_ITERATIONS) != MV_OK) {
369 DEBUG_LEVELING(DEBUG_LEVEL_ERROR,
370 ("polling failed for Old WL result\n"));
371 return MV_FAIL;
372 }
373 }
374
375 return MV_OK;
376 }
377
378 /*
379 * Legacy Dynamic read leveling
380 */
ddr3_tip_legacy_dynamic_read_leveling(u32 dev_num)381 int ddr3_tip_legacy_dynamic_read_leveling(u32 dev_num)
382 {
383 u32 c_cs, if_id, cs_mask = 0;
384 u32 max_cs = ddr3_tip_max_cs_get(dev_num);
385 struct mv_ddr_topology_map *tm = mv_ddr_topology_map_get();
386
387 /*
388 * In TRAINIUNG reg (0x15b0) write 0x80000040 | cs_mask:
389 * Trn_start
390 * cs_mask = 0x1 <<20 Trn_CS0 - CS0 is included in the DDR3 training
391 * cs_mask = 0x1 <<21 Trn_CS1 - CS1 is included in the DDR3 training
392 * cs_mask = 0x1 <<22 Trn_CS2 - CS2 is included in the DDR3 training
393 * cs_mask = 0x1 <<23 Trn_CS3 - CS3 is included in the DDR3 training
394 * Trn_auto_seq = Read Leveling using training pattern
395 */
396 for (c_cs = 0; c_cs < max_cs; c_cs++)
397 cs_mask = cs_mask | 1 << (20 + c_cs);
398
399 CHECK_STATUS(ddr3_tip_if_write
400 (dev_num, ACCESS_TYPE_MULTICAST, 0, TRAINING_REG,
401 (0x80000040 | cs_mask), 0xffffffff));
402 mdelay(100);
403
404 for (if_id = 0; if_id < MAX_INTERFACE_NUM; if_id++) {
405 VALIDATE_IF_ACTIVE(tm->if_act_mask, if_id);
406 if (ddr3_tip_if_polling
407 (dev_num, ACCESS_TYPE_UNICAST, if_id, 0,
408 (u32)0x80000000, TRAINING_REG,
409 MAX_POLLING_ITERATIONS) != MV_OK) {
410 DEBUG_LEVELING(DEBUG_LEVEL_ERROR,
411 ("polling failed for Old RL result\n"));
412 return MV_FAIL;
413 }
414 }
415
416 return MV_OK;
417 }
418
419 /*
420 * Dynamic per bit read leveling
421 */
ddr3_tip_dynamic_per_bit_read_leveling(u32 dev_num,u32 freq)422 int ddr3_tip_dynamic_per_bit_read_leveling(u32 dev_num, u32 freq)
423 {
424 u32 data, mask;
425 u32 bus_num, if_id, cl_val, bit_num;
426 u32 curr_numb, curr_min_delay;
427 int adll_array[3] = { 0, -0xa, 0x14 };
428 u32 phyreg3_arr[MAX_INTERFACE_NUM][MAX_BUS_NUM];
429 enum hws_speed_bin speed_bin_index;
430 int is_any_pup_fail = 0;
431 int break_loop = 0;
432 u32 cs_enable_reg_val[MAX_INTERFACE_NUM]; /* save current CS value */
433 u32 data_read[MAX_INTERFACE_NUM];
434 int per_bit_rl_pup_status[MAX_INTERFACE_NUM][MAX_BUS_NUM];
435 u32 data2_write[MAX_INTERFACE_NUM][MAX_BUS_NUM];
436 struct pattern_info *pattern_table = ddr3_tip_get_pattern_table();
437 u16 *mask_results_dq_reg_map = ddr3_tip_get_mask_results_dq_reg();
438 u32 octets_per_if_num = ddr3_tip_dev_attr_get(dev_num, MV_ATTR_OCTET_PER_INTERFACE);
439 struct mv_ddr_topology_map *tm = mv_ddr_topology_map_get();
440
441 for (if_id = 0; if_id < MAX_INTERFACE_NUM; if_id++) {
442 VALIDATE_IF_ACTIVE(tm->if_act_mask, if_id);
443 for (bus_num = 0;
444 bus_num <= octets_per_if_num; bus_num++) {
445 VALIDATE_BUS_ACTIVE(tm->bus_act_mask, bus_num);
446 per_bit_rl_pup_status[if_id][bus_num] = 0;
447 data2_write[if_id][bus_num] = 0;
448 /* read current value of phy register 0x3 */
449 CHECK_STATUS(ddr3_tip_bus_read
450 (dev_num, if_id, ACCESS_TYPE_UNICAST,
451 bus_num, DDR_PHY_DATA,
452 CRX_PHY_REG(0),
453 &phyreg3_arr[if_id][bus_num]));
454 }
455 }
456
457 /* NEW RL machine */
458 for (if_id = 0; if_id <= MAX_INTERFACE_NUM - 1; if_id++) {
459 VALIDATE_IF_ACTIVE(tm->if_act_mask, if_id);
460 training_result[training_stage][if_id] = TEST_SUCCESS;
461
462 /* save current cs enable reg val */
463 CHECK_STATUS(ddr3_tip_if_read
464 (dev_num, ACCESS_TYPE_UNICAST, if_id,
465 DUAL_DUNIT_CFG_REG, &cs_enable_reg_val[if_id],
466 MASK_ALL_BITS));
467 /* enable single cs */
468 CHECK_STATUS(ddr3_tip_if_write
469 (dev_num, ACCESS_TYPE_UNICAST, if_id,
470 DUAL_DUNIT_CFG_REG, (1 << 3), (1 << 3)));
471 }
472
473 ddr3_tip_reset_fifo_ptr(dev_num);
474 for (curr_numb = 0; curr_numb < 3; curr_numb++) {
475 /*
476 * Phase 1: Load pattern (using ODPG)
477 *
478 * enter Read Leveling mode
479 * only 27 bits are masked
480 * assuming non multi-CS configuration
481 * write to CS = 0 for the non multi CS configuration, note that
482 * the results shall be read back to the required CS !!!
483 */
484
485 /* BUS count is 0 shifted 26 */
486 CHECK_STATUS(ddr3_tip_if_write
487 (dev_num, ACCESS_TYPE_MULTICAST, PARAM_NOT_CARE,
488 ODPG_DATA_CTRL_REG, 0x3, 0x3));
489 CHECK_STATUS(ddr3_tip_configure_odpg
490 (dev_num, ACCESS_TYPE_MULTICAST, PARAM_NOT_CARE, 0,
491 pattern_table[PATTERN_TEST].num_of_phases_tx, 0,
492 pattern_table[PATTERN_TEST].num_of_phases_rx, 0,
493 0, 0, STRESS_NONE, DURATION_SINGLE));
494
495 /* load pattern to ODPG */
496 ddr3_tip_load_pattern_to_odpg(dev_num, ACCESS_TYPE_MULTICAST,
497 PARAM_NOT_CARE, PATTERN_TEST,
498 pattern_table[PATTERN_TEST].
499 start_addr);
500
501 /*
502 * Phase 2: ODPG to Read Leveling mode
503 */
504
505 /* General Training Opcode register */
506 CHECK_STATUS(ddr3_tip_if_write
507 (dev_num, ACCESS_TYPE_MULTICAST, PARAM_NOT_CARE,
508 ODPG_WR_RD_MODE_ENA_REG, 0,
509 MASK_ALL_BITS));
510 CHECK_STATUS(ddr3_tip_if_write
511 (dev_num, ACCESS_TYPE_MULTICAST, PARAM_NOT_CARE,
512 GENERAL_TRAINING_OPCODE_REG, 0x301b01, 0x3c3fef));
513
514 /* Object1 opcode register 0 & 1 */
515 for (if_id = 0; if_id <= MAX_INTERFACE_NUM - 1; if_id++) {
516 VALIDATE_IF_ACTIVE(tm->if_act_mask, if_id);
517 speed_bin_index =
518 tm->interface_params[if_id].speed_bin_index;
519 cl_val =
520 cas_latency_table[speed_bin_index].cl_val[freq];
521 data = (cl_val << 17) | (0x3 << 25);
522 mask = (0xff << 9) | (0x1f << 17) | (0x3 << 25);
523 CHECK_STATUS(ddr3_tip_if_write
524 (dev_num, ACCESS_TYPE_UNICAST, if_id,
525 OPCODE_REG0_REG(1), data, mask));
526 }
527
528 /* Set iteration count to max value */
529 CHECK_STATUS(ddr3_tip_if_write
530 (dev_num, ACCESS_TYPE_MULTICAST, PARAM_NOT_CARE,
531 OPCODE_REG1_REG(1), 0xd00, 0xd00));
532
533 /*
534 * Phase 2: Mask config
535 */
536
537 ddr3_tip_dynamic_per_bit_read_leveling_seq(dev_num);
538
539 /*
540 * Phase 3: Read Leveling execution
541 */
542
543 /* temporary jira dunit=14751 */
544 CHECK_STATUS(ddr3_tip_if_write
545 (dev_num, ACCESS_TYPE_MULTICAST, PARAM_NOT_CARE,
546 TRAINING_DBG_1_REG, 0, (u32)(1 << 31)));
547 /* configure phy reset value */
548 CHECK_STATUS(ddr3_tip_if_write
549 (dev_num, ACCESS_TYPE_MULTICAST, PARAM_NOT_CARE,
550 TRAINING_DBG_3_REG, (0x7f << 24),
551 (u32)(0xff << 24)));
552 /* data pup rd reset enable */
553 CHECK_STATUS(ddr3_tip_if_write
554 (dev_num, ACCESS_TYPE_MULTICAST, PARAM_NOT_CARE,
555 SDRAM_CFG_REG, 0, (1 << 30)));
556 /* data pup rd reset disable */
557 CHECK_STATUS(ddr3_tip_if_write
558 (dev_num, ACCESS_TYPE_MULTICAST, PARAM_NOT_CARE,
559 SDRAM_CFG_REG, (1 << 30), (1 << 30)));
560 /* training SW override & training RL mode */
561 CHECK_STATUS(ddr3_tip_if_write
562 (dev_num, ACCESS_TYPE_MULTICAST, PARAM_NOT_CARE,
563 TRAINING_SW_2_REG, 0x1, 0x9));
564 /* training enable */
565 CHECK_STATUS(ddr3_tip_if_write
566 (dev_num, ACCESS_TYPE_MULTICAST, PARAM_NOT_CARE,
567 TRAINING_REG, (1 << 24) | (1 << 20),
568 (1 << 24) | (1 << 20)));
569 CHECK_STATUS(ddr3_tip_if_write
570 (dev_num, ACCESS_TYPE_MULTICAST, PARAM_NOT_CARE,
571 TRAINING_REG, (u32)(1 << 31), (u32)(1 << 31)));
572
573 /* trigger training */
574 mv_ddr_training_enable();
575
576 /* check for training done */
577 if (mv_ddr_is_training_done(MAX_POLLING_ITERATIONS, &data) != MV_OK) {
578 DEBUG_LEVELING(DEBUG_LEVEL_ERROR, ("training done failed\n"));
579 return MV_FAIL;
580 }
581 /* check for training pass */
582 if (data != PASS)
583 DEBUG_LEVELING(DEBUG_LEVEL_INFO, ("training result failed\n"));
584
585 /* disable odpg; switch back to functional mode */
586 mv_ddr_odpg_disable();
587
588 if (mv_ddr_is_odpg_done(MAX_POLLING_ITERATIONS) != MV_OK) {
589 DEBUG_LEVELING(DEBUG_LEVEL_ERROR, ("odpg disable failed\n"));
590 return MV_FAIL;
591 }
592
593 ddr3_tip_if_write(0, ACCESS_TYPE_MULTICAST, PARAM_NOT_CARE,
594 ODPG_DATA_CTRL_REG, 0, MASK_ALL_BITS);
595
596 /* double loop on bus, pup */
597 for (if_id = 0; if_id <= MAX_INTERFACE_NUM - 1; if_id++) {
598 VALIDATE_IF_ACTIVE(tm->if_act_mask, if_id);
599 /* check training done */
600 for (bus_num = 0;
601 bus_num < octets_per_if_num;
602 bus_num++) {
603 VALIDATE_BUS_ACTIVE(tm->bus_act_mask, bus_num);
604
605 if (per_bit_rl_pup_status[if_id][bus_num]
606 == 0) {
607 curr_min_delay = 0;
608 for (bit_num = 0; bit_num < 8;
609 bit_num++) {
610 if (ddr3_tip_if_polling
611 (dev_num,
612 ACCESS_TYPE_UNICAST,
613 if_id, (1 << 25),
614 (1 << 25),
615 mask_results_dq_reg_map
616 [bus_num * 8 + bit_num],
617 MAX_POLLING_ITERATIONS) !=
618 MV_OK) {
619 DEBUG_LEVELING
620 (DEBUG_LEVEL_ERROR,
621 ("\n_r_l: DDR3 poll failed(2) for bus %d bit %d\n",
622 bus_num,
623 bit_num));
624 } else {
625 /* read result per pup */
626 CHECK_STATUS
627 (ddr3_tip_if_read
628 (dev_num,
629 ACCESS_TYPE_UNICAST,
630 if_id,
631 mask_results_dq_reg_map
632 [bus_num * 8 +
633 bit_num],
634 data_read,
635 MASK_ALL_BITS));
636 data =
637 (data_read
638 [if_id] &
639 0x1f) |
640 ((data_read
641 [if_id] &
642 0xe0) << 1);
643 if (curr_min_delay == 0)
644 curr_min_delay =
645 data;
646 else if (data <
647 curr_min_delay)
648 curr_min_delay =
649 data;
650 if (data > data2_write[if_id][bus_num])
651 data2_write
652 [if_id]
653 [bus_num] =
654 data;
655 }
656 }
657
658 if (data2_write[if_id][bus_num] <=
659 (curr_min_delay +
660 MAX_DQ_READ_LEVELING_DELAY)) {
661 per_bit_rl_pup_status[if_id]
662 [bus_num] = 1;
663 }
664 }
665 }
666 }
667
668 /* check if there is need to search new phyreg3 value */
669 if (curr_numb < 2) {
670 /* if there is DLL that is not checked yet */
671 for (if_id = 0; if_id <= MAX_INTERFACE_NUM - 1;
672 if_id++) {
673 VALIDATE_IF_ACTIVE(tm->if_act_mask, if_id);
674 for (bus_num = 0;
675 bus_num < octets_per_if_num;
676 bus_num++) {
677 VALIDATE_BUS_ACTIVE(tm->bus_act_mask,
678 bus_num);
679 if (per_bit_rl_pup_status[if_id]
680 [bus_num] != 1) {
681 /* go to next ADLL value */
682 CHECK_STATUS
683 (ddr3_tip_bus_write
684 (dev_num,
685 ACCESS_TYPE_UNICAST,
686 if_id,
687 ACCESS_TYPE_UNICAST,
688 bus_num, DDR_PHY_DATA,
689 CRX_PHY_REG(0),
690 (phyreg3_arr[if_id]
691 [bus_num] +
692 adll_array[curr_numb])));
693 break_loop = 1;
694 break;
695 }
696 }
697 if (break_loop)
698 break;
699 }
700 } /* if (curr_numb < 2) */
701 if (!break_loop)
702 break;
703 } /* for ( curr_numb = 0; curr_numb <3; curr_numb++) */
704
705 for (if_id = 0; if_id <= MAX_INTERFACE_NUM - 1; if_id++) {
706 VALIDATE_IF_ACTIVE(tm->if_act_mask, if_id);
707 for (bus_num = 0; bus_num < octets_per_if_num;
708 bus_num++) {
709 VALIDATE_BUS_ACTIVE(tm->bus_act_mask, bus_num);
710 if (per_bit_rl_pup_status[if_id][bus_num] == 1)
711 ddr3_tip_bus_write(dev_num,
712 ACCESS_TYPE_UNICAST,
713 if_id,
714 ACCESS_TYPE_UNICAST,
715 bus_num, DDR_PHY_DATA,
716 RL_PHY_REG(effective_cs),
717 data2_write[if_id]
718 [bus_num]);
719 else
720 is_any_pup_fail = 1;
721 }
722
723 /* TBD flow does not support multi CS */
724 /*
725 * cs_bitmask = tm->interface_params[if_id].
726 * as_bus_params[bus_num].cs_bitmask;
727 */
728 /* divide by 4 is used for retrieving the CS number */
729 /*
730 * TBD BC2 - what is the PHY address for other
731 * CS ddr3_tip_write_cs_result() ???
732 */
733 /*
734 * find what should be written to PHY
735 * - max delay that is less than threshold
736 */
737 if (is_any_pup_fail == 1) {
738 training_result[training_stage][if_id] = TEST_FAILED;
739 if (debug_mode == 0)
740 return MV_FAIL;
741 }
742 }
743 DEBUG_LEVELING(DEBUG_LEVEL_INFO, ("RL exit read leveling\n"));
744
745 /*
746 * Phase 3: Exit Read Leveling
747 */
748
749 CHECK_STATUS(ddr3_tip_if_write
750 (dev_num, ACCESS_TYPE_MULTICAST, PARAM_NOT_CARE,
751 TRAINING_SW_2_REG, (1 << 3), (1 << 3)));
752 CHECK_STATUS(ddr3_tip_if_write
753 (dev_num, ACCESS_TYPE_MULTICAST, PARAM_NOT_CARE,
754 TRAINING_SW_1_REG, (1 << 16), (1 << 16)));
755 /* set ODPG to functional */
756 CHECK_STATUS(ddr3_tip_if_write
757 (dev_num, ACCESS_TYPE_MULTICAST, PARAM_NOT_CARE,
758 ODPG_DATA_CTRL_REG, 0x0, MASK_ALL_BITS));
759 /*
760 * Copy the result from the effective CS search to the real
761 * Functional CS
762 */
763 ddr3_tip_write_cs_result(dev_num, RL_PHY_REG(0));
764 CHECK_STATUS(ddr3_tip_if_write
765 (dev_num, ACCESS_TYPE_MULTICAST, PARAM_NOT_CARE,
766 ODPG_DATA_CTRL_REG, 0x0, MASK_ALL_BITS));
767
768 for (if_id = 0; if_id <= MAX_INTERFACE_NUM - 1; if_id++) {
769 VALIDATE_IF_ACTIVE(tm->if_act_mask, if_id);
770 /* restore cs enable value */
771 CHECK_STATUS(ddr3_tip_if_write
772 (dev_num, ACCESS_TYPE_UNICAST, if_id,
773 DUAL_DUNIT_CFG_REG, cs_enable_reg_val[if_id],
774 MASK_ALL_BITS));
775 if (odt_config != 0) {
776 CHECK_STATUS(ddr3_tip_write_additional_odt_setting
777 (dev_num, if_id));
778 }
779 }
780
781 for (if_id = 0; if_id <= MAX_INTERFACE_NUM - 1; if_id++) {
782 VALIDATE_IF_ACTIVE(tm->if_act_mask, if_id);
783 if (training_result[training_stage][if_id] == TEST_FAILED)
784 return MV_FAIL;
785 }
786
787 return MV_OK;
788 }
789
ddr3_tip_calc_cs_mask(u32 dev_num,u32 if_id,u32 effective_cs,u32 * cs_mask)790 int ddr3_tip_calc_cs_mask(u32 dev_num, u32 if_id, u32 effective_cs,
791 u32 *cs_mask)
792 {
793 u32 all_bus_cs = 0, same_bus_cs;
794 u32 bus_cnt;
795 u32 octets_per_if_num = ddr3_tip_dev_attr_get(dev_num, MV_ATTR_OCTET_PER_INTERFACE);
796 struct mv_ddr_topology_map *tm = mv_ddr_topology_map_get();
797
798 *cs_mask = same_bus_cs = CS_BIT_MASK;
799
800 /*
801 * In some of the devices (such as BC2), the CS is per pup and there
802 * for mixed mode is valid on like other devices where CS configuration
803 * is per interface.
804 * In order to know that, we do 'Or' and 'And' operation between all
805 * CS (of the pups).
806 * If they are they are not the same then it's mixed mode so all CS
807 * should be configured (when configuring the MRS)
808 */
809 for (bus_cnt = 0; bus_cnt < octets_per_if_num; bus_cnt++) {
810 VALIDATE_BUS_ACTIVE(tm->bus_act_mask, bus_cnt);
811
812 all_bus_cs |= tm->interface_params[if_id].
813 as_bus_params[bus_cnt].cs_bitmask;
814 same_bus_cs &= tm->interface_params[if_id].
815 as_bus_params[bus_cnt].cs_bitmask;
816
817 /* cs enable is active low */
818 *cs_mask &= ~tm->interface_params[if_id].
819 as_bus_params[bus_cnt].cs_bitmask;
820 }
821
822 if (all_bus_cs == same_bus_cs)
823 *cs_mask = (*cs_mask | (~(1 << effective_cs))) & CS_BIT_MASK;
824
825 return MV_OK;
826 }
827
828 /*
829 * Dynamic write leveling
830 */
ddr3_tip_dynamic_write_leveling(u32 dev_num,int phase_remove)831 int ddr3_tip_dynamic_write_leveling(u32 dev_num, int phase_remove)
832 {
833 u32 reg_data = 0, temp = 0, iter, if_id, bus_cnt;
834 u32 cs_enable_reg_val[MAX_INTERFACE_NUM] = { 0 };
835 u32 cs_mask[MAX_INTERFACE_NUM];
836 u32 read_data_sample_delay_vals[MAX_INTERFACE_NUM] = { 0 };
837 u32 read_data_ready_delay_vals[MAX_INTERFACE_NUM] = { 0 };
838 /* 0 for failure */
839 u32 res_values[MAX_INTERFACE_NUM * MAX_BUS_NUM] = { 0 };
840 u32 test_res = 0; /* 0 - success for all pup */
841 u32 data_read[MAX_INTERFACE_NUM];
842 u8 wl_values[NUM_OF_CS][MAX_BUS_NUM][MAX_INTERFACE_NUM];
843 u16 *mask_results_pup_reg_map = ddr3_tip_get_mask_results_pup_reg_map();
844 u32 cs_mask0[MAX_INTERFACE_NUM] = { 0 };
845 u32 max_cs = ddr3_tip_max_cs_get(dev_num);
846 u32 octets_per_if_num = ddr3_tip_dev_attr_get(dev_num, MV_ATTR_OCTET_PER_INTERFACE);
847 struct mv_ddr_topology_map *tm = mv_ddr_topology_map_get();
848
849 for (if_id = 0; if_id <= MAX_INTERFACE_NUM - 1; if_id++) {
850 VALIDATE_IF_ACTIVE(tm->if_act_mask, if_id);
851
852 training_result[training_stage][if_id] = TEST_SUCCESS;
853
854 /* save Read Data Sample Delay */
855 CHECK_STATUS(ddr3_tip_if_read
856 (dev_num, ACCESS_TYPE_UNICAST, if_id,
857 RD_DATA_SMPL_DLYS_REG,
858 read_data_sample_delay_vals, MASK_ALL_BITS));
859 /* save Read Data Ready Delay */
860 CHECK_STATUS(ddr3_tip_if_read
861 (dev_num, ACCESS_TYPE_UNICAST, if_id,
862 RD_DATA_RDY_DLYS_REG, read_data_ready_delay_vals,
863 MASK_ALL_BITS));
864 /* save current cs reg val */
865 CHECK_STATUS(ddr3_tip_if_read
866 (dev_num, ACCESS_TYPE_UNICAST, if_id,
867 DUAL_DUNIT_CFG_REG, cs_enable_reg_val, MASK_ALL_BITS));
868 }
869
870 if (ddr3_tip_dev_attr_get(dev_num, MV_ATTR_TIP_REV) < MV_TIP_REV_3) {
871 /* Enable multi-CS */
872 CHECK_STATUS(ddr3_tip_if_write
873 (dev_num, ACCESS_TYPE_UNICAST, if_id,
874 DUAL_DUNIT_CFG_REG, 0, (1 << 3)));
875 }
876
877 /*
878 * Phase 1: DRAM 2 Write Leveling mode
879 */
880
881 /*Assert 10 refresh commands to DRAM to all CS */
882 for (iter = 0; iter < WL_ITERATION_NUM; iter++) {
883 for (if_id = 0; if_id <= MAX_INTERFACE_NUM - 1; if_id++) {
884 VALIDATE_IF_ACTIVE(tm->if_act_mask, if_id);
885 CHECK_STATUS(ddr3_tip_if_write
886 (dev_num, ACCESS_TYPE_UNICAST,
887 if_id, SDRAM_OP_REG,
888 (u32)((~(0xf) << 8) | 0x2), 0xf1f));
889 }
890 }
891 /* check controller back to normal */
892 for (if_id = 0; if_id <= MAX_INTERFACE_NUM - 1; if_id++) {
893 VALIDATE_IF_ACTIVE(tm->if_act_mask, if_id);
894 if (ddr3_tip_if_polling
895 (dev_num, ACCESS_TYPE_UNICAST, if_id, 0, 0x1f,
896 SDRAM_OP_REG, MAX_POLLING_ITERATIONS) != MV_OK) {
897 DEBUG_LEVELING(DEBUG_LEVEL_ERROR,
898 ("WL: DDR3 poll failed(3)"));
899 }
900 }
901
902 for (effective_cs = 0; effective_cs < max_cs; effective_cs++) {
903 /*enable write leveling to all cs - Q off , WL n */
904 /* calculate interface cs mask */
905 CHECK_STATUS(ddr3_tip_write_mrs_cmd(dev_num, cs_mask0, MR_CMD1,
906 0x1000, 0x1080));
907
908 for (if_id = 0; if_id <= MAX_INTERFACE_NUM - 1; if_id++) {
909 VALIDATE_IF_ACTIVE(tm->if_act_mask, if_id);
910 /* cs enable is active low */
911 ddr3_tip_calc_cs_mask(dev_num, if_id, effective_cs,
912 &cs_mask[if_id]);
913 }
914
915 if (ddr3_tip_dev_attr_get(dev_num, MV_ATTR_TIP_REV) >= MV_TIP_REV_3) {
916 /* Enable Output buffer to relevant CS - Q on , WL on */
917 CHECK_STATUS(ddr3_tip_write_mrs_cmd
918 (dev_num, cs_mask, MR_CMD1, 0x80, 0x1080));
919
920 /*enable odt for relevant CS */
921 CHECK_STATUS(ddr3_tip_if_write
922 (dev_num, ACCESS_TYPE_MULTICAST, PARAM_NOT_CARE,
923 0x1498, (0x3 << (effective_cs * 2)), 0xf));
924 } else {
925 /* FIXME: should be the same as _CPU case */
926 CHECK_STATUS(ddr3_tip_write_mrs_cmd
927 (dev_num, cs_mask, MR_CMD1, 0xc0, 0x12c4));
928 }
929
930 /*
931 * Phase 2: Set training IP to write leveling mode
932 */
933
934 CHECK_STATUS(ddr3_tip_dynamic_write_leveling_seq(dev_num));
935
936 /* phase 3: trigger training */
937 mv_ddr_training_enable();
938
939 /* check for training done */
940 if (mv_ddr_is_training_done(MAX_POLLING_ITERATIONS, data_read) != MV_OK) {
941 DEBUG_LEVELING(DEBUG_LEVEL_ERROR, ("training done failed\n"));
942 } else { /* check for training pass */
943 reg_data = data_read[0];
944 #if defined(CONFIG_ARMADA_38X) /* JIRA #1498 for 16 bit with ECC */
945 if (tm->bus_act_mask == 0xb) /* set to data to 0 to skip the check */
946 reg_data = 0;
947 #endif
948 if (reg_data != PASS)
949 DEBUG_LEVELING(DEBUG_LEVEL_INFO, ("training result failed\n"));
950
951 /* check for training completion per bus */
952 for (bus_cnt = 0; bus_cnt < octets_per_if_num; bus_cnt++) {
953 VALIDATE_BUS_ACTIVE(tm->bus_act_mask, bus_cnt);
954 /* training status */
955 ddr3_tip_if_read(0, ACCESS_TYPE_UNICAST, 0,
956 mask_results_pup_reg_map[bus_cnt],
957 data_read, MASK_ALL_BITS);
958 reg_data = data_read[0];
959 DEBUG_LEVELING(DEBUG_LEVEL_TRACE, ("WL: IF %d BUS %d reg 0x%x\n",
960 0, bus_cnt, reg_data));
961 if ((reg_data & (1 << 25)) == 0)
962 res_values[bus_cnt] = 1;
963 ddr3_tip_if_read(0, ACCESS_TYPE_UNICAST, 0,
964 mask_results_pup_reg_map[bus_cnt],
965 data_read, 0xff);
966 /*
967 * Save the read value that should be
968 * write to PHY register
969 */
970 wl_values[effective_cs][bus_cnt][0] = (u8)data_read[0];
971 }
972 }
973
974 /*
975 * Phase 3.5: Validate result
976 */
977 for (if_id = 0; if_id < MAX_INTERFACE_NUM; if_id++) {
978 VALIDATE_IF_ACTIVE(tm->if_act_mask, if_id);
979 for (bus_cnt = 0; bus_cnt < octets_per_if_num; bus_cnt++) {
980 VALIDATE_BUS_ACTIVE(tm->bus_act_mask, bus_cnt);
981 /*
982 * Read result control register according to subphy
983 * "16" below is for a half-phase
984 */
985 reg_data = wl_values[effective_cs][bus_cnt][if_id] + 16;
986 /*
987 * Write to WL register: ADLL [4:0], Phase [8:6],
988 * Centralization ADLL [15:10] + 0x10
989 */
990 reg_data = (reg_data & 0x1f) |
991 (((reg_data & 0xe0) >> 5) << 6) |
992 (((reg_data & 0x1f) + phy_reg1_val) << 10);
993 /* Search with WL CS0 subphy reg */
994 ddr3_tip_bus_write(dev_num, ACCESS_TYPE_UNICAST, if_id,
995 ACCESS_TYPE_UNICAST, bus_cnt,
996 DDR_PHY_DATA, WL_PHY_REG(0), reg_data);
997 /*
998 * Check for change in data read from DRAM.
999 * If changed, fix the result
1000 */
1001 CHECK_STATUS(ddr3_tip_if_read
1002 (dev_num,
1003 ACCESS_TYPE_UNICAST,
1004 if_id,
1005 TRAINING_WL_REG,
1006 data_read, MASK_ALL_BITS));
1007 if (((data_read[if_id] & (1 << (bus_cnt + 20))) >>
1008 (bus_cnt + 20)) == 0) {
1009 DEBUG_LEVELING(
1010 DEBUG_LEVEL_ERROR,
1011 ("WLValues was changed from 0x%X",
1012 wl_values[effective_cs]
1013 [bus_cnt][if_id]));
1014 wl_values[effective_cs]
1015 [bus_cnt][if_id] += 32;
1016 DEBUG_LEVELING(
1017 DEBUG_LEVEL_ERROR,
1018 ("to 0x%X",
1019 wl_values[effective_cs]
1020 [bus_cnt][if_id]));
1021 }
1022 }
1023 }
1024
1025 /*
1026 * Phase 4: Exit write leveling mode
1027 */
1028
1029 /* disable DQs toggling */
1030 CHECK_STATUS(ddr3_tip_if_write
1031 (dev_num, ACCESS_TYPE_MULTICAST, PARAM_NOT_CARE,
1032 WL_DQS_PATTERN_REG, 0x0, 0x1));
1033
1034 /* Update MRS 1 (WL off) */
1035 if (ddr3_tip_dev_attr_get(dev_num, MV_ATTR_TIP_REV) >= MV_TIP_REV_3) {
1036 CHECK_STATUS(ddr3_tip_write_mrs_cmd(dev_num, cs_mask0, MR_CMD1,
1037 0x1000, 0x1080));
1038 } else {
1039 /* FIXME: should be same as _CPU case */
1040 CHECK_STATUS(ddr3_tip_write_mrs_cmd(dev_num, cs_mask0, MR_CMD1,
1041 0x1000, 0x12c4));
1042 }
1043
1044 /* Update MRS 1 (return to functional mode - Q on , WL off) */
1045 CHECK_STATUS(ddr3_tip_write_mrs_cmd
1046 (dev_num, cs_mask0, MR_CMD1, 0x0, 0x1080));
1047
1048 /* set phy to normal mode */
1049 CHECK_STATUS(ddr3_tip_if_write
1050 (dev_num, ACCESS_TYPE_MULTICAST, PARAM_NOT_CARE,
1051 TRAINING_SW_2_REG, 0x5, 0x7));
1052
1053 /* exit sw override mode */
1054 CHECK_STATUS(ddr3_tip_if_write
1055 (dev_num, ACCESS_TYPE_MULTICAST, PARAM_NOT_CARE,
1056 TRAINING_SW_2_REG, 0x4, 0x7));
1057 }
1058
1059 /*
1060 * Phase 5: Load WL values to each PHY
1061 */
1062
1063 for (effective_cs = 0; effective_cs < max_cs; effective_cs++) {
1064 for (if_id = 0; if_id <= MAX_INTERFACE_NUM - 1; if_id++) {
1065 VALIDATE_IF_ACTIVE(tm->if_act_mask, if_id);
1066 test_res = 0;
1067 for (bus_cnt = 0;
1068 bus_cnt < octets_per_if_num;
1069 bus_cnt++) {
1070 VALIDATE_BUS_ACTIVE(tm->bus_act_mask, bus_cnt);
1071 /* check if result == pass */
1072 if (res_values
1073 [(if_id *
1074 octets_per_if_num) +
1075 bus_cnt] == 0) {
1076 /*
1077 * read result control register
1078 * according to pup
1079 */
1080 reg_data =
1081 wl_values[effective_cs][bus_cnt]
1082 [if_id];
1083 /*
1084 * Write into write leveling register
1085 * ([4:0] ADLL, [8:6] Phase, [15:10]
1086 * (centralization) ADLL + 0x10)
1087 */
1088 reg_data =
1089 (reg_data & 0x1f) |
1090 (((reg_data & 0xe0) >> 5) << 6) |
1091 (((reg_data & 0x1f) +
1092 phy_reg1_val) << 10);
1093 /*
1094 * in case phase remove should be executed
1095 * need to remove more than one phase.
1096 * this will take place only in low frequency,
1097 * where there could be more than one phase between sub-phys
1098 */
1099 if (phase_remove == 1) {
1100 temp = (reg_data >> WR_LVL_PH_SEL_OFFS) & WR_LVL_PH_SEL_PHASE1;
1101 reg_data &= ~(WR_LVL_PH_SEL_MASK << WR_LVL_PH_SEL_OFFS);
1102 reg_data |= (temp << WR_LVL_PH_SEL_OFFS);
1103 }
1104
1105 ddr3_tip_bus_write(
1106 dev_num,
1107 ACCESS_TYPE_UNICAST,
1108 if_id,
1109 ACCESS_TYPE_UNICAST,
1110 bus_cnt,
1111 DDR_PHY_DATA,
1112 WL_PHY_REG(effective_cs),
1113 reg_data);
1114 } else {
1115 test_res = 1;
1116 /*
1117 * read result control register
1118 * according to pup
1119 */
1120 CHECK_STATUS(ddr3_tip_if_read
1121 (dev_num,
1122 ACCESS_TYPE_UNICAST,
1123 if_id,
1124 mask_results_pup_reg_map
1125 [bus_cnt], data_read,
1126 0xff));
1127 reg_data = data_read[if_id];
1128 DEBUG_LEVELING(
1129 DEBUG_LEVEL_ERROR,
1130 ("WL: IF %d BUS %d failed, reg 0x%x\n",
1131 if_id, bus_cnt, reg_data));
1132 }
1133 }
1134
1135 if (test_res != 0) {
1136 training_result[training_stage][if_id] =
1137 TEST_FAILED;
1138 }
1139 }
1140 }
1141 /* Set to 0 after each loop to avoid illegal value may be used */
1142 effective_cs = 0;
1143
1144 /*
1145 * Copy the result from the effective CS search to the real
1146 * Functional CS
1147 */
1148 /* ddr3_tip_write_cs_result(dev_num, WL_PHY_REG(0); */
1149 /* restore saved values */
1150 for (if_id = 0; if_id <= MAX_INTERFACE_NUM - 1; if_id++) {
1151 VALIDATE_IF_ACTIVE(tm->if_act_mask, if_id);
1152 /* restore Read Data Sample Delay */
1153 CHECK_STATUS(ddr3_tip_if_write
1154 (dev_num, ACCESS_TYPE_UNICAST, if_id,
1155 RD_DATA_SMPL_DLYS_REG,
1156 read_data_sample_delay_vals[if_id],
1157 MASK_ALL_BITS));
1158
1159 /* restore Read Data Ready Delay */
1160 CHECK_STATUS(ddr3_tip_if_write
1161 (dev_num, ACCESS_TYPE_UNICAST, if_id,
1162 RD_DATA_RDY_DLYS_REG,
1163 read_data_ready_delay_vals[if_id],
1164 MASK_ALL_BITS));
1165
1166 /* enable multi cs */
1167 CHECK_STATUS(ddr3_tip_if_write
1168 (dev_num, ACCESS_TYPE_UNICAST, if_id,
1169 DUAL_DUNIT_CFG_REG, cs_enable_reg_val[if_id],
1170 MASK_ALL_BITS));
1171 }
1172
1173 if (ddr3_tip_dev_attr_get(dev_num, MV_ATTR_TIP_REV) >= MV_TIP_REV_3) {
1174 /* Disable modt0 for CS0 training - need to adjust for multi-CS
1175 * in case of ddr4 set 0xf else 0
1176 */
1177 if (odt_config != 0) {
1178 CHECK_STATUS(ddr3_tip_if_write(dev_num, ACCESS_TYPE_MULTICAST, PARAM_NOT_CARE,
1179 SDRAM_ODT_CTRL_HIGH_REG, 0x0, 0xf));
1180 }
1181 else {
1182 CHECK_STATUS(ddr3_tip_if_write(dev_num, ACCESS_TYPE_MULTICAST, PARAM_NOT_CARE,
1183 SDRAM_ODT_CTRL_HIGH_REG, 0xf, 0xf));
1184 }
1185
1186 }
1187
1188 for (if_id = 0; if_id <= MAX_INTERFACE_NUM - 1; if_id++) {
1189 VALIDATE_IF_ACTIVE(tm->if_act_mask, if_id);
1190 if (training_result[training_stage][if_id] == TEST_FAILED)
1191 return MV_FAIL;
1192 }
1193
1194 return MV_OK;
1195 }
1196
1197 /*
1198 * Dynamic write leveling supplementary
1199 */
ddr3_tip_dynamic_write_leveling_supp(u32 dev_num)1200 int ddr3_tip_dynamic_write_leveling_supp(u32 dev_num)
1201 {
1202 int adll_offset;
1203 u32 if_id, bus_id, data, data_tmp;
1204 int is_if_fail = 0;
1205 u32 octets_per_if_num = ddr3_tip_dev_attr_get(dev_num, MV_ATTR_OCTET_PER_INTERFACE);
1206 struct mv_ddr_topology_map *tm = mv_ddr_topology_map_get();
1207
1208 for (if_id = 0; if_id <= MAX_INTERFACE_NUM - 1; if_id++) {
1209 VALIDATE_IF_ACTIVE(tm->if_act_mask, if_id);
1210 is_if_fail = 0;
1211
1212 for (bus_id = 0; bus_id < octets_per_if_num; bus_id++) {
1213 VALIDATE_BUS_ACTIVE(tm->bus_act_mask, bus_id);
1214 wr_supp_res[if_id][bus_id].is_pup_fail = 1;
1215 CHECK_STATUS(ddr3_tip_bus_read
1216 (dev_num, if_id, ACCESS_TYPE_UNICAST,
1217 bus_id, DDR_PHY_DATA,
1218 CTX_PHY_REG(effective_cs),
1219 &data));
1220 DEBUG_LEVELING(
1221 DEBUG_LEVEL_TRACE,
1222 ("WL Supp: adll_offset=0 data delay = %d\n",
1223 data));
1224 if (ddr3_tip_wl_supp_align_phase_shift
1225 (dev_num, if_id, bus_id) == MV_OK) {
1226 DEBUG_LEVELING(
1227 DEBUG_LEVEL_TRACE,
1228 ("WL Supp: IF %d bus_id %d adll_offset=0 Success !\n",
1229 if_id, bus_id));
1230 continue;
1231 }
1232
1233 /* change adll */
1234 adll_offset = 5;
1235 CHECK_STATUS(ddr3_tip_bus_write
1236 (dev_num, ACCESS_TYPE_UNICAST, if_id,
1237 ACCESS_TYPE_UNICAST, bus_id, DDR_PHY_DATA,
1238 CTX_PHY_REG(effective_cs),
1239 data + adll_offset));
1240 CHECK_STATUS(ddr3_tip_bus_read
1241 (dev_num, if_id, ACCESS_TYPE_UNICAST,
1242 bus_id, DDR_PHY_DATA,
1243 CTX_PHY_REG(effective_cs),
1244 &data_tmp));
1245 DEBUG_LEVELING(
1246 DEBUG_LEVEL_TRACE,
1247 ("WL Supp: adll_offset= %d data delay = %d\n",
1248 adll_offset, data_tmp));
1249
1250 if (ddr3_tip_wl_supp_align_phase_shift
1251 (dev_num, if_id, bus_id) == MV_OK) {
1252 DEBUG_LEVELING(
1253 DEBUG_LEVEL_TRACE,
1254 ("WL Supp: IF %d bus_id %d adll_offset= %d Success !\n",
1255 if_id, bus_id, adll_offset));
1256 continue;
1257 }
1258
1259 /* change adll */
1260 adll_offset = -5;
1261 CHECK_STATUS(ddr3_tip_bus_write
1262 (dev_num, ACCESS_TYPE_UNICAST, if_id,
1263 ACCESS_TYPE_UNICAST, bus_id, DDR_PHY_DATA,
1264 CTX_PHY_REG(effective_cs),
1265 data + adll_offset));
1266 CHECK_STATUS(ddr3_tip_bus_read
1267 (dev_num, if_id, ACCESS_TYPE_UNICAST,
1268 bus_id, DDR_PHY_DATA,
1269 CTX_PHY_REG(effective_cs),
1270 &data_tmp));
1271 DEBUG_LEVELING(
1272 DEBUG_LEVEL_TRACE,
1273 ("WL Supp: adll_offset= %d data delay = %d\n",
1274 adll_offset, data_tmp));
1275 if (ddr3_tip_wl_supp_align_phase_shift
1276 (dev_num, if_id, bus_id) == MV_OK) {
1277 DEBUG_LEVELING(
1278 DEBUG_LEVEL_TRACE,
1279 ("WL Supp: IF %d bus_id %d adll_offset= %d Success !\n",
1280 if_id, bus_id, adll_offset));
1281 continue;
1282 } else {
1283 DEBUG_LEVELING(
1284 DEBUG_LEVEL_ERROR,
1285 ("WL Supp: IF %d bus_id %d Failed !\n",
1286 if_id, bus_id));
1287 is_if_fail = 1;
1288 }
1289 }
1290
1291 if (is_if_fail == 1) {
1292 DEBUG_LEVELING(DEBUG_LEVEL_ERROR,
1293 ("WL Supp: CS# %d: IF %d failed\n",
1294 effective_cs, if_id));
1295 training_result[training_stage][if_id] = TEST_FAILED;
1296 } else {
1297 training_result[training_stage][if_id] = TEST_SUCCESS;
1298 }
1299 }
1300
1301 for (if_id = 0; if_id <= MAX_INTERFACE_NUM - 1; if_id++) {
1302 VALIDATE_IF_ACTIVE(tm->if_act_mask, if_id);
1303 if (training_result[training_stage][if_id] == TEST_FAILED)
1304 return MV_FAIL;
1305 }
1306
1307 return MV_OK;
1308 }
1309
1310 /*
1311 * Phase Shift
1312 */
ddr3_tip_wl_supp_align_phase_shift(u32 dev_num,u32 if_id,u32 bus_id)1313 static int ddr3_tip_wl_supp_align_phase_shift(u32 dev_num, u32 if_id,
1314 u32 bus_id)
1315 {
1316 u32 original_phase;
1317 u32 data, write_data;
1318
1319 wr_supp_res[if_id][bus_id].stage = PHASE_SHIFT;
1320 if (ddr3_tip_xsb_compare_test
1321 (dev_num, if_id, bus_id, 0) == MV_OK)
1322 return MV_OK;
1323
1324 /* Read current phase */
1325 CHECK_STATUS(ddr3_tip_bus_read
1326 (dev_num, if_id, ACCESS_TYPE_UNICAST, bus_id,
1327 DDR_PHY_DATA, WL_PHY_REG(effective_cs), &data));
1328 original_phase = (data >> 6) & 0x7;
1329
1330 /* Set phase (0x0[6-8]) -2 */
1331 if (original_phase >= 1) {
1332 if (original_phase == 1)
1333 write_data = data & ~0x1df;
1334 else
1335 write_data = (data & ~0x1c0) |
1336 ((original_phase - 2) << 6);
1337 ddr3_tip_bus_write(dev_num, ACCESS_TYPE_UNICAST, if_id,
1338 ACCESS_TYPE_UNICAST, bus_id, DDR_PHY_DATA,
1339 WL_PHY_REG(effective_cs), write_data);
1340 if (ddr3_tip_xsb_compare_test
1341 (dev_num, if_id, bus_id, -2) == MV_OK)
1342 return MV_OK;
1343 }
1344
1345 /* Set phase (0x0[6-8]) +2 */
1346 if (original_phase <= 5) {
1347 write_data = (data & ~0x1c0) |
1348 ((original_phase + 2) << 6);
1349 ddr3_tip_bus_write(dev_num, ACCESS_TYPE_UNICAST, if_id,
1350 ACCESS_TYPE_UNICAST, bus_id, DDR_PHY_DATA,
1351 WL_PHY_REG(effective_cs), write_data);
1352 if (ddr3_tip_xsb_compare_test
1353 (dev_num, if_id, bus_id, 2) == MV_OK)
1354 return MV_OK;
1355 }
1356
1357 /* Set phase (0x0[6-8]) +4 */
1358 if (original_phase <= 3) {
1359 write_data = (data & ~0x1c0) |
1360 ((original_phase + 4) << 6);
1361 ddr3_tip_bus_write(dev_num, ACCESS_TYPE_UNICAST, if_id,
1362 ACCESS_TYPE_UNICAST, bus_id, DDR_PHY_DATA,
1363 WL_PHY_REG(effective_cs), write_data);
1364 if (ddr3_tip_xsb_compare_test
1365 (dev_num, if_id, bus_id, 4) == MV_OK)
1366 return MV_OK;
1367 }
1368
1369 /* Set phase (0x0[6-8]) +6 */
1370 if (original_phase <= 1) {
1371 write_data = (data & ~0x1c0) |
1372 ((original_phase + 6) << 6);
1373 ddr3_tip_bus_write(dev_num, ACCESS_TYPE_UNICAST, if_id,
1374 ACCESS_TYPE_UNICAST, bus_id, DDR_PHY_DATA,
1375 WL_PHY_REG(effective_cs), write_data);
1376 if (ddr3_tip_xsb_compare_test
1377 (dev_num, if_id, bus_id, 6) == MV_OK)
1378 return MV_OK;
1379 }
1380
1381 /* Write original WL result back */
1382 ddr3_tip_bus_write(dev_num, ACCESS_TYPE_UNICAST, if_id,
1383 ACCESS_TYPE_UNICAST, bus_id, DDR_PHY_DATA,
1384 WL_PHY_REG(effective_cs), data);
1385 wr_supp_res[if_id][bus_id].is_pup_fail = 1;
1386
1387 return MV_FAIL;
1388 }
1389
1390 /*
1391 * Compare Test
1392 */
ddr3_tip_xsb_compare_test(u32 dev_num,u32 if_id,u32 bus_id,u32 edge_offset)1393 static int ddr3_tip_xsb_compare_test(u32 dev_num, u32 if_id, u32 bus_id,
1394 u32 edge_offset)
1395 {
1396 u32 num_of_succ_byte_compare, word_in_pattern;
1397 u32 word_offset, i, num_of_word_mult;
1398 u32 read_pattern[TEST_PATTERN_LENGTH * 2];
1399 struct pattern_info *pattern_table = ddr3_tip_get_pattern_table();
1400 u32 pattern_test_pattern_table[8];
1401 struct mv_ddr_topology_map *tm = mv_ddr_topology_map_get();
1402
1403 /* 3 below for INTERFACE_BUS_MASK_16BIT */
1404 num_of_word_mult = (tm->bus_act_mask == 3) ? 1 : 2;
1405
1406 for (i = 0; i < 8; i++) {
1407 pattern_test_pattern_table[i] =
1408 pattern_table_get_word(dev_num, PATTERN_TEST, (u8)i);
1409 }
1410
1411 /* External write, read and compare */
1412 CHECK_STATUS(ddr3_tip_load_pattern_to_mem(dev_num, PATTERN_TEST));
1413
1414 CHECK_STATUS(ddr3_tip_reset_fifo_ptr(dev_num));
1415
1416 CHECK_STATUS(ddr3_tip_ext_read
1417 (dev_num, if_id,
1418 ((pattern_table[PATTERN_TEST].start_addr << 3) +
1419 ((SDRAM_CS_SIZE + 1) * effective_cs)), 1, read_pattern));
1420
1421 DEBUG_LEVELING(
1422 DEBUG_LEVEL_TRACE,
1423 ("XSB-compt CS#%d: IF %d bus_id %d 0x%x 0x%x 0x%x 0x%x 0x%x 0x%x 0x%x 0x%x\n",
1424 effective_cs, if_id, bus_id,
1425 read_pattern[0], read_pattern[1],
1426 read_pattern[2], read_pattern[3],
1427 read_pattern[4], read_pattern[5],
1428 read_pattern[6], read_pattern[7]));
1429
1430 /* compare byte per pup */
1431 num_of_succ_byte_compare = 0;
1432 for (word_in_pattern = start_xsb_offset;
1433 word_in_pattern < (TEST_PATTERN_LENGTH * num_of_word_mult);
1434 word_in_pattern++) {
1435 word_offset = word_in_pattern;
1436 if ((word_offset > (TEST_PATTERN_LENGTH * 2 - 1)))
1437 continue;
1438
1439 if ((read_pattern[word_in_pattern] & pup_mask_table[bus_id]) ==
1440 (pattern_test_pattern_table[word_offset] &
1441 pup_mask_table[bus_id]))
1442 num_of_succ_byte_compare++;
1443 }
1444
1445 if ((TEST_PATTERN_LENGTH * num_of_word_mult - start_xsb_offset) ==
1446 num_of_succ_byte_compare) {
1447 wr_supp_res[if_id][bus_id].stage = edge_offset;
1448 DEBUG_LEVELING(DEBUG_LEVEL_TRACE,
1449 ("supplementary: shift to %d for if %d pup %d success\n",
1450 edge_offset, if_id, bus_id));
1451 wr_supp_res[if_id][bus_id].is_pup_fail = 0;
1452
1453 return MV_OK;
1454 } else {
1455 DEBUG_LEVELING(
1456 DEBUG_LEVEL_TRACE,
1457 ("XSB-compt CS#%d: IF %d bus_id %d num_of_succ_byte_compare %d - Fail!\n",
1458 effective_cs, if_id, bus_id, num_of_succ_byte_compare));
1459
1460 DEBUG_LEVELING(
1461 DEBUG_LEVEL_TRACE,
1462 ("XSB-compt: expected 0x%x 0x%x 0x%x 0x%x 0x%x 0x%x 0x%x 0x%x\n",
1463 pattern_test_pattern_table[0],
1464 pattern_test_pattern_table[1],
1465 pattern_test_pattern_table[2],
1466 pattern_test_pattern_table[3],
1467 pattern_test_pattern_table[4],
1468 pattern_test_pattern_table[5],
1469 pattern_test_pattern_table[6],
1470 pattern_test_pattern_table[7]));
1471 DEBUG_LEVELING(
1472 DEBUG_LEVEL_TRACE,
1473 ("XSB-compt: recieved 0x%x 0x%x 0x%x 0x%x 0x%x 0x%x 0x%x 0x%x\n",
1474 read_pattern[0], read_pattern[1],
1475 read_pattern[2], read_pattern[3],
1476 read_pattern[4], read_pattern[5],
1477 read_pattern[6], read_pattern[7]));
1478
1479 return MV_FAIL;
1480 }
1481 }
1482
1483 /*
1484 * Dynamic write leveling sequence
1485 */
ddr3_tip_dynamic_write_leveling_seq(u32 dev_num)1486 static int ddr3_tip_dynamic_write_leveling_seq(u32 dev_num)
1487 {
1488 u32 bus_id, dq_id;
1489 u16 *mask_results_pup_reg_map = ddr3_tip_get_mask_results_pup_reg_map();
1490 u16 *mask_results_dq_reg_map = ddr3_tip_get_mask_results_dq_reg();
1491 u32 octets_per_if_num = ddr3_tip_dev_attr_get(dev_num, MV_ATTR_OCTET_PER_INTERFACE);
1492 struct mv_ddr_topology_map *tm = mv_ddr_topology_map_get();
1493
1494 CHECK_STATUS(ddr3_tip_if_write
1495 (dev_num, ACCESS_TYPE_MULTICAST, PARAM_NOT_CARE,
1496 TRAINING_SW_2_REG, 0x1, 0x5));
1497 CHECK_STATUS(ddr3_tip_if_write
1498 (dev_num, ACCESS_TYPE_MULTICAST, PARAM_NOT_CARE,
1499 TRAINING_WL_REG, 0x50, 0xff));
1500 CHECK_STATUS(ddr3_tip_if_write
1501 (dev_num, ACCESS_TYPE_MULTICAST, PARAM_NOT_CARE,
1502 TRAINING_WL_REG, 0x5c, 0xff));
1503 CHECK_STATUS(ddr3_tip_if_write
1504 (dev_num, ACCESS_TYPE_MULTICAST, PARAM_NOT_CARE,
1505 GENERAL_TRAINING_OPCODE_REG, 0x381b82, 0x3c3faf));
1506 CHECK_STATUS(ddr3_tip_if_write
1507 (dev_num, ACCESS_TYPE_MULTICAST, PARAM_NOT_CARE,
1508 OPCODE_REG0_REG(1), (0x3 << 25), (0x3ffff << 9)));
1509 CHECK_STATUS(ddr3_tip_if_write
1510 (dev_num, ACCESS_TYPE_MULTICAST, PARAM_NOT_CARE,
1511 OPCODE_REG1_REG(1), 0x80, 0xffff));
1512 CHECK_STATUS(ddr3_tip_if_write
1513 (dev_num, ACCESS_TYPE_MULTICAST, PARAM_NOT_CARE,
1514 WL_DONE_CNTR_REF_REG, 0x14, 0xff));
1515 CHECK_STATUS(ddr3_tip_if_write
1516 (dev_num, ACCESS_TYPE_MULTICAST, PARAM_NOT_CARE,
1517 TRAINING_WL_REG, 0xff5c, 0xffff));
1518
1519 /* mask PBS */
1520 for (dq_id = 0; dq_id < MAX_DQ_NUM; dq_id++) {
1521 CHECK_STATUS(ddr3_tip_if_write
1522 (dev_num, ACCESS_TYPE_MULTICAST, PARAM_NOT_CARE,
1523 mask_results_dq_reg_map[dq_id], 0x1 << 24,
1524 0x1 << 24));
1525 }
1526
1527 /* Mask all results */
1528 for (bus_id = 0; bus_id < octets_per_if_num; bus_id++) {
1529 CHECK_STATUS(ddr3_tip_if_write
1530 (dev_num, ACCESS_TYPE_MULTICAST, PARAM_NOT_CARE,
1531 mask_results_pup_reg_map[bus_id], 0x1 << 24,
1532 0x1 << 24));
1533 }
1534
1535 /* Unmask only wanted */
1536 for (bus_id = 0; bus_id < octets_per_if_num; bus_id++) {
1537 VALIDATE_BUS_ACTIVE(tm->bus_act_mask, bus_id);
1538 CHECK_STATUS(ddr3_tip_if_write
1539 (dev_num, ACCESS_TYPE_MULTICAST, PARAM_NOT_CARE,
1540 mask_results_pup_reg_map[bus_id], 0, 0x1 << 24));
1541 }
1542
1543 CHECK_STATUS(ddr3_tip_if_write
1544 (dev_num, ACCESS_TYPE_MULTICAST, PARAM_NOT_CARE,
1545 WL_DQS_PATTERN_REG, 0x1, 0x1));
1546
1547 return MV_OK;
1548 }
1549
1550 /*
1551 * Dynamic read leveling sequence
1552 */
ddr3_tip_dynamic_read_leveling_seq(u32 dev_num)1553 static int ddr3_tip_dynamic_read_leveling_seq(u32 dev_num)
1554 {
1555 u32 bus_id, dq_id;
1556 u16 *mask_results_pup_reg_map = ddr3_tip_get_mask_results_pup_reg_map();
1557 u16 *mask_results_dq_reg_map = ddr3_tip_get_mask_results_dq_reg();
1558 u32 octets_per_if_num = ddr3_tip_dev_attr_get(dev_num, MV_ATTR_OCTET_PER_INTERFACE);
1559 struct mv_ddr_topology_map *tm = mv_ddr_topology_map_get();
1560
1561 /* mask PBS */
1562 for (dq_id = 0; dq_id < MAX_DQ_NUM; dq_id++) {
1563 CHECK_STATUS(ddr3_tip_if_write
1564 (dev_num, ACCESS_TYPE_MULTICAST, PARAM_NOT_CARE,
1565 mask_results_dq_reg_map[dq_id], 0x1 << 24,
1566 0x1 << 24));
1567 }
1568
1569 /* Mask all results */
1570 for (bus_id = 0; bus_id < octets_per_if_num; bus_id++) {
1571 CHECK_STATUS(ddr3_tip_if_write
1572 (dev_num, ACCESS_TYPE_MULTICAST, PARAM_NOT_CARE,
1573 mask_results_pup_reg_map[bus_id], 0x1 << 24,
1574 0x1 << 24));
1575 }
1576
1577 /* Unmask only wanted */
1578 for (bus_id = 0; bus_id < octets_per_if_num; bus_id++) {
1579 VALIDATE_BUS_ACTIVE(tm->bus_act_mask, bus_id);
1580 CHECK_STATUS(ddr3_tip_if_write
1581 (dev_num, ACCESS_TYPE_MULTICAST, PARAM_NOT_CARE,
1582 mask_results_pup_reg_map[bus_id], 0, 0x1 << 24));
1583 }
1584
1585 return MV_OK;
1586 }
1587
1588 /*
1589 * Dynamic read leveling sequence
1590 */
ddr3_tip_dynamic_per_bit_read_leveling_seq(u32 dev_num)1591 static int ddr3_tip_dynamic_per_bit_read_leveling_seq(u32 dev_num)
1592 {
1593 u32 bus_id, dq_id;
1594 u16 *mask_results_pup_reg_map = ddr3_tip_get_mask_results_pup_reg_map();
1595 u16 *mask_results_dq_reg_map = ddr3_tip_get_mask_results_dq_reg();
1596 u32 octets_per_if_num = ddr3_tip_dev_attr_get(dev_num, MV_ATTR_OCTET_PER_INTERFACE);
1597 struct mv_ddr_topology_map *tm = mv_ddr_topology_map_get();
1598
1599 /* mask PBS */
1600 for (dq_id = 0; dq_id < MAX_DQ_NUM; dq_id++) {
1601 CHECK_STATUS(ddr3_tip_if_write
1602 (dev_num, ACCESS_TYPE_MULTICAST, PARAM_NOT_CARE,
1603 mask_results_dq_reg_map[dq_id], 0x1 << 24,
1604 0x1 << 24));
1605 }
1606
1607 /* Mask all results */
1608 for (bus_id = 0; bus_id < octets_per_if_num; bus_id++) {
1609 CHECK_STATUS(ddr3_tip_if_write
1610 (dev_num, ACCESS_TYPE_MULTICAST, PARAM_NOT_CARE,
1611 mask_results_pup_reg_map[bus_id], 0x1 << 24,
1612 0x1 << 24));
1613 }
1614
1615 /* Unmask only wanted */
1616 for (dq_id = 0; dq_id < MAX_DQ_NUM; dq_id++) {
1617 VALIDATE_BUS_ACTIVE(tm->bus_act_mask, dq_id / 8);
1618 CHECK_STATUS(ddr3_tip_if_write
1619 (dev_num, ACCESS_TYPE_MULTICAST, PARAM_NOT_CARE,
1620 mask_results_dq_reg_map[dq_id], 0x0 << 24,
1621 0x1 << 24));
1622 }
1623
1624 return MV_OK;
1625 }
1626
1627 /*
1628 * Print write leveling supplementary results
1629 */
ddr3_tip_print_wl_supp_result(u32 dev_num)1630 int ddr3_tip_print_wl_supp_result(u32 dev_num)
1631 {
1632 u32 bus_id = 0, if_id = 0;
1633 u32 octets_per_if_num = ddr3_tip_dev_attr_get(dev_num, MV_ATTR_OCTET_PER_INTERFACE);
1634 struct mv_ddr_topology_map *tm = mv_ddr_topology_map_get();
1635
1636 DEBUG_LEVELING(DEBUG_LEVEL_INFO,
1637 ("I/F0 PUP0 Result[0 - success, 1-fail] ...\n"));
1638
1639 for (if_id = 0; if_id <= MAX_INTERFACE_NUM - 1; if_id++) {
1640 VALIDATE_IF_ACTIVE(tm->if_act_mask, if_id);
1641 for (bus_id = 0; bus_id < octets_per_if_num;
1642 bus_id++) {
1643 VALIDATE_BUS_ACTIVE(tm->bus_act_mask, bus_id);
1644 DEBUG_LEVELING(DEBUG_LEVEL_INFO,
1645 ("%d ,", wr_supp_res[if_id]
1646 [bus_id].is_pup_fail));
1647 }
1648 }
1649 DEBUG_LEVELING(
1650 DEBUG_LEVEL_INFO,
1651 ("I/F0 PUP0 Stage[0-phase_shift, 1-clock_shift, 2-align_shift] ...\n"));
1652
1653 for (if_id = 0; if_id <= MAX_INTERFACE_NUM - 1; if_id++) {
1654 VALIDATE_IF_ACTIVE(tm->if_act_mask, if_id);
1655 for (bus_id = 0; bus_id < octets_per_if_num;
1656 bus_id++) {
1657 VALIDATE_BUS_ACTIVE(tm->bus_act_mask, bus_id);
1658 DEBUG_LEVELING(DEBUG_LEVEL_INFO,
1659 ("%d ,", wr_supp_res[if_id]
1660 [bus_id].stage));
1661 }
1662 }
1663
1664 return MV_OK;
1665 }
1666
1667 #define RD_FIFO_PTR_LOW_STAT_INDIR_ADDR 0x9a
1668 #define RD_FIFO_PTR_HIGH_STAT_INDIR_ADDR 0x9b
1669 /* position of falling dqs edge in fifo; walking 1 */
1670 #define RD_FIFO_DQS_FALL_EDGE_POS_0 0x1
1671 #define RD_FIFO_DQS_FALL_EDGE_POS_1 0x2
1672 #define RD_FIFO_DQS_FALL_EDGE_POS_2 0x4
1673 #define RD_FIFO_DQS_FALL_EDGE_POS_3 0x8
1674 #define RD_FIFO_DQS_FALL_EDGE_POS_4 0x10 /* lock */
1675 /* position of rising dqs edge in fifo; walking 0 */
1676 #define RD_FIFO_DQS_RISE_EDGE_POS_0 0x1fff
1677 #define RD_FIFO_DQS_RISE_EDGE_POS_1 0x3ffe
1678 #define RD_FIFO_DQS_RISE_EDGE_POS_2 0x3ffd
1679 #define RD_FIFO_DQS_RISE_EDGE_POS_3 0x3ffb
1680 #define RD_FIFO_DQS_RISE_EDGE_POS_4 0x3ff7 /* lock */
1681 #define TEST_ADDR 0x8
1682 #define TAPS_PER_UI 32
1683 #define UI_PER_RD_SAMPLE 4
1684 #define TAPS_PER_RD_SAMPLE ((UI_PER_RD_SAMPLE) * (TAPS_PER_UI))
1685 #define MAX_RD_SAMPLES 32
1686 #define MAX_RL_VALUE ((MAX_RD_SAMPLES) * (TAPS_PER_RD_SAMPLE))
1687 #define RD_FIFO_DLY 8
1688 #define STEP_SIZE 64
1689 #define RL_JITTER_WIDTH_LMT 20
1690 #define ADLL_TAPS_IN_CYCLE 64
1691
1692 enum rl_dqs_burst_state {
1693 RL_AHEAD = 0,
1694 RL_INSIDE,
1695 RL_BEHIND
1696 };
mv_ddr_rl_dqs_burst(u32 dev_num,u32 if_id,u32 freq)1697 int mv_ddr_rl_dqs_burst(u32 dev_num, u32 if_id, u32 freq)
1698 {
1699 enum rl_dqs_burst_state rl_state[NUM_OF_CS][MAX_BUS_NUM][MAX_INTERFACE_NUM] = { { {0} } };
1700 enum hws_ddr_phy subphy_type = DDR_PHY_DATA;
1701 struct mv_ddr_topology_map *tm = mv_ddr_topology_map_get();
1702 int cl_val = tm->interface_params[0].cas_l;
1703 int rl_adll_val, rl_phase_val, sdr_cycle_incr, rd_sample, rd_ready;
1704 int final_rd_sample, final_rd_ready;
1705 int i, subphy_id, step;
1706 int pass_lock_num = 0;
1707 int init_pass_lock_num;
1708 int phase_delta;
1709 int min_phase, max_phase;
1710 u32 max_cs = ddr3_tip_max_cs_get(dev_num);
1711 u32 rl_values[NUM_OF_CS][MAX_BUS_NUM][MAX_INTERFACE_NUM] = { { {0} } };
1712 u32 rl_min_values[NUM_OF_CS][MAX_BUS_NUM][MAX_INTERFACE_NUM] = { { {0} } };
1713 u32 rl_max_values[NUM_OF_CS][MAX_BUS_NUM][MAX_INTERFACE_NUM] = { { {0} } };
1714 u32 rl_val, rl_min_val[NUM_OF_CS], rl_max_val[NUM_OF_CS];
1715 u32 reg_val_low, reg_val_high;
1716 u32 reg_val, reg_mask;
1717 uintptr_t test_addr = TEST_ADDR;
1718
1719 /* initialization */
1720 if (ddr3_if_ecc_enabled()) {
1721 ddr3_tip_if_read(dev_num, ACCESS_TYPE_UNICAST, if_id, TRAINING_SW_2_REG,
1722 ®_val, MASK_ALL_BITS);
1723 reg_mask = (TRAINING_ECC_MUX_MASK << TRAINING_ECC_MUX_OFFS) |
1724 (TRAINING_SW_OVRD_MASK << TRAINING_SW_OVRD_OFFS);
1725 reg_val &= ~reg_mask;
1726 reg_val |= (TRAINING_ECC_MUX_DIS << TRAINING_ECC_MUX_OFFS) |
1727 (TRAINING_SW_OVRD_ENA << TRAINING_SW_OVRD_OFFS);
1728 ddr3_tip_if_write(0, ACCESS_TYPE_MULTICAST, PARAM_NOT_CARE, TRAINING_SW_2_REG,
1729 reg_val, MASK_ALL_BITS);
1730 ddr3_tip_if_read(dev_num, ACCESS_TYPE_UNICAST, if_id, TRAINING_REG,
1731 ®_val, MASK_ALL_BITS);
1732 reg_mask = (TRN_START_MASK << TRN_START_OFFS);
1733 reg_val &= ~reg_mask;
1734 reg_val |= TRN_START_ENA << TRN_START_OFFS;
1735 ddr3_tip_if_write(0, ACCESS_TYPE_MULTICAST, PARAM_NOT_CARE, TRAINING_REG,
1736 reg_val, MASK_ALL_BITS);
1737 }
1738
1739 for (effective_cs = 0; effective_cs < max_cs; effective_cs++)
1740 for (subphy_id = 0; subphy_id < MAX_BUS_NUM; subphy_id++)
1741 for (if_id = 0; if_id < MAX_INTERFACE_NUM; if_id++)
1742 if (IS_BUS_ACTIVE(tm->bus_act_mask, subphy_id) == 0)
1743 pass_lock_num++; /* increment on inactive subphys */
1744
1745 init_pass_lock_num = pass_lock_num / max_cs;
1746 for (effective_cs = 0; effective_cs < max_cs; effective_cs++) {
1747 for (if_id = 0; if_id < MAX_INTERFACE_NUM; if_id++) {
1748 VALIDATE_IF_ACTIVE(tm->if_act_mask, if_id);
1749 training_result[training_stage][if_id] = TEST_SUCCESS;
1750 }
1751 }
1752
1753 /* search for dqs edges per subphy */
1754 if_id = 0;
1755 for (effective_cs = 0; effective_cs < max_cs; effective_cs++) {
1756 pass_lock_num = init_pass_lock_num;
1757 ddr3_tip_if_write(0, ACCESS_TYPE_MULTICAST, PARAM_NOT_CARE, ODPG_DATA_CTRL_REG,
1758 effective_cs << ODPG_DATA_CS_OFFS,
1759 ODPG_DATA_CS_MASK << ODPG_DATA_CS_OFFS);
1760 rl_min_val[effective_cs] = MAX_RL_VALUE;
1761 rl_max_val[effective_cs] = 0;
1762 step = STEP_SIZE;
1763 for (i = 0; i < MAX_RL_VALUE; i += step) {
1764 rl_val = 0;
1765 sdr_cycle_incr = i / TAPS_PER_RD_SAMPLE; /* sdr cycle increment */
1766 rd_sample = cl_val + 2 * sdr_cycle_incr;
1767 /* fifo out to in delay in search is constant */
1768 rd_ready = rd_sample + RD_FIFO_DLY;
1769
1770 ddr3_tip_if_write(0, ACCESS_TYPE_UNICAST, 0, RD_DATA_SMPL_DLYS_REG,
1771 rd_sample << RD_SMPL_DLY_CS_OFFS(effective_cs),
1772 RD_SMPL_DLY_CS_MASK << RD_SMPL_DLY_CS_OFFS(effective_cs));
1773 ddr3_tip_if_write(0, ACCESS_TYPE_UNICAST, 0, RD_DATA_RDY_DLYS_REG,
1774 rd_ready << RD_RDY_DLY_CS_OFFS(effective_cs),
1775 RD_RDY_DLY_CS_MASK << RD_RDY_DLY_CS_OFFS(effective_cs));
1776
1777 /* one sdr (single data rate) cycle incremented on every four phases of ddr clock */
1778 sdr_cycle_incr = i % TAPS_PER_RD_SAMPLE;
1779 rl_adll_val = sdr_cycle_incr % MAX_RD_SAMPLES;
1780 rl_phase_val = sdr_cycle_incr / MAX_RD_SAMPLES;
1781 rl_val = ((rl_adll_val & RL_REF_DLY_MASK) << RL_REF_DLY_OFFS) |
1782 ((rl_phase_val & RL_PH_SEL_MASK) << RL_PH_SEL_OFFS);
1783
1784 /* write to all subphys (even to not connected or locked) */
1785 ddr3_tip_bus_write(dev_num, ACCESS_TYPE_MULTICAST, PARAM_NOT_CARE, ACCESS_TYPE_MULTICAST,
1786 0, DDR_PHY_DATA, RL_PHY_REG(effective_cs), rl_val);
1787
1788 /* reset read fifo assertion */
1789 ddr3_tip_if_write(dev_num, ACCESS_TYPE_MULTICAST, if_id, SDRAM_CFG_REG,
1790 DATA_PUP_RD_RESET_ENA << DATA_PUP_RD_RESET_OFFS,
1791 DATA_PUP_RD_RESET_MASK << DATA_PUP_RD_RESET_OFFS);
1792
1793 /* reset read fifo deassertion */
1794 ddr3_tip_if_write(dev_num, ACCESS_TYPE_MULTICAST, if_id, SDRAM_CFG_REG,
1795 DATA_PUP_RD_RESET_DIS << DATA_PUP_RD_RESET_OFFS,
1796 DATA_PUP_RD_RESET_MASK << DATA_PUP_RD_RESET_OFFS);
1797
1798 /* perform one read burst */
1799 if (MV_DDR_IS_64BIT_DRAM_MODE(tm->bus_act_mask))
1800 readq(test_addr);
1801 else
1802 readl(test_addr);
1803
1804 /* progress read ptr; decide on rl state per byte */
1805 for (subphy_id = 0; subphy_id < MAX_BUS_NUM; subphy_id++) {
1806 if (rl_state[effective_cs][subphy_id][if_id] == RL_BEHIND)
1807 continue; /* skip locked subphys */
1808 ddr3_tip_bus_read(dev_num, if_id, ACCESS_TYPE_UNICAST, subphy_id, DDR_PHY_DATA,
1809 RD_FIFO_PTR_LOW_STAT_INDIR_ADDR, ®_val_low);
1810 ddr3_tip_bus_read(dev_num, if_id, ACCESS_TYPE_UNICAST, subphy_id, DDR_PHY_DATA,
1811 RD_FIFO_PTR_HIGH_STAT_INDIR_ADDR, ®_val_high);
1812 DEBUG_LEVELING(DEBUG_LEVEL_TRACE,
1813 ("%s: cs %d, step %d, subphy %d, state %d, low 0x%04x, high 0x%04x; move to ",
1814 __func__, effective_cs, i, subphy_id,
1815 rl_state[effective_cs][subphy_id][if_id],
1816 reg_val_low, reg_val_high));
1817
1818 switch (rl_state[effective_cs][subphy_id][if_id]) {
1819 case RL_AHEAD:
1820 /* improve search resolution getting closer to the window */
1821 if (reg_val_low == RD_FIFO_DQS_FALL_EDGE_POS_4 &&
1822 reg_val_high == RD_FIFO_DQS_RISE_EDGE_POS_4) {
1823 rl_state[effective_cs][subphy_id][if_id] = RL_INSIDE;
1824 rl_values[effective_cs][subphy_id][if_id] = i;
1825 rl_min_values[effective_cs][subphy_id][if_id] = i;
1826 DEBUG_LEVELING(DEBUG_LEVEL_TRACE,
1827 ("new state %d\n",
1828 rl_state[effective_cs][subphy_id][if_id]));
1829 } else if (reg_val_low == RD_FIFO_DQS_FALL_EDGE_POS_3 &&
1830 reg_val_high == RD_FIFO_DQS_RISE_EDGE_POS_3) {
1831 step = (step < 2) ? step : 2;
1832 } else if (reg_val_low == RD_FIFO_DQS_FALL_EDGE_POS_2 &&
1833 reg_val_high == RD_FIFO_DQS_RISE_EDGE_POS_2) {
1834 step = (step < 16) ? step : 16;
1835 } else if (reg_val_low == RD_FIFO_DQS_FALL_EDGE_POS_1 &&
1836 reg_val_high == RD_FIFO_DQS_RISE_EDGE_POS_1) {
1837 step = (step < 32) ? step : 32;
1838 } else if (reg_val_low == RD_FIFO_DQS_FALL_EDGE_POS_0 &&
1839 reg_val_high == RD_FIFO_DQS_RISE_EDGE_POS_0) {
1840 step = (step < 64) ? step : 64;
1841 } else {
1842 /* otherwise, step is unchanged */
1843 }
1844 break;
1845 case RL_INSIDE:
1846 if (reg_val_low == RD_FIFO_DQS_FALL_EDGE_POS_4 &&
1847 reg_val_high == RD_FIFO_DQS_RISE_EDGE_POS_4) {
1848 rl_max_values[effective_cs][subphy_id][if_id] = i;
1849 if ((rl_max_values[effective_cs][subphy_id][if_id] -
1850 rl_min_values[effective_cs][subphy_id][if_id]) >
1851 ADLL_TAPS_IN_CYCLE) {
1852 rl_state[effective_cs][subphy_id][if_id] = RL_BEHIND;
1853 rl_values[effective_cs][subphy_id][if_id] =
1854 (i + rl_values[effective_cs][subphy_id][if_id]) / 2;
1855 pass_lock_num++;
1856 DEBUG_LEVELING(DEBUG_LEVEL_TRACE,
1857 ("new lock %d\n", pass_lock_num));
1858 if (rl_min_val[effective_cs] >
1859 rl_values[effective_cs][subphy_id][if_id])
1860 rl_min_val[effective_cs] =
1861 rl_values[effective_cs][subphy_id][if_id];
1862 if (rl_max_val[effective_cs] <
1863 rl_values[effective_cs][subphy_id][if_id])
1864 rl_max_val[effective_cs] =
1865 rl_values[effective_cs][subphy_id][if_id];
1866 step = 2;
1867 }
1868 }
1869 if (reg_val_low != RD_FIFO_DQS_FALL_EDGE_POS_4 ||
1870 reg_val_high != RD_FIFO_DQS_RISE_EDGE_POS_4) {
1871 if ((i - rl_values[effective_cs][subphy_id][if_id]) <
1872 RL_JITTER_WIDTH_LMT) {
1873 /* inside the jitter; not valid segment */
1874 rl_state[effective_cs][subphy_id][if_id] = RL_AHEAD;
1875 DEBUG_LEVELING(DEBUG_LEVEL_TRACE,
1876 ("new state %d; jitter on mask\n",
1877 rl_state[effective_cs][subphy_id][if_id]));
1878 } else { /* finished valid segment */
1879 rl_state[effective_cs][subphy_id][if_id] = RL_BEHIND;
1880 rl_values[effective_cs][subphy_id][if_id] =
1881 (i + rl_values[effective_cs][subphy_id][if_id]) / 2;
1882 DEBUG_LEVELING(DEBUG_LEVEL_TRACE,
1883 ("new state %d, solution %d\n",
1884 rl_state[effective_cs][subphy_id][if_id],
1885 rl_values[effective_cs][subphy_id][if_id]));
1886 pass_lock_num++;
1887 DEBUG_LEVELING(DEBUG_LEVEL_TRACE,
1888 ("new lock %d\n", pass_lock_num));
1889 if (rl_min_val[effective_cs] >
1890 rl_values[effective_cs][subphy_id][if_id])
1891 rl_min_val[effective_cs] =
1892 rl_values[effective_cs][subphy_id][if_id];
1893 if (rl_max_val[effective_cs] <
1894 rl_values[effective_cs][subphy_id][if_id])
1895 rl_max_val[effective_cs] =
1896 rl_values[effective_cs][subphy_id][if_id];
1897 step = 2;
1898 }
1899 }
1900 break;
1901 case RL_BEHIND: /* do nothing */
1902 break;
1903 }
1904 DEBUG_LEVELING(DEBUG_LEVEL_TRACE, ("\n"));
1905 }
1906 DEBUG_LEVELING(DEBUG_LEVEL_TRACE, ("pass_lock_num %d\n", pass_lock_num));
1907 /* exit condition */
1908 if (pass_lock_num == MAX_BUS_NUM)
1909 break;
1910 } /* for-loop on i */
1911
1912 if (pass_lock_num != MAX_BUS_NUM) {
1913 DEBUG_LEVELING(DEBUG_LEVEL_ERROR,
1914 ("%s: cs %d, pass_lock_num %d, max_bus_num %d, init_pass_lock_num %d\n",
1915 __func__, effective_cs, pass_lock_num, MAX_BUS_NUM, init_pass_lock_num));
1916 for (subphy_id = 0; subphy_id < MAX_BUS_NUM; subphy_id++) {
1917 VALIDATE_BUS_ACTIVE(tm->bus_act_mask, subphy_id);
1918 DEBUG_LEVELING(DEBUG_LEVEL_ERROR,
1919 ("%s: subphy %d %s\n",
1920 __func__, subphy_id,
1921 (rl_state[effective_cs][subphy_id][if_id] == RL_BEHIND) ?
1922 "locked" : "not locked"));
1923 }
1924 }
1925 } /* for-loop on effective_cs */
1926
1927 /* post-processing read leveling results */
1928 if_id = 0;
1929 for (effective_cs = 0; effective_cs < max_cs; effective_cs++) {
1930 phase_delta = 0;
1931 i = rl_min_val[effective_cs];
1932 sdr_cycle_incr = i / TAPS_PER_RD_SAMPLE; /* sdr cycle increment */
1933 rd_sample = cl_val + 2 * sdr_cycle_incr;
1934 rd_ready = rd_sample + RD_FIFO_DLY;
1935 min_phase = (rl_min_val[effective_cs] - (sdr_cycle_incr * TAPS_PER_RD_SAMPLE)) % MAX_RD_SAMPLES;
1936 max_phase = (rl_max_val[effective_cs] - (sdr_cycle_incr * TAPS_PER_RD_SAMPLE)) % MAX_RD_SAMPLES;
1937 final_rd_sample = rd_sample;
1938 final_rd_ready = rd_ready;
1939
1940 ddr3_tip_if_write(0, ACCESS_TYPE_UNICAST, 0, RD_DATA_SMPL_DLYS_REG,
1941 rd_sample << RD_SMPL_DLY_CS_OFFS(effective_cs),
1942 RD_SMPL_DLY_CS_MASK << RD_SMPL_DLY_CS_OFFS(effective_cs));
1943 ddr3_tip_if_write(0, ACCESS_TYPE_UNICAST, 0, RD_DATA_RDY_DLYS_REG,
1944 rd_ready << RD_RDY_DLY_CS_OFFS(effective_cs),
1945 RD_RDY_DLY_CS_MASK << RD_RDY_DLY_CS_OFFS(effective_cs));
1946 DEBUG_LEVELING(DEBUG_LEVEL_INFO,
1947 ("%s: cs %d, min phase %d, max phase %d, read sample %d\n",
1948 __func__, effective_cs, min_phase, max_phase, rd_sample));
1949
1950 for (subphy_id = 0; subphy_id < MAX_BUS_NUM; subphy_id++) {
1951 VALIDATE_BUS_ACTIVE(tm->bus_act_mask, subphy_id);
1952 /* reduce sdr cycle per cs; extract rl adll and phase values */
1953 i = rl_values[effective_cs][subphy_id][if_id] - (sdr_cycle_incr * TAPS_PER_RD_SAMPLE);
1954 rl_adll_val = i % MAX_RD_SAMPLES;
1955 rl_phase_val = i / MAX_RD_SAMPLES;
1956 rl_phase_val -= phase_delta;
1957 DEBUG_LEVELING(DEBUG_LEVEL_INFO,
1958 ("%s: final results: cs %d, subphy %d, read sample %d read ready %d, rl_phase_val %d, rl_adll_val %d\n",
1959 __func__, effective_cs, subphy_id, final_rd_sample,
1960 final_rd_ready, rl_phase_val, rl_adll_val));
1961
1962 rl_val = ((rl_adll_val & RL_REF_DLY_MASK) << RL_REF_DLY_OFFS) |
1963 ((rl_phase_val & RL_PH_SEL_MASK) << RL_PH_SEL_OFFS);
1964 ddr3_tip_bus_write(dev_num, ACCESS_TYPE_MULTICAST, PARAM_NOT_CARE, ACCESS_TYPE_UNICAST,
1965 subphy_id, subphy_type, RL_PHY_REG(effective_cs), rl_val);
1966 }
1967 } /* for-loop on effective cs */
1968
1969 for (if_id = 0; if_id < MAX_INTERFACE_NUM; if_id++) {
1970 VALIDATE_IF_ACTIVE(tm->if_act_mask, if_id);
1971 if (odt_config != 0)
1972 CHECK_STATUS(ddr3_tip_write_additional_odt_setting(dev_num, if_id));
1973 }
1974
1975 /* reset read fifo assertion */
1976 ddr3_tip_if_write(dev_num, ACCESS_TYPE_MULTICAST, if_id, SDRAM_CFG_REG,
1977 DATA_PUP_RD_RESET_ENA << DATA_PUP_RD_RESET_OFFS,
1978 DATA_PUP_RD_RESET_MASK << DATA_PUP_RD_RESET_OFFS);
1979
1980 /* reset read fifo deassertion */
1981 ddr3_tip_if_write(dev_num, ACCESS_TYPE_MULTICAST, if_id, SDRAM_CFG_REG,
1982 DATA_PUP_RD_RESET_DIS << DATA_PUP_RD_RESET_OFFS,
1983 DATA_PUP_RD_RESET_MASK << DATA_PUP_RD_RESET_OFFS);
1984
1985 return MV_OK;
1986 }
1987