1 // SPDX-License-Identifier: GPL-2.0+
2 /*
3 * Freescale GPMI NAND Flash Driver
4 *
5 * Copyright (C) 2008-2011 Freescale Semiconductor, Inc.
6 * Copyright (C) 2008 Embedded Alley Solutions, Inc.
7 */
8 #include <linux/delay.h>
9 #include <linux/clk.h>
10 #include <linux/slab.h>
11
12 #include "gpmi-nand.h"
13 #include "gpmi-regs.h"
14 #include "bch-regs.h"
15
16 /* Converts time to clock cycles */
17 #define TO_CYCLES(duration, period) DIV_ROUND_UP_ULL(duration, period)
18
19 #define MXS_SET_ADDR 0x4
20 #define MXS_CLR_ADDR 0x8
21 /*
22 * Clear the bit and poll it cleared. This is usually called with
23 * a reset address and mask being either SFTRST(bit 31) or CLKGATE
24 * (bit 30).
25 */
clear_poll_bit(void __iomem * addr,u32 mask)26 static int clear_poll_bit(void __iomem *addr, u32 mask)
27 {
28 int timeout = 0x400;
29
30 /* clear the bit */
31 writel(mask, addr + MXS_CLR_ADDR);
32
33 /*
34 * SFTRST needs 3 GPMI clocks to settle, the reference manual
35 * recommends to wait 1us.
36 */
37 udelay(1);
38
39 /* poll the bit becoming clear */
40 while ((readl(addr) & mask) && --timeout)
41 /* nothing */;
42
43 return !timeout;
44 }
45
46 #define MODULE_CLKGATE (1 << 30)
47 #define MODULE_SFTRST (1 << 31)
48 /*
49 * The current mxs_reset_block() will do two things:
50 * [1] enable the module.
51 * [2] reset the module.
52 *
53 * In most of the cases, it's ok.
54 * But in MX23, there is a hardware bug in the BCH block (see erratum #2847).
55 * If you try to soft reset the BCH block, it becomes unusable until
56 * the next hard reset. This case occurs in the NAND boot mode. When the board
57 * boots by NAND, the ROM of the chip will initialize the BCH blocks itself.
58 * So If the driver tries to reset the BCH again, the BCH will not work anymore.
59 * You will see a DMA timeout in this case. The bug has been fixed
60 * in the following chips, such as MX28.
61 *
62 * To avoid this bug, just add a new parameter `just_enable` for
63 * the mxs_reset_block(), and rewrite it here.
64 */
gpmi_reset_block(void __iomem * reset_addr,bool just_enable)65 static int gpmi_reset_block(void __iomem *reset_addr, bool just_enable)
66 {
67 int ret;
68 int timeout = 0x400;
69
70 /* clear and poll SFTRST */
71 ret = clear_poll_bit(reset_addr, MODULE_SFTRST);
72 if (unlikely(ret))
73 goto error;
74
75 /* clear CLKGATE */
76 writel(MODULE_CLKGATE, reset_addr + MXS_CLR_ADDR);
77
78 if (!just_enable) {
79 /* set SFTRST to reset the block */
80 writel(MODULE_SFTRST, reset_addr + MXS_SET_ADDR);
81 udelay(1);
82
83 /* poll CLKGATE becoming set */
84 while ((!(readl(reset_addr) & MODULE_CLKGATE)) && --timeout)
85 /* nothing */;
86 if (unlikely(!timeout))
87 goto error;
88 }
89
90 /* clear and poll SFTRST */
91 ret = clear_poll_bit(reset_addr, MODULE_SFTRST);
92 if (unlikely(ret))
93 goto error;
94
95 /* clear and poll CLKGATE */
96 ret = clear_poll_bit(reset_addr, MODULE_CLKGATE);
97 if (unlikely(ret))
98 goto error;
99
100 return 0;
101
102 error:
103 pr_err("%s(%p): module reset timeout\n", __func__, reset_addr);
104 return -ETIMEDOUT;
105 }
106
__gpmi_enable_clk(struct gpmi_nand_data * this,bool v)107 static int __gpmi_enable_clk(struct gpmi_nand_data *this, bool v)
108 {
109 struct clk *clk;
110 int ret;
111 int i;
112
113 for (i = 0; i < GPMI_CLK_MAX; i++) {
114 clk = this->resources.clock[i];
115 if (!clk)
116 break;
117
118 if (v) {
119 ret = clk_prepare_enable(clk);
120 if (ret)
121 goto err_clk;
122 } else {
123 clk_disable_unprepare(clk);
124 }
125 }
126 return 0;
127
128 err_clk:
129 for (; i > 0; i--)
130 clk_disable_unprepare(this->resources.clock[i - 1]);
131 return ret;
132 }
133
gpmi_enable_clk(struct gpmi_nand_data * this)134 int gpmi_enable_clk(struct gpmi_nand_data *this)
135 {
136 return __gpmi_enable_clk(this, true);
137 }
138
gpmi_disable_clk(struct gpmi_nand_data * this)139 int gpmi_disable_clk(struct gpmi_nand_data *this)
140 {
141 return __gpmi_enable_clk(this, false);
142 }
143
gpmi_init(struct gpmi_nand_data * this)144 int gpmi_init(struct gpmi_nand_data *this)
145 {
146 struct resources *r = &this->resources;
147 int ret;
148
149 ret = gpmi_enable_clk(this);
150 if (ret)
151 return ret;
152 ret = gpmi_reset_block(r->gpmi_regs, false);
153 if (ret)
154 goto err_out;
155
156 /*
157 * Reset BCH here, too. We got failures otherwise :(
158 * See later BCH reset for explanation of MX23 and MX28 handling
159 */
160 ret = gpmi_reset_block(r->bch_regs,
161 GPMI_IS_MX23(this) || GPMI_IS_MX28(this));
162 if (ret)
163 goto err_out;
164
165 /* Choose NAND mode. */
166 writel(BM_GPMI_CTRL1_GPMI_MODE, r->gpmi_regs + HW_GPMI_CTRL1_CLR);
167
168 /* Set the IRQ polarity. */
169 writel(BM_GPMI_CTRL1_ATA_IRQRDY_POLARITY,
170 r->gpmi_regs + HW_GPMI_CTRL1_SET);
171
172 /* Disable Write-Protection. */
173 writel(BM_GPMI_CTRL1_DEV_RESET, r->gpmi_regs + HW_GPMI_CTRL1_SET);
174
175 /* Select BCH ECC. */
176 writel(BM_GPMI_CTRL1_BCH_MODE, r->gpmi_regs + HW_GPMI_CTRL1_SET);
177
178 /*
179 * Decouple the chip select from dma channel. We use dma0 for all
180 * the chips.
181 */
182 writel(BM_GPMI_CTRL1_DECOUPLE_CS, r->gpmi_regs + HW_GPMI_CTRL1_SET);
183
184 gpmi_disable_clk(this);
185 return 0;
186 err_out:
187 gpmi_disable_clk(this);
188 return ret;
189 }
190
191 /* This function is very useful. It is called only when the bug occur. */
gpmi_dump_info(struct gpmi_nand_data * this)192 void gpmi_dump_info(struct gpmi_nand_data *this)
193 {
194 struct resources *r = &this->resources;
195 struct bch_geometry *geo = &this->bch_geometry;
196 u32 reg;
197 int i;
198
199 dev_err(this->dev, "Show GPMI registers :\n");
200 for (i = 0; i <= HW_GPMI_DEBUG / 0x10 + 1; i++) {
201 reg = readl(r->gpmi_regs + i * 0x10);
202 dev_err(this->dev, "offset 0x%.3x : 0x%.8x\n", i * 0x10, reg);
203 }
204
205 /* start to print out the BCH info */
206 dev_err(this->dev, "Show BCH registers :\n");
207 for (i = 0; i <= HW_BCH_VERSION / 0x10 + 1; i++) {
208 reg = readl(r->bch_regs + i * 0x10);
209 dev_err(this->dev, "offset 0x%.3x : 0x%.8x\n", i * 0x10, reg);
210 }
211 dev_err(this->dev, "BCH Geometry :\n"
212 "GF length : %u\n"
213 "ECC Strength : %u\n"
214 "Page Size in Bytes : %u\n"
215 "Metadata Size in Bytes : %u\n"
216 "ECC Chunk Size in Bytes: %u\n"
217 "ECC Chunk Count : %u\n"
218 "Payload Size in Bytes : %u\n"
219 "Auxiliary Size in Bytes: %u\n"
220 "Auxiliary Status Offset: %u\n"
221 "Block Mark Byte Offset : %u\n"
222 "Block Mark Bit Offset : %u\n",
223 geo->gf_len,
224 geo->ecc_strength,
225 geo->page_size,
226 geo->metadata_size,
227 geo->ecc_chunk_size,
228 geo->ecc_chunk_count,
229 geo->payload_size,
230 geo->auxiliary_size,
231 geo->auxiliary_status_offset,
232 geo->block_mark_byte_offset,
233 geo->block_mark_bit_offset);
234 }
235
236 /* Configures the geometry for BCH. */
bch_set_geometry(struct gpmi_nand_data * this)237 int bch_set_geometry(struct gpmi_nand_data *this)
238 {
239 struct resources *r = &this->resources;
240 struct bch_geometry *bch_geo = &this->bch_geometry;
241 unsigned int block_count;
242 unsigned int block_size;
243 unsigned int metadata_size;
244 unsigned int ecc_strength;
245 unsigned int page_size;
246 unsigned int gf_len;
247 int ret;
248
249 ret = common_nfc_set_geometry(this);
250 if (ret)
251 return ret;
252
253 block_count = bch_geo->ecc_chunk_count - 1;
254 block_size = bch_geo->ecc_chunk_size;
255 metadata_size = bch_geo->metadata_size;
256 ecc_strength = bch_geo->ecc_strength >> 1;
257 page_size = bch_geo->page_size;
258 gf_len = bch_geo->gf_len;
259
260 ret = gpmi_enable_clk(this);
261 if (ret)
262 return ret;
263
264 /*
265 * Due to erratum #2847 of the MX23, the BCH cannot be soft reset on this
266 * chip, otherwise it will lock up. So we skip resetting BCH on the MX23.
267 * and MX28.
268 */
269 ret = gpmi_reset_block(r->bch_regs,
270 GPMI_IS_MX23(this) || GPMI_IS_MX28(this));
271 if (ret)
272 goto err_out;
273
274 /* Configure layout 0. */
275 writel(BF_BCH_FLASH0LAYOUT0_NBLOCKS(block_count)
276 | BF_BCH_FLASH0LAYOUT0_META_SIZE(metadata_size)
277 | BF_BCH_FLASH0LAYOUT0_ECC0(ecc_strength, this)
278 | BF_BCH_FLASH0LAYOUT0_GF(gf_len, this)
279 | BF_BCH_FLASH0LAYOUT0_DATA0_SIZE(block_size, this),
280 r->bch_regs + HW_BCH_FLASH0LAYOUT0);
281
282 writel(BF_BCH_FLASH0LAYOUT1_PAGE_SIZE(page_size)
283 | BF_BCH_FLASH0LAYOUT1_ECCN(ecc_strength, this)
284 | BF_BCH_FLASH0LAYOUT1_GF(gf_len, this)
285 | BF_BCH_FLASH0LAYOUT1_DATAN_SIZE(block_size, this),
286 r->bch_regs + HW_BCH_FLASH0LAYOUT1);
287
288 /* Set *all* chip selects to use layout 0. */
289 writel(0, r->bch_regs + HW_BCH_LAYOUTSELECT);
290
291 /* Enable interrupts. */
292 writel(BM_BCH_CTRL_COMPLETE_IRQ_EN,
293 r->bch_regs + HW_BCH_CTRL_SET);
294
295 gpmi_disable_clk(this);
296 return 0;
297 err_out:
298 gpmi_disable_clk(this);
299 return ret;
300 }
301
302 /*
303 * <1> Firstly, we should know what's the GPMI-clock means.
304 * The GPMI-clock is the internal clock in the gpmi nand controller.
305 * If you set 100MHz to gpmi nand controller, the GPMI-clock's period
306 * is 10ns. Mark the GPMI-clock's period as GPMI-clock-period.
307 *
308 * <2> Secondly, we should know what's the frequency on the nand chip pins.
309 * The frequency on the nand chip pins is derived from the GPMI-clock.
310 * We can get it from the following equation:
311 *
312 * F = G / (DS + DH)
313 *
314 * F : the frequency on the nand chip pins.
315 * G : the GPMI clock, such as 100MHz.
316 * DS : GPMI_HW_GPMI_TIMING0:DATA_SETUP
317 * DH : GPMI_HW_GPMI_TIMING0:DATA_HOLD
318 *
319 * <3> Thirdly, when the frequency on the nand chip pins is above 33MHz,
320 * the nand EDO(extended Data Out) timing could be applied.
321 * The GPMI implements a feedback read strobe to sample the read data.
322 * The feedback read strobe can be delayed to support the nand EDO timing
323 * where the read strobe may deasserts before the read data is valid, and
324 * read data is valid for some time after read strobe.
325 *
326 * The following figure illustrates some aspects of a NAND Flash read:
327 *
328 * |<---tREA---->|
329 * | |
330 * | | |
331 * |<--tRP-->| |
332 * | | |
333 * __ ___|__________________________________
334 * RDN \________/ |
335 * |
336 * /---------\
337 * Read Data --------------< >---------
338 * \---------/
339 * | |
340 * |<-D->|
341 * FeedbackRDN ________ ____________
342 * \___________/
343 *
344 * D stands for delay, set in the HW_GPMI_CTRL1:RDN_DELAY.
345 *
346 *
347 * <4> Now, we begin to describe how to compute the right RDN_DELAY.
348 *
349 * 4.1) From the aspect of the nand chip pins:
350 * Delay = (tREA + C - tRP) {1}
351 *
352 * tREA : the maximum read access time.
353 * C : a constant to adjust the delay. default is 4000ps.
354 * tRP : the read pulse width, which is exactly:
355 * tRP = (GPMI-clock-period) * DATA_SETUP
356 *
357 * 4.2) From the aspect of the GPMI nand controller:
358 * Delay = RDN_DELAY * 0.125 * RP {2}
359 *
360 * RP : the DLL reference period.
361 * if (GPMI-clock-period > DLL_THRETHOLD)
362 * RP = GPMI-clock-period / 2;
363 * else
364 * RP = GPMI-clock-period;
365 *
366 * Set the HW_GPMI_CTRL1:HALF_PERIOD if GPMI-clock-period
367 * is greater DLL_THRETHOLD. In other SOCs, the DLL_THRETHOLD
368 * is 16000ps, but in mx6q, we use 12000ps.
369 *
370 * 4.3) since {1} equals {2}, we get:
371 *
372 * (tREA + 4000 - tRP) * 8
373 * RDN_DELAY = ----------------------- {3}
374 * RP
375 */
gpmi_nfc_compute_timings(struct gpmi_nand_data * this,const struct nand_sdr_timings * sdr)376 static void gpmi_nfc_compute_timings(struct gpmi_nand_data *this,
377 const struct nand_sdr_timings *sdr)
378 {
379 struct gpmi_nfc_hardware_timing *hw = &this->hw;
380 unsigned int dll_threshold_ps = this->devdata->max_chain_delay;
381 unsigned int period_ps, reference_period_ps;
382 unsigned int data_setup_cycles, data_hold_cycles, addr_setup_cycles;
383 unsigned int tRP_ps;
384 bool use_half_period;
385 int sample_delay_ps, sample_delay_factor;
386 u16 busy_timeout_cycles;
387 u8 wrn_dly_sel;
388
389 if (sdr->tRC_min >= 30000) {
390 /* ONFI non-EDO modes [0-3] */
391 hw->clk_rate = 22000000;
392 wrn_dly_sel = BV_GPMI_CTRL1_WRN_DLY_SEL_4_TO_8NS;
393 } else if (sdr->tRC_min >= 25000) {
394 /* ONFI EDO mode 4 */
395 hw->clk_rate = 80000000;
396 wrn_dly_sel = BV_GPMI_CTRL1_WRN_DLY_SEL_NO_DELAY;
397 } else {
398 /* ONFI EDO mode 5 */
399 hw->clk_rate = 100000000;
400 wrn_dly_sel = BV_GPMI_CTRL1_WRN_DLY_SEL_NO_DELAY;
401 }
402
403 /* SDR core timings are given in picoseconds */
404 period_ps = div_u64((u64)NSEC_PER_SEC * 1000, hw->clk_rate);
405
406 addr_setup_cycles = TO_CYCLES(sdr->tALS_min, period_ps);
407 data_setup_cycles = TO_CYCLES(sdr->tDS_min, period_ps);
408 data_hold_cycles = TO_CYCLES(sdr->tDH_min, period_ps);
409 busy_timeout_cycles = TO_CYCLES(sdr->tWB_max + sdr->tR_max, period_ps);
410
411 hw->timing0 = BF_GPMI_TIMING0_ADDRESS_SETUP(addr_setup_cycles) |
412 BF_GPMI_TIMING0_DATA_HOLD(data_hold_cycles) |
413 BF_GPMI_TIMING0_DATA_SETUP(data_setup_cycles);
414 hw->timing1 = BF_GPMI_TIMING1_BUSY_TIMEOUT(busy_timeout_cycles * 4096);
415
416 /*
417 * Derive NFC ideal delay from {3}:
418 *
419 * (tREA + 4000 - tRP) * 8
420 * RDN_DELAY = -----------------------
421 * RP
422 */
423 if (period_ps > dll_threshold_ps) {
424 use_half_period = true;
425 reference_period_ps = period_ps / 2;
426 } else {
427 use_half_period = false;
428 reference_period_ps = period_ps;
429 }
430
431 tRP_ps = data_setup_cycles * period_ps;
432 sample_delay_ps = (sdr->tREA_max + 4000 - tRP_ps) * 8;
433 if (sample_delay_ps > 0)
434 sample_delay_factor = sample_delay_ps / reference_period_ps;
435 else
436 sample_delay_factor = 0;
437
438 hw->ctrl1n = BF_GPMI_CTRL1_WRN_DLY_SEL(wrn_dly_sel);
439 if (sample_delay_factor)
440 hw->ctrl1n |= BF_GPMI_CTRL1_RDN_DELAY(sample_delay_factor) |
441 BM_GPMI_CTRL1_DLL_ENABLE |
442 (use_half_period ? BM_GPMI_CTRL1_HALF_PERIOD : 0);
443 }
444
gpmi_nfc_apply_timings(struct gpmi_nand_data * this)445 void gpmi_nfc_apply_timings(struct gpmi_nand_data *this)
446 {
447 struct gpmi_nfc_hardware_timing *hw = &this->hw;
448 struct resources *r = &this->resources;
449 void __iomem *gpmi_regs = r->gpmi_regs;
450 unsigned int dll_wait_time_us;
451
452 clk_set_rate(r->clock[0], hw->clk_rate);
453
454 writel(hw->timing0, gpmi_regs + HW_GPMI_TIMING0);
455 writel(hw->timing1, gpmi_regs + HW_GPMI_TIMING1);
456
457 /*
458 * Clear several CTRL1 fields, DLL must be disabled when setting
459 * RDN_DELAY or HALF_PERIOD.
460 */
461 writel(BM_GPMI_CTRL1_CLEAR_MASK, gpmi_regs + HW_GPMI_CTRL1_CLR);
462 writel(hw->ctrl1n, gpmi_regs + HW_GPMI_CTRL1_SET);
463
464 /* Wait 64 clock cycles before using the GPMI after enabling the DLL */
465 dll_wait_time_us = USEC_PER_SEC / hw->clk_rate * 64;
466 if (!dll_wait_time_us)
467 dll_wait_time_us = 1;
468
469 /* Wait for the DLL to settle. */
470 udelay(dll_wait_time_us);
471 }
472
gpmi_setup_data_interface(struct mtd_info * mtd,int chipnr,const struct nand_data_interface * conf)473 int gpmi_setup_data_interface(struct mtd_info *mtd, int chipnr,
474 const struct nand_data_interface *conf)
475 {
476 struct nand_chip *chip = mtd_to_nand(mtd);
477 struct gpmi_nand_data *this = nand_get_controller_data(chip);
478 const struct nand_sdr_timings *sdr;
479
480 /* Retrieve required NAND timings */
481 sdr = nand_get_sdr_timings(conf);
482 if (IS_ERR(sdr))
483 return PTR_ERR(sdr);
484
485 /* Only MX6 GPMI controller can reach EDO timings */
486 if (sdr->tRC_min <= 25000 && !GPMI_IS_MX6(this))
487 return -ENOTSUPP;
488
489 /* Stop here if this call was just a check */
490 if (chipnr < 0)
491 return 0;
492
493 /* Do the actual derivation of the controller timings */
494 gpmi_nfc_compute_timings(this, sdr);
495
496 this->hw.must_apply_timings = true;
497
498 return 0;
499 }
500
501 /* Clears a BCH interrupt. */
gpmi_clear_bch(struct gpmi_nand_data * this)502 void gpmi_clear_bch(struct gpmi_nand_data *this)
503 {
504 struct resources *r = &this->resources;
505 writel(BM_BCH_CTRL_COMPLETE_IRQ, r->bch_regs + HW_BCH_CTRL_CLR);
506 }
507
508 /* Returns the Ready/Busy status of the given chip. */
gpmi_is_ready(struct gpmi_nand_data * this,unsigned chip)509 int gpmi_is_ready(struct gpmi_nand_data *this, unsigned chip)
510 {
511 struct resources *r = &this->resources;
512 uint32_t mask = 0;
513 uint32_t reg = 0;
514
515 if (GPMI_IS_MX23(this)) {
516 mask = MX23_BM_GPMI_DEBUG_READY0 << chip;
517 reg = readl(r->gpmi_regs + HW_GPMI_DEBUG);
518 } else if (GPMI_IS_MX28(this) || GPMI_IS_MX6(this)) {
519 /*
520 * In the imx6, all the ready/busy pins are bound
521 * together. So we only need to check chip 0.
522 */
523 if (GPMI_IS_MX6(this))
524 chip = 0;
525
526 /* MX28 shares the same R/B register as MX6Q. */
527 mask = MX28_BF_GPMI_STAT_READY_BUSY(1 << chip);
528 reg = readl(r->gpmi_regs + HW_GPMI_STAT);
529 } else
530 dev_err(this->dev, "unknown arch.\n");
531 return reg & mask;
532 }
533
gpmi_send_command(struct gpmi_nand_data * this)534 int gpmi_send_command(struct gpmi_nand_data *this)
535 {
536 struct dma_chan *channel = get_dma_chan(this);
537 struct dma_async_tx_descriptor *desc;
538 struct scatterlist *sgl;
539 int chip = this->current_chip;
540 int ret;
541 u32 pio[3];
542
543 /* [1] send out the PIO words */
544 pio[0] = BF_GPMI_CTRL0_COMMAND_MODE(BV_GPMI_CTRL0_COMMAND_MODE__WRITE)
545 | BM_GPMI_CTRL0_WORD_LENGTH
546 | BF_GPMI_CTRL0_CS(chip, this)
547 | BF_GPMI_CTRL0_LOCK_CS(LOCK_CS_ENABLE, this)
548 | BF_GPMI_CTRL0_ADDRESS(BV_GPMI_CTRL0_ADDRESS__NAND_CLE)
549 | BM_GPMI_CTRL0_ADDRESS_INCREMENT
550 | BF_GPMI_CTRL0_XFER_COUNT(this->command_length);
551 pio[1] = pio[2] = 0;
552 desc = dmaengine_prep_slave_sg(channel,
553 (struct scatterlist *)pio,
554 ARRAY_SIZE(pio), DMA_TRANS_NONE, 0);
555 if (!desc)
556 return -EINVAL;
557
558 /* [2] send out the COMMAND + ADDRESS string stored in @buffer */
559 sgl = &this->cmd_sgl;
560
561 sg_init_one(sgl, this->cmd_buffer, this->command_length);
562 dma_map_sg(this->dev, sgl, 1, DMA_TO_DEVICE);
563 desc = dmaengine_prep_slave_sg(channel,
564 sgl, 1, DMA_MEM_TO_DEV,
565 DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
566 if (!desc)
567 return -EINVAL;
568
569 /* [3] submit the DMA */
570 ret = start_dma_without_bch_irq(this, desc);
571
572 dma_unmap_sg(this->dev, sgl, 1, DMA_TO_DEVICE);
573
574 return ret;
575 }
576
gpmi_send_data(struct gpmi_nand_data * this,const void * buf,int len)577 int gpmi_send_data(struct gpmi_nand_data *this, const void *buf, int len)
578 {
579 struct dma_async_tx_descriptor *desc;
580 struct dma_chan *channel = get_dma_chan(this);
581 int chip = this->current_chip;
582 int ret;
583 uint32_t command_mode;
584 uint32_t address;
585 u32 pio[2];
586
587 /* [1] PIO */
588 command_mode = BV_GPMI_CTRL0_COMMAND_MODE__WRITE;
589 address = BV_GPMI_CTRL0_ADDRESS__NAND_DATA;
590
591 pio[0] = BF_GPMI_CTRL0_COMMAND_MODE(command_mode)
592 | BM_GPMI_CTRL0_WORD_LENGTH
593 | BF_GPMI_CTRL0_CS(chip, this)
594 | BF_GPMI_CTRL0_LOCK_CS(LOCK_CS_ENABLE, this)
595 | BF_GPMI_CTRL0_ADDRESS(address)
596 | BF_GPMI_CTRL0_XFER_COUNT(len);
597 pio[1] = 0;
598 desc = dmaengine_prep_slave_sg(channel, (struct scatterlist *)pio,
599 ARRAY_SIZE(pio), DMA_TRANS_NONE, 0);
600 if (!desc)
601 return -EINVAL;
602
603 /* [2] send DMA request */
604 prepare_data_dma(this, buf, len, DMA_TO_DEVICE);
605 desc = dmaengine_prep_slave_sg(channel, &this->data_sgl,
606 1, DMA_MEM_TO_DEV,
607 DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
608 if (!desc)
609 return -EINVAL;
610
611 /* [3] submit the DMA */
612 ret = start_dma_without_bch_irq(this, desc);
613
614 dma_unmap_sg(this->dev, &this->data_sgl, 1, DMA_TO_DEVICE);
615
616 return ret;
617 }
618
gpmi_read_data(struct gpmi_nand_data * this,void * buf,int len)619 int gpmi_read_data(struct gpmi_nand_data *this, void *buf, int len)
620 {
621 struct dma_async_tx_descriptor *desc;
622 struct dma_chan *channel = get_dma_chan(this);
623 int chip = this->current_chip;
624 int ret;
625 u32 pio[2];
626 bool direct;
627
628 /* [1] : send PIO */
629 pio[0] = BF_GPMI_CTRL0_COMMAND_MODE(BV_GPMI_CTRL0_COMMAND_MODE__READ)
630 | BM_GPMI_CTRL0_WORD_LENGTH
631 | BF_GPMI_CTRL0_CS(chip, this)
632 | BF_GPMI_CTRL0_LOCK_CS(LOCK_CS_ENABLE, this)
633 | BF_GPMI_CTRL0_ADDRESS(BV_GPMI_CTRL0_ADDRESS__NAND_DATA)
634 | BF_GPMI_CTRL0_XFER_COUNT(len);
635 pio[1] = 0;
636 desc = dmaengine_prep_slave_sg(channel,
637 (struct scatterlist *)pio,
638 ARRAY_SIZE(pio), DMA_TRANS_NONE, 0);
639 if (!desc)
640 return -EINVAL;
641
642 /* [2] : send DMA request */
643 direct = prepare_data_dma(this, buf, len, DMA_FROM_DEVICE);
644 desc = dmaengine_prep_slave_sg(channel, &this->data_sgl,
645 1, DMA_DEV_TO_MEM,
646 DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
647 if (!desc)
648 return -EINVAL;
649
650 /* [3] : submit the DMA */
651
652 ret = start_dma_without_bch_irq(this, desc);
653
654 dma_unmap_sg(this->dev, &this->data_sgl, 1, DMA_FROM_DEVICE);
655 if (!direct)
656 memcpy(buf, this->data_buffer_dma, len);
657
658 return ret;
659 }
660
gpmi_send_page(struct gpmi_nand_data * this,dma_addr_t payload,dma_addr_t auxiliary)661 int gpmi_send_page(struct gpmi_nand_data *this,
662 dma_addr_t payload, dma_addr_t auxiliary)
663 {
664 struct bch_geometry *geo = &this->bch_geometry;
665 uint32_t command_mode;
666 uint32_t address;
667 uint32_t ecc_command;
668 uint32_t buffer_mask;
669 struct dma_async_tx_descriptor *desc;
670 struct dma_chan *channel = get_dma_chan(this);
671 int chip = this->current_chip;
672 u32 pio[6];
673
674 /* A DMA descriptor that does an ECC page read. */
675 command_mode = BV_GPMI_CTRL0_COMMAND_MODE__WRITE;
676 address = BV_GPMI_CTRL0_ADDRESS__NAND_DATA;
677 ecc_command = BV_GPMI_ECCCTRL_ECC_CMD__BCH_ENCODE;
678 buffer_mask = BV_GPMI_ECCCTRL_BUFFER_MASK__BCH_PAGE |
679 BV_GPMI_ECCCTRL_BUFFER_MASK__BCH_AUXONLY;
680
681 pio[0] = BF_GPMI_CTRL0_COMMAND_MODE(command_mode)
682 | BM_GPMI_CTRL0_WORD_LENGTH
683 | BF_GPMI_CTRL0_CS(chip, this)
684 | BF_GPMI_CTRL0_LOCK_CS(LOCK_CS_ENABLE, this)
685 | BF_GPMI_CTRL0_ADDRESS(address)
686 | BF_GPMI_CTRL0_XFER_COUNT(0);
687 pio[1] = 0;
688 pio[2] = BM_GPMI_ECCCTRL_ENABLE_ECC
689 | BF_GPMI_ECCCTRL_ECC_CMD(ecc_command)
690 | BF_GPMI_ECCCTRL_BUFFER_MASK(buffer_mask);
691 pio[3] = geo->page_size;
692 pio[4] = payload;
693 pio[5] = auxiliary;
694
695 desc = dmaengine_prep_slave_sg(channel,
696 (struct scatterlist *)pio,
697 ARRAY_SIZE(pio), DMA_TRANS_NONE,
698 DMA_CTRL_ACK);
699 if (!desc)
700 return -EINVAL;
701
702 return start_dma_with_bch_irq(this, desc);
703 }
704
gpmi_read_page(struct gpmi_nand_data * this,dma_addr_t payload,dma_addr_t auxiliary)705 int gpmi_read_page(struct gpmi_nand_data *this,
706 dma_addr_t payload, dma_addr_t auxiliary)
707 {
708 struct bch_geometry *geo = &this->bch_geometry;
709 uint32_t command_mode;
710 uint32_t address;
711 uint32_t ecc_command;
712 uint32_t buffer_mask;
713 struct dma_async_tx_descriptor *desc;
714 struct dma_chan *channel = get_dma_chan(this);
715 int chip = this->current_chip;
716 u32 pio[6];
717
718 /* [1] Wait for the chip to report ready. */
719 command_mode = BV_GPMI_CTRL0_COMMAND_MODE__WAIT_FOR_READY;
720 address = BV_GPMI_CTRL0_ADDRESS__NAND_DATA;
721
722 pio[0] = BF_GPMI_CTRL0_COMMAND_MODE(command_mode)
723 | BM_GPMI_CTRL0_WORD_LENGTH
724 | BF_GPMI_CTRL0_CS(chip, this)
725 | BF_GPMI_CTRL0_LOCK_CS(LOCK_CS_ENABLE, this)
726 | BF_GPMI_CTRL0_ADDRESS(address)
727 | BF_GPMI_CTRL0_XFER_COUNT(0);
728 pio[1] = 0;
729 desc = dmaengine_prep_slave_sg(channel,
730 (struct scatterlist *)pio, 2,
731 DMA_TRANS_NONE, 0);
732 if (!desc)
733 return -EINVAL;
734
735 /* [2] Enable the BCH block and read. */
736 command_mode = BV_GPMI_CTRL0_COMMAND_MODE__READ;
737 address = BV_GPMI_CTRL0_ADDRESS__NAND_DATA;
738 ecc_command = BV_GPMI_ECCCTRL_ECC_CMD__BCH_DECODE;
739 buffer_mask = BV_GPMI_ECCCTRL_BUFFER_MASK__BCH_PAGE
740 | BV_GPMI_ECCCTRL_BUFFER_MASK__BCH_AUXONLY;
741
742 pio[0] = BF_GPMI_CTRL0_COMMAND_MODE(command_mode)
743 | BM_GPMI_CTRL0_WORD_LENGTH
744 | BF_GPMI_CTRL0_CS(chip, this)
745 | BF_GPMI_CTRL0_LOCK_CS(LOCK_CS_ENABLE, this)
746 | BF_GPMI_CTRL0_ADDRESS(address)
747 | BF_GPMI_CTRL0_XFER_COUNT(geo->page_size);
748
749 pio[1] = 0;
750 pio[2] = BM_GPMI_ECCCTRL_ENABLE_ECC
751 | BF_GPMI_ECCCTRL_ECC_CMD(ecc_command)
752 | BF_GPMI_ECCCTRL_BUFFER_MASK(buffer_mask);
753 pio[3] = geo->page_size;
754 pio[4] = payload;
755 pio[5] = auxiliary;
756 desc = dmaengine_prep_slave_sg(channel,
757 (struct scatterlist *)pio,
758 ARRAY_SIZE(pio), DMA_TRANS_NONE,
759 DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
760 if (!desc)
761 return -EINVAL;
762
763 /* [3] Disable the BCH block */
764 command_mode = BV_GPMI_CTRL0_COMMAND_MODE__WAIT_FOR_READY;
765 address = BV_GPMI_CTRL0_ADDRESS__NAND_DATA;
766
767 pio[0] = BF_GPMI_CTRL0_COMMAND_MODE(command_mode)
768 | BM_GPMI_CTRL0_WORD_LENGTH
769 | BF_GPMI_CTRL0_CS(chip, this)
770 | BF_GPMI_CTRL0_LOCK_CS(LOCK_CS_ENABLE, this)
771 | BF_GPMI_CTRL0_ADDRESS(address)
772 | BF_GPMI_CTRL0_XFER_COUNT(geo->page_size);
773 pio[1] = 0;
774 pio[2] = 0; /* clear GPMI_HW_GPMI_ECCCTRL, disable the BCH. */
775 desc = dmaengine_prep_slave_sg(channel,
776 (struct scatterlist *)pio, 3,
777 DMA_TRANS_NONE,
778 DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
779 if (!desc)
780 return -EINVAL;
781
782 /* [4] submit the DMA */
783 return start_dma_with_bch_irq(this, desc);
784 }
785
786 /**
787 * gpmi_copy_bits - copy bits from one memory region to another
788 * @dst: destination buffer
789 * @dst_bit_off: bit offset we're starting to write at
790 * @src: source buffer
791 * @src_bit_off: bit offset we're starting to read from
792 * @nbits: number of bits to copy
793 *
794 * This functions copies bits from one memory region to another, and is used by
795 * the GPMI driver to copy ECC sections which are not guaranteed to be byte
796 * aligned.
797 *
798 * src and dst should not overlap.
799 *
800 */
gpmi_copy_bits(u8 * dst,size_t dst_bit_off,const u8 * src,size_t src_bit_off,size_t nbits)801 void gpmi_copy_bits(u8 *dst, size_t dst_bit_off,
802 const u8 *src, size_t src_bit_off,
803 size_t nbits)
804 {
805 size_t i;
806 size_t nbytes;
807 u32 src_buffer = 0;
808 size_t bits_in_src_buffer = 0;
809
810 if (!nbits)
811 return;
812
813 /*
814 * Move src and dst pointers to the closest byte pointer and store bit
815 * offsets within a byte.
816 */
817 src += src_bit_off / 8;
818 src_bit_off %= 8;
819
820 dst += dst_bit_off / 8;
821 dst_bit_off %= 8;
822
823 /*
824 * Initialize the src_buffer value with bits available in the first
825 * byte of data so that we end up with a byte aligned src pointer.
826 */
827 if (src_bit_off) {
828 src_buffer = src[0] >> src_bit_off;
829 if (nbits >= (8 - src_bit_off)) {
830 bits_in_src_buffer += 8 - src_bit_off;
831 } else {
832 src_buffer &= GENMASK(nbits - 1, 0);
833 bits_in_src_buffer += nbits;
834 }
835 nbits -= bits_in_src_buffer;
836 src++;
837 }
838
839 /* Calculate the number of bytes that can be copied from src to dst. */
840 nbytes = nbits / 8;
841
842 /* Try to align dst to a byte boundary. */
843 if (dst_bit_off) {
844 if (bits_in_src_buffer < (8 - dst_bit_off) && nbytes) {
845 src_buffer |= src[0] << bits_in_src_buffer;
846 bits_in_src_buffer += 8;
847 src++;
848 nbytes--;
849 }
850
851 if (bits_in_src_buffer >= (8 - dst_bit_off)) {
852 dst[0] &= GENMASK(dst_bit_off - 1, 0);
853 dst[0] |= src_buffer << dst_bit_off;
854 src_buffer >>= (8 - dst_bit_off);
855 bits_in_src_buffer -= (8 - dst_bit_off);
856 dst_bit_off = 0;
857 dst++;
858 if (bits_in_src_buffer > 7) {
859 bits_in_src_buffer -= 8;
860 dst[0] = src_buffer;
861 dst++;
862 src_buffer >>= 8;
863 }
864 }
865 }
866
867 if (!bits_in_src_buffer && !dst_bit_off) {
868 /*
869 * Both src and dst pointers are byte aligned, thus we can
870 * just use the optimized memcpy function.
871 */
872 if (nbytes)
873 memcpy(dst, src, nbytes);
874 } else {
875 /*
876 * src buffer is not byte aligned, hence we have to copy each
877 * src byte to the src_buffer variable before extracting a byte
878 * to store in dst.
879 */
880 for (i = 0; i < nbytes; i++) {
881 src_buffer |= src[i] << bits_in_src_buffer;
882 dst[i] = src_buffer;
883 src_buffer >>= 8;
884 }
885 }
886 /* Update dst and src pointers */
887 dst += nbytes;
888 src += nbytes;
889
890 /*
891 * nbits is the number of remaining bits. It should not exceed 8 as
892 * we've already copied as much bytes as possible.
893 */
894 nbits %= 8;
895
896 /*
897 * If there's no more bits to copy to the destination and src buffer
898 * was already byte aligned, then we're done.
899 */
900 if (!nbits && !bits_in_src_buffer)
901 return;
902
903 /* Copy the remaining bits to src_buffer */
904 if (nbits)
905 src_buffer |= (*src & GENMASK(nbits - 1, 0)) <<
906 bits_in_src_buffer;
907 bits_in_src_buffer += nbits;
908
909 /*
910 * In case there were not enough bits to get a byte aligned dst buffer
911 * prepare the src_buffer variable to match the dst organization (shift
912 * src_buffer by dst_bit_off and retrieve the least significant bits
913 * from dst).
914 */
915 if (dst_bit_off)
916 src_buffer = (src_buffer << dst_bit_off) |
917 (*dst & GENMASK(dst_bit_off - 1, 0));
918 bits_in_src_buffer += dst_bit_off;
919
920 /*
921 * Keep most significant bits from dst if we end up with an unaligned
922 * number of bits.
923 */
924 nbytes = bits_in_src_buffer / 8;
925 if (bits_in_src_buffer % 8) {
926 src_buffer |= (dst[nbytes] &
927 GENMASK(7, bits_in_src_buffer % 8)) <<
928 (nbytes * 8);
929 nbytes++;
930 }
931
932 /* Copy the remaining bytes to dst */
933 for (i = 0; i < nbytes; i++) {
934 dst[i] = src_buffer;
935 src_buffer >>= 8;
936 }
937 }
938