projects / openwrt / staging / jow.git / blob
? search:


ed209050123b7ecf984a34c22859954aa22e6840
[openwrt/staging/jow.git] /
1 From 8170bafa8936e9fbfdce992932a63bd20eca3bc3 Mon Sep 17 00:00:00 2001
2 From: Chuanhong Guo <gch981213@gmail.com>
3 Date: Sat, 2 Apr 2022 10:16:11 +0800
4 Subject: [PATCH v6 2/5] spi: add driver for MTK SPI NAND Flash Interface
5
6 This driver implements support for the SPI-NAND mode of MTK NAND Flash
7 Interface as a SPI-MEM controller with pipelined ECC capability.
8
9 Signed-off-by: Chuanhong Guo <gch981213@gmail.com>
10 Tested-by: Daniel Golle <daniel@makrotopia.org>
11 ---
12 Change since v1:
13 fix CI warnings
14
15 Changes since v2:
16 use streamed DMA api to avoid an extra memory copy during read
17 make ECC engine config a per-nand context
18 take user-requested ECC strength into account
19
20 Change since v3: none
21 Changes since v4:
22 fix missing OOB write
23 print page format with dev_dbg
24 replace uint*_t copied from vendor driver with u*
25
26 Changes since v5:
27 add missing nfi mode register configuration in probe
28 fix an off-by-one bug in mtk_snand_mac_io
29
30 drivers/spi/Kconfig | 10 +
31 drivers/spi/Makefile | 1 +
32 drivers/spi/spi-mtk-snfi.c | 1470 ++++++++++++++++++++++++++++++++++++
33 3 files changed, 1481 insertions(+)
34 create mode 100644 drivers/spi/spi-mtk-snfi.c
35
36 --- a/drivers/spi/Kconfig
37 +++ b/drivers/spi/Kconfig
38 @@ -529,6 +529,16 @@ config SPI_MTK_NOR
39 SPI interface as well as several SPI NOR specific instructions
40 via SPI MEM interface.
41
42 +config SPI_MTK_SNFI
43 + tristate "MediaTek SPI NAND Flash Interface"
44 + depends on ARCH_MEDIATEK || COMPILE_TEST
45 + depends on MTD_NAND_ECC_MEDIATEK
46 + help
47 + This enables support for SPI-NAND mode on the MediaTek NAND
48 + Flash Interface found on MediaTek ARM SoCs. This controller
49 + is implemented as a SPI-MEM controller with pipelined ECC
50 + capcability.
51 +
52 config SPI_NPCM_FIU
53 tristate "Nuvoton NPCM FLASH Interface Unit"
54 depends on ARCH_NPCM || COMPILE_TEST
55 --- a/drivers/spi/Makefile
56 +++ b/drivers/spi/Makefile
57 @@ -71,6 +71,7 @@ obj-$(CONFIG_SPI_MPC52xx) += spi-mpc52x
58 obj-$(CONFIG_SPI_MT65XX) += spi-mt65xx.o
59 obj-$(CONFIG_SPI_MT7621) += spi-mt7621.o
60 obj-$(CONFIG_SPI_MTK_NOR) += spi-mtk-nor.o
61 +obj-$(CONFIG_SPI_MTK_SNFI) += spi-mtk-snfi.o
62 obj-$(CONFIG_SPI_MXIC) += spi-mxic.o
63 obj-$(CONFIG_SPI_MXS) += spi-mxs.o
64 obj-$(CONFIG_SPI_NPCM_FIU) += spi-npcm-fiu.o
65 --- /dev/null
66 +++ b/drivers/spi/spi-mtk-snfi.c
67 @@ -0,0 +1,1470 @@
68 +// SPDX-License-Identifier: GPL-2.0
69 +//
70 +// Driver for the SPI-NAND mode of Mediatek NAND Flash Interface
71 +//
72 +// Copyright (c) 2022 Chuanhong Guo <gch981213@gmail.com>
73 +//
74 +// This driver is based on the SPI-NAND mtd driver from Mediatek SDK:
75 +//
76 +// Copyright (C) 2020 MediaTek Inc.
77 +// Author: Weijie Gao <weijie.gao@mediatek.com>
78 +//
79 +// This controller organize the page data as several interleaved sectors
80 +// like the following: (sizeof(FDM + ECC) = snf->nfi_cfg.spare_size)
81 +// +---------+------+------+---------+------+------+-----+
82 +// | Sector1 | FDM1 | ECC1 | Sector2 | FDM2 | ECC2 | ... |
83 +// +---------+------+------+---------+------+------+-----+
84 +// With auto-format turned on, DMA only returns this part:
85 +// +---------+---------+-----+
86 +// | Sector1 | Sector2 | ... |
87 +// +---------+---------+-----+
88 +// The FDM data will be filled to the registers, and ECC parity data isn't
89 +// accessible.
90 +// With auto-format off, all ((Sector+FDM+ECC)*nsectors) will be read over DMA
91 +// in it's original order shown in the first table. ECC can't be turned on when
92 +// auto-format is off.
93 +//
94 +// However, Linux SPI-NAND driver expects the data returned as:
95 +// +------+-----+
96 +// | Page | OOB |
97 +// +------+-----+
98 +// where the page data is continuously stored instead of interleaved.
99 +// So we assume all instructions matching the page_op template between ECC
100 +// prepare_io_req and finish_io_req are for page cache r/w.
101 +// Here's how this spi-mem driver operates when reading:
102 +// 1. Always set snf->autofmt = true in prepare_io_req (even when ECC is off).
103 +// 2. Perform page ops and let the controller fill the DMA bounce buffer with
104 +// de-interleaved sector data and set FDM registers.
105 +// 3. Return the data as:
106 +// +---------+---------+-----+------+------+-----+
107 +// | Sector1 | Sector2 | ... | FDM1 | FDM2 | ... |
108 +// +---------+---------+-----+------+------+-----+
109 +// 4. For other matching spi_mem ops outside a prepare/finish_io_req pair,
110 +// read the data with auto-format off into the bounce buffer and copy
111 +// needed data to the buffer specified in the request.
112 +//
113 +// Write requests operates in a similar manner.
114 +// As a limitation of this strategy, we won't be able to access any ECC parity
115 +// data at all in Linux.
116 +//
117 +// Here's the bad block mark situation on MTK chips:
118 +// In older chips like mt7622, MTK uses the first FDM byte in the first sector
119 +// as the bad block mark. After de-interleaving, this byte appears at [pagesize]
120 +// in the returned data, which is the BBM position expected by kernel. However,
121 +// the conventional bad block mark is the first byte of the OOB, which is part
122 +// of the last sector data in the interleaved layout. Instead of fixing their
123 +// hardware, MTK decided to address this inconsistency in software. On these
124 +// later chips, the BootROM expects the following:
125 +// 1. The [pagesize] byte on a nand page is used as BBM, which will appear at
126 +// (page_size - (nsectors - 1) * spare_size) in the DMA buffer.
127 +// 2. The original byte stored at that position in the DMA buffer will be stored
128 +// as the first byte of the FDM section in the last sector.
129 +// We can't disagree with the BootROM, so after de-interleaving, we need to
130 +// perform the following swaps in read:
131 +// 1. Store the BBM at [page_size - (nsectors - 1) * spare_size] to [page_size],
132 +// which is the expected BBM position by kernel.
133 +// 2. Store the page data byte at [pagesize + (nsectors-1) * fdm] back to
134 +// [page_size - (nsectors - 1) * spare_size]
135 +// Similarly, when writing, we need to perform swaps in the other direction.
136 +
137 +#include <linux/kernel.h>
138 +#include <linux/module.h>
139 +#include <linux/init.h>
140 +#include <linux/device.h>
141 +#include <linux/mutex.h>
142 +#include <linux/clk.h>
143 +#include <linux/interrupt.h>
144 +#include <linux/dma-mapping.h>
145 +#include <linux/iopoll.h>
146 +#include <linux/of_platform.h>
147 +#include <linux/mtd/nand-ecc-mtk.h>
148 +#include <linux/spi/spi.h>
149 +#include <linux/spi/spi-mem.h>
150 +#include <linux/mtd/nand.h>
151 +
152 +// NFI registers
153 +#define NFI_CNFG 0x000
154 +#define CNFG_OP_MODE_S 12
155 +#define CNFG_OP_MODE_CUST 6
156 +#define CNFG_OP_MODE_PROGRAM 3
157 +#define CNFG_AUTO_FMT_EN BIT(9)
158 +#define CNFG_HW_ECC_EN BIT(8)
159 +#define CNFG_DMA_BURST_EN BIT(2)
160 +#define CNFG_READ_MODE BIT(1)
161 +#define CNFG_DMA_MODE BIT(0)
162 +
163 +#define NFI_PAGEFMT 0x0004
164 +#define NFI_SPARE_SIZE_LS_S 16
165 +#define NFI_FDM_ECC_NUM_S 12
166 +#define NFI_FDM_NUM_S 8
167 +#define NFI_SPARE_SIZE_S 4
168 +#define NFI_SEC_SEL_512 BIT(2)
169 +#define NFI_PAGE_SIZE_S 0
170 +#define NFI_PAGE_SIZE_512_2K 0
171 +#define NFI_PAGE_SIZE_2K_4K 1
172 +#define NFI_PAGE_SIZE_4K_8K 2
173 +#define NFI_PAGE_SIZE_8K_16K 3
174 +
175 +#define NFI_CON 0x008
176 +#define CON_SEC_NUM_S 12
177 +#define CON_BWR BIT(9)
178 +#define CON_BRD BIT(8)
179 +#define CON_NFI_RST BIT(1)
180 +#define CON_FIFO_FLUSH BIT(0)
181 +
182 +#define NFI_INTR_EN 0x010
183 +#define NFI_INTR_STA 0x014
184 +#define NFI_IRQ_INTR_EN BIT(31)
185 +#define NFI_IRQ_CUS_READ BIT(8)
186 +#define NFI_IRQ_CUS_PG BIT(7)
187 +
188 +#define NFI_CMD 0x020
189 +#define NFI_CMD_DUMMY_READ 0x00
190 +#define NFI_CMD_DUMMY_WRITE 0x80
191 +
192 +#define NFI_STRDATA 0x040
193 +#define STR_DATA BIT(0)
194 +
195 +#define NFI_STA 0x060
196 +#define NFI_NAND_FSM GENMASK(28, 24)
197 +#define NFI_FSM GENMASK(19, 16)
198 +#define READ_EMPTY BIT(12)
199 +
200 +#define NFI_FIFOSTA 0x064
201 +#define FIFO_WR_REMAIN_S 8
202 +#define FIFO_RD_REMAIN_S 0
203 +
204 +#define NFI_ADDRCNTR 0x070
205 +#define SEC_CNTR GENMASK(16, 12)
206 +#define SEC_CNTR_S 12
207 +#define NFI_SEC_CNTR(val) (((val)&SEC_CNTR) >> SEC_CNTR_S)
208 +
209 +#define NFI_STRADDR 0x080
210 +
211 +#define NFI_BYTELEN 0x084
212 +#define BUS_SEC_CNTR(val) (((val)&SEC_CNTR) >> SEC_CNTR_S)
213 +
214 +#define NFI_FDM0L 0x0a0
215 +#define NFI_FDM0M 0x0a4
216 +#define NFI_FDML(n) (NFI_FDM0L + (n)*8)
217 +#define NFI_FDMM(n) (NFI_FDM0M + (n)*8)
218 +
219 +#define NFI_DEBUG_CON1 0x220
220 +#define WBUF_EN BIT(2)
221 +
222 +#define NFI_MASTERSTA 0x224
223 +#define MAS_ADDR GENMASK(11, 9)
224 +#define MAS_RD GENMASK(8, 6)
225 +#define MAS_WR GENMASK(5, 3)
226 +#define MAS_RDDLY GENMASK(2, 0)
227 +#define NFI_MASTERSTA_MASK_7622 (MAS_ADDR | MAS_RD | MAS_WR | MAS_RDDLY)
228 +
229 +// SNFI registers
230 +#define SNF_MAC_CTL 0x500
231 +#define MAC_XIO_SEL BIT(4)
232 +#define SF_MAC_EN BIT(3)
233 +#define SF_TRIG BIT(2)
234 +#define WIP_READY BIT(1)
235 +#define WIP BIT(0)
236 +
237 +#define SNF_MAC_OUTL 0x504
238 +#define SNF_MAC_INL 0x508
239 +
240 +#define SNF_RD_CTL2 0x510
241 +#define DATA_READ_DUMMY_S 8
242 +#define DATA_READ_MAX_DUMMY 0xf
243 +#define DATA_READ_CMD_S 0
244 +
245 +#define SNF_RD_CTL3 0x514
246 +
247 +#define SNF_PG_CTL1 0x524
248 +#define PG_LOAD_CMD_S 8
249 +
250 +#define SNF_PG_CTL2 0x528
251 +
252 +#define SNF_MISC_CTL 0x538
253 +#define SW_RST BIT(28)
254 +#define FIFO_RD_LTC_S 25
255 +#define PG_LOAD_X4_EN BIT(20)
256 +#define DATA_READ_MODE_S 16
257 +#define DATA_READ_MODE GENMASK(18, 16)
258 +#define DATA_READ_MODE_X1 0
259 +#define DATA_READ_MODE_X2 1
260 +#define DATA_READ_MODE_X4 2
261 +#define DATA_READ_MODE_DUAL 5
262 +#define DATA_READ_MODE_QUAD 6
263 +#define PG_LOAD_CUSTOM_EN BIT(7)
264 +#define DATARD_CUSTOM_EN BIT(6)
265 +#define CS_DESELECT_CYC_S 0
266 +
267 +#define SNF_MISC_CTL2 0x53c
268 +#define PROGRAM_LOAD_BYTE_NUM_S 16
269 +#define READ_DATA_BYTE_NUM_S 11
270 +
271 +#define SNF_DLY_CTL3 0x548
272 +#define SFCK_SAM_DLY_S 0
273 +
274 +#define SNF_STA_CTL1 0x550
275 +#define CUS_PG_DONE BIT(28)
276 +#define CUS_READ_DONE BIT(27)
277 +#define SPI_STATE_S 0
278 +#define SPI_STATE GENMASK(3, 0)
279 +
280 +#define SNF_CFG 0x55c
281 +#define SPI_MODE BIT(0)
282 +
283 +#define SNF_GPRAM 0x800
284 +#define SNF_GPRAM_SIZE 0xa0
285 +
286 +#define SNFI_POLL_INTERVAL 1000000
287 +
288 +static const u8 mt7622_spare_sizes[] = { 16, 26, 27, 28 };
289 +
290 +struct mtk_snand_caps {
291 + u16 sector_size;
292 + u16 max_sectors;
293 + u16 fdm_size;
294 + u16 fdm_ecc_size;
295 + u16 fifo_size;
296 +
297 + bool bbm_swap;
298 + bool empty_page_check;
299 + u32 mastersta_mask;
300 +
301 + const u8 *spare_sizes;
302 + u32 num_spare_size;
303 +};
304 +
305 +static const struct mtk_snand_caps mt7622_snand_caps = {
306 + .sector_size = 512,
307 + .max_sectors = 8,
308 + .fdm_size = 8,
309 + .fdm_ecc_size = 1,
310 + .fifo_size = 32,
311 + .bbm_swap = false,
312 + .empty_page_check = false,
313 + .mastersta_mask = NFI_MASTERSTA_MASK_7622,
314 + .spare_sizes = mt7622_spare_sizes,
315 + .num_spare_size = ARRAY_SIZE(mt7622_spare_sizes)
316 +};
317 +
318 +static const struct mtk_snand_caps mt7629_snand_caps = {
319 + .sector_size = 512,
320 + .max_sectors = 8,
321 + .fdm_size = 8,
322 + .fdm_ecc_size = 1,
323 + .fifo_size = 32,
324 + .bbm_swap = true,
325 + .empty_page_check = false,
326 + .mastersta_mask = NFI_MASTERSTA_MASK_7622,
327 + .spare_sizes = mt7622_spare_sizes,
328 + .num_spare_size = ARRAY_SIZE(mt7622_spare_sizes)
329 +};
330 +
331 +struct mtk_snand_conf {
332 + size_t page_size;
333 + size_t oob_size;
334 + u8 nsectors;
335 + u8 spare_size;
336 +};
337 +
338 +struct mtk_snand {
339 + struct spi_controller *ctlr;
340 + struct device *dev;
341 + struct clk *nfi_clk;
342 + struct clk *pad_clk;
343 + void __iomem *nfi_base;
344 + int irq;
345 + struct completion op_done;
346 + const struct mtk_snand_caps *caps;
347 + struct mtk_ecc_config *ecc_cfg;
348 + struct mtk_ecc *ecc;
349 + struct mtk_snand_conf nfi_cfg;
350 + struct mtk_ecc_stats ecc_stats;
351 + struct nand_ecc_engine ecc_eng;
352 + bool autofmt;
353 + u8 *buf;
354 + size_t buf_len;
355 +};
356 +
357 +static struct mtk_snand *nand_to_mtk_snand(struct nand_device *nand)
358 +{
359 + struct nand_ecc_engine *eng = nand->ecc.engine;
360 +
361 + return container_of(eng, struct mtk_snand, ecc_eng);
362 +}
363 +
364 +static inline int snand_prepare_bouncebuf(struct mtk_snand *snf, size_t size)
365 +{
366 + if (snf->buf_len >= size)
367 + return 0;
368 + kfree(snf->buf);
369 + snf->buf = kmalloc(size, GFP_KERNEL);
370 + if (!snf->buf)
371 + return -ENOMEM;
372 + snf->buf_len = size;
373 + memset(snf->buf, 0xff, snf->buf_len);
374 + return 0;
375 +}
376 +
377 +static inline u32 nfi_read32(struct mtk_snand *snf, u32 reg)
378 +{
379 + return readl(snf->nfi_base + reg);
380 +}
381 +
382 +static inline void nfi_write32(struct mtk_snand *snf, u32 reg, u32 val)
383 +{
384 + writel(val, snf->nfi_base + reg);
385 +}
386 +
387 +static inline void nfi_write16(struct mtk_snand *snf, u32 reg, u16 val)
388 +{
389 + writew(val, snf->nfi_base + reg);
390 +}
391 +
392 +static inline void nfi_rmw32(struct mtk_snand *snf, u32 reg, u32 clr, u32 set)
393 +{
394 + u32 val;
395 +
396 + val = readl(snf->nfi_base + reg);
397 + val &= ~clr;
398 + val |= set;
399 + writel(val, snf->nfi_base + reg);
400 +}
401 +
402 +static void nfi_read_data(struct mtk_snand *snf, u32 reg, u8 *data, u32 len)
403 +{
404 + u32 i, val = 0, es = sizeof(u32);
405 +
406 + for (i = reg; i < reg + len; i++) {
407 + if (i == reg || i % es == 0)
408 + val = nfi_read32(snf, i & ~(es - 1));
409 +
410 + *data++ = (u8)(val >> (8 * (i % es)));
411 + }
412 +}
413 +
414 +static int mtk_nfi_reset(struct mtk_snand *snf)
415 +{
416 + u32 val, fifo_mask;
417 + int ret;
418 +
419 + nfi_write32(snf, NFI_CON, CON_FIFO_FLUSH | CON_NFI_RST);
420 +
421 + ret = readw_poll_timeout(snf->nfi_base + NFI_MASTERSTA, val,
422 + !(val & snf->caps->mastersta_mask), 0,
423 + SNFI_POLL_INTERVAL);
424 + if (ret) {
425 + dev_err(snf->dev, "NFI master is still busy after reset\n");
426 + return ret;
427 + }
428 +
429 + ret = readl_poll_timeout(snf->nfi_base + NFI_STA, val,
430 + !(val & (NFI_FSM | NFI_NAND_FSM)), 0,
431 + SNFI_POLL_INTERVAL);
432 + if (ret) {
433 + dev_err(snf->dev, "Failed to reset NFI\n");
434 + return ret;
435 + }
436 +
437 + fifo_mask = ((snf->caps->fifo_size - 1) << FIFO_RD_REMAIN_S) |
438 + ((snf->caps->fifo_size - 1) << FIFO_WR_REMAIN_S);
439 + ret = readw_poll_timeout(snf->nfi_base + NFI_FIFOSTA, val,
440 + !(val & fifo_mask), 0, SNFI_POLL_INTERVAL);
441 + if (ret) {
442 + dev_err(snf->dev, "NFI FIFOs are not empty\n");
443 + return ret;
444 + }
445 +
446 + return 0;
447 +}
448 +
449 +static int mtk_snand_mac_reset(struct mtk_snand *snf)
450 +{
451 + int ret;
452 + u32 val;
453 +
454 + nfi_rmw32(snf, SNF_MISC_CTL, 0, SW_RST);
455 +
456 + ret = readl_poll_timeout(snf->nfi_base + SNF_STA_CTL1, val,
457 + !(val & SPI_STATE), 0, SNFI_POLL_INTERVAL);
458 + if (ret)
459 + dev_err(snf->dev, "Failed to reset SNFI MAC\n");
460 +
461 + nfi_write32(snf, SNF_MISC_CTL,
462 + (2 << FIFO_RD_LTC_S) | (10 << CS_DESELECT_CYC_S));
463 +
464 + return ret;
465 +}
466 +
467 +static int mtk_snand_mac_trigger(struct mtk_snand *snf, u32 outlen, u32 inlen)
468 +{
469 + int ret;
470 + u32 val;
471 +
472 + nfi_write32(snf, SNF_MAC_CTL, SF_MAC_EN);
473 + nfi_write32(snf, SNF_MAC_OUTL, outlen);
474 + nfi_write32(snf, SNF_MAC_INL, inlen);
475 +
476 + nfi_write32(snf, SNF_MAC_CTL, SF_MAC_EN | SF_TRIG);
477 +
478 + ret = readl_poll_timeout(snf->nfi_base + SNF_MAC_CTL, val,
479 + val & WIP_READY, 0, SNFI_POLL_INTERVAL);
480 + if (ret) {
481 + dev_err(snf->dev, "Timed out waiting for WIP_READY\n");
482 + goto cleanup;
483 + }
484 +
485 + ret = readl_poll_timeout(snf->nfi_base + SNF_MAC_CTL, val, !(val & WIP),
486 + 0, SNFI_POLL_INTERVAL);
487 + if (ret)
488 + dev_err(snf->dev, "Timed out waiting for WIP cleared\n");
489 +
490 +cleanup:
491 + nfi_write32(snf, SNF_MAC_CTL, 0);
492 +
493 + return ret;
494 +}
495 +
496 +static int mtk_snand_mac_io(struct mtk_snand *snf, const struct spi_mem_op *op)
497 +{
498 + u32 rx_len = 0;
499 + u32 reg_offs = 0;
500 + u32 val = 0;
501 + const u8 *tx_buf = NULL;
502 + u8 *rx_buf = NULL;
503 + int i, ret;
504 + u8 b;
505 +
506 + if (op->data.dir == SPI_MEM_DATA_IN) {
507 + rx_len = op->data.nbytes;
508 + rx_buf = op->data.buf.in;
509 + } else {
510 + tx_buf = op->data.buf.out;
511 + }
512 +
513 + mtk_snand_mac_reset(snf);
514 +
515 + for (i = 0; i < op->cmd.nbytes; i++, reg_offs++) {
516 + b = (op->cmd.opcode >> ((op->cmd.nbytes - i - 1) * 8)) & 0xff;
517 + val |= b << (8 * (reg_offs % 4));
518 + if (reg_offs % 4 == 3) {
519 + nfi_write32(snf, SNF_GPRAM + reg_offs - 3, val);
520 + val = 0;
521 + }
522 + }
523 +
524 + for (i = 0; i < op->addr.nbytes; i++, reg_offs++) {
525 + b = (op->addr.val >> ((op->addr.nbytes - i - 1) * 8)) & 0xff;
526 + val |= b << (8 * (reg_offs % 4));
527 + if (reg_offs % 4 == 3) {
528 + nfi_write32(snf, SNF_GPRAM + reg_offs - 3, val);
529 + val = 0;
530 + }
531 + }
532 +
533 + for (i = 0; i < op->dummy.nbytes; i++, reg_offs++) {
534 + if (reg_offs % 4 == 3) {
535 + nfi_write32(snf, SNF_GPRAM + reg_offs - 3, val);
536 + val = 0;
537 + }
538 + }
539 +
540 + if (op->data.dir == SPI_MEM_DATA_OUT) {
541 + for (i = 0; i < op->data.nbytes; i++, reg_offs++) {
542 + val |= tx_buf[i] << (8 * (reg_offs % 4));
543 + if (reg_offs % 4 == 3) {
544 + nfi_write32(snf, SNF_GPRAM + reg_offs - 3, val);
545 + val = 0;
546 + }
547 + }
548 + }
549 +
550 + if (reg_offs % 4)
551 + nfi_write32(snf, SNF_GPRAM + (reg_offs & ~3), val);
552 +
553 + for (i = 0; i < reg_offs; i += 4)
554 + dev_dbg(snf->dev, "%d: %08X", i,
555 + nfi_read32(snf, SNF_GPRAM + i));
556 +
557 + dev_dbg(snf->dev, "SNF TX: %u RX: %u", reg_offs, rx_len);
558 +
559 + ret = mtk_snand_mac_trigger(snf, reg_offs, rx_len);
560 + if (ret)
561 + return ret;
562 +
563 + if (!rx_len)
564 + return 0;
565 +
566 + nfi_read_data(snf, SNF_GPRAM + reg_offs, rx_buf, rx_len);
567 + return 0;
568 +}
569 +
570 +static int mtk_snand_setup_pagefmt(struct mtk_snand *snf, u32 page_size,
571 + u32 oob_size)
572 +{
573 + int spare_idx = -1;
574 + u32 spare_size, spare_size_shift, pagesize_idx;
575 + u32 sector_size_512;
576 + u8 nsectors;
577 + int i;
578 +
579 + // skip if it's already configured as required.
580 + if (snf->nfi_cfg.page_size == page_size &&
581 + snf->nfi_cfg.oob_size == oob_size)
582 + return 0;
583 +
584 + nsectors = page_size / snf->caps->sector_size;
585 + if (nsectors > snf->caps->max_sectors) {
586 + dev_err(snf->dev, "too many sectors required.\n");
587 + goto err;
588 + }
589 +
590 + if (snf->caps->sector_size == 512) {
591 + sector_size_512 = NFI_SEC_SEL_512;
592 + spare_size_shift = NFI_SPARE_SIZE_S;
593 + } else {
594 + sector_size_512 = 0;
595 + spare_size_shift = NFI_SPARE_SIZE_LS_S;
596 + }
597 +
598 + switch (page_size) {
599 + case SZ_512:
600 + pagesize_idx = NFI_PAGE_SIZE_512_2K;
601 + break;
602 + case SZ_2K:
603 + if (snf->caps->sector_size == 512)
604 + pagesize_idx = NFI_PAGE_SIZE_2K_4K;
605 + else
606 + pagesize_idx = NFI_PAGE_SIZE_512_2K;
607 + break;
608 + case SZ_4K:
609 + if (snf->caps->sector_size == 512)
610 + pagesize_idx = NFI_PAGE_SIZE_4K_8K;
611 + else
612 + pagesize_idx = NFI_PAGE_SIZE_2K_4K;
613 + break;
614 + case SZ_8K:
615 + if (snf->caps->sector_size == 512)
616 + pagesize_idx = NFI_PAGE_SIZE_8K_16K;
617 + else
618 + pagesize_idx = NFI_PAGE_SIZE_4K_8K;
619 + break;
620 + case SZ_16K:
621 + pagesize_idx = NFI_PAGE_SIZE_8K_16K;
622 + break;
623 + default:
624 + dev_err(snf->dev, "unsupported page size.\n");
625 + goto err;
626 + }
627 +
628 + spare_size = oob_size / nsectors;
629 + // If we're using the 1KB sector size, HW will automatically double the
630 + // spare size. We should only use half of the value in this case.
631 + if (snf->caps->sector_size == 1024)
632 + spare_size /= 2;
633 +
634 + for (i = snf->caps->num_spare_size - 1; i >= 0; i--) {
635 + if (snf->caps->spare_sizes[i] <= spare_size) {
636 + spare_size = snf->caps->spare_sizes[i];
637 + if (snf->caps->sector_size == 1024)
638 + spare_size *= 2;
639 + spare_idx = i;
640 + break;
641 + }
642 + }
643 +
644 + if (spare_idx < 0) {
645 + dev_err(snf->dev, "unsupported spare size: %u\n", spare_size);
646 + goto err;
647 + }
648 +
649 + nfi_write32(snf, NFI_PAGEFMT,
650 + (snf->caps->fdm_ecc_size << NFI_FDM_ECC_NUM_S) |
651 + (snf->caps->fdm_size << NFI_FDM_NUM_S) |
652 + (spare_idx << spare_size_shift) |
653 + (pagesize_idx << NFI_PAGE_SIZE_S) |
654 + sector_size_512);
655 +
656 + snf->nfi_cfg.page_size = page_size;
657 + snf->nfi_cfg.oob_size = oob_size;
658 + snf->nfi_cfg.nsectors = nsectors;
659 + snf->nfi_cfg.spare_size = spare_size;
660 +
661 + dev_dbg(snf->dev, "page format: (%u + %u) * %u\n",
662 + snf->caps->sector_size, spare_size, nsectors);
663 + return snand_prepare_bouncebuf(snf, page_size + oob_size);
664 +err:
665 + dev_err(snf->dev, "page size %u + %u is not supported\n", page_size,
666 + oob_size);
667 + return -EOPNOTSUPP;
668 +}
669 +
670 +static int mtk_snand_ooblayout_ecc(struct mtd_info *mtd, int section,
671 + struct mtd_oob_region *oobecc)
672 +{
673 + // ECC area is not accessible
674 + return -ERANGE;
675 +}
676 +
677 +static int mtk_snand_ooblayout_free(struct mtd_info *mtd, int section,
678 + struct mtd_oob_region *oobfree)
679 +{
680 + struct nand_device *nand = mtd_to_nanddev(mtd);
681 + struct mtk_snand *ms = nand_to_mtk_snand(nand);
682 +
683 + if (section >= ms->nfi_cfg.nsectors)
684 + return -ERANGE;
685 +
686 + oobfree->length = ms->caps->fdm_size - 1;
687 + oobfree->offset = section * ms->caps->fdm_size + 1;
688 + return 0;
689 +}
690 +
691 +static const struct mtd_ooblayout_ops mtk_snand_ooblayout = {
692 + .ecc = mtk_snand_ooblayout_ecc,
693 + .free = mtk_snand_ooblayout_free,
694 +};
695 +
696 +static int mtk_snand_ecc_init_ctx(struct nand_device *nand)
697 +{
698 + struct mtk_snand *snf = nand_to_mtk_snand(nand);
699 + struct nand_ecc_props *conf = &nand->ecc.ctx.conf;
700 + struct nand_ecc_props *reqs = &nand->ecc.requirements;
701 + struct nand_ecc_props *user = &nand->ecc.user_conf;
702 + struct mtd_info *mtd = nanddev_to_mtd(nand);
703 + int step_size = 0, strength = 0, desired_correction = 0, steps;
704 + bool ecc_user = false;
705 + int ret;
706 + u32 parity_bits, max_ecc_bytes;
707 + struct mtk_ecc_config *ecc_cfg;
708 +
709 + ret = mtk_snand_setup_pagefmt(snf, nand->memorg.pagesize,
710 + nand->memorg.oobsize);
711 + if (ret)
712 + return ret;
713 +
714 + ecc_cfg = kzalloc(sizeof(*ecc_cfg), GFP_KERNEL);
715 + if (!ecc_cfg)
716 + return -ENOMEM;
717 +
718 + nand->ecc.ctx.priv = ecc_cfg;
719 +
720 + if (user->step_size && user->strength) {
721 + step_size = user->step_size;
722 + strength = user->strength;
723 + ecc_user = true;
724 + } else if (reqs->step_size && reqs->strength) {
725 + step_size = reqs->step_size;
726 + strength = reqs->strength;
727 + }
728 +
729 + if (step_size && strength) {
730 + steps = mtd->writesize / step_size;
731 + desired_correction = steps * strength;
732 + strength = desired_correction / snf->nfi_cfg.nsectors;
733 + }
734 +
735 + ecc_cfg->mode = ECC_NFI_MODE;
736 + ecc_cfg->sectors = snf->nfi_cfg.nsectors;
737 + ecc_cfg->len = snf->caps->sector_size + snf->caps->fdm_ecc_size;
738 +
739 + // calculate the max possible strength under current page format
740 + parity_bits = mtk_ecc_get_parity_bits(snf->ecc);
741 + max_ecc_bytes = snf->nfi_cfg.spare_size - snf->caps->fdm_size;
742 + ecc_cfg->strength = max_ecc_bytes * 8 / parity_bits;
743 + mtk_ecc_adjust_strength(snf->ecc, &ecc_cfg->strength);
744 +
745 + // if there's a user requested strength, find the minimum strength that
746 + // meets the requirement. Otherwise use the maximum strength which is
747 + // expected by BootROM.
748 + if (ecc_user && strength) {
749 + u32 s_next = ecc_cfg->strength - 1;
750 +
751 + while (1) {
752 + mtk_ecc_adjust_strength(snf->ecc, &s_next);
753 + if (s_next >= ecc_cfg->strength)
754 + break;
755 + if (s_next < strength)
756 + break;
757 + s_next = ecc_cfg->strength - 1;
758 + }
759 + }
760 +
761 + mtd_set_ooblayout(mtd, &mtk_snand_ooblayout);
762 +
763 + conf->step_size = snf->caps->sector_size;
764 + conf->strength = ecc_cfg->strength;
765 +
766 + if (ecc_cfg->strength < strength)
767 + dev_warn(snf->dev, "unable to fulfill ECC of %u bits.\n",
768 + strength);
769 + dev_info(snf->dev, "ECC strength: %u bits per %u bytes\n",
770 + ecc_cfg->strength, snf->caps->sector_size);
771 +
772 + return 0;
773 +}
774 +
775 +static void mtk_snand_ecc_cleanup_ctx(struct nand_device *nand)
776 +{
777 + struct mtk_ecc_config *ecc_cfg = nand_to_ecc_ctx(nand);
778 +
779 + kfree(ecc_cfg);
780 +}
781 +
782 +static int mtk_snand_ecc_prepare_io_req(struct nand_device *nand,
783 + struct nand_page_io_req *req)
784 +{
785 + struct mtk_snand *snf = nand_to_mtk_snand(nand);
786 + struct mtk_ecc_config *ecc_cfg = nand_to_ecc_ctx(nand);
787 + int ret;
788 +
789 + ret = mtk_snand_setup_pagefmt(snf, nand->memorg.pagesize,
790 + nand->memorg.oobsize);
791 + if (ret)
792 + return ret;
793 + snf->autofmt = true;
794 + snf->ecc_cfg = ecc_cfg;
795 + return 0;
796 +}
797 +
798 +static int mtk_snand_ecc_finish_io_req(struct nand_device *nand,
799 + struct nand_page_io_req *req)
800 +{
801 + struct mtk_snand *snf = nand_to_mtk_snand(nand);
802 + struct mtd_info *mtd = nanddev_to_mtd(nand);
803 +
804 + snf->ecc_cfg = NULL;
805 + snf->autofmt = false;
806 + if ((req->mode == MTD_OPS_RAW) || (req->type != NAND_PAGE_READ))
807 + return 0;
808 +
809 + if (snf->ecc_stats.failed)
810 + mtd->ecc_stats.failed += snf->ecc_stats.failed;
811 + mtd->ecc_stats.corrected += snf->ecc_stats.corrected;
812 + return snf->ecc_stats.failed ? -EBADMSG : snf->ecc_stats.bitflips;
813 +}
814 +
815 +static struct nand_ecc_engine_ops mtk_snfi_ecc_engine_ops = {
816 + .init_ctx = mtk_snand_ecc_init_ctx,
817 + .cleanup_ctx = mtk_snand_ecc_cleanup_ctx,
818 + .prepare_io_req = mtk_snand_ecc_prepare_io_req,
819 + .finish_io_req = mtk_snand_ecc_finish_io_req,
820 +};
821 +
822 +static void mtk_snand_read_fdm(struct mtk_snand *snf, u8 *buf)
823 +{
824 + u32 vall, valm;
825 + u8 *oobptr = buf;
826 + int i, j;
827 +
828 + for (i = 0; i < snf->nfi_cfg.nsectors; i++) {
829 + vall = nfi_read32(snf, NFI_FDML(i));
830 + valm = nfi_read32(snf, NFI_FDMM(i));
831 +
832 + for (j = 0; j < snf->caps->fdm_size; j++)
833 + oobptr[j] = (j >= 4 ? valm : vall) >> ((j % 4) * 8);
834 +
835 + oobptr += snf->caps->fdm_size;
836 + }
837 +}
838 +
839 +static void mtk_snand_write_fdm(struct mtk_snand *snf, const u8 *buf)
840 +{
841 + u32 fdm_size = snf->caps->fdm_size;
842 + const u8 *oobptr = buf;
843 + u32 vall, valm;
844 + int i, j;
845 +
846 + for (i = 0; i < snf->nfi_cfg.nsectors; i++) {
847 + vall = 0;
848 + valm = 0;
849 +
850 + for (j = 0; j < 8; j++) {
851 + if (j < 4)
852 + vall |= (j < fdm_size ? oobptr[j] : 0xff)
853 + << (j * 8);
854 + else
855 + valm |= (j < fdm_size ? oobptr[j] : 0xff)
856 + << ((j - 4) * 8);
857 + }
858 +
859 + nfi_write32(snf, NFI_FDML(i), vall);
860 + nfi_write32(snf, NFI_FDMM(i), valm);
861 +
862 + oobptr += fdm_size;
863 + }
864 +}
865 +
866 +static void mtk_snand_bm_swap(struct mtk_snand *snf, u8 *buf)
867 +{
868 + u32 buf_bbm_pos, fdm_bbm_pos;
869 +
870 + if (!snf->caps->bbm_swap || snf->nfi_cfg.nsectors == 1)
871 + return;
872 +
873 + // swap [pagesize] byte on nand with the first fdm byte
874 + // in the last sector.
875 + buf_bbm_pos = snf->nfi_cfg.page_size -
876 + (snf->nfi_cfg.nsectors - 1) * snf->nfi_cfg.spare_size;
877 + fdm_bbm_pos = snf->nfi_cfg.page_size +
878 + (snf->nfi_cfg.nsectors - 1) * snf->caps->fdm_size;
879 +
880 + swap(snf->buf[fdm_bbm_pos], buf[buf_bbm_pos]);
881 +}
882 +
883 +static void mtk_snand_fdm_bm_swap(struct mtk_snand *snf)
884 +{
885 + u32 fdm_bbm_pos1, fdm_bbm_pos2;
886 +
887 + if (!snf->caps->bbm_swap || snf->nfi_cfg.nsectors == 1)
888 + return;
889 +
890 + // swap the first fdm byte in the first and the last sector.
891 + fdm_bbm_pos1 = snf->nfi_cfg.page_size;
892 + fdm_bbm_pos2 = snf->nfi_cfg.page_size +
893 + (snf->nfi_cfg.nsectors - 1) * snf->caps->fdm_size;
894 + swap(snf->buf[fdm_bbm_pos1], snf->buf[fdm_bbm_pos2]);
895 +}
896 +
897 +static int mtk_snand_read_page_cache(struct mtk_snand *snf,
898 + const struct spi_mem_op *op)
899 +{
900 + u8 *buf = snf->buf;
901 + u8 *buf_fdm = buf + snf->nfi_cfg.page_size;
902 + // the address part to be sent by the controller
903 + u32 op_addr = op->addr.val;
904 + // where to start copying data from bounce buffer
905 + u32 rd_offset = 0;
906 + u32 dummy_clk = (op->dummy.nbytes * BITS_PER_BYTE / op->dummy.buswidth);
907 + u32 op_mode = 0;
908 + u32 dma_len = snf->buf_len;
909 + int ret = 0;
910 + u32 rd_mode, rd_bytes, val;
911 + dma_addr_t buf_dma;
912 +
913 + if (snf->autofmt) {
914 + u32 last_bit;
915 + u32 mask;
916 +
917 + dma_len = snf->nfi_cfg.page_size;
918 + op_mode = CNFG_AUTO_FMT_EN;
919 + if (op->data.ecc)
920 + op_mode |= CNFG_HW_ECC_EN;
921 + // extract the plane bit:
922 + // Find the highest bit set in (pagesize+oobsize).
923 + // Bits higher than that in op->addr are kept and sent over SPI
924 + // Lower bits are used as an offset for copying data from DMA
925 + // bounce buffer.
926 + last_bit = fls(snf->nfi_cfg.page_size + snf->nfi_cfg.oob_size);
927 + mask = (1 << last_bit) - 1;
928 + rd_offset = op_addr & mask;
929 + op_addr &= ~mask;
930 +
931 + // check if we can dma to the caller memory
932 + if (rd_offset == 0 && op->data.nbytes >= snf->nfi_cfg.page_size)
933 + buf = op->data.buf.in;
934 + }
935 + mtk_snand_mac_reset(snf);
936 + mtk_nfi_reset(snf);
937 +
938 + // command and dummy cycles
939 + nfi_write32(snf, SNF_RD_CTL2,
940 + (dummy_clk << DATA_READ_DUMMY_S) |
941 + (op->cmd.opcode << DATA_READ_CMD_S));
942 +
943 + // read address
944 + nfi_write32(snf, SNF_RD_CTL3, op_addr);
945 +
946 + // Set read op_mode
947 + if (op->data.buswidth == 4)
948 + rd_mode = op->addr.buswidth == 4 ? DATA_READ_MODE_QUAD :
949 + DATA_READ_MODE_X4;
950 + else if (op->data.buswidth == 2)
951 + rd_mode = op->addr.buswidth == 2 ? DATA_READ_MODE_DUAL :
952 + DATA_READ_MODE_X2;
953 + else
954 + rd_mode = DATA_READ_MODE_X1;
955 + rd_mode <<= DATA_READ_MODE_S;
956 + nfi_rmw32(snf, SNF_MISC_CTL, DATA_READ_MODE,
957 + rd_mode | DATARD_CUSTOM_EN);
958 +
959 + // Set bytes to read
960 + rd_bytes = (snf->nfi_cfg.spare_size + snf->caps->sector_size) *
961 + snf->nfi_cfg.nsectors;
962 + nfi_write32(snf, SNF_MISC_CTL2,
963 + (rd_bytes << PROGRAM_LOAD_BYTE_NUM_S) | rd_bytes);
964 +
965 + // NFI read prepare
966 + nfi_write16(snf, NFI_CNFG,
967 + (CNFG_OP_MODE_CUST << CNFG_OP_MODE_S) | CNFG_DMA_BURST_EN |
968 + CNFG_READ_MODE | CNFG_DMA_MODE | op_mode);
969 +
970 + nfi_write32(snf, NFI_CON, (snf->nfi_cfg.nsectors << CON_SEC_NUM_S));
971 +
972 + buf_dma = dma_map_single(snf->dev, buf, dma_len, DMA_FROM_DEVICE);
973 + if (dma_mapping_error(snf->dev, buf_dma)) {
974 + dev_err(snf->dev, "DMA mapping failed.\n");
975 + goto cleanup;
976 + }
977 + nfi_write32(snf, NFI_STRADDR, buf_dma);
978 + if (op->data.ecc) {
979 + snf->ecc_cfg->op = ECC_DECODE;
980 + ret = mtk_ecc_enable(snf->ecc, snf->ecc_cfg);
981 + if (ret)
982 + goto cleanup_dma;
983 + }
984 + // Prepare for custom read interrupt
985 + nfi_write32(snf, NFI_INTR_EN, NFI_IRQ_INTR_EN | NFI_IRQ_CUS_READ);
986 + reinit_completion(&snf->op_done);
987 +
988 + // Trigger NFI into custom mode
989 + nfi_write16(snf, NFI_CMD, NFI_CMD_DUMMY_READ);
990 +
991 + // Start DMA read
992 + nfi_rmw32(snf, NFI_CON, 0, CON_BRD);
993 + nfi_write16(snf, NFI_STRDATA, STR_DATA);
994 +
995 + if (!wait_for_completion_timeout(
996 + &snf->op_done, usecs_to_jiffies(SNFI_POLL_INTERVAL))) {
997 + dev_err(snf->dev, "DMA timed out for reading from cache.\n");
998 + ret = -ETIMEDOUT;
999 + goto cleanup;
1000 + }
1001 +
1002 + // Wait for BUS_SEC_CNTR returning expected value
1003 + ret = readl_poll_timeout(snf->nfi_base + NFI_BYTELEN, val,
1004 + BUS_SEC_CNTR(val) >= snf->nfi_cfg.nsectors, 0,
1005 + SNFI_POLL_INTERVAL);
1006 + if (ret) {
1007 + dev_err(snf->dev, "Timed out waiting for BUS_SEC_CNTR\n");
1008 + goto cleanup2;
1009 + }
1010 +
1011 + // Wait for bus becoming idle
1012 + ret = readl_poll_timeout(snf->nfi_base + NFI_MASTERSTA, val,
1013 + !(val & snf->caps->mastersta_mask), 0,
1014 + SNFI_POLL_INTERVAL);
1015 + if (ret) {
1016 + dev_err(snf->dev, "Timed out waiting for bus becoming idle\n");
1017 + goto cleanup2;
1018 + }
1019 +
1020 + if (op->data.ecc) {
1021 + ret = mtk_ecc_wait_done(snf->ecc, ECC_DECODE);
1022 + if (ret) {
1023 + dev_err(snf->dev, "wait ecc done timeout\n");
1024 + goto cleanup2;
1025 + }
1026 + // save status before disabling ecc
1027 + mtk_ecc_get_stats(snf->ecc, &snf->ecc_stats,
1028 + snf->nfi_cfg.nsectors);
1029 + }
1030 +
1031 + dma_unmap_single(snf->dev, buf_dma, dma_len, DMA_FROM_DEVICE);
1032 +
1033 + if (snf->autofmt) {
1034 + mtk_snand_read_fdm(snf, buf_fdm);
1035 + if (snf->caps->bbm_swap) {
1036 + mtk_snand_bm_swap(snf, buf);
1037 + mtk_snand_fdm_bm_swap(snf);
1038 + }
1039 + }
1040 +
1041 + // copy data back
1042 + if (nfi_read32(snf, NFI_STA) & READ_EMPTY) {
1043 + memset(op->data.buf.in, 0xff, op->data.nbytes);
1044 + snf->ecc_stats.bitflips = 0;
1045 + snf->ecc_stats.failed = 0;
1046 + snf->ecc_stats.corrected = 0;
1047 + } else {
1048 + if (buf == op->data.buf.in) {
1049 + u32 cap_len = snf->buf_len - snf->nfi_cfg.page_size;
1050 + u32 req_left = op->data.nbytes - snf->nfi_cfg.page_size;
1051 +
1052 + if (req_left)
1053 + memcpy(op->data.buf.in + snf->nfi_cfg.page_size,
1054 + buf_fdm,
1055 + cap_len < req_left ? cap_len : req_left);
1056 + } else if (rd_offset < snf->buf_len) {
1057 + u32 cap_len = snf->buf_len - rd_offset;
1058 +
1059 + if (op->data.nbytes < cap_len)
1060 + cap_len = op->data.nbytes;
1061 + memcpy(op->data.buf.in, snf->buf + rd_offset, cap_len);
1062 + }
1063 + }
1064 +cleanup2:
1065 + if (op->data.ecc)
1066 + mtk_ecc_disable(snf->ecc);
1067 +cleanup_dma:
1068 + // unmap dma only if any error happens. (otherwise it's done before
1069 + // data copying)
1070 + if (ret)
1071 + dma_unmap_single(snf->dev, buf_dma, dma_len, DMA_FROM_DEVICE);
1072 +cleanup:
1073 + // Stop read
1074 + nfi_write32(snf, NFI_CON, 0);
1075 + nfi_write16(snf, NFI_CNFG, 0);
1076 +
1077 + // Clear SNF done flag
1078 + nfi_rmw32(snf, SNF_STA_CTL1, 0, CUS_READ_DONE);
1079 + nfi_write32(snf, SNF_STA_CTL1, 0);
1080 +
1081 + // Disable interrupt
1082 + nfi_read32(snf, NFI_INTR_STA);
1083 + nfi_write32(snf, NFI_INTR_EN, 0);
1084 +
1085 + nfi_rmw32(snf, SNF_MISC_CTL, DATARD_CUSTOM_EN, 0);
1086 + return ret;
1087 +}
1088 +
1089 +static int mtk_snand_write_page_cache(struct mtk_snand *snf,
1090 + const struct spi_mem_op *op)
1091 +{
1092 + // the address part to be sent by the controller
1093 + u32 op_addr = op->addr.val;
1094 + // where to start copying data from bounce buffer
1095 + u32 wr_offset = 0;
1096 + u32 op_mode = 0;
1097 + int ret = 0;
1098 + u32 wr_mode = 0;
1099 + u32 dma_len = snf->buf_len;
1100 + u32 wr_bytes, val;
1101 + size_t cap_len;
1102 + dma_addr_t buf_dma;
1103 +
1104 + if (snf->autofmt) {
1105 + u32 last_bit;
1106 + u32 mask;
1107 +
1108 + dma_len = snf->nfi_cfg.page_size;
1109 + op_mode = CNFG_AUTO_FMT_EN;
1110 + if (op->data.ecc)
1111 + op_mode |= CNFG_HW_ECC_EN;
1112 +
1113 + last_bit = fls(snf->nfi_cfg.page_size + snf->nfi_cfg.oob_size);
1114 + mask = (1 << last_bit) - 1;
1115 + wr_offset = op_addr & mask;
1116 + op_addr &= ~mask;
1117 + }
1118 + mtk_snand_mac_reset(snf);
1119 + mtk_nfi_reset(snf);
1120 +
1121 + if (wr_offset)
1122 + memset(snf->buf, 0xff, wr_offset);
1123 +
1124 + cap_len = snf->buf_len - wr_offset;
1125 + if (op->data.nbytes < cap_len)
1126 + cap_len = op->data.nbytes;
1127 + memcpy(snf->buf + wr_offset, op->data.buf.out, cap_len);
1128 + if (snf->autofmt) {
1129 + if (snf->caps->bbm_swap) {
1130 + mtk_snand_fdm_bm_swap(snf);
1131 + mtk_snand_bm_swap(snf, snf->buf);
1132 + }
1133 + mtk_snand_write_fdm(snf, snf->buf + snf->nfi_cfg.page_size);
1134 + }
1135 +
1136 + // Command
1137 + nfi_write32(snf, SNF_PG_CTL1, (op->cmd.opcode << PG_LOAD_CMD_S));
1138 +
1139 + // write address
1140 + nfi_write32(snf, SNF_PG_CTL2, op_addr);
1141 +
1142 + // Set read op_mode
1143 + if (op->data.buswidth == 4)
1144 + wr_mode = PG_LOAD_X4_EN;
1145 +
1146 + nfi_rmw32(snf, SNF_MISC_CTL, PG_LOAD_X4_EN,
1147 + wr_mode | PG_LOAD_CUSTOM_EN);
1148 +
1149 + // Set bytes to write
1150 + wr_bytes = (snf->nfi_cfg.spare_size + snf->caps->sector_size) *
1151 + snf->nfi_cfg.nsectors;
1152 + nfi_write32(snf, SNF_MISC_CTL2,
1153 + (wr_bytes << PROGRAM_LOAD_BYTE_NUM_S) | wr_bytes);
1154 +
1155 + // NFI write prepare
1156 + nfi_write16(snf, NFI_CNFG,
1157 + (CNFG_OP_MODE_PROGRAM << CNFG_OP_MODE_S) |
1158 + CNFG_DMA_BURST_EN | CNFG_DMA_MODE | op_mode);
1159 +
1160 + nfi_write32(snf, NFI_CON, (snf->nfi_cfg.nsectors << CON_SEC_NUM_S));
1161 + buf_dma = dma_map_single(snf->dev, snf->buf, dma_len, DMA_TO_DEVICE);
1162 + if (dma_mapping_error(snf->dev, buf_dma)) {
1163 + dev_err(snf->dev, "DMA mapping failed.\n");
1164 + goto cleanup;
1165 + }
1166 + nfi_write32(snf, NFI_STRADDR, buf_dma);
1167 + if (op->data.ecc) {
1168 + snf->ecc_cfg->op = ECC_ENCODE;
1169 + ret = mtk_ecc_enable(snf->ecc, snf->ecc_cfg);
1170 + if (ret)
1171 + goto cleanup_dma;
1172 + }
1173 + // Prepare for custom write interrupt
1174 + nfi_write32(snf, NFI_INTR_EN, NFI_IRQ_INTR_EN | NFI_IRQ_CUS_PG);
1175 + reinit_completion(&snf->op_done);
1176 + ;
1177 +
1178 + // Trigger NFI into custom mode
1179 + nfi_write16(snf, NFI_CMD, NFI_CMD_DUMMY_WRITE);
1180 +
1181 + // Start DMA write
1182 + nfi_rmw32(snf, NFI_CON, 0, CON_BWR);
1183 + nfi_write16(snf, NFI_STRDATA, STR_DATA);
1184 +
1185 + if (!wait_for_completion_timeout(
1186 + &snf->op_done, usecs_to_jiffies(SNFI_POLL_INTERVAL))) {
1187 + dev_err(snf->dev, "DMA timed out for program load.\n");
1188 + ret = -ETIMEDOUT;
1189 + goto cleanup_ecc;
1190 + }
1191 +
1192 + // Wait for NFI_SEC_CNTR returning expected value
1193 + ret = readl_poll_timeout(snf->nfi_base + NFI_ADDRCNTR, val,
1194 + NFI_SEC_CNTR(val) >= snf->nfi_cfg.nsectors, 0,
1195 + SNFI_POLL_INTERVAL);
1196 + if (ret)
1197 + dev_err(snf->dev, "Timed out waiting for NFI_SEC_CNTR\n");
1198 +
1199 +cleanup_ecc:
1200 + if (op->data.ecc)
1201 + mtk_ecc_disable(snf->ecc);
1202 +cleanup_dma:
1203 + dma_unmap_single(snf->dev, buf_dma, dma_len, DMA_TO_DEVICE);
1204 +cleanup:
1205 + // Stop write
1206 + nfi_write32(snf, NFI_CON, 0);
1207 + nfi_write16(snf, NFI_CNFG, 0);
1208 +
1209 + // Clear SNF done flag
1210 + nfi_rmw32(snf, SNF_STA_CTL1, 0, CUS_PG_DONE);
1211 + nfi_write32(snf, SNF_STA_CTL1, 0);
1212 +
1213 + // Disable interrupt
1214 + nfi_read32(snf, NFI_INTR_STA);
1215 + nfi_write32(snf, NFI_INTR_EN, 0);
1216 +
1217 + nfi_rmw32(snf, SNF_MISC_CTL, PG_LOAD_CUSTOM_EN, 0);
1218 +
1219 + return ret;
1220 +}
1221 +
1222 +/**
1223 + * mtk_snand_is_page_ops() - check if the op is a controller supported page op.
1224 + * @op spi-mem op to check
1225 + *
1226 + * Check whether op can be executed with read_from_cache or program_load
1227 + * mode in the controller.
1228 + * This controller can execute typical Read From Cache and Program Load
1229 + * instructions found on SPI-NAND with 2-byte address.
1230 + * DTR and cmd buswidth & nbytes should be checked before calling this.
1231 + *
1232 + * Return: true if the op matches the instruction template
1233 + */
1234 +static bool mtk_snand_is_page_ops(const struct spi_mem_op *op)
1235 +{
1236 + if (op->addr.nbytes != 2)
1237 + return false;
1238 +
1239 + if (op->addr.buswidth != 1 && op->addr.buswidth != 2 &&
1240 + op->addr.buswidth != 4)
1241 + return false;
1242 +
1243 + // match read from page instructions
1244 + if (op->data.dir == SPI_MEM_DATA_IN) {
1245 + // check dummy cycle first
1246 + if (op->dummy.nbytes * BITS_PER_BYTE / op->dummy.buswidth >
1247 + DATA_READ_MAX_DUMMY)
1248 + return false;
1249 + // quad io / quad out
1250 + if ((op->addr.buswidth == 4 || op->addr.buswidth == 1) &&
1251 + op->data.buswidth == 4)
1252 + return true;
1253 +
1254 + // dual io / dual out
1255 + if ((op->addr.buswidth == 2 || op->addr.buswidth == 1) &&
1256 + op->data.buswidth == 2)
1257 + return true;
1258 +
1259 + // standard spi
1260 + if (op->addr.buswidth == 1 && op->data.buswidth == 1)
1261 + return true;
1262 + } else if (op->data.dir == SPI_MEM_DATA_OUT) {
1263 + // check dummy cycle first
1264 + if (op->dummy.nbytes)
1265 + return false;
1266 + // program load quad out
1267 + if (op->addr.buswidth == 1 && op->data.buswidth == 4)
1268 + return true;
1269 + // standard spi
1270 + if (op->addr.buswidth == 1 && op->data.buswidth == 1)
1271 + return true;
1272 + }
1273 + return false;
1274 +}
1275 +
1276 +static bool mtk_snand_supports_op(struct spi_mem *mem,
1277 + const struct spi_mem_op *op)
1278 +{
1279 + if (!spi_mem_default_supports_op(mem, op))
1280 + return false;
1281 + if (op->cmd.nbytes != 1 || op->cmd.buswidth != 1)
1282 + return false;
1283 + if (mtk_snand_is_page_ops(op))
1284 + return true;
1285 + return ((op->addr.nbytes == 0 || op->addr.buswidth == 1) &&
1286 + (op->dummy.nbytes == 0 || op->dummy.buswidth == 1) &&
1287 + (op->data.nbytes == 0 || op->data.buswidth == 1));
1288 +}
1289 +
1290 +static int mtk_snand_adjust_op_size(struct spi_mem *mem, struct spi_mem_op *op)
1291 +{
1292 + struct mtk_snand *ms = spi_controller_get_devdata(mem->spi->master);
1293 + // page ops transfer size must be exactly ((sector_size + spare_size) *
1294 + // nsectors). Limit the op size if the caller requests more than that.
1295 + // exec_op will read more than needed and discard the leftover if the
1296 + // caller requests less data.
1297 + if (mtk_snand_is_page_ops(op)) {
1298 + size_t l;
1299 + // skip adjust_op_size for page ops
1300 + if (ms->autofmt)
1301 + return 0;
1302 + l = ms->caps->sector_size + ms->nfi_cfg.spare_size;
1303 + l *= ms->nfi_cfg.nsectors;
1304 + if (op->data.nbytes > l)
1305 + op->data.nbytes = l;
1306 + } else {
1307 + size_t hl = op->cmd.nbytes + op->addr.nbytes + op->dummy.nbytes;
1308 +
1309 + if (hl >= SNF_GPRAM_SIZE)
1310 + return -EOPNOTSUPP;
1311 + if (op->data.nbytes > SNF_GPRAM_SIZE - hl)
1312 + op->data.nbytes = SNF_GPRAM_SIZE - hl;
1313 + }
1314 + return 0;
1315 +}
1316 +
1317 +static int mtk_snand_exec_op(struct spi_mem *mem, const struct spi_mem_op *op)
1318 +{
1319 + struct mtk_snand *ms = spi_controller_get_devdata(mem->spi->master);
1320 +
1321 + dev_dbg(ms->dev, "OP %02x ADDR %08llX@%d:%u DATA %d:%u", op->cmd.opcode,
1322 + op->addr.val, op->addr.buswidth, op->addr.nbytes,
1323 + op->data.buswidth, op->data.nbytes);
1324 + if (mtk_snand_is_page_ops(op)) {
1325 + if (op->data.dir == SPI_MEM_DATA_IN)
1326 + return mtk_snand_read_page_cache(ms, op);
1327 + else
1328 + return mtk_snand_write_page_cache(ms, op);
1329 + } else {
1330 + return mtk_snand_mac_io(ms, op);
1331 + }
1332 +}
1333 +
1334 +static const struct spi_controller_mem_ops mtk_snand_mem_ops = {
1335 + .adjust_op_size = mtk_snand_adjust_op_size,
1336 + .supports_op = mtk_snand_supports_op,
1337 + .exec_op = mtk_snand_exec_op,
1338 +};
1339 +
1340 +static const struct spi_controller_mem_caps mtk_snand_mem_caps = {
1341 + .ecc = true,
1342 +};
1343 +
1344 +static irqreturn_t mtk_snand_irq(int irq, void *id)
1345 +{
1346 + struct mtk_snand *snf = id;
1347 + u32 sta, ien;
1348 +
1349 + sta = nfi_read32(snf, NFI_INTR_STA);
1350 + ien = nfi_read32(snf, NFI_INTR_EN);
1351 +
1352 + if (!(sta & ien))
1353 + return IRQ_NONE;
1354 +
1355 + nfi_write32(snf, NFI_INTR_EN, 0);
1356 + complete(&snf->op_done);
1357 + return IRQ_HANDLED;
1358 +}
1359 +
1360 +static const struct of_device_id mtk_snand_ids[] = {
1361 + { .compatible = "mediatek,mt7622-snand", .data = &mt7622_snand_caps },
1362 + { .compatible = "mediatek,mt7629-snand", .data = &mt7629_snand_caps },
1363 + {},
1364 +};
1365 +
1366 +MODULE_DEVICE_TABLE(of, mtk_snand_ids);
1367 +
1368 +static int mtk_snand_enable_clk(struct mtk_snand *ms)
1369 +{
1370 + int ret;
1371 +
1372 + ret = clk_prepare_enable(ms->nfi_clk);
1373 + if (ret) {
1374 + dev_err(ms->dev, "unable to enable nfi clk\n");
1375 + return ret;
1376 + }
1377 + ret = clk_prepare_enable(ms->pad_clk);
1378 + if (ret) {
1379 + dev_err(ms->dev, "unable to enable pad clk\n");
1380 + goto err1;
1381 + }
1382 + return 0;
1383 +err1:
1384 + clk_disable_unprepare(ms->nfi_clk);
1385 + return ret;
1386 +}
1387 +
1388 +static void mtk_snand_disable_clk(struct mtk_snand *ms)
1389 +{
1390 + clk_disable_unprepare(ms->pad_clk);
1391 + clk_disable_unprepare(ms->nfi_clk);
1392 +}
1393 +
1394 +static int mtk_snand_probe(struct platform_device *pdev)
1395 +{
1396 + struct device_node *np = pdev->dev.of_node;
1397 + const struct of_device_id *dev_id;
1398 + struct spi_controller *ctlr;
1399 + struct mtk_snand *ms;
1400 + int ret;
1401 +
1402 + dev_id = of_match_node(mtk_snand_ids, np);
1403 + if (!dev_id)
1404 + return -EINVAL;
1405 +
1406 + ctlr = devm_spi_alloc_master(&pdev->dev, sizeof(*ms));
1407 + if (!ctlr)
1408 + return -ENOMEM;
1409 + platform_set_drvdata(pdev, ctlr);
1410 +
1411 + ms = spi_controller_get_devdata(ctlr);
1412 +
1413 + ms->ctlr = ctlr;
1414 + ms->caps = dev_id->data;
1415 +
1416 + ms->ecc = of_mtk_ecc_get(np);
1417 + if (IS_ERR(ms->ecc))
1418 + return PTR_ERR(ms->ecc);
1419 + else if (!ms->ecc)
1420 + return -ENODEV;
1421 +
1422 + ms->nfi_base = devm_platform_ioremap_resource(pdev, 0);
1423 + if (IS_ERR(ms->nfi_base)) {
1424 + ret = PTR_ERR(ms->nfi_base);
1425 + goto release_ecc;
1426 + }
1427 +
1428 + ms->dev = &pdev->dev;
1429 +
1430 + ms->nfi_clk = devm_clk_get(&pdev->dev, "nfi_clk");
1431 + if (IS_ERR(ms->nfi_clk)) {
1432 + ret = PTR_ERR(ms->nfi_clk);
1433 + dev_err(&pdev->dev, "unable to get nfi_clk, err = %d\n", ret);
1434 + goto release_ecc;
1435 + }
1436 +
1437 + ms->pad_clk = devm_clk_get(&pdev->dev, "pad_clk");
1438 + if (IS_ERR(ms->pad_clk)) {
1439 + ret = PTR_ERR(ms->pad_clk);
1440 + dev_err(&pdev->dev, "unable to get pad_clk, err = %d\n", ret);
1441 + goto release_ecc;
1442 + }
1443 +
1444 + ret = mtk_snand_enable_clk(ms);
1445 + if (ret)
1446 + goto release_ecc;
1447 +
1448 + init_completion(&ms->op_done);
1449 +
1450 + ms->irq = platform_get_irq(pdev, 0);
1451 + if (ms->irq < 0) {
1452 + ret = ms->irq;
1453 + goto disable_clk;
1454 + }
1455 + ret = devm_request_irq(ms->dev, ms->irq, mtk_snand_irq, 0x0,
1456 + "mtk-snand", ms);
1457 + if (ret) {
1458 + dev_err(ms->dev, "failed to request snfi irq\n");
1459 + goto disable_clk;
1460 + }
1461 +
1462 + ret = dma_set_mask(ms->dev, DMA_BIT_MASK(32));
1463 + if (ret) {
1464 + dev_err(ms->dev, "failed to set dma mask\n");
1465 + goto disable_clk;
1466 + }
1467 +
1468 + // switch to SNFI mode
1469 + nfi_write32(ms, SNF_CFG, SPI_MODE);
1470 +
1471 + // setup an initial page format for ops matching page_cache_op template
1472 + // before ECC is called.
1473 + ret = mtk_snand_setup_pagefmt(ms, ms->caps->sector_size,
1474 + ms->caps->spare_sizes[0]);
1475 + if (ret) {
1476 + dev_err(ms->dev, "failed to set initial page format\n");
1477 + goto disable_clk;
1478 + }
1479 +
1480 + // setup ECC engine
1481 + ms->ecc_eng.dev = &pdev->dev;
1482 + ms->ecc_eng.integration = NAND_ECC_ENGINE_INTEGRATION_PIPELINED;
1483 + ms->ecc_eng.ops = &mtk_snfi_ecc_engine_ops;
1484 + ms->ecc_eng.priv = ms;
1485 +
1486 + ret = nand_ecc_register_on_host_hw_engine(&ms->ecc_eng);
1487 + if (ret) {
1488 + dev_err(&pdev->dev, "failed to register ecc engine.\n");
1489 + goto disable_clk;
1490 + }
1491 +
1492 + ctlr->num_chipselect = 1;
1493 + ctlr->mem_ops = &mtk_snand_mem_ops;
1494 + ctlr->mem_caps = &mtk_snand_mem_caps;
1495 + ctlr->bits_per_word_mask = SPI_BPW_MASK(8);
1496 + ctlr->mode_bits = SPI_RX_DUAL | SPI_RX_QUAD | SPI_TX_DUAL | SPI_TX_QUAD;
1497 + ctlr->dev.of_node = pdev->dev.of_node;
1498 + ret = spi_register_controller(ctlr);
1499 + if (ret) {
1500 + dev_err(&pdev->dev, "spi_register_controller failed.\n");
1501 + goto disable_clk;
1502 + }
1503 +
1504 + return 0;
1505 +disable_clk:
1506 + mtk_snand_disable_clk(ms);
1507 +release_ecc:
1508 + mtk_ecc_release(ms->ecc);
1509 + return ret;
1510 +}
1511 +
1512 +static int mtk_snand_remove(struct platform_device *pdev)
1513 +{
1514 + struct spi_controller *ctlr = platform_get_drvdata(pdev);
1515 + struct mtk_snand *ms = spi_controller_get_devdata(ctlr);
1516 +
1517 + spi_unregister_controller(ctlr);
1518 + mtk_snand_disable_clk(ms);
1519 + mtk_ecc_release(ms->ecc);
1520 + kfree(ms->buf);
1521 + return 0;
1522 +}
1523 +
1524 +static struct platform_driver mtk_snand_driver = {
1525 + .probe = mtk_snand_probe,
1526 + .remove = mtk_snand_remove,
1527 + .driver = {
1528 + .name = "mtk-snand",
1529 + .of_match_table = mtk_snand_ids,
1530 + },
1531 +};
1532 +
1533 +module_platform_driver(mtk_snand_driver);
1534 +
1535 +MODULE_LICENSE("GPL");
1536 +MODULE_AUTHOR("Chuanhong Guo <gch981213@gmail.com>");
1537 +MODULE_DESCRIPTION("MeidaTek SPI-NAND Flash Controller Driver");
Staging tree of Jo-Philipp Wich