[J-u-boot.git] / drivers / mmc / dw_mmc.c

// SPDX-License-Identifier: GPL-2.0+
/*
 * (C) Copyright 2012 SAMSUNG Electronics
 * Jaehoon Chung <[email protected]>
 * Rajeshawari Shinde <[email protected]>
 */

#include <bouncebuf.h>
#include <common.h>
#include <cpu_func.h>
#include <errno.h>
#include <log.h>
#include <malloc.h>
#include <memalign.h>
#include <mmc.h>
#include <dwmmc.h>
#include <wait_bit.h>
#include <asm/cache.h>
#include <linux/delay.h>
#include <power/regulator.h>

#define PAGE_SIZE 4096

static int dwmci_wait_reset(struct dwmci_host *host, u32 value)
{
	unsigned long timeout = 1000;
	u32 ctrl;

	dwmci_writel(host, DWMCI_CTRL, value);

	while (timeout--) {
		ctrl = dwmci_readl(host, DWMCI_CTRL);
		if (!(ctrl & DWMCI_RESET_ALL))
			return 1;
	}
	return 0;
}

static void dwmci_set_idma_desc(struct dwmci_idmac *idmac,
		u32 desc0, u32 desc1, u32 desc2)
{
	struct dwmci_idmac *desc = idmac;

	desc->flags = desc0;
	desc->cnt = desc1;
	desc->addr = desc2;
	desc->next_addr = (ulong)desc + sizeof(struct dwmci_idmac);
}

static void dwmci_prepare_data(struct dwmci_host *host,
			       struct mmc_data *data,
			       struct dwmci_idmac *cur_idmac,
			       void *bounce_buffer)
{
	unsigned long ctrl;
	unsigned int i = 0, flags, cnt, blk_cnt;
	ulong data_start, data_end;


	blk_cnt = data->blocks;

	dwmci_wait_reset(host, DWMCI_CTRL_FIFO_RESET);

	/* Clear IDMAC interrupt */
	dwmci_writel(host, DWMCI_IDSTS, 0xFFFFFFFF);

	data_start = (ulong)cur_idmac;
	dwmci_writel(host, DWMCI_DBADDR, (ulong)cur_idmac);

	do {
		flags = DWMCI_IDMAC_OWN | DWMCI_IDMAC_CH ;
		flags |= (i == 0) ? DWMCI_IDMAC_FS : 0;
		if (blk_cnt <= 8) {
			flags |= DWMCI_IDMAC_LD;
			cnt = data->blocksize * blk_cnt;
		} else
			cnt = data->blocksize * 8;

		dwmci_set_idma_desc(cur_idmac, flags, cnt,
				    (ulong)bounce_buffer + (i * PAGE_SIZE));

		cur_idmac++;
		if (blk_cnt <= 8)
			break;
		blk_cnt -= 8;
		i++;
	} while(1);

	data_end = (ulong)cur_idmac;
	flush_dcache_range(data_start, roundup(data_end, ARCH_DMA_MINALIGN));

	ctrl = dwmci_readl(host, DWMCI_CTRL);
	ctrl |= DWMCI_IDMAC_EN | DWMCI_DMA_EN;
	dwmci_writel(host, DWMCI_CTRL, ctrl);

	ctrl = dwmci_readl(host, DWMCI_BMOD);
	ctrl |= DWMCI_BMOD_IDMAC_FB | DWMCI_BMOD_IDMAC_EN;
	dwmci_writel(host, DWMCI_BMOD, ctrl);

	dwmci_writel(host, DWMCI_BLKSIZ, data->blocksize);
	dwmci_writel(host, DWMCI_BYTCNT, data->blocksize * data->blocks);
}

static int dwmci_fifo_ready(struct dwmci_host *host, u32 bit, u32 *len)
{
	u32 timeout = 20000;

	*len = dwmci_readl(host, DWMCI_STATUS);
	while (--timeout && (*len & bit)) {
		udelay(200);
		*len = dwmci_readl(host, DWMCI_STATUS);
	}

	if (!timeout) {
		debug("%s: FIFO underflow timeout\n", __func__);
		return -ETIMEDOUT;
	}

	return 0;
}

static unsigned int dwmci_get_timeout(struct mmc *mmc, const unsigned int size)
{
	unsigned int timeout;

	timeout = size * 8;	/* counting in bits */
	timeout *= 10;		/* wait 10 times as long */
	timeout /= mmc->clock;
	timeout /= mmc->bus_width;
	timeout /= mmc->ddr_mode ? 2 : 1;
	timeout *= 1000;	/* counting in msec */
	timeout = (timeout < 1000) ? 1000 : timeout;

	return timeout;
}

static int dwmci_data_transfer(struct dwmci_host *host, struct mmc_data *data)
{
	struct mmc *mmc = host->mmc;
	int ret = 0;
	u32 timeout, mask, size, i, len = 0;
	u32 *buf = NULL;
	ulong start = get_timer(0);
	u32 fifo_depth = (((host->fifoth_val & RX_WMARK_MASK) >>
			    RX_WMARK_SHIFT) + 1) * 2;

	size = data->blocksize * data->blocks;
	if (data->flags == MMC_DATA_READ)
		buf = (unsigned int *)data->dest;
	else
		buf = (unsigned int *)data->src;

	timeout = dwmci_get_timeout(mmc, size);

	size /= 4;

	for (;;) {
		mask = dwmci_readl(host, DWMCI_RINTSTS);
		/* Error during data transfer. */
		if (mask & (DWMCI_DATA_ERR | DWMCI_DATA_TOUT)) {
			debug("%s: DATA ERROR!\n", __func__);
			ret = -EINVAL;
			break;
		}

		if (host->fifo_mode && size) {
			len = 0;
			if (data->flags == MMC_DATA_READ &&
			    (mask & (DWMCI_INTMSK_RXDR | DWMCI_INTMSK_DTO))) {
				dwmci_writel(host, DWMCI_RINTSTS,
					     mask & (DWMCI_INTMSK_RXDR |
						     DWMCI_INTMSK_DTO));
				while (size) {
					ret = dwmci_fifo_ready(host,
							DWMCI_FIFO_EMPTY,
							&len);
					if (ret < 0)
						break;

					len = (len >> DWMCI_FIFO_SHIFT) &
						    DWMCI_FIFO_MASK;
					len = min(size, len);
					for (i = 0; i < len; i++)
						*buf++ =
						dwmci_readl(host, DWMCI_DATA);
					size = size > len ? (size - len) : 0;
				}
			} else if (data->flags == MMC_DATA_WRITE &&
				   (mask & DWMCI_INTMSK_TXDR)) {
				while (size) {
					ret = dwmci_fifo_ready(host,
							DWMCI_FIFO_FULL,
							&len);
					if (ret < 0)
						break;

					len = fifo_depth - ((len >>
						   DWMCI_FIFO_SHIFT) &
						   DWMCI_FIFO_MASK);
					len = min(size, len);
					for (i = 0; i < len; i++)
						dwmci_writel(host, DWMCI_DATA,
							     *buf++);
					size = size > len ? (size - len) : 0;
				}
				dwmci_writel(host, DWMCI_RINTSTS,
					     DWMCI_INTMSK_TXDR);
			}
		}

		/* Data arrived correctly. */
		if (mask & DWMCI_INTMSK_DTO) {
			ret = 0;
			break;
		}

		/* Check for timeout. */
		if (get_timer(start) > timeout) {
			debug("%s: Timeout waiting for data!\n",
			      __func__);
			ret = -ETIMEDOUT;
			break;
		}
	}

	dwmci_writel(host, DWMCI_RINTSTS, mask);

	return ret;
}

static int dwmci_set_transfer_mode(struct dwmci_host *host,
		struct mmc_data *data)
{
	unsigned long mode;

	mode = DWMCI_CMD_DATA_EXP;
	if (data->flags & MMC_DATA_WRITE)
		mode |= DWMCI_CMD_RW;

	return mode;
}

#ifdef CONFIG_DM_MMC
static int dwmci_send_cmd(struct udevice *dev, struct mmc_cmd *cmd,
		   struct mmc_data *data)
{
	struct mmc *mmc = mmc_get_mmc_dev(dev);
#else
static int dwmci_send_cmd(struct mmc *mmc, struct mmc_cmd *cmd,
		struct mmc_data *data)
{
#endif
	struct dwmci_host *host = mmc->priv;
	ALLOC_CACHE_ALIGN_BUFFER(struct dwmci_idmac, cur_idmac,
				 data ? DIV_ROUND_UP(data->blocks, 8) : 0);
	int ret = 0, flags = 0, i;
	unsigned int timeout = 500;
	u32 retry = 100000;
	u32 mask, ctrl;
	ulong start = get_timer(0);
	struct bounce_buffer bbstate;

	while (dwmci_readl(host, DWMCI_STATUS) & DWMCI_BUSY) {
		if (get_timer(start) > timeout) {
			debug("%s: Timeout on data busy\n", __func__);
			return -ETIMEDOUT;
		}
	}

	dwmci_writel(host, DWMCI_RINTSTS, DWMCI_INTMSK_ALL);

	if (data) {
		if (host->fifo_mode) {
			dwmci_writel(host, DWMCI_BLKSIZ, data->blocksize);
			dwmci_writel(host, DWMCI_BYTCNT,
				     data->blocksize * data->blocks);
			dwmci_wait_reset(host, DWMCI_CTRL_FIFO_RESET);
		} else {
			if (data->flags == MMC_DATA_READ) {
				ret = bounce_buffer_start(&bbstate,
						(void*)data->dest,
						data->blocksize *
						data->blocks, GEN_BB_WRITE);
			} else {
				ret = bounce_buffer_start(&bbstate,
						(void*)data->src,
						data->blocksize *
						data->blocks, GEN_BB_READ);
			}

			if (ret)
				return ret;

			dwmci_prepare_data(host, data, cur_idmac,
					   bbstate.bounce_buffer);
		}
	}

	dwmci_writel(host, DWMCI_CMDARG, cmd->cmdarg);

	if (data)
		flags = dwmci_set_transfer_mode(host, data);

	if ((cmd->resp_type & MMC_RSP_136) && (cmd->resp_type & MMC_RSP_BUSY))
		return -EBUSY;

	if (cmd->cmdidx == MMC_CMD_STOP_TRANSMISSION)
		flags |= DWMCI_CMD_ABORT_STOP;
	else
		flags |= DWMCI_CMD_PRV_DAT_WAIT;

	if (cmd->resp_type & MMC_RSP_PRESENT) {
		flags |= DWMCI_CMD_RESP_EXP;
		if (cmd->resp_type & MMC_RSP_136)
			flags |= DWMCI_CMD_RESP_LENGTH;
	}

	if (cmd->resp_type & MMC_RSP_CRC)
		flags |= DWMCI_CMD_CHECK_CRC;

	flags |= (cmd->cmdidx | DWMCI_CMD_START | DWMCI_CMD_USE_HOLD_REG);

	debug("Sending CMD%d\n",cmd->cmdidx);

	dwmci_writel(host, DWMCI_CMD, flags);

	for (i = 0; i < retry; i++) {
		mask = dwmci_readl(host, DWMCI_RINTSTS);
		if (mask & DWMCI_INTMSK_CDONE) {
			if (!data)
				dwmci_writel(host, DWMCI_RINTSTS, mask);
			break;
		}
	}

	if (i == retry) {
		debug("%s: Timeout.\n", __func__);
		return -ETIMEDOUT;
	}

	if (mask & DWMCI_INTMSK_RTO) {
		/*
		 * Timeout here is not necessarily fatal. (e)MMC cards
		 * will splat here when they receive CMD55 as they do
		 * not support this command and that is exactly the way
		 * to tell them apart from SD cards. Thus, this output
		 * below shall be debug(). eMMC cards also do not favor
		 * CMD8, please keep that in mind.
		 */
		debug("%s: Response Timeout.\n", __func__);
		return -ETIMEDOUT;
	} else if (mask & DWMCI_INTMSK_RE) {
		debug("%s: Response Error.\n", __func__);
		return -EIO;
	} else if ((cmd->resp_type & MMC_RSP_CRC) &&
		   (mask & DWMCI_INTMSK_RCRC)) {
		debug("%s: Response CRC Error.\n", __func__);
		return -EIO;
	}


	if (cmd->resp_type & MMC_RSP_PRESENT) {
		if (cmd->resp_type & MMC_RSP_136) {
			cmd->response[0] = dwmci_readl(host, DWMCI_RESP3);
			cmd->response[1] = dwmci_readl(host, DWMCI_RESP2);
			cmd->response[2] = dwmci_readl(host, DWMCI_RESP1);
			cmd->response[3] = dwmci_readl(host, DWMCI_RESP0);
		} else {
			cmd->response[0] = dwmci_readl(host, DWMCI_RESP0);
		}
	}

	if (data) {
		ret = dwmci_data_transfer(host, data);

		/* only dma mode need it */
		if (!host->fifo_mode) {
			if (data->flags == MMC_DATA_READ)
				mask = DWMCI_IDINTEN_RI;
			else
				mask = DWMCI_IDINTEN_TI;
			ret = wait_for_bit_le32(host->ioaddr + DWMCI_IDSTS,
						mask, true, 1000, false);
			if (ret)
				debug("%s: DWMCI_IDINTEN mask 0x%x timeout.\n",
				      __func__, mask);
			/* clear interrupts */
			dwmci_writel(host, DWMCI_IDSTS, DWMCI_IDINTEN_MASK);

			ctrl = dwmci_readl(host, DWMCI_CTRL);
			ctrl &= ~(DWMCI_DMA_EN);
			dwmci_writel(host, DWMCI_CTRL, ctrl);
			bounce_buffer_stop(&bbstate);
		}
	}

	udelay(100);

	return ret;
}

static int dwmci_setup_bus(struct dwmci_host *host, u32 freq)
{
	u32 div, status;
	int timeout = 10000;
	unsigned long sclk;

	if ((freq == host->clock) || (freq == 0))
		return 0;
	/*
	 * If host->get_mmc_clk isn't defined,
	 * then assume that host->bus_hz is source clock value.
	 * host->bus_hz should be set by user.
	 */
	if (host->get_mmc_clk)
		sclk = host->get_mmc_clk(host, freq);
	else if (host->bus_hz)
		sclk = host->bus_hz;
	else {
		debug("%s: Didn't get source clock value.\n", __func__);
		return -EINVAL;
	}

	if (sclk == freq)
		div = 0;	/* bypass mode */
	else
		div = DIV_ROUND_UP(sclk, 2 * freq);

	dwmci_writel(host, DWMCI_CLKENA, 0);
	dwmci_writel(host, DWMCI_CLKSRC, 0);

	dwmci_writel(host, DWMCI_CLKDIV, div);
	dwmci_writel(host, DWMCI_CMD, DWMCI_CMD_PRV_DAT_WAIT |
			DWMCI_CMD_UPD_CLK | DWMCI_CMD_START);

	do {
		status = dwmci_readl(host, DWMCI_CMD);
		if (timeout-- < 0) {
			debug("%s: Timeout!\n", __func__);
			return -ETIMEDOUT;
		}
	} while (status & DWMCI_CMD_START);

	dwmci_writel(host, DWMCI_CLKENA, DWMCI_CLKEN_ENABLE |
			DWMCI_CLKEN_LOW_PWR);

	dwmci_writel(host, DWMCI_CMD, DWMCI_CMD_PRV_DAT_WAIT |
			DWMCI_CMD_UPD_CLK | DWMCI_CMD_START);

	timeout = 10000;
	do {
		status = dwmci_readl(host, DWMCI_CMD);
		if (timeout-- < 0) {
			debug("%s: Timeout!\n", __func__);
			return -ETIMEDOUT;
		}
	} while (status & DWMCI_CMD_START);

	host->clock = freq;

	return 0;
}

#ifdef CONFIG_DM_MMC
static int dwmci_set_ios(struct udevice *dev)
{
	struct mmc *mmc = mmc_get_mmc_dev(dev);
#else
static int dwmci_set_ios(struct mmc *mmc)
{
#endif
	struct dwmci_host *host = (struct dwmci_host *)mmc->priv;
	u32 ctype, regs;

	debug("Buswidth = %d, clock: %d\n", mmc->bus_width, mmc->clock);

	dwmci_setup_bus(host, mmc->clock);
	switch (mmc->bus_width) {
	case 8:
		ctype = DWMCI_CTYPE_8BIT;
		break;
	case 4:
		ctype = DWMCI_CTYPE_4BIT;
		break;
	default:
		ctype = DWMCI_CTYPE_1BIT;
		break;
	}

	dwmci_writel(host, DWMCI_CTYPE, ctype);

	regs = dwmci_readl(host, DWMCI_UHS_REG);
	if (mmc->ddr_mode)
		regs |= DWMCI_DDR_MODE;
	else
		regs &= ~DWMCI_DDR_MODE;

	dwmci_writel(host, DWMCI_UHS_REG, regs);

	if (host->clksel) {
		int ret;

		ret = host->clksel(host);
		if (ret)
			return ret;
	}

#if CONFIG_IS_ENABLED(DM_REGULATOR)
	if (mmc->vqmmc_supply) {
		int ret;

		ret = regulator_set_enable_if_allowed(mmc->vqmmc_supply, false);
		if (ret)
			return ret;

		if (mmc->signal_voltage == MMC_SIGNAL_VOLTAGE_180)
			regulator_set_value(mmc->vqmmc_supply, 1800000);
		else
			regulator_set_value(mmc->vqmmc_supply, 3300000);

		ret = regulator_set_enable_if_allowed(mmc->vqmmc_supply, true);
		if (ret)
			return ret;
	}
#endif

	return 0;
}

static int dwmci_init(struct mmc *mmc)
{
	struct dwmci_host *host = mmc->priv;

	if (host->board_init)
		host->board_init(host);

	dwmci_writel(host, DWMCI_PWREN, 1);

	if (!dwmci_wait_reset(host, DWMCI_RESET_ALL)) {
		debug("%s[%d] Fail-reset!!\n", __func__, __LINE__);
		return -EIO;
	}

	/* Enumerate at 400KHz */
	dwmci_setup_bus(host, mmc->cfg->f_min);

	dwmci_writel(host, DWMCI_RINTSTS, 0xFFFFFFFF);
	dwmci_writel(host, DWMCI_INTMASK, 0);

	dwmci_writel(host, DWMCI_TMOUT, 0xFFFFFFFF);

	dwmci_writel(host, DWMCI_IDINTEN, 0);
	dwmci_writel(host, DWMCI_BMOD, 1);

	if (!host->fifoth_val) {
		uint32_t fifo_size;

		fifo_size = dwmci_readl(host, DWMCI_FIFOTH);
		fifo_size = ((fifo_size & RX_WMARK_MASK) >> RX_WMARK_SHIFT) + 1;
		host->fifoth_val = MSIZE(0x2) | RX_WMARK(fifo_size / 2 - 1) |
				TX_WMARK(fifo_size / 2);
	}
	dwmci_writel(host, DWMCI_FIFOTH, host->fifoth_val);

	dwmci_writel(host, DWMCI_CLKENA, 0);
	dwmci_writel(host, DWMCI_CLKSRC, 0);

	if (!host->fifo_mode)
		dwmci_writel(host, DWMCI_IDINTEN, DWMCI_IDINTEN_MASK);

	return 0;
}

#ifdef CONFIG_DM_MMC
int dwmci_probe(struct udevice *dev)
{
	struct mmc *mmc = mmc_get_mmc_dev(dev);

	return dwmci_init(mmc);
}

const struct dm_mmc_ops dm_dwmci_ops = {
	.send_cmd	= dwmci_send_cmd,
	.set_ios	= dwmci_set_ios,
};

#else
static const struct mmc_ops dwmci_ops = {
	.send_cmd	= dwmci_send_cmd,
	.set_ios	= dwmci_set_ios,
	.init		= dwmci_init,
};
#endif

void dwmci_setup_cfg(struct mmc_config *cfg, struct dwmci_host *host,
		u32 max_clk, u32 min_clk)
{
	cfg->name = host->name;
#ifndef CONFIG_DM_MMC
	cfg->ops = &dwmci_ops;
#endif
	cfg->f_min = min_clk;
	cfg->f_max = max_clk;

	cfg->voltages = MMC_VDD_32_33 | MMC_VDD_33_34 | MMC_VDD_165_195;

	cfg->host_caps = host->caps;

	if (host->buswidth == 8) {
		cfg->host_caps |= MMC_MODE_8BIT;
		cfg->host_caps &= ~MMC_MODE_4BIT;
	} else {
		cfg->host_caps |= MMC_MODE_4BIT;
		cfg->host_caps &= ~MMC_MODE_8BIT;
	}
	cfg->host_caps |= MMC_MODE_HS | MMC_MODE_HS_52MHz;

	cfg->b_max = CONFIG_SYS_MMC_MAX_BLK_COUNT;
}

#ifdef CONFIG_BLK
int dwmci_bind(struct udevice *dev, struct mmc *mmc, struct mmc_config *cfg)
{
	return mmc_bind(dev, mmc, cfg);
}
#else
int add_dwmci(struct dwmci_host *host, u32 max_clk, u32 min_clk)
{
	dwmci_setup_cfg(&host->cfg, host, max_clk, min_clk);

	host->mmc = mmc_create(&host->cfg, host);
	if (host->mmc == NULL)
		return -1;

	return 0;
}
#endif
Commit	Line	Data
83d290c5	1	// SPDX-License-Identifier: GPL-2.0+
757bff49 JC	2	/*
	3	* (C) Copyright 2012 SAMSUNG Electronics
	4	* Jaehoon Chung <[email protected]>
	5	* Rajeshawari Shinde <[email protected]>
757bff49 JC	6	*/
757bff49 JC	7
2a7a210e	8	#include <bouncebuf.h>
757bff49	9	#include <common.h>
1eb69ae4	10	#include <cpu_func.h>
1c87ffe8	11	#include <errno.h>
f7ae49fc	12	#include <log.h>
757bff49	13	#include <malloc.h>
cf92e05c	14	#include <memalign.h>
757bff49 JC	15	#include <mmc.h>
757bff49 JC	16	#include <dwmmc.h>
7997599e	17	#include <wait_bit.h>
90526e9f	18	#include <asm/cache.h>
c05ed00a	19	#include <linux/delay.h>
2b157019	20	#include <power/regulator.h>
757bff49 JC	21
	22	#define PAGE_SIZE 4096
	23
	24	static int dwmci_wait_reset(struct dwmci_host *host, u32 value)
	25	{
	26	unsigned long timeout = 1000;
	27	u32 ctrl;
	28
	29	dwmci_writel(host, DWMCI_CTRL, value);
	30
	31	while (timeout--) {
	32	ctrl = dwmci_readl(host, DWMCI_CTRL);
	33	if (!(ctrl & DWMCI_RESET_ALL))
	34	return 1;
	35	}
	36	return 0;
	37	}
	38
	39	static void dwmci_set_idma_desc(struct dwmci_idmac *idmac,
	40	u32 desc0, u32 desc1, u32 desc2)
	41	{
	42	struct dwmci_idmac *desc = idmac;
	43
	44	desc->flags = desc0;
	45	desc->cnt = desc1;
	46	desc->addr = desc2;
41f7be3c	47	desc->next_addr = (ulong)desc + sizeof(struct dwmci_idmac);
757bff49 JC	48	}
	49
	50	static void dwmci_prepare_data(struct dwmci_host *host,
2a7a210e AB	51	struct mmc_data *data,
	52	struct dwmci_idmac *cur_idmac,
	53	void *bounce_buffer)
757bff49 JC	54	{
	55	unsigned long ctrl;
	56	unsigned int i = 0, flags, cnt, blk_cnt;
2a7a210e	57	ulong data_start, data_end;
757bff49 JC	58
	59
	60	blk_cnt = data->blocks;
	61
	62	dwmci_wait_reset(host, DWMCI_CTRL_FIFO_RESET);
	63
7997599e LFT	64	/* Clear IDMAC interrupt */
	65	dwmci_writel(host, DWMCI_IDSTS, 0xFFFFFFFF);
	66
757bff49	67	data_start = (ulong)cur_idmac;
41f7be3c	68	dwmci_writel(host, DWMCI_DBADDR, (ulong)cur_idmac);
757bff49	69
757bff49 JC	70	do {
	71	flags = DWMCI_IDMAC_OWN \| DWMCI_IDMAC_CH ;
	72	flags \|= (i == 0) ? DWMCI_IDMAC_FS : 0;
	73	if (blk_cnt <= 8) {
	74	flags \|= DWMCI_IDMAC_LD;
	75	cnt = data->blocksize * blk_cnt;
	76	} else
	77	cnt = data->blocksize * 8;
	78
	79	dwmci_set_idma_desc(cur_idmac, flags, cnt,
41f7be3c	80	(ulong)bounce_buffer + (i * PAGE_SIZE));
757bff49	81
bdb5df1a	82	cur_idmac++;
21bd5761	83	if (blk_cnt <= 8)
757bff49 JC	84	break;
757bff49 JC	85	blk_cnt -= 8;
757bff49 JC	86	i++;
	87	} while(1);
	88
	89	data_end = (ulong)cur_idmac;
bdb5df1a	90	flush_dcache_range(data_start, roundup(data_end, ARCH_DMA_MINALIGN));
757bff49 JC	91
	92	ctrl = dwmci_readl(host, DWMCI_CTRL);
	93	ctrl \|= DWMCI_IDMAC_EN \| DWMCI_DMA_EN;
	94	dwmci_writel(host, DWMCI_CTRL, ctrl);
	95
	96	ctrl = dwmci_readl(host, DWMCI_BMOD);
	97	ctrl \|= DWMCI_BMOD_IDMAC_FB \| DWMCI_BMOD_IDMAC_EN;
	98	dwmci_writel(host, DWMCI_BMOD, ctrl);
	99
	100	dwmci_writel(host, DWMCI_BLKSIZ, data->blocksize);
	101	dwmci_writel(host, DWMCI_BYTCNT, data->blocksize * data->blocks);
	102	}
	103
05fa06b9 HS	104	static int dwmci_fifo_ready(struct dwmci_host host, u32 bit, u32 len)
	105	{
	106	u32 timeout = 20000;
	107
	108	*len = dwmci_readl(host, DWMCI_STATUS);
	109	while (--timeout && (*len & bit)) {
	110	udelay(200);
	111	*len = dwmci_readl(host, DWMCI_STATUS);
	112	}
	113
	114	if (!timeout) {
	115	debug("%s: FIFO underflow timeout\n", __func__);
	116	return -ETIMEDOUT;
	117	}
	118
	119	return 0;
	120	}
	121
4e16f0a6 MV	122	static unsigned int dwmci_get_timeout(struct mmc *mmc, const unsigned int size)
	123	{
	124	unsigned int timeout;
	125
c077c057 KY	126	timeout = size * 8; /* counting in bits */
c077c057 KY	127	timeout = 10; / wait 10 times as long */
4e16f0a6 MV	128	timeout /= mmc->clock;
	129	timeout /= mmc->bus_width;
	130	timeout /= mmc->ddr_mode ? 2 : 1;
c077c057	131	timeout = 1000; / counting in msec */
4e16f0a6 MV	132	timeout = (timeout < 1000) ? 1000 : timeout;
	133
	134	return timeout;
	135	}
	136
a65f51b9	137	static int dwmci_data_transfer(struct dwmci_host host, struct mmc_data data)
f382eb83	138	{
4e16f0a6	139	struct mmc *mmc = host->mmc;
f382eb83	140	int ret = 0;
4e16f0a6	141	u32 timeout, mask, size, i, len = 0;
a65f51b9	142	u32 *buf = NULL;
f382eb83	143	ulong start = get_timer(0);
a65f51b9	144	u32 fifo_depth = (((host->fifoth_val & RX_WMARK_MASK) >>
	145	RX_WMARK_SHIFT) + 1) * 2;
	146
4e16f0a6	147	size = data->blocksize * data->blocks;
a65f51b9	148	if (data->flags == MMC_DATA_READ)
	149	buf = (unsigned int *)data->dest;
	150	else
	151	buf = (unsigned int *)data->src;
f382eb83	152
4e16f0a6 MV	153	timeout = dwmci_get_timeout(mmc, size);
	154
	155	size /= 4;
	156
f382eb83	157	for (;;) {
	158	mask = dwmci_readl(host, DWMCI_RINTSTS);
	159	/* Error during data transfer. */
	160	if (mask & (DWMCI_DATA_ERR \| DWMCI_DATA_TOUT)) {
	161	debug("%s: DATA ERROR!\n", __func__);
	162	ret = -EINVAL;
	163	break;
	164	}
	165
a65f51b9	166	if (host->fifo_mode && size) {
720724d0	167	len = 0;
2b429033	168	if (data->flags == MMC_DATA_READ &&
8cb9d3ed LFT	169	(mask & (DWMCI_INTMSK_RXDR \| DWMCI_INTMSK_DTO))) {
8cb9d3ed LFT	170	dwmci_writel(host, DWMCI_RINTSTS,
7ff2f30b JK	171	mask & (DWMCI_INTMSK_RXDR \|
7ff2f30b JK	172	DWMCI_INTMSK_DTO));
2b429033	173	while (size) {
05fa06b9 HS	174	ret = dwmci_fifo_ready(host,
	175	DWMCI_FIFO_EMPTY,
	176	&len);
	177	if (ret < 0)
	178	break;
	179
a65f51b9	180	len = (len >> DWMCI_FIFO_SHIFT) &
a65f51b9	181	DWMCI_FIFO_MASK;
2990e07a	182	len = min(size, len);
a65f51b9	183	for (i = 0; i < len; i++)
	184	*buf++ =
	185	dwmci_readl(host, DWMCI_DATA);
2b429033	186	size = size > len ? (size - len) : 0;
a65f51b9	187	}
2b429033 JC	188	} else if (data->flags == MMC_DATA_WRITE &&
	189	(mask & DWMCI_INTMSK_TXDR)) {
	190	while (size) {
05fa06b9 HS	191	ret = dwmci_fifo_ready(host,
	192	DWMCI_FIFO_FULL,
	193	&len);
	194	if (ret < 0)
	195	break;
	196
a65f51b9	197	len = fifo_depth - ((len >>
	198	DWMCI_FIFO_SHIFT) &
	199	DWMCI_FIFO_MASK);
2990e07a	200	len = min(size, len);
a65f51b9	201	for (i = 0; i < len; i++)
	202	dwmci_writel(host, DWMCI_DATA,
	203	*buf++);
2b429033	204	size = size > len ? (size - len) : 0;
a65f51b9	205	}
2b429033 JC	206	dwmci_writel(host, DWMCI_RINTSTS,
2b429033 JC	207	DWMCI_INTMSK_TXDR);
a65f51b9	208	}
a65f51b9	209	}
a65f51b9	210
f382eb83	211	/* Data arrived correctly. */
	212	if (mask & DWMCI_INTMSK_DTO) {
	213	ret = 0;
	214	break;
	215	}
	216
	217	/* Check for timeout. */
	218	if (get_timer(start) > timeout) {
	219	debug("%s: Timeout waiting for data!\n",
	220	__func__);
915ffa52	221	ret = -ETIMEDOUT;
f382eb83	222	break;
	223	}
	224	}
	225
	226	dwmci_writel(host, DWMCI_RINTSTS, mask);
	227
	228	return ret;
	229	}
	230
757bff49 JC	231	static int dwmci_set_transfer_mode(struct dwmci_host *host,
	232	struct mmc_data *data)
	233	{
	234	unsigned long mode;
	235
	236	mode = DWMCI_CMD_DATA_EXP;
	237	if (data->flags & MMC_DATA_WRITE)
	238	mode \|= DWMCI_CMD_RW;
	239
	240	return mode;
	241	}
	242
e7881d85	243	#ifdef CONFIG_DM_MMC
5628347f	244	static int dwmci_send_cmd(struct udevice dev, struct mmc_cmd cmd,
691272fe SG	245	struct mmc_data *data)
	246	{
	247	struct mmc *mmc = mmc_get_mmc_dev(dev);
	248	#else
757bff49 JC	249	static int dwmci_send_cmd(struct mmc mmc, struct mmc_cmd cmd,
	250	struct mmc_data *data)
	251	{
691272fe	252	#endif
93bfd616	253	struct dwmci_host *host = mmc->priv;
2136d226	254	ALLOC_CACHE_ALIGN_BUFFER(struct dwmci_idmac, cur_idmac,
21bd5761	255	data ? DIV_ROUND_UP(data->blocks, 8) : 0);
9042d974	256	int ret = 0, flags = 0, i;
02ebd42c	257	unsigned int timeout = 500;
9b5b8b6e	258	u32 retry = 100000;
757bff49	259	u32 mask, ctrl;
9c50e35f	260	ulong start = get_timer(0);
2a7a210e	261	struct bounce_buffer bbstate;
757bff49 JC	262
757bff49 JC	263	while (dwmci_readl(host, DWMCI_STATUS) & DWMCI_BUSY) {
9c50e35f	264	if (get_timer(start) > timeout) {
1c87ffe8	265	debug("%s: Timeout on data busy\n", __func__);
915ffa52	266	return -ETIMEDOUT;
757bff49	267	}
757bff49 JC	268	}
	269
	270	dwmci_writel(host, DWMCI_RINTSTS, DWMCI_INTMSK_ALL);
	271
2a7a210e	272	if (data) {
a65f51b9	273	if (host->fifo_mode) {
	274	dwmci_writel(host, DWMCI_BLKSIZ, data->blocksize);
	275	dwmci_writel(host, DWMCI_BYTCNT,
	276	data->blocksize * data->blocks);
	277	dwmci_wait_reset(host, DWMCI_CTRL_FIFO_RESET);
2a7a210e	278	} else {
a65f51b9	279	if (data->flags == MMC_DATA_READ) {
6ad5aec4 MV	280	ret = bounce_buffer_start(&bbstate,
6ad5aec4 MV	281	(void*)data->dest,
a65f51b9	282	data->blocksize *
	283	data->blocks, GEN_BB_WRITE);
	284	} else {
6ad5aec4 MV	285	ret = bounce_buffer_start(&bbstate,
6ad5aec4 MV	286	(void*)data->src,
a65f51b9	287	data->blocksize *
	288	data->blocks, GEN_BB_READ);
	289	}
6ad5aec4 MV	290
	291	if (ret)
	292	return ret;
	293
a65f51b9	294	dwmci_prepare_data(host, data, cur_idmac,
a65f51b9	295	bbstate.bounce_buffer);
2a7a210e	296	}
2a7a210e	297	}
757bff49	298
757bff49 JC	299	dwmci_writel(host, DWMCI_CMDARG, cmd->cmdarg);
	300
	301	if (data)
	302	flags = dwmci_set_transfer_mode(host, data);
	303
	304	if ((cmd->resp_type & MMC_RSP_136) && (cmd->resp_type & MMC_RSP_BUSY))
66d0b7e1	305	return -EBUSY;
757bff49 JC	306
	307	if (cmd->cmdidx == MMC_CMD_STOP_TRANSMISSION)
	308	flags \|= DWMCI_CMD_ABORT_STOP;
	309	else
	310	flags \|= DWMCI_CMD_PRV_DAT_WAIT;
	311
	312	if (cmd->resp_type & MMC_RSP_PRESENT) {
	313	flags \|= DWMCI_CMD_RESP_EXP;
	314	if (cmd->resp_type & MMC_RSP_136)
	315	flags \|= DWMCI_CMD_RESP_LENGTH;
	316	}
	317
	318	if (cmd->resp_type & MMC_RSP_CRC)
	319	flags \|= DWMCI_CMD_CHECK_CRC;
	320
	321	flags \|= (cmd->cmdidx \| DWMCI_CMD_START \| DWMCI_CMD_USE_HOLD_REG);
	322
	323	debug("Sending CMD%d\n",cmd->cmdidx);
	324
	325	dwmci_writel(host, DWMCI_CMD, flags);
	326
	327	for (i = 0; i < retry; i++) {
	328	mask = dwmci_readl(host, DWMCI_RINTSTS);
	329	if (mask & DWMCI_INTMSK_CDONE) {
	330	if (!data)
	331	dwmci_writel(host, DWMCI_RINTSTS, mask);
	332	break;
	333	}
	334	}
	335
f33c9305	336	if (i == retry) {
1c87ffe8	337	debug("%s: Timeout.\n", __func__);
915ffa52	338	return -ETIMEDOUT;
f33c9305	339	}
757bff49 JC	340
757bff49 JC	341	if (mask & DWMCI_INTMSK_RTO) {
f33c9305 PM	342	/*
	343	* Timeout here is not necessarily fatal. (e)MMC cards
	344	* will splat here when they receive CMD55 as they do
	345	* not support this command and that is exactly the way
	346	* to tell them apart from SD cards. Thus, this output
	347	* below shall be debug(). eMMC cards also do not favor
	348	* CMD8, please keep that in mind.
	349	*/
	350	debug("%s: Response Timeout.\n", __func__);
915ffa52	351	return -ETIMEDOUT;
757bff49	352	} else if (mask & DWMCI_INTMSK_RE) {
1c87ffe8 SG	353	debug("%s: Response Error.\n", __func__);
1c87ffe8 SG	354	return -EIO;
26cc40d8 MV	355	} else if ((cmd->resp_type & MMC_RSP_CRC) &&
	356	(mask & DWMCI_INTMSK_RCRC)) {
	357	debug("%s: Response CRC Error.\n", __func__);
	358	return -EIO;
757bff49 JC	359	}
	360
	361
	362	if (cmd->resp_type & MMC_RSP_PRESENT) {
	363	if (cmd->resp_type & MMC_RSP_136) {
	364	cmd->response[0] = dwmci_readl(host, DWMCI_RESP3);
	365	cmd->response[1] = dwmci_readl(host, DWMCI_RESP2);
	366	cmd->response[2] = dwmci_readl(host, DWMCI_RESP1);
	367	cmd->response[3] = dwmci_readl(host, DWMCI_RESP0);
	368	} else {
	369	cmd->response[0] = dwmci_readl(host, DWMCI_RESP0);
	370	}
	371	}
	372
	373	if (data) {
a65f51b9	374	ret = dwmci_data_transfer(host, data);
	375
	376	/* only dma mode need it */
	377	if (!host->fifo_mode) {
7997599e LFT	378	if (data->flags == MMC_DATA_READ)
	379	mask = DWMCI_IDINTEN_RI;
	380	else
	381	mask = DWMCI_IDINTEN_TI;
	382	ret = wait_for_bit_le32(host->ioaddr + DWMCI_IDSTS,
	383	mask, true, 1000, false);
	384	if (ret)
	385	debug("%s: DWMCI_IDINTEN mask 0x%x timeout.\n",
	386	__func__, mask);
	387	/* clear interrupts */
	388	dwmci_writel(host, DWMCI_IDSTS, DWMCI_IDINTEN_MASK);
	389
a65f51b9	390	ctrl = dwmci_readl(host, DWMCI_CTRL);
	391	ctrl &= ~(DWMCI_DMA_EN);
	392	dwmci_writel(host, DWMCI_CTRL, ctrl);
	393	bounce_buffer_stop(&bbstate);
	394	}
757bff49 JC	395	}
	396
	397	udelay(100);
	398
9042d974	399	return ret;
757bff49 JC	400	}
	401
	402	static int dwmci_setup_bus(struct dwmci_host *host, u32 freq)
	403	{
	404	u32 div, status;
	405	int timeout = 10000;
	406	unsigned long sclk;
	407
9c50e35f	408	if ((freq == host->clock) \|\| (freq == 0))
757bff49	409	return 0;
757bff49	410	/*
f33c9305	411	* If host->get_mmc_clk isn't defined,
757bff49	412	* then assume that host->bus_hz is source clock value.
f33c9305	413	* host->bus_hz should be set by user.
757bff49	414	*/
b44fe83a	415	if (host->get_mmc_clk)
e3563f2e	416	sclk = host->get_mmc_clk(host, freq);
757bff49 JC	417	else if (host->bus_hz)
	418	sclk = host->bus_hz;
	419	else {
1c87ffe8	420	debug("%s: Didn't get source clock value.\n", __func__);
757bff49 JC	421	return -EINVAL;
	422	}
	423
6ace153d CLS	424	if (sclk == freq)
	425	div = 0; /* bypass mode */
	426	else
	427	div = DIV_ROUND_UP(sclk, 2 * freq);
757bff49 JC	428
	429	dwmci_writel(host, DWMCI_CLKENA, 0);
	430	dwmci_writel(host, DWMCI_CLKSRC, 0);
	431
	432	dwmci_writel(host, DWMCI_CLKDIV, div);
	433	dwmci_writel(host, DWMCI_CMD, DWMCI_CMD_PRV_DAT_WAIT \|
	434	DWMCI_CMD_UPD_CLK \| DWMCI_CMD_START);
	435
	436	do {
	437	status = dwmci_readl(host, DWMCI_CMD);
	438	if (timeout-- < 0) {
1c87ffe8	439	debug("%s: Timeout!\n", __func__);
757bff49 JC	440	return -ETIMEDOUT;
	441	}
	442	} while (status & DWMCI_CMD_START);
	443
	444	dwmci_writel(host, DWMCI_CLKENA, DWMCI_CLKEN_ENABLE \|
	445	DWMCI_CLKEN_LOW_PWR);
	446
	447	dwmci_writel(host, DWMCI_CMD, DWMCI_CMD_PRV_DAT_WAIT \|
	448	DWMCI_CMD_UPD_CLK \| DWMCI_CMD_START);
	449
	450	timeout = 10000;
	451	do {
	452	status = dwmci_readl(host, DWMCI_CMD);
	453	if (timeout-- < 0) {
1c87ffe8	454	debug("%s: Timeout!\n", __func__);
757bff49 JC	455	return -ETIMEDOUT;
	456	}
	457	} while (status & DWMCI_CMD_START);
	458
	459	host->clock = freq;
	460
	461	return 0;
	462	}
	463
e7881d85	464	#ifdef CONFIG_DM_MMC
5628347f	465	static int dwmci_set_ios(struct udevice *dev)
691272fe SG	466	{
	467	struct mmc *mmc = mmc_get_mmc_dev(dev);
	468	#else
07b0b9c0	469	static int dwmci_set_ios(struct mmc *mmc)
757bff49	470	{
691272fe	471	#endif
045bdcd0 JC	472	struct dwmci_host host = (struct dwmci_host )mmc->priv;
045bdcd0 JC	473	u32 ctype, regs;
757bff49	474
f33c9305	475	debug("Buswidth = %d, clock: %d\n", mmc->bus_width, mmc->clock);
757bff49 JC	476
	477	dwmci_setup_bus(host, mmc->clock);
	478	switch (mmc->bus_width) {
	479	case 8:
	480	ctype = DWMCI_CTYPE_8BIT;
	481	break;
	482	case 4:
	483	ctype = DWMCI_CTYPE_4BIT;
	484	break;
	485	default:
	486	ctype = DWMCI_CTYPE_1BIT;
	487	break;
	488	}
	489
	490	dwmci_writel(host, DWMCI_CTYPE, ctype);
	491
045bdcd0	492	regs = dwmci_readl(host, DWMCI_UHS_REG);
2b8a9692	493	if (mmc->ddr_mode)
045bdcd0 JC	494	regs \|= DWMCI_DDR_MODE;
045bdcd0 JC	495	else
afc9e2b5	496	regs &= ~DWMCI_DDR_MODE;
045bdcd0 JC	497
	498	dwmci_writel(host, DWMCI_UHS_REG, regs);
	499
d456dfba SCL	500	if (host->clksel) {
	501	int ret;
	502
	503	ret = host->clksel(host);
	504	if (ret)
	505	return ret;
	506	}
07b0b9c0	507
2b157019 UR	508	#if CONFIG_IS_ENABLED(DM_REGULATOR)
	509	if (mmc->vqmmc_supply) {
	510	int ret;
	511
01b2917a JK	512	ret = regulator_set_enable_if_allowed(mmc->vqmmc_supply, false);
	513	if (ret)
	514	return ret;
	515
2b157019 UR	516	if (mmc->signal_voltage == MMC_SIGNAL_VOLTAGE_180)
	517	regulator_set_value(mmc->vqmmc_supply, 1800000);
	518	else
	519	regulator_set_value(mmc->vqmmc_supply, 3300000);
	520
	521	ret = regulator_set_enable_if_allowed(mmc->vqmmc_supply, true);
	522	if (ret)
	523	return ret;
	524	}
	525	#endif
	526
691272fe	527	return 0;
757bff49 JC	528	}
	529
	530	static int dwmci_init(struct mmc *mmc)
	531	{
93bfd616	532	struct dwmci_host *host = mmc->priv;
757bff49	533
18ab6755 JC	534	if (host->board_init)
18ab6755 JC	535	host->board_init(host);
6f0b7caa	536
757bff49 JC	537	dwmci_writel(host, DWMCI_PWREN, 1);
	538
	539	if (!dwmci_wait_reset(host, DWMCI_RESET_ALL)) {
1c87ffe8 SG	540	debug("%s[%d] Fail-reset!!\n", __func__, __LINE__);
1c87ffe8 SG	541	return -EIO;
757bff49 JC	542	}
757bff49 JC	543
9c50e35f	544	/* Enumerate at 400KHz */
93bfd616	545	dwmci_setup_bus(host, mmc->cfg->f_min);
9c50e35f	546
757bff49 JC	547	dwmci_writel(host, DWMCI_RINTSTS, 0xFFFFFFFF);
	548	dwmci_writel(host, DWMCI_INTMASK, 0);
	549
	550	dwmci_writel(host, DWMCI_TMOUT, 0xFFFFFFFF);
	551
	552	dwmci_writel(host, DWMCI_IDINTEN, 0);
	553	dwmci_writel(host, DWMCI_BMOD, 1);
	554
760177df SG	555	if (!host->fifoth_val) {
	556	uint32_t fifo_size;
	557
	558	fifo_size = dwmci_readl(host, DWMCI_FIFOTH);
	559	fifo_size = ((fifo_size & RX_WMARK_MASK) >> RX_WMARK_SHIFT) + 1;
	560	host->fifoth_val = MSIZE(0x2) \| RX_WMARK(fifo_size / 2 - 1) \|
	561	TX_WMARK(fifo_size / 2);
9c50e35f	562	}
760177df	563	dwmci_writel(host, DWMCI_FIFOTH, host->fifoth_val);
757bff49 JC	564
	565	dwmci_writel(host, DWMCI_CLKENA, 0);
	566	dwmci_writel(host, DWMCI_CLKSRC, 0);
	567
7997599e LFT	568	if (!host->fifo_mode)
	569	dwmci_writel(host, DWMCI_IDINTEN, DWMCI_IDINTEN_MASK);
	570
757bff49 JC	571	return 0;
	572	}
	573
e7881d85	574	#ifdef CONFIG_DM_MMC
691272fe SG	575	int dwmci_probe(struct udevice *dev)
	576	{
	577	struct mmc *mmc = mmc_get_mmc_dev(dev);
	578
	579	return dwmci_init(mmc);
	580	}
	581
	582	const struct dm_mmc_ops dm_dwmci_ops = {
	583	.send_cmd = dwmci_send_cmd,
	584	.set_ios = dwmci_set_ios,
	585	};
	586
	587	#else
ab769f22 PA	588	static const struct mmc_ops dwmci_ops = {
	589	.send_cmd = dwmci_send_cmd,
	590	.set_ios = dwmci_set_ios,
	591	.init = dwmci_init,
	592	};
691272fe	593	#endif
ab769f22	594
e5113c33 JC	595	void dwmci_setup_cfg(struct mmc_config cfg, struct dwmci_host host,
e5113c33 JC	596	u32 max_clk, u32 min_clk)
757bff49	597	{
e5113c33	598	cfg->name = host->name;
e7881d85	599	#ifndef CONFIG_DM_MMC
5e6ff810	600	cfg->ops = &dwmci_ops;
691272fe	601	#endif
5e6ff810 SG	602	cfg->f_min = min_clk;
5e6ff810 SG	603	cfg->f_max = max_clk;
757bff49	604
5e6ff810	605	cfg->voltages = MMC_VDD_32_33 \| MMC_VDD_33_34 \| MMC_VDD_165_195;
757bff49	606
e5113c33	607	cfg->host_caps = host->caps;
757bff49	608
e5113c33	609	if (host->buswidth == 8) {
5e6ff810 SG	610	cfg->host_caps \|= MMC_MODE_8BIT;
5e6ff810 SG	611	cfg->host_caps &= ~MMC_MODE_4BIT;
757bff49	612	} else {
5e6ff810 SG	613	cfg->host_caps \|= MMC_MODE_4BIT;
5e6ff810 SG	614	cfg->host_caps &= ~MMC_MODE_8BIT;
757bff49	615	}
5e6ff810 SG	616	cfg->host_caps \|= MMC_MODE_HS \| MMC_MODE_HS_52MHz;
	617
	618	cfg->b_max = CONFIG_SYS_MMC_MAX_BLK_COUNT;
	619	}
93bfd616	620
5e6ff810 SG	621	#ifdef CONFIG_BLK
	622	int dwmci_bind(struct udevice dev, struct mmc mmc, struct mmc_config *cfg)
	623	{
	624	return mmc_bind(dev, mmc, cfg);
	625	}
	626	#else
	627	int add_dwmci(struct dwmci_host *host, u32 max_clk, u32 min_clk)
	628	{
e5113c33	629	dwmci_setup_cfg(&host->cfg, host, max_clk, min_clk);
757bff49	630
93bfd616 PA	631	host->mmc = mmc_create(&host->cfg, host);
	632	if (host->mmc == NULL)
	633	return -1;
757bff49	634
93bfd616	635	return 0;
757bff49	636	}
5e6ff810	637	#endif