drivers/mmc/host/rtsx_pci_sdmmc.c

   1 /* Realtek PCI-Express SD/MMC Card Interface driver
   2  *
   3  * Copyright(c) 2009-2013 Realtek Semiconductor Corp. All rights reserved.
   4  *
   5  * This program is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License as published by the
   7  * Free Software Foundation; either version 2, or (at your option) any
   8  * later version.
   9  *
  10  * This program is distributed in the hope that it will be useful, but
  11  * WITHOUT ANY WARRANTY; without even the implied warranty of
  12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  13  * General Public License for more details.
  14  *
  15  * You should have received a copy of the GNU General Public License along
  16  * with this program; if not, see <http://www.gnu.org/licenses/>.
  17  *
  18  * Author:
  19  *   Wei WANG <wei_wang@realsil.com.cn>
  20  */
  21
  22 #include <linux/module.h>
  23 #include <linux/slab.h>
  24 #include <linux/highmem.h>
  25 #include <linux/delay.h>
  26 #include <linux/platform_device.h>
  27 #include <linux/mmc/host.h>
  28 #include <linux/mmc/mmc.h>
  29 #include <linux/mmc/sd.h>
  30 #include <linux/mmc/card.h>
  31 #include <linux/mfd/rtsx_pci.h>
  32 #include <asm/unaligned.h>
  33
  34 struct realtek_next {
  35         unsigned int    sg_count;
  36         s32             cookie;
  37 };
  38
  39 struct realtek_pci_sdmmc {
  40         struct platform_device  *pdev;
  41         struct rtsx_pcr         *pcr;
  42         struct mmc_host         *mmc;
  43         struct mmc_request      *mrq;
  44         struct mmc_command      *cmd;
  45         struct mmc_data         *data;
  46
  47         spinlock_t              lock;
  48         struct timer_list       timer;
  49         struct tasklet_struct   cmd_tasklet;
  50         struct tasklet_struct   data_tasklet;
  51         struct tasklet_struct   finish_tasklet;
  52
  53         u8                      rsp_type;
  54         u8                      rsp_len;
  55         int                     sg_count;
  56         u8                      ssc_depth;
  57         unsigned int            clock;
  58         bool                    vpclk;
  59         bool                    double_clk;
  60         bool                    eject;
  61         bool                    initial_mode;
  62         int                     power_state;
  63 #define SDMMC_POWER_ON          1
  64 #define SDMMC_POWER_OFF         0
  65
  66         struct realtek_next     next_data;
  67 };
  68
  69 static int sd_start_multi_rw(struct realtek_pci_sdmmc *host,
  70                 struct mmc_request *mrq);
  71
  72 static inline struct device *sdmmc_dev(struct realtek_pci_sdmmc *host)
  73 {
  74         return &(host->pdev->dev);
  75 }
  76
  77 static inline void sd_clear_error(struct realtek_pci_sdmmc *host)
  78 {
  79         rtsx_pci_write_register(host->pcr, CARD_STOP,
  80                         SD_STOP | SD_CLR_ERR, SD_STOP | SD_CLR_ERR);
  81 }
  82
  83 #ifdef DEBUG
  84 static void sd_print_debug_regs(struct realtek_pci_sdmmc *host)
  85 {
  86         struct rtsx_pcr *pcr = host->pcr;
  87         u16 i;
  88         u8 *ptr;
  89
  90         /* Print SD host internal registers */
  91         rtsx_pci_init_cmd(pcr);
  92         for (i = 0xFDA0; i <= 0xFDAE; i++)
  93                 rtsx_pci_add_cmd(pcr, READ_REG_CMD, i, 0, 0);
  94         for (i = 0xFD52; i <= 0xFD69; i++)
  95                 rtsx_pci_add_cmd(pcr, READ_REG_CMD, i, 0, 0);
  96         rtsx_pci_send_cmd(pcr, 100);
  97
  98         ptr = rtsx_pci_get_cmd_data(pcr);
  99         for (i = 0xFDA0; i <= 0xFDAE; i++)
 100                 dev_dbg(sdmmc_dev(host), "0x%04X: 0x%02x\n", i, *(ptr++));
 101         for (i = 0xFD52; i <= 0xFD69; i++)
 102                 dev_dbg(sdmmc_dev(host), "0x%04X: 0x%02x\n", i, *(ptr++));
 103 }
 104 #else
 105 #define sd_print_debug_regs(host)
 106 #endif /* DEBUG */
 107
 108 static void sd_isr_done_transfer(struct platform_device *pdev)
 109 {
 110         struct realtek_pci_sdmmc *host = platform_get_drvdata(pdev);
 111
 112         spin_lock(&host->lock);
 113         if (host->cmd)
 114                 tasklet_schedule(&host->cmd_tasklet);
 115         if (host->data)
 116                 tasklet_schedule(&host->data_tasklet);
 117         spin_unlock(&host->lock);
 118 }
 119
 120 static void sd_request_timeout(unsigned long host_addr)
 121 {
 122         struct realtek_pci_sdmmc *host = (struct realtek_pci_sdmmc *)host_addr;
 123         unsigned long flags;
 124
 125         spin_lock_irqsave(&host->lock, flags);
 126
 127         if (!host->mrq) {
 128                 dev_err(sdmmc_dev(host), "error: no request exist\n");
 129                 goto out;
 130         }
 131
 132         if (host->cmd)
 133                 host->cmd->error = -ETIMEDOUT;
 134         if (host->data)
 135                 host->data->error = -ETIMEDOUT;
 136
 137         dev_dbg(sdmmc_dev(host), "timeout for request\n");
 138
 139 out:
 140         tasklet_schedule(&host->finish_tasklet);
 141         spin_unlock_irqrestore(&host->lock, flags);
 142 }
 143
 144 static void sd_finish_request(unsigned long host_addr)
 145 {
 146         struct realtek_pci_sdmmc *host = (struct realtek_pci_sdmmc *)host_addr;
 147         struct rtsx_pcr *pcr = host->pcr;
 148         struct mmc_request *mrq;
 149         struct mmc_command *cmd;
 150         struct mmc_data *data;
 151         unsigned long flags;
 152         bool any_error;
 153
 154         spin_lock_irqsave(&host->lock, flags);
 155
 156         del_timer(&host->timer);
 157         mrq = host->mrq;
 158         if (!mrq) {
 159                 dev_err(sdmmc_dev(host), "error: no request need finish\n");
 160                 goto out;
 161         }
 162
 163         cmd = mrq->cmd;
 164         data = mrq->data;
 165
 166         any_error = (mrq->sbc && mrq->sbc->error) ||
 167                 (mrq->stop && mrq->stop->error) ||
 168                 (cmd && cmd->error) || (data && data->error);
 169
 170         if (any_error) {
 171                 rtsx_pci_stop_cmd(pcr);
 172                 sd_clear_error(host);
 173         }
 174
 175         if (data) {
 176                 if (any_error)
 177                         data->bytes_xfered = 0;
 178                 else
 179                         data->bytes_xfered = data->blocks * data->blksz;
 180
 181                 if (!data->host_cookie)
 182                         rtsx_pci_dma_unmap_sg(pcr, data->sg, data->sg_len,
 183                                         data->flags & MMC_DATA_READ);
 184
 185         }
 186
 187         host->mrq = NULL;
 188         host->cmd = NULL;
 189         host->data = NULL;
 190
 191 out:
 192         spin_unlock_irqrestore(&host->lock, flags);
 193         mutex_unlock(&pcr->pcr_mutex);
 194         mmc_request_done(host->mmc, mrq);
 195 }
 196
 197 static int sd_read_data(struct realtek_pci_sdmmc *host, u8 *cmd, u16 byte_cnt,
 198                 u8 *buf, int buf_len, int timeout)
 199 {
 200         struct rtsx_pcr *pcr = host->pcr;
 201         int err, i;
 202         u8 trans_mode;
 203
 204         dev_dbg(sdmmc_dev(host), "%s: SD/MMC CMD%d\n", __func__, cmd[0] - 0x40);
 205
 206         if (!buf)
 207                 buf_len = 0;
 208
 209         if ((cmd[0] & 0x3F) == MMC_SEND_TUNING_BLOCK)
 210                 trans_mode = SD_TM_AUTO_TUNING;
 211         else
 212                 trans_mode = SD_TM_NORMAL_READ;
 213
 214         rtsx_pci_init_cmd(pcr);
 215
 216         for (i = 0; i < 5; i++)
 217                 rtsx_pci_add_cmd(pcr, WRITE_REG_CMD, SD_CMD0 + i, 0xFF, cmd[i]);
 218
 219         rtsx_pci_add_cmd(pcr, WRITE_REG_CMD, SD_BYTE_CNT_L, 0xFF, (u8)byte_cnt);
 220         rtsx_pci_add_cmd(pcr, WRITE_REG_CMD, SD_BYTE_CNT_H,
 221                         0xFF, (u8)(byte_cnt >> 8));
 222         rtsx_pci_add_cmd(pcr, WRITE_REG_CMD, SD_BLOCK_CNT_L, 0xFF, 1);
 223         rtsx_pci_add_cmd(pcr, WRITE_REG_CMD, SD_BLOCK_CNT_H, 0xFF, 0);
 224
 225         rtsx_pci_add_cmd(pcr, WRITE_REG_CMD, SD_CFG2, 0xFF,
 226                         SD_CALCULATE_CRC7 | SD_CHECK_CRC16 |
 227                         SD_NO_WAIT_BUSY_END | SD_CHECK_CRC7 | SD_RSP_LEN_6);
 228         if (trans_mode != SD_TM_AUTO_TUNING)
 229                 rtsx_pci_add_cmd(pcr, WRITE_REG_CMD,
 230                                 CARD_DATA_SOURCE, 0x01, PINGPONG_BUFFER);
 231
 232         rtsx_pci_add_cmd(pcr, WRITE_REG_CMD, SD_TRANSFER,
 233                         0xFF, trans_mode | SD_TRANSFER_START);
 234         rtsx_pci_add_cmd(pcr, CHECK_REG_CMD, SD_TRANSFER,
 235                         SD_TRANSFER_END, SD_TRANSFER_END);
 236
 237         err = rtsx_pci_send_cmd(pcr, timeout);
 238         if (err < 0) {
 239                 sd_print_debug_regs(host);
 240                 dev_dbg(sdmmc_dev(host),
 241                         "rtsx_pci_send_cmd fail (err = %d)\n", err);
 242                 return err;
 243         }
 244
 245         if (buf && buf_len) {
 246                 err = rtsx_pci_read_ppbuf(pcr, buf, buf_len);
 247                 if (err < 0) {
 248                         dev_dbg(sdmmc_dev(host),
 249                                 "rtsx_pci_read_ppbuf fail (err = %d)\n", err);
 250                         return err;
 251                 }
 252         }
 253
 254         return 0;
 255 }
 256
 257 static int sd_write_data(struct realtek_pci_sdmmc *host, u8 *cmd, u16 byte_cnt,
 258                 u8 *buf, int buf_len, int timeout)
 259 {
 260         struct rtsx_pcr *pcr = host->pcr;
 261         int err, i;
 262         u8 trans_mode;
 263
 264         if (!buf)
 265                 buf_len = 0;
 266
 267         if (buf && buf_len) {
 268                 err = rtsx_pci_write_ppbuf(pcr, buf, buf_len);
 269                 if (err < 0) {
 270                         dev_dbg(sdmmc_dev(host),
 271                                 "rtsx_pci_write_ppbuf fail (err = %d)\n", err);
 272                         return err;
 273                 }
 274         }
 275
 276         trans_mode = cmd ? SD_TM_AUTO_WRITE_2 : SD_TM_AUTO_WRITE_3;
 277         rtsx_pci_init_cmd(pcr);
 278
 279         if (cmd) {
 280                 dev_dbg(sdmmc_dev(host), "%s: SD/MMC CMD %d\n", __func__,
 281                                 cmd[0] - 0x40);
 282
 283                 for (i = 0; i < 5; i++)
 284                         rtsx_pci_add_cmd(pcr, WRITE_REG_CMD,
 285                                         SD_CMD0 + i, 0xFF, cmd[i]);
 286         }
 287
 288         rtsx_pci_add_cmd(pcr, WRITE_REG_CMD, SD_BYTE_CNT_L, 0xFF, (u8)byte_cnt);
 289         rtsx_pci_add_cmd(pcr, WRITE_REG_CMD, SD_BYTE_CNT_H,
 290                         0xFF, (u8)(byte_cnt >> 8));
 291         rtsx_pci_add_cmd(pcr, WRITE_REG_CMD, SD_BLOCK_CNT_L, 0xFF, 1);
 292         rtsx_pci_add_cmd(pcr, WRITE_REG_CMD, SD_BLOCK_CNT_H, 0xFF, 0);
 293
 294         rtsx_pci_add_cmd(pcr, WRITE_REG_CMD, SD_CFG2, 0xFF,
 295                 SD_CALCULATE_CRC7 | SD_CHECK_CRC16 |
 296                 SD_NO_WAIT_BUSY_END | SD_CHECK_CRC7 | SD_RSP_LEN_6);
 297
 298         rtsx_pci_add_cmd(pcr, WRITE_REG_CMD, SD_TRANSFER, 0xFF,
 299                         trans_mode | SD_TRANSFER_START);
 300         rtsx_pci_add_cmd(pcr, CHECK_REG_CMD, SD_TRANSFER,
 301                         SD_TRANSFER_END, SD_TRANSFER_END);
 302
 303         err = rtsx_pci_send_cmd(pcr, timeout);
 304         if (err < 0) {
 305                 sd_print_debug_regs(host);
 306                 dev_dbg(sdmmc_dev(host),
 307                         "rtsx_pci_send_cmd fail (err = %d)\n", err);
 308                 return err;
 309         }
 310
 311         return 0;
 312 }
 313
 314 static void sd_send_cmd(struct realtek_pci_sdmmc *host, struct mmc_command *cmd)
 315 {
 316         struct rtsx_pcr *pcr = host->pcr;
 317         u8 cmd_idx = (u8)cmd->opcode;
 318         u32 arg = cmd->arg;
 319         int err = 0;
 320         int timeout = 100;
 321         int i;
 322         u8 rsp_type;
 323         int rsp_len = 5;
 324         unsigned long flags;
 325
 326         if (host->cmd)
 327                 dev_err(sdmmc_dev(host), "error: cmd already exist\n");
 328
 329         host->cmd = cmd;
 330
 331         dev_dbg(sdmmc_dev(host), "%s: SD/MMC CMD %d, arg = 0x%08x\n",
 332                         __func__, cmd_idx, arg);
 333
 334         /* Response type:
 335          * R0
 336          * R1, R5, R6, R7
 337          * R1b
 338          * R2
 339          * R3, R4
 340          */
 341         switch (mmc_resp_type(cmd)) {
 342         case MMC_RSP_NONE:
 343                 rsp_type = SD_RSP_TYPE_R0;
 344                 rsp_len = 0;
 345                 break;
 346         case MMC_RSP_R1:
 347                 rsp_type = SD_RSP_TYPE_R1;
 348                 break;
 349         case MMC_RSP_R1B:
 350                 rsp_type = SD_RSP_TYPE_R1b;
 351                 break;
 352         case MMC_RSP_R2:
 353                 rsp_type = SD_RSP_TYPE_R2;
 354                 rsp_len = 16;
 355                 break;
 356         case MMC_RSP_R3:
 357                 rsp_type = SD_RSP_TYPE_R3;
 358                 break;
 359         default:
 360                 dev_dbg(sdmmc_dev(host), "cmd->flag is not valid\n");
 361                 err = -EINVAL;
 362                 goto out;
 363         }
 364         host->rsp_type = rsp_type;
 365         host->rsp_len = rsp_len;
 366
 367         if (rsp_type == SD_RSP_TYPE_R1b)
 368                 timeout = 3000;
 369
 370         if (cmd->opcode == SD_SWITCH_VOLTAGE) {
 371                 err = rtsx_pci_write_register(pcr, SD_BUS_STAT,
 372                                 0xFF, SD_CLK_TOGGLE_EN);
 373                 if (err < 0)
 374                         goto out;
 375         }
 376
 377         rtsx_pci_init_cmd(pcr);
 378
 379         rtsx_pci_add_cmd(pcr, WRITE_REG_CMD, SD_CMD0, 0xFF, 0x40 | cmd_idx);
 380         rtsx_pci_add_cmd(pcr, WRITE_REG_CMD, SD_CMD1, 0xFF, (u8)(arg >> 24));
 381         rtsx_pci_add_cmd(pcr, WRITE_REG_CMD, SD_CMD2, 0xFF, (u8)(arg >> 16));
 382         rtsx_pci_add_cmd(pcr, WRITE_REG_CMD, SD_CMD3, 0xFF, (u8)(arg >> 8));
 383         rtsx_pci_add_cmd(pcr, WRITE_REG_CMD, SD_CMD4, 0xFF, (u8)arg);
 384
 385         rtsx_pci_add_cmd(pcr, WRITE_REG_CMD, SD_CFG2, 0xFF, rsp_type);
 386         rtsx_pci_add_cmd(pcr, WRITE_REG_CMD, CARD_DATA_SOURCE,
 387                         0x01, PINGPONG_BUFFER);
 388         rtsx_pci_add_cmd(pcr, WRITE_REG_CMD, SD_TRANSFER,
 389                         0xFF, SD_TM_CMD_RSP | SD_TRANSFER_START);
 390         rtsx_pci_add_cmd(pcr, CHECK_REG_CMD, SD_TRANSFER,
 391                      SD_TRANSFER_END | SD_STAT_IDLE,
 392                      SD_TRANSFER_END | SD_STAT_IDLE);
 393
 394         if (rsp_type == SD_RSP_TYPE_R2) {
 395                 /* Read data from ping-pong buffer */
 396                 for (i = PPBUF_BASE2; i < PPBUF_BASE2 + 16; i++)
 397                         rtsx_pci_add_cmd(pcr, READ_REG_CMD, (u16)i, 0, 0);
 398         } else if (rsp_type != SD_RSP_TYPE_R0) {
 399                 /* Read data from SD_CMDx registers */
 400                 for (i = SD_CMD0; i <= SD_CMD4; i++)
 401                         rtsx_pci_add_cmd(pcr, READ_REG_CMD, (u16)i, 0, 0);
 402         }
 403
 404         rtsx_pci_add_cmd(pcr, READ_REG_CMD, SD_STAT1, 0, 0);
 405
 406         mod_timer(&host->timer, jiffies + msecs_to_jiffies(timeout));
 407
 408         spin_lock_irqsave(&pcr->lock, flags);
 409         pcr->trans_result = TRANS_NOT_READY;
 410         rtsx_pci_send_cmd_no_wait(pcr);
 411         spin_unlock_irqrestore(&pcr->lock, flags);
 412
 413         return;
 414
 415 out:
 416         cmd->error = err;
 417         tasklet_schedule(&host->finish_tasklet);
 418 }
 419
 420 static void sd_get_rsp(unsigned long host_addr)
 421 {
 422         struct realtek_pci_sdmmc *host = (struct realtek_pci_sdmmc *)host_addr;
 423         struct rtsx_pcr *pcr = host->pcr;
 424         struct mmc_command *cmd;
 425         int i, err = 0, stat_idx;
 426         u8 *ptr, rsp_type;
 427         unsigned long flags;
 428
 429         spin_lock_irqsave(&host->lock, flags);
 430
 431         cmd = host->cmd;
 432         host->cmd = NULL;
 433
 434         if (!cmd) {
 435                 dev_err(sdmmc_dev(host), "error: cmd not exist\n");
 436                 goto out;
 437         }
 438
 439         spin_lock(&pcr->lock);
 440         if (pcr->trans_result == TRANS_NO_DEVICE)
 441                 err = -ENODEV;
 442         else if (pcr->trans_result != TRANS_RESULT_OK)
 443                 err = -EINVAL;
 444         spin_unlock(&pcr->lock);
 445
 446         if (err < 0)
 447                 goto out;
 448
 449         rsp_type = host->rsp_type;
 450         stat_idx = host->rsp_len;
 451
 452         if (rsp_type == SD_RSP_TYPE_R0) {
 453                 err = 0;
 454                 goto out;
 455         }
 456
 457         /* Eliminate returned value of CHECK_REG_CMD */
 458         ptr = rtsx_pci_get_cmd_data(pcr) + 1;
 459
 460         /* Check (Start,Transmission) bit of Response */
 461         if ((ptr[0] & 0xC0) != 0) {
 462                 err = -EILSEQ;
 463                 dev_dbg(sdmmc_dev(host), "Invalid response bit\n");
 464                 goto out;
 465         }
 466
 467         /* Check CRC7 */
 468         if (!(rsp_type & SD_NO_CHECK_CRC7)) {
 469                 if (ptr[stat_idx] & SD_CRC7_ERR) {
 470                         err = -EILSEQ;
 471                         dev_dbg(sdmmc_dev(host), "CRC7 error\n");
 472                         goto out;
 473                 }
 474         }
 475
 476         if (rsp_type == SD_RSP_TYPE_R2) {
 477                 for (i = 0; i < 4; i++) {
 478                         cmd->resp[i] = get_unaligned_be32(ptr + 1 + i * 4);
 479                         dev_dbg(sdmmc_dev(host), "cmd->resp[%d] = 0x%08x\n",
 480                                         i, cmd->resp[i]);
 481                 }
 482         } else {
 483                 cmd->resp[0] = get_unaligned_be32(ptr + 1);
 484                 dev_dbg(sdmmc_dev(host), "cmd->resp[0] = 0x%08x\n",
 485                                 cmd->resp[0]);
 486         }
 487
 488         if (cmd == host->mrq->sbc) {
 489                 sd_send_cmd(host, host->mrq->cmd);
 490                 spin_unlock_irqrestore(&host->lock, flags);
 491                 return;
 492         }
 493
 494         if (cmd == host->mrq->stop)
 495                 goto out;
 496
 497         if (cmd->data) {
 498                 sd_start_multi_rw(host, host->mrq);
 499                 spin_unlock_irqrestore(&host->lock, flags);
 500                 return;
 501         }
 502
 503 out:
 504         cmd->error = err;
 505
 506         tasklet_schedule(&host->finish_tasklet);
 507         spin_unlock_irqrestore(&host->lock, flags);
 508 }
 509
 510 static int sd_pre_dma_transfer(struct realtek_pci_sdmmc *host,
 511                         struct mmc_data *data, struct realtek_next *next)
 512 {
 513         struct rtsx_pcr *pcr = host->pcr;
 514         int read = data->flags & MMC_DATA_READ;
 515         int sg_count = 0;
 516
 517         if (!next && data->host_cookie &&
 518                 data->host_cookie != host->next_data.cookie) {
 519                 dev_err(sdmmc_dev(host),
 520                         "error: invalid cookie data[%d] host[%d]\n",
 521                         data->host_cookie, host->next_data.cookie);
 522                 data->host_cookie = 0;
 523         }
 524
 525         if (next || (!next && data->host_cookie != host->next_data.cookie))
 526                 sg_count = rtsx_pci_dma_map_sg(pcr,
 527                                 data->sg, data->sg_len, read);
 528         else
 529                 sg_count = host->next_data.sg_count;
 530
 531         if (next) {
 532                 next->sg_count = sg_count;
 533                 if (++next->cookie < 0)
 534                         next->cookie = 1;
 535                 data->host_cookie = next->cookie;
 536         }
 537
 538         return sg_count;
 539 }
 540
 541 static void sdmmc_pre_req(struct mmc_host *mmc, struct mmc_request *mrq,
 542                 bool is_first_req)
 543 {
 544         struct realtek_pci_sdmmc *host = mmc_priv(mmc);
 545         struct mmc_data *data = mrq->data;
 546
 547         if (data->host_cookie) {
 548                 dev_err(sdmmc_dev(host),
 549                         "error: descard already cookie data[%d]\n",
 550                         data->host_cookie);
 551                 data->host_cookie = 0;
 552         }
 553
 554         dev_dbg(sdmmc_dev(host), "dma sg prepared: %d\n",
 555                 sd_pre_dma_transfer(host, data, &host->next_data));
 556 }
 557
 558 static void sdmmc_post_req(struct mmc_host *mmc, struct mmc_request *mrq,
 559                 int err)
 560 {
 561         struct realtek_pci_sdmmc *host = mmc_priv(mmc);
 562         struct rtsx_pcr *pcr = host->pcr;
 563         struct mmc_data *data = mrq->data;
 564         int read = data->flags & MMC_DATA_READ;
 565
 566         rtsx_pci_dma_unmap_sg(pcr, data->sg, data->sg_len, read);
 567         data->host_cookie = 0;
 568 }
 569
 570 static int sd_start_multi_rw(struct realtek_pci_sdmmc *host,
 571                 struct mmc_request *mrq)
 572 {
 573         struct rtsx_pcr *pcr = host->pcr;
 574         struct mmc_host *mmc = host->mmc;
 575         struct mmc_card *card = mmc->card;
 576         struct mmc_data *data = mrq->data;
 577         int uhs = mmc_card_uhs(card);
 578         int read = data->flags & MMC_DATA_READ;
 579         u8 cfg2, trans_mode;
 580         int err;
 581         size_t data_len = data->blksz * data->blocks;
 582
 583         if (host->data)
 584                 dev_err(sdmmc_dev(host), "error: data already exist\n");
 585
 586         host->data = data;
 587
 588         if (read) {
 589                 cfg2 = SD_CALCULATE_CRC7 | SD_CHECK_CRC16 |
 590                         SD_NO_WAIT_BUSY_END | SD_CHECK_CRC7 | SD_RSP_LEN_0;
 591                 trans_mode = SD_TM_AUTO_READ_3;
 592         } else {
 593                 cfg2 = SD_NO_CALCULATE_CRC7 | SD_CHECK_CRC16 |
 594                         SD_NO_WAIT_BUSY_END | SD_NO_CHECK_CRC7 | SD_RSP_LEN_0;
 595                 trans_mode = SD_TM_AUTO_WRITE_3;
 596         }
 597
 598         if (!uhs)
 599                 cfg2 |= SD_NO_CHECK_WAIT_CRC_TO;
 600
 601         rtsx_pci_init_cmd(pcr);
 602
 603         rtsx_pci_add_cmd(pcr, WRITE_REG_CMD, SD_BYTE_CNT_L, 0xFF, 0x00);
 604         rtsx_pci_add_cmd(pcr, WRITE_REG_CMD, SD_BYTE_CNT_H, 0xFF, 0x02);
 605         rtsx_pci_add_cmd(pcr, WRITE_REG_CMD, SD_BLOCK_CNT_L,
 606                         0xFF, (u8)data->blocks);
 607         rtsx_pci_add_cmd(pcr, WRITE_REG_CMD, SD_BLOCK_CNT_H,
 608                         0xFF, (u8)(data->blocks >> 8));
 609
 610         rtsx_pci_add_cmd(pcr, WRITE_REG_CMD, IRQSTAT0,
 611                         DMA_DONE_INT, DMA_DONE_INT);
 612         rtsx_pci_add_cmd(pcr, WRITE_REG_CMD, DMATC3,
 613                         0xFF, (u8)(data_len >> 24));
 614         rtsx_pci_add_cmd(pcr, WRITE_REG_CMD, DMATC2,
 615                         0xFF, (u8)(data_len >> 16));
 616         rtsx_pci_add_cmd(pcr, WRITE_REG_CMD, DMATC1,
 617                         0xFF, (u8)(data_len >> 8));
 618         rtsx_pci_add_cmd(pcr, WRITE_REG_CMD, DMATC0, 0xFF, (u8)data_len);
 619         if (read) {
 620                 rtsx_pci_add_cmd(pcr, WRITE_REG_CMD, DMACTL,
 621                                 0x03 | DMA_PACK_SIZE_MASK,
 622                                 DMA_DIR_FROM_CARD | DMA_EN | DMA_512);
 623         } else {
 624                 rtsx_pci_add_cmd(pcr, WRITE_REG_CMD, DMACTL,
 625                                 0x03 | DMA_PACK_SIZE_MASK,
 626                                 DMA_DIR_TO_CARD | DMA_EN | DMA_512);
 627         }
 628
 629         rtsx_pci_add_cmd(pcr, WRITE_REG_CMD, CARD_DATA_SOURCE,
 630                         0x01, RING_BUFFER);
 631
 632         rtsx_pci_add_cmd(pcr, WRITE_REG_CMD, SD_CFG2, 0xFF, cfg2);
 633         rtsx_pci_add_cmd(pcr, WRITE_REG_CMD, SD_TRANSFER, 0xFF,
 634                         trans_mode | SD_TRANSFER_START);
 635         rtsx_pci_add_cmd(pcr, CHECK_REG_CMD, SD_TRANSFER,
 636                         SD_TRANSFER_END, SD_TRANSFER_END);
 637
 638         mod_timer(&host->timer, jiffies + 10 * HZ);
 639         rtsx_pci_send_cmd_no_wait(pcr);
 640
 641         err = rtsx_pci_dma_transfer(pcr, data->sg, host->sg_count, read);
 642         if (err < 0) {
 643                 data->error = err;
 644                 tasklet_schedule(&host->finish_tasklet);
 645         }
 646         return 0;
 647 }
 648
 649 static void sd_finish_multi_rw(unsigned long host_addr)
 650 {
 651         struct realtek_pci_sdmmc *host = (struct realtek_pci_sdmmc *)host_addr;
 652         struct rtsx_pcr *pcr = host->pcr;
 653         struct mmc_data *data;
 654         int err = 0;
 655         unsigned long flags;
 656
 657         spin_lock_irqsave(&host->lock, flags);
 658
 659         if (!host->data) {
 660                 dev_err(sdmmc_dev(host), "error: no data exist\n");
 661                 goto out;
 662         }
 663
 664         data = host->data;
 665         host->data = NULL;
 666
 667         if (pcr->trans_result == TRANS_NO_DEVICE)
 668                 err = -ENODEV;
 669         else if (pcr->trans_result != TRANS_RESULT_OK)
 670                 err = -EINVAL;
 671
 672         if (err < 0) {
 673                 data->error = err;
 674                 goto out;
 675         }
 676
 677         if (!host->mrq->sbc && data->stop) {
 678                 sd_send_cmd(host, data->stop);
 679                 spin_unlock_irqrestore(&host->lock, flags);
 680                 return;
 681         }
 682
 683 out:
 684         tasklet_schedule(&host->finish_tasklet);
 685         spin_unlock_irqrestore(&host->lock, flags);
 686 }
 687
 688 static inline void sd_enable_initial_mode(struct realtek_pci_sdmmc *host)
 689 {
 690         rtsx_pci_write_register(host->pcr, SD_CFG1,
 691                         SD_CLK_DIVIDE_MASK, SD_CLK_DIVIDE_128);
 692 }
 693
 694 static inline void sd_disable_initial_mode(struct realtek_pci_sdmmc *host)
 695 {
 696         rtsx_pci_write_register(host->pcr, SD_CFG1,
 697                         SD_CLK_DIVIDE_MASK, SD_CLK_DIVIDE_0);
 698 }
 699
 700 static void sd_normal_rw(struct realtek_pci_sdmmc *host,
 701                 struct mmc_request *mrq)
 702 {
 703         struct mmc_command *cmd = mrq->cmd;
 704         struct mmc_data *data = mrq->data;
 705         u8 _cmd[5], *buf;
 706
 707         _cmd[0] = 0x40 | (u8)cmd->opcode;
 708         put_unaligned_be32(cmd->arg, (u32 *)(&_cmd[1]));
 709
 710         buf = kzalloc(data->blksz, GFP_NOIO);
 711         if (!buf) {
 712                 cmd->error = -ENOMEM;
 713                 return;
 714         }
 715
 716         if (data->flags & MMC_DATA_READ) {
 717                 if (host->initial_mode)
 718                         sd_disable_initial_mode(host);
 719
 720                 cmd->error = sd_read_data(host, _cmd, (u16)data->blksz, buf,
 721                                 data->blksz, 200);
 722
 723                 if (host->initial_mode)
 724                         sd_enable_initial_mode(host);
 725
 726                 sg_copy_from_buffer(data->sg, data->sg_len, buf, data->blksz);
 727         } else {
 728                 sg_copy_to_buffer(data->sg, data->sg_len, buf, data->blksz);
 729
 730                 cmd->error = sd_write_data(host, _cmd, (u16)data->blksz, buf,
 731                                 data->blksz, 200);
 732         }
 733
 734         kfree(buf);
 735 }
 736
 737 static int sd_change_phase(struct realtek_pci_sdmmc *host,
 738                 u8 sample_point, bool rx)
 739 {
 740         struct rtsx_pcr *pcr = host->pcr;
 741         int err;
 742
 743         dev_dbg(sdmmc_dev(host), "%s(%s): sample_point = %d\n",
 744                         __func__, rx ? "RX" : "TX", sample_point);
 745
 746         rtsx_pci_init_cmd(pcr);
 747
 748         rtsx_pci_add_cmd(pcr, WRITE_REG_CMD, CLK_CTL, CHANGE_CLK, CHANGE_CLK);
 749         if (rx)
 750                 rtsx_pci_add_cmd(pcr, WRITE_REG_CMD,
 751                                 SD_VPRX_CTL, 0x1F, sample_point);
 752         else
 753                 rtsx_pci_add_cmd(pcr, WRITE_REG_CMD,
 754                                 SD_VPTX_CTL, 0x1F, sample_point);
 755         rtsx_pci_add_cmd(pcr, WRITE_REG_CMD, SD_VPCLK0_CTL, PHASE_NOT_RESET, 0);
 756         rtsx_pci_add_cmd(pcr, WRITE_REG_CMD, SD_VPCLK0_CTL,
 757                         PHASE_NOT_RESET, PHASE_NOT_RESET);
 758         rtsx_pci_add_cmd(pcr, WRITE_REG_CMD, CLK_CTL, CHANGE_CLK, 0);
 759         rtsx_pci_add_cmd(pcr, WRITE_REG_CMD, SD_CFG1, SD_ASYNC_FIFO_NOT_RST, 0);
 760
 761         err = rtsx_pci_send_cmd(pcr, 100);
 762         if (err < 0)
 763                 return err;
 764
 765         return 0;
 766 }
 767
 768 static inline u32 test_phase_bit(u32 phase_map, unsigned int bit)
 769 {
 770         bit %= RTSX_PHASE_MAX;
 771         return phase_map & (1 << bit);
 772 }
 773
 774 static int sd_get_phase_len(u32 phase_map, unsigned int start_bit)
 775 {
 776         int i;
 777
 778         for (i = 0; i < RTSX_PHASE_MAX; i++) {
 779                 if (test_phase_bit(phase_map, start_bit + i) == 0)
 780                         return i;
 781         }
 782         return RTSX_PHASE_MAX;
 783 }
 784
 785 static u8 sd_search_final_phase(struct realtek_pci_sdmmc *host, u32 phase_map)
 786 {
 787         int start = 0, len = 0;
 788         int start_final = 0, len_final = 0;
 789         u8 final_phase = 0xFF;
 790
 791         if (phase_map == 0) {
 792                 dev_err(sdmmc_dev(host), "phase error: [map:%x]\n", phase_map);
 793                 return final_phase;
 794         }
 795
 796         while (start < RTSX_PHASE_MAX) {
 797                 len = sd_get_phase_len(phase_map, start);
 798                 if (len_final < len) {
 799                         start_final = start;
 800                         len_final = len;
 801                 }
 802                 start += len ? len : 1;
 803         }
 804
 805         final_phase = (start_final + len_final / 2) % RTSX_PHASE_MAX;
 806         dev_dbg(sdmmc_dev(host), "phase: [map:%x] [maxlen:%d] [final:%d]\n",
 807                 phase_map, len_final, final_phase);
 808
 809         return final_phase;
 810 }
 811
 812 static void sd_wait_data_idle(struct realtek_pci_sdmmc *host)
 813 {
 814         int err, i;
 815         u8 val = 0;
 816
 817         for (i = 0; i < 100; i++) {
 818                 err = rtsx_pci_read_register(host->pcr, SD_DATA_STATE, &val);
 819                 if (val & SD_DATA_IDLE)
 820                         return;
 821
 822                 udelay(100);
 823         }
 824 }
 825
 826 static int sd_tuning_rx_cmd(struct realtek_pci_sdmmc *host,
 827                 u8 opcode, u8 sample_point)
 828 {
 829         int err;
 830         u8 cmd[5] = {0};
 831
 832         err = sd_change_phase(host, sample_point, true);
 833         if (err < 0)
 834                 return err;
 835
 836         cmd[0] = 0x40 | opcode;
 837         err = sd_read_data(host, cmd, 0x40, NULL, 0, 100);
 838         if (err < 0) {
 839                 /* Wait till SD DATA IDLE */
 840                 sd_wait_data_idle(host);
 841                 sd_clear_error(host);
 842                 return err;
 843         }
 844
 845         return 0;
 846 }
 847
 848 static int sd_tuning_phase(struct realtek_pci_sdmmc *host,
 849                 u8 opcode, u32 *phase_map)
 850 {
 851         int err, i;
 852         u32 raw_phase_map = 0;
 853
 854         for (i = 0; i < RTSX_PHASE_MAX; i++) {
 855                 err = sd_tuning_rx_cmd(host, opcode, (u8)i);
 856                 if (err == 0)
 857                         raw_phase_map |= 1 << i;
 858         }
 859
 860         if (phase_map)
 861                 *phase_map = raw_phase_map;
 862
 863         return 0;
 864 }
 865
 866 static int sd_tuning_rx(struct realtek_pci_sdmmc *host, u8 opcode)
 867 {
 868         int err, i;
 869         u32 raw_phase_map[RX_TUNING_CNT] = {0}, phase_map;
 870         u8 final_phase;
 871
 872         for (i = 0; i < RX_TUNING_CNT; i++) {
 873                 err = sd_tuning_phase(host, opcode, &(raw_phase_map[i]));
 874                 if (err < 0)
 875                         return err;
 876
 877                 if (raw_phase_map[i] == 0)
 878                         break;
 879         }
 880
 881         phase_map = 0xFFFFFFFF;
 882         for (i = 0; i < RX_TUNING_CNT; i++) {
 883                 dev_dbg(sdmmc_dev(host), "RX raw_phase_map[%d] = 0x%08x\n",
 884                                 i, raw_phase_map[i]);
 885                 phase_map &= raw_phase_map[i];
 886         }
 887         dev_dbg(sdmmc_dev(host), "RX phase_map = 0x%08x\n", phase_map);
 888
 889         if (phase_map) {
 890                 final_phase = sd_search_final_phase(host, phase_map);
 891                 if (final_phase == 0xFF)
 892                         return -EINVAL;
 893
 894                 err = sd_change_phase(host, final_phase, true);
 895                 if (err < 0)
 896                         return err;
 897         } else {
 898                 return -EINVAL;
 899         }
 900
 901         return 0;
 902 }
 903
 904 static inline bool sd_use_muti_rw(struct mmc_command *cmd)
 905 {
 906         return mmc_op_multi(cmd->opcode) ||
 907                 (cmd->opcode == MMC_READ_SINGLE_BLOCK) ||
 908                 (cmd->opcode == MMC_WRITE_BLOCK);
 909 }
 910
 911 static void sdmmc_request(struct mmc_host *mmc, struct mmc_request *mrq)
 912 {
 913         struct realtek_pci_sdmmc *host = mmc_priv(mmc);
 914         struct rtsx_pcr *pcr = host->pcr;
 915         struct mmc_command *cmd = mrq->cmd;
 916         struct mmc_data *data = mrq->data;
 917         unsigned int data_size = 0;
 918         int err;
 919         unsigned long flags;
 920
 921         mutex_lock(&pcr->pcr_mutex);
 922         spin_lock_irqsave(&host->lock, flags);
 923
 924         if (host->mrq)
 925                 dev_err(sdmmc_dev(host), "error: request already exist\n");
 926         host->mrq = mrq;
 927
 928         if (host->eject) {
 929                 cmd->error = -ENOMEDIUM;
 930                 goto finish;
 931         }
 932
 933         err = rtsx_pci_card_exclusive_check(host->pcr, RTSX_SD_CARD);
 934         if (err) {
 935                 cmd->error = err;
 936                 goto finish;
 937         }
 938
 939         rtsx_pci_start_run(pcr);
 940
 941         rtsx_pci_switch_clock(pcr, host->clock, host->ssc_depth,
 942                         host->initial_mode, host->double_clk, host->vpclk);
 943         rtsx_pci_write_register(pcr, CARD_SELECT, 0x07, SD_MOD_SEL);
 944         rtsx_pci_write_register(pcr, CARD_SHARE_MODE,
 945                         CARD_SHARE_MASK, CARD_SHARE_48_SD);
 946
 947         if (mrq->data)
 948                 data_size = data->blocks * data->blksz;
 949
 950         if (sd_use_muti_rw(cmd))
 951                 host->sg_count = sd_pre_dma_transfer(host, data, NULL);
 952
 953         if (!data_size || sd_use_muti_rw(cmd)) {
 954                 if (mrq->sbc)
 955                         sd_send_cmd(host, mrq->sbc);
 956                 else
 957                         sd_send_cmd(host, cmd);
 958                 spin_unlock_irqrestore(&host->lock, flags);
 959         } else {
 960                 spin_unlock_irqrestore(&host->lock, flags);
 961                 sd_normal_rw(host, mrq);
 962                 tasklet_schedule(&host->finish_tasklet);
 963         }
 964         return;
 965
 966 finish:
 967         tasklet_schedule(&host->finish_tasklet);
 968         spin_unlock_irqrestore(&host->lock, flags);
 969 }
 970
 971 static int sd_set_bus_width(struct realtek_pci_sdmmc *host,
 972                 unsigned char bus_width)
 973 {
 974         int err = 0;
 975         u8 width[] = {
 976                 [MMC_BUS_WIDTH_1] = SD_BUS_WIDTH_1BIT,
 977                 [MMC_BUS_WIDTH_4] = SD_BUS_WIDTH_4BIT,
 978                 [MMC_BUS_WIDTH_8] = SD_BUS_WIDTH_8BIT,
 979         };
 980
 981         if (bus_width <= MMC_BUS_WIDTH_8)
 982                 err = rtsx_pci_write_register(host->pcr, SD_CFG1,
 983                                 0x03, width[bus_width]);
 984
 985         return err;
 986 }
 987
 988 static int sd_power_on(struct realtek_pci_sdmmc *host)
 989 {
 990         struct rtsx_pcr *pcr = host->pcr;
 991         int err;
 992
 993         if (host->power_state == SDMMC_POWER_ON)
 994                 return 0;
 995
 996         rtsx_pci_init_cmd(pcr);
 997         rtsx_pci_add_cmd(pcr, WRITE_REG_CMD, CARD_SELECT, 0x07, SD_MOD_SEL);
 998         rtsx_pci_add_cmd(pcr, WRITE_REG_CMD, CARD_SHARE_MODE,
 999                         CARD_SHARE_MASK, CARD_SHARE_48_SD);
1000         rtsx_pci_add_cmd(pcr, WRITE_REG_CMD, CARD_CLK_EN,
1001                         SD_CLK_EN, SD_CLK_EN);
1002         err = rtsx_pci_send_cmd(pcr, 100);
1003         if (err < 0)
1004                 return err;
1005
1006         err = rtsx_pci_card_pull_ctl_enable(pcr, RTSX_SD_CARD);
1007         if (err < 0)
1008                 return err;
1009
1010         err = rtsx_pci_card_power_on(pcr, RTSX_SD_CARD);
1011         if (err < 0)
1012                 return err;
1013
1014         err = rtsx_pci_write_register(pcr, CARD_OE, SD_OUTPUT_EN, SD_OUTPUT_EN);
1015         if (err < 0)
1016                 return err;
1017
1018         host->power_state = SDMMC_POWER_ON;
1019         return 0;
1020 }
1021
1022 static int sd_power_off(struct realtek_pci_sdmmc *host)
1023 {
1024         struct rtsx_pcr *pcr = host->pcr;
1025         int err;
1026
1027         host->power_state = SDMMC_POWER_OFF;
1028
1029         rtsx_pci_init_cmd(pcr);
1030
1031         rtsx_pci_add_cmd(pcr, WRITE_REG_CMD, CARD_CLK_EN, SD_CLK_EN, 0);
1032         rtsx_pci_add_cmd(pcr, WRITE_REG_CMD, CARD_OE, SD_OUTPUT_EN, 0);
1033
1034         err = rtsx_pci_send_cmd(pcr, 100);
1035         if (err < 0)
1036                 return err;
1037
1038         err = rtsx_pci_card_power_off(pcr, RTSX_SD_CARD);
1039         if (err < 0)
1040                 return err;
1041
1042         return rtsx_pci_card_pull_ctl_disable(pcr, RTSX_SD_CARD);
1043 }
1044
1045 static int sd_set_power_mode(struct realtek_pci_sdmmc *host,
1046                 unsigned char power_mode)
1047 {
1048         int err;
1049
1050         if (power_mode == MMC_POWER_OFF)
1051                 err = sd_power_off(host);
1052         else
1053                 err = sd_power_on(host);
1054
1055         return err;
1056 }
1057
1058 static int sd_set_timing(struct realtek_pci_sdmmc *host, unsigned char timing)
1059 {
1060         struct rtsx_pcr *pcr = host->pcr;
1061         int err = 0;
1062
1063         rtsx_pci_init_cmd(pcr);
1064
1065         switch (timing) {
1066         case MMC_TIMING_UHS_SDR104:
1067         case MMC_TIMING_UHS_SDR50:
1068                 rtsx_pci_add_cmd(pcr, WRITE_REG_CMD, SD_CFG1,
1069                                 0x0C | SD_ASYNC_FIFO_NOT_RST,
1070                                 SD_30_MODE | SD_ASYNC_FIFO_NOT_RST);
1071                 rtsx_pci_add_cmd(pcr, WRITE_REG_CMD, CLK_CTL,
1072                                 CLK_LOW_FREQ, CLK_LOW_FREQ);
1073                 rtsx_pci_add_cmd(pcr, WRITE_REG_CMD, CARD_CLK_SOURCE, 0xFF,
1074                                 CRC_VAR_CLK0 | SD30_FIX_CLK | SAMPLE_VAR_CLK1);
1075                 rtsx_pci_add_cmd(pcr, WRITE_REG_CMD, CLK_CTL, CLK_LOW_FREQ, 0);
1076                 break;
1077
1078         case MMC_TIMING_UHS_DDR50:
1079                 rtsx_pci_add_cmd(pcr, WRITE_REG_CMD, SD_CFG1,
1080                                 0x0C | SD_ASYNC_FIFO_NOT_RST,
1081                                 SD_DDR_MODE | SD_ASYNC_FIFO_NOT_RST);
1082                 rtsx_pci_add_cmd(pcr, WRITE_REG_CMD, CLK_CTL,
1083                                 CLK_LOW_FREQ, CLK_LOW_FREQ);
1084                 rtsx_pci_add_cmd(pcr, WRITE_REG_CMD, CARD_CLK_SOURCE, 0xFF,
1085                                 CRC_VAR_CLK0 | SD30_FIX_CLK | SAMPLE_VAR_CLK1);
1086                 rtsx_pci_add_cmd(pcr, WRITE_REG_CMD, CLK_CTL, CLK_LOW_FREQ, 0);
1087                 rtsx_pci_add_cmd(pcr, WRITE_REG_CMD, SD_PUSH_POINT_CTL,
1088                                 DDR_VAR_TX_CMD_DAT, DDR_VAR_TX_CMD_DAT);
1089                 rtsx_pci_add_cmd(pcr, WRITE_REG_CMD, SD_SAMPLE_POINT_CTL,
1090                                 DDR_VAR_RX_DAT | DDR_VAR_RX_CMD,
1091                                 DDR_VAR_RX_DAT | DDR_VAR_RX_CMD);
1092                 break;
1093
1094         case MMC_TIMING_MMC_HS:
1095         case MMC_TIMING_SD_HS:
1096                 rtsx_pci_add_cmd(pcr, WRITE_REG_CMD, SD_CFG1,
1097                                 0x0C, SD_20_MODE);
1098                 rtsx_pci_add_cmd(pcr, WRITE_REG_CMD, CLK_CTL,
1099                                 CLK_LOW_FREQ, CLK_LOW_FREQ);
1100                 rtsx_pci_add_cmd(pcr, WRITE_REG_CMD, CARD_CLK_SOURCE, 0xFF,
1101                                 CRC_FIX_CLK | SD30_VAR_CLK0 | SAMPLE_VAR_CLK1);
1102                 rtsx_pci_add_cmd(pcr, WRITE_REG_CMD, CLK_CTL, CLK_LOW_FREQ, 0);
1103                 rtsx_pci_add_cmd(pcr, WRITE_REG_CMD, SD_PUSH_POINT_CTL,
1104                                 SD20_TX_SEL_MASK, SD20_TX_14_AHEAD);
1105                 rtsx_pci_add_cmd(pcr, WRITE_REG_CMD, SD_SAMPLE_POINT_CTL,
1106                                 SD20_RX_SEL_MASK, SD20_RX_14_DELAY);
1107                 break;
1108
1109         default:
1110                 rtsx_pci_add_cmd(pcr, WRITE_REG_CMD,
1111                                 SD_CFG1, 0x0C, SD_20_MODE);
1112                 rtsx_pci_add_cmd(pcr, WRITE_REG_CMD, CLK_CTL,
1113                                 CLK_LOW_FREQ, CLK_LOW_FREQ);
1114                 rtsx_pci_add_cmd(pcr, WRITE_REG_CMD, CARD_CLK_SOURCE, 0xFF,
1115                                 CRC_FIX_CLK | SD30_VAR_CLK0 | SAMPLE_VAR_CLK1);
1116                 rtsx_pci_add_cmd(pcr, WRITE_REG_CMD, CLK_CTL, CLK_LOW_FREQ, 0);
1117                 rtsx_pci_add_cmd(pcr, WRITE_REG_CMD,
1118                                 SD_PUSH_POINT_CTL, 0xFF, 0);
1119                 rtsx_pci_add_cmd(pcr, WRITE_REG_CMD, SD_SAMPLE_POINT_CTL,
1120                                 SD20_RX_SEL_MASK, SD20_RX_POS_EDGE);
1121                 break;
1122         }
1123
1124         err = rtsx_pci_send_cmd(pcr, 100);
1125
1126         return err;
1127 }
1128
1129 static void sdmmc_set_ios(struct mmc_host *mmc, struct mmc_ios *ios)
1130 {
1131         struct realtek_pci_sdmmc *host = mmc_priv(mmc);
1132         struct rtsx_pcr *pcr = host->pcr;
1133
1134         if (host->eject)
1135                 return;
1136
1137         if (rtsx_pci_card_exclusive_check(host->pcr, RTSX_SD_CARD))
1138                 return;
1139
1140         mutex_lock(&pcr->pcr_mutex);
1141
1142         rtsx_pci_start_run(pcr);
1143
1144         sd_set_bus_width(host, ios->bus_width);
1145         sd_set_power_mode(host, ios->power_mode);
1146         sd_set_timing(host, ios->timing);
1147
1148         host->vpclk = false;
1149         host->double_clk = true;
1150
1151         switch (ios->timing) {
1152         case MMC_TIMING_UHS_SDR104:
1153         case MMC_TIMING_UHS_SDR50:
1154                 host->ssc_depth = RTSX_SSC_DEPTH_2M;
1155                 host->vpclk = true;
1156                 host->double_clk = false;
1157                 break;
1158         case MMC_TIMING_UHS_DDR50:
1159         case MMC_TIMING_UHS_SDR25:
1160                 host->ssc_depth = RTSX_SSC_DEPTH_1M;
1161                 break;
1162         default:
1163                 host->ssc_depth = RTSX_SSC_DEPTH_500K;
1164                 break;
1165         }
1166
1167         host->initial_mode = (ios->clock <= 1000000) ? true : false;
1168
1169         host->clock = ios->clock;
1170         rtsx_pci_switch_clock(pcr, ios->clock, host->ssc_depth,
1171                         host->initial_mode, host->double_clk, host->vpclk);
1172
1173         mutex_unlock(&pcr->pcr_mutex);
1174 }
1175
1176 static int sdmmc_get_ro(struct mmc_host *mmc)
1177 {
1178         struct realtek_pci_sdmmc *host = mmc_priv(mmc);
1179         struct rtsx_pcr *pcr = host->pcr;
1180         int ro = 0;
1181         u32 val;
1182
1183         if (host->eject)
1184                 return -ENOMEDIUM;
1185
1186         mutex_lock(&pcr->pcr_mutex);
1187
1188         rtsx_pci_start_run(pcr);
1189
1190         /* Check SD mechanical write-protect switch */
1191         val = rtsx_pci_readl(pcr, RTSX_BIPR);
1192         dev_dbg(sdmmc_dev(host), "%s: RTSX_BIPR = 0x%08x\n", __func__, val);
1193         if (val & SD_WRITE_PROTECT)
1194                 ro = 1;
1195
1196         mutex_unlock(&pcr->pcr_mutex);
1197
1198         return ro;
1199 }
1200
1201 static int sdmmc_get_cd(struct mmc_host *mmc)
1202 {
1203         struct realtek_pci_sdmmc *host = mmc_priv(mmc);
1204         struct rtsx_pcr *pcr = host->pcr;
1205         int cd = 0;
1206         u32 val;
1207
1208         if (host->eject)
1209                 return -ENOMEDIUM;
1210
1211         mutex_lock(&pcr->pcr_mutex);
1212
1213         rtsx_pci_start_run(pcr);
1214
1215         /* Check SD card detect */
1216         val = rtsx_pci_card_exist(pcr);
1217         dev_dbg(sdmmc_dev(host), "%s: RTSX_BIPR = 0x%08x\n", __func__, val);
1218         if (val & SD_EXIST)
1219                 cd = 1;
1220
1221         mutex_unlock(&pcr->pcr_mutex);
1222
1223         return cd;
1224 }
1225
1226 static int sd_wait_voltage_stable_1(struct realtek_pci_sdmmc *host)
1227 {
1228         struct rtsx_pcr *pcr = host->pcr;
1229         int err;
1230         u8 stat;
1231
1232         /* Reference to Signal Voltage Switch Sequence in SD spec.
1233          * Wait for a period of time so that the card can drive SD_CMD and
1234          * SD_DAT[3:0] to low after sending back CMD11 response.
1235          */
1236         mdelay(1);
1237
1238         /* SD_CMD, SD_DAT[3:0] should be driven to low by card;
1239          * If either one of SD_CMD,SD_DAT[3:0] is not low,
1240          * abort the voltage switch sequence;
1241          */
1242         err = rtsx_pci_read_register(pcr, SD_BUS_STAT, &stat);
1243         if (err < 0)
1244                 return err;
1245
1246         if (stat & (SD_CMD_STATUS | SD_DAT3_STATUS | SD_DAT2_STATUS |
1247                                 SD_DAT1_STATUS | SD_DAT0_STATUS))
1248                 return -EINVAL;
1249
1250         /* Stop toggle SD clock */
1251         err = rtsx_pci_write_register(pcr, SD_BUS_STAT,
1252                         0xFF, SD_CLK_FORCE_STOP);
1253         if (err < 0)
1254                 return err;
1255
1256         return 0;
1257 }
1258
1259 static int sd_wait_voltage_stable_2(struct realtek_pci_sdmmc *host)
1260 {
1261         struct rtsx_pcr *pcr = host->pcr;
1262         int err;
1263         u8 stat, mask, val;
1264
1265         /* Wait 1.8V output of voltage regulator in card stable */
1266         msleep(50);
1267
1268         /* Toggle SD clock again */
1269         err = rtsx_pci_write_register(pcr, SD_BUS_STAT, 0xFF, SD_CLK_TOGGLE_EN);
1270         if (err < 0)
1271                 return err;
1272
1273         /* Wait for a period of time so that the card can drive
1274          * SD_DAT[3:0] to high at 1.8V
1275          */
1276         msleep(20);
1277
1278         /* SD_CMD, SD_DAT[3:0] should be pulled high by host */
1279         err = rtsx_pci_read_register(pcr, SD_BUS_STAT, &stat);
1280         if (err < 0)
1281                 return err;
1282
1283         mask = SD_CMD_STATUS | SD_DAT3_STATUS | SD_DAT2_STATUS |
1284                 SD_DAT1_STATUS | SD_DAT0_STATUS;
1285         val = SD_CMD_STATUS | SD_DAT3_STATUS | SD_DAT2_STATUS |
1286                 SD_DAT1_STATUS | SD_DAT0_STATUS;
1287         if ((stat & mask) != val) {
1288                 dev_dbg(sdmmc_dev(host),
1289                         "%s: SD_BUS_STAT = 0x%x\n", __func__, stat);
1290                 rtsx_pci_write_register(pcr, SD_BUS_STAT,
1291                                 SD_CLK_TOGGLE_EN | SD_CLK_FORCE_STOP, 0);
1292                 rtsx_pci_write_register(pcr, CARD_CLK_EN, 0xFF, 0);
1293                 return -EINVAL;
1294         }
1295
1296         return 0;
1297 }
1298
1299 static int sdmmc_switch_voltage(struct mmc_host *mmc, struct mmc_ios *ios)
1300 {
1301         struct realtek_pci_sdmmc *host = mmc_priv(mmc);
1302         struct rtsx_pcr *pcr = host->pcr;
1303         int err = 0;
1304         u8 voltage;
1305
1306         dev_dbg(sdmmc_dev(host), "%s: signal_voltage = %d\n",
1307                         __func__, ios->signal_voltage);
1308
1309         if (host->eject)
1310                 return -ENOMEDIUM;
1311
1312         err = rtsx_pci_card_exclusive_check(host->pcr, RTSX_SD_CARD);
1313         if (err)
1314                 return err;
1315
1316         mutex_lock(&pcr->pcr_mutex);
1317
1318         rtsx_pci_start_run(pcr);
1319
1320         if (ios->signal_voltage == MMC_SIGNAL_VOLTAGE_330)
1321                 voltage = OUTPUT_3V3;
1322         else
1323                 voltage = OUTPUT_1V8;
1324
1325         if (voltage == OUTPUT_1V8) {
1326                 err = sd_wait_voltage_stable_1(host);
1327                 if (err < 0)
1328                         goto out;
1329         }
1330
1331         err = rtsx_pci_switch_output_voltage(pcr, voltage);
1332         if (err < 0)
1333                 goto out;
1334
1335         if (voltage == OUTPUT_1V8) {
1336                 err = sd_wait_voltage_stable_2(host);
1337                 if (err < 0)
1338                         goto out;
1339         }
1340
1341 out:
1342         /* Stop toggle SD clock in idle */
1343         err = rtsx_pci_write_register(pcr, SD_BUS_STAT,
1344                         SD_CLK_TOGGLE_EN | SD_CLK_FORCE_STOP, 0);
1345
1346         mutex_unlock(&pcr->pcr_mutex);
1347
1348         return err;
1349 }
1350
1351 static int sdmmc_execute_tuning(struct mmc_host *mmc, u32 opcode)
1352 {
1353         struct realtek_pci_sdmmc *host = mmc_priv(mmc);
1354         struct rtsx_pcr *pcr = host->pcr;
1355         int err = 0;
1356
1357         if (host->eject)
1358                 return -ENOMEDIUM;
1359
1360         err = rtsx_pci_card_exclusive_check(host->pcr, RTSX_SD_CARD);
1361         if (err)
1362                 return err;
1363
1364         mutex_lock(&pcr->pcr_mutex);
1365
1366         rtsx_pci_start_run(pcr);
1367
1368         /* Set initial TX phase */
1369         switch (mmc->ios.timing) {
1370         case MMC_TIMING_UHS_SDR104:
1371                 err = sd_change_phase(host, SDR104_TX_PHASE(pcr), false);
1372                 break;
1373
1374         case MMC_TIMING_UHS_SDR50:
1375                 err = sd_change_phase(host, SDR50_TX_PHASE(pcr), false);
1376                 break;
1377
1378         case MMC_TIMING_UHS_DDR50:
1379                 err = sd_change_phase(host, DDR50_TX_PHASE(pcr), false);
1380                 break;
1381
1382         default:
1383                 err = 0;
1384         }
1385
1386         if (err)
1387                 goto out;
1388
1389         /* Tuning RX phase */
1390         if ((mmc->ios.timing == MMC_TIMING_UHS_SDR104) ||
1391                         (mmc->ios.timing == MMC_TIMING_UHS_SDR50))
1392                 err = sd_tuning_rx(host, opcode);
1393         else if (mmc->ios.timing == MMC_TIMING_UHS_DDR50)
1394                 err = sd_change_phase(host, DDR50_RX_PHASE(pcr), true);
1395
1396 out:
1397         mutex_unlock(&pcr->pcr_mutex);
1398
1399         return err;
1400 }
1401
1402 static const struct mmc_host_ops realtek_pci_sdmmc_ops = {
1403         .pre_req = sdmmc_pre_req,
1404         .post_req = sdmmc_post_req,
1405         .request = sdmmc_request,
1406         .set_ios = sdmmc_set_ios,
1407         .get_ro = sdmmc_get_ro,
1408         .get_cd = sdmmc_get_cd,
1409         .start_signal_voltage_switch = sdmmc_switch_voltage,
1410         .execute_tuning = sdmmc_execute_tuning,
1411 };
1412
1413 static void init_extra_caps(struct realtek_pci_sdmmc *host)
1414 {
1415         struct mmc_host *mmc = host->mmc;
1416         struct rtsx_pcr *pcr = host->pcr;
1417
1418         dev_dbg(sdmmc_dev(host), "pcr->extra_caps = 0x%x\n", pcr->extra_caps);
1419
1420         if (pcr->extra_caps & EXTRA_CAPS_SD_SDR50)
1421                 mmc->caps |= MMC_CAP_UHS_SDR50;
1422         if (pcr->extra_caps & EXTRA_CAPS_SD_SDR104)
1423                 mmc->caps |= MMC_CAP_UHS_SDR104;
1424         if (pcr->extra_caps & EXTRA_CAPS_SD_DDR50)
1425                 mmc->caps |= MMC_CAP_UHS_DDR50;
1426         if (pcr->extra_caps & EXTRA_CAPS_MMC_HSDDR)
1427                 mmc->caps |= MMC_CAP_1_8V_DDR;
1428         if (pcr->extra_caps & EXTRA_CAPS_MMC_8BIT)
1429                 mmc->caps |= MMC_CAP_8_BIT_DATA;
1430 }
1431
1432 static void realtek_init_host(struct realtek_pci_sdmmc *host)
1433 {
1434         struct mmc_host *mmc = host->mmc;
1435
1436         mmc->f_min = 250000;
1437         mmc->f_max = 208000000;
1438         mmc->ocr_avail = MMC_VDD_32_33 | MMC_VDD_33_34 | MMC_VDD_165_195;
1439         mmc->caps = MMC_CAP_4_BIT_DATA | MMC_CAP_SD_HIGHSPEED |
1440                 MMC_CAP_MMC_HIGHSPEED | MMC_CAP_BUS_WIDTH_TEST |
1441                 MMC_CAP_UHS_SDR12 | MMC_CAP_UHS_SDR25;
1442         mmc->max_current_330 = 400;
1443         mmc->max_current_180 = 800;
1444         mmc->ops = &realtek_pci_sdmmc_ops;
1445
1446         init_extra_caps(host);
1447
1448         mmc->max_segs = 256;
1449         mmc->max_seg_size = 65536;
1450         mmc->max_blk_size = 512;
1451         mmc->max_blk_count = 65535;
1452         mmc->max_req_size = 524288;
1453 }
1454
1455 static void rtsx_pci_sdmmc_card_event(struct platform_device *pdev)
1456 {
1457         struct realtek_pci_sdmmc *host = platform_get_drvdata(pdev);
1458
1459         mmc_detect_change(host->mmc, 0);
1460 }
1461
1462 static int rtsx_pci_sdmmc_drv_probe(struct platform_device *pdev)
1463 {
1464         struct mmc_host *mmc;
1465         struct realtek_pci_sdmmc *host;
1466         struct rtsx_pcr *pcr;
1467         struct pcr_handle *handle = pdev->dev.platform_data;
1468         unsigned long host_addr;
1469
1470         if (!handle)
1471                 return -ENXIO;
1472
1473         pcr = handle->pcr;
1474         if (!pcr)
1475                 return -ENXIO;
1476
1477         dev_dbg(&(pdev->dev), ": Realtek PCI-E SDMMC controller found\n");
1478
1479         mmc = mmc_alloc_host(sizeof(*host), &pdev->dev);
1480         if (!mmc)
1481                 return -ENOMEM;
1482
1483         host = mmc_priv(mmc);
1484         host->pcr = pcr;
1485         host->mmc = mmc;
1486         host->pdev = pdev;
1487         host->power_state = SDMMC_POWER_OFF;
1488         platform_set_drvdata(pdev, host);
1489         pcr->slots[RTSX_SD_CARD].p_dev = pdev;
1490         pcr->slots[RTSX_SD_CARD].card_event = rtsx_pci_sdmmc_card_event;
1491
1492         host_addr = (unsigned long)host;
1493         host->next_data.cookie = 1;
1494         setup_timer(&host->timer, sd_request_timeout, host_addr);
1495         tasklet_init(&host->cmd_tasklet, sd_get_rsp, host_addr);
1496         tasklet_init(&host->data_tasklet, sd_finish_multi_rw, host_addr);
1497         tasklet_init(&host->finish_tasklet, sd_finish_request, host_addr);
1498         spin_lock_init(&host->lock);
1499
1500         pcr->slots[RTSX_SD_CARD].done_transfer = sd_isr_done_transfer;
1501         realtek_init_host(host);
1502
1503         mmc_add_host(mmc);
1504
1505         return 0;
1506 }
1507
1508 static int rtsx_pci_sdmmc_drv_remove(struct platform_device *pdev)
1509 {
1510         struct realtek_pci_sdmmc *host = platform_get_drvdata(pdev);
1511         struct rtsx_pcr *pcr;
1512         struct mmc_host *mmc;
1513         struct mmc_request *mrq;
1514         unsigned long flags;
1515
1516         if (!host)
1517                 return 0;
1518
1519         pcr = host->pcr;
1520         pcr->slots[RTSX_SD_CARD].p_dev = NULL;
1521         pcr->slots[RTSX_SD_CARD].card_event = NULL;
1522         pcr->slots[RTSX_SD_CARD].done_transfer = NULL;
1523         mmc = host->mmc;
1524         mrq = host->mrq;
1525
1526         spin_lock_irqsave(&host->lock, flags);
1527         if (host->mrq) {
1528                 dev_dbg(&(pdev->dev),
1529                         "%s: Controller removed during transfer\n",
1530                         mmc_hostname(mmc));
1531
1532                 if (mrq->sbc)
1533                         mrq->sbc->error = -ENOMEDIUM;
1534                 if (mrq->cmd)
1535                         mrq->cmd->error = -ENOMEDIUM;
1536                 if (mrq->stop)
1537                         mrq->stop->error = -ENOMEDIUM;
1538                 if (mrq->data)
1539                         mrq->data->error = -ENOMEDIUM;
1540
1541                 tasklet_schedule(&host->finish_tasklet);
1542         }
1543         spin_unlock_irqrestore(&host->lock, flags);
1544
1545         del_timer_sync(&host->timer);
1546         tasklet_kill(&host->cmd_tasklet);
1547         tasklet_kill(&host->data_tasklet);
1548         tasklet_kill(&host->finish_tasklet);
1549
1550         mmc_remove_host(mmc);
1551         host->eject = true;
1552
1553         mmc_free_host(mmc);
1554
1555         dev_dbg(&(pdev->dev),
1556                 ": Realtek PCI-E SDMMC controller has been removed\n");
1557
1558         return 0;
1559 }
1560
1561 static struct platform_device_id rtsx_pci_sdmmc_ids[] = {
1562         {
1563                 .name = DRV_NAME_RTSX_PCI_SDMMC,
1564         }, {
1565                 /* sentinel */
1566         }
1567 };
1568 MODULE_DEVICE_TABLE(platform, rtsx_pci_sdmmc_ids);
1569
1570 static struct platform_driver rtsx_pci_sdmmc_driver = {
1571         .probe          = rtsx_pci_sdmmc_drv_probe,
1572         .remove         = rtsx_pci_sdmmc_drv_remove,
1573         .id_table       = rtsx_pci_sdmmc_ids,
1574         .driver         = {
1575                 .owner  = THIS_MODULE,
1576                 .name   = DRV_NAME_RTSX_PCI_SDMMC,
1577         },
1578 };
1579 module_platform_driver(rtsx_pci_sdmmc_driver);
1580
1581 MODULE_LICENSE("GPL");
1582 MODULE_AUTHOR("Wei WANG <wei_wang@realsil.com.cn>");
1583 MODULE_DESCRIPTION("Realtek PCI-E SD/MMC Card Host Driver");