drivers/memory/tegra/tegra20-emc.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * Tegra20 External Memory Controller driver
   4  *
   5  * Author: Dmitry Osipenko <digetx@gmail.com>
   6  */
   7
   8 #include <linux/clk.h>
   9 #include <linux/clk/tegra.h>
  10 #include <linux/completion.h>
  11 #include <linux/err.h>
  12 #include <linux/interrupt.h>
  13 #include <linux/io.h>
  14 #include <linux/kernel.h>
  15 #include <linux/module.h>
  16 #include <linux/of.h>
  17 #include <linux/platform_device.h>
  18 #include <linux/sort.h>
  19 #include <linux/types.h>
  20
  21 #include <soc/tegra/fuse.h>
  22
  23 #define EMC_INTSTATUS                           0x000
  24 #define EMC_INTMASK                             0x004
  25 #define EMC_DBG                                 0x008
  26 #define EMC_TIMING_CONTROL                      0x028
  27 #define EMC_RC                                  0x02c
  28 #define EMC_RFC                                 0x030
  29 #define EMC_RAS                                 0x034
  30 #define EMC_RP                                  0x038
  31 #define EMC_R2W                                 0x03c
  32 #define EMC_W2R                                 0x040
  33 #define EMC_R2P                                 0x044
  34 #define EMC_W2P                                 0x048
  35 #define EMC_RD_RCD                              0x04c
  36 #define EMC_WR_RCD                              0x050
  37 #define EMC_RRD                                 0x054
  38 #define EMC_REXT                                0x058
  39 #define EMC_WDV                                 0x05c
  40 #define EMC_QUSE                                0x060
  41 #define EMC_QRST                                0x064
  42 #define EMC_QSAFE                               0x068
  43 #define EMC_RDV                                 0x06c
  44 #define EMC_REFRESH                             0x070
  45 #define EMC_BURST_REFRESH_NUM                   0x074
  46 #define EMC_PDEX2WR                             0x078
  47 #define EMC_PDEX2RD                             0x07c
  48 #define EMC_PCHG2PDEN                           0x080
  49 #define EMC_ACT2PDEN                            0x084
  50 #define EMC_AR2PDEN                             0x088
  51 #define EMC_RW2PDEN                             0x08c
  52 #define EMC_TXSR                                0x090
  53 #define EMC_TCKE                                0x094
  54 #define EMC_TFAW                                0x098
  55 #define EMC_TRPAB                               0x09c
  56 #define EMC_TCLKSTABLE                          0x0a0
  57 #define EMC_TCLKSTOP                            0x0a4
  58 #define EMC_TREFBW                              0x0a8
  59 #define EMC_QUSE_EXTRA                          0x0ac
  60 #define EMC_ODT_WRITE                           0x0b0
  61 #define EMC_ODT_READ                            0x0b4
  62 #define EMC_FBIO_CFG5                           0x104
  63 #define EMC_FBIO_CFG6                           0x114
  64 #define EMC_AUTO_CAL_INTERVAL                   0x2a8
  65 #define EMC_CFG_2                               0x2b8
  66 #define EMC_CFG_DIG_DLL                         0x2bc
  67 #define EMC_DLL_XFORM_DQS                       0x2c0
  68 #define EMC_DLL_XFORM_QUSE                      0x2c4
  69 #define EMC_ZCAL_REF_CNT                        0x2e0
  70 #define EMC_ZCAL_WAIT_CNT                       0x2e4
  71 #define EMC_CFG_CLKTRIM_0                       0x2d0
  72 #define EMC_CFG_CLKTRIM_1                       0x2d4
  73 #define EMC_CFG_CLKTRIM_2                       0x2d8
  74
  75 #define EMC_CLKCHANGE_REQ_ENABLE                BIT(0)
  76 #define EMC_CLKCHANGE_PD_ENABLE                 BIT(1)
  77 #define EMC_CLKCHANGE_SR_ENABLE                 BIT(2)
  78
  79 #define EMC_TIMING_UPDATE                       BIT(0)
  80
  81 #define EMC_REFRESH_OVERFLOW_INT                BIT(3)
  82 #define EMC_CLKCHANGE_COMPLETE_INT              BIT(4)
  83
  84 #define EMC_DBG_READ_MUX_ASSEMBLY               BIT(0)
  85 #define EMC_DBG_WRITE_MUX_ACTIVE                BIT(1)
  86 #define EMC_DBG_FORCE_UPDATE                    BIT(2)
  87 #define EMC_DBG_READ_DQM_CTRL                   BIT(9)
  88 #define EMC_DBG_CFG_PRIORITY                    BIT(24)
  89
  90 static const u16 emc_timing_registers[] = {
  91         EMC_RC,
  92         EMC_RFC,
  93         EMC_RAS,
  94         EMC_RP,
  95         EMC_R2W,
  96         EMC_W2R,
  97         EMC_R2P,
  98         EMC_W2P,
  99         EMC_RD_RCD,
 100         EMC_WR_RCD,
 101         EMC_RRD,
 102         EMC_REXT,
 103         EMC_WDV,
 104         EMC_QUSE,
 105         EMC_QRST,
 106         EMC_QSAFE,
 107         EMC_RDV,
 108         EMC_REFRESH,
 109         EMC_BURST_REFRESH_NUM,
 110         EMC_PDEX2WR,
 111         EMC_PDEX2RD,
 112         EMC_PCHG2PDEN,
 113         EMC_ACT2PDEN,
 114         EMC_AR2PDEN,
 115         EMC_RW2PDEN,
 116         EMC_TXSR,
 117         EMC_TCKE,
 118         EMC_TFAW,
 119         EMC_TRPAB,
 120         EMC_TCLKSTABLE,
 121         EMC_TCLKSTOP,
 122         EMC_TREFBW,
 123         EMC_QUSE_EXTRA,
 124         EMC_FBIO_CFG6,
 125         EMC_ODT_WRITE,
 126         EMC_ODT_READ,
 127         EMC_FBIO_CFG5,
 128         EMC_CFG_DIG_DLL,
 129         EMC_DLL_XFORM_DQS,
 130         EMC_DLL_XFORM_QUSE,
 131         EMC_ZCAL_REF_CNT,
 132         EMC_ZCAL_WAIT_CNT,
 133         EMC_AUTO_CAL_INTERVAL,
 134         EMC_CFG_CLKTRIM_0,
 135         EMC_CFG_CLKTRIM_1,
 136         EMC_CFG_CLKTRIM_2,
 137 };
 138
 139 struct emc_timing {
 140         unsigned long rate;
 141         u32 data[ARRAY_SIZE(emc_timing_registers)];
 142 };
 143
 144 struct tegra_emc {
 145         struct device *dev;
 146         struct completion clk_handshake_complete;
 147         struct notifier_block clk_nb;
 148         struct clk *clk;
 149         void __iomem *regs;
 150
 151         struct emc_timing *timings;
 152         unsigned int num_timings;
 153 };
 154
 155 static irqreturn_t tegra_emc_isr(int irq, void *data)
 156 {
 157         struct tegra_emc *emc = data;
 158         u32 intmask = EMC_REFRESH_OVERFLOW_INT | EMC_CLKCHANGE_COMPLETE_INT;
 159         u32 status;
 160
 161         status = readl_relaxed(emc->regs + EMC_INTSTATUS) & intmask;
 162         if (!status)
 163                 return IRQ_NONE;
 164
 165         /* notify about EMC-CAR handshake completion */
 166         if (status & EMC_CLKCHANGE_COMPLETE_INT)
 167                 complete(&emc->clk_handshake_complete);
 168
 169         /* notify about HW problem */
 170         if (status & EMC_REFRESH_OVERFLOW_INT)
 171                 dev_err_ratelimited(emc->dev,
 172                                     "refresh request overflow timeout\n");
 173
 174         /* clear interrupts */
 175         writel_relaxed(status, emc->regs + EMC_INTSTATUS);
 176
 177         return IRQ_HANDLED;
 178 }
 179
 180 static struct emc_timing *tegra_emc_find_timing(struct tegra_emc *emc,
 181                                                 unsigned long rate)
 182 {
 183         struct emc_timing *timing = NULL;
 184         unsigned int i;
 185
 186         for (i = 0; i < emc->num_timings; i++) {
 187                 if (emc->timings[i].rate >= rate) {
 188                         timing = &emc->timings[i];
 189                         break;
 190                 }
 191         }
 192
 193         if (!timing) {
 194                 dev_err(emc->dev, "no timing for rate %lu\n", rate);
 195                 return NULL;
 196         }
 197
 198         return timing;
 199 }
 200
 201 static int emc_prepare_timing_change(struct tegra_emc *emc, unsigned long rate)
 202 {
 203         struct emc_timing *timing = tegra_emc_find_timing(emc, rate);
 204         unsigned int i;
 205
 206         if (!timing)
 207                 return -EINVAL;
 208
 209         dev_dbg(emc->dev, "%s: using timing rate %lu for requested rate %lu\n",
 210                 __func__, timing->rate, rate);
 211
 212         /* program shadow registers */
 213         for (i = 0; i < ARRAY_SIZE(timing->data); i++)
 214                 writel_relaxed(timing->data[i],
 215                                emc->regs + emc_timing_registers[i]);
 216
 217         /* wait until programming has settled */
 218         readl_relaxed(emc->regs + emc_timing_registers[i - 1]);
 219
 220         reinit_completion(&emc->clk_handshake_complete);
 221
 222         return 0;
 223 }
 224
 225 static int emc_complete_timing_change(struct tegra_emc *emc, bool flush)
 226 {
 227         unsigned long timeout;
 228
 229         dev_dbg(emc->dev, "%s: flush %d\n", __func__, flush);
 230
 231         if (flush) {
 232                 /* manually initiate memory timing update */
 233                 writel_relaxed(EMC_TIMING_UPDATE,
 234                                emc->regs + EMC_TIMING_CONTROL);
 235                 return 0;
 236         }
 237
 238         timeout = wait_for_completion_timeout(&emc->clk_handshake_complete,
 239                                               msecs_to_jiffies(100));
 240         if (timeout == 0) {
 241                 dev_err(emc->dev, "EMC-CAR handshake failed\n");
 242                 return -EIO;
 243         }
 244
 245         return 0;
 246 }
 247
 248 static int tegra_emc_clk_change_notify(struct notifier_block *nb,
 249                                        unsigned long msg, void *data)
 250 {
 251         struct tegra_emc *emc = container_of(nb, struct tegra_emc, clk_nb);
 252         struct clk_notifier_data *cnd = data;
 253         int err;
 254
 255         switch (msg) {
 256         case PRE_RATE_CHANGE:
 257                 err = emc_prepare_timing_change(emc, cnd->new_rate);
 258                 break;
 259
 260         case ABORT_RATE_CHANGE:
 261                 err = emc_prepare_timing_change(emc, cnd->old_rate);
 262                 if (err)
 263                         break;
 264
 265                 err = emc_complete_timing_change(emc, true);
 266                 break;
 267
 268         case POST_RATE_CHANGE:
 269                 err = emc_complete_timing_change(emc, false);
 270                 break;
 271
 272         default:
 273                 return NOTIFY_DONE;
 274         }
 275
 276         return notifier_from_errno(err);
 277 }
 278
 279 static int load_one_timing_from_dt(struct tegra_emc *emc,
 280                                    struct emc_timing *timing,
 281                                    struct device_node *node)
 282 {
 283         u32 rate;
 284         int err;
 285
 286         if (!of_device_is_compatible(node, "nvidia,tegra20-emc-table")) {
 287                 dev_err(emc->dev, "incompatible DT node: %pOF\n", node);
 288                 return -EINVAL;
 289         }
 290
 291         err = of_property_read_u32(node, "clock-frequency", &rate);
 292         if (err) {
 293                 dev_err(emc->dev, "timing %pOF: failed to read rate: %d\n",
 294                         node, err);
 295                 return err;
 296         }
 297
 298         err = of_property_read_u32_array(node, "nvidia,emc-registers",
 299                                          timing->data,
 300                                          ARRAY_SIZE(emc_timing_registers));
 301         if (err) {
 302                 dev_err(emc->dev,
 303                         "timing %pOF: failed to read emc timing data: %d\n",
 304                         node, err);
 305                 return err;
 306         }
 307
 308         /*
 309          * The EMC clock rate is twice the bus rate, and the bus rate is
 310          * measured in kHz.
 311          */
 312         timing->rate = rate * 2 * 1000;
 313
 314         dev_dbg(emc->dev, "%s: %pOF: EMC rate %lu\n",
 315                 __func__, node, timing->rate);
 316
 317         return 0;
 318 }
 319
 320 static int cmp_timings(const void *_a, const void *_b)
 321 {
 322         const struct emc_timing *a = _a;
 323         const struct emc_timing *b = _b;
 324
 325         if (a->rate < b->rate)
 326                 return -1;
 327
 328         if (a->rate > b->rate)
 329                 return 1;
 330
 331         return 0;
 332 }
 333
 334 static int tegra_emc_load_timings_from_dt(struct tegra_emc *emc,
 335                                           struct device_node *node)
 336 {
 337         struct device_node *child;
 338         struct emc_timing *timing;
 339         int child_count;
 340         int err;
 341
 342         child_count = of_get_child_count(node);
 343         if (!child_count) {
 344                 dev_err(emc->dev, "no memory timings in DT node: %pOF\n", node);
 345                 return -EINVAL;
 346         }
 347
 348         emc->timings = devm_kcalloc(emc->dev, child_count, sizeof(*timing),
 349                                     GFP_KERNEL);
 350         if (!emc->timings)
 351                 return -ENOMEM;
 352
 353         emc->num_timings = child_count;
 354         timing = emc->timings;
 355
 356         for_each_child_of_node(node, child) {
 357                 err = load_one_timing_from_dt(emc, timing++, child);
 358                 if (err) {
 359                         of_node_put(child);
 360                         return err;
 361                 }
 362         }
 363
 364         sort(emc->timings, emc->num_timings, sizeof(*timing), cmp_timings,
 365              NULL);
 366
 367         dev_info(emc->dev,
 368                  "got %u timings for RAM code %u (min %luMHz max %luMHz)\n",
 369                  emc->num_timings,
 370                  tegra_read_ram_code(),
 371                  emc->timings[0].rate / 1000000,
 372                  emc->timings[emc->num_timings - 1].rate / 1000000);
 373
 374         return 0;
 375 }
 376
 377 static struct device_node *
 378 tegra_emc_find_node_by_ram_code(struct device *dev)
 379 {
 380         struct device_node *np;
 381         u32 value, ram_code;
 382         int err;
 383
 384         if (!of_property_read_bool(dev->of_node, "nvidia,use-ram-code"))
 385                 return of_node_get(dev->of_node);
 386
 387         ram_code = tegra_read_ram_code();
 388
 389         for (np = of_find_node_by_name(dev->of_node, "emc-tables"); np;
 390              np = of_find_node_by_name(np, "emc-tables")) {
 391                 err = of_property_read_u32(np, "nvidia,ram-code", &value);
 392                 if (err || value != ram_code) {
 393                         of_node_put(np);
 394                         continue;
 395                 }
 396
 397                 return np;
 398         }
 399
 400         dev_err(dev, "no memory timings for RAM code %u found in device tree\n",
 401                 ram_code);
 402
 403         return NULL;
 404 }
 405
 406 static int emc_setup_hw(struct tegra_emc *emc)
 407 {
 408         u32 intmask = EMC_REFRESH_OVERFLOW_INT | EMC_CLKCHANGE_COMPLETE_INT;
 409         u32 emc_cfg, emc_dbg;
 410
 411         emc_cfg = readl_relaxed(emc->regs + EMC_CFG_2);
 412
 413         /*
 414          * Depending on a memory type, DRAM should enter either self-refresh
 415          * or power-down state on EMC clock change.
 416          */
 417         if (!(emc_cfg & EMC_CLKCHANGE_PD_ENABLE) &&
 418             !(emc_cfg & EMC_CLKCHANGE_SR_ENABLE)) {
 419                 dev_err(emc->dev,
 420                         "bootloader didn't specify DRAM auto-suspend mode\n");
 421                 return -EINVAL;
 422         }
 423
 424         /* enable EMC and CAR to handshake on PLL divider/source changes */
 425         emc_cfg |= EMC_CLKCHANGE_REQ_ENABLE;
 426         writel_relaxed(emc_cfg, emc->regs + EMC_CFG_2);
 427
 428         /* initialize interrupt */
 429         writel_relaxed(intmask, emc->regs + EMC_INTMASK);
 430         writel_relaxed(intmask, emc->regs + EMC_INTSTATUS);
 431
 432         /* ensure that unwanted debug features are disabled */
 433         emc_dbg = readl_relaxed(emc->regs + EMC_DBG);
 434         emc_dbg |= EMC_DBG_CFG_PRIORITY;
 435         emc_dbg &= ~EMC_DBG_READ_MUX_ASSEMBLY;
 436         emc_dbg &= ~EMC_DBG_WRITE_MUX_ACTIVE;
 437         emc_dbg &= ~EMC_DBG_FORCE_UPDATE;
 438         writel_relaxed(emc_dbg, emc->regs + EMC_DBG);
 439
 440         return 0;
 441 }
 442
 443 static long emc_round_rate(unsigned long rate,
 444                            unsigned long min_rate,
 445                            unsigned long max_rate,
 446                            void *arg)
 447 {
 448         struct emc_timing *timing = NULL;
 449         struct tegra_emc *emc = arg;
 450         unsigned int i;
 451
 452         min_rate = min(min_rate, emc->timings[emc->num_timings - 1].rate);
 453
 454         for (i = 0; i < emc->num_timings; i++) {
 455                 if (emc->timings[i].rate < rate && i != emc->num_timings - 1)
 456                         continue;
 457
 458                 if (emc->timings[i].rate > max_rate) {
 459                         i = max(i, 1u) - 1;
 460
 461                         if (emc->timings[i].rate < min_rate)
 462                                 break;
 463                 }
 464
 465                 if (emc->timings[i].rate < min_rate)
 466                         continue;
 467
 468                 timing = &emc->timings[i];
 469                 break;
 470         }
 471
 472         if (!timing) {
 473                 dev_err(emc->dev, "no timing for rate %lu min %lu max %lu\n",
 474                         rate, min_rate, max_rate);
 475                 return -EINVAL;
 476         }
 477
 478         return timing->rate;
 479 }
 480
 481 static int tegra_emc_probe(struct platform_device *pdev)
 482 {
 483         struct device_node *np;
 484         struct tegra_emc *emc;
 485         struct resource *res;
 486         int irq, err;
 487
 488         /* driver has nothing to do in a case of memory timing absence */
 489         if (of_get_child_count(pdev->dev.of_node) == 0) {
 490                 dev_info(&pdev->dev,
 491                          "EMC device tree node doesn't have memory timings\n");
 492                 return 0;
 493         }
 494
 495         irq = platform_get_irq(pdev, 0);
 496         if (irq < 0) {
 497                 dev_err(&pdev->dev, "interrupt not specified\n");
 498                 dev_err(&pdev->dev, "please update your device tree\n");
 499                 return irq;
 500         }
 501
 502         np = tegra_emc_find_node_by_ram_code(&pdev->dev);
 503         if (!np)
 504                 return -EINVAL;
 505
 506         emc = devm_kzalloc(&pdev->dev, sizeof(*emc), GFP_KERNEL);
 507         if (!emc) {
 508                 of_node_put(np);
 509                 return -ENOMEM;
 510         }
 511
 512         init_completion(&emc->clk_handshake_complete);
 513         emc->clk_nb.notifier_call = tegra_emc_clk_change_notify;
 514         emc->dev = &pdev->dev;
 515
 516         err = tegra_emc_load_timings_from_dt(emc, np);
 517         of_node_put(np);
 518         if (err)
 519                 return err;
 520
 521         res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
 522         emc->regs = devm_ioremap_resource(&pdev->dev, res);
 523         if (IS_ERR(emc->regs))
 524                 return PTR_ERR(emc->regs);
 525
 526         err = emc_setup_hw(emc);
 527         if (err)
 528                 return err;
 529
 530         err = devm_request_irq(&pdev->dev, irq, tegra_emc_isr, 0,
 531                                dev_name(&pdev->dev), emc);
 532         if (err) {
 533                 dev_err(&pdev->dev, "failed to request IRQ#%u: %d\n", irq, err);
 534                 return err;
 535         }
 536
 537         tegra20_clk_set_emc_round_callback(emc_round_rate, emc);
 538
 539         emc->clk = devm_clk_get(&pdev->dev, "emc");
 540         if (IS_ERR(emc->clk)) {
 541                 err = PTR_ERR(emc->clk);
 542                 dev_err(&pdev->dev, "failed to get emc clock: %d\n", err);
 543                 goto unset_cb;
 544         }
 545
 546         err = clk_notifier_register(emc->clk, &emc->clk_nb);
 547         if (err) {
 548                 dev_err(&pdev->dev, "failed to register clk notifier: %d\n",
 549                         err);
 550                 goto unset_cb;
 551         }
 552
 553         return 0;
 554
 555 unset_cb:
 556         tegra20_clk_set_emc_round_callback(NULL, NULL);
 557
 558         return err;
 559 }
 560
 561 static const struct of_device_id tegra_emc_of_match[] = {
 562         { .compatible = "nvidia,tegra20-emc", },
 563         {},
 564 };
 565
 566 static struct platform_driver tegra_emc_driver = {
 567         .probe = tegra_emc_probe,
 568         .driver = {
 569                 .name = "tegra20-emc",
 570                 .of_match_table = tegra_emc_of_match,
 571                 .suppress_bind_attrs = true,
 572         },
 573 };
 574
 575 static int __init tegra_emc_init(void)
 576 {
 577         return platform_driver_register(&tegra_emc_driver);
 578 }
 579 subsys_initcall(tegra_emc_init);