drivers/memory/emif.c

   1 /*
   2  * EMIF driver
   3  *
   4  * Copyright (C) 2012 Texas Instruments, Inc.
   5  *
   6  * Aneesh V <aneesh@ti.com>
   7  * Santosh Shilimkar <santosh.shilimkar@ti.com>
   8  *
   9  * This program is free software; you can redistribute it and/or modify
  10  * it under the terms of the GNU General Public License version 2 as
  11  * published by the Free Software Foundation.
  12  */
  13 #include <linux/err.h>
  14 #include <linux/kernel.h>
  15 #include <linux/reboot.h>
  16 #include <linux/platform_data/emif_plat.h>
  17 #include <linux/io.h>
  18 #include <linux/device.h>
  19 #include <linux/platform_device.h>
  20 #include <linux/interrupt.h>
  21 #include <linux/slab.h>
  22 #include <linux/of.h>
  23 #include <linux/debugfs.h>
  24 #include <linux/seq_file.h>
  25 #include <linux/module.h>
  26 #include <linux/list.h>
  27 #include <linux/spinlock.h>
  28 #include <linux/pm.h>
  29 #include <memory/jedec_ddr.h>
  30 #include "emif.h"
  31 #include "of_memory.h"
  32
  33 /**
  34  * struct emif_data - Per device static data for driver's use
  35  * @duplicate:                  Whether the DDR devices attached to this EMIF
  36  *                              instance are exactly same as that on EMIF1. In
  37  *                              this case we can save some memory and processing
  38  * @temperature_level:          Maximum temperature of LPDDR2 devices attached
  39  *                              to this EMIF - read from MR4 register. If there
  40  *                              are two devices attached to this EMIF, this
  41  *                              value is the maximum of the two temperature
  42  *                              levels.
  43  * @node:                       node in the device list
  44  * @base:                       base address of memory-mapped IO registers.
  45  * @dev:                        device pointer.
  46  * @addressing                  table with addressing information from the spec
  47  * @regs_cache:                 An array of 'struct emif_regs' that stores
  48  *                              calculated register values for different
  49  *                              frequencies, to avoid re-calculating them on
  50  *                              each DVFS transition.
  51  * @curr_regs:                  The set of register values used in the last
  52  *                              frequency change (i.e. corresponding to the
  53  *                              frequency in effect at the moment)
  54  * @plat_data:                  Pointer to saved platform data.
  55  * @debugfs_root:               dentry to the root folder for EMIF in debugfs
  56  * @np_ddr:                     Pointer to ddr device tree node
  57  */
  58 struct emif_data {
  59         u8                              duplicate;
  60         u8                              temperature_level;
  61         u8                              lpmode;
  62         struct list_head                node;
  63         unsigned long                   irq_state;
  64         void __iomem                    *base;
  65         struct device                   *dev;
  66         const struct lpddr2_addressing  *addressing;
  67         struct emif_regs                *regs_cache[EMIF_MAX_NUM_FREQUENCIES];
  68         struct emif_regs                *curr_regs;
  69         struct emif_platform_data       *plat_data;
  70         struct dentry                   *debugfs_root;
  71         struct device_node              *np_ddr;
  72 };
  73
  74 static struct emif_data *emif1;
  75 static spinlock_t       emif_lock;
  76 static unsigned long    irq_state;
  77 static u32              t_ck; /* DDR clock period in ps */
  78 static LIST_HEAD(device_list);
  79
  80 #ifdef CONFIG_DEBUG_FS
  81 static void do_emif_regdump_show(struct seq_file *s, struct emif_data *emif,
  82         struct emif_regs *regs)
  83 {
  84         u32 type = emif->plat_data->device_info->type;
  85         u32 ip_rev = emif->plat_data->ip_rev;
  86
  87         seq_printf(s, "EMIF register cache dump for %dMHz\n",
  88                 regs->freq/1000000);
  89
  90         seq_printf(s, "ref_ctrl_shdw\t: 0x%08x\n", regs->ref_ctrl_shdw);
  91         seq_printf(s, "sdram_tim1_shdw\t: 0x%08x\n", regs->sdram_tim1_shdw);
  92         seq_printf(s, "sdram_tim2_shdw\t: 0x%08x\n", regs->sdram_tim2_shdw);
  93         seq_printf(s, "sdram_tim3_shdw\t: 0x%08x\n", regs->sdram_tim3_shdw);
  94
  95         if (ip_rev == EMIF_4D) {
  96                 seq_printf(s, "read_idle_ctrl_shdw_normal\t: 0x%08x\n",
  97                         regs->read_idle_ctrl_shdw_normal);
  98                 seq_printf(s, "read_idle_ctrl_shdw_volt_ramp\t: 0x%08x\n",
  99                         regs->read_idle_ctrl_shdw_volt_ramp);
 100         } else if (ip_rev == EMIF_4D5) {
 101                 seq_printf(s, "dll_calib_ctrl_shdw_normal\t: 0x%08x\n",
 102                         regs->dll_calib_ctrl_shdw_normal);
 103                 seq_printf(s, "dll_calib_ctrl_shdw_volt_ramp\t: 0x%08x\n",
 104                         regs->dll_calib_ctrl_shdw_volt_ramp);
 105         }
 106
 107         if (type == DDR_TYPE_LPDDR2_S2 || type == DDR_TYPE_LPDDR2_S4) {
 108                 seq_printf(s, "ref_ctrl_shdw_derated\t: 0x%08x\n",
 109                         regs->ref_ctrl_shdw_derated);
 110                 seq_printf(s, "sdram_tim1_shdw_derated\t: 0x%08x\n",
 111                         regs->sdram_tim1_shdw_derated);
 112                 seq_printf(s, "sdram_tim3_shdw_derated\t: 0x%08x\n",
 113                         regs->sdram_tim3_shdw_derated);
 114         }
 115 }
 116
 117 static int emif_regdump_show(struct seq_file *s, void *unused)
 118 {
 119         struct emif_data        *emif   = s->private;
 120         struct emif_regs        **regs_cache;
 121         int                     i;
 122
 123         if (emif->duplicate)
 124                 regs_cache = emif1->regs_cache;
 125         else
 126                 regs_cache = emif->regs_cache;
 127
 128         for (i = 0; i < EMIF_MAX_NUM_FREQUENCIES && regs_cache[i]; i++) {
 129                 do_emif_regdump_show(s, emif, regs_cache[i]);
 130                 seq_printf(s, "\n");
 131         }
 132
 133         return 0;
 134 }
 135
 136 static int emif_regdump_open(struct inode *inode, struct file *file)
 137 {
 138         return single_open(file, emif_regdump_show, inode->i_private);
 139 }
 140
 141 static const struct file_operations emif_regdump_fops = {
 142         .open                   = emif_regdump_open,
 143         .read                   = seq_read,
 144         .release                = single_release,
 145 };
 146
 147 static int emif_mr4_show(struct seq_file *s, void *unused)
 148 {
 149         struct emif_data *emif = s->private;
 150
 151         seq_printf(s, "MR4=%d\n", emif->temperature_level);
 152         return 0;
 153 }
 154
 155 static int emif_mr4_open(struct inode *inode, struct file *file)
 156 {
 157         return single_open(file, emif_mr4_show, inode->i_private);
 158 }
 159
 160 static const struct file_operations emif_mr4_fops = {
 161         .open                   = emif_mr4_open,
 162         .read                   = seq_read,
 163         .release                = single_release,
 164 };
 165
 166 static int __init_or_module emif_debugfs_init(struct emif_data *emif)
 167 {
 168         struct dentry   *dentry;
 169         int             ret;
 170
 171         dentry = debugfs_create_dir(dev_name(emif->dev), NULL);
 172         if (!dentry) {
 173                 ret = -ENOMEM;
 174                 goto err0;
 175         }
 176         emif->debugfs_root = dentry;
 177
 178         dentry = debugfs_create_file("regcache_dump", S_IRUGO,
 179                         emif->debugfs_root, emif, &emif_regdump_fops);
 180         if (!dentry) {
 181                 ret = -ENOMEM;
 182                 goto err1;
 183         }
 184
 185         dentry = debugfs_create_file("mr4", S_IRUGO,
 186                         emif->debugfs_root, emif, &emif_mr4_fops);
 187         if (!dentry) {
 188                 ret = -ENOMEM;
 189                 goto err1;
 190         }
 191
 192         return 0;
 193 err1:
 194         debugfs_remove_recursive(emif->debugfs_root);
 195 err0:
 196         return ret;
 197 }
 198
 199 static void __exit emif_debugfs_exit(struct emif_data *emif)
 200 {
 201         debugfs_remove_recursive(emif->debugfs_root);
 202         emif->debugfs_root = NULL;
 203 }
 204 #else
 205 static inline int __init_or_module emif_debugfs_init(struct emif_data *emif)
 206 {
 207         return 0;
 208 }
 209
 210 static inline void __exit emif_debugfs_exit(struct emif_data *emif)
 211 {
 212 }
 213 #endif
 214
 215 /*
 216  * Calculate the period of DDR clock from frequency value
 217  */
 218 static void set_ddr_clk_period(u32 freq)
 219 {
 220         /* Divide 10^12 by frequency to get period in ps */
 221         t_ck = (u32)DIV_ROUND_UP_ULL(1000000000000ull, freq);
 222 }
 223
 224 /*
 225  * Get bus width used by EMIF. Note that this may be different from the
 226  * bus width of the DDR devices used. For instance two 16-bit DDR devices
 227  * may be connected to a given CS of EMIF. In this case bus width as far
 228  * as EMIF is concerned is 32, where as the DDR bus width is 16 bits.
 229  */
 230 static u32 get_emif_bus_width(struct emif_data *emif)
 231 {
 232         u32             width;
 233         void __iomem    *base = emif->base;
 234
 235         width = (readl(base + EMIF_SDRAM_CONFIG) & NARROW_MODE_MASK)
 236                         >> NARROW_MODE_SHIFT;
 237         width = width == 0 ? 32 : 16;
 238
 239         return width;
 240 }
 241
 242 /*
 243  * Get the CL from SDRAM_CONFIG register
 244  */
 245 static u32 get_cl(struct emif_data *emif)
 246 {
 247         u32             cl;
 248         void __iomem    *base = emif->base;
 249
 250         cl = (readl(base + EMIF_SDRAM_CONFIG) & CL_MASK) >> CL_SHIFT;
 251
 252         return cl;
 253 }
 254
 255 static void set_lpmode(struct emif_data *emif, u8 lpmode)
 256 {
 257         u32 temp;
 258         void __iomem *base = emif->base;
 259
 260         temp = readl(base + EMIF_POWER_MANAGEMENT_CONTROL);
 261         temp &= ~LP_MODE_MASK;
 262         temp |= (lpmode << LP_MODE_SHIFT);
 263         writel(temp, base + EMIF_POWER_MANAGEMENT_CONTROL);
 264 }
 265
 266 static void do_freq_update(void)
 267 {
 268         struct emif_data *emif;
 269
 270         /*
 271          * Workaround for errata i728: Disable LPMODE during FREQ_UPDATE
 272          *
 273          * i728 DESCRIPTION:
 274          * The EMIF automatically puts the SDRAM into self-refresh mode
 275          * after the EMIF has not performed accesses during
 276          * EMIF_PWR_MGMT_CTRL[7:4] REG_SR_TIM number of DDR clock cycles
 277          * and the EMIF_PWR_MGMT_CTRL[10:8] REG_LP_MODE bit field is set
 278          * to 0x2. If during a small window the following three events
 279          * occur:
 280          * - The SR_TIMING counter expires
 281          * - And frequency change is requested
 282          * - And OCP access is requested
 283          * Then it causes instable clock on the DDR interface.
 284          *
 285          * WORKAROUND
 286          * To avoid the occurrence of the three events, the workaround
 287          * is to disable the self-refresh when requesting a frequency
 288          * change. Before requesting a frequency change the software must
 289          * program EMIF_PWR_MGMT_CTRL[10:8] REG_LP_MODE to 0x0. When the
 290          * frequency change has been done, the software can reprogram
 291          * EMIF_PWR_MGMT_CTRL[10:8] REG_LP_MODE to 0x2
 292          */
 293         list_for_each_entry(emif, &device_list, node) {
 294                 if (emif->lpmode == EMIF_LP_MODE_SELF_REFRESH)
 295                         set_lpmode(emif, EMIF_LP_MODE_DISABLE);
 296         }
 297
 298         /*
 299          * TODO: Do FREQ_UPDATE here when an API
 300          * is available for this as part of the new
 301          * clock framework
 302          */
 303
 304         list_for_each_entry(emif, &device_list, node) {
 305                 if (emif->lpmode == EMIF_LP_MODE_SELF_REFRESH)
 306                         set_lpmode(emif, EMIF_LP_MODE_SELF_REFRESH);
 307         }
 308 }
 309
 310 /* Find addressing table entry based on the device's type and density */
 311 static const struct lpddr2_addressing *get_addressing_table(
 312         const struct ddr_device_info *device_info)
 313 {
 314         u32             index, type, density;
 315
 316         type = device_info->type;
 317         density = device_info->density;
 318
 319         switch (type) {
 320         case DDR_TYPE_LPDDR2_S4:
 321                 index = density - 1;
 322                 break;
 323         case DDR_TYPE_LPDDR2_S2:
 324                 switch (density) {
 325                 case DDR_DENSITY_1Gb:
 326                 case DDR_DENSITY_2Gb:
 327                         index = density + 3;
 328                         break;
 329                 default:
 330                         index = density - 1;
 331                 }
 332                 break;
 333         default:
 334                 return NULL;
 335         }
 336
 337         return &lpddr2_jedec_addressing_table[index];
 338 }
 339
 340 /*
 341  * Find the the right timing table from the array of timing
 342  * tables of the device using DDR clock frequency
 343  */
 344 static const struct lpddr2_timings *get_timings_table(struct emif_data *emif,
 345                 u32 freq)
 346 {
 347         u32                             i, min, max, freq_nearest;
 348         const struct lpddr2_timings     *timings = NULL;
 349         const struct lpddr2_timings     *timings_arr = emif->plat_data->timings;
 350         struct                          device *dev = emif->dev;
 351
 352         /* Start with a very high frequency - 1GHz */
 353         freq_nearest = 1000000000;
 354
 355         /*
 356          * Find the timings table such that:
 357          *  1. the frequency range covers the required frequency(safe) AND
 358          *  2. the max_freq is closest to the required frequency(optimal)
 359          */
 360         for (i = 0; i < emif->plat_data->timings_arr_size; i++) {
 361                 max = timings_arr[i].max_freq;
 362                 min = timings_arr[i].min_freq;
 363                 if ((freq >= min) && (freq <= max) && (max < freq_nearest)) {
 364                         freq_nearest = max;
 365                         timings = &timings_arr[i];
 366                 }
 367         }
 368
 369         if (!timings)
 370                 dev_err(dev, "%s: couldn't find timings for - %dHz\n",
 371                         __func__, freq);
 372
 373         dev_dbg(dev, "%s: timings table: freq %d, speed bin freq %d\n",
 374                 __func__, freq, freq_nearest);
 375
 376         return timings;
 377 }
 378
 379 static u32 get_sdram_ref_ctrl_shdw(u32 freq,
 380                 const struct lpddr2_addressing *addressing)
 381 {
 382         u32 ref_ctrl_shdw = 0, val = 0, freq_khz, t_refi;
 383
 384         /* Scale down frequency and t_refi to avoid overflow */
 385         freq_khz = freq / 1000;
 386         t_refi = addressing->tREFI_ns / 100;
 387
 388         /*
 389          * refresh rate to be set is 'tREFI(in us) * freq in MHz
 390          * division by 10000 to account for change in units
 391          */
 392         val = t_refi * freq_khz / 10000;
 393         ref_ctrl_shdw |= val << REFRESH_RATE_SHIFT;
 394
 395         return ref_ctrl_shdw;
 396 }
 397
 398 static u32 get_sdram_tim_1_shdw(const struct lpddr2_timings *timings,
 399                 const struct lpddr2_min_tck *min_tck,
 400                 const struct lpddr2_addressing *addressing)
 401 {
 402         u32 tim1 = 0, val = 0;
 403
 404         val = max(min_tck->tWTR, DIV_ROUND_UP(timings->tWTR, t_ck)) - 1;
 405         tim1 |= val << T_WTR_SHIFT;
 406
 407         if (addressing->num_banks == B8)
 408                 val = DIV_ROUND_UP(timings->tFAW, t_ck*4);
 409         else
 410                 val = max(min_tck->tRRD, DIV_ROUND_UP(timings->tRRD, t_ck));
 411         tim1 |= (val - 1) << T_RRD_SHIFT;
 412
 413         val = DIV_ROUND_UP(timings->tRAS_min + timings->tRPab, t_ck) - 1;
 414         tim1 |= val << T_RC_SHIFT;
 415
 416         val = max(min_tck->tRASmin, DIV_ROUND_UP(timings->tRAS_min, t_ck));
 417         tim1 |= (val - 1) << T_RAS_SHIFT;
 418
 419         val = max(min_tck->tWR, DIV_ROUND_UP(timings->tWR, t_ck)) - 1;
 420         tim1 |= val << T_WR_SHIFT;
 421
 422         val = max(min_tck->tRCD, DIV_ROUND_UP(timings->tRCD, t_ck)) - 1;
 423         tim1 |= val << T_RCD_SHIFT;
 424
 425         val = max(min_tck->tRPab, DIV_ROUND_UP(timings->tRPab, t_ck)) - 1;
 426         tim1 |= val << T_RP_SHIFT;
 427
 428         return tim1;
 429 }
 430
 431 static u32 get_sdram_tim_1_shdw_derated(const struct lpddr2_timings *timings,
 432                 const struct lpddr2_min_tck *min_tck,
 433                 const struct lpddr2_addressing *addressing)
 434 {
 435         u32 tim1 = 0, val = 0;
 436
 437         val = max(min_tck->tWTR, DIV_ROUND_UP(timings->tWTR, t_ck)) - 1;
 438         tim1 = val << T_WTR_SHIFT;
 439
 440         /*
 441          * tFAW is approximately 4 times tRRD. So add 1875*4 = 7500ps
 442          * to tFAW for de-rating
 443          */
 444         if (addressing->num_banks == B8) {
 445                 val = DIV_ROUND_UP(timings->tFAW + 7500, 4 * t_ck) - 1;
 446         } else {
 447                 val = DIV_ROUND_UP(timings->tRRD + 1875, t_ck);
 448                 val = max(min_tck->tRRD, val) - 1;
 449         }
 450         tim1 |= val << T_RRD_SHIFT;
 451
 452         val = DIV_ROUND_UP(timings->tRAS_min + timings->tRPab + 1875, t_ck);
 453         tim1 |= (val - 1) << T_RC_SHIFT;
 454
 455         val = DIV_ROUND_UP(timings->tRAS_min + 1875, t_ck);
 456         val = max(min_tck->tRASmin, val) - 1;
 457         tim1 |= val << T_RAS_SHIFT;
 458
 459         val = max(min_tck->tWR, DIV_ROUND_UP(timings->tWR, t_ck)) - 1;
 460         tim1 |= val << T_WR_SHIFT;
 461
 462         val = max(min_tck->tRCD, DIV_ROUND_UP(timings->tRCD + 1875, t_ck));
 463         tim1 |= (val - 1) << T_RCD_SHIFT;
 464
 465         val = max(min_tck->tRPab, DIV_ROUND_UP(timings->tRPab + 1875, t_ck));
 466         tim1 |= (val - 1) << T_RP_SHIFT;
 467
 468         return tim1;
 469 }
 470
 471 static u32 get_sdram_tim_2_shdw(const struct lpddr2_timings *timings,
 472                 const struct lpddr2_min_tck *min_tck,
 473                 const struct lpddr2_addressing *addressing,
 474                 u32 type)
 475 {
 476         u32 tim2 = 0, val = 0;
 477
 478         val = min_tck->tCKE - 1;
 479         tim2 |= val << T_CKE_SHIFT;
 480
 481         val = max(min_tck->tRTP, DIV_ROUND_UP(timings->tRTP, t_ck)) - 1;
 482         tim2 |= val << T_RTP_SHIFT;
 483
 484         /* tXSNR = tRFCab_ps + 10 ns(tRFCab_ps for LPDDR2). */
 485         val = DIV_ROUND_UP(addressing->tRFCab_ps + 10000, t_ck) - 1;
 486         tim2 |= val << T_XSNR_SHIFT;
 487
 488         /* XSRD same as XSNR for LPDDR2 */
 489         tim2 |= val << T_XSRD_SHIFT;
 490
 491         val = max(min_tck->tXP, DIV_ROUND_UP(timings->tXP, t_ck)) - 1;
 492         tim2 |= val << T_XP_SHIFT;
 493
 494         return tim2;
 495 }
 496
 497 static u32 get_sdram_tim_3_shdw(const struct lpddr2_timings *timings,
 498                 const struct lpddr2_min_tck *min_tck,
 499                 const struct lpddr2_addressing *addressing,
 500                 u32 type, u32 ip_rev, u32 derated)
 501 {
 502         u32 tim3 = 0, val = 0, t_dqsck;
 503
 504         val = timings->tRAS_max_ns / addressing->tREFI_ns - 1;
 505         val = val > 0xF ? 0xF : val;
 506         tim3 |= val << T_RAS_MAX_SHIFT;
 507
 508         val = DIV_ROUND_UP(addressing->tRFCab_ps, t_ck) - 1;
 509         tim3 |= val << T_RFC_SHIFT;
 510
 511         t_dqsck = (derated == EMIF_DERATED_TIMINGS) ?
 512                 timings->tDQSCK_max_derated : timings->tDQSCK_max;
 513         if (ip_rev == EMIF_4D5)
 514                 val = DIV_ROUND_UP(t_dqsck + 1000, t_ck) - 1;
 515         else
 516                 val = DIV_ROUND_UP(t_dqsck, t_ck) - 1;
 517
 518         tim3 |= val << T_TDQSCKMAX_SHIFT;
 519
 520         val = DIV_ROUND_UP(timings->tZQCS, t_ck) - 1;
 521         tim3 |= val << ZQ_ZQCS_SHIFT;
 522
 523         val = DIV_ROUND_UP(timings->tCKESR, t_ck);
 524         val = max(min_tck->tCKESR, val) - 1;
 525         tim3 |= val << T_CKESR_SHIFT;
 526
 527         if (ip_rev == EMIF_4D5) {
 528                 tim3 |= (EMIF_T_CSTA - 1) << T_CSTA_SHIFT;
 529
 530                 val = DIV_ROUND_UP(EMIF_T_PDLL_UL, 128) - 1;
 531                 tim3 |= val << T_PDLL_UL_SHIFT;
 532         }
 533
 534         return tim3;
 535 }
 536
 537 static u32 get_zq_config_reg(const struct lpddr2_addressing *addressing,
 538                 bool cs1_used, bool cal_resistors_per_cs)
 539 {
 540         u32 zq = 0, val = 0;
 541
 542         val = EMIF_ZQCS_INTERVAL_US * 1000 / addressing->tREFI_ns;
 543         zq |= val << ZQ_REFINTERVAL_SHIFT;
 544
 545         val = DIV_ROUND_UP(T_ZQCL_DEFAULT_NS, T_ZQCS_DEFAULT_NS) - 1;
 546         zq |= val << ZQ_ZQCL_MULT_SHIFT;
 547
 548         val = DIV_ROUND_UP(T_ZQINIT_DEFAULT_NS, T_ZQCL_DEFAULT_NS) - 1;
 549         zq |= val << ZQ_ZQINIT_MULT_SHIFT;
 550
 551         zq |= ZQ_SFEXITEN_ENABLE << ZQ_SFEXITEN_SHIFT;
 552
 553         if (cal_resistors_per_cs)
 554                 zq |= ZQ_DUALCALEN_ENABLE << ZQ_DUALCALEN_SHIFT;
 555         else
 556                 zq |= ZQ_DUALCALEN_DISABLE << ZQ_DUALCALEN_SHIFT;
 557
 558         zq |= ZQ_CS0EN_MASK; /* CS0 is used for sure */
 559
 560         val = cs1_used ? 1 : 0;
 561         zq |= val << ZQ_CS1EN_SHIFT;
 562
 563         return zq;
 564 }
 565
 566 static u32 get_temp_alert_config(const struct lpddr2_addressing *addressing,
 567                 const struct emif_custom_configs *custom_configs, bool cs1_used,
 568                 u32 sdram_io_width, u32 emif_bus_width)
 569 {
 570         u32 alert = 0, interval, devcnt;
 571
 572         if (custom_configs && (custom_configs->mask &
 573                                 EMIF_CUSTOM_CONFIG_TEMP_ALERT_POLL_INTERVAL))
 574                 interval = custom_configs->temp_alert_poll_interval_ms;
 575         else
 576                 interval = TEMP_ALERT_POLL_INTERVAL_DEFAULT_MS;
 577
 578         interval *= 1000000;                    /* Convert to ns */
 579         interval /= addressing->tREFI_ns;       /* Convert to refresh cycles */
 580         alert |= (interval << TA_REFINTERVAL_SHIFT);
 581
 582         /*
 583          * sdram_io_width is in 'log2(x) - 1' form. Convert emif_bus_width
 584          * also to this form and subtract to get TA_DEVCNT, which is
 585          * in log2(x) form.
 586          */
 587         emif_bus_width = __fls(emif_bus_width) - 1;
 588         devcnt = emif_bus_width - sdram_io_width;
 589         alert |= devcnt << TA_DEVCNT_SHIFT;
 590
 591         /* DEVWDT is in 'log2(x) - 3' form */
 592         alert |= (sdram_io_width - 2) << TA_DEVWDT_SHIFT;
 593
 594         alert |= 1 << TA_SFEXITEN_SHIFT;
 595         alert |= 1 << TA_CS0EN_SHIFT;
 596         alert |= (cs1_used ? 1 : 0) << TA_CS1EN_SHIFT;
 597
 598         return alert;
 599 }
 600
 601 static u32 get_read_idle_ctrl_shdw(u8 volt_ramp)
 602 {
 603         u32 idle = 0, val = 0;
 604
 605         /*
 606          * Maximum value in normal conditions and increased frequency
 607          * when voltage is ramping
 608          */
 609         if (volt_ramp)
 610                 val = READ_IDLE_INTERVAL_DVFS / t_ck / 64 - 1;
 611         else
 612                 val = 0x1FF;
 613
 614         /*
 615          * READ_IDLE_CTRL register in EMIF4D has same offset and fields
 616          * as DLL_CALIB_CTRL in EMIF4D5, so use the same shifts
 617          */
 618         idle |= val << DLL_CALIB_INTERVAL_SHIFT;
 619         idle |= EMIF_READ_IDLE_LEN_VAL << ACK_WAIT_SHIFT;
 620
 621         return idle;
 622 }
 623
 624 static u32 get_dll_calib_ctrl_shdw(u8 volt_ramp)
 625 {
 626         u32 calib = 0, val = 0;
 627
 628         if (volt_ramp == DDR_VOLTAGE_RAMPING)
 629                 val = DLL_CALIB_INTERVAL_DVFS / t_ck / 16 - 1;
 630         else
 631                 val = 0; /* Disabled when voltage is stable */
 632
 633         calib |= val << DLL_CALIB_INTERVAL_SHIFT;
 634         calib |= DLL_CALIB_ACK_WAIT_VAL << ACK_WAIT_SHIFT;
 635
 636         return calib;
 637 }
 638
 639 static u32 get_ddr_phy_ctrl_1_attilaphy_4d(const struct lpddr2_timings *timings,
 640         u32 freq, u8 RL)
 641 {
 642         u32 phy = EMIF_DDR_PHY_CTRL_1_BASE_VAL_ATTILAPHY, val = 0;
 643
 644         val = RL + DIV_ROUND_UP(timings->tDQSCK_max, t_ck) - 1;
 645         phy |= val << READ_LATENCY_SHIFT_4D;
 646
 647         if (freq <= 100000000)
 648                 val = EMIF_DLL_SLAVE_DLY_CTRL_100_MHZ_AND_LESS_ATTILAPHY;
 649         else if (freq <= 200000000)
 650                 val = EMIF_DLL_SLAVE_DLY_CTRL_200_MHZ_ATTILAPHY;
 651         else
 652                 val = EMIF_DLL_SLAVE_DLY_CTRL_400_MHZ_ATTILAPHY;
 653
 654         phy |= val << DLL_SLAVE_DLY_CTRL_SHIFT_4D;
 655
 656         return phy;
 657 }
 658
 659 static u32 get_phy_ctrl_1_intelliphy_4d5(u32 freq, u8 cl)
 660 {
 661         u32 phy = EMIF_DDR_PHY_CTRL_1_BASE_VAL_INTELLIPHY, half_delay;
 662
 663         /*
 664          * DLL operates at 266 MHz. If DDR frequency is near 266 MHz,
 665          * half-delay is not needed else set half-delay
 666          */
 667         if (freq >= 265000000 && freq < 267000000)
 668                 half_delay = 0;
 669         else
 670                 half_delay = 1;
 671
 672         phy |= half_delay << DLL_HALF_DELAY_SHIFT_4D5;
 673         phy |= ((cl + DIV_ROUND_UP(EMIF_PHY_TOTAL_READ_LATENCY_INTELLIPHY_PS,
 674                         t_ck) - 1) << READ_LATENCY_SHIFT_4D5);
 675
 676         return phy;
 677 }
 678
 679 static u32 get_ext_phy_ctrl_2_intelliphy_4d5(void)
 680 {
 681         u32 fifo_we_slave_ratio;
 682
 683         fifo_we_slave_ratio =  DIV_ROUND_CLOSEST(
 684                 EMIF_INTELLI_PHY_DQS_GATE_OPENING_DELAY_PS * 256 , t_ck);
 685
 686         return fifo_we_slave_ratio | fifo_we_slave_ratio << 11 |
 687                 fifo_we_slave_ratio << 22;
 688 }
 689
 690 static u32 get_ext_phy_ctrl_3_intelliphy_4d5(void)
 691 {
 692         u32 fifo_we_slave_ratio;
 693
 694         fifo_we_slave_ratio =  DIV_ROUND_CLOSEST(
 695                 EMIF_INTELLI_PHY_DQS_GATE_OPENING_DELAY_PS * 256 , t_ck);
 696
 697         return fifo_we_slave_ratio >> 10 | fifo_we_slave_ratio << 1 |
 698                 fifo_we_slave_ratio << 12 | fifo_we_slave_ratio << 23;
 699 }
 700
 701 static u32 get_ext_phy_ctrl_4_intelliphy_4d5(void)
 702 {
 703         u32 fifo_we_slave_ratio;
 704
 705         fifo_we_slave_ratio =  DIV_ROUND_CLOSEST(
 706                 EMIF_INTELLI_PHY_DQS_GATE_OPENING_DELAY_PS * 256 , t_ck);
 707
 708         return fifo_we_slave_ratio >> 9 | fifo_we_slave_ratio << 2 |
 709                 fifo_we_slave_ratio << 13;
 710 }
 711
 712 static u32 get_pwr_mgmt_ctrl(u32 freq, struct emif_data *emif, u32 ip_rev)
 713 {
 714         u32 pwr_mgmt_ctrl       = 0, timeout;
 715         u32 lpmode              = EMIF_LP_MODE_SELF_REFRESH;
 716         u32 timeout_perf        = EMIF_LP_MODE_TIMEOUT_PERFORMANCE;
 717         u32 timeout_pwr         = EMIF_LP_MODE_TIMEOUT_POWER;
 718         u32 freq_threshold      = EMIF_LP_MODE_FREQ_THRESHOLD;
 719         u32 mask;
 720         u8 shift;
 721
 722         struct emif_custom_configs *cust_cfgs = emif->plat_data->custom_configs;
 723
 724         if (cust_cfgs && (cust_cfgs->mask & EMIF_CUSTOM_CONFIG_LPMODE)) {
 725                 lpmode          = cust_cfgs->lpmode;
 726                 timeout_perf    = cust_cfgs->lpmode_timeout_performance;
 727                 timeout_pwr     = cust_cfgs->lpmode_timeout_power;
 728                 freq_threshold  = cust_cfgs->lpmode_freq_threshold;
 729         }
 730
 731         /* Timeout based on DDR frequency */
 732         timeout = freq >= freq_threshold ? timeout_perf : timeout_pwr;
 733
 734         /*
 735          * The value to be set in register is "log2(timeout) - 3"
 736          * if timeout < 16 load 0 in register
 737          * if timeout is not a power of 2, round to next highest power of 2
 738          */
 739         if (timeout < 16) {
 740                 timeout = 0;
 741         } else {
 742                 if (timeout & (timeout - 1))
 743                         timeout <<= 1;
 744                 timeout = __fls(timeout) - 3;
 745         }
 746
 747         switch (lpmode) {
 748         case EMIF_LP_MODE_CLOCK_STOP:
 749                 shift = CS_TIM_SHIFT;
 750                 mask = CS_TIM_MASK;
 751                 break;
 752         case EMIF_LP_MODE_SELF_REFRESH:
 753                 /* Workaround for errata i735 */
 754                 if (timeout < 6)
 755                         timeout = 6;
 756
 757                 shift = SR_TIM_SHIFT;
 758                 mask = SR_TIM_MASK;
 759                 break;
 760         case EMIF_LP_MODE_PWR_DN:
 761                 shift = PD_TIM_SHIFT;
 762                 mask = PD_TIM_MASK;
 763                 break;
 764         case EMIF_LP_MODE_DISABLE:
 765         default:
 766                 mask = 0;
 767                 shift = 0;
 768                 break;
 769         }
 770         /* Round to maximum in case of overflow, BUT warn! */
 771         if (lpmode != EMIF_LP_MODE_DISABLE && timeout > mask >> shift) {
 772                 pr_err("TIMEOUT Overflow - lpmode=%d perf=%d pwr=%d freq=%d\n",
 773                        lpmode,
 774                        timeout_perf,
 775                        timeout_pwr,
 776                        freq_threshold);
 777                 WARN(1, "timeout=0x%02x greater than 0x%02x. Using max\n",
 778                      timeout, mask >> shift);
 779                 timeout = mask >> shift;
 780         }
 781
 782         /* Setup required timing */
 783         pwr_mgmt_ctrl = (timeout << shift) & mask;
 784         /* setup a default mask for rest of the modes */
 785         pwr_mgmt_ctrl |= (SR_TIM_MASK | CS_TIM_MASK | PD_TIM_MASK) &
 786                           ~mask;
 787
 788         /* No CS_TIM in EMIF_4D5 */
 789         if (ip_rev == EMIF_4D5)
 790                 pwr_mgmt_ctrl &= ~CS_TIM_MASK;
 791
 792         pwr_mgmt_ctrl |= lpmode << LP_MODE_SHIFT;
 793
 794         return pwr_mgmt_ctrl;
 795 }
 796
 797 /*
 798  * Get the temperature level of the EMIF instance:
 799  * Reads the MR4 register of attached SDRAM parts to find out the temperature
 800  * level. If there are two parts attached(one on each CS), then the temperature
 801  * level for the EMIF instance is the higher of the two temperatures.
 802  */
 803 static void get_temperature_level(struct emif_data *emif)
 804 {
 805         u32             temp, temperature_level;
 806         void __iomem    *base;
 807
 808         base = emif->base;
 809
 810         /* Read mode register 4 */
 811         writel(DDR_MR4, base + EMIF_LPDDR2_MODE_REG_CONFIG);
 812         temperature_level = readl(base + EMIF_LPDDR2_MODE_REG_DATA);
 813         temperature_level = (temperature_level & MR4_SDRAM_REF_RATE_MASK) >>
 814                                 MR4_SDRAM_REF_RATE_SHIFT;
 815
 816         if (emif->plat_data->device_info->cs1_used) {
 817                 writel(DDR_MR4 | CS_MASK, base + EMIF_LPDDR2_MODE_REG_CONFIG);
 818                 temp = readl(base + EMIF_LPDDR2_MODE_REG_DATA);
 819                 temp = (temp & MR4_SDRAM_REF_RATE_MASK)
 820                                 >> MR4_SDRAM_REF_RATE_SHIFT;
 821                 temperature_level = max(temp, temperature_level);
 822         }
 823
 824         /* treat everything less than nominal(3) in MR4 as nominal */
 825         if (unlikely(temperature_level < SDRAM_TEMP_NOMINAL))
 826                 temperature_level = SDRAM_TEMP_NOMINAL;
 827
 828         /* if we get reserved value in MR4 persist with the existing value */
 829         if (likely(temperature_level != SDRAM_TEMP_RESERVED_4))
 830                 emif->temperature_level = temperature_level;
 831 }
 832
 833 /*
 834  * Program EMIF shadow registers that are not dependent on temperature
 835  * or voltage
 836  */
 837 static void setup_registers(struct emif_data *emif, struct emif_regs *regs)
 838 {
 839         void __iomem    *base = emif->base;
 840
 841         writel(regs->sdram_tim2_shdw, base + EMIF_SDRAM_TIMING_2_SHDW);
 842         writel(regs->phy_ctrl_1_shdw, base + EMIF_DDR_PHY_CTRL_1_SHDW);
 843         writel(regs->pwr_mgmt_ctrl_shdw,
 844                base + EMIF_POWER_MANAGEMENT_CTRL_SHDW);
 845
 846         /* Settings specific for EMIF4D5 */
 847         if (emif->plat_data->ip_rev != EMIF_4D5)
 848                 return;
 849         writel(regs->ext_phy_ctrl_2_shdw, base + EMIF_EXT_PHY_CTRL_2_SHDW);
 850         writel(regs->ext_phy_ctrl_3_shdw, base + EMIF_EXT_PHY_CTRL_3_SHDW);
 851         writel(regs->ext_phy_ctrl_4_shdw, base + EMIF_EXT_PHY_CTRL_4_SHDW);
 852 }
 853
 854 /*
 855  * When voltage ramps dll calibration and forced read idle should
 856  * happen more often
 857  */
 858 static void setup_volt_sensitive_regs(struct emif_data *emif,
 859                 struct emif_regs *regs, u32 volt_state)
 860 {
 861         u32             calib_ctrl;
 862         void __iomem    *base = emif->base;
 863
 864         /*
 865          * EMIF_READ_IDLE_CTRL in EMIF4D refers to the same register as
 866          * EMIF_DLL_CALIB_CTRL in EMIF4D5 and dll_calib_ctrl_shadow_*
 867          * is an alias of the respective read_idle_ctrl_shdw_* (members of
 868          * a union). So, the below code takes care of both cases
 869          */
 870         if (volt_state == DDR_VOLTAGE_RAMPING)
 871                 calib_ctrl = regs->dll_calib_ctrl_shdw_volt_ramp;
 872         else
 873                 calib_ctrl = regs->dll_calib_ctrl_shdw_normal;
 874
 875         writel(calib_ctrl, base + EMIF_DLL_CALIB_CTRL_SHDW);
 876 }
 877
 878 /*
 879  * setup_temperature_sensitive_regs() - set the timings for temperature
 880  * sensitive registers. This happens once at initialisation time based
 881  * on the temperature at boot time and subsequently based on the temperature
 882  * alert interrupt. Temperature alert can happen when the temperature
 883  * increases or drops. So this function can have the effect of either
 884  * derating the timings or going back to nominal values.
 885  */
 886 static void setup_temperature_sensitive_regs(struct emif_data *emif,
 887                 struct emif_regs *regs)
 888 {
 889         u32             tim1, tim3, ref_ctrl, type;
 890         void __iomem    *base = emif->base;
 891         u32             temperature;
 892
 893         type = emif->plat_data->device_info->type;
 894
 895         tim1 = regs->sdram_tim1_shdw;
 896         tim3 = regs->sdram_tim3_shdw;
 897         ref_ctrl = regs->ref_ctrl_shdw;
 898
 899         /* No de-rating for non-lpddr2 devices */
 900         if (type != DDR_TYPE_LPDDR2_S2 && type != DDR_TYPE_LPDDR2_S4)
 901                 goto out;
 902
 903         temperature = emif->temperature_level;
 904         if (temperature == SDRAM_TEMP_HIGH_DERATE_REFRESH) {
 905                 ref_ctrl = regs->ref_ctrl_shdw_derated;
 906         } else if (temperature == SDRAM_TEMP_HIGH_DERATE_REFRESH_AND_TIMINGS) {
 907                 tim1 = regs->sdram_tim1_shdw_derated;
 908                 tim3 = regs->sdram_tim3_shdw_derated;
 909                 ref_ctrl = regs->ref_ctrl_shdw_derated;
 910         }
 911
 912 out:
 913         writel(tim1, base + EMIF_SDRAM_TIMING_1_SHDW);
 914         writel(tim3, base + EMIF_SDRAM_TIMING_3_SHDW);
 915         writel(ref_ctrl, base + EMIF_SDRAM_REFRESH_CTRL_SHDW);
 916 }
 917
 918 static irqreturn_t handle_temp_alert(void __iomem *base, struct emif_data *emif)
 919 {
 920         u32             old_temp_level;
 921         irqreturn_t     ret = IRQ_HANDLED;
 922         struct emif_custom_configs *custom_configs;
 923
 924         spin_lock_irqsave(&emif_lock, irq_state);
 925         old_temp_level = emif->temperature_level;
 926         get_temperature_level(emif);
 927
 928         if (unlikely(emif->temperature_level == old_temp_level)) {
 929                 goto out;
 930         } else if (!emif->curr_regs) {
 931                 dev_err(emif->dev, "temperature alert before registers are calculated, not de-rating timings\n");
 932                 goto out;
 933         }
 934
 935         custom_configs = emif->plat_data->custom_configs;
 936
 937         /*
 938          * IF we detect higher than "nominal rating" from DDR sensor
 939          * on an unsupported DDR part, shutdown system
 940          */
 941         if (custom_configs && !(custom_configs->mask &
 942                                 EMIF_CUSTOM_CONFIG_EXTENDED_TEMP_PART)) {
 943                 if (emif->temperature_level >= SDRAM_TEMP_HIGH_DERATE_REFRESH) {
 944                         dev_err(emif->dev,
 945                                 "%s:NOT Extended temperature capable memory."
 946                                 "Converting MR4=0x%02x as shutdown event\n",
 947                                 __func__, emif->temperature_level);
 948                         /*
 949                          * Temperature far too high - do kernel_power_off()
 950                          * from thread context
 951                          */
 952                         emif->temperature_level = SDRAM_TEMP_VERY_HIGH_SHUTDOWN;
 953                         ret = IRQ_WAKE_THREAD;
 954                         goto out;
 955                 }
 956         }
 957
 958         if (emif->temperature_level < old_temp_level ||
 959                 emif->temperature_level == SDRAM_TEMP_VERY_HIGH_SHUTDOWN) {
 960                 /*
 961                  * Temperature coming down - defer handling to thread OR
 962                  * Temperature far too high - do kernel_power_off() from
 963                  * thread context
 964                  */
 965                 ret = IRQ_WAKE_THREAD;
 966         } else {
 967                 /* Temperature is going up - handle immediately */
 968                 setup_temperature_sensitive_regs(emif, emif->curr_regs);
 969                 do_freq_update();
 970         }
 971
 972 out:
 973         spin_unlock_irqrestore(&emif_lock, irq_state);
 974         return ret;
 975 }
 976
 977 static irqreturn_t emif_interrupt_handler(int irq, void *dev_id)
 978 {
 979         u32                     interrupts;
 980         struct emif_data        *emif = dev_id;
 981         void __iomem            *base = emif->base;
 982         struct device           *dev = emif->dev;
 983         irqreturn_t             ret = IRQ_HANDLED;
 984
 985         /* Save the status and clear it */
 986         interrupts = readl(base + EMIF_SYSTEM_OCP_INTERRUPT_STATUS);
 987         writel(interrupts, base + EMIF_SYSTEM_OCP_INTERRUPT_STATUS);
 988
 989         /*
 990          * Handle temperature alert
 991          * Temperature alert should be same for all ports
 992          * So, it's enough to process it only for one of the ports
 993          */
 994         if (interrupts & TA_SYS_MASK)
 995                 ret = handle_temp_alert(base, emif);
 996
 997         if (interrupts & ERR_SYS_MASK)
 998                 dev_err(dev, "Access error from SYS port - %x\n", interrupts);
 999
1000         if (emif->plat_data->hw_caps & EMIF_HW_CAPS_LL_INTERFACE) {
1001                 /* Save the status and clear it */
1002                 interrupts = readl(base + EMIF_LL_OCP_INTERRUPT_STATUS);
1003                 writel(interrupts, base + EMIF_LL_OCP_INTERRUPT_STATUS);
1004
1005                 if (interrupts & ERR_LL_MASK)
1006                         dev_err(dev, "Access error from LL port - %x\n",
1007                                 interrupts);
1008         }
1009
1010         return ret;
1011 }
1012
1013 static irqreturn_t emif_threaded_isr(int irq, void *dev_id)
1014 {
1015         struct emif_data        *emif = dev_id;
1016
1017         if (emif->temperature_level == SDRAM_TEMP_VERY_HIGH_SHUTDOWN) {
1018                 dev_emerg(emif->dev, "SDRAM temperature exceeds operating limit.. Needs shut down!!!\n");
1019
1020                 /* If we have Power OFF ability, use it, else try restarting */
1021                 if (pm_power_off) {
1022                         kernel_power_off();
1023                 } else {
1024                         WARN(1, "FIXME: NO pm_power_off!!! trying restart\n");
1025                         kernel_restart("SDRAM Over-temp Emergency restart");
1026                 }
1027                 return IRQ_HANDLED;
1028         }
1029
1030         spin_lock_irqsave(&emif_lock, irq_state);
1031
1032         if (emif->curr_regs) {
1033                 setup_temperature_sensitive_regs(emif, emif->curr_regs);
1034                 do_freq_update();
1035         } else {
1036                 dev_err(emif->dev, "temperature alert before registers are calculated, not de-rating timings\n");
1037         }
1038
1039         spin_unlock_irqrestore(&emif_lock, irq_state);
1040
1041         return IRQ_HANDLED;
1042 }
1043
1044 static void clear_all_interrupts(struct emif_data *emif)
1045 {
1046         void __iomem    *base = emif->base;
1047
1048         writel(readl(base + EMIF_SYSTEM_OCP_INTERRUPT_STATUS),
1049                 base + EMIF_SYSTEM_OCP_INTERRUPT_STATUS);
1050         if (emif->plat_data->hw_caps & EMIF_HW_CAPS_LL_INTERFACE)
1051                 writel(readl(base + EMIF_LL_OCP_INTERRUPT_STATUS),
1052                         base + EMIF_LL_OCP_INTERRUPT_STATUS);
1053 }
1054
1055 static void disable_and_clear_all_interrupts(struct emif_data *emif)
1056 {
1057         void __iomem            *base = emif->base;
1058
1059         /* Disable all interrupts */
1060         writel(readl(base + EMIF_SYSTEM_OCP_INTERRUPT_ENABLE_SET),
1061                 base + EMIF_SYSTEM_OCP_INTERRUPT_ENABLE_CLEAR);
1062         if (emif->plat_data->hw_caps & EMIF_HW_CAPS_LL_INTERFACE)
1063                 writel(readl(base + EMIF_LL_OCP_INTERRUPT_ENABLE_SET),
1064                         base + EMIF_LL_OCP_INTERRUPT_ENABLE_CLEAR);
1065
1066         /* Clear all interrupts */
1067         clear_all_interrupts(emif);
1068 }
1069
1070 static int __init_or_module setup_interrupts(struct emif_data *emif, u32 irq)
1071 {
1072         u32             interrupts, type;
1073         void __iomem    *base = emif->base;
1074
1075         type = emif->plat_data->device_info->type;
1076
1077         clear_all_interrupts(emif);
1078
1079         /* Enable interrupts for SYS interface */
1080         interrupts = EN_ERR_SYS_MASK;
1081         if (type == DDR_TYPE_LPDDR2_S2 || type == DDR_TYPE_LPDDR2_S4)
1082                 interrupts |= EN_TA_SYS_MASK;
1083         writel(interrupts, base + EMIF_SYSTEM_OCP_INTERRUPT_ENABLE_SET);
1084
1085         /* Enable interrupts for LL interface */
1086         if (emif->plat_data->hw_caps & EMIF_HW_CAPS_LL_INTERFACE) {
1087                 /* TA need not be enabled for LL */
1088                 interrupts = EN_ERR_LL_MASK;
1089                 writel(interrupts, base + EMIF_LL_OCP_INTERRUPT_ENABLE_SET);
1090         }
1091
1092         /* setup IRQ handlers */
1093         return devm_request_threaded_irq(emif->dev, irq,
1094                                     emif_interrupt_handler,
1095                                     emif_threaded_isr,
1096                                     0, dev_name(emif->dev),
1097                                     emif);
1098
1099 }
1100
1101 static void __init_or_module emif_onetime_settings(struct emif_data *emif)
1102 {
1103         u32                             pwr_mgmt_ctrl, zq, temp_alert_cfg;
1104         void __iomem                    *base = emif->base;
1105         const struct lpddr2_addressing  *addressing;
1106         const struct ddr_device_info    *device_info;
1107
1108         device_info = emif->plat_data->device_info;
1109         addressing = get_addressing_table(device_info);
1110
1111         /*
1112          * Init power management settings
1113          * We don't know the frequency yet. Use a high frequency
1114          * value for a conservative timeout setting
1115          */
1116         pwr_mgmt_ctrl = get_pwr_mgmt_ctrl(1000000000, emif,
1117                         emif->plat_data->ip_rev);
1118         emif->lpmode = (pwr_mgmt_ctrl & LP_MODE_MASK) >> LP_MODE_SHIFT;
1119         writel(pwr_mgmt_ctrl, base + EMIF_POWER_MANAGEMENT_CONTROL);
1120
1121         /* Init ZQ calibration settings */
1122         zq = get_zq_config_reg(addressing, device_info->cs1_used,
1123                 device_info->cal_resistors_per_cs);
1124         writel(zq, base + EMIF_SDRAM_OUTPUT_IMPEDANCE_CALIBRATION_CONFIG);
1125
1126         /* Check temperature level temperature level*/
1127         get_temperature_level(emif);
1128         if (emif->temperature_level == SDRAM_TEMP_VERY_HIGH_SHUTDOWN)
1129                 dev_emerg(emif->dev, "SDRAM temperature exceeds operating limit.. Needs shut down!!!\n");
1130
1131         /* Init temperature polling */
1132         temp_alert_cfg = get_temp_alert_config(addressing,
1133                 emif->plat_data->custom_configs, device_info->cs1_used,
1134                 device_info->io_width, get_emif_bus_width(emif));
1135         writel(temp_alert_cfg, base + EMIF_TEMPERATURE_ALERT_CONFIG);
1136
1137         /*
1138          * Program external PHY control registers that are not frequency
1139          * dependent
1140          */
1141         if (emif->plat_data->phy_type != EMIF_PHY_TYPE_INTELLIPHY)
1142                 return;
1143         writel(EMIF_EXT_PHY_CTRL_1_VAL, base + EMIF_EXT_PHY_CTRL_1_SHDW);
1144         writel(EMIF_EXT_PHY_CTRL_5_VAL, base + EMIF_EXT_PHY_CTRL_5_SHDW);
1145         writel(EMIF_EXT_PHY_CTRL_6_VAL, base + EMIF_EXT_PHY_CTRL_6_SHDW);
1146         writel(EMIF_EXT_PHY_CTRL_7_VAL, base + EMIF_EXT_PHY_CTRL_7_SHDW);
1147         writel(EMIF_EXT_PHY_CTRL_8_VAL, base + EMIF_EXT_PHY_CTRL_8_SHDW);
1148         writel(EMIF_EXT_PHY_CTRL_9_VAL, base + EMIF_EXT_PHY_CTRL_9_SHDW);
1149         writel(EMIF_EXT_PHY_CTRL_10_VAL, base + EMIF_EXT_PHY_CTRL_10_SHDW);
1150         writel(EMIF_EXT_PHY_CTRL_11_VAL, base + EMIF_EXT_PHY_CTRL_11_SHDW);
1151         writel(EMIF_EXT_PHY_CTRL_12_VAL, base + EMIF_EXT_PHY_CTRL_12_SHDW);
1152         writel(EMIF_EXT_PHY_CTRL_13_VAL, base + EMIF_EXT_PHY_CTRL_13_SHDW);
1153         writel(EMIF_EXT_PHY_CTRL_14_VAL, base + EMIF_EXT_PHY_CTRL_14_SHDW);
1154         writel(EMIF_EXT_PHY_CTRL_15_VAL, base + EMIF_EXT_PHY_CTRL_15_SHDW);
1155         writel(EMIF_EXT_PHY_CTRL_16_VAL, base + EMIF_EXT_PHY_CTRL_16_SHDW);
1156         writel(EMIF_EXT_PHY_CTRL_17_VAL, base + EMIF_EXT_PHY_CTRL_17_SHDW);
1157         writel(EMIF_EXT_PHY_CTRL_18_VAL, base + EMIF_EXT_PHY_CTRL_18_SHDW);
1158         writel(EMIF_EXT_PHY_CTRL_19_VAL, base + EMIF_EXT_PHY_CTRL_19_SHDW);
1159         writel(EMIF_EXT_PHY_CTRL_20_VAL, base + EMIF_EXT_PHY_CTRL_20_SHDW);
1160         writel(EMIF_EXT_PHY_CTRL_21_VAL, base + EMIF_EXT_PHY_CTRL_21_SHDW);
1161         writel(EMIF_EXT_PHY_CTRL_22_VAL, base + EMIF_EXT_PHY_CTRL_22_SHDW);
1162         writel(EMIF_EXT_PHY_CTRL_23_VAL, base + EMIF_EXT_PHY_CTRL_23_SHDW);
1163         writel(EMIF_EXT_PHY_CTRL_24_VAL, base + EMIF_EXT_PHY_CTRL_24_SHDW);
1164 }
1165
1166 static void get_default_timings(struct emif_data *emif)
1167 {
1168         struct emif_platform_data *pd = emif->plat_data;
1169
1170         pd->timings             = lpddr2_jedec_timings;
1171         pd->timings_arr_size    = ARRAY_SIZE(lpddr2_jedec_timings);
1172
1173         dev_warn(emif->dev, "%s: using default timings\n", __func__);
1174 }
1175
1176 static int is_dev_data_valid(u32 type, u32 density, u32 io_width, u32 phy_type,
1177                 u32 ip_rev, struct device *dev)
1178 {
1179         int valid;
1180
1181         valid = (type == DDR_TYPE_LPDDR2_S4 ||
1182                         type == DDR_TYPE_LPDDR2_S2)
1183                 && (density >= DDR_DENSITY_64Mb
1184                         && density <= DDR_DENSITY_8Gb)
1185                 && (io_width >= DDR_IO_WIDTH_8
1186                         && io_width <= DDR_IO_WIDTH_32);
1187
1188         /* Combinations of EMIF and PHY revisions that we support today */
1189         switch (ip_rev) {
1190         case EMIF_4D:
1191                 valid = valid && (phy_type == EMIF_PHY_TYPE_ATTILAPHY);
1192                 break;
1193         case EMIF_4D5:
1194                 valid = valid && (phy_type == EMIF_PHY_TYPE_INTELLIPHY);
1195                 break;
1196         default:
1197                 valid = 0;
1198         }
1199
1200         if (!valid)
1201                 dev_err(dev, "%s: invalid DDR details\n", __func__);
1202         return valid;
1203 }
1204
1205 static int is_custom_config_valid(struct emif_custom_configs *cust_cfgs,
1206                 struct device *dev)
1207 {
1208         int valid = 1;
1209
1210         if ((cust_cfgs->mask & EMIF_CUSTOM_CONFIG_LPMODE) &&
1211                 (cust_cfgs->lpmode != EMIF_LP_MODE_DISABLE))
1212                 valid = cust_cfgs->lpmode_freq_threshold &&
1213                         cust_cfgs->lpmode_timeout_performance &&
1214                         cust_cfgs->lpmode_timeout_power;
1215
1216         if (cust_cfgs->mask & EMIF_CUSTOM_CONFIG_TEMP_ALERT_POLL_INTERVAL)
1217                 valid = valid && cust_cfgs->temp_alert_poll_interval_ms;
1218
1219         if (!valid)
1220                 dev_warn(dev, "%s: invalid custom configs\n", __func__);
1221
1222         return valid;
1223 }
1224
1225 #if defined(CONFIG_OF)
1226 static void __init_or_module of_get_custom_configs(struct device_node *np_emif,
1227                 struct emif_data *emif)
1228 {
1229         struct emif_custom_configs      *cust_cfgs = NULL;
1230         int                             len;
1231         const int                       *lpmode, *poll_intvl;
1232
1233         lpmode = of_get_property(np_emif, "low-power-mode", &len);
1234         poll_intvl = of_get_property(np_emif, "temp-alert-poll-interval", &len);
1235
1236         if (lpmode || poll_intvl)
1237                 cust_cfgs = devm_kzalloc(emif->dev, sizeof(*cust_cfgs),
1238                         GFP_KERNEL);
1239
1240         if (!cust_cfgs)
1241                 return;
1242
1243         if (lpmode) {
1244                 cust_cfgs->mask |= EMIF_CUSTOM_CONFIG_LPMODE;
1245                 cust_cfgs->lpmode = *lpmode;
1246                 of_property_read_u32(np_emif,
1247                                 "low-power-mode-timeout-performance",
1248                                 &cust_cfgs->lpmode_timeout_performance);
1249                 of_property_read_u32(np_emif,
1250                                 "low-power-mode-timeout-power",
1251                                 &cust_cfgs->lpmode_timeout_power);
1252                 of_property_read_u32(np_emif,
1253                                 "low-power-mode-freq-threshold",
1254                                 &cust_cfgs->lpmode_freq_threshold);
1255         }
1256
1257         if (poll_intvl) {
1258                 cust_cfgs->mask |=
1259                                 EMIF_CUSTOM_CONFIG_TEMP_ALERT_POLL_INTERVAL;
1260                 cust_cfgs->temp_alert_poll_interval_ms = *poll_intvl;
1261         }
1262
1263         if (of_find_property(np_emif, "extended-temp-part", &len))
1264                 cust_cfgs->mask |= EMIF_CUSTOM_CONFIG_EXTENDED_TEMP_PART;
1265
1266         if (!is_custom_config_valid(cust_cfgs, emif->dev)) {
1267                 devm_kfree(emif->dev, cust_cfgs);
1268                 return;
1269         }
1270
1271         emif->plat_data->custom_configs = cust_cfgs;
1272 }
1273
1274 static void __init_or_module of_get_ddr_info(struct device_node *np_emif,
1275                 struct device_node *np_ddr,
1276                 struct ddr_device_info *dev_info)
1277 {
1278         u32 density = 0, io_width = 0;
1279         int len;
1280
1281         if (of_find_property(np_emif, "cs1-used", &len))
1282                 dev_info->cs1_used = true;
1283
1284         if (of_find_property(np_emif, "cal-resistor-per-cs", &len))
1285                 dev_info->cal_resistors_per_cs = true;
1286
1287         if (of_device_is_compatible(np_ddr , "jedec,lpddr2-s4"))
1288                 dev_info->type = DDR_TYPE_LPDDR2_S4;
1289         else if (of_device_is_compatible(np_ddr , "jedec,lpddr2-s2"))
1290                 dev_info->type = DDR_TYPE_LPDDR2_S2;
1291
1292         of_property_read_u32(np_ddr, "density", &density);
1293         of_property_read_u32(np_ddr, "io-width", &io_width);
1294
1295         /* Convert from density in Mb to the density encoding in jedc_ddr.h */
1296         if (density & (density - 1))
1297                 dev_info->density = 0;
1298         else
1299                 dev_info->density = __fls(density) - 5;
1300
1301         /* Convert from io_width in bits to io_width encoding in jedc_ddr.h */
1302         if (io_width & (io_width - 1))
1303                 dev_info->io_width = 0;
1304         else
1305                 dev_info->io_width = __fls(io_width) - 1;
1306 }
1307
1308 static struct emif_data * __init_or_module of_get_memory_device_details(
1309                 struct device_node *np_emif, struct device *dev)
1310 {
1311         struct emif_data                *emif = NULL;
1312         struct ddr_device_info          *dev_info = NULL;
1313         struct emif_platform_data       *pd = NULL;
1314         struct device_node              *np_ddr;
1315         int                             len;
1316
1317         np_ddr = of_parse_phandle(np_emif, "device-handle", 0);
1318         if (!np_ddr)
1319                 goto error;
1320         emif    = devm_kzalloc(dev, sizeof(struct emif_data), GFP_KERNEL);
1321         pd      = devm_kzalloc(dev, sizeof(*pd), GFP_KERNEL);
1322         dev_info = devm_kzalloc(dev, sizeof(*dev_info), GFP_KERNEL);
1323
1324         if (!emif || !pd || !dev_info) {
1325                 dev_err(dev, "%s: Out of memory!!\n",
1326                         __func__);
1327                 goto error;
1328         }
1329
1330         emif->plat_data         = pd;
1331         pd->device_info         = dev_info;
1332         emif->dev               = dev;
1333         emif->np_ddr            = np_ddr;
1334         emif->temperature_level = SDRAM_TEMP_NOMINAL;
1335
1336         if (of_device_is_compatible(np_emif, "ti,emif-4d"))
1337                 emif->plat_data->ip_rev = EMIF_4D;
1338         else if (of_device_is_compatible(np_emif, "ti,emif-4d5"))
1339                 emif->plat_data->ip_rev = EMIF_4D5;
1340
1341         of_property_read_u32(np_emif, "phy-type", &pd->phy_type);
1342
1343         if (of_find_property(np_emif, "hw-caps-ll-interface", &len))
1344                 pd->hw_caps |= EMIF_HW_CAPS_LL_INTERFACE;
1345
1346         of_get_ddr_info(np_emif, np_ddr, dev_info);
1347         if (!is_dev_data_valid(pd->device_info->type, pd->device_info->density,
1348                         pd->device_info->io_width, pd->phy_type, pd->ip_rev,
1349                         emif->dev)) {
1350                 dev_err(dev, "%s: invalid device data!!\n", __func__);
1351                 goto error;
1352         }
1353         /*
1354          * For EMIF instances other than EMIF1 see if the devices connected
1355          * are exactly same as on EMIF1(which is typically the case). If so,
1356          * mark it as a duplicate of EMIF1. This will save some memory and
1357          * computation.
1358          */
1359         if (emif1 && emif1->np_ddr == np_ddr) {
1360                 emif->duplicate = true;
1361                 goto out;
1362         } else if (emif1) {
1363                 dev_warn(emif->dev, "%s: Non-symmetric DDR geometry\n",
1364                         __func__);
1365         }
1366
1367         of_get_custom_configs(np_emif, emif);
1368         emif->plat_data->timings = of_get_ddr_timings(np_ddr, emif->dev,
1369                                         emif->plat_data->device_info->type,
1370                                         &emif->plat_data->timings_arr_size);
1371
1372         emif->plat_data->min_tck = of_get_min_tck(np_ddr, emif->dev);
1373         goto out;
1374
1375 error:
1376         return NULL;
1377 out:
1378         return emif;
1379 }
1380
1381 #else
1382
1383 static struct emif_data * __init_or_module of_get_memory_device_details(
1384                 struct device_node *np_emif, struct device *dev)
1385 {
1386         return NULL;
1387 }
1388 #endif
1389
1390 static struct emif_data *__init_or_module get_device_details(
1391                 struct platform_device *pdev)
1392 {
1393         u32                             size;
1394         struct emif_data                *emif = NULL;
1395         struct ddr_device_info          *dev_info;
1396         struct emif_custom_configs      *cust_cfgs;
1397         struct emif_platform_data       *pd;
1398         struct device                   *dev;
1399         void                            *temp;
1400
1401         pd = pdev->dev.platform_data;
1402         dev = &pdev->dev;
1403
1404         if (!(pd && pd->device_info && is_dev_data_valid(pd->device_info->type,
1405                         pd->device_info->density, pd->device_info->io_width,
1406                         pd->phy_type, pd->ip_rev, dev))) {
1407                 dev_err(dev, "%s: invalid device data\n", __func__);
1408                 goto error;
1409         }
1410
1411         emif    = devm_kzalloc(dev, sizeof(*emif), GFP_KERNEL);
1412         temp    = devm_kzalloc(dev, sizeof(*pd), GFP_KERNEL);
1413         dev_info = devm_kzalloc(dev, sizeof(*dev_info), GFP_KERNEL);
1414
1415         if (!emif || !pd || !dev_info) {
1416                 dev_err(dev, "%s:%d: allocation error\n", __func__, __LINE__);
1417                 goto error;
1418         }
1419
1420         memcpy(temp, pd, sizeof(*pd));
1421         pd = temp;
1422         memcpy(dev_info, pd->device_info, sizeof(*dev_info));
1423
1424         pd->device_info         = dev_info;
1425         emif->plat_data         = pd;
1426         emif->dev               = dev;
1427         emif->temperature_level = SDRAM_TEMP_NOMINAL;
1428
1429         /*
1430          * For EMIF instances other than EMIF1 see if the devices connected
1431          * are exactly same as on EMIF1(which is typically the case). If so,
1432          * mark it as a duplicate of EMIF1 and skip copying timings data.
1433          * This will save some memory and some computation later.
1434          */
1435         emif->duplicate = emif1 && (memcmp(dev_info,
1436                 emif1->plat_data->device_info,
1437                 sizeof(struct ddr_device_info)) == 0);
1438
1439         if (emif->duplicate) {
1440                 pd->timings = NULL;
1441                 pd->min_tck = NULL;
1442                 goto out;
1443         } else if (emif1) {
1444                 dev_warn(emif->dev, "%s: Non-symmetric DDR geometry\n",
1445                         __func__);
1446         }
1447
1448         /*
1449          * Copy custom configs - ignore allocation error, if any, as
1450          * custom_configs is not very critical
1451          */
1452         cust_cfgs = pd->custom_configs;
1453         if (cust_cfgs && is_custom_config_valid(cust_cfgs, dev)) {
1454                 temp = devm_kzalloc(dev, sizeof(*cust_cfgs), GFP_KERNEL);
1455                 if (temp)
1456                         memcpy(temp, cust_cfgs, sizeof(*cust_cfgs));
1457                 else
1458                         dev_warn(dev, "%s:%d: allocation error\n", __func__,
1459                                 __LINE__);
1460                 pd->custom_configs = temp;
1461         }
1462
1463         /*
1464          * Copy timings and min-tck values from platform data. If it is not
1465          * available or if memory allocation fails, use JEDEC defaults
1466          */
1467         size = sizeof(struct lpddr2_timings) * pd->timings_arr_size;
1468         if (pd->timings) {
1469                 temp = devm_kzalloc(dev, size, GFP_KERNEL);
1470                 if (temp) {
1471                         memcpy(temp, pd->timings, sizeof(*pd->timings));
1472                         pd->timings = temp;
1473                 } else {
1474                         dev_warn(dev, "%s:%d: allocation error\n", __func__,
1475                                 __LINE__);
1476                         get_default_timings(emif);
1477                 }
1478         } else {
1479                 get_default_timings(emif);
1480         }
1481
1482         if (pd->min_tck) {
1483                 temp = devm_kzalloc(dev, sizeof(*pd->min_tck), GFP_KERNEL);
1484                 if (temp) {
1485                         memcpy(temp, pd->min_tck, sizeof(*pd->min_tck));
1486                         pd->min_tck = temp;
1487                 } else {
1488                         dev_warn(dev, "%s:%d: allocation error\n", __func__,
1489                                 __LINE__);
1490                         pd->min_tck = &lpddr2_jedec_min_tck;
1491                 }
1492         } else {
1493                 pd->min_tck = &lpddr2_jedec_min_tck;
1494         }
1495
1496 out:
1497         return emif;
1498
1499 error:
1500         return NULL;
1501 }
1502
1503 static int __init_or_module emif_probe(struct platform_device *pdev)
1504 {
1505         struct emif_data        *emif;
1506         struct resource         *res;
1507         int                     irq;
1508
1509         if (pdev->dev.of_node)
1510                 emif = of_get_memory_device_details(pdev->dev.of_node, &pdev->dev);
1511         else
1512                 emif = get_device_details(pdev);
1513
1514         if (!emif) {
1515                 pr_err("%s: error getting device data\n", __func__);
1516                 goto error;
1517         }
1518
1519         list_add(&emif->node, &device_list);
1520         emif->addressing = get_addressing_table(emif->plat_data->device_info);
1521
1522         /* Save pointers to each other in emif and device structures */
1523         emif->dev = &pdev->dev;
1524         platform_set_drvdata(pdev, emif);
1525
1526         res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
1527         if (!res) {
1528                 dev_err(emif->dev, "%s: error getting memory resource\n",
1529                         __func__);
1530                 goto error;
1531         }
1532
1533         emif->base = devm_ioremap_resource(emif->dev, res);
1534         if (IS_ERR(emif->base))
1535                 goto error;
1536
1537         irq = platform_get_irq(pdev, 0);
1538         if (irq < 0) {
1539                 dev_err(emif->dev, "%s: error getting IRQ resource - %d\n",
1540                         __func__, irq);
1541                 goto error;
1542         }
1543
1544         emif_onetime_settings(emif);
1545         emif_debugfs_init(emif);
1546         disable_and_clear_all_interrupts(emif);
1547         setup_interrupts(emif, irq);
1548
1549         /* One-time actions taken on probing the first device */
1550         if (!emif1) {
1551                 emif1 = emif;
1552                 spin_lock_init(&emif_lock);
1553
1554                 /*
1555                  * TODO: register notifiers for frequency and voltage
1556                  * change here once the respective frameworks are
1557                  * available
1558                  */
1559         }
1560
1561         dev_info(&pdev->dev, "%s: device configured with addr = %p and IRQ%d\n",
1562                 __func__, emif->base, irq);
1563
1564         return 0;
1565 error:
1566         return -ENODEV;
1567 }
1568
1569 static int __exit emif_remove(struct platform_device *pdev)
1570 {
1571         struct emif_data *emif = platform_get_drvdata(pdev);
1572
1573         emif_debugfs_exit(emif);
1574
1575         return 0;
1576 }
1577
1578 static void emif_shutdown(struct platform_device *pdev)
1579 {
1580         struct emif_data        *emif = platform_get_drvdata(pdev);
1581
1582         disable_and_clear_all_interrupts(emif);
1583 }
1584
1585 static int get_emif_reg_values(struct emif_data *emif, u32 freq,
1586                 struct emif_regs *regs)
1587 {
1588         u32                             cs1_used, ip_rev, phy_type;
1589         u32                             cl, type;
1590         const struct lpddr2_timings     *timings;
1591         const struct lpddr2_min_tck     *min_tck;
1592         const struct ddr_device_info    *device_info;
1593         const struct lpddr2_addressing  *addressing;
1594         struct emif_data                *emif_for_calc;
1595         struct device                   *dev;
1596         const struct emif_custom_configs *custom_configs;
1597
1598         dev = emif->dev;
1599         /*
1600          * If the devices on this EMIF instance is duplicate of EMIF1,
1601          * use EMIF1 details for the calculation
1602          */
1603         emif_for_calc   = emif->duplicate ? emif1 : emif;
1604         timings         = get_timings_table(emif_for_calc, freq);
1605         addressing      = emif_for_calc->addressing;
1606         if (!timings || !addressing) {
1607                 dev_err(dev, "%s: not enough data available for %dHz",
1608                         __func__, freq);
1609                 return -1;
1610         }
1611
1612         device_info     = emif_for_calc->plat_data->device_info;
1613         type            = device_info->type;
1614         cs1_used        = device_info->cs1_used;
1615         ip_rev          = emif_for_calc->plat_data->ip_rev;
1616         phy_type        = emif_for_calc->plat_data->phy_type;
1617
1618         min_tck         = emif_for_calc->plat_data->min_tck;
1619         custom_configs  = emif_for_calc->plat_data->custom_configs;
1620
1621         set_ddr_clk_period(freq);
1622
1623         regs->ref_ctrl_shdw = get_sdram_ref_ctrl_shdw(freq, addressing);
1624         regs->sdram_tim1_shdw = get_sdram_tim_1_shdw(timings, min_tck,
1625                         addressing);
1626         regs->sdram_tim2_shdw = get_sdram_tim_2_shdw(timings, min_tck,
1627                         addressing, type);
1628         regs->sdram_tim3_shdw = get_sdram_tim_3_shdw(timings, min_tck,
1629                 addressing, type, ip_rev, EMIF_NORMAL_TIMINGS);
1630
1631         cl = get_cl(emif);
1632
1633         if (phy_type == EMIF_PHY_TYPE_ATTILAPHY && ip_rev == EMIF_4D) {
1634                 regs->phy_ctrl_1_shdw = get_ddr_phy_ctrl_1_attilaphy_4d(
1635                         timings, freq, cl);
1636         } else if (phy_type == EMIF_PHY_TYPE_INTELLIPHY && ip_rev == EMIF_4D5) {
1637                 regs->phy_ctrl_1_shdw = get_phy_ctrl_1_intelliphy_4d5(freq, cl);
1638                 regs->ext_phy_ctrl_2_shdw = get_ext_phy_ctrl_2_intelliphy_4d5();
1639                 regs->ext_phy_ctrl_3_shdw = get_ext_phy_ctrl_3_intelliphy_4d5();
1640                 regs->ext_phy_ctrl_4_shdw = get_ext_phy_ctrl_4_intelliphy_4d5();
1641         } else {
1642                 return -1;
1643         }
1644
1645         /* Only timeout values in pwr_mgmt_ctrl_shdw register */
1646         regs->pwr_mgmt_ctrl_shdw =
1647                 get_pwr_mgmt_ctrl(freq, emif_for_calc, ip_rev) &
1648                 (CS_TIM_MASK | SR_TIM_MASK | PD_TIM_MASK);
1649
1650         if (ip_rev & EMIF_4D) {
1651                 regs->read_idle_ctrl_shdw_normal =
1652                         get_read_idle_ctrl_shdw(DDR_VOLTAGE_STABLE);
1653
1654                 regs->read_idle_ctrl_shdw_volt_ramp =
1655                         get_read_idle_ctrl_shdw(DDR_VOLTAGE_RAMPING);
1656         } else if (ip_rev & EMIF_4D5) {
1657                 regs->dll_calib_ctrl_shdw_normal =
1658                         get_dll_calib_ctrl_shdw(DDR_VOLTAGE_STABLE);
1659
1660                 regs->dll_calib_ctrl_shdw_volt_ramp =
1661                         get_dll_calib_ctrl_shdw(DDR_VOLTAGE_RAMPING);
1662         }
1663
1664         if (type == DDR_TYPE_LPDDR2_S2 || type == DDR_TYPE_LPDDR2_S4) {
1665                 regs->ref_ctrl_shdw_derated = get_sdram_ref_ctrl_shdw(freq / 4,
1666                         addressing);
1667
1668                 regs->sdram_tim1_shdw_derated =
1669                         get_sdram_tim_1_shdw_derated(timings, min_tck,
1670                                 addressing);
1671
1672                 regs->sdram_tim3_shdw_derated = get_sdram_tim_3_shdw(timings,
1673                         min_tck, addressing, type, ip_rev,
1674                         EMIF_DERATED_TIMINGS);
1675         }
1676
1677         regs->freq = freq;
1678
1679         return 0;
1680 }
1681
1682 /*
1683  * get_regs() - gets the cached emif_regs structure for a given EMIF instance
1684  * given frequency(freq):
1685  *
1686  * As an optimisation, every EMIF instance other than EMIF1 shares the
1687  * register cache with EMIF1 if the devices connected on this instance
1688  * are same as that on EMIF1(indicated by the duplicate flag)
1689  *
1690  * If we do not have an entry corresponding to the frequency given, we
1691  * allocate a new entry and calculate the values
1692  *
1693  * Upon finding the right reg dump, save it in curr_regs. It can be
1694  * directly used for thermal de-rating and voltage ramping changes.
1695  */
1696 static struct emif_regs *get_regs(struct emif_data *emif, u32 freq)
1697 {
1698         int                     i;
1699         struct emif_regs        **regs_cache;
1700         struct emif_regs        *regs = NULL;
1701         struct device           *dev;
1702
1703         dev = emif->dev;
1704         if (emif->curr_regs && emif->curr_regs->freq == freq) {
1705                 dev_dbg(dev, "%s: using curr_regs - %u Hz", __func__, freq);
1706                 return emif->curr_regs;
1707         }
1708
1709         if (emif->duplicate)
1710                 regs_cache = emif1->regs_cache;
1711         else
1712                 regs_cache = emif->regs_cache;
1713
1714         for (i = 0; i < EMIF_MAX_NUM_FREQUENCIES && regs_cache[i]; i++) {
1715                 if (regs_cache[i]->freq == freq) {
1716                         regs = regs_cache[i];
1717                         dev_dbg(dev,
1718                                 "%s: reg dump found in reg cache for %u Hz\n",
1719                                 __func__, freq);
1720                         break;
1721                 }
1722         }
1723
1724         /*
1725          * If we don't have an entry for this frequency in the cache create one
1726          * and calculate the values
1727          */
1728         if (!regs) {
1729                 regs = devm_kzalloc(emif->dev, sizeof(*regs), GFP_ATOMIC);
1730                 if (!regs)
1731                         return NULL;
1732
1733                 if (get_emif_reg_values(emif, freq, regs)) {
1734                         devm_kfree(emif->dev, regs);
1735                         return NULL;
1736                 }
1737
1738                 /*
1739                  * Now look for an un-used entry in the cache and save the
1740                  * newly created struct. If there are no free entries
1741                  * over-write the last entry
1742                  */
1743                 for (i = 0; i < EMIF_MAX_NUM_FREQUENCIES && regs_cache[i]; i++)
1744                         ;
1745
1746                 if (i >= EMIF_MAX_NUM_FREQUENCIES) {
1747                         dev_warn(dev, "%s: regs_cache full - reusing a slot!!\n",
1748                                 __func__);
1749                         i = EMIF_MAX_NUM_FREQUENCIES - 1;
1750                         devm_kfree(emif->dev, regs_cache[i]);
1751                 }
1752                 regs_cache[i] = regs;
1753         }
1754
1755         return regs;
1756 }
1757
1758 static void do_volt_notify_handling(struct emif_data *emif, u32 volt_state)
1759 {
1760         dev_dbg(emif->dev, "%s: voltage notification : %d", __func__,
1761                 volt_state);
1762
1763         if (!emif->curr_regs) {
1764                 dev_err(emif->dev,
1765                         "%s: volt-notify before registers are ready: %d\n",
1766                         __func__, volt_state);
1767                 return;
1768         }
1769
1770         setup_volt_sensitive_regs(emif, emif->curr_regs, volt_state);
1771 }
1772
1773 /*
1774  * TODO: voltage notify handling should be hooked up to
1775  * regulator framework as soon as the necessary support
1776  * is available in mainline kernel. This function is un-used
1777  * right now.
1778  */
1779 static void __attribute__((unused)) volt_notify_handling(u32 volt_state)
1780 {
1781         struct emif_data *emif;
1782
1783         spin_lock_irqsave(&emif_lock, irq_state);
1784
1785         list_for_each_entry(emif, &device_list, node)
1786                 do_volt_notify_handling(emif, volt_state);
1787         do_freq_update();
1788
1789         spin_unlock_irqrestore(&emif_lock, irq_state);
1790 }
1791
1792 static void do_freq_pre_notify_handling(struct emif_data *emif, u32 new_freq)
1793 {
1794         struct emif_regs *regs;
1795
1796         regs = get_regs(emif, new_freq);
1797         if (!regs)
1798                 return;
1799
1800         emif->curr_regs = regs;
1801
1802         /*
1803          * Update the shadow registers:
1804          * Temperature and voltage-ramp sensitive settings are also configured
1805          * in terms of DDR cycles. So, we need to update them too when there
1806          * is a freq change
1807          */
1808         dev_dbg(emif->dev, "%s: setting up shadow registers for %uHz",
1809                 __func__, new_freq);
1810         setup_registers(emif, regs);
1811         setup_temperature_sensitive_regs(emif, regs);
1812         setup_volt_sensitive_regs(emif, regs, DDR_VOLTAGE_STABLE);
1813
1814         /*
1815          * Part of workaround for errata i728. See do_freq_update()
1816          * for more details
1817          */
1818         if (emif->lpmode == EMIF_LP_MODE_SELF_REFRESH)
1819                 set_lpmode(emif, EMIF_LP_MODE_DISABLE);
1820 }
1821
1822 /*
1823  * TODO: frequency notify handling should be hooked up to
1824  * clock framework as soon as the necessary support is
1825  * available in mainline kernel. This function is un-used
1826  * right now.
1827  */
1828 static void __attribute__((unused)) freq_pre_notify_handling(u32 new_freq)
1829 {
1830         struct emif_data *emif;
1831
1832         /*
1833          * NOTE: we are taking the spin-lock here and releases it
1834          * only in post-notifier. This doesn't look good and
1835          * Sparse complains about it, but this seems to be
1836          * un-avoidable. We need to lock a sequence of events
1837          * that is split between EMIF and clock framework.
1838          *
1839          * 1. EMIF driver updates EMIF timings in shadow registers in the
1840          *    frequency pre-notify callback from clock framework
1841          * 2. clock framework sets up the registers for the new frequency
1842          * 3. clock framework initiates a hw-sequence that updates
1843          *    the frequency EMIF timings synchronously.
1844          *
1845          * All these 3 steps should be performed as an atomic operation
1846          * vis-a-vis similar sequence in the EMIF interrupt handler
1847          * for temperature events. Otherwise, there could be race
1848          * conditions that could result in incorrect EMIF timings for
1849          * a given frequency
1850          */
1851         spin_lock_irqsave(&emif_lock, irq_state);
1852
1853         list_for_each_entry(emif, &device_list, node)
1854                 do_freq_pre_notify_handling(emif, new_freq);
1855 }
1856
1857 static void do_freq_post_notify_handling(struct emif_data *emif)
1858 {
1859         /*
1860          * Part of workaround for errata i728. See do_freq_update()
1861          * for more details
1862          */
1863         if (emif->lpmode == EMIF_LP_MODE_SELF_REFRESH)
1864                 set_lpmode(emif, EMIF_LP_MODE_SELF_REFRESH);
1865 }
1866
1867 /*
1868  * TODO: frequency notify handling should be hooked up to
1869  * clock framework as soon as the necessary support is
1870  * available in mainline kernel. This function is un-used
1871  * right now.
1872  */
1873 static void __attribute__((unused)) freq_post_notify_handling(void)
1874 {
1875         struct emif_data *emif;
1876
1877         list_for_each_entry(emif, &device_list, node)
1878                 do_freq_post_notify_handling(emif);
1879
1880         /*
1881          * Lock is done in pre-notify handler. See freq_pre_notify_handling()
1882          * for more details
1883          */
1884         spin_unlock_irqrestore(&emif_lock, irq_state);
1885 }
1886
1887 #if defined(CONFIG_OF)
1888 static const struct of_device_id emif_of_match[] = {
1889                 { .compatible = "ti,emif-4d" },
1890                 { .compatible = "ti,emif-4d5" },
1891                 {},
1892 };
1893 MODULE_DEVICE_TABLE(of, emif_of_match);
1894 #endif
1895
1896 static struct platform_driver emif_driver = {
1897         .remove         = __exit_p(emif_remove),
1898         .shutdown       = emif_shutdown,
1899         .driver = {
1900                 .name = "emif",
1901                 .of_match_table = of_match_ptr(emif_of_match),
1902         },
1903 };
1904
1905 module_platform_driver_probe(emif_driver, emif_probe);
1906
1907 MODULE_DESCRIPTION("TI EMIF SDRAM Controller Driver");
1908 MODULE_LICENSE("GPL");
1909 MODULE_ALIAS("platform:emif");
1910 MODULE_AUTHOR("Texas Instruments Inc");