arch/mips/mach-octeon/include/mach/octeon_ddr.h

   1 /* SPDX-License-Identifier: GPL-2.0 */
   2 /*
   3  * Copyright (C) 2020 Marvell International Ltd.
   4  */
   5
   6 #ifndef __OCTEON_DDR_H_
   7 #define __OCTEON_DDR_H_
   8
   9 #include <env.h>
  10 #include <linux/compat.h>
  11 #include <linux/delay.h>
  12 #include <linux/io.h>
  13 #include <mach/octeon-model.h>
  14 #include <mach/cvmx-lmcx-defs.h>
  15 #include <mach/cvmx-regs.h>
  16 #include <mach/cvmx-l2c-defs.h>
  17
  18 /* Some "external" (non-LMC) registers */
  19 #define CVMX_IPD_CLK_COUNT              0x00014F0000000338
  20 #define CVMX_FPA_CLK_COUNT              0x00012800000000F0
  21
  22 #define CVMX_NODE_MEM_SHIFT     40
  23
  24 #define DDR_INTERFACE_MAX       4
  25
  26 /* Private data struct */
  27 struct ddr_priv {
  28         void __iomem *lmc_base;
  29         void __iomem *l2c_base;
  30
  31         bool ddr_clock_initialized[DDR_INTERFACE_MAX];
  32         bool ddr_memory_preserved;
  33         u32 flags;
  34
  35         struct ram_info info;
  36 };
  37
  38 /* Short cut to convert a number to megabytes */
  39 #define MB(X)                   ((u64)(X) * (u64)(1024 * 1024))
  40
  41 #define octeon_is_cpuid(x)      (__OCTEON_IS_MODEL_COMPILE__(x, read_c0_prid()))
  42
  43 #define strtoull                simple_strtoull
  44
  45 /* Access LMC registers */
  46 static inline u64 lmc_rd(struct ddr_priv *priv, u64 addr)
  47 {
  48         return ioread64(priv->lmc_base + addr);
  49 }
  50
  51 static inline void lmc_wr(struct ddr_priv *priv, u64 addr, u64 val)
  52 {
  53         iowrite64(val, priv->lmc_base + addr);
  54 }
  55
  56 /* Access L2C registers */
  57 static inline u64 l2c_rd(struct ddr_priv *priv, u64 addr)
  58 {
  59         return ioread64(priv->l2c_base + addr);
  60 }
  61
  62 static inline void l2c_wr(struct ddr_priv *priv, u64 addr, u64 val)
  63 {
  64         iowrite64(val, priv->l2c_base + addr);
  65 }
  66
  67 /* Failsafe mode */
  68 #define FLAG_FAILSAFE_MODE              0x01000
  69 /* Note that the DDR clock initialized flags must be contiguous */
  70 /* Clock for DDR 0 initialized */
  71 #define FLAG_DDR0_CLK_INITIALIZED       0x02000
  72 /* Clock for DDR 1 initialized */
  73 #define FLAG_DDR1_CLK_INITIALIZED       0x04000
  74 /* Clock for DDR 2 initialized */
  75 #define FLAG_DDR2_CLK_INITIALIZED       0x08000
  76 /* Clock for DDR 3 initialized */
  77 #define FLAG_DDR3_CLK_INITIALIZED       0x10000
  78 /* Loaded into RAM externally */
  79 #define FLAG_RAM_RESIDENT               0x20000
  80 /* Verbose DDR information */
  81 #define FLAG_DDR_VERBOSE                0x40000
  82 /* Check env. for DDR variables */
  83 #define FLAG_DDR_DEBUG                  0x80000
  84 #define FLAG_DDR_TRACE_INIT             0x100000
  85 #define FLAG_MEMORY_PRESERVED           0x200000
  86 #define FLAG_DFM_VERBOSE                0x400000
  87 #define FLAG_DFM_TRACE_INIT             0x800000
  88 /* DFM memory clock initialized */
  89 #define FLAG_DFM_CLK_INITIALIZED        0x1000000
  90 /* EEPROM clock descr. missing */
  91 #define FLAG_CLOCK_DESC_MISSING         0x2000000
  92 /* EEPROM board descr. missing */
  93 #define FLAG_BOARD_DESC_MISSING         0x4000000
  94 #define FLAG_DDR_PROMPT                 0x8000000
  95
  96 #ifndef DDR_NO_DEBUG
  97 static inline int ddr_verbose(struct ddr_priv *priv)
  98 {
  99         return !!(priv->flags & FLAG_DDR_VERBOSE);
 100 }
 101
 102 static inline char *ddr_getenv_debug(struct ddr_priv *priv, char *name)
 103 {
 104         if (priv->flags & FLAG_FAILSAFE_MODE)
 105                 return NULL;
 106
 107         if (priv->flags & FLAG_DDR_DEBUG)
 108                 return env_get(name);
 109
 110         return NULL;
 111 }
 112 #else
 113 static inline int ddr_verbose(void)
 114 {
 115         return 0;
 116 }
 117 #endif
 118
 119 /* turn the variable name into a string */
 120 #define CVMX_TMP_STR(x) CVMX_TMP_STR2(x)
 121 #define CVMX_TMP_STR2(x) #x
 122
 123 #define CVMX_CACHE(op, address, offset)                                 \
 124         asm volatile ("cache " CVMX_TMP_STR(op) ", "                    \
 125                       CVMX_TMP_STR(offset) "(%[rbase])"                 \
 126                       : : [rbase] "d" (address))
 127
 128 /* unlock the state */
 129 #define CVMX_CACHE_WBIL2(address, offset)       \
 130         CVMX_CACHE(23, address, offset)
 131
 132 /* complete prefetches, invalidate entire dcache */
 133 #define CVMX_DCACHE_INVALIDATE                                  \
 134         { CVMX_SYNC; asm volatile ("cache 9, 0($0)" : : ); }
 135
 136 #define DEBUG_VALIDATE_BITMASK 0
 137 #if DEBUG_VALIDATE_BITMASK
 138 #define debug_bitmask_print printf
 139 #else
 140 #define debug_bitmask_print(...)
 141 #endif
 142
 143 #define RLEVEL_BITMASK_TRAILING_BITS_ERROR      5
 144 // FIXME? now less than TOOLONG
 145 #define RLEVEL_BITMASK_BUBBLE_BITS_ERROR        11
 146 #define RLEVEL_BITMASK_NARROW_ERROR             6
 147 #define RLEVEL_BITMASK_BLANK_ERROR              100
 148 #define RLEVEL_BITMASK_TOOLONG_ERROR            12
 149 #define RLEVEL_NONSEQUENTIAL_DELAY_ERROR        50
 150 #define RLEVEL_ADJACENT_DELAY_ERROR             30
 151
 152 /*
 153  * Apply a filter to the BITMASK results returned from Octeon
 154  * read-leveling to determine the most likely delay result.  This
 155  * computed delay may be used to qualify the delay result returned by
 156  * Octeon. Accumulate an error penalty for invalid characteristics of
 157  * the bitmask so that they can be used to select the most reliable
 158  * results.
 159  *
 160  * The algorithm searches for the largest contiguous MASK within a
 161  * maximum RANGE of bits beginning with the MSB.
 162  *
 163  * 1. a MASK with a WIDTH less than 4 will be penalized
 164  * 2. Bubbles in the bitmask that occur before or after the MASK
 165  *    will be penalized
 166  * 3. If there are no trailing bubbles then extra bits that occur
 167  *    beyond the maximum RANGE will be penalized.
 168  *
 169  *   +++++++++++++++++++++++++++++++++++++++++++++++++++
 170  *   +                                                 +
 171  *   +   e.g. bitmask = 27B00                          +
 172  *   +                                                 +
 173  *   +   63                  +--- mstart           0   +
 174  *   +   |                   |                     |   +
 175  *   +   |         +---------+     +--- fb         |   +
 176  *   +   |         |  range  |     |               |   +
 177  *   +   V         V         V     V               V   +
 178  *   +                                                 +
 179  *   +   0 0 ... 1 0 0 1 1 1 1 0 1 1 0 0 0 0 0 0 0 0   +
 180  *   +                                                 +
 181  *   +           ^     ^     ^                         +
 182  *   +           |     | mask|                         +
 183  *   +     lb ---+     +-----+                         +
 184  *   +                  width                          +
 185  *   +                                                 +
 186  *   +++++++++++++++++++++++++++++++++++++++++++++++++++
 187  */
 188
 189 struct rlevel_bitmask {
 190         u64 bm;
 191         u8 mstart;
 192         u8 width;
 193         int errs;
 194 };
 195
 196 #define MASKRANGE_BITS  6
 197 #define MASKRANGE       ((1 << MASKRANGE_BITS) - 1)
 198
 199 /* data field addresses in the DDR2 SPD eeprom */
 200 enum ddr2_spd_addrs {
 201         DDR2_SPD_BYTES_PROGRAMMED       = 0,
 202         DDR2_SPD_TOTAL_BYTES            = 1,
 203         DDR2_SPD_MEM_TYPE               = 2,
 204         DDR2_SPD_NUM_ROW_BITS           = 3,
 205         DDR2_SPD_NUM_COL_BITS           = 4,
 206         DDR2_SPD_NUM_RANKS              = 5,
 207         DDR2_SPD_CYCLE_CLX              = 9,
 208         DDR2_SPD_CONFIG_TYPE            = 11,
 209         DDR2_SPD_REFRESH                = 12,
 210         DDR2_SPD_SDRAM_WIDTH            = 13,
 211         DDR2_SPD_BURST_LENGTH           = 16,
 212         DDR2_SPD_NUM_BANKS              = 17,
 213         DDR2_SPD_CAS_LATENCY            = 18,
 214         DDR2_SPD_DIMM_TYPE              = 20,
 215         DDR2_SPD_CYCLE_CLX1             = 23,
 216         DDR2_SPD_CYCLE_CLX2             = 25,
 217         DDR2_SPD_TRP                    = 27,
 218         DDR2_SPD_TRRD                   = 28,
 219         DDR2_SPD_TRCD                   = 29,
 220         DDR2_SPD_TRAS                   = 30,
 221         DDR2_SPD_TWR                    = 36,
 222         DDR2_SPD_TWTR                   = 37,
 223         DDR2_SPD_TRFC_EXT               = 40,
 224         DDR2_SPD_TRFC                   = 42,
 225         DDR2_SPD_CHECKSUM               = 63,
 226         DDR2_SPD_MFR_ID                 = 64
 227 };
 228
 229 /* data field addresses in the DDR2 SPD eeprom */
 230 enum ddr3_spd_addrs {
 231         DDR3_SPD_BYTES_PROGRAMMED                       =  0,
 232         DDR3_SPD_REVISION                               =  1,
 233         DDR3_SPD_KEY_BYTE_DEVICE_TYPE                   =  2,
 234         DDR3_SPD_KEY_BYTE_MODULE_TYPE                   =  3,
 235         DDR3_SPD_DENSITY_BANKS                          =  4,
 236         DDR3_SPD_ADDRESSING_ROW_COL_BITS                =  5,
 237         DDR3_SPD_NOMINAL_VOLTAGE                        =  6,
 238         DDR3_SPD_MODULE_ORGANIZATION                    =  7,
 239         DDR3_SPD_MEMORY_BUS_WIDTH                       =  8,
 240         DDR3_SPD_FINE_TIMEBASE_DIVIDEND_DIVISOR         =  9,
 241         DDR3_SPD_MEDIUM_TIMEBASE_DIVIDEND               = 10,
 242         DDR3_SPD_MEDIUM_TIMEBASE_DIVISOR                = 11,
 243         DDR3_SPD_MINIMUM_CYCLE_TIME_TCKMIN              = 12,
 244         DDR3_SPD_CAS_LATENCIES_LSB                      = 14,
 245         DDR3_SPD_CAS_LATENCIES_MSB                      = 15,
 246         DDR3_SPD_MIN_CAS_LATENCY_TAAMIN                 = 16,
 247         DDR3_SPD_MIN_WRITE_RECOVERY_TWRMIN              = 17,
 248         DDR3_SPD_MIN_RAS_CAS_DELAY_TRCDMIN              = 18,
 249         DDR3_SPD_MIN_ROW_ACTIVE_DELAY_TRRDMIN           = 19,
 250         DDR3_SPD_MIN_ROW_PRECHARGE_DELAY_TRPMIN         = 20,
 251         DDR3_SPD_UPPER_NIBBLES_TRAS_TRC                 = 21,
 252         DDR3_SPD_MIN_ACTIVE_PRECHARGE_LSB_TRASMIN       = 22,
 253         DDR3_SPD_MIN_ACTIVE_REFRESH_LSB_TRCMIN          = 23,
 254         DDR3_SPD_MIN_REFRESH_RECOVERY_LSB_TRFCMIN       = 24,
 255         DDR3_SPD_MIN_REFRESH_RECOVERY_MSB_TRFCMIN       = 25,
 256         DDR3_SPD_MIN_INTERNAL_WRITE_READ_CMD_TWTRMIN    = 26,
 257         DDR3_SPD_MIN_INTERNAL_READ_PRECHARGE_CMD_TRTPMIN = 27,
 258         DDR3_SPD_UPPER_NIBBLE_TFAW                      = 28,
 259         DDR3_SPD_MIN_FOUR_ACTIVE_WINDOW_TFAWMIN         = 29,
 260         DDR3_SPD_SDRAM_OPTIONAL_FEATURES                = 30,
 261         DDR3_SPD_SDRAM_THERMAL_REFRESH_OPTIONS          = 31,
 262         DDR3_SPD_MODULE_THERMAL_SENSOR                  = 32,
 263         DDR3_SPD_SDRAM_DEVICE_TYPE                      = 33,
 264         DDR3_SPD_MINIMUM_CYCLE_TIME_FINE_TCKMIN         = 34,
 265         DDR3_SPD_MIN_CAS_LATENCY_FINE_TAAMIN            = 35,
 266         DDR3_SPD_MIN_RAS_CAS_DELAY_FINE_TRCDMIN         = 36,
 267         DDR3_SPD_MIN_ROW_PRECHARGE_DELAY_FINE_TRPMIN    = 37,
 268         DDR3_SPD_MIN_ACTIVE_REFRESH_LSB_FINE_TRCMIN     = 38,
 269         DDR3_SPD_REFERENCE_RAW_CARD                     = 62,
 270         DDR3_SPD_ADDRESS_MAPPING                        = 63,
 271         DDR3_SPD_REGISTER_MANUFACTURER_ID_LSB           = 65,
 272         DDR3_SPD_REGISTER_MANUFACTURER_ID_MSB           = 66,
 273         DDR3_SPD_REGISTER_REVISION_NUMBER               = 67,
 274         DDR3_SPD_MODULE_SERIAL_NUMBER                   = 122,
 275         DDR3_SPD_CYCLICAL_REDUNDANCY_CODE_LOWER_NIBBLE  = 126,
 276         DDR3_SPD_CYCLICAL_REDUNDANCY_CODE_UPPER_NIBBLE  = 127,
 277         DDR3_SPD_MODULE_PART_NUMBER                     = 128
 278 };
 279
 280 /* data field addresses in the DDR4 SPD eeprom */
 281 enum ddr4_spd_addrs {
 282         DDR4_SPD_BYTES_PROGRAMMED                       =  0,
 283         DDR4_SPD_REVISION                               =  1,
 284         DDR4_SPD_KEY_BYTE_DEVICE_TYPE                   =  2,
 285         DDR4_SPD_KEY_BYTE_MODULE_TYPE                   =  3,
 286         DDR4_SPD_DENSITY_BANKS                          =  4,
 287         DDR4_SPD_ADDRESSING_ROW_COL_BITS                =  5,
 288         DDR4_SPD_PACKAGE_TYPE                           =  6,
 289         DDR4_SPD_OPTIONAL_FEATURES                      =  7,
 290         DDR4_SPD_THERMAL_REFRESH_OPTIONS                =  8,
 291         DDR4_SPD_OTHER_OPTIONAL_FEATURES                =  9,
 292         DDR4_SPD_SECONDARY_PACKAGE_TYPE                 = 10,
 293         DDR4_SPD_MODULE_NOMINAL_VOLTAGE                 = 11,
 294         DDR4_SPD_MODULE_ORGANIZATION                    = 12,
 295         DDR4_SPD_MODULE_MEMORY_BUS_WIDTH                = 13,
 296         DDR4_SPD_MODULE_THERMAL_SENSOR                  = 14,
 297         DDR4_SPD_RESERVED_BYTE15                        = 15,
 298         DDR4_SPD_RESERVED_BYTE16                        = 16,
 299         DDR4_SPD_TIMEBASES                              = 17,
 300         DDR4_SPD_MINIMUM_CYCLE_TIME_TCKAVGMIN           = 18,
 301         DDR4_SPD_MAXIMUM_CYCLE_TIME_TCKAVGMAX           = 19,
 302         DDR4_SPD_CAS_LATENCIES_BYTE0                    = 20,
 303         DDR4_SPD_CAS_LATENCIES_BYTE1                    = 21,
 304         DDR4_SPD_CAS_LATENCIES_BYTE2                    = 22,
 305         DDR4_SPD_CAS_LATENCIES_BYTE3                    = 23,
 306         DDR4_SPD_MIN_CAS_LATENCY_TAAMIN                 = 24,
 307         DDR4_SPD_MIN_RAS_CAS_DELAY_TRCDMIN              = 25,
 308         DDR4_SPD_MIN_ROW_PRECHARGE_DELAY_TRPMIN         = 26,
 309         DDR4_SPD_UPPER_NIBBLES_TRAS_TRC                 = 27,
 310         DDR4_SPD_MIN_ACTIVE_PRECHARGE_LSB_TRASMIN       = 28,
 311         DDR4_SPD_MIN_ACTIVE_REFRESH_LSB_TRCMIN          = 29,
 312         DDR4_SPD_MIN_REFRESH_RECOVERY_LSB_TRFC1MIN      = 30,
 313         DDR4_SPD_MIN_REFRESH_RECOVERY_MSB_TRFC1MIN      = 31,
 314         DDR4_SPD_MIN_REFRESH_RECOVERY_LSB_TRFC2MIN      = 32,
 315         DDR4_SPD_MIN_REFRESH_RECOVERY_MSB_TRFC2MIN      = 33,
 316         DDR4_SPD_MIN_REFRESH_RECOVERY_LSB_TRFC4MIN      = 34,
 317         DDR4_SPD_MIN_REFRESH_RECOVERY_MSB_TRFC4MIN      = 35,
 318         DDR4_SPD_MIN_FOUR_ACTIVE_WINDOW_MSN_TFAWMIN     = 36,
 319         DDR4_SPD_MIN_FOUR_ACTIVE_WINDOW_LSB_TFAWMIN     = 37,
 320         DDR4_SPD_MIN_ROW_ACTIVE_DELAY_SAME_TRRD_SMIN    = 38,
 321         DDR4_SPD_MIN_ROW_ACTIVE_DELAY_DIFF_TRRD_LMIN    = 39,
 322         DDR4_SPD_MIN_CAS_TO_CAS_DELAY_TCCD_LMIN         = 40,
 323         DDR4_SPD_MIN_CAS_TO_CAS_DELAY_FINE_TCCD_LMIN    = 117,
 324         DDR4_SPD_MIN_ACT_TO_ACT_DELAY_SAME_FINE_TRRD_LMIN = 118,
 325         DDR4_SPD_MIN_ACT_TO_ACT_DELAY_DIFF_FINE_TRRD_SMIN = 119,
 326         DDR4_SPD_MIN_ACT_TO_ACT_REFRESH_DELAY_FINE_TRCMIN = 120,
 327         DDR4_SPD_MIN_ROW_PRECHARGE_DELAY_FINE_TRPMIN    = 121,
 328         DDR4_SPD_MIN_RAS_TO_CAS_DELAY_FINE_TRCDMIN      = 122,
 329         DDR4_SPD_MIN_CAS_LATENCY_FINE_TAAMIN            = 123,
 330         DDR4_SPD_MAX_CYCLE_TIME_FINE_TCKAVGMAX          = 124,
 331         DDR4_SPD_MIN_CYCLE_TIME_FINE_TCKAVGMIN          = 125,
 332         DDR4_SPD_CYCLICAL_REDUNDANCY_CODE_LOWER_NIBBLE  = 126,
 333         DDR4_SPD_CYCLICAL_REDUNDANCY_CODE_UPPER_NIBBLE  = 127,
 334         DDR4_SPD_REFERENCE_RAW_CARD                     = 130,
 335         DDR4_SPD_UDIMM_ADDR_MAPPING_FROM_EDGE           = 131,
 336         DDR4_SPD_REGISTER_MANUFACTURER_ID_LSB           = 133,
 337         DDR4_SPD_REGISTER_MANUFACTURER_ID_MSB           = 134,
 338         DDR4_SPD_REGISTER_REVISION_NUMBER               = 135,
 339         DDR4_SPD_RDIMM_ADDR_MAPPING_FROM_REGISTER_TO_DRAM = 136,
 340         DDR4_SPD_RDIMM_REGISTER_DRIVE_STRENGTH_CTL      = 137,
 341         DDR4_SPD_RDIMM_REGISTER_DRIVE_STRENGTH_CK       = 138,
 342 };
 343
 344 #define SPD_EEPROM_SIZE         (DDR4_SPD_RDIMM_REGISTER_DRIVE_STRENGTH_CK + 1)
 345
 346 struct impedence_values {
 347         unsigned char *rodt_ohms;
 348         unsigned char *rtt_nom_ohms;
 349         unsigned char *rtt_nom_table;
 350         unsigned char *rtt_wr_ohms;
 351         unsigned char *dic_ohms;
 352         short *drive_strength;
 353         short *dqx_strength;
 354 };
 355
 356 #define RODT_OHMS_COUNT        8
 357 #define RTT_NOM_OHMS_COUNT     8
 358 #define RTT_NOM_TABLE_COUNT    8
 359 #define RTT_WR_OHMS_COUNT      8
 360 #define DIC_OHMS_COUNT         3
 361 #define DRIVE_STRENGTH_COUNT  15
 362
 363 /*
 364  * Structure that provides DIMM information, either in the form of an SPD
 365  * TWSI address, or a pointer to an array that contains SPD data. One of
 366  * the two fields must be valid.
 367  */
 368 struct dimm_config {
 369         u16 spd_addrs[2]; /* TWSI address of SPD, 0 if not used */
 370         u8 *spd_ptrs[2]; /* pointer to SPD data array, NULL if not used */
 371         int spd_cached[2];
 372         u8 spd_data[2][SPD_EEPROM_SIZE];
 373 };
 374
 375 struct dimm_odt_config {
 376         u8 odt_ena;            /* FIX: dqx_ctl for Octeon 3 DDR4 */
 377         u64 odt_mask;          /* FIX: wodt_mask for Octeon 3 */
 378         union cvmx_lmcx_modereg_params1 modereg_params1;
 379         union cvmx_lmcx_modereg_params2 modereg_params2;
 380         u8 qs_dic;             /* FIX: rodt_ctl for Octeon 3 */
 381         u64 rodt_ctl;          /* FIX: rodt_mask for Octeon 3 */
 382         u8 dic;
 383 };
 384
 385 struct ddr_delay_config {
 386         u32 ddr_board_delay;
 387         u8 lmc_delay_clk;
 388         u8 lmc_delay_cmd;
 389         u8 lmc_delay_dq;
 390 };
 391
 392 /*
 393  * The parameters below make up the custom_lmc_config data structure.
 394  * This structure is used to customize the way that the LMC DRAM
 395  * Controller is configured for a particular board design.
 396  *
 397  * The HRM describes LMC Read Leveling which supports automatic
 398  * selection of per byte-lane delays.  When measuring the read delays
 399  * the LMC configuration software sweeps through a range of settings
 400  * for LMC0_COMP_CTL2[RODT_CTL], the Octeon II on-die-termination
 401  * resistance and LMC0_MODEREG_PARAMS1[RTT_NOM_XX], the DRAM
 402  * on-die-termination resistance.  The minimum and maximum parameters
 403  * for rtt_nom_idx and rodt_ctl listed below determine the ranges of
 404  * ODT settings used for the measurements.  Note that for rtt_nom an
 405  * index is used into a sorted table rather than the direct csr setting
 406  * in order to optimize the sweep.
 407  *
 408  * .min_rtt_nom_idx: 1=120ohms, 2=60ohms, 3=40ohms, 4=30ohms, 5=20ohms
 409  * .max_rtt_nom_idx: 1=120ohms, 2=60ohms, 3=40ohms, 4=30ohms, 5=20ohms
 410  * .min_rodt_ctl: 1=20ohms, 2=30ohms, 3=40ohms, 4=60ohms, 5=120ohms
 411  * .max_rodt_ctl: 1=20ohms, 2=30ohms, 3=40ohms, 4=60ohms, 5=120ohms
 412  *
 413  * The settings below control the Octeon II drive strength for the CK,
 414  * ADD/CMD, and DQ/DQS signals.  1=24ohms, 2=26.67ohms, 3=30ohms,
 415  * 4=34.3ohms, 5=40ohms, 6=48ohms, 6=60ohms.
 416  *
 417  * .dqx_ctl: Drive strength control for DDR_DQX/DDR_DQS_X_P/N drivers.
 418  * .ck_ctl: Drive strength control for
 419  * DDR_CK_X_P/DDR_DIMMX_CSX_L/DDR_DIMMX_ODT_X drivers.
 420  * .cmd_ctl: Drive strength control for CMD/A/RESET_L/CKEX drivers.
 421  *
 422  * The LMC controller software selects the most optimal CAS Latency
 423  * that complies with the appropriate SPD values and the frequency
 424  * that the DRAMS are being operated.  When operating the DRAMs at
 425  * frequencies substantially lower than their rated frequencies it
 426  * might be necessary to limit the minimum CAS Latency the LMC
 427  * controller software is allowed to select in order to make the DRAM
 428  * work reliably.
 429  *
 430  * .min_cas_latency: Minimum allowed CAS Latency
 431  *
 432  * The value used for LMC0_RLEVEL_CTL[OFFSET_EN] determine how the
 433  * read-leveling information that the Octeon II gathers is interpreted
 434  * to determine the per-byte read delays.
 435  *
 436  * .offset_en: Value used for LMC0_RLEVEL_CTL[OFFSET_EN].
 437  * .offset_udimm: Value used for LMC0_RLEVEL_CTL[OFFSET] for UDIMMS.
 438  * .offset_rdimm: Value used for LMC0_RLEVEL_CTL[OFFSET] for RDIMMS.
 439  *
 440  * The LMC configuration software sweeps through a range of ODT
 441  * settings while measuring the per-byte read delays.  During those
 442  * measurements the software makes an assessment of the quality of the
 443  * measurements in order to determine which measurements provide the
 444  * most accurate delays.  The automatic settings provide the option to
 445  * allow that same assessment to determine the most optimal RODT_CTL
 446  * and/or RTT_NOM settings.
 447  *
 448  * The automatic approach might provide the best means to determine
 449  * the settings used for initial poweron of a new design.  However,
 450  * the final settings should be determined by board analysis, testing,
 451  * and experience.
 452  *
 453  * .ddr_rtt_nom_auto: 1 means automatically set RTT_NOM value.
 454  * .ddr_rodt_ctl_auto: 1 means automatically set RODT_CTL value.
 455  *
 456  * .rlevel_compute: Enables software interpretation of per-byte read
 457  * delays using the measurements collected by the
 458  * Octeon II rather than completely relying on the
 459  * Octeon II to determine the delays.  1=software
 460  * computation is recomended since a more complete
 461  * analysis is implemented in software.
 462  *
 463  * .rlevel_comp_offset: Set to 2 unless instructed differently by Cavium.
 464  *
 465  * .rlevel_average_loops: Determines the number of times the read-leveling
 466  * sequence is run for each rank.  The results is
 467  * then averaged across the number of loops. The
 468  * default setting is 1.
 469  *
 470  * .ddr2t_udimm:
 471  * .ddr2t_rdimm: Turn on the DDR 2T mode. 2-cycle window for CMD and
 472  * address. This mode helps relieve setup time pressure
 473  * on the address and command bus. Please refer to
 474  * Micron's tech note tn_47_01 titled DDR2-533 Memory
 475  * Design Guide for Two Dimm Unbuffered Systems for
 476  * physical details.
 477  *
 478  * .disable_sequential_delay_check: As result of the flyby topology
 479  * prescribed in the JEDEC specifications the byte delays should
 480  * maintain a consistent increasing or decreasing trend across
 481  * the bytes on standard dimms.  This setting can be used disable
 482  * that check for unusual circumstances where the check is not
 483  * useful.
 484  *
 485  * .maximum_adjacent_rlevel_delay_increment: An additional sequential
 486  * delay check for the delays that result from the flyby
 487  * topology. This value specifies the maximum difference between
 488  * the delays of adjacent bytes.  A value of 0 disables this
 489  * check.
 490  *
 491  * .fprch2 Front Porch Enable: When set, the turn-off
 492  * time for the default DDR_DQ/DQS drivers is FPRCH2 CKs earlier.
 493  * 00 = 0 CKs
 494  * 01 = 1 CKs
 495  * 10 = 2 CKs
 496  *
 497  * .parity: The parity input signal PAR_IN on each dimm must be
 498  * strapped high or low on the board.  This bit is programmed
 499  * into LMC0_DIMM_CTL[PARITY] and it must be set to match the
 500  * board strapping.  This signal is typically strapped low.
 501  *
 502  * .mode32b: Enable 32-bit datapath mode.  Set to 1 if only 32 DQ pins
 503  * are used. (cn61xx, cn71xx)
 504  *
 505  * .measured_vref: Set to 1 to measure VREF; set to 0 to compute VREF.
 506  *
 507  * .dram_connection: Set to 1 if discrete DRAMs; set to 0 if using DIMMs.
 508  * This changes the algorithms used to compute VREF.
 509  *
 510  * .dll_write_offset: FIXME: Add description
 511  * .dll_read_offset:  FIXME: Add description
 512  */
 513
 514 struct rlevel_table {
 515         const char part[20];
 516         int speed;
 517         u64 rl_rank[4][4];
 518 };
 519
 520 struct ddr3_custom_config {
 521         u8 min_rtt_nom_idx;
 522         u8 max_rtt_nom_idx;
 523         u8 min_rodt_ctl;
 524         u8 max_rodt_ctl;
 525         u8 dqx_ctl;
 526         u8 ck_ctl;
 527         u8 cmd_ctl;
 528         u8 ctl_ctl;
 529         u8 min_cas_latency;
 530         u8 offset_en;
 531         u8 offset_udimm;
 532         u8 offset_rdimm;
 533         u8 rlevel_compute;
 534         u8 ddr_rtt_nom_auto;
 535         u8 ddr_rodt_ctl_auto;
 536         u8 rlevel_comp_offset_udimm;
 537         u8 rlevel_comp_offset_rdimm;
 538         int8_t ptune_offset;
 539         int8_t ntune_offset;
 540         u8 rlevel_average_loops;
 541         u8 ddr2t_udimm;
 542         u8 ddr2t_rdimm;
 543         u8 disable_sequential_delay_check;
 544         u8 maximum_adjacent_rlevel_delay_increment;
 545         u8 parity;
 546         u8 fprch2;
 547         u8 mode32b;
 548         u8 measured_vref;
 549         u8 dram_connection;
 550         const int8_t *dll_write_offset;
 551         const int8_t *dll_read_offset;
 552         struct rlevel_table *rl_tbl;
 553 };
 554
 555 #define DDR_CFG_T_MAX_DIMMS     5
 556
 557 struct ddr_conf {
 558         struct dimm_config dimm_config_table[DDR_CFG_T_MAX_DIMMS];
 559         struct dimm_odt_config odt_1rank_config[4];
 560         struct dimm_odt_config odt_2rank_config[4];
 561         struct dimm_odt_config odt_4rank_config[4];
 562         struct ddr_delay_config unbuffered;
 563         struct ddr_delay_config registered;
 564         struct ddr3_custom_config custom_lmc_config;
 565 };
 566
 567 /* Divide and round results to the nearest integer. */
 568 static inline u64 divide_nint(u64 dividend, u64 divisor)
 569 {
 570         u64 quotent, remainder;
 571
 572         quotent   = dividend / divisor;
 573         remainder = dividend % divisor;
 574         return (quotent + ((remainder * 2) >= divisor));
 575 }
 576
 577 /* Divide and round results up to the next higher integer. */
 578 static inline u64 divide_roundup(u64 dividend, u64 divisor)
 579 {
 580         return ((dividend + divisor - 1) / divisor);
 581 }
 582
 583 enum ddr_type {
 584         DDR3_DRAM = 3,
 585         DDR4_DRAM = 4,
 586 };
 587
 588 #define rttnom_none   0         /* Rtt_Nom disabled */
 589 #define rttnom_60ohm  1         /* RZQ/4  = 240/4  =  60 ohms */
 590 #define rttnom_120ohm 2         /* RZQ/2  = 240/2  = 120 ohms */
 591 #define rttnom_40ohm  3         /* RZQ/6  = 240/6  =  40 ohms */
 592 #define rttnom_20ohm  4         /* RZQ/12 = 240/12 =  20 ohms */
 593 #define rttnom_30ohm  5         /* RZQ/8  = 240/8  =  30 ohms */
 594 #define rttnom_rsrv1  6         /* Reserved */
 595 #define rttnom_rsrv2  7         /* Reserved */
 596
 597 #define rttwr_none    0         /* Dynamic ODT off */
 598 #define rttwr_60ohm   1         /* RZQ/4  = 240/4  =  60 ohms */
 599 #define rttwr_120ohm  2         /* RZQ/2  = 240/2  = 120 ohms */
 600 #define rttwr_rsrv1   3         /* Reserved */
 601
 602 #define dic_40ohm     0         /* RZQ/6  = 240/6  =  40 ohms */
 603 #define dic_34ohm     1         /* RZQ/7  = 240/7  =  34 ohms */
 604
 605 #define driver_24_ohm   1
 606 #define driver_27_ohm   2
 607 #define driver_30_ohm   3
 608 #define driver_34_ohm   4
 609 #define driver_40_ohm   5
 610 #define driver_48_ohm   6
 611 #define driver_60_ohm   7
 612
 613 #define rodt_ctl_none     0
 614 #define rodt_ctl_20_ohm   1
 615 #define rodt_ctl_30_ohm   2
 616 #define rodt_ctl_40_ohm   3
 617 #define rodt_ctl_60_ohm   4
 618 #define rodt_ctl_120_ohm  5
 619
 620 #define ddr4_rttnom_none   0         /* Rtt_Nom disabled */
 621 #define ddr4_rttnom_60ohm  1         /* RZQ/4  = 240/4  =  60 ohms */
 622 #define ddr4_rttnom_120ohm 2         /* RZQ/2  = 240/2  = 120 ohms */
 623 #define ddr4_rttnom_40ohm  3         /* RZQ/6  = 240/6  =  40 ohms */
 624 #define ddr4_rttnom_240ohm 4         /* RZQ/1  = 240/1  = 240 ohms */
 625 #define ddr4_rttnom_48ohm  5         /* RZQ/5  = 240/5  =  48 ohms */
 626 #define ddr4_rttnom_80ohm  6         /* RZQ/3  = 240/3  =  80 ohms */
 627 #define ddr4_rttnom_34ohm  7         /* RZQ/7  = 240/7  =  34 ohms */
 628
 629 #define ddr4_rttwr_none    0         /* Dynamic ODT off */
 630 #define ddr4_rttwr_120ohm  1         /* RZQ/2  = 240/2  = 120 ohms */
 631 #define ddr4_rttwr_240ohm  2         /* RZQ/1  = 240/1  = 240 ohms */
 632 #define ddr4_rttwr_hiz     3         /* HiZ */
 633 /* This setting is available for cn78xx pass 2, and cn73xx & cnf75xx pass 1 */
 634 #define ddr4_rttwr_80ohm   4         /* RZQ/3  = 240/3  =  80 ohms */
 635
 636 #define ddr4_dic_34ohm     0         /* RZQ/7  = 240/7  =  34 ohms */
 637 #define ddr4_dic_48ohm     1         /* RZQ/5  = 240/5  =  48 ohms */
 638
 639 #define ddr4_rttpark_none   0         /* Rtt_Park disabled */
 640 #define ddr4_rttpark_60ohm  1         /* RZQ/4  = 240/4  =  60 ohms */
 641 #define ddr4_rttpark_120ohm 2         /* RZQ/2  = 240/2  = 120 ohms */
 642 #define ddr4_rttpark_40ohm  3         /* RZQ/6  = 240/6  =  40 ohms */
 643 #define ddr4_rttpark_240ohm 4         /* RZQ/1  = 240/1  = 240 ohms */
 644 #define ddr4_rttpark_48ohm  5         /* RZQ/5  = 240/5  =  48 ohms */
 645 #define ddr4_rttpark_80ohm  6         /* RZQ/3  = 240/3  =  80 ohms */
 646 #define ddr4_rttpark_34ohm  7         /* RZQ/7  = 240/7  =  34 ohms */
 647
 648 #define ddr4_driver_26_ohm   2
 649 #define ddr4_driver_30_ohm   3
 650 #define ddr4_driver_34_ohm   4
 651 #define ddr4_driver_40_ohm   5
 652 #define ddr4_driver_48_ohm   6
 653
 654 #define ddr4_dqx_driver_24_ohm   1
 655 #define ddr4_dqx_driver_27_ohm   2
 656 #define ddr4_dqx_driver_30_ohm   3
 657 #define ddr4_dqx_driver_34_ohm   4
 658 #define ddr4_dqx_driver_40_ohm   5
 659 #define ddr4_dqx_driver_48_ohm   6
 660 #define ddr4_dqx_driver_60_ohm   7
 661
 662 #define ddr4_rodt_ctl_none     0
 663 #define ddr4_rodt_ctl_40_ohm   1
 664 #define ddr4_rodt_ctl_60_ohm   2
 665 #define ddr4_rodt_ctl_80_ohm   3
 666 #define ddr4_rodt_ctl_120_ohm  4
 667 #define ddr4_rodt_ctl_240_ohm  5
 668 #define ddr4_rodt_ctl_34_ohm   6
 669 #define ddr4_rodt_ctl_48_ohm   7
 670
 671 #define DIMM_CONFIG_TERMINATOR  { {0, 0}, {NULL, NULL} }
 672
 673 #define SET_DDR_DLL_CTL3(field, expr)                           \
 674         do {                                                    \
 675                 if (octeon_is_cpuid(OCTEON_CN66XX) ||           \
 676                     octeon_is_cpuid(OCTEON_CN63XX))             \
 677                         ddr_dll_ctl3.cn63xx.field = (expr);     \
 678                 else if (octeon_is_cpuid(OCTEON_CN68XX) ||      \
 679                          octeon_is_cpuid(OCTEON_CN61XX) ||      \
 680                          octeon_is_cpuid(OCTEON_CNF71XX))       \
 681                         ddr_dll_ctl3.cn61xx.field = (expr);     \
 682                 else if (octeon_is_cpuid(OCTEON_CN70XX) ||      \
 683                          octeon_is_cpuid(OCTEON_CN78XX))        \
 684                         ddr_dll_ctl3.cn70xx.field = (expr);     \
 685                 else if (octeon_is_cpuid(OCTEON_CN73XX) ||      \
 686                          octeon_is_cpuid(OCTEON_CNF75XX))       \
 687                         ddr_dll_ctl3.cn73xx.field = (expr);     \
 688                 else                                            \
 689                         debug("%s(): " #field                   \
 690                               "not set for unknown chip\n",     \
 691                               __func__);                        \
 692         } while (0)
 693
 694 #define ENCODE_DLL90_BYTE_SEL(byte_sel)                                 \
 695         (octeon_is_cpuid(OCTEON_CN70XX) ? ((9 + 7 - (byte_sel)) % 9) :  \
 696          ((byte_sel) + 1))
 697
 698 /**
 699  * If debugging is disabled the ddr_print macro is not compatible
 700  * with this macro.
 701  */
 702 # define GET_DDR_DLL_CTL3(field)                \
 703         ((octeon_is_cpuid(OCTEON_CN66XX) ||     \
 704           octeon_is_cpuid(OCTEON_CN63XX)) ?     \
 705          ddr_dll_ctl3.cn63xx.field :            \
 706          (octeon_is_cpuid(OCTEON_CN68XX) ||     \
 707           octeon_is_cpuid(OCTEON_CN61XX) ||     \
 708           octeon_is_cpuid(OCTEON_CNF71XX)) ?    \
 709          ddr_dll_ctl3.cn61xx.field :            \
 710          (octeon_is_cpuid(OCTEON_CN70XX) ||     \
 711           octeon_is_cpuid(OCTEON_CN78XX)) ?     \
 712          ddr_dll_ctl3.cn70xx.field :            \
 713          (octeon_is_cpuid(OCTEON_CN73XX) ||     \
 714           octeon_is_cpuid(OCTEON_CNF75XX)) ?    \
 715          ddr_dll_ctl3.cn73xx.field : 0)
 716
 717 extern const char *ddr3_dimm_types[];
 718 extern const char *ddr4_dimm_types[];
 719
 720 extern const struct dimm_odt_config disable_odt_config[];
 721
 722 #define RLEVEL_BYTE_BITS        6
 723 #define RLEVEL_BYTE_MSK         ((1ULL << 6) - 1)
 724
 725 /* Prototypes */
 726 int get_ddr_type(struct dimm_config *dimm_config, int upper_dimm);
 727 int get_dimm_module_type(struct dimm_config *dimm_config, int upper_dimm,
 728                          int ddr_type);
 729 int read_spd(struct dimm_config *dimm_config, int dimm_index, int spd_field);
 730 int read_spd_init(struct dimm_config *dimm_config, int dimm_index);
 731 void report_dimm(struct dimm_config *dimm_config, int upper_dimm,
 732                  int dimm, int if_num);
 733 int validate_dimm(struct ddr_priv *priv, struct dimm_config *dimm_config,
 734                   int dimm_index);
 735 char *printable_rank_spec(char *buffer, int num_ranks, int dram_width,
 736                           int spd_package);
 737
 738 bool ddr_memory_preserved(struct ddr_priv *priv);
 739
 740 int get_wl_rank(union cvmx_lmcx_wlevel_rankx *lmc_wlevel_rank, int byte);
 741 int get_rl_rank(union cvmx_lmcx_rlevel_rankx *lmc_rlevel_rank, int byte);
 742 void upd_wl_rank(union cvmx_lmcx_wlevel_rankx *lmc_wlevel_rank, int byte,
 743                  int delay);
 744 void upd_rl_rank(union cvmx_lmcx_rlevel_rankx *lmc_rlevel_rank, int byte,
 745                  int delay);
 746
 747 int compute_ddr3_rlevel_delay(u8 mstart, u8 width,
 748                               union cvmx_lmcx_rlevel_ctl rlevel_ctl);
 749
 750 int encode_row_lsb_ddr3(int row_lsb);
 751 int encode_pbank_lsb_ddr3(int pbank_lsb);
 752
 753 int initialize_ddr_clock(struct ddr_priv *priv, struct ddr_conf *ddr_conf,
 754                          u32 cpu_hertz, u32 ddr_hertz, u32 ddr_ref_hertz,
 755                          int if_num, u32 if_mask);
 756
 757 void process_custom_dll_offsets(struct ddr_priv *priv, int if_num,
 758                                 const char *enable_str,
 759                                 const int8_t *offsets, const char *byte_str,
 760                                 int mode);
 761 int nonseq_del(struct rlevel_byte_data *rlevel_byte, int start, int end,
 762                int max_adj_delay_inc);
 763 int roundup_ddr3_wlevel_bitmask(int bitmask);
 764
 765 void oct3_ddr3_seq(struct ddr_priv *priv, int rank_mask, int if_num,
 766                    int sequence);
 767 void ddr_init_seq(struct ddr_priv *priv, int rank_mask, int if_num);
 768
 769 void rlevel_to_wlevel(union cvmx_lmcx_rlevel_rankx *lmc_rlevel_rank,
 770                       union cvmx_lmcx_wlevel_rankx *lmc_wlevel_rank, int byte);
 771
 772 int validate_ddr3_rlevel_bitmask(struct rlevel_bitmask *rlevel_bitmask_p,
 773                                  int ddr_type);
 774
 775 void change_dll_offset_enable(struct ddr_priv *priv, int if_num, int change);
 776 unsigned short load_dll_offset(struct ddr_priv *priv, int if_num,
 777                                int dll_offset_mode,
 778                                int byte_offset, int byte);
 779
 780 u64 lmc_ddr3_rl_dbg_read(struct ddr_priv *priv, int if_num, int idx);
 781 u64 lmc_ddr3_wl_dbg_read(struct ddr_priv *priv, int if_num, int idx);
 782
 783 void cvmx_maybe_tune_node(struct ddr_priv *priv, u32 ddr_speed);
 784 void cvmx_dbi_switchover(struct ddr_priv *priv);
 785
 786 int init_octeon3_ddr3_interface(struct ddr_priv *priv,
 787                                 struct ddr_conf *ddr_conf,
 788                                 u32 ddr_hertz, u32 cpu_hertz, u32 ddr_ref_hertz,
 789                                 int if_num, u32 if_mask);
 790
 791 char *lookup_env(struct ddr_priv *priv, const char *format, ...);
 792 char *lookup_env_ull(struct ddr_priv *priv, const char *format, ...);
 793
 794 /* Each board provides a board-specific config table via this function */
 795 struct ddr_conf *octeon_ddr_conf_table_get(int *count, int *def_ddr_freq);
 796
 797 #endif /* __OCTEON_DDR_H_ */