arch/x86/cpu/quark/smc.c

   1 /*
   2  * Copyright (C) 2013, Intel Corporation
   3  * Copyright (C) 2015, Bin Meng <bmeng.cn@gmail.com>
   4  *
   5  * Ported from Intel released Quark UEFI BIOS
   6  * QuarkSocPkg/QuarkNorthCluster/MemoryInit/Pei
   7  *
   8  * SPDX-License-Identifier:     Intel
   9  */
  10
  11 #include <common.h>
  12 #include <pci.h>
  13 #include <asm/arch/device.h>
  14 #include <asm/arch/mrc.h>
  15 #include <asm/arch/msg_port.h>
  16 #include "mrc_util.h"
  17 #include "hte.h"
  18 #include "smc.h"
  19
  20 /* t_rfc values (in picoseconds) per density */
  21 static const uint32_t t_rfc[5] = {
  22         90000,  /* 512Mb */
  23         110000, /* 1Gb */
  24         160000, /* 2Gb */
  25         300000, /* 4Gb */
  26         350000, /* 8Gb */
  27 };
  28
  29 /* t_ck clock period in picoseconds per speed index 800, 1066, 1333 */
  30 static const uint32_t t_ck[3] = {
  31         2500,
  32         1875,
  33         1500
  34 };
  35
  36 /* Global variables */
  37 static const uint16_t ddr_wclk[] = {193, 158};
  38 static const uint16_t ddr_wctl[] = {1, 217};
  39 static const uint16_t ddr_wcmd[] = {1, 220};
  40
  41 #ifdef BACKUP_RCVN
  42 static const uint16_t ddr_rcvn[] = {129, 498};
  43 #endif
  44
  45 #ifdef BACKUP_WDQS
  46 static const uint16_t ddr_wdqs[] = {65, 289};
  47 #endif
  48
  49 #ifdef BACKUP_RDQS
  50 static const uint8_t ddr_rdqs[] = {32, 24};
  51 #endif
  52
  53 #ifdef BACKUP_WDQ
  54 static const uint16_t ddr_wdq[] = {32, 257};
  55 #endif
  56
  57 /* Stop self refresh driven by MCU */
  58 void clear_self_refresh(struct mrc_params *mrc_params)
  59 {
  60         ENTERFN();
  61
  62         /* clear the PMSTS Channel Self Refresh bits */
  63         mrc_write_mask(MEM_CTLR, PMSTS, BIT0, BIT0);
  64
  65         LEAVEFN();
  66 }
  67
  68 /* It will initialize timing registers in the MCU (DTR0..DTR4) */
  69 void prog_ddr_timing_control(struct mrc_params *mrc_params)
  70 {
  71         uint8_t tcl, wl;
  72         uint8_t trp, trcd, tras, twr, twtr, trrd, trtp, tfaw;
  73         uint32_t tck;
  74         u32 dtr0, dtr1, dtr2, dtr3, dtr4;
  75         u32 tmp1, tmp2;
  76
  77         ENTERFN();
  78
  79         /* mcu_init starts */
  80         mrc_post_code(0x02, 0x00);
  81
  82         dtr0 = msg_port_read(MEM_CTLR, DTR0);
  83         dtr1 = msg_port_read(MEM_CTLR, DTR1);
  84         dtr2 = msg_port_read(MEM_CTLR, DTR2);
  85         dtr3 = msg_port_read(MEM_CTLR, DTR3);
  86         dtr4 = msg_port_read(MEM_CTLR, DTR4);
  87
  88         tck = t_ck[mrc_params->ddr_speed];      /* Clock in picoseconds */
  89         tcl = mrc_params->params.cl;            /* CAS latency in clocks */
  90         trp = tcl;      /* Per CAT MRC */
  91         trcd = tcl;     /* Per CAT MRC */
  92         tras = MCEIL(mrc_params->params.ras, tck);
  93
  94         /* Per JEDEC: tWR=15000ps DDR2/3 from 800-1600 */
  95         twr = MCEIL(15000, tck);
  96
  97         twtr = MCEIL(mrc_params->params.wtr, tck);
  98         trrd = MCEIL(mrc_params->params.rrd, tck);
  99         trtp = 4;       /* Valid for 800 and 1066, use 5 for 1333 */
 100         tfaw = MCEIL(mrc_params->params.faw, tck);
 101
 102         wl = 5 + mrc_params->ddr_speed;
 103
 104         dtr0 &= ~(BIT0 | BIT1);
 105         dtr0 |= mrc_params->ddr_speed;
 106         dtr0 &= ~(BIT12 | BIT13 | BIT14);
 107         tmp1 = tcl - 5;
 108         dtr0 |= ((tcl - 5) << 12);
 109         dtr0 &= ~(BIT4 | BIT5 | BIT6 | BIT7);
 110         dtr0 |= ((trp - 5) << 4);       /* 5 bit DRAM Clock */
 111         dtr0 &= ~(BIT8 | BIT9 | BIT10 | BIT11);
 112         dtr0 |= ((trcd - 5) << 8);      /* 5 bit DRAM Clock */
 113
 114         dtr1 &= ~(BIT0 | BIT1 | BIT2);
 115         tmp2 = wl - 3;
 116         dtr1 |= (wl - 3);
 117         dtr1 &= ~(BIT8 | BIT9 | BIT10 | BIT11);
 118         dtr1 |= ((wl + 4 + twr - 14) << 8);     /* Change to tWTP */
 119         dtr1 &= ~(BIT28 | BIT29 | BIT30);
 120         dtr1 |= ((MMAX(trtp, 4) - 3) << 28);    /* 4 bit DRAM Clock */
 121         dtr1 &= ~(BIT24 | BIT25);
 122         dtr1 |= ((trrd - 4) << 24);             /* 4 bit DRAM Clock */
 123         dtr1 &= ~(BIT4 | BIT5);
 124         dtr1 |= (1 << 4);
 125         dtr1 &= ~(BIT20 | BIT21 | BIT22 | BIT23);
 126         dtr1 |= ((tras - 14) << 20);            /* 6 bit DRAM Clock */
 127         dtr1 &= ~(BIT16 | BIT17 | BIT18 | BIT19);
 128         dtr1 |= ((((tfaw + 1) >> 1) - 5) << 16);/* 4 bit DRAM Clock */
 129         /* Set 4 Clock CAS to CAS delay (multi-burst) */
 130         dtr1 &= ~(BIT12 | BIT13);
 131
 132         dtr2 &= ~(BIT0 | BIT1 | BIT2);
 133         dtr2 |= 1;
 134         dtr2 &= ~(BIT8 | BIT9 | BIT10);
 135         dtr2 |= (2 << 8);
 136         dtr2 &= ~(BIT16 | BIT17 | BIT18 | BIT19);
 137         dtr2 |= (2 << 16);
 138
 139         dtr3 &= ~(BIT0 | BIT1 | BIT2);
 140         dtr3 |= 2;
 141         dtr3 &= ~(BIT4 | BIT5 | BIT6);
 142         dtr3 |= (2 << 4);
 143
 144         dtr3 &= ~(BIT8 | BIT9 | BIT10 | BIT11);
 145         if (mrc_params->ddr_speed == DDRFREQ_800) {
 146                 /* Extended RW delay (+1) */
 147                 dtr3 |= ((tcl - 5 + 1) << 8);
 148         } else if (mrc_params->ddr_speed == DDRFREQ_1066) {
 149                 /* Extended RW delay (+1) */
 150                 dtr3 |= ((tcl - 5 + 1) << 8);
 151         }
 152
 153         dtr3 &= ~(BIT13 | BIT14 | BIT15 | BIT16);
 154         dtr3 |= ((4 + wl + twtr - 11) << 13);
 155
 156         dtr3 &= ~(BIT22 | BIT23);
 157         if (mrc_params->ddr_speed == DDRFREQ_800)
 158                 dtr3 |= ((MMAX(0, 1 - 1)) << 22);
 159         else
 160                 dtr3 |= ((MMAX(0, 2 - 1)) << 22);
 161
 162         dtr4 &= ~(BIT0 | BIT1);
 163         dtr4 |= 1;
 164         dtr4 &= ~(BIT4 | BIT5 | BIT6);
 165         dtr4 |= (1 << 4);
 166         dtr4 &= ~(BIT8 | BIT9 | BIT10);
 167         dtr4 |= ((1 + tmp1 - tmp2 + 2) << 8);
 168         dtr4 &= ~(BIT12 | BIT13 | BIT14);
 169         dtr4 |= ((1 + tmp1 - tmp2 + 2) << 12);
 170         dtr4 &= ~(BIT15 | BIT16);
 171
 172         msg_port_write(MEM_CTLR, DTR0, dtr0);
 173         msg_port_write(MEM_CTLR, DTR1, dtr1);
 174         msg_port_write(MEM_CTLR, DTR2, dtr2);
 175         msg_port_write(MEM_CTLR, DTR3, dtr3);
 176         msg_port_write(MEM_CTLR, DTR4, dtr4);
 177
 178         LEAVEFN();
 179 }
 180
 181 /* Configure MCU before jedec init sequence */
 182 void prog_decode_before_jedec(struct mrc_params *mrc_params)
 183 {
 184         u32 drp;
 185         u32 drfc;
 186         u32 dcal;
 187         u32 dsch;
 188         u32 dpmc0;
 189
 190         ENTERFN();
 191
 192         /* Disable power saving features */
 193         dpmc0 = msg_port_read(MEM_CTLR, DPMC0);
 194         dpmc0 |= (BIT24 | BIT25);
 195         dpmc0 &= ~(BIT16 | BIT17 | BIT18);
 196         dpmc0 &= ~BIT23;
 197         msg_port_write(MEM_CTLR, DPMC0, dpmc0);
 198
 199         /* Disable out of order transactions */
 200         dsch = msg_port_read(MEM_CTLR, DSCH);
 201         dsch |= (BIT8 | BIT12);
 202         msg_port_write(MEM_CTLR, DSCH, dsch);
 203
 204         /* Disable issuing the REF command */
 205         drfc = msg_port_read(MEM_CTLR, DRFC);
 206         drfc &= ~(BIT12 | BIT13 | BIT14);
 207         msg_port_write(MEM_CTLR, DRFC, drfc);
 208
 209         /* Disable ZQ calibration short */
 210         dcal = msg_port_read(MEM_CTLR, DCAL);
 211         dcal &= ~(BIT8 | BIT9 | BIT10);
 212         dcal &= ~(BIT12 | BIT13);
 213         msg_port_write(MEM_CTLR, DCAL, dcal);
 214
 215         /*
 216          * Training performed in address mode 0, rank population has limited
 217          * impact, however simulator complains if enabled non-existing rank.
 218          */
 219         drp = 0;
 220         if (mrc_params->rank_enables & 1)
 221                 drp |= BIT0;
 222         if (mrc_params->rank_enables & 2)
 223                 drp |= BIT1;
 224         msg_port_write(MEM_CTLR, DRP, drp);
 225
 226         LEAVEFN();
 227 }
 228
 229 /*
 230  * After Cold Reset, BIOS should set COLDWAKE bit to 1 before
 231  * sending the WAKE message to the Dunit.
 232  *
 233  * For Standby Exit, or any other mode in which the DRAM is in
 234  * SR, this bit must be set to 0.
 235  */
 236 void perform_ddr_reset(struct mrc_params *mrc_params)
 237 {
 238         ENTERFN();
 239
 240         /* Set COLDWAKE bit before sending the WAKE message */
 241         mrc_write_mask(MEM_CTLR, DRMC, BIT16, BIT16);
 242
 243         /* Send wake command to DUNIT (MUST be done before JEDEC) */
 244         dram_wake_command();
 245
 246         /* Set default value */
 247         msg_port_write(MEM_CTLR, DRMC,
 248                        (mrc_params->rd_odt_value == 0 ? BIT12 : 0));
 249
 250         LEAVEFN();
 251 }
 252
 253
 254 /*
 255  * This function performs some initialization on the DDRIO unit.
 256  * This function is dependent on BOARD_ID, DDR_SPEED, and CHANNEL_ENABLES.
 257  */
 258 void ddrphy_init(struct mrc_params *mrc_params)
 259 {
 260         uint32_t temp;
 261         uint8_t ch;     /* channel counter */
 262         uint8_t rk;     /* rank counter */
 263         uint8_t bl_grp; /*  byte lane group counter (2 BLs per module) */
 264         uint8_t bl_divisor = 1; /* byte lane divisor */
 265         /* For DDR3 --> 0 == 800, 1 == 1066, 2 == 1333 */
 266         uint8_t speed = mrc_params->ddr_speed & (BIT1 | BIT0);
 267         uint8_t cas;
 268         uint8_t cwl;
 269
 270         ENTERFN();
 271
 272         cas = mrc_params->params.cl;
 273         cwl = 5 + mrc_params->ddr_speed;
 274
 275         /* ddrphy_init starts */
 276         mrc_post_code(0x03, 0x00);
 277
 278         /*
 279          * HSD#231531
 280          * Make sure IOBUFACT is deasserted before initializing the DDR PHY
 281          *
 282          * HSD#234845
 283          * Make sure WRPTRENABLE is deasserted before initializing the DDR PHY
 284          */
 285         for (ch = 0; ch < NUM_CHANNELS; ch++) {
 286                 if (mrc_params->channel_enables & (1 << ch)) {
 287                         /* Deassert DDRPHY Initialization Complete */
 288                         mrc_alt_write_mask(DDRPHY,
 289                                 (CMDPMCONFIG0 + (ch * DDRIOCCC_CH_OFFSET)),
 290                                 ~BIT20, BIT20); /* SPID_INIT_COMPLETE=0 */
 291                         /* Deassert IOBUFACT */
 292                         mrc_alt_write_mask(DDRPHY,
 293                                 (CMDCFGREG0 + (ch * DDRIOCCC_CH_OFFSET)),
 294                                 ~BIT2, BIT2);   /* IOBUFACTRST_N=0 */
 295                         /* Disable WRPTR */
 296                         mrc_alt_write_mask(DDRPHY,
 297                                 (CMDPTRREG + (ch * DDRIOCCC_CH_OFFSET)),
 298                                 ~BIT0, BIT0);   /* WRPTRENABLE=0 */
 299                 }
 300         }
 301
 302         /* Put PHY in reset */
 303         mrc_alt_write_mask(DDRPHY, MASTERRSTN, 0, BIT0);
 304
 305         /* Initialize DQ01, DQ23, CMD, CLK-CTL, COMP modules */
 306
 307         /* STEP0 */
 308         mrc_post_code(0x03, 0x10);
 309         for (ch = 0; ch < NUM_CHANNELS; ch++) {
 310                 if (mrc_params->channel_enables & (1 << ch)) {
 311                         /* DQ01-DQ23 */
 312                         for (bl_grp = 0;
 313                              bl_grp < ((NUM_BYTE_LANES / bl_divisor) / 2);
 314                              bl_grp++) {
 315                                 /* Analog MUX select - IO2xCLKSEL */
 316                                 mrc_alt_write_mask(DDRPHY,
 317                                         (DQOBSCKEBBCTL +
 318                                         (bl_grp * DDRIODQ_BL_OFFSET) +
 319                                         (ch * DDRIODQ_CH_OFFSET)),
 320                                         ((bl_grp) ? (0x00) : (BIT22)), (BIT22));
 321
 322                                 /* ODT Strength */
 323                                 switch (mrc_params->rd_odt_value) {
 324                                 case 1:
 325                                         temp = 0x3;
 326                                         break;  /* 60 ohm */
 327                                 case 2:
 328                                         temp = 0x3;
 329                                         break;  /* 120 ohm */
 330                                 case 3:
 331                                         temp = 0x3;
 332                                         break;  /* 180 ohm */
 333                                 default:
 334                                         temp = 0x3;
 335                                         break;  /* 120 ohm */
 336                                 }
 337
 338                                 /* ODT strength */
 339                                 mrc_alt_write_mask(DDRPHY,
 340                                         (B0RXIOBUFCTL +
 341                                         (bl_grp * DDRIODQ_BL_OFFSET) +
 342                                         (ch * DDRIODQ_CH_OFFSET)),
 343                                         (temp << 5), (BIT6 | BIT5));
 344                                 /* ODT strength */
 345                                 mrc_alt_write_mask(DDRPHY,
 346                                         (B1RXIOBUFCTL +
 347                                         (bl_grp * DDRIODQ_BL_OFFSET) +
 348                                         (ch * DDRIODQ_CH_OFFSET)),
 349                                         (temp << 5), (BIT6 | BIT5));
 350
 351                                 /* Dynamic ODT/DIFFAMP */
 352                                 temp = (((cas) << 24) | ((cas) << 16) |
 353                                         ((cas) << 8) | ((cas) << 0));
 354                                 switch (speed) {
 355                                 case 0:
 356                                         temp -= 0x01010101;
 357                                         break;  /* 800 */
 358                                 case 1:
 359                                         temp -= 0x02020202;
 360                                         break;  /* 1066 */
 361                                 case 2:
 362                                         temp -= 0x03030303;
 363                                         break;  /* 1333 */
 364                                 case 3:
 365                                         temp -= 0x04040404;
 366                                         break;  /* 1600 */
 367                                 }
 368
 369                                 /* Launch Time: ODT, DIFFAMP, ODT, DIFFAMP */
 370                                 mrc_alt_write_mask(DDRPHY,
 371                                         (B01LATCTL1 +
 372                                         (bl_grp * DDRIODQ_BL_OFFSET) +
 373                                         (ch * DDRIODQ_CH_OFFSET)),
 374                                         temp,
 375                                         (BIT28 | BIT27 | BIT26 | BIT25 | BIT24 |
 376                                         BIT20 | BIT19 | BIT18 | BIT17 | BIT16 |
 377                                         BIT12 | BIT11 | BIT10 | BIT9 | BIT8 |
 378                                         BIT4 | BIT3 | BIT2 | BIT1 | BIT0));
 379                                 switch (speed) {
 380                                 /* HSD#234715 */
 381                                 case 0:
 382                                         temp = ((0x06 << 16) | (0x07 << 8));
 383                                         break;  /* 800 */
 384                                 case 1:
 385                                         temp = ((0x07 << 16) | (0x08 << 8));
 386                                         break;  /* 1066 */
 387                                 case 2:
 388                                         temp = ((0x09 << 16) | (0x0A << 8));
 389                                         break;  /* 1333 */
 390                                 case 3:
 391                                         temp = ((0x0A << 16) | (0x0B << 8));
 392                                         break;  /* 1600 */
 393                                 }
 394
 395                                 /* On Duration: ODT, DIFFAMP */
 396                                 mrc_alt_write_mask(DDRPHY,
 397                                         (B0ONDURCTL +
 398                                         (bl_grp * DDRIODQ_BL_OFFSET) +
 399                                         (ch * DDRIODQ_CH_OFFSET)),
 400                                         temp,
 401                                         (BIT21 | BIT20 | BIT19 | BIT18 | BIT17 |
 402                                         BIT16 | BIT13 | BIT12 | BIT11 | BIT10 |
 403                                         BIT9 | BIT8));
 404                                 /* On Duration: ODT, DIFFAMP */
 405                                 mrc_alt_write_mask(DDRPHY,
 406                                         (B1ONDURCTL +
 407                                         (bl_grp * DDRIODQ_BL_OFFSET) +
 408                                         (ch * DDRIODQ_CH_OFFSET)),
 409                                         temp,
 410                                         (BIT21 | BIT20 | BIT19 | BIT18 | BIT17 |
 411                                         BIT16 | BIT13 | BIT12 | BIT11 | BIT10 |
 412                                         BIT9 | BIT8));
 413
 414                                 switch (mrc_params->rd_odt_value) {
 415                                 case 0:
 416                                         /* override DIFFAMP=on, ODT=off */
 417                                         temp = ((0x3F << 16) | (0x3f << 10));
 418                                         break;
 419                                 default:
 420                                         /* override DIFFAMP=on, ODT=on */
 421                                         temp = ((0x3F << 16) | (0x2A << 10));
 422                                         break;
 423                                 }
 424
 425                                 /* Override: DIFFAMP, ODT */
 426                                 mrc_alt_write_mask(DDRPHY,
 427                                         (B0OVRCTL +
 428                                         (bl_grp * DDRIODQ_BL_OFFSET) +
 429                                         (ch * DDRIODQ_CH_OFFSET)),
 430                                         temp,
 431                                         (BIT21 | BIT20 | BIT19 | BIT18 | BIT17 |
 432                                         BIT16 | BIT15 | BIT14 | BIT13 | BIT12 |
 433                                         BIT11 | BIT10));
 434                                 /* Override: DIFFAMP, ODT */
 435                                 mrc_alt_write_mask(DDRPHY,
 436                                         (B1OVRCTL +
 437                                         (bl_grp * DDRIODQ_BL_OFFSET) +
 438                                         (ch * DDRIODQ_CH_OFFSET)),
 439                                         temp,
 440                                         (BIT21 | BIT20 | BIT19 | BIT18 | BIT17 |
 441                                         BIT16 | BIT15 | BIT14 | BIT13 | BIT12 |
 442                                         BIT11 | BIT10));
 443
 444                                 /* DLL Setup */
 445
 446                                 /* 1xCLK Domain Timings: tEDP,RCVEN,WDQS (PO) */
 447                                 mrc_alt_write_mask(DDRPHY,
 448                                         (B0LATCTL0 +
 449                                         (bl_grp * DDRIODQ_BL_OFFSET) +
 450                                         (ch * DDRIODQ_CH_OFFSET)),
 451                                         (((cas + 7) << 16) | ((cas - 4) << 8) |
 452                                         ((cwl - 2) << 0)),
 453                                         (BIT21 | BIT20 | BIT19 | BIT18 | BIT17 |
 454                                         BIT16 | BIT12 | BIT11 | BIT10 | BIT9 |
 455                                         BIT8 | BIT4 | BIT3 | BIT2 | BIT1 |
 456                                         BIT0));
 457                                 mrc_alt_write_mask(DDRPHY,
 458                                         (B1LATCTL0 +
 459                                         (bl_grp * DDRIODQ_BL_OFFSET) +
 460                                         (ch * DDRIODQ_CH_OFFSET)),
 461                                         (((cas + 7) << 16) | ((cas - 4) << 8) |
 462                                         ((cwl - 2) << 0)),
 463                                         (BIT21 | BIT20 | BIT19 | BIT18 | BIT17 |
 464                                         BIT16 | BIT12 | BIT11 | BIT10 | BIT9 |
 465                                         BIT8 | BIT4 | BIT3 | BIT2 | BIT1 |
 466                                         BIT0));
 467
 468                                 /* RCVEN Bypass (PO) */
 469                                 mrc_alt_write_mask(DDRPHY,
 470                                         (B0RXIOBUFCTL +
 471                                         (bl_grp * DDRIODQ_BL_OFFSET) +
 472                                         (ch * DDRIODQ_CH_OFFSET)),
 473                                         ((0x0 << 7) | (0x0 << 0)),
 474                                         (BIT7 | BIT0));
 475                                 mrc_alt_write_mask(DDRPHY,
 476                                         (B1RXIOBUFCTL +
 477                                         (bl_grp * DDRIODQ_BL_OFFSET) +
 478                                         (ch * DDRIODQ_CH_OFFSET)),
 479                                         ((0x0 << 7) | (0x0 << 0)),
 480                                         (BIT7 | BIT0));
 481
 482                                 /* TX */
 483                                 mrc_alt_write_mask(DDRPHY,
 484                                         (DQCTL +
 485                                         (bl_grp * DDRIODQ_BL_OFFSET) +
 486                                         (ch * DDRIODQ_CH_OFFSET)),
 487                                         (BIT16), (BIT16));
 488                                 mrc_alt_write_mask(DDRPHY,
 489                                         (B01PTRCTL1 +
 490                                         (bl_grp * DDRIODQ_BL_OFFSET) +
 491                                         (ch * DDRIODQ_CH_OFFSET)),
 492                                         (BIT8), (BIT8));
 493
 494                                 /* RX (PO) */
 495                                 /* Internal Vref Code, Enable#, Ext_or_Int (1=Ext) */
 496                                 mrc_alt_write_mask(DDRPHY,
 497                                         (B0VREFCTL +
 498                                         (bl_grp * DDRIODQ_BL_OFFSET) +
 499                                         (ch * DDRIODQ_CH_OFFSET)),
 500                                         ((0x03 << 2) | (0x0 << 1) | (0x0 << 0)),
 501                                         (BIT7 | BIT6 | BIT5 | BIT4 | BIT3 |
 502                                         BIT2 | BIT1 | BIT0));
 503                                 /* Internal Vref Code, Enable#, Ext_or_Int (1=Ext) */
 504                                 mrc_alt_write_mask(DDRPHY,
 505                                         (B1VREFCTL +
 506                                         (bl_grp * DDRIODQ_BL_OFFSET) +
 507                                         (ch * DDRIODQ_CH_OFFSET)),
 508                                         ((0x03 << 2) | (0x0 << 1) | (0x0 << 0)),
 509                                         (BIT7 | BIT6 | BIT5 | BIT4 | BIT3 |
 510                                         BIT2 | BIT1 | BIT0));
 511                                 /* Per-Bit De-Skew Enable */
 512                                 mrc_alt_write_mask(DDRPHY,
 513                                         (B0RXIOBUFCTL +
 514                                         (bl_grp * DDRIODQ_BL_OFFSET) +
 515                                         (ch * DDRIODQ_CH_OFFSET)),
 516                                         (0), (BIT4));
 517                                 /* Per-Bit De-Skew Enable */
 518                                 mrc_alt_write_mask(DDRPHY,
 519                                         (B1RXIOBUFCTL +
 520                                         (bl_grp * DDRIODQ_BL_OFFSET) +
 521                                         (ch * DDRIODQ_CH_OFFSET)),
 522                                         (0), (BIT4));
 523                         }
 524
 525                         /* CLKEBB */
 526                         mrc_alt_write_mask(DDRPHY,
 527                                 (CMDOBSCKEBBCTL + (ch * DDRIOCCC_CH_OFFSET)),
 528                                 0, (BIT23));
 529
 530                         /* Enable tristate control of cmd/address bus */
 531                         mrc_alt_write_mask(DDRPHY,
 532                                 (CMDCFGREG0 + (ch * DDRIOCCC_CH_OFFSET)),
 533                                 0, (BIT1 | BIT0));
 534
 535                         /* ODT RCOMP */
 536                         mrc_alt_write_mask(DDRPHY,
 537                                 (CMDRCOMPODT + (ch * DDRIOCCC_CH_OFFSET)),
 538                                 ((0x03 << 5) | (0x03 << 0)),
 539                                 (BIT9 | BIT8 | BIT7 | BIT6 | BIT5 | BIT4 |
 540                                 BIT3 | BIT2 | BIT1 | BIT0));
 541
 542                         /* CMDPM* registers must be programmed in this order */
 543
 544                         /* Turn On Delays: SFR (regulator), MPLL */
 545                         mrc_alt_write_mask(DDRPHY,
 546                                 (CMDPMDLYREG4 + (ch * DDRIOCCC_CH_OFFSET)),
 547                                 ((0xFFFFU << 16) | (0xFFFF << 0)),
 548                                 0xFFFFFFFF);
 549                         /*
 550                          * Delays: ASSERT_IOBUFACT_to_ALLON0_for_PM_MSG_3,
 551                          * VREG (MDLL) Turn On, ALLON0_to_DEASSERT_IOBUFACT
 552                          * for_PM_MSG_gt0, MDLL Turn On
 553                          */
 554                         mrc_alt_write_mask(DDRPHY,
 555                                 (CMDPMDLYREG3 + (ch * DDRIOCCC_CH_OFFSET)),
 556                                 ((0xFU << 28) | (0xFFF << 16) | (0xF << 12) |
 557                                 (0x616 << 0)), 0xFFFFFFFF);
 558                         /* MPLL Divider Reset Delays */
 559                         mrc_alt_write_mask(DDRPHY,
 560                                 (CMDPMDLYREG2 + (ch * DDRIOCCC_CH_OFFSET)),
 561                                 ((0xFFU << 24) | (0xFF << 16) | (0xFF << 8) |
 562                                 (0xFF << 0)), 0xFFFFFFFF);
 563                         /* Turn Off Delays: VREG, Staggered MDLL, MDLL, PI */
 564                         mrc_alt_write_mask(DDRPHY,
 565                                 (CMDPMDLYREG1 + (ch * DDRIOCCC_CH_OFFSET)),
 566                                 ((0xFFU << 24) | (0xFF << 16) | (0xFF << 8) |
 567                                 (0xFF << 0)), 0xFFFFFFFF);
 568                         /* Turn On Delays: MPLL, Staggered MDLL, PI, IOBUFACT */
 569                         mrc_alt_write_mask(DDRPHY,
 570                                 (CMDPMDLYREG0 + (ch * DDRIOCCC_CH_OFFSET)),
 571                                 ((0xFFU << 24) | (0xFF << 16) | (0xFF << 8) |
 572                                 (0xFF << 0)), 0xFFFFFFFF);
 573                         /* Allow PUnit signals */
 574                         mrc_alt_write_mask(DDRPHY,
 575                                 (CMDPMCONFIG0 + (ch * DDRIOCCC_CH_OFFSET)),
 576                                 ((0x6 << 8) | BIT6 | (0x4 << 0)),
 577                                 (BIT31 | BIT30 | BIT29 | BIT28 | BIT27 | BIT26 |
 578                                 BIT25 | BIT24 | BIT23 | BIT22 | BIT21 | BIT11 |
 579                                 BIT10 | BIT9 | BIT8 | BIT6 | BIT3 | BIT2 |
 580                                 BIT1 | BIT0));
 581                         /* DLL_VREG Bias Trim, VREF Tuning for DLL_VREG */
 582                         mrc_alt_write_mask(DDRPHY,
 583                                 (CMDMDLLCTL + (ch * DDRIOCCC_CH_OFFSET)),
 584                                 ((0x3 << 4) | (0x7 << 0)),
 585                                 (BIT6 | BIT5 | BIT4 | BIT3 | BIT2 | BIT1 |
 586                                 BIT0));
 587
 588                         /* CLK-CTL */
 589                         mrc_alt_write_mask(DDRPHY,
 590                                 (CCOBSCKEBBCTL + (ch * DDRIOCCC_CH_OFFSET)),
 591                                 0, BIT24);      /* CLKEBB */
 592                         /* Buffer Enable: CS,CKE,ODT,CLK */
 593                         mrc_alt_write_mask(DDRPHY,
 594                                 (CCCFGREG0 + (ch * DDRIOCCC_CH_OFFSET)),
 595                                 ((0x0 << 16) | (0x0 << 12) | (0x0 << 8) |
 596                                 (0xF << 4) | BIT0),
 597                                 (BIT19 | BIT18 | BIT17 | BIT16 | BIT15 | BIT14 |
 598                                 BIT13 | BIT12 | BIT11 | BIT10 | BIT9 | BIT8 |
 599                                 BIT7 | BIT6 | BIT5 | BIT4 | BIT0));
 600                         /* ODT RCOMP */
 601                         mrc_alt_write_mask(DDRPHY,
 602                                 (CCRCOMPODT + (ch * DDRIOCCC_CH_OFFSET)),
 603                                 ((0x03 << 8) | (0x03 << 0)),
 604                                 (BIT12 | BIT11 | BIT10 | BIT9 | BIT8 | BIT4 |
 605                                 BIT3 | BIT2 | BIT1 | BIT0));
 606                         /* DLL_VREG Bias Trim, VREF Tuning for DLL_VREG */
 607                         mrc_alt_write_mask(DDRPHY,
 608                                 (CCMDLLCTL + (ch * DDRIOCCC_CH_OFFSET)),
 609                                 ((0x3 << 4) | (0x7 << 0)),
 610                                 (BIT6 | BIT5 | BIT4 | BIT3 | BIT2 | BIT1 |
 611                                 BIT0));
 612
 613                         /*
 614                          * COMP (RON channel specific)
 615                          * - DQ/DQS/DM RON: 32 Ohm
 616                          * - CTRL/CMD RON: 27 Ohm
 617                          * - CLK RON: 26 Ohm
 618                          */
 619                         /* RCOMP Vref PU/PD */
 620                         mrc_alt_write_mask(DDRPHY,
 621                                 (DQVREFCH0 +  (ch * DDRCOMP_CH_OFFSET)),
 622                                 ((0x08 << 24) | (0x03 << 16)),
 623                                 (BIT29 | BIT28 | BIT27 | BIT26 | BIT25 |
 624                                 BIT24 | BIT21 | BIT20 | BIT19 | BIT18 |
 625                                 BIT17 | BIT16));
 626                         /* RCOMP Vref PU/PD */
 627                         mrc_alt_write_mask(DDRPHY,
 628                                 (CMDVREFCH0 + (ch * DDRCOMP_CH_OFFSET)),
 629                                 ((0x0C << 24) | (0x03 << 16)),
 630                                 (BIT29 | BIT28 | BIT27 | BIT26 | BIT25 |
 631                                 BIT24 | BIT21 | BIT20 | BIT19 | BIT18 |
 632                                 BIT17 | BIT16));
 633                         /* RCOMP Vref PU/PD */
 634                         mrc_alt_write_mask(DDRPHY,
 635                                 (CLKVREFCH0 + (ch * DDRCOMP_CH_OFFSET)),
 636                                 ((0x0F << 24) | (0x03 << 16)),
 637                                 (BIT29 | BIT28 | BIT27 | BIT26 | BIT25 |
 638                                 BIT24 | BIT21 | BIT20 | BIT19 | BIT18 |
 639                                 BIT17 | BIT16));
 640                         /* RCOMP Vref PU/PD */
 641                         mrc_alt_write_mask(DDRPHY,
 642                                 (DQSVREFCH0 + (ch * DDRCOMP_CH_OFFSET)),
 643                                 ((0x08 << 24) | (0x03 << 16)),
 644                                 (BIT29 | BIT28 | BIT27 | BIT26 | BIT25 |
 645                                 BIT24 | BIT21 | BIT20 | BIT19 | BIT18 |
 646                                 BIT17 | BIT16));
 647                         /* RCOMP Vref PU/PD */
 648                         mrc_alt_write_mask(DDRPHY,
 649                                 (CTLVREFCH0 + (ch * DDRCOMP_CH_OFFSET)),
 650                                 ((0x0C << 24) | (0x03 << 16)),
 651                                 (BIT29 | BIT28 | BIT27 | BIT26 | BIT25 |
 652                                 BIT24 | BIT21 | BIT20 | BIT19 | BIT18 |
 653                                 BIT17 | BIT16));
 654
 655                         /* DQS Swapped Input Enable */
 656                         mrc_alt_write_mask(DDRPHY,
 657                                 (COMPEN1CH0 + (ch * DDRCOMP_CH_OFFSET)),
 658                                 (BIT19 | BIT17),
 659                                 (BIT31 | BIT30 | BIT19 | BIT17 |
 660                                 BIT15 | BIT14));
 661
 662                         /* ODT VREF = 1.5 x 274/360+274 = 0.65V (code of ~50) */
 663                         /* ODT Vref PU/PD */
 664                         mrc_alt_write_mask(DDRPHY,
 665                                 (DQVREFCH0 + (ch * DDRCOMP_CH_OFFSET)),
 666                                 ((0x32 << 8) | (0x03 << 0)),
 667                                 (BIT13 | BIT12 | BIT11 | BIT10 | BIT9 | BIT8 |
 668                                 BIT5 | BIT4 | BIT3 | BIT2 | BIT1 | BIT0));
 669                         /* ODT Vref PU/PD */
 670                         mrc_alt_write_mask(DDRPHY,
 671                                 (DQSVREFCH0 + (ch * DDRCOMP_CH_OFFSET)),
 672                                 ((0x32 << 8) | (0x03 << 0)),
 673                                 (BIT13 | BIT12 | BIT11 | BIT10 | BIT9 | BIT8 |
 674                                 BIT5 | BIT4 | BIT3 | BIT2 | BIT1 | BIT0));
 675                         /* ODT Vref PU/PD */
 676                         mrc_alt_write_mask(DDRPHY,
 677                                 (CLKVREFCH0 + (ch * DDRCOMP_CH_OFFSET)),
 678                                 ((0x0E << 8) | (0x05 << 0)),
 679                                 (BIT13 | BIT12 | BIT11 | BIT10 | BIT9 | BIT8 |
 680                                 BIT5 | BIT4 | BIT3 | BIT2 | BIT1 | BIT0));
 681
 682                         /*
 683                          * Slew rate settings are frequency specific,
 684                          * numbers below are for 800Mhz (speed == 0)
 685                          * - DQ/DQS/DM/CLK SR: 4V/ns,
 686                          * - CTRL/CMD SR: 1.5V/ns
 687                          */
 688                         temp = (0x0E << 16) | (0x0E << 12) | (0x08 << 8) |
 689                                 (0x0B << 4) | (0x0B << 0);
 690                         /* DCOMP Delay Select: CTL,CMD,CLK,DQS,DQ */
 691                         mrc_alt_write_mask(DDRPHY,
 692                                 (DLYSELCH0 + (ch * DDRCOMP_CH_OFFSET)),
 693                                 temp,
 694                                 (BIT19 | BIT18 | BIT17 | BIT16 | BIT15 |
 695                                 BIT14 | BIT13 | BIT12 | BIT11 | BIT10 |
 696                                 BIT9 | BIT8 | BIT7 | BIT6 | BIT5 | BIT4 |
 697                                 BIT3 | BIT2 | BIT1 | BIT0));
 698                         /* TCO Vref CLK,DQS,DQ */
 699                         mrc_alt_write_mask(DDRPHY,
 700                                 (TCOVREFCH0 + (ch * DDRCOMP_CH_OFFSET)),
 701                                 ((0x05 << 16) | (0x05 << 8) | (0x05 << 0)),
 702                                 (BIT21 | BIT20 | BIT19 | BIT18 | BIT17 |
 703                                 BIT16 | BIT13 | BIT12 | BIT11 | BIT10 |
 704                                 BIT9 | BIT8 | BIT5 | BIT4 | BIT3 | BIT2 |
 705                                 BIT1 | BIT0));
 706                         /* ODTCOMP CMD/CTL PU/PD */
 707                         mrc_alt_write_mask(DDRPHY,
 708                                 (CCBUFODTCH0 + (ch * DDRCOMP_CH_OFFSET)),
 709                                 ((0x03 << 8) | (0x03 << 0)),
 710                                 (BIT12 | BIT11 | BIT10 | BIT9 | BIT8 |
 711                                 BIT4 | BIT3 | BIT2 | BIT1 | BIT0));
 712                         /* COMP */
 713                         mrc_alt_write_mask(DDRPHY,
 714                                 (COMPEN0CH0 + (ch * DDRCOMP_CH_OFFSET)),
 715                                 0, (BIT31 | BIT30 | BIT8));
 716
 717 #ifdef BACKUP_COMPS
 718                         /* DQ COMP Overrides */
 719                         /* RCOMP PU */
 720                         mrc_alt_write_mask(DDRPHY,
 721                                 (DQDRVPUCTLCH0 + (ch * DDRCOMP_CH_OFFSET)),
 722                                 (BIT31 | (0x0A << 16)),
 723                                 (BIT31 | BIT20 | BIT19 |
 724                                 BIT18 | BIT17 | BIT16));
 725                         /* RCOMP PD */
 726                         mrc_alt_write_mask(DDRPHY,
 727                                 (DQDRVPDCTLCH0 + (ch * DDRCOMP_CH_OFFSET)),
 728                                 (BIT31 | (0x0A << 16)),
 729                                 (BIT31 | BIT20 | BIT19 |
 730                                 BIT18 | BIT17 | BIT16));
 731                         /* DCOMP PU */
 732                         mrc_alt_write_mask(DDRPHY,
 733                                 (DQDLYPUCTLCH0 + (ch * DDRCOMP_CH_OFFSET)),
 734                                 (BIT31 | (0x10 << 16)),
 735                                 (BIT31 | BIT20 | BIT19 |
 736                                 BIT18 | BIT17 | BIT16));
 737                         /* DCOMP PD */
 738                         mrc_alt_write_mask(DDRPHY,
 739                                 (DQDLYPDCTLCH0 + (ch * DDRCOMP_CH_OFFSET)),
 740                                 (BIT31 | (0x10 << 16)),
 741                                 (BIT31 | BIT20 | BIT19 |
 742                                 BIT18 | BIT17 | BIT16));
 743                         /* ODTCOMP PU */
 744                         mrc_alt_write_mask(DDRPHY,
 745                                 (DQODTPUCTLCH0 + (ch * DDRCOMP_CH_OFFSET)),
 746                                 (BIT31 | (0x0B << 16)),
 747                                 (BIT31 | BIT20 | BIT19 |
 748                                 BIT18 | BIT17 | BIT16));
 749                         /* ODTCOMP PD */
 750                         mrc_alt_write_mask(DDRPHY,
 751                                 (DQODTPDCTLCH0 + (ch * DDRCOMP_CH_OFFSET)),
 752                                 (BIT31 | (0x0B << 16)),
 753                                 (BIT31 | BIT20 | BIT19 |
 754                                 BIT18 | BIT17 | BIT16));
 755                         /* TCOCOMP PU */
 756                         mrc_alt_write_mask(DDRPHY,
 757                                 (DQTCOPUCTLCH0 + (ch * DDRCOMP_CH_OFFSET)),
 758                                 (BIT31), (BIT31));
 759                         /* TCOCOMP PD */
 760                         mrc_alt_write_mask(DDRPHY,
 761                                 (DQTCOPDCTLCH0 + (ch * DDRCOMP_CH_OFFSET)),
 762                                 (BIT31), (BIT31));
 763
 764                         /* DQS COMP Overrides */
 765                         /* RCOMP PU */
 766                         mrc_alt_write_mask(DDRPHY,
 767                                 (DQSDRVPUCTLCH0 + (ch * DDRCOMP_CH_OFFSET)),
 768                                 (BIT31 | (0x0A << 16)),
 769                                 (BIT31 | BIT20 | BIT19 |
 770                                 BIT18 | BIT17 | BIT16));
 771                         /* RCOMP PD */
 772                         mrc_alt_write_mask(DDRPHY,
 773                                 (DQSDRVPDCTLCH0 + (ch * DDRCOMP_CH_OFFSET)),
 774                                 (BIT31 | (0x0A << 16)),
 775                                 (BIT31 | BIT20 | BIT19 |
 776                                 BIT18 | BIT17 | BIT16));
 777                         /* DCOMP PU */
 778                         mrc_alt_write_mask(DDRPHY,
 779                                 (DQSDLYPUCTLCH0 + (ch * DDRCOMP_CH_OFFSET)),
 780                                 (BIT31 | (0x10 << 16)),
 781                                 (BIT31 | BIT20 | BIT19 |
 782                                 BIT18 | BIT17 | BIT16));
 783                         /* DCOMP PD */
 784                         mrc_alt_write_mask(DDRPHY,
 785                                 (DQSDLYPDCTLCH0 + (ch * DDRCOMP_CH_OFFSET)),
 786                                 (BIT31 | (0x10 << 16)),
 787                                 (BIT31 | BIT20 | BIT19 |
 788                                 BIT18 | BIT17 | BIT16));
 789                         /* ODTCOMP PU */
 790                         mrc_alt_write_mask(DDRPHY,
 791                                 (DQSODTPUCTLCH0 + (ch * DDRCOMP_CH_OFFSET)),
 792                                 (BIT31 | (0x0B << 16)),
 793                                 (BIT31 | BIT20 | BIT19 |
 794                                 BIT18 | BIT17 | BIT16));
 795                         /* ODTCOMP PD */
 796                         mrc_alt_write_mask(DDRPHY,
 797                                 (DQSODTPDCTLCH0 + (ch * DDRCOMP_CH_OFFSET)),
 798                                 (BIT31 | (0x0B << 16)),
 799                                 (BIT31 | BIT20 | BIT19 |
 800                                 BIT18 | BIT17 | BIT16));
 801                         /* TCOCOMP PU */
 802                         mrc_alt_write_mask(DDRPHY,
 803                                 (DQSTCOPUCTLCH0 + (ch * DDRCOMP_CH_OFFSET)),
 804                                 (BIT31), (BIT31));
 805                         /* TCOCOMP PD */
 806                         mrc_alt_write_mask(DDRPHY,
 807                                 (DQSTCOPDCTLCH0 + (ch * DDRCOMP_CH_OFFSET)),
 808                                 (BIT31), (BIT31));
 809
 810                         /* CLK COMP Overrides */
 811                         /* RCOMP PU */
 812                         mrc_alt_write_mask(DDRPHY,
 813                                 (CLKDRVPUCTLCH0 + (ch * DDRCOMP_CH_OFFSET)),
 814                                 (BIT31 | (0x0C << 16)),
 815                                 (BIT31 | (0x0B << 16)),
 816                                 (BIT31 | BIT20 | BIT19 |
 817                                 BIT18 | BIT17 | BIT16));
 818                         /* RCOMP PD */
 819                         mrc_alt_write_mask(DDRPHY,
 820                                 (CLKDRVPDCTLCH0 + (ch * DDRCOMP_CH_OFFSET)),
 821                                 (BIT31 | (0x0C << 16)),
 822                                 (BIT31 | (0x0B << 16)),
 823                                 (BIT31 | BIT20 | BIT19 |
 824                                 BIT18 | BIT17 | BIT16));
 825                         /* DCOMP PU */
 826                         mrc_alt_write_mask(DDRPHY,
 827                                 (CLKDLYPUCTLCH0 + (ch * DDRCOMP_CH_OFFSET)),
 828                                 (BIT31 | (0x07 << 16)),
 829                                 (BIT31 | (0x0B << 16)),
 830                                 (BIT31 | BIT20 | BIT19 |
 831                                 BIT18 | BIT17 | BIT16));
 832                         /* DCOMP PD */
 833                         mrc_alt_write_mask(DDRPHY,
 834                                 (CLKDLYPDCTLCH0 + (ch * DDRCOMP_CH_OFFSET)),
 835                                 (BIT31 | (0x07 << 16)),
 836                                 (BIT31 | (0x0B << 16)),
 837                                 (BIT31 | BIT20 | BIT19 |
 838                                 BIT18 | BIT17 | BIT16));
 839                         /* ODTCOMP PU */
 840                         mrc_alt_write_mask(DDRPHY,
 841                                 (CLKODTPUCTLCH0 + (ch * DDRCOMP_CH_OFFSET)),
 842                                 (BIT31 | (0x0B << 16)),
 843                                 (BIT31 | (0x0B << 16)),
 844                                 (BIT31 | BIT20 | BIT19 |
 845                                 BIT18 | BIT17 | BIT16));
 846                         /* ODTCOMP PD */
 847                         mrc_alt_write_mask(DDRPHY,
 848                                 (CLKODTPDCTLCH0 + (ch * DDRCOMP_CH_OFFSET)),
 849                                 (BIT31 | (0x0B << 16)),
 850                                 (BIT31 | (0x0B << 16)),
 851                                 (BIT31 | BIT20 | BIT19 |
 852                                 BIT18 | BIT17 | BIT16));
 853                         /* TCOCOMP PU */
 854                         mrc_alt_write_mask(DDRPHY,
 855                                 (CLKTCOPUCTLCH0 + (ch * DDRCOMP_CH_OFFSET)),
 856                                 (BIT31), (BIT31));
 857                         /* TCOCOMP PD */
 858                         mrc_alt_write_mask(DDRPHY,
 859                                 (CLKTCOPDCTLCH0 + (ch * DDRCOMP_CH_OFFSET)),
 860                                 (BIT31), (BIT31));
 861
 862                         /* CMD COMP Overrides */
 863                         /* RCOMP PU */
 864                         mrc_alt_write_mask(DDRPHY,
 865                                 (CMDDRVPUCTLCH0 + (ch * DDRCOMP_CH_OFFSET)),
 866                                 (BIT31 | (0x0D << 16)),
 867                                 (BIT31 | BIT21 | BIT20 | BIT19 |
 868                                 BIT18 | BIT17 | BIT16));
 869                         /* RCOMP PD */
 870                         mrc_alt_write_mask(DDRPHY,
 871                                 (CMDDRVPDCTLCH0 + (ch * DDRCOMP_CH_OFFSET)),
 872                                 (BIT31 | (0x0D << 16)),
 873                                 (BIT31 | BIT21 | BIT20 | BIT19 |
 874                                 BIT18 | BIT17 | BIT16));
 875                         /* DCOMP PU */
 876                         mrc_alt_write_mask(DDRPHY,
 877                                 (CMDDLYPUCTLCH0 + (ch * DDRCOMP_CH_OFFSET)),
 878                                 (BIT31 | (0x0A << 16)),
 879                                 (BIT31 | BIT20 | BIT19 |
 880                                 BIT18 | BIT17 | BIT16));
 881                         /* DCOMP PD */
 882                         mrc_alt_write_mask(DDRPHY,
 883                                 (CMDDLYPDCTLCH0 + (ch * DDRCOMP_CH_OFFSET)),
 884                                 (BIT31 | (0x0A << 16)),
 885                                 (BIT31 | BIT20 | BIT19 |
 886                                 BIT18 | BIT17 | BIT16));
 887
 888                         /* CTL COMP Overrides */
 889                         /* RCOMP PU */
 890                         mrc_alt_write_mask(DDRPHY,
 891                                 (CTLDRVPUCTLCH0 + (ch * DDRCOMP_CH_OFFSET)),
 892                                 (BIT31 | (0x0D << 16)),
 893                                 (BIT31 | BIT21 | BIT20 | BIT19 |
 894                                 BIT18 | BIT17 | BIT16));
 895                         /* RCOMP PD */
 896                         mrc_alt_write_mask(DDRPHY,
 897                                 (CTLDRVPDCTLCH0 + (ch * DDRCOMP_CH_OFFSET)),
 898                                 (BIT31 | (0x0D << 16)),
 899                                 (BIT31 | BIT21 | BIT20 | BIT19 |
 900                                 BIT18 | BIT17 | BIT16));
 901                         /* DCOMP PU */
 902                         mrc_alt_write_mask(DDRPHY,
 903                                 (CTLDLYPUCTLCH0 + (ch * DDRCOMP_CH_OFFSET)),
 904                                 (BIT31 | (0x0A << 16)),
 905                                 (BIT31 | BIT20 | BIT19 |
 906                                 BIT18 | BIT17 | BIT16));
 907                         /* DCOMP PD */
 908                         mrc_alt_write_mask(DDRPHY,
 909                                 (CTLDLYPDCTLCH0 + (ch * DDRCOMP_CH_OFFSET)),
 910                                 (BIT31 | (0x0A << 16)),
 911                                 (BIT31 | BIT20 | BIT19 |
 912                                 BIT18 | BIT17 | BIT16));
 913 #else
 914                         /* DQ TCOCOMP Overrides */
 915                         /* TCOCOMP PU */
 916                         mrc_alt_write_mask(DDRPHY,
 917                                 (DQTCOPUCTLCH0 + (ch * DDRCOMP_CH_OFFSET)),
 918                                 (BIT31 | (0x1F << 16)),
 919                                 (BIT31 | BIT20 | BIT19 |
 920                                 BIT18 | BIT17 | BIT16));
 921                         /* TCOCOMP PD */
 922                         mrc_alt_write_mask(DDRPHY,
 923                                 (DQTCOPDCTLCH0 + (ch * DDRCOMP_CH_OFFSET)),
 924                                 (BIT31 | (0x1F << 16)),
 925                                 (BIT31 | BIT20 | BIT19 |
 926                                 BIT18 | BIT17 | BIT16));
 927
 928                         /* DQS TCOCOMP Overrides */
 929                         /* TCOCOMP PU */
 930                         mrc_alt_write_mask(DDRPHY,
 931                                 (DQSTCOPUCTLCH0 + (ch * DDRCOMP_CH_OFFSET)),
 932                                 (BIT31 | (0x1F << 16)),
 933                                 (BIT31 | BIT20 | BIT19 |
 934                                 BIT18 | BIT17 | BIT16));
 935                         /* TCOCOMP PD */
 936                         mrc_alt_write_mask(DDRPHY,
 937                                 (DQSTCOPDCTLCH0 + (ch * DDRCOMP_CH_OFFSET)),
 938                                 (BIT31 | (0x1F << 16)),
 939                                 (BIT31 | BIT20 | BIT19 |
 940                                 BIT18 | BIT17 | BIT16));
 941
 942                         /* CLK TCOCOMP Overrides */
 943                         /* TCOCOMP PU */
 944                         mrc_alt_write_mask(DDRPHY,
 945                                 (CLKTCOPUCTLCH0 + (ch * DDRCOMP_CH_OFFSET)),
 946                                 (BIT31 | (0x1F << 16)),
 947                                 (BIT31 | BIT20 | BIT19 |
 948                                 BIT18 | BIT17 | BIT16));
 949                         /* TCOCOMP PD */
 950                         mrc_alt_write_mask(DDRPHY,
 951                                 (CLKTCOPDCTLCH0 + (ch * DDRCOMP_CH_OFFSET)),
 952                                 (BIT31 | (0x1F << 16)),
 953                                 (BIT31 | BIT20 | BIT19 |
 954                                 BIT18 | BIT17 | BIT16));
 955 #endif
 956
 957                         /* program STATIC delays */
 958 #ifdef BACKUP_WCMD
 959                         set_wcmd(ch, ddr_wcmd[PLATFORM_ID]);
 960 #else
 961                         set_wcmd(ch, ddr_wclk[PLATFORM_ID] + HALF_CLK);
 962 #endif
 963
 964                         for (rk = 0; rk < NUM_RANKS; rk++) {
 965                                 if (mrc_params->rank_enables & (1<<rk)) {
 966                                         set_wclk(ch, rk, ddr_wclk[PLATFORM_ID]);
 967 #ifdef BACKUP_WCTL
 968                                         set_wctl(ch, rk, ddr_wctl[PLATFORM_ID]);
 969 #else
 970                                         set_wctl(ch, rk, ddr_wclk[PLATFORM_ID] + HALF_CLK);
 971 #endif
 972                                 }
 973                         }
 974                 }
 975         }
 976
 977         /* COMP (non channel specific) */
 978         /* RCOMP: Dither PU Enable */
 979         mrc_alt_write_mask(DDRPHY, (DQANADRVPUCTL), (BIT30), (BIT30));
 980         /* RCOMP: Dither PD Enable */
 981         mrc_alt_write_mask(DDRPHY, (DQANADRVPDCTL), (BIT30), (BIT30));
 982         /* RCOMP: Dither PU Enable */
 983         mrc_alt_write_mask(DDRPHY, (CMDANADRVPUCTL), (BIT30), (BIT30));
 984         /* RCOMP: Dither PD Enable */
 985         mrc_alt_write_mask(DDRPHY, (CMDANADRVPDCTL), (BIT30), (BIT30));
 986         /* RCOMP: Dither PU Enable */
 987         mrc_alt_write_mask(DDRPHY, (CLKANADRVPUCTL), (BIT30), (BIT30));
 988         /* RCOMP: Dither PD Enable */
 989         mrc_alt_write_mask(DDRPHY, (CLKANADRVPDCTL), (BIT30), (BIT30));
 990         /* RCOMP: Dither PU Enable */
 991         mrc_alt_write_mask(DDRPHY, (DQSANADRVPUCTL), (BIT30), (BIT30));
 992         /* RCOMP: Dither PD Enable */
 993         mrc_alt_write_mask(DDRPHY, (DQSANADRVPDCTL), (BIT30), (BIT30));
 994         /* RCOMP: Dither PU Enable */
 995         mrc_alt_write_mask(DDRPHY, (CTLANADRVPUCTL), (BIT30), (BIT30));
 996         /* RCOMP: Dither PD Enable */
 997         mrc_alt_write_mask(DDRPHY, (CTLANADRVPDCTL), (BIT30), (BIT30));
 998         /* ODT: Dither PU Enable */
 999         mrc_alt_write_mask(DDRPHY, (DQANAODTPUCTL), (BIT30), (BIT30));
1000         /* ODT: Dither PD Enable */
1001         mrc_alt_write_mask(DDRPHY, (DQANAODTPDCTL), (BIT30), (BIT30));
1002         /* ODT: Dither PU Enable */
1003         mrc_alt_write_mask(DDRPHY, (CLKANAODTPUCTL), (BIT30), (BIT30));
1004         /* ODT: Dither PD Enable */
1005         mrc_alt_write_mask(DDRPHY, (CLKANAODTPDCTL), (BIT30), (BIT30));
1006         /* ODT: Dither PU Enable */
1007         mrc_alt_write_mask(DDRPHY, (DQSANAODTPUCTL), (BIT30), (BIT30));
1008         /* ODT: Dither PD Enable */
1009         mrc_alt_write_mask(DDRPHY, (DQSANAODTPDCTL), (BIT30), (BIT30));
1010         /* DCOMP: Dither PU Enable */
1011         mrc_alt_write_mask(DDRPHY, (DQANADLYPUCTL), (BIT30), (BIT30));
1012         /* DCOMP: Dither PD Enable */
1013         mrc_alt_write_mask(DDRPHY, (DQANADLYPDCTL), (BIT30), (BIT30));
1014         /* DCOMP: Dither PU Enable */
1015         mrc_alt_write_mask(DDRPHY, (CMDANADLYPUCTL), (BIT30), (BIT30));
1016         /* DCOMP: Dither PD Enable */
1017         mrc_alt_write_mask(DDRPHY, (CMDANADLYPDCTL), (BIT30), (BIT30));
1018         /* DCOMP: Dither PU Enable */
1019         mrc_alt_write_mask(DDRPHY, (CLKANADLYPUCTL), (BIT30), (BIT30));
1020         /* DCOMP: Dither PD Enable */
1021         mrc_alt_write_mask(DDRPHY, (CLKANADLYPDCTL), (BIT30), (BIT30));
1022         /* DCOMP: Dither PU Enable */
1023         mrc_alt_write_mask(DDRPHY, (DQSANADLYPUCTL), (BIT30), (BIT30));
1024         /* DCOMP: Dither PD Enable */
1025         mrc_alt_write_mask(DDRPHY, (DQSANADLYPDCTL), (BIT30), (BIT30));
1026         /* DCOMP: Dither PU Enable */
1027         mrc_alt_write_mask(DDRPHY, (CTLANADLYPUCTL), (BIT30), (BIT30));
1028         /* DCOMP: Dither PD Enable */
1029         mrc_alt_write_mask(DDRPHY, (CTLANADLYPDCTL), (BIT30), (BIT30));
1030         /* TCO: Dither PU Enable */
1031         mrc_alt_write_mask(DDRPHY, (DQANATCOPUCTL), (BIT30), (BIT30));
1032         /* TCO: Dither PD Enable */
1033         mrc_alt_write_mask(DDRPHY, (DQANATCOPDCTL), (BIT30), (BIT30));
1034         /* TCO: Dither PU Enable */
1035         mrc_alt_write_mask(DDRPHY, (CLKANATCOPUCTL), (BIT30), (BIT30));
1036         /* TCO: Dither PD Enable */
1037         mrc_alt_write_mask(DDRPHY, (CLKANATCOPDCTL), (BIT30), (BIT30));
1038         /* TCO: Dither PU Enable */
1039         mrc_alt_write_mask(DDRPHY, (DQSANATCOPUCTL), (BIT30), (BIT30));
1040         /* TCO: Dither PD Enable */
1041         mrc_alt_write_mask(DDRPHY, (DQSANATCOPDCTL), (BIT30), (BIT30));
1042         /* TCOCOMP: Pulse Count */
1043         mrc_alt_write_mask(DDRPHY, (TCOCNTCTRL), (0x1 << 0), (BIT1 | BIT0));
1044         /* ODT: CMD/CTL PD/PU */
1045         mrc_alt_write_mask(DDRPHY,
1046                 (CHNLBUFSTATIC), ((0x03 << 24) | (0x03 << 16)),
1047                 (BIT28 | BIT27 | BIT26 | BIT25 | BIT24 |
1048                 BIT20 | BIT19 | BIT18 | BIT17 | BIT16));
1049         /* Set 1us counter */
1050         mrc_alt_write_mask(DDRPHY,
1051                 (MSCNTR), (0x64 << 0),
1052                 (BIT7 | BIT6 | BIT5 | BIT4 | BIT3 | BIT2 | BIT1 | BIT0));
1053         mrc_alt_write_mask(DDRPHY,
1054                 (LATCH1CTL), (0x1 << 28),
1055                 (BIT30 | BIT29 | BIT28));
1056
1057         /* Release PHY from reset */
1058         mrc_alt_write_mask(DDRPHY, MASTERRSTN, BIT0, BIT0);
1059
1060         /* STEP1 */
1061         mrc_post_code(0x03, 0x11);
1062
1063         for (ch = 0; ch < NUM_CHANNELS; ch++) {
1064                 if (mrc_params->channel_enables & (1 << ch)) {
1065                         /* DQ01-DQ23 */
1066                         for (bl_grp = 0;
1067                              bl_grp < ((NUM_BYTE_LANES / bl_divisor) / 2);
1068                              bl_grp++) {
1069                                 mrc_alt_write_mask(DDRPHY,
1070                                         (DQMDLLCTL +
1071                                         (bl_grp * DDRIODQ_BL_OFFSET) +
1072                                         (ch * DDRIODQ_CH_OFFSET)),
1073                                         (BIT13),
1074                                         (BIT13));       /* Enable VREG */
1075                                 delay_n(3);
1076                         }
1077
1078                         /* ECC */
1079                         mrc_alt_write_mask(DDRPHY, (ECCMDLLCTL),
1080                                 (BIT13), (BIT13));      /* Enable VREG */
1081                         delay_n(3);
1082                         /* CMD */
1083                         mrc_alt_write_mask(DDRPHY,
1084                                 (CMDMDLLCTL + (ch * DDRIOCCC_CH_OFFSET)),
1085                                 (BIT13), (BIT13));      /* Enable VREG */
1086                         delay_n(3);
1087                         /* CLK-CTL */
1088                         mrc_alt_write_mask(DDRPHY,
1089                                 (CCMDLLCTL + (ch * DDRIOCCC_CH_OFFSET)),
1090                                 (BIT13), (BIT13));      /* Enable VREG */
1091                         delay_n(3);
1092                 }
1093         }
1094
1095         /* STEP2 */
1096         mrc_post_code(0x03, 0x12);
1097         delay_n(200);
1098
1099         for (ch = 0; ch < NUM_CHANNELS; ch++) {
1100                 if (mrc_params->channel_enables & (1 << ch)) {
1101                         /* DQ01-DQ23 */
1102                         for (bl_grp = 0;
1103                              bl_grp < ((NUM_BYTE_LANES / bl_divisor) / 2);
1104                              bl_grp++) {
1105                                 mrc_alt_write_mask(DDRPHY,
1106                                         (DQMDLLCTL +
1107                                         (bl_grp * DDRIODQ_BL_OFFSET) +
1108                                         (ch * DDRIODQ_CH_OFFSET)),
1109                                         (BIT17),
1110                                         (BIT17));       /* Enable MCDLL */
1111                                 delay_n(50);
1112                         }
1113
1114                 /* ECC */
1115                 mrc_alt_write_mask(DDRPHY, (ECCMDLLCTL),
1116                         (BIT17), (BIT17));      /* Enable MCDLL */
1117                 delay_n(50);
1118                 /* CMD */
1119                 mrc_alt_write_mask(DDRPHY,
1120                         (CMDMDLLCTL + (ch * DDRIOCCC_CH_OFFSET)),
1121                         (BIT18), (BIT18));      /* Enable MCDLL */
1122                 delay_n(50);
1123                 /* CLK-CTL */
1124                 mrc_alt_write_mask(DDRPHY,
1125                         (CCMDLLCTL + (ch * DDRIOCCC_CH_OFFSET)),
1126                         (BIT18), (BIT18));      /* Enable MCDLL */
1127                 delay_n(50);
1128                 }
1129         }
1130
1131         /* STEP3: */
1132         mrc_post_code(0x03, 0x13);
1133         delay_n(100);
1134
1135         for (ch = 0; ch < NUM_CHANNELS; ch++) {
1136                 if (mrc_params->channel_enables & (1 << ch)) {
1137                         /* DQ01-DQ23 */
1138                         for (bl_grp = 0;
1139                              bl_grp < ((NUM_BYTE_LANES / bl_divisor) / 2);
1140                              bl_grp++) {
1141 #ifdef FORCE_16BIT_DDRIO
1142                                 temp = ((bl_grp) &&
1143                                         (mrc_params->channel_width == X16)) ?
1144                                         ((0x1 << 12) | (0x1 << 8) |
1145                                         (0xF << 4) | (0xF << 0)) :
1146                                         ((0xF << 12) | (0xF << 8) |
1147                                         (0xF << 4) | (0xF << 0));
1148 #else
1149                                 temp = ((0xF << 12) | (0xF << 8) |
1150                                         (0xF << 4) | (0xF << 0));
1151 #endif
1152                                 /* Enable TXDLL */
1153                                 mrc_alt_write_mask(DDRPHY,
1154                                         (DQDLLTXCTL +
1155                                         (bl_grp * DDRIODQ_BL_OFFSET) +
1156                                         (ch * DDRIODQ_CH_OFFSET)),
1157                                         temp, 0xFFFF);
1158                                 delay_n(3);
1159                                 /* Enable RXDLL */
1160                                 mrc_alt_write_mask(DDRPHY,
1161                                         (DQDLLRXCTL +
1162                                         (bl_grp * DDRIODQ_BL_OFFSET) +
1163                                         (ch * DDRIODQ_CH_OFFSET)),
1164                                         (BIT3 | BIT2 | BIT1 | BIT0),
1165                                         (BIT3 | BIT2 | BIT1 | BIT0));
1166                                 delay_n(3);
1167                                 /* Enable RXDLL Overrides BL0 */
1168                                 mrc_alt_write_mask(DDRPHY,
1169                                         (B0OVRCTL +
1170                                         (bl_grp * DDRIODQ_BL_OFFSET) +
1171                                         (ch * DDRIODQ_CH_OFFSET)),
1172                                         (BIT3 | BIT2 | BIT1 | BIT0),
1173                                         (BIT3 | BIT2 | BIT1 | BIT0));
1174                         }
1175
1176                         /* ECC */
1177                         temp = ((0xF << 12) | (0xF << 8) |
1178                                 (0xF << 4) | (0xF << 0));
1179                         mrc_alt_write_mask(DDRPHY, (ECCDLLTXCTL),
1180                                 temp, 0xFFFF);
1181                         delay_n(3);
1182
1183                         /* CMD (PO) */
1184                         mrc_alt_write_mask(DDRPHY,
1185                                 (CMDDLLTXCTL + (ch * DDRIOCCC_CH_OFFSET)),
1186                                 temp, 0xFFFF);
1187                         delay_n(3);
1188                 }
1189         }
1190
1191         /* STEP4 */
1192         mrc_post_code(0x03, 0x14);
1193
1194         for (ch = 0; ch < NUM_CHANNELS; ch++) {
1195                 if (mrc_params->channel_enables & (1 << ch)) {
1196                         /* Host To Memory Clock Alignment (HMC) for 800/1066 */
1197                         for (bl_grp = 0;
1198                              bl_grp < ((NUM_BYTE_LANES / bl_divisor) / 2);
1199                              bl_grp++) {
1200                                 /* CLK_ALIGN_MOD_ID */
1201                                 mrc_alt_write_mask(DDRPHY,
1202                                         (DQCLKALIGNREG2 +
1203                                         (bl_grp * DDRIODQ_BL_OFFSET) +
1204                                         (ch * DDRIODQ_CH_OFFSET)),
1205                                         (bl_grp) ? (0x3) : (0x1),
1206                                         (BIT3 | BIT2 | BIT1 | BIT0));
1207                         }
1208
1209                         mrc_alt_write_mask(DDRPHY,
1210                                 (ECCCLKALIGNREG2 + (ch * DDRIODQ_CH_OFFSET)),
1211                                 0x2,
1212                                 (BIT3 | BIT2 | BIT1 | BIT0));
1213                         mrc_alt_write_mask(DDRPHY,
1214                                 (CMDCLKALIGNREG2 + (ch * DDRIODQ_CH_OFFSET)),
1215                                 0x0,
1216                                 (BIT3 | BIT2 | BIT1 | BIT0));
1217                         mrc_alt_write_mask(DDRPHY,
1218                                 (CCCLKALIGNREG2 + (ch * DDRIODQ_CH_OFFSET)),
1219                                 0x2,
1220                                 (BIT3 | BIT2 | BIT1 | BIT0));
1221                         mrc_alt_write_mask(DDRPHY,
1222                                 (CMDCLKALIGNREG0 + (ch * DDRIOCCC_CH_OFFSET)),
1223                                 (0x2 << 4), (BIT5 | BIT4));
1224                         /*
1225                          * NUM_SAMPLES, MAX_SAMPLES,
1226                          * MACRO_PI_STEP, MICRO_PI_STEP
1227                          */
1228                         mrc_alt_write_mask(DDRPHY,
1229                                 (CMDCLKALIGNREG1 + (ch * DDRIOCCC_CH_OFFSET)),
1230                                 ((0x18 << 16) | (0x10 << 8) |
1231                                 (0x8 << 2) | (0x1 << 0)),
1232                                 (BIT22 | BIT21 | BIT20 | BIT19 | BIT18 | BIT17 |
1233                                 BIT16 | BIT14 | BIT13 | BIT12 | BIT11 | BIT10 |
1234                                 BIT9 | BIT8 | BIT7 | BIT6 | BIT5 | BIT4 | BIT3 |
1235                                 BIT2 | BIT1 | BIT0));
1236                         /* TOTAL_NUM_MODULES, FIRST_U_PARTITION */
1237                         mrc_alt_write_mask(DDRPHY,
1238                                 (CMDCLKALIGNREG2 + (ch * DDRIOCCC_CH_OFFSET)),
1239                                 ((0x10 << 16) | (0x4 << 8) | (0x2 << 4)),
1240                                 (BIT20 | BIT19 | BIT18 | BIT17 | BIT16 |
1241                                 BIT11 | BIT10 | BIT9 | BIT8 | BIT7 | BIT6 |
1242                                 BIT5 | BIT4));
1243 #ifdef HMC_TEST
1244                         /* START_CLK_ALIGN=1 */
1245                         mrc_alt_write_mask(DDRPHY,
1246                                 (CMDCLKALIGNREG0 + (ch * DDRIOCCC_CH_OFFSET)),
1247                                 BIT24, BIT24);
1248                         while (msg_port_alt_read(DDRPHY,
1249                                 (CMDCLKALIGNREG0 + (ch * DDRIOCCC_CH_OFFSET))) &
1250                                 BIT24)
1251                                 ;       /* wait for START_CLK_ALIGN=0 */
1252 #endif
1253
1254                         /* Set RD/WR Pointer Seperation & COUNTEN & FIFOPTREN */
1255                         mrc_alt_write_mask(DDRPHY,
1256                                 (CMDPTRREG + (ch * DDRIOCCC_CH_OFFSET)),
1257                                 BIT0, BIT0);    /* WRPTRENABLE=1 */
1258
1259                         /* COMP initial */
1260                         /* enable bypass for CLK buffer (PO) */
1261                         mrc_alt_write_mask(DDRPHY,
1262                                 (COMPEN0CH0 + (ch * DDRCOMP_CH_OFFSET)),
1263                                 BIT5, BIT5);
1264                         /* Initial COMP Enable */
1265                         mrc_alt_write_mask(DDRPHY, (CMPCTRL),
1266                                 (BIT0), (BIT0));
1267                         /* wait for Initial COMP Enable = 0 */
1268                         while (msg_port_alt_read(DDRPHY, (CMPCTRL)) & BIT0)
1269                                 ;
1270                         /* disable bypass for CLK buffer (PO) */
1271                         mrc_alt_write_mask(DDRPHY,
1272                                 (COMPEN0CH0 + (ch * DDRCOMP_CH_OFFSET)),
1273                                 ~BIT5, BIT5);
1274
1275                         /* IOBUFACT */
1276
1277                         /* STEP4a */
1278                         mrc_alt_write_mask(DDRPHY,
1279                                 (CMDCFGREG0 + (ch * DDRIOCCC_CH_OFFSET)),
1280                                 BIT2, BIT2);    /* IOBUFACTRST_N=1 */
1281
1282                         /* DDRPHY initialization complete */
1283                         mrc_alt_write_mask(DDRPHY,
1284                                 (CMDPMCONFIG0 + (ch * DDRIOCCC_CH_OFFSET)),
1285                                 BIT20, BIT20);  /* SPID_INIT_COMPLETE=1 */
1286                 }
1287         }
1288
1289         LEAVEFN();
1290 }
1291
1292 /* This function performs JEDEC initialization on all enabled channels */
1293 void perform_jedec_init(struct mrc_params *mrc_params)
1294 {
1295         uint8_t twr, wl, rank;
1296         uint32_t tck;
1297         u32 dtr0;
1298         u32 drp;
1299         u32 drmc;
1300         u32 mrs0_cmd = 0;
1301         u32 emrs1_cmd = 0;
1302         u32 emrs2_cmd = 0;
1303         u32 emrs3_cmd = 0;
1304
1305         ENTERFN();
1306
1307         /* jedec_init starts */
1308         mrc_post_code(0x04, 0x00);
1309
1310         /* DDR3_RESET_SET=0, DDR3_RESET_RESET=1 */
1311         mrc_alt_write_mask(DDRPHY, CCDDR3RESETCTL, BIT1, (BIT8 | BIT1));
1312
1313         /* Assert RESET# for 200us */
1314         delay_u(200);
1315
1316         /* DDR3_RESET_SET=1, DDR3_RESET_RESET=0 */
1317         mrc_alt_write_mask(DDRPHY, CCDDR3RESETCTL, BIT8, (BIT8 | BIT1));
1318
1319         dtr0 = msg_port_read(MEM_CTLR, DTR0);
1320
1321         /*
1322          * Set CKEVAL for populated ranks
1323          * then send NOP to each rank (#4550197)
1324          */
1325
1326         drp = msg_port_read(MEM_CTLR, DRP);
1327         drp &= 0x3;
1328
1329         drmc = msg_port_read(MEM_CTLR, DRMC);
1330         drmc &= 0xFFFFFFFC;
1331         drmc |= (BIT4 | drp);
1332
1333         msg_port_write(MEM_CTLR, DRMC, drmc);
1334
1335         for (rank = 0; rank < NUM_RANKS; rank++) {
1336                 /* Skip to next populated rank */
1337                 if ((mrc_params->rank_enables & (1 << rank)) == 0)
1338                         continue;
1339
1340                 dram_init_command(DCMD_NOP(rank));
1341         }
1342
1343         msg_port_write(MEM_CTLR, DRMC,
1344                 (mrc_params->rd_odt_value == 0 ? BIT12 : 0));
1345
1346         /*
1347          * setup for emrs 2
1348          * BIT[15:11] --> Always "0"
1349          * BIT[10:09] --> Rtt_WR: want "Dynamic ODT Off" (0)
1350          * BIT[08]    --> Always "0"
1351          * BIT[07]    --> SRT: use sr_temp_range
1352          * BIT[06]    --> ASR: want "Manual SR Reference" (0)
1353          * BIT[05:03] --> CWL: use oem_tCWL
1354          * BIT[02:00] --> PASR: want "Full Array" (0)
1355          */
1356         emrs2_cmd |= (2 << 3);
1357         wl = 5 + mrc_params->ddr_speed;
1358         emrs2_cmd |= ((wl - 5) << 9);
1359         emrs2_cmd |= (mrc_params->sr_temp_range << 13);
1360
1361         /*
1362          * setup for emrs 3
1363          * BIT[15:03] --> Always "0"
1364          * BIT[02]    --> MPR: want "Normal Operation" (0)
1365          * BIT[01:00] --> MPR_Loc: want "Predefined Pattern" (0)
1366          */
1367         emrs3_cmd |= (3 << 3);
1368
1369         /*
1370          * setup for emrs 1
1371          * BIT[15:13]     --> Always "0"
1372          * BIT[12:12]     --> Qoff: want "Output Buffer Enabled" (0)
1373          * BIT[11:11]     --> TDQS: want "Disabled" (0)
1374          * BIT[10:10]     --> Always "0"
1375          * BIT[09,06,02]  --> Rtt_nom: use rtt_nom_value
1376          * BIT[08]        --> Always "0"
1377          * BIT[07]        --> WR_LVL: want "Disabled" (0)
1378          * BIT[05,01]     --> DIC: use ron_value
1379          * BIT[04:03]     --> AL: additive latency want "0" (0)
1380          * BIT[00]        --> DLL: want "Enable" (0)
1381          *
1382          * (BIT5|BIT1) set Ron value
1383          * 00 --> RZQ/6 (40ohm)
1384          * 01 --> RZQ/7 (34ohm)
1385          * 1* --> RESERVED
1386          *
1387          * (BIT9|BIT6|BIT2) set Rtt_nom value
1388          * 000 --> Disabled
1389          * 001 --> RZQ/4 ( 60ohm)
1390          * 010 --> RZQ/2 (120ohm)
1391          * 011 --> RZQ/6 ( 40ohm)
1392          * 1** --> RESERVED
1393          */
1394         emrs1_cmd |= (1 << 3);
1395         emrs1_cmd &= ~BIT6;
1396
1397         if (mrc_params->ron_value == 0)
1398                 emrs1_cmd |= BIT7;
1399         else
1400                 emrs1_cmd &= ~BIT7;
1401
1402         if (mrc_params->rtt_nom_value == 0)
1403                 emrs1_cmd |= (DDR3_EMRS1_RTTNOM_40 << 6);
1404         else if (mrc_params->rtt_nom_value == 1)
1405                 emrs1_cmd |= (DDR3_EMRS1_RTTNOM_60 << 6);
1406         else if (mrc_params->rtt_nom_value == 2)
1407                 emrs1_cmd |= (DDR3_EMRS1_RTTNOM_120 << 6);
1408
1409         /* save MRS1 value (excluding control fields) */
1410         mrc_params->mrs1 = emrs1_cmd >> 6;
1411
1412         /*
1413          * setup for mrs 0
1414          * BIT[15:13]     --> Always "0"
1415          * BIT[12]        --> PPD: for Quark (1)
1416          * BIT[11:09]     --> WR: use oem_tWR
1417          * BIT[08]        --> DLL: want "Reset" (1, self clearing)
1418          * BIT[07]        --> MODE: want "Normal" (0)
1419          * BIT[06:04,02]  --> CL: use oem_tCAS
1420          * BIT[03]        --> RD_BURST_TYPE: want "Interleave" (1)
1421          * BIT[01:00]     --> BL: want "8 Fixed" (0)
1422          * WR:
1423          * 0 --> 16
1424          * 1 --> 5
1425          * 2 --> 6
1426          * 3 --> 7
1427          * 4 --> 8
1428          * 5 --> 10
1429          * 6 --> 12
1430          * 7 --> 14
1431          * CL:
1432          * BIT[02:02] "0" if oem_tCAS <= 11 (1866?)
1433          * BIT[06:04] use oem_tCAS-4
1434          */
1435         mrs0_cmd |= BIT14;
1436         mrs0_cmd |= BIT18;
1437         mrs0_cmd |= ((((dtr0 >> 12) & 7) + 1) << 10);
1438
1439         tck = t_ck[mrc_params->ddr_speed];
1440         /* Per JEDEC: tWR=15000ps DDR2/3 from 800-1600 */
1441         twr = MCEIL(15000, tck);
1442         mrs0_cmd |= ((twr - 4) << 15);
1443
1444         for (rank = 0; rank < NUM_RANKS; rank++) {
1445                 /* Skip to next populated rank */
1446                 if ((mrc_params->rank_enables & (1 << rank)) == 0)
1447                         continue;
1448
1449                 emrs2_cmd |= (rank << 22);
1450                 dram_init_command(emrs2_cmd);
1451
1452                 emrs3_cmd |= (rank << 22);
1453                 dram_init_command(emrs3_cmd);
1454
1455                 emrs1_cmd |= (rank << 22);
1456                 dram_init_command(emrs1_cmd);
1457
1458                 mrs0_cmd |= (rank << 22);
1459                 dram_init_command(mrs0_cmd);
1460
1461                 dram_init_command(DCMD_ZQCL(rank));
1462         }
1463
1464         LEAVEFN();
1465 }
1466
1467 /*
1468  * Dunit Initialization Complete
1469  *
1470  * Indicates that initialization of the Dunit has completed.
1471  *
1472  * Memory accesses are permitted and maintenance operation begins.
1473  * Until this bit is set to a 1, the memory controller will not accept
1474  * DRAM requests from the MEMORY_MANAGER or HTE.
1475  */
1476 void set_ddr_init_complete(struct mrc_params *mrc_params)
1477 {
1478         u32 dco;
1479
1480         ENTERFN();
1481
1482         dco = msg_port_read(MEM_CTLR, DCO);
1483         dco &= ~BIT28;
1484         dco |= BIT31;
1485         msg_port_write(MEM_CTLR, DCO, dco);
1486
1487         LEAVEFN();
1488 }
1489
1490 /*
1491  * This function will retrieve relevant timing data
1492  *
1493  * This data will be used on subsequent boots to speed up boot times
1494  * and is required for Suspend To RAM capabilities.
1495  */
1496 void restore_timings(struct mrc_params *mrc_params)
1497 {
1498         uint8_t ch, rk, bl;
1499         const struct mrc_timings *mt = &mrc_params->timings;
1500
1501         for (ch = 0; ch < NUM_CHANNELS; ch++) {
1502                 for (rk = 0; rk < NUM_RANKS; rk++) {
1503                         for (bl = 0; bl < NUM_BYTE_LANES; bl++) {
1504                                 set_rcvn(ch, rk, bl, mt->rcvn[ch][rk][bl]);
1505                                 set_rdqs(ch, rk, bl, mt->rdqs[ch][rk][bl]);
1506                                 set_wdqs(ch, rk, bl, mt->wdqs[ch][rk][bl]);
1507                                 set_wdq(ch, rk, bl, mt->wdq[ch][rk][bl]);
1508                                 if (rk == 0) {
1509                                         /* VREF (RANK0 only) */
1510                                         set_vref(ch, bl, mt->vref[ch][bl]);
1511                                 }
1512                         }
1513                         set_wctl(ch, rk, mt->wctl[ch][rk]);
1514                 }
1515                 set_wcmd(ch, mt->wcmd[ch]);
1516         }
1517 }
1518
1519 /*
1520  * Configure default settings normally set as part of read training
1521  *
1522  * Some defaults have to be set earlier as they may affect earlier
1523  * training steps.
1524  */
1525 void default_timings(struct mrc_params *mrc_params)
1526 {
1527         uint8_t ch, rk, bl;
1528
1529         for (ch = 0; ch < NUM_CHANNELS; ch++) {
1530                 for (rk = 0; rk < NUM_RANKS; rk++) {
1531                         for (bl = 0; bl < NUM_BYTE_LANES; bl++) {
1532                                 set_rdqs(ch, rk, bl, 24);
1533                                 if (rk == 0) {
1534                                         /* VREF (RANK0 only) */
1535                                         set_vref(ch, bl, 32);
1536                                 }
1537                         }
1538                 }
1539         }
1540 }
1541
1542 /*
1543  * This function will perform our RCVEN Calibration Algorithm.
1544  * We will only use the 2xCLK domain timings to perform RCVEN Calibration.
1545  * All byte lanes will be calibrated "simultaneously" per channel per rank.
1546  */
1547 void rcvn_cal(struct mrc_params *mrc_params)
1548 {
1549         uint8_t ch;     /* channel counter */
1550         uint8_t rk;     /* rank counter */
1551         uint8_t bl;     /* byte lane counter */
1552         uint8_t bl_divisor = (mrc_params->channel_width == X16) ? 2 : 1;
1553
1554 #ifdef R2R_SHARING
1555         /* used to find placement for rank2rank sharing configs */
1556         uint32_t final_delay[NUM_CHANNELS][NUM_BYTE_LANES];
1557 #ifndef BACKUP_RCVN
1558         /* used to find placement for rank2rank sharing configs */
1559         uint32_t num_ranks_enabled = 0;
1560 #endif
1561 #endif
1562
1563 #ifdef BACKUP_RCVN
1564 #else
1565         uint32_t temp;
1566         /* absolute PI value to be programmed on the byte lane */
1567         uint32_t delay[NUM_BYTE_LANES];
1568         u32 dtr1, dtr1_save;
1569 #endif
1570
1571         ENTERFN();
1572
1573         /* rcvn_cal starts */
1574         mrc_post_code(0x05, 0x00);
1575
1576 #ifndef BACKUP_RCVN
1577         /* need separate burst to sample DQS preamble */
1578         dtr1 = msg_port_read(MEM_CTLR, DTR1);
1579         dtr1_save = dtr1;
1580         dtr1 |= BIT12;
1581         msg_port_write(MEM_CTLR, DTR1, dtr1);
1582 #endif
1583
1584 #ifdef R2R_SHARING
1585         /* need to set "final_delay[][]" elements to "0" */
1586         memset((void *)(final_delay), 0x00, (size_t)sizeof(final_delay));
1587 #endif
1588
1589         /* loop through each enabled channel */
1590         for (ch = 0; ch < NUM_CHANNELS; ch++) {
1591                 if (mrc_params->channel_enables & (1 << ch)) {
1592                         /* perform RCVEN Calibration on a per rank basis */
1593                         for (rk = 0; rk < NUM_RANKS; rk++) {
1594                                 if (mrc_params->rank_enables & (1 << rk)) {
1595                                         /*
1596                                          * POST_CODE here indicates the current
1597                                          * channel and rank being calibrated
1598                                          */
1599                                         mrc_post_code(0x05, (0x10 + ((ch << 4) | rk)));
1600
1601 #ifdef BACKUP_RCVN
1602                                         /* et hard-coded timing values */
1603                                         for (bl = 0; bl < (NUM_BYTE_LANES / bl_divisor); bl++)
1604                                                 set_rcvn(ch, rk, bl, ddr_rcvn[PLATFORM_ID]);
1605 #else
1606                                         /* enable FIFORST */
1607                                         for (bl = 0; bl < (NUM_BYTE_LANES / bl_divisor); bl += 2) {
1608                                                 mrc_alt_write_mask(DDRPHY,
1609                                                         (B01PTRCTL1 +
1610                                                         ((bl >> 1) * DDRIODQ_BL_OFFSET) +
1611                                                         (ch * DDRIODQ_CH_OFFSET)),
1612                                                         0, BIT8);
1613                                         }
1614                                         /* initialize the starting delay to 128 PI (cas +1 CLK) */
1615                                         for (bl = 0; bl < (NUM_BYTE_LANES / bl_divisor); bl++) {
1616                                                 /* 1x CLK domain timing is cas-4 */
1617                                                 delay[bl] = (4 + 1) * FULL_CLK;
1618
1619                                                 set_rcvn(ch, rk, bl, delay[bl]);
1620                                         }
1621
1622                                         /* now find the rising edge */
1623                                         find_rising_edge(mrc_params, delay, ch, rk, true);
1624
1625                                         /* Now increase delay by 32 PI (1/4 CLK) to place in center of high pulse */
1626                                         for (bl = 0; bl < (NUM_BYTE_LANES / bl_divisor); bl++) {
1627                                                 delay[bl] += QRTR_CLK;
1628                                                 set_rcvn(ch, rk, bl, delay[bl]);
1629                                         }
1630                                         /* Now decrement delay by 128 PI (1 CLK) until we sample a "0" */
1631                                         do {
1632                                                 temp = sample_dqs(mrc_params, ch, rk, true);
1633                                                 for (bl = 0; bl < (NUM_BYTE_LANES / bl_divisor); bl++) {
1634                                                         if (temp & (1 << bl)) {
1635                                                                 if (delay[bl] >= FULL_CLK) {
1636                                                                         delay[bl] -= FULL_CLK;
1637                                                                         set_rcvn(ch, rk, bl, delay[bl]);
1638                                                                 } else {
1639                                                                         /* not enough delay */
1640                                                                         training_message(ch, rk, bl);
1641                                                                         mrc_post_code(0xEE, 0x50);
1642                                                                 }
1643                                                         }
1644                                                 }
1645                                         } while (temp & 0xFF);
1646
1647 #ifdef R2R_SHARING
1648                                         /* increment "num_ranks_enabled" */
1649                                         num_ranks_enabled++;
1650                                         /* Finally increment delay by 32 PI (1/4 CLK) to place in center of preamble */
1651                                         for (bl = 0; bl < (NUM_BYTE_LANES / bl_divisor); bl++) {
1652                                                 delay[bl] += QRTR_CLK;
1653                                                 /* add "delay[]" values to "final_delay[][]" for rolling average */
1654                                                 final_delay[ch][bl] += delay[bl];
1655                                                 /* set timing based on rolling average values */
1656                                                 set_rcvn(ch, rk, bl, ((final_delay[ch][bl]) / num_ranks_enabled));
1657                                         }
1658 #else
1659                                         /* Finally increment delay by 32 PI (1/4 CLK) to place in center of preamble */
1660                                         for (bl = 0; bl < (NUM_BYTE_LANES / bl_divisor); bl++) {
1661                                                 delay[bl] += QRTR_CLK;
1662                                                 set_rcvn(ch, rk, bl, delay[bl]);
1663                                         }
1664 #endif
1665
1666                                         /* disable FIFORST */
1667                                         for (bl = 0; bl < (NUM_BYTE_LANES / bl_divisor); bl += 2) {
1668                                                 mrc_alt_write_mask(DDRPHY,
1669                                                         (B01PTRCTL1 +
1670                                                         ((bl >> 1) * DDRIODQ_BL_OFFSET) +
1671                                                         (ch * DDRIODQ_CH_OFFSET)),
1672                                                         BIT8, BIT8);
1673                                         }
1674 #endif
1675                                 }
1676                         }
1677                 }
1678         }
1679
1680 #ifndef BACKUP_RCVN
1681         /* restore original */
1682         msg_port_write(MEM_CTLR, DTR1, dtr1_save);
1683 #endif
1684
1685         LEAVEFN();
1686 }
1687
1688 /*
1689  * This function will perform the Write Levelling algorithm
1690  * (align WCLK and WDQS).
1691  *
1692  * This algorithm will act on each rank in each channel separately.
1693  */
1694 void wr_level(struct mrc_params *mrc_params)
1695 {
1696         uint8_t ch;     /* channel counter */
1697         uint8_t rk;     /* rank counter */
1698         uint8_t bl;     /* byte lane counter */
1699         uint8_t bl_divisor = (mrc_params->channel_width == X16) ? 2 : 1;
1700
1701 #ifdef R2R_SHARING
1702         /* used to find placement for rank2rank sharing configs */
1703         uint32_t final_delay[NUM_CHANNELS][NUM_BYTE_LANES];
1704 #ifndef BACKUP_WDQS
1705         /* used to find placement for rank2rank sharing configs */
1706         uint32_t num_ranks_enabled = 0;
1707 #endif
1708 #endif
1709
1710 #ifdef BACKUP_WDQS
1711 #else
1712         /* determines stop condition for CRS_WR_LVL */
1713         bool all_edges_found;
1714         /* absolute PI value to be programmed on the byte lane */
1715         uint32_t delay[NUM_BYTE_LANES];
1716         /*
1717          * static makes it so the data is loaded in the heap once by shadow(),
1718          * where non-static copies the data onto the stack every time this
1719          * function is called
1720          */
1721         uint32_t address;       /* address to be checked during COARSE_WR_LVL */
1722         u32 dtr4, dtr4_save;
1723 #endif
1724
1725         ENTERFN();
1726
1727         /* wr_level starts */
1728         mrc_post_code(0x06, 0x00);
1729
1730 #ifdef R2R_SHARING
1731         /* need to set "final_delay[][]" elements to "0" */
1732         memset((void *)(final_delay), 0x00, (size_t)sizeof(final_delay));
1733 #endif
1734
1735         /* loop through each enabled channel */
1736         for (ch = 0; ch < NUM_CHANNELS; ch++) {
1737                 if (mrc_params->channel_enables & (1 << ch)) {
1738                         /* perform WRITE LEVELING algorithm on a per rank basis */
1739                         for (rk = 0; rk < NUM_RANKS; rk++) {
1740                                 if (mrc_params->rank_enables & (1 << rk)) {
1741                                         /*
1742                                          * POST_CODE here indicates the current
1743                                          * rank and channel being calibrated
1744                                          */
1745                                         mrc_post_code(0x06, (0x10 + ((ch << 4) | rk)));
1746
1747 #ifdef BACKUP_WDQS
1748                                         for (bl = 0; bl < (NUM_BYTE_LANES / bl_divisor); bl++) {
1749                                                 set_wdqs(ch, rk, bl, ddr_wdqs[PLATFORM_ID]);
1750                                                 set_wdq(ch, rk, bl, (ddr_wdqs[PLATFORM_ID] - QRTR_CLK));
1751                                         }
1752 #else
1753                                         /*
1754                                          * perform a single PRECHARGE_ALL command to
1755                                          * make DRAM state machine go to IDLE state
1756                                          */
1757                                         dram_init_command(DCMD_PREA(rk));
1758
1759                                         /*
1760                                          * enable Write Levelling Mode
1761                                          * (EMRS1 w/ Write Levelling Mode Enable)
1762                                          */
1763                                         dram_init_command(DCMD_MRS1(rk, 0x0082));
1764
1765                                         /*
1766                                          * set ODT DRAM Full Time Termination
1767                                          * disable in MCU
1768                                          */
1769
1770                                         dtr4 = msg_port_read(MEM_CTLR, DTR4);
1771                                         dtr4_save = dtr4;
1772                                         dtr4 |= BIT15;
1773                                         msg_port_write(MEM_CTLR, DTR4, dtr4);
1774
1775                                         for (bl = 0; bl < ((NUM_BYTE_LANES / bl_divisor) / 2); bl++) {
1776                                                 /*
1777                                                  * Enable Sandy Bridge Mode (WDQ Tri-State) &
1778                                                  * Ensure 5 WDQS pulses during Write Leveling
1779                                                  */
1780                                                 mrc_alt_write_mask(DDRPHY,
1781                                                         DQCTL + (DDRIODQ_BL_OFFSET * bl) + (DDRIODQ_CH_OFFSET * ch),
1782                                                         (BIT28 | BIT8 | BIT6 | BIT4 | BIT2),
1783                                                         (BIT28 | BIT9 | BIT8 | BIT7 | BIT6 | BIT5 | BIT4 | BIT3 | BIT2));
1784                                         }
1785
1786                                         /* Write Leveling Mode enabled in IO */
1787                                         mrc_alt_write_mask(DDRPHY,
1788                                                 CCDDR3RESETCTL + (DDRIOCCC_CH_OFFSET * ch),
1789                                                 BIT16, BIT16);
1790
1791                                         /* Initialize the starting delay to WCLK */
1792                                         for (bl = 0; bl < (NUM_BYTE_LANES / bl_divisor); bl++) {
1793                                                 /*
1794                                                  * CLK0 --> RK0
1795                                                  * CLK1 --> RK1
1796                                                  */
1797                                                 delay[bl] = get_wclk(ch, rk);
1798
1799                                                 set_wdqs(ch, rk, bl, delay[bl]);
1800                                         }
1801
1802                                         /* now find the rising edge */
1803                                         find_rising_edge(mrc_params, delay, ch, rk, false);
1804
1805                                         /* disable Write Levelling Mode */
1806                                         mrc_alt_write_mask(DDRPHY,
1807                                                 CCDDR3RESETCTL + (DDRIOCCC_CH_OFFSET * ch),
1808                                                 0, BIT16);
1809
1810                                         for (bl = 0; bl < ((NUM_BYTE_LANES / bl_divisor) / 2); bl++) {
1811                                                 /* Disable Sandy Bridge Mode & Ensure 4 WDQS pulses during normal operation */
1812                                                 mrc_alt_write_mask(DDRPHY,
1813                                                         DQCTL + (DDRIODQ_BL_OFFSET * bl) + (DDRIODQ_CH_OFFSET * ch),
1814                                                         (BIT8 | BIT6 | BIT4 | BIT2),
1815                                                         (BIT28 | BIT9 | BIT8 | BIT7 | BIT6 | BIT5 | BIT4 | BIT3 | BIT2));
1816                                         }
1817
1818                                         /* restore original DTR4 */
1819                                         msg_port_write(MEM_CTLR, DTR4, dtr4_save);
1820
1821                                         /*
1822                                          * restore original value
1823                                          * (Write Levelling Mode Disable)
1824                                          */
1825                                         dram_init_command(DCMD_MRS1(rk, mrc_params->mrs1));
1826
1827                                         /*
1828                                          * perform a single PRECHARGE_ALL command to
1829                                          * make DRAM state machine go to IDLE state
1830                                          */
1831                                         dram_init_command(DCMD_PREA(rk));
1832
1833                                         mrc_post_code(0x06, (0x30 + ((ch << 4) | rk)));
1834
1835                                         /*
1836                                          * COARSE WRITE LEVEL:
1837                                          * check that we're on the correct clock edge
1838                                          */
1839
1840                                         /* hte reconfiguration request */
1841                                         mrc_params->hte_setup = 1;
1842
1843                                         /* start CRS_WR_LVL with WDQS = WDQS + 128 PI */
1844                                         for (bl = 0; bl < (NUM_BYTE_LANES / bl_divisor); bl++) {
1845                                                 delay[bl] = get_wdqs(ch, rk, bl) + FULL_CLK;
1846                                                 set_wdqs(ch, rk, bl, delay[bl]);
1847                                                 /*
1848                                                  * program WDQ timings based on WDQS
1849                                                  * (WDQ = WDQS - 32 PI)
1850                                                  */
1851                                                 set_wdq(ch, rk, bl, (delay[bl] - QRTR_CLK));
1852                                         }
1853
1854                                         /* get an address in the targeted channel/rank */
1855                                         address = get_addr(ch, rk);
1856                                         do {
1857                                                 uint32_t coarse_result = 0x00;
1858                                                 uint32_t coarse_result_mask = byte_lane_mask(mrc_params);
1859                                                 /* assume pass */
1860                                                 all_edges_found = true;
1861
1862                                                 mrc_params->hte_setup = 1;
1863                                                 coarse_result = check_rw_coarse(mrc_params, address);
1864
1865                                                 /* check for failures and margin the byte lane back 128 PI (1 CLK) */
1866                                                 for (bl = 0; bl < (NUM_BYTE_LANES / bl_divisor); bl++) {
1867                                                         if (coarse_result & (coarse_result_mask << bl)) {
1868                                                                 all_edges_found = false;
1869                                                                 delay[bl] -= FULL_CLK;
1870                                                                 set_wdqs(ch, rk, bl, delay[bl]);
1871                                                                 /* program WDQ timings based on WDQS (WDQ = WDQS - 32 PI) */
1872                                                                 set_wdq(ch, rk, bl, (delay[bl] - QRTR_CLK));
1873                                                         }
1874                                                 }
1875                                         } while (!all_edges_found);
1876
1877 #ifdef R2R_SHARING
1878                                         /* increment "num_ranks_enabled" */
1879                                          num_ranks_enabled++;
1880                                         /* accumulate "final_delay[][]" values from "delay[]" values for rolling average */
1881                                         for (bl = 0; bl < (NUM_BYTE_LANES / bl_divisor); bl++) {
1882                                                 final_delay[ch][bl] += delay[bl];
1883                                                 set_wdqs(ch, rk, bl, ((final_delay[ch][bl]) / num_ranks_enabled));
1884                                                 /* program WDQ timings based on WDQS (WDQ = WDQS - 32 PI) */
1885                                                 set_wdq(ch, rk, bl, ((final_delay[ch][bl]) / num_ranks_enabled) - QRTR_CLK);
1886                                         }
1887 #endif
1888 #endif
1889                                 }
1890                         }
1891                 }
1892         }
1893
1894         LEAVEFN();
1895 }
1896
1897 void prog_page_ctrl(struct mrc_params *mrc_params)
1898 {
1899         u32 dpmc0;
1900
1901         ENTERFN();
1902
1903         dpmc0 = msg_port_read(MEM_CTLR, DPMC0);
1904         dpmc0 &= ~(BIT16 | BIT17 | BIT18);
1905         dpmc0 |= (4 << 16);
1906         dpmc0 |= BIT21;
1907         msg_port_write(MEM_CTLR, DPMC0, dpmc0);
1908 }
1909
1910 /*
1911  * This function will perform the READ TRAINING Algorithm on all
1912  * channels/ranks/byte_lanes simultaneously to minimize execution time.
1913  *
1914  * The idea here is to train the VREF and RDQS (and eventually RDQ) values
1915  * to achieve maximum READ margins. The algorithm will first determine the
1916  * X coordinate (RDQS setting). This is done by collapsing the VREF eye
1917  * until we find a minimum required RDQS eye for VREF_MIN and VREF_MAX.
1918  * Then we take the averages of the RDQS eye at VREF_MIN and VREF_MAX,
1919  * then average those; this will be the final X coordinate. The algorithm
1920  * will then determine the Y coordinate (VREF setting). This is done by
1921  * collapsing the RDQS eye until we find a minimum required VREF eye for
1922  * RDQS_MIN and RDQS_MAX. Then we take the averages of the VREF eye at
1923  * RDQS_MIN and RDQS_MAX, then average those; this will be the final Y
1924  * coordinate.
1925  *
1926  * NOTE: this algorithm assumes the eye curves have a one-to-one relationship,
1927  * meaning for each X the curve has only one Y and vice-a-versa.
1928  */
1929 void rd_train(struct mrc_params *mrc_params)
1930 {
1931         uint8_t ch;     /* channel counter */
1932         uint8_t rk;     /* rank counter */
1933         uint8_t bl;     /* byte lane counter */
1934         uint8_t bl_divisor = (mrc_params->channel_width == X16) ? 2 : 1;
1935 #ifdef BACKUP_RDQS
1936 #else
1937         uint8_t side_x; /* tracks LEFT/RIGHT approach vectors */
1938         uint8_t side_y; /* tracks BOTTOM/TOP approach vectors */
1939         /* X coordinate data (passing RDQS values) for approach vectors */
1940         uint8_t x_coordinate[2][2][NUM_CHANNELS][NUM_RANKS][NUM_BYTE_LANES];
1941         /* Y coordinate data (passing VREF values) for approach vectors */
1942         uint8_t y_coordinate[2][2][NUM_CHANNELS][NUM_BYTE_LANES];
1943         /* centered X (RDQS) */
1944         uint8_t x_center[NUM_CHANNELS][NUM_RANKS][NUM_BYTE_LANES];
1945         /* centered Y (VREF) */
1946         uint8_t y_center[NUM_CHANNELS][NUM_BYTE_LANES];
1947         uint32_t address;       /* target address for check_bls_ex() */
1948         uint32_t result;        /* result of check_bls_ex() */
1949         uint32_t bl_mask;       /* byte lane mask for result checking */
1950 #ifdef R2R_SHARING
1951         /* used to find placement for rank2rank sharing configs */
1952         uint32_t final_delay[NUM_CHANNELS][NUM_BYTE_LANES];
1953         /* used to find placement for rank2rank sharing configs */
1954         uint32_t num_ranks_enabled = 0;
1955 #endif
1956 #endif
1957
1958         /* rd_train starts */
1959         mrc_post_code(0x07, 0x00);
1960
1961         ENTERFN();
1962
1963 #ifdef BACKUP_RDQS
1964         for (ch = 0; ch < NUM_CHANNELS; ch++) {
1965                 if (mrc_params->channel_enables & (1 << ch)) {
1966                         for (rk = 0; rk < NUM_RANKS; rk++) {
1967                                 if (mrc_params->rank_enables & (1 << rk)) {
1968                                         for (bl = 0;
1969                                              bl < (NUM_BYTE_LANES / bl_divisor);
1970                                              bl++) {
1971                                                 set_rdqs(ch, rk, bl, ddr_rdqs[PLATFORM_ID]);
1972                                         }
1973                                 }
1974                         }
1975                 }
1976         }
1977 #else
1978         /* initialize x/y_coordinate arrays */
1979         for (ch = 0; ch < NUM_CHANNELS; ch++) {
1980                 if (mrc_params->channel_enables & (1 << ch)) {
1981                         for (rk = 0; rk < NUM_RANKS; rk++) {
1982                                 if (mrc_params->rank_enables & (1 << rk)) {
1983                                         for (bl = 0;
1984                                              bl < (NUM_BYTE_LANES / bl_divisor);
1985                                              bl++) {
1986                                                 /* x_coordinate */
1987                                                 x_coordinate[L][B][ch][rk][bl] = RDQS_MIN;
1988                                                 x_coordinate[R][B][ch][rk][bl] = RDQS_MAX;
1989                                                 x_coordinate[L][T][ch][rk][bl] = RDQS_MIN;
1990                                                 x_coordinate[R][T][ch][rk][bl] = RDQS_MAX;
1991                                                 /* y_coordinate */
1992                                                 y_coordinate[L][B][ch][bl] = VREF_MIN;
1993                                                 y_coordinate[R][B][ch][bl] = VREF_MIN;
1994                                                 y_coordinate[L][T][ch][bl] = VREF_MAX;
1995                                                 y_coordinate[R][T][ch][bl] = VREF_MAX;
1996                                         }
1997                                 }
1998                         }
1999                 }
2000         }
2001
2002         /* initialize other variables */
2003         bl_mask = byte_lane_mask(mrc_params);
2004         address = get_addr(0, 0);
2005
2006 #ifdef R2R_SHARING
2007         /* need to set "final_delay[][]" elements to "0" */
2008         memset((void *)(final_delay), 0x00, (size_t)sizeof(final_delay));
2009 #endif
2010
2011         /* look for passing coordinates */
2012         for (side_y = B; side_y <= T; side_y++) {
2013                 for (side_x = L; side_x <= R; side_x++) {
2014                         mrc_post_code(0x07, (0x10 + (side_y * 2) + (side_x)));
2015
2016                         /* find passing values */
2017                         for (ch = 0; ch < NUM_CHANNELS; ch++) {
2018                                 if (mrc_params->channel_enables & (0x1 << ch)) {
2019                                         for (rk = 0; rk < NUM_RANKS; rk++) {
2020                                                 if (mrc_params->rank_enables &
2021                                                         (0x1 << rk)) {
2022                                                         /* set x/y_coordinate search starting settings */
2023                                                         for (bl = 0;
2024                                                              bl < (NUM_BYTE_LANES / bl_divisor);
2025                                                              bl++) {
2026                                                                 set_rdqs(ch, rk, bl,
2027                                                                          x_coordinate[side_x][side_y][ch][rk][bl]);
2028                                                                 set_vref(ch, bl,
2029                                                                          y_coordinate[side_x][side_y][ch][bl]);
2030                                                         }
2031
2032                                                         /* get an address in the target channel/rank */
2033                                                         address = get_addr(ch, rk);
2034
2035                                                         /* request HTE reconfiguration */
2036                                                         mrc_params->hte_setup = 1;
2037
2038                                                         /* test the settings */
2039                                                         do {
2040                                                                 /* result[07:00] == failing byte lane (MAX 8) */
2041                                                                 result = check_bls_ex(mrc_params, address);
2042
2043                                                                 /* check for failures */
2044                                                                 if (result & 0xFF) {
2045                                                                         /* at least 1 byte lane failed */
2046                                                                         for (bl = 0; bl < (NUM_BYTE_LANES / bl_divisor); bl++) {
2047                                                                                 if (result &
2048                                                                                         (bl_mask << bl)) {
2049                                                                                         /* adjust the RDQS values accordingly */
2050                                                                                         if (side_x == L)
2051                                                                                                 x_coordinate[L][side_y][ch][rk][bl] += RDQS_STEP;
2052                                                                                         else
2053                                                                                                 x_coordinate[R][side_y][ch][rk][bl] -= RDQS_STEP;
2054
2055                                                                                         /* check that we haven't closed the RDQS_EYE too much */
2056                                                                                         if ((x_coordinate[L][side_y][ch][rk][bl] > (RDQS_MAX - MIN_RDQS_EYE)) ||
2057                                                                                                 (x_coordinate[R][side_y][ch][rk][bl] < (RDQS_MIN + MIN_RDQS_EYE)) ||
2058                                                                                                 (x_coordinate[L][side_y][ch][rk][bl] ==
2059                                                                                                 x_coordinate[R][side_y][ch][rk][bl])) {
2060                                                                                                 /*
2061                                                                                                  * not enough RDQS margin available at this VREF
2062                                                                                                  * update VREF values accordingly
2063                                                                                                  */
2064                                                                                                 if (side_y == B)
2065                                                                                                         y_coordinate[side_x][B][ch][bl] += VREF_STEP;
2066                                                                                                 else
2067                                                                                                         y_coordinate[side_x][T][ch][bl] -= VREF_STEP;
2068
2069                                                                                                 /* check that we haven't closed the VREF_EYE too much */
2070                                                                                                 if ((y_coordinate[side_x][B][ch][bl] > (VREF_MAX - MIN_VREF_EYE)) ||
2071                                                                                                         (y_coordinate[side_x][T][ch][bl] < (VREF_MIN + MIN_VREF_EYE)) ||
2072                                                                                                         (y_coordinate[side_x][B][ch][bl] == y_coordinate[side_x][T][ch][bl])) {
2073                                                                                                         /* VREF_EYE collapsed below MIN_VREF_EYE */
2074                                                                                                         training_message(ch, rk, bl);
2075                                                                                                         mrc_post_code(0xEE, (0x70 + (side_y * 2) + (side_x)));
2076                                                                                                 } else {
2077                                                                                                         /* update the VREF setting */
2078                                                                                                         set_vref(ch, bl, y_coordinate[side_x][side_y][ch][bl]);
2079                                                                                                         /* reset the X coordinate to begin the search at the new VREF */
2080                                                                                                         x_coordinate[side_x][side_y][ch][rk][bl] =
2081                                                                                                                 (side_x == L) ? (RDQS_MIN) : (RDQS_MAX);
2082                                                                                                 }
2083                                                                                         }
2084
2085                                                                                         /* update the RDQS setting */
2086                                                                                         set_rdqs(ch, rk, bl, x_coordinate[side_x][side_y][ch][rk][bl]);
2087                                                                                 }
2088                                                                         }
2089                                                                 }
2090                                                         } while (result & 0xFF);
2091                                                 }
2092                                         }
2093                                 }
2094                         }
2095                 }
2096         }
2097
2098         mrc_post_code(0x07, 0x20);
2099
2100         /* find final RDQS (X coordinate) & final VREF (Y coordinate) */
2101         for (ch = 0; ch < NUM_CHANNELS; ch++) {
2102                 if (mrc_params->channel_enables & (1 << ch)) {
2103                         for (rk = 0; rk < NUM_RANKS; rk++) {
2104                                 if (mrc_params->rank_enables & (1 << rk)) {
2105                                         for (bl = 0; bl < (NUM_BYTE_LANES / bl_divisor); bl++) {
2106                                                 uint32_t temp1;
2107                                                 uint32_t temp2;
2108
2109                                                 /* x_coordinate */
2110                                                 DPF(D_INFO,
2111                                                     "RDQS T/B eye rank%d lane%d : %d-%d %d-%d\n",
2112                                                     rk, bl,
2113                                                     x_coordinate[L][T][ch][rk][bl],
2114                                                     x_coordinate[R][T][ch][rk][bl],
2115                                                     x_coordinate[L][B][ch][rk][bl],
2116                                                     x_coordinate[R][B][ch][rk][bl]);
2117
2118                                                 /* average the TOP side LEFT & RIGHT values */
2119                                                 temp1 = (x_coordinate[R][T][ch][rk][bl] + x_coordinate[L][T][ch][rk][bl]) / 2;
2120                                                 /* average the BOTTOM side LEFT & RIGHT values */
2121                                                 temp2 = (x_coordinate[R][B][ch][rk][bl] + x_coordinate[L][B][ch][rk][bl]) / 2;
2122                                                 /* average the above averages */
2123                                                 x_center[ch][rk][bl] = (uint8_t) ((temp1 + temp2) / 2);
2124
2125                                                 /* y_coordinate */
2126                                                 DPF(D_INFO,
2127                                                     "VREF R/L eye lane%d : %d-%d %d-%d\n",
2128                                                     bl,
2129                                                     y_coordinate[R][B][ch][bl],
2130                                                     y_coordinate[R][T][ch][bl],
2131                                                     y_coordinate[L][B][ch][bl],
2132                                                     y_coordinate[L][T][ch][bl]);
2133
2134                                                 /* average the RIGHT side TOP & BOTTOM values */
2135                                                 temp1 = (y_coordinate[R][T][ch][bl] + y_coordinate[R][B][ch][bl]) / 2;
2136                                                 /* average the LEFT side TOP & BOTTOM values */
2137                                                 temp2 = (y_coordinate[L][T][ch][bl] + y_coordinate[L][B][ch][bl]) / 2;
2138                                                 /* average the above averages */
2139                                                 y_center[ch][bl] = (uint8_t) ((temp1 + temp2) / 2);
2140                                         }
2141                                 }
2142                         }
2143                 }
2144         }
2145
2146 #ifdef RX_EYE_CHECK
2147         /* perform an eye check */
2148         for (side_y = B; side_y <= T; side_y++) {
2149                 for (side_x = L; side_x <= R; side_x++) {
2150                         mrc_post_code(0x07, (0x30 + (side_y * 2) + (side_x)));
2151
2152                         /* update the settings for the eye check */
2153                         for (ch = 0; ch < NUM_CHANNELS; ch++) {
2154                                 if (mrc_params->channel_enables & (1 << ch)) {
2155                                         for (rk = 0; rk < NUM_RANKS; rk++) {
2156                                                 if (mrc_params->rank_enables & (1 << rk)) {
2157                                                         for (bl = 0; bl < (NUM_BYTE_LANES / bl_divisor); bl++) {
2158                                                                 if (side_x == L)
2159                                                                         set_rdqs(ch, rk, bl, (x_center[ch][rk][bl] - (MIN_RDQS_EYE / 2)));
2160                                                                 else
2161                                                                         set_rdqs(ch, rk, bl, (x_center[ch][rk][bl] + (MIN_RDQS_EYE / 2)));
2162
2163                                                                 if (side_y == B)
2164                                                                         set_vref(ch, bl, (y_center[ch][bl] - (MIN_VREF_EYE / 2)));
2165                                                                 else
2166                                                                         set_vref(ch, bl, (y_center[ch][bl] + (MIN_VREF_EYE / 2)));
2167                                                         }
2168                                                 }
2169                                         }
2170                                 }
2171                         }
2172
2173                         /* request HTE reconfiguration */
2174                         mrc_params->hte_setup = 1;
2175
2176                         /* check the eye */
2177                         if (check_bls_ex(mrc_params, address) & 0xFF) {
2178                                 /* one or more byte lanes failed */
2179                                 mrc_post_code(0xEE, (0x74 + (side_x * 2) + (side_y)));
2180                         }
2181                 }
2182         }
2183 #endif
2184
2185         mrc_post_code(0x07, 0x40);
2186
2187         /* set final placements */
2188         for (ch = 0; ch < NUM_CHANNELS; ch++) {
2189                 if (mrc_params->channel_enables & (1 << ch)) {
2190                         for (rk = 0; rk < NUM_RANKS; rk++) {
2191                                 if (mrc_params->rank_enables & (1 << rk)) {
2192 #ifdef R2R_SHARING
2193                                         /* increment "num_ranks_enabled" */
2194                                         num_ranks_enabled++;
2195 #endif
2196                                         for (bl = 0; bl < (NUM_BYTE_LANES / bl_divisor); bl++) {
2197                                                 /* x_coordinate */
2198 #ifdef R2R_SHARING
2199                                                 final_delay[ch][bl] += x_center[ch][rk][bl];
2200                                                 set_rdqs(ch, rk, bl, ((final_delay[ch][bl]) / num_ranks_enabled));
2201 #else
2202                                                 set_rdqs(ch, rk, bl, x_center[ch][rk][bl]);
2203 #endif
2204                                                 /* y_coordinate */
2205                                                 set_vref(ch, bl, y_center[ch][bl]);
2206                                         }
2207                                 }
2208                         }
2209                 }
2210         }
2211 #endif
2212
2213         LEAVEFN();
2214 }
2215
2216 /*
2217  * This function will perform the WRITE TRAINING Algorithm on all
2218  * channels/ranks/byte_lanes simultaneously to minimize execution time.
2219  *
2220  * The idea here is to train the WDQ timings to achieve maximum WRITE margins.
2221  * The algorithm will start with WDQ at the current WDQ setting (tracks WDQS
2222  * in WR_LVL) +/- 32 PIs (+/- 1/4 CLK) and collapse the eye until all data
2223  * patterns pass. This is because WDQS will be aligned to WCLK by the
2224  * Write Leveling algorithm and WDQ will only ever have a 1/2 CLK window
2225  * of validity.
2226  */
2227 void wr_train(struct mrc_params *mrc_params)
2228 {
2229         uint8_t ch;     /* channel counter */
2230         uint8_t rk;     /* rank counter */
2231         uint8_t bl;     /* byte lane counter */
2232         uint8_t bl_divisor = (mrc_params->channel_width == X16) ? 2 : 1;
2233 #ifdef BACKUP_WDQ
2234 #else
2235         uint8_t side;           /* LEFT/RIGHT side indicator (0=L, 1=R) */
2236         uint32_t temp;          /* temporary DWORD */
2237         /* 2 arrays, for L & R side passing delays */
2238         uint32_t delay[2][NUM_CHANNELS][NUM_RANKS][NUM_BYTE_LANES];
2239         uint32_t address;       /* target address for check_bls_ex() */
2240         uint32_t result;        /* result of check_bls_ex() */
2241         uint32_t bl_mask;       /* byte lane mask for result checking */
2242 #ifdef R2R_SHARING
2243         /* used to find placement for rank2rank sharing configs */
2244         uint32_t final_delay[NUM_CHANNELS][NUM_BYTE_LANES];
2245         /* used to find placement for rank2rank sharing configs */
2246         uint32_t num_ranks_enabled = 0;
2247 #endif
2248 #endif
2249
2250         /* wr_train starts */
2251         mrc_post_code(0x08, 0x00);
2252
2253         ENTERFN();
2254
2255 #ifdef BACKUP_WDQ
2256         for (ch = 0; ch < NUM_CHANNELS; ch++) {
2257                 if (mrc_params->channel_enables & (1 << ch)) {
2258                         for (rk = 0; rk < NUM_RANKS; rk++) {
2259                                 if (mrc_params->rank_enables & (1 << rk)) {
2260                                         for (bl = 0;
2261                                              bl < (NUM_BYTE_LANES / bl_divisor);
2262                                              bl++) {
2263                                                 set_wdq(ch, rk, bl, ddr_wdq[PLATFORM_ID]);
2264                                         }
2265                                 }
2266                         }
2267                 }
2268         }
2269 #else
2270         /* initialize "delay" */
2271         for (ch = 0; ch < NUM_CHANNELS; ch++) {
2272                 if (mrc_params->channel_enables & (1 << ch)) {
2273                         for (rk = 0; rk < NUM_RANKS; rk++) {
2274                                 if (mrc_params->rank_enables & (1 << rk)) {
2275                                         for (bl = 0;
2276                                              bl < (NUM_BYTE_LANES / bl_divisor);
2277                                              bl++) {
2278                                                 /*
2279                                                  * want to start with
2280                                                  * WDQ = (WDQS - QRTR_CLK)
2281                                                  * +/- QRTR_CLK
2282                                                  */
2283                                                 temp = get_wdqs(ch, rk, bl) - QRTR_CLK;
2284                                                 delay[L][ch][rk][bl] = temp - QRTR_CLK;
2285                                                 delay[R][ch][rk][bl] = temp + QRTR_CLK;
2286                                         }
2287                                 }
2288                         }
2289                 }
2290         }
2291
2292         /* initialize other variables */
2293         bl_mask = byte_lane_mask(mrc_params);
2294         address = get_addr(0, 0);
2295
2296 #ifdef R2R_SHARING
2297         /* need to set "final_delay[][]" elements to "0" */
2298         memset((void *)(final_delay), 0x00, (size_t)sizeof(final_delay));
2299 #endif
2300
2301         /*
2302          * start algorithm on the LEFT side and train each channel/bl
2303          * until no failures are observed, then repeat for the RIGHT side.
2304          */
2305         for (side = L; side <= R; side++) {
2306                 mrc_post_code(0x08, (0x10 + (side)));
2307
2308                 /* set starting values */
2309                 for (ch = 0; ch < NUM_CHANNELS; ch++) {
2310                         if (mrc_params->channel_enables & (1 << ch)) {
2311                                 for (rk = 0; rk < NUM_RANKS; rk++) {
2312                                         if (mrc_params->rank_enables &
2313                                                 (1 << rk)) {
2314                                                 for (bl = 0;
2315                                                      bl < (NUM_BYTE_LANES / bl_divisor);
2316                                                      bl++) {
2317                                                         set_wdq(ch, rk, bl, delay[side][ch][rk][bl]);
2318                                                 }
2319                                         }
2320                                 }
2321                         }
2322                 }
2323
2324                 /* find passing values */
2325                 for (ch = 0; ch < NUM_CHANNELS; ch++) {
2326                         if (mrc_params->channel_enables & (1 << ch)) {
2327                                 for (rk = 0; rk < NUM_RANKS; rk++) {
2328                                         if (mrc_params->rank_enables &
2329                                                 (1 << rk)) {
2330                                                 /* get an address in the target channel/rank */
2331                                                 address = get_addr(ch, rk);
2332
2333                                                 /* request HTE reconfiguration */
2334                                                 mrc_params->hte_setup = 1;
2335
2336                                                 /* check the settings */
2337                                                 do {
2338                                                         /* result[07:00] == failing byte lane (MAX 8) */
2339                                                         result = check_bls_ex(mrc_params, address);
2340                                                         /* check for failures */
2341                                                         if (result & 0xFF) {
2342                                                                 /* at least 1 byte lane failed */
2343                                                                 for (bl = 0; bl < (NUM_BYTE_LANES / bl_divisor); bl++) {
2344                                                                         if (result &
2345                                                                                 (bl_mask << bl)) {
2346                                                                                 if (side == L)
2347                                                                                         delay[L][ch][rk][bl] += WDQ_STEP;
2348                                                                                 else
2349                                                                                         delay[R][ch][rk][bl] -= WDQ_STEP;
2350
2351                                                                                 /* check for algorithm failure */
2352                                                                                 if (delay[L][ch][rk][bl] != delay[R][ch][rk][bl]) {
2353                                                                                         /*
2354                                                                                          * margin available
2355                                                                                          * update delay setting
2356                                                                                          */
2357                                                                                         set_wdq(ch, rk, bl,
2358                                                                                                 delay[side][ch][rk][bl]);
2359                                                                                 } else {
2360                                                                                         /*
2361                                                                                          * no margin available
2362                                                                                          * notify the user and halt
2363                                                                                          */
2364                                                                                         training_message(ch, rk, bl);
2365                                                                                         mrc_post_code(0xEE, (0x80 + side));
2366                                                                                 }
2367                                                                         }
2368                                                                 }
2369                                                         }
2370                                                 /* stop when all byte lanes pass */
2371                                                 } while (result & 0xFF);
2372                                         }
2373                                 }
2374                         }
2375                 }
2376         }
2377
2378         /* program WDQ to the middle of passing window */
2379         for (ch = 0; ch < NUM_CHANNELS; ch++) {
2380                 if (mrc_params->channel_enables & (1 << ch)) {
2381                         for (rk = 0; rk < NUM_RANKS; rk++) {
2382                                 if (mrc_params->rank_enables & (1 << rk)) {
2383 #ifdef R2R_SHARING
2384                                         /* increment "num_ranks_enabled" */
2385                                         num_ranks_enabled++;
2386 #endif
2387                                         for (bl = 0; bl < (NUM_BYTE_LANES / bl_divisor); bl++) {
2388                                                 DPF(D_INFO,
2389                                                     "WDQ eye rank%d lane%d : %d-%d\n",
2390                                                     rk, bl,
2391                                                     delay[L][ch][rk][bl],
2392                                                     delay[R][ch][rk][bl]);
2393
2394                                                 temp = (delay[R][ch][rk][bl] + delay[L][ch][rk][bl]) / 2;
2395
2396 #ifdef R2R_SHARING
2397                                                 final_delay[ch][bl] += temp;
2398                                                 set_wdq(ch, rk, bl,
2399                                                         ((final_delay[ch][bl]) / num_ranks_enabled));
2400 #else
2401                                                 set_wdq(ch, rk, bl, temp);
2402 #endif
2403                                         }
2404                                 }
2405                         }
2406                 }
2407         }
2408 #endif
2409
2410         LEAVEFN();
2411 }
2412
2413 /*
2414  * This function will store relevant timing data
2415  *
2416  * This data will be used on subsequent boots to speed up boot times
2417  * and is required for Suspend To RAM capabilities.
2418  */
2419 void store_timings(struct mrc_params *mrc_params)
2420 {
2421         uint8_t ch, rk, bl;
2422         struct mrc_timings *mt = &mrc_params->timings;
2423
2424         for (ch = 0; ch < NUM_CHANNELS; ch++) {
2425                 for (rk = 0; rk < NUM_RANKS; rk++) {
2426                         for (bl = 0; bl < NUM_BYTE_LANES; bl++) {
2427                                 mt->rcvn[ch][rk][bl] = get_rcvn(ch, rk, bl);
2428                                 mt->rdqs[ch][rk][bl] = get_rdqs(ch, rk, bl);
2429                                 mt->wdqs[ch][rk][bl] = get_wdqs(ch, rk, bl);
2430                                 mt->wdq[ch][rk][bl] = get_wdq(ch, rk, bl);
2431
2432                                 if (rk == 0)
2433                                         mt->vref[ch][bl] = get_vref(ch, bl);
2434                         }
2435
2436                         mt->wctl[ch][rk] = get_wctl(ch, rk);
2437                 }
2438
2439                 mt->wcmd[ch] = get_wcmd(ch);
2440         }
2441
2442         /* need to save for a case of changing frequency after warm reset */
2443         mt->ddr_speed = mrc_params->ddr_speed;
2444 }
2445
2446 /*
2447  * The purpose of this function is to ensure the SEC comes out of reset
2448  * and IA initiates the SEC enabling Memory Scrambling.
2449  */
2450 void enable_scrambling(struct mrc_params *mrc_params)
2451 {
2452         uint32_t lfsr = 0;
2453         uint8_t i;
2454
2455         if (mrc_params->scrambling_enables == 0)
2456                 return;
2457
2458         ENTERFN();
2459
2460         /* 32 bit seed is always stored in BIOS NVM */
2461         lfsr = mrc_params->timings.scrambler_seed;
2462
2463         if (mrc_params->boot_mode == BM_COLD) {
2464                 /*
2465                  * factory value is 0 and in first boot,
2466                  * a clock based seed is loaded.
2467                  */
2468                 if (lfsr == 0) {
2469                         /*
2470                          * get seed from system clock
2471                          * and make sure it is not all 1's
2472                          */
2473                         lfsr = rdtsc() & 0x0FFFFFFF;
2474                 } else {
2475                         /*
2476                          * Need to replace scrambler
2477                          *
2478                          * get next 32bit LFSR 16 times which is the last
2479                          * part of the previous scrambler vector
2480                          */
2481                         for (i = 0; i < 16; i++)
2482                                 lfsr32(&lfsr);
2483                 }
2484
2485                 /* save new seed */
2486                 mrc_params->timings.scrambler_seed = lfsr;
2487         }
2488
2489         /*
2490          * In warm boot or S3 exit, we have the previous seed.
2491          * In cold boot, we have the last 32bit LFSR which is the new seed.
2492          */
2493         lfsr32(&lfsr);  /* shift to next value */
2494         msg_port_write(MEM_CTLR, SCRMSEED, (lfsr & 0x0003FFFF));
2495
2496         for (i = 0; i < 2; i++)
2497                 msg_port_write(MEM_CTLR, SCRMLO + i, (lfsr & 0xAAAAAAAA));
2498
2499         LEAVEFN();
2500 }
2501
2502 /*
2503  * Configure MCU Power Management Control Register
2504  * and Scheduler Control Register
2505  */
2506 void prog_ddr_control(struct mrc_params *mrc_params)
2507 {
2508         u32 dsch;
2509         u32 dpmc0;
2510
2511         ENTERFN();
2512
2513         dsch = msg_port_read(MEM_CTLR, DSCH);
2514         dsch &= ~(BIT8 | BIT9 | BIT12);
2515         msg_port_write(MEM_CTLR, DSCH, dsch);
2516
2517         dpmc0 = msg_port_read(MEM_CTLR, DPMC0);
2518         dpmc0 &= ~BIT25;
2519         dpmc0 |= (mrc_params->power_down_disable << 25);
2520         dpmc0 &= ~BIT24;
2521         dpmc0 &= ~(BIT16 | BIT17 | BIT18);
2522         dpmc0 |= (4 << 16);
2523         dpmc0 |= BIT21;
2524         msg_port_write(MEM_CTLR, DPMC0, dpmc0);
2525
2526         /* CMDTRIST = 2h - CMD/ADDR are tristated when no valid command */
2527         mrc_write_mask(MEM_CTLR, DPMC1, 2 << 4, BIT4 | BIT5);
2528
2529         LEAVEFN();
2530 }
2531
2532 /*
2533  * After training complete configure MCU Rank Population Register
2534  * specifying: ranks enabled, device width, density, address mode
2535  */
2536 void prog_dra_drb(struct mrc_params *mrc_params)
2537 {
2538         u32 drp;
2539         u32 dco;
2540         u8 density = mrc_params->params.density;
2541
2542         ENTERFN();
2543
2544         dco = msg_port_read(MEM_CTLR, DCO);
2545         dco &= ~BIT31;
2546         msg_port_write(MEM_CTLR, DCO, dco);
2547
2548         drp = 0;
2549         if (mrc_params->rank_enables & 1)
2550                 drp |= BIT0;
2551         if (mrc_params->rank_enables & 2)
2552                 drp |= BIT1;
2553         if (mrc_params->dram_width == X16) {
2554                 drp |= (1 << 4);
2555                 drp |= (1 << 9);
2556         }
2557
2558         /*
2559          * Density encoding in struct dram_params: 0=512Mb, 1=Gb, 2=2Gb, 3=4Gb
2560          * has to be mapped RANKDENSx encoding (0=1Gb)
2561          */
2562         if (density == 0)
2563                 density = 4;
2564
2565         drp |= ((density - 1) << 6);
2566         drp |= ((density - 1) << 11);
2567
2568         /* Address mode can be overwritten if ECC enabled */
2569         drp |= (mrc_params->address_mode << 14);
2570
2571         msg_port_write(MEM_CTLR, DRP, drp);
2572
2573         dco &= ~BIT28;
2574         dco |= BIT31;
2575         msg_port_write(MEM_CTLR, DCO, dco);
2576
2577         LEAVEFN();
2578 }
2579
2580 /* Send DRAM wake command */
2581 void perform_wake(struct mrc_params *mrc_params)
2582 {
2583         ENTERFN();
2584
2585         dram_wake_command();
2586
2587         LEAVEFN();
2588 }
2589
2590 /*
2591  * Configure refresh rate and short ZQ calibration interval
2592  * Activate dynamic self refresh
2593  */
2594 void change_refresh_period(struct mrc_params *mrc_params)
2595 {
2596         u32 drfc;
2597         u32 dcal;
2598         u32 dpmc0;
2599
2600         ENTERFN();
2601
2602         drfc = msg_port_read(MEM_CTLR, DRFC);
2603         drfc &= ~(BIT12 | BIT13 | BIT14);
2604         drfc |= (mrc_params->refresh_rate << 12);
2605         drfc |= BIT21;
2606         msg_port_write(MEM_CTLR, DRFC, drfc);
2607
2608         dcal = msg_port_read(MEM_CTLR, DCAL);
2609         dcal &= ~(BIT8 | BIT9 | BIT10);
2610         dcal |= (3 << 8);       /* 63ms */
2611         msg_port_write(MEM_CTLR, DCAL, dcal);
2612
2613         dpmc0 = msg_port_read(MEM_CTLR, DPMC0);
2614         dpmc0 |= (BIT23 | BIT29);
2615         msg_port_write(MEM_CTLR, DPMC0, dpmc0);
2616
2617         LEAVEFN();
2618 }
2619
2620 /*
2621  * Configure DDRPHY for Auto-Refresh, Periodic Compensations,
2622  * Dynamic Diff-Amp, ZQSPERIOD, Auto-Precharge, CKE Power-Down
2623  */
2624 void set_auto_refresh(struct mrc_params *mrc_params)
2625 {
2626         uint32_t channel;
2627         uint32_t rank;
2628         uint32_t bl;
2629         uint32_t bl_divisor = 1;
2630         uint32_t temp;
2631
2632         ENTERFN();
2633
2634         /*
2635          * Enable Auto-Refresh, Periodic Compensations, Dynamic Diff-Amp,
2636          * ZQSPERIOD, Auto-Precharge, CKE Power-Down
2637          */
2638         for (channel = 0; channel < NUM_CHANNELS; channel++) {
2639                 if (mrc_params->channel_enables & (1 << channel)) {
2640                         /* Enable Periodic RCOMPS */
2641                         mrc_alt_write_mask(DDRPHY, CMPCTRL, BIT1, BIT1);
2642
2643                         /* Enable Dynamic DiffAmp & Set Read ODT Value */
2644                         switch (mrc_params->rd_odt_value) {
2645                         case 0:
2646                                 temp = 0x3F;    /* OFF */
2647                                 break;
2648                         default:
2649                                 temp = 0x00;    /* Auto */
2650                                 break;
2651                         }
2652
2653                         for (bl = 0; bl < ((NUM_BYTE_LANES / bl_divisor) / 2); bl++) {
2654                                 /* Override: DIFFAMP, ODT */
2655                                 mrc_alt_write_mask(DDRPHY,
2656                                         (B0OVRCTL + (bl * DDRIODQ_BL_OFFSET) +
2657                                         (channel * DDRIODQ_CH_OFFSET)),
2658                                         (0x00 << 16) | (temp << 10),
2659                                         (BIT21 | BIT20 | BIT19 | BIT18 |
2660                                          BIT17 | BIT16 | BIT15 | BIT14 |
2661                                          BIT13 | BIT12 | BIT11 | BIT10));
2662
2663                                 /* Override: DIFFAMP, ODT */
2664                                 mrc_alt_write_mask(DDRPHY,
2665                                         (B1OVRCTL + (bl * DDRIODQ_BL_OFFSET) +
2666                                         (channel * DDRIODQ_CH_OFFSET)),
2667                                         (0x00 << 16) | (temp << 10),
2668                                         (BIT21 | BIT20 | BIT19 | BIT18 |
2669                                          BIT17 | BIT16 | BIT15 | BIT14 |
2670                                          BIT13 | BIT12 | BIT11 | BIT10));
2671                         }
2672
2673                         /* Issue ZQCS command */
2674                         for (rank = 0; rank < NUM_RANKS; rank++) {
2675                                 if (mrc_params->rank_enables & (1 << rank))
2676                                         dram_init_command(DCMD_ZQCS(rank));
2677                         }
2678                 }
2679         }
2680
2681         clear_pointers();
2682
2683         LEAVEFN();
2684 }
2685
2686 /*
2687  * Depending on configuration enables ECC support
2688  *
2689  * Available memory size is decreased, and updated with 0s
2690  * in order to clear error status. Address mode 2 forced.
2691  */
2692 void ecc_enable(struct mrc_params *mrc_params)
2693 {
2694         u32 drp;
2695         u32 dsch;
2696         u32 ecc_ctrl;
2697
2698         if (mrc_params->ecc_enables == 0)
2699                 return;
2700
2701         ENTERFN();
2702
2703         /* Configuration required in ECC mode */
2704         drp = msg_port_read(MEM_CTLR, DRP);
2705         drp &= ~(BIT14 | BIT15);
2706         drp |= BIT15;
2707         drp |= BIT13;
2708         msg_port_write(MEM_CTLR, DRP, drp);
2709
2710         /* Disable new request bypass */
2711         dsch = msg_port_read(MEM_CTLR, DSCH);
2712         dsch |= BIT12;
2713         msg_port_write(MEM_CTLR, DSCH, dsch);
2714
2715         /* Enable ECC */
2716         ecc_ctrl = (BIT0 | BIT1 | BIT17);
2717         msg_port_write(MEM_CTLR, DECCCTRL, ecc_ctrl);
2718
2719         /* Assume 8 bank memory, one bank is gone for ECC */
2720         mrc_params->mem_size -= mrc_params->mem_size / 8;
2721
2722         /* For S3 resume memory content has to be preserved */
2723         if (mrc_params->boot_mode != BM_S3) {
2724                 select_hte();
2725                 hte_mem_init(mrc_params, MRC_MEM_INIT);
2726                 select_mem_mgr();
2727         }
2728
2729         LEAVEFN();
2730 }
2731
2732 /*
2733  * Execute memory test
2734  * if error detected it is indicated in mrc_params->status
2735  */
2736 void memory_test(struct mrc_params *mrc_params)
2737 {
2738         uint32_t result = 0;
2739
2740         ENTERFN();
2741
2742         select_hte();
2743         result = hte_mem_init(mrc_params, MRC_MEM_TEST);
2744         select_mem_mgr();
2745
2746         DPF(D_INFO, "Memory test result %x\n", result);
2747         mrc_params->status = ((result == 0) ? MRC_SUCCESS : MRC_E_MEMTEST);
2748         LEAVEFN();
2749 }
2750
2751 /* Lock MCU registers at the end of initialization sequence */
2752 void lock_registers(struct mrc_params *mrc_params)
2753 {
2754         u32 dco;
2755
2756         ENTERFN();
2757
2758         dco = msg_port_read(MEM_CTLR, DCO);
2759         dco &= ~(BIT28 | BIT29);
2760         dco |= (BIT0 | BIT8);
2761         msg_port_write(MEM_CTLR, DCO, dco);
2762
2763         LEAVEFN();
2764 }