drivers/ddr/marvell/axp/ddr3_dqs.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * Copyright (C) Marvell International Ltd. and its affiliates
   4  */
   5
   6 #include <common.h>
   7 #include <i2c.h>
   8 #include <spl.h>
   9 #include <asm/io.h>
  10 #include <asm/arch/cpu.h>
  11 #include <asm/arch/soc.h>
  12
  13 #include "ddr3_hw_training.h"
  14
  15 /*
  16  * Debug
  17  */
  18 #define DEBUG_DQS_C(s, d, l) \
  19         DEBUG_DQS_S(s); DEBUG_DQS_D(d, l); DEBUG_DQS_S("\n")
  20 #define DEBUG_DQS_FULL_C(s, d, l) \
  21         DEBUG_DQS_FULL_S(s); DEBUG_DQS_FULL_D(d, l); DEBUG_DQS_FULL_S("\n")
  22 #define DEBUG_DQS_RESULTS_C(s, d, l) \
  23         DEBUG_DQS_RESULTS_S(s); DEBUG_DQS_RESULTS_D(d, l); DEBUG_DQS_RESULTS_S("\n")
  24 #define DEBUG_PER_DQ_C(s, d, l) \
  25         puts(s); printf("%x", d); puts("\n")
  26
  27 #define DEBUG_DQS_RESULTS_S(s) \
  28         debug_cond(ddr3_get_log_level() >= MV_LOG_LEVEL_2, "%s", s)
  29 #define DEBUG_DQS_RESULTS_D(d, l) \
  30         debug_cond(ddr3_get_log_level() >= MV_LOG_LEVEL_2, "%x", d)
  31
  32 #define DEBUG_PER_DQ_S(s) \
  33         debug_cond(ddr3_get_log_level() >= MV_LOG_LEVEL_3, "%s", s)
  34 #define DEBUG_PER_DQ_D(d, l) \
  35         debug_cond(ddr3_get_log_level() >= MV_LOG_LEVEL_3, "%x", d)
  36 #define DEBUG_PER_DQ_DD(d, l) \
  37         debug_cond(ddr3_get_log_level() >= MV_LOG_LEVEL_3, "%d", d)
  38
  39 #ifdef MV_DEBUG_DQS
  40 #define DEBUG_DQS_S(s)                  puts(s)
  41 #define DEBUG_DQS_D(d, l)               printf("%x", d)
  42 #else
  43 #define DEBUG_DQS_S(s)
  44 #define DEBUG_DQS_D(d, l)
  45 #endif
  46
  47 #ifdef MV_DEBUG_DQS_FULL
  48 #define DEBUG_DQS_FULL_S(s)             puts(s)
  49 #define DEBUG_DQS_FULL_D(d, l)          printf("%x", d)
  50 #else
  51 #define DEBUG_DQS_FULL_S(s)
  52 #define DEBUG_DQS_FULL_D(d, l)
  53 #endif
  54
  55 /* State machine for centralization - find low & high limit */
  56 enum {
  57         PUP_ADLL_LIMITS_STATE_FAIL,
  58         PUP_ADLL_LIMITS_STATE_PASS,
  59         PUP_ADLL_LIMITS_STATE_FAIL_AFTER_PASS,
  60 };
  61
  62 /* Hold centralization low results */
  63 static int centralization_low_limit[MAX_PUP_NUM] = { 0 };
  64 /* Hold centralization high results */
  65 static int centralization_high_limit[MAX_PUP_NUM] = { 0 };
  66
  67 int ddr3_find_adll_limits(MV_DRAM_INFO *dram_info, u32 cs, u32 ecc, int is_tx);
  68 int ddr3_check_window_limits(u32 pup, int high_limit, int low_limit, int is_tx,
  69                           int *size_valid);
  70 static int ddr3_center_calc(MV_DRAM_INFO *dram_info, u32 cs, u32 ecc,
  71                             int is_tx);
  72 int ddr3_special_pattern_i_search(MV_DRAM_INFO *dram_info, u32 cs, u32 ecc,
  73                               int is_tx, u32 special_pattern_pup);
  74 int ddr3_special_pattern_ii_search(MV_DRAM_INFO *dram_info, u32 cs, u32 ecc,
  75                                    int is_tx, u32 special_pattern_pup);
  76 int ddr3_set_dqs_centralization_results(MV_DRAM_INFO *dram_info, u32 cs, u32 ecc,
  77                                     int is_tx);
  78
  79 #ifdef MV88F78X60
  80 extern u32 killer_pattern_32b[DQ_NUM][LEN_SPECIAL_PATTERN];
  81 extern u32 killer_pattern_64b[DQ_NUM][LEN_SPECIAL_PATTERN];
  82 extern int per_bit_data[MAX_PUP_NUM][DQ_NUM];
  83 #else
  84 extern u32 killer_pattern[DQ_NUM][LEN_16BIT_KILLER_PATTERN];
  85 extern u32 killer_pattern_32b[DQ_NUM][LEN_SPECIAL_PATTERN];
  86 #if defined(MV88F672X)
  87 extern int per_bit_data[MAX_PUP_NUM][DQ_NUM];
  88 #endif
  89 #endif
  90 extern u32 special_pattern[DQ_NUM][LEN_SPECIAL_PATTERN];
  91
  92 static u32 *ddr3_dqs_choose_pattern(MV_DRAM_INFO *dram_info, u32 victim_dq)
  93 {
  94         u32 *pattern_ptr;
  95
  96         /* Choose pattern */
  97         switch (dram_info->ddr_width) {
  98 #if defined(MV88F672X)
  99         case 16:
 100                 pattern_ptr = (u32 *)&killer_pattern[victim_dq];
 101                 break;
 102 #endif
 103         case 32:
 104                 pattern_ptr = (u32 *)&killer_pattern_32b[victim_dq];
 105                 break;
 106 #if defined(MV88F78X60)
 107         case 64:
 108                 pattern_ptr = (u32 *)&killer_pattern_64b[victim_dq];
 109                 break;
 110 #endif
 111         default:
 112 #if defined(MV88F78X60)
 113                 pattern_ptr = (u32 *)&killer_pattern_32b[victim_dq];
 114 #else
 115                 pattern_ptr = (u32 *)&killer_pattern[victim_dq];
 116 #endif
 117                 break;
 118         }
 119
 120         return pattern_ptr;
 121 }
 122
 123 /*
 124  * Name:     ddr3_dqs_centralization_rx
 125  * Desc:     Execute the DQS centralization RX phase.
 126  * Args:     dram_info
 127  * Notes:
 128  * Returns:  MV_OK if success, other error code if fail.
 129  */
 130 int ddr3_dqs_centralization_rx(MV_DRAM_INFO *dram_info)
 131 {
 132         u32 cs, ecc, reg;
 133         int status;
 134
 135         DEBUG_DQS_S("DDR3 - DQS Centralization RX - Starting procedure\n");
 136
 137         /* Enable SW override */
 138         reg = reg_read(REG_DRAM_TRAINING_2_ADDR) |
 139                 (1 << REG_DRAM_TRAINING_2_SW_OVRD_OFFS);
 140
 141         /* [0] = 1 - Enable SW override  */
 142         /* 0x15B8 - Training SW 2 Register */
 143         reg_write(REG_DRAM_TRAINING_2_ADDR, reg);
 144         DEBUG_DQS_S("DDR3 - DQS Centralization RX - SW Override Enabled\n");
 145
 146         reg = (1 << REG_DRAM_TRAINING_AUTO_OFFS);
 147         reg_write(REG_DRAM_TRAINING_ADDR, reg); /* 0x15B0 - Training Register */
 148
 149         /* Loop for each CS */
 150         for (cs = 0; cs < MAX_CS; cs++) {
 151                 if (dram_info->cs_ena & (1 << cs)) {
 152                         DEBUG_DQS_FULL_C("DDR3 - DQS Centralization RX - CS - ",
 153                                          (u32) cs, 1);
 154
 155                         for (ecc = 0; ecc < (dram_info->ecc_ena + 1); ecc++) {
 156
 157                                 /* ECC Support - Switch ECC Mux on ecc=1 */
 158                                 reg = reg_read(REG_DRAM_TRAINING_2_ADDR) &
 159                                         ~(1 << REG_DRAM_TRAINING_2_ECC_MUX_OFFS);
 160                                 reg |= (dram_info->ecc_ena *
 161                                         ecc << REG_DRAM_TRAINING_2_ECC_MUX_OFFS);
 162                                 reg_write(REG_DRAM_TRAINING_2_ADDR, reg);
 163
 164                                 if (ecc)
 165                                         DEBUG_DQS_FULL_S("DDR3 - DQS Centralization RX - ECC Mux Enabled\n");
 166                                 else
 167                                         DEBUG_DQS_FULL_S("DDR3 - DQS Centralization RX - ECC Mux Disabled\n");
 168
 169                                 DEBUG_DQS_FULL_S("DDR3 - DQS Centralization RX - Find all limits\n");
 170
 171                                 status = ddr3_find_adll_limits(dram_info, cs,
 172                                                                ecc, 0);
 173                                 if (MV_OK != status)
 174                                         return status;
 175
 176                                 DEBUG_DQS_FULL_S("DDR3 - DQS Centralization RX - Start calculating center\n");
 177
 178                                 status = ddr3_center_calc(dram_info, cs, ecc,
 179                                                           0);
 180                                 if (MV_OK != status)
 181                                         return status;
 182                         }
 183                 }
 184         }
 185
 186         /* ECC Support - Disable ECC MUX */
 187         reg = reg_read(REG_DRAM_TRAINING_2_ADDR) &
 188                 ~(1 << REG_DRAM_TRAINING_2_ECC_MUX_OFFS);
 189         reg_write(REG_DRAM_TRAINING_2_ADDR, reg);
 190
 191         /* Disable SW override - Must be in a different stage */
 192         /* [0]=0 - Enable SW override  */
 193         reg = reg_read(REG_DRAM_TRAINING_2_ADDR);
 194         reg &= ~(1 << REG_DRAM_TRAINING_2_SW_OVRD_OFFS);
 195         /* 0x15B8 - Training SW 2 Register */
 196         reg_write(REG_DRAM_TRAINING_2_ADDR, reg);
 197
 198         reg = reg_read(REG_DRAM_TRAINING_1_ADDR) |
 199                 (1 << REG_DRAM_TRAINING_1_TRNBPOINT_OFFS);
 200         reg_write(REG_DRAM_TRAINING_1_ADDR, reg);
 201
 202         return MV_OK;
 203 }
 204
 205 /*
 206  * Name:     ddr3_dqs_centralization_tx
 207  * Desc:     Execute the DQS centralization TX phase.
 208  * Args:     dram_info
 209  * Notes:
 210  * Returns:  MV_OK if success, other error code if fail.
 211  */
 212 int ddr3_dqs_centralization_tx(MV_DRAM_INFO *dram_info)
 213 {
 214         u32 cs, ecc, reg;
 215         int status;
 216
 217         DEBUG_DQS_S("DDR3 - DQS Centralization TX - Starting procedure\n");
 218
 219         /* Enable SW override */
 220         reg = reg_read(REG_DRAM_TRAINING_2_ADDR) |
 221                 (1 << REG_DRAM_TRAINING_2_SW_OVRD_OFFS);
 222
 223         /* [0] = 1 - Enable SW override  */
 224         /* 0x15B8 - Training SW 2 Register */
 225         reg_write(REG_DRAM_TRAINING_2_ADDR, reg);
 226         DEBUG_DQS_S("DDR3 - DQS Centralization TX - SW Override Enabled\n");
 227
 228         reg = (1 << REG_DRAM_TRAINING_AUTO_OFFS);
 229         reg_write(REG_DRAM_TRAINING_ADDR, reg); /* 0x15B0 - Training Register */
 230
 231         /* Loop for each CS */
 232         for (cs = 0; cs < MAX_CS; cs++) {
 233                 if (dram_info->cs_ena & (1 << cs)) {
 234                         DEBUG_DQS_FULL_C("DDR3 - DQS Centralization TX - CS - ",
 235                                          (u32) cs, 1);
 236                         for (ecc = 0; ecc < (dram_info->ecc_ena + 1); ecc++) {
 237                                 /* ECC Support - Switch ECC Mux on ecc=1 */
 238                                 reg = reg_read(REG_DRAM_TRAINING_2_ADDR) &
 239                                         ~(1 << REG_DRAM_TRAINING_2_ECC_MUX_OFFS);
 240                                 reg |= (dram_info->ecc_ena *
 241                                         ecc << REG_DRAM_TRAINING_2_ECC_MUX_OFFS);
 242                                 reg_write(REG_DRAM_TRAINING_2_ADDR, reg);
 243
 244                                 if (ecc)
 245                                         DEBUG_DQS_FULL_S("DDR3 - DQS Centralization TX - ECC Mux Enabled\n");
 246                                 else
 247                                         DEBUG_DQS_FULL_S("DDR3 - DQS Centralization TX - ECC Mux Disabled\n");
 248
 249                                 DEBUG_DQS_FULL_S("DDR3 - DQS Centralization TX - Find all limits\n");
 250
 251                                 status = ddr3_find_adll_limits(dram_info, cs,
 252                                                                ecc, 1);
 253                                 if (MV_OK != status)
 254                                         return status;
 255
 256                                 DEBUG_DQS_FULL_S("DDR3 - DQS Centralization TX - Start calculating center\n");
 257
 258                                 status = ddr3_center_calc(dram_info, cs, ecc,
 259                                                           1);
 260                                 if (MV_OK != status)
 261                                         return status;
 262                         }
 263                 }
 264         }
 265
 266         /* ECC Support - Disable ECC MUX */
 267         reg = reg_read(REG_DRAM_TRAINING_2_ADDR) &
 268                 ~(1 << REG_DRAM_TRAINING_2_ECC_MUX_OFFS);
 269         reg_write(REG_DRAM_TRAINING_2_ADDR, reg);
 270
 271         /* Disable SW override - Must be in a different stage */
 272         /* [0]=0 - Enable SW override  */
 273         reg = reg_read(REG_DRAM_TRAINING_2_ADDR);
 274         reg &= ~(1 << REG_DRAM_TRAINING_2_SW_OVRD_OFFS);
 275         /* 0x15B8 - Training SW 2 Register */
 276         reg_write(REG_DRAM_TRAINING_2_ADDR, reg);
 277
 278         reg = reg_read(REG_DRAM_TRAINING_1_ADDR) |
 279                 (1 << REG_DRAM_TRAINING_1_TRNBPOINT_OFFS);
 280         reg_write(REG_DRAM_TRAINING_1_ADDR, reg);
 281
 282         return MV_OK;
 283 }
 284
 285 /*
 286  * Name:     ddr3_find_adll_limits
 287  * Desc:     Execute the Find ADLL limits phase.
 288  * Args:     dram_info
 289  *           cs
 290  *           ecc_ena
 291  *           is_tx             Indicate whether Rx or Tx
 292  * Notes:
 293  * Returns:  MV_OK if success, other error code if fail.
 294  */
 295 int ddr3_find_adll_limits(MV_DRAM_INFO *dram_info, u32 cs, u32 ecc, int is_tx)
 296 {
 297         u32 victim_dq, pup, tmp;
 298         u32 adll_addr;
 299         u32 max_pup;            /* maximal pup index */
 300         u32 pup_mask = 0;
 301         u32 unlock_pup;         /* bit array of un locked pups */
 302         u32 new_unlock_pup;     /* bit array of compare failed pups */
 303         u32 curr_adll;
 304         u32 adll_start_val;     /* adll start loop value - for rx or tx limit */
 305         u32 high_limit; /* holds found High Limit */
 306         u32 low_limit;          /* holds found Low Limit */
 307         int win_valid;
 308         int update_win;
 309         u32 sdram_offset;
 310         u32 uj, cs_count, cs_tmp, ii;
 311         u32 *pattern_ptr;
 312         u32 dq;
 313         u32 adll_end_val;       /* adll end of loop val - for rx or tx limit */
 314         u8 analog_pbs[DQ_NUM][MAX_PUP_NUM][DQ_NUM][2];
 315         u8 analog_pbs_sum[MAX_PUP_NUM][DQ_NUM][2];
 316         int pup_adll_limit_state[MAX_PUP_NUM];  /* hold state of each pup */
 317
 318         adll_addr = ((is_tx == 1) ? PUP_DQS_WR : PUP_DQS_RD);
 319         adll_end_val = ((is_tx == 1) ? ADLL_MIN : ADLL_MAX);
 320         adll_start_val = ((is_tx == 1) ? ADLL_MAX : ADLL_MIN);
 321         max_pup = (ecc + (1 - ecc) * dram_info->num_of_std_pups);
 322
 323         DEBUG_DQS_FULL_S("DDR3 - DQS Find Limits - Starting Find ADLL Limits\n");
 324
 325         /* init the array */
 326         for (pup = 0; pup < max_pup; pup++) {
 327                 centralization_low_limit[pup] = ADLL_MIN;
 328                 centralization_high_limit[pup] = ADLL_MAX;
 329         }
 330
 331         /* Killer Pattern */
 332         cs_count = 0;
 333         for (cs_tmp = 0; cs_tmp < cs; cs_tmp++) {
 334                 if (dram_info->cs_ena & (1 << cs_tmp))
 335                         cs_count++;
 336         }
 337         sdram_offset = cs_count * (SDRAM_CS_SIZE + 1);
 338         sdram_offset += ((is_tx == 1) ?
 339                          SDRAM_DQS_TX_OFFS : SDRAM_DQS_RX_OFFS);
 340
 341         /* Prepare pup masks */
 342         for (pup = 0; pup < max_pup; pup++)
 343                 pup_mask |= (1 << pup);
 344
 345         for (pup = 0; pup < max_pup; pup++) {
 346                 for (dq = 0; dq < DQ_NUM; dq++) {
 347                         analog_pbs_sum[pup][dq][0] = adll_start_val;
 348                         analog_pbs_sum[pup][dq][1] = adll_end_val;
 349                 }
 350         }
 351
 352         /* Loop - use different pattern for each victim_dq */
 353         for (victim_dq = 0; victim_dq < DQ_NUM; victim_dq++) {
 354                 DEBUG_DQS_FULL_C("DDR3 - DQS Find Limits - Victim DQ - ",
 355                                  (u32)victim_dq, 1);
 356                 /*
 357                  * The pups 3 bit arrays represent state machine. with
 358                  * 3 stages for each pup.
 359                  * 1. fail and didn't get pass in earlier compares.
 360                  * 2. pass compare
 361                  * 3. fail after pass - end state.
 362                  * The window limits are the adll values where the adll
 363                  * was in the pass stage.
 364                  */
 365
 366                 /* Set all states to Fail (1st state) */
 367                 for (pup = 0; pup < max_pup; pup++)
 368                         pup_adll_limit_state[pup] = PUP_ADLL_LIMITS_STATE_FAIL;
 369
 370                 /* Set current valid pups */
 371                 unlock_pup = pup_mask;
 372
 373                 /* Set ADLL to start value */
 374                 curr_adll = adll_start_val;
 375
 376 #if defined(MV88F78X60)
 377                 for (pup = 0; pup < max_pup; pup++) {
 378                         for (dq = 0; dq < DQ_NUM; dq++) {
 379                                 analog_pbs[victim_dq][pup][dq][0] =
 380                                         adll_start_val;
 381                                 analog_pbs[victim_dq][pup][dq][1] =
 382                                         adll_end_val;
 383                                 per_bit_data[pup][dq] = 0;
 384                         }
 385                 }
 386 #endif
 387
 388                 for (uj = 0; uj < ADLL_MAX; uj++) {
 389                         DEBUG_DQS_FULL_C("DDR3 - DQS Find Limits - Setting ADLL to ",
 390                                          curr_adll, 2);
 391                         for (pup = 0; pup < max_pup; pup++) {
 392                                 if (IS_PUP_ACTIVE(unlock_pup, pup) == 1) {
 393                                         tmp = ((is_tx == 1) ? curr_adll +
 394                                                dram_info->wl_val[cs]
 395                                                [pup * (1 - ecc) + ecc * ECC_PUP]
 396                                                [D] : curr_adll);
 397                                         ddr3_write_pup_reg(adll_addr, cs, pup +
 398                                                 (ecc * ECC_PUP), 0, tmp);
 399                                 }
 400                         }
 401
 402                         /* Choose pattern */
 403                         pattern_ptr = ddr3_dqs_choose_pattern(dram_info,
 404                                                               victim_dq);
 405
 406                         /* '1' - means pup failed, '0' - means pup pass */
 407                         new_unlock_pup = 0;
 408
 409                         /* Read and compare results for Victim_DQ# */
 410                         for (ii = 0; ii < 3; ii++) {
 411                                 u32 tmp = 0;
 412                                 if (MV_OK != ddr3_sdram_dqs_compare(dram_info,
 413                                                            unlock_pup, &tmp,
 414                                                            pattern_ptr,
 415                                                            LEN_KILLER_PATTERN,
 416                                                            sdram_offset +
 417                                                            LEN_KILLER_PATTERN *
 418                                                            4 * victim_dq,
 419                                                            is_tx, 0, NULL,
 420                                                            0))
 421                                         return MV_DDR3_TRAINING_ERR_DRAM_COMPARE;
 422
 423                                 new_unlock_pup |= tmp;
 424                         }
 425
 426                         pup = 0;
 427                         DEBUG_DQS_FULL_C("DDR3 - DQS Find Limits - UnlockPup: ",
 428                                          unlock_pup, 2);
 429                         DEBUG_DQS_FULL_C("DDR3 - DQS Find Limits - NewUnlockPup: ",
 430                                          new_unlock_pup, 2);
 431
 432                         /* Update pup state */
 433                         for (pup = 0; pup < max_pup; pup++) {
 434                                 if (IS_PUP_ACTIVE(unlock_pup, pup) == 0) {
 435                                         DEBUG_DQS_FULL_C("DDR3 - DQS Find Limits - Skipping pup ",
 436                                                          pup, 1);
 437                                         continue;
 438                                 }
 439
 440                                 /*
 441                                  * Still didn't find the window limit of the pup
 442                                  */
 443                                 if (IS_PUP_ACTIVE(new_unlock_pup, pup) == 1) {
 444                                         /* Current compare result == fail */
 445                                         if (pup_adll_limit_state[pup] ==
 446                                             PUP_ADLL_LIMITS_STATE_PASS) {
 447                                                 /*
 448                                                  * If now it failed but passed
 449                                                  * earlier
 450                                                  */
 451                                                 DEBUG_DQS_S("DDR3 - DQS Find Limits - PASS to FAIL: CS - ");
 452                                                 DEBUG_DQS_D(cs, 1);
 453                                                 DEBUG_DQS_S(", DQ - ");
 454                                                 DEBUG_DQS_D(victim_dq, 1);
 455                                                 DEBUG_DQS_S(", Pup - ");
 456                                                 DEBUG_DQS_D(pup, 1);
 457                                                 DEBUG_DQS_S(", ADLL - ");
 458                                                 DEBUG_DQS_D(curr_adll, 2);
 459                                                 DEBUG_DQS_S("\n");
 460
 461 #if defined(MV88F78X60)
 462                                                 for (dq = 0; dq < DQ_NUM; dq++) {
 463                                                         if ((analog_pbs[victim_dq][pup][dq][0] != adll_start_val)
 464                                                             && (analog_pbs[victim_dq][pup]
 465                                                                 [dq][1] == adll_end_val))
 466                                                                 analog_pbs
 467                                                                         [victim_dq]
 468                                                                         [pup][dq]
 469                                                                         [1] =
 470                                                                         curr_adll;
 471                                                 }
 472 #endif
 473                                                 win_valid = 1;
 474                                                 update_win = 0;
 475
 476                                                 /* Keep min / max limit value */
 477                                                 if (is_tx == 0) {
 478                                                         /* RX - found upper limit */
 479                                                         if (centralization_high_limit[pup] >
 480                                                             (curr_adll - 1)) {
 481                                                                 high_limit =
 482                                                                         curr_adll - 1;
 483                                                                 low_limit =
 484                                                                         centralization_low_limit[pup];
 485                                                                 update_win = 1;
 486                                                         }
 487                                                 } else {
 488                                                         /* TX - found lower limit */
 489                                                         if (centralization_low_limit[pup] < (curr_adll + 1)) {
 490                                                                 high_limit =
 491                                                                         centralization_high_limit
 492                                                                         [pup];
 493                                                                 low_limit =
 494                                                                         curr_adll + 1;
 495                                                                 update_win =
 496                                                                         1;
 497                                                         }
 498                                                 }
 499
 500                                                 if (update_win == 1) {
 501                                                         /*
 502                                                          * Before updating
 503                                                          * window limits we need
 504                                                          * to check that the
 505                                                          * limits are valid
 506                                                          */
 507                                                         if (MV_OK !=
 508                                                             ddr3_check_window_limits
 509                                                             (pup, high_limit,
 510                                                              low_limit, is_tx,
 511                                                              &win_valid))
 512                                                                 return MV_DDR3_TRAINING_ERR_WIN_LIMITS;
 513
 514                                                         if (win_valid == 1) {
 515                                                                 /*
 516                                                                  * Window limits
 517                                                                  * should be
 518                                                                  * updated
 519                                                                  */
 520                                                                 centralization_low_limit
 521                                                                         [pup] =
 522                                                                         low_limit;
 523                                                                 centralization_high_limit
 524                                                                         [pup] =
 525                                                                         high_limit;
 526                                                         }
 527                                                 }
 528
 529                                                 if (win_valid == 1) {
 530                                                         /* Found end of window - lock the pup */
 531                                                         pup_adll_limit_state[pup] =
 532                                                                 PUP_ADLL_LIMITS_STATE_FAIL_AFTER_PASS;
 533                                                         unlock_pup &= ~(1 << pup);
 534                                                 } else {
 535                                                         /* Probably false pass - reset status */
 536                                                         pup_adll_limit_state[pup] =
 537                                                                 PUP_ADLL_LIMITS_STATE_FAIL;
 538
 539 #if defined(MV88F78X60)
 540                                                         /* Clear logging array of win size (per Dq) */
 541                                                         for (dq = 0;
 542                                                              dq < DQ_NUM;
 543                                                              dq++) {
 544                                                                 analog_pbs
 545                                                                         [victim_dq]
 546                                                                         [pup][dq]
 547                                                                         [0] =
 548                                                                         adll_start_val;
 549                                                                 analog_pbs
 550                                                                         [victim_dq]
 551                                                                         [pup][dq]
 552                                                                         [1] =
 553                                                                         adll_end_val;
 554                                                                 per_bit_data
 555                                                                         [pup][dq]
 556                                                                         = 0;
 557                                                         }
 558 #endif
 559                                                 }
 560                                         }
 561                                 } else {
 562                                         /* Current compare result == pass */
 563                                         if (pup_adll_limit_state[pup] ==
 564                                             PUP_ADLL_LIMITS_STATE_FAIL) {
 565                                                 /* If now it passed but failed earlier */
 566                                                 DEBUG_DQS_S("DDR3 - DQS Find Limits - FAIL to PASS: CS - ");
 567                                                 DEBUG_DQS_D(cs, 1);
 568                                                 DEBUG_DQS_S(", DQ - ");
 569                                                 DEBUG_DQS_D(victim_dq, 1);
 570                                                 DEBUG_DQS_S(", Pup - ");
 571                                                 DEBUG_DQS_D(pup, 1);
 572                                                 DEBUG_DQS_S(", ADLL - ");
 573                                                 DEBUG_DQS_D(curr_adll, 2);
 574                                                 DEBUG_DQS_S("\n");
 575
 576 #if defined(MV88F78X60)
 577                                                 for (dq = 0; dq < DQ_NUM;
 578                                                      dq++) {
 579                                                         if (analog_pbs[victim_dq][pup][dq][0] == adll_start_val)
 580                                                                 analog_pbs
 581                                                                     [victim_dq]
 582                                                                     [pup][dq]
 583                                                                     [0] =
 584                                                                     curr_adll;
 585                                                 }
 586 #endif
 587                                                 /* Found start of window */
 588                                                 pup_adll_limit_state[pup] =
 589                                                     PUP_ADLL_LIMITS_STATE_PASS;
 590
 591                                                 /* Keep min / max limit value */
 592                                                 if (is_tx == 0) {
 593                                                         /* RX - found low limit */
 594                                                         if (centralization_low_limit[pup] <= curr_adll)
 595                                                                 centralization_low_limit
 596                                                                     [pup] =
 597                                                                     curr_adll;
 598                                                 } else {
 599                                                         /* TX - found high limit */
 600                                                         if (centralization_high_limit[pup] >= curr_adll)
 601                                                                 centralization_high_limit
 602                                                                     [pup] =
 603                                                                     curr_adll;
 604                                                 }
 605                                         }
 606                                 }
 607                         }
 608
 609                         if (unlock_pup == 0) {
 610                                 /* Found limit to all pups */
 611                                 DEBUG_DQS_FULL_S("DDR3 - DQS Find Limits - found PUP limit\n");
 612                                 break;
 613                         }
 614
 615                         /*
 616                          * Increment / decrement (Move to right / left
 617                          * one phase - ADLL) dqs RX / TX delay (for all un
 618                          * lock pups
 619                          */
 620                         if (is_tx == 0)
 621                                 curr_adll++;
 622                         else
 623                                 curr_adll--;
 624                 }
 625
 626                 if (unlock_pup != 0) {
 627                         /*
 628                          * Found pups that didn't reach to the end of the
 629                          * state machine
 630                          */
 631                         DEBUG_DQS_C("DDR3 - DQS Find Limits - Pups that didn't reached end of the state machine: ",
 632                                     unlock_pup, 1);
 633
 634                         for (pup = 0; pup < max_pup; pup++) {
 635                                 if (IS_PUP_ACTIVE(unlock_pup, pup) == 1) {
 636                                         if (pup_adll_limit_state[pup] ==
 637                                             PUP_ADLL_LIMITS_STATE_FAIL) {
 638                                                 /* ERROR - found fail for all window size */
 639                                                 DEBUG_DQS_S("DDR3 - DQS Find Limits - Got FAIL for the complete range on pup - ");
 640                                                 DEBUG_DQS_D(pup, 1);
 641                                                 DEBUG_DQS_C(" victim DQ ",
 642                                                             victim_dq, 1);
 643
 644                                                 /* For debug - set min limit to illegal limit */
 645                                                 centralization_low_limit[pup]
 646                                                         = ADLL_ERROR;
 647                                                 /*
 648                                                  * In case the pup is in mode
 649                                                  * PASS - the limit is the min
 650                                                  * / max adll, no need to
 651                                                  * update because of the results
 652                                                  * array default value
 653                                                  */
 654                                                 return MV_DDR3_TRAINING_ERR_PUP_RANGE;
 655                                         }
 656                                 }
 657                         }
 658                 }
 659         }
 660
 661         DEBUG_DQS_S("DDR3 - DQS Find Limits - DQ values per victim results:\n");
 662         for (victim_dq = 0; victim_dq < DQ_NUM; victim_dq++) {
 663                 for (pup = 0; pup < max_pup; pup++) {
 664                         DEBUG_DQS_S("Victim DQ-");
 665                         DEBUG_DQS_D(victim_dq, 1);
 666                         DEBUG_DQS_S(", PUP-");
 667                         DEBUG_DQS_D(pup, 1);
 668                         for (dq = 0; dq < DQ_NUM; dq++) {
 669                                 DEBUG_DQS_S(", DQ-");
 670                                 DEBUG_DQS_D(dq, 1);
 671                                 DEBUG_DQS_S(",S-");
 672                                 DEBUG_DQS_D(analog_pbs[victim_dq][pup][dq]
 673                                             [0], 2);
 674                                 DEBUG_DQS_S(",E-");
 675                                 DEBUG_DQS_D(analog_pbs[victim_dq][pup][dq]
 676                                             [1], 2);
 677
 678                                 if (is_tx == 0) {
 679                                         if (analog_pbs[victim_dq][pup][dq][0]
 680                                             > analog_pbs_sum[pup][dq][0])
 681                                                 analog_pbs_sum[pup][dq][0] =
 682                                                     analog_pbs[victim_dq][pup]
 683                                                     [dq][0];
 684                                         if (analog_pbs[victim_dq][pup][dq][1]
 685                                             < analog_pbs_sum[pup][dq][1])
 686                                                 analog_pbs_sum[pup][dq][1] =
 687                                                     analog_pbs[victim_dq][pup]
 688                                                     [dq][1];
 689                                 } else {
 690                                         if (analog_pbs[victim_dq][pup][dq][0]
 691                                             < analog_pbs_sum[pup][dq][0])
 692                                                 analog_pbs_sum[pup][dq][0] =
 693                                                     analog_pbs[victim_dq][pup]
 694                                                     [dq][0];
 695                                         if (analog_pbs[victim_dq][pup][dq][1]
 696                                             > analog_pbs_sum[pup][dq][1])
 697                                                 analog_pbs_sum[pup][dq][1] =
 698                                                     analog_pbs[victim_dq][pup]
 699                                                     [dq][1];
 700                                 }
 701                         }
 702                         DEBUG_DQS_S("\n");
 703                 }
 704         }
 705
 706         if (ddr3_get_log_level() >= MV_LOG_LEVEL_3) {
 707                 u32 dq;
 708
 709                 DEBUG_PER_DQ_S("\n########## LOG LEVEL 3(Windows margins per-DQ) ##########\n");
 710                 if (is_tx) {
 711                         DEBUG_PER_DQ_C("DDR3 - TX  CS: ", cs, 1);
 712                 } else {
 713                         DEBUG_PER_DQ_C("DDR3 - RX  CS: ", cs, 1);
 714                 }
 715
 716                 if (ecc == 0) {
 717                         DEBUG_PER_DQ_S("\n DATA RESULTS:\n");
 718                 } else {
 719                         DEBUG_PER_DQ_S("\n ECC RESULTS:\n");
 720                 }
 721
 722                 /* Since all dq has the same value we take 0 as representive */
 723                 dq = 0;
 724                 for (pup = 0; pup < max_pup; pup++) {
 725                         if (ecc == 0) {
 726                                 DEBUG_PER_DQ_S("\nBYTE:");
 727                                 DEBUG_PER_DQ_D(pup, 1);
 728                                 DEBUG_PER_DQ_S("\n");
 729                         } else {
 730                                 DEBUG_PER_DQ_S("\nECC BYTE:\n");
 731                         }
 732                         DEBUG_PER_DQ_S("  DQ's        LOW       HIGH       WIN-SIZE\n");
 733                         DEBUG_PER_DQ_S("============================================\n");
 734                         for (victim_dq = 0; victim_dq < DQ_NUM; victim_dq++) {
 735                                 if (ecc == 0) {
 736                                         DEBUG_PER_DQ_S("DQ[");
 737                                         DEBUG_PER_DQ_DD((victim_dq +
 738                                                          DQ_NUM * pup), 2);
 739                                         DEBUG_PER_DQ_S("]");
 740                                 } else {
 741                                         DEBUG_PER_DQ_S("CB[");
 742                                         DEBUG_PER_DQ_DD(victim_dq, 2);
 743                                         DEBUG_PER_DQ_S("]");
 744                                 }
 745                                 if (is_tx) {
 746                                         DEBUG_PER_DQ_S("      0x");
 747                                         DEBUG_PER_DQ_D(analog_pbs[victim_dq][pup][dq][1], 2);   /* low value */
 748                                         DEBUG_PER_DQ_S("        0x");
 749                                         DEBUG_PER_DQ_D(analog_pbs[victim_dq][pup][dq][0], 2);   /* high value */
 750                                         DEBUG_PER_DQ_S("        0x");
 751                                         DEBUG_PER_DQ_D(analog_pbs[victim_dq][pup][dq][0] - analog_pbs[victim_dq][pup][dq][1], 2);       /* win-size */
 752                                 } else {
 753                                         DEBUG_PER_DQ_S("     0x");
 754                                         DEBUG_PER_DQ_D(analog_pbs[victim_dq][pup][dq][0], 2);   /* low value */
 755                                         DEBUG_PER_DQ_S("       0x");
 756                                         DEBUG_PER_DQ_D((analog_pbs[victim_dq][pup][dq][1] - 1), 2);     /* high value */
 757                                         DEBUG_PER_DQ_S("       0x");
 758                                         DEBUG_PER_DQ_D(analog_pbs[victim_dq][pup][dq][1] - analog_pbs[victim_dq][pup][dq][0], 2);       /* win-size */
 759                                 }
 760                                 DEBUG_PER_DQ_S("\n");
 761                         }
 762                 }
 763                 DEBUG_PER_DQ_S("\n");
 764         }
 765
 766         if (is_tx) {
 767                 DEBUG_DQS_S("DDR3 - DQS TX - Find Limits - DQ values Summary:\n");
 768         } else {
 769                 DEBUG_DQS_S("DDR3 - DQS RX - Find Limits - DQ values Summary:\n");
 770         }
 771
 772         for (pup = 0; pup < max_pup; pup++) {
 773                 DEBUG_DQS_S("PUP-");
 774                 DEBUG_DQS_D(pup, 1);
 775                 for (dq = 0; dq < DQ_NUM; dq++) {
 776                         DEBUG_DQS_S(", DQ-");
 777                         DEBUG_DQS_D(dq, 1);
 778                         DEBUG_DQS_S(",S-");
 779                         DEBUG_DQS_D(analog_pbs_sum[pup][dq][0], 2);
 780                         DEBUG_DQS_S(",E-");
 781                         DEBUG_DQS_D(analog_pbs_sum[pup][dq][1], 2);
 782                 }
 783                 DEBUG_DQS_S("\n");
 784         }
 785
 786         if (is_tx) {
 787                 DEBUG_DQS_S("DDR3 - DQS TX - Find Limits - DQ values Summary:\n");
 788         } else {
 789                 DEBUG_DQS_S("DDR3 - DQS RX - Find Limits - DQ values Summary:\n");
 790         }
 791
 792         for (pup = 0; pup < max_pup; pup++) {
 793                 if (max_pup == 1) {
 794                         /* For ECC PUP */
 795                         DEBUG_DQS_S("DDR3 - DQS8");
 796                 } else {
 797                         DEBUG_DQS_S("DDR3 - DQS");
 798                         DEBUG_DQS_D(pup, 1);
 799                 }
 800
 801                 for (dq = 0; dq < DQ_NUM; dq++) {
 802                         DEBUG_DQS_S(", DQ-");
 803                         DEBUG_DQS_D(dq, 1);
 804                         DEBUG_DQS_S("::S-");
 805                         DEBUG_DQS_D(analog_pbs_sum[pup][dq][0], 2);
 806                         DEBUG_DQS_S(",E-");
 807                         DEBUG_DQS_D(analog_pbs_sum[pup][dq][1], 2);
 808                 }
 809                 DEBUG_DQS_S("\n");
 810         }
 811
 812         DEBUG_DQS_S("DDR3 - DQS Find Limits - Ended\n");
 813
 814         return MV_OK;
 815 }
 816
 817 /*
 818  * Name:     ddr3_check_window_limits
 819  * Desc:     Check window High & Low limits.
 820  * Args:     pup                pup index
 821  *           high_limit           window high limit
 822  *           low_limit            window low limit
 823  *           is_tx                Indicate whether Rx or Tx
 824  *           size_valid          Indicate whether window size is valid
 825  * Notes:
 826  * Returns:  MV_OK if success, other error code if fail.
 827  */
 828 int ddr3_check_window_limits(u32 pup, int high_limit, int low_limit, int is_tx,
 829                              int *size_valid)
 830 {
 831         DEBUG_DQS_FULL_S("DDR3 - DQS Check Win Limits - Starting\n");
 832
 833         if (low_limit > high_limit) {
 834                 DEBUG_DQS_S("DDR3 - DQS Check Win Limits - Pup ");
 835                 DEBUG_DQS_D(pup, 1);
 836                 DEBUG_DQS_S(" Low Limit grater than High Limit\n");
 837                 *size_valid = 0;
 838                 return MV_OK;
 839         }
 840
 841         /*
 842          * Check that window size is valid, if not it was probably false pass
 843          * before
 844          */
 845         if ((high_limit - low_limit) < MIN_WIN_SIZE) {
 846                 /*
 847                  * Since window size is too small probably there was false
 848                  * pass
 849                  */
 850                 *size_valid = 0;
 851
 852                 DEBUG_DQS_S("DDR3 - DQS Check Win Limits - Pup ");
 853                 DEBUG_DQS_D(pup, 1);
 854                 DEBUG_DQS_S(" Window size is smaller than MIN_WIN_SIZE\n");
 855
 856         } else if ((high_limit - low_limit) > ADLL_MAX) {
 857                 *size_valid = 0;
 858
 859                 DEBUG_DQS_S("DDR3 - DQS Check Win Limits - Pup ");
 860                 DEBUG_DQS_D(pup, 1);
 861                 DEBUG_DQS_S
 862                     (" Window size is bigger than max ADLL taps (31)  Exiting.\n");
 863
 864                 return MV_FAIL;
 865
 866         } else {
 867                 *size_valid = 1;
 868
 869                 DEBUG_DQS_FULL_S("DDR3 - DQS Check Win Limits - Pup ");
 870                 DEBUG_DQS_FULL_D(pup, 1);
 871                 DEBUG_DQS_FULL_C(" window size is ", (high_limit - low_limit),
 872                                  2);
 873         }
 874
 875         return MV_OK;
 876 }
 877
 878 /*
 879  * Name:     ddr3_center_calc
 880  * Desc:     Execute the calculate the center of windows phase.
 881  * Args:     pDram Info
 882  *           is_tx             Indicate whether Rx or Tx
 883  * Notes:
 884  * Returns:  MV_OK if success, other error code if fail.
 885  */
 886 static int ddr3_center_calc(MV_DRAM_INFO *dram_info, u32 cs, u32 ecc,
 887                             int is_tx)
 888 {
 889         /* bit array of pups that need specail search */
 890         u32 special_pattern_i_pup = 0;
 891         u32 special_pattern_ii_pup = 0;
 892         u32 pup;
 893         u32 max_pup;
 894
 895         max_pup = (ecc + (1 - ecc) * dram_info->num_of_std_pups);
 896
 897         for (pup = 0; pup < max_pup; pup++) {
 898                 if (is_tx == 0) {
 899                         /* Check special pattern I */
 900                         /*
 901                          * Special pattern Low limit search - relevant only
 902                          * for Rx, win size < threshold and low limit = 0
 903                          */
 904                         if (((centralization_high_limit[pup] -
 905                               centralization_low_limit[pup]) < VALID_WIN_THRS)
 906                             && (centralization_low_limit[pup] == MIN_DELAY))
 907                                 special_pattern_i_pup |= (1 << pup);
 908
 909                         /* Check special pattern II */
 910                         /*
 911                          * Special pattern High limit search - relevant only
 912                          * for Rx, win size < threshold and high limit = 31
 913                          */
 914                         if (((centralization_high_limit[pup] -
 915                               centralization_low_limit[pup]) < VALID_WIN_THRS)
 916                             && (centralization_high_limit[pup] == MAX_DELAY))
 917                                 special_pattern_ii_pup |= (1 << pup);
 918                 }
 919         }
 920
 921         /* Run special pattern Low limit search - for relevant pup */
 922         if (special_pattern_i_pup != 0) {
 923                 DEBUG_DQS_S("DDR3 - DQS Center Calc - Entering special pattern I for Low limit search\n");
 924                 if (MV_OK !=
 925                     ddr3_special_pattern_i_search(dram_info, cs, ecc, is_tx,
 926                                               special_pattern_i_pup))
 927                         return MV_DDR3_TRAINING_ERR_DQS_LOW_LIMIT_SEARCH;
 928         }
 929
 930         /* Run special pattern High limit search - for relevant pup */
 931         if (special_pattern_ii_pup != 0) {
 932                 DEBUG_DQS_S("DDR3 - DQS Center Calc - Entering special pattern II for High limit search\n");
 933                 if (MV_OK !=
 934                     ddr3_special_pattern_ii_search(dram_info, cs, ecc, is_tx,
 935                                                    special_pattern_ii_pup))
 936                         return MV_DDR3_TRAINING_ERR_DQS_HIGH_LIMIT_SEARCH;
 937         }
 938
 939         /* Set adll to center = (General_High_limit + General_Low_limit)/2 */
 940         return ddr3_set_dqs_centralization_results(dram_info, cs, ecc, is_tx);
 941 }
 942
 943 /*
 944  * Name:     ddr3_special_pattern_i_search
 945  * Desc:     Execute special pattern low limit search.
 946  * Args:
 947  *           special_pattern_pup  The pups that need the special search
 948  * Notes:
 949  * Returns:  MV_OK if success, other error code if fail.
 950  */
 951 int ddr3_special_pattern_i_search(MV_DRAM_INFO *dram_info, u32 cs, u32 ecc,
 952                                   int is_tx, u32 special_pattern_pup)
 953 {
 954         u32 victim_dq;          /* loop index - victim DQ */
 955         u32 adll_idx;
 956         u32 pup;
 957         u32 unlock_pup;         /* bit array of the unlock pups  */
 958         u32 first_fail; /* bit array - of pups that  get first fail */
 959         u32 new_lockup_pup;     /* bit array of compare failed pups */
 960         u32 pass_pup;           /* bit array of compare pass pup */
 961         u32 sdram_offset;
 962         u32 max_pup;
 963         u32 comp_val;
 964         u32 special_res[MAX_PUP_NUM];   /* hold tmp results */
 965
 966         DEBUG_DQS_S("DDR3 - DQS - Special Pattern I Search - Starting\n");
 967
 968         max_pup = ecc + (1 - ecc) * dram_info->num_of_std_pups;
 969
 970         /* Init the temporary results to max ADLL value */
 971         for (pup = 0; pup < max_pup; pup++)
 972                 special_res[pup] = ADLL_MAX;
 973
 974         /* Run special pattern for all DQ - use the same pattern */
 975         for (victim_dq = 0; victim_dq < DQ_NUM; victim_dq++) {
 976                 unlock_pup = special_pattern_pup;
 977                 first_fail = 0;
 978
 979                 sdram_offset = cs * SDRAM_CS_SIZE + SDRAM_DQS_RX_OFFS +
 980                         LEN_KILLER_PATTERN * 4 * victim_dq;
 981
 982                 for (pup = 0; pup < max_pup; pup++) {
 983                         /* Set adll value per PUP. adll = high limit per pup */
 984                         if (IS_PUP_ACTIVE(unlock_pup, pup)) {
 985                                 /* only for pups that need special search */
 986                                 ddr3_write_pup_reg(PUP_DQS_RD, cs,
 987                                                    pup + (ecc * ECC_PUP), 0,
 988                                                    centralization_high_limit
 989                                                    [pup]);
 990                         }
 991                 }
 992
 993                 adll_idx = 0;
 994                 do {
 995                         /*
 996                          * Perform read and compare simultaneously for all
 997                          * un-locked MC use the special pattern mask
 998                          */
 999                         new_lockup_pup = 0;
1000
1001                         if (MV_OK !=
1002                             ddr3_sdram_dqs_compare(dram_info, unlock_pup,
1003                                                    &new_lockup_pup,
1004                                                    special_pattern
1005                                                    [victim_dq],
1006                                                    LEN_SPECIAL_PATTERN,
1007                                                    sdram_offset, 0,
1008                                                    0, NULL, 1))
1009                                 return MV_FAIL;
1010
1011                         DEBUG_DQS_S("DDR3 - DQS - Special I - ADLL value is: ");
1012                         DEBUG_DQS_D(adll_idx, 2);
1013                         DEBUG_DQS_S(", UnlockPup: ");
1014                         DEBUG_DQS_D(unlock_pup, 2);
1015                         DEBUG_DQS_S(", NewLockPup: ");
1016                         DEBUG_DQS_D(new_lockup_pup, 2);
1017                         DEBUG_DQS_S("\n");
1018
1019                         if (unlock_pup != new_lockup_pup)
1020                                 DEBUG_DQS_S("DDR3 - DQS - Special I - Some Pup passed!\n");
1021
1022                         /* Search for pups with passed compare & already fail */
1023                         pass_pup = first_fail & ~new_lockup_pup & unlock_pup;
1024                         first_fail |= new_lockup_pup;
1025                         unlock_pup &= ~pass_pup;
1026
1027                         /* Get pass pups */
1028                         if (pass_pup != 0) {
1029                                 for (pup = 0; pup < max_pup; pup++) {
1030                                         if (IS_PUP_ACTIVE(pass_pup, pup) ==
1031                                             1) {
1032                                                 /* If pup passed and has first fail = 1 */
1033                                                 /* keep min value of ADLL max value - current adll */
1034                                                 /* (centralization_high_limit[pup] + adll_idx) = current adll !!! */
1035                                                 comp_val =
1036                                                     (ADLL_MAX -
1037                                                      (centralization_high_limit
1038                                                       [pup] + adll_idx));
1039
1040                                                 DEBUG_DQS_C
1041                                                     ("DDR3 - DQS - Special I - Pup - ",
1042                                                      pup, 1);
1043                                                 DEBUG_DQS_C
1044                                                     (" comp_val = ",
1045                                                      comp_val, 2);
1046
1047                                                 if (comp_val <
1048                                                     special_res[pup]) {
1049                                                         special_res[pup] =
1050                                                             comp_val;
1051                                                         centralization_low_limit
1052                                                             [pup] =
1053                                                             (-1) *
1054                                                             comp_val;
1055
1056                                                         DEBUG_DQS_C
1057                                                             ("DDR3 - DQS - Special I - Pup - ",
1058                                                              pup, 1);
1059                                                         DEBUG_DQS_C
1060                                                             (" Changed Low limit to ",
1061                                                              centralization_low_limit
1062                                                              [pup], 2);
1063                                                 }
1064                                         }
1065                                 }
1066                         }
1067
1068                         /*
1069                          * Did all PUP found missing window?
1070                          * Check for each pup if adll (different for each pup)
1071                          * reach maximum if reach max value - lock the pup
1072                          * if not - increment (Move to right one phase - ADLL)
1073                          * dqs RX delay
1074                          */
1075                         adll_idx++;
1076                         for (pup = 0; pup < max_pup; pup++) {
1077                                 if (IS_PUP_ACTIVE(unlock_pup, pup) == 1) {
1078                                         /* Check only unlocked pups */
1079                                         if ((centralization_high_limit[pup] +
1080                                              adll_idx) >= ADLL_MAX) {
1081                                                 /* reach maximum - lock the pup */
1082                                                 DEBUG_DQS_C("DDR3 - DQS - Special I - reach maximum - lock pup ",
1083                                                             pup, 1);
1084                                                 unlock_pup &= ~(1 << pup);
1085                                         } else {
1086                                                 /* Didn't reach maximum - increment ADLL */
1087                                                 ddr3_write_pup_reg(PUP_DQS_RD,
1088                                                                    cs,
1089                                                                    pup +
1090                                                                    (ecc *
1091                                                                     ECC_PUP), 0,
1092                                                                    (centralization_high_limit
1093                                                                     [pup] +
1094                                                                     adll_idx));
1095                                         }
1096                                 }
1097                         }
1098                 } while (unlock_pup != 0);
1099         }
1100
1101         return MV_OK;
1102 }
1103
1104 /*
1105  * Name:     ddr3_special_pattern_ii_search
1106  * Desc:     Execute special pattern high limit search.
1107  * Args:
1108  *           special_pattern_pup  The pups that need the special search
1109  * Notes:
1110  * Returns:  MV_OK if success, other error code if fail.
1111  */
1112 int ddr3_special_pattern_ii_search(MV_DRAM_INFO *dram_info, u32 cs, u32 ecc,
1113                                    int is_tx, u32 special_pattern_pup)
1114 {
1115         u32 victim_dq;          /* loop index - victim DQ */
1116         u32 adll_idx;
1117         u32 pup;
1118         u32 unlock_pup;         /* bit array of the unlock pups  */
1119         u32 first_fail; /* bit array - of pups that  get first fail */
1120         u32 new_lockup_pup;     /* bit array of compare failed pups */
1121         u32 pass_pup;           /* bit array of compare pass pup */
1122         u32 sdram_offset;
1123         u32 max_pup;
1124         u32 comp_val;
1125         u32 special_res[MAX_PUP_NUM];   /* hold tmp results */
1126
1127         DEBUG_DQS_S("DDR3 - DQS - Special Pattern II Search - Starting\n");
1128
1129         max_pup = (ecc + (1 - ecc) * dram_info->num_of_std_pups);
1130
1131         /* init the tmporary results to max ADLL value */
1132         for (pup = 0; pup < max_pup; pup++)
1133                 special_res[pup] = ADLL_MAX;
1134
1135         sdram_offset = cs * SDRAM_CS_SIZE + SDRAM_DQS_RX_OFFS;
1136
1137         /* run special pattern for all DQ - use the same pattern */
1138         for (victim_dq = 0; victim_dq < DQ_NUM; victim_dq++) {
1139                 unlock_pup = special_pattern_pup;
1140                 first_fail = 0;
1141
1142                 for (pup = 0; pup < max_pup; pup++) {
1143                         /* Set adll value per PUP. adll = 0 */
1144                         if (IS_PUP_ACTIVE(unlock_pup, pup)) {
1145                                 /* Only for pups that need special search */
1146                                 ddr3_write_pup_reg(PUP_DQS_RD, cs,
1147                                                    pup + (ecc * ECC_PUP), 0,
1148                                                    ADLL_MIN);
1149                         }
1150                 }
1151
1152                 adll_idx = 0;
1153                 do {
1154                         /*
1155                          * Perform read and compare simultaneously for all
1156                          * un-locked MC use the special pattern mask
1157                          */
1158                         new_lockup_pup = 0;
1159
1160                         if (MV_OK != ddr3_sdram_dqs_compare(
1161                                     dram_info, unlock_pup, &new_lockup_pup,
1162                                     special_pattern[victim_dq],
1163                                     LEN_SPECIAL_PATTERN,
1164                                     sdram_offset, 0, 0, NULL, 0))
1165                                 return MV_FAIL;
1166
1167                         DEBUG_DQS_S("DDR3 - DQS - Special II - ADLL value is ");
1168                         DEBUG_DQS_D(adll_idx, 2);
1169                         DEBUG_DQS_S("unlock_pup ");
1170                         DEBUG_DQS_D(unlock_pup, 1);
1171                         DEBUG_DQS_S("new_lockup_pup ");
1172                         DEBUG_DQS_D(new_lockup_pup, 1);
1173                         DEBUG_DQS_S("\n");
1174
1175                         if (unlock_pup != new_lockup_pup) {
1176                                 DEBUG_DQS_S("DDR3 - DQS - Special II - Some Pup passed!\n");
1177                         }
1178
1179                         /* Search for pups with passed compare & already fail */
1180                         pass_pup = first_fail & ~new_lockup_pup & unlock_pup;
1181                         first_fail |= new_lockup_pup;
1182                         unlock_pup &= ~pass_pup;
1183
1184                         /* Get pass pups */
1185                         if (pass_pup != 0) {
1186                                 for (pup = 0; pup < max_pup; pup++) {
1187                                         if (IS_PUP_ACTIVE(pass_pup, pup) ==
1188                                             1) {
1189                                                 /* If pup passed and has first fail = 1 */
1190                                                 /* keep min value of ADLL max value - current adll */
1191                                                 /* (adll_idx) = current adll !!! */
1192                                                 comp_val = adll_idx;
1193
1194                                                 DEBUG_DQS_C("DDR3 - DQS - Special II - Pup - ",
1195                                                             pup, 1);
1196                                                 DEBUG_DQS_C(" comp_val = ",
1197                                                             comp_val, 1);
1198
1199                                                 if (comp_val <
1200                                                     special_res[pup]) {
1201                                                         special_res[pup] =
1202                                                             comp_val;
1203                                                         centralization_high_limit
1204                                                             [pup] =
1205                                                             ADLL_MAX +
1206                                                             comp_val;
1207
1208                                                         DEBUG_DQS_C
1209                                                             ("DDR3 - DQS - Special II - Pup - ",
1210                                                              pup, 1);
1211                                                         DEBUG_DQS_C
1212                                                             (" Changed High limit to ",
1213                                                              centralization_high_limit
1214                                                              [pup], 2);
1215                                                 }
1216                                         }
1217                                 }
1218                         }
1219
1220                         /*
1221                          * Did all PUP found missing window?
1222                          * Check for each pup if adll (different for each pup)
1223                          * reach maximum if reach max value - lock the pup
1224                          * if not - increment (Move to right one phase - ADLL)
1225                          * dqs RX delay
1226                          */
1227                         adll_idx++;
1228                         for (pup = 0; pup < max_pup; pup++) {
1229                                 if (IS_PUP_ACTIVE(unlock_pup, pup) == 1) {
1230                                         /* Check only unlocked pups */
1231                                         if ((adll_idx) >= ADLL_MAX) {
1232                                                 /* Reach maximum - lock the pup */
1233                                                 DEBUG_DQS_C("DDR3 - DQS - Special II - reach maximum - lock pup ",
1234                                                             pup, 1);
1235                                                 unlock_pup &= ~(1 << pup);
1236                                         } else {
1237                                                 /* Didn't reach maximum - increment ADLL */
1238                                                 ddr3_write_pup_reg(PUP_DQS_RD,
1239                                                                    cs,
1240                                                                    pup +
1241                                                                    (ecc *
1242                                                                     ECC_PUP), 0,
1243                                                                    (adll_idx));
1244                                         }
1245                                 }
1246                         }
1247                 } while (unlock_pup != 0);
1248         }
1249
1250         return MV_OK;
1251 }
1252
1253 /*
1254  * Name:     ddr3_set_dqs_centralization_results
1255  * Desc:     Set to HW the DQS centralization phase results.
1256  * Args:
1257  *           is_tx             Indicates whether to set Tx or RX results
1258  * Notes:
1259  * Returns:  MV_OK if success, other error code if fail.
1260  */
1261 int ddr3_set_dqs_centralization_results(MV_DRAM_INFO *dram_info, u32 cs,
1262                                         u32 ecc, int is_tx)
1263 {
1264         u32 pup, pup_num;
1265         int addl_val;
1266         u32 max_pup;
1267
1268         max_pup = (ecc + (1 - ecc) * dram_info->num_of_std_pups);
1269
1270         DEBUG_DQS_RESULTS_S("\n############ LOG LEVEL 2(Windows margins) ############\n");
1271
1272         if (is_tx) {
1273                 DEBUG_DQS_RESULTS_C("DDR3 - DQS TX - Set Dqs Centralization Results - CS: ",
1274                                     cs, 1);
1275         } else {
1276                 DEBUG_DQS_RESULTS_C("DDR3 - DQS RX - Set Dqs Centralization Results - CS: ",
1277                                     cs, 1);
1278         }
1279
1280         /* Set adll to center = (General_High_limit + General_Low_limit)/2 */
1281         DEBUG_DQS_RESULTS_S("\nDQS    LOW     HIGH     WIN-SIZE      Set\n");
1282         DEBUG_DQS_RESULTS_S("==============================================\n");
1283         for (pup = 0; pup < max_pup; pup++) {
1284                 addl_val = (centralization_high_limit[pup] +
1285                             centralization_low_limit[pup]) / 2;
1286
1287                 pup_num = pup * (1 - ecc) + ecc * ECC_PUP;
1288
1289                 DEBUG_DQS_RESULTS_D(pup_num, 1);
1290                 DEBUG_DQS_RESULTS_S("     0x");
1291                 DEBUG_DQS_RESULTS_D(centralization_low_limit[pup], 2);
1292                 DEBUG_DQS_RESULTS_S("      0x");
1293                 DEBUG_DQS_RESULTS_D(centralization_high_limit[pup], 2);
1294                 DEBUG_DQS_RESULTS_S("      0x");
1295                 DEBUG_DQS_RESULTS_D(centralization_high_limit[pup] -
1296                                     centralization_low_limit[pup], 2);
1297                 DEBUG_DQS_RESULTS_S("       0x");
1298                 DEBUG_DQS_RESULTS_D(addl_val, 2);
1299                 DEBUG_DQS_RESULTS_S("\n");
1300
1301                 if (addl_val < ADLL_MIN) {
1302                         addl_val = ADLL_MIN;
1303                         DEBUG_DQS_RESULTS_S("DDR3 - DQS - Setting ADLL value for Pup to MIN (since it was lower than 0)\n");
1304                 }
1305
1306                 if (addl_val > ADLL_MAX) {
1307                         addl_val = ADLL_MAX;
1308                         DEBUG_DQS_RESULTS_S("DDR3 - DQS - Setting ADLL value for Pup to MAX (since it was higher than 31)\n");
1309                 }
1310
1311                 if (is_tx) {
1312                         ddr3_write_pup_reg(PUP_DQS_WR, cs, pup_num, 0,
1313                                            addl_val +
1314                                            dram_info->wl_val[cs][pup_num][D]);
1315                 } else {
1316                         ddr3_write_pup_reg(PUP_DQS_RD, cs, pup_num, 0,
1317                                            addl_val);
1318                 }
1319         }
1320
1321         return MV_OK;
1322 }
1323
1324 /*
1325  * Set training patterns
1326  */
1327 int ddr3_load_dqs_patterns(MV_DRAM_INFO *dram_info)
1328 {
1329         u32 cs, cs_count, cs_tmp, victim_dq;
1330         u32 sdram_addr;
1331         u32 *pattern_ptr;
1332
1333         /* Loop for each CS */
1334         for (cs = 0; cs < MAX_CS; cs++) {
1335                 if (dram_info->cs_ena & (1 << cs)) {
1336                         cs_count = 0;
1337                         for (cs_tmp = 0; cs_tmp < cs; cs_tmp++) {
1338                                 if (dram_info->cs_ena & (1 << cs_tmp))
1339                                         cs_count++;
1340                         }
1341
1342                         /* Init killer pattern */
1343                         sdram_addr = (cs_count * (SDRAM_CS_SIZE + 1) +
1344                                       SDRAM_DQS_RX_OFFS);
1345                         for (victim_dq = 0; victim_dq < DQ_NUM; victim_dq++) {
1346                                 pattern_ptr = ddr3_dqs_choose_pattern(dram_info,
1347                                                                       victim_dq);
1348                                 if (MV_OK != ddr3_sdram_dqs_compare(
1349                                             dram_info, (u32)NULL, NULL,
1350                                             pattern_ptr, LEN_KILLER_PATTERN,
1351                                             sdram_addr + LEN_KILLER_PATTERN *
1352                                             4 * victim_dq, 1, 0, NULL,
1353                                             0))
1354                                         return MV_DDR3_TRAINING_ERR_DQS_PATTERN;
1355                         }
1356
1357                         /* Init special-killer pattern */
1358                         sdram_addr = (cs_count * (SDRAM_CS_SIZE + 1) +
1359                                       SDRAM_DQS_RX_SPECIAL_OFFS);
1360                         for (victim_dq = 0; victim_dq < DQ_NUM; victim_dq++) {
1361                                 if (MV_OK != ddr3_sdram_dqs_compare(
1362                                             dram_info, (u32)NULL, NULL,
1363                                             special_pattern[victim_dq],
1364                                             LEN_KILLER_PATTERN, sdram_addr +
1365                                             LEN_KILLER_PATTERN * 4 * victim_dq,
1366                                             1, 0, NULL, 0))
1367                                         return MV_DDR3_TRAINING_ERR_DQS_PATTERN;
1368                         }
1369                 }
1370         }
1371
1372         return MV_OK;
1373 }