drivers/crypto/hisilicon/sec2/sec_main.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /* Copyright (c) 2019 HiSilicon Limited. */
   3
   4 #include <linux/acpi.h>
   5 #include <linux/aer.h>
   6 #include <linux/bitops.h>
   7 #include <linux/debugfs.h>
   8 #include <linux/init.h>
   9 #include <linux/io.h>
  10 #include <linux/kernel.h>
  11 #include <linux/module.h>
  12 #include <linux/pci.h>
  13 #include <linux/seq_file.h>
  14 #include <linux/topology.h>
  15
  16 #include "sec.h"
  17
  18 #define SEC_VF_NUM                      63
  19 #define SEC_QUEUE_NUM_V1                4096
  20 #define SEC_QUEUE_NUM_V2                1024
  21 #define SEC_PF_PCI_DEVICE_ID            0xa255
  22 #define SEC_VF_PCI_DEVICE_ID            0xa256
  23
  24 #define SEC_XTS_MIV_ENABLE_REG          0x301384
  25 #define SEC_XTS_MIV_ENABLE_MSK          0x7FFFFFFF
  26 #define SEC_XTS_MIV_DISABLE_MSK         0xFFFFFFFF
  27 #define SEC_BD_ERR_CHK_EN1              0xfffff7fd
  28 #define SEC_BD_ERR_CHK_EN2              0xffffbfff
  29
  30 #define SEC_SQE_SIZE                    128
  31 #define SEC_SQ_SIZE                     (SEC_SQE_SIZE * QM_Q_DEPTH)
  32 #define SEC_PF_DEF_Q_NUM                64
  33 #define SEC_PF_DEF_Q_BASE               0
  34 #define SEC_CTX_Q_NUM_DEF               24
  35
  36 #define SEC_CTRL_CNT_CLR_CE             0x301120
  37 #define SEC_CTRL_CNT_CLR_CE_BIT         BIT(0)
  38 #define SEC_ENGINE_PF_CFG_OFF           0x300000
  39 #define SEC_ACC_COMMON_REG_OFF          0x1000
  40 #define SEC_CORE_INT_SOURCE             0x301010
  41 #define SEC_CORE_INT_MASK               0x301000
  42 #define SEC_CORE_INT_STATUS             0x301008
  43 #define SEC_CORE_SRAM_ECC_ERR_INFO      0x301C14
  44 #define SEC_ECC_NUM(err)                        (((err) >> 16) & 0xFF)
  45 #define SEC_ECC_ADDR(err)                       ((err) >> 0)
  46 #define SEC_CORE_INT_DISABLE            0x0
  47 #define SEC_CORE_INT_ENABLE             0x1ff
  48
  49 #define SEC_RAS_CE_REG                  0x50
  50 #define SEC_RAS_FE_REG                  0x54
  51 #define SEC_RAS_NFE_REG                 0x58
  52 #define SEC_RAS_CE_ENB_MSK              0x88
  53 #define SEC_RAS_FE_ENB_MSK              0x0
  54 #define SEC_RAS_NFE_ENB_MSK             0x177
  55 #define SEC_RAS_DISABLE                 0x0
  56 #define SEC_MEM_START_INIT_REG          0x0100
  57 #define SEC_MEM_INIT_DONE_REG           0x0104
  58 #define SEC_QM_ABNORMAL_INT_MASK        0x100004
  59
  60 #define SEC_CONTROL_REG                 0x0200
  61 #define SEC_TRNG_EN_SHIFT               8
  62 #define SEC_CLK_GATE_ENABLE             BIT(3)
  63 #define SEC_CLK_GATE_DISABLE            (~BIT(3))
  64 #define SEC_AXI_SHUTDOWN_ENABLE BIT(12)
  65 #define SEC_AXI_SHUTDOWN_DISABLE        0xFFFFEFFF
  66
  67 #define SEC_INTERFACE_USER_CTRL0_REG    0x0220
  68 #define SEC_INTERFACE_USER_CTRL1_REG    0x0224
  69 #define SEC_BD_ERR_CHK_EN_REG1          0x0384
  70 #define SEC_BD_ERR_CHK_EN_REG2          0x038c
  71
  72 #define SEC_USER0_SMMU_NORMAL           (BIT(23) | BIT(15))
  73 #define SEC_USER1_SMMU_NORMAL           (BIT(31) | BIT(23) | BIT(15) | BIT(7))
  74 #define SEC_CORE_INT_STATUS_M_ECC       BIT(2)
  75
  76 #define SEC_DELAY_10_US                 10
  77 #define SEC_POLL_TIMEOUT_US             1000
  78 #define SEC_VF_CNT_MASK                 0xffffffc0
  79 #define SEC_DBGFS_VAL_MAX_LEN           20
  80
  81 #define SEC_ADDR(qm, offset) ((qm)->io_base + (offset) + \
  82                              SEC_ENGINE_PF_CFG_OFF + SEC_ACC_COMMON_REG_OFF)
  83
  84 struct sec_hw_error {
  85         u32 int_msk;
  86         const char *msg;
  87 };
  88
  89 static const char sec_name[] = "hisi_sec2";
  90 static struct dentry *sec_debugfs_root;
  91 static LIST_HEAD(sec_list);
  92 static DEFINE_MUTEX(sec_list_lock);
  93
  94 static const struct sec_hw_error sec_hw_errors[] = {
  95         {.int_msk = BIT(0), .msg = "sec_axi_rresp_err_rint"},
  96         {.int_msk = BIT(1), .msg = "sec_axi_bresp_err_rint"},
  97         {.int_msk = BIT(2), .msg = "sec_ecc_2bit_err_rint"},
  98         {.int_msk = BIT(3), .msg = "sec_ecc_1bit_err_rint"},
  99         {.int_msk = BIT(4), .msg = "sec_req_trng_timeout_rint"},
 100         {.int_msk = BIT(5), .msg = "sec_fsm_hbeat_rint"},
 101         {.int_msk = BIT(6), .msg = "sec_channel_req_rng_timeout_rint"},
 102         {.int_msk = BIT(7), .msg = "sec_bd_err_rint"},
 103         {.int_msk = BIT(8), .msg = "sec_chain_buff_err_rint"},
 104         { /* sentinel */ }
 105 };
 106
 107 struct sec_dev *sec_find_device(int node)
 108 {
 109 #define SEC_NUMA_MAX_DISTANCE   100
 110         int min_distance = SEC_NUMA_MAX_DISTANCE;
 111         int dev_node = 0, free_qp_num = 0;
 112         struct sec_dev *sec, *ret = NULL;
 113         struct hisi_qm *qm;
 114         struct device *dev;
 115
 116         mutex_lock(&sec_list_lock);
 117         list_for_each_entry(sec, &sec_list, list) {
 118                 qm = &sec->qm;
 119                 dev = &qm->pdev->dev;
 120 #ifdef CONFIG_NUMA
 121                 dev_node = dev->numa_node;
 122                 if (dev_node < 0)
 123                         dev_node = 0;
 124 #endif
 125                 if (node_distance(dev_node, node) < min_distance) {
 126                         free_qp_num = hisi_qm_get_free_qp_num(qm);
 127                         if (free_qp_num >= sec->ctx_q_num) {
 128                                 ret = sec;
 129                                 min_distance = node_distance(dev_node, node);
 130                         }
 131                 }
 132         }
 133         mutex_unlock(&sec_list_lock);
 134
 135         return ret;
 136 }
 137
 138 static const char * const sec_dbg_file_name[] = {
 139         [SEC_CURRENT_QM] = "current_qm",
 140         [SEC_CLEAR_ENABLE] = "clear_enable",
 141 };
 142
 143 static struct debugfs_reg32 sec_dfx_regs[] = {
 144         {"SEC_PF_ABNORMAL_INT_SOURCE    ",  0x301010},
 145         {"SEC_SAA_EN                    ",  0x301270},
 146         {"SEC_BD_LATENCY_MIN            ",  0x301600},
 147         {"SEC_BD_LATENCY_MAX            ",  0x301608},
 148         {"SEC_BD_LATENCY_AVG            ",  0x30160C},
 149         {"SEC_BD_NUM_IN_SAA0            ",  0x301670},
 150         {"SEC_BD_NUM_IN_SAA1            ",  0x301674},
 151         {"SEC_BD_NUM_IN_SEC             ",  0x301680},
 152         {"SEC_ECC_1BIT_CNT              ",  0x301C00},
 153         {"SEC_ECC_1BIT_INFO             ",  0x301C04},
 154         {"SEC_ECC_2BIT_CNT              ",  0x301C10},
 155         {"SEC_ECC_2BIT_INFO             ",  0x301C14},
 156         {"SEC_BD_SAA0                   ",  0x301C20},
 157         {"SEC_BD_SAA1                   ",  0x301C24},
 158         {"SEC_BD_SAA2                   ",  0x301C28},
 159         {"SEC_BD_SAA3                   ",  0x301C2C},
 160         {"SEC_BD_SAA4                   ",  0x301C30},
 161         {"SEC_BD_SAA5                   ",  0x301C34},
 162         {"SEC_BD_SAA6                   ",  0x301C38},
 163         {"SEC_BD_SAA7                   ",  0x301C3C},
 164         {"SEC_BD_SAA8                   ",  0x301C40},
 165 };
 166
 167 static int sec_pf_q_num_set(const char *val, const struct kernel_param *kp)
 168 {
 169         struct pci_dev *pdev;
 170         u32 n, q_num;
 171         u8 rev_id;
 172         int ret;
 173
 174         if (!val)
 175                 return -EINVAL;
 176
 177         pdev = pci_get_device(PCI_VENDOR_ID_HUAWEI,
 178                               SEC_PF_PCI_DEVICE_ID, NULL);
 179         if (!pdev) {
 180                 q_num = min_t(u32, SEC_QUEUE_NUM_V1, SEC_QUEUE_NUM_V2);
 181                 pr_info("No device, suppose queue number is %d!\n", q_num);
 182         } else {
 183                 rev_id = pdev->revision;
 184
 185                 switch (rev_id) {
 186                 case QM_HW_V1:
 187                         q_num = SEC_QUEUE_NUM_V1;
 188                         break;
 189                 case QM_HW_V2:
 190                         q_num = SEC_QUEUE_NUM_V2;
 191                         break;
 192                 default:
 193                         return -EINVAL;
 194                 }
 195         }
 196
 197         ret = kstrtou32(val, 10, &n);
 198         if (ret || !n || n > q_num)
 199                 return -EINVAL;
 200
 201         return param_set_int(val, kp);
 202 }
 203
 204 static const struct kernel_param_ops sec_pf_q_num_ops = {
 205         .set = sec_pf_q_num_set,
 206         .get = param_get_int,
 207 };
 208 static u32 pf_q_num = SEC_PF_DEF_Q_NUM;
 209 module_param_cb(pf_q_num, &sec_pf_q_num_ops, &pf_q_num, 0444);
 210 MODULE_PARM_DESC(pf_q_num, "Number of queues in PF(v1 0-4096, v2 0-1024)");
 211
 212 static int sec_ctx_q_num_set(const char *val, const struct kernel_param *kp)
 213 {
 214         u32 ctx_q_num;
 215         int ret;
 216
 217         if (!val)
 218                 return -EINVAL;
 219
 220         ret = kstrtou32(val, 10, &ctx_q_num);
 221         if (ret)
 222                 return -EINVAL;
 223
 224         if (!ctx_q_num || ctx_q_num > QM_Q_DEPTH || ctx_q_num & 0x1) {
 225                 pr_err("ctx queue num[%u] is invalid!\n", ctx_q_num);
 226                 return -EINVAL;
 227         }
 228
 229         return param_set_int(val, kp);
 230 }
 231
 232 static const struct kernel_param_ops sec_ctx_q_num_ops = {
 233         .set = sec_ctx_q_num_set,
 234         .get = param_get_int,
 235 };
 236 static u32 ctx_q_num = SEC_CTX_Q_NUM_DEF;
 237 module_param_cb(ctx_q_num, &sec_ctx_q_num_ops, &ctx_q_num, 0444);
 238 MODULE_PARM_DESC(ctx_q_num, "Number of queue in ctx (2, 4, 6, ..., 1024)");
 239
 240 static const struct pci_device_id sec_dev_ids[] = {
 241         { PCI_DEVICE(PCI_VENDOR_ID_HUAWEI, SEC_PF_PCI_DEVICE_ID) },
 242         { PCI_DEVICE(PCI_VENDOR_ID_HUAWEI, SEC_VF_PCI_DEVICE_ID) },
 243         { 0, }
 244 };
 245 MODULE_DEVICE_TABLE(pci, sec_dev_ids);
 246
 247 static inline void sec_add_to_list(struct sec_dev *sec)
 248 {
 249         mutex_lock(&sec_list_lock);
 250         list_add_tail(&sec->list, &sec_list);
 251         mutex_unlock(&sec_list_lock);
 252 }
 253
 254 static inline void sec_remove_from_list(struct sec_dev *sec)
 255 {
 256         mutex_lock(&sec_list_lock);
 257         list_del(&sec->list);
 258         mutex_unlock(&sec_list_lock);
 259 }
 260
 261 static u8 sec_get_endian(struct sec_dev *sec)
 262 {
 263         struct hisi_qm *qm = &sec->qm;
 264         u32 reg;
 265
 266         /*
 267          * As for VF, it is a wrong way to get endian setting by
 268          * reading a register of the engine
 269          */
 270         if (qm->pdev->is_virtfn) {
 271                 dev_err_ratelimited(&qm->pdev->dev,
 272                                     "cannot access a register in VF!\n");
 273                 return SEC_LE;
 274         }
 275         reg = readl_relaxed(qm->io_base + SEC_ENGINE_PF_CFG_OFF +
 276                             SEC_ACC_COMMON_REG_OFF + SEC_CONTROL_REG);
 277
 278         /* BD little endian mode */
 279         if (!(reg & BIT(0)))
 280                 return SEC_LE;
 281
 282         /* BD 32-bits big endian mode */
 283         else if (!(reg & BIT(1)))
 284                 return SEC_32BE;
 285
 286         /* BD 64-bits big endian mode */
 287         else
 288                 return SEC_64BE;
 289 }
 290
 291 static int sec_engine_init(struct sec_dev *sec)
 292 {
 293         struct hisi_qm *qm = &sec->qm;
 294         int ret;
 295         u32 reg;
 296
 297         /* disable clock gate control */
 298         reg = readl_relaxed(SEC_ADDR(qm, SEC_CONTROL_REG));
 299         reg &= SEC_CLK_GATE_DISABLE;
 300         writel_relaxed(reg, SEC_ADDR(qm, SEC_CONTROL_REG));
 301
 302         writel_relaxed(0x1, SEC_ADDR(qm, SEC_MEM_START_INIT_REG));
 303
 304         ret = readl_relaxed_poll_timeout(SEC_ADDR(qm, SEC_MEM_INIT_DONE_REG),
 305                                          reg, reg & 0x1, SEC_DELAY_10_US,
 306                                          SEC_POLL_TIMEOUT_US);
 307         if (ret) {
 308                 dev_err(&qm->pdev->dev, "fail to init sec mem\n");
 309                 return ret;
 310         }
 311
 312         reg = readl_relaxed(SEC_ADDR(qm, SEC_CONTROL_REG));
 313         reg |= (0x1 << SEC_TRNG_EN_SHIFT);
 314         writel_relaxed(reg, SEC_ADDR(qm, SEC_CONTROL_REG));
 315
 316         reg = readl_relaxed(SEC_ADDR(qm, SEC_INTERFACE_USER_CTRL0_REG));
 317         reg |= SEC_USER0_SMMU_NORMAL;
 318         writel_relaxed(reg, SEC_ADDR(qm, SEC_INTERFACE_USER_CTRL0_REG));
 319
 320         reg = readl_relaxed(SEC_ADDR(qm, SEC_INTERFACE_USER_CTRL1_REG));
 321         reg |= SEC_USER1_SMMU_NORMAL;
 322         writel_relaxed(reg, SEC_ADDR(qm, SEC_INTERFACE_USER_CTRL1_REG));
 323
 324         writel_relaxed(SEC_BD_ERR_CHK_EN1,
 325                        SEC_ADDR(qm, SEC_BD_ERR_CHK_EN_REG1));
 326         writel_relaxed(SEC_BD_ERR_CHK_EN2,
 327                        SEC_ADDR(qm, SEC_BD_ERR_CHK_EN_REG2));
 328
 329         /* enable clock gate control */
 330         reg = readl_relaxed(SEC_ADDR(qm, SEC_CONTROL_REG));
 331         reg |= SEC_CLK_GATE_ENABLE;
 332         writel_relaxed(reg, SEC_ADDR(qm, SEC_CONTROL_REG));
 333
 334         /* config endian */
 335         reg = readl_relaxed(SEC_ADDR(qm, SEC_CONTROL_REG));
 336         reg |= sec_get_endian(sec);
 337         writel_relaxed(reg, SEC_ADDR(qm, SEC_CONTROL_REG));
 338
 339         /* Enable sm4 xts mode multiple iv */
 340         writel_relaxed(SEC_XTS_MIV_ENABLE_MSK,
 341                        qm->io_base + SEC_XTS_MIV_ENABLE_REG);
 342
 343         return 0;
 344 }
 345
 346 static int sec_set_user_domain_and_cache(struct sec_dev *sec)
 347 {
 348         struct hisi_qm *qm = &sec->qm;
 349
 350         /* qm user domain */
 351         writel(AXUSER_BASE, qm->io_base + QM_ARUSER_M_CFG_1);
 352         writel(ARUSER_M_CFG_ENABLE, qm->io_base + QM_ARUSER_M_CFG_ENABLE);
 353         writel(AXUSER_BASE, qm->io_base + QM_AWUSER_M_CFG_1);
 354         writel(AWUSER_M_CFG_ENABLE, qm->io_base + QM_AWUSER_M_CFG_ENABLE);
 355         writel(WUSER_M_CFG_ENABLE, qm->io_base + QM_WUSER_M_CFG_ENABLE);
 356
 357         /* qm cache */
 358         writel(AXI_M_CFG, qm->io_base + QM_AXI_M_CFG);
 359         writel(AXI_M_CFG_ENABLE, qm->io_base + QM_AXI_M_CFG_ENABLE);
 360
 361         /* disable FLR triggered by BME(bus master enable) */
 362         writel(PEH_AXUSER_CFG, qm->io_base + QM_PEH_AXUSER_CFG);
 363         writel(PEH_AXUSER_CFG_ENABLE, qm->io_base + QM_PEH_AXUSER_CFG_ENABLE);
 364
 365         /* enable sqc,cqc writeback */
 366         writel(SQC_CACHE_ENABLE | CQC_CACHE_ENABLE | SQC_CACHE_WB_ENABLE |
 367                CQC_CACHE_WB_ENABLE | FIELD_PREP(SQC_CACHE_WB_THRD, 1) |
 368                FIELD_PREP(CQC_CACHE_WB_THRD, 1), qm->io_base + QM_CACHE_CTL);
 369
 370         return sec_engine_init(sec);
 371 }
 372
 373 /* sec_debug_regs_clear() - clear the sec debug regs */
 374 static void sec_debug_regs_clear(struct hisi_qm *qm)
 375 {
 376         /* clear current_qm */
 377         writel(0x0, qm->io_base + QM_DFX_MB_CNT_VF);
 378         writel(0x0, qm->io_base + QM_DFX_DB_CNT_VF);
 379
 380         /* clear rdclr_en */
 381         writel(0x0, qm->io_base + SEC_CTRL_CNT_CLR_CE);
 382
 383         hisi_qm_debug_regs_clear(qm);
 384 }
 385
 386 static void sec_hw_error_enable(struct sec_dev *sec)
 387 {
 388         struct hisi_qm *qm = &sec->qm;
 389         u32 val;
 390
 391         if (qm->ver == QM_HW_V1) {
 392                 writel(SEC_CORE_INT_DISABLE, qm->io_base + SEC_CORE_INT_MASK);
 393                 dev_info(&qm->pdev->dev, "V1 not support hw error handle\n");
 394                 return;
 395         }
 396
 397         val = readl(qm->io_base + SEC_CONTROL_REG);
 398
 399         /* clear SEC hw error source if having */
 400         writel(SEC_CORE_INT_DISABLE, qm->io_base + SEC_CORE_INT_SOURCE);
 401
 402         /* enable SEC hw error interrupts */
 403         writel(SEC_CORE_INT_ENABLE, qm->io_base + SEC_CORE_INT_MASK);
 404
 405         /* enable RAS int */
 406         writel(SEC_RAS_CE_ENB_MSK, qm->io_base + SEC_RAS_CE_REG);
 407         writel(SEC_RAS_FE_ENB_MSK, qm->io_base + SEC_RAS_FE_REG);
 408         writel(SEC_RAS_NFE_ENB_MSK, qm->io_base + SEC_RAS_NFE_REG);
 409
 410         /* enable SEC block master OOO when m-bit error occur */
 411         val = val | SEC_AXI_SHUTDOWN_ENABLE;
 412
 413         writel(val, qm->io_base + SEC_CONTROL_REG);
 414 }
 415
 416 static void sec_hw_error_disable(struct sec_dev *sec)
 417 {
 418         struct hisi_qm *qm = &sec->qm;
 419         u32 val;
 420
 421         val = readl(qm->io_base + SEC_CONTROL_REG);
 422
 423         /* disable RAS int */
 424         writel(SEC_RAS_DISABLE, qm->io_base + SEC_RAS_CE_REG);
 425         writel(SEC_RAS_DISABLE, qm->io_base + SEC_RAS_FE_REG);
 426         writel(SEC_RAS_DISABLE, qm->io_base + SEC_RAS_NFE_REG);
 427
 428         /* disable SEC hw error interrupts */
 429         writel(SEC_CORE_INT_DISABLE, qm->io_base + SEC_CORE_INT_MASK);
 430
 431         /* disable SEC block master OOO when m-bit error occur */
 432         val = val & SEC_AXI_SHUTDOWN_DISABLE;
 433
 434         writel(val, qm->io_base + SEC_CONTROL_REG);
 435 }
 436
 437 static void sec_hw_error_init(struct sec_dev *sec)
 438 {
 439         if (sec->qm.fun_type == QM_HW_VF)
 440                 return;
 441
 442         hisi_qm_hw_error_init(&sec->qm, QM_BASE_CE,
 443                               QM_BASE_NFE | QM_ACC_DO_TASK_TIMEOUT
 444                               | QM_ACC_WB_NOT_READY_TIMEOUT, 0,
 445                               QM_DB_RANDOM_INVALID);
 446         sec_hw_error_enable(sec);
 447 }
 448
 449 static void sec_hw_error_uninit(struct sec_dev *sec)
 450 {
 451         if (sec->qm.fun_type == QM_HW_VF)
 452                 return;
 453
 454         sec_hw_error_disable(sec);
 455         writel(GENMASK(12, 0), sec->qm.io_base + SEC_QM_ABNORMAL_INT_MASK);
 456 }
 457
 458 static u32 sec_current_qm_read(struct sec_debug_file *file)
 459 {
 460         struct hisi_qm *qm = file->qm;
 461
 462         return readl(qm->io_base + QM_DFX_MB_CNT_VF);
 463 }
 464
 465 static int sec_current_qm_write(struct sec_debug_file *file, u32 val)
 466 {
 467         struct hisi_qm *qm = file->qm;
 468         struct sec_dev *sec = container_of(qm, struct sec_dev, qm);
 469         u32 vfq_num;
 470         u32 tmp;
 471
 472         if (val > sec->num_vfs)
 473                 return -EINVAL;
 474
 475         /* According PF or VF Dev ID to calculation curr_qm_qp_num and store */
 476         if (!val) {
 477                 qm->debug.curr_qm_qp_num = qm->qp_num;
 478         } else {
 479                 vfq_num = (qm->ctrl_qp_num - qm->qp_num) / sec->num_vfs;
 480
 481                 if (val == sec->num_vfs)
 482                         qm->debug.curr_qm_qp_num =
 483                                 qm->ctrl_qp_num - qm->qp_num -
 484                                 (sec->num_vfs - 1) * vfq_num;
 485                 else
 486                         qm->debug.curr_qm_qp_num = vfq_num;
 487         }
 488
 489         writel(val, qm->io_base + QM_DFX_MB_CNT_VF);
 490         writel(val, qm->io_base + QM_DFX_DB_CNT_VF);
 491
 492         tmp = val |
 493               (readl(qm->io_base + QM_DFX_SQE_CNT_VF_SQN) & CURRENT_Q_MASK);
 494         writel(tmp, qm->io_base + QM_DFX_SQE_CNT_VF_SQN);
 495
 496         tmp = val |
 497               (readl(qm->io_base + QM_DFX_CQE_CNT_VF_CQN) & CURRENT_Q_MASK);
 498         writel(tmp, qm->io_base + QM_DFX_CQE_CNT_VF_CQN);
 499
 500         return 0;
 501 }
 502
 503 static u32 sec_clear_enable_read(struct sec_debug_file *file)
 504 {
 505         struct hisi_qm *qm = file->qm;
 506
 507         return readl(qm->io_base + SEC_CTRL_CNT_CLR_CE) &
 508                         SEC_CTRL_CNT_CLR_CE_BIT;
 509 }
 510
 511 static int sec_clear_enable_write(struct sec_debug_file *file, u32 val)
 512 {
 513         struct hisi_qm *qm = file->qm;
 514         u32 tmp;
 515
 516         if (val != 1 && val)
 517                 return -EINVAL;
 518
 519         tmp = (readl(qm->io_base + SEC_CTRL_CNT_CLR_CE) &
 520                ~SEC_CTRL_CNT_CLR_CE_BIT) | val;
 521         writel(tmp, qm->io_base + SEC_CTRL_CNT_CLR_CE);
 522
 523         return 0;
 524 }
 525
 526 static ssize_t sec_debug_read(struct file *filp, char __user *buf,
 527                                size_t count, loff_t *pos)
 528 {
 529         struct sec_debug_file *file = filp->private_data;
 530         char tbuf[SEC_DBGFS_VAL_MAX_LEN];
 531         u32 val;
 532         int ret;
 533
 534         spin_lock_irq(&file->lock);
 535
 536         switch (file->index) {
 537         case SEC_CURRENT_QM:
 538                 val = sec_current_qm_read(file);
 539                 break;
 540         case SEC_CLEAR_ENABLE:
 541                 val = sec_clear_enable_read(file);
 542                 break;
 543         default:
 544                 spin_unlock_irq(&file->lock);
 545                 return -EINVAL;
 546         }
 547
 548         spin_unlock_irq(&file->lock);
 549         ret = snprintf(tbuf, SEC_DBGFS_VAL_MAX_LEN, "%u\n", val);
 550
 551         return simple_read_from_buffer(buf, count, pos, tbuf, ret);
 552 }
 553
 554 static ssize_t sec_debug_write(struct file *filp, const char __user *buf,
 555                                size_t count, loff_t *pos)
 556 {
 557         struct sec_debug_file *file = filp->private_data;
 558         char tbuf[SEC_DBGFS_VAL_MAX_LEN];
 559         unsigned long val;
 560         int len, ret;
 561
 562         if (*pos != 0)
 563                 return 0;
 564
 565         if (count >= SEC_DBGFS_VAL_MAX_LEN)
 566                 return -ENOSPC;
 567
 568         len = simple_write_to_buffer(tbuf, SEC_DBGFS_VAL_MAX_LEN - 1,
 569                                      pos, buf, count);
 570         if (len < 0)
 571                 return len;
 572
 573         tbuf[len] = '\0';
 574         if (kstrtoul(tbuf, 0, &val))
 575                 return -EFAULT;
 576
 577         spin_lock_irq(&file->lock);
 578
 579         switch (file->index) {
 580         case SEC_CURRENT_QM:
 581                 ret = sec_current_qm_write(file, val);
 582                 if (ret)
 583                         goto err_input;
 584                 break;
 585         case SEC_CLEAR_ENABLE:
 586                 ret = sec_clear_enable_write(file, val);
 587                 if (ret)
 588                         goto err_input;
 589                 break;
 590         default:
 591                 ret = -EINVAL;
 592                 goto err_input;
 593         }
 594
 595         spin_unlock_irq(&file->lock);
 596
 597         return count;
 598
 599  err_input:
 600         spin_unlock_irq(&file->lock);
 601         return ret;
 602 }
 603
 604 static const struct file_operations sec_dbg_fops = {
 605         .owner = THIS_MODULE,
 606         .open = simple_open,
 607         .read = sec_debug_read,
 608         .write = sec_debug_write,
 609 };
 610
 611 static int debugfs_atomic64_t_get(void *data, u64 *val)
 612 {
 613         *val = atomic64_read((atomic64_t *)data);
 614         return 0;
 615 }
 616 DEFINE_DEBUGFS_ATTRIBUTE(fops_atomic64_t_ro, debugfs_atomic64_t_get, NULL,
 617                         "%lld\n");
 618
 619 static int sec_core_debug_init(struct sec_dev *sec)
 620 {
 621         struct hisi_qm *qm = &sec->qm;
 622         struct device *dev = &qm->pdev->dev;
 623         struct sec_dfx *dfx = &sec->debug.dfx;
 624         struct debugfs_regset32 *regset;
 625         struct dentry *tmp_d;
 626
 627         tmp_d = debugfs_create_dir("sec_dfx", sec->qm.debug.debug_root);
 628
 629         regset = devm_kzalloc(dev, sizeof(*regset), GFP_KERNEL);
 630         if (!regset)
 631                 return -ENOENT;
 632
 633         regset->regs = sec_dfx_regs;
 634         regset->nregs = ARRAY_SIZE(sec_dfx_regs);
 635         regset->base = qm->io_base;
 636
 637         debugfs_create_regset32("regs", 0444, tmp_d, regset);
 638
 639         debugfs_create_file("send_cnt", 0444, tmp_d, &dfx->send_cnt,
 640                             &fops_atomic64_t_ro);
 641
 642         debugfs_create_file("recv_cnt", 0444, tmp_d, &dfx->recv_cnt,
 643                             &fops_atomic64_t_ro);
 644
 645         return 0;
 646 }
 647
 648 static int sec_debug_init(struct sec_dev *sec)
 649 {
 650         int i;
 651
 652         for (i = SEC_CURRENT_QM; i < SEC_DEBUG_FILE_NUM; i++) {
 653                 spin_lock_init(&sec->debug.files[i].lock);
 654                 sec->debug.files[i].index = i;
 655                 sec->debug.files[i].qm = &sec->qm;
 656
 657                 debugfs_create_file(sec_dbg_file_name[i], 0600,
 658                                     sec->qm.debug.debug_root,
 659                                     sec->debug.files + i,
 660                                     &sec_dbg_fops);
 661         }
 662
 663         return sec_core_debug_init(sec);
 664 }
 665
 666 static int sec_debugfs_init(struct sec_dev *sec)
 667 {
 668         struct hisi_qm *qm = &sec->qm;
 669         struct device *dev = &qm->pdev->dev;
 670         int ret;
 671
 672         qm->debug.debug_root = debugfs_create_dir(dev_name(dev),
 673                                                   sec_debugfs_root);
 674         ret = hisi_qm_debug_init(qm);
 675         if (ret)
 676                 goto failed_to_create;
 677
 678         if (qm->pdev->device == SEC_PF_PCI_DEVICE_ID) {
 679                 ret = sec_debug_init(sec);
 680                 if (ret)
 681                         goto failed_to_create;
 682         }
 683
 684         return 0;
 685
 686 failed_to_create:
 687         debugfs_remove_recursive(sec_debugfs_root);
 688
 689         return ret;
 690 }
 691
 692 static void sec_debugfs_exit(struct sec_dev *sec)
 693 {
 694         debugfs_remove_recursive(sec->qm.debug.debug_root);
 695 }
 696
 697 static int sec_pf_probe_init(struct sec_dev *sec)
 698 {
 699         struct hisi_qm *qm = &sec->qm;
 700         int ret;
 701
 702         switch (qm->ver) {
 703         case QM_HW_V1:
 704                 qm->ctrl_qp_num = SEC_QUEUE_NUM_V1;
 705                 break;
 706
 707         case QM_HW_V2:
 708                 qm->ctrl_qp_num = SEC_QUEUE_NUM_V2;
 709                 break;
 710
 711         default:
 712                 return -EINVAL;
 713         }
 714
 715         ret = sec_set_user_domain_and_cache(sec);
 716         if (ret)
 717                 return ret;
 718
 719         sec_hw_error_init(sec);
 720         sec_debug_regs_clear(qm);
 721
 722         return 0;
 723 }
 724
 725 static int sec_qm_init(struct hisi_qm *qm, struct pci_dev *pdev)
 726 {
 727         enum qm_hw_ver rev_id;
 728
 729         rev_id = hisi_qm_get_hw_version(pdev);
 730         if (rev_id == QM_HW_UNKNOWN)
 731                 return -ENODEV;
 732
 733         qm->pdev = pdev;
 734         qm->ver = rev_id;
 735
 736         qm->sqe_size = SEC_SQE_SIZE;
 737         qm->dev_name = sec_name;
 738         qm->fun_type = (pdev->device == SEC_PF_PCI_DEVICE_ID) ?
 739                         QM_HW_PF : QM_HW_VF;
 740         qm->use_dma_api = true;
 741
 742         return hisi_qm_init(qm);
 743 }
 744
 745 static void sec_qm_uninit(struct hisi_qm *qm)
 746 {
 747         hisi_qm_uninit(qm);
 748 }
 749
 750 static int sec_probe_init(struct hisi_qm *qm, struct sec_dev *sec)
 751 {
 752         if (qm->fun_type == QM_HW_PF) {
 753                 qm->qp_base = SEC_PF_DEF_Q_BASE;
 754                 qm->qp_num = pf_q_num;
 755                 qm->debug.curr_qm_qp_num = pf_q_num;
 756
 757                 return sec_pf_probe_init(sec);
 758         } else if (qm->fun_type == QM_HW_VF) {
 759                 /*
 760                  * have no way to get qm configure in VM in v1 hardware,
 761                  * so currently force PF to uses SEC_PF_DEF_Q_NUM, and force
 762                  * to trigger only one VF in v1 hardware.
 763                  * v2 hardware has no such problem.
 764                  */
 765                 if (qm->ver == QM_HW_V1) {
 766                         qm->qp_base = SEC_PF_DEF_Q_NUM;
 767                         qm->qp_num = SEC_QUEUE_NUM_V1 - SEC_PF_DEF_Q_NUM;
 768                 } else if (qm->ver == QM_HW_V2) {
 769                         /* v2 starts to support get vft by mailbox */
 770                         return hisi_qm_get_vft(qm, &qm->qp_base, &qm->qp_num);
 771                 }
 772         } else {
 773                 return -ENODEV;
 774         }
 775
 776         return 0;
 777 }
 778
 779 static void sec_probe_uninit(struct sec_dev *sec)
 780 {
 781         sec_hw_error_uninit(sec);
 782 }
 783
 784 static int sec_probe(struct pci_dev *pdev, const struct pci_device_id *id)
 785 {
 786         struct sec_dev *sec;
 787         struct hisi_qm *qm;
 788         int ret;
 789
 790         sec = devm_kzalloc(&pdev->dev, sizeof(*sec), GFP_KERNEL);
 791         if (!sec)
 792                 return -ENOMEM;
 793
 794         pci_set_drvdata(pdev, sec);
 795
 796         sec->ctx_q_num = ctx_q_num;
 797
 798         qm = &sec->qm;
 799
 800         ret = sec_qm_init(qm, pdev);
 801         if (ret) {
 802                 pci_err(pdev, "Failed to pre init qm!\n");
 803                 return ret;
 804         }
 805
 806         ret = sec_probe_init(qm, sec);
 807         if (ret) {
 808                 pci_err(pdev, "Failed to probe!\n");
 809                 goto err_qm_uninit;
 810         }
 811
 812         ret = hisi_qm_start(qm);
 813         if (ret) {
 814                 pci_err(pdev, "Failed to start sec qm!\n");
 815                 goto err_probe_uninit;
 816         }
 817
 818         ret = sec_debugfs_init(sec);
 819         if (ret)
 820                 pci_warn(pdev, "Failed to init debugfs!\n");
 821
 822         sec_add_to_list(sec);
 823
 824         ret = sec_register_to_crypto();
 825         if (ret < 0) {
 826                 pr_err("Failed to register driver to crypto.\n");
 827                 goto err_remove_from_list;
 828         }
 829
 830         return 0;
 831
 832 err_remove_from_list:
 833         sec_remove_from_list(sec);
 834         sec_debugfs_exit(sec);
 835         hisi_qm_stop(qm);
 836
 837 err_probe_uninit:
 838         sec_probe_uninit(sec);
 839
 840 err_qm_uninit:
 841         sec_qm_uninit(qm);
 842
 843         return ret;
 844 }
 845
 846 /* now we only support equal assignment */
 847 static int sec_vf_q_assign(struct sec_dev *sec, u32 num_vfs)
 848 {
 849         struct hisi_qm *qm = &sec->qm;
 850         u32 qp_num = qm->qp_num;
 851         u32 q_base = qp_num;
 852         u32 q_num, remain_q_num;
 853         int i, j, ret;
 854
 855         if (!num_vfs)
 856                 return -EINVAL;
 857
 858         remain_q_num = qm->ctrl_qp_num - qp_num;
 859         q_num = remain_q_num / num_vfs;
 860
 861         for (i = 1; i <= num_vfs; i++) {
 862                 if (i == num_vfs)
 863                         q_num += remain_q_num % num_vfs;
 864                 ret = hisi_qm_set_vft(qm, i, q_base, q_num);
 865                 if (ret) {
 866                         for (j = i; j > 0; j--)
 867                                 hisi_qm_set_vft(qm, j, 0, 0);
 868                         return ret;
 869                 }
 870                 q_base += q_num;
 871         }
 872
 873         return 0;
 874 }
 875
 876 static int sec_clear_vft_config(struct sec_dev *sec)
 877 {
 878         struct hisi_qm *qm = &sec->qm;
 879         u32 num_vfs = sec->num_vfs;
 880         int ret;
 881         u32 i;
 882
 883         for (i = 1; i <= num_vfs; i++) {
 884                 ret = hisi_qm_set_vft(qm, i, 0, 0);
 885                 if (ret)
 886                         return ret;
 887         }
 888
 889         sec->num_vfs = 0;
 890
 891         return 0;
 892 }
 893
 894 static int sec_sriov_enable(struct pci_dev *pdev, int max_vfs)
 895 {
 896         struct sec_dev *sec = pci_get_drvdata(pdev);
 897         int pre_existing_vfs, ret;
 898         u32 num_vfs;
 899
 900         pre_existing_vfs = pci_num_vf(pdev);
 901
 902         if (pre_existing_vfs) {
 903                 pci_err(pdev, "Can't enable VF. Please disable at first!\n");
 904                 return 0;
 905         }
 906
 907         num_vfs = min_t(u32, max_vfs, SEC_VF_NUM);
 908
 909         ret = sec_vf_q_assign(sec, num_vfs);
 910         if (ret) {
 911                 pci_err(pdev, "Can't assign queues for VF!\n");
 912                 return ret;
 913         }
 914
 915         sec->num_vfs = num_vfs;
 916
 917         ret = pci_enable_sriov(pdev, num_vfs);
 918         if (ret) {
 919                 pci_err(pdev, "Can't enable VF!\n");
 920                 sec_clear_vft_config(sec);
 921                 return ret;
 922         }
 923
 924         return num_vfs;
 925 }
 926
 927 static int sec_sriov_disable(struct pci_dev *pdev)
 928 {
 929         struct sec_dev *sec = pci_get_drvdata(pdev);
 930
 931         if (pci_vfs_assigned(pdev)) {
 932                 pci_err(pdev, "Can't disable VFs while VFs are assigned!\n");
 933                 return -EPERM;
 934         }
 935
 936         /* remove in sec_pci_driver will be called to free VF resources */
 937         pci_disable_sriov(pdev);
 938
 939         return sec_clear_vft_config(sec);
 940 }
 941
 942 static int sec_sriov_configure(struct pci_dev *pdev, int num_vfs)
 943 {
 944         if (num_vfs)
 945                 return sec_sriov_enable(pdev, num_vfs);
 946         else
 947                 return sec_sriov_disable(pdev);
 948 }
 949
 950 static void sec_remove(struct pci_dev *pdev)
 951 {
 952         struct sec_dev *sec = pci_get_drvdata(pdev);
 953         struct hisi_qm *qm = &sec->qm;
 954
 955         sec_unregister_from_crypto();
 956
 957         sec_remove_from_list(sec);
 958
 959         if (qm->fun_type == QM_HW_PF && sec->num_vfs)
 960                 (void)sec_sriov_disable(pdev);
 961
 962         sec_debugfs_exit(sec);
 963
 964         (void)hisi_qm_stop(qm);
 965
 966         if (qm->fun_type == QM_HW_PF)
 967                 sec_debug_regs_clear(qm);
 968
 969         sec_probe_uninit(sec);
 970
 971         sec_qm_uninit(qm);
 972 }
 973
 974 static void sec_log_hw_error(struct sec_dev *sec, u32 err_sts)
 975 {
 976         const struct sec_hw_error *errs = sec_hw_errors;
 977         struct device *dev = &sec->qm.pdev->dev;
 978         u32 err_val;
 979
 980         while (errs->msg) {
 981                 if (errs->int_msk & err_sts) {
 982                         dev_err(dev, "%s [error status=0x%x] found\n",
 983                                 errs->msg, errs->int_msk);
 984
 985                         if (SEC_CORE_INT_STATUS_M_ECC & err_sts) {
 986                                 err_val = readl(sec->qm.io_base +
 987                                                 SEC_CORE_SRAM_ECC_ERR_INFO);
 988                                 dev_err(dev, "multi ecc sram num=0x%x\n",
 989                                         SEC_ECC_NUM(err_val));
 990                                 dev_err(dev, "multi ecc sram addr=0x%x\n",
 991                                         SEC_ECC_ADDR(err_val));
 992                         }
 993                 }
 994                 errs++;
 995         }
 996 }
 997
 998 static pci_ers_result_t sec_hw_error_handle(struct sec_dev *sec)
 999 {
1000         u32 err_sts;
1001
1002         /* read err sts */
1003         err_sts = readl(sec->qm.io_base + SEC_CORE_INT_STATUS);
1004         if (err_sts) {
1005                 sec_log_hw_error(sec, err_sts);
1006
1007                 /* clear error interrupts */
1008                 writel(err_sts, sec->qm.io_base + SEC_CORE_INT_SOURCE);
1009
1010                 return PCI_ERS_RESULT_NEED_RESET;
1011         }
1012
1013         return PCI_ERS_RESULT_RECOVERED;
1014 }
1015
1016 static pci_ers_result_t sec_process_hw_error(struct pci_dev *pdev)
1017 {
1018         struct sec_dev *sec = pci_get_drvdata(pdev);
1019         pci_ers_result_t qm_ret, sec_ret;
1020
1021         if (!sec) {
1022                 pci_err(pdev, "Can't recover error during device init\n");
1023                 return PCI_ERS_RESULT_NONE;
1024         }
1025
1026         /* log qm error */
1027         qm_ret = hisi_qm_hw_error_handle(&sec->qm);
1028
1029         /* log sec error */
1030         sec_ret = sec_hw_error_handle(sec);
1031
1032         return (qm_ret == PCI_ERS_RESULT_NEED_RESET ||
1033                 sec_ret == PCI_ERS_RESULT_NEED_RESET) ?
1034                 PCI_ERS_RESULT_NEED_RESET : PCI_ERS_RESULT_RECOVERED;
1035 }
1036
1037 static pci_ers_result_t sec_error_detected(struct pci_dev *pdev,
1038                                            pci_channel_state_t state)
1039 {
1040         if (pdev->is_virtfn)
1041                 return PCI_ERS_RESULT_NONE;
1042
1043         pci_info(pdev, "PCI error detected, state(=%d)!!\n", state);
1044         if (state == pci_channel_io_perm_failure)
1045                 return PCI_ERS_RESULT_DISCONNECT;
1046
1047         return sec_process_hw_error(pdev);
1048 }
1049
1050 static const struct pci_error_handlers sec_err_handler = {
1051         .error_detected = sec_error_detected,
1052 };
1053
1054 static struct pci_driver sec_pci_driver = {
1055         .name = "hisi_sec2",
1056         .id_table = sec_dev_ids,
1057         .probe = sec_probe,
1058         .remove = sec_remove,
1059         .err_handler = &sec_err_handler,
1060         .sriov_configure = sec_sriov_configure,
1061 };
1062
1063 static void sec_register_debugfs(void)
1064 {
1065         if (!debugfs_initialized())
1066                 return;
1067
1068         sec_debugfs_root = debugfs_create_dir("hisi_sec2", NULL);
1069 }
1070
1071 static void sec_unregister_debugfs(void)
1072 {
1073         debugfs_remove_recursive(sec_debugfs_root);
1074 }
1075
1076 static int __init sec_init(void)
1077 {
1078         int ret;
1079
1080         sec_register_debugfs();
1081
1082         ret = pci_register_driver(&sec_pci_driver);
1083         if (ret < 0) {
1084                 sec_unregister_debugfs();
1085                 pr_err("Failed to register pci driver.\n");
1086                 return ret;
1087         }
1088
1089         return 0;
1090 }
1091
1092 static void __exit sec_exit(void)
1093 {
1094         pci_unregister_driver(&sec_pci_driver);
1095         sec_unregister_debugfs();
1096 }
1097
1098 module_init(sec_init);
1099 module_exit(sec_exit);
1100
1101 MODULE_LICENSE("GPL v2");
1102 MODULE_AUTHOR("Zaibo Xu <xuzaibo@huawei.com>");
1103 MODULE_AUTHOR("Longfang Liu <liulongfang@huawei.com>");
1104 MODULE_AUTHOR("Wei Zhang <zhangwei375@huawei.com>");
1105 MODULE_DESCRIPTION("Driver for HiSilicon SEC accelerator");