drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c

   1 // SPDX-License-Identifier: GPL-2.0 OR MIT
   2 /*
   3  * Copyright 2014-2022 Advanced Micro Devices, Inc.
   4  *
   5  * Permission is hereby granted, free of charge, to any person obtaining a
   6  * copy of this software and associated documentation files (the "Software"),
   7  * to deal in the Software without restriction, including without limitation
   8  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   9  * and/or sell copies of the Software, and to permit persons to whom the
  10  * Software is furnished to do so, subject to the following conditions:
  11  *
  12  * The above copyright notice and this permission notice shall be included in
  13  * all copies or substantial portions of the Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  19  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  20  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  21  * OTHER DEALINGS IN THE SOFTWARE.
  22  *
  23  */
  24
  25 #include <linux/ratelimit.h>
  26 #include <linux/printk.h>
  27 #include <linux/slab.h>
  28 #include <linux/list.h>
  29 #include <linux/types.h>
  30 #include <linux/bitops.h>
  31 #include <linux/sched.h>
  32 #include "kfd_priv.h"
  33 #include "kfd_device_queue_manager.h"
  34 #include "kfd_mqd_manager.h"
  35 #include "cik_regs.h"
  36 #include "kfd_kernel_queue.h"
  37 #include "amdgpu_amdkfd.h"
  38 #include "mes_api_def.h"
  39 #include "kfd_debug.h"
  40
  41 /* Size of the per-pipe EOP queue */
  42 #define CIK_HPD_EOP_BYTES_LOG2 11
  43 #define CIK_HPD_EOP_BYTES (1U << CIK_HPD_EOP_BYTES_LOG2)
  44
  45 static int set_pasid_vmid_mapping(struct device_queue_manager *dqm,
  46                                   u32 pasid, unsigned int vmid);
  47
  48 static int execute_queues_cpsch(struct device_queue_manager *dqm,
  49                                 enum kfd_unmap_queues_filter filter,
  50                                 uint32_t filter_param,
  51                                 uint32_t grace_period);
  52 static int unmap_queues_cpsch(struct device_queue_manager *dqm,
  53                                 enum kfd_unmap_queues_filter filter,
  54                                 uint32_t filter_param,
  55                                 uint32_t grace_period,
  56                                 bool reset);
  57
  58 static int map_queues_cpsch(struct device_queue_manager *dqm);
  59
  60 static void deallocate_sdma_queue(struct device_queue_manager *dqm,
  61                                 struct queue *q);
  62
  63 static inline void deallocate_hqd(struct device_queue_manager *dqm,
  64                                 struct queue *q);
  65 static int allocate_hqd(struct device_queue_manager *dqm, struct queue *q);
  66 static int allocate_sdma_queue(struct device_queue_manager *dqm,
  67                                 struct queue *q, const uint32_t *restore_sdma_id);
  68 static void kfd_process_hw_exception(struct work_struct *work);
  69
  70 static inline
  71 enum KFD_MQD_TYPE get_mqd_type_from_queue_type(enum kfd_queue_type type)
  72 {
  73         if (type == KFD_QUEUE_TYPE_SDMA || type == KFD_QUEUE_TYPE_SDMA_XGMI)
  74                 return KFD_MQD_TYPE_SDMA;
  75         return KFD_MQD_TYPE_CP;
  76 }
  77
  78 static bool is_pipe_enabled(struct device_queue_manager *dqm, int mec, int pipe)
  79 {
  80         int i;
  81         int pipe_offset = (mec * dqm->dev->kfd->shared_resources.num_pipe_per_mec
  82                 + pipe) * dqm->dev->kfd->shared_resources.num_queue_per_pipe;
  83
  84         /* queue is available for KFD usage if bit is 1 */
  85         for (i = 0; i <  dqm->dev->kfd->shared_resources.num_queue_per_pipe; ++i)
  86                 if (test_bit(pipe_offset + i,
  87                               dqm->dev->kfd->shared_resources.cp_queue_bitmap))
  88                         return true;
  89         return false;
  90 }
  91
  92 unsigned int get_cp_queues_num(struct device_queue_manager *dqm)
  93 {
  94         return bitmap_weight(dqm->dev->kfd->shared_resources.cp_queue_bitmap,
  95                                 KGD_MAX_QUEUES);
  96 }
  97
  98 unsigned int get_queues_per_pipe(struct device_queue_manager *dqm)
  99 {
 100         return dqm->dev->kfd->shared_resources.num_queue_per_pipe;
 101 }
 102
 103 unsigned int get_pipes_per_mec(struct device_queue_manager *dqm)
 104 {
 105         return dqm->dev->kfd->shared_resources.num_pipe_per_mec;
 106 }
 107
 108 static unsigned int get_num_all_sdma_engines(struct device_queue_manager *dqm)
 109 {
 110         return kfd_get_num_sdma_engines(dqm->dev) +
 111                 kfd_get_num_xgmi_sdma_engines(dqm->dev);
 112 }
 113
 114 unsigned int get_num_sdma_queues(struct device_queue_manager *dqm)
 115 {
 116         return kfd_get_num_sdma_engines(dqm->dev) *
 117                 dqm->dev->kfd->device_info.num_sdma_queues_per_engine;
 118 }
 119
 120 unsigned int get_num_xgmi_sdma_queues(struct device_queue_manager *dqm)
 121 {
 122         return kfd_get_num_xgmi_sdma_engines(dqm->dev) *
 123                 dqm->dev->kfd->device_info.num_sdma_queues_per_engine;
 124 }
 125
 126 static void init_sdma_bitmaps(struct device_queue_manager *dqm)
 127 {
 128         bitmap_zero(dqm->sdma_bitmap, KFD_MAX_SDMA_QUEUES);
 129         bitmap_set(dqm->sdma_bitmap, 0, get_num_sdma_queues(dqm));
 130
 131         bitmap_zero(dqm->xgmi_sdma_bitmap, KFD_MAX_SDMA_QUEUES);
 132         bitmap_set(dqm->xgmi_sdma_bitmap, 0, get_num_xgmi_sdma_queues(dqm));
 133
 134         /* Mask out the reserved queues */
 135         bitmap_andnot(dqm->sdma_bitmap, dqm->sdma_bitmap,
 136                       dqm->dev->kfd->device_info.reserved_sdma_queues_bitmap,
 137                       KFD_MAX_SDMA_QUEUES);
 138 }
 139
 140 void program_sh_mem_settings(struct device_queue_manager *dqm,
 141                                         struct qcm_process_device *qpd)
 142 {
 143         uint32_t xcc_mask = dqm->dev->xcc_mask;
 144         int xcc_id;
 145
 146         for_each_inst(xcc_id, xcc_mask)
 147                 dqm->dev->kfd2kgd->program_sh_mem_settings(
 148                         dqm->dev->adev, qpd->vmid, qpd->sh_mem_config,
 149                         qpd->sh_mem_ape1_base, qpd->sh_mem_ape1_limit,
 150                         qpd->sh_mem_bases, xcc_id);
 151 }
 152
 153 static void kfd_hws_hang(struct device_queue_manager *dqm)
 154 {
 155         /*
 156          * Issue a GPU reset if HWS is unresponsive
 157          */
 158         dqm->is_hws_hang = true;
 159
 160         /* It's possible we're detecting a HWS hang in the
 161          * middle of a GPU reset. No need to schedule another
 162          * reset in this case.
 163          */
 164         if (!dqm->is_resetting)
 165                 schedule_work(&dqm->hw_exception_work);
 166 }
 167
 168 static int convert_to_mes_queue_type(int queue_type)
 169 {
 170         int mes_queue_type;
 171
 172         switch (queue_type) {
 173         case KFD_QUEUE_TYPE_COMPUTE:
 174                 mes_queue_type = MES_QUEUE_TYPE_COMPUTE;
 175                 break;
 176         case KFD_QUEUE_TYPE_SDMA:
 177                 mes_queue_type = MES_QUEUE_TYPE_SDMA;
 178                 break;
 179         default:
 180                 WARN(1, "Invalid queue type %d", queue_type);
 181                 mes_queue_type = -EINVAL;
 182                 break;
 183         }
 184
 185         return mes_queue_type;
 186 }
 187
 188 static int add_queue_mes(struct device_queue_manager *dqm, struct queue *q,
 189                          struct qcm_process_device *qpd)
 190 {
 191         struct amdgpu_device *adev = (struct amdgpu_device *)dqm->dev->adev;
 192         struct kfd_process_device *pdd = qpd_to_pdd(qpd);
 193         struct mes_add_queue_input queue_input;
 194         int r, queue_type;
 195         uint64_t wptr_addr_off;
 196
 197         if (dqm->is_hws_hang)
 198                 return -EIO;
 199
 200         memset(&queue_input, 0x0, sizeof(struct mes_add_queue_input));
 201         queue_input.process_id = qpd->pqm->process->pasid;
 202         queue_input.page_table_base_addr =  qpd->page_table_base;
 203         queue_input.process_va_start = 0;
 204         queue_input.process_va_end = adev->vm_manager.max_pfn - 1;
 205         /* MES unit for quantum is 100ns */
 206         queue_input.process_quantum = KFD_MES_PROCESS_QUANTUM;  /* Equivalent to 10ms. */
 207         queue_input.process_context_addr = pdd->proc_ctx_gpu_addr;
 208         queue_input.gang_quantum = KFD_MES_GANG_QUANTUM; /* Equivalent to 1ms */
 209         queue_input.gang_context_addr = q->gang_ctx_gpu_addr;
 210         queue_input.inprocess_gang_priority = q->properties.priority;
 211         queue_input.gang_global_priority_level =
 212                                         AMDGPU_MES_PRIORITY_LEVEL_NORMAL;
 213         queue_input.doorbell_offset = q->properties.doorbell_off;
 214         queue_input.mqd_addr = q->gart_mqd_addr;
 215         queue_input.wptr_addr = (uint64_t)q->properties.write_ptr;
 216
 217         if (q->wptr_bo) {
 218                 wptr_addr_off = (uint64_t)q->properties.write_ptr & (PAGE_SIZE - 1);
 219                 queue_input.wptr_mc_addr = ((uint64_t)q->wptr_bo->tbo.resource->start << PAGE_SHIFT) + wptr_addr_off;
 220         }
 221
 222         queue_input.is_kfd_process = 1;
 223         queue_input.is_aql_queue = (q->properties.format == KFD_QUEUE_FORMAT_AQL);
 224         queue_input.queue_size = q->properties.queue_size >> 2;
 225
 226         queue_input.paging = false;
 227         queue_input.tba_addr = qpd->tba_addr;
 228         queue_input.tma_addr = qpd->tma_addr;
 229         queue_input.trap_en = KFD_GC_VERSION(q->device) < IP_VERSION(11, 0, 0) ||
 230                               KFD_GC_VERSION(q->device) > IP_VERSION(11, 0, 3);
 231         queue_input.skip_process_ctx_clear = qpd->pqm->process->debug_trap_enabled;
 232
 233         queue_type = convert_to_mes_queue_type(q->properties.type);
 234         if (queue_type < 0) {
 235                 pr_err("Queue type not supported with MES, queue:%d\n",
 236                                 q->properties.type);
 237                 return -EINVAL;
 238         }
 239         queue_input.queue_type = (uint32_t)queue_type;
 240
 241         if (q->gws) {
 242                 queue_input.gws_base = 0;
 243                 queue_input.gws_size = qpd->num_gws;
 244         }
 245
 246         amdgpu_mes_lock(&adev->mes);
 247         r = adev->mes.funcs->add_hw_queue(&adev->mes, &queue_input);
 248         amdgpu_mes_unlock(&adev->mes);
 249         if (r) {
 250                 pr_err("failed to add hardware queue to MES, doorbell=0x%x\n",
 251                         q->properties.doorbell_off);
 252                 pr_err("MES might be in unrecoverable state, issue a GPU reset\n");
 253                 kfd_hws_hang(dqm);
 254 }
 255
 256         return r;
 257 }
 258
 259 static int remove_queue_mes(struct device_queue_manager *dqm, struct queue *q,
 260                         struct qcm_process_device *qpd)
 261 {
 262         struct amdgpu_device *adev = (struct amdgpu_device *)dqm->dev->adev;
 263         int r;
 264         struct mes_remove_queue_input queue_input;
 265
 266         if (dqm->is_hws_hang)
 267                 return -EIO;
 268
 269         memset(&queue_input, 0x0, sizeof(struct mes_remove_queue_input));
 270         queue_input.doorbell_offset = q->properties.doorbell_off;
 271         queue_input.gang_context_addr = q->gang_ctx_gpu_addr;
 272
 273         amdgpu_mes_lock(&adev->mes);
 274         r = adev->mes.funcs->remove_hw_queue(&adev->mes, &queue_input);
 275         amdgpu_mes_unlock(&adev->mes);
 276
 277         if (r) {
 278                 pr_err("failed to remove hardware queue from MES, doorbell=0x%x\n",
 279                         q->properties.doorbell_off);
 280                 pr_err("MES might be in unrecoverable state, issue a GPU reset\n");
 281                 kfd_hws_hang(dqm);
 282         }
 283
 284         return r;
 285 }
 286
 287 static int remove_all_queues_mes(struct device_queue_manager *dqm)
 288 {
 289         struct device_process_node *cur;
 290         struct qcm_process_device *qpd;
 291         struct queue *q;
 292         int retval = 0;
 293
 294         list_for_each_entry(cur, &dqm->queues, list) {
 295                 qpd = cur->qpd;
 296                 list_for_each_entry(q, &qpd->queues_list, list) {
 297                         if (q->properties.is_active) {
 298                                 retval = remove_queue_mes(dqm, q, qpd);
 299                                 if (retval) {
 300                                         pr_err("%s: Failed to remove queue %d for dev %d",
 301                                                 __func__,
 302                                                 q->properties.queue_id,
 303                                                 dqm->dev->id);
 304                                         return retval;
 305                                 }
 306                         }
 307                 }
 308         }
 309
 310         return retval;
 311 }
 312
 313 static void increment_queue_count(struct device_queue_manager *dqm,
 314                                   struct qcm_process_device *qpd,
 315                                   struct queue *q)
 316 {
 317         dqm->active_queue_count++;
 318         if (q->properties.type == KFD_QUEUE_TYPE_COMPUTE ||
 319             q->properties.type == KFD_QUEUE_TYPE_DIQ)
 320                 dqm->active_cp_queue_count++;
 321
 322         if (q->properties.is_gws) {
 323                 dqm->gws_queue_count++;
 324                 qpd->mapped_gws_queue = true;
 325         }
 326 }
 327
 328 static void decrement_queue_count(struct device_queue_manager *dqm,
 329                                   struct qcm_process_device *qpd,
 330                                   struct queue *q)
 331 {
 332         dqm->active_queue_count--;
 333         if (q->properties.type == KFD_QUEUE_TYPE_COMPUTE ||
 334             q->properties.type == KFD_QUEUE_TYPE_DIQ)
 335                 dqm->active_cp_queue_count--;
 336
 337         if (q->properties.is_gws) {
 338                 dqm->gws_queue_count--;
 339                 qpd->mapped_gws_queue = false;
 340         }
 341 }
 342
 343 /*
 344  * Allocate a doorbell ID to this queue.
 345  * If doorbell_id is passed in, make sure requested ID is valid then allocate it.
 346  */
 347 static int allocate_doorbell(struct qcm_process_device *qpd,
 348                              struct queue *q,
 349                              uint32_t const *restore_id)
 350 {
 351         struct kfd_node *dev = qpd->dqm->dev;
 352
 353         if (!KFD_IS_SOC15(dev)) {
 354                 /* On pre-SOC15 chips we need to use the queue ID to
 355                  * preserve the user mode ABI.
 356                  */
 357
 358                 if (restore_id && *restore_id != q->properties.queue_id)
 359                         return -EINVAL;
 360
 361                 q->doorbell_id = q->properties.queue_id;
 362         } else if (q->properties.type == KFD_QUEUE_TYPE_SDMA ||
 363                         q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI) {
 364                 /* For SDMA queues on SOC15 with 8-byte doorbell, use static
 365                  * doorbell assignments based on the engine and queue id.
 366                  * The doobell index distance between RLC (2*i) and (2*i+1)
 367                  * for a SDMA engine is 512.
 368                  */
 369
 370                 uint32_t *idx_offset = dev->kfd->shared_resources.sdma_doorbell_idx;
 371
 372                 /*
 373                  * q->properties.sdma_engine_id corresponds to the virtual
 374                  * sdma engine number. However, for doorbell allocation,
 375                  * we need the physical sdma engine id in order to get the
 376                  * correct doorbell offset.
 377                  */
 378                 uint32_t valid_id = idx_offset[qpd->dqm->dev->node_id *
 379                                                get_num_all_sdma_engines(qpd->dqm) +
 380                                                q->properties.sdma_engine_id]
 381                                                 + (q->properties.sdma_queue_id & 1)
 382                                                 * KFD_QUEUE_DOORBELL_MIRROR_OFFSET
 383                                                 + (q->properties.sdma_queue_id >> 1);
 384
 385                 if (restore_id && *restore_id != valid_id)
 386                         return -EINVAL;
 387                 q->doorbell_id = valid_id;
 388         } else {
 389                 /* For CP queues on SOC15 */
 390                 if (restore_id) {
 391                         /* make sure that ID is free  */
 392                         if (__test_and_set_bit(*restore_id, qpd->doorbell_bitmap))
 393                                 return -EINVAL;
 394
 395                         q->doorbell_id = *restore_id;
 396                 } else {
 397                         /* or reserve a free doorbell ID */
 398                         unsigned int found;
 399
 400                         found = find_first_zero_bit(qpd->doorbell_bitmap,
 401                                                 KFD_MAX_NUM_OF_QUEUES_PER_PROCESS);
 402                         if (found >= KFD_MAX_NUM_OF_QUEUES_PER_PROCESS) {
 403                                 pr_debug("No doorbells available");
 404                                 return -EBUSY;
 405                         }
 406                         set_bit(found, qpd->doorbell_bitmap);
 407                         q->doorbell_id = found;
 408                 }
 409         }
 410
 411         q->properties.doorbell_off =
 412                 kfd_get_doorbell_dw_offset_in_bar(dev->kfd, qpd_to_pdd(qpd),
 413                                           q->doorbell_id);
 414         return 0;
 415 }
 416
 417 static void deallocate_doorbell(struct qcm_process_device *qpd,
 418                                 struct queue *q)
 419 {
 420         unsigned int old;
 421         struct kfd_node *dev = qpd->dqm->dev;
 422
 423         if (!KFD_IS_SOC15(dev) ||
 424             q->properties.type == KFD_QUEUE_TYPE_SDMA ||
 425             q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI)
 426                 return;
 427
 428         old = test_and_clear_bit(q->doorbell_id, qpd->doorbell_bitmap);
 429         WARN_ON(!old);
 430 }
 431
 432 static void program_trap_handler_settings(struct device_queue_manager *dqm,
 433                                 struct qcm_process_device *qpd)
 434 {
 435         uint32_t xcc_mask = dqm->dev->xcc_mask;
 436         int xcc_id;
 437
 438         if (dqm->dev->kfd2kgd->program_trap_handler_settings)
 439                 for_each_inst(xcc_id, xcc_mask)
 440                         dqm->dev->kfd2kgd->program_trap_handler_settings(
 441                                 dqm->dev->adev, qpd->vmid, qpd->tba_addr,
 442                                 qpd->tma_addr, xcc_id);
 443 }
 444
 445 static int allocate_vmid(struct device_queue_manager *dqm,
 446                         struct qcm_process_device *qpd,
 447                         struct queue *q)
 448 {
 449         int allocated_vmid = -1, i;
 450
 451         for (i = dqm->dev->vm_info.first_vmid_kfd;
 452                         i <= dqm->dev->vm_info.last_vmid_kfd; i++) {
 453                 if (!dqm->vmid_pasid[i]) {
 454                         allocated_vmid = i;
 455                         break;
 456                 }
 457         }
 458
 459         if (allocated_vmid < 0) {
 460                 pr_err("no more vmid to allocate\n");
 461                 return -ENOSPC;
 462         }
 463
 464         pr_debug("vmid allocated: %d\n", allocated_vmid);
 465
 466         dqm->vmid_pasid[allocated_vmid] = q->process->pasid;
 467
 468         set_pasid_vmid_mapping(dqm, q->process->pasid, allocated_vmid);
 469
 470         qpd->vmid = allocated_vmid;
 471         q->properties.vmid = allocated_vmid;
 472
 473         program_sh_mem_settings(dqm, qpd);
 474
 475         if (KFD_IS_SOC15(dqm->dev) && dqm->dev->kfd->cwsr_enabled)
 476                 program_trap_handler_settings(dqm, qpd);
 477
 478         /* qpd->page_table_base is set earlier when register_process()
 479          * is called, i.e. when the first queue is created.
 480          */
 481         dqm->dev->kfd2kgd->set_vm_context_page_table_base(dqm->dev->adev,
 482                         qpd->vmid,
 483                         qpd->page_table_base);
 484         /* invalidate the VM context after pasid and vmid mapping is set up */
 485         kfd_flush_tlb(qpd_to_pdd(qpd), TLB_FLUSH_LEGACY);
 486
 487         if (dqm->dev->kfd2kgd->set_scratch_backing_va)
 488                 dqm->dev->kfd2kgd->set_scratch_backing_va(dqm->dev->adev,
 489                                 qpd->sh_hidden_private_base, qpd->vmid);
 490
 491         return 0;
 492 }
 493
 494 static int flush_texture_cache_nocpsch(struct kfd_node *kdev,
 495                                 struct qcm_process_device *qpd)
 496 {
 497         const struct packet_manager_funcs *pmf = qpd->dqm->packet_mgr.pmf;
 498         int ret;
 499
 500         if (!qpd->ib_kaddr)
 501                 return -ENOMEM;
 502
 503         ret = pmf->release_mem(qpd->ib_base, (uint32_t *)qpd->ib_kaddr);
 504         if (ret)
 505                 return ret;
 506
 507         return amdgpu_amdkfd_submit_ib(kdev->adev, KGD_ENGINE_MEC1, qpd->vmid,
 508                                 qpd->ib_base, (uint32_t *)qpd->ib_kaddr,
 509                                 pmf->release_mem_size / sizeof(uint32_t));
 510 }
 511
 512 static void deallocate_vmid(struct device_queue_manager *dqm,
 513                                 struct qcm_process_device *qpd,
 514                                 struct queue *q)
 515 {
 516         /* On GFX v7, CP doesn't flush TC at dequeue */
 517         if (q->device->adev->asic_type == CHIP_HAWAII)
 518                 if (flush_texture_cache_nocpsch(q->device, qpd))
 519                         pr_err("Failed to flush TC\n");
 520
 521         kfd_flush_tlb(qpd_to_pdd(qpd), TLB_FLUSH_LEGACY);
 522
 523         /* Release the vmid mapping */
 524         set_pasid_vmid_mapping(dqm, 0, qpd->vmid);
 525         dqm->vmid_pasid[qpd->vmid] = 0;
 526
 527         qpd->vmid = 0;
 528         q->properties.vmid = 0;
 529 }
 530
 531 static int create_queue_nocpsch(struct device_queue_manager *dqm,
 532                                 struct queue *q,
 533                                 struct qcm_process_device *qpd,
 534                                 const struct kfd_criu_queue_priv_data *qd,
 535                                 const void *restore_mqd, const void *restore_ctl_stack)
 536 {
 537         struct mqd_manager *mqd_mgr;
 538         int retval;
 539
 540         dqm_lock(dqm);
 541
 542         if (dqm->total_queue_count >= max_num_of_queues_per_device) {
 543                 pr_warn("Can't create new usermode queue because %d queues were already created\n",
 544                                 dqm->total_queue_count);
 545                 retval = -EPERM;
 546                 goto out_unlock;
 547         }
 548
 549         if (list_empty(&qpd->queues_list)) {
 550                 retval = allocate_vmid(dqm, qpd, q);
 551                 if (retval)
 552                         goto out_unlock;
 553         }
 554         q->properties.vmid = qpd->vmid;
 555         /*
 556          * Eviction state logic: mark all queues as evicted, even ones
 557          * not currently active. Restoring inactive queues later only
 558          * updates the is_evicted flag but is a no-op otherwise.
 559          */
 560         q->properties.is_evicted = !!qpd->evicted;
 561
 562         q->properties.tba_addr = qpd->tba_addr;
 563         q->properties.tma_addr = qpd->tma_addr;
 564
 565         mqd_mgr = dqm->mqd_mgrs[get_mqd_type_from_queue_type(
 566                         q->properties.type)];
 567         if (q->properties.type == KFD_QUEUE_TYPE_COMPUTE) {
 568                 retval = allocate_hqd(dqm, q);
 569                 if (retval)
 570                         goto deallocate_vmid;
 571                 pr_debug("Loading mqd to hqd on pipe %d, queue %d\n",
 572                         q->pipe, q->queue);
 573         } else if (q->properties.type == KFD_QUEUE_TYPE_SDMA ||
 574                 q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI) {
 575                 retval = allocate_sdma_queue(dqm, q, qd ? &qd->sdma_id : NULL);
 576                 if (retval)
 577                         goto deallocate_vmid;
 578                 dqm->asic_ops.init_sdma_vm(dqm, q, qpd);
 579         }
 580
 581         retval = allocate_doorbell(qpd, q, qd ? &qd->doorbell_id : NULL);
 582         if (retval)
 583                 goto out_deallocate_hqd;
 584
 585         /* Temporarily release dqm lock to avoid a circular lock dependency */
 586         dqm_unlock(dqm);
 587         q->mqd_mem_obj = mqd_mgr->allocate_mqd(mqd_mgr->dev, &q->properties);
 588         dqm_lock(dqm);
 589
 590         if (!q->mqd_mem_obj) {
 591                 retval = -ENOMEM;
 592                 goto out_deallocate_doorbell;
 593         }
 594
 595         if (qd)
 596                 mqd_mgr->restore_mqd(mqd_mgr, &q->mqd, q->mqd_mem_obj, &q->gart_mqd_addr,
 597                                      &q->properties, restore_mqd, restore_ctl_stack,
 598                                      qd->ctl_stack_size);
 599         else
 600                 mqd_mgr->init_mqd(mqd_mgr, &q->mqd, q->mqd_mem_obj,
 601                                         &q->gart_mqd_addr, &q->properties);
 602
 603         if (q->properties.is_active) {
 604                 if (!dqm->sched_running) {
 605                         WARN_ONCE(1, "Load non-HWS mqd while stopped\n");
 606                         goto add_queue_to_list;
 607                 }
 608
 609                 if (WARN(q->process->mm != current->mm,
 610                                         "should only run in user thread"))
 611                         retval = -EFAULT;
 612                 else
 613                         retval = mqd_mgr->load_mqd(mqd_mgr, q->mqd, q->pipe,
 614                                         q->queue, &q->properties, current->mm);
 615                 if (retval)
 616                         goto out_free_mqd;
 617         }
 618
 619 add_queue_to_list:
 620         list_add(&q->list, &qpd->queues_list);
 621         qpd->queue_count++;
 622         if (q->properties.is_active)
 623                 increment_queue_count(dqm, qpd, q);
 624
 625         /*
 626          * Unconditionally increment this counter, regardless of the queue's
 627          * type or whether the queue is active.
 628          */
 629         dqm->total_queue_count++;
 630         pr_debug("Total of %d queues are accountable so far\n",
 631                         dqm->total_queue_count);
 632         goto out_unlock;
 633
 634 out_free_mqd:
 635         mqd_mgr->free_mqd(mqd_mgr, q->mqd, q->mqd_mem_obj);
 636 out_deallocate_doorbell:
 637         deallocate_doorbell(qpd, q);
 638 out_deallocate_hqd:
 639         if (q->properties.type == KFD_QUEUE_TYPE_COMPUTE)
 640                 deallocate_hqd(dqm, q);
 641         else if (q->properties.type == KFD_QUEUE_TYPE_SDMA ||
 642                 q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI)
 643                 deallocate_sdma_queue(dqm, q);
 644 deallocate_vmid:
 645         if (list_empty(&qpd->queues_list))
 646                 deallocate_vmid(dqm, qpd, q);
 647 out_unlock:
 648         dqm_unlock(dqm);
 649         return retval;
 650 }
 651
 652 static int allocate_hqd(struct device_queue_manager *dqm, struct queue *q)
 653 {
 654         bool set;
 655         int pipe, bit, i;
 656
 657         set = false;
 658
 659         for (pipe = dqm->next_pipe_to_allocate, i = 0;
 660                         i < get_pipes_per_mec(dqm);
 661                         pipe = ((pipe + 1) % get_pipes_per_mec(dqm)), ++i) {
 662
 663                 if (!is_pipe_enabled(dqm, 0, pipe))
 664                         continue;
 665
 666                 if (dqm->allocated_queues[pipe] != 0) {
 667                         bit = ffs(dqm->allocated_queues[pipe]) - 1;
 668                         dqm->allocated_queues[pipe] &= ~(1 << bit);
 669                         q->pipe = pipe;
 670                         q->queue = bit;
 671                         set = true;
 672                         break;
 673                 }
 674         }
 675
 676         if (!set)
 677                 return -EBUSY;
 678
 679         pr_debug("hqd slot - pipe %d, queue %d\n", q->pipe, q->queue);
 680         /* horizontal hqd allocation */
 681         dqm->next_pipe_to_allocate = (pipe + 1) % get_pipes_per_mec(dqm);
 682
 683         return 0;
 684 }
 685
 686 static inline void deallocate_hqd(struct device_queue_manager *dqm,
 687                                 struct queue *q)
 688 {
 689         dqm->allocated_queues[q->pipe] |= (1 << q->queue);
 690 }
 691
 692 #define SQ_IND_CMD_CMD_KILL             0x00000003
 693 #define SQ_IND_CMD_MODE_BROADCAST       0x00000001
 694
 695 static int dbgdev_wave_reset_wavefronts(struct kfd_node *dev, struct kfd_process *p)
 696 {
 697         int status = 0;
 698         unsigned int vmid;
 699         uint16_t queried_pasid;
 700         union SQ_CMD_BITS reg_sq_cmd;
 701         union GRBM_GFX_INDEX_BITS reg_gfx_index;
 702         struct kfd_process_device *pdd;
 703         int first_vmid_to_scan = dev->vm_info.first_vmid_kfd;
 704         int last_vmid_to_scan = dev->vm_info.last_vmid_kfd;
 705         uint32_t xcc_mask = dev->xcc_mask;
 706         int xcc_id;
 707
 708         reg_sq_cmd.u32All = 0;
 709         reg_gfx_index.u32All = 0;
 710
 711         pr_debug("Killing all process wavefronts\n");
 712
 713         if (!dev->kfd2kgd->get_atc_vmid_pasid_mapping_info) {
 714                 pr_err("no vmid pasid mapping supported \n");
 715                 return -EOPNOTSUPP;
 716         }
 717
 718         /* Scan all registers in the range ATC_VMID8_PASID_MAPPING ..
 719          * ATC_VMID15_PASID_MAPPING
 720          * to check which VMID the current process is mapped to.
 721          */
 722
 723         for (vmid = first_vmid_to_scan; vmid <= last_vmid_to_scan; vmid++) {
 724                 status = dev->kfd2kgd->get_atc_vmid_pasid_mapping_info
 725                                 (dev->adev, vmid, &queried_pasid);
 726
 727                 if (status && queried_pasid == p->pasid) {
 728                         pr_debug("Killing wave fronts of vmid %d and pasid 0x%x\n",
 729                                         vmid, p->pasid);
 730                         break;
 731                 }
 732         }
 733
 734         if (vmid > last_vmid_to_scan) {
 735                 pr_err("Didn't find vmid for pasid 0x%x\n", p->pasid);
 736                 return -EFAULT;
 737         }
 738
 739         /* taking the VMID for that process on the safe way using PDD */
 740         pdd = kfd_get_process_device_data(dev, p);
 741         if (!pdd)
 742                 return -EFAULT;
 743
 744         reg_gfx_index.bits.sh_broadcast_writes = 1;
 745         reg_gfx_index.bits.se_broadcast_writes = 1;
 746         reg_gfx_index.bits.instance_broadcast_writes = 1;
 747         reg_sq_cmd.bits.mode = SQ_IND_CMD_MODE_BROADCAST;
 748         reg_sq_cmd.bits.cmd = SQ_IND_CMD_CMD_KILL;
 749         reg_sq_cmd.bits.vm_id = vmid;
 750
 751         for_each_inst(xcc_id, xcc_mask)
 752                 dev->kfd2kgd->wave_control_execute(
 753                         dev->adev, reg_gfx_index.u32All,
 754                         reg_sq_cmd.u32All, xcc_id);
 755
 756         return 0;
 757 }
 758
 759 /* Access to DQM has to be locked before calling destroy_queue_nocpsch_locked
 760  * to avoid asynchronized access
 761  */
 762 static int destroy_queue_nocpsch_locked(struct device_queue_manager *dqm,
 763                                 struct qcm_process_device *qpd,
 764                                 struct queue *q)
 765 {
 766         int retval;
 767         struct mqd_manager *mqd_mgr;
 768
 769         mqd_mgr = dqm->mqd_mgrs[get_mqd_type_from_queue_type(
 770                         q->properties.type)];
 771
 772         if (q->properties.type == KFD_QUEUE_TYPE_COMPUTE)
 773                 deallocate_hqd(dqm, q);
 774         else if (q->properties.type == KFD_QUEUE_TYPE_SDMA)
 775                 deallocate_sdma_queue(dqm, q);
 776         else if (q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI)
 777                 deallocate_sdma_queue(dqm, q);
 778         else {
 779                 pr_debug("q->properties.type %d is invalid\n",
 780                                 q->properties.type);
 781                 return -EINVAL;
 782         }
 783         dqm->total_queue_count--;
 784
 785         deallocate_doorbell(qpd, q);
 786
 787         if (!dqm->sched_running) {
 788                 WARN_ONCE(1, "Destroy non-HWS queue while stopped\n");
 789                 return 0;
 790         }
 791
 792         retval = mqd_mgr->destroy_mqd(mqd_mgr, q->mqd,
 793                                 KFD_PREEMPT_TYPE_WAVEFRONT_RESET,
 794                                 KFD_UNMAP_LATENCY_MS,
 795                                 q->pipe, q->queue);
 796         if (retval == -ETIME)
 797                 qpd->reset_wavefronts = true;
 798
 799         list_del(&q->list);
 800         if (list_empty(&qpd->queues_list)) {
 801                 if (qpd->reset_wavefronts) {
 802                         pr_warn("Resetting wave fronts (nocpsch) on dev %p\n",
 803                                         dqm->dev);
 804                         /* dbgdev_wave_reset_wavefronts has to be called before
 805                          * deallocate_vmid(), i.e. when vmid is still in use.
 806                          */
 807                         dbgdev_wave_reset_wavefronts(dqm->dev,
 808                                         qpd->pqm->process);
 809                         qpd->reset_wavefronts = false;
 810                 }
 811
 812                 deallocate_vmid(dqm, qpd, q);
 813         }
 814         qpd->queue_count--;
 815         if (q->properties.is_active)
 816                 decrement_queue_count(dqm, qpd, q);
 817
 818         return retval;
 819 }
 820
 821 static int destroy_queue_nocpsch(struct device_queue_manager *dqm,
 822                                 struct qcm_process_device *qpd,
 823                                 struct queue *q)
 824 {
 825         int retval;
 826         uint64_t sdma_val = 0;
 827         struct kfd_process_device *pdd = qpd_to_pdd(qpd);
 828         struct mqd_manager *mqd_mgr =
 829                 dqm->mqd_mgrs[get_mqd_type_from_queue_type(q->properties.type)];
 830
 831         /* Get the SDMA queue stats */
 832         if ((q->properties.type == KFD_QUEUE_TYPE_SDMA) ||
 833             (q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI)) {
 834                 retval = read_sdma_queue_counter((uint64_t __user *)q->properties.read_ptr,
 835                                                         &sdma_val);
 836                 if (retval)
 837                         pr_err("Failed to read SDMA queue counter for queue: %d\n",
 838                                 q->properties.queue_id);
 839         }
 840
 841         dqm_lock(dqm);
 842         retval = destroy_queue_nocpsch_locked(dqm, qpd, q);
 843         if (!retval)
 844                 pdd->sdma_past_activity_counter += sdma_val;
 845         dqm_unlock(dqm);
 846
 847         mqd_mgr->free_mqd(mqd_mgr, q->mqd, q->mqd_mem_obj);
 848
 849         return retval;
 850 }
 851
 852 static int update_queue(struct device_queue_manager *dqm, struct queue *q,
 853                         struct mqd_update_info *minfo)
 854 {
 855         int retval = 0;
 856         struct mqd_manager *mqd_mgr;
 857         struct kfd_process_device *pdd;
 858         bool prev_active = false;
 859
 860         dqm_lock(dqm);
 861         pdd = kfd_get_process_device_data(q->device, q->process);
 862         if (!pdd) {
 863                 retval = -ENODEV;
 864                 goto out_unlock;
 865         }
 866         mqd_mgr = dqm->mqd_mgrs[get_mqd_type_from_queue_type(
 867                         q->properties.type)];
 868
 869         /* Save previous activity state for counters */
 870         prev_active = q->properties.is_active;
 871
 872         /* Make sure the queue is unmapped before updating the MQD */
 873         if (dqm->sched_policy != KFD_SCHED_POLICY_NO_HWS) {
 874                 if (!dqm->dev->kfd->shared_resources.enable_mes)
 875                         retval = unmap_queues_cpsch(dqm,
 876                                                     KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES, 0, USE_DEFAULT_GRACE_PERIOD, false);
 877                 else if (prev_active)
 878                         retval = remove_queue_mes(dqm, q, &pdd->qpd);
 879
 880                 if (retval) {
 881                         pr_err("unmap queue failed\n");
 882                         goto out_unlock;
 883                 }
 884         } else if (prev_active &&
 885                    (q->properties.type == KFD_QUEUE_TYPE_COMPUTE ||
 886                     q->properties.type == KFD_QUEUE_TYPE_SDMA ||
 887                     q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI)) {
 888
 889                 if (!dqm->sched_running) {
 890                         WARN_ONCE(1, "Update non-HWS queue while stopped\n");
 891                         goto out_unlock;
 892                 }
 893
 894                 retval = mqd_mgr->destroy_mqd(mqd_mgr, q->mqd,
 895                                 (dqm->dev->kfd->cwsr_enabled ?
 896                                  KFD_PREEMPT_TYPE_WAVEFRONT_SAVE :
 897                                  KFD_PREEMPT_TYPE_WAVEFRONT_DRAIN),
 898                                 KFD_UNMAP_LATENCY_MS, q->pipe, q->queue);
 899                 if (retval) {
 900                         pr_err("destroy mqd failed\n");
 901                         goto out_unlock;
 902                 }
 903         }
 904
 905         mqd_mgr->update_mqd(mqd_mgr, q->mqd, &q->properties, minfo);
 906
 907         /*
 908          * check active state vs. the previous state and modify
 909          * counter accordingly. map_queues_cpsch uses the
 910          * dqm->active_queue_count to determine whether a new runlist must be
 911          * uploaded.
 912          */
 913         if (q->properties.is_active && !prev_active) {
 914                 increment_queue_count(dqm, &pdd->qpd, q);
 915         } else if (!q->properties.is_active && prev_active) {
 916                 decrement_queue_count(dqm, &pdd->qpd, q);
 917         } else if (q->gws && !q->properties.is_gws) {
 918                 if (q->properties.is_active) {
 919                         dqm->gws_queue_count++;
 920                         pdd->qpd.mapped_gws_queue = true;
 921                 }
 922                 q->properties.is_gws = true;
 923         } else if (!q->gws && q->properties.is_gws) {
 924                 if (q->properties.is_active) {
 925                         dqm->gws_queue_count--;
 926                         pdd->qpd.mapped_gws_queue = false;
 927                 }
 928                 q->properties.is_gws = false;
 929         }
 930
 931         if (dqm->sched_policy != KFD_SCHED_POLICY_NO_HWS) {
 932                 if (!dqm->dev->kfd->shared_resources.enable_mes)
 933                         retval = map_queues_cpsch(dqm);
 934                 else if (q->properties.is_active)
 935                         retval = add_queue_mes(dqm, q, &pdd->qpd);
 936         } else if (q->properties.is_active &&
 937                  (q->properties.type == KFD_QUEUE_TYPE_COMPUTE ||
 938                   q->properties.type == KFD_QUEUE_TYPE_SDMA ||
 939                   q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI)) {
 940                 if (WARN(q->process->mm != current->mm,
 941                          "should only run in user thread"))
 942                         retval = -EFAULT;
 943                 else
 944                         retval = mqd_mgr->load_mqd(mqd_mgr, q->mqd,
 945                                                    q->pipe, q->queue,
 946                                                    &q->properties, current->mm);
 947         }
 948
 949 out_unlock:
 950         dqm_unlock(dqm);
 951         return retval;
 952 }
 953
 954 /* suspend_single_queue does not lock the dqm like the
 955  * evict_process_queues_cpsch or evict_process_queues_nocpsch. You should
 956  * lock the dqm before calling, and unlock after calling.
 957  *
 958  * The reason we don't lock the dqm is because this function may be
 959  * called on multiple queues in a loop, so rather than locking/unlocking
 960  * multiple times, we will just keep the dqm locked for all of the calls.
 961  */
 962 static int suspend_single_queue(struct device_queue_manager *dqm,
 963                                       struct kfd_process_device *pdd,
 964                                       struct queue *q)
 965 {
 966         bool is_new;
 967
 968         if (q->properties.is_suspended)
 969                 return 0;
 970
 971         pr_debug("Suspending PASID %u queue [%i]\n",
 972                         pdd->process->pasid,
 973                         q->properties.queue_id);
 974
 975         is_new = q->properties.exception_status & KFD_EC_MASK(EC_QUEUE_NEW);
 976
 977         if (is_new || q->properties.is_being_destroyed) {
 978                 pr_debug("Suspend: skip %s queue id %i\n",
 979                                 is_new ? "new" : "destroyed",
 980                                 q->properties.queue_id);
 981                 return -EBUSY;
 982         }
 983
 984         q->properties.is_suspended = true;
 985         if (q->properties.is_active) {
 986                 if (dqm->dev->kfd->shared_resources.enable_mes) {
 987                         int r = remove_queue_mes(dqm, q, &pdd->qpd);
 988
 989                         if (r)
 990                                 return r;
 991                 }
 992
 993                 decrement_queue_count(dqm, &pdd->qpd, q);
 994                 q->properties.is_active = false;
 995         }
 996
 997         return 0;
 998 }
 999
1000 /* resume_single_queue does not lock the dqm like the functions
1001  * restore_process_queues_cpsch or restore_process_queues_nocpsch. You should
1002  * lock the dqm before calling, and unlock after calling.
1003  *
1004  * The reason we don't lock the dqm is because this function may be
1005  * called on multiple queues in a loop, so rather than locking/unlocking
1006  * multiple times, we will just keep the dqm locked for all of the calls.
1007  */
1008 static int resume_single_queue(struct device_queue_manager *dqm,
1009                                       struct qcm_process_device *qpd,
1010                                       struct queue *q)
1011 {
1012         struct kfd_process_device *pdd;
1013
1014         if (!q->properties.is_suspended)
1015                 return 0;
1016
1017         pdd = qpd_to_pdd(qpd);
1018
1019         pr_debug("Restoring from suspend PASID %u queue [%i]\n",
1020                             pdd->process->pasid,
1021                             q->properties.queue_id);
1022
1023         q->properties.is_suspended = false;
1024
1025         if (QUEUE_IS_ACTIVE(q->properties)) {
1026                 if (dqm->dev->kfd->shared_resources.enable_mes) {
1027                         int r = add_queue_mes(dqm, q, &pdd->qpd);
1028
1029                         if (r)
1030                                 return r;
1031                 }
1032
1033                 q->properties.is_active = true;
1034                 increment_queue_count(dqm, qpd, q);
1035         }
1036
1037         return 0;
1038 }
1039
1040 static int evict_process_queues_nocpsch(struct device_queue_manager *dqm,
1041                                         struct qcm_process_device *qpd)
1042 {
1043         struct queue *q;
1044         struct mqd_manager *mqd_mgr;
1045         struct kfd_process_device *pdd;
1046         int retval, ret = 0;
1047
1048         dqm_lock(dqm);
1049         if (qpd->evicted++ > 0) /* already evicted, do nothing */
1050                 goto out;
1051
1052         pdd = qpd_to_pdd(qpd);
1053         pr_debug_ratelimited("Evicting PASID 0x%x queues\n",
1054                             pdd->process->pasid);
1055
1056         pdd->last_evict_timestamp = get_jiffies_64();
1057         /* Mark all queues as evicted. Deactivate all active queues on
1058          * the qpd.
1059          */
1060         list_for_each_entry(q, &qpd->queues_list, list) {
1061                 q->properties.is_evicted = true;
1062                 if (!q->properties.is_active)
1063                         continue;
1064
1065                 mqd_mgr = dqm->mqd_mgrs[get_mqd_type_from_queue_type(
1066                                 q->properties.type)];
1067                 q->properties.is_active = false;
1068                 decrement_queue_count(dqm, qpd, q);
1069
1070                 if (WARN_ONCE(!dqm->sched_running, "Evict when stopped\n"))
1071                         continue;
1072
1073                 retval = mqd_mgr->destroy_mqd(mqd_mgr, q->mqd,
1074                                 (dqm->dev->kfd->cwsr_enabled ?
1075                                  KFD_PREEMPT_TYPE_WAVEFRONT_SAVE :
1076                                  KFD_PREEMPT_TYPE_WAVEFRONT_DRAIN),
1077                                 KFD_UNMAP_LATENCY_MS, q->pipe, q->queue);
1078                 if (retval && !ret)
1079                         /* Return the first error, but keep going to
1080                          * maintain a consistent eviction state
1081                          */
1082                         ret = retval;
1083         }
1084
1085 out:
1086         dqm_unlock(dqm);
1087         return ret;
1088 }
1089
1090 static int evict_process_queues_cpsch(struct device_queue_manager *dqm,
1091                                       struct qcm_process_device *qpd)
1092 {
1093         struct queue *q;
1094         struct kfd_process_device *pdd;
1095         int retval = 0;
1096
1097         dqm_lock(dqm);
1098         if (qpd->evicted++ > 0) /* already evicted, do nothing */
1099                 goto out;
1100
1101         pdd = qpd_to_pdd(qpd);
1102
1103         /* The debugger creates processes that temporarily have not acquired
1104          * all VMs for all devices and has no VMs itself.
1105          * Skip queue eviction on process eviction.
1106          */
1107         if (!pdd->drm_priv)
1108                 goto out;
1109
1110         pr_debug_ratelimited("Evicting PASID 0x%x queues\n",
1111                             pdd->process->pasid);
1112
1113         /* Mark all queues as evicted. Deactivate all active queues on
1114          * the qpd.
1115          */
1116         list_for_each_entry(q, &qpd->queues_list, list) {
1117                 q->properties.is_evicted = true;
1118                 if (!q->properties.is_active)
1119                         continue;
1120
1121                 q->properties.is_active = false;
1122                 decrement_queue_count(dqm, qpd, q);
1123
1124                 if (dqm->dev->kfd->shared_resources.enable_mes) {
1125                         retval = remove_queue_mes(dqm, q, qpd);
1126                         if (retval) {
1127                                 pr_err("Failed to evict queue %d\n",
1128                                         q->properties.queue_id);
1129                                 goto out;
1130                         }
1131                 }
1132         }
1133         pdd->last_evict_timestamp = get_jiffies_64();
1134         if (!dqm->dev->kfd->shared_resources.enable_mes)
1135                 retval = execute_queues_cpsch(dqm,
1136                                               qpd->is_debug ?
1137                                               KFD_UNMAP_QUEUES_FILTER_ALL_QUEUES :
1138                                               KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES, 0,
1139                                               USE_DEFAULT_GRACE_PERIOD);
1140
1141 out:
1142         dqm_unlock(dqm);
1143         return retval;
1144 }
1145
1146 static int restore_process_queues_nocpsch(struct device_queue_manager *dqm,
1147                                           struct qcm_process_device *qpd)
1148 {
1149         struct mm_struct *mm = NULL;
1150         struct queue *q;
1151         struct mqd_manager *mqd_mgr;
1152         struct kfd_process_device *pdd;
1153         uint64_t pd_base;
1154         uint64_t eviction_duration;
1155         int retval, ret = 0;
1156
1157         pdd = qpd_to_pdd(qpd);
1158         /* Retrieve PD base */
1159         pd_base = amdgpu_amdkfd_gpuvm_get_process_page_dir(pdd->drm_priv);
1160
1161         dqm_lock(dqm);
1162         if (WARN_ON_ONCE(!qpd->evicted)) /* already restored, do nothing */
1163                 goto out;
1164         if (qpd->evicted > 1) { /* ref count still > 0, decrement & quit */
1165                 qpd->evicted--;
1166                 goto out;
1167         }
1168
1169         pr_debug_ratelimited("Restoring PASID 0x%x queues\n",
1170                             pdd->process->pasid);
1171
1172         /* Update PD Base in QPD */
1173         qpd->page_table_base = pd_base;
1174         pr_debug("Updated PD address to 0x%llx\n", pd_base);
1175
1176         if (!list_empty(&qpd->queues_list)) {
1177                 dqm->dev->kfd2kgd->set_vm_context_page_table_base(
1178                                 dqm->dev->adev,
1179                                 qpd->vmid,
1180                                 qpd->page_table_base);
1181                 kfd_flush_tlb(pdd, TLB_FLUSH_LEGACY);
1182         }
1183
1184         /* Take a safe reference to the mm_struct, which may otherwise
1185          * disappear even while the kfd_process is still referenced.
1186          */
1187         mm = get_task_mm(pdd->process->lead_thread);
1188         if (!mm) {
1189                 ret = -EFAULT;
1190                 goto out;
1191         }
1192
1193         /* Remove the eviction flags. Activate queues that are not
1194          * inactive for other reasons.
1195          */
1196         list_for_each_entry(q, &qpd->queues_list, list) {
1197                 q->properties.is_evicted = false;
1198                 if (!QUEUE_IS_ACTIVE(q->properties))
1199                         continue;
1200
1201                 mqd_mgr = dqm->mqd_mgrs[get_mqd_type_from_queue_type(
1202                                 q->properties.type)];
1203                 q->properties.is_active = true;
1204                 increment_queue_count(dqm, qpd, q);
1205
1206                 if (WARN_ONCE(!dqm->sched_running, "Restore when stopped\n"))
1207                         continue;
1208
1209                 retval = mqd_mgr->load_mqd(mqd_mgr, q->mqd, q->pipe,
1210                                        q->queue, &q->properties, mm);
1211                 if (retval && !ret)
1212                         /* Return the first error, but keep going to
1213                          * maintain a consistent eviction state
1214                          */
1215                         ret = retval;
1216         }
1217         qpd->evicted = 0;
1218         eviction_duration = get_jiffies_64() - pdd->last_evict_timestamp;
1219         atomic64_add(eviction_duration, &pdd->evict_duration_counter);
1220 out:
1221         if (mm)
1222                 mmput(mm);
1223         dqm_unlock(dqm);
1224         return ret;
1225 }
1226
1227 static int restore_process_queues_cpsch(struct device_queue_manager *dqm,
1228                                         struct qcm_process_device *qpd)
1229 {
1230         struct queue *q;
1231         struct kfd_process_device *pdd;
1232         uint64_t eviction_duration;
1233         int retval = 0;
1234
1235         pdd = qpd_to_pdd(qpd);
1236
1237         dqm_lock(dqm);
1238         if (WARN_ON_ONCE(!qpd->evicted)) /* already restored, do nothing */
1239                 goto out;
1240         if (qpd->evicted > 1) { /* ref count still > 0, decrement & quit */
1241                 qpd->evicted--;
1242                 goto out;
1243         }
1244
1245         /* The debugger creates processes that temporarily have not acquired
1246          * all VMs for all devices and has no VMs itself.
1247          * Skip queue restore on process restore.
1248          */
1249         if (!pdd->drm_priv)
1250                 goto vm_not_acquired;
1251
1252         pr_debug_ratelimited("Restoring PASID 0x%x queues\n",
1253                             pdd->process->pasid);
1254
1255         /* Update PD Base in QPD */
1256         qpd->page_table_base = amdgpu_amdkfd_gpuvm_get_process_page_dir(pdd->drm_priv);
1257         pr_debug("Updated PD address to 0x%llx\n", qpd->page_table_base);
1258
1259         /* activate all active queues on the qpd */
1260         list_for_each_entry(q, &qpd->queues_list, list) {
1261                 q->properties.is_evicted = false;
1262                 if (!QUEUE_IS_ACTIVE(q->properties))
1263                         continue;
1264
1265                 q->properties.is_active = true;
1266                 increment_queue_count(dqm, &pdd->qpd, q);
1267
1268                 if (dqm->dev->kfd->shared_resources.enable_mes) {
1269                         retval = add_queue_mes(dqm, q, qpd);
1270                         if (retval) {
1271                                 pr_err("Failed to restore queue %d\n",
1272                                         q->properties.queue_id);
1273                                 goto out;
1274                         }
1275                 }
1276         }
1277         if (!dqm->dev->kfd->shared_resources.enable_mes)
1278                 retval = execute_queues_cpsch(dqm,
1279                                               KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES, 0, USE_DEFAULT_GRACE_PERIOD);
1280         eviction_duration = get_jiffies_64() - pdd->last_evict_timestamp;
1281         atomic64_add(eviction_duration, &pdd->evict_duration_counter);
1282 vm_not_acquired:
1283         qpd->evicted = 0;
1284 out:
1285         dqm_unlock(dqm);
1286         return retval;
1287 }
1288
1289 static int register_process(struct device_queue_manager *dqm,
1290                                         struct qcm_process_device *qpd)
1291 {
1292         struct device_process_node *n;
1293         struct kfd_process_device *pdd;
1294         uint64_t pd_base;
1295         int retval;
1296
1297         n = kzalloc(sizeof(*n), GFP_KERNEL);
1298         if (!n)
1299                 return -ENOMEM;
1300
1301         n->qpd = qpd;
1302
1303         pdd = qpd_to_pdd(qpd);
1304         /* Retrieve PD base */
1305         pd_base = amdgpu_amdkfd_gpuvm_get_process_page_dir(pdd->drm_priv);
1306
1307         dqm_lock(dqm);
1308         list_add(&n->list, &dqm->queues);
1309
1310         /* Update PD Base in QPD */
1311         qpd->page_table_base = pd_base;
1312         pr_debug("Updated PD address to 0x%llx\n", pd_base);
1313
1314         retval = dqm->asic_ops.update_qpd(dqm, qpd);
1315
1316         dqm->processes_count++;
1317
1318         dqm_unlock(dqm);
1319
1320         /* Outside the DQM lock because under the DQM lock we can't do
1321          * reclaim or take other locks that others hold while reclaiming.
1322          */
1323         kfd_inc_compute_active(dqm->dev);
1324
1325         return retval;
1326 }
1327
1328 static int unregister_process(struct device_queue_manager *dqm,
1329                                         struct qcm_process_device *qpd)
1330 {
1331         int retval;
1332         struct device_process_node *cur, *next;
1333
1334         pr_debug("qpd->queues_list is %s\n",
1335                         list_empty(&qpd->queues_list) ? "empty" : "not empty");
1336
1337         retval = 0;
1338         dqm_lock(dqm);
1339
1340         list_for_each_entry_safe(cur, next, &dqm->queues, list) {
1341                 if (qpd == cur->qpd) {
1342                         list_del(&cur->list);
1343                         kfree(cur);
1344                         dqm->processes_count--;
1345                         goto out;
1346                 }
1347         }
1348         /* qpd not found in dqm list */
1349         retval = 1;
1350 out:
1351         dqm_unlock(dqm);
1352
1353         /* Outside the DQM lock because under the DQM lock we can't do
1354          * reclaim or take other locks that others hold while reclaiming.
1355          */
1356         if (!retval)
1357                 kfd_dec_compute_active(dqm->dev);
1358
1359         return retval;
1360 }
1361
1362 static int
1363 set_pasid_vmid_mapping(struct device_queue_manager *dqm, u32 pasid,
1364                         unsigned int vmid)
1365 {
1366         uint32_t xcc_mask = dqm->dev->xcc_mask;
1367         int xcc_id, ret;
1368
1369         for_each_inst(xcc_id, xcc_mask) {
1370                 ret = dqm->dev->kfd2kgd->set_pasid_vmid_mapping(
1371                         dqm->dev->adev, pasid, vmid, xcc_id);
1372                 if (ret)
1373                         break;
1374         }
1375
1376         return ret;
1377 }
1378
1379 static void init_interrupts(struct device_queue_manager *dqm)
1380 {
1381         uint32_t xcc_mask = dqm->dev->xcc_mask;
1382         unsigned int i, xcc_id;
1383
1384         for_each_inst(xcc_id, xcc_mask) {
1385                 for (i = 0 ; i < get_pipes_per_mec(dqm) ; i++) {
1386                         if (is_pipe_enabled(dqm, 0, i)) {
1387                                 dqm->dev->kfd2kgd->init_interrupts(
1388                                         dqm->dev->adev, i, xcc_id);
1389                         }
1390                 }
1391         }
1392 }
1393
1394 static int initialize_nocpsch(struct device_queue_manager *dqm)
1395 {
1396         int pipe, queue;
1397
1398         pr_debug("num of pipes: %d\n", get_pipes_per_mec(dqm));
1399
1400         dqm->allocated_queues = kcalloc(get_pipes_per_mec(dqm),
1401                                         sizeof(unsigned int), GFP_KERNEL);
1402         if (!dqm->allocated_queues)
1403                 return -ENOMEM;
1404
1405         mutex_init(&dqm->lock_hidden);
1406         INIT_LIST_HEAD(&dqm->queues);
1407         dqm->active_queue_count = dqm->next_pipe_to_allocate = 0;
1408         dqm->active_cp_queue_count = 0;
1409         dqm->gws_queue_count = 0;
1410
1411         for (pipe = 0; pipe < get_pipes_per_mec(dqm); pipe++) {
1412                 int pipe_offset = pipe * get_queues_per_pipe(dqm);
1413
1414                 for (queue = 0; queue < get_queues_per_pipe(dqm); queue++)
1415                         if (test_bit(pipe_offset + queue,
1416                                      dqm->dev->kfd->shared_resources.cp_queue_bitmap))
1417                                 dqm->allocated_queues[pipe] |= 1 << queue;
1418         }
1419
1420         memset(dqm->vmid_pasid, 0, sizeof(dqm->vmid_pasid));
1421
1422         init_sdma_bitmaps(dqm);
1423
1424         return 0;
1425 }
1426
1427 static void uninitialize(struct device_queue_manager *dqm)
1428 {
1429         int i;
1430
1431         WARN_ON(dqm->active_queue_count > 0 || dqm->processes_count > 0);
1432
1433         kfree(dqm->allocated_queues);
1434         for (i = 0 ; i < KFD_MQD_TYPE_MAX ; i++)
1435                 kfree(dqm->mqd_mgrs[i]);
1436         mutex_destroy(&dqm->lock_hidden);
1437 }
1438
1439 static int start_nocpsch(struct device_queue_manager *dqm)
1440 {
1441         int r = 0;
1442
1443         pr_info("SW scheduler is used");
1444         init_interrupts(dqm);
1445
1446         if (dqm->dev->adev->asic_type == CHIP_HAWAII)
1447                 r = pm_init(&dqm->packet_mgr, dqm);
1448         if (!r)
1449                 dqm->sched_running = true;
1450
1451         return r;
1452 }
1453
1454 static int stop_nocpsch(struct device_queue_manager *dqm)
1455 {
1456         dqm_lock(dqm);
1457         if (!dqm->sched_running) {
1458                 dqm_unlock(dqm);
1459                 return 0;
1460         }
1461
1462         if (dqm->dev->adev->asic_type == CHIP_HAWAII)
1463                 pm_uninit(&dqm->packet_mgr, false);
1464         dqm->sched_running = false;
1465         dqm_unlock(dqm);
1466
1467         return 0;
1468 }
1469
1470 static void pre_reset(struct device_queue_manager *dqm)
1471 {
1472         dqm_lock(dqm);
1473         dqm->is_resetting = true;
1474         dqm_unlock(dqm);
1475 }
1476
1477 static int allocate_sdma_queue(struct device_queue_manager *dqm,
1478                                 struct queue *q, const uint32_t *restore_sdma_id)
1479 {
1480         int bit;
1481
1482         if (q->properties.type == KFD_QUEUE_TYPE_SDMA) {
1483                 if (bitmap_empty(dqm->sdma_bitmap, KFD_MAX_SDMA_QUEUES)) {
1484                         pr_err("No more SDMA queue to allocate\n");
1485                         return -ENOMEM;
1486                 }
1487
1488                 if (restore_sdma_id) {
1489                         /* Re-use existing sdma_id */
1490                         if (!test_bit(*restore_sdma_id, dqm->sdma_bitmap)) {
1491                                 pr_err("SDMA queue already in use\n");
1492                                 return -EBUSY;
1493                         }
1494                         clear_bit(*restore_sdma_id, dqm->sdma_bitmap);
1495                         q->sdma_id = *restore_sdma_id;
1496                 } else {
1497                         /* Find first available sdma_id */
1498                         bit = find_first_bit(dqm->sdma_bitmap,
1499                                              get_num_sdma_queues(dqm));
1500                         clear_bit(bit, dqm->sdma_bitmap);
1501                         q->sdma_id = bit;
1502                 }
1503
1504                 q->properties.sdma_engine_id =
1505                         q->sdma_id % kfd_get_num_sdma_engines(dqm->dev);
1506                 q->properties.sdma_queue_id = q->sdma_id /
1507                                 kfd_get_num_sdma_engines(dqm->dev);
1508         } else if (q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI) {
1509                 if (bitmap_empty(dqm->xgmi_sdma_bitmap, KFD_MAX_SDMA_QUEUES)) {
1510                         pr_err("No more XGMI SDMA queue to allocate\n");
1511                         return -ENOMEM;
1512                 }
1513                 if (restore_sdma_id) {
1514                         /* Re-use existing sdma_id */
1515                         if (!test_bit(*restore_sdma_id, dqm->xgmi_sdma_bitmap)) {
1516                                 pr_err("SDMA queue already in use\n");
1517                                 return -EBUSY;
1518                         }
1519                         clear_bit(*restore_sdma_id, dqm->xgmi_sdma_bitmap);
1520                         q->sdma_id = *restore_sdma_id;
1521                 } else {
1522                         bit = find_first_bit(dqm->xgmi_sdma_bitmap,
1523                                              get_num_xgmi_sdma_queues(dqm));
1524                         clear_bit(bit, dqm->xgmi_sdma_bitmap);
1525                         q->sdma_id = bit;
1526                 }
1527                 /* sdma_engine_id is sdma id including
1528                  * both PCIe-optimized SDMAs and XGMI-
1529                  * optimized SDMAs. The calculation below
1530                  * assumes the first N engines are always
1531                  * PCIe-optimized ones
1532                  */
1533                 q->properties.sdma_engine_id =
1534                         kfd_get_num_sdma_engines(dqm->dev) +
1535                         q->sdma_id % kfd_get_num_xgmi_sdma_engines(dqm->dev);
1536                 q->properties.sdma_queue_id = q->sdma_id /
1537                         kfd_get_num_xgmi_sdma_engines(dqm->dev);
1538         }
1539
1540         pr_debug("SDMA engine id: %d\n", q->properties.sdma_engine_id);
1541         pr_debug("SDMA queue id: %d\n", q->properties.sdma_queue_id);
1542
1543         return 0;
1544 }
1545
1546 static void deallocate_sdma_queue(struct device_queue_manager *dqm,
1547                                 struct queue *q)
1548 {
1549         if (q->properties.type == KFD_QUEUE_TYPE_SDMA) {
1550                 if (q->sdma_id >= get_num_sdma_queues(dqm))
1551                         return;
1552                 set_bit(q->sdma_id, dqm->sdma_bitmap);
1553         } else if (q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI) {
1554                 if (q->sdma_id >= get_num_xgmi_sdma_queues(dqm))
1555                         return;
1556                 set_bit(q->sdma_id, dqm->xgmi_sdma_bitmap);
1557         }
1558 }
1559
1560 /*
1561  * Device Queue Manager implementation for cp scheduler
1562  */
1563
1564 static int set_sched_resources(struct device_queue_manager *dqm)
1565 {
1566         int i, mec;
1567         struct scheduling_resources res;
1568
1569         res.vmid_mask = dqm->dev->compute_vmid_bitmap;
1570
1571         res.queue_mask = 0;
1572         for (i = 0; i < KGD_MAX_QUEUES; ++i) {
1573                 mec = (i / dqm->dev->kfd->shared_resources.num_queue_per_pipe)
1574                         / dqm->dev->kfd->shared_resources.num_pipe_per_mec;
1575
1576                 if (!test_bit(i, dqm->dev->kfd->shared_resources.cp_queue_bitmap))
1577                         continue;
1578
1579                 /* only acquire queues from the first MEC */
1580                 if (mec > 0)
1581                         continue;
1582
1583                 /* This situation may be hit in the future if a new HW
1584                  * generation exposes more than 64 queues. If so, the
1585                  * definition of res.queue_mask needs updating
1586                  */
1587                 if (WARN_ON(i >= (sizeof(res.queue_mask)*8))) {
1588                         pr_err("Invalid queue enabled by amdgpu: %d\n", i);
1589                         break;
1590                 }
1591
1592                 res.queue_mask |= 1ull
1593                         << amdgpu_queue_mask_bit_to_set_resource_bit(
1594                                 dqm->dev->adev, i);
1595         }
1596         res.gws_mask = ~0ull;
1597         res.oac_mask = res.gds_heap_base = res.gds_heap_size = 0;
1598
1599         pr_debug("Scheduling resources:\n"
1600                         "vmid mask: 0x%8X\n"
1601                         "queue mask: 0x%8llX\n",
1602                         res.vmid_mask, res.queue_mask);
1603
1604         return pm_send_set_resources(&dqm->packet_mgr, &res);
1605 }
1606
1607 static int initialize_cpsch(struct device_queue_manager *dqm)
1608 {
1609         pr_debug("num of pipes: %d\n", get_pipes_per_mec(dqm));
1610
1611         mutex_init(&dqm->lock_hidden);
1612         INIT_LIST_HEAD(&dqm->queues);
1613         dqm->active_queue_count = dqm->processes_count = 0;
1614         dqm->active_cp_queue_count = 0;
1615         dqm->gws_queue_count = 0;
1616         dqm->active_runlist = false;
1617         INIT_WORK(&dqm->hw_exception_work, kfd_process_hw_exception);
1618         dqm->trap_debug_vmid = 0;
1619
1620         init_sdma_bitmaps(dqm);
1621
1622         if (dqm->dev->kfd2kgd->get_iq_wait_times)
1623                 dqm->dev->kfd2kgd->get_iq_wait_times(dqm->dev->adev,
1624                                         &dqm->wait_times);
1625         return 0;
1626 }
1627
1628 static int start_cpsch(struct device_queue_manager *dqm)
1629 {
1630         int retval;
1631
1632         retval = 0;
1633
1634         dqm_lock(dqm);
1635
1636         if (!dqm->dev->kfd->shared_resources.enable_mes) {
1637                 retval = pm_init(&dqm->packet_mgr, dqm);
1638                 if (retval)
1639                         goto fail_packet_manager_init;
1640
1641                 retval = set_sched_resources(dqm);
1642                 if (retval)
1643                         goto fail_set_sched_resources;
1644         }
1645         pr_debug("Allocating fence memory\n");
1646
1647         /* allocate fence memory on the gart */
1648         retval = kfd_gtt_sa_allocate(dqm->dev, sizeof(*dqm->fence_addr),
1649                                         &dqm->fence_mem);
1650
1651         if (retval)
1652                 goto fail_allocate_vidmem;
1653
1654         dqm->fence_addr = (uint64_t *)dqm->fence_mem->cpu_ptr;
1655         dqm->fence_gpu_addr = dqm->fence_mem->gpu_addr;
1656
1657         init_interrupts(dqm);
1658
1659         /* clear hang status when driver try to start the hw scheduler */
1660         dqm->is_hws_hang = false;
1661         dqm->is_resetting = false;
1662         dqm->sched_running = true;
1663
1664         if (!dqm->dev->kfd->shared_resources.enable_mes)
1665                 execute_queues_cpsch(dqm, KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES, 0, USE_DEFAULT_GRACE_PERIOD);
1666         dqm_unlock(dqm);
1667
1668         return 0;
1669 fail_allocate_vidmem:
1670 fail_set_sched_resources:
1671         if (!dqm->dev->kfd->shared_resources.enable_mes)
1672                 pm_uninit(&dqm->packet_mgr, false);
1673 fail_packet_manager_init:
1674         dqm_unlock(dqm);
1675         return retval;
1676 }
1677
1678 static int stop_cpsch(struct device_queue_manager *dqm)
1679 {
1680         bool hanging;
1681
1682         dqm_lock(dqm);
1683         if (!dqm->sched_running) {
1684                 dqm_unlock(dqm);
1685                 return 0;
1686         }
1687
1688         if (!dqm->is_hws_hang) {
1689                 if (!dqm->dev->kfd->shared_resources.enable_mes)
1690                         unmap_queues_cpsch(dqm, KFD_UNMAP_QUEUES_FILTER_ALL_QUEUES, 0, USE_DEFAULT_GRACE_PERIOD, false);
1691                 else
1692                         remove_all_queues_mes(dqm);
1693         }
1694
1695         hanging = dqm->is_hws_hang || dqm->is_resetting;
1696         dqm->sched_running = false;
1697
1698         if (!dqm->dev->kfd->shared_resources.enable_mes)
1699                 pm_release_ib(&dqm->packet_mgr);
1700
1701         kfd_gtt_sa_free(dqm->dev, dqm->fence_mem);
1702         if (!dqm->dev->kfd->shared_resources.enable_mes)
1703                 pm_uninit(&dqm->packet_mgr, hanging);
1704         dqm_unlock(dqm);
1705
1706         return 0;
1707 }
1708
1709 static int create_kernel_queue_cpsch(struct device_queue_manager *dqm,
1710                                         struct kernel_queue *kq,
1711                                         struct qcm_process_device *qpd)
1712 {
1713         dqm_lock(dqm);
1714         if (dqm->total_queue_count >= max_num_of_queues_per_device) {
1715                 pr_warn("Can't create new kernel queue because %d queues were already created\n",
1716                                 dqm->total_queue_count);
1717                 dqm_unlock(dqm);
1718                 return -EPERM;
1719         }
1720
1721         /*
1722          * Unconditionally increment this counter, regardless of the queue's
1723          * type or whether the queue is active.
1724          */
1725         dqm->total_queue_count++;
1726         pr_debug("Total of %d queues are accountable so far\n",
1727                         dqm->total_queue_count);
1728
1729         list_add(&kq->list, &qpd->priv_queue_list);
1730         increment_queue_count(dqm, qpd, kq->queue);
1731         qpd->is_debug = true;
1732         execute_queues_cpsch(dqm, KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES, 0,
1733                         USE_DEFAULT_GRACE_PERIOD);
1734         dqm_unlock(dqm);
1735
1736         return 0;
1737 }
1738
1739 static void destroy_kernel_queue_cpsch(struct device_queue_manager *dqm,
1740                                         struct kernel_queue *kq,
1741                                         struct qcm_process_device *qpd)
1742 {
1743         dqm_lock(dqm);
1744         list_del(&kq->list);
1745         decrement_queue_count(dqm, qpd, kq->queue);
1746         qpd->is_debug = false;
1747         execute_queues_cpsch(dqm, KFD_UNMAP_QUEUES_FILTER_ALL_QUEUES, 0,
1748                         USE_DEFAULT_GRACE_PERIOD);
1749         /*
1750          * Unconditionally decrement this counter, regardless of the queue's
1751          * type.
1752          */
1753         dqm->total_queue_count--;
1754         pr_debug("Total of %d queues are accountable so far\n",
1755                         dqm->total_queue_count);
1756         dqm_unlock(dqm);
1757 }
1758
1759 static int create_queue_cpsch(struct device_queue_manager *dqm, struct queue *q,
1760                         struct qcm_process_device *qpd,
1761                         const struct kfd_criu_queue_priv_data *qd,
1762                         const void *restore_mqd, const void *restore_ctl_stack)
1763 {
1764         int retval;
1765         struct mqd_manager *mqd_mgr;
1766
1767         if (dqm->total_queue_count >= max_num_of_queues_per_device) {
1768                 pr_warn("Can't create new usermode queue because %d queues were already created\n",
1769                                 dqm->total_queue_count);
1770                 retval = -EPERM;
1771                 goto out;
1772         }
1773
1774         if (q->properties.type == KFD_QUEUE_TYPE_SDMA ||
1775                 q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI) {
1776                 dqm_lock(dqm);
1777                 retval = allocate_sdma_queue(dqm, q, qd ? &qd->sdma_id : NULL);
1778                 dqm_unlock(dqm);
1779                 if (retval)
1780                         goto out;
1781         }
1782
1783         retval = allocate_doorbell(qpd, q, qd ? &qd->doorbell_id : NULL);
1784         if (retval)
1785                 goto out_deallocate_sdma_queue;
1786
1787         mqd_mgr = dqm->mqd_mgrs[get_mqd_type_from_queue_type(
1788                         q->properties.type)];
1789
1790         if (q->properties.type == KFD_QUEUE_TYPE_SDMA ||
1791                 q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI)
1792                 dqm->asic_ops.init_sdma_vm(dqm, q, qpd);
1793         q->properties.tba_addr = qpd->tba_addr;
1794         q->properties.tma_addr = qpd->tma_addr;
1795         q->mqd_mem_obj = mqd_mgr->allocate_mqd(mqd_mgr->dev, &q->properties);
1796         if (!q->mqd_mem_obj) {
1797                 retval = -ENOMEM;
1798                 goto out_deallocate_doorbell;
1799         }
1800
1801         dqm_lock(dqm);
1802         /*
1803          * Eviction state logic: mark all queues as evicted, even ones
1804          * not currently active. Restoring inactive queues later only
1805          * updates the is_evicted flag but is a no-op otherwise.
1806          */
1807         q->properties.is_evicted = !!qpd->evicted;
1808         q->properties.is_dbg_wa = qpd->pqm->process->debug_trap_enabled &&
1809                         KFD_GC_VERSION(q->device) >= IP_VERSION(11, 0, 0) &&
1810                         KFD_GC_VERSION(q->device) <= IP_VERSION(11, 0, 3);
1811
1812         if (qd)
1813                 mqd_mgr->restore_mqd(mqd_mgr, &q->mqd, q->mqd_mem_obj, &q->gart_mqd_addr,
1814                                      &q->properties, restore_mqd, restore_ctl_stack,
1815                                      qd->ctl_stack_size);
1816         else
1817                 mqd_mgr->init_mqd(mqd_mgr, &q->mqd, q->mqd_mem_obj,
1818                                         &q->gart_mqd_addr, &q->properties);
1819
1820         list_add(&q->list, &qpd->queues_list);
1821         qpd->queue_count++;
1822
1823         if (q->properties.is_active) {
1824                 increment_queue_count(dqm, qpd, q);
1825
1826                 if (!dqm->dev->kfd->shared_resources.enable_mes)
1827                         retval = execute_queues_cpsch(dqm,
1828                                         KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES, 0, USE_DEFAULT_GRACE_PERIOD);
1829                 else
1830                         retval = add_queue_mes(dqm, q, qpd);
1831                 if (retval)
1832                         goto cleanup_queue;
1833         }
1834
1835         /*
1836          * Unconditionally increment this counter, regardless of the queue's
1837          * type or whether the queue is active.
1838          */
1839         dqm->total_queue_count++;
1840
1841         pr_debug("Total of %d queues are accountable so far\n",
1842                         dqm->total_queue_count);
1843
1844         dqm_unlock(dqm);
1845         return retval;
1846
1847 cleanup_queue:
1848         qpd->queue_count--;
1849         list_del(&q->list);
1850         if (q->properties.is_active)
1851                 decrement_queue_count(dqm, qpd, q);
1852         mqd_mgr->free_mqd(mqd_mgr, q->mqd, q->mqd_mem_obj);
1853         dqm_unlock(dqm);
1854 out_deallocate_doorbell:
1855         deallocate_doorbell(qpd, q);
1856 out_deallocate_sdma_queue:
1857         if (q->properties.type == KFD_QUEUE_TYPE_SDMA ||
1858                 q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI) {
1859                 dqm_lock(dqm);
1860                 deallocate_sdma_queue(dqm, q);
1861                 dqm_unlock(dqm);
1862         }
1863 out:
1864         return retval;
1865 }
1866
1867 int amdkfd_fence_wait_timeout(uint64_t *fence_addr,
1868                                 uint64_t fence_value,
1869                                 unsigned int timeout_ms)
1870 {
1871         unsigned long end_jiffies = msecs_to_jiffies(timeout_ms) + jiffies;
1872
1873         while (*fence_addr != fence_value) {
1874                 if (time_after(jiffies, end_jiffies)) {
1875                         pr_err("qcm fence wait loop timeout expired\n");
1876                         /* In HWS case, this is used to halt the driver thread
1877                          * in order not to mess up CP states before doing
1878                          * scandumps for FW debugging.
1879                          */
1880                         while (halt_if_hws_hang)
1881                                 schedule();
1882
1883                         return -ETIME;
1884                 }
1885                 schedule();
1886         }
1887
1888         return 0;
1889 }
1890
1891 /* dqm->lock mutex has to be locked before calling this function */
1892 static int map_queues_cpsch(struct device_queue_manager *dqm)
1893 {
1894         int retval;
1895
1896         if (!dqm->sched_running)
1897                 return 0;
1898         if (dqm->active_queue_count <= 0 || dqm->processes_count <= 0)
1899                 return 0;
1900         if (dqm->active_runlist)
1901                 return 0;
1902
1903         retval = pm_send_runlist(&dqm->packet_mgr, &dqm->queues);
1904         pr_debug("%s sent runlist\n", __func__);
1905         if (retval) {
1906                 pr_err("failed to execute runlist\n");
1907                 return retval;
1908         }
1909         dqm->active_runlist = true;
1910
1911         return retval;
1912 }
1913
1914 /* dqm->lock mutex has to be locked before calling this function */
1915 static int unmap_queues_cpsch(struct device_queue_manager *dqm,
1916                                 enum kfd_unmap_queues_filter filter,
1917                                 uint32_t filter_param,
1918                                 uint32_t grace_period,
1919                                 bool reset)
1920 {
1921         int retval = 0;
1922         struct mqd_manager *mqd_mgr;
1923
1924         if (!dqm->sched_running)
1925                 return 0;
1926         if (dqm->is_hws_hang || dqm->is_resetting)
1927                 return -EIO;
1928         if (!dqm->active_runlist)
1929                 return retval;
1930
1931         if (grace_period != USE_DEFAULT_GRACE_PERIOD) {
1932                 retval = pm_update_grace_period(&dqm->packet_mgr, grace_period);
1933                 if (retval)
1934                         return retval;
1935         }
1936
1937         retval = pm_send_unmap_queue(&dqm->packet_mgr, filter, filter_param, reset);
1938         if (retval)
1939                 return retval;
1940
1941         *dqm->fence_addr = KFD_FENCE_INIT;
1942         pm_send_query_status(&dqm->packet_mgr, dqm->fence_gpu_addr,
1943                                 KFD_FENCE_COMPLETED);
1944         /* should be timed out */
1945         retval = amdkfd_fence_wait_timeout(dqm->fence_addr, KFD_FENCE_COMPLETED,
1946                                 queue_preemption_timeout_ms);
1947         if (retval) {
1948                 pr_err("The cp might be in an unrecoverable state due to an unsuccessful queues preemption\n");
1949                 kfd_hws_hang(dqm);
1950                 return retval;
1951         }
1952
1953         /* In the current MEC firmware implementation, if compute queue
1954          * doesn't response to the preemption request in time, HIQ will
1955          * abandon the unmap request without returning any timeout error
1956          * to driver. Instead, MEC firmware will log the doorbell of the
1957          * unresponding compute queue to HIQ.MQD.queue_doorbell_id fields.
1958          * To make sure the queue unmap was successful, driver need to
1959          * check those fields
1960          */
1961         mqd_mgr = dqm->mqd_mgrs[KFD_MQD_TYPE_HIQ];
1962         if (mqd_mgr->read_doorbell_id(dqm->packet_mgr.priv_queue->queue->mqd)) {
1963                 pr_err("HIQ MQD's queue_doorbell_id0 is not 0, Queue preemption time out\n");
1964                 while (halt_if_hws_hang)
1965                         schedule();
1966                 return -ETIME;
1967         }
1968
1969         /* We need to reset the grace period value for this device */
1970         if (grace_period != USE_DEFAULT_GRACE_PERIOD) {
1971                 if (pm_update_grace_period(&dqm->packet_mgr,
1972                                         USE_DEFAULT_GRACE_PERIOD))
1973                         pr_err("Failed to reset grace period\n");
1974         }
1975
1976         pm_release_ib(&dqm->packet_mgr);
1977         dqm->active_runlist = false;
1978
1979         return retval;
1980 }
1981
1982 /* only for compute queue */
1983 static int reset_queues_cpsch(struct device_queue_manager *dqm,
1984                         uint16_t pasid)
1985 {
1986         int retval;
1987
1988         dqm_lock(dqm);
1989
1990         retval = unmap_queues_cpsch(dqm, KFD_UNMAP_QUEUES_FILTER_BY_PASID,
1991                         pasid, USE_DEFAULT_GRACE_PERIOD, true);
1992
1993         dqm_unlock(dqm);
1994         return retval;
1995 }
1996
1997 /* dqm->lock mutex has to be locked before calling this function */
1998 static int execute_queues_cpsch(struct device_queue_manager *dqm,
1999                                 enum kfd_unmap_queues_filter filter,
2000                                 uint32_t filter_param,
2001                                 uint32_t grace_period)
2002 {
2003         int retval;
2004
2005         if (dqm->is_hws_hang)
2006                 return -EIO;
2007         retval = unmap_queues_cpsch(dqm, filter, filter_param, grace_period, false);
2008         if (retval)
2009                 return retval;
2010
2011         return map_queues_cpsch(dqm);
2012 }
2013
2014 static int wait_on_destroy_queue(struct device_queue_manager *dqm,
2015                                  struct queue *q)
2016 {
2017         struct kfd_process_device *pdd = kfd_get_process_device_data(q->device,
2018                                                                 q->process);
2019         int ret = 0;
2020
2021         if (pdd->qpd.is_debug)
2022                 return ret;
2023
2024         q->properties.is_being_destroyed = true;
2025
2026         if (pdd->process->debug_trap_enabled && q->properties.is_suspended) {
2027                 dqm_unlock(dqm);
2028                 mutex_unlock(&q->process->mutex);
2029                 ret = wait_event_interruptible(dqm->destroy_wait,
2030                                                 !q->properties.is_suspended);
2031
2032                 mutex_lock(&q->process->mutex);
2033                 dqm_lock(dqm);
2034         }
2035
2036         return ret;
2037 }
2038
2039 static int destroy_queue_cpsch(struct device_queue_manager *dqm,
2040                                 struct qcm_process_device *qpd,
2041                                 struct queue *q)
2042 {
2043         int retval;
2044         struct mqd_manager *mqd_mgr;
2045         uint64_t sdma_val = 0;
2046         struct kfd_process_device *pdd = qpd_to_pdd(qpd);
2047
2048         /* Get the SDMA queue stats */
2049         if ((q->properties.type == KFD_QUEUE_TYPE_SDMA) ||
2050             (q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI)) {
2051                 retval = read_sdma_queue_counter((uint64_t __user *)q->properties.read_ptr,
2052                                                         &sdma_val);
2053                 if (retval)
2054                         pr_err("Failed to read SDMA queue counter for queue: %d\n",
2055                                 q->properties.queue_id);
2056         }
2057
2058         /* remove queue from list to prevent rescheduling after preemption */
2059         dqm_lock(dqm);
2060
2061         retval = wait_on_destroy_queue(dqm, q);
2062
2063         if (retval) {
2064                 dqm_unlock(dqm);
2065                 return retval;
2066         }
2067
2068         if (qpd->is_debug) {
2069                 /*
2070                  * error, currently we do not allow to destroy a queue
2071                  * of a currently debugged process
2072                  */
2073                 retval = -EBUSY;
2074                 goto failed_try_destroy_debugged_queue;
2075
2076         }
2077
2078         mqd_mgr = dqm->mqd_mgrs[get_mqd_type_from_queue_type(
2079                         q->properties.type)];
2080
2081         deallocate_doorbell(qpd, q);
2082
2083         if ((q->properties.type == KFD_QUEUE_TYPE_SDMA) ||
2084             (q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI)) {
2085                 deallocate_sdma_queue(dqm, q);
2086                 pdd->sdma_past_activity_counter += sdma_val;
2087         }
2088
2089         list_del(&q->list);
2090         qpd->queue_count--;
2091         if (q->properties.is_active) {
2092                 decrement_queue_count(dqm, qpd, q);
2093                 if (!dqm->dev->kfd->shared_resources.enable_mes) {
2094                         retval = execute_queues_cpsch(dqm,
2095                                                       KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES, 0,
2096                                                       USE_DEFAULT_GRACE_PERIOD);
2097                         if (retval == -ETIME)
2098                                 qpd->reset_wavefronts = true;
2099                 } else {
2100                         retval = remove_queue_mes(dqm, q, qpd);
2101                 }
2102         }
2103
2104         /*
2105          * Unconditionally decrement this counter, regardless of the queue's
2106          * type
2107          */
2108         dqm->total_queue_count--;
2109         pr_debug("Total of %d queues are accountable so far\n",
2110                         dqm->total_queue_count);
2111
2112         dqm_unlock(dqm);
2113
2114         /*
2115          * Do free_mqd and raise delete event after dqm_unlock(dqm) to avoid
2116          * circular locking
2117          */
2118         kfd_dbg_ev_raise(KFD_EC_MASK(EC_DEVICE_QUEUE_DELETE),
2119                                 qpd->pqm->process, q->device,
2120                                 -1, false, NULL, 0);
2121
2122         mqd_mgr->free_mqd(mqd_mgr, q->mqd, q->mqd_mem_obj);
2123
2124         return retval;
2125
2126 failed_try_destroy_debugged_queue:
2127
2128         dqm_unlock(dqm);
2129         return retval;
2130 }
2131
2132 /*
2133  * Low bits must be 0000/FFFF as required by HW, high bits must be 0 to
2134  * stay in user mode.
2135  */
2136 #define APE1_FIXED_BITS_MASK 0xFFFF80000000FFFFULL
2137 /* APE1 limit is inclusive and 64K aligned. */
2138 #define APE1_LIMIT_ALIGNMENT 0xFFFF
2139
2140 static bool set_cache_memory_policy(struct device_queue_manager *dqm,
2141                                    struct qcm_process_device *qpd,
2142                                    enum cache_policy default_policy,
2143                                    enum cache_policy alternate_policy,
2144                                    void __user *alternate_aperture_base,
2145                                    uint64_t alternate_aperture_size)
2146 {
2147         bool retval = true;
2148
2149         if (!dqm->asic_ops.set_cache_memory_policy)
2150                 return retval;
2151
2152         dqm_lock(dqm);
2153
2154         if (alternate_aperture_size == 0) {
2155                 /* base > limit disables APE1 */
2156                 qpd->sh_mem_ape1_base = 1;
2157                 qpd->sh_mem_ape1_limit = 0;
2158         } else {
2159                 /*
2160                  * In FSA64, APE1_Base[63:0] = { 16{SH_MEM_APE1_BASE[31]},
2161                  *                      SH_MEM_APE1_BASE[31:0], 0x0000 }
2162                  * APE1_Limit[63:0] = { 16{SH_MEM_APE1_LIMIT[31]},
2163                  *                      SH_MEM_APE1_LIMIT[31:0], 0xFFFF }
2164                  * Verify that the base and size parameters can be
2165                  * represented in this format and convert them.
2166                  * Additionally restrict APE1 to user-mode addresses.
2167                  */
2168
2169                 uint64_t base = (uintptr_t)alternate_aperture_base;
2170                 uint64_t limit = base + alternate_aperture_size - 1;
2171
2172                 if (limit <= base || (base & APE1_FIXED_BITS_MASK) != 0 ||
2173                    (limit & APE1_FIXED_BITS_MASK) != APE1_LIMIT_ALIGNMENT) {
2174                         retval = false;
2175                         goto out;
2176                 }
2177
2178                 qpd->sh_mem_ape1_base = base >> 16;
2179                 qpd->sh_mem_ape1_limit = limit >> 16;
2180         }
2181
2182         retval = dqm->asic_ops.set_cache_memory_policy(
2183                         dqm,
2184                         qpd,
2185                         default_policy,
2186                         alternate_policy,
2187                         alternate_aperture_base,
2188                         alternate_aperture_size);
2189
2190         if ((dqm->sched_policy == KFD_SCHED_POLICY_NO_HWS) && (qpd->vmid != 0))
2191                 program_sh_mem_settings(dqm, qpd);
2192
2193         pr_debug("sh_mem_config: 0x%x, ape1_base: 0x%x, ape1_limit: 0x%x\n",
2194                 qpd->sh_mem_config, qpd->sh_mem_ape1_base,
2195                 qpd->sh_mem_ape1_limit);
2196
2197 out:
2198         dqm_unlock(dqm);
2199         return retval;
2200 }
2201
2202 static int process_termination_nocpsch(struct device_queue_manager *dqm,
2203                 struct qcm_process_device *qpd)
2204 {
2205         struct queue *q;
2206         struct device_process_node *cur, *next_dpn;
2207         int retval = 0;
2208         bool found = false;
2209
2210         dqm_lock(dqm);
2211
2212         /* Clear all user mode queues */
2213         while (!list_empty(&qpd->queues_list)) {
2214                 struct mqd_manager *mqd_mgr;
2215                 int ret;
2216
2217                 q = list_first_entry(&qpd->queues_list, struct queue, list);
2218                 mqd_mgr = dqm->mqd_mgrs[get_mqd_type_from_queue_type(
2219                                 q->properties.type)];
2220                 ret = destroy_queue_nocpsch_locked(dqm, qpd, q);
2221                 if (ret)
2222                         retval = ret;
2223                 dqm_unlock(dqm);
2224                 mqd_mgr->free_mqd(mqd_mgr, q->mqd, q->mqd_mem_obj);
2225                 dqm_lock(dqm);
2226         }
2227
2228         /* Unregister process */
2229         list_for_each_entry_safe(cur, next_dpn, &dqm->queues, list) {
2230                 if (qpd == cur->qpd) {
2231                         list_del(&cur->list);
2232                         kfree(cur);
2233                         dqm->processes_count--;
2234                         found = true;
2235                         break;
2236                 }
2237         }
2238
2239         dqm_unlock(dqm);
2240
2241         /* Outside the DQM lock because under the DQM lock we can't do
2242          * reclaim or take other locks that others hold while reclaiming.
2243          */
2244         if (found)
2245                 kfd_dec_compute_active(dqm->dev);
2246
2247         return retval;
2248 }
2249
2250 static int get_wave_state(struct device_queue_manager *dqm,
2251                           struct queue *q,
2252                           void __user *ctl_stack,
2253                           u32 *ctl_stack_used_size,
2254                           u32 *save_area_used_size)
2255 {
2256         struct mqd_manager *mqd_mgr;
2257
2258         dqm_lock(dqm);
2259
2260         mqd_mgr = dqm->mqd_mgrs[KFD_MQD_TYPE_CP];
2261
2262         if (q->properties.type != KFD_QUEUE_TYPE_COMPUTE ||
2263             q->properties.is_active || !q->device->kfd->cwsr_enabled ||
2264             !mqd_mgr->get_wave_state) {
2265                 dqm_unlock(dqm);
2266                 return -EINVAL;
2267         }
2268
2269         dqm_unlock(dqm);
2270
2271         /*
2272          * get_wave_state is outside the dqm lock to prevent circular locking
2273          * and the queue should be protected against destruction by the process
2274          * lock.
2275          */
2276         return mqd_mgr->get_wave_state(mqd_mgr, q->mqd, &q->properties,
2277                         ctl_stack, ctl_stack_used_size, save_area_used_size);
2278 }
2279
2280 static void get_queue_checkpoint_info(struct device_queue_manager *dqm,
2281                         const struct queue *q,
2282                         u32 *mqd_size,
2283                         u32 *ctl_stack_size)
2284 {
2285         struct mqd_manager *mqd_mgr;
2286         enum KFD_MQD_TYPE mqd_type =
2287                         get_mqd_type_from_queue_type(q->properties.type);
2288
2289         dqm_lock(dqm);
2290         mqd_mgr = dqm->mqd_mgrs[mqd_type];
2291         *mqd_size = mqd_mgr->mqd_size;
2292         *ctl_stack_size = 0;
2293
2294         if (q->properties.type == KFD_QUEUE_TYPE_COMPUTE && mqd_mgr->get_checkpoint_info)
2295                 mqd_mgr->get_checkpoint_info(mqd_mgr, q->mqd, ctl_stack_size);
2296
2297         dqm_unlock(dqm);
2298 }
2299
2300 static int checkpoint_mqd(struct device_queue_manager *dqm,
2301                           const struct queue *q,
2302                           void *mqd,
2303                           void *ctl_stack)
2304 {
2305         struct mqd_manager *mqd_mgr;
2306         int r = 0;
2307         enum KFD_MQD_TYPE mqd_type =
2308                         get_mqd_type_from_queue_type(q->properties.type);
2309
2310         dqm_lock(dqm);
2311
2312         if (q->properties.is_active || !q->device->kfd->cwsr_enabled) {
2313                 r = -EINVAL;
2314                 goto dqm_unlock;
2315         }
2316
2317         mqd_mgr = dqm->mqd_mgrs[mqd_type];
2318         if (!mqd_mgr->checkpoint_mqd) {
2319                 r = -EOPNOTSUPP;
2320                 goto dqm_unlock;
2321         }
2322
2323         mqd_mgr->checkpoint_mqd(mqd_mgr, q->mqd, mqd, ctl_stack);
2324
2325 dqm_unlock:
2326         dqm_unlock(dqm);
2327         return r;
2328 }
2329
2330 static int process_termination_cpsch(struct device_queue_manager *dqm,
2331                 struct qcm_process_device *qpd)
2332 {
2333         int retval;
2334         struct queue *q;
2335         struct kernel_queue *kq, *kq_next;
2336         struct mqd_manager *mqd_mgr;
2337         struct device_process_node *cur, *next_dpn;
2338         enum kfd_unmap_queues_filter filter =
2339                 KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES;
2340         bool found = false;
2341
2342         retval = 0;
2343
2344         dqm_lock(dqm);
2345
2346         /* Clean all kernel queues */
2347         list_for_each_entry_safe(kq, kq_next, &qpd->priv_queue_list, list) {
2348                 list_del(&kq->list);
2349                 decrement_queue_count(dqm, qpd, kq->queue);
2350                 qpd->is_debug = false;
2351                 dqm->total_queue_count--;
2352                 filter = KFD_UNMAP_QUEUES_FILTER_ALL_QUEUES;
2353         }
2354
2355         /* Clear all user mode queues */
2356         list_for_each_entry(q, &qpd->queues_list, list) {
2357                 if (q->properties.type == KFD_QUEUE_TYPE_SDMA)
2358                         deallocate_sdma_queue(dqm, q);
2359                 else if (q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI)
2360                         deallocate_sdma_queue(dqm, q);
2361
2362                 if (q->properties.is_active) {
2363                         decrement_queue_count(dqm, qpd, q);
2364
2365                         if (dqm->dev->kfd->shared_resources.enable_mes) {
2366                                 retval = remove_queue_mes(dqm, q, qpd);
2367                                 if (retval)
2368                                         pr_err("Failed to remove queue %d\n",
2369                                                 q->properties.queue_id);
2370                         }
2371                 }
2372
2373                 dqm->total_queue_count--;
2374         }
2375
2376         /* Unregister process */
2377         list_for_each_entry_safe(cur, next_dpn, &dqm->queues, list) {
2378                 if (qpd == cur->qpd) {
2379                         list_del(&cur->list);
2380                         kfree(cur);
2381                         dqm->processes_count--;
2382                         found = true;
2383                         break;
2384                 }
2385         }
2386
2387         if (!dqm->dev->kfd->shared_resources.enable_mes)
2388                 retval = execute_queues_cpsch(dqm, filter, 0, USE_DEFAULT_GRACE_PERIOD);
2389
2390         if ((!dqm->is_hws_hang) && (retval || qpd->reset_wavefronts)) {
2391                 pr_warn("Resetting wave fronts (cpsch) on dev %p\n", dqm->dev);
2392                 dbgdev_wave_reset_wavefronts(dqm->dev, qpd->pqm->process);
2393                 qpd->reset_wavefronts = false;
2394         }
2395
2396         /* Lastly, free mqd resources.
2397          * Do free_mqd() after dqm_unlock to avoid circular locking.
2398          */
2399         while (!list_empty(&qpd->queues_list)) {
2400                 q = list_first_entry(&qpd->queues_list, struct queue, list);
2401                 mqd_mgr = dqm->mqd_mgrs[get_mqd_type_from_queue_type(
2402                                 q->properties.type)];
2403                 list_del(&q->list);
2404                 qpd->queue_count--;
2405                 dqm_unlock(dqm);
2406                 mqd_mgr->free_mqd(mqd_mgr, q->mqd, q->mqd_mem_obj);
2407                 dqm_lock(dqm);
2408         }
2409         dqm_unlock(dqm);
2410
2411         /* Outside the DQM lock because under the DQM lock we can't do
2412          * reclaim or take other locks that others hold while reclaiming.
2413          */
2414         if (found)
2415                 kfd_dec_compute_active(dqm->dev);
2416
2417         return retval;
2418 }
2419
2420 static int init_mqd_managers(struct device_queue_manager *dqm)
2421 {
2422         int i, j;
2423         struct mqd_manager *mqd_mgr;
2424
2425         for (i = 0; i < KFD_MQD_TYPE_MAX; i++) {
2426                 mqd_mgr = dqm->asic_ops.mqd_manager_init(i, dqm->dev);
2427                 if (!mqd_mgr) {
2428                         pr_err("mqd manager [%d] initialization failed\n", i);
2429                         goto out_free;
2430                 }
2431                 dqm->mqd_mgrs[i] = mqd_mgr;
2432         }
2433
2434         return 0;
2435
2436 out_free:
2437         for (j = 0; j < i; j++) {
2438                 kfree(dqm->mqd_mgrs[j]);
2439                 dqm->mqd_mgrs[j] = NULL;
2440         }
2441
2442         return -ENOMEM;
2443 }
2444
2445 /* Allocate one hiq mqd (HWS) and all SDMA mqd in a continuous trunk*/
2446 static int allocate_hiq_sdma_mqd(struct device_queue_manager *dqm)
2447 {
2448         int retval;
2449         struct kfd_node *dev = dqm->dev;
2450         struct kfd_mem_obj *mem_obj = &dqm->hiq_sdma_mqd;
2451         uint32_t size = dqm->mqd_mgrs[KFD_MQD_TYPE_SDMA]->mqd_size *
2452                 get_num_all_sdma_engines(dqm) *
2453                 dev->kfd->device_info.num_sdma_queues_per_engine +
2454                 (dqm->mqd_mgrs[KFD_MQD_TYPE_HIQ]->mqd_size *
2455                 NUM_XCC(dqm->dev->xcc_mask));
2456
2457         retval = amdgpu_amdkfd_alloc_gtt_mem(dev->adev, size,
2458                 &(mem_obj->gtt_mem), &(mem_obj->gpu_addr),
2459                 (void *)&(mem_obj->cpu_ptr), false);
2460
2461         return retval;
2462 }
2463
2464 struct device_queue_manager *device_queue_manager_init(struct kfd_node *dev)
2465 {
2466         struct device_queue_manager *dqm;
2467
2468         pr_debug("Loading device queue manager\n");
2469
2470         dqm = kzalloc(sizeof(*dqm), GFP_KERNEL);
2471         if (!dqm)
2472                 return NULL;
2473
2474         switch (dev->adev->asic_type) {
2475         /* HWS is not available on Hawaii. */
2476         case CHIP_HAWAII:
2477         /* HWS depends on CWSR for timely dequeue. CWSR is not
2478          * available on Tonga.
2479          *
2480          * FIXME: This argument also applies to Kaveri.
2481          */
2482         case CHIP_TONGA:
2483                 dqm->sched_policy = KFD_SCHED_POLICY_NO_HWS;
2484                 break;
2485         default:
2486                 dqm->sched_policy = sched_policy;
2487                 break;
2488         }
2489
2490         dqm->dev = dev;
2491         switch (dqm->sched_policy) {
2492         case KFD_SCHED_POLICY_HWS:
2493         case KFD_SCHED_POLICY_HWS_NO_OVERSUBSCRIPTION:
2494                 /* initialize dqm for cp scheduling */
2495                 dqm->ops.create_queue = create_queue_cpsch;
2496                 dqm->ops.initialize = initialize_cpsch;
2497                 dqm->ops.start = start_cpsch;
2498                 dqm->ops.stop = stop_cpsch;
2499                 dqm->ops.pre_reset = pre_reset;
2500                 dqm->ops.destroy_queue = destroy_queue_cpsch;
2501                 dqm->ops.update_queue = update_queue;
2502                 dqm->ops.register_process = register_process;
2503                 dqm->ops.unregister_process = unregister_process;
2504                 dqm->ops.uninitialize = uninitialize;
2505                 dqm->ops.create_kernel_queue = create_kernel_queue_cpsch;
2506                 dqm->ops.destroy_kernel_queue = destroy_kernel_queue_cpsch;
2507                 dqm->ops.set_cache_memory_policy = set_cache_memory_policy;
2508                 dqm->ops.process_termination = process_termination_cpsch;
2509                 dqm->ops.evict_process_queues = evict_process_queues_cpsch;
2510                 dqm->ops.restore_process_queues = restore_process_queues_cpsch;
2511                 dqm->ops.get_wave_state = get_wave_state;
2512                 dqm->ops.reset_queues = reset_queues_cpsch;
2513                 dqm->ops.get_queue_checkpoint_info = get_queue_checkpoint_info;
2514                 dqm->ops.checkpoint_mqd = checkpoint_mqd;
2515                 break;
2516         case KFD_SCHED_POLICY_NO_HWS:
2517                 /* initialize dqm for no cp scheduling */
2518                 dqm->ops.start = start_nocpsch;
2519                 dqm->ops.stop = stop_nocpsch;
2520                 dqm->ops.pre_reset = pre_reset;
2521                 dqm->ops.create_queue = create_queue_nocpsch;
2522                 dqm->ops.destroy_queue = destroy_queue_nocpsch;
2523                 dqm->ops.update_queue = update_queue;
2524                 dqm->ops.register_process = register_process;
2525                 dqm->ops.unregister_process = unregister_process;
2526                 dqm->ops.initialize = initialize_nocpsch;
2527                 dqm->ops.uninitialize = uninitialize;
2528                 dqm->ops.set_cache_memory_policy = set_cache_memory_policy;
2529                 dqm->ops.process_termination = process_termination_nocpsch;
2530                 dqm->ops.evict_process_queues = evict_process_queues_nocpsch;
2531                 dqm->ops.restore_process_queues =
2532                         restore_process_queues_nocpsch;
2533                 dqm->ops.get_wave_state = get_wave_state;
2534                 dqm->ops.get_queue_checkpoint_info = get_queue_checkpoint_info;
2535                 dqm->ops.checkpoint_mqd = checkpoint_mqd;
2536                 break;
2537         default:
2538                 pr_err("Invalid scheduling policy %d\n", dqm->sched_policy);
2539                 goto out_free;
2540         }
2541
2542         switch (dev->adev->asic_type) {
2543         case CHIP_CARRIZO:
2544                 device_queue_manager_init_vi(&dqm->asic_ops);
2545                 break;
2546
2547         case CHIP_KAVERI:
2548                 device_queue_manager_init_cik(&dqm->asic_ops);
2549                 break;
2550
2551         case CHIP_HAWAII:
2552                 device_queue_manager_init_cik_hawaii(&dqm->asic_ops);
2553                 break;
2554
2555         case CHIP_TONGA:
2556         case CHIP_FIJI:
2557         case CHIP_POLARIS10:
2558         case CHIP_POLARIS11:
2559         case CHIP_POLARIS12:
2560         case CHIP_VEGAM:
2561                 device_queue_manager_init_vi_tonga(&dqm->asic_ops);
2562                 break;
2563
2564         default:
2565                 if (KFD_GC_VERSION(dev) >= IP_VERSION(11, 0, 0))
2566                         device_queue_manager_init_v11(&dqm->asic_ops);
2567                 else if (KFD_GC_VERSION(dev) >= IP_VERSION(10, 1, 1))
2568                         device_queue_manager_init_v10_navi10(&dqm->asic_ops);
2569                 else if (KFD_GC_VERSION(dev) >= IP_VERSION(9, 0, 1))
2570                         device_queue_manager_init_v9(&dqm->asic_ops);
2571                 else {
2572                         WARN(1, "Unexpected ASIC family %u",
2573                              dev->adev->asic_type);
2574                         goto out_free;
2575                 }
2576         }
2577
2578         if (init_mqd_managers(dqm))
2579                 goto out_free;
2580
2581         if (!dev->kfd->shared_resources.enable_mes && allocate_hiq_sdma_mqd(dqm)) {
2582                 pr_err("Failed to allocate hiq sdma mqd trunk buffer\n");
2583                 goto out_free;
2584         }
2585
2586         if (!dqm->ops.initialize(dqm)) {
2587                 init_waitqueue_head(&dqm->destroy_wait);
2588                 return dqm;
2589         }
2590
2591 out_free:
2592         kfree(dqm);
2593         return NULL;
2594 }
2595
2596 static void deallocate_hiq_sdma_mqd(struct kfd_node *dev,
2597                                     struct kfd_mem_obj *mqd)
2598 {
2599         WARN(!mqd, "No hiq sdma mqd trunk to free");
2600
2601         amdgpu_amdkfd_free_gtt_mem(dev->adev, mqd->gtt_mem);
2602 }
2603
2604 void device_queue_manager_uninit(struct device_queue_manager *dqm)
2605 {
2606         dqm->ops.stop(dqm);
2607         dqm->ops.uninitialize(dqm);
2608         if (!dqm->dev->kfd->shared_resources.enable_mes)
2609                 deallocate_hiq_sdma_mqd(dqm->dev, &dqm->hiq_sdma_mqd);
2610         kfree(dqm);
2611 }
2612
2613 int kfd_dqm_evict_pasid(struct device_queue_manager *dqm, u32 pasid)
2614 {
2615         struct kfd_process_device *pdd;
2616         struct kfd_process *p = kfd_lookup_process_by_pasid(pasid);
2617         int ret = 0;
2618
2619         if (!p)
2620                 return -EINVAL;
2621         WARN(debug_evictions, "Evicting pid %d", p->lead_thread->pid);
2622         pdd = kfd_get_process_device_data(dqm->dev, p);
2623         if (pdd)
2624                 ret = dqm->ops.evict_process_queues(dqm, &pdd->qpd);
2625         kfd_unref_process(p);
2626
2627         return ret;
2628 }
2629
2630 static void kfd_process_hw_exception(struct work_struct *work)
2631 {
2632         struct device_queue_manager *dqm = container_of(work,
2633                         struct device_queue_manager, hw_exception_work);
2634         amdgpu_amdkfd_gpu_reset(dqm->dev->adev);
2635 }
2636
2637 int reserve_debug_trap_vmid(struct device_queue_manager *dqm,
2638                                 struct qcm_process_device *qpd)
2639 {
2640         int r;
2641         int updated_vmid_mask;
2642
2643         if (dqm->sched_policy == KFD_SCHED_POLICY_NO_HWS) {
2644                 pr_err("Unsupported on sched_policy: %i\n", dqm->sched_policy);
2645                 return -EINVAL;
2646         }
2647
2648         dqm_lock(dqm);
2649
2650         if (dqm->trap_debug_vmid != 0) {
2651                 pr_err("Trap debug id already reserved\n");
2652                 r = -EBUSY;
2653                 goto out_unlock;
2654         }
2655
2656         r = unmap_queues_cpsch(dqm, KFD_UNMAP_QUEUES_FILTER_ALL_QUEUES, 0,
2657                         USE_DEFAULT_GRACE_PERIOD, false);
2658         if (r)
2659                 goto out_unlock;
2660
2661         updated_vmid_mask = dqm->dev->kfd->shared_resources.compute_vmid_bitmap;
2662         updated_vmid_mask &= ~(1 << dqm->dev->vm_info.last_vmid_kfd);
2663
2664         dqm->dev->kfd->shared_resources.compute_vmid_bitmap = updated_vmid_mask;
2665         dqm->trap_debug_vmid = dqm->dev->vm_info.last_vmid_kfd;
2666         r = set_sched_resources(dqm);
2667         if (r)
2668                 goto out_unlock;
2669
2670         r = map_queues_cpsch(dqm);
2671         if (r)
2672                 goto out_unlock;
2673
2674         pr_debug("Reserved VMID for trap debug: %i\n", dqm->trap_debug_vmid);
2675
2676 out_unlock:
2677         dqm_unlock(dqm);
2678         return r;
2679 }
2680
2681 /*
2682  * Releases vmid for the trap debugger
2683  */
2684 int release_debug_trap_vmid(struct device_queue_manager *dqm,
2685                         struct qcm_process_device *qpd)
2686 {
2687         int r;
2688         int updated_vmid_mask;
2689         uint32_t trap_debug_vmid;
2690
2691         if (dqm->sched_policy == KFD_SCHED_POLICY_NO_HWS) {
2692                 pr_err("Unsupported on sched_policy: %i\n", dqm->sched_policy);
2693                 return -EINVAL;
2694         }
2695
2696         dqm_lock(dqm);
2697         trap_debug_vmid = dqm->trap_debug_vmid;
2698         if (dqm->trap_debug_vmid == 0) {
2699                 pr_err("Trap debug id is not reserved\n");
2700                 r = -EINVAL;
2701                 goto out_unlock;
2702         }
2703
2704         r = unmap_queues_cpsch(dqm, KFD_UNMAP_QUEUES_FILTER_ALL_QUEUES, 0,
2705                         USE_DEFAULT_GRACE_PERIOD, false);
2706         if (r)
2707                 goto out_unlock;
2708
2709         updated_vmid_mask = dqm->dev->kfd->shared_resources.compute_vmid_bitmap;
2710         updated_vmid_mask |= (1 << dqm->dev->vm_info.last_vmid_kfd);
2711
2712         dqm->dev->kfd->shared_resources.compute_vmid_bitmap = updated_vmid_mask;
2713         dqm->trap_debug_vmid = 0;
2714         r = set_sched_resources(dqm);
2715         if (r)
2716                 goto out_unlock;
2717
2718         r = map_queues_cpsch(dqm);
2719         if (r)
2720                 goto out_unlock;
2721
2722         pr_debug("Released VMID for trap debug: %i\n", trap_debug_vmid);
2723
2724 out_unlock:
2725         dqm_unlock(dqm);
2726         return r;
2727 }
2728
2729 #define QUEUE_NOT_FOUND         -1
2730 /* invalidate queue operation in array */
2731 static void q_array_invalidate(uint32_t num_queues, uint32_t *queue_ids)
2732 {
2733         int i;
2734
2735         for (i = 0; i < num_queues; i++)
2736                 queue_ids[i] |= KFD_DBG_QUEUE_INVALID_MASK;
2737 }
2738
2739 /* find queue index in array */
2740 static int q_array_get_index(unsigned int queue_id,
2741                 uint32_t num_queues,
2742                 uint32_t *queue_ids)
2743 {
2744         int i;
2745
2746         for (i = 0; i < num_queues; i++)
2747                 if (queue_id == (queue_ids[i] & ~KFD_DBG_QUEUE_INVALID_MASK))
2748                         return i;
2749
2750         return QUEUE_NOT_FOUND;
2751 }
2752
2753 struct copy_context_work_handler_workarea {
2754         struct work_struct copy_context_work;
2755         struct kfd_process *p;
2756 };
2757
2758 static void copy_context_work_handler (struct work_struct *work)
2759 {
2760         struct copy_context_work_handler_workarea *workarea;
2761         struct mqd_manager *mqd_mgr;
2762         struct queue *q;
2763         struct mm_struct *mm;
2764         struct kfd_process *p;
2765         uint32_t tmp_ctl_stack_used_size, tmp_save_area_used_size;
2766         int i;
2767
2768         workarea = container_of(work,
2769                         struct copy_context_work_handler_workarea,
2770                         copy_context_work);
2771
2772         p = workarea->p;
2773         mm = get_task_mm(p->lead_thread);
2774
2775         if (!mm)
2776                 return;
2777
2778         kthread_use_mm(mm);
2779         for (i = 0; i < p->n_pdds; i++) {
2780                 struct kfd_process_device *pdd = p->pdds[i];
2781                 struct device_queue_manager *dqm = pdd->dev->dqm;
2782                 struct qcm_process_device *qpd = &pdd->qpd;
2783
2784                 list_for_each_entry(q, &qpd->queues_list, list) {
2785                         mqd_mgr = dqm->mqd_mgrs[KFD_MQD_TYPE_CP];
2786
2787                         /* We ignore the return value from get_wave_state
2788                          * because
2789                          * i) right now, it always returns 0, and
2790                          * ii) if we hit an error, we would continue to the
2791                          *      next queue anyway.
2792                          */
2793                         mqd_mgr->get_wave_state(mqd_mgr,
2794                                         q->mqd,
2795                                         &q->properties,
2796                                         (void __user *) q->properties.ctx_save_restore_area_address,
2797                                         &tmp_ctl_stack_used_size,
2798                                         &tmp_save_area_used_size);
2799                 }
2800         }
2801         kthread_unuse_mm(mm);
2802         mmput(mm);
2803 }
2804
2805 static uint32_t *get_queue_ids(uint32_t num_queues, uint32_t *usr_queue_id_array)
2806 {
2807         size_t array_size = num_queues * sizeof(uint32_t);
2808         uint32_t *queue_ids = NULL;
2809
2810         if (!usr_queue_id_array)
2811                 return NULL;
2812
2813         queue_ids = kzalloc(array_size, GFP_KERNEL);
2814         if (!queue_ids)
2815                 return ERR_PTR(-ENOMEM);
2816
2817         if (copy_from_user(queue_ids, usr_queue_id_array, array_size))
2818                 return ERR_PTR(-EFAULT);
2819
2820         return queue_ids;
2821 }
2822
2823 int resume_queues(struct kfd_process *p,
2824                 uint32_t num_queues,
2825                 uint32_t *usr_queue_id_array)
2826 {
2827         uint32_t *queue_ids = NULL;
2828         int total_resumed = 0;
2829         int i;
2830
2831         if (usr_queue_id_array) {
2832                 queue_ids = get_queue_ids(num_queues, usr_queue_id_array);
2833
2834                 if (IS_ERR(queue_ids))
2835                         return PTR_ERR(queue_ids);
2836
2837                 /* mask all queues as invalid.  unmask per successful request */
2838                 q_array_invalidate(num_queues, queue_ids);
2839         }
2840
2841         for (i = 0; i < p->n_pdds; i++) {
2842                 struct kfd_process_device *pdd = p->pdds[i];
2843                 struct device_queue_manager *dqm = pdd->dev->dqm;
2844                 struct qcm_process_device *qpd = &pdd->qpd;
2845                 struct queue *q;
2846                 int r, per_device_resumed = 0;
2847
2848                 dqm_lock(dqm);
2849
2850                 /* unmask queues that resume or already resumed as valid */
2851                 list_for_each_entry(q, &qpd->queues_list, list) {
2852                         int q_idx = QUEUE_NOT_FOUND;
2853
2854                         if (queue_ids)
2855                                 q_idx = q_array_get_index(
2856                                                 q->properties.queue_id,
2857                                                 num_queues,
2858                                                 queue_ids);
2859
2860                         if (!queue_ids || q_idx != QUEUE_NOT_FOUND) {
2861                                 int err = resume_single_queue(dqm, &pdd->qpd, q);
2862
2863                                 if (queue_ids) {
2864                                         if (!err) {
2865                                                 queue_ids[q_idx] &=
2866                                                         ~KFD_DBG_QUEUE_INVALID_MASK;
2867                                         } else {
2868                                                 queue_ids[q_idx] |=
2869                                                         KFD_DBG_QUEUE_ERROR_MASK;
2870                                                 break;
2871                                         }
2872                                 }
2873
2874                                 if (dqm->dev->kfd->shared_resources.enable_mes) {
2875                                         wake_up_all(&dqm->destroy_wait);
2876                                         if (!err)
2877                                                 total_resumed++;
2878                                 } else {
2879                                         per_device_resumed++;
2880                                 }
2881                         }
2882                 }
2883
2884                 if (!per_device_resumed) {
2885                         dqm_unlock(dqm);
2886                         continue;
2887                 }
2888
2889                 r = execute_queues_cpsch(dqm,
2890                                         KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES,
2891                                         0,
2892                                         USE_DEFAULT_GRACE_PERIOD);
2893                 if (r) {
2894                         pr_err("Failed to resume process queues\n");
2895                         if (queue_ids) {
2896                                 list_for_each_entry(q, &qpd->queues_list, list) {
2897                                         int q_idx = q_array_get_index(
2898                                                         q->properties.queue_id,
2899                                                         num_queues,
2900                                                         queue_ids);
2901
2902                                         /* mask queue as error on resume fail */
2903                                         if (q_idx != QUEUE_NOT_FOUND)
2904                                                 queue_ids[q_idx] |=
2905                                                         KFD_DBG_QUEUE_ERROR_MASK;
2906                                 }
2907                         }
2908                 } else {
2909                         wake_up_all(&dqm->destroy_wait);
2910                         total_resumed += per_device_resumed;
2911                 }
2912
2913                 dqm_unlock(dqm);
2914         }
2915
2916         if (queue_ids) {
2917                 if (copy_to_user((void __user *)usr_queue_id_array, queue_ids,
2918                                 num_queues * sizeof(uint32_t)))
2919                         pr_err("copy_to_user failed on queue resume\n");
2920
2921                 kfree(queue_ids);
2922         }
2923
2924         return total_resumed;
2925 }
2926
2927 int suspend_queues(struct kfd_process *p,
2928                         uint32_t num_queues,
2929                         uint32_t grace_period,
2930                         uint64_t exception_clear_mask,
2931                         uint32_t *usr_queue_id_array)
2932 {
2933         uint32_t *queue_ids = get_queue_ids(num_queues, usr_queue_id_array);
2934         int total_suspended = 0;
2935         int i;
2936
2937         if (IS_ERR(queue_ids))
2938                 return PTR_ERR(queue_ids);
2939
2940         /* mask all queues as invalid.  umask on successful request */
2941         q_array_invalidate(num_queues, queue_ids);
2942
2943         for (i = 0; i < p->n_pdds; i++) {
2944                 struct kfd_process_device *pdd = p->pdds[i];
2945                 struct device_queue_manager *dqm = pdd->dev->dqm;
2946                 struct qcm_process_device *qpd = &pdd->qpd;
2947                 struct queue *q;
2948                 int r, per_device_suspended = 0;
2949
2950                 mutex_lock(&p->event_mutex);
2951                 dqm_lock(dqm);
2952
2953                 /* unmask queues that suspend or already suspended */
2954                 list_for_each_entry(q, &qpd->queues_list, list) {
2955                         int q_idx = q_array_get_index(q->properties.queue_id,
2956                                                         num_queues,
2957                                                         queue_ids);
2958
2959                         if (q_idx != QUEUE_NOT_FOUND) {
2960                                 int err = suspend_single_queue(dqm, pdd, q);
2961                                 bool is_mes = dqm->dev->kfd->shared_resources.enable_mes;
2962
2963                                 if (!err) {
2964                                         queue_ids[q_idx] &= ~KFD_DBG_QUEUE_INVALID_MASK;
2965                                         if (exception_clear_mask && is_mes)
2966                                                 q->properties.exception_status &=
2967                                                         ~exception_clear_mask;
2968
2969                                         if (is_mes)
2970                                                 total_suspended++;
2971                                         else
2972                                                 per_device_suspended++;
2973                                 } else if (err != -EBUSY) {
2974                                         r = err;
2975                                         queue_ids[q_idx] |= KFD_DBG_QUEUE_ERROR_MASK;
2976                                         break;
2977                                 }
2978                         }
2979                 }
2980
2981                 if (!per_device_suspended) {
2982                         dqm_unlock(dqm);
2983                         mutex_unlock(&p->event_mutex);
2984                         if (total_suspended)
2985                                 amdgpu_amdkfd_debug_mem_fence(dqm->dev->adev);
2986                         continue;
2987                 }
2988
2989                 r = execute_queues_cpsch(dqm,
2990                         KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES, 0,
2991                         grace_period);
2992
2993                 if (r)
2994                         pr_err("Failed to suspend process queues.\n");
2995                 else
2996                         total_suspended += per_device_suspended;
2997
2998                 list_for_each_entry(q, &qpd->queues_list, list) {
2999                         int q_idx = q_array_get_index(q->properties.queue_id,
3000                                                 num_queues, queue_ids);
3001
3002                         if (q_idx == QUEUE_NOT_FOUND)
3003                                 continue;
3004
3005                         /* mask queue as error on suspend fail */
3006                         if (r)
3007                                 queue_ids[q_idx] |= KFD_DBG_QUEUE_ERROR_MASK;
3008                         else if (exception_clear_mask)
3009                                 q->properties.exception_status &=
3010                                                         ~exception_clear_mask;
3011                 }
3012
3013                 dqm_unlock(dqm);
3014                 mutex_unlock(&p->event_mutex);
3015                 amdgpu_device_flush_hdp(dqm->dev->adev, NULL);
3016         }
3017
3018         if (total_suspended) {
3019                 struct copy_context_work_handler_workarea copy_context_worker;
3020
3021                 INIT_WORK_ONSTACK(
3022                                 &copy_context_worker.copy_context_work,
3023                                 copy_context_work_handler);
3024
3025                 copy_context_worker.p = p;
3026
3027                 schedule_work(&copy_context_worker.copy_context_work);
3028
3029
3030                 flush_work(&copy_context_worker.copy_context_work);
3031                 destroy_work_on_stack(&copy_context_worker.copy_context_work);
3032         }
3033
3034         if (copy_to_user((void __user *)usr_queue_id_array, queue_ids,
3035                         num_queues * sizeof(uint32_t)))
3036                 pr_err("copy_to_user failed on queue suspend\n");
3037
3038         kfree(queue_ids);
3039
3040         return total_suspended;
3041 }
3042
3043 static uint32_t set_queue_type_for_user(struct queue_properties *q_props)
3044 {
3045         switch (q_props->type) {
3046         case KFD_QUEUE_TYPE_COMPUTE:
3047                 return q_props->format == KFD_QUEUE_FORMAT_PM4
3048                                         ? KFD_IOC_QUEUE_TYPE_COMPUTE
3049                                         : KFD_IOC_QUEUE_TYPE_COMPUTE_AQL;
3050         case KFD_QUEUE_TYPE_SDMA:
3051                 return KFD_IOC_QUEUE_TYPE_SDMA;
3052         case KFD_QUEUE_TYPE_SDMA_XGMI:
3053                 return KFD_IOC_QUEUE_TYPE_SDMA_XGMI;
3054         default:
3055                 WARN_ONCE(true, "queue type not recognized!");
3056                 return 0xffffffff;
3057         };
3058 }
3059
3060 void set_queue_snapshot_entry(struct queue *q,
3061                               uint64_t exception_clear_mask,
3062                               struct kfd_queue_snapshot_entry *qss_entry)
3063 {
3064         qss_entry->ring_base_address = q->properties.queue_address;
3065         qss_entry->write_pointer_address = (uint64_t)q->properties.write_ptr;
3066         qss_entry->read_pointer_address = (uint64_t)q->properties.read_ptr;
3067         qss_entry->ctx_save_restore_address =
3068                                 q->properties.ctx_save_restore_area_address;
3069         qss_entry->ctx_save_restore_area_size =
3070                                 q->properties.ctx_save_restore_area_size;
3071         qss_entry->exception_status = q->properties.exception_status;
3072         qss_entry->queue_id = q->properties.queue_id;
3073         qss_entry->gpu_id = q->device->id;
3074         qss_entry->ring_size = (uint32_t)q->properties.queue_size;
3075         qss_entry->queue_type = set_queue_type_for_user(&q->properties);
3076         q->properties.exception_status &= ~exception_clear_mask;
3077 }
3078
3079 int debug_lock_and_unmap(struct device_queue_manager *dqm)
3080 {
3081         int r;
3082
3083         if (dqm->sched_policy == KFD_SCHED_POLICY_NO_HWS) {
3084                 pr_err("Unsupported on sched_policy: %i\n", dqm->sched_policy);
3085                 return -EINVAL;
3086         }
3087
3088         if (!kfd_dbg_is_per_vmid_supported(dqm->dev))
3089                 return 0;
3090
3091         dqm_lock(dqm);
3092
3093         r = unmap_queues_cpsch(dqm, KFD_UNMAP_QUEUES_FILTER_ALL_QUEUES, 0, 0, false);
3094         if (r)
3095                 dqm_unlock(dqm);
3096
3097         return r;
3098 }
3099
3100 int debug_map_and_unlock(struct device_queue_manager *dqm)
3101 {
3102         int r;
3103
3104         if (dqm->sched_policy == KFD_SCHED_POLICY_NO_HWS) {
3105                 pr_err("Unsupported on sched_policy: %i\n", dqm->sched_policy);
3106                 return -EINVAL;
3107         }
3108
3109         if (!kfd_dbg_is_per_vmid_supported(dqm->dev))
3110                 return 0;
3111
3112         r = map_queues_cpsch(dqm);
3113
3114         dqm_unlock(dqm);
3115
3116         return r;
3117 }
3118
3119 int debug_refresh_runlist(struct device_queue_manager *dqm)
3120 {
3121         int r = debug_lock_and_unmap(dqm);
3122
3123         if (r)
3124                 return r;
3125
3126         return debug_map_and_unlock(dqm);
3127 }
3128
3129 #if defined(CONFIG_DEBUG_FS)
3130
3131 static void seq_reg_dump(struct seq_file *m,
3132                          uint32_t (*dump)[2], uint32_t n_regs)
3133 {
3134         uint32_t i, count;
3135
3136         for (i = 0, count = 0; i < n_regs; i++) {
3137                 if (count == 0 ||
3138                     dump[i-1][0] + sizeof(uint32_t) != dump[i][0]) {
3139                         seq_printf(m, "%s    %08x: %08x",
3140                                    i ? "\n" : "",
3141                                    dump[i][0], dump[i][1]);
3142                         count = 7;
3143                 } else {
3144                         seq_printf(m, " %08x", dump[i][1]);
3145                         count--;
3146                 }
3147         }
3148
3149         seq_puts(m, "\n");
3150 }
3151
3152 int dqm_debugfs_hqds(struct seq_file *m, void *data)
3153 {
3154         struct device_queue_manager *dqm = data;
3155         uint32_t xcc_mask = dqm->dev->xcc_mask;
3156         uint32_t (*dump)[2], n_regs;
3157         int pipe, queue;
3158         int r = 0, xcc_id;
3159         uint32_t sdma_engine_start;
3160
3161         if (!dqm->sched_running) {
3162                 seq_puts(m, " Device is stopped\n");
3163                 return 0;
3164         }
3165
3166         for_each_inst(xcc_id, xcc_mask) {
3167                 r = dqm->dev->kfd2kgd->hqd_dump(dqm->dev->adev,
3168                                                 KFD_CIK_HIQ_PIPE,
3169                                                 KFD_CIK_HIQ_QUEUE, &dump,
3170                                                 &n_regs, xcc_id);
3171                 if (!r) {
3172                         seq_printf(
3173                                 m,
3174                                 "   Inst %d, HIQ on MEC %d Pipe %d Queue %d\n",
3175                                 xcc_id,
3176                                 KFD_CIK_HIQ_PIPE / get_pipes_per_mec(dqm) + 1,
3177                                 KFD_CIK_HIQ_PIPE % get_pipes_per_mec(dqm),
3178                                 KFD_CIK_HIQ_QUEUE);
3179                         seq_reg_dump(m, dump, n_regs);
3180
3181                         kfree(dump);
3182                 }
3183
3184                 for (pipe = 0; pipe < get_pipes_per_mec(dqm); pipe++) {
3185                         int pipe_offset = pipe * get_queues_per_pipe(dqm);
3186
3187                         for (queue = 0; queue < get_queues_per_pipe(dqm); queue++) {
3188                                 if (!test_bit(pipe_offset + queue,
3189                                       dqm->dev->kfd->shared_resources.cp_queue_bitmap))
3190                                         continue;
3191
3192                                 r = dqm->dev->kfd2kgd->hqd_dump(dqm->dev->adev,
3193                                                                 pipe, queue,
3194                                                                 &dump, &n_regs,
3195                                                                 xcc_id);
3196                                 if (r)
3197                                         break;
3198
3199                                 seq_printf(m,
3200                                            " Inst %d,  CP Pipe %d, Queue %d\n",
3201                                            xcc_id, pipe, queue);
3202                                 seq_reg_dump(m, dump, n_regs);
3203
3204                                 kfree(dump);
3205                         }
3206                 }
3207         }
3208
3209         sdma_engine_start = dqm->dev->node_id * get_num_all_sdma_engines(dqm);
3210         for (pipe = sdma_engine_start;
3211              pipe < (sdma_engine_start + get_num_all_sdma_engines(dqm));
3212              pipe++) {
3213                 for (queue = 0;
3214                      queue < dqm->dev->kfd->device_info.num_sdma_queues_per_engine;
3215                      queue++) {
3216                         r = dqm->dev->kfd2kgd->hqd_sdma_dump(
3217                                 dqm->dev->adev, pipe, queue, &dump, &n_regs);
3218                         if (r)
3219                                 break;
3220
3221                         seq_printf(m, "  SDMA Engine %d, RLC %d\n",
3222                                   pipe, queue);
3223                         seq_reg_dump(m, dump, n_regs);
3224
3225                         kfree(dump);
3226                 }
3227         }
3228
3229         return r;
3230 }
3231
3232 int dqm_debugfs_hang_hws(struct device_queue_manager *dqm)
3233 {
3234         int r = 0;
3235
3236         dqm_lock(dqm);
3237         r = pm_debugfs_hang_hws(&dqm->packet_mgr);
3238         if (r) {
3239                 dqm_unlock(dqm);
3240                 return r;
3241         }
3242         dqm->active_runlist = true;
3243         r = execute_queues_cpsch(dqm, KFD_UNMAP_QUEUES_FILTER_ALL_QUEUES,
3244                                 0, USE_DEFAULT_GRACE_PERIOD);
3245         dqm_unlock(dqm);
3246
3247         return r;
3248 }
3249
3250 #endif