drivers/accel/ivpu/ivpu_job.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * Copyright (C) 2020-2023 Intel Corporation
   4  */
   5
   6 #include <drm/drm_file.h>
   7
   8 #include <linux/bitfield.h>
   9 #include <linux/highmem.h>
  10 #include <linux/kthread.h>
  11 #include <linux/pci.h>
  12 #include <linux/module.h>
  13 #include <uapi/drm/ivpu_accel.h>
  14
  15 #include "ivpu_drv.h"
  16 #include "ivpu_hw.h"
  17 #include "ivpu_ipc.h"
  18 #include "ivpu_job.h"
  19 #include "ivpu_jsm_msg.h"
  20 #include "ivpu_pm.h"
  21
  22 #define CMD_BUF_IDX          0
  23 #define JOB_ID_JOB_MASK      GENMASK(7, 0)
  24 #define JOB_ID_CONTEXT_MASK  GENMASK(31, 8)
  25 #define JOB_MAX_BUFFER_COUNT 65535
  26
  27 static unsigned int ivpu_tdr_timeout_ms;
  28 module_param_named(tdr_timeout_ms, ivpu_tdr_timeout_ms, uint, 0644);
  29 MODULE_PARM_DESC(tdr_timeout_ms, "Timeout for device hang detection, in milliseconds, 0 - default");
  30
  31 static void ivpu_cmdq_ring_db(struct ivpu_device *vdev, struct ivpu_cmdq *cmdq)
  32 {
  33         ivpu_hw_reg_db_set(vdev, cmdq->db_id);
  34 }
  35
  36 static struct ivpu_cmdq *ivpu_cmdq_alloc(struct ivpu_file_priv *file_priv, u16 engine)
  37 {
  38         struct ivpu_device *vdev = file_priv->vdev;
  39         struct vpu_job_queue_header *jobq_header;
  40         struct ivpu_cmdq *cmdq;
  41
  42         cmdq = kzalloc(sizeof(*cmdq), GFP_KERNEL);
  43         if (!cmdq)
  44                 return NULL;
  45
  46         cmdq->mem = ivpu_bo_alloc_internal(vdev, 0, SZ_4K, DRM_IVPU_BO_WC);
  47         if (!cmdq->mem)
  48                 goto cmdq_free;
  49
  50         cmdq->db_id = file_priv->ctx.id + engine * ivpu_get_context_count(vdev);
  51         cmdq->entry_count = (u32)((cmdq->mem->base.size - sizeof(struct vpu_job_queue_header)) /
  52                                   sizeof(struct vpu_job_queue_entry));
  53
  54         cmdq->jobq = (struct vpu_job_queue *)cmdq->mem->kvaddr;
  55         jobq_header = &cmdq->jobq->header;
  56         jobq_header->engine_idx = engine;
  57         jobq_header->head = 0;
  58         jobq_header->tail = 0;
  59         wmb(); /* Flush WC buffer for jobq->header */
  60
  61         return cmdq;
  62
  63 cmdq_free:
  64         kfree(cmdq);
  65         return NULL;
  66 }
  67
  68 static void ivpu_cmdq_free(struct ivpu_file_priv *file_priv, struct ivpu_cmdq *cmdq)
  69 {
  70         if (!cmdq)
  71                 return;
  72
  73         ivpu_bo_free_internal(cmdq->mem);
  74         kfree(cmdq);
  75 }
  76
  77 static struct ivpu_cmdq *ivpu_cmdq_acquire(struct ivpu_file_priv *file_priv, u16 engine)
  78 {
  79         struct ivpu_device *vdev = file_priv->vdev;
  80         struct ivpu_cmdq *cmdq = file_priv->cmdq[engine];
  81         int ret;
  82
  83         lockdep_assert_held(&file_priv->lock);
  84
  85         if (!cmdq) {
  86                 cmdq = ivpu_cmdq_alloc(file_priv, engine);
  87                 if (!cmdq)
  88                         return NULL;
  89                 file_priv->cmdq[engine] = cmdq;
  90         }
  91
  92         if (cmdq->db_registered)
  93                 return cmdq;
  94
  95         ret = ivpu_jsm_register_db(vdev, file_priv->ctx.id, cmdq->db_id,
  96                                    cmdq->mem->vpu_addr, cmdq->mem->base.size);
  97         if (ret)
  98                 return NULL;
  99
 100         cmdq->db_registered = true;
 101
 102         return cmdq;
 103 }
 104
 105 static void ivpu_cmdq_release_locked(struct ivpu_file_priv *file_priv, u16 engine)
 106 {
 107         struct ivpu_cmdq *cmdq = file_priv->cmdq[engine];
 108
 109         lockdep_assert_held(&file_priv->lock);
 110
 111         if (cmdq) {
 112                 file_priv->cmdq[engine] = NULL;
 113                 if (cmdq->db_registered)
 114                         ivpu_jsm_unregister_db(file_priv->vdev, cmdq->db_id);
 115
 116                 ivpu_cmdq_free(file_priv, cmdq);
 117         }
 118 }
 119
 120 void ivpu_cmdq_release_all(struct ivpu_file_priv *file_priv)
 121 {
 122         int i;
 123
 124         mutex_lock(&file_priv->lock);
 125
 126         for (i = 0; i < IVPU_NUM_ENGINES; i++)
 127                 ivpu_cmdq_release_locked(file_priv, i);
 128
 129         mutex_unlock(&file_priv->lock);
 130 }
 131
 132 /*
 133  * Mark the doorbell as unregistered and reset job queue pointers.
 134  * This function needs to be called when the VPU hardware is restarted
 135  * and FW looses job queue state. The next time job queue is used it
 136  * will be registered again.
 137  */
 138 static void ivpu_cmdq_reset_locked(struct ivpu_file_priv *file_priv, u16 engine)
 139 {
 140         struct ivpu_cmdq *cmdq = file_priv->cmdq[engine];
 141
 142         lockdep_assert_held(&file_priv->lock);
 143
 144         if (cmdq) {
 145                 cmdq->db_registered = false;
 146                 cmdq->jobq->header.head = 0;
 147                 cmdq->jobq->header.tail = 0;
 148                 wmb(); /* Flush WC buffer for jobq header */
 149         }
 150 }
 151
 152 static void ivpu_cmdq_reset_all(struct ivpu_file_priv *file_priv)
 153 {
 154         int i;
 155
 156         mutex_lock(&file_priv->lock);
 157
 158         for (i = 0; i < IVPU_NUM_ENGINES; i++)
 159                 ivpu_cmdq_reset_locked(file_priv, i);
 160
 161         mutex_unlock(&file_priv->lock);
 162 }
 163
 164 void ivpu_cmdq_reset_all_contexts(struct ivpu_device *vdev)
 165 {
 166         struct ivpu_file_priv *file_priv;
 167         unsigned long ctx_id;
 168
 169         xa_for_each(&vdev->context_xa, ctx_id, file_priv) {
 170                 file_priv = ivpu_file_priv_get_by_ctx_id(vdev, ctx_id);
 171                 if (!file_priv)
 172                         continue;
 173
 174                 ivpu_cmdq_reset_all(file_priv);
 175
 176                 ivpu_file_priv_put(&file_priv);
 177         }
 178 }
 179
 180 static int ivpu_cmdq_push_job(struct ivpu_cmdq *cmdq, struct ivpu_job *job)
 181 {
 182         struct ivpu_device *vdev = job->vdev;
 183         struct vpu_job_queue_header *header = &cmdq->jobq->header;
 184         struct vpu_job_queue_entry *entry;
 185         u32 tail = READ_ONCE(header->tail);
 186         u32 next_entry = (tail + 1) % cmdq->entry_count;
 187
 188         /* Check if there is space left in job queue */
 189         if (next_entry == header->head) {
 190                 ivpu_dbg(vdev, JOB, "Job queue full: ctx %d engine %d db %d head %d tail %d\n",
 191                          job->file_priv->ctx.id, job->engine_idx, cmdq->db_id, header->head, tail);
 192                 return -EBUSY;
 193         }
 194
 195         entry = &cmdq->jobq->job[tail];
 196         entry->batch_buf_addr = job->cmd_buf_vpu_addr;
 197         entry->job_id = job->job_id;
 198         entry->flags = 0;
 199         wmb(); /* Ensure that tail is updated after filling entry */
 200         header->tail = next_entry;
 201         wmb(); /* Flush WC buffer for jobq header */
 202
 203         return 0;
 204 }
 205
 206 struct ivpu_fence {
 207         struct dma_fence base;
 208         spinlock_t lock; /* protects base */
 209         struct ivpu_device *vdev;
 210 };
 211
 212 static inline struct ivpu_fence *to_vpu_fence(struct dma_fence *fence)
 213 {
 214         return container_of(fence, struct ivpu_fence, base);
 215 }
 216
 217 static const char *ivpu_fence_get_driver_name(struct dma_fence *fence)
 218 {
 219         return DRIVER_NAME;
 220 }
 221
 222 static const char *ivpu_fence_get_timeline_name(struct dma_fence *fence)
 223 {
 224         struct ivpu_fence *ivpu_fence = to_vpu_fence(fence);
 225
 226         return dev_name(ivpu_fence->vdev->drm.dev);
 227 }
 228
 229 static const struct dma_fence_ops ivpu_fence_ops = {
 230         .get_driver_name = ivpu_fence_get_driver_name,
 231         .get_timeline_name = ivpu_fence_get_timeline_name,
 232 };
 233
 234 static struct dma_fence *ivpu_fence_create(struct ivpu_device *vdev)
 235 {
 236         struct ivpu_fence *fence;
 237
 238         fence = kzalloc(sizeof(*fence), GFP_KERNEL);
 239         if (!fence)
 240                 return NULL;
 241
 242         fence->vdev = vdev;
 243         spin_lock_init(&fence->lock);
 244         dma_fence_init(&fence->base, &ivpu_fence_ops, &fence->lock, dma_fence_context_alloc(1), 1);
 245
 246         return &fence->base;
 247 }
 248
 249 static void job_get(struct ivpu_job *job, struct ivpu_job **link)
 250 {
 251         struct ivpu_device *vdev = job->vdev;
 252
 253         kref_get(&job->ref);
 254         *link = job;
 255
 256         ivpu_dbg(vdev, KREF, "Job get: id %u refcount %u\n", job->job_id, kref_read(&job->ref));
 257 }
 258
 259 static void job_release(struct kref *ref)
 260 {
 261         struct ivpu_job *job = container_of(ref, struct ivpu_job, ref);
 262         struct ivpu_device *vdev = job->vdev;
 263         u32 i;
 264
 265         for (i = 0; i < job->bo_count; i++)
 266                 if (job->bos[i])
 267                         drm_gem_object_put(&job->bos[i]->base);
 268
 269         dma_fence_put(job->done_fence);
 270         ivpu_file_priv_put(&job->file_priv);
 271
 272         ivpu_dbg(vdev, KREF, "Job released: id %u\n", job->job_id);
 273         kfree(job);
 274
 275         /* Allow the VPU to get suspended, must be called after ivpu_file_priv_put() */
 276         ivpu_rpm_put(vdev);
 277 }
 278
 279 static void job_put(struct ivpu_job *job)
 280 {
 281         struct ivpu_device *vdev = job->vdev;
 282
 283         ivpu_dbg(vdev, KREF, "Job put: id %u refcount %u\n", job->job_id, kref_read(&job->ref));
 284         kref_put(&job->ref, job_release);
 285 }
 286
 287 static struct ivpu_job *
 288 ivpu_create_job(struct ivpu_file_priv *file_priv, u32 engine_idx, u32 bo_count)
 289 {
 290         struct ivpu_device *vdev = file_priv->vdev;
 291         struct ivpu_job *job;
 292         int ret;
 293
 294         ret = ivpu_rpm_get(vdev);
 295         if (ret < 0)
 296                 return NULL;
 297
 298         job = kzalloc(struct_size(job, bos, bo_count), GFP_KERNEL);
 299         if (!job)
 300                 goto err_rpm_put;
 301
 302         kref_init(&job->ref);
 303
 304         job->vdev = vdev;
 305         job->engine_idx = engine_idx;
 306         job->bo_count = bo_count;
 307         job->done_fence = ivpu_fence_create(vdev);
 308         if (!job->done_fence) {
 309                 ivpu_warn_ratelimited(vdev, "Failed to create a fence\n");
 310                 goto err_free_job;
 311         }
 312
 313         job->file_priv = ivpu_file_priv_get(file_priv);
 314
 315         ivpu_dbg(vdev, JOB, "Job created: ctx %2d engine %d", file_priv->ctx.id, job->engine_idx);
 316
 317         return job;
 318
 319 err_free_job:
 320         kfree(job);
 321 err_rpm_put:
 322         ivpu_rpm_put(vdev);
 323         return NULL;
 324 }
 325
 326 static int ivpu_job_done(struct ivpu_device *vdev, u32 job_id, u32 job_status)
 327 {
 328         struct ivpu_job *job;
 329
 330         job = xa_erase(&vdev->submitted_jobs_xa, job_id);
 331         if (!job)
 332                 return -ENOENT;
 333
 334         if (job->file_priv->has_mmu_faults)
 335                 job_status = VPU_JSM_STATUS_ABORTED;
 336
 337         job->bos[CMD_BUF_IDX]->job_status = job_status;
 338         dma_fence_signal(job->done_fence);
 339
 340         ivpu_dbg(vdev, JOB, "Job complete:  id %3u ctx %2d engine %d status 0x%x\n",
 341                  job->job_id, job->file_priv->ctx.id, job->engine_idx, job_status);
 342
 343         job_put(job);
 344         return 0;
 345 }
 346
 347 static void ivpu_job_done_message(struct ivpu_device *vdev, void *msg)
 348 {
 349         struct vpu_ipc_msg_payload_job_done *payload;
 350         struct vpu_jsm_msg *job_ret_msg = msg;
 351         int ret;
 352
 353         payload = (struct vpu_ipc_msg_payload_job_done *)&job_ret_msg->payload;
 354
 355         ret = ivpu_job_done(vdev, payload->job_id, payload->job_status);
 356         if (ret)
 357                 ivpu_err(vdev, "Failed to finish job %d: %d\n", payload->job_id, ret);
 358 }
 359
 360 void ivpu_jobs_abort_all(struct ivpu_device *vdev)
 361 {
 362         struct ivpu_job *job;
 363         unsigned long id;
 364
 365         xa_for_each(&vdev->submitted_jobs_xa, id, job)
 366                 ivpu_job_done(vdev, id, VPU_JSM_STATUS_ABORTED);
 367 }
 368
 369 static int ivpu_direct_job_submission(struct ivpu_job *job)
 370 {
 371         struct ivpu_file_priv *file_priv = job->file_priv;
 372         struct ivpu_device *vdev = job->vdev;
 373         struct xa_limit job_id_range;
 374         struct ivpu_cmdq *cmdq;
 375         int ret;
 376
 377         mutex_lock(&file_priv->lock);
 378
 379         cmdq = ivpu_cmdq_acquire(job->file_priv, job->engine_idx);
 380         if (!cmdq) {
 381                 ivpu_warn(vdev, "Failed get job queue, ctx %d engine %d\n",
 382                           file_priv->ctx.id, job->engine_idx);
 383                 ret = -EINVAL;
 384                 goto err_unlock;
 385         }
 386
 387         job_id_range.min = FIELD_PREP(JOB_ID_CONTEXT_MASK, (file_priv->ctx.id - 1));
 388         job_id_range.max = job_id_range.min | JOB_ID_JOB_MASK;
 389
 390         job_get(job, &job);
 391         ret = xa_alloc(&vdev->submitted_jobs_xa, &job->job_id, job, job_id_range, GFP_KERNEL);
 392         if (ret) {
 393                 ivpu_warn_ratelimited(vdev, "Failed to allocate job id: %d\n", ret);
 394                 goto err_job_put;
 395         }
 396
 397         ret = ivpu_cmdq_push_job(cmdq, job);
 398         if (ret)
 399                 goto err_xa_erase;
 400
 401         ivpu_dbg(vdev, JOB, "Job submitted: id %3u addr 0x%llx ctx %2d engine %d next %d\n",
 402                  job->job_id, job->cmd_buf_vpu_addr, file_priv->ctx.id,
 403                  job->engine_idx, cmdq->jobq->header.tail);
 404
 405         if (ivpu_test_mode == IVPU_TEST_MODE_NULL_HW) {
 406                 ivpu_job_done(vdev, job->job_id, VPU_JSM_STATUS_SUCCESS);
 407                 cmdq->jobq->header.head = cmdq->jobq->header.tail;
 408                 wmb(); /* Flush WC buffer for jobq header */
 409         } else {
 410                 ivpu_cmdq_ring_db(vdev, cmdq);
 411         }
 412
 413         mutex_unlock(&file_priv->lock);
 414         return 0;
 415
 416 err_xa_erase:
 417         xa_erase(&vdev->submitted_jobs_xa, job->job_id);
 418 err_job_put:
 419         job_put(job);
 420 err_unlock:
 421         mutex_unlock(&file_priv->lock);
 422         return ret;
 423 }
 424
 425 static int
 426 ivpu_job_prepare_bos_for_submit(struct drm_file *file, struct ivpu_job *job, u32 *buf_handles,
 427                                 u32 buf_count, u32 commands_offset)
 428 {
 429         struct ivpu_file_priv *file_priv = file->driver_priv;
 430         struct ivpu_device *vdev = file_priv->vdev;
 431         struct ww_acquire_ctx acquire_ctx;
 432         enum dma_resv_usage usage;
 433         struct ivpu_bo *bo;
 434         int ret;
 435         u32 i;
 436
 437         for (i = 0; i < buf_count; i++) {
 438                 struct drm_gem_object *obj = drm_gem_object_lookup(file, buf_handles[i]);
 439
 440                 if (!obj)
 441                         return -ENOENT;
 442
 443                 job->bos[i] = to_ivpu_bo(obj);
 444
 445                 ret = ivpu_bo_pin(job->bos[i]);
 446                 if (ret)
 447                         return ret;
 448         }
 449
 450         bo = job->bos[CMD_BUF_IDX];
 451         if (!dma_resv_test_signaled(bo->base.resv, DMA_RESV_USAGE_READ)) {
 452                 ivpu_warn(vdev, "Buffer is already in use\n");
 453                 return -EBUSY;
 454         }
 455
 456         if (commands_offset >= bo->base.size) {
 457                 ivpu_warn(vdev, "Invalid command buffer offset %u\n", commands_offset);
 458                 return -EINVAL;
 459         }
 460
 461         job->cmd_buf_vpu_addr = bo->vpu_addr + commands_offset;
 462
 463         ret = drm_gem_lock_reservations((struct drm_gem_object **)job->bos, buf_count,
 464                                         &acquire_ctx);
 465         if (ret) {
 466                 ivpu_warn(vdev, "Failed to lock reservations: %d\n", ret);
 467                 return ret;
 468         }
 469
 470         for (i = 0; i < buf_count; i++) {
 471                 ret = dma_resv_reserve_fences(job->bos[i]->base.resv, 1);
 472                 if (ret) {
 473                         ivpu_warn(vdev, "Failed to reserve fences: %d\n", ret);
 474                         goto unlock_reservations;
 475                 }
 476         }
 477
 478         for (i = 0; i < buf_count; i++) {
 479                 usage = (i == CMD_BUF_IDX) ? DMA_RESV_USAGE_WRITE : DMA_RESV_USAGE_BOOKKEEP;
 480                 dma_resv_add_fence(job->bos[i]->base.resv, job->done_fence, usage);
 481         }
 482
 483 unlock_reservations:
 484         drm_gem_unlock_reservations((struct drm_gem_object **)job->bos, buf_count, &acquire_ctx);
 485
 486         wmb(); /* Flush write combining buffers */
 487
 488         return ret;
 489 }
 490
 491 int ivpu_submit_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
 492 {
 493         struct ivpu_file_priv *file_priv = file->driver_priv;
 494         struct ivpu_device *vdev = file_priv->vdev;
 495         struct drm_ivpu_submit *params = data;
 496         struct ivpu_job *job;
 497         u32 *buf_handles;
 498         int idx, ret;
 499
 500         if (params->engine > DRM_IVPU_ENGINE_COPY)
 501                 return -EINVAL;
 502
 503         if (params->buffer_count == 0 || params->buffer_count > JOB_MAX_BUFFER_COUNT)
 504                 return -EINVAL;
 505
 506         if (!IS_ALIGNED(params->commands_offset, 8))
 507                 return -EINVAL;
 508
 509         if (!file_priv->ctx.id)
 510                 return -EINVAL;
 511
 512         if (file_priv->has_mmu_faults)
 513                 return -EBADFD;
 514
 515         buf_handles = kcalloc(params->buffer_count, sizeof(u32), GFP_KERNEL);
 516         if (!buf_handles)
 517                 return -ENOMEM;
 518
 519         ret = copy_from_user(buf_handles,
 520                              (void __user *)params->buffers_ptr,
 521                              params->buffer_count * sizeof(u32));
 522         if (ret) {
 523                 ret = -EFAULT;
 524                 goto free_handles;
 525         }
 526
 527         if (!drm_dev_enter(&vdev->drm, &idx)) {
 528                 ret = -ENODEV;
 529                 goto free_handles;
 530         }
 531
 532         ivpu_dbg(vdev, JOB, "Submit ioctl: ctx %u buf_count %u\n",
 533                  file_priv->ctx.id, params->buffer_count);
 534
 535         job = ivpu_create_job(file_priv, params->engine, params->buffer_count);
 536         if (!job) {
 537                 ivpu_err(vdev, "Failed to create job\n");
 538                 ret = -ENOMEM;
 539                 goto dev_exit;
 540         }
 541
 542         ret = ivpu_job_prepare_bos_for_submit(file, job, buf_handles, params->buffer_count,
 543                                               params->commands_offset);
 544         if (ret) {
 545                 ivpu_err(vdev, "Failed to prepare job, ret %d\n", ret);
 546                 goto job_put;
 547         }
 548
 549         ret = ivpu_direct_job_submission(job);
 550         if (ret) {
 551                 dma_fence_signal(job->done_fence);
 552                 ivpu_err(vdev, "Failed to submit job to the HW, ret %d\n", ret);
 553         }
 554
 555 job_put:
 556         job_put(job);
 557 dev_exit:
 558         drm_dev_exit(idx);
 559 free_handles:
 560         kfree(buf_handles);
 561
 562         return ret;
 563 }
 564
 565 static int ivpu_job_done_thread(void *arg)
 566 {
 567         struct ivpu_device *vdev = (struct ivpu_device *)arg;
 568         struct ivpu_ipc_consumer cons;
 569         struct vpu_jsm_msg jsm_msg;
 570         bool jobs_submitted;
 571         unsigned int timeout;
 572         int ret;
 573
 574         ivpu_dbg(vdev, JOB, "Started %s\n", __func__);
 575
 576         ivpu_ipc_consumer_add(vdev, &cons, VPU_IPC_CHAN_JOB_RET);
 577
 578         while (!kthread_should_stop()) {
 579                 timeout = ivpu_tdr_timeout_ms ? ivpu_tdr_timeout_ms : vdev->timeout.tdr;
 580                 jobs_submitted = !xa_empty(&vdev->submitted_jobs_xa);
 581                 ret = ivpu_ipc_receive(vdev, &cons, NULL, &jsm_msg, timeout);
 582                 if (!ret) {
 583                         ivpu_job_done_message(vdev, &jsm_msg);
 584                 } else if (ret == -ETIMEDOUT) {
 585                         if (jobs_submitted && !xa_empty(&vdev->submitted_jobs_xa)) {
 586                                 ivpu_err(vdev, "TDR detected, timeout %d ms", timeout);
 587                                 ivpu_hw_diagnose_failure(vdev);
 588                                 ivpu_pm_schedule_recovery(vdev);
 589                         }
 590                 }
 591         }
 592
 593         ivpu_ipc_consumer_del(vdev, &cons);
 594
 595         ivpu_jobs_abort_all(vdev);
 596
 597         ivpu_dbg(vdev, JOB, "Stopped %s\n", __func__);
 598         return 0;
 599 }
 600
 601 int ivpu_job_done_thread_init(struct ivpu_device *vdev)
 602 {
 603         struct task_struct *thread;
 604
 605         thread = kthread_run(&ivpu_job_done_thread, (void *)vdev, "ivpu_job_done_thread");
 606         if (IS_ERR(thread)) {
 607                 ivpu_err(vdev, "Failed to start job completion thread\n");
 608                 return -EIO;
 609         }
 610
 611         get_task_struct(thread);
 612         wake_up_process(thread);
 613
 614         vdev->job_done_thread = thread;
 615
 616         return 0;
 617 }
 618
 619 void ivpu_job_done_thread_fini(struct ivpu_device *vdev)
 620 {
 621         kthread_stop(vdev->job_done_thread);
 622         put_task_struct(vdev->job_done_thread);
 623 }