drivers/lightnvm/pblk-gc.c

   1 /*
   2  * Copyright (C) 2016 CNEX Labs
   3  * Initial release: Javier Gonzalez <javier@cnexlabs.com>
   4  *                  Matias Bjorling <matias@cnexlabs.com>
   5  *
   6  * This program is free software; you can redistribute it and/or
   7  * modify it under the terms of the GNU General Public License version
   8  * 2 as published by the Free Software Foundation.
   9  *
  10  * This program is distributed in the hope that it will be useful, but
  11  * WITHOUT ANY WARRANTY; without even the implied warranty of
  12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  13  * General Public License for more details.
  14  *
  15  * pblk-gc.c - pblk's garbage collector
  16  */
  17
  18 #include "pblk.h"
  19 #include <linux/delay.h>
  20
  21 static void pblk_gc_free_gc_rq(struct pblk_gc_rq *gc_rq)
  22 {
  23         if (gc_rq->data)
  24                 vfree(gc_rq->data);
  25         kfree(gc_rq);
  26 }
  27
  28 static int pblk_gc_write(struct pblk *pblk)
  29 {
  30         struct pblk_gc *gc = &pblk->gc;
  31         struct pblk_gc_rq *gc_rq, *tgc_rq;
  32         LIST_HEAD(w_list);
  33
  34         spin_lock(&gc->w_lock);
  35         if (list_empty(&gc->w_list)) {
  36                 spin_unlock(&gc->w_lock);
  37                 return 1;
  38         }
  39
  40         list_cut_position(&w_list, &gc->w_list, gc->w_list.prev);
  41         gc->w_entries = 0;
  42         spin_unlock(&gc->w_lock);
  43
  44         list_for_each_entry_safe(gc_rq, tgc_rq, &w_list, list) {
  45                 pblk_write_gc_to_cache(pblk, gc_rq);
  46                 list_del(&gc_rq->list);
  47                 kref_put(&gc_rq->line->ref, pblk_line_put);
  48                 pblk_gc_free_gc_rq(gc_rq);
  49         }
  50
  51         return 0;
  52 }
  53
  54 static void pblk_gc_writer_kick(struct pblk_gc *gc)
  55 {
  56         wake_up_process(gc->gc_writer_ts);
  57 }
  58
  59 static void pblk_put_line_back(struct pblk *pblk, struct pblk_line *line)
  60 {
  61         struct pblk_line_mgmt *l_mg = &pblk->l_mg;
  62         struct list_head *move_list;
  63
  64         spin_lock(&line->lock);
  65         WARN_ON(line->state != PBLK_LINESTATE_GC);
  66         line->state = PBLK_LINESTATE_CLOSED;
  67         move_list = pblk_line_gc_list(pblk, line);
  68         spin_unlock(&line->lock);
  69
  70         if (move_list) {
  71                 spin_lock(&l_mg->gc_lock);
  72                 list_add_tail(&line->list, move_list);
  73                 spin_unlock(&l_mg->gc_lock);
  74         }
  75 }
  76
  77 static void pblk_gc_line_ws(struct work_struct *work)
  78 {
  79         struct pblk_line_ws *gc_rq_ws = container_of(work,
  80                                                 struct pblk_line_ws, ws);
  81         struct pblk *pblk = gc_rq_ws->pblk;
  82         struct nvm_tgt_dev *dev = pblk->dev;
  83         struct nvm_geo *geo = &dev->geo;
  84         struct pblk_gc *gc = &pblk->gc;
  85         struct pblk_line *line = gc_rq_ws->line;
  86         struct pblk_gc_rq *gc_rq = gc_rq_ws->priv;
  87         int ret;
  88
  89         up(&gc->gc_sem);
  90
  91         gc_rq->data = vmalloc(gc_rq->nr_secs * geo->sec_size);
  92         if (!gc_rq->data) {
  93                 pr_err("pblk: could not GC line:%d (%d/%d)\n",
  94                                         line->id, *line->vsc, gc_rq->nr_secs);
  95                 goto out;
  96         }
  97
  98         /* Read from GC victim block */
  99         ret = pblk_submit_read_gc(pblk, gc_rq);
 100         if (ret) {
 101                 pr_err("pblk: failed GC read in line:%d (err:%d)\n",
 102                                                                 line->id, ret);
 103                 goto out;
 104         }
 105
 106         if (!gc_rq->secs_to_gc)
 107                 goto out;
 108
 109 retry:
 110         spin_lock(&gc->w_lock);
 111         if (gc->w_entries >= PBLK_GC_RQ_QD) {
 112                 spin_unlock(&gc->w_lock);
 113                 pblk_gc_writer_kick(&pblk->gc);
 114                 usleep_range(128, 256);
 115                 goto retry;
 116         }
 117         gc->w_entries++;
 118         list_add_tail(&gc_rq->list, &gc->w_list);
 119         spin_unlock(&gc->w_lock);
 120
 121         pblk_gc_writer_kick(&pblk->gc);
 122
 123         kfree(gc_rq_ws);
 124         return;
 125
 126 out:
 127         pblk_gc_free_gc_rq(gc_rq);
 128         kref_put(&line->ref, pblk_line_put);
 129         kfree(gc_rq_ws);
 130 }
 131
 132 static void pblk_gc_line_prepare_ws(struct work_struct *work)
 133 {
 134         struct pblk_line_ws *line_ws = container_of(work, struct pblk_line_ws,
 135                                                                         ws);
 136         struct pblk *pblk = line_ws->pblk;
 137         struct pblk_line *line = line_ws->line;
 138         struct pblk_line_mgmt *l_mg = &pblk->l_mg;
 139         struct pblk_line_meta *lm = &pblk->lm;
 140         struct pblk_gc *gc = &pblk->gc;
 141         struct line_emeta *emeta_buf;
 142         struct pblk_line_ws *gc_rq_ws;
 143         struct pblk_gc_rq *gc_rq;
 144         __le64 *lba_list;
 145         unsigned long *invalid_bitmap;
 146         int sec_left, nr_secs, bit;
 147         int ret;
 148
 149         invalid_bitmap = kmalloc(lm->sec_bitmap_len, GFP_KERNEL);
 150         if (!invalid_bitmap) {
 151                 pr_err("pblk: could not allocate GC invalid bitmap\n");
 152                 goto fail_free_ws;
 153         }
 154
 155         emeta_buf = pblk_malloc(lm->emeta_len[0], l_mg->emeta_alloc_type,
 156                                                                 GFP_KERNEL);
 157         if (!emeta_buf) {
 158                 pr_err("pblk: cannot use GC emeta\n");
 159                 goto fail_free_bitmap;
 160         }
 161
 162         ret = pblk_line_read_emeta(pblk, line, emeta_buf);
 163         if (ret) {
 164                 pr_err("pblk: line %d read emeta failed (%d)\n", line->id, ret);
 165                 goto fail_free_emeta;
 166         }
 167
 168         /* If this read fails, it means that emeta is corrupted. For now, leave
 169          * the line untouched. TODO: Implement a recovery routine that scans and
 170          * moves all sectors on the line.
 171          */
 172
 173         ret = pblk_recov_check_emeta(pblk, emeta_buf);
 174         if (ret) {
 175                 pr_err("pblk: inconsistent emeta (line %d)\n", line->id);
 176                 goto fail_free_emeta;
 177         }
 178
 179         lba_list = emeta_to_lbas(pblk, emeta_buf);
 180         if (!lba_list) {
 181                 pr_err("pblk: could not interpret emeta (line %d)\n", line->id);
 182                 goto fail_free_emeta;
 183         }
 184
 185         spin_lock(&line->lock);
 186         bitmap_copy(invalid_bitmap, line->invalid_bitmap, lm->sec_per_line);
 187         sec_left = pblk_line_vsc(line);
 188         spin_unlock(&line->lock);
 189
 190         if (sec_left < 0) {
 191                 pr_err("pblk: corrupted GC line (%d)\n", line->id);
 192                 goto fail_free_emeta;
 193         }
 194
 195         bit = -1;
 196 next_rq:
 197         gc_rq = kmalloc(sizeof(struct pblk_gc_rq), GFP_KERNEL);
 198         if (!gc_rq)
 199                 goto fail_free_emeta;
 200
 201         nr_secs = 0;
 202         do {
 203                 bit = find_next_zero_bit(invalid_bitmap, lm->sec_per_line,
 204                                                                 bit + 1);
 205                 if (bit > line->emeta_ssec)
 206                         break;
 207
 208                 gc_rq->paddr_list[nr_secs] = bit;
 209                 gc_rq->lba_list[nr_secs++] = le64_to_cpu(lba_list[bit]);
 210         } while (nr_secs < pblk->max_write_pgs);
 211
 212         if (unlikely(!nr_secs)) {
 213                 kfree(gc_rq);
 214                 goto out;
 215         }
 216
 217         gc_rq->nr_secs = nr_secs;
 218         gc_rq->line = line;
 219
 220         gc_rq_ws = kmalloc(sizeof(struct pblk_line_ws), GFP_KERNEL);
 221         if (!gc_rq_ws)
 222                 goto fail_free_gc_rq;
 223
 224         gc_rq_ws->pblk = pblk;
 225         gc_rq_ws->line = line;
 226         gc_rq_ws->priv = gc_rq;
 227
 228         /* The write GC path can be much slower than the read GC one due to
 229          * the budget imposed by the rate-limiter. Balance in case that we get
 230          * back pressure from the write GC path.
 231          */
 232         while (down_timeout(&gc->gc_sem, msecs_to_jiffies(30000)))
 233                 io_schedule();
 234
 235         kref_get(&line->ref);
 236
 237         INIT_WORK(&gc_rq_ws->ws, pblk_gc_line_ws);
 238         queue_work(gc->gc_line_reader_wq, &gc_rq_ws->ws);
 239
 240         sec_left -= nr_secs;
 241         if (sec_left > 0)
 242                 goto next_rq;
 243
 244 out:
 245         pblk_mfree(emeta_buf, l_mg->emeta_alloc_type);
 246         kfree(line_ws);
 247         kfree(invalid_bitmap);
 248
 249         kref_put(&line->ref, pblk_line_put);
 250         atomic_dec(&gc->read_inflight_gc);
 251
 252         return;
 253
 254 fail_free_gc_rq:
 255         kfree(gc_rq);
 256 fail_free_emeta:
 257         pblk_mfree(emeta_buf, l_mg->emeta_alloc_type);
 258 fail_free_bitmap:
 259         kfree(invalid_bitmap);
 260 fail_free_ws:
 261         kfree(line_ws);
 262
 263         pblk_put_line_back(pblk, line);
 264         kref_put(&line->ref, pblk_line_put);
 265         atomic_dec(&gc->read_inflight_gc);
 266
 267         pr_err("pblk: Failed to GC line %d\n", line->id);
 268 }
 269
 270 static int pblk_gc_line(struct pblk *pblk, struct pblk_line *line)
 271 {
 272         struct pblk_gc *gc = &pblk->gc;
 273         struct pblk_line_ws *line_ws;
 274
 275         pr_debug("pblk: line '%d' being reclaimed for GC\n", line->id);
 276
 277         line_ws = kmalloc(sizeof(struct pblk_line_ws), GFP_KERNEL);
 278         if (!line_ws)
 279                 return -ENOMEM;
 280
 281         line_ws->pblk = pblk;
 282         line_ws->line = line;
 283
 284         atomic_inc(&gc->pipeline_gc);
 285         INIT_WORK(&line_ws->ws, pblk_gc_line_prepare_ws);
 286         queue_work(gc->gc_reader_wq, &line_ws->ws);
 287
 288         return 0;
 289 }
 290
 291 static void pblk_gc_reader_kick(struct pblk_gc *gc)
 292 {
 293         wake_up_process(gc->gc_reader_ts);
 294 }
 295
 296 static void pblk_gc_kick(struct pblk *pblk)
 297 {
 298         struct pblk_gc *gc = &pblk->gc;
 299
 300         pblk_gc_writer_kick(gc);
 301         pblk_gc_reader_kick(gc);
 302
 303         /* If we're shutting down GC, let's not start it up again */
 304         if (gc->gc_enabled) {
 305                 wake_up_process(gc->gc_ts);
 306                 mod_timer(&gc->gc_timer,
 307                           jiffies + msecs_to_jiffies(GC_TIME_MSECS));
 308         }
 309 }
 310
 311 static int pblk_gc_read(struct pblk *pblk)
 312 {
 313         struct pblk_gc *gc = &pblk->gc;
 314         struct pblk_line *line;
 315
 316         spin_lock(&gc->r_lock);
 317         if (list_empty(&gc->r_list)) {
 318                 spin_unlock(&gc->r_lock);
 319                 return 1;
 320         }
 321
 322         line = list_first_entry(&gc->r_list, struct pblk_line, list);
 323         list_del(&line->list);
 324         spin_unlock(&gc->r_lock);
 325
 326         pblk_gc_kick(pblk);
 327
 328         if (pblk_gc_line(pblk, line))
 329                 pr_err("pblk: failed to GC line %d\n", line->id);
 330
 331         return 0;
 332 }
 333
 334 static struct pblk_line *pblk_gc_get_victim_line(struct pblk *pblk,
 335                                                  struct list_head *group_list)
 336 {
 337         struct pblk_line *line, *victim;
 338         int line_vsc, victim_vsc;
 339
 340         victim = list_first_entry(group_list, struct pblk_line, list);
 341         list_for_each_entry(line, group_list, list) {
 342                 line_vsc = le32_to_cpu(*line->vsc);
 343                 victim_vsc = le32_to_cpu(*victim->vsc);
 344                 if (line_vsc < victim_vsc)
 345                         victim = line;
 346         }
 347
 348         return victim;
 349 }
 350
 351 static bool pblk_gc_should_run(struct pblk_gc *gc, struct pblk_rl *rl)
 352 {
 353         unsigned int nr_blocks_free, nr_blocks_need;
 354
 355         nr_blocks_need = pblk_rl_high_thrs(rl);
 356         nr_blocks_free = pblk_rl_nr_free_blks(rl);
 357
 358         /* This is not critical, no need to take lock here */
 359         return ((gc->gc_active) && (nr_blocks_need > nr_blocks_free));
 360 }
 361
 362 void pblk_gc_free_full_lines(struct pblk *pblk)
 363 {
 364         struct pblk_line_mgmt *l_mg = &pblk->l_mg;
 365         struct pblk_gc *gc = &pblk->gc;
 366         struct pblk_line *line;
 367
 368         do {
 369                 spin_lock(&l_mg->gc_lock);
 370                 if (list_empty(&l_mg->gc_full_list)) {
 371                         spin_unlock(&l_mg->gc_lock);
 372                         return;
 373                 }
 374
 375                 line = list_first_entry(&l_mg->gc_full_list,
 376                                                         struct pblk_line, list);
 377
 378                 spin_lock(&line->lock);
 379                 WARN_ON(line->state != PBLK_LINESTATE_CLOSED);
 380                 line->state = PBLK_LINESTATE_GC;
 381                 spin_unlock(&line->lock);
 382
 383                 list_del(&line->list);
 384                 spin_unlock(&l_mg->gc_lock);
 385
 386                 atomic_inc(&gc->pipeline_gc);
 387                 kref_put(&line->ref, pblk_line_put);
 388         } while (1);
 389 }
 390
 391 /*
 392  * Lines with no valid sectors will be returned to the free list immediately. If
 393  * GC is activated - either because the free block count is under the determined
 394  * threshold, or because it is being forced from user space - only lines with a
 395  * high count of invalid sectors will be recycled.
 396  */
 397 static void pblk_gc_run(struct pblk *pblk)
 398 {
 399         struct pblk_line_mgmt *l_mg = &pblk->l_mg;
 400         struct pblk_gc *gc = &pblk->gc;
 401         struct pblk_line *line;
 402         struct list_head *group_list;
 403         bool run_gc;
 404         int read_inflight_gc, gc_group = 0, prev_group = 0;
 405
 406         pblk_gc_free_full_lines(pblk);
 407
 408         run_gc = pblk_gc_should_run(&pblk->gc, &pblk->rl);
 409         if (!run_gc || (atomic_read(&gc->read_inflight_gc) >= PBLK_GC_L_QD))
 410                 return;
 411
 412 next_gc_group:
 413         group_list = l_mg->gc_lists[gc_group++];
 414
 415         do {
 416                 spin_lock(&l_mg->gc_lock);
 417                 if (list_empty(group_list)) {
 418                         spin_unlock(&l_mg->gc_lock);
 419                         break;
 420                 }
 421
 422                 line = pblk_gc_get_victim_line(pblk, group_list);
 423
 424                 spin_lock(&line->lock);
 425                 WARN_ON(line->state != PBLK_LINESTATE_CLOSED);
 426                 line->state = PBLK_LINESTATE_GC;
 427                 spin_unlock(&line->lock);
 428
 429                 list_del(&line->list);
 430                 spin_unlock(&l_mg->gc_lock);
 431
 432                 spin_lock(&gc->r_lock);
 433                 list_add_tail(&line->list, &gc->r_list);
 434                 spin_unlock(&gc->r_lock);
 435
 436                 read_inflight_gc = atomic_inc_return(&gc->read_inflight_gc);
 437                 pblk_gc_reader_kick(gc);
 438
 439                 prev_group = 1;
 440
 441                 /* No need to queue up more GC lines than we can handle */
 442                 run_gc = pblk_gc_should_run(&pblk->gc, &pblk->rl);
 443                 if (!run_gc || read_inflight_gc >= PBLK_GC_L_QD)
 444                         break;
 445         } while (1);
 446
 447         if (!prev_group && pblk->rl.rb_state > gc_group &&
 448                                                 gc_group < PBLK_GC_NR_LISTS)
 449                 goto next_gc_group;
 450 }
 451
 452 static void pblk_gc_timer(struct timer_list *t)
 453 {
 454         struct pblk *pblk = from_timer(pblk, t, gc.gc_timer);
 455
 456         pblk_gc_kick(pblk);
 457 }
 458
 459 static int pblk_gc_ts(void *data)
 460 {
 461         struct pblk *pblk = data;
 462
 463         while (!kthread_should_stop()) {
 464                 pblk_gc_run(pblk);
 465                 set_current_state(TASK_INTERRUPTIBLE);
 466                 io_schedule();
 467         }
 468
 469         return 0;
 470 }
 471
 472 static int pblk_gc_writer_ts(void *data)
 473 {
 474         struct pblk *pblk = data;
 475
 476         while (!kthread_should_stop()) {
 477                 if (!pblk_gc_write(pblk))
 478                         continue;
 479                 set_current_state(TASK_INTERRUPTIBLE);
 480                 io_schedule();
 481         }
 482
 483         return 0;
 484 }
 485
 486 static int pblk_gc_reader_ts(void *data)
 487 {
 488         struct pblk *pblk = data;
 489         struct pblk_gc *gc = &pblk->gc;
 490
 491         while (!kthread_should_stop()) {
 492                 if (!pblk_gc_read(pblk))
 493                         continue;
 494                 set_current_state(TASK_INTERRUPTIBLE);
 495                 io_schedule();
 496         }
 497
 498 #ifdef CONFIG_NVM_DEBUG
 499         pr_info("pblk: flushing gc pipeline, %d lines left\n",
 500                 atomic_read(&gc->pipeline_gc));
 501 #endif
 502
 503         do {
 504                 if (!atomic_read(&gc->pipeline_gc))
 505                         break;
 506
 507                 schedule();
 508         } while (1);
 509
 510         return 0;
 511 }
 512
 513 static void pblk_gc_start(struct pblk *pblk)
 514 {
 515         pblk->gc.gc_active = 1;
 516         pr_debug("pblk: gc start\n");
 517 }
 518
 519 void pblk_gc_should_start(struct pblk *pblk)
 520 {
 521         struct pblk_gc *gc = &pblk->gc;
 522
 523         if (gc->gc_enabled && !gc->gc_active) {
 524                 pblk_gc_start(pblk);
 525                 pblk_gc_kick(pblk);
 526         }
 527 }
 528
 529 /*
 530  * If flush_wq == 1 then no lock should be held by the caller since
 531  * flush_workqueue can sleep
 532  */
 533 static void pblk_gc_stop(struct pblk *pblk, int flush_wq)
 534 {
 535         pblk->gc.gc_active = 0;
 536         pr_debug("pblk: gc stop\n");
 537 }
 538
 539 void pblk_gc_should_stop(struct pblk *pblk)
 540 {
 541         struct pblk_gc *gc = &pblk->gc;
 542
 543         if (gc->gc_active && !gc->gc_forced)
 544                 pblk_gc_stop(pblk, 0);
 545 }
 546
 547 void pblk_gc_should_kick(struct pblk *pblk)
 548 {
 549         pblk_rl_update_rates(&pblk->rl);
 550 }
 551
 552 void pblk_gc_sysfs_state_show(struct pblk *pblk, int *gc_enabled,
 553                               int *gc_active)
 554 {
 555         struct pblk_gc *gc = &pblk->gc;
 556
 557         spin_lock(&gc->lock);
 558         *gc_enabled = gc->gc_enabled;
 559         *gc_active = gc->gc_active;
 560         spin_unlock(&gc->lock);
 561 }
 562
 563 int pblk_gc_sysfs_force(struct pblk *pblk, int force)
 564 {
 565         struct pblk_gc *gc = &pblk->gc;
 566
 567         if (force < 0 || force > 1)
 568                 return -EINVAL;
 569
 570         spin_lock(&gc->lock);
 571         gc->gc_forced = force;
 572
 573         if (force)
 574                 gc->gc_enabled = 1;
 575         else
 576                 gc->gc_enabled = 0;
 577         spin_unlock(&gc->lock);
 578
 579         pblk_gc_should_start(pblk);
 580
 581         return 0;
 582 }
 583
 584 int pblk_gc_init(struct pblk *pblk)
 585 {
 586         struct pblk_gc *gc = &pblk->gc;
 587         int ret;
 588
 589         gc->gc_ts = kthread_create(pblk_gc_ts, pblk, "pblk-gc-ts");
 590         if (IS_ERR(gc->gc_ts)) {
 591                 pr_err("pblk: could not allocate GC main kthread\n");
 592                 return PTR_ERR(gc->gc_ts);
 593         }
 594
 595         gc->gc_writer_ts = kthread_create(pblk_gc_writer_ts, pblk,
 596                                                         "pblk-gc-writer-ts");
 597         if (IS_ERR(gc->gc_writer_ts)) {
 598                 pr_err("pblk: could not allocate GC writer kthread\n");
 599                 ret = PTR_ERR(gc->gc_writer_ts);
 600                 goto fail_free_main_kthread;
 601         }
 602
 603         gc->gc_reader_ts = kthread_create(pblk_gc_reader_ts, pblk,
 604                                                         "pblk-gc-reader-ts");
 605         if (IS_ERR(gc->gc_reader_ts)) {
 606                 pr_err("pblk: could not allocate GC reader kthread\n");
 607                 ret = PTR_ERR(gc->gc_reader_ts);
 608                 goto fail_free_writer_kthread;
 609         }
 610
 611         timer_setup(&gc->gc_timer, pblk_gc_timer, 0);
 612         mod_timer(&gc->gc_timer, jiffies + msecs_to_jiffies(GC_TIME_MSECS));
 613
 614         gc->gc_active = 0;
 615         gc->gc_forced = 0;
 616         gc->gc_enabled = 1;
 617         gc->w_entries = 0;
 618         atomic_set(&gc->read_inflight_gc, 0);
 619         atomic_set(&gc->pipeline_gc, 0);
 620
 621         /* Workqueue that reads valid sectors from a line and submit them to the
 622          * GC writer to be recycled.
 623          */
 624         gc->gc_line_reader_wq = alloc_workqueue("pblk-gc-line-reader-wq",
 625                         WQ_MEM_RECLAIM | WQ_UNBOUND, PBLK_GC_MAX_READERS);
 626         if (!gc->gc_line_reader_wq) {
 627                 pr_err("pblk: could not allocate GC line reader workqueue\n");
 628                 ret = -ENOMEM;
 629                 goto fail_free_reader_kthread;
 630         }
 631
 632         /* Workqueue that prepare lines for GC */
 633         gc->gc_reader_wq = alloc_workqueue("pblk-gc-line_wq",
 634                                         WQ_MEM_RECLAIM | WQ_UNBOUND, 1);
 635         if (!gc->gc_reader_wq) {
 636                 pr_err("pblk: could not allocate GC reader workqueue\n");
 637                 ret = -ENOMEM;
 638                 goto fail_free_reader_line_wq;
 639         }
 640
 641         spin_lock_init(&gc->lock);
 642         spin_lock_init(&gc->w_lock);
 643         spin_lock_init(&gc->r_lock);
 644
 645         sema_init(&gc->gc_sem, PBLK_GC_RQ_QD);
 646
 647         INIT_LIST_HEAD(&gc->w_list);
 648         INIT_LIST_HEAD(&gc->r_list);
 649
 650         return 0;
 651
 652 fail_free_reader_line_wq:
 653         destroy_workqueue(gc->gc_line_reader_wq);
 654 fail_free_reader_kthread:
 655         kthread_stop(gc->gc_reader_ts);
 656 fail_free_writer_kthread:
 657         kthread_stop(gc->gc_writer_ts);
 658 fail_free_main_kthread:
 659         kthread_stop(gc->gc_ts);
 660
 661         return ret;
 662 }
 663
 664 void pblk_gc_exit(struct pblk *pblk)
 665 {
 666         struct pblk_gc *gc = &pblk->gc;
 667
 668         gc->gc_enabled = 0;
 669         del_timer_sync(&gc->gc_timer);
 670         pblk_gc_stop(pblk, 1);
 671
 672         if (gc->gc_ts)
 673                 kthread_stop(gc->gc_ts);
 674
 675         if (gc->gc_reader_ts)
 676                 kthread_stop(gc->gc_reader_ts);
 677
 678         flush_workqueue(gc->gc_reader_wq);
 679         if (gc->gc_reader_wq)
 680                 destroy_workqueue(gc->gc_reader_wq);
 681
 682         flush_workqueue(gc->gc_line_reader_wq);
 683         if (gc->gc_line_reader_wq)
 684                 destroy_workqueue(gc->gc_line_reader_wq);
 685
 686         if (gc->gc_writer_ts)
 687                 kthread_stop(gc->gc_writer_ts);
 688 }