drivers/staging/media/atomisp/pci/atomisp2/hmm/hmm_bo.c

   1 /*
   2  * Support for Medifield PNW Camera Imaging ISP subsystem.
   3  *
   4  * Copyright (c) 2010 Intel Corporation. All Rights Reserved.
   5  *
   6  * Copyright (c) 2010 Silicon Hive www.siliconhive.com.
   7  *
   8  * This program is free software; you can redistribute it and/or
   9  * modify it under the terms of the GNU General Public License version
  10  * 2 as published by the Free Software Foundation.
  11  *
  12  * This program is distributed in the hope that it will be useful,
  13  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  14  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  15  * GNU General Public License for more details.
  16  *
  17  *
  18  */
  19 /*
  20  * This file contains functions for buffer object structure management
  21  */
  22 #include <linux/kernel.h>
  23 #include <linux/types.h>
  24 #include <linux/gfp.h>          /* for GFP_ATOMIC */
  25 #include <linux/mm.h>
  26 #include <linux/mm_types.h>
  27 #include <linux/hugetlb.h>
  28 #include <linux/highmem.h>
  29 #include <linux/slab.h>         /* for kmalloc */
  30 #include <linux/module.h>
  31 #include <linux/moduleparam.h>
  32 #include <linux/string.h>
  33 #include <linux/list.h>
  34 #include <linux/errno.h>
  35 #include <linux/io.h>
  36 #include <asm/current.h>
  37 #include <linux/sched/signal.h>
  38 #include <linux/file.h>
  39
  40 #include <asm/set_memory.h>
  41
  42 #include "atomisp_internal.h"
  43 #include "hmm/hmm_common.h"
  44 #include "hmm/hmm_pool.h"
  45 #include "hmm/hmm_bo.h"
  46
  47 static unsigned int order_to_nr(unsigned int order)
  48 {
  49         return 1U << order;
  50 }
  51
  52 static unsigned int nr_to_order_bottom(unsigned int nr)
  53 {
  54         return fls(nr) - 1;
  55 }
  56
  57 static struct hmm_buffer_object *__bo_alloc(struct kmem_cache *bo_cache)
  58 {
  59         struct hmm_buffer_object *bo;
  60
  61         bo = kmem_cache_alloc(bo_cache, GFP_KERNEL);
  62         if (!bo)
  63                 dev_err(atomisp_dev, "%s: failed!\n", __func__);
  64
  65         return bo;
  66 }
  67
  68 static int __bo_init(struct hmm_bo_device *bdev, struct hmm_buffer_object *bo,
  69                                         unsigned int pgnr)
  70 {
  71         check_bodev_null_return(bdev, -EINVAL);
  72         var_equal_return(hmm_bo_device_inited(bdev), 0, -EINVAL,
  73                         "hmm_bo_device not inited yet.\n");
  74         /* prevent zero size buffer object */
  75         if (pgnr == 0) {
  76                 dev_err(atomisp_dev, "0 size buffer is not allowed.\n");
  77                 return -EINVAL;
  78         }
  79
  80         memset(bo, 0, sizeof(*bo));
  81         mutex_init(&bo->mutex);
  82
  83         /* init the bo->list HEAD as an element of entire_bo_list */
  84         INIT_LIST_HEAD(&bo->list);
  85
  86         bo->bdev = bdev;
  87         bo->vmap_addr = NULL;
  88         bo->status = HMM_BO_FREE;
  89         bo->start = bdev->start;
  90         bo->pgnr = pgnr;
  91         bo->end = bo->start + pgnr_to_size(pgnr);
  92         bo->prev = NULL;
  93         bo->next = NULL;
  94
  95         return 0;
  96 }
  97
  98 static struct hmm_buffer_object *__bo_search_and_remove_from_free_rbtree(
  99                                 struct rb_node *node, unsigned int pgnr)
 100 {
 101         struct hmm_buffer_object *this, *ret_bo, *temp_bo;
 102
 103         this = rb_entry(node, struct hmm_buffer_object, node);
 104         if (this->pgnr == pgnr ||
 105                 (this->pgnr > pgnr && this->node.rb_left == NULL)) {
 106                 goto remove_bo_and_return;
 107         } else {
 108                 if (this->pgnr < pgnr) {
 109                         if (!this->node.rb_right)
 110                                 return NULL;
 111                         ret_bo = __bo_search_and_remove_from_free_rbtree(
 112                                 this->node.rb_right, pgnr);
 113                 } else {
 114                         ret_bo = __bo_search_and_remove_from_free_rbtree(
 115                                 this->node.rb_left, pgnr);
 116                 }
 117                 if (!ret_bo) {
 118                         if (this->pgnr > pgnr)
 119                                 goto remove_bo_and_return;
 120                         else
 121                                 return NULL;
 122                 }
 123                 return ret_bo;
 124         }
 125
 126 remove_bo_and_return:
 127         /* NOTE: All nodes on free rbtree have a 'prev' that points to NULL.
 128          * 1. check if 'this->next' is NULL:
 129          *      yes: erase 'this' node and rebalance rbtree, return 'this'.
 130          */
 131         if (this->next == NULL) {
 132                 rb_erase(&this->node, &this->bdev->free_rbtree);
 133                 return this;
 134         }
 135         /* NOTE: if 'this->next' is not NULL, always return 'this->next' bo.
 136          * 2. check if 'this->next->next' is NULL:
 137          *      yes: change the related 'next/prev' pointer,
 138          *              return 'this->next' but the rbtree stays unchanged.
 139          */
 140         temp_bo = this->next;
 141         this->next = temp_bo->next;
 142         if (temp_bo->next)
 143                 temp_bo->next->prev = this;
 144         temp_bo->next = NULL;
 145         temp_bo->prev = NULL;
 146         return temp_bo;
 147 }
 148
 149 static struct hmm_buffer_object *__bo_search_by_addr(struct rb_root *root,
 150                                                         ia_css_ptr start)
 151 {
 152         struct rb_node *n = root->rb_node;
 153         struct hmm_buffer_object *bo;
 154
 155         do {
 156                 bo = rb_entry(n, struct hmm_buffer_object, node);
 157
 158                 if (bo->start > start) {
 159                         if (n->rb_left == NULL)
 160                                 return NULL;
 161                         n = n->rb_left;
 162                 } else if (bo->start < start) {
 163                         if (n->rb_right == NULL)
 164                                 return NULL;
 165                         n = n->rb_right;
 166                 } else {
 167                         return bo;
 168                 }
 169         } while (n);
 170
 171         return NULL;
 172 }
 173
 174 static struct hmm_buffer_object *__bo_search_by_addr_in_range(
 175                 struct rb_root *root, unsigned int start)
 176 {
 177         struct rb_node *n = root->rb_node;
 178         struct hmm_buffer_object *bo;
 179
 180         do {
 181                 bo = rb_entry(n, struct hmm_buffer_object, node);
 182
 183                 if (bo->start > start) {
 184                         if (n->rb_left == NULL)
 185                                 return NULL;
 186                         n = n->rb_left;
 187                 } else {
 188                         if (bo->end > start)
 189                                 return bo;
 190                         if (n->rb_right == NULL)
 191                                 return NULL;
 192                         n = n->rb_right;
 193                 }
 194         } while (n);
 195
 196         return NULL;
 197 }
 198
 199 static void __bo_insert_to_free_rbtree(struct rb_root *root,
 200                                         struct hmm_buffer_object *bo)
 201 {
 202         struct rb_node **new = &(root->rb_node);
 203         struct rb_node *parent = NULL;
 204         struct hmm_buffer_object *this;
 205         unsigned int pgnr = bo->pgnr;
 206
 207         while (*new) {
 208                 parent = *new;
 209                 this = container_of(*new, struct hmm_buffer_object, node);
 210
 211                 if (pgnr < this->pgnr) {
 212                         new = &((*new)->rb_left);
 213                 } else if (pgnr > this->pgnr) {
 214                         new = &((*new)->rb_right);
 215                 } else {
 216                         bo->prev = this;
 217                         bo->next = this->next;
 218                         if (this->next)
 219                                 this->next->prev = bo;
 220                         this->next = bo;
 221                         bo->status = (bo->status & ~HMM_BO_MASK) | HMM_BO_FREE;
 222                         return;
 223                 }
 224         }
 225
 226         bo->status = (bo->status & ~HMM_BO_MASK) | HMM_BO_FREE;
 227
 228         rb_link_node(&bo->node, parent, new);
 229         rb_insert_color(&bo->node, root);
 230 }
 231
 232 static void __bo_insert_to_alloc_rbtree(struct rb_root *root,
 233                                         struct hmm_buffer_object *bo)
 234 {
 235         struct rb_node **new = &(root->rb_node);
 236         struct rb_node *parent = NULL;
 237         struct hmm_buffer_object *this;
 238         unsigned int start = bo->start;
 239
 240         while (*new) {
 241                 parent = *new;
 242                 this = container_of(*new, struct hmm_buffer_object, node);
 243
 244                 if (start < this->start)
 245                         new = &((*new)->rb_left);
 246                 else
 247                         new = &((*new)->rb_right);
 248         }
 249
 250         kref_init(&bo->kref);
 251         bo->status = (bo->status & ~HMM_BO_MASK) | HMM_BO_ALLOCED;
 252
 253         rb_link_node(&bo->node, parent, new);
 254         rb_insert_color(&bo->node, root);
 255 }
 256
 257 static struct hmm_buffer_object *__bo_break_up(struct hmm_bo_device *bdev,
 258                                         struct hmm_buffer_object *bo,
 259                                         unsigned int pgnr)
 260 {
 261         struct hmm_buffer_object *new_bo;
 262         unsigned long flags;
 263         int ret;
 264
 265         new_bo = __bo_alloc(bdev->bo_cache);
 266         if (!new_bo) {
 267                 dev_err(atomisp_dev, "%s: __bo_alloc failed!\n", __func__);
 268                 return NULL;
 269         }
 270         ret = __bo_init(bdev, new_bo, pgnr);
 271         if (ret) {
 272                 dev_err(atomisp_dev, "%s: __bo_init failed!\n", __func__);
 273                 kmem_cache_free(bdev->bo_cache, new_bo);
 274                 return NULL;
 275         }
 276
 277         new_bo->start = bo->start;
 278         new_bo->end = new_bo->start + pgnr_to_size(pgnr);
 279         bo->start = new_bo->end;
 280         bo->pgnr = bo->pgnr - pgnr;
 281
 282         spin_lock_irqsave(&bdev->list_lock, flags);
 283         list_add_tail(&new_bo->list, &bo->list);
 284         spin_unlock_irqrestore(&bdev->list_lock, flags);
 285
 286         return new_bo;
 287 }
 288
 289 static void __bo_take_off_handling(struct hmm_buffer_object *bo)
 290 {
 291         struct hmm_bo_device *bdev = bo->bdev;
 292         /* There are 4 situations when we take off a known bo from free rbtree:
 293          * 1. if bo->next && bo->prev == NULL, bo is a rbtree node
 294          *      and does not have a linked list after bo, to take off this bo,
 295          *      we just need erase bo directly and rebalance the free rbtree
 296          */
 297         if (bo->prev == NULL && bo->next == NULL) {
 298                 rb_erase(&bo->node, &bdev->free_rbtree);
 299         /* 2. when bo->next != NULL && bo->prev == NULL, bo is a rbtree node,
 300          *      and has a linked list,to take off this bo we need erase bo
 301          *      first, then, insert bo->next into free rbtree and rebalance
 302          *      the free rbtree
 303          */
 304         } else if (bo->prev == NULL && bo->next != NULL) {
 305                 bo->next->prev = NULL;
 306                 rb_erase(&bo->node, &bdev->free_rbtree);
 307                 __bo_insert_to_free_rbtree(&bdev->free_rbtree, bo->next);
 308                 bo->next = NULL;
 309         /* 3. when bo->prev != NULL && bo->next == NULL, bo is not a rbtree
 310          *      node, bo is the last element of the linked list after rbtree
 311          *      node, to take off this bo, we just need set the "prev/next"
 312          *      pointers to NULL, the free rbtree stays unchaged
 313          */
 314         } else if (bo->prev != NULL && bo->next == NULL) {
 315                 bo->prev->next = NULL;
 316                 bo->prev = NULL;
 317         /* 4. when bo->prev != NULL && bo->next != NULL ,bo is not a rbtree
 318          *      node, bo is in the middle of the linked list after rbtree node,
 319          *      to take off this bo, we just set take the "prev/next" pointers
 320          *      to NULL, the free rbtree stays unchaged
 321          */
 322         } else {
 323                 bo->next->prev = bo->prev;
 324                 bo->prev->next = bo->next;
 325                 bo->next = NULL;
 326                 bo->prev = NULL;
 327         }
 328 }
 329
 330 static struct hmm_buffer_object *__bo_merge(struct hmm_buffer_object *bo,
 331                                         struct hmm_buffer_object *next_bo)
 332 {
 333         struct hmm_bo_device *bdev;
 334         unsigned long flags;
 335
 336         bdev = bo->bdev;
 337         next_bo->start = bo->start;
 338         next_bo->pgnr = next_bo->pgnr + bo->pgnr;
 339
 340         spin_lock_irqsave(&bdev->list_lock, flags);
 341         list_del(&bo->list);
 342         spin_unlock_irqrestore(&bdev->list_lock, flags);
 343
 344         kmem_cache_free(bo->bdev->bo_cache, bo);
 345
 346         return next_bo;
 347 }
 348
 349 /*
 350  * hmm_bo_device functions.
 351  */
 352 int hmm_bo_device_init(struct hmm_bo_device *bdev,
 353                                 struct isp_mmu_client *mmu_driver,
 354                                 unsigned int vaddr_start,
 355                                 unsigned int size)
 356 {
 357         struct hmm_buffer_object *bo;
 358         unsigned long flags;
 359         int ret;
 360
 361         check_bodev_null_return(bdev, -EINVAL);
 362
 363         ret = isp_mmu_init(&bdev->mmu, mmu_driver);
 364         if (ret) {
 365                 dev_err(atomisp_dev, "isp_mmu_init failed.\n");
 366                 return ret;
 367         }
 368
 369         bdev->start = vaddr_start;
 370         bdev->pgnr = size_to_pgnr_ceil(size);
 371         bdev->size = pgnr_to_size(bdev->pgnr);
 372
 373         spin_lock_init(&bdev->list_lock);
 374         mutex_init(&bdev->rbtree_mutex);
 375
 376         bdev->flag = HMM_BO_DEVICE_INITED;
 377
 378         INIT_LIST_HEAD(&bdev->entire_bo_list);
 379         bdev->allocated_rbtree = RB_ROOT;
 380         bdev->free_rbtree = RB_ROOT;
 381
 382         bdev->bo_cache = kmem_cache_create("bo_cache",
 383                                 sizeof(struct hmm_buffer_object), 0, 0, NULL);
 384         if (!bdev->bo_cache) {
 385                 dev_err(atomisp_dev, "%s: create cache failed!\n", __func__);
 386                 isp_mmu_exit(&bdev->mmu);
 387                 return -ENOMEM;
 388         }
 389
 390         bo = __bo_alloc(bdev->bo_cache);
 391         if (!bo) {
 392                 dev_err(atomisp_dev, "%s: __bo_alloc failed!\n", __func__);
 393                 isp_mmu_exit(&bdev->mmu);
 394                 return -ENOMEM;
 395         }
 396
 397         ret = __bo_init(bdev, bo, bdev->pgnr);
 398         if (ret) {
 399                 dev_err(atomisp_dev, "%s: __bo_init failed!\n", __func__);
 400                 kmem_cache_free(bdev->bo_cache, bo);
 401                 isp_mmu_exit(&bdev->mmu);
 402                 return -EINVAL;
 403         }
 404
 405         spin_lock_irqsave(&bdev->list_lock, flags);
 406         list_add_tail(&bo->list, &bdev->entire_bo_list);
 407         spin_unlock_irqrestore(&bdev->list_lock, flags);
 408
 409         __bo_insert_to_free_rbtree(&bdev->free_rbtree, bo);
 410
 411         return 0;
 412 }
 413
 414 struct hmm_buffer_object *hmm_bo_alloc(struct hmm_bo_device *bdev,
 415                                         unsigned int pgnr)
 416 {
 417         struct hmm_buffer_object *bo, *new_bo;
 418         struct rb_root *root = &bdev->free_rbtree;
 419
 420         check_bodev_null_return(bdev, NULL);
 421         var_equal_return(hmm_bo_device_inited(bdev), 0, NULL,
 422                         "hmm_bo_device not inited yet.\n");
 423
 424         if (pgnr == 0) {
 425                 dev_err(atomisp_dev, "0 size buffer is not allowed.\n");
 426                 return NULL;
 427         }
 428
 429         mutex_lock(&bdev->rbtree_mutex);
 430         bo = __bo_search_and_remove_from_free_rbtree(root->rb_node, pgnr);
 431         if (!bo) {
 432                 mutex_unlock(&bdev->rbtree_mutex);
 433                 dev_err(atomisp_dev, "%s: Out of Memory! hmm_bo_alloc failed",
 434                         __func__);
 435                 return NULL;
 436         }
 437
 438         if (bo->pgnr > pgnr) {
 439                 new_bo = __bo_break_up(bdev, bo, pgnr);
 440                 if (!new_bo) {
 441                         mutex_unlock(&bdev->rbtree_mutex);
 442                         dev_err(atomisp_dev, "%s: __bo_break_up failed!\n",
 443                                 __func__);
 444                         return NULL;
 445                 }
 446
 447                 __bo_insert_to_alloc_rbtree(&bdev->allocated_rbtree, new_bo);
 448                 __bo_insert_to_free_rbtree(&bdev->free_rbtree, bo);
 449
 450                 mutex_unlock(&bdev->rbtree_mutex);
 451                 return new_bo;
 452         }
 453
 454         __bo_insert_to_alloc_rbtree(&bdev->allocated_rbtree, bo);
 455
 456         mutex_unlock(&bdev->rbtree_mutex);
 457         return bo;
 458 }
 459
 460 void hmm_bo_release(struct hmm_buffer_object *bo)
 461 {
 462         struct hmm_bo_device *bdev = bo->bdev;
 463         struct hmm_buffer_object *next_bo, *prev_bo;
 464
 465         mutex_lock(&bdev->rbtree_mutex);
 466
 467         /*
 468          * FIX ME:
 469          *
 470          * how to destroy the bo when it is stilled MMAPED?
 471          *
 472          * ideally, this will not happened as hmm_bo_release
 473          * will only be called when kref reaches 0, and in mmap
 474          * operation the hmm_bo_ref will eventually be called.
 475          * so, if this happened, something goes wrong.
 476          */
 477         if (bo->status & HMM_BO_MMAPED) {
 478                 mutex_unlock(&bdev->rbtree_mutex);
 479                 dev_dbg(atomisp_dev, "destroy bo which is MMAPED, do nothing\n");
 480                 return;
 481         }
 482
 483         if (bo->status & HMM_BO_BINDED) {
 484                 dev_warn(atomisp_dev, "the bo is still binded, unbind it first...\n");
 485                 hmm_bo_unbind(bo);
 486         }
 487
 488         if (bo->status & HMM_BO_PAGE_ALLOCED) {
 489                 dev_warn(atomisp_dev, "the pages is not freed, free pages first\n");
 490                 hmm_bo_free_pages(bo);
 491         }
 492         if (bo->status & HMM_BO_VMAPED || bo->status & HMM_BO_VMAPED_CACHED) {
 493                 dev_warn(atomisp_dev, "the vunmap is not done, do it...\n");
 494                 hmm_bo_vunmap(bo);
 495         }
 496
 497         rb_erase(&bo->node, &bdev->allocated_rbtree);
 498
 499         prev_bo = list_entry(bo->list.prev, struct hmm_buffer_object, list);
 500         next_bo = list_entry(bo->list.next, struct hmm_buffer_object, list);
 501
 502         if (bo->list.prev != &bdev->entire_bo_list &&
 503                 prev_bo->end == bo->start &&
 504                 (prev_bo->status & HMM_BO_MASK) == HMM_BO_FREE) {
 505                 __bo_take_off_handling(prev_bo);
 506                 bo = __bo_merge(prev_bo, bo);
 507         }
 508
 509         if (bo->list.next != &bdev->entire_bo_list &&
 510                 next_bo->start == bo->end &&
 511                 (next_bo->status & HMM_BO_MASK) == HMM_BO_FREE) {
 512                 __bo_take_off_handling(next_bo);
 513                 bo = __bo_merge(bo, next_bo);
 514         }
 515
 516         __bo_insert_to_free_rbtree(&bdev->free_rbtree, bo);
 517
 518         mutex_unlock(&bdev->rbtree_mutex);
 519         return;
 520 }
 521
 522 void hmm_bo_device_exit(struct hmm_bo_device *bdev)
 523 {
 524         struct hmm_buffer_object *bo;
 525         unsigned long flags;
 526
 527         dev_dbg(atomisp_dev, "%s: entering!\n", __func__);
 528
 529         check_bodev_null_return_void(bdev);
 530
 531         /*
 532          * release all allocated bos even they a in use
 533          * and all bos will be merged into a big bo
 534          */
 535         while (!RB_EMPTY_ROOT(&bdev->allocated_rbtree))
 536                 hmm_bo_release(
 537                         rbtree_node_to_hmm_bo(bdev->allocated_rbtree.rb_node));
 538
 539         dev_dbg(atomisp_dev, "%s: finished releasing all allocated bos!\n",
 540                 __func__);
 541
 542         /* free all bos to release all ISP virtual memory */
 543         while (!list_empty(&bdev->entire_bo_list)) {
 544                 bo = list_to_hmm_bo(bdev->entire_bo_list.next);
 545
 546                 spin_lock_irqsave(&bdev->list_lock, flags);
 547                 list_del(&bo->list);
 548                 spin_unlock_irqrestore(&bdev->list_lock, flags);
 549
 550                 kmem_cache_free(bdev->bo_cache, bo);
 551         }
 552
 553         dev_dbg(atomisp_dev, "%s: finished to free all bos!\n", __func__);
 554
 555         kmem_cache_destroy(bdev->bo_cache);
 556
 557         isp_mmu_exit(&bdev->mmu);
 558 }
 559
 560 int hmm_bo_device_inited(struct hmm_bo_device *bdev)
 561 {
 562         check_bodev_null_return(bdev, -EINVAL);
 563
 564         return bdev->flag == HMM_BO_DEVICE_INITED;
 565 }
 566
 567 int hmm_bo_allocated(struct hmm_buffer_object *bo)
 568 {
 569         check_bo_null_return(bo, 0);
 570
 571         return bo->status & HMM_BO_ALLOCED;
 572 }
 573
 574 struct hmm_buffer_object *hmm_bo_device_search_start(
 575         struct hmm_bo_device *bdev, ia_css_ptr vaddr)
 576 {
 577         struct hmm_buffer_object *bo;
 578
 579         check_bodev_null_return(bdev, NULL);
 580
 581         mutex_lock(&bdev->rbtree_mutex);
 582         bo = __bo_search_by_addr(&bdev->allocated_rbtree, vaddr);
 583         if (!bo) {
 584                 mutex_unlock(&bdev->rbtree_mutex);
 585                 dev_err(atomisp_dev, "%s can not find bo with addr: 0x%x\n",
 586                         __func__, vaddr);
 587                 return NULL;
 588         }
 589         mutex_unlock(&bdev->rbtree_mutex);
 590
 591         return bo;
 592 }
 593
 594 struct hmm_buffer_object *hmm_bo_device_search_in_range(
 595         struct hmm_bo_device *bdev, unsigned int vaddr)
 596 {
 597         struct hmm_buffer_object *bo;
 598
 599         check_bodev_null_return(bdev, NULL);
 600
 601         mutex_lock(&bdev->rbtree_mutex);
 602         bo = __bo_search_by_addr_in_range(&bdev->allocated_rbtree, vaddr);
 603         if (!bo) {
 604                 mutex_unlock(&bdev->rbtree_mutex);
 605                 dev_err(atomisp_dev, "%s can not find bo contain addr: 0x%x\n",
 606                         __func__, vaddr);
 607                 return NULL;
 608         }
 609         mutex_unlock(&bdev->rbtree_mutex);
 610
 611         return bo;
 612 }
 613
 614 struct hmm_buffer_object *hmm_bo_device_search_vmap_start(
 615         struct hmm_bo_device *bdev, const void *vaddr)
 616 {
 617         struct list_head *pos;
 618         struct hmm_buffer_object *bo;
 619         unsigned long flags;
 620
 621         check_bodev_null_return(bdev, NULL);
 622
 623         spin_lock_irqsave(&bdev->list_lock, flags);
 624         list_for_each(pos, &bdev->entire_bo_list) {
 625                 bo = list_to_hmm_bo(pos);
 626                 /* pass bo which has no vm_node allocated */
 627                 if ((bo->status & HMM_BO_MASK) == HMM_BO_FREE)
 628                         continue;
 629                 if (bo->vmap_addr == vaddr)
 630                         goto found;
 631         }
 632         spin_unlock_irqrestore(&bdev->list_lock, flags);
 633         return NULL;
 634 found:
 635         spin_unlock_irqrestore(&bdev->list_lock, flags);
 636         return bo;
 637
 638 }
 639
 640
 641 static void free_private_bo_pages(struct hmm_buffer_object *bo,
 642                                 struct hmm_pool *dypool,
 643                                 struct hmm_pool *repool,
 644                                 int free_pgnr)
 645 {
 646         int i, ret;
 647
 648         for (i = 0; i < free_pgnr; i++) {
 649                 switch (bo->page_obj[i].type) {
 650                 case HMM_PAGE_TYPE_RESERVED:
 651                         if (repool->pops
 652                             && repool->pops->pool_free_pages) {
 653                                 repool->pops->pool_free_pages(repool->pool_info,
 654                                                         &bo->page_obj[i]);
 655                                 hmm_mem_stat.res_cnt--;
 656                         }
 657                         break;
 658                 /*
 659                  * HMM_PAGE_TYPE_GENERAL indicates that pages are from system
 660                  * memory, so when free them, they should be put into dynamic
 661                  * pool.
 662                  */
 663                 case HMM_PAGE_TYPE_DYNAMIC:
 664                 case HMM_PAGE_TYPE_GENERAL:
 665                         if (dypool->pops
 666                             && dypool->pops->pool_inited
 667                             && dypool->pops->pool_inited(dypool->pool_info)) {
 668                                 if (dypool->pops->pool_free_pages)
 669                                         dypool->pops->pool_free_pages(
 670                                                               dypool->pool_info,
 671                                                               &bo->page_obj[i]);
 672                                 break;
 673                         }
 674
 675                         /*
 676                          * if dynamic memory pool doesn't exist, need to free
 677                          * pages to system directly.
 678                          */
 679                 default:
 680                         ret = set_pages_wb(bo->page_obj[i].page, 1);
 681                         if (ret)
 682                                 dev_err(atomisp_dev,
 683                                                 "set page to WB err ...ret = %d\n",
 684                                                         ret);
 685                         /*
 686                         W/A: set_pages_wb seldom return value = -EFAULT
 687                         indicate that address of page is not in valid
 688                         range(0xffff880000000000~0xffffc7ffffffffff)
 689                         then, _free_pages would panic; Do not know why page
 690                         address be valid,it maybe memory corruption by lowmemory
 691                         */
 692                         if (!ret) {
 693                                 __free_pages(bo->page_obj[i].page, 0);
 694                                 hmm_mem_stat.sys_size--;
 695                         }
 696                         break;
 697                 }
 698         }
 699
 700         return;
 701 }
 702
 703 /*Allocate pages which will be used only by ISP*/
 704 static int alloc_private_pages(struct hmm_buffer_object *bo,
 705                                 int from_highmem,
 706                                 bool cached,
 707                                 struct hmm_pool *dypool,
 708                                 struct hmm_pool *repool)
 709 {
 710         int ret;
 711         unsigned int pgnr, order, blk_pgnr, alloc_pgnr;
 712         struct page *pages;
 713         gfp_t gfp = GFP_NOWAIT | __GFP_NOWARN; /* REVISIT: need __GFP_FS too? */
 714         int i, j;
 715         int failure_number = 0;
 716         bool reduce_order = false;
 717         bool lack_mem = true;
 718
 719         if (from_highmem)
 720                 gfp |= __GFP_HIGHMEM;
 721
 722         pgnr = bo->pgnr;
 723
 724         bo->page_obj = kmalloc_array(pgnr, sizeof(struct hmm_page_object),
 725                                 GFP_KERNEL);
 726         if (unlikely(!bo->page_obj))
 727                 return -ENOMEM;
 728
 729         i = 0;
 730         alloc_pgnr = 0;
 731
 732         /*
 733          * get physical pages from dynamic pages pool.
 734          */
 735         if (dypool->pops && dypool->pops->pool_alloc_pages) {
 736                 alloc_pgnr = dypool->pops->pool_alloc_pages(dypool->pool_info,
 737                                                         bo->page_obj, pgnr,
 738                                                         cached);
 739                 hmm_mem_stat.dyc_size -= alloc_pgnr;
 740
 741                 if (alloc_pgnr == pgnr)
 742                         return 0;
 743         }
 744
 745         pgnr -= alloc_pgnr;
 746         i += alloc_pgnr;
 747
 748         /*
 749          * get physical pages from reserved pages pool for atomisp.
 750          */
 751         if (repool->pops && repool->pops->pool_alloc_pages) {
 752                 alloc_pgnr = repool->pops->pool_alloc_pages(repool->pool_info,
 753                                                         &bo->page_obj[i], pgnr,
 754                                                         cached);
 755                 hmm_mem_stat.res_cnt += alloc_pgnr;
 756                 if (alloc_pgnr == pgnr)
 757                         return 0;
 758         }
 759
 760         pgnr -= alloc_pgnr;
 761         i += alloc_pgnr;
 762
 763         while (pgnr) {
 764                 order = nr_to_order_bottom(pgnr);
 765                 /*
 766                  * if be short of memory, we will set order to 0
 767                  * everytime.
 768                  */
 769                 if (lack_mem)
 770                         order = HMM_MIN_ORDER;
 771                 else if (order > HMM_MAX_ORDER)
 772                         order = HMM_MAX_ORDER;
 773 retry:
 774                 /*
 775                  * When order > HMM_MIN_ORDER, for performance reasons we don't
 776                  * want alloc_pages() to sleep. In case it fails and fallbacks
 777                  * to HMM_MIN_ORDER or in case the requested order is originally
 778                  * the minimum value, we can allow alloc_pages() to sleep for
 779                  * robustness purpose.
 780                  *
 781                  * REVISIT: why __GFP_FS is necessary?
 782                  */
 783                 if (order == HMM_MIN_ORDER) {
 784                         gfp &= ~GFP_NOWAIT;
 785                         gfp |= __GFP_RECLAIM | __GFP_FS;
 786                 }
 787
 788                 pages = alloc_pages(gfp, order);
 789                 if (unlikely(!pages)) {
 790                         /*
 791                          * in low memory case, if allocation page fails,
 792                          * we turn to try if order=0 allocation could
 793                          * succeed. if order=0 fails too, that means there is
 794                          * no memory left.
 795                          */
 796                         if (order == HMM_MIN_ORDER) {
 797                                 dev_err(atomisp_dev,
 798                                         "%s: cannot allocate pages\n",
 799                                          __func__);
 800                                 goto cleanup;
 801                         }
 802                         order = HMM_MIN_ORDER;
 803                         failure_number++;
 804                         reduce_order = true;
 805                         /*
 806                          * if fail two times continuously, we think be short
 807                          * of memory now.
 808                          */
 809                         if (failure_number == 2) {
 810                                 lack_mem = true;
 811                                 failure_number = 0;
 812                         }
 813                         goto retry;
 814                 } else {
 815                         blk_pgnr = order_to_nr(order);
 816
 817                         if (!cached) {
 818                                 /*
 819                                  * set memory to uncacheable -- UC_MINUS
 820                                  */
 821                                 ret = set_pages_uc(pages, blk_pgnr);
 822                                 if (ret) {
 823                                         dev_err(atomisp_dev,
 824                                                      "set page uncacheable"
 825                                                         "failed.\n");
 826
 827                                         __free_pages(pages, order);
 828
 829                                         goto cleanup;
 830                                 }
 831                         }
 832
 833                         for (j = 0; j < blk_pgnr; j++) {
 834                                 bo->page_obj[i].page = pages + j;
 835                                 bo->page_obj[i++].type = HMM_PAGE_TYPE_GENERAL;
 836                         }
 837
 838                         pgnr -= blk_pgnr;
 839                         hmm_mem_stat.sys_size += blk_pgnr;
 840
 841                         /*
 842                          * if order is not reduced this time, clear
 843                          * failure_number.
 844                          */
 845                         if (reduce_order)
 846                                 reduce_order = false;
 847                         else
 848                                 failure_number = 0;
 849                 }
 850         }
 851
 852         return 0;
 853 cleanup:
 854         alloc_pgnr = i;
 855         free_private_bo_pages(bo, dypool, repool, alloc_pgnr);
 856
 857         kfree(bo->page_obj);
 858
 859         return -ENOMEM;
 860 }
 861
 862 static void free_private_pages(struct hmm_buffer_object *bo,
 863                                 struct hmm_pool *dypool,
 864                                 struct hmm_pool *repool)
 865 {
 866         free_private_bo_pages(bo, dypool, repool, bo->pgnr);
 867
 868         kfree(bo->page_obj);
 869 }
 870
 871 /*
 872  * Hacked from kernel function __get_user_pages in mm/memory.c
 873  *
 874  * Handle buffers allocated by other kernel space driver and mmaped into user
 875  * space, function Ignore the VM_PFNMAP and VM_IO flag in VMA structure
 876  *
 877  * Get physical pages from user space virtual address and update into page list
 878  */
 879 static int __get_pfnmap_pages(struct task_struct *tsk, struct mm_struct *mm,
 880                               unsigned long start, int nr_pages,
 881                               unsigned int gup_flags, struct page **pages,
 882                               struct vm_area_struct **vmas)
 883 {
 884         int i, ret;
 885         unsigned long vm_flags;
 886
 887         if (nr_pages <= 0)
 888                 return 0;
 889
 890         VM_BUG_ON(!!pages != !!(gup_flags & FOLL_GET));
 891
 892         /*
 893          * Require read or write permissions.
 894          * If FOLL_FORCE is set, we only require the "MAY" flags.
 895          */
 896         vm_flags  = (gup_flags & FOLL_WRITE) ?
 897                         (VM_WRITE | VM_MAYWRITE) : (VM_READ | VM_MAYREAD);
 898         vm_flags &= (gup_flags & FOLL_FORCE) ?
 899                         (VM_MAYREAD | VM_MAYWRITE) : (VM_READ | VM_WRITE);
 900         i = 0;
 901
 902         do {
 903                 struct vm_area_struct *vma;
 904
 905                 vma = find_vma(mm, start);
 906                 if (!vma) {
 907                         dev_err(atomisp_dev, "find_vma failed\n");
 908                         return i ? : -EFAULT;
 909                 }
 910
 911                 if (is_vm_hugetlb_page(vma)) {
 912                         /*
 913                         i = follow_hugetlb_page(mm, vma, pages, vmas,
 914                                         &start, &nr_pages, i, gup_flags);
 915                         */
 916                         continue;
 917                 }
 918
 919                 do {
 920                         struct page *page;
 921                         unsigned long pfn;
 922
 923                         /*
 924                          * If we have a pending SIGKILL, don't keep faulting
 925                          * pages and potentially allocating memory.
 926                          */
 927                         if (unlikely(fatal_signal_pending(current))) {
 928                                 dev_err(atomisp_dev,
 929                                         "fatal_signal_pending in %s\n",
 930                                         __func__);
 931                                 return i ? i : -ERESTARTSYS;
 932                         }
 933
 934                         ret = follow_pfn(vma, start, &pfn);
 935                         if (ret) {
 936                                 dev_err(atomisp_dev, "follow_pfn() failed\n");
 937                                 return i ? : -EFAULT;
 938                         }
 939
 940                         page = pfn_to_page(pfn);
 941                         if (IS_ERR(page))
 942                                 return i ? i : PTR_ERR(page);
 943                         if (pages) {
 944                                 pages[i] = page;
 945                                 get_page(page);
 946                                 flush_anon_page(vma, page, start);
 947                                 flush_dcache_page(page);
 948                         }
 949                         if (vmas)
 950                                 vmas[i] = vma;
 951                         i++;
 952                         start += PAGE_SIZE;
 953                         nr_pages--;
 954                 } while (nr_pages && start < vma->vm_end);
 955         } while (nr_pages);
 956
 957         return i;
 958 }
 959
 960 static int get_pfnmap_pages(struct task_struct *tsk, struct mm_struct *mm,
 961                      unsigned long start, int nr_pages, int write, int force,
 962                      struct page **pages, struct vm_area_struct **vmas)
 963 {
 964         int flags = FOLL_TOUCH;
 965
 966         if (pages)
 967                 flags |= FOLL_GET;
 968         if (write)
 969                 flags |= FOLL_WRITE;
 970         if (force)
 971                 flags |= FOLL_FORCE;
 972
 973         return __get_pfnmap_pages(tsk, mm, start, nr_pages, flags, pages, vmas);
 974 }
 975
 976 /*
 977  * Convert user space virtual address into pages list
 978  */
 979 static int alloc_user_pages(struct hmm_buffer_object *bo,
 980                               void *userptr, bool cached)
 981 {
 982         int page_nr;
 983         int i;
 984         struct vm_area_struct *vma;
 985         struct page **pages;
 986
 987         pages = kmalloc_array(bo->pgnr, sizeof(struct page *), GFP_KERNEL);
 988         if (unlikely(!pages))
 989                 return -ENOMEM;
 990
 991         bo->page_obj = kmalloc_array(bo->pgnr, sizeof(struct hmm_page_object),
 992                 GFP_KERNEL);
 993         if (unlikely(!bo->page_obj)) {
 994                 kfree(pages);
 995                 return -ENOMEM;
 996         }
 997
 998         mutex_unlock(&bo->mutex);
 999         down_read(&current->mm->mmap_sem);
1000         vma = find_vma(current->mm, (unsigned long)userptr);
1001         up_read(&current->mm->mmap_sem);
1002         if (vma == NULL) {
1003                 dev_err(atomisp_dev, "find_vma failed\n");
1004                 kfree(bo->page_obj);
1005                 kfree(pages);
1006                 mutex_lock(&bo->mutex);
1007                 return -EFAULT;
1008         }
1009         mutex_lock(&bo->mutex);
1010         /*
1011          * Handle frame buffer allocated in other kerenl space driver
1012          * and map to user space
1013          */
1014         if (vma->vm_flags & (VM_IO | VM_PFNMAP)) {
1015                 page_nr = get_pfnmap_pages(current, current->mm,
1016                                            (unsigned long)userptr,
1017                                            (int)(bo->pgnr), 1, 0,
1018                                            pages, NULL);
1019                 bo->mem_type = HMM_BO_MEM_TYPE_PFN;
1020         } else {
1021                 /*Handle frame buffer allocated in user space*/
1022                 mutex_unlock(&bo->mutex);
1023                 page_nr = get_user_pages_fast((unsigned long)userptr,
1024                                          (int)(bo->pgnr), 1, pages);
1025                 mutex_lock(&bo->mutex);
1026                 bo->mem_type = HMM_BO_MEM_TYPE_USER;
1027         }
1028
1029         /* can be written by caller, not forced */
1030         if (page_nr != bo->pgnr) {
1031                 dev_err(atomisp_dev,
1032                                 "get_user_pages err: bo->pgnr = %d, "
1033                                 "pgnr actually pinned = %d.\n",
1034                                 bo->pgnr, page_nr);
1035                 goto out_of_mem;
1036         }
1037
1038         for (i = 0; i < bo->pgnr; i++) {
1039                 bo->page_obj[i].page = pages[i];
1040                 bo->page_obj[i].type = HMM_PAGE_TYPE_GENERAL;
1041         }
1042         hmm_mem_stat.usr_size += bo->pgnr;
1043         kfree(pages);
1044
1045         return 0;
1046
1047 out_of_mem:
1048         for (i = 0; i < page_nr; i++)
1049                 put_page(pages[i]);
1050         kfree(pages);
1051         kfree(bo->page_obj);
1052
1053         return -ENOMEM;
1054 }
1055
1056 static void free_user_pages(struct hmm_buffer_object *bo)
1057 {
1058         int i;
1059
1060         for (i = 0; i < bo->pgnr; i++)
1061                 put_page(bo->page_obj[i].page);
1062         hmm_mem_stat.usr_size -= bo->pgnr;
1063
1064         kfree(bo->page_obj);
1065 }
1066
1067 /*
1068  * allocate/free physical pages for the bo.
1069  *
1070  * type indicate where are the pages from. currently we have 3 types
1071  * of memory: HMM_BO_PRIVATE, HMM_BO_USER, HMM_BO_SHARE.
1072  *
1073  * from_highmem is only valid when type is HMM_BO_PRIVATE, it will
1074  * try to alloc memory from highmem if from_highmem is set.
1075  *
1076  * userptr is only valid when type is HMM_BO_USER, it indicates
1077  * the start address from user space task.
1078  *
1079  * from_highmem and userptr will both be ignored when type is
1080  * HMM_BO_SHARE.
1081  */
1082 int hmm_bo_alloc_pages(struct hmm_buffer_object *bo,
1083                        enum hmm_bo_type type, int from_highmem,
1084                        void *userptr, bool cached)
1085 {
1086         int ret = -EINVAL;
1087
1088         check_bo_null_return(bo, -EINVAL);
1089
1090         mutex_lock(&bo->mutex);
1091         check_bo_status_no_goto(bo, HMM_BO_PAGE_ALLOCED, status_err);
1092
1093         /*
1094          * TO DO:
1095          * add HMM_BO_USER type
1096          */
1097         if (type == HMM_BO_PRIVATE) {
1098                 ret = alloc_private_pages(bo, from_highmem,
1099                                 cached, &dynamic_pool, &reserved_pool);
1100         } else if (type == HMM_BO_USER) {
1101                 ret = alloc_user_pages(bo, userptr, cached);
1102         } else {
1103                 dev_err(atomisp_dev, "invalid buffer type.\n");
1104                 ret = -EINVAL;
1105         }
1106         if (ret)
1107                 goto alloc_err;
1108
1109         bo->type = type;
1110
1111         bo->status |= HMM_BO_PAGE_ALLOCED;
1112
1113         mutex_unlock(&bo->mutex);
1114
1115         return 0;
1116
1117 alloc_err:
1118         mutex_unlock(&bo->mutex);
1119         dev_err(atomisp_dev, "alloc pages err...\n");
1120         return ret;
1121 status_err:
1122         mutex_unlock(&bo->mutex);
1123         dev_err(atomisp_dev,
1124                         "buffer object has already page allocated.\n");
1125         return -EINVAL;
1126 }
1127
1128 /*
1129  * free physical pages of the bo.
1130  */
1131 void hmm_bo_free_pages(struct hmm_buffer_object *bo)
1132 {
1133         check_bo_null_return_void(bo);
1134
1135         mutex_lock(&bo->mutex);
1136
1137         check_bo_status_yes_goto(bo, HMM_BO_PAGE_ALLOCED, status_err2);
1138
1139         /* clear the flag anyway. */
1140         bo->status &= (~HMM_BO_PAGE_ALLOCED);
1141
1142         if (bo->type == HMM_BO_PRIVATE)
1143                 free_private_pages(bo, &dynamic_pool, &reserved_pool);
1144         else if (bo->type == HMM_BO_USER)
1145                 free_user_pages(bo);
1146         else
1147                 dev_err(atomisp_dev, "invalid buffer type.\n");
1148         mutex_unlock(&bo->mutex);
1149
1150         return;
1151
1152 status_err2:
1153         mutex_unlock(&bo->mutex);
1154         dev_err(atomisp_dev,
1155                         "buffer object not page allocated yet.\n");
1156 }
1157
1158 int hmm_bo_page_allocated(struct hmm_buffer_object *bo)
1159 {
1160         check_bo_null_return(bo, 0);
1161
1162         return bo->status & HMM_BO_PAGE_ALLOCED;
1163 }
1164
1165 /*
1166  * get physical page info of the bo.
1167  */
1168 int hmm_bo_get_page_info(struct hmm_buffer_object *bo,
1169                          struct hmm_page_object **page_obj, int *pgnr)
1170 {
1171         check_bo_null_return(bo, -EINVAL);
1172
1173         mutex_lock(&bo->mutex);
1174
1175         check_bo_status_yes_goto(bo, HMM_BO_PAGE_ALLOCED, status_err);
1176
1177         *page_obj = bo->page_obj;
1178         *pgnr = bo->pgnr;
1179
1180         mutex_unlock(&bo->mutex);
1181
1182         return 0;
1183
1184 status_err:
1185         dev_err(atomisp_dev,
1186                         "buffer object not page allocated yet.\n");
1187         mutex_unlock(&bo->mutex);
1188         return -EINVAL;
1189 }
1190
1191 /*
1192  * bind the physical pages to a virtual address space.
1193  */
1194 int hmm_bo_bind(struct hmm_buffer_object *bo)
1195 {
1196         int ret;
1197         unsigned int virt;
1198         struct hmm_bo_device *bdev;
1199         unsigned int i;
1200
1201         check_bo_null_return(bo, -EINVAL);
1202
1203         mutex_lock(&bo->mutex);
1204
1205         check_bo_status_yes_goto(bo,
1206                                    HMM_BO_PAGE_ALLOCED | HMM_BO_ALLOCED,
1207                                    status_err1);
1208
1209         check_bo_status_no_goto(bo, HMM_BO_BINDED, status_err2);
1210
1211         bdev = bo->bdev;
1212
1213         virt = bo->start;
1214
1215         for (i = 0; i < bo->pgnr; i++) {
1216                 ret =
1217                     isp_mmu_map(&bdev->mmu, virt,
1218                                 page_to_phys(bo->page_obj[i].page), 1);
1219                 if (ret)
1220                         goto map_err;
1221                 virt += (1 << PAGE_SHIFT);
1222         }
1223
1224         /*
1225          * flush TBL here.
1226          *
1227          * theoretically, we donot need to flush TLB as we didnot change
1228          * any existed address mappings, but for Silicon Hive's MMU, its
1229          * really a bug here. I guess when fetching PTEs (page table entity)
1230          * to TLB, its MMU will fetch additional INVALID PTEs automatically
1231          * for performance issue. EX, we only set up 1 page address mapping,
1232          * meaning updating 1 PTE, but the MMU fetches 4 PTE at one time,
1233          * so the additional 3 PTEs are invalid.
1234          */
1235         if (bo->start != 0x0)
1236                 isp_mmu_flush_tlb_range(&bdev->mmu, bo->start,
1237                                                 (bo->pgnr << PAGE_SHIFT));
1238
1239         bo->status |= HMM_BO_BINDED;
1240
1241         mutex_unlock(&bo->mutex);
1242
1243         return 0;
1244
1245 map_err:
1246         /* unbind the physical pages with related virtual address space */
1247         virt = bo->start;
1248         for ( ; i > 0; i--) {
1249                 isp_mmu_unmap(&bdev->mmu, virt, 1);
1250                 virt += pgnr_to_size(1);
1251         }
1252
1253         mutex_unlock(&bo->mutex);
1254         dev_err(atomisp_dev,
1255                         "setup MMU address mapping failed.\n");
1256         return ret;
1257
1258 status_err2:
1259         mutex_unlock(&bo->mutex);
1260         dev_err(atomisp_dev, "buffer object already binded.\n");
1261         return -EINVAL;
1262 status_err1:
1263         mutex_unlock(&bo->mutex);
1264         dev_err(atomisp_dev,
1265                      "buffer object vm_node or page not allocated.\n");
1266         return -EINVAL;
1267 }
1268
1269 /*
1270  * unbind the physical pages with related virtual address space.
1271  */
1272 void hmm_bo_unbind(struct hmm_buffer_object *bo)
1273 {
1274         unsigned int virt;
1275         struct hmm_bo_device *bdev;
1276         unsigned int i;
1277
1278         check_bo_null_return_void(bo);
1279
1280         mutex_lock(&bo->mutex);
1281
1282         check_bo_status_yes_goto(bo,
1283                                    HMM_BO_PAGE_ALLOCED |
1284                                    HMM_BO_ALLOCED |
1285                                    HMM_BO_BINDED, status_err);
1286
1287         bdev = bo->bdev;
1288
1289         virt = bo->start;
1290
1291         for (i = 0; i < bo->pgnr; i++) {
1292                 isp_mmu_unmap(&bdev->mmu, virt, 1);
1293                 virt += pgnr_to_size(1);
1294         }
1295
1296         /*
1297          * flush TLB as the address mapping has been removed and
1298          * related TLBs should be invalidated.
1299          */
1300         isp_mmu_flush_tlb_range(&bdev->mmu, bo->start,
1301                                 (bo->pgnr << PAGE_SHIFT));
1302
1303         bo->status &= (~HMM_BO_BINDED);
1304
1305         mutex_unlock(&bo->mutex);
1306
1307         return;
1308
1309 status_err:
1310         mutex_unlock(&bo->mutex);
1311         dev_err(atomisp_dev,
1312                      "buffer vm or page not allocated or not binded yet.\n");
1313 }
1314
1315 int hmm_bo_binded(struct hmm_buffer_object *bo)
1316 {
1317         int ret;
1318
1319         check_bo_null_return(bo, 0);
1320
1321         mutex_lock(&bo->mutex);
1322
1323         ret = bo->status & HMM_BO_BINDED;
1324
1325         mutex_unlock(&bo->mutex);
1326
1327         return ret;
1328 }
1329
1330 void *hmm_bo_vmap(struct hmm_buffer_object *bo, bool cached)
1331 {
1332         struct page **pages;
1333         int i;
1334
1335         check_bo_null_return(bo, NULL);
1336
1337         mutex_lock(&bo->mutex);
1338         if (((bo->status & HMM_BO_VMAPED) && !cached) ||
1339             ((bo->status & HMM_BO_VMAPED_CACHED) && cached)) {
1340                 mutex_unlock(&bo->mutex);
1341                 return bo->vmap_addr;
1342         }
1343
1344         /* cached status need to be changed, so vunmap first */
1345         if (bo->status & HMM_BO_VMAPED || bo->status & HMM_BO_VMAPED_CACHED) {
1346                 vunmap(bo->vmap_addr);
1347                 bo->vmap_addr = NULL;
1348                 bo->status &= ~(HMM_BO_VMAPED | HMM_BO_VMAPED_CACHED);
1349         }
1350
1351         pages = kmalloc_array(bo->pgnr, sizeof(*pages), GFP_KERNEL);
1352         if (unlikely(!pages)) {
1353                 mutex_unlock(&bo->mutex);
1354                 return NULL;
1355         }
1356
1357         for (i = 0; i < bo->pgnr; i++)
1358                 pages[i] = bo->page_obj[i].page;
1359
1360         bo->vmap_addr = vmap(pages, bo->pgnr, VM_MAP,
1361                 cached ? PAGE_KERNEL : PAGE_KERNEL_NOCACHE);
1362         if (unlikely(!bo->vmap_addr)) {
1363                 kfree(pages);
1364                 mutex_unlock(&bo->mutex);
1365                 dev_err(atomisp_dev, "vmap failed...\n");
1366                 return NULL;
1367         }
1368         bo->status |= (cached ? HMM_BO_VMAPED_CACHED : HMM_BO_VMAPED);
1369
1370         kfree(pages);
1371
1372         mutex_unlock(&bo->mutex);
1373         return bo->vmap_addr;
1374 }
1375
1376 void hmm_bo_flush_vmap(struct hmm_buffer_object *bo)
1377 {
1378         check_bo_null_return_void(bo);
1379
1380         mutex_lock(&bo->mutex);
1381         if (!(bo->status & HMM_BO_VMAPED_CACHED) || !bo->vmap_addr) {
1382                 mutex_unlock(&bo->mutex);
1383                 return;
1384         }
1385
1386         clflush_cache_range(bo->vmap_addr, bo->pgnr * PAGE_SIZE);
1387         mutex_unlock(&bo->mutex);
1388 }
1389
1390 void hmm_bo_vunmap(struct hmm_buffer_object *bo)
1391 {
1392         check_bo_null_return_void(bo);
1393
1394         mutex_lock(&bo->mutex);
1395         if (bo->status & HMM_BO_VMAPED || bo->status & HMM_BO_VMAPED_CACHED) {
1396                 vunmap(bo->vmap_addr);
1397                 bo->vmap_addr = NULL;
1398                 bo->status &= ~(HMM_BO_VMAPED | HMM_BO_VMAPED_CACHED);
1399         }
1400
1401         mutex_unlock(&bo->mutex);
1402         return;
1403 }
1404
1405 void hmm_bo_ref(struct hmm_buffer_object *bo)
1406 {
1407         check_bo_null_return_void(bo);
1408
1409         kref_get(&bo->kref);
1410 }
1411
1412 static void kref_hmm_bo_release(struct kref *kref)
1413 {
1414         if (!kref)
1415                 return;
1416
1417         hmm_bo_release(kref_to_hmm_bo(kref));
1418 }
1419
1420 void hmm_bo_unref(struct hmm_buffer_object *bo)
1421 {
1422         check_bo_null_return_void(bo);
1423
1424         kref_put(&bo->kref, kref_hmm_bo_release);
1425 }
1426
1427 static void hmm_bo_vm_open(struct vm_area_struct *vma)
1428 {
1429         struct hmm_buffer_object *bo =
1430             (struct hmm_buffer_object *)vma->vm_private_data;
1431
1432         check_bo_null_return_void(bo);
1433
1434         hmm_bo_ref(bo);
1435
1436         mutex_lock(&bo->mutex);
1437
1438         bo->status |= HMM_BO_MMAPED;
1439
1440         bo->mmap_count++;
1441
1442         mutex_unlock(&bo->mutex);
1443 }
1444
1445 static void hmm_bo_vm_close(struct vm_area_struct *vma)
1446 {
1447         struct hmm_buffer_object *bo =
1448             (struct hmm_buffer_object *)vma->vm_private_data;
1449
1450         check_bo_null_return_void(bo);
1451
1452         hmm_bo_unref(bo);
1453
1454         mutex_lock(&bo->mutex);
1455
1456         bo->mmap_count--;
1457
1458         if (!bo->mmap_count) {
1459                 bo->status &= (~HMM_BO_MMAPED);
1460                 vma->vm_private_data = NULL;
1461         }
1462
1463         mutex_unlock(&bo->mutex);
1464 }
1465
1466 static const struct vm_operations_struct hmm_bo_vm_ops = {
1467         .open = hmm_bo_vm_open,
1468         .close = hmm_bo_vm_close,
1469 };
1470
1471 /*
1472  * mmap the bo to user space.
1473  */
1474 int hmm_bo_mmap(struct vm_area_struct *vma, struct hmm_buffer_object *bo)
1475 {
1476         unsigned int start, end;
1477         unsigned int virt;
1478         unsigned int pgnr, i;
1479         unsigned int pfn;
1480
1481         check_bo_null_return(bo, -EINVAL);
1482
1483         check_bo_status_yes_goto(bo, HMM_BO_PAGE_ALLOCED, status_err);
1484
1485         pgnr = bo->pgnr;
1486         start = vma->vm_start;
1487         end = vma->vm_end;
1488
1489         /*
1490          * check vma's virtual address space size and buffer object's size.
1491          * must be the same.
1492          */
1493         if ((start + pgnr_to_size(pgnr)) != end) {
1494                 dev_warn(atomisp_dev,
1495                              "vma's address space size not equal"
1496                              " to buffer object's size");
1497                 return -EINVAL;
1498         }
1499
1500         virt = vma->vm_start;
1501         for (i = 0; i < pgnr; i++) {
1502                 pfn = page_to_pfn(bo->page_obj[i].page);
1503                 if (remap_pfn_range(vma, virt, pfn, PAGE_SIZE, PAGE_SHARED)) {
1504                         dev_warn(atomisp_dev,
1505                                         "remap_pfn_range failed:"
1506                                         " virt = 0x%x, pfn = 0x%x,"
1507                                         " mapped_pgnr = %d\n", virt, pfn, 1);
1508                         return -EINVAL;
1509                 }
1510                 virt += PAGE_SIZE;
1511         }
1512
1513         vma->vm_private_data = bo;
1514
1515         vma->vm_ops = &hmm_bo_vm_ops;
1516         vma->vm_flags |= VM_IO|VM_DONTEXPAND|VM_DONTDUMP;
1517
1518         /*
1519          * call hmm_bo_vm_open explictly.
1520          */
1521         hmm_bo_vm_open(vma);
1522
1523         return 0;
1524
1525 status_err:
1526         dev_err(atomisp_dev, "buffer page not allocated yet.\n");
1527         return -EINVAL;
1528 }