IB/srp: try to use larger FMR sizes to cover our mappings

Now that we can get larger SG lists, we can take advantage of HCAs that
allow us to use larger FMR sizes. In many cases, we can use up to 512
entries, so start there and work our way down.

Signed-off-by: David Dillow <dillowda@ornl.gov>

diff --git a/drivers/infiniband/ulp/srp/ib_srp.c b/drivers/infiniband/ulp/srp/ib_srp.c
index 4ec7dddbbf4901ea7297b031676ea80dc87616dc..376d640487d2ce942ff58d348a38a8a9737c5699 100644 (file)
--- a/drivers/infiniband/ulp/srp/ib_srp.c
+++ b/drivers/infiniband/ulp/srp/ib_srp.c
@@ -2300,7 +2300,7 @@ static void srp_add_one(struct ib_device *device)
        struct ib_device_attr *dev_attr;
        struct ib_fmr_pool_param fmr_param;
        struct srp_host *host;
-       int fmr_page_shift, s, e, p;
+       int max_pages_per_fmr, fmr_page_shift, s, e, p;
 
        dev_attr = kmalloc(sizeof *dev_attr, GFP_KERNEL);
        if (!dev_attr)
@@ -2340,17 +2340,24 @@ static void srp_add_one(struct ib_device *device)
        if (IS_ERR(srp_dev->mr))
                goto err_pd;
 
-       memset(&fmr_param, 0, sizeof fmr_param);
-       fmr_param.pool_size         = SRP_FMR_POOL_SIZE;
-       fmr_param.dirty_watermark   = SRP_FMR_DIRTY_SIZE;
-       fmr_param.cache             = 1;
-       fmr_param.max_pages_per_fmr = SRP_FMR_SIZE;
-       fmr_param.page_shift        = fmr_page_shift;
-       fmr_param.access            = (IB_ACCESS_LOCAL_WRITE |
-                                      IB_ACCESS_REMOTE_WRITE |
-                                      IB_ACCESS_REMOTE_READ);
-
-       srp_dev->fmr_pool = ib_create_fmr_pool(srp_dev->pd, &fmr_param);
+       for (max_pages_per_fmr = SRP_FMR_SIZE;
+                       max_pages_per_fmr >= SRP_FMR_MIN_SIZE;
+                       max_pages_per_fmr /= 2, srp_dev->fmr_max_size /= 2) {
+               memset(&fmr_param, 0, sizeof fmr_param);
+               fmr_param.pool_size         = SRP_FMR_POOL_SIZE;
+               fmr_param.dirty_watermark   = SRP_FMR_DIRTY_SIZE;
+               fmr_param.cache             = 1;
+               fmr_param.max_pages_per_fmr = max_pages_per_fmr;
+               fmr_param.page_shift        = fmr_page_shift;
+               fmr_param.access            = (IB_ACCESS_LOCAL_WRITE |
+                                              IB_ACCESS_REMOTE_WRITE |
+                                              IB_ACCESS_REMOTE_READ);
+
+               srp_dev->fmr_pool = ib_create_fmr_pool(srp_dev->pd, &fmr_param);
+               if (!IS_ERR(srp_dev->fmr_pool))
+                       break;
+       }
+
        if (IS_ERR(srp_dev->fmr_pool))
                srp_dev->fmr_pool = NULL;
 

diff --git a/drivers/infiniband/ulp/srp/ib_srp.h b/drivers/infiniband/ulp/srp/ib_srp.h
index cf696218eeebb957453021c31396a110b2e707d7..020caf0c3789ed0d6bd14fa95708132a3bdb8c94 100644 (file)
--- a/drivers/infiniband/ulp/srp/ib_srp.h
+++ b/drivers/infiniband/ulp/srp/ib_srp.h
@@ -69,7 +69,8 @@ enum {
        SRP_TAG_NO_REQ          = ~0U,
        SRP_TAG_TSK_MGMT        = 1U << 31,
 
-       SRP_FMR_SIZE            = 256,
+       SRP_FMR_SIZE            = 512,
+       SRP_FMR_MIN_SIZE        = 128,
        SRP_FMR_POOL_SIZE       = 1024,
        SRP_FMR_DIRTY_SIZE      = SRP_FMR_POOL_SIZE / 4,