LU-7054 o2iblnd: less intense allocating retry

ko2iblnd may retry too frequent for growing pools, all schedulers
are spinning if another thread is in progress of allocating a new
pool and can't finish right away because of high system load.

Signed-off-by: Liang Zhen <liang.zhen@intel.com>
Change-Id: I21be43c6f77b1ae13d500ecbd6795b6d0099d2f1
Reviewed-on: http://review.whamcloud.com/16470
Tested-by: Jenkins
Tested-by: Maloo <hpdd-maloo@intel.com>
Reviewed-by: Doug Oucharek <doug.s.oucharek@intel.com>
Reviewed-by: James Simmons <uja.ornl@yahoo.com>
Reviewed-by: Oleg Drokin <oleg.drokin@intel.com>

diff --git a/lnet/klnds/o2iblnd/o2iblnd.c b/lnet/klnds/o2iblnd/o2iblnd.c
index 293bb0c..d9761ee 100644 (file)
--- a/lnet/klnds/o2iblnd/o2iblnd.c
+++ b/lnet/klnds/o2iblnd/o2iblnd.c
@@ -1306,6 +1306,7 @@ kiblnd_current_hdev(kib_dev_t *dev)
                if (i++ % 50 == 0)
                        CDEBUG(D_NET, "%s: Wait for failover\n",
                               dev->ibd_ifname);
                if (i++ % 50 == 0)
                        CDEBUG(D_NET, "%s: Wait for failover\n",
                               dev->ibd_ifname);

+               set_current_state(TASK_INTERRUPTIBLE);

                schedule_timeout(cfs_time_seconds(1) / 100);
 
                read_lock_irqsave(&kiblnd_data.kib_global_lock, flags);
                schedule_timeout(cfs_time_seconds(1) / 100);
 
                read_lock_irqsave(&kiblnd_data.kib_global_lock, flags);


@@ -1793,6 +1794,9 @@ kiblnd_pool_alloc_node(kib_poolset_t *ps)
        struct list_head        *node;
        kib_pool_t              *pool;
        int                     rc;
        struct list_head        *node;
        kib_pool_t              *pool;
        int                     rc;

+       unsigned int            interval = 1;
+       cfs_time_t              time_before;
+       unsigned int            trips = 0;

 
 again:
        spin_lock(&ps->ps_lock);
 
 again:
        spin_lock(&ps->ps_lock);


@@ -1817,10 +1821,17 @@ again:
        if (ps->ps_increasing) {
                /* another thread is allocating a new pool */
                spin_unlock(&ps->ps_lock);
        if (ps->ps_increasing) {
                /* another thread is allocating a new pool */
                spin_unlock(&ps->ps_lock);

+               trips++;

                 CDEBUG(D_NET, "Another thread is allocating new "
                 CDEBUG(D_NET, "Another thread is allocating new "

-                       "%s pool, waiting for her to complete\n",
-                       ps->ps_name);
-               schedule();
+                      "%s pool, waiting %d HZs for her to complete."
+                      "trips = %d\n",
+                      ps->ps_name, interval, trips);
+
+               set_current_state(TASK_INTERRUPTIBLE);
+               schedule_timeout(interval);
+               if (interval < cfs_time_seconds(1))
+                       interval *= 2;
+

                 goto again;
         }
 
                 goto again;
         }
 


@@ -1834,8 +1845,10 @@ again:
        spin_unlock(&ps->ps_lock);
 
        CDEBUG(D_NET, "%s pool exhausted, allocate new pool\n", ps->ps_name);
        spin_unlock(&ps->ps_lock);
 
        CDEBUG(D_NET, "%s pool exhausted, allocate new pool\n", ps->ps_name);

-
+       time_before = cfs_time_current();

        rc = ps->ps_pool_create(ps, ps->ps_pool_size, &pool);
        rc = ps->ps_pool_create(ps, ps->ps_pool_size, &pool);

+       CDEBUG(D_NET, "ps_pool_create took %lu HZ to complete",
+              cfs_time_current() - time_before);

 
        spin_lock(&ps->ps_lock);
        ps->ps_increasing = 0;
 
        spin_lock(&ps->ps_lock);
        ps->ps_increasing = 0;