Whamcloud - gitweb
LU-56 lnet: tuning wildcard portals rotor
authorLiang Zhen <liang@whamcloud.com>
Wed, 27 Jun 2012 01:49:48 +0000 (09:49 +0800)
committerAndreas Dilger <adilger@whamcloud.com>
Fri, 29 Jun 2012 22:18:15 +0000 (18:18 -0400)
By default, PUT messages arrived on wildcard Portals always prefer
to match buffers on current CPT, this patch added a few more
options for dispatching message match which's called portals rotor:

- OFF: Turn off message rotor
- ON: round-robin dispatch all PUT messages to different CPTS
- RR_RT: round-robin dispatch routed PUT message to different CPTs
- HASH_RT: dispatch routed PUT message to CPTs by hashing source NID

User can change portals rotor by /proc interface at runtime.

Signed-off-by: Liang Zhen <liang@whamcloud.com>
Change-Id: Ie2f03255d9b00b005ca5365315fff350d45a6933
Reviewed-on: http://review.whamcloud.com/3193
Reviewed-by: Doug Oucharek <doug@whamcloud.com>
Reviewed-by: Bobi Jam <bobijam@whamcloud.com>
Tested-by: Hudson
Tested-by: Maloo <whamcloud.maloo@gmail.com>
Reviewed-by: Andreas Dilger <adilger@whamcloud.com>
lnet/include/lnet/lib-lnet.h
lnet/include/lnet/lib-types.h
lnet/lnet/lib-ptl.c
lnet/lnet/router_proc.c

index bcdea8f..547a4b3 100644 (file)
@@ -803,8 +803,6 @@ struct lnet_match_table *lnet_mt_of_attach(unsigned int index,
                                           lnet_process_id_t id, __u64 mbits,
                                           __u64 ignore_bits,
                                           lnet_ins_pos_t pos);
-struct lnet_match_table *lnet_mt_of_match(unsigned int index,
-                                         lnet_process_id_t id, __u64 mbits);
 int lnet_mt_match_md(struct lnet_match_table *mtable,
                     struct lnet_match_info *info, struct lnet_msg *msg);
 
index ca4512a..f4ef8ff 100644 (file)
@@ -614,6 +614,16 @@ struct lnet_match_table {
        cfs_list_t              *mt_mhash;      /* matching hash */
 };
 
+/* these are only useful for wildcard portal */
+/* Turn off message rotor for wildcard portals */
+#define        LNET_PTL_ROTOR_OFF      0
+/* round-robin dispatch all PUT messages for wildcard portals */
+#define        LNET_PTL_ROTOR_ON       1
+/* round-robin dispatch routed PUT message for wildcard portals */
+#define        LNET_PTL_ROTOR_RR_RT    2
+/* dispatch routed PUT message by hashing source NID for wildcard portals */
+#define        LNET_PTL_ROTOR_HASH_RT  3
+
 typedef struct lnet_portal {
 #ifdef __KERNEL__
        cfs_spinlock_t          ptl_lock;
index 1cfc87c..9882e5f 100644 (file)
@@ -39,7 +39,7 @@
 #include <lnet/lib-lnet.h>
 
 /* NB: add /proc interfaces in upcoming patches */
-int    portal_rotor;
+int    portal_rotor    = LNET_PTL_ROTOR_HASH_RT;
 CFS_MODULE_PARM(portal_rotor, "i", int, 0644,
                "redirect PUTs to different cpu-partitions");
 
@@ -259,34 +259,42 @@ lnet_mt_of_attach(unsigned int index, lnet_process_id_t id,
        }
 }
 
-struct lnet_match_table *
-lnet_mt_of_match(unsigned int index, lnet_process_id_t id, __u64 mbits)
+static struct lnet_match_table *
+lnet_mt_of_match(struct lnet_match_info *info, struct lnet_msg *msg)
 {
        struct lnet_match_table *mtable;
        struct lnet_portal      *ptl;
        int                     nmaps;
        int                     rotor;
+       int                     routed;
        int                     cpt;
 
        /* NB: called w/o lock */
-       LASSERT(index < the_lnet.ln_nportals);
-       ptl = the_lnet.ln_portals[index];
+       LASSERT(info->mi_portal < the_lnet.ln_nportals);
+       ptl = the_lnet.ln_portals[info->mi_portal];
 
        LASSERT(lnet_ptl_is_wildcard(ptl) || lnet_ptl_is_unique(ptl));
 
-       mtable = lnet_match2mt(ptl, id, mbits);
+       mtable = lnet_match2mt(ptl, info->mi_id, info->mi_mbits);
        if (mtable != NULL)
                return mtable;
 
        /* it's a wildcard portal */
-       if (!portal_rotor) {
+       routed = LNET_NIDNET(msg->msg_hdr.src_nid) !=
+                LNET_NIDNET(msg->msg_hdr.dest_nid);
+
+       if (portal_rotor == LNET_PTL_ROTOR_OFF ||
+           (portal_rotor != LNET_PTL_ROTOR_ON && !routed)) {
                cpt = lnet_cpt_current();
                if (ptl->ptl_mtables[cpt]->mt_enabled)
                        return ptl->ptl_mtables[cpt];
        }
 
-       rotor = ptl->ptl_rotor++;
-       cpt = rotor % LNET_CPT_NUMBER;
+       rotor = ptl->ptl_rotor++; /* get round-robin factor */
+       if (portal_rotor == LNET_PTL_ROTOR_HASH_RT && routed)
+               cpt = lnet_cpt_of_nid(msg->msg_hdr.src_nid);
+       else
+               cpt = rotor % LNET_CPT_NUMBER;
 
        if (!ptl->ptl_mtables[cpt]->mt_enabled) {
                /* is there any active entry for this portal? */
@@ -491,8 +499,7 @@ lnet_ptl_match_md(struct lnet_match_info *info, struct lnet_msg *msg)
        if (rc != 0) /* matched or delayed early message */
                return rc;
 
-       mtable = lnet_mt_of_match(info->mi_portal,
-                                 info->mi_id, info->mi_mbits);
+       mtable = lnet_mt_of_match(info, msg);
        lnet_res_lock(mtable->mt_cpt);
 
        if (the_lnet.ln_shutdown) {
index 97585ce..f64bea2 100644 (file)
@@ -41,6 +41,7 @@ enum {
         PSDEV_LNET_PEERS,
         PSDEV_LNET_BUFFERS,
         PSDEV_LNET_NIS,
+       PSDEV_LNET_PTL_ROTOR,
 };
 #else
 #define CTL_LNET           CTL_UNNUMBERED
@@ -50,6 +51,7 @@ enum {
 #define PSDEV_LNET_PEERS   CTL_UNNUMBERED
 #define PSDEV_LNET_BUFFERS CTL_UNNUMBERED
 #define PSDEV_LNET_NIS     CTL_UNNUMBERED
+#define PSDEV_LNET_PTL_ROTOR   CTL_UNNUMBERED
 #endif
 
 #define LNET_LOFFT_BITS                (sizeof(loff_t) * 8)
@@ -744,6 +746,107 @@ int LL_PROC_PROTO(proc_lnet_nis)
         return rc;
 }
 
+struct lnet_portal_rotors {
+       int             pr_value;
+       const char      *pr_name;
+       const char      *pr_desc;
+};
+
+static struct lnet_portal_rotors       portal_rotors[] = {
+       {
+               .pr_value = LNET_PTL_ROTOR_OFF,
+               .pr_name  = "OFF",
+               .pr_desc  = "Turn off message rotor for wildcard portals"
+       },
+       {
+               .pr_value = LNET_PTL_ROTOR_ON,
+               .pr_name  = "ON",
+               .pr_desc  = "round-robin dispatch all PUT messages for "
+                           "wildcard portals"
+       },
+       {
+               .pr_value = LNET_PTL_ROTOR_RR_RT,
+               .pr_name  = "RR_RT",
+               .pr_desc  = "round-robin dispatch routed PUT message for "
+                           "wildcard portals"
+       },
+       {
+               .pr_value = LNET_PTL_ROTOR_HASH_RT,
+               .pr_name  = "HASH_RT",
+               .pr_desc  = "dispatch routed PUT message by hashing source "
+                           "NID for wildcard portals"
+       },
+       {
+               .pr_value = -1,
+               .pr_name  = NULL,
+               .pr_desc  = NULL
+       },
+};
+
+extern int portal_rotor;
+
+static int __proc_lnet_portal_rotor(void *data, int write,
+                                   loff_t pos, void *buffer, int nob)
+{
+       const int       buf_len = 128;
+       char            *buf;
+       char            *tmp;
+       int             rc;
+       int             i;
+
+       LIBCFS_ALLOC(buf, buf_len);
+       if (buf == NULL)
+               return -ENOMEM;
+
+       if (!write) {
+               lnet_res_lock(0);
+
+               for (i = 0; portal_rotors[i].pr_value >= 0; i++) {
+                       if (portal_rotors[i].pr_value == portal_rotor)
+                               break;
+               }
+
+               LASSERT(portal_rotors[i].pr_value == portal_rotor);
+               lnet_res_unlock(0);
+
+               rc = snprintf(buf, buf_len,
+                             "{\n\tportals: all\n"
+                             "\trotor: %s\n\tdescription: %s\n}",
+                             portal_rotors[i].pr_name,
+                             portal_rotors[i].pr_desc);
+
+               if (pos >= min_t(int, rc, buf_len)) {
+                       rc = 0;
+               } else {
+                       rc = cfs_trace_copyout_string(buffer, nob,
+                                       buf + pos, "\n");
+               }
+               goto out;
+       }
+
+       rc = cfs_trace_copyin_string(buf, buf_len, buffer, nob);
+       if (rc < 0)
+               goto out;
+
+       tmp = cfs_trimwhite(buf);
+
+       rc = -EINVAL;
+       lnet_res_lock(0);
+       for (i = 0; portal_rotors[i].pr_name != NULL; i++) {
+               if (cfs_strncasecmp(portal_rotors[i].pr_name, tmp,
+                                   strlen(portal_rotors[i].pr_name)) == 0) {
+                       portal_rotor = portal_rotors[i].pr_value;
+                       rc = 0;
+                       break;
+               }
+       }
+       lnet_res_unlock(0);
+out:
+       LIBCFS_FREE(buf, buf_len);
+       return rc;
+}
+DECLARE_PROC_HANDLER(proc_lnet_portal_rotor);
+
 static cfs_sysctl_table_t lnet_table[] = {
         /*
          * NB No .strategy entries have been provided since sysctl(8) prefers
@@ -786,8 +889,14 @@ static cfs_sysctl_table_t lnet_table[] = {
                 .proc_handler = &proc_lnet_nis,
         },
         {
-                INIT_CTL_NAME(0)
-        }
+               INIT_CTL_NAME(PSDEV_LNET_PTL_ROTOR)
+               .procname = "portal_rotor",
+               .mode     = 0644,
+               .proc_handler = &proc_lnet_portal_rotor,
+       },
+       {
+               INIT_CTL_NAME(0)
+       }
 };
 
 static cfs_sysctl_table_t top_table[] = {