LU-5718 lnet: add offset for selftest brw

In current lnet selftest, both client and server side bulk have
no offset and we can only test page aligned IO, this patch changed
this:

- user can set brw offset by lst add_test ... brw off=OFFSET ...
- offset is only effective on client side so far
- to simply implementation, offset needs to be eight bytes aligned

Signed-off-by: Liang Zhen <liang.zhen@intel.com>
Change-Id: I34690d2c8fbb627b827291afcd46d7aa59831a3f
Reviewed-on: http://review.whamcloud.com/12496
Tested-by: Jenkins
Tested-by: Maloo <hpdd-maloo@intel.com>
Reviewed-by: Doug Oucharek <doug.s.oucharek@intel.com>
Reviewed-by: James Simmons <uja.ornl@yahoo.com>
Reviewed-by: Oleg Drokin <oleg.drokin@intel.com>

diff --git a/lnet/include/lnet/lnetst.h b/lnet/include/lnet/lnetst.h
index 62a1131..d1bc80c 100644 (file)
--- a/lnet/include/lnet/lnetst.h
+++ b/lnet/include/lnet/lnetst.h
@@ -486,6 +486,8 @@ typedef struct {
        int                     blk_size;               /* size (bytes) */
        int                     blk_time;               /* time of running the test*/
        int                     blk_flags;              /* reserved flags */
+       int                     blk_cli_off;            /* bulk offset on client */
+       int                     blk_srv_off;            /* reserved: bulk offset on server */
 } lst_test_bulk_param_t;
 
 typedef struct {

diff --git a/lnet/selftest/brw_test.c b/lnet/selftest/brw_test.c
index 1379c53..9fa6beb 100644 (file)
--- a/lnet/selftest/brw_test.c
+++ b/lnet/selftest/brw_test.c
@@ -48,6 +48,10 @@ static int brw_inject_errors;
 module_param(brw_inject_errors, int, 0644);
 MODULE_PARM_DESC(brw_inject_errors, "# data errors to inject randomly, zero by default");
 
+#define BRW_POISON     0xbeefbeefbeefbeefULL
+#define BRW_MAGIC      0xeeb0eeb1eeb2eeb3ULL
+#define BRW_MSIZE      sizeof(__u64)
+
 static void
 brw_client_fini (sfw_test_instance_t *tsi)
 {
@@ -71,6 +75,7 @@ brw_client_init (sfw_test_instance_t *tsi)
 {
        sfw_session_t    *sn = tsi->tsi_batch->bat_session;
        int               flags;
+       int               off;
        int               npg;
        int               len;
        int               opc;
@@ -89,6 +94,7 @@ brw_client_init (sfw_test_instance_t *tsi)
                /* NB: this is not going to work for variable page size,
                 * but we have to keep it for compatibility */
                len   = npg * PAGE_SIZE;
+               off   = 0;
 
        } else {
                test_bulk_req_v1_t  *breq = &tsi->tsi_u.bulk_v1;
@@ -100,9 +106,13 @@ brw_client_init (sfw_test_instance_t *tsi)
                opc   = breq->blk_opc;
                flags = breq->blk_flags;
                len   = breq->blk_len;
-               npg   = (len + PAGE_SIZE - 1) >> PAGE_SHIFT;
+               off   = breq->blk_offset & ~PAGE_MASK;
+               npg   = (off + len + PAGE_SIZE - 1) >> PAGE_SHIFT;
        }
 
+       if (off % BRW_MSIZE != 0)
+               return -EINVAL;
+
        if (npg > LNET_MAX_IOV || npg <= 0)
                return -EINVAL;
 
@@ -115,7 +125,7 @@ brw_client_init (sfw_test_instance_t *tsi)
 
        list_for_each_entry(tsu, &tsi->tsi_units, tsu_list) {
                bulk = srpc_alloc_bulk(lnet_cpt_of_nid(tsu->tsu_dest.nid),
-                                      npg, len, opc == LST_BRW_READ);
+                                      off, npg, len, opc == LST_BRW_READ);
                if (bulk == NULL) {
                        brw_client_fini(tsi);
                        return -ENOMEM;
@@ -131,7 +141,7 @@ brw_client_init (sfw_test_instance_t *tsi)
 #define BRW_MAGIC       0xeeb0eeb1eeb2eeb3ULL
 #define BRW_MSIZE       sizeof(__u64)
 
-static int brw_inject_one_error(void)
+int brw_inject_one_error(void)
 {
        struct timeval tv;
 
@@ -145,111 +155,127 @@ static int brw_inject_one_error(void)
 }
 
 static void
-brw_fill_page(struct page *pg, int pattern, __u64 magic)
+brw_fill_page(struct page *pg, int off, int len, int pattern, __u64 magic)
 {
-       char *addr = page_address(pg);
-        int   i;
+       char *addr = page_address(pg) + off;
+       int   i;
 
-        LASSERT (addr != NULL);
+       LASSERT(addr != NULL);
+       LASSERT(off % BRW_MSIZE == 0 && len % BRW_MSIZE == 0);
 
-        if (pattern == LST_BRW_CHECK_NONE) return;
+       if (pattern == LST_BRW_CHECK_NONE)
+               return;
 
-        if (magic == BRW_MAGIC)
-                magic += brw_inject_one_error();
+       if (magic == BRW_MAGIC)
+               magic += brw_inject_one_error();
 
-        if (pattern == LST_BRW_CHECK_SIMPLE) {
-                memcpy(addr, &magic, BRW_MSIZE);
-               addr += PAGE_SIZE - BRW_MSIZE;
-                memcpy(addr, &magic, BRW_MSIZE);
-                return;
-        }
-
-        if (pattern == LST_BRW_CHECK_FULL) {
-               for (i = 0; i < PAGE_SIZE / BRW_MSIZE; i++)
-                        memcpy(addr + i * BRW_MSIZE, &magic, BRW_MSIZE);
-                return;
-        }
+       if (pattern == LST_BRW_CHECK_SIMPLE) {
+               memcpy(addr, &magic, BRW_MSIZE);
+               if (len > BRW_MSIZE) {
+                       addr += len - BRW_MSIZE;
+                       memcpy(addr, &magic, BRW_MSIZE);
+               }
+               return;
+       }
 
-        LBUG ();
-        return;
+       if (pattern == LST_BRW_CHECK_FULL) {
+               for (i = 0; i < len; i += BRW_MSIZE)
+                       memcpy(addr + i, &magic, BRW_MSIZE);
+               return;
+       }
+       LBUG();
 }
 
 static int
-brw_check_page(struct page *pg, int pattern, __u64 magic)
+brw_check_page(struct page *pg, int off, int len, int pattern, __u64 magic)
 {
-       char  *addr = page_address(pg);
-        __u64  data = 0; /* make compiler happy */
-        int    i;
+       char  *addr = page_address(pg) + off;
+       __u64  data = 0; /* make compiler happy */
+       int    i;
 
-        LASSERT (addr != NULL);
-
-        if (pattern == LST_BRW_CHECK_NONE)
-                return 0;
+       LASSERT(addr != NULL);
+       LASSERT(off % BRW_MSIZE == 0 && len % BRW_MSIZE == 0);
 
-        if (pattern == LST_BRW_CHECK_SIMPLE) {
-                data = *((__u64 *) addr);
-                if (data != magic) goto bad_data;
-
-               addr += PAGE_SIZE - BRW_MSIZE;
-                data = *((__u64 *) addr);
-                if (data != magic) goto bad_data;
+       if (pattern == LST_BRW_CHECK_NONE)
+               return 0;
 
-                return 0;
-        }
+       if (pattern == LST_BRW_CHECK_SIMPLE) {
+               data = *((__u64 *) addr);
+               if (data != magic)
+                       goto bad_data;
 
-        if (pattern == LST_BRW_CHECK_FULL) {
-               for (i = 0; i < PAGE_SIZE / BRW_MSIZE; i++) {
-                        data = *(((__u64 *) addr) + i);
-                        if (data != magic) goto bad_data;
-                }
+               if (len > BRW_MSIZE) {
+                       addr += len - BRW_MSIZE;
+                       data = *((__u64 *) addr);
+                       if (data != magic)
+                               goto bad_data;
+               }
+               return 0;
+       }
 
-                return 0;
-        }
+       if (pattern == LST_BRW_CHECK_FULL) {
+               for (i = 0; i < len; i += BRW_MSIZE) {
+                       data = *(__u64 *)(addr + i);
+                       if (data != magic)
+                               goto bad_data;
+               }
+               return 0;
+       }
 
-        LBUG ();
+       LBUG();
 
 bad_data:
        CERROR ("Bad data in page %p: %#llx, %#llx expected\n",
-                pg, data, magic);
-        return 1;
+               pg, data, magic);
+       return 1;
 }
 
 static void
 brw_fill_bulk(srpc_bulk_t *bk, int pattern, __u64 magic)
 {
-        int         i;
+       int          i;
        struct page *pg;
 
-        for (i = 0; i < bk->bk_niov; i++) {
-                pg = bk->bk_iovs[i].kiov_page;
-                brw_fill_page(pg, pattern, magic);
-        }
+       for (i = 0; i < bk->bk_niov; i++) {
+               int     off;
+               int     len;
+
+               pg = bk->bk_iovs[i].kiov_page;
+               off = bk->bk_iovs[i].kiov_offset;
+               len = bk->bk_iovs[i].kiov_len;
+               brw_fill_page(pg, off, len, pattern, magic);
+       }
 }
 
 static int
 brw_check_bulk(srpc_bulk_t *bk, int pattern, __u64 magic)
 {
-        int         i;
+       int          i;
        struct page *pg;
 
-        for (i = 0; i < bk->bk_niov; i++) {
-                pg = bk->bk_iovs[i].kiov_page;
-                if (brw_check_page(pg, pattern, magic) != 0) {
-                        CERROR ("Bulk page %p (%d/%d) is corrupted!\n",
-                                pg, i, bk->bk_niov);
-                        return 1;
-                }
-        }
+       for (i = 0; i < bk->bk_niov; i++) {
+               int     off;
+               int     len;
+
+               pg = bk->bk_iovs[i].kiov_page;
+               off = bk->bk_iovs[i].kiov_offset;
+               len = bk->bk_iovs[i].kiov_len;
+               if (brw_check_page(pg, off, len, pattern, magic) != 0) {
+                       CERROR("Bulk page %p (%d/%d) is corrupted!\n",
+                              pg, i, bk->bk_niov);
+                       return 1;
+               }
+       }
 
-        return 0;
+       return 0;
 }
 
 static int
-brw_client_prep_rpc (sfw_test_unit_t *tsu,
-                     lnet_process_id_t dest, srpc_client_rpc_t **rpcpp)
+brw_client_prep_rpc(sfw_test_unit_t *tsu,
+                   lnet_process_id_t dest, srpc_client_rpc_t **rpcpp)
 {
-        srpc_bulk_t         *bulk = tsu->tsu_private;
-        sfw_test_instance_t *tsi = tsu->tsu_instance;
+       srpc_bulk_t         *bulk = tsu->tsu_private;
+       sfw_test_instance_t *tsi = tsu->tsu_instance;
        sfw_session_t       *sn = tsi->tsi_batch->bat_session;
        srpc_client_rpc_t   *rpc;
        srpc_brw_reqst_t    *req;
@@ -272,6 +298,7 @@ brw_client_prep_rpc (sfw_test_unit_t *tsu,
 
        } else {
                test_bulk_req_v1_t  *breq = &tsi->tsi_u.bulk_v1;
+               int                  off;
 
                /* I should never get this step if it's unknown feature
                 * because make_session will reject unknown feature */
@@ -280,7 +307,8 @@ brw_client_prep_rpc (sfw_test_unit_t *tsu,
                opc   = breq->blk_opc;
                flags = breq->blk_flags;
                len   = breq->blk_len;
-               npg   = (len + PAGE_SIZE - 1) >> PAGE_SHIFT;
+               off   = breq->blk_offset;
+               npg   = (off + len + PAGE_SIZE - 1) >> PAGE_SHIFT;
        }
 
        rc = sfw_create_test_rpc(tsu, dest, sn->sn_features, npg, len, &rpc);

diff --git a/lnet/selftest/conrpc.c b/lnet/selftest/conrpc.c
index 8b372a2..68a565a 100644 (file)
--- a/lnet/selftest/conrpc.c
+++ b/lnet/selftest/conrpc.c
@@ -797,14 +797,15 @@ lstcon_bulkrpc_v0_prep(lst_test_bulk_param_t *param, srpc_test_reqst_t *req)
 }
 
 static int
-lstcon_bulkrpc_v1_prep(lst_test_bulk_param_t *param, srpc_test_reqst_t *req)
+lstcon_bulkrpc_v1_prep(lst_test_bulk_param_t *param, bool is_client,
+                      srpc_test_reqst_t *req)
 {
        test_bulk_req_v1_t *brq = &req->tsr_u.bulk_v1;
 
        brq->blk_opc    = param->blk_opc;
        brq->blk_flags  = param->blk_flags;
        brq->blk_len    = param->blk_size;
-       brq->blk_offset = 0; /* reserved */
+       brq->blk_offset = is_client ? param->blk_cli_off : param->blk_srv_off;
 
        return 0;
 }
@@ -904,7 +905,8 @@ lstcon_testrpc_prep(lstcon_node_t *nd, int transop, unsigned feats,
                                                    &test->tes_param[0], trq);
                } else {
                        rc = lstcon_bulkrpc_v1_prep((lst_test_bulk_param_t *)
-                                                   &test->tes_param[0], trq);
+                                                   &test->tes_param[0],
+                                                   trq->tsr_is_client, trq);
                }
 
                 break;

diff --git a/lnet/selftest/framework.c b/lnet/selftest/framework.c
index 1999962..478a4fa 100644 (file)
--- a/lnet/selftest/framework.c
+++ b/lnet/selftest/framework.c
@@ -1125,7 +1125,7 @@ sfw_alloc_pages(struct srpc_server_rpc *rpc, int cpt, int npages, int len,
        LASSERT(rpc->srpc_bulk == NULL);
        LASSERT(npages > 0 && npages <= LNET_MAX_IOV);
 
-       rpc->srpc_bulk = srpc_alloc_bulk(cpt, npages, len, sink);
+       rpc->srpc_bulk = srpc_alloc_bulk(cpt, 0, npages, len, sink);
        if (rpc->srpc_bulk == NULL)
                return -ENOMEM;
 

diff --git a/lnet/selftest/rpc.c b/lnet/selftest/rpc.c
index bec99df..aa766c2 100644 (file)
--- a/lnet/selftest/rpc.c
+++ b/lnet/selftest/rpc.c
@@ -88,14 +88,12 @@ void srpc_set_counters (const srpc_counters_t *cnt)
 }
 
 static int
-srpc_add_bulk_page(srpc_bulk_t *bk, struct page *pg, int i, int nob)
+srpc_add_bulk_page(srpc_bulk_t *bk, struct page *pg, int i, int off, int nob)
 {
-       nob = min_t(int, nob, (int)PAGE_SIZE);
+       LASSERT(off < PAGE_SIZE);
+       LASSERT(nob > 0 && nob <= PAGE_SIZE);
 
-       LASSERT(nob > 0);
-       LASSERT(i >= 0 && i < bk->bk_niov);
-
-       bk->bk_iovs[i].kiov_offset = 0;
+       bk->bk_iovs[i].kiov_offset = off;
        bk->bk_iovs[i].kiov_page   = pg;
        bk->bk_iovs[i].kiov_len    = nob;
        return nob;
@@ -121,7 +119,8 @@ srpc_free_bulk (srpc_bulk_t *bk)
 }
 
 srpc_bulk_t *
-srpc_alloc_bulk(int cpt, unsigned bulk_npg, unsigned bulk_len, int sink)
+srpc_alloc_bulk(int cpt, unsigned bulk_off, unsigned bulk_npg,
+               unsigned bulk_len, int sink)
 {
        srpc_bulk_t  *bk;
        int           i;
@@ -151,8 +150,12 @@ srpc_alloc_bulk(int cpt, unsigned bulk_npg, unsigned bulk_len, int sink)
                        return NULL;
                }
 
-               nob = srpc_add_bulk_page(bk, pg, i, bulk_len);
+               nob = min_t(unsigned, bulk_off + bulk_len, PAGE_SIZE) -
+                     bulk_off;
+
+               srpc_add_bulk_page(bk, pg, i, bulk_off, nob);
                bulk_len -= nob;
+               bulk_off = 0;
        }
 
        return bk;

diff --git a/lnet/selftest/rpc.h b/lnet/selftest/rpc.h
index 9211644..b3cbbdc 100644 (file)
--- a/lnet/selftest/rpc.h
+++ b/lnet/selftest/rpc.h
@@ -181,7 +181,7 @@ typedef struct {
        __u16                   blk_flags;
        /** data length */
        __u32                   blk_len;
-       /** reserved: offset */
+       /** bulk offset */
        __u32                   blk_offset;
 } WIRE_ATTR test_bulk_req_v1_t;
 

diff --git a/lnet/selftest/selftest.h b/lnet/selftest/selftest.h
index 29b1af2..a0e8f94 100644 (file)
--- a/lnet/selftest/selftest.h
+++ b/lnet/selftest/selftest.h
@@ -436,8 +436,8 @@ srpc_create_client_rpc(lnet_process_id_t peer, int service,
 void srpc_post_rpc(srpc_client_rpc_t *rpc);
 void srpc_abort_rpc(srpc_client_rpc_t *rpc, int why);
 void srpc_free_bulk(srpc_bulk_t *bk);
-srpc_bulk_t *srpc_alloc_bulk(int cpt, unsigned bulk_npg, unsigned bulk_len,
-                            int sink);
+srpc_bulk_t *srpc_alloc_bulk(int cpt, unsigned off, unsigned bulk_npg,
+                            unsigned bulk_len, int sink);
 int srpc_send_rpc(swi_workitem_t *wi);
 int srpc_send_reply(srpc_server_rpc_t *rpc);
 int srpc_add_service(srpc_service_t *sv);

diff --git a/lnet/utils/lst.c b/lnet/utils/lst.c
index c84a208..a0e7229 100644 (file)
--- a/lnet/utils/lst.c
+++ b/lnet/utils/lst.c
@@ -2955,6 +2955,7 @@ lst_get_bulk_param(int argc, char **argv, lst_test_bulk_param_t *bulk)
         bulk->blk_size  = 4096;
         bulk->blk_opc   = LST_BRW_READ;
         bulk->blk_flags = LST_BRW_CHECK_NONE;
+       bulk->blk_srv_off = bulk->blk_cli_off = 0;
 
         while (i < argc) {
                 if (strcasestr(argv[i], "check=") == argv[i] ||
@@ -2996,6 +2997,27 @@ lst_get_bulk_param(int argc, char **argv, lst_test_bulk_param_t *bulk)
                                 return -1;
                         }
 
+               } else if (strcasestr(argv[i], "off=") == argv[i]) {
+                       int     off;
+
+                       tok = strchr(argv[i], '=') + 1;
+
+                       off = strtol(tok, &end, 0);
+                       /* NB: align with sizeof(__u64) to simplify page
+                        * checking implementation */
+                       if (off < 0 || off % sizeof(__u64) != 0) {
+                               fprintf(stderr,
+                                       "Invalid offset %s/%d, it should be "
+                                       "postive value and multiple of %d\n",
+                                       tok, off, (int)sizeof(__u64));
+                               return -1;
+                       }
+
+                       /* NB: blk_srv_off is reserved so far */
+                       bulk->blk_cli_off = bulk->blk_srv_off = off;
+                       if (end == NULL)
+                               return 0;
+
                 } else if (strcasecmp(argv[i], "read") == 0 ||
                            strcasecmp(argv[i], "r") == 0) {
                         bulk->blk_opc = LST_BRW_READ;