LU-12910 osc: allow increasing osc.*.short_io_bytes

The osc.*.short_io_bytes parameter was mixing up the default and
maximum parameter values, and did not allow increasing the parameter
beyond the default.

Allow it to be increased to the maximum value, which depends on the
client PAGE_SIZE, and the amount of free space in the maximally-sized
OST RPC.  Since the maximum size is system dependent, allow some
grace when setting the parameter, so that a single tunable parameter
can work on a variety of different systems.

However, if it is larger than the maximum RDMA size (which is already
too large) return an error, as it means something is wrong.

Add a test case to exercise the osc.*.short_io_bytes parameter.

Signed-off-by: Andreas Dilger <adilger@whamcloud.com>
Change-Id: I2ce73af5963a0f9e0f1079dd2f91a4495a3ebbe5
Reviewed-on: https://review.whamcloud.com/36587
Tested-by: jenkins <devops@whamcloud.com>
Tested-by: Maloo <maloo@whamcloud.com>
Reviewed-by: Wang Shilong <wshilong@ddn.com>
Reviewed-by: Olaf Faaland-LLNL <faaland1@llnl.gov>
Reviewed-by: Oleg Drokin <green@whamcloud.com>

diff --git a/lustre/include/lustre_net.h b/lustre/include/lustre_net.h
index f8f50d5..8bee050 100644 (file)
--- a/lustre/include/lustre_net.h
+++ b/lustre/include/lustre_net.h
@@ -467,21 +467,23 @@
 
 /**
  * OST_IO_MAXREQSIZE ~=
 
 /**
  * OST_IO_MAXREQSIZE ~=

- *     lustre_msg + ptlrpc_body + obdo + obd_ioobj +
- *     DT_MAX_BRW_PAGES * niobuf_remote
+ *     lustre_msg + ptlrpc_body + obdo + obd_ioobj +
+ *     DT_MAX_BRW_PAGES * niobuf_remote

  *
  * - single object with 16 pages is 512 bytes
  *
  * - single object with 16 pages is 512 bytes

- * - OST_IO_MAXREQSIZE must be at least 1 page of cookies plus some spillover
+ * - OST_IO_MAXREQSIZE must be at least 1 niobuf per page of data

  * - Must be a multiple of 1024
  * - Must be a multiple of 1024

+ * - should allow a reasonably large SHORT_IO_BYTES size (64KB)

  */
  */

-#define _OST_MAXREQSIZE_BASE ((unsigned long)(sizeof(struct lustre_msg) + \
-                                   sizeof(struct ptlrpc_body) +          \
-                                   sizeof(struct obdo) +                 \
-                                   sizeof(struct obd_ioobj) +            \
-                                   sizeof(struct niobuf_remote)))
-#define _OST_MAXREQSIZE_SUM ((unsigned long)(_OST_MAXREQSIZE_BASE +      \
-                                  sizeof(struct niobuf_remote) *         \
-                                  (DT_MAX_BRW_PAGES - 1)))
+#define _OST_MAXREQSIZE_BASE ((unsigned long)(sizeof(struct lustre_msg)   + \
+                            /* lm_buflens */ sizeof(__u32) * 4           + \
+                                             sizeof(struct ptlrpc_body)  + \
+                                             sizeof(struct obdo)         + \
+                                             sizeof(struct obd_ioobj)    + \
+                                             sizeof(struct niobuf_remote)))
+#define _OST_MAXREQSIZE_SUM ((unsigned long)(_OST_MAXREQSIZE_BASE        + \
+                                            sizeof(struct niobuf_remote) * \
+                                            DT_MAX_BRW_PAGES))

 /**
  * FIEMAP request can be 4K+ for now
  */
 /**
  * FIEMAP request can be 4K+ for now
  */


@@ -491,13 +493,14 @@
                                    (1024UL - 1)) + 1)
 /* Safe estimate of free space in standard RPC, provides upper limit for # of
  * bytes of i/o to pack in RPC (skipping bulk transfer). */
                                    (1024UL - 1)) + 1)
 /* Safe estimate of free space in standard RPC, provides upper limit for # of
  * bytes of i/o to pack in RPC (skipping bulk transfer). */

-#define OST_SHORT_IO_SPACE     (OST_IO_MAXREQSIZE - _OST_MAXREQSIZE_BASE)
+#define OST_MAX_SHORT_IO_BYTES ((OST_IO_MAXREQSIZE - _OST_MAXREQSIZE_BASE) & \
+                                PAGE_MASK)

 
 /* Actual size used for short i/o buffer.  Calculation means this:
  * At least one page (for large PAGE_SIZE), or 16 KiB, but not more
  * than the available space aligned to a page boundary. */
 
 /* Actual size used for short i/o buffer.  Calculation means this:
  * At least one page (for large PAGE_SIZE), or 16 KiB, but not more
  * than the available space aligned to a page boundary. */

-#define OBD_MAX_SHORT_IO_BYTES min(max(PAGE_SIZE, 16UL * 1024UL), \
-                                   OST_SHORT_IO_SPACE & PAGE_MASK)
+#define OBD_DEF_SHORT_IO_BYTES min(max(PAGE_SIZE, 16UL * 1024UL), \
+                                   OST_MAX_SHORT_IO_BYTES)

 
 #define OST_MAXREPSIZE         (9 * 1024)
 #define OST_IO_MAXREPSIZE      OST_MAXREPSIZE
 
 #define OST_MAXREPSIZE         (9 * 1024)
 #define OST_IO_MAXREPSIZE      OST_MAXREPSIZE

diff --git a/lustre/ldlm/ldlm_lib.c b/lustre/ldlm/ldlm_lib.c
index 6c311e2..3ffe771 100644 (file)
--- a/lustre/ldlm/ldlm_lib.c
+++ b/lustre/ldlm/ldlm_lib.c
@@ -434,7 +434,7 @@ int client_obd_setup(struct obd_device *obddev, struct lustre_cfg *lcfg)
         */
        cli->cl_max_pages_per_rpc = PTLRPC_MAX_BRW_PAGES;
 
         */
        cli->cl_max_pages_per_rpc = PTLRPC_MAX_BRW_PAGES;
 

-       cli->cl_max_short_io_bytes = OBD_MAX_SHORT_IO_BYTES;
+       cli->cl_max_short_io_bytes = OBD_DEF_SHORT_IO_BYTES;

 
        /*
         * set cl_chunkbits default value to PAGE_SHIFT,
 
        /*
         * set cl_chunkbits default value to PAGE_SHIFT,

diff --git a/lustre/obdclass/lprocfs_status.c b/lustre/obdclass/lprocfs_status.c
index 7030e8e..9763b8d 100644 (file)
--- a/lustre/obdclass/lprocfs_status.c
+++ b/lustre/obdclass/lprocfs_status.c
@@ -2338,25 +2338,31 @@ ssize_t short_io_bytes_store(struct kobject *kobj, struct attribute *attr,
        struct obd_device *dev = container_of(kobj, struct obd_device,
                                              obd_kset.kobj);
        struct client_obd *cli = &dev->u.cli;
        struct obd_device *dev = container_of(kobj, struct obd_device,
                                              obd_kset.kobj);
        struct client_obd *cli = &dev->u.cli;

-       u32 val;
+       char kernbuf[32];
+       s64 val;

        int rc;
 
        int rc;
 

+       if (count >= sizeof(kernbuf))
+               return -EINVAL;
+

        LPROCFS_CLIMP_CHECK(dev);
 
        LPROCFS_CLIMP_CHECK(dev);
 

-       rc = kstrtouint(buffer, 0, &val);
+       memcpy(kernbuf, buffer, count);
+       kernbuf[count] = '\0';
+       rc = lu_str_to_s64(kernbuf, count, &val, '1');

        if (rc)
                GOTO(out, rc);
 
        if (rc)
                GOTO(out, rc);
 

-       if (val && (val < MIN_SHORT_IO_BYTES || val > OBD_MAX_SHORT_IO_BYTES))
+       if (val == -1)
+               val = OBD_DEF_SHORT_IO_BYTES;
+
+       if (val && (val < MIN_SHORT_IO_BYTES || val > LNET_MTU))

                GOTO(out, rc = -ERANGE);
 
        rc = count;
 
        spin_lock(&cli->cl_loi_list_lock);
                GOTO(out, rc = -ERANGE);
 
        rc = count;
 
        spin_lock(&cli->cl_loi_list_lock);

-       if (val > (cli->cl_max_pages_per_rpc << PAGE_SHIFT))
-               rc = -ERANGE;
-       else
-               cli->cl_max_short_io_bytes = val;
+       cli->cl_max_short_io_bytes = min_t(u64, val, OST_MAX_SHORT_IO_BYTES);

        spin_unlock(&cli->cl_loi_list_lock);
 
 out:
        spin_unlock(&cli->cl_loi_list_lock);
 
 out:

diff --git a/lustre/tests/sanity.sh b/lustre/tests/sanity.sh
index 63a86c3..df92c36 100644 (file)
--- a/lustre/tests/sanity.sh
+++ b/lustre/tests/sanity.sh
@@ -17422,7 +17422,7 @@ test_247e() {
 }
 run_test 247e "mount .. as fileset"
 
 }
 run_test 247e "mount .. as fileset"
 

-test_248() {
+test_248a() {

        local fast_read_sav=$($LCTL get_param -n llite.*.fast_read 2>/dev/null)
        [ -z "$fast_read_sav" ] && skip "no fast read support"
 
        local fast_read_sav=$($LCTL get_param -n llite.*.fast_read 2>/dev/null)
        [ -z "$fast_read_sav" ] && skip "no fast read support"
 


@@ -17473,7 +17473,89 @@ test_248() {
        $LCTL set_param -n llite.*.fast_read=$fast_read_sav
        rm -f $DIR/$tfile
 }
        $LCTL set_param -n llite.*.fast_read=$fast_read_sav
        rm -f $DIR/$tfile
 }

-run_test 248 "fast read verification"
+run_test 248a "fast read verification"
+
+test_248b() {
+       # Default short_io_bytes=16384, try both smaller and larger sizes.
+       # Lustre O_DIRECT read and write needs to be a multiple of PAGE_SIZE.
+       # 6017024 = 2^12*13*113 = 47008*128 = 11752*512 = 4096*1469 = 53248*113
+       echo "bs=53248 count=113 normal buffered write"
+       dd if=/dev/urandom of=$TMP/$tfile.0 bs=53248 count=113 ||
+               error "dd of initial data file failed"
+       stack_trap "rm -f $DIR/$tfile.[0-3] $TMP/$tfile.[0-3]" EXIT
+
+       echo "bs=47008 count=128 oflag=dsync normal write $tfile.0"
+       dd if=$TMP/$tfile.0 of=$DIR/$tfile.0 bs=47008 count=128 oflag=dsync ||
+               error "dd with sync normal writes failed"
+       cmp $TMP/$tfile.0 $DIR/$tfile.0 || error "compare $DIR/$tfile.0 failed"
+
+       echo "bs=11752 count=512 oflag=dsync small write $tfile.1"
+       dd if=$TMP/$tfile.0 of=$DIR/$tfile.1 bs=11752 count=512 oflag=dsync ||
+               error "dd with sync small writes failed"
+       cmp $TMP/$tfile.0 $DIR/$tfile.1 || error "compare $DIR/$tfile.1 failed"
+
+       cancel_lru_locks osc
+
+       # calculate the small O_DIRECT size and count for the client PAGE_SIZE
+       local num=$((13 * 113 / (PAGE_SIZE / 4096)))
+       echo "bs=$PAGE_SIZE count=$num iflag=direct small read $tfile.1"
+       dd if=$DIR/$tfile.1 of=$TMP/$tfile.1 bs=$PAGE_SIZE count=$num \
+               iflag=direct || error "dd with O_DIRECT small read failed"
+       # adjust bytes checked to handle larger PAGE_SIZE for ARM/PPC
+       cmp --bytes=$((PAGE_SIZE * num)) $TMP/$tfile.0 $TMP/$tfile.1 ||
+               error "compare $TMP/$tfile.1 failed"
+
+       local save=$($LCTL get_param -n osc.*OST000*.short_io_bytes | head -n 1)
+       stack_trap "$LCTL set_param osc.$FSNAME-*.short_io_bytes=$save" EXIT
+
+       # just to see what the maximum tunable value is, and test parsing
+       echo "test invalid parameter 2MB"
+       $LCTL set_param osc.$FSNAME-OST0000*.short_io_bytes=2M &&
+               error "too-large short_io_bytes allowed"
+       echo "test maximum parameter 512KB"
+       # if we can set a larger short_io_bytes, run test regardless of version
+       if ! $LCTL set_param osc.$FSNAME-OST0000*.short_io_bytes=512K; then
+               # older clients may not allow setting it this large, that's OK
+               [ $CLIENT_VERSION -ge $(version_code 2.13.50) ] ||
+                       skip "Need at least client version 2.13.50"
+               error "medium short_io_bytes failed"
+       fi
+       $LCTL get_param osc.$FSNAME-OST0000*.short_io_bytes
+       size=$($LCTL get_param -n osc.$FSNAME-OST0000*.short_io_bytes)
+
+       echo "test large parameter 64KB"
+       $LCTL set_param osc.$FSNAME-*.short_io_bytes=65536
+       $LCTL get_param osc.$FSNAME-OST0000*.short_io_bytes
+
+       echo "bs=47008 count=128 oflag=dsync large write $tfile.2"
+       dd if=$TMP/$tfile.0 of=$DIR/$tfile.2 bs=47008 count=128 oflag=dsync ||
+               error "dd with sync large writes failed"
+       cmp $TMP/$tfile.0 $DIR/$tfile.2 || error "compare $DIR/$tfile.2 failed"
+
+       # calculate the large O_DIRECT size and count for the client PAGE_SIZE
+       local size=$(((4096 * 13 + PAGE_SIZE - 1) / PAGE_SIZE * PAGE_SIZE))
+       num=$((113 * 4096 / PAGE_SIZE))
+       echo "bs=$size count=$num oflag=direct large write $tfile.3"
+       dd if=$TMP/$tfile.0 of=$DIR/$tfile.3 bs=$size count=$num oflag=direct ||
+               error "dd with O_DIRECT large writes failed"
+       cmp --bytes=$((size * num)) $TMP/$tfile.0 $DIR/$tfile.3 ||
+               error "compare $DIR/$tfile.3 failed"
+
+       cancel_lru_locks osc
+
+       echo "bs=$size count=$num iflag=direct large read $tfile.2"
+       dd if=$DIR/$tfile.2 of=$TMP/$tfile.2 bs=$size count=$num iflag=direct ||
+               error "dd with O_DIRECT large read failed"
+       cmp --bytes=$((size * num)) $TMP/$tfile.0 $TMP/$tfile.2 ||
+               error "compare $TMP/$tfile.2 failed"
+
+       echo "bs=$size count=$num iflag=direct large read $tfile.3"
+       dd if=$DIR/$tfile.3 of=$TMP/$tfile.3 bs=$size count=$num iflag=direct ||
+               error "dd with O_DIRECT large read failed"
+       cmp --bytes=$((size * num)) $TMP/$tfile.0 $TMP/$tfile.3 ||
+               error "compare $TMP/$tfile.3 failed"
+}
+run_test 248b "test short_io read and write for both small and large sizes"

 
 test_249() { # LU-7890
        [ $MDS1_VERSION -lt $(version_code 2.8.53) ] &&
 
 test_249() { # LU-7890
        [ $MDS1_VERSION -lt $(version_code 2.8.53) ] &&