LU-325 using preallocated objects if OST has enough disk space

during creating files, if ENOSPC is returned by one OST, the preallocated
objects in the corresponding OSC should still can be used if there is enough
blocks but no inode available in the OST.

Change-Id: I9d57461698d1c0fb3a492188fb4ce1519d65efc7
Signed-off-by: Hongchao Zhang <hongchao.zhang@whamcloud.com>
Reviewed-on: http://review.whamcloud.com/1051
Tested-by: Hudson
Reviewed-by: Johann Lombardi <johann@whamcloud.com>
Tested-by: Maloo <whamcloud.maloo@gmail.com>
Reviewed-by: Fan Yong <yong.fan@whamcloud.com>
Reviewed-by: Oleg Drokin <green@whamcloud.com>

diff --git a/lustre/include/lustre/lustre_idl.h b/lustre/include/lustre/lustre_idl.h
index b4f485b..d04bba4 100644 (file)
--- a/lustre/include/lustre/lustre_idl.h
+++ b/lustre/include/lustre/lustre_idl.h
@@ -1267,6 +1267,7 @@ enum obdo_flags {
         OBD_FL_SHRINK_GRANT = 0x00020000, /* object shrink the grant */
         OBD_FL_MMAP         = 0x00040000, /* object is mmapped on the client */
         OBD_FL_RECOV_RESEND = 0x00080000, /* recoverable resent */
+        OBD_FL_NOSPC_BLK    = 0x00100000, /* no more block space on OST */
 
         OBD_FL_CKSUM_ALL    = OBD_FL_CKSUM_CRC32 | OBD_FL_CKSUM_ADLER,
 

diff --git a/lustre/obdfilter/filter.c b/lustre/obdfilter/filter.c
index 609aaeb..d6a9fd8 100644 (file)
--- a/lustre/obdfilter/filter.c
+++ b/lustre/obdfilter/filter.c
@@ -3833,6 +3833,13 @@ static int filter_precreate(struct obd_device *obd, struct obdo *oa,
                                LPU64"\n", obd->obd_name, osfs->os_bavail <<
                                obd->u.obt.obt_vfsmnt->mnt_sb->s_blocksize_bits);
                         *num = 0;
+                        if (oa->o_valid & OBD_MD_FLFLAGS)
+                                oa->o_flags |= OBD_FL_NOSPC_BLK;
+                        else {
+                                oa->o_valid |= OBD_MD_FLFLAGS;
+                                oa->o_flags = OBD_FL_NOSPC_BLK;
+                        }
+
                         rc = -ENOSPC;
                 }
                 OBD_FREE(osfs, sizeof(*osfs));
@@ -3937,9 +3944,21 @@ static int filter_precreate(struct obd_device *obd, struct obdo *oa,
                         CERROR("create failed rc = %d\n", rc);
                         if (rc == -ENOSPC) {
                                 os_ffree = filter_calc_free_inodes(obd);
-                                if (os_ffree != -1)
+                                if (os_ffree == -1) 
+                                        GOTO(cleanup, rc);
+
+                                if (obd->obd_osfs.os_bavail <
+                                    (obd->obd_osfs.os_blocks >> 10)) {
+                                        if (oa->o_valid & OBD_MD_FLFLAGS)
+                                                oa->o_flags |= OBD_FL_NOSPC_BLK;
+                                        else {
+                                                oa->o_valid |= OBD_MD_FLFLAGS;
+                                                oa->o_flags = OBD_FL_NOSPC_BLK;
+                                        }
+
                                         CERROR("%s: free inode "LPU64"\n",
                                                obd->obd_name, os_ffree);
+                                }
                         }
                         GOTO(cleanup, rc);
                 }

diff --git a/lustre/osc/osc_create.c b/lustre/osc/osc_create.c
index c18183e..b4d5cad 100644 (file)
--- a/lustre/osc/osc_create.c
+++ b/lustre/osc/osc_create.c
@@ -132,6 +132,12 @@ static int osc_interpret_create(const struct lu_env *env,
                         if (body && rc == -ENOSPC) {
                                 oscc->oscc_last_id = body->oa.o_id;
                                 oscc->oscc_grow_count = OST_MIN_PRECREATE;
+
+                                if ((body->oa.o_valid & OBD_MD_FLFLAGS) &&
+                                    (body->oa.o_flags & OBD_FL_NOSPC_BLK))
+                                        oscc->oscc_flags |= OSCC_FLAG_NOSPC_BLK;
+                                else
+                                        rc = 0;
                         }
                 }
                 cfs_spin_unlock(&oscc->oscc_lock);
@@ -359,7 +365,7 @@ int osc_precreate(struct obd_export *exp)
 
         /* Handle critical states first */
         cfs_spin_lock(&oscc->oscc_lock);
-        if (oscc->oscc_flags & OSCC_FLAG_NOSPC ||
+        if (oscc->oscc_flags & OSCC_FLAG_NOSPC_BLK ||
             oscc->oscc_flags & OSCC_FLAG_RDONLY ||
             oscc->oscc_flags & OSCC_FLAG_EXITING)
                 GOTO(out, rc = 1000);
@@ -374,6 +380,9 @@ int osc_precreate(struct obd_export *exp)
         /* Return 0, if we have at least one object - bug 22884 */
         rc = oscc_has_objects_nolock(oscc, 1) ? 0 : 1;
 
+        if (oscc->oscc_flags & OSCC_FLAG_NOSPC)
+                GOTO(out, (rc == 0) ? 0 : 1000);
+
         /* Do not check for OSCC_FLAG_CREATING flag here, let
          * osc_precreate() call oscc_internal_create() and
          * adjust oscc_grow_count bug21563 */
@@ -408,7 +417,7 @@ static int handle_async_create(struct ptlrpc_request *req, int rc)
         if (oscc->oscc_flags & OSCC_FLAG_EXITING)
                 GOTO(out_wake, rc = -EIO);
 
-        if (oscc->oscc_flags & OSCC_FLAG_NOSPC)
+        if (oscc->oscc_flags & OSCC_FLAG_NOSPC_BLK)
                 GOTO(out_wake, rc = -ENOSPC);
 
         if (oscc->oscc_flags & OSCC_FLAG_RDONLY)
@@ -577,8 +586,12 @@ int osc_create(struct obd_export *exp, struct obdo *oa,
                 if (rc == 0 || rc == -ENOSPC) {
                         struct obd_connect_data *ocd;
 
-                        if (rc == -ENOSPC)
+                        if (rc == -ENOSPC) {
                                 oscc->oscc_flags |= OSCC_FLAG_NOSPC;
+                                if ((oa->o_valid & OBD_MD_FLFLAGS) &&
+                                    (oa->o_flags & OBD_FL_NOSPC_BLK))
+                                        oscc->oscc_flags |= OSCC_FLAG_NOSPC_BLK;
+                        }
                         oscc->oscc_flags &= ~OSCC_FLAG_RECOVERING;
 
                         oscc->oscc_last_id = oa->o_id;
@@ -639,7 +652,7 @@ int osc_create(struct obd_export *exp, struct obdo *oa,
                         break;
                 }
 
-                if (oscc->oscc_flags & OSCC_FLAG_NOSPC) {
+                if (oscc->oscc_flags & OSCC_FLAG_NOSPC_BLK) {
                         rc = -ENOSPC;
                         cfs_spin_unlock(&oscc->oscc_lock);
                         break;
@@ -682,6 +695,12 @@ int osc_create(struct obd_export *exp, struct obdo *oa,
                         break;
                 }
 
+                if (oscc->oscc_flags & OSCC_FLAG_NOSPC) {
+                        rc = -ENOSPC;
+                        cfs_spin_unlock(&oscc->oscc_lock);
+                        break;
+                }
+
                 cfs_spin_unlock(&oscc->oscc_lock);
         }
 

diff --git a/lustre/osc/osc_internal.h b/lustre/osc/osc_internal.h
index 2a94ebb..9b11a9d 100644 (file)
--- a/lustre/osc/osc_internal.h
+++ b/lustre/osc/osc_internal.h
@@ -106,6 +106,7 @@ struct osc_cache_waiter {
 #define OSCC_FLAG_EXITING            0x20
 #define OSCC_FLAG_DEGRADED           0x40
 #define OSCC_FLAG_RDONLY             0x80
+#define OSCC_FLAG_NOSPC_BLK          0x100 /* no more block space on OST */
 
 int osc_precreate(struct obd_export *exp);
 int osc_create(struct obd_export *exp, struct obdo *oa,

diff --git a/lustre/osc/osc_request.c b/lustre/osc/osc_request.c
index 4b5f2c3..b6948c1 100644 (file)
--- a/lustre/osc/osc_request.c
+++ b/lustre/osc/osc_request.c
@@ -3632,8 +3632,15 @@ static int osc_statfs_interpret(const struct lu_env *env,
                      ((msfs->os_ffree < 32) || (msfs->os_bavail < used))))
                 cli->cl_oscc.oscc_flags |= OSCC_FLAG_NOSPC;
         else if (unlikely(((cli->cl_oscc.oscc_flags & OSCC_FLAG_NOSPC) != 0) &&
-                (msfs->os_ffree > 64) && (msfs->os_bavail > (used << 1))))
-                        cli->cl_oscc.oscc_flags &= ~OSCC_FLAG_NOSPC;
+                          (msfs->os_ffree > 64) &&
+                          (msfs->os_bavail > (used << 1)))) {
+                cli->cl_oscc.oscc_flags &= ~(OSCC_FLAG_NOSPC |
+                                             OSCC_FLAG_NOSPC_BLK);
+        }
+
+        if (unlikely(((cli->cl_oscc.oscc_flags & OSCC_FLAG_NOSPC) != 0) &&
+                     (msfs->os_bavail < used)))
+                cli->cl_oscc.oscc_flags |= OSCC_FLAG_NOSPC_BLK;
 
         cfs_spin_unlock(&cli->cl_oscc.oscc_lock);
 
@@ -4381,7 +4388,8 @@ static int osc_import_event(struct obd_device *obd,
                         struct osc_creator *oscc = &obd->u.cli.cl_oscc;
 
                         cfs_spin_lock(&oscc->oscc_lock);
-                        oscc->oscc_flags &= ~OSCC_FLAG_NOSPC;
+                        oscc->oscc_flags &= ~(OSCC_FLAG_NOSPC |
+                                              OSCC_FLAG_NOSPC_BLK);
                         cfs_spin_unlock(&oscc->oscc_lock);
                 }
                 rc = obd_notify_observer(obd, obd, OBD_NOTIFY_ACTIVE, NULL);

diff --git a/lustre/tests/sanity.sh b/lustre/tests/sanity.sh
index 2bd1db9..4cbd691 100644 (file)
--- a/lustre/tests/sanity.sh
+++ b/lustre/tests/sanity.sh
@@ -8231,6 +8231,53 @@ test_219() {
 }
 run_test 219 "LU-394: Write partial won't cause uncontiguous pages vec at LND"
 
+test_220() { #LU-325
+       local OSTIDX=0
+
+       mkdir -p $DIR/$tdir
+       local OST=$(lfs osts | grep ${OSTIDX}": " | \
+               awk '{print $2}' | sed -e 's/_UUID$//')
+
+        # on the mdt's osc
+       local mdtosc_proc1=$(get_mdtosc_proc_path $SINGLEMDS $OST)
+       local last_id=$(do_facet $SINGLEMDS lctl get_param -n \
+                       osc.$mdtosc_proc1.prealloc_last_id)
+       local next_id=$(do_facet $SINGLEMDS lctl get_param -n \
+                       osc.$mdtosc_proc1.prealloc_next_id)
+
+       $LFS df -i
+
+       do_facet mgs $LCTL pool_new $FSNAME.$TESTNAME || return 1
+       do_facet mgs $LCTL pool_add $FSNAME.$TESTNAME $OST || return 2
+
+       $SETSTRIPE $DIR/$tdir -i $OSTIDX -c 1 -p $FSNAME.$TESTNAME
+
+       echo "preallocated objects in MDS is $((last_id - next_id))" \
+             "($last_id - $next_id)"
+
+       count=$($LFS df -i $MOUNT | grep ^$OST | awk '{print $4}')
+       echo "OST still has $count objects"
+
+       free=$((count + last_id - next_id))
+       echo "create $free files..."
+       createmany -o $DIR/$tdir/f $next_id $free || return 3
+
+       local last_id=$(do_facet mds${MDSIDX} lctl get_param -n \
+                       osc.$mdtosc_proc1.prealloc_last_id)
+       local next_id=$(do_facet mds${MDSIDX} lctl get_param -n \
+                       osc.$mdtosc_proc1.prealloc_next_id)
+
+       echo "after creation, last_id=$last_id, next_id=$next_id"
+       $LFS df -i
+
+       echo "cleanup..."
+
+       do_facet mgs $LCTL pool_remove $FSNAME.$TESTNAME $OST || return 4
+       do_facet mgs $LCTL pool_destroy $FSNAME.$TESTNAME || return 5
+       rm -fr $DIR/$tdir
+}
+run_test 220 "the preallocated objects in MDS still can be used if ENOSPC is returned by OST with enough disk space"
+
 #
 # tests that do cleanup/setup should be run at the end
 #