Whamcloud - gitweb
Branch HEAD
authorfanyong <fanyong>
Wed, 25 Mar 2009 03:34:13 +0000 (03:34 +0000)
committerfanyong <fanyong>
Wed, 25 Mar 2009 03:34:13 +0000 (03:34 +0000)
b=18741
i=tianzy
i=huanghua

1) abort all dqacq/dqrel requests when lqc_import is invalid.
2) scripts fix.

13 files changed:
lustre/include/lustre_net.h
lustre/include/md_object.h
lustre/include/obd.h
lustre/ldlm/ldlm_lockd.c
lustre/mdd/mdd_lov.c
lustre/mds/mds_lov.c
lustre/mdt/mdt_handler.c
lustre/ptlrpc/import.c
lustre/ptlrpc/ptlrpc_module.c
lustre/quota/quota_context.c
lustre/quota/quota_ctl.c
lustre/quota/quota_interface.c
lustre/tests/sanity-quota.sh

index 30bf3d8..dace885 100644 (file)
@@ -927,6 +927,7 @@ int ptlrpc_replay_req(struct ptlrpc_request *req);
 int ptlrpc_unregister_reply(struct ptlrpc_request *req, int async);
 void ptlrpc_restart_req(struct ptlrpc_request *req);
 void ptlrpc_abort_inflight(struct obd_import *imp);
+void ptlrpc_cleanup_imp(struct obd_import *imp);
 void ptlrpc_abort_set(struct ptlrpc_request_set *set);
 
 struct ptlrpc_request_set *ptlrpc_prep_set(void);
index 0a16b78..f69fac5 100644 (file)
@@ -411,11 +411,13 @@ struct md_device_operations {
 };
 
 enum md_upcall_event {
-        /**sync the md layer*/
+        /** Sync the md layer*/
         MD_LOV_SYNC = (1 << 0),
         /** Just for split, no need trans, for replay */
         MD_NO_TRANS = (1 << 1),
-        MD_LOV_CONFIG = (1 << 2)
+        MD_LOV_CONFIG = (1 << 2),
+        /** Trigger quota recovery */
+        MD_LOV_QUOTA = (1 << 3)
 };
 
 struct md_upcall {
index a894998..4925fc8 100644 (file)
@@ -925,7 +925,9 @@ enum obd_notify_event {
         OBD_NOTIFY_SYNC_NONBLOCK,
         OBD_NOTIFY_SYNC,
         /* Configuration event */
-        OBD_NOTIFY_CONFIG
+        OBD_NOTIFY_CONFIG,
+        /* Trigger quota recovery */
+        OBD_NOTIFY_QUOTA
 };
 
 /* bit-mask flags for config events */
index 3aabe5e..ebe2807 100644 (file)
@@ -1692,21 +1692,6 @@ static int ldlm_callback_handler(struct ptlrpc_request *req)
         req_capsule_init(&req->rq_pill, req, RCL_SERVER);
 
         if (req->rq_export == NULL) {
-                struct ldlm_request *dlm_req;
-
-                CDEBUG(D_RPCTRACE, "operation %d from %s with bad "
-                       "export cookie "LPX64"; this is "
-                       "normal if this node rebooted with a lock held\n",
-                       lustre_msg_get_opc(req->rq_reqmsg),
-                       libcfs_id2str(req->rq_peer),
-                       lustre_msg_get_handle(req->rq_reqmsg)->cookie);
-
-                req_capsule_set(&req->rq_pill, &RQF_LDLM_CALLBACK);
-                dlm_req = req_capsule_client_get(&req->rq_pill, &RMF_DLM_REQ);
-                if (dlm_req != NULL)
-                        CDEBUG(D_RPCTRACE, "--> lock cookie: "LPX64"\n",
-                               dlm_req->lock_handle[0].cookie);
-
                 ldlm_callback_reply(req, -ENOTCONN);
                 RETURN(0);
         }
index 197ffd6..982313a 100644 (file)
@@ -77,6 +77,11 @@ static int mdd_notify(struct obd_device *host, struct obd_device *watched,
                 case OBD_NOTIFY_CONFIG:
                         rc = md_do_upcall(NULL, &mdd->mdd_md_dev, MD_LOV_CONFIG);
                         break;
+#ifdef HAVE_QUOTA_SUPPORT
+                case OBD_NOTIFY_QUOTA:
+                        rc = md_do_upcall(NULL, &mdd->mdd_md_dev, MD_LOV_QUOTA);
+                        break;
+#endif
                 default:
                         CDEBUG(D_INFO, "Unhandled notification %#x\n", ev);
         }
index 51b2fbc..afb7564 100644 (file)
@@ -64,15 +64,16 @@ static void mds_lov_dump_objids(const char *label, struct obd_device *obd)
                 GOTO(skip_bitmap, i);
         }
 
-        for(i=0;i<((mds->mds_lov_page_dirty->size/BITS_PER_LONG)+1);i++)
-                CDEBUG(D_INFO, "%u - %lx\n", i, mds->mds_lov_page_dirty->data[i]);
+        for(i = 0; i < mds->mds_lov_page_dirty->size / BITS_PER_LONG + 1; i++)
+                CDEBUG(D_INFO, "%u - %lx\n", i,
+                       mds->mds_lov_page_dirty->data[i]);
 skip_bitmap:
         if (mds->mds_lov_page_array == NULL) {
                 CERROR("not init page array!\n");
                 GOTO(skip_array, i);
 
         }
-        for(i=0;i<MDS_LOV_OBJID_PAGES_COUNT;i++) {
+        for(i = 0;i < MDS_LOV_OBJID_PAGES_COUNT; i++) {
                 obd_id *data = mds->mds_lov_page_array[i];
 
                 if (data == NULL)
@@ -81,7 +82,8 @@ skip_bitmap:
                 for(j=0; j < OBJID_PER_PAGE(); j++) {
                         if (data[j] == 0)
                                 continue;
-                        CDEBUG(D_INFO,"objid page %u idx %u - %llu \n", i,j,data[j]);
+                        CDEBUG(D_INFO,"objid page %u idx %u - "LPU64" \n",
+                               i, j, data[j]);
                 }
         }
 skip_array:
@@ -150,7 +152,7 @@ void mds_lov_destroy_objids(struct obd_device *obd)
         }
 
         if (mds->mds_lov_objid_filp) {
-                rc = filp_close((struct file *)mds->mds_lov_objid_filp, 0);
+                rc = filp_close((struct file *)mds->mds_lov_objid_filp, NULL);
                 mds->mds_lov_objid_filp = NULL;
                 if (rc)
                         CERROR("%s file won't close, rc=%d\n", LOV_OBJID, rc);
@@ -301,7 +303,6 @@ void mds_lov_update_objids(struct obd_device *obd, struct lov_mds_md *lmm)
 }
 EXPORT_SYMBOL(mds_lov_update_objids);
 
-
 static int mds_lov_update_from_read(struct mds_obd *mds, obd_id *data,
                                     __u32 count)
 {
@@ -332,7 +333,7 @@ static int mds_lov_read_objids(struct obd_device *obd)
 {
         struct mds_obd *mds = &obd->u.mds;
         loff_t off = 0;
-        int i, rc, count = 0, page = 0;
+        int i, rc = 0, count = 0, page = 0;
         unsigned long size;
         ENTRY;
 
@@ -345,8 +346,8 @@ static int mds_lov_read_objids(struct obd_device *obd)
 
         page = (size / (OBJID_PER_PAGE() * sizeof(obd_id))) + 1;
         CDEBUG(D_INFO, "file size %lu pages %d\n", size, page);
-
         for (i = 0; i < page; i++) {
+                obd_id *data;
                 loff_t off_old = off;
 
                 LASSERT(mds->mds_lov_page_array[i] == NULL);
@@ -354,21 +355,23 @@ static int mds_lov_read_objids(struct obd_device *obd)
                 if (mds->mds_lov_page_array[i] == NULL)
                         GOTO(out, rc = -ENOMEM);
 
-                rc = fsfilt_read_record(obd, mds->mds_lov_objid_filp, mds->mds_lov_page_array[i],
+                data = mds->mds_lov_page_array[i];
+
+                rc = fsfilt_read_record(obd, mds->mds_lov_objid_filp, data,
                                         OBJID_PER_PAGE()*sizeof(obd_id), &off);
                 if (rc < 0) {
                         CERROR("Error reading objids %d\n", rc);
                         GOTO(out, rc);
                 }
 
-                count += (off - off_old)/sizeof(obd_id);
-                if (mds_lov_update_from_read(mds, mds->mds_lov_page_array[i], count)) {
-                         CERROR("Can't update mds data\n");
-                         GOTO(out, rc = -EIO);
-                 }
+                count += (off - off_old) / sizeof(obd_id);
+                if (mds_lov_update_from_read(mds, data, count)) {
+                        CERROR("Can't update mds data\n");
+                        GOTO(out, rc = -EIO);
+                }
 
                 if (off == off_old)
-                        break; // eof
+                        break; /* eof */
         }
         mds->mds_lov_objid_lastpage = i;
         mds->mds_lov_objid_lastidx = count % OBJID_PER_PAGE();
@@ -378,13 +381,13 @@ static int mds_lov_read_objids(struct obd_device *obd)
 out:
         mds_lov_dump_objids("read",obd);
 
-        RETURN(0);
+        RETURN(rc);
 }
 
 int mds_lov_write_objids(struct obd_device *obd)
 {
         struct mds_obd *mds = &obd->u.mds;
-        int i, rc = 0;
+        int i = 0, rc = 0;
         ENTRY;
 
         if (cfs_bitmap_check_empty(mds->mds_lov_page_dirty))
@@ -403,6 +406,7 @@ int mds_lov_write_objids(struct obd_device *obd)
                 if (i == mds->mds_lov_objid_lastpage)
                         size = (mds->mds_lov_objid_lastidx+1) * sizeof(obd_id);
 
+                CDEBUG(D_INFO, "write %lld - %u\n", off, size);
                 rc = fsfilt_write_record(obd, mds->mds_lov_objid_filp, data,
                                          size, &off, 0);
                 if (rc < 0)
@@ -441,8 +445,14 @@ static int mds_lov_get_objid(struct obd_device * obd,
                 if (rc)
                         GOTO(out, rc);
 
+                /* workaround for clean filter */
+                if (data[off] == 0)
+                        data[off] = 1;
+
                 cfs_bitmap_set(mds->mds_lov_page_dirty, page);
         }
+        CDEBUG(D_INFO, "idx "LPU64" - %p - %d/%d - "LPU64"\n",
+               idx, data, page, off, data[off]);
 out:
         RETURN(rc);
 }
@@ -466,6 +476,7 @@ int mds_lov_clear_orphans(struct mds_obd *mds, struct obd_uuid *ost_uuid)
         oa.o_valid = OBD_MD_FLFLAGS | OBD_MD_FLGROUP;
         if (ost_uuid != NULL)
                 oti.oti_ost_uuid = ost_uuid;
+
         rc = obd_create(mds->mds_osc_exp, &oa, &empty_ea, &oti);
 
         RETURN(rc);
@@ -523,7 +534,6 @@ static int mds_lov_update_desc(struct obd_device *obd, int idx,
         if (rc != 0)
                 GOTO(out, rc );
 
-
         /* If we added a target we have to reconnect the llogs */
         /* We only _need_ to do this at first add (idx), or the first time
            after recovery.  However, it should now be safe to call anytime. */
@@ -801,15 +811,17 @@ static int __mds_lov_synchronize(void *data)
                 GOTO(out, rc);
         }
 
+#ifdef HAVE_QUOTA_SUPPORT
         if (obd->obd_upcall.onu_owner) {
                 /*
                  * This is a hack for mds_notify->mdd_notify. When the mds obd
                  * in mdd is removed, This hack should be removed.
                  */
                  LASSERT(obd->obd_upcall.onu_upcall != NULL);
-                 rc = obd->obd_upcall.onu_upcall(obd, NULL, OBD_NOTIFY_ACTIVE,
+                 rc = obd->obd_upcall.onu_upcall(obd, NULL, OBD_NOTIFY_QUOTA,
                                                  obd->obd_upcall.onu_owner);
         }
+#endif
         EXIT;
 out:
         up_read(&mds->mds_notify_lock);
@@ -907,7 +919,6 @@ int mds_notify(struct obd_device *obd, struct obd_device *watched,
         case OBD_NOTIFY_SYNC_NONBLOCK:
                 /* sync event should be pass lov idx as argument */
                 break;
-        case OBD_NOTIFY_CONFIG:
         default:
                 RETURN(0);
         }
index 0099f50..9f0ea54 100644 (file)
@@ -5269,10 +5269,6 @@ static int mdt_upcall(const struct lu_env *env, struct md_device *md,
                         CDEBUG(D_INFO, "get max mdsize %d max cookiesize %d\n",
                                      m->mdt_max_mdsize, m->mdt_max_cookiesize);
                         mdt_allow_cli(m, CONFIG_SYNC);
-#ifdef HAVE_QUOTA_SUPPORT
-                        if (md->md_lu_dev.ld_obd->obd_recovering == 0)
-                                next->md_ops->mdo_quota.mqo_recovery(env, next);
-#endif
                         break;
                 case MD_NO_TRANS:
                         mti = lu_context_key_get(&env->le_ctx, &mdt_thread_key);
@@ -5283,6 +5279,12 @@ static int mdt_upcall(const struct lu_env *env, struct md_device *md,
                         /* Check that MDT is not yet configured */
                         LASSERT(!m->mdt_fl_cfglog);
                         break;
+#ifdef HAVE_QUOTA_SUPPORT
+                case MD_LOV_QUOTA:
+                        if (md->md_lu_dev.ld_obd->obd_recovering == 0)
+                                next->md_ops->mdo_quota.mqo_recovery(env, next);
+                        break;
+#endif
                 default:
                         CERROR("invalid event\n");
                         rc = -EINVAL;
index baa4fb8..098443c 100644 (file)
@@ -1427,6 +1427,18 @@ out:
         RETURN(rc);
 }
 
+void ptlrpc_cleanup_imp(struct obd_import *imp)
+{
+        ENTRY;
+
+        spin_lock(&imp->imp_lock);
+        IMPORT_SET_STATE_NOLOCK(imp, LUSTRE_IMP_CLOSED);
+        imp->imp_generation++;
+        spin_unlock(&imp->imp_lock);
+        ptlrpc_abort_inflight(imp);
+
+        EXIT;
+}
 
 /* Adaptive Timeout utils */
 extern unsigned int at_min, at_max, at_history;
index 7a9e4d5..fe75eb9 100644 (file)
@@ -190,6 +190,7 @@ EXPORT_SYMBOL(ptlrpc_prep_bulk_exp);
 EXPORT_SYMBOL(ptlrpc_free_bulk);
 EXPORT_SYMBOL(ptlrpc_prep_bulk_page);
 EXPORT_SYMBOL(ptlrpc_abort_inflight);
+EXPORT_SYMBOL(ptlrpc_cleanup_imp);
 EXPORT_SYMBOL(ptlrpc_retain_replayable_request);
 EXPORT_SYMBOL(ptlrpc_next_xid);
 EXPORT_SYMBOL(ptlrpc_req_set_repsize);
index 6bcbcc5..cccc3fd 100644 (file)
@@ -499,7 +499,6 @@ static void compute_lqs_after_removing_qunit(struct lustre_qunit *qunit)
                 /* this is for schedule_dqacq */
                 lqs_putref(lqs);
         }
-
 }
 
 static void remove_qunit_nolock(struct lustre_qunit *qunit)
index 5e8cd88..be5bab6 100644 (file)
@@ -151,6 +151,7 @@ int filter_quota_ctl(struct obd_device *unused, struct obd_export *exp,
                 }
                 if (oqctl->qc_cmd == Q_FINVALIDATE &&
                     (obt->obt_qctxt.lqc_flags & UGQUOTA2LQC(oqctl->qc_type))) {
+                        atomic_inc(&obt->obt_quotachecking);
                         rc = -EBUSY;
                         break;
                 }
index 59c6e70..ef40fd8 100644 (file)
@@ -137,6 +137,7 @@ static int filter_quota_clearinfo(struct obd_export *exp, struct obd_device *obd
                 spin_lock(&qctxt->lqc_lock);
                 qctxt->lqc_import = NULL;
                 spin_unlock(&qctxt->lqc_lock);
+                ptlrpc_cleanup_imp(exp->exp_imp_reverse);
                 dqacq_interrupt(qctxt);
                 CDEBUG(D_QUOTA, "%s: lqc_import of obd(%p) is invalid now.\n",
                        obd->obd_name, obd);
index c9a2177..068073d 100644 (file)
@@ -173,9 +173,32 @@ run_test_with_stat() {
 #        resetquota -g groupname
 
 resetquota() {
-       [ "$#" != 2 ] && error "resetquota: wrong number of arguments: $#"
-       [ "$1" != "-u" -a "$1" != "-g" ] && error "resetquota: wrong specifier $1 passed"
-       $LFS setquota "$1" "$2" -b 0 -B 0 -i 0 -I 0 $MOUNT || error "resetquota failed"
+        [ "$#" != 2 ] && error "resetquota: wrong number of arguments: $#"
+        [ "$1" != "-u" -a "$1" != "-g" ] && error "resetquota: wrong specifier $1 passed"
+
+        count=0
+        if at_is_valid && at_is_enabled; then
+           timeout=$(at_max_get mds)
+        else
+           timeout=$(lctl get_param -n timeout)
+        fi
+
+        while [ $((count++)) -lt $timeout ]; do
+                $LFS setquota "$1" "$2" -b 0 -B 0 -i 0 -I 0 $MOUNT
+                RC=$?
+                if [ $RC -ne 0 ]; then
+                        if [ $RC -eq 240 ]; then # 240 means -EBUSY
+                                log "resetquota is blocked for quota master recovery, retry after 1 sec"
+                                sleep 1
+                                continue
+                        else
+                                error "resetquota failed: $RC"
+                        fi
+                fi
+                break
+        done
+
+        [ $count -lt $timeout ] || error "resetquota timeout: $timeout"
 }
 
 quota_scan() {
@@ -215,12 +238,20 @@ quota_show_check() {
 
         if [ "$LOCAL_BF" == "a" -o "$LOCAL_BF" == "b" ]; then
                USAGE="`$LFS quota -$LOCAL_UG $LOCAL_ID $DIR | awk '/^.*'$PATTERN'.*[[:digit:]+][[:space:]+]/ { print $2 }'`"
-                [ $USAGE -ne 0 ] && quota_log $LOCAL_UG $LOCAL_ID "System is not clean for block ($LOCAL_UG:$LOCAL_ID:$USAGE)."
+                if [ -z $USAGE ]; then
+                        quota_error $LOCAL_UG $LOCAL_ID "System is error when query quota for block ($LOCAL_UG:$LOCAL_ID)."
+                else
+                        [ $USAGE -ne 0 ] && quota_log $LOCAL_UG $LOCAL_ID "System is not clean for block ($LOCAL_UG:$LOCAL_ID:$USAGE)."
+                fi
         fi
 
         if [ "$LOCAL_BF" == "a" -o "$LOCAL_BF" == "f" ]; then
                USAGE="`$LFS quota -$LOCAL_UG $LOCAL_ID $DIR | awk '/^.*'$PATTERN'.*[[:digit:]+][[:space:]+]/ { print $5 }'`"
-                [ $USAGE -ne 0 ] && quota_log $LOCAL_UG $LOCAL_ID "System is not clean for file ($LOCAL_UG:$LOCAL_ID:$USAGE)."
+                if [ -z $USAGE ]; then
+                        quota_error $LOCAL_UG $LOCAL_ID "System is error when query quota for file ($LOCAL_UG:$LOCAL_ID)."
+                else
+                        [ $USAGE -ne 0 ] && quota_log $LOCAL_UG $LOCAL_ID "System is not clean for file ($LOCAL_UG:$LOCAL_ID:$USAGE)."
+                fi
         fi
 }
 
@@ -1397,15 +1428,18 @@ test_18() {
            sleep 1
        done
         log "(dd_pid=$DDPID, time=$count, timeout=$timeout)"
+        sync
+        cancel_lru_locks mdc
+        cancel_lru_locks osc
 
         testfile_size=$(stat -c %s $TESTFILE)
         [ $testfile_size -ne $((BLK_SZ * 1024 * 100)) ] && \
            quota_error u $TSTUSR "expect $((BLK_SZ * 1024 * 100)), got ${testfile_size}. Verifying file failed!"
-       rm -f $TESTFILE
-       sync; sleep 3; sync;
+        $SHOW_QUOTA_USER
+        rm -f $TESTFILE
+        sync
 
        resetquota -u $TSTUSR
-
        set_blk_unitsz $((128 * 1024))
        set_blk_tunesz $((128 * 1024 / 2))
 }
@@ -1455,12 +1489,10 @@ test_18a() {
         log "(dd_pid=$DDPID, time=$count, timeout=$timeout)"
 
         lustre_fail mds 0
-
        rm -f $TESTFILE
-       sync; sleep 3; sync;
+       sync
 
        resetquota -u $TSTUSR
-
        set_blk_unitsz $((128 * 1024))
        set_blk_tunesz $((128 * 1024 / 2))
 }
@@ -1525,15 +1557,20 @@ test_18bc_sub() {
             sleep 1
         done
         log "(dd_pid=$DDPID, time=$count, timeout=$timeout)"
-        sync; sleep 1; sync
+        sync
+        cancel_lru_locks mdc
+        cancel_lru_locks osc
 
         testfile_size=$(stat -c %s $TESTFILE)
         [ $testfile_size -ne $((BLK_SZ * 1024 * 100)) ] && \
            quota_error u $TSTUSR "expect $((BLK_SZ * 1024 * 100)), got ${testfile_size}. Verifying file failed!"
         $SHOW_QUOTA_USER
-        resetquota -u $TSTUSR
-        rm -rf $TESTFILE
-        sync; sleep 1; sync
+        rm -f $TESTFILE
+        sync
+
+       resetquota -u $TSTUSR
+       set_blk_unitsz $((128 * 1024))
+       set_blk_tunesz $((128 * 1024 / 2))
 }
 
 # test when mds does failover, the ost still could work well
@@ -1729,21 +1766,6 @@ test_22() {
 }
 run_test_with_stat 22 "test if quota_type saved as permanent parameter ===="
 
-# It is triggered when test_23 failed, diagnostic for bug 18293
-test_23_dumppage()
-{
-        NUM=$1
-        DUMPPAGE=`find /proc/fs/${FSNAME}/llite/ -name dump_page_cache`
-        qtime=`date +%s`
-        cat $DUMPPAGE > $TMP/sanity-quota_test_23_${qtime}_${NUM}.log
-        fsize=`stat -c%s $TMP/sanity-quota_test_23_${qtime}_${NUM}.log`
-        if [ $fsize -eq 0 ]; then
-                rm -f $TMP/sanity-quota_test_23_${qtime}_${NUM}.log
-        else
-                error "some IO error was found during directIO"
-        fi
-}
-
 test_23_sub() {
        mkdir -p $DIR/$tdir
        chmod 0777 $DIR/$tdir
@@ -1766,15 +1788,15 @@ test_23_sub() {
        log "    Step1: trigger quota with 0_DIRECT"
        log "      Write half of file"
        $RUNAS $DIRECTIO write $TESTFILE 0 $(($LIMIT/1024/2)) $bs_unit || \
-                (quota_error u $TSTUSR "(1) write failure, but expect success: $LIMIT" && test_23_dumppage 1)
+                quota_error u $TSTUSR "(1) write failure, but expect success: $LIMIT"
        log "      Write out of block quota ..."
        $RUNAS $DIRECTIO write $TESTFILE $(($LIMIT/1024/2)) $(($LIMIT/1024/2)) $bs_unit && \
-                quota_error u $TSTUSR "(2) write success, but expect EDQUOT: $LIMIT" && test_23_dumppage 2
+                quota_error u $TSTUSR "(2) write success, but expect EDQUOT: $LIMIT"
        log "    Step1: done"
 
        log "    Step2: rewrite should succeed"
-       $RUNAS $DIRECTIO write $TESTFILE $(($LIMIT/1024/2)) 1 $bs_unit || \
-                (quota_error u $TSTUSR "(3) write failure, but expect success: $LIMIT" && test_23_dumppage 3)
+       $RUNAS $DIRECTIO write $TESTFILE 0 1 $bs_unit || \
+                quota_error u $TSTUSR "(3) write failure, but expect success: $LIMIT"
        log "    Step2: done"
 
        rm -f $TESTFILE