From c50dca76220901956b8904c4d1081663a735a6fe Mon Sep 17 00:00:00 2001 From: fanyong Date: Wed, 25 Mar 2009 03:34:13 +0000 Subject: [PATCH] Branch HEAD b=18741 i=tianzy i=huanghua 1) abort all dqacq/dqrel requests when lqc_import is invalid. 2) scripts fix. --- lustre/include/lustre_net.h | 1 + lustre/include/md_object.h | 6 ++- lustre/include/obd.h | 4 +- lustre/ldlm/ldlm_lockd.c | 15 ------- lustre/mdd/mdd_lov.c | 5 +++ lustre/mds/mds_lov.c | 51 ++++++++++++++---------- lustre/mdt/mdt_handler.c | 10 +++-- lustre/ptlrpc/import.c | 12 ++++++ lustre/ptlrpc/ptlrpc_module.c | 1 + lustre/quota/quota_context.c | 1 - lustre/quota/quota_ctl.c | 1 + lustre/quota/quota_interface.c | 1 + lustre/tests/sanity-quota.sh | 90 ++++++++++++++++++++++++++---------------- 13 files changed, 121 insertions(+), 77 deletions(-) diff --git a/lustre/include/lustre_net.h b/lustre/include/lustre_net.h index 30bf3d8..dace885 100644 --- a/lustre/include/lustre_net.h +++ b/lustre/include/lustre_net.h @@ -927,6 +927,7 @@ int ptlrpc_replay_req(struct ptlrpc_request *req); int ptlrpc_unregister_reply(struct ptlrpc_request *req, int async); void ptlrpc_restart_req(struct ptlrpc_request *req); void ptlrpc_abort_inflight(struct obd_import *imp); +void ptlrpc_cleanup_imp(struct obd_import *imp); void ptlrpc_abort_set(struct ptlrpc_request_set *set); struct ptlrpc_request_set *ptlrpc_prep_set(void); diff --git a/lustre/include/md_object.h b/lustre/include/md_object.h index 0a16b78..f69fac5 100644 --- a/lustre/include/md_object.h +++ b/lustre/include/md_object.h @@ -411,11 +411,13 @@ struct md_device_operations { }; enum md_upcall_event { - /**sync the md layer*/ + /** Sync the md layer*/ MD_LOV_SYNC = (1 << 0), /** Just for split, no need trans, for replay */ MD_NO_TRANS = (1 << 1), - MD_LOV_CONFIG = (1 << 2) + MD_LOV_CONFIG = (1 << 2), + /** Trigger quota recovery */ + MD_LOV_QUOTA = (1 << 3) }; struct md_upcall { diff --git a/lustre/include/obd.h b/lustre/include/obd.h index a894998..4925fc8 100644 --- a/lustre/include/obd.h +++ b/lustre/include/obd.h @@ -925,7 +925,9 @@ enum obd_notify_event { OBD_NOTIFY_SYNC_NONBLOCK, OBD_NOTIFY_SYNC, /* Configuration event */ - OBD_NOTIFY_CONFIG + OBD_NOTIFY_CONFIG, + /* Trigger quota recovery */ + OBD_NOTIFY_QUOTA }; /* bit-mask flags for config events */ diff --git a/lustre/ldlm/ldlm_lockd.c b/lustre/ldlm/ldlm_lockd.c index 3aabe5e..ebe2807 100644 --- a/lustre/ldlm/ldlm_lockd.c +++ b/lustre/ldlm/ldlm_lockd.c @@ -1692,21 +1692,6 @@ static int ldlm_callback_handler(struct ptlrpc_request *req) req_capsule_init(&req->rq_pill, req, RCL_SERVER); if (req->rq_export == NULL) { - struct ldlm_request *dlm_req; - - CDEBUG(D_RPCTRACE, "operation %d from %s with bad " - "export cookie "LPX64"; this is " - "normal if this node rebooted with a lock held\n", - lustre_msg_get_opc(req->rq_reqmsg), - libcfs_id2str(req->rq_peer), - lustre_msg_get_handle(req->rq_reqmsg)->cookie); - - req_capsule_set(&req->rq_pill, &RQF_LDLM_CALLBACK); - dlm_req = req_capsule_client_get(&req->rq_pill, &RMF_DLM_REQ); - if (dlm_req != NULL) - CDEBUG(D_RPCTRACE, "--> lock cookie: "LPX64"\n", - dlm_req->lock_handle[0].cookie); - ldlm_callback_reply(req, -ENOTCONN); RETURN(0); } diff --git a/lustre/mdd/mdd_lov.c b/lustre/mdd/mdd_lov.c index 197ffd6..982313a 100644 --- a/lustre/mdd/mdd_lov.c +++ b/lustre/mdd/mdd_lov.c @@ -77,6 +77,11 @@ static int mdd_notify(struct obd_device *host, struct obd_device *watched, case OBD_NOTIFY_CONFIG: rc = md_do_upcall(NULL, &mdd->mdd_md_dev, MD_LOV_CONFIG); break; +#ifdef HAVE_QUOTA_SUPPORT + case OBD_NOTIFY_QUOTA: + rc = md_do_upcall(NULL, &mdd->mdd_md_dev, MD_LOV_QUOTA); + break; +#endif default: CDEBUG(D_INFO, "Unhandled notification %#x\n", ev); } diff --git a/lustre/mds/mds_lov.c b/lustre/mds/mds_lov.c index 51b2fbc..afb7564 100644 --- a/lustre/mds/mds_lov.c +++ b/lustre/mds/mds_lov.c @@ -64,15 +64,16 @@ static void mds_lov_dump_objids(const char *label, struct obd_device *obd) GOTO(skip_bitmap, i); } - for(i=0;i<((mds->mds_lov_page_dirty->size/BITS_PER_LONG)+1);i++) - CDEBUG(D_INFO, "%u - %lx\n", i, mds->mds_lov_page_dirty->data[i]); + for(i = 0; i < mds->mds_lov_page_dirty->size / BITS_PER_LONG + 1; i++) + CDEBUG(D_INFO, "%u - %lx\n", i, + mds->mds_lov_page_dirty->data[i]); skip_bitmap: if (mds->mds_lov_page_array == NULL) { CERROR("not init page array!\n"); GOTO(skip_array, i); } - for(i=0;imds_lov_page_array[i]; if (data == NULL) @@ -81,7 +82,8 @@ skip_bitmap: for(j=0; j < OBJID_PER_PAGE(); j++) { if (data[j] == 0) continue; - CDEBUG(D_INFO,"objid page %u idx %u - %llu \n", i,j,data[j]); + CDEBUG(D_INFO,"objid page %u idx %u - "LPU64" \n", + i, j, data[j]); } } skip_array: @@ -150,7 +152,7 @@ void mds_lov_destroy_objids(struct obd_device *obd) } if (mds->mds_lov_objid_filp) { - rc = filp_close((struct file *)mds->mds_lov_objid_filp, 0); + rc = filp_close((struct file *)mds->mds_lov_objid_filp, NULL); mds->mds_lov_objid_filp = NULL; if (rc) CERROR("%s file won't close, rc=%d\n", LOV_OBJID, rc); @@ -301,7 +303,6 @@ void mds_lov_update_objids(struct obd_device *obd, struct lov_mds_md *lmm) } EXPORT_SYMBOL(mds_lov_update_objids); - static int mds_lov_update_from_read(struct mds_obd *mds, obd_id *data, __u32 count) { @@ -332,7 +333,7 @@ static int mds_lov_read_objids(struct obd_device *obd) { struct mds_obd *mds = &obd->u.mds; loff_t off = 0; - int i, rc, count = 0, page = 0; + int i, rc = 0, count = 0, page = 0; unsigned long size; ENTRY; @@ -345,8 +346,8 @@ static int mds_lov_read_objids(struct obd_device *obd) page = (size / (OBJID_PER_PAGE() * sizeof(obd_id))) + 1; CDEBUG(D_INFO, "file size %lu pages %d\n", size, page); - for (i = 0; i < page; i++) { + obd_id *data; loff_t off_old = off; LASSERT(mds->mds_lov_page_array[i] == NULL); @@ -354,21 +355,23 @@ static int mds_lov_read_objids(struct obd_device *obd) if (mds->mds_lov_page_array[i] == NULL) GOTO(out, rc = -ENOMEM); - rc = fsfilt_read_record(obd, mds->mds_lov_objid_filp, mds->mds_lov_page_array[i], + data = mds->mds_lov_page_array[i]; + + rc = fsfilt_read_record(obd, mds->mds_lov_objid_filp, data, OBJID_PER_PAGE()*sizeof(obd_id), &off); if (rc < 0) { CERROR("Error reading objids %d\n", rc); GOTO(out, rc); } - count += (off - off_old)/sizeof(obd_id); - if (mds_lov_update_from_read(mds, mds->mds_lov_page_array[i], count)) { - CERROR("Can't update mds data\n"); - GOTO(out, rc = -EIO); - } + count += (off - off_old) / sizeof(obd_id); + if (mds_lov_update_from_read(mds, data, count)) { + CERROR("Can't update mds data\n"); + GOTO(out, rc = -EIO); + } if (off == off_old) - break; // eof + break; /* eof */ } mds->mds_lov_objid_lastpage = i; mds->mds_lov_objid_lastidx = count % OBJID_PER_PAGE(); @@ -378,13 +381,13 @@ static int mds_lov_read_objids(struct obd_device *obd) out: mds_lov_dump_objids("read",obd); - RETURN(0); + RETURN(rc); } int mds_lov_write_objids(struct obd_device *obd) { struct mds_obd *mds = &obd->u.mds; - int i, rc = 0; + int i = 0, rc = 0; ENTRY; if (cfs_bitmap_check_empty(mds->mds_lov_page_dirty)) @@ -403,6 +406,7 @@ int mds_lov_write_objids(struct obd_device *obd) if (i == mds->mds_lov_objid_lastpage) size = (mds->mds_lov_objid_lastidx+1) * sizeof(obd_id); + CDEBUG(D_INFO, "write %lld - %u\n", off, size); rc = fsfilt_write_record(obd, mds->mds_lov_objid_filp, data, size, &off, 0); if (rc < 0) @@ -441,8 +445,14 @@ static int mds_lov_get_objid(struct obd_device * obd, if (rc) GOTO(out, rc); + /* workaround for clean filter */ + if (data[off] == 0) + data[off] = 1; + cfs_bitmap_set(mds->mds_lov_page_dirty, page); } + CDEBUG(D_INFO, "idx "LPU64" - %p - %d/%d - "LPU64"\n", + idx, data, page, off, data[off]); out: RETURN(rc); } @@ -466,6 +476,7 @@ int mds_lov_clear_orphans(struct mds_obd *mds, struct obd_uuid *ost_uuid) oa.o_valid = OBD_MD_FLFLAGS | OBD_MD_FLGROUP; if (ost_uuid != NULL) oti.oti_ost_uuid = ost_uuid; + rc = obd_create(mds->mds_osc_exp, &oa, &empty_ea, &oti); RETURN(rc); @@ -523,7 +534,6 @@ static int mds_lov_update_desc(struct obd_device *obd, int idx, if (rc != 0) GOTO(out, rc ); - /* If we added a target we have to reconnect the llogs */ /* We only _need_ to do this at first add (idx), or the first time after recovery. However, it should now be safe to call anytime. */ @@ -801,15 +811,17 @@ static int __mds_lov_synchronize(void *data) GOTO(out, rc); } +#ifdef HAVE_QUOTA_SUPPORT if (obd->obd_upcall.onu_owner) { /* * This is a hack for mds_notify->mdd_notify. When the mds obd * in mdd is removed, This hack should be removed. */ LASSERT(obd->obd_upcall.onu_upcall != NULL); - rc = obd->obd_upcall.onu_upcall(obd, NULL, OBD_NOTIFY_ACTIVE, + rc = obd->obd_upcall.onu_upcall(obd, NULL, OBD_NOTIFY_QUOTA, obd->obd_upcall.onu_owner); } +#endif EXIT; out: up_read(&mds->mds_notify_lock); @@ -907,7 +919,6 @@ int mds_notify(struct obd_device *obd, struct obd_device *watched, case OBD_NOTIFY_SYNC_NONBLOCK: /* sync event should be pass lov idx as argument */ break; - case OBD_NOTIFY_CONFIG: default: RETURN(0); } diff --git a/lustre/mdt/mdt_handler.c b/lustre/mdt/mdt_handler.c index 0099f50..9f0ea54 100644 --- a/lustre/mdt/mdt_handler.c +++ b/lustre/mdt/mdt_handler.c @@ -5269,10 +5269,6 @@ static int mdt_upcall(const struct lu_env *env, struct md_device *md, CDEBUG(D_INFO, "get max mdsize %d max cookiesize %d\n", m->mdt_max_mdsize, m->mdt_max_cookiesize); mdt_allow_cli(m, CONFIG_SYNC); -#ifdef HAVE_QUOTA_SUPPORT - if (md->md_lu_dev.ld_obd->obd_recovering == 0) - next->md_ops->mdo_quota.mqo_recovery(env, next); -#endif break; case MD_NO_TRANS: mti = lu_context_key_get(&env->le_ctx, &mdt_thread_key); @@ -5283,6 +5279,12 @@ static int mdt_upcall(const struct lu_env *env, struct md_device *md, /* Check that MDT is not yet configured */ LASSERT(!m->mdt_fl_cfglog); break; +#ifdef HAVE_QUOTA_SUPPORT + case MD_LOV_QUOTA: + if (md->md_lu_dev.ld_obd->obd_recovering == 0) + next->md_ops->mdo_quota.mqo_recovery(env, next); + break; +#endif default: CERROR("invalid event\n"); rc = -EINVAL; diff --git a/lustre/ptlrpc/import.c b/lustre/ptlrpc/import.c index baa4fb8..098443c 100644 --- a/lustre/ptlrpc/import.c +++ b/lustre/ptlrpc/import.c @@ -1427,6 +1427,18 @@ out: RETURN(rc); } +void ptlrpc_cleanup_imp(struct obd_import *imp) +{ + ENTRY; + + spin_lock(&imp->imp_lock); + IMPORT_SET_STATE_NOLOCK(imp, LUSTRE_IMP_CLOSED); + imp->imp_generation++; + spin_unlock(&imp->imp_lock); + ptlrpc_abort_inflight(imp); + + EXIT; +} /* Adaptive Timeout utils */ extern unsigned int at_min, at_max, at_history; diff --git a/lustre/ptlrpc/ptlrpc_module.c b/lustre/ptlrpc/ptlrpc_module.c index 7a9e4d5..fe75eb9 100644 --- a/lustre/ptlrpc/ptlrpc_module.c +++ b/lustre/ptlrpc/ptlrpc_module.c @@ -190,6 +190,7 @@ EXPORT_SYMBOL(ptlrpc_prep_bulk_exp); EXPORT_SYMBOL(ptlrpc_free_bulk); EXPORT_SYMBOL(ptlrpc_prep_bulk_page); EXPORT_SYMBOL(ptlrpc_abort_inflight); +EXPORT_SYMBOL(ptlrpc_cleanup_imp); EXPORT_SYMBOL(ptlrpc_retain_replayable_request); EXPORT_SYMBOL(ptlrpc_next_xid); EXPORT_SYMBOL(ptlrpc_req_set_repsize); diff --git a/lustre/quota/quota_context.c b/lustre/quota/quota_context.c index 6bcbcc5..cccc3fd 100644 --- a/lustre/quota/quota_context.c +++ b/lustre/quota/quota_context.c @@ -499,7 +499,6 @@ static void compute_lqs_after_removing_qunit(struct lustre_qunit *qunit) /* this is for schedule_dqacq */ lqs_putref(lqs); } - } static void remove_qunit_nolock(struct lustre_qunit *qunit) diff --git a/lustre/quota/quota_ctl.c b/lustre/quota/quota_ctl.c index 5e8cd88..be5bab6 100644 --- a/lustre/quota/quota_ctl.c +++ b/lustre/quota/quota_ctl.c @@ -151,6 +151,7 @@ int filter_quota_ctl(struct obd_device *unused, struct obd_export *exp, } if (oqctl->qc_cmd == Q_FINVALIDATE && (obt->obt_qctxt.lqc_flags & UGQUOTA2LQC(oqctl->qc_type))) { + atomic_inc(&obt->obt_quotachecking); rc = -EBUSY; break; } diff --git a/lustre/quota/quota_interface.c b/lustre/quota/quota_interface.c index 59c6e70..ef40fd8 100644 --- a/lustre/quota/quota_interface.c +++ b/lustre/quota/quota_interface.c @@ -137,6 +137,7 @@ static int filter_quota_clearinfo(struct obd_export *exp, struct obd_device *obd spin_lock(&qctxt->lqc_lock); qctxt->lqc_import = NULL; spin_unlock(&qctxt->lqc_lock); + ptlrpc_cleanup_imp(exp->exp_imp_reverse); dqacq_interrupt(qctxt); CDEBUG(D_QUOTA, "%s: lqc_import of obd(%p) is invalid now.\n", obd->obd_name, obd); diff --git a/lustre/tests/sanity-quota.sh b/lustre/tests/sanity-quota.sh index c9a2177..068073d 100644 --- a/lustre/tests/sanity-quota.sh +++ b/lustre/tests/sanity-quota.sh @@ -173,9 +173,32 @@ run_test_with_stat() { # resetquota -g groupname resetquota() { - [ "$#" != 2 ] && error "resetquota: wrong number of arguments: $#" - [ "$1" != "-u" -a "$1" != "-g" ] && error "resetquota: wrong specifier $1 passed" - $LFS setquota "$1" "$2" -b 0 -B 0 -i 0 -I 0 $MOUNT || error "resetquota failed" + [ "$#" != 2 ] && error "resetquota: wrong number of arguments: $#" + [ "$1" != "-u" -a "$1" != "-g" ] && error "resetquota: wrong specifier $1 passed" + + count=0 + if at_is_valid && at_is_enabled; then + timeout=$(at_max_get mds) + else + timeout=$(lctl get_param -n timeout) + fi + + while [ $((count++)) -lt $timeout ]; do + $LFS setquota "$1" "$2" -b 0 -B 0 -i 0 -I 0 $MOUNT + RC=$? + if [ $RC -ne 0 ]; then + if [ $RC -eq 240 ]; then # 240 means -EBUSY + log "resetquota is blocked for quota master recovery, retry after 1 sec" + sleep 1 + continue + else + error "resetquota failed: $RC" + fi + fi + break + done + + [ $count -lt $timeout ] || error "resetquota timeout: $timeout" } quota_scan() { @@ -215,12 +238,20 @@ quota_show_check() { if [ "$LOCAL_BF" == "a" -o "$LOCAL_BF" == "b" ]; then USAGE="`$LFS quota -$LOCAL_UG $LOCAL_ID $DIR | awk '/^.*'$PATTERN'.*[[:digit:]+][[:space:]+]/ { print $2 }'`" - [ $USAGE -ne 0 ] && quota_log $LOCAL_UG $LOCAL_ID "System is not clean for block ($LOCAL_UG:$LOCAL_ID:$USAGE)." + if [ -z $USAGE ]; then + quota_error $LOCAL_UG $LOCAL_ID "System is error when query quota for block ($LOCAL_UG:$LOCAL_ID)." + else + [ $USAGE -ne 0 ] && quota_log $LOCAL_UG $LOCAL_ID "System is not clean for block ($LOCAL_UG:$LOCAL_ID:$USAGE)." + fi fi if [ "$LOCAL_BF" == "a" -o "$LOCAL_BF" == "f" ]; then USAGE="`$LFS quota -$LOCAL_UG $LOCAL_ID $DIR | awk '/^.*'$PATTERN'.*[[:digit:]+][[:space:]+]/ { print $5 }'`" - [ $USAGE -ne 0 ] && quota_log $LOCAL_UG $LOCAL_ID "System is not clean for file ($LOCAL_UG:$LOCAL_ID:$USAGE)." + if [ -z $USAGE ]; then + quota_error $LOCAL_UG $LOCAL_ID "System is error when query quota for file ($LOCAL_UG:$LOCAL_ID)." + else + [ $USAGE -ne 0 ] && quota_log $LOCAL_UG $LOCAL_ID "System is not clean for file ($LOCAL_UG:$LOCAL_ID:$USAGE)." + fi fi } @@ -1397,15 +1428,18 @@ test_18() { sleep 1 done log "(dd_pid=$DDPID, time=$count, timeout=$timeout)" + sync + cancel_lru_locks mdc + cancel_lru_locks osc testfile_size=$(stat -c %s $TESTFILE) [ $testfile_size -ne $((BLK_SZ * 1024 * 100)) ] && \ quota_error u $TSTUSR "expect $((BLK_SZ * 1024 * 100)), got ${testfile_size}. Verifying file failed!" - rm -f $TESTFILE - sync; sleep 3; sync; + $SHOW_QUOTA_USER + rm -f $TESTFILE + sync resetquota -u $TSTUSR - set_blk_unitsz $((128 * 1024)) set_blk_tunesz $((128 * 1024 / 2)) } @@ -1455,12 +1489,10 @@ test_18a() { log "(dd_pid=$DDPID, time=$count, timeout=$timeout)" lustre_fail mds 0 - rm -f $TESTFILE - sync; sleep 3; sync; + sync resetquota -u $TSTUSR - set_blk_unitsz $((128 * 1024)) set_blk_tunesz $((128 * 1024 / 2)) } @@ -1525,15 +1557,20 @@ test_18bc_sub() { sleep 1 done log "(dd_pid=$DDPID, time=$count, timeout=$timeout)" - sync; sleep 1; sync + sync + cancel_lru_locks mdc + cancel_lru_locks osc testfile_size=$(stat -c %s $TESTFILE) [ $testfile_size -ne $((BLK_SZ * 1024 * 100)) ] && \ quota_error u $TSTUSR "expect $((BLK_SZ * 1024 * 100)), got ${testfile_size}. Verifying file failed!" $SHOW_QUOTA_USER - resetquota -u $TSTUSR - rm -rf $TESTFILE - sync; sleep 1; sync + rm -f $TESTFILE + sync + + resetquota -u $TSTUSR + set_blk_unitsz $((128 * 1024)) + set_blk_tunesz $((128 * 1024 / 2)) } # test when mds does failover, the ost still could work well @@ -1729,21 +1766,6 @@ test_22() { } run_test_with_stat 22 "test if quota_type saved as permanent parameter ====" -# It is triggered when test_23 failed, diagnostic for bug 18293 -test_23_dumppage() -{ - NUM=$1 - DUMPPAGE=`find /proc/fs/${FSNAME}/llite/ -name dump_page_cache` - qtime=`date +%s` - cat $DUMPPAGE > $TMP/sanity-quota_test_23_${qtime}_${NUM}.log - fsize=`stat -c%s $TMP/sanity-quota_test_23_${qtime}_${NUM}.log` - if [ $fsize -eq 0 ]; then - rm -f $TMP/sanity-quota_test_23_${qtime}_${NUM}.log - else - error "some IO error was found during directIO" - fi -} - test_23_sub() { mkdir -p $DIR/$tdir chmod 0777 $DIR/$tdir @@ -1766,15 +1788,15 @@ test_23_sub() { log " Step1: trigger quota with 0_DIRECT" log " Write half of file" $RUNAS $DIRECTIO write $TESTFILE 0 $(($LIMIT/1024/2)) $bs_unit || \ - (quota_error u $TSTUSR "(1) write failure, but expect success: $LIMIT" && test_23_dumppage 1) + quota_error u $TSTUSR "(1) write failure, but expect success: $LIMIT" log " Write out of block quota ..." $RUNAS $DIRECTIO write $TESTFILE $(($LIMIT/1024/2)) $(($LIMIT/1024/2)) $bs_unit && \ - quota_error u $TSTUSR "(2) write success, but expect EDQUOT: $LIMIT" && test_23_dumppage 2 + quota_error u $TSTUSR "(2) write success, but expect EDQUOT: $LIMIT" log " Step1: done" log " Step2: rewrite should succeed" - $RUNAS $DIRECTIO write $TESTFILE $(($LIMIT/1024/2)) 1 $bs_unit || \ - (quota_error u $TSTUSR "(3) write failure, but expect success: $LIMIT" && test_23_dumppage 3) + $RUNAS $DIRECTIO write $TESTFILE 0 1 $bs_unit || \ + quota_error u $TSTUSR "(3) write failure, but expect success: $LIMIT" log " Step2: done" rm -f $TESTFILE -- 1.8.3.1