From 1e382adf2e1220f309811a7bfd05725de39d6b43 Mon Sep 17 00:00:00 2001 From: yury Date: Thu, 12 Oct 2006 14:11:48 +0000 Subject: [PATCH] - zero out used locks in MDT before finishing req handling; - fixes in seq-mgr recovery (meta and super); - cleanups, more debug and comments, fixes in comments. --- lustre/fid/fid_handler.c | 54 ++++++++++++++++++++++---------------- lustre/fld/fld_handler.c | 5 ++++ lustre/include/lustre/lustre_idl.h | 5 ++-- lustre/include/obd.h | 2 +- lustre/mdt/mdt_handler.c | 16 ++++++----- lustre/obdclass/obd_config.c | 20 +++++++++----- lustre/tests/replay-single.sh | 18 +++++++++++++ 7 files changed, 81 insertions(+), 39 deletions(-) diff --git a/lustre/fid/fid_handler.c b/lustre/fid/fid_handler.c index 703c47b..c257a9d 100644 --- a/lustre/fid/fid_handler.c +++ b/lustre/fid/fid_handler.c @@ -130,16 +130,15 @@ static int __seq_server_alloc_super(struct lu_server_seq *seq, LASSERT(range_is_sane(space)); if (in != NULL) { - CDEBUG(D_INFO|D_WARNING, "%s: Recovery started. Use input " - "(last allocated) range "DRANGE"\n", seq->lss_name, - PRANGE(in)); + CDEBUG(D_INFO|D_WARNING, "%s: Input seq range: " + DRANGE"\n", seq->lss_name, PRANGE(in)); - if (in->lr_start > space->lr_start) - space->lr_start = in->lr_start; + if (in->lr_end > space->lr_start) + space->lr_start = in->lr_end; *out = *in; - CDEBUG(D_INFO|D_WARNING, "%s: Recovery finished. Recovered " - "space: "DRANGE"\n", seq->lss_name, PRANGE(space)); + CDEBUG(D_INFO|D_WARNING, "%s: Recovered space: "DRANGE"\n", + seq->lss_name, PRANGE(space)); } else { if (range_space(space) < seq->lss_width) { CWARN("%s: Sequences space to be exhausted soon. " @@ -158,7 +157,8 @@ static int __seq_server_alloc_super(struct lu_server_seq *seq, rc = seq_store_write(seq, env); if (rc) { - CERROR("%s: Can't save state, rc %d\n", seq->lss_name, rc); + CERROR("%s: Can't save state, rc %d\n", + seq->lss_name, rc); RETURN(rc); } @@ -199,35 +199,44 @@ static int __seq_server_alloc_meta(struct lu_server_seq *seq, * it is allocated from new super. */ if (in != NULL) { - CDEBUG(D_INFO|D_WARNING, "%s: Recovery started. Use input " - "(last allocated) range "DRANGE"\n", seq->lss_name, - PRANGE(in)); + CDEBUG(D_INFO|D_WARNING, "%s: Input seq range: " + DRANGE"\n", seq->lss_name, PRANGE(in)); if (range_is_exhausted(space)) { - LASSERT(in->lr_start > space->lr_start); - /* * Server cannot send empty range to client, this is why * we check here that range from client is "newer" than * exhausted super. */ - space->lr_start = in->lr_start; + LASSERT(in->lr_end > space->lr_start); + + /* + * Start is set to end of last allocated, because it + * *is* already allocated so we take that into account + * and do not use for other allocations. + */ + space->lr_start = in->lr_end; - space->lr_end = space->lr_start + - seq->lss_width; + /* + * End is set to in->lr_start + super sequence + * allocation unit. That is because in->lr_start is + * first seq in new allocated range from controller + * before failure. + */ + space->lr_end = in->lr_start + LUSTRE_SEQ_SUPER_WIDTH; } else { /* - * Update super start by start from client's range. End - * should not be changed if range was not exhausted. + * Update super start by end from client's range. Super + * end should not be changed if range was not exhausted. */ - if (in->lr_start > space->lr_start) - space->lr_start = in->lr_start; + if (in->lr_end > space->lr_start) + space->lr_start = in->lr_end; } *out = *in; - CDEBUG(D_INFO|D_WARNING, "%s: Recovery finished. Recovered " - "super: "DRANGE"\n", seq->lss_name, PRANGE(space)); + CDEBUG(D_INFO|D_WARNING, "%s: Recovered space: "DRANGE"\n", + seq->lss_name, PRANGE(space)); } else { /* * XXX: Avoid cascading RPCs using kind of async preallocation @@ -251,6 +260,7 @@ static int __seq_server_alloc_meta(struct lu_server_seq *seq, *space = seq->lss_cli->lcs_space; LASSERT(range_is_sane(space)); } + range_alloc(out, space, seq->lss_width); } diff --git a/lustre/fld/fld_handler.c b/lustre/fld/fld_handler.c index 738289e..375982c 100644 --- a/lustre/fld/fld_handler.c +++ b/lustre/fld/fld_handler.c @@ -164,6 +164,11 @@ static int fld_server_handle(struct lu_server_fld *fld, rc = -EINVAL; break; } + if (rc) { + CERROR("%s: FLD req (opc: %d, seq: "LPX64", mds: " + LPU64") handle error %d\n", fld->lsf_name, + opc, mf->mf_seq, mf->mf_mds, rc); + } RETURN(rc); } diff --git a/lustre/include/lustre/lustre_idl.h b/lustre/include/lustre/lustre_idl.h index 84fc13b..e634970 100644 --- a/lustre/include/lustre/lustre_idl.h +++ b/lustre/include/lustre/lustre_idl.h @@ -182,7 +182,7 @@ static inline int range_is_exhausted(struct lu_range *r) return range_space(r) == 0; } -#define DRANGE "[%16.16"LPF64"x-%16.16"LPF64"x]" +#define DRANGE "[0x%16.16"LPF64"x-0x%16.16"LPF64"x]" #define PRANGE(range) \ (range)->lr_start, \ @@ -201,7 +201,6 @@ struct lu_fid { */ enum { LUSTRE_ROOT_FID_SEQ = 1ULL, /* XXX: should go into mkfs. */ - LUSTRE_ROOT_FID_OID = 2UL, /* XXX: should go into mkfs. */ /* initial fid id value */ LUSTRE_FID_INIT_OID = 1UL @@ -242,7 +241,7 @@ static inline int fid_is_sane(const struct lu_fid *fid) fid_seq_is_sane(fid_seq(fid)) && fid_oid(fid) != 0; } -#define DFID "[%16.16"LPF64"x/%8.8x:%8.8x]" +#define DFID "[0x%16.16"LPF64"x/0x%8.8x:0x%8.8x]" #define PFID(fid) \ fid_seq(fid), \ diff --git a/lustre/include/obd.h b/lustre/include/obd.h index 84c9452..b3f01af 100644 --- a/lustre/include/obd.h +++ b/lustre/include/obd.h @@ -879,7 +879,7 @@ struct obd_device { obd_force:1, /* cleanup with > 0 obd refcount */ obd_fail:1, /* cleanup with failover */ obd_async_recov:1, /* allow asyncronous orphan cleanup */ - obd_configured:1; /* config llog parsed completely */ + obd_configured:1; /* llog configuration finished successfully */ atomic_t obd_refcount; cfs_waitq_t obd_refcount_waitq; cfs_proc_dir_entry_t *obd_proc_entry; diff --git a/lustre/mdt/mdt_handler.c b/lustre/mdt/mdt_handler.c index 2b57da3..dc9993c 100644 --- a/lustre/mdt/mdt_handler.c +++ b/lustre/mdt/mdt_handler.c @@ -2128,10 +2128,10 @@ int mdt_intent_lock_replace(struct mdt_thread_info *info, RETURN(ELDLM_LOCK_REPLACED); } -static void mdt_fixup_resent(struct req_capsule *pill, - struct ldlm_lock *new_lock, - struct ldlm_lock **old_lock, - struct mdt_lock_handle *lh) +static void mdt_intent_fixup_resent(struct req_capsule *pill, + struct ldlm_lock *new_lock, + struct ldlm_lock **old_lock, + struct mdt_lock_handle *lh) { struct ptlrpc_request *req = pill->rc_req; struct obd_export *exp = req->rq_export; @@ -2227,7 +2227,7 @@ static int mdt_intent_getattr(enum mdt_it_code opcode, mdt_set_disposition(info, ldlm_rep, DISP_IT_EXECD); /* Get lock from request for possible resent case. */ - mdt_fixup_resent(&info->mti_pill, *lockp, &new_lock, lhc); + mdt_intent_fixup_resent(&info->mti_pill, *lockp, &new_lock, lhc); ldlm_rep->lock_policy_res2 = mdt_getattr_name_lock(info, lhc, child_bits, ldlm_rep); @@ -2235,7 +2235,8 @@ static int mdt_intent_getattr(enum mdt_it_code opcode, if (mdt_get_disposition(ldlm_rep, DISP_LOOKUP_NEG)) ldlm_rep->lock_policy_res2 = 0; if (!mdt_get_disposition(ldlm_rep, DISP_LOOKUP_POS) || - ldlm_rep->lock_policy_res2) { + ldlm_rep->lock_policy_res2) { + lhc->mlh_lh.cookie = 0ull; GOTO(out_ucred, rc = ELDLM_LOCK_ABORTED); } @@ -2276,7 +2277,7 @@ static int mdt_intent_reint(enum mdt_it_code opcode, } /* Get lock from request for possible resent case. */ - mdt_fixup_resent(&info->mti_pill, *lockp, NULL, lhc); + mdt_intent_fixup_resent(&info->mti_pill, *lockp, NULL, lhc); rc = mdt_reint_internal(info, lhc, opc); @@ -2296,6 +2297,7 @@ static int mdt_intent_reint(enum mdt_it_code opcode, } rep->lock_policy_res2 = clear_serious(rc); + lhc->mlh_lh.cookie = 0ull; RETURN(ELDLM_LOCK_ABORTED); } diff --git a/lustre/obdclass/obd_config.c b/lustre/obdclass/obd_config.c index d76d60a..4043f65 100644 --- a/lustre/obdclass/obd_config.c +++ b/lustre/obdclass/obd_config.c @@ -977,8 +977,8 @@ static int class_config_llog_handler(struct llog_handle * handle, (lcfg->lcfg_command != LCFG_MARKER)) { CWARN("Config not inside markers, ignoring! " "(inst: %s, uuid: %s, flags: %#x)\n", - clli->cfg_instance, clli->cfg_uuid.uuid, - clli->cfg_flags); + clli->cfg_instance ? clli->cfg_instance : "", + clli->cfg_uuid.uuid, clli->cfg_flags); clli->cfg_flags |= CFG_F_SKIP; } if (clli->cfg_flags & CFG_F_SKIP) { @@ -1062,7 +1062,7 @@ out: RETURN(rc); } -void class_config_notify_end(const char *name) +static void class_config_notify_end(const char *name) { struct obd_device *obd; ENTRY; @@ -1077,7 +1077,14 @@ void class_config_notify_end(const char *name) if (obd) obd->obd_configured = 1; else { - CWARN("OBD \"%s\" is not found - skipped.\n", name); + /* + * XXX: @name is actually not OBD name, rather log name and it + * may not be the same as OBD name for clients. For servers it + * is true currently but anyway not reliable. This is why this + * it is dirty hack. + */ + CDEBUG(D_CONFIG, "OBD \"%s\" is not found - skipped.\n", + name); } EXIT; } @@ -1114,9 +1121,10 @@ int class_config_parse_llog(struct llog_ctxt *ctxt, char *name, CDEBUG(D_CONFIG, "Notify config log %s parse finish\n", name); } else { - CDEBUG(D_CONFIG, "Log %s did not parse, obd is not " - "configured.\n", name); + CDEBUG(D_CONFIG, "Log %s did not parse successfully, " + "OBD is not configured.\n", name); } + if (cfg) cfg->cfg_last_idx = cd.last_idx; diff --git a/lustre/tests/replay-single.sh b/lustre/tests/replay-single.sh index 703bf03..82e4ea0 100755 --- a/lustre/tests/replay-single.sh +++ b/lustre/tests/replay-single.sh @@ -77,6 +77,24 @@ seq_get_width() cat $file } +# This test should pass for single-mds and multi-mds configs. +# But for different configurations it tests different things. +# +# single-mds +# ---------- +# (1) fld_create replay should happen; +# +# (2) fld_create replay should not return -EEXISTS, if it does +# this means sequence manager recovery code is buggy and allocated +# same sequence two times after recovery. +# +# multi-mds +# --------- +# (1) fld_create replay may not happen, because its home MDS is +# MDS2 which is not involved to revovery; +# +# (2) as fld_create does not happen on MDS1, it does not may any +# problem. test_0c() { local label=`mdsdevlabel 1` [ -z "$label" ] && echo "No label for mds1" && return 1 -- 1.8.3.1