From 1829e654506c5b798eae5ef4a91948f1411513b4 Mon Sep 17 00:00:00 2001 From: nathan Date: Wed, 11 Mar 2009 15:50:24 +0000 Subject: [PATCH] b=18676 i=johann i=manoj - fix test script (this bug) - automatically start and stop changelogs based on if there are any registered users - fix "purge to record 0 = all records" - add test for changelog_mask setting --- lustre/mdd/mdd_device.c | 89 +++++++++++++++++++++++++++++++++++------- lustre/mdd/mdd_internal.h | 1 + lustre/mdd/mdd_lproc.c | 26 +++--------- lustre/tests/recovery-small.sh | 48 +++++++++++++---------- lustre/tests/sanity.sh | 52 +++++++++++++++++------- 5 files changed, 146 insertions(+), 70 deletions(-) diff --git a/lustre/mdd/mdd_device.c b/lustre/mdd/mdd_device.c index 9de0d80..8586475 100644 --- a/lustre/mdd/mdd_device.c +++ b/lustre/mdd/mdd_device.c @@ -192,7 +192,7 @@ static int mdd_changelog_llog_init(struct mdd_device *mdd) CERROR("changelog init failed: %d\n", rc); return rc; } - CDEBUG(D_INODE, "changelog starting index="LPU64"\n", + CDEBUG(D_IOCTL, "changelog starting index="LPU64"\n", mdd->mdd_cl.mc_index); /* Find last changelog user id */ @@ -214,7 +214,12 @@ static int mdd_changelog_llog_init(struct mdd_device *mdd) CERROR("changelog user init failed: %d\n", rc); return rc; } - return 0; + + /* If we have registered users, assume we want changelogs on */ + if (mdd->mdd_cl.mc_lastuser > 0) + rc = mdd_changelog_on(mdd, 1); + + return rc; } static int mdd_changelog_init(const struct lu_env *env, struct mdd_device *mdd) @@ -244,6 +249,34 @@ static void mdd_changelog_fini(const struct lu_env *env, struct mdd_device *mdd) mdd->mdd_cl.mc_flags = 0; } +/* Start / stop recording */ +int mdd_changelog_on(struct mdd_device *mdd, int on) +{ + int rc = 0; + + if ((on == 1) && ((mdd->mdd_cl.mc_flags & CLM_ON) == 0)) { + LCONSOLE_INFO("%s: changelog on\n", mdd2obd_dev(mdd)->obd_name); + if (mdd->mdd_cl.mc_flags & CLM_ERR) { + CERROR("Changelogs cannot be enabled due to error " + "condition (see %s log).\n", + mdd2obd_dev(mdd)->obd_name); + rc = -ESRCH; + } else { + spin_lock(&mdd->mdd_cl.mc_lock); + mdd->mdd_cl.mc_flags |= CLM_ON; + spin_unlock(&mdd->mdd_cl.mc_lock); + rc = mdd_changelog_write_header(mdd, CLM_START); + } + } else if ((on == 0) && ((mdd->mdd_cl.mc_flags & CLM_ON) == CLM_ON)) { + LCONSOLE_INFO("%s: changelog off\n",mdd2obd_dev(mdd)->obd_name); + rc = mdd_changelog_write_header(mdd, CLM_FINI); + spin_lock(&mdd->mdd_cl.mc_lock); + mdd->mdd_cl.mc_flags &= ~CLM_ON; + spin_unlock(&mdd->mdd_cl.mc_lock); + } + return rc; +} + /** Add a changelog entry \a rec to the changelog llog * \param mdd * \param rec @@ -1081,9 +1114,13 @@ static int mdd_changelog_user_register(struct mdd_device *mdd, int *id) RETURN(-ENOMEM); } + /* Assume we want it on since somebody registered */ + rc = mdd_changelog_on(mdd, 1); + if (rc) + GOTO(out, rc); + rec->cur_hdr.lrh_len = sizeof(*rec); rec->cur_hdr.lrh_type = CHANGELOG_USER_REC; - rec->cur_endrec = 0ULL; spin_lock(&mdd->mdd_cl.mc_user_lock); if (mdd->mdd_cl.mc_lastuser == (unsigned int)(-1)) { spin_unlock(&mdd->mdd_cl.mc_user_lock); @@ -1091,10 +1128,12 @@ static int mdd_changelog_user_register(struct mdd_device *mdd, int *id) GOTO(out, rc = -EOVERFLOW); } *id = rec->cur_id = ++mdd->mdd_cl.mc_lastuser; + rec->cur_endrec = mdd->mdd_cl.mc_index; spin_unlock(&mdd->mdd_cl.mc_user_lock); + rc = llog_add(ctxt, &rec->cur_hdr, NULL, NULL, 0); - CDEBUG(D_INODE, "Registered changelog user %d\n", *id); + CDEBUG(D_IOCTL, "Registered changelog user %d\n", *id); out: OBD_FREE_PTR(rec); llog_ctxt_put(ctxt); @@ -1106,8 +1145,10 @@ struct mdd_changelog_user_data { __u64 mcud_minrec; /**< lowest changelog recno still referenced */ __u32 mcud_id; __u32 mcud_minid; /**< user id with lowest rec reference */ + __u32 mcud_usercount; int mcud_found:1; }; +#define MCUD_UNREGISTER -1LL /** Two things: * 1. Find the smallest record everyone is willing to purge @@ -1126,7 +1167,10 @@ static int mdd_changelog_user_purge_cb(struct llog_handle *llh, rec = (struct llog_changelog_user_rec *)hdr; - /* If we have a new endrec for this id, use it for the min check */ + mcud->mcud_usercount++; + + /* If we have a new endrec for this id, use it for the following + min check instead of its old value */ if (rec->cur_id == mcud->mcud_id) rec->cur_endrec = max(rec->cur_endrec, mcud->mcud_endrec); @@ -1142,13 +1186,15 @@ static int mdd_changelog_user_purge_cb(struct llog_handle *llh, /* Update this user's record */ mcud->mcud_found = 1; - /* Special case: unregister this user if endrec == -1 */ - if (mcud->mcud_endrec == -1) { + /* Special case: unregister this user */ + if (mcud->mcud_endrec == MCUD_UNREGISTER) { struct llog_cookie cookie; cookie.lgc_lgl = llh->lgh_id; cookie.lgc_index = hdr->lrh_index; rc = llog_cat_cancel_records(llh->u.phd.phd_cat_handle, 1, &cookie); + if (rc == 0) + mcud->mcud_usercount--; RETURN(rc); } @@ -1174,19 +1220,28 @@ static int mdd_changelog_user_purge(struct mdd_device *mdd, int id, CDEBUG(D_IOCTL, "Purge request: id=%d, endrec="LPD64"\n", id, endrec); + data.mcud_id = id; + data.mcud_minid = 0; + data.mcud_minrec = 0; + data.mcud_usercount = 0; + data.mcud_endrec = endrec; + spin_lock(&mdd->mdd_cl.mc_lock); + endrec = mdd->mdd_cl.mc_index; + spin_unlock(&mdd->mdd_cl.mc_lock); + if ((data.mcud_endrec == 0) || + ((data.mcud_endrec > endrec) && + (data.mcud_endrec != MCUD_UNREGISTER))) + data.mcud_endrec = endrec; + ctxt = llog_get_context(mdd2obd_dev(mdd),LLOG_CHANGELOG_USER_ORIG_CTXT); if (ctxt == NULL) return -ENXIO; LASSERT(ctxt->loc_handle->lgh_hdr->llh_flags & LLOG_F_IS_CAT); - data.mcud_id = id; - data.mcud_endrec = endrec; - data.mcud_minid = 0; - data.mcud_minrec = 0; rc = llog_cat_process(ctxt->loc_handle, mdd_changelog_user_purge_cb, (void *)&data, 0, 0); if ((rc >= 0) && (data.mcud_minrec > 0)) { - CDEBUG(D_INODE, "Purging CL entries up to "LPD64 + CDEBUG(D_IOCTL, "Purging changelog entries up to "LPD64 ", referenced by "CHANGELOG_USER_PREFIX"%d\n", data.mcud_minrec, data.mcud_minid); rc = mdd_changelog_llog_cancel(mdd, data.mcud_minrec); @@ -1195,6 +1250,8 @@ static int mdd_changelog_user_purge(struct mdd_device *mdd, int id, rc); } + llog_ctxt_put(ctxt); + if (!data.mcud_found) { CWARN("No entry for user %d. Last changelog reference is " LPD64" by changelog user %d\n", data.mcud_id, @@ -1202,7 +1259,10 @@ static int mdd_changelog_user_purge(struct mdd_device *mdd, int id, rc = -ENOENT; } - llog_ctxt_put(ctxt); + if (!rc && data.mcud_usercount == 0) + /* No more users; turn changelogs off */ + rc = mdd_changelog_on(mdd, 0); + RETURN (rc); } @@ -1251,7 +1311,8 @@ static int mdd_iocontrol(const struct lu_env *env, struct md_device *m, rc = mdd_changelog_user_register(mdd, &data->ioc_u32_1); break; case OBD_IOC_CHANGELOG_DEREG: - rc = mdd_changelog_user_purge(mdd, data->ioc_u32_1, -1); + rc = mdd_changelog_user_purge(mdd, data->ioc_u32_1, + MCUD_UNREGISTER); break; default: rc = -EOPNOTSUPP; diff --git a/lustre/mdd/mdd_internal.h b/lustre/mdd/mdd_internal.h index 4c6bb0e..9b657e1 100644 --- a/lustre/mdd/mdd_internal.h +++ b/lustre/mdd/mdd_internal.h @@ -448,6 +448,7 @@ int mdd_changelog_llog_write(struct mdd_device *mdd, struct thandle *handle); int mdd_changelog_llog_cancel(struct mdd_device *mdd, long long endrec); int mdd_changelog_write_header(struct mdd_device *mdd, int markerflags); +int mdd_changelog_on(struct mdd_device *mdd, int on); /* mdd_permission.c */ #define mdd_cap_t(x) (x) diff --git a/lustre/mdd/mdd_lproc.c b/lustre/mdd/mdd_lproc.c index 6e043bf..903a675 100644 --- a/lustre/mdd/mdd_lproc.c +++ b/lustre/mdd/mdd_lproc.c @@ -288,27 +288,11 @@ static int mdd_changelog_write(struct file *file, const char *buffer, if (kernbuf[count - 1] == '\n') kernbuf[count - 1] = '\0'; + /* Forced on/off/purge rec, independent of changelog users! */ if (strcmp(kernbuf, "on") == 0) { - LCONSOLE_INFO("changelog on\n"); - if (mdd->mdd_cl.mc_flags & CLM_ERR) { - CERROR("Changelogs cannot be enabled due to error " - "condition.\n"); - } else { - spin_lock(&mdd->mdd_cl.mc_lock); - mdd->mdd_cl.mc_flags |= CLM_ON; - spin_unlock(&mdd->mdd_cl.mc_lock); - rc = mdd_changelog_write_header(mdd, CLM_START); - if (rc) - return rc; - } + rc = mdd_changelog_on(mdd, 1); } else if (strcmp(kernbuf, "off") == 0) { - LCONSOLE_INFO("changelog off\n"); - rc = mdd_changelog_write_header(mdd, CLM_FINI); - if (rc) - return rc; - spin_lock(&mdd->mdd_cl.mc_lock); - mdd->mdd_cl.mc_flags &= ~CLM_ON; - spin_unlock(&mdd->mdd_cl.mc_lock); + rc = mdd_changelog_on(mdd, 0); } else { /* purge to an index */ long long unsigned endrec; @@ -320,10 +304,10 @@ static int mdd_changelog_write(struct file *file, const char *buffer, LCONSOLE_INFO("changelog purge to %llu\n", endrec); rc = mdd_changelog_llog_cancel(mdd, endrec); - if (rc < 0) - return rc; } + if (rc < 0) + return rc; return count; out_usage: diff --git a/lustre/tests/recovery-small.sh b/lustre/tests/recovery-small.sh index 943b41d..dc9088d 100755 --- a/lustre/tests/recovery-small.sh +++ b/lustre/tests/recovery-small.sh @@ -983,6 +983,7 @@ test_59() { # bug 10589 run_test 59 "Read cancel race on client eviction" err17935 () { + # we assume that all md changes are in the MDT0 changelog if [ $MDSCOUNT -gt 1 ]; then error_ignore 17935 $* else @@ -991,17 +992,17 @@ err17935 () { } test_60() { - remote_mds && { skip "remote MDS" && return 0; } + MDT0=$($LCTL get_param -n mdc.*.mds_server_uuid | \ + awk '{gsub(/_UUID/,""); print $1}' | head -1) NUM_FILES=15000 mkdir -p $DIR/$tdir - # Enable and clear changelog - $LCTL conf_param ${mds1_svc}.mdd.changelog=on - $LCTL set_param -n mdd.*.changelog on - $LFS changelog_clear $FSNAME 0 + # Register (and start) changelog + USER=$(do_facet $SINGLEMDS lctl --device $MDT0 changelog_register -n) + echo "Registered as $MDT0 changelog user $USER" - # Create NUM_FILES in the background + # Generate a large number of changelog entries createmany -o $DIR/$tdir/$tfile $NUM_FILES sync sleep 5 @@ -1011,27 +1012,34 @@ test_60() { CLIENT_PID=$! sleep 1 - # Failover the MDS while creates are happening + # Failover the MDS while unlinks are happening facet_failover $SINGLEMDS # Wait for unlinkmany to finish wait $CLIENT_PID - # Check if NUM_FILES create/unlink events were recorded + # Check if all the create/unlink events were recorded # in the changelog - $LFS changelog $FSNAME >> $DIR/$tdir/changelog + $LFS changelog $MDT0 >> $DIR/$tdir/changelog local cl_count=$(grep UNLNK $DIR/$tdir/changelog | wc -l) - echo "$cl_count unlinks in changelog" - - [ $cl_count -eq $NUM_FILES ] || err17935 "Recorded ${cl_count} unlinks out -of $NUM_FILES" - - # Also make sure we can clear large changelogs - lctl set_param -n mdd.*.changelog off - $LFS changelog_clear $FSNAME 0 - - cl_count=$($LFS changelog $FSNAME | wc -l) - [ $cl_count -eq 1 ] || error "Changelog not empty: $cl_count entries" + echo "$cl_count unlinks in $MDT0 changelog" + + do_facet $SINGLEMDS lctl --device $MDT0 changelog_deregister $USER + USERS=$(( $(do_facet $SINGLEMDS lctl get_param -n \ + mdd.$MDT0.changelog_users | wc -l) - 2 )) + if [ $USERS -eq 0 ]; then + [ $cl_count -eq $NUM_FILES ] || \ + err17935 "Recorded ${cl_count} unlinks out of $NUM_FILES" + # Also make sure we can clear large changelogs + cl_count=$($LFS changelog $FSNAME | wc -l) + [ $cl_count -le 2 ] || \ + error "Changelog not empty: $cl_count entries" + else + # If there are other users, there may be other unlinks in the log + [ $cl_count -ge $NUM_FILES ] || \ + err17935 "Recorded ${cl_count} unlinks out of $NUM_FILES" + echo "$USERS other changelog users; can't verify clear" + fi } run_test 60 "Add Changelog entries during MDS failover" diff --git a/lustre/tests/sanity.sh b/lustre/tests/sanity.sh index 3220383..f715a71 100644 --- a/lustre/tests/sanity.sh +++ b/lustre/tests/sanity.sh @@ -6003,7 +6003,6 @@ err17935 () { fi } test_160() { - do_facet $SINGLEMDS lctl set_param mdd.$MDT0.changelog on USER=$(do_facet $SINGLEMDS lctl --device $MDT0 changelog_register -n) echo "Registered as changelog user $USER" do_facet $SINGLEMDS lctl get_param -n mdd.$MDT0.changelog_users | \ @@ -6018,8 +6017,17 @@ test_160() { ln -s $DIR/$tdir/pics/2008/portland.jpg $DIR/$tdir/pics/desktop.jpg rm $DIR/$tdir/pics/desktop.jpg - # verify contents $LFS changelog $MDT0 | tail -5 + + echo "verifying changelog mask" + $LCTL set_param mdd.$MDT0.changelog_mask="-mkdir" + mkdir -p $DIR/$tdir/pics/2009/sofia + $LCTL set_param mdd.$MDT0.changelog_mask="+mkdir" + mkdir $DIR/$tdir/pics/2009/zachary + DIRS=$($LFS changelog $MDT0 | tail -5 | grep -c MKDIR) + [ $DIRS -eq 1 ] || err17935 "changelog mask count $DIRS != 1" + + # verify contents echo "verifying target fid" fidc=$($LFS changelog $MDT0 | grep timestamp | grep "CREAT" | \ tail -1 | awk '{print $5}') @@ -6033,26 +6041,40 @@ test_160() { [ "$fidc" == "p=$fidf" ] || \ err17935 "pfid in changelog $fidc != dir fid $fidf" - echo "verifying user clear" - USERS=$(( $(do_facet $SINGLEMDS lctl get_param -n \ - mdd.$MDT0.changelog_users | wc -l) - 2 )) - FIRST_REC=$($LFS changelog $MDT0 | head -1 | awk '{print $1}') - $LFS changelog_clear $MDT0 $USER $(($FIRST_REC + 5)) - USER_REC=$(do_facet $SINGLEMDS lctl get_param -n \ + USER_REC1=$(do_facet $SINGLEMDS lctl get_param -n \ mdd.$MDT0.changelog_users | grep $USER | awk '{print $2}') - [ $USER_REC == $(($FIRST_REC + 5)) ] || \ - err17935 "user index should be $(($FIRST_REC + 5)); is $USER_REC" - CLEAR_REC=$($LFS changelog $MDT0 | head -1 | awk '{print $1}') - [ $CLEAR_REC == $(($FIRST_REC + 6)) -o $USERS -gt 1 ] || \ - err17935 "first index should be $(($FIRST_REC + 6)); is $PURGE_REC" + $LFS changelog_clear $MDT0 $USER $(($USER_REC1 + 5)) + USER_REC2=$(do_facet $SINGLEMDS lctl get_param -n \ + mdd.$MDT0.changelog_users | grep $USER | awk '{print $2}') + echo "verifying user clear: $(( $USER_REC1 + 5 )) == $USER_REC2" + [ $USER_REC2 == $(($USER_REC1 + 5)) ] || \ + err17935 "user index should be $(($USER_REC1 + 5)); is $USER_REC2" + + MIN_REC=$(do_facet $SINGLEMDS lctl get_param mdd.$MDT0.changelog_users | \ + awk 'min == "" || $2 < min {min = $2}; END {print min}') + FIRST_REC=$($LFS changelog $MDT0 | head -1 | awk '{print $1}') + echo "verifying min purge: $(( $MIN_REC + 1 )) == $FIRST_REC" + [ $FIRST_REC == $(($MIN_REC + 1)) ] || \ + err17935 "first index should be $(($MIN_REC + 1)); is $FIRST_REC" echo "verifying user deregister" do_facet $SINGLEMDS lctl --device $MDT0 changelog_deregister $USER do_facet $SINGLEMDS lctl get_param -n mdd.$MDT0.changelog_users | \ grep -q $USER && error "User $USER still found in changelog_users" - [ $USERS -eq 1 ] && \ - do_facet $SINGLEMDS lctl set_param mdd.$MDT0.changelog off || true + USERS=$(( $(do_facet $SINGLEMDS lctl get_param -n \ + mdd.$MDT0.changelog_users | wc -l) - 2 )) + if [ $USERS -eq 0 ]; then + LAST_REC1=$(do_facet $SINGLEMDS lctl get_param -n \ + mdd.$MDT0.changelog_users | head -1 | awk '{print $3}') + touch $DIR/$tdir/chloe + LAST_REC2=$(do_facet $SINGLEMDS lctl get_param -n \ + mdd.$MDT0.changelog_users | head -1 | awk '{print $3}') + echo "verify changelogs are off if we were the only user: $LAST_REC1 == $LAST_REC2" + [ $LAST_REC1 == $LAST_REC2 ] || error "changelogs not off" + else + echo "$USERS other changelog users; can't verify off" + fi } run_test 160 "changelog sanity" -- 1.8.3.1