From dd4b034540d7dda499ebbb8c465d3435ad46b82a Mon Sep 17 00:00:00 2001 From: Bruno Faccini Date: Wed, 16 Mar 2016 10:10:47 +0100 Subject: [PATCH] LU-4640 mdt: implement Remove Archive on Last Unlink policy This reverts commit d9f95aa201341d972eeb610471e3c45f1ba12202 which had already reverted original commit 74d92933108dc64b110a843352cf3336dca249d0 for LU-4640, to rebase and re-integrate original patch changes. In addition, necessary changes (LU-7881) have been added to make sanity-hsm/test_26b comply with LU-7136 recent changes in sanity-hsm internal framework/functions. And also to comply with concurrent changes that had caused object to become unavailable earlier than expected in original patch version, by moving mdt_handle_last_unlink() call in mdt_mfd_close() now. Signed-off-by: Bruno Faccini Change-Id: I172d778dad494d36a6f5d03e261b2ba584fb6542 Reviewed-on: https://review.whamcloud.com/18946 Reviewed-by: Quentin Bouget Tested-by: Jenkins Tested-by: Maloo Reviewed-by: Li Xi Reviewed-by: Oleg Drokin --- lustre/mdt/mdt_coordinator.c | 41 ++++++- lustre/mdt/mdt_hsm_cdt_requests.c | 5 +- lustre/mdt/mdt_internal.h | 5 +- lustre/mdt/mdt_lib.c | 82 ++++++++++++-- lustre/mdt/mdt_open.c | 15 +-- lustre/tests/sanity-hsm.sh | 225 ++++++++++++++++++++++++++++++++++++++ 6 files changed, 349 insertions(+), 24 deletions(-) diff --git a/lustre/mdt/mdt_coordinator.c b/lustre/mdt/mdt_coordinator.c index 3e43a16..73463ea 100644 --- a/lustre/mdt/mdt_coordinator.c +++ b/lustre/mdt/mdt_coordinator.c @@ -920,6 +920,13 @@ int mdt_hsm_cdt_init(struct mdt_device *mdt) cdt->cdt_policy = CDT_DEFAULT_POLICY; cdt->cdt_active_req_timeout = 3600; + /* Initialize cdt_compound_id here to allow its usage for + * delayed requests from RAoLU policy */ + atomic_set(&cdt->cdt_compound_id, cfs_time_current_sec()); + + /* by default do not remove archives on last unlink */ + cdt->cdt_remove_archive_on_last_unlink = false; + RETURN(0); out_env: @@ -987,7 +994,6 @@ static int mdt_hsm_cdt_start(struct mdt_device *mdt) CLASSERT(1 << (CDT_POLICY_SHIFT_COUNT - 1) == CDT_POLICY_LAST); cdt->cdt_policy = CDT_DEFAULT_POLICY; - atomic_set(&cdt->cdt_compound_id, cfs_time_current_sec()); /* just need to be larger than previous one */ /* cdt_last_cookie is protected by cdt_llog_lock */ cdt->cdt_last_cookie = cfs_time_current_sec(); @@ -2228,6 +2234,36 @@ mdt_hsm_other_request_mask_seq_write(struct file *file, const char __user *buf, &cdt->cdt_other_request_mask); } +static int mdt_hsm_cdt_raolu_seq_show(struct seq_file *m, void *data) +{ + struct mdt_device *mdt = m->private; + struct coordinator *cdt = &mdt->mdt_coordinator; + ENTRY; + + seq_printf(m, "%d\n", (int)cdt->cdt_remove_archive_on_last_unlink); + RETURN(0); +} + +static ssize_t +mdt_hsm_cdt_raolu_seq_write(struct file *file, const char __user *buffer, + size_t count, loff_t *off) + +{ + struct seq_file *m = file->private_data; + struct mdt_device *mdt = m->private; + struct coordinator *cdt = &mdt->mdt_coordinator; + __s64 val; + int rc; + ENTRY; + + rc = lprocfs_str_to_s64(buffer, count, &val); + if (rc < 0) + RETURN(rc); + + cdt->cdt_remove_archive_on_last_unlink = val; + RETURN(count); +} + LPROC_SEQ_FOPS(mdt_hsm_cdt_loop_period); LPROC_SEQ_FOPS(mdt_hsm_cdt_grace_delay); LPROC_SEQ_FOPS(mdt_hsm_cdt_active_req_timeout); @@ -2236,6 +2272,7 @@ LPROC_SEQ_FOPS(mdt_hsm_cdt_default_archive_id); LPROC_SEQ_FOPS(mdt_hsm_user_request_mask); LPROC_SEQ_FOPS(mdt_hsm_group_request_mask); LPROC_SEQ_FOPS(mdt_hsm_other_request_mask); +LPROC_SEQ_FOPS(mdt_hsm_cdt_raolu); static struct lprocfs_vars lprocfs_mdt_hsm_vars[] = { { .name = "agents", @@ -2263,5 +2300,7 @@ static struct lprocfs_vars lprocfs_mdt_hsm_vars[] = { .fops = &mdt_hsm_group_request_mask_fops, }, { .name = "other_request_mask", .fops = &mdt_hsm_other_request_mask_fops, }, + { .name = "remove_archive_on_last_unlink", + .fops = &mdt_hsm_cdt_raolu_fops, }, { 0 } }; diff --git a/lustre/mdt/mdt_hsm_cdt_requests.c b/lustre/mdt/mdt_hsm_cdt_requests.c index e13fdaf..8b6dec7 100644 --- a/lustre/mdt/mdt_hsm_cdt_requests.c +++ b/lustre/mdt/mdt_hsm_cdt_requests.c @@ -458,8 +458,9 @@ struct cdt_agent_req *mdt_cdt_update_request(struct coordinator *cdt, car->car_req_update = cfs_time_current_sec(); - /* update progress done by copy tool */ - if (pgs->hpk_errval == 0 && pgs->hpk_extent.length != 0) { + /* update data move progress done by copy tool */ + if (car->car_hai->hai_action != HSMA_REMOVE && pgs->hpk_errval == 0 && + pgs->hpk_extent.length != 0) { rc = hsm_update_work(&car->car_progress, &pgs->hpk_extent); if (rc) { mdt_cdt_put_request(car); diff --git a/lustre/mdt/mdt_internal.h b/lustre/mdt/mdt_internal.h index 5810af8..20273e3 100644 --- a/lustre/mdt/mdt_internal.h +++ b/lustre/mdt/mdt_internal.h @@ -176,6 +176,9 @@ struct coordinator { __u64 cdt_user_request_mask; __u64 cdt_group_request_mask; __u64 cdt_other_request_mask; + + /* Remove archive on last unlink policy */ + bool cdt_remove_archive_on_last_unlink; }; /* mdt state flag bits */ @@ -735,7 +738,7 @@ int mdt_add_dirty_flag(struct mdt_thread_info *info, struct mdt_object *mo, struct md_attr *ma); int mdt_fix_reply(struct mdt_thread_info *info); int mdt_handle_last_unlink(struct mdt_thread_info *, struct mdt_object *, - const struct md_attr *); + struct md_attr *); void mdt_reconstruct_open(struct mdt_thread_info *, struct mdt_lock_handle *); struct lu_buf *mdt_buf(const struct lu_env *env, void *area, ssize_t len); diff --git a/lustre/mdt/mdt_lib.c b/lustre/mdt/mdt_lib.c index d03b0ba..bdad584 100644 --- a/lustre/mdt/mdt_lib.c +++ b/lustre/mdt/mdt_lib.c @@ -779,25 +779,89 @@ int mdt_fix_reply(struct mdt_thread_info *info) /* if object is dying, pack the lov/llog data, - * parameter info->mti_attr should be valid at this point! */ + * parameter info->mti_attr should be valid at this point! + * Also implements RAoLU policy */ int mdt_handle_last_unlink(struct mdt_thread_info *info, struct mdt_object *mo, - const struct md_attr *ma) + struct md_attr *ma) { - struct mdt_body *repbody; + struct mdt_body *repbody = NULL; const struct lu_attr *la = &ma->ma_attr; + struct coordinator *cdt = &info->mti_mdt->mdt_coordinator; + int rc; + __u64 need = 0; + struct hsm_action_item hai = { + .hai_len = sizeof(hai), + .hai_action = HSMA_REMOVE, + .hai_extent.length = -1, + .hai_cookie = 0, + .hai_gid = 0, + }; + __u64 compound_id; + int archive_id; + ENTRY; - repbody = req_capsule_server_get(info->mti_pill, &RMF_MDT_BODY); - LASSERT(repbody != NULL); + if (mdt_info_req(info) != NULL) { + repbody = req_capsule_server_get(info->mti_pill, &RMF_MDT_BODY); + LASSERT(repbody != NULL); + } else { + CDEBUG(D_INFO, "not running in a request/reply context\n"); + } - if (ma->ma_valid & MA_INODE) + if ((ma->ma_valid & MA_INODE) && repbody != NULL) mdt_pack_attr2body(info, repbody, la, mdt_object_fid(mo)); - if (ma->ma_valid & MA_LOV) { + if (ma->ma_valid & MA_LOV) { CERROR("No need in LOV EA upon unlink\n"); dump_stack(); - } - repbody->mbo_eadatasize = 0; + } + if (repbody != NULL) + repbody->mbo_eadatasize = 0; + + /* Only check unlinked and archived if RAoLU and upon last close */ + if (!cdt->cdt_remove_archive_on_last_unlink || + atomic_read(&mo->mot_open_count) != 0) + RETURN(0); + + need |= (MA_INODE | MA_HSM) & ~ma->ma_valid; + if (need != 0) { + /* ma->ma_valid is missing either MA_INODE, MA_HSM, or both, + * try setting them */ + ma->ma_need |= need; + rc = mdt_attr_get_complex(info, mo, ma); + if (rc) { + CERROR("%s: unable to fetch missing attributes of" + DFID": rc=%d\n", mdt_obd_name(info->mti_mdt), + PFID(mdt_object_fid(mo)), rc); + RETURN(0); + } + } + /* Assume ma->ma_valid & MA_INODE is true, cf. mdt_attr_get_complex() */ + if (ma->ma_attr.la_nlink != 0) + RETURN(0); + + /* ma->ma_valid & MA_HSM may still be false */ + if (!(ma->ma_valid & MA_HSM && ma->ma_hsm.mh_flags & HS_EXISTS)) + RETURN(0); + + /* RAoLU policy is active, last close on file has occured, + * file is unlinked, file is archived, so create remove request + * for copytool! + * If CDT is not running, requests will be logged for later. */ + compound_id = atomic_inc_return(&cdt->cdt_compound_id); + if (ma->ma_hsm.mh_arch_id != 0) + archive_id = ma->ma_hsm.mh_arch_id; + else + archive_id = cdt->cdt_default_archive_id; + + hai.hai_fid = *mdt_object_fid(mo); + + rc = mdt_agent_record_add(info->mti_env, info->mti_mdt, + compound_id, archive_id, 0, &hai); + if (rc) + CERROR("%s: unable to add HSM remove request for "DFID + ": rc=%d\n", mdt_obd_name(info->mti_mdt), + PFID(mdt_object_fid(mo)), rc); RETURN(0); } diff --git a/lustre/mdt/mdt_open.c b/lustre/mdt/mdt_open.c index 3eff74b..9469b97 100644 --- a/lustre/mdt/mdt_open.c +++ b/lustre/mdt/mdt_open.c @@ -2073,6 +2073,10 @@ int mdt_mfd_close(struct mdt_thread_info *info, struct mdt_file_data *mfd) ma->ma_need |= MA_INODE; ma->ma_valid &= ~MA_INODE; + LASSERT(atomic_read(&o->mot_open_count) > 0); + atomic_dec(&o->mot_open_count); + mdt_handle_last_unlink(info, o, ma); + if (!MFD_CLOSED(mode)) rc = mo_close(info->mti_env, next, ma, mode); @@ -2082,8 +2086,6 @@ int mdt_mfd_close(struct mdt_thread_info *info, struct mdt_file_data *mfd) atomic_dec(&o->mot_lease_count); } - LASSERT(atomic_read(&o->mot_open_count) > 0); - atomic_dec(&o->mot_open_count); mdt_mfd_free(mfd); mdt_object_put(info->mti_env, o); @@ -2095,8 +2097,6 @@ int mdt_close_internal(struct mdt_thread_info *info, struct ptlrpc_request *req, { struct mdt_export_data *med; struct mdt_file_data *mfd; - struct mdt_object *o; - struct md_attr *ma = &info->mti_attr; int ret = 0; int rc = 0; ENTRY; @@ -2115,14 +2115,7 @@ int mdt_close_internal(struct mdt_thread_info *info, struct ptlrpc_request *req, class_handle_unhash(&mfd->mfd_handle); list_del_init(&mfd->mfd_list); spin_unlock(&med->med_open_lock); - - /* Do not lose object before last unlink. */ - o = mfd->mfd_object; - mdt_object_get(info->mti_env, o); ret = mdt_mfd_close(info, mfd); - if (repbody != NULL) - rc = mdt_handle_last_unlink(info, o, ma); - mdt_object_put(info->mti_env, o); } RETURN(rc ? rc : ret); diff --git a/lustre/tests/sanity-hsm.sh b/lustre/tests/sanity-hsm.sh index a1cdb1b..02b5175 100755 --- a/lustre/tests/sanity-hsm.sh +++ b/lustre/tests/sanity-hsm.sh @@ -738,6 +738,15 @@ get_request_count() { "awk -vn=0 '/'$fid'.*action='$request'/ {n++}; END {print n}'" } +# Ensure the number of HSM request for a given FID is correct +# assert_request_count FID REQUEST_TYPE COUNT [ERROR_MSG] +assert_request_count() { + local request_count=$(get_request_count $1 $2) + local default_error_msg=("expected $3 '$2' request(s) for '$1', found " + "'$request_count'") + [ $request_count -eq $3 ] || error "${4:-"${default_error_msg[@]}"}" +} + wait_all_done() { local timeout=$1 local fid=$2 @@ -2340,6 +2349,222 @@ test_26() { } run_test 26 "Remove the archive of a valid file" +cleanup_test_26a() { + trap 0 + set_hsm_param remove_archive_on_last_unlink 0 + set_hsm_param loop_period $orig_loop_period + set_hsm_param grace_delay $orig_grace_delay + copytool_cleanup +} + +test_26a() { + local raolu=$(get_hsm_param remove_archive_on_last_unlink) + [[ $raolu -eq 0 ]] || error "RAoLU policy should be off" + + # test needs a running copytool + copytool_setup + + mkdir -p $DIR/$tdir + local f=$DIR/$tdir/$tfile + local fid=$(copy_file /etc/passwd $f) + + $LFS hsm_archive --archive $HSM_ARCHIVE_NUMBER $f + wait_request_state $fid ARCHIVE SUCCEED + + local f2=$DIR/$tdir/${tfile}_2 + local fid2=$(copy_file /etc/passwd $f2) + + $LFS hsm_archive --archive $HSM_ARCHIVE_NUMBER $f2 + wait_request_state $fid2 ARCHIVE SUCCEED + + local f3=$DIR/$tdir/${tfile}_3 + local fid3=$(copy_file /etc/passwd $f3) + + $LFS hsm_archive --archive $HSM_ARCHIVE_NUMBER $f3 + wait_request_state $fid3 ARCHIVE SUCCEED + + trap cleanup_test_26a EXIT + + # set a long grace_delay vs short loop_period + local orig_loop_period=$(get_hsm_param loop_period) + local orig_grace_delay=$(get_hsm_param grace_delay) + set_hsm_param loop_period 10 + set_hsm_param grace_delay 100 + + rm -f $f + + set_hsm_param remove_archive_on_last_unlink 1 + + ln "$f3" "$f3"_bis || error "Unable to create hard-link" + rm -f $f3 + + rm -f $f2 + + set_hsm_param remove_archive_on_last_unlink 0 + + wait_request_state $fid2 REMOVE SUCCEED + + assert_request_count $fid REMOVE 0 \ + "Unexpected archived data remove request for $f" + assert_request_count $fid3 REMOVE 0 \ + "Unexpected archived data remove request for $f3" + + cleanup_test_26a +} +run_test 26a "Remove Archive On Last Unlink (RAoLU) policy" + +cleanup_test_26b() { + trap 0 + set_hsm_param remove_archive_on_last_unlink 0 + copytool_cleanup +} + +test_26b() { + + # test needs a running copytool + copytool_setup + + mkdir -p $DIR/$tdir + local f=$DIR/$tdir/$tfile + local fid=$(copy_file /etc/passwd $f) + + $LFS hsm_archive --archive $HSM_ARCHIVE_NUMBER $f + wait_request_state $fid ARCHIVE SUCCEED + + trap cleanup_test_26b EXIT + + set_hsm_param remove_archive_on_last_unlink 1 + + cdt_shutdown + cdt_check_state stopped + + rm -f $f + + set_hsm_param remove_archive_on_last_unlink 0 + + wait_request_state $fid REMOVE WAITING + + cdt_enable + # copytool must re-register + kill_copytools + wait_copytools || error "copytool failed to stop" + HSM_ARCHIVE_PURGE=false copytool_setup + + wait_request_state $fid REMOVE SUCCEED + + cleanup_test_26b +} +run_test 26b "RAoLU policy when CDT off" + +cleanup_test_26c() { + trap 0 + set_hsm_param remove_archive_on_last_unlink 0 + set_hsm_param loop_period $orig_loop_period + set_hsm_param grace_delay $orig_grace_delay + copytool_cleanup +} + +test_26c() { + + # test needs a running copytool + copytool_setup + + mkdir -p $DIR/$tdir + local f=$DIR/$tdir/$tfile + local fid=$(copy_file /etc/passwd $f) + + $LFS hsm_archive --archive $HSM_ARCHIVE_NUMBER $f + wait_request_state $fid ARCHIVE SUCCEED + + local f2=$DIR/$tdir/${tfile}_2 + local fid2=$(copy_file /etc/passwd $f2) + + $LFS hsm_archive --archive $HSM_ARCHIVE_NUMBER $f2 + wait_request_state $fid2 ARCHIVE SUCCEED + + trap cleanup_test_26c EXIT + + # set a long grace_delay vs short loop_period + local orig_loop_period=$(get_hsm_param loop_period) + local orig_grace_delay=$(get_hsm_param grace_delay) + set_hsm_param loop_period 10 + set_hsm_param grace_delay 100 + + set_hsm_param remove_archive_on_last_unlink 1 + + multiop_bg_pause $f O_c || error "open $f failed" + local pid=$! + + rm -f $f + rm -f $f2 + + wait_request_state $fid2 REMOVE SUCCEED + assert_request_count $fid REMOVE 0 \ + "Unexpected archived data remove request for $f" + + kill -USR1 $pid || error "multiop early exit" + # should reach autotest timeout if multiop fails to trap + # signal, close file, and exit ... + wait $pid || error + + set_hsm_param remove_archive_on_last_unlink 0 + + wait_request_state $fid REMOVE SUCCEED + + cleanup_test_26c +} +run_test 26c "RAoLU effective when file closed" + +cleanup_test_26d() { + trap 0 + set_hsm_param remove_archive_on_last_unlink 0 + set_hsm_param loop_period $orig_loop_period + set_hsm_param grace_delay $orig_grace_delay + copytool_cleanup +} + +test_26d() { + + # test needs a running copytool + copytool_setup + + mkdir -p $DIR/$tdir + local f=$DIR/$tdir/$tfile + local fid=$(copy_file /etc/motd $f 1) + + $LFS hsm_archive $f || error "could not archive file" + wait_request_state $fid ARCHIVE SUCCEED + + trap cleanup_test_26d EXIT + + # set a long grace_delay vs short loop_period + local orig_loop_period=$(get_hsm_param loop_period) + local orig_grace_delay=$(get_hsm_param grace_delay) + set_hsm_param loop_period 10 + set_hsm_param grace_delay 100 + + set_hsm_param remove_archive_on_last_unlink 1 + + multiop_bg_pause $f O_c || error "multiop failed" + local MULTIPID=$! + + rm -f $f + + mds_evict_client + + set_hsm_param remove_archive_on_last_unlink 0 + + wait_request_state $fid REMOVE SUCCEED + + client_up || client_up || true + + kill -USR1 $MULTIPID + wait $MULTIPID || error "multiop close failed" + + cleanup_test_26d +} +run_test 26d "RAoLU when Client eviction" + test_27a() { # test needs a running copytool copytool_setup -- 1.8.3.1