From 74d92933108dc64b110a843352cf3336dca249d0 Mon Sep 17 00:00:00 2001 From: Bruno Faccini Date: Thu, 9 Apr 2015 10:31:54 +0200 Subject: [PATCH] LU-4640 mdt: implement Remove Archive on Last Unlink policy This patch introduces RAoLU policy where an implicit remove request will be sent to archive/Agent upon last close of an unlinked file. Policy can be enabled/disabled using an lprocfs tunable. If CDT not running, requests will be queued. test_26[a-c] related tests have been also added in sanity-hsm. This patch also contains a small fix to prevent unnecessary progress infos to be gathered for REMOVE requests. Patch now also handles cases where unlinked file is closed from mdt_export_cleanup() after Client eviction. And specific test_26d has been added in sanity-hsm. Signed-off-by: Bruno Faccini Change-Id: I7affb20b2834bcd0618412349fc3adc7f6744de0 Reviewed-on: http://review.whamcloud.com/14384 Tested-by: Jenkins Tested-by: Maloo Reviewed-by: John L. Hammond Reviewed-by: Aurelien Degremont Reviewed-by: Oleg Drokin --- lustre/mdt/mdt_coordinator.c | 41 +++++- lustre/mdt/mdt_handler.c | 7 + lustre/mdt/mdt_hsm_cdt_requests.c | 5 +- lustre/mdt/mdt_internal.h | 5 +- lustre/mdt/mdt_lib.c | 101 +++++++++++++-- lustre/mdt/mdt_open.c | 3 +- lustre/tests/sanity-hsm.sh | 264 ++++++++++++++++++++++++++++++++++++++ 7 files changed, 412 insertions(+), 14 deletions(-) diff --git a/lustre/mdt/mdt_coordinator.c b/lustre/mdt/mdt_coordinator.c index 27ff1e3..daf8d95 100644 --- a/lustre/mdt/mdt_coordinator.c +++ b/lustre/mdt/mdt_coordinator.c @@ -915,6 +915,13 @@ int mdt_hsm_cdt_init(struct mdt_device *mdt) cdt->cdt_policy = CDT_DEFAULT_POLICY; cdt->cdt_active_req_timeout = 3600; + /* Initialize cdt_compound_id here to allow its usage for + * delayed requests from RAoLU policy */ + atomic_set(&cdt->cdt_compound_id, cfs_time_current_sec()); + + /* by default do not remove archives on last unlink */ + cdt->cdt_remove_archive_on_last_unlink = false; + RETURN(0); } @@ -966,7 +973,6 @@ int mdt_hsm_cdt_start(struct mdt_device *mdt) cdt->cdt_state = CDT_INIT; - atomic_set(&cdt->cdt_compound_id, cfs_time_current_sec()); /* just need to be larger than previous one */ /* cdt_last_cookie is protected by cdt_llog_lock */ cdt->cdt_last_cookie = cfs_time_current_sec(); @@ -2206,6 +2212,36 @@ mdt_hsm_other_request_mask_seq_write(struct file *file, const char __user *buf, &cdt->cdt_other_request_mask); } +static int mdt_hsm_cdt_raolu_seq_show(struct seq_file *m, void *data) +{ + struct mdt_device *mdt = m->private; + struct coordinator *cdt = &mdt->mdt_coordinator; + ENTRY; + + seq_printf(m, "%d\n", (int)cdt->cdt_remove_archive_on_last_unlink); + RETURN(0); +} + +static ssize_t +mdt_hsm_cdt_raolu_seq_write(struct file *file, const char __user *buffer, + size_t count, loff_t *off) + +{ + struct seq_file *m = file->private_data; + struct mdt_device *mdt = m->private; + struct coordinator *cdt = &mdt->mdt_coordinator; + int val; + int rc; + ENTRY; + + rc = lprocfs_write_helper(buffer, count, &val); + if (rc < 0) + RETURN(rc); + + cdt->cdt_remove_archive_on_last_unlink = val; + RETURN(count); +} + LPROC_SEQ_FOPS(mdt_hsm_cdt_loop_period); LPROC_SEQ_FOPS(mdt_hsm_cdt_grace_delay); LPROC_SEQ_FOPS(mdt_hsm_cdt_active_req_timeout); @@ -2214,6 +2250,7 @@ LPROC_SEQ_FOPS(mdt_hsm_cdt_default_archive_id); LPROC_SEQ_FOPS(mdt_hsm_user_request_mask); LPROC_SEQ_FOPS(mdt_hsm_group_request_mask); LPROC_SEQ_FOPS(mdt_hsm_other_request_mask); +LPROC_SEQ_FOPS(mdt_hsm_cdt_raolu); static struct lprocfs_vars lprocfs_mdt_hsm_vars[] = { { .name = "agents", @@ -2241,5 +2278,7 @@ static struct lprocfs_vars lprocfs_mdt_hsm_vars[] = { .fops = &mdt_hsm_group_request_mask_fops, }, { .name = "other_request_mask", .fops = &mdt_hsm_other_request_mask_fops, }, + { .name = "remove_archive_on_last_unlink", + .fops = &mdt_hsm_cdt_raolu_fops, }, { 0 } }; diff --git a/lustre/mdt/mdt_handler.c b/lustre/mdt/mdt_handler.c index 3f16666..82cb920 100644 --- a/lustre/mdt/mdt_handler.c +++ b/lustre/mdt/mdt_handler.c @@ -5060,6 +5060,7 @@ static int mdt_export_cleanup(struct obd_export *exp) if (!list_empty(&closing_list)) { struct md_attr *ma = &info->mti_attr; + struct mdt_object *o; /* Close any open files (which may also cause orphan * unlinking). */ @@ -5089,7 +5090,13 @@ static int mdt_export_cleanup(struct obd_export *exp) ma->ma_valid = MA_FLAGS; ma->ma_attr_flags |= MDS_KEEP_ORPHAN; } + + /* Do not lose object before last unlink. */ + o = mfd->mfd_object; + mdt_object_get(info->mti_env, o); mdt_mfd_close(info, mfd); + mdt_handle_last_unlink(info, o, ma); + mdt_object_put(info->mti_env, o); } } info->mti_mdt = NULL; diff --git a/lustre/mdt/mdt_hsm_cdt_requests.c b/lustre/mdt/mdt_hsm_cdt_requests.c index bbd7f44..2b672a6 100644 --- a/lustre/mdt/mdt_hsm_cdt_requests.c +++ b/lustre/mdt/mdt_hsm_cdt_requests.c @@ -434,8 +434,9 @@ struct cdt_agent_req *mdt_cdt_update_request(struct coordinator *cdt, car->car_req_update = cfs_time_current_sec(); - /* update progress done by copy tool */ - if (pgs->hpk_errval == 0 && pgs->hpk_extent.length != 0) { + /* update data move progress done by copy tool */ + if (car->car_hai->hai_action != HSMA_REMOVE && pgs->hpk_errval == 0 && + pgs->hpk_extent.length != 0) { rc = hsm_update_work(&car->car_progress, &pgs->hpk_extent); if (rc) { mdt_cdt_put_request(car); diff --git a/lustre/mdt/mdt_internal.h b/lustre/mdt/mdt_internal.h index d6256fd..e32d754 100644 --- a/lustre/mdt/mdt_internal.h +++ b/lustre/mdt/mdt_internal.h @@ -148,6 +148,9 @@ struct coordinator { __u64 cdt_user_request_mask; __u64 cdt_group_request_mask; __u64 cdt_other_request_mask; + + /* Remove archive on last unlink policy */ + bool cdt_remove_archive_on_last_unlink; }; /* mdt state flag bits */ @@ -700,7 +703,7 @@ int mdt_add_dirty_flag(struct mdt_thread_info *info, struct mdt_object *mo, struct md_attr *ma); int mdt_fix_reply(struct mdt_thread_info *info); int mdt_handle_last_unlink(struct mdt_thread_info *, struct mdt_object *, - const struct md_attr *); + struct md_attr *); void mdt_reconstruct_open(struct mdt_thread_info *, struct mdt_lock_handle *); struct lu_buf *mdt_buf(const struct lu_env *env, void *area, ssize_t len); diff --git a/lustre/mdt/mdt_lib.c b/lustre/mdt/mdt_lib.c index b018176..62d9b8b 100644 --- a/lustre/mdt/mdt_lib.c +++ b/lustre/mdt/mdt_lib.c @@ -809,23 +809,108 @@ int mdt_fix_reply(struct mdt_thread_info *info) /* if object is dying, pack the lov/llog data, * parameter info->mti_attr should be valid at this point! */ int mdt_handle_last_unlink(struct mdt_thread_info *info, struct mdt_object *mo, - const struct md_attr *ma) + struct md_attr *ma) { - struct mdt_body *repbody; + struct mdt_body *repbody = NULL; const struct lu_attr *la = &ma->ma_attr; + struct coordinator *cdt = &info->mti_mdt->mdt_coordinator; + int rc; + __u64 need = 0; + struct hsm_action_item hai = { + .hai_len = sizeof(hai), + .hai_action = HSMA_REMOVE, + .hai_extent.length = -1, + .hai_cookie = 0, + .hai_gid = 0, + }; + __u64 compound_id; + int archive_id; + ENTRY; - repbody = req_capsule_server_get(info->mti_pill, &RMF_MDT_BODY); - LASSERT(repbody != NULL); + if (mdt_info_req(info) != NULL) { + repbody = req_capsule_server_get(info->mti_pill, &RMF_MDT_BODY); + LASSERT(repbody != NULL); + } else { + CDEBUG(D_INFO, "not running in a request/reply context\n"); + } - if (ma->ma_valid & MA_INODE) + if ((ma->ma_valid & MA_INODE) && repbody != NULL) mdt_pack_attr2body(info, repbody, la, mdt_object_fid(mo)); - if (ma->ma_valid & MA_LOV) { + if (ma->ma_valid & MA_LOV) { CERROR("No need in LOV EA upon unlink\n"); dump_stack(); - } - repbody->mbo_eadatasize = 0; + } + if (repbody != NULL) + repbody->mbo_eadatasize = 0; + + /* Only check unlinked and archived if RAoLU and upon last close */ + if (!cdt->cdt_remove_archive_on_last_unlink || + atomic_read(&mo->mot_open_count) != 0) + RETURN(0); + + if (ma->ma_valid & MA_INODE) { + if (ma->ma_attr.la_nlink != 0) + RETURN(0); + } else { + need |= MA_INODE; + } + + if (ma->ma_valid & MA_HSM) { + if (!(ma->ma_hsm.mh_flags & HS_EXISTS)) + RETURN(0); + } else { + need |= MA_HSM; + } + + if (need != 0) { + ma->ma_need |= need; + rc = mdt_attr_get_complex(info, mo, ma); + if (rc) { + CERROR("%s: unable to fetch missing attributes of" + DFID": rc=%d\n", mdt_obd_name(info->mti_mdt), + PFID(mdt_object_fid(mo)), rc); + RETURN(0); + } + + if (need & MA_INODE) { + if (ma->ma_valid & MA_INODE) { + if (ma->ma_attr.la_nlink != 0) + RETURN(0); + } else { + RETURN(0); + } + } + + if (need & MA_HSM) { + if (ma->ma_valid & MA_HSM) { + if (!(ma->ma_hsm.mh_flags & HS_EXISTS)) + RETURN(0); + } else { + RETURN(0); + } + } + } + + /* RAoLU policy is active, last close on file has occured, + * file is unlinked, file is archived, so create remove request + * for copytool! + * If CDT is not running, requests will be logged for later. */ + compound_id = atomic_inc_return(&cdt->cdt_compound_id); + if (ma->ma_hsm.mh_arch_id != 0) + archive_id = ma->ma_hsm.mh_arch_id; + else + archive_id = cdt->cdt_default_archive_id; + + hai.hai_fid = *mdt_object_fid(mo); + + rc = mdt_agent_record_add(info->mti_env, info->mti_mdt, + compound_id, archive_id, 0, &hai); + if (rc) + CERROR("%s: unable to add HSM remove request for "DFID + ": rc=%d\n", mdt_obd_name(info->mti_mdt), + PFID(mdt_object_fid(mo)), rc); RETURN(0); } diff --git a/lustre/mdt/mdt_open.c b/lustre/mdt/mdt_open.c index 324e3c0..ce9be37 100644 --- a/lustre/mdt/mdt_open.c +++ b/lustre/mdt/mdt_open.c @@ -2032,8 +2032,7 @@ int mdt_close_internal(struct mdt_thread_info *info, struct ptlrpc_request *req, o = mfd->mfd_object; mdt_object_get(info->mti_env, o); ret = mdt_mfd_close(info, mfd); - if (repbody != NULL) - rc = mdt_handle_last_unlink(info, o, ma); + rc = mdt_handle_last_unlink(info, o, ma); mdt_object_put(info->mti_env, o); } diff --git a/lustre/tests/sanity-hsm.sh b/lustre/tests/sanity-hsm.sh index f73a502..bdabe10 100755 --- a/lustre/tests/sanity-hsm.sh +++ b/lustre/tests/sanity-hsm.sh @@ -2255,6 +2255,270 @@ test_26() { } run_test 26 "Remove the archive of a valid file" +cleanup_test_26a() { + trap 0 + set_hsm_param remove_archive_on_last_unlink 0 + set_hsm_param loop_period $orig_loop_period + set_hsm_param grace_delay $orig_grace_delay + copytool_cleanup +} + +test_26a() { + local raolu=$(get_hsm_param remove_archive_on_last_unlink) + [[ $raolu -eq 0 ]] || error "RAoLU policy should be off" + + # test needs a running copytool + copytool_setup + + mkdir -p $DIR/$tdir + local f=$DIR/$tdir/$tfile + local fid=$(copy_file /etc/passwd $f) + + $LFS hsm_archive --archive $HSM_ARCHIVE_NUMBER $f + wait_request_state $fid ARCHIVE SUCCEED + + rm -f $f + + local f2=$DIR/$tdir/${tfile}_2 + local fid2=$(copy_file /etc/passwd $f2) + + $LFS hsm_archive --archive $HSM_ARCHIVE_NUMBER $f2 + wait_request_state $fid2 ARCHIVE SUCCEED + + cat $f2 > /dev/null + + local f3=$DIR/$tdir/${tfile}_3 + local fid3=$(copy_file /etc/passwd $f3) + + $LFS hsm_archive --archive $HSM_ARCHIVE_NUMBER $f3 + wait_request_state $fid3 ARCHIVE SUCCEED + + local f4=$DIR/$tdir/${tfile}_4 + local fid4=$(copy_file /etc/passwd $f4) + + $LFS hsm_archive --archive $HSM_ARCHIVE_NUMBER $f4 + wait_request_state $fid4 ARCHIVE SUCCEED + + trap cleanup_test_26a EXIT + + # set a long grace_delay vs short loop_period + local orig_loop_period=$(get_hsm_param loop_period) + local orig_grace_delay=$(get_hsm_param grace_delay) + set_hsm_param loop_period 10 + set_hsm_param grace_delay 100 + + set_hsm_param remove_archive_on_last_unlink 1 + + rm -f $f3 + cat $f4 > /dev/null + local f4bis=$DIR/$tdir/${tfile}_4bis + ln $f4 $f4bis + [[ $? -eq 0 ]] || error "Unable to create hard-link" + rm -f $f4 + + # Since CDT is not signaled for RAoLU requests to be sure it + # will wake-up to send remove request and copytool will process + # it, wait for loop_period + some extra-time. + local loop_period=$(get_hsm_param loop_period) + sleep $((loop_period + 5)) + + set_hsm_param remove_archive_on_last_unlink 0 + + do_facet $SINGLEAGT ls $HSM_ARCHIVE'/*/*/*/*/*/*/'$fid + [[ $? -eq 0 ]] || error "File being removed on archive" + + do_facet $SINGLEAGT ls $HSM_ARCHIVE'/*/*/*/*/*/*/'$fid2 + [[ $? -eq 0 ]] || error "File being removed on archive" + + do_facet $SINGLEAGT ls $HSM_ARCHIVE'/*/*/*/*/*/*/'$fid3 + [[ $? -eq 0 ]] && error "File not being removed on archive" + + do_facet $SINGLEAGT ls $HSM_ARCHIVE'/*/*/*/*/*/*/'$fid4 + [[ $? -eq 0 ]] || error "File being removed on archive" + + # previous actions elapsed time should be < grace_delay + wait_request_state $fid3 REMOVE SUCCEED + + cleanup_test_26a +} +run_test 26a "Remove Archive On Last Unlink (RAoLU) policy" + +cleanup_test_26b() { + trap 0 + set_hsm_param remove_archive_on_last_unlink 0 + copytool_cleanup +} + +test_26b() { + + # test needs a running copytool + copytool_setup + + mkdir -p $DIR/$tdir + local f=$DIR/$tdir/$tfile + local fid=$(copy_file /etc/passwd $f) + + $LFS hsm_archive --archive $HSM_ARCHIVE_NUMBER $f + wait_request_state $fid ARCHIVE SUCCEED + + trap cleanup_test_26b EXIT + + set_hsm_param remove_archive_on_last_unlink 1 + + cdt_shutdown + cdt_check_state stopped + + rm -f $f + + set_hsm_param remove_archive_on_last_unlink 0 + + wait_request_state $fid REMOVE WAITING + + cdt_enable + # copytool must re-register + search_and_kill_copytool + sleep 5 + search_copytools && error "Copytool should have stopped" + HSM_ARCHIVE_PURGE=false copytool_setup + + wait_request_state $fid REMOVE SUCCEED + do_facet $SINGLEAGT ls $HSM_ARCHIVE'/*/*/*/*/*/*/'$fid + [[ $? -eq 0 ]] && error "File not being removed on archive" + + cleanup_test_26b +} +run_test 26b "RAoLU policy when CDT off" + +cleanup_test_26c() { + trap 0 + set_hsm_param remove_archive_on_last_unlink 0 + set_hsm_param loop_period $orig_loop_period + set_hsm_param grace_delay $orig_grace_delay + copytool_cleanup +} + +test_26c() { + + # test needs a running copytool + copytool_setup + + mkdir -p $DIR/$tdir + local f=$DIR/$tdir/$tfile + local fid=$(copy_file /etc/passwd $f) + + $LFS hsm_archive --archive $HSM_ARCHIVE_NUMBER $f + wait_request_state $fid ARCHIVE SUCCEED + + trap cleanup_test_26c EXIT + + # set a long grace_delay vs short loop_period + local orig_loop_period=$(get_hsm_param loop_period) + local orig_grace_delay=$(get_hsm_param grace_delay) + set_hsm_param loop_period 10 + set_hsm_param grace_delay 100 + + set_hsm_param remove_archive_on_last_unlink 1 + + multiop_bg_pause $f O_c || error "open $f failed" + local pid=$! + # give multiop a chance to open + sleep 2 + + rm -f $f + + # Since CDT is not signaled for RAoLU requests to be sure it + # will wake-up to send remove request and copytool will process + # it, wait for loop_period + some extra-time. + local loop_period=$(get_hsm_param loop_period) + sleep $((loop_period + 5)) + + do_facet $SINGLEAGT ls $HSM_ARCHIVE'/*/*/*/*/*/*/'$fid + [[ $? -eq 0 ]] || error "File being removed on archive" + + kill -USR1 $pid || error "multiop early exit" + # should reach autotest timeout if multiop fails to trap + # signal, close file, and exit ... + wait $pid || error + + # again, wait for loop_period + some extra-time, to allow + # CDT enough time to handle remove request. + sleep $((loop_period + 5)) + + set_hsm_param remove_archive_on_last_unlink 0 + + do_facet $SINGLEAGT ls $HSM_ARCHIVE'/*/*/*/*/*/*/'$fid + [[ $? -eq 0 ]] && error "File not being removed on archive" + + # previous actions elapsed time should be < grace_delay + wait_request_state $fid REMOVE SUCCEED + + cleanup_test_26c +} +run_test 26c "RAoLU effective when file closed" + +cleanup_test_26d() { + trap 0 + set_hsm_param remove_archive_on_last_unlink 0 + set_hsm_param loop_period $orig_loop_period + set_hsm_param grace_delay $orig_grace_delay + copytool_cleanup +} + +test_26d() { + + # test needs a running copytool + copytool_setup + + mkdir -p $DIR/$tdir + local f=$DIR/$tdir/$tfile + local fid=$(copy_file /etc/motd $f 1) + + $LFS hsm_archive $f || error "could not archive file" + wait_request_state $fid ARCHIVE SUCCEED + check_hsm_flags $f "0x00000009" + + trap cleanup_test_26d EXIT + + # set a long grace_delay vs short loop_period + local orig_loop_period=$(get_hsm_param loop_period) + local orig_grace_delay=$(get_hsm_param grace_delay) + set_hsm_param loop_period 10 + set_hsm_param grace_delay 100 + + set_hsm_param remove_archive_on_last_unlink 1 + + multiop_bg_pause $f O_c || error "multiop failed" + local MULTIPID=$! + # give multiop a chance to open + sleep 2 + + rm -f $f + + mds_evict_client + + # Since CDT is not signaled for RAoLU requests to be sure it + # will wake-up to send remove request and copytool will process + # it, wait for loop_period + some extra-time. + local loop_period=$(get_hsm_param loop_period) + sleep $((loop_period + 5)) + + set_hsm_param remove_archive_on_last_unlink 0 + + do_facet $SINGLEAGT ls $HSM_ARCHIVE'/*/*/*/*/*/*/'$fid + [[ $? -eq 0 ]] && error "File not being removed on archive" + + # previous actions elapsed time should be < grace_delay + wait_request_state $fid REMOVE SUCCEED + + client_up || client_up || true + + kill -USR1 $MULTIPID + wait $MULTIPID || error "multiop close failed" + + cleanup_test_26d +} +run_test 26d "RAoLU when Client eviction" + test_27a() { # test needs a running copytool copytool_setup -- 1.8.3.1