From: Bobi Jam Date: Sat, 22 Sep 2018 09:14:46 +0000 (+0800) Subject: LU-10258 lfs: lfs mirror write command X-Git-Tag: 2.12.0-RC1~118 X-Git-Url: https://git.whamcloud.com/?p=fs%2Flustre-release.git;a=commitdiff_plain;h=14171e787dd00902af7c9edf10a6b54aab45f7b2;ds=sidebyside LU-10258 lfs: lfs mirror write command Rename "lfs mirror dump" command to "lfs mirror read". Add "lfs mirror write" command to write a mirror's content of a mirrored file. Usage: lfs mirror write {--mirror-id|-N } [--inputfile|-i ] Options: --mirror-id|-N This option indicates the content of which mirror specified by mirror_id needs to be written. The mirror_id is the numerical unique identifier for a mirror. --inputfile|-i The path name of the input file, if not specified, the standard input stream will be used. The input stream or input_file cannot be the same mirrored file as the mirrored_file. This command will issue a RESYNC lease write lock to notify MDS to prepare destination mirror for the write (instantiate components of the mirror), then client copy data from a file or STDIN to the specified mirror of the mirrored file. After the data copy, a RESYNC_DONE lease unlock is issued to MDS to update the layout of the mirrored file. Signed-off-by: Bobi Jam Change-Id: I02022349d4ce871319903a8714ffc4534186c0e4 Reviewed-on: https://review.whamcloud.com/33219 Tested-by: Jenkins Reviewed-by: Andreas Dilger Tested-by: Maloo Reviewed-by: Jian Yu Reviewed-by: Oleg Drokin --- diff --git a/lustre/doc/lfs-mirror-dump.1 b/lustre/doc/lfs-mirror-read.1 similarity index 65% rename from lustre/doc/lfs-mirror-dump.1 rename to lustre/doc/lfs-mirror-read.1 index becc796..44318a7 100644 --- a/lustre/doc/lfs-mirror-dump.1 +++ b/lustre/doc/lfs-mirror-read.1 @@ -1,13 +1,13 @@ -.TH LFS-MIRROR-DUMP 1 2018-08-16 "Lustre" "Lustre Utilities" +.TH LFS-MIRROR-READ 1 2018-08-16 "Lustre" "Lustre Utilities" .SH NAME -lfs mirror dump \- dump a mirror's content of a mirrored file +lfs mirror read \- read a mirror's content of a mirrored file .SH SYNOPSIS -.B lfs mirror dump -<\fB\-\-mirror-id|\-N\fR <\fImirror_id\fR>> +.B lfs mirror read +{\fB\-\-mirror-id|\-N\fR <\fImirror_id\fR>} [\fB\-\-outfile|\-o\fR <\fIoutput_file\fR>] <\fImirrored_file\fR> .SH DESCRIPTION -This command dumps the content of a file's mirror, the file is specified by the +This command reads the content of a file's mirror, the file is specified by the path name \fImirrored_file\fR, the mirror is specified by its mirror ID \fImirror_id\fR. .br @@ -17,7 +17,7 @@ otherwise the standard output stream will be used. .TP .BR \-\-mirror-id|\-N\fR\ <\fImirror_id\fR> This option indicates the content of which mirror specified by \fImirror_id\fR -needs to be dumped. The \fImirror_id\fR is the numerical unique identifier for +needs to be read. The \fImirror_id\fR is the numerical unique identifier for a mirror. .TP .BR \-\-outfile|\-o\fR\ <\fIoutput_file\fR> @@ -25,14 +25,14 @@ The path name of the output file, if not specified, the standard output stream will be used. .SH EXAMPLES .TP -.B lfs mirror dump --mirror-id 3 /mnt/lustre/file1 +.B lfs mirror read --mirror-id 3 /mnt/lustre/file1 Dump the content of mirror with mirror ID 3 for /mnt/lustre/file1 to STDOUT. .TP -.B lfs mirror dump -N 2 -o /tmp/m2 /mnt/lustre/file1 +.B lfs mirror read -N 2 -o /tmp/m2 /mnt/lustre/file1 Dump the content of mirror with mirror ID 2 for /mnt/lustre/file1 to file /tmp/m2. .SH AUTHOR -The \fBlfs mirror dump\fR command is part of the Lustre filesystem. +The \fBlfs mirror read\fR command is part of the Lustre filesystem. .SH SEE ALSO .BR lfs (1), .BR lfs-setstripe (1), @@ -41,3 +41,4 @@ The \fBlfs mirror dump\fR command is part of the Lustre filesystem. .BR lfs-mirror-extend (1), .BR lfs-mirror-split (1), .BR lfs-mirror-verify (1) +.BR lfs-mirror-write (1) diff --git a/lustre/doc/lfs-mirror-write.1 b/lustre/doc/lfs-mirror-write.1 new file mode 100644 index 0000000..c0030ef --- /dev/null +++ b/lustre/doc/lfs-mirror-write.1 @@ -0,0 +1,43 @@ +.TH LFS-MIRROR-WRITE 1 2018-09-23 "Lustre" "Lustre Utilities" +.SH NAME +lfs mirror write \- write a mirror's content of a mirrored file +.SH SYNOPSIS +.B lfs mirror write +{\fB\-\-mirror-id|\-N\fR <\fImirror_id\fR>} +[\fB\-\-inputfile|\-i\fR <\fIinput_file\fR>] +<\fImirrored_file\fR> +.SH DESCRIPTION +This command writes the content of a file's mirror, the file is specified by the +path name \fImirrored_file\fR, the mirror is specified by its mirror ID +\fImirror_id\fR. +.SH OPTIONS +.TP +.BR \-\-mirror-id|\-N\fR\ <\fImirror_id\fR> +This option indicates the content of which mirror specified by \fImirror_id\fR +needs to be written. The \fImirror_id\fR is the numerical unique identifier for +a mirror. +.TP +.BR \-\-inputfile|\-i\fR\ <\fIinput_file\fR> +The path name of the input file, if not specified, the standard input stream +will be used. The input stream or input_file cannot be the same mirrored file +as the \fImirrored_file\fR. +.SH EXAMPLES +.TP +.B lfs mirror write --mirror-id 3 /mnt/lustre/file1 < /tmp/m2 +Write the content of /mnt/m2 to the mirror with miro ID 3 for +/mnt/lustre/file1. +.TP +.B lfs mirror write -N2 -i /tmp/m2 /mnt/lustre/file1 +Write the content of /mnt/m2 to the mirror with mirror ID 2 for +/mnt/lustre/file1. +.SH AUTHOR +The \fBlfs mirror write\fR command is part of the Lustre filesystem. +.SH SEE ALSO +.BR lfs (1), +.BR lfs-setstripe (1), +.BR lfs-getstripe (1), +.BR lfs-mirror-create (1), +.BR lfs-mirror-extend (1), +.BR lfs-mirror-split (1), +.BR lfs-mirror-verify (1), +.BR lfs-mirror-read (1) diff --git a/lustre/include/md_object.h b/lustre/include/md_object.h index 242e90d..718be0c 100644 --- a/lustre/include/md_object.h +++ b/lustre/include/md_object.h @@ -181,6 +181,7 @@ enum md_layout_opc { */ struct md_layout_change { enum md_layout_opc mlc_opc; + __u16 mlc_mirror_id; struct layout_intent *mlc_intent; struct lu_buf mlc_buf; struct lustre_som_attrs mlc_som; diff --git a/lustre/include/uapi/linux/lustre/lustre_idl.h b/lustre/include/uapi/linux/lustre/lustre_idl.h index ff8ab1a..1275d41 100644 --- a/lustre/include/uapi/linux/lustre/lustre_idl.h +++ b/lustre/include/uapi/linux/lustre/lustre_idl.h @@ -2101,7 +2101,8 @@ struct mdt_rec_resync { __u32 rs_padding6; /* rr_flags */ __u32 rs_padding7; /* rr_flags_h */ __u32 rs_padding8; /* rr_umask */ - __u32 rs_padding9; /* rr_padding_4 */ + __u16 rs_mirror_id; + __u16 rs_padding9; /* rr_padding_4 */ }; /* @@ -2135,7 +2136,8 @@ struct mdt_rec_reint { __u32 rr_flags; __u32 rr_flags_h; __u32 rr_umask; - __u32 rr_padding_4; /* also fix lustre_swab_mdt_rec_reint */ + __u16 rr_mirror_id; + __u16 rr_padding_4; /* also fix lustre_swab_mdt_rec_reint */ }; /* lmv structures */ diff --git a/lustre/include/uapi/linux/lustre/lustre_user.h b/lustre/include/uapi/linux/lustre/lustre_user.h index 6913d99..1ca6449 100644 --- a/lustre/include/uapi/linux/lustre/lustre_user.h +++ b/lustre/include/uapi/linux/lustre/lustre_user.h @@ -361,6 +361,16 @@ struct ll_ioc_lease { __u32 lil_ids[0]; }; +struct ll_ioc_lease_id { + __u32 lil_mode; + __u32 lil_flags; + __u32 lil_count; + __u16 lil_mirror_id; + __u16 lil_padding1; + __u64 lil_padding2; + __u32 lil_ids[0]; +}; + /* * The ioctl naming rules: * LL_* - works on the currently opened filehandle instead of parent dir diff --git a/lustre/llite/file.c b/lustre/llite/file.c index 73d032b..a1b43b6 100644 --- a/lustre/llite/file.c +++ b/lustre/llite/file.c @@ -1137,10 +1137,11 @@ static int ll_lease_close(struct obd_client_handle *och, struct inode *inode, * After lease is taken, send the RPC MDS_REINT_RESYNC to the MDT */ static int ll_lease_file_resync(struct obd_client_handle *och, - struct inode *inode) + struct inode *inode, unsigned long arg) { struct ll_sb_info *sbi = ll_i2sbi(inode); struct md_op_data *op_data; + struct ll_ioc_lease_id ioc; __u64 data_version_unused; int rc; ENTRY; @@ -1150,6 +1151,10 @@ static int ll_lease_file_resync(struct obd_client_handle *och, if (IS_ERR(op_data)) RETURN(PTR_ERR(op_data)); + if (copy_from_user(&ioc, (struct ll_ioc_lease_id __user *)arg, + sizeof(ioc))) + RETURN(-EFAULT); + /* before starting file resync, it's necessary to clean up page cache * in client memory, otherwise once the layout version is increased, * writing back cached data will be denied the OSTs. */ @@ -1158,6 +1163,7 @@ static int ll_lease_file_resync(struct obd_client_handle *och, GOTO(out, rc); op_data->op_lease_handle = och->och_lease_handle; + op_data->op_mirror_id = ioc.lil_mirror_id; rc = md_file_resync(sbi->ll_md_exp, op_data); if (rc) GOTO(out, rc); @@ -3274,7 +3280,7 @@ static long ll_file_set_lease(struct file *file, struct ll_ioc_lease *ioc, RETURN(PTR_ERR(och)); if (ioc->lil_flags & LL_LEASE_RESYNC) { - rc = ll_lease_file_resync(och, inode); + rc = ll_lease_file_resync(och, inode, arg); if (rc) { ll_lease_close(och, inode, NULL); RETURN(rc); diff --git a/lustre/lod/lod_object.c b/lustre/lod/lod_object.c index babe445..c9571a2 100644 --- a/lustre/lod/lod_object.c +++ b/lustre/lod/lod_object.c @@ -6136,6 +6136,30 @@ static int lod_primary_pick(const struct lu_env *env, struct lod_object *lo, RETURN(picked); } +static int lod_prepare_resync_mirror(const struct lu_env *env, + struct lod_object *lo, + __u16 mirror_id) +{ + struct lod_thread_info *info = lod_env_info(env); + struct lod_layout_component *lod_comp; + int i; + + for (i = 0; i < lo->ldo_mirror_count; i++) { + if (lo->ldo_mirrors[i].lme_id != mirror_id) + continue; + + lod_foreach_mirror_comp(lod_comp, lo, i) { + if (lod_comp_inited(lod_comp)) + continue; + + info->lti_comp_idx[info->lti_count++] = + lod_comp_index(lo, lod_comp); + } + } + + return 0; +} + /** * figure out the components should be instantiated for resync. */ @@ -6251,23 +6275,33 @@ static int lod_declare_update_rdonly(const struct lu_env *env, * prep uninited all components assuming any non-stale mirror * could be picked as the primary mirror. */ - for (i = 0; i < lo->ldo_mirror_count; i++) { - if (lo->ldo_mirrors[i].lme_stale) - continue; + if (mlc->mlc_mirror_id == 0) { + /* normal resync */ + for (i = 0; i < lo->ldo_mirror_count; i++) { + if (lo->ldo_mirrors[i].lme_stale) + continue; - lod_foreach_mirror_comp(lod_comp, lo, i) { - if (!lod_comp_inited(lod_comp)) - break; + lod_foreach_mirror_comp(lod_comp, lo, i) { + if (!lod_comp_inited(lod_comp)) + break; - if (extent.e_end < lod_comp->llc_extent.e_end) - extent.e_end = - lod_comp->llc_extent.e_end; + if (extent.e_end < + lod_comp->llc_extent.e_end) + extent.e_end = + lod_comp->llc_extent.e_end; + } } + rc = lod_prepare_resync(env, lo, &extent); + if (rc) + GOTO(out, rc); + } else { + /* mirror write, try to init its all components */ + rc = lod_prepare_resync_mirror(env, lo, + mlc->mlc_mirror_id); + if (rc) + GOTO(out, rc); } - rc = lod_prepare_resync(env, lo, &extent); - if (rc) - GOTO(out, rc); /* change the file state to SYNC_PENDING */ lo->ldo_flr_state = LCM_FL_SYNC_PENDING; } @@ -6391,16 +6425,26 @@ static int lod_declare_update_write_pending(const struct lu_env *env, lod_comp_index(lo, lod_comp); } } else { /* MD_LAYOUT_RESYNC */ - lod_foreach_mirror_comp(lod_comp, lo, primary) { - if (!lod_comp_inited(lod_comp)) - break; + if (mlc->mlc_mirror_id == 0) { + /* normal resync */ + lod_foreach_mirror_comp(lod_comp, lo, primary) { + if (!lod_comp_inited(lod_comp)) + break; - extent.e_end = lod_comp->llc_extent.e_end; + extent.e_end = lod_comp->llc_extent.e_end; + } + + rc = lod_prepare_resync(env, lo, &extent); + if (rc) + GOTO(out, rc); + } else { + /* mirror write, try to init its all components */ + rc = lod_prepare_resync_mirror(env, lo, + mlc->mlc_mirror_id); + if (rc) + GOTO(out, rc); } - rc = lod_prepare_resync(env, lo, &extent); - if (rc) - GOTO(out, rc); /* change the file state to SYNC_PENDING */ lo->ldo_flr_state = LCM_FL_SYNC_PENDING; } diff --git a/lustre/mdc/mdc_reint.c b/lustre/mdc/mdc_reint.c index e0a97b1..5868101 100644 --- a/lustre/mdc/mdc_reint.c +++ b/lustre/mdc/mdc_reint.c @@ -462,6 +462,7 @@ int mdc_file_resync(struct obd_export *exp, struct md_op_data *op_data) rec->rs_cap = op_data->op_cap; rec->rs_fid = op_data->op_fid1; rec->rs_bias = op_data->op_bias; + rec->rs_mirror_id = op_data->op_mirror_id; lock = ldlm_handle2lock(&op_data->op_lease_handle); if (lock != NULL) { diff --git a/lustre/mdt/mdt_internal.h b/lustre/mdt/mdt_internal.h index 7cc7b8e..f448f31 100644 --- a/lustre/mdt/mdt_internal.h +++ b/lustre/mdt/mdt_internal.h @@ -360,6 +360,7 @@ struct mdt_reint_record { void *rr_eadata; int rr_eadatalen; __u32 rr_flags; + __u16 rr_mirror_id; }; enum mdt_reint_flag { diff --git a/lustre/mdt/mdt_lib.c b/lustre/mdt/mdt_lib.c index baaebdc..dddf727 100644 --- a/lustre/mdt/mdt_lib.c +++ b/lustre/mdt/mdt_lib.c @@ -1670,6 +1670,7 @@ static int mdt_resync_unpack(struct mdt_thread_info *info) uc->uc_cap = rec->rs_cap; rr->rr_fid1 = &rec->rs_fid; + rr->rr_mirror_id = rec->rs_mirror_id; /* cookie doesn't need to be swapped but it has been swapped * in lustre_swab_mdt_rec_reint() as rr_mtime, so here it needs diff --git a/lustre/mdt/mdt_reint.c b/lustre/mdt/mdt_reint.c index 69bb85a..e6ac0a8 100644 --- a/lustre/mdt/mdt_reint.c +++ b/lustre/mdt/mdt_reint.c @@ -2648,7 +2648,7 @@ static int mdt_reint_resync(struct mdt_thread_info *info, struct mdt_object *mo; struct ldlm_lock *lease; struct mdt_body *repbody; - struct md_layout_change layout = { 0 }; + struct md_layout_change layout = { .mlc_mirror_id = rr->rr_mirror_id }; bool lease_broken; int rc, rc2; ENTRY; diff --git a/lustre/ptlrpc/pack_generic.c b/lustre/ptlrpc/pack_generic.c index 20ae792..1493726 100644 --- a/lustre/ptlrpc/pack_generic.c +++ b/lustre/ptlrpc/pack_generic.c @@ -2068,6 +2068,7 @@ void lustre_swab_mdt_rec_reint (struct mdt_rec_reint *rr) __swab32s(&rr->rr_flags); __swab32s(&rr->rr_flags_h); __swab32s(&rr->rr_umask); + __swab16s(&rr->rr_mirror_id); CLASSERT(offsetof(typeof(*rr), rr_padding_4) != 0); }; diff --git a/lustre/ptlrpc/wiretest.c b/lustre/ptlrpc/wiretest.c index ad29829..12ec0c1 100644 --- a/lustre/ptlrpc/wiretest.c +++ b/lustre/ptlrpc/wiretest.c @@ -3193,9 +3193,13 @@ void lustre_assert_wire_constants(void) (long long)(int)offsetof(struct mdt_rec_resync, rs_padding8)); LASSERTF((int)sizeof(((struct mdt_rec_resync *)0)->rs_padding8) == 4, "found %lld\n", (long long)(int)sizeof(((struct mdt_rec_resync *)0)->rs_padding8)); - LASSERTF((int)offsetof(struct mdt_rec_resync, rs_padding9) == 132, "found %lld\n", + LASSERTF((int)offsetof(struct mdt_rec_resync, rs_mirror_id) == 132, "found %lld\n", + (long long)(int)offsetof(struct mdt_rec_resync, rs_mirror_id)); + LASSERTF((int)sizeof(((struct mdt_rec_resync *)0)->rs_mirror_id) == 2, "found %lld\n", + (long long)(int)sizeof(((struct mdt_rec_resync *)0)->rs_mirror_id)); + LASSERTF((int)offsetof(struct mdt_rec_resync, rs_padding9) == 134, "found %lld\n", (long long)(int)offsetof(struct mdt_rec_resync, rs_padding9)); - LASSERTF((int)sizeof(((struct mdt_rec_resync *)0)->rs_padding9) == 4, "found %lld\n", + LASSERTF((int)sizeof(((struct mdt_rec_resync *)0)->rs_padding9) == 2, "found %lld\n", (long long)(int)sizeof(((struct mdt_rec_resync *)0)->rs_padding9)); /* Checks for struct mdt_rec_reint */ @@ -3289,9 +3293,13 @@ void lustre_assert_wire_constants(void) (long long)(int)offsetof(struct mdt_rec_reint, rr_umask)); LASSERTF((int)sizeof(((struct mdt_rec_reint *)0)->rr_umask) == 4, "found %lld\n", (long long)(int)sizeof(((struct mdt_rec_reint *)0)->rr_umask)); - LASSERTF((int)offsetof(struct mdt_rec_reint, rr_padding_4) == 132, "found %lld\n", + LASSERTF((int)offsetof(struct mdt_rec_reint, rr_mirror_id) == 132, "found %lld\n", + (long long)(int)offsetof(struct mdt_rec_reint, rr_mirror_id)); + LASSERTF((int)sizeof(((struct mdt_rec_reint *)0)->rr_mirror_id) == 2, "found %lld\n", + (long long)(int)sizeof(((struct mdt_rec_reint *)0)->rr_mirror_id)); + LASSERTF((int)offsetof(struct mdt_rec_reint, rr_padding_4) == 134, "found %lld\n", (long long)(int)offsetof(struct mdt_rec_reint, rr_padding_4)); - LASSERTF((int)sizeof(((struct mdt_rec_reint *)0)->rr_padding_4) == 4, "found %lld\n", + LASSERTF((int)sizeof(((struct mdt_rec_reint *)0)->rr_padding_4) == 2, "found %lld\n", (long long)(int)sizeof(((struct mdt_rec_reint *)0)->rr_padding_4)); /* Checks for struct lmv_desc */ diff --git a/lustre/tests/sanity-flr.sh b/lustre/tests/sanity-flr.sh index 60a154c..2631eca 100644 --- a/lustre/tests/sanity-flr.sh +++ b/lustre/tests/sanity-flr.sh @@ -1273,8 +1273,11 @@ test_37() local tf=$DIR/$tfile local tf2=$DIR/$tfile-2 local tf3=$DIR/$tfile-3 + local tf4=$DIR/$tfile-4 create_files_37 $((RANDOM + 15 * 1048576)) $tf $tf2 $tf3 + rm -f $tf4 + cp $tf $tf4 # assume the mirror id will be 1, 2, and 3 declare -A checksums @@ -1297,11 +1300,20 @@ test_37() local sum for i in ${mirror_array[@]}; do - sum=$($LFS mirror dump -N $i $tf | md5sum) + $LCTL set_param ldlm.namespaces.*.lru_size=clear > /dev/null + sum=$($LFS mirror read -N $i $tf | md5sum) [ "$sum" = "${checksums[$i]}" ] || error "$i: mismatch: \'${checksums[$i]}\' vs. \'$sum\'" done + # verify mirror write + echo "Verifying mirror write .." + $LFS mirror write -N2 $tf < $tf4 + + sum=$($LFS mirror read -N2 $tf | md5sum) + [[ "$sum" = "${checksums[1]}" ]] || + error "2: mismatch \'${checksums[1]}\' vs. \'$sum\'" + # verify mirror copy, write to this mirrored file will invalidate # the other two mirrors echo "Verifying mirror copy .." @@ -1320,7 +1332,7 @@ test_37() # verify copying is successful by checking checksums remount_client $MOUNT for i in ${mirror_array[@]}; do - sum=$($LFS mirror dump -N $i $tf | md5sum) + sum=$($LFS mirror read -N $i $tf | md5sum) [ "$sum" = "${checksums[1]}" ] || error "$i: mismatch checksum after copy" done @@ -1379,7 +1391,7 @@ test_38() { local valid_mirror stale_mirror id mirror_cksum for id in "${mirror_array[@]}"; do - mirror_cksum=$($LFS mirror dump -N $id $tf | md5sum) + mirror_cksum=$($LFS mirror read -N $id $tf | md5sum) [ "$ref_cksum" == "$mirror_cksum" ] && { valid_mirror=$id; continue; } @@ -1392,7 +1404,7 @@ test_38() { mirror_io resync $tf || error "resync failed" verify_flr_state $tf "ro" - mirror_cksum=$($LFS mirror dump -N $stale_mirror $tf | md5sum) + mirror_cksum=$($LFS mirror read -N $stale_mirror $tf | md5sum) [ "$file_cksum" = "$ref_cksum" ] || error "resync failed" # case 2: inject an error to make mirror_io exit after changing @@ -1518,7 +1530,7 @@ test_41() { echo " **verify $tf-1 data consistency in all mirrors" for i in 1 2 3; do - local sum=$($LFS mirror dump -N$i $tf-1 | md5sum) + local sum=$($LFS mirror read -N$i $tf-1 | md5sum) [[ "$sum" = "$sum0" ]] || error "$tf-1.$i: checksum mismatch: $sum != $sum0" done @@ -1997,8 +2009,8 @@ test_48() { verify_flr_state $tf "wp" verify_comp_attr lcme_flags $tf 0x20003 nosync,stale - local sum1=$($LFS mirror dump -N1 $tf | md5sum) - local sum2=$($LFS mirror dump -N2 $tf | md5sum) + local sum1=$($LFS mirror read -N1 $tf | md5sum) + local sum2=$($LFS mirror read -N2 $tf | md5sum) echo " ** verify mirror 2 doesn't change" echo "original checksum: $sum0" @@ -2015,8 +2027,8 @@ test_48() { verify_flr_state $tf "ro" verify_comp_attr lcme_flags $tf 0x20003 nosync,^stale - sum1=$($LFS mirror dump -N1 $tf | md5sum) - sum2=$($LFS mirror dump -N2 $tf | md5sum) + sum1=$($LFS mirror read -N1 $tf | md5sum) + sum2=$($LFS mirror read -N2 $tf | md5sum) echo " ** verify mirror 2 resync-ed" echo "original checksum: $sum0" @@ -2161,9 +2173,9 @@ test_200() { mirror_io resync $tf get_mirror_ids $tf - local csum=$($LFS mirror dump -N ${mirror_array[0]} $tf | md5sum) + local csum=$($LFS mirror read -N ${mirror_array[0]} $tf | md5sum) for id in ${mirror_array[@]:1}; do - [ "$($LFS mirror dump -N $id $tf | md5sum)" = "$csum" ] || + [ "$($LFS mirror read -N $id $tf | md5sum)" = "$csum" ] || error "checksum error for mirror $id" done diff --git a/lustre/utils/lfs.c b/lustre/utils/lfs.c index db45030..885c9b7 100644 --- a/lustre/utils/lfs.c +++ b/lustre/utils/lfs.c @@ -124,7 +124,8 @@ static int lfs_mirror_list_commands(int argc, char **argv); static int lfs_list_commands(int argc, char **argv); static inline int lfs_mirror_resync(int argc, char **argv); static inline int lfs_mirror_verify(int argc, char **argv); -static inline int lfs_mirror_dump(int argc, char **argv); +static inline int lfs_mirror_read(int argc, char **argv); +static inline int lfs_mirror_write(int argc, char **argv); enum setstripe_origin { SO_SETSTRIPE, @@ -284,10 +285,14 @@ command_t mirror_cmdlist[] = { "\t mirror will be stored into. If not specified,\n" "\t a new file named .mirror~\n" "\t will be used.\n" }, - { .pc_name = "dump", .pc_func = lfs_mirror_dump, - .pc_help = "Dump the content of a specified mirror of a file.\n" - "usage: lfs mirror dump <--mirror-id|-N " - "[--outfile|-o ] \n" }, + { .pc_name = "read", .pc_func = lfs_mirror_read, + .pc_help = "Read the content of a specified mirror of a file.\n" + "usage: lfs mirror read <--mirror-id|-N " + "[--outfile|-o ] \n" }, + { .pc_name = "write", .pc_func = lfs_mirror_write, + .pc_help = "Write to a specified mirror of a file.\n" + "usage: lfs mirror write <--mirror-id|-N " + "[--inputfile|-i ] \n" }, { .pc_name = "resync", .pc_func = lfs_mirror_resync, .pc_help = "Resynchronizes out-of-sync mirrored file(s).\n" "usage: lfs mirror resync [--only ] " @@ -297,7 +302,7 @@ command_t mirror_cmdlist[] = { "usage: lfs mirror verify " "[--only ] " "[--verbose|-v] [ ...]\n"}, - { .pc_name = "--list-commands", .pc_func = lfs_mirror_list_commands, + { .pc_name = "list-commands", .pc_func = lfs_mirror_list_commands, .pc_help = "list commands supported by lfs mirror"}, { .pc_name = "help", .pc_func = Parser_help, .pc_help = "help" }, { .pc_name = "exit", .pc_func = Parser_quit, .pc_help = "quit" }, @@ -586,7 +591,8 @@ command_t cmdlist[] = { "lfs mirror extend - add mirror(s) to an existing file\n" "lfs mirror split - split a mirror from an existing mirrored file\n" "lfs mirror resync - resynchronize out-of-sync mirrored file(s)\n" - "lfs mirror dump - dump a mirror content of a mirrored file\n" + "lfs mirror read - read a mirror content of a mirrored file\n" + "lfs mirror write - write to a mirror of a mirrored file\n" "lfs mirror verify - verify mirrored file(s)\n"}, {"getsom", lfs_getsom, 0, "To list the SOM info for a given file.\n" "usage: getsom [-s] [-b] [-f] \n" @@ -8288,7 +8294,7 @@ static inline int lfs_mirror_resync(int argc, char **argv) /* ignore previous file's error, continue with next file */ /* reset ioc */ - memset(ioc, 0, sizeof(__u32) * 4096); + memset(ioc, 0, sizeof(*ioc) + sizeof(__u32) * 4096); } free(ioc); @@ -8296,21 +8302,76 @@ error: return rc; } -static inline int lfs_mirror_dump(int argc, char **argv) +static inline int verify_mirror_id_by_fd(int fd, __u16 mirror_id) +{ + struct llapi_layout *layout; + int rc; + + layout = llapi_layout_get_by_fd(fd, 0); + if (layout == NULL) { + fprintf(stderr, "could not get layout.\n"); + return -EINVAL; + } + + rc = llapi_layout_comp_iterate(layout, find_mirror_id, &mirror_id); + if (rc < 0) { + fprintf(stderr, "failed to iterate layout\n"); + llapi_layout_free(layout); + + return rc; + } else if (rc == LLAPI_LAYOUT_ITER_CONT) { + fprintf(stderr, "does not find mirror with ID %u\n", mirror_id); + llapi_layout_free(layout); + + return -EINVAL; + } + llapi_layout_free(layout); + + return 0; +} + +/** + * Check whether two files are the same file + * \retval 0 same file + * \retval 1 not the same file + * \retval <0 error code + */ +static inline int check_same_file(const char *f1, const char *f2) +{ + struct stat stbuf1; + struct stat stbuf2; + + if (stat(f1, &stbuf1) < 0) { + fprintf(stderr, "%s: cannot stat file '%s': %s\n", + progname, f1, strerror(errno)); + return -errno; + } + + if (stat(f2, &stbuf2) < 0) { + fprintf(stderr, "%s: cannot stat file '%s': %s\n", + progname, f2, strerror(errno)); + return -errno; + } + + if (stbuf1.st_rdev == stbuf2.st_rdev && + stbuf1.st_ino == stbuf2.st_ino) + return 0; + + return 1; +} + +static inline int lfs_mirror_read(int argc, char **argv) { int rc = CMD_HELP; __u16 mirror_id = 0; const char *outfile = NULL; char *fname; - struct stat stbuf; int fd = 0; - struct llapi_layout *layout; int outfd; int c; - const size_t buflen = 4 << 20; void *buf; + const size_t buflen = 4 << 20; off_t pos; - struct option long_opts[] = { { .val = 'N', .name = "mirror-id", .has_arg = required_argument }, { .val = 'o', .name = "outfile", .has_arg = required_argument }, @@ -8332,6 +8393,10 @@ static inline int lfs_mirror_dump(int argc, char **argv) case 'o': outfile = optarg; break; + default: + fprintf(stderr, "%s: option '%s' unrecognized.\n", + progname, argv[optind - 1]); + return -EINVAL; } } @@ -8353,56 +8418,39 @@ static inline int lfs_mirror_dump(int argc, char **argv) /* open mirror file */ fname = argv[optind]; - if (stat(fname, &stbuf) < 0) { - fprintf(stderr, "%s %s: cannot stat file '%s': %s.\n", - progname, argv[0], fname, strerror(errno)); - return rc; - } - - if (!S_ISREG(stbuf.st_mode)) { - fprintf(stderr, "%s %s: '%s' is not a regular file.\n", - progname, argv[0], fname); - return rc; + if (outfile) { + rc = check_same_file(fname, outfile); + if (rc == 0) { + fprintf(stderr, + "%s %s: output file cannot be the mirrored file\n", + progname, argv[0]); + return -EINVAL; + } + if (rc < 0) + return rc; } fd = open(fname, O_DIRECT | O_RDONLY); if (fd < 0) { - fprintf(stderr, "%s %s: cannot open '%s': %s.\n", + fprintf(stderr, "%s %s: cannot open '%s': %s\n", progname, argv[0], fname, strerror(errno)); return rc; } /* verify mirror id */ - layout = llapi_layout_get_by_fd(fd, 0); - if (layout == NULL) { - fprintf(stderr, "%s %s: could not get layout of file '%s'.\n", - progname, argv[0], fname); - rc = -EINVAL; - goto close_fd; - } - - rc = llapi_layout_comp_iterate(layout, find_mirror_id, &mirror_id); - if (rc < 0) { - fprintf(stderr, "%s %s: failed to iterate layout of '%s'.\n", - progname, argv[0], fname); - llapi_layout_free(layout); - goto close_fd; - } else if (rc == LLAPI_LAYOUT_ITER_CONT) { + rc = verify_mirror_id_by_fd(fd, mirror_id); + if (rc) { fprintf(stderr, - "%s %s: file '%s' does not contain mirror with ID %u\n", - progname, argv[0], fname, mirror_id); - llapi_layout_free(layout); - - rc = -EINVAL; + "%s %s: cannot find mirror with ID %u in '%s'\n", + progname, argv[0], mirror_id, fname); goto close_fd; } - llapi_layout_free(layout); /* open output file */ if (outfile) { outfd = open(outfile, O_EXCL | O_WRONLY | O_CREAT, 0644); if (outfd < 0) { - fprintf(stderr, "%s %s: cannot create file '%s':%s.\n", + fprintf(stderr, "%s %s: cannot create file '%s': %s\n", progname, argv[0], outfile, strerror(errno)); rc = -errno; goto close_fd; @@ -8414,7 +8462,7 @@ static inline int lfs_mirror_dump(int argc, char **argv) /* allocate buffer */ rc = posix_memalign(&buf, sysconf(_SC_PAGESIZE), buflen); if (rc) { - fprintf(stderr, "%s %s: posix_memalign returns %d.\n", + fprintf(stderr, "%s %s: posix_memalign returns %d\n", progname, argv[0], rc); goto close_outfd; } @@ -8428,7 +8476,7 @@ static inline int lfs_mirror_dump(int argc, char **argv) if (bytes_read < 0) { rc = bytes_read; fprintf(stderr, - "%s %s: fail to read data from mirror %u:%s.\n", + "%s %s: fail to read data from mirror %u: %s\n", progname, argv[0], mirror_id, strerror(-rc)); goto free_buf; } @@ -8444,7 +8492,7 @@ static inline int lfs_mirror_dump(int argc, char **argv) bytes_read - written); if (written2 < 0) { fprintf(stderr, - "%s %s: fail to write %s:%s.\n", + "%s %s: fail to write %s: %s\n", progname, argv[0], outfile ? : "STDOUT", strerror(errno)); rc = -errno; @@ -8478,6 +8526,205 @@ close_fd: return rc; } +static inline int lfs_mirror_write(int argc, char **argv) +{ + int rc = CMD_HELP; + __u16 mirror_id = 0; + const char *inputfile = NULL; + char *fname; + int fd = 0; + int inputfd; + int c; + void *buf; + const size_t buflen = 4 << 20; + off_t pos; + size_t page_size = sysconf(_SC_PAGESIZE); + struct ll_ioc_lease_id ioc; + + struct option long_opts[] = { + { .val = 'N', .name = "mirror-id", .has_arg = required_argument }, + { .val = 'i', .name = "inputfile", .has_arg = required_argument }, + { .name = NULL } }; + + while ((c = getopt_long(argc, argv, "N:i:", long_opts, NULL)) >= 0) { + char *end; + + switch (c) { + case 'N': + mirror_id = strtoul(optarg, &end, 0); + if (*end != '\0' || mirror_id == 0) { + fprintf(stderr, + "%s %s: invalid mirror ID '%s'\n", + progname, argv[0], optarg); + return rc; + } + break; + case 'i': + inputfile = optarg; + break; + default: + fprintf(stderr, "%s: option '%s' unrecognized\n", + progname, argv[optind - 1]); + return -EINVAL; + } + } + + if (argc == optind) { + fprintf(stderr, "%s %s: no mirrored file provided\n", + progname, argv[0]); + return rc; + } else if (argc > optind + 1) { + fprintf(stderr, "%s %s: too many files\n", progname, argv[0]); + return rc; + } + + if (mirror_id == 0) { + fprintf(stderr, "%s %s: no valid mirror ID is provided\n", + progname, argv[0]); + return rc; + } + + /* open mirror file */ + fname = argv[optind]; + + if (inputfile) { + rc = check_same_file(fname, inputfile); + if (rc == 0) { + fprintf(stderr, + "%s %s: input file cannot be the mirrored file\n", + progname, argv[0]); + return -EINVAL; + } + if (rc < 0) + return rc; + } + + fd = open(fname, O_DIRECT | O_WRONLY); + if (fd < 0) { + fprintf(stderr, "%s %s: cannot open '%s': %s\n", + progname, argv[0], fname, strerror(errno)); + return rc; + } + + /* verify mirror id */ + rc = verify_mirror_id_by_fd(fd, mirror_id); + if (rc) { + fprintf(stderr, + "%s %s: cannot find mirror with ID %u in '%s'\n", + progname, argv[0], mirror_id, fname); + goto close_fd; + } + + /* open input file */ + if (inputfile) { + inputfd = open(inputfile, O_RDONLY, 0644); + if (inputfd < 0) { + fprintf(stderr, "%s %s: cannot open file '%s': %s\n", + progname, argv[0], inputfile, strerror(errno)); + rc = -errno; + goto close_fd; + } + } else { + inputfd = STDIN_FILENO; + } + + /* allocate buffer */ + rc = posix_memalign(&buf, page_size, buflen); + if (rc) { + fprintf(stderr, "%s %s: posix_memalign returns %d\n", + progname, argv[0], rc); + goto close_inputfd; + } + + /* prepare target mirror components instantiation */ + ioc.lil_mode = LL_LEASE_WRLCK; + ioc.lil_flags = LL_LEASE_RESYNC; + ioc.lil_mirror_id = mirror_id; + rc = llapi_lease_set(fd, (struct ll_ioc_lease *)&ioc); + if (rc < 0) { + fprintf(stderr, + "%s %s: '%s' llapi_lease_get_ext failed: %s\n", + progname, argv[0], fname, strerror(errno)); + goto free_buf; + } + + pos = 0; + while (1) { + ssize_t bytes_read; + ssize_t written; + size_t to_write; + + rc = llapi_lease_check(fd); + if (rc != LL_LEASE_WRLCK) { + fprintf(stderr, "%s %s: '%s' lost lease lock\n", + progname, argv[0], fname); + goto free_buf; + } + + bytes_read = read(inputfd, buf, buflen); + if (bytes_read < 0) { + rc = bytes_read; + fprintf(stderr, + "%s %s: fail to read data from '%s': %s\n", + progname, argv[0], inputfile ? : "STDIN", + strerror(errno)); + rc = -errno; + goto free_buf; + } + + /* EOF reached */ + if (bytes_read == 0) + break; + + /* round up to page align to make direct IO happy. */ + to_write = (bytes_read + page_size - 1) & ~(page_size - 1); + + written = llapi_mirror_write(fd, mirror_id, buf, to_write, + pos); + if (written < 0) { + rc = written; + fprintf(stderr, + "%s %s: fail to write to mirror %u: %s\n", + progname, argv[0], mirror_id, + strerror(-rc)); + goto free_buf; + } + + pos += bytes_read; + } + + if (pos & (page_size - 1)) { + rc = llapi_mirror_truncate(fd, mirror_id, pos); + if (rc < 0) + goto free_buf; + } + + ioc.lil_mode = LL_LEASE_UNLCK; + ioc.lil_flags = LL_LEASE_RESYNC_DONE; + ioc.lil_count = 0; + rc = llapi_lease_set(fd, (struct ll_ioc_lease *)&ioc); + if (rc <= 0) { + if (rc == 0) + rc = -EBUSY; + fprintf(stderr, + "%s %s: release lease lock of '%s' failed: %s\n", + progname, argv[0], fname, strerror(errno)); + goto free_buf; + } + + rc = 0; + +free_buf: + free(buf); +close_inputfd: + if (inputfile) + close(inputfd); +close_fd: + close(fd); + + return rc; +} + /** * struct verify_chunk - Mirror chunk to be verified. * @chunk: [start, end) of the chunk. @@ -9074,7 +9321,7 @@ static inline int lfs_mirror_verify(int argc, char **argv) verbose++; break; default: - fprintf(stderr, "%s: options '%s' unrecognized.\n", + fprintf(stderr, "%s: option '%s' unrecognized.\n", progname, argv[optind - 1]); rc = -EINVAL; goto error; diff --git a/lustre/utils/wirecheck.c b/lustre/utils/wirecheck.c index c14a965..065e5bc 100644 --- a/lustre/utils/wirecheck.c +++ b/lustre/utils/wirecheck.c @@ -1351,6 +1351,7 @@ check_mdt_rec_resync(void) CHECK_MEMBER(mdt_rec_resync, rs_padding6); CHECK_MEMBER(mdt_rec_resync, rs_padding7); CHECK_MEMBER(mdt_rec_resync, rs_padding8); + CHECK_MEMBER(mdt_rec_resync, rs_mirror_id); CHECK_MEMBER(mdt_rec_resync, rs_padding9); } @@ -1381,6 +1382,7 @@ check_mdt_rec_reint(void) CHECK_MEMBER(mdt_rec_reint, rr_flags); CHECK_MEMBER(mdt_rec_reint, rr_flags_h); CHECK_MEMBER(mdt_rec_reint, rr_umask); + CHECK_MEMBER(mdt_rec_reint, rr_mirror_id); CHECK_MEMBER(mdt_rec_reint, rr_padding_4); } diff --git a/lustre/utils/wiretest.c b/lustre/utils/wiretest.c index f34e6f4..7c5fe3f 100644 --- a/lustre/utils/wiretest.c +++ b/lustre/utils/wiretest.c @@ -3215,9 +3215,13 @@ void lustre_assert_wire_constants(void) (long long)(int)offsetof(struct mdt_rec_resync, rs_padding8)); LASSERTF((int)sizeof(((struct mdt_rec_resync *)0)->rs_padding8) == 4, "found %lld\n", (long long)(int)sizeof(((struct mdt_rec_resync *)0)->rs_padding8)); - LASSERTF((int)offsetof(struct mdt_rec_resync, rs_padding9) == 132, "found %lld\n", + LASSERTF((int)offsetof(struct mdt_rec_resync, rs_mirror_id) == 132, "found %lld\n", + (long long)(int)offsetof(struct mdt_rec_resync, rs_mirror_id)); + LASSERTF((int)sizeof(((struct mdt_rec_resync *)0)->rs_mirror_id) == 2, "found %lld\n", + (long long)(int)sizeof(((struct mdt_rec_resync *)0)->rs_mirror_id)); + LASSERTF((int)offsetof(struct mdt_rec_resync, rs_padding9) == 134, "found %lld\n", (long long)(int)offsetof(struct mdt_rec_resync, rs_padding9)); - LASSERTF((int)sizeof(((struct mdt_rec_resync *)0)->rs_padding9) == 4, "found %lld\n", + LASSERTF((int)sizeof(((struct mdt_rec_resync *)0)->rs_padding9) == 2, "found %lld\n", (long long)(int)sizeof(((struct mdt_rec_resync *)0)->rs_padding9)); /* Checks for struct mdt_rec_reint */ @@ -3311,9 +3315,13 @@ void lustre_assert_wire_constants(void) (long long)(int)offsetof(struct mdt_rec_reint, rr_umask)); LASSERTF((int)sizeof(((struct mdt_rec_reint *)0)->rr_umask) == 4, "found %lld\n", (long long)(int)sizeof(((struct mdt_rec_reint *)0)->rr_umask)); - LASSERTF((int)offsetof(struct mdt_rec_reint, rr_padding_4) == 132, "found %lld\n", + LASSERTF((int)offsetof(struct mdt_rec_reint, rr_mirror_id) == 132, "found %lld\n", + (long long)(int)offsetof(struct mdt_rec_reint, rr_mirror_id)); + LASSERTF((int)sizeof(((struct mdt_rec_reint *)0)->rr_mirror_id) == 2, "found %lld\n", + (long long)(int)sizeof(((struct mdt_rec_reint *)0)->rr_mirror_id)); + LASSERTF((int)offsetof(struct mdt_rec_reint, rr_padding_4) == 134, "found %lld\n", (long long)(int)offsetof(struct mdt_rec_reint, rr_padding_4)); - LASSERTF((int)sizeof(((struct mdt_rec_reint *)0)->rr_padding_4) == 4, "found %lld\n", + LASSERTF((int)sizeof(((struct mdt_rec_reint *)0)->rr_padding_4) == 2, "found %lld\n", (long long)(int)sizeof(((struct mdt_rec_reint *)0)->rr_padding_4)); /* Checks for struct lmv_desc */