From 1fd63fcb045c91aa259068d3077b481652da0270 Mon Sep 17 00:00:00 2001
From: Alex Zhuravlev <bzzz@whamcloud.com>
Date: Tue, 19 Mar 2019 18:31:33 +0300
Subject: [PATCH] LU-12090 utils: lfs rmfid

a new RPC_REINT_RMFID has been introduced by the patch.
it's supposed to be used with corresponding llapi_rmfid()
to unlink a batch of MDS files by their FIDs. the caller
has to have permission to modify parent dir(s) and the objects
themselves.

Change-Id: I50421d85babc74d448842acea489321a5d40052d
Signed-off-by: Alex Zhuravlev <bzzz@whamcloud.com>
Reviewed-on: https://review.whamcloud.com/34449
Reviewed-by: Li Xi <lixi@ddn.com>
Tested-by: jenkins <devops@whamcloud.com>
Tested-by: Maloo <maloo@whamcloud.com>
Reviewed-by: Andreas Dilger <adilger@whamcloud.com>
Reviewed-by: Patrick Farrell <pfarrell@whamcloud.com>
Reviewed-by: Oleg Drokin <green@whamcloud.com>
---
 lustre/doc/lfs-rmfid.1                         |  25 +++
 lustre/doc/llapi_rmfid.3                       |  50 ++++++
 lustre/include/lustre/lustreapi.h              |   1 +
 lustre/include/lustre_req_layout.h             |   2 +
 lustre/include/obd.h                           |   2 +
 lustre/include/obd_class.h                     |  12 ++
 lustre/include/obd_support.h                   |   1 +
 lustre/include/uapi/linux/lustre/lustre_idl.h  |   1 +
 lustre/include/uapi/linux/lustre/lustre_user.h |  10 ++
 lustre/llite/dir.c                             |  50 ++++++
 lustre/lmv/lmv_obd.c                           |  91 ++++++++++
 lustre/mdc/mdc_request.c                       | 122 ++++++++++---
 lustre/mdt/mdt_coordinator.c                   |   2 +-
 lustre/mdt/mdt_handler.c                       | 214 +++++++++++++++++++++++
 lustre/mdt/mdt_internal.h                      |   1 +
 lustre/ptlrpc/layout.c                         |  25 +++
 lustre/ptlrpc/lproc_ptlrpc.c                   |   1 +
 lustre/ptlrpc/wiretest.c                       |   4 +-
 lustre/tests/sanity.sh                         | 228 +++++++++++++++++++++++++
 lustre/utils/lfs.c                             |  82 +++++++++
 lustre/utils/liblustreapi_util.c               |  25 +++
 lustre/utils/wiretest.c                        |   4 +-
 22 files changed, 927 insertions(+), 26 deletions(-)
 create mode 100644 lustre/doc/lfs-rmfid.1
 create mode 100644 lustre/doc/llapi_rmfid.3

diff --git a/lustre/doc/lfs-rmfid.1 b/lustre/doc/lfs-rmfid.1
new file mode 100644
index 0000000..38b5b62
--- /dev/null
+++ b/lustre/doc/lfs-rmfid.1
@@ -0,0 +1,25 @@
+.TH LFS-RMFID 1 2017-07-25 "Lustre" "Lustre Utilities"
+.SH NAME
+lfs rmfid \- remove file by FID
+.SH SYNOPSIS
+.B lfs rmfid
+<\fIdirectory\fR> <\fIFID1\fR> [<\fIFID2\fR>...]
+.SH DESCRIPTION
+This command removes file(s) specified by the \fIFID\fR.
+.br
+In some cases it is more optimal to remove files by their FIDs.
+The \fBdirectory\fR specifies a mountpoint of Lustre filesystem where given
+\fBFIDs\fR are stored. The mountpoint should be mounted with
+\fBuser_fid2path\fR mount option and the caller has to have a right to
+modify the given files. rmfid will be trying to remove all hardlinks of the
+given file. FIDs can be wrapped with square brackets.
+.SH EXAMPLES
+.TP
+.B lfs rmfid /mnt/lustre [0x200000400:0x1:0x0] [0x200000402:0x20:0x0]
+Remove files with FIDs [0x200000400:0x1:0x0] [0x200000402:0x20:0x0]
+.SH AUTHOR
+The \fBlfs rmfid\fR command is part of the Lustre filesystem.
+.SH SEE ALSO
+.BR lfs (1),
+.BR lfs-path2fid (1),
+.BR lfs-fid2path (1)
diff --git a/lustre/doc/llapi_rmfid.3 b/lustre/doc/llapi_rmfid.3
new file mode 100644
index 0000000..d49c9f4
--- /dev/null
+++ b/lustre/doc/llapi_rmfid.3
@@ -0,0 +1,50 @@
+.TH llapi_rmfid 3 "2014 Oct 13" "Lustre User API"
+.SH NAME
+llapi_rmfid \- Remove files by their FIDs in Lustre.
+.SH SYNOPSIS
+.nf
+.B #include <lustre/lustreapi.h>
+.PP
+.BI "int llapi_rmfid(const char *" path ", struct fid_array *" fa ");
+
+.sp
+.fi
+.SH DESCRIPTION
+.PP
+.BR llapi_rmfid()
+tries to remove
+.I fa->fa_nr
+Lustre files by FIDs stored in
+.I fa->fa_fids
+All file's hardlinks are subject to removal. This functionality is available
+only for root or regular users on filesystems mounted with
+.I user_fid2path
+mount option to delete files that they own and are in a directory in which
+they have write permission.
+
+.SH RETURN VALUES
+.LP
+.B llapi_rmfid()
+return 0 on success or a negative errno value on failure. Result for each file
+is stored in the corresponding
+.I fa->fa_fid[N].f_ver
+.SH ERRORS
+.TP 15
+.TP
+.SM -ENOENT
+.I file
+does not exist.
+.TP
+.SM -EBUSY
+file is open and can't be removed
+.TP
+.SM -EPERM
+The file cannot be open by user or CAP_DAC_READ_SEARCH is not granted.
+.TP
+.SM -EINVAL
+Invalid FID is passed
+.TP
+.SM -ENOMEM
+Not enough memory to process the request
+.SH "SEE ALSO"
+.BR lustreapi (7)
diff --git a/lustre/include/lustre/lustreapi.h b/lustre/include/lustre/lustreapi.h
index 97d10bb..0b6e81c 100644
--- a/lustre/include/lustre/lustreapi.h
+++ b/lustre/include/lustre/lustreapi.h
@@ -409,6 +409,7 @@ int llapi_path2parent(const char *path, unsigned int linkno,
 		      struct lu_fid *parent_fid, char *name, size_t name_size);
 int llapi_fd2parent(int fd, unsigned int linkno, struct lu_fid *parent_fid,
 		    char *name, size_t name_size);
+int llapi_rmfid(const char *path, struct fid_array *fa);
 int llapi_chomp_string(char *buf);
 int llapi_open_by_fid(const char *dir, const struct lu_fid *fid,
 		      int open_flags);
diff --git a/lustre/include/lustre_req_layout.h b/lustre/include/lustre_req_layout.h
index dd75309..8b2c924 100644
--- a/lustre/include/lustre_req_layout.h
+++ b/lustre/include/lustre_req_layout.h
@@ -179,6 +179,7 @@ extern struct req_format RQF_QUOTA_DQACQ;
 extern struct req_format RQF_MDS_SWAP_LAYOUTS;
 extern struct req_format RQF_MDS_REINT_MIGRATE;
 extern struct req_format RQF_MDS_REINT_RESYNC;
+extern struct req_format RQF_MDS_RMFID;
 /* MDS hsm formats */
 extern struct req_format RQF_MDS_HSM_STATE_GET;
 extern struct req_format RQF_MDS_HSM_STATE_SET;
@@ -256,6 +257,7 @@ extern struct req_msg_field RMF_IDX_INFO;
 extern struct req_msg_field RMF_CLOSE_DATA;
 extern struct req_msg_field RMF_FILE_SECCTX_NAME;
 extern struct req_msg_field RMF_FILE_SECCTX;
+extern struct req_msg_field RMF_FID_ARRAY;
 
 /*
  * connection handle received in MDS_CONNECT request.
diff --git a/lustre/include/obd.h b/lustre/include/obd.h
index 368dda1..f26b59f 100644
--- a/lustre/include/obd.h
+++ b/lustre/include/obd.h
@@ -1156,6 +1156,8 @@ struct md_ops {
 				  struct lu_fid *fid);
 	int (*m_unpackmd)(struct obd_export *exp, struct lmv_stripe_md **plsm,
 			  const union lmv_mds_md *lmv, size_t lmv_size);
+	int (*m_rmfid)(struct obd_export *exp, struct fid_array *fa, int *rcs,
+		       struct ptlrpc_request_set *set);
 };
 
 static inline struct md_open_data *obd_mod_alloc(void)
diff --git a/lustre/include/obd_class.h b/lustre/include/obd_class.h
index 9be6bc0..c7c0be7 100644
--- a/lustre/include/obd_class.h
+++ b/lustre/include/obd_class.h
@@ -1783,6 +1783,18 @@ static inline int md_unpackmd(struct obd_export *exp,
 	return MDP(exp->exp_obd, unpackmd)(exp, plsm, lmm, lmm_size);
 }
 
+static inline int md_rmfid(struct obd_export *exp, struct fid_array *fa,
+			   int *rcs, struct ptlrpc_request_set *set)
+{
+	int rc;
+
+	rc = exp_check_ops(exp);
+	if (rc)
+		return rc;
+
+	return MDP(exp->exp_obd, rmfid)(exp, fa, rcs, set);
+}
+
 /* OBD Metadata Support */
 
 extern int obd_init_caches(void);
diff --git a/lustre/include/obd_support.h b/lustre/include/obd_support.h
index 964bf35..73d3191 100644
--- a/lustre/include/obd_support.h
+++ b/lustre/include/obd_support.h
@@ -251,6 +251,7 @@ extern char obd_jobid_var[];
 #define OBD_FAIL_MDS_LOV_CREATE_RACE	 0x163
 #define OBD_FAIL_MDS_HSM_CDT_DELAY	 0x164
 #define OBD_FAIL_MDS_ORPHAN_DELETE	 0x165
+#define OBD_FAIL_MDS_RMFID_NET		 0x166
 
 /* layout lock */
 #define OBD_FAIL_MDS_NO_LL_GETATTR	 0x170
diff --git a/lustre/include/uapi/linux/lustre/lustre_idl.h b/lustre/include/uapi/linux/lustre/lustre_idl.h
index e5e9633..515b081 100644
--- a/lustre/include/uapi/linux/lustre/lustre_idl.h
+++ b/lustre/include/uapi/linux/lustre/lustre_idl.h
@@ -1643,6 +1643,7 @@ enum mds_cmd {
 	MDS_HSM_CT_REGISTER	= 59,
 	MDS_HSM_CT_UNREGISTER	= 60,
 	MDS_SWAP_LAYOUTS	= 61,
+	MDS_RMFID		= 62,
 	MDS_LAST_OPC
 };
 
diff --git a/lustre/include/uapi/linux/lustre/lustre_user.h b/lustre/include/uapi/linux/lustre/lustre_user.h
index bec3191..df8db0c 100644
--- a/lustre/include/uapi/linux/lustre/lustre_user.h
+++ b/lustre/include/uapi/linux/lustre/lustre_user.h
@@ -480,6 +480,7 @@ struct ll_ioc_lease_id {
 #define LL_IOC_LMV_SETSTRIPE		_IOWR('f', 240, struct lmv_user_md)
 #define LL_IOC_LMV_GETSTRIPE		_IOWR('f', 241, struct lmv_user_md)
 #define LL_IOC_REMOVE_ENTRY		_IOWR('f', 242, __u64)
+#define LL_IOC_RMFID			_IOR('f', 242, struct fid_array)
 #define LL_IOC_SET_LEASE		_IOWR('f', 243, struct ll_ioc_lease)
 #define LL_IOC_SET_LEASE_OLD		_IOWR('f', 243, long)
 #define LL_IOC_GET_LEASE		_IO('f', 244)
@@ -2405,6 +2406,15 @@ struct lu_pcc_state {
 	char	pccs_path[PATH_MAX];
 };
 
+struct fid_array {
+	__u32 fa_nr;
+	/* make header's size equal lu_fid */
+	__u32 fa_padding0;
+	__u64 fa_padding1;
+	struct lu_fid fa_fids[0];
+};
+#define OBD_MAX_FIDS_IN_ARRAY	4096
+
 #if defined(__cplusplus)
 }
 #endif
diff --git a/lustre/llite/dir.c b/lustre/llite/dir.c
index 57db3ac..9b69fdf 100644
--- a/lustre/llite/dir.c
+++ b/lustre/llite/dir.c
@@ -1229,6 +1229,54 @@ out:
         RETURN(rc);
 }
 
+int ll_rmfid(struct file *file, void __user *arg)
+{
+	const struct fid_array __user *ufa = arg;
+	struct fid_array *lfa = NULL;
+	size_t size;
+	unsigned nr;
+	int i, rc, *rcs = NULL;
+	ENTRY;
+
+	if (!cfs_capable(CFS_CAP_DAC_READ_SEARCH) &&
+	    !(ll_i2sbi(file_inode(file))->ll_flags & LL_SBI_USER_FID2PATH))
+		RETURN(-EPERM);
+	/* Only need to get the buflen */
+	if (get_user(nr, &ufa->fa_nr))
+		RETURN(-EFAULT);
+	/* DoS protection */
+	if (nr > OBD_MAX_FIDS_IN_ARRAY)
+		RETURN(-E2BIG);
+
+	size = offsetof(struct fid_array, fa_fids[nr]);
+	OBD_ALLOC(lfa, size);
+	if (!lfa)
+		RETURN(-ENOMEM);
+	OBD_ALLOC(rcs, sizeof(int) * nr);
+	if (!rcs)
+		GOTO(free_lfa, rc = -ENOMEM);
+
+	if (copy_from_user(lfa, arg, size))
+		GOTO(free_rcs, rc = -EFAULT);
+
+	/* Call mdc_iocontrol */
+	rc = md_rmfid(ll_i2mdexp(file_inode(file)), lfa, rcs, NULL);
+	if (!rc) {
+		for (i = 0; i < nr; i++)
+			if (rcs[i])
+				lfa->fa_fids[i].f_ver = rcs[i];
+		if (copy_to_user(arg, lfa, size))
+			rc = -EFAULT;
+	}
+
+free_rcs:
+	OBD_FREE(rcs, sizeof(int) * nr);
+free_lfa:
+	OBD_FREE(lfa, size);
+
+	RETURN(rc);
+}
+
 /* This function tries to get a single name component,
  * to send to the server. No actual path traversal involved,
  * so we limit to NAME_MAX */
@@ -1603,6 +1651,8 @@ out_rmdir:
                         ll_putname(filename);
 		RETURN(rc);
 	}
+	case LL_IOC_RMFID:
+		RETURN(ll_rmfid(file, (void __user *)arg));
 	case LL_IOC_LOV_SWAP_LAYOUTS:
 		RETURN(-EPERM);
 	case IOC_OBD_STATFS:
diff --git a/lustre/lmv/lmv_obd.c b/lustre/lmv/lmv_obd.c
index 4cd844f..73fece5 100644
--- a/lustre/lmv/lmv_obd.c
+++ b/lustre/lmv/lmv_obd.c
@@ -2981,6 +2981,96 @@ static int lmv_get_info(const struct lu_env *env, struct obd_export *exp,
         RETURN(-EINVAL);
 }
 
+static int lmv_rmfid(struct obd_export *exp, struct fid_array *fa,
+		     int *__rcs, struct ptlrpc_request_set *_set)
+{
+	struct obd_device *obddev = class_exp2obd(exp);
+	struct ptlrpc_request_set *set = _set;
+	struct lmv_obd *lmv = &obddev->u.lmv;
+	int tgt_count = lmv->desc.ld_tgt_count;
+	struct fid_array *fat, **fas = NULL;
+	int i, rc, **rcs = NULL;
+
+	if (!set) {
+		set = ptlrpc_prep_set();
+		if (!set)
+			RETURN(-ENOMEM);
+	}
+
+	/* split FIDs by targets */
+	OBD_ALLOC(fas, sizeof(fas) * tgt_count);
+	if (fas == NULL)
+		GOTO(out, rc = -ENOMEM);
+	OBD_ALLOC(rcs, sizeof(int *) * tgt_count);
+	if (rcs == NULL)
+		GOTO(out_fas, rc = -ENOMEM);
+
+	for (i = 0; i < fa->fa_nr; i++) {
+		unsigned int idx;
+
+		rc = lmv_fld_lookup(lmv, &fa->fa_fids[i], &idx);
+		if (rc) {
+			CDEBUG(D_OTHER, "can't lookup "DFID": rc = %d\n",
+			       PFID(&fa->fa_fids[i]), rc);
+			continue;
+		}
+		LASSERT(idx < tgt_count);
+		if (!fas[idx])
+			OBD_ALLOC(fas[idx], offsetof(struct fid_array,
+				  fa_fids[fa->fa_nr]));
+		if (!fas[idx])
+			GOTO(out, rc = -ENOMEM);
+		if (!rcs[idx])
+			OBD_ALLOC(rcs[idx], sizeof(int) * fa->fa_nr);
+		if (!rcs[idx])
+			GOTO(out, rc = -ENOMEM);
+
+		fat = fas[idx];
+		fat->fa_fids[fat->fa_nr++] = fa->fa_fids[i];
+	}
+
+	for (i = 0; i < tgt_count; i++) {
+		fat = fas[i];
+		if (!fat || fat->fa_nr == 0)
+			continue;
+		rc = md_rmfid(lmv->tgts[i]->ltd_exp, fat, rcs[i], set);
+	}
+
+	rc = ptlrpc_set_wait(NULL, set);
+	if (rc == 0) {
+		int j = 0;
+		for (i = 0; i < tgt_count; i++) {
+			fat = fas[i];
+			if (!fat || fat->fa_nr == 0)
+				continue;
+			/* copy FIDs back */
+			memcpy(fa->fa_fids + j, fat->fa_fids,
+			       fat->fa_nr * sizeof(struct lu_fid));
+			/* copy rcs back */
+			memcpy(__rcs + j, rcs[i], fat->fa_nr * sizeof(**rcs));
+			j += fat->fa_nr;
+		}
+	}
+	if (set != _set)
+		ptlrpc_set_destroy(set);
+
+out:
+	for (i = 0; i < tgt_count; i++) {
+		if (fas && fas[i])
+			OBD_FREE(fas[i], offsetof(struct fid_array,
+						fa_fids[fa->fa_nr]));
+		if (rcs && rcs[i])
+			OBD_FREE(rcs[i], sizeof(int) * fa->fa_nr);
+	}
+	if (rcs)
+		OBD_FREE(rcs, sizeof(int *) * tgt_count);
+out_fas:
+	if (fas)
+		OBD_FREE(fas, sizeof(fas) * tgt_count);
+
+	RETURN(rc);
+}
+
 /**
  * Asynchronously set by key a value associated with a LMV device.
  *
@@ -3587,6 +3677,7 @@ struct md_ops lmv_md_ops = {
 	.m_revalidate_lock      = lmv_revalidate_lock,
 	.m_get_fid_from_lsm	= lmv_get_fid_from_lsm,
 	.m_unpackmd		= lmv_unpackmd,
+	.m_rmfid		= lmv_rmfid,
 };
 
 static int __init lmv_init(void)
diff --git a/lustre/mdc/mdc_request.c b/lustre/mdc/mdc_request.c
index 47bf7ec..ad1327c 100644
--- a/lustre/mdc/mdc_request.c
+++ b/lustre/mdc/mdc_request.c
@@ -2574,6 +2574,81 @@ static int mdc_fsync(struct obd_export *exp, const struct lu_fid *fid,
         RETURN(rc);
 }
 
+struct mdc_rmfid_args {
+	int *mra_rcs;
+	int mra_nr;
+};
+
+int mdc_rmfid_interpret(const struct lu_env *env, struct ptlrpc_request *req,
+			  void *args, int rc)
+{
+	struct mdc_rmfid_args *aa;
+	int *rcs, size;
+	ENTRY;
+
+	if (!rc) {
+		aa = ptlrpc_req_async_args(req);
+
+		size = req_capsule_get_size(&req->rq_pill, &RMF_RCS,
+					    RCL_SERVER);
+		LASSERT(size == sizeof(int) * aa->mra_nr);
+		rcs = req_capsule_server_get(&req->rq_pill, &RMF_RCS);
+		LASSERT(rcs);
+		LASSERT(aa->mra_rcs);
+		LASSERT(aa->mra_nr);
+		memcpy(aa->mra_rcs, rcs, size);
+	}
+
+	RETURN(rc);
+}
+
+static int mdc_rmfid(struct obd_export *exp, struct fid_array *fa,
+		     int *rcs, struct ptlrpc_request_set *set)
+{
+	struct ptlrpc_request *req;
+	struct mdc_rmfid_args *aa;
+	struct mdt_body *b;
+	struct lu_fid *tmp;
+	int rc, flen;
+	ENTRY;
+
+	req = ptlrpc_request_alloc(class_exp2cliimp(exp), &RQF_MDS_RMFID);
+	if (req == NULL)
+		RETURN(-ENOMEM);
+
+	flen = fa->fa_nr * sizeof(struct lu_fid);
+	req_capsule_set_size(&req->rq_pill, &RMF_FID_ARRAY,
+			     RCL_CLIENT, flen);
+	req_capsule_set_size(&req->rq_pill, &RMF_FID_ARRAY,
+			     RCL_SERVER, flen);
+	req_capsule_set_size(&req->rq_pill, &RMF_RCS,
+			     RCL_SERVER, fa->fa_nr * sizeof(__u32));
+	rc = ptlrpc_request_pack(req, LUSTRE_MDS_VERSION, MDS_RMFID);
+	if (rc) {
+		ptlrpc_request_free(req);
+		RETURN(rc);
+	}
+	tmp = req_capsule_client_get(&req->rq_pill, &RMF_FID_ARRAY);
+	memcpy(tmp, fa->fa_fids, flen);
+
+	mdc_pack_body(req, NULL, 0, 0, -1, 0);
+	b = req_capsule_client_get(&req->rq_pill, &RMF_MDT_BODY);
+	b->mbo_ctime = ktime_get_real_seconds();
+
+	ptlrpc_request_set_replen(req);
+
+	LASSERT(rcs);
+	aa = ptlrpc_req_async_args(req);
+	aa->mra_rcs = rcs;
+	aa->mra_nr = fa->fa_nr;
+	req->rq_interpret_reply = mdc_rmfid_interpret;
+
+	ptlrpc_set_add_req(set, req);
+	ptlrpc_check_set(NULL, set);
+
+	RETURN(rc);
+}
+
 static int mdc_import_event(struct obd_device *obd, struct obd_import *imp,
 			    enum obd_import_event event)
 {
@@ -2856,32 +2931,33 @@ static struct obd_ops mdc_obd_ops = {
 
 static struct md_ops mdc_md_ops = {
 	.m_get_root	    = mdc_get_root,
-        .m_null_inode	    = mdc_null_inode,
-        .m_close            = mdc_close,
-        .m_create           = mdc_create,
-        .m_enqueue          = mdc_enqueue,
-        .m_getattr          = mdc_getattr,
-        .m_getattr_name     = mdc_getattr_name,
-        .m_intent_lock      = mdc_intent_lock,
-        .m_link             = mdc_link,
-        .m_rename           = mdc_rename,
-        .m_setattr          = mdc_setattr,
-        .m_setxattr         = mdc_setxattr,
-        .m_getxattr         = mdc_getxattr,
+	.m_null_inode	    = mdc_null_inode,
+	.m_close            = mdc_close,
+	.m_create           = mdc_create,
+	.m_enqueue          = mdc_enqueue,
+	.m_getattr          = mdc_getattr,
+	.m_getattr_name     = mdc_getattr_name,
+	.m_intent_lock      = mdc_intent_lock,
+	.m_link             = mdc_link,
+	.m_rename           = mdc_rename,
+	.m_setattr          = mdc_setattr,
+	.m_setxattr         = mdc_setxattr,
+	.m_getxattr         = mdc_getxattr,
 	.m_fsync		= mdc_fsync,
 	.m_file_resync		= mdc_file_resync,
 	.m_read_page		= mdc_read_page,
-        .m_unlink           = mdc_unlink,
-        .m_cancel_unused    = mdc_cancel_unused,
-        .m_init_ea_size     = mdc_init_ea_size,
-        .m_set_lock_data    = mdc_set_lock_data,
-        .m_lock_match       = mdc_lock_match,
-        .m_get_lustre_md    = mdc_get_lustre_md,
-        .m_free_lustre_md   = mdc_free_lustre_md,
-        .m_set_open_replay_data = mdc_set_open_replay_data,
-        .m_clear_open_replay_data = mdc_clear_open_replay_data,
-        .m_intent_getattr_async = mdc_intent_getattr_async,
-        .m_revalidate_lock      = mdc_revalidate_lock
+	.m_unlink           = mdc_unlink,
+	.m_cancel_unused    = mdc_cancel_unused,
+	.m_init_ea_size     = mdc_init_ea_size,
+	.m_set_lock_data    = mdc_set_lock_data,
+	.m_lock_match       = mdc_lock_match,
+	.m_get_lustre_md    = mdc_get_lustre_md,
+	.m_free_lustre_md   = mdc_free_lustre_md,
+	.m_set_open_replay_data = mdc_set_open_replay_data,
+	.m_clear_open_replay_data = mdc_clear_open_replay_data,
+	.m_intent_getattr_async = mdc_intent_getattr_async,
+	.m_revalidate_lock      = mdc_revalidate_lock,
+	.m_rmfid		= mdc_rmfid,
 };
 
 static int __init mdc_init(void)
diff --git a/lustre/mdt/mdt_coordinator.c b/lustre/mdt/mdt_coordinator.c
index a23e536..c9bfb19 100644
--- a/lustre/mdt/mdt_coordinator.c
+++ b/lustre/mdt/mdt_coordinator.c
@@ -970,7 +970,7 @@ static int mdt_hsm_pending_restore(struct mdt_thread_info *mti)
 	RETURN(rc);
 }
 
-static int hsm_init_ucred(struct lu_ucred *uc)
+int hsm_init_ucred(struct lu_ucred *uc)
 {
 	ENTRY;
 
diff --git a/lustre/mdt/mdt_handler.c b/lustre/mdt/mdt_handler.c
index 9167c83..5ae5c63 100644
--- a/lustre/mdt/mdt_handler.c
+++ b/lustre/mdt/mdt_handler.c
@@ -1974,6 +1974,219 @@ out_shrink:
 	return rc;
 }
 
+static int mdt_rmfid_unlink(struct mdt_thread_info *info,
+			    const struct lu_fid *pfid,
+			    const struct lu_name *name,
+			    struct mdt_object *obj, s64 ctime)
+{
+	struct lu_fid *child_fid = &info->mti_tmp_fid1;
+	struct ldlm_enqueue_info *einfo = &info->mti_einfo[0];
+	struct mdt_device *mdt = info->mti_mdt;
+	struct md_attr *ma = &info->mti_attr;
+	struct mdt_lock_handle *parent_lh;
+	struct mdt_lock_handle *child_lh;
+	struct mdt_object *pobj;
+	bool cos_incompat = false;
+	int rc;
+	ENTRY;
+
+	pobj = mdt_object_find(info->mti_env, mdt, pfid);
+	if (IS_ERR(pobj))
+		GOTO(out, rc = PTR_ERR(pobj));
+
+	parent_lh = &info->mti_lh[MDT_LH_PARENT];
+	mdt_lock_pdo_init(parent_lh, LCK_PW, name);
+	rc = mdt_object_lock(info, pobj, parent_lh, MDS_INODELOCK_UPDATE);
+	if (rc != 0)
+		GOTO(put_parent, rc);
+
+	if (mdt_object_remote(pobj))
+		cos_incompat = true;
+
+	rc = mdo_lookup(info->mti_env, mdt_object_child(pobj),
+			name, child_fid, &info->mti_spec);
+	if (rc != 0)
+		GOTO(unlock_parent, rc);
+
+	if (!lu_fid_eq(child_fid, mdt_object_fid(obj)))
+		GOTO(unlock_parent, rc = -EREMCHG);
+
+	child_lh = &info->mti_lh[MDT_LH_CHILD];
+	mdt_lock_reg_init(child_lh, LCK_EX);
+	rc = mdt_reint_striped_lock(info, obj, child_lh,
+				    MDS_INODELOCK_LOOKUP | MDS_INODELOCK_UPDATE,
+				    einfo, cos_incompat);
+	if (rc != 0)
+		GOTO(unlock_parent, rc);
+
+	if (atomic_read(&obj->mot_open_count)) {
+		CDEBUG(D_OTHER, "object "DFID" open, skip\n",
+		       PFID(mdt_object_fid(obj)));
+		GOTO(unlock_child, rc = -EBUSY);
+	}
+
+	ma->ma_need = 0;
+	ma->ma_valid = MA_INODE;
+	ma->ma_attr.la_valid = LA_CTIME;
+	ma->ma_attr.la_ctime = ctime;
+
+	mutex_lock(&obj->mot_lov_mutex);
+
+	rc = mdo_unlink(info->mti_env, mdt_object_child(pobj),
+			mdt_object_child(obj), name, ma, 0);
+
+	mutex_unlock(&obj->mot_lov_mutex);
+
+unlock_child:
+	mdt_reint_striped_unlock(info, obj, child_lh, einfo, 1);
+unlock_parent:
+	mdt_object_unlock(info, pobj, parent_lh, 1);
+put_parent:
+	mdt_object_put(info->mti_env, pobj);
+out:
+	RETURN(rc);
+}
+
+static int mdt_rmfid_check_permission(struct mdt_thread_info *info,
+					struct mdt_object *obj)
+{
+	struct lu_ucred *uc = lu_ucred(info->mti_env);
+	struct md_attr *ma = &info->mti_attr;
+	struct lu_attr *la = &ma->ma_attr;
+	int rc = 0;
+	ENTRY;
+
+	ma->ma_need = MA_INODE;
+	rc = mo_attr_get(info->mti_env, mdt_object_child(obj), ma);
+	if (rc)
+		GOTO(out, rc);
+
+	if (la->la_flags & LUSTRE_IMMUTABLE_FL)
+			rc = -EACCES;
+
+	if (md_capable(uc, CFS_CAP_DAC_OVERRIDE))
+		RETURN(0);
+	if (uc->uc_fsuid == la->la_uid) {
+		if ((la->la_mode & S_IWUSR) == 0)
+			rc = -EACCES;
+	} else if (uc->uc_fsgid == la->la_gid) {
+		if ((la->la_mode & S_IWGRP) == 0)
+			rc = -EACCES;
+	} else if ((la->la_mode & S_IWOTH) == 0) {
+			rc = -EACCES;
+	}
+
+out:
+	RETURN(rc);
+}
+
+static int mdt_rmfid_one(struct mdt_thread_info *info, struct lu_fid *fid,
+			 s64 ctime)
+{
+	struct mdt_device *mdt = info->mti_mdt;
+	struct mdt_object *obj = NULL;
+	struct linkea_data ldata = { NULL };
+	struct lu_buf *buf = &info->mti_big_buf;
+	struct lu_name *name = &info->mti_name;
+	struct lu_fid *pfid = &info->mti_tmp_fid1;
+	struct link_ea_header *leh;
+	struct link_ea_entry *lee;
+	int reclen, count, rc = 0;
+	ENTRY;
+
+	if (!fid_is_sane(fid))
+		GOTO(out, rc = -EINVAL);
+
+	if (!fid_is_namespace_visible(fid))
+		GOTO(out, rc = -EINVAL);
+
+	obj = mdt_object_find(info->mti_env, mdt, fid);
+	if (IS_ERR(obj))
+		GOTO(out, rc = PTR_ERR(obj));
+
+	if (mdt_object_remote(obj))
+		GOTO(out, rc = -EREMOTE);
+	if (!mdt_object_exists(obj) || lu_object_is_dying(&obj->mot_header))
+		GOTO(out, rc = -ENOENT);
+
+	rc = mdt_rmfid_check_permission(info, obj);
+	if (rc)
+		GOTO(out, rc);
+
+	/* take LinkEA */
+	buf = lu_buf_check_and_alloc(buf, PATH_MAX);
+	if (!buf->lb_buf)
+		GOTO(out, rc = -ENOMEM);
+
+	ldata.ld_buf = buf;
+	rc = mdt_links_read(info, obj, &ldata);
+	if (rc)
+		GOTO(out, rc);
+
+	leh = buf->lb_buf;
+	lee = (struct link_ea_entry *)(leh + 1);
+	for (count = 0; count < leh->leh_reccount; count++) {
+		/* remove every hardlink */
+		linkea_entry_unpack(lee, &reclen, name, pfid);
+		lee = (struct link_ea_entry *) ((char *)lee + reclen);
+		rc = mdt_rmfid_unlink(info, pfid, name, obj, ctime);
+		if (rc)
+			break;
+	}
+
+out:
+	if (obj && !IS_ERR(obj))
+		mdt_object_put(info->mti_env, obj);
+	if (info->mti_big_buf.lb_buf)
+		lu_buf_free(&info->mti_big_buf);
+
+	RETURN(rc);
+}
+
+static int mdt_rmfid(struct tgt_session_info *tsi)
+{
+	struct mdt_thread_info *mti = tsi2mdt_info(tsi);
+	struct mdt_body *reqbody;
+	struct lu_fid *fids, *rfids;
+	int bufsize, rc;
+	__u32 *rcs;
+	int i, nr;
+	ENTRY;
+
+	reqbody = req_capsule_client_get(tsi->tsi_pill, &RMF_MDT_BODY);
+	if (reqbody == NULL)
+		RETURN(-EPROTO);
+	bufsize = req_capsule_get_size(tsi->tsi_pill, &RMF_FID_ARRAY,
+				       RCL_CLIENT);
+	nr = bufsize / sizeof(struct lu_fid);
+	if (nr * sizeof(struct lu_fid) != bufsize)
+		RETURN(-EINVAL);
+	req_capsule_set_size(tsi->tsi_pill, &RMF_RCS,
+			     RCL_SERVER, nr * sizeof(__u32));
+	req_capsule_set_size(tsi->tsi_pill, &RMF_FID_ARRAY,
+			     RCL_SERVER, nr * sizeof(struct lu_fid));
+	rc = req_capsule_server_pack(tsi->tsi_pill);
+	if (rc)
+		GOTO(out, rc = err_serious(rc));
+	fids = req_capsule_client_get(tsi->tsi_pill, &RMF_FID_ARRAY);
+	if (fids == NULL)
+		RETURN(-EPROTO);
+	rcs = req_capsule_server_get(tsi->tsi_pill, &RMF_RCS);
+	LASSERT(rcs);
+	rfids = req_capsule_server_get(tsi->tsi_pill, &RMF_FID_ARRAY);
+	LASSERT(rfids);
+
+	mdt_init_ucred(mti, reqbody);
+	for (i = 0; i < nr; i++) {
+		rfids[i] = fids[i];
+		rcs[i] = mdt_rmfid_one(mti, fids + i, reqbody->mbo_ctime);
+	}
+	mdt_exit_ucred(mti);
+
+out:
+	RETURN(rc);
+}
+
 static int mdt_iocontrol(unsigned int cmd, struct obd_export *exp, int len,
 			 void *karg, void __user *uarg);
 
@@ -4941,6 +5154,7 @@ TGT_MDT_HDL(HAS_BODY | HAS_REPLY, MDS_HSM_REQUEST,
 TGT_MDT_HDL(HAS_KEY | HAS_BODY | HAS_REPLY | IS_MUTABLE,
 	    MDS_SWAP_LAYOUTS,
 	    mdt_swap_layouts),
+TGT_MDT_HDL(IS_MUTABLE,		MDS_RMFID,	mdt_rmfid),
 };
 
 static struct tgt_handler mdt_io_ops[] = {
diff --git a/lustre/mdt/mdt_internal.h b/lustre/mdt/mdt_internal.h
index f6a04f1..a9b68d1 100644
--- a/lustre/mdt/mdt_internal.h
+++ b/lustre/mdt/mdt_internal.h
@@ -926,6 +926,7 @@ int mdt_intent_lock_replace(struct mdt_thread_info *info,
 			    struct mdt_lock_handle *lh,
 			    __u64 flags, int result);
 
+int hsm_init_ucred(struct lu_ucred *uc);
 int mdt_hsm_attr_set(struct mdt_thread_info *info, struct mdt_object *obj,
 		     const struct md_hsm *mh);
 
diff --git a/lustre/ptlrpc/layout.c b/lustre/ptlrpc/layout.c
index 7bada20..3324543 100644
--- a/lustre/ptlrpc/layout.c
+++ b/lustre/ptlrpc/layout.c
@@ -337,6 +337,21 @@ static const struct req_msg_field *mdt_swap_layouts[] = {
 	&RMF_DLM_REQ
 };
 
+static const struct req_msg_field *mds_rmfid_client[] = {
+	&RMF_PTLRPC_BODY,
+	&RMF_MDT_BODY,
+	&RMF_FID_ARRAY,
+	&RMF_CAPA1,
+	&RMF_CAPA2,
+};
+
+static const struct req_msg_field *mds_rmfid_server[] = {
+	&RMF_PTLRPC_BODY,
+	&RMF_MDT_BODY,
+	&RMF_FID_ARRAY,
+	&RMF_RCS,
+};
+
 static const struct req_msg_field *obd_connect_client[] = {
 	&RMF_PTLRPC_BODY,
 	&RMF_TGTUUID,
@@ -784,6 +799,7 @@ static struct req_format *req_formats[] = {
 	&RQF_MDS_HSM_ACTION,
 	&RQF_MDS_HSM_REQUEST,
 	&RQF_MDS_SWAP_LAYOUTS,
+	&RQF_MDS_RMFID,
 	&RQF_OUT_UPDATE,
 	&RQF_OST_CONNECT,
 	&RQF_OST_DISCONNECT,
@@ -994,6 +1010,10 @@ struct req_msg_field RMF_NAME =
         DEFINE_MSGF("name", RMF_F_STRING, -1, NULL, NULL);
 EXPORT_SYMBOL(RMF_NAME);
 
+struct req_msg_field RMF_FID_ARRAY =
+	DEFINE_MSGF("fid_array", 0, -1, NULL, NULL);
+EXPORT_SYMBOL(RMF_FID_ARRAY);
+
 struct req_msg_field RMF_SYMTGT =
         DEFINE_MSGF("symtgt", RMF_F_STRING, -1, NULL, NULL);
 EXPORT_SYMBOL(RMF_SYMTGT);
@@ -1622,6 +1642,11 @@ struct req_format RQF_MDS_SWAP_LAYOUTS =
 			mdt_swap_layouts, empty);
 EXPORT_SYMBOL(RQF_MDS_SWAP_LAYOUTS);
 
+struct req_format RQF_MDS_RMFID =
+	DEFINE_REQ_FMT0("MDS_RMFID", mds_rmfid_client,
+			mds_rmfid_server);
+EXPORT_SYMBOL(RQF_MDS_RMFID);
+
 struct req_format RQF_LLOG_ORIGIN_HANDLE_CREATE =
         DEFINE_REQ_FMT0("LLOG_ORIGIN_HANDLE_CREATE",
                         llog_origin_handle_create_client, llogd_body_only);
diff --git a/lustre/ptlrpc/lproc_ptlrpc.c b/lustre/ptlrpc/lproc_ptlrpc.c
index 5929fe3..546aa3c 100644
--- a/lustre/ptlrpc/lproc_ptlrpc.c
+++ b/lustre/ptlrpc/lproc_ptlrpc.c
@@ -95,6 +95,7 @@ static struct ll_rpc_opcode {
 	{ MDS_HSM_CT_REGISTER, "mds_hsm_ct_register" },
 	{ MDS_HSM_CT_UNREGISTER, "mds_hsm_ct_unregister" },
 	{ MDS_SWAP_LAYOUTS,	"mds_swap_layouts" },
+	{ MDS_RMFID,		"mds_rmfid" },
         { LDLM_ENQUEUE,     "ldlm_enqueue" },
         { LDLM_CONVERT,     "ldlm_convert" },
         { LDLM_CANCEL,      "ldlm_cancel" },
diff --git a/lustre/ptlrpc/wiretest.c b/lustre/ptlrpc/wiretest.c
index 9d94ec5..c7f1681 100644
--- a/lustre/ptlrpc/wiretest.c
+++ b/lustre/ptlrpc/wiretest.c
@@ -175,7 +175,9 @@ void lustre_assert_wire_constants(void)
 		 (long long)MDS_HSM_CT_UNREGISTER);
 	LASSERTF(MDS_SWAP_LAYOUTS == 61, "found %lld\n",
 		 (long long)MDS_SWAP_LAYOUTS);
-	LASSERTF(MDS_LAST_OPC == 62, "found %lld\n",
+	LASSERTF(MDS_RMFID == 62, "found %lld\n",
+		 (long long)MDS_RMFID);
+	LASSERTF(MDS_LAST_OPC == 63, "found %lld\n",
 		 (long long)MDS_LAST_OPC);
 	LASSERTF(REINT_SETATTR == 1, "found %lld\n",
 		 (long long)REINT_SETATTR);
diff --git a/lustre/tests/sanity.sh b/lustre/tests/sanity.sh
index 1b7f3e3..974b52d 100644
--- a/lustre/tests/sanity.sh
+++ b/lustre/tests/sanity.sh
@@ -20444,6 +20444,234 @@ test_420()
 }
 run_test 420 "clear SGID bit on non-directories for non-members"
 
+test_421a() {
+	local cnt
+	local fid1
+	local fid2
+
+	[ $MDS1_VERSION -lt $(version_code 2.12.54) ] &&
+		skip "Need MDS version at least 2.12.54"
+
+	test_mkdir $DIR/$tdir
+	createmany -o $DIR/$tdir/f 3
+	cnt=$(ls -1 $DIR/$tdir | wc -l)
+	[ $cnt != 3 ] && error "unexpected #files: $cnt"
+
+	fid1=$(lfs path2fid $DIR/$tdir/f1)
+	fid2=$(lfs path2fid $DIR/$tdir/f2)
+	$LFS rmfid $DIR $fid1 $fid2 || error "rmfid failed"
+
+	stat $DIR/$tdir/f1 && error "f1 still visible on the client"
+	stat $DIR/$tdir/f2 && error "f2 still visible on the client"
+
+	cnt=$(ls -1 $DIR/$tdir | wc -l)
+	[ $cnt == 1 ] || error "unexpected #files after: $cnt"
+
+	rm -f $DIR/$tdir/f3 || error "can't remove f3"
+	createmany -o $DIR/$tdir/f 3
+	cnt=$(ls -1 $DIR/$tdir | wc -l)
+	[ $cnt != 3 ] && error "unexpected #files: $cnt"
+
+	fid1=$(lfs path2fid $DIR/$tdir/f1)
+	fid2=$(lfs path2fid $DIR/$tdir/f2)
+	echo "remove using fsname $FSNAME"
+	$LFS rmfid $FSNAME $fid1 $fid2 || error "rmfid with fsname failed"
+
+	cnt=$(ls -1 $DIR/$tdir | wc -l)
+	[ $cnt == 1 ] || error "unexpected #files after: $cnt"
+}
+run_test 421a "simple rm by fid"
+
+test_421b() {
+	local cnt
+	local FID1
+	local FID2
+
+	[ $MDS1_VERSION -lt $(version_code 2.12.54) ] &&
+		skip "Need MDS version at least 2.12.54"
+
+	test_mkdir $DIR/$tdir
+	createmany -o $DIR/$tdir/f 3
+	multiop_bg_pause $DIR/$tdir/f1 o_c || error "multiop failed to start"
+	MULTIPID=$!
+
+	FID1=$(lfs path2fid $DIR/$tdir/f1)
+	FID2=$(lfs path2fid $DIR/$tdir/f2)
+	$LFS rmfid $DIR $FID1 $FID2 && error "rmfid didn't fail"
+
+	kill -USR1 $MULTIPID
+	wait
+
+	cnt=$(ls $DIR/$tdir | wc -l)
+	[ $cnt == 2 ] || error "unexpected #files after: $cnt"
+}
+run_test 421b "rm by fid on open file"
+
+test_421c() {
+	local cnt
+	local FIDS
+
+	[ $MDS1_VERSION -lt $(version_code 2.12.54) ] &&
+		skip "Need MDS version at least 2.12.54"
+
+	test_mkdir $DIR/$tdir
+	createmany -o $DIR/$tdir/f 3
+	touch $DIR/$tdir/$tfile
+	createmany -l$DIR/$tdir/$tfile $DIR/$tdir/h 180
+	cnt=$(ls -1 $DIR/$tdir | wc -l)
+	[ $cnt != 184 ] && error "unexpected #files: $cnt"
+
+	FID1=$(lfs path2fid $DIR/$tdir/$tfile)
+	$LFS rmfid $DIR $FID1 || error "rmfid failed"
+
+	cnt=$(ls $DIR/$tdir | wc -l)
+	[ $cnt == 3 ] || error "unexpected #files after: $cnt"
+}
+run_test 421c "rm by fid against hardlinked files"
+
+test_421d() {
+	local cnt
+	local FIDS
+
+	[ $MDS1_VERSION -lt $(version_code 2.12.54) ] &&
+		skip "Need MDS version at least 2.12.54"
+
+	test_mkdir $DIR/$tdir
+	createmany -o $DIR/$tdir/f 4097
+	cnt=$(ls -1 $DIR/$tdir | wc -l)
+	[ $cnt != 4097 ] && error "unexpected #files: $cnt"
+
+	FIDS=$(lfs path2fid $DIR/$tdir/f* | sed "s/[/][^:]*://g")
+	$LFS rmfid $DIR $FIDS || error "rmfid failed"
+
+	cnt=$(ls $DIR/$tdir | wc -l)
+	rm -rf $DIR/$tdir
+	[ $cnt == 0 ] || error "unexpected #files after: $cnt"
+}
+run_test 421d "rmfid en masse"
+
+test_421e() {
+	local cnt
+	local FID
+
+	[ $MDSCOUNT -lt 2 ] && skip "needs >= 2 MDTs"
+	[ $MDS1_VERSION -lt $(version_code 2.12.54) ] &&
+		skip "Need MDS version at least 2.12.54"
+
+	mkdir -p $DIR/$tdir
+	$LFS setdirstripe -c$MDSCOUNT $DIR/$tdir/striped_dir
+	createmany -o $DIR/$tdir/striped_dir/f 512
+	cnt=$(ls -1 $DIR/$tdir/striped_dir | wc -l)
+	[ $cnt != 512 ] && error "unexpected #files: $cnt"
+
+	FIDS=$(lfs path2fid $DIR/$tdir/striped_dir/f* |
+		sed "s/[/][^:]*://g")
+	$LFS rmfid $DIR $FIDS || error "rmfid failed"
+
+	cnt=$(ls $DIR/$tdir/striped_dir | wc -l)
+	rm -rf $DIR/$tdir
+	[ $cnt == 0 ] || error "unexpected #files after: $cnt"
+}
+run_test 421e "rmfid in DNE"
+
+test_421f() {
+	local cnt
+	local FID
+
+	[ $MDS1_VERSION -lt $(version_code 2.12.54) ] &&
+		skip "Need MDS version at least 2.12.54"
+
+	test_mkdir $DIR/$tdir
+	touch $DIR/$tdir/f
+	cnt=$(ls -1 $DIR/$tdir | wc -l)
+	[ $cnt != 1 ] && error "unexpected #files: $cnt"
+
+	FID=$(lfs path2fid $DIR/$tdir/f)
+	$RUNAS $LFS rmfid $DIR $FID && error "rmfid didn't fail (1)"
+	# rmfid should fail
+	cnt=$(ls -1 $DIR/$tdir | wc -l)
+	[ $cnt != 1 ] && error "unexpected #files after (2): $cnt"
+
+	chmod a+rw $DIR/$tdir
+	ls -la $DIR/$tdir
+	$RUNAS $LFS rmfid $DIR $FID && error "rmfid didn't fail (2)"
+	# rmfid should fail
+	cnt=$(ls -1 $DIR/$tdir | wc -l)
+	[ $cnt != 1 ] && error "unexpected #files after (3): $cnt"
+
+	rm -f $DIR/$tdir/f
+	$RUNAS touch $DIR/$tdir/f
+	FID=$(lfs path2fid $DIR/$tdir/f)
+	echo "rmfid as root"
+	$LFS rmfid $DIR $FID || error "rmfid as root failed"
+	cnt=$(ls -1 $DIR/$tdir | wc -l)
+	[ $cnt == 0 ] || error "unexpected #files after (4): $cnt"
+
+	rm -f $DIR/$tdir/f
+	$RUNAS touch $DIR/$tdir/f
+	cnt=$(ls -1 $DIR/$tdir | wc -l)
+	[ $cnt != 1 ] && error "unexpected #files (4): $cnt"
+	FID=$(lfs path2fid $DIR/$tdir/f)
+	# rmfid w/o user_fid2path mount option should fail
+	$RUNAS $LFS rmfid $DIR $FID && error "rmfid didn't fail(3)"
+	cnt=$(ls -1 $DIR/$tdir | wc -l)
+	[ $cnt == 1 ] || error "unexpected #files after (5): $cnt"
+
+	umount_client $MOUNT || "failed to umount client"
+	mount_client $MOUNT "$MOUNT_OPTS,user_fid2path" ||
+		"failed to mount client'"
+
+	$RUNAS $LFS rmfid $DIR $FID || error "rmfid failed"
+	# rmfid should succeed
+	cnt=$(ls -1 $DIR/$tdir | wc -l)
+	[ $cnt == 0 ] || error "unexpected #files after (6): $cnt"
+
+	# rmfid shouldn't allow to remove files due to dir's permission
+	chmod a+rwx $DIR/$tdir
+	touch $DIR/$tdir/f
+	ls -la $DIR/$tdir
+	FID=$(lfs path2fid $DIR/$tdir/f)
+	$RUNAS $LFS rmfid $DIR $FID && error "rmfid didn't fail"
+
+	umount_client $MOUNT || "failed to umount client"
+	mount_client $MOUNT "$MOUNT_OPTS" ||
+		"failed to mount client'"
+
+}
+run_test 421f "rmfid checks permissions"
+
+test_421g() {
+	local cnt
+	local FIDS
+
+	[ $MDSCOUNT -lt 2 ] && skip "needs >= 2 MDTs"
+	[ $MDS1_VERSION -lt $(version_code 2.12.54) ] &&
+		skip "Need MDS version at least 2.12.54"
+
+	mkdir -p $DIR/$tdir
+	$LFS setdirstripe -c$MDSCOUNT $DIR/$tdir/striped_dir
+	createmany -o $DIR/$tdir/striped_dir/f 512
+	cnt=$(ls -1 $DIR/$tdir/striped_dir | wc -l)
+	[ $cnt != 512 ] && error "unexpected #files: $cnt"
+
+	FIDS=$(lfs path2fid $DIR/$tdir/striped_dir/f* |
+		sed "s/[/][^:]*://g")
+
+	rm -f $DIR/$tdir/striped_dir/f1*
+	cnt=$(ls -1 $DIR/$tdir/striped_dir | wc -l)
+	removed=$((512 - cnt))
+
+	# few files have been just removed, so we expect
+	# rmfid to fail on their fids
+	errors=$($LFS rmfid $DIR $FIDS 2>&1 | wc -l)
+	[ $removed != $errors ] && error "$errors != $removed"
+
+	cnt=$(ls $DIR/$tdir/striped_dir | wc -l)
+	rm -rf $DIR/$tdir
+	[ $cnt == 0 ] || error "unexpected #files after: $cnt"
+}
+run_test 421g "rmfid to return errors properly"
+
 prep_801() {
 	[[ $(lustre_version_code mds1) -lt $(version_code 2.9.55) ]] ||
 	[[ $OST1_VERSION -lt $(version_code 2.9.55) ]] &&
diff --git a/lustre/utils/lfs.c b/lustre/utils/lfs.c
index f1af46a..d261f4d 100644
--- a/lustre/utils/lfs.c
+++ b/lustre/utils/lfs.c
@@ -103,6 +103,7 @@ static int lfs_changelog(int argc, char **argv);
 static int lfs_changelog_clear(int argc, char **argv);
 static int lfs_fid2path(int argc, char **argv);
 static int lfs_path2fid(int argc, char **argv);
+static int lfs_rmfid(int argc, char **argv);
 static int lfs_data_version(int argc, char **argv);
 static int lfs_hsm_state(int argc, char **argv);
 static int lfs_hsm_set(int argc, char **argv);
@@ -557,6 +558,8 @@ command_t cmdlist[] = {
 		/* [ --rec <recno> ] */ },
 	{"path2fid", lfs_path2fid, 0, "Display the fid(s) for a given path(s).\n"
 	 "usage: path2fid [--parents] <path> ..."},
+	{"rmfid", lfs_rmfid, 0, "Remove file(s) by FID(s)\n"
+	 "usage: rmfid <fsname|rootpath> <fid> ..."},
 	{"data_version", lfs_data_version, 0, "Display file data version for "
 	 "a given path.\n" "usage: data_version -[n|r|w] <path>"},
 	{"hsm_state", lfs_hsm_state, 0, "Display the HSM information (states, "
@@ -7701,6 +7704,85 @@ static int lfs_path2fid(int argc, char **argv)
 	return rc;
 }
 
+#define MAX_ERRNO	4095
+#define IS_ERR_VALUE(x) ((unsigned long)(x) >= (unsigned long)-MAX_ERRNO)
+
+static int lfs_rmfid_and_show_errors(const char *device, struct fid_array *fa)
+{
+	int rc, rc2 = 0, k;
+
+	rc = llapi_rmfid(device, fa);
+	if (rc) {
+		fprintf(stderr, "rmfid(): rc = %d\n", rc);
+		return rc;
+	}
+
+	for (k = 0; k < fa->fa_nr; k++) {
+		rc = (__s32)fa->fa_fids[k].f_ver;
+		if (!IS_ERR_VALUE(rc))
+			continue;
+		if (!rc2 && rc)
+			rc2 = rc;
+		if (!rc)
+			continue;
+		fa->fa_fids[k].f_ver = 0;
+		fprintf(stderr, "rmfid("DFID"): rc = %d\n",
+			PFID(&fa->fa_fids[k]), rc);
+	}
+
+	return rc2;
+}
+
+static int lfs_rmfid(int argc, char **argv)
+{
+	char *fidstr, *device;
+	int rc = 0, rc2, nr;
+	struct fid_array *fa;
+
+	if (optind > argc - 1) {
+		fprintf(stderr, "%s rmfid: missing dirname\n", progname);
+		return CMD_HELP;
+	}
+
+	device = argv[optind++];
+
+	nr = argc - optind;
+	fa = malloc(offsetof(struct fid_array, fa_fids[nr + 1]));
+	if (fa == NULL)
+		return -ENOMEM;
+
+	fa->fa_nr = 0;
+	rc = 0;
+	while (optind < argc) {
+		int found;
+
+		fidstr = argv[optind++];
+		while (*fidstr == '[')
+			fidstr++;
+		found = sscanf(fidstr, SFID, RFID(&fa->fa_fids[fa->fa_nr]));
+		if (found != 3) {
+			fprintf(stderr, "unrecognized FID: %s\n",
+				argv[optind - 1]);
+			exit(1);
+		}
+		fa->fa_nr++;
+		if (fa->fa_nr == OBD_MAX_FIDS_IN_ARRAY) {
+			/* start another batch */
+			rc2 = lfs_rmfid_and_show_errors(device, fa);
+			if (rc2 && !rc)
+				rc = rc2;
+			fa->fa_nr = 0;
+		}
+	}
+	if (fa->fa_nr) {
+		rc2 = lfs_rmfid_and_show_errors(device, fa);
+		if (rc2 && !rc)
+			rc = rc2;
+	}
+
+	return rc;
+}
+
 static int lfs_data_version(int argc, char **argv)
 {
 	char *path;
diff --git a/lustre/utils/liblustreapi_util.c b/lustre/utils/liblustreapi_util.c
index d206b02..59f7e98 100644
--- a/lustre/utils/liblustreapi_util.c
+++ b/lustre/utils/liblustreapi_util.c
@@ -34,6 +34,7 @@
 #include <stddef.h>
 #include <stdio.h>
 #include <stdlib.h>
+#include <sys/ioctl.h>
 #include <string.h>
 #include <unistd.h>
 #include <sys/time.h>
@@ -286,3 +287,27 @@ int llapi_search_ost(const char *fsname, const char *poolname,
 	return llapi_search_tgt(fsname, poolname, ostname, false);
 }
 
+int llapi_rmfid(const char *path, struct fid_array *fa)
+{
+	char rootpath[PATH_MAX];
+	int fd, rc;
+
+retry_open:
+	fd = open(path, O_RDONLY | O_NONBLOCK | O_NOFOLLOW);
+	if (fd < 0) {
+		if (errno == ENOENT && path != rootpath) {
+			rc = llapi_search_rootpath(rootpath, path);
+			if (!rc) {
+				path = rootpath;
+				goto retry_open;
+			}
+		} else {
+			return -errno;
+		}
+	}
+
+	rc = ioctl(fd, LL_IOC_RMFID, fa);
+	close(fd);
+
+	return rc ? -errno : 0;
+}
diff --git a/lustre/utils/wiretest.c b/lustre/utils/wiretest.c
index dda345b..f9e76c2 100644
--- a/lustre/utils/wiretest.c
+++ b/lustre/utils/wiretest.c
@@ -196,7 +196,9 @@ void lustre_assert_wire_constants(void)
 		 (long long)MDS_HSM_CT_UNREGISTER);
 	LASSERTF(MDS_SWAP_LAYOUTS == 61, "found %lld\n",
 		 (long long)MDS_SWAP_LAYOUTS);
-	LASSERTF(MDS_LAST_OPC == 62, "found %lld\n",
+	LASSERTF(MDS_RMFID == 62, "found %lld\n",
+		 (long long)MDS_RMFID);
+	LASSERTF(MDS_LAST_OPC == 63, "found %lld\n",
 		 (long long)MDS_LAST_OPC);
 	LASSERTF(REINT_SETATTR == 1, "found %lld\n",
 		 (long long)REINT_SETATTR);
-- 
1.8.3.1