From: Alex Zhuravlev Date: Mon, 19 Oct 2015 10:18:44 +0000 (+0300) Subject: LU-2222 mdt: restore evict-by-nid functionality X-Git-Tag: 2.7.63~10 X-Git-Url: https://git.whamcloud.com/gitweb?a=commitdiff_plain;h=refs%2Fchanges%2F67%2F16867%2F8;p=fs%2Flustre-release.git LU-2222 mdt: restore evict-by-nid functionality Writing a NID or UUID to mdt.*.evict_tgt_nids will evict clients with NID or UUID specified all the targets (OSTs and MDTs). Change-Id: I66a60a6c81fbac1571f5685111df7b00a306be36 Signed-off-by: Alex Zhuravlev Reviewed-on: http://review.whamcloud.com/16867 Tested-by: Jenkins Reviewed-by: Andreas Dilger Tested-by: Maloo Reviewed-by: Niu Yawei Reviewed-by: Oleg Drokin --- diff --git a/lustre/lod/lod_dev.c b/lustre/lod/lod_dev.c index e95d48e..be0dfbf 100644 --- a/lustre/lod/lod_dev.c +++ b/lustre/lod/lod_dev.c @@ -1848,11 +1848,70 @@ static int lod_obd_get_info(const struct lu_env *env, struct obd_export *exp, RETURN(rc); } +static int lod_obd_set_info_async(const struct lu_env *env, + struct obd_export *exp, + __u32 keylen, void *key, + __u32 vallen, void *val, + struct ptlrpc_request_set *set) +{ + struct obd_device *obd = class_exp2obd(exp); + struct lod_device *d; + struct lod_tgt_desc *tgt; + int no_set = 0; + int i, rc = 0, rc2; + ENTRY; + + if (set == NULL) { + no_set = 1; + set = ptlrpc_prep_set(); + if (!set) + RETURN(-ENOMEM); + } + + d = lu2lod_dev(obd->obd_lu_dev); + lod_getref(&d->lod_ost_descs); + lod_foreach_ost(d, i) { + tgt = OST_TGT(d, i); + LASSERT(tgt && tgt->ltd_tgt); + if (!tgt->ltd_active) + continue; + + rc2 = obd_set_info_async(env, tgt->ltd_exp, keylen, key, + vallen, val, set); + if (rc2 != 0 && rc == 0) + rc = rc2; + } + lod_putref(d, &d->lod_ost_descs); + + lod_getref(&d->lod_mdt_descs); + lod_foreach_mdt(d, i) { + tgt = MDT_TGT(d, i); + LASSERT(tgt && tgt->ltd_tgt); + if (!tgt->ltd_active) + continue; + rc2 = obd_set_info_async(env, tgt->ltd_exp, keylen, key, + vallen, val, set); + if (rc2 != 0 && rc == 0) + rc = rc2; + } + lod_putref(d, &d->lod_mdt_descs); + + + if (no_set) { + rc2 = ptlrpc_set_wait(set); + if (rc2 == 0 && rc == 0) + rc = rc2; + ptlrpc_set_destroy(set); + } + RETURN(rc); +} + static struct obd_ops lod_obd_device_ops = { .o_owner = THIS_MODULE, .o_connect = lod_obd_connect, .o_disconnect = lod_obd_disconnect, .o_get_info = lod_obd_get_info, + .o_set_info_async = lod_obd_set_info_async, .o_pool_new = lod_pool_new, .o_pool_rem = lod_pool_remove, .o_pool_add = lod_pool_add, diff --git a/lustre/mdd/mdd_device.c b/lustre/mdd/mdd_device.c index b6d67b5..b3b9d96 100644 --- a/lustre/mdd/mdd_device.c +++ b/lustre/mdd/mdd_device.c @@ -1234,11 +1234,32 @@ static int mdd_obd_get_info(const struct lu_env *env, struct obd_export *exp, RETURN(rc); } +static int mdd_obd_set_info_async(const struct lu_env *env, + struct obd_export *exp, + __u32 keylen, void *key, + __u32 vallen, void *val, + struct ptlrpc_request_set *set) +{ + struct obd_device *obd = exp->exp_obd; + struct mdd_device *mdd; + int rc; + + if (!obd->obd_set_up || obd->obd_stopping) + RETURN(-EAGAIN); + + mdd = lu2mdd_dev(obd->obd_lu_dev); + LASSERT(mdd); + rc = obd_set_info_async(env, mdd->mdd_child_exp, keylen, key, + vallen, val, set); + RETURN(rc); +} + static struct obd_ops mdd_obd_device_ops = { .o_owner = THIS_MODULE, .o_connect = mdd_obd_connect, .o_disconnect = mdd_obd_disconnect, .o_get_info = mdd_obd_get_info, + .o_set_info_async = mdd_obd_set_info_async, }; static int mdd_changelog_user_register(const struct lu_env *env, diff --git a/lustre/mdt/mdt_handler.c b/lustre/mdt/mdt_handler.c index 03bd675..ef4eeda 100644 --- a/lustre/mdt/mdt_handler.c +++ b/lustre/mdt/mdt_handler.c @@ -1630,6 +1630,9 @@ static int mdt_set_info(struct tgt_session_info *tsi) rc = mdt_iocontrol(OBD_IOC_CHANGELOG_CLEAR, req->rq_export, vallen, val, NULL); + } else if (KEY_IS(KEY_EVICT_BY_NID)) { + if (vallen > 0) + obd_export_evict_by_nid(req->rq_export->exp_obd, val); } else { RETURN(-EINVAL); } @@ -4336,7 +4339,7 @@ static int mdt_init0(const struct lu_env *env, struct mdt_device *m, LASSERT(obd != NULL); m->mdt_max_mdsize = MAX_MD_SIZE; /* 4 stripes */ - + m->mdt_opts.mo_evict_tgt_nids = 1; m->mdt_opts.mo_cos = MDT_COS_DEFAULT; /* default is coordinator off, it is started through conf_param diff --git a/lustre/mdt/mdt_internal.h b/lustre/mdt/mdt_internal.h index 6477e24..2e163b9 100644 --- a/lustre/mdt/mdt_internal.h +++ b/lustre/mdt/mdt_internal.h @@ -176,6 +176,7 @@ struct mdt_device { unsigned int mo_user_xattr:1, mo_acl:1, mo_cos:1, + mo_evict_tgt_nids:1, mo_coordinator:1; } mdt_opts; /* mdt state flags */ diff --git a/lustre/mdt/mdt_lproc.c b/lustre/mdt/mdt_lproc.c index 3fa5883..fe1d9c4 100644 --- a/lustre/mdt/mdt_lproc.c +++ b/lustre/mdt/mdt_lproc.c @@ -421,44 +421,85 @@ LPROC_SEQ_FOPS_RO(mdt_site_stats); #define BUFLEN (UUID_MAX + 4) static ssize_t -lprocfs_mds_evict_client_seq_write(struct file *file, - const char __user *buffer, +lprocfs_mds_evict_client_seq_write(struct file *file, const char __user *buf, size_t count, loff_t *off) { + struct seq_file *m = file->private_data; + struct obd_device *obd = m->private; + struct mdt_device *mdt = mdt_dev(obd->obd_lu_dev); char *kbuf; char *tmpbuf; + int rc = 0; - OBD_ALLOC(kbuf, BUFLEN); - if (kbuf == NULL) - return -ENOMEM; - - /* - * OBD_ALLOC() will zero kbuf, but we only copy BUFLEN - 1 - * bytes into kbuf, to ensure that the string is NUL-terminated. - * UUID_MAX should include a trailing NUL already. - */ - if (copy_from_user(kbuf, buffer, - min_t(unsigned long, BUFLEN - 1, count))) { - count = -EFAULT; - goto out; - } - tmpbuf = cfs_firststr(kbuf, min_t(unsigned long, BUFLEN - 1, count)); + OBD_ALLOC(kbuf, BUFLEN); + if (kbuf == NULL) + return -ENOMEM; - if (strncmp(tmpbuf, "nid:", 4) != 0) { - count = lprocfs_evict_client_seq_write(file, buffer, count, - off); - goto out; - } + /* + * OBD_ALLOC() will zero kbuf, but we only copy BUFLEN - 1 + * bytes into kbuf, to ensure that the string is NUL-terminated. + * UUID_MAX should include a trailing NUL already. + */ + if (copy_from_user(kbuf, buf, min_t(unsigned long, BUFLEN - 1, count))) + GOTO(out, rc = -EFAULT); + tmpbuf = cfs_firststr(kbuf, min_t(unsigned long, BUFLEN - 1, count)); + + if (strncmp(tmpbuf, "nid:", 4) != 0) { + count = lprocfs_evict_client_seq_write(file, buf, count, off); + goto out; + } + + if (mdt->mdt_opts.mo_evict_tgt_nids) { + rc = obd_set_info_async(NULL, mdt->mdt_child_exp, + sizeof(KEY_EVICT_BY_NID), + KEY_EVICT_BY_NID, + strlen(tmpbuf + 4) + 1, + tmpbuf + 4, NULL); + if (rc) + CERROR("Failed to evict nid %s from OSTs: rc %d\n", + tmpbuf + 4, rc); + } + + /* See the comments in function lprocfs_wr_evict_client() + * in ptlrpc/lproc_ptlrpc.c for details. - jay */ + class_incref(obd, __func__, current); + obd_export_evict_by_nid(obd, tmpbuf + 4); + class_decref(obd, __func__, current); - CERROR("NOT implement evict client by nid %s\n", tmpbuf); out: - OBD_FREE(kbuf, BUFLEN); - return count; + OBD_FREE(kbuf, BUFLEN); + return rc < 0 ? rc : count; } #undef BUFLEN +static int mdt_evict_tgt_nids_seq_show(struct seq_file *m, void *data) +{ + struct obd_device *obd = m->private; + struct mdt_device *mdt = mdt_dev(obd->obd_lu_dev); + + return seq_printf(m, "%u\n", mdt->mdt_opts.mo_evict_tgt_nids); +} + +static ssize_t +mdt_evict_tgt_nids_seq_write(struct file *file, const char __user *buffer, + size_t count, loff_t *off) +{ + struct seq_file *m = file->private_data; + struct obd_device *obd = m->private; + struct mdt_device *mdt = mdt_dev(obd->obd_lu_dev); + int val, rc; + + rc = lprocfs_write_helper(buffer, count, &val); + if (rc) + return rc; + mdt->mdt_opts.mo_evict_tgt_nids = !!val; + return count; +} +LPROC_SEQ_FOPS(mdt_evict_tgt_nids); + + static int mdt_sec_level_seq_show(struct seq_file *m, void *data) { struct obd_device *obd = m->private; @@ -668,6 +709,8 @@ static struct lprocfs_vars lprocfs_mdt_obd_vars[] = { .fops = &mdt_site_stats_fops }, { .name = "evict_client", .fops = &mdt_mds_evict_client_fops }, + { .name = "evict_tgt_nids", + .fops = &mdt_evict_tgt_nids_fops }, { .name = "hash_stats", .fops = &mdt_hash_fops }, { .name = "sec_level", diff --git a/lustre/osp/osp_dev.c b/lustre/osp/osp_dev.c index 787aeb6..d85bbbe 100644 --- a/lustre/osp/osp_dev.c +++ b/lustre/osp/osp_dev.c @@ -1690,23 +1690,6 @@ static int osp_iocontrol(unsigned int cmd, struct obd_export *exp, int len, } -static int osp_set_info_async(const struct lu_env *env, - struct obd_export *exp, - u32 keylen, void *key, - u32 vallen, void *val, - struct ptlrpc_request_set *set) -{ - ENTRY; - - if (KEY_IS(KEY_SPTLRPC_CONF)) { - sptlrpc_conf_client_adapt(exp->exp_obd); - RETURN(0); - } - - CERROR("Unknown key %s\n", (char *)key); - RETURN(-EINVAL); -} - /** * Implementation of obd_ops::o_get_info * @@ -1749,6 +1732,58 @@ static int osp_obd_get_info(const struct lu_env *env, struct obd_export *exp, RETURN(rc); } +static int osp_obd_set_info_async(const struct lu_env *env, + struct obd_export *exp, + u32 keylen, void *key, + u32 vallen, void *val, + struct ptlrpc_request_set *set) +{ + struct obd_device *obd = exp->exp_obd; + struct obd_import *imp = obd->u.cli.cl_import; + struct osp_device *osp; + struct ptlrpc_request *req; + char *tmp; + int rc; + + if (KEY_IS(KEY_SPTLRPC_CONF)) { + sptlrpc_conf_client_adapt(exp->exp_obd); + RETURN(0); + } + + LASSERT(set != NULL); + if (!obd->obd_set_up || obd->obd_stopping) + RETURN(-EAGAIN); + osp = lu2osp_dev(obd->obd_lu_dev); + + req = ptlrpc_request_alloc(imp, &RQF_OBD_SET_INFO); + if (req == NULL) + RETURN(-ENOMEM); + + req_capsule_set_size(&req->rq_pill, &RMF_SETINFO_KEY, + RCL_CLIENT, keylen); + req_capsule_set_size(&req->rq_pill, &RMF_SETINFO_VAL, + RCL_CLIENT, vallen); + if (osp->opd_connect_mdt) + rc = ptlrpc_request_pack(req, LUSTRE_MDS_VERSION, MDS_SET_INFO); + else + rc = ptlrpc_request_pack(req, LUSTRE_OST_VERSION, OST_SET_INFO); + if (rc) { + ptlrpc_request_free(req); + RETURN(rc); + } + + tmp = req_capsule_client_get(&req->rq_pill, &RMF_SETINFO_KEY); + memcpy(tmp, key, keylen); + tmp = req_capsule_client_get(&req->rq_pill, &RMF_SETINFO_VAL); + memcpy(tmp, val, vallen); + + ptlrpc_request_set_replen(req); + ptlrpc_set_add_req(set, req); + ptlrpc_check_set(NULL, set); + + RETURN(0); +} + /** * Implementation of obd_ops: o_fid_alloc * @@ -1842,13 +1877,13 @@ static struct obd_ops osp_obd_device_ops = { .o_connect = osp_obd_connect, .o_disconnect = osp_obd_disconnect, .o_get_info = osp_obd_get_info, + .o_set_info_async = osp_obd_set_info_async, .o_import_event = osp_import_event, .o_iocontrol = osp_iocontrol, .o_statfs = osp_obd_statfs, .o_fid_init = client_fid_init, .o_fid_fini = client_fid_fini, .o_fid_alloc = osp_fid_alloc, - .o_set_info_async = osp_set_info_async, }; struct llog_operations osp_mds_ost_orig_logops; diff --git a/lustre/tests/conf-sanity.sh b/lustre/tests/conf-sanity.sh index b7f23ec..fb97eeb 100644 --- a/lustre/tests/conf-sanity.sh +++ b/lustre/tests/conf-sanity.sh @@ -6039,6 +6039,70 @@ test_90d() { } run_test 90d "check one close RPC is allowed above max_mod_rpcs_in_flight" +check_uuid_on_ost() { + local nid=$1 + do_facet ost1 "$LCTL get_param obdfilter.${FSNAME}*.exports.'$nid'.uuid" +} + +check_uuid_on_mdt() { + local nid=$1 + do_facet $SINGLEMDS "$LCTL get_param mdt.${FSNAME}*.exports.'$nid'.uuid" +} + +test_91() { + local uuid + local nid + local found + + load_modules + start_mds || error "MDS start failed" + start_ost || error "unable to start OST" + mount_client $MOUNT || error "client start failed" + check_mount || error "check_mount failed" + + if remote_mds; then + nid=$($LCTL list_nids | head -1 | sed "s/\./\\\./g") + else + nid="0@lo" + fi + uuid=$(get_client_uuid $MOUNT) + + echo "list nids on mdt:" + do_facet $SINGLEMDS "$LCTL list_param mdt.${FSNAME}*.exports.*" + echo "uuid from $nid:" + do_facet $SINGLEMDS "$LCTL get_param mdt.${FSNAME}*.exports.'$nid'.uuid" + + found=$(check_uuid_on_mdt $nid | grep $uuid) + [ -z "$found" ] && error "can't find $uuid $nid on MDT" + found=$(check_uuid_on_ost $nid | grep $uuid) + [ -z "$found" ] && error "can't find $uuid $nid on OST" + + # umount the client so it won't reconnect + manual_umount_client --force || error "failed to umount $?" + # shouldn't disappear on MDS after forced umount + found=$(check_uuid_on_mdt $nid | grep $uuid) + [ -z "$found" ] && error "can't find $uuid $nid" + + echo "evict $nid" + do_facet $SINGLEMDS \ + "$LCTL set_param -n mdt.${mds1_svc}.evict_client nid:$nid" + + found=$(check_uuid_on_mdt $nid | grep $uuid) + [ -n "$found" ] && error "found $uuid $nid on MDT" + found=$(check_uuid_on_ost $nid | grep $uuid) + [ -n "$found" ] && error "found $uuid $nid on OST" + + # check it didn't reconnect (being umounted) + sleep $((TIMEOUT+1)) + found=$(check_uuid_on_mdt $nid | grep $uuid) + [ -n "$found" ] && error "found $uuid $nid on MDT" + found=$(check_uuid_on_ost $nid | grep $uuid) + [ -n "$found" ] && error "found $uuid $nid on OST" + + cleanup +} +run_test 91 "evict-by-nid support" + if ! combined_mgs_mds ; then stop mgs fi