From aa725c74dc931bb9b6caec258516e7a4c8790a5a Mon Sep 17 00:00:00 2001 From: rread Date: Wed, 30 Oct 2002 01:40:16 +0000 Subject: [PATCH] * create a minimal MDT - only setup once on a node - one thead pool for all MDS's - theads call mds_handle directly * changes to lconf to setup/cleanup MDT --- lustre/include/linux/lustre_mds.h | 1 + lustre/mds/handler.c | 93 ++++++++++++++++++++++++++------------- lustre/utils/lconf | 34 ++++++++++++-- 3 files changed, 95 insertions(+), 33 deletions(-) diff --git a/lustre/include/linux/lustre_mds.h b/lustre/include/linux/lustre_mds.h index 7eb76c7..16c9c6f 100644 --- a/lustre/include/linux/lustre_mds.h +++ b/lustre/include/linux/lustre_mds.h @@ -41,6 +41,7 @@ struct ptlrpc_request; struct obd_device; #define LUSTRE_MDS_NAME "mds" +#define LUSTRE_MDT_NAME "mdt" #define LUSTRE_MDC_NAME "mdc" struct mds_update_record { diff --git a/lustre/mds/handler.c b/lustre/mds/handler.c index e5d7396..c61591f 100644 --- a/lustre/mds/handler.c +++ b/lustre/mds/handler.c @@ -981,7 +981,7 @@ int mds_handle(struct ptlrpc_request *req) if (req->rq_reqmsg->opc != MDS_CONNECT && req->rq_export == NULL) GOTO(out, rc = -ENOTCONN); - LASSERT(!strcmp(req->rq_obd->obd_type->typ_name, "mds")); + LASSERT(!strcmp(req->rq_obd->obd_type->typ_name, LUSTRE_MDT_NAME)); switch (req->rq_reqmsg->opc) { case MDS_CONNECT: @@ -1166,11 +1166,9 @@ static int mds_recover(struct obd_device *obddev) return rc; } -#define MDS_NUM_THREADS 8 /* mount the file system (secretly) */ static int mds_setup(struct obd_device *obddev, obd_count len, void *buf) { - int i; struct obd_ioctl_data* data = buf; struct mds_obd *mds = &obddev->u.mds; struct vfsmount *mnt; @@ -1205,46 +1203,24 @@ static int mds_setup(struct obd_device *obddev, obd_count len, void *buf) CERROR("MDS filesystem method init failed: rc = %d\n", rc); GOTO(err_put, rc); } -#warning move this to module init - mds->mds_service = ptlrpc_init_svc(MDS_NEVENTS, MDS_NBUFS, - MDS_BUFSIZE, MDS_MAXREQSIZE, - MDS_REQUEST_PORTAL, MDC_REPLY_PORTAL, - "self", mds_handle, "mds"); - if (!mds->mds_service) { - CERROR("failed to start service\n"); - GOTO(err_fs, rc = -EINVAL); - } obddev->obd_namespace = ldlm_namespace_new("mds_server", LDLM_NAMESPACE_SERVER); if (obddev->obd_namespace == NULL) { mds_cleanup(obddev); - GOTO(err_svc, rc = -ENOMEM); + GOTO(err_fs, rc = -ENOMEM); } - for (i = 0; i < MDS_NUM_THREADS; i++) { - char name[32]; - sprintf(name, "lustre_MDS_%02d", i); - rc = ptlrpc_start_thread(obddev, mds->mds_service, name); - if (rc) { - CERROR("cannot start MDS thread #%d: rc %d\n", i, rc); - GOTO(err_thread, rc); - } - } rc = mds_recover(obddev); if (rc) - GOTO(err_thread, rc); + GOTO(err_fs, rc); ptlrpc_init_client(LDLM_REQUEST_PORTAL, LDLM_REPLY_PORTAL, "mds_ldlm_client", &obddev->obd_ldlm_client); RETURN(0); -err_thread: - ptlrpc_stop_all_threads(mds->mds_service); -err_svc: - ptlrpc_unregister_service(mds->mds_service); err_fs: mds_fs_cleanup(obddev); err_put: @@ -1266,9 +1242,6 @@ static int mds_cleanup(struct obd_device *obddev) struct obd_run_ctxt saved; ENTRY; - ptlrpc_stop_all_threads(mds->mds_service); - ptlrpc_unregister_service(mds->mds_service); - sb = mds->mds_sb; if (!mds->mds_sb) RETURN(0); @@ -1449,6 +1422,59 @@ static int ldlm_intent_policy(struct ldlm_lock *lock, void *req_cookie, } +#define MDT_NUM_THREADS 8 +static int mdt_setup(struct obd_device *obddev, obd_count len, void *buf) +{ + int i; + // struct obd_ioctl_data* data = buf; + struct mds_obd *mds = &obddev->u.mds; + int rc = 0; + ENTRY; + + MOD_INC_USE_COUNT; + + mds->mds_service = ptlrpc_init_svc(MDS_NEVENTS, MDS_NBUFS, + MDS_BUFSIZE, MDS_MAXREQSIZE, + MDS_REQUEST_PORTAL, MDC_REPLY_PORTAL, + "self", mds_handle, "mds"); + if (!mds->mds_service) { + CERROR("failed to start service\n"); + GOTO(err_dec, rc = -EINVAL); + } + + for (i = 0; i < MDT_NUM_THREADS; i++) { + char name[32]; + sprintf(name, "lustre_MDT_%02d", i); + rc = ptlrpc_start_thread(obddev, mds->mds_service, name); + if (rc) { + CERROR("cannot start MDT thread #%d: rc %d\n", i, rc); + GOTO(err_thread, rc); + } + } + + RETURN(0); + +err_thread: + ptlrpc_stop_all_threads(mds->mds_service); + ptlrpc_unregister_service(mds->mds_service); +err_dec: + MOD_DEC_USE_COUNT; + RETURN(rc); +} + + +static int mdt_cleanup(struct obd_device *obddev) +{ + struct mds_obd *mds = &obddev->u.mds; + ENTRY; + + ptlrpc_stop_all_threads(mds->mds_service); + ptlrpc_unregister_service(mds->mds_service); + + MOD_DEC_USE_COUNT; + RETURN(0); +} + extern int mds_iocontrol(long cmd, struct lustre_handle *conn, int len, void *karg, void *uarg); @@ -1461,6 +1487,12 @@ static struct obd_ops mds_obd_ops = { o_iocontrol: mds_iocontrol }; +static struct obd_ops mdt_obd_ops = { + o_setup: mdt_setup, + o_cleanup: mdt_cleanup, +}; + + static int __init mds_init(void) { mds_file_cache = kmem_cache_create("ll_mds_file_data", @@ -1470,6 +1502,7 @@ static int __init mds_init(void) return -ENOMEM; class_register_type(&mds_obd_ops, LUSTRE_MDS_NAME); + class_register_type(&mdt_obd_ops, LUSTRE_MDT_NAME); ldlm_register_intent(ldlm_intent_policy); return 0; } diff --git a/lustre/utils/lconf b/lustre/utils/lconf index a054ce0..a89bc27 100755 --- a/lustre/utils/lconf +++ b/lustre/utils/lconf @@ -410,6 +410,14 @@ class LCTLInterface: quit""" % (dump_file) self.run(cmds) + # get list of devices + def device_list(self): + cmds = """ + device_list + quit""" + rc, out = self.run(cmds) + return out + # ============================================================ # Various system-level functions # (ideally moved to their own module) @@ -623,7 +631,17 @@ def get_local_address(net_type, wildcard): fixme("automatic local address for GM") return local - + +def is_prepared(uuid): + """Return true if a device exists for the uuid""" + # expect this format: + # 1 UP ldlm ldlm ldlm_UUID 2 + out = lctl.device_list() + for s in out: + if uuid == string.split(s)[4]: + return 1 + return 0 + # ============================================================ # Classes to prepare and cleanup the various objects @@ -778,7 +796,7 @@ class Network(Module): lctl.network(self.net_type, self.nid) - lctl.newdev(attach = "ptlrpc RPCDEV") + lctl.newdev(attach = "ptlrpc RPCDEV RPCDEV_UUID") def cleanup(self): self.info(self.net_type, self.nid, self.port) @@ -806,7 +824,7 @@ class Network(Module): cleanup_error(e.rc) try: - lctl.cleanup("RPCDEV", "") + lctl.cleanup("RPCDEV", "RPCDEV_UUID") except CommandError, e: print "cleanup failed: ", self.name e.dump() @@ -930,9 +948,19 @@ class MDS(Module): def prepare(self): self.info(self.devname, self.fstype, self.format) blkdev = block_dev(self.devname, self.size, self.fstype, self.format) + if not is_prepared('MDT_UUID'): + lctl.newdev(attach="mdt %s %s" % ('MDT', 'MDT_UUID'), + setup ="") lctl.newdev(attach="mds %s %s" % (self.name, self.uuid), setup ="%s %s" %(blkdev, self.fstype)) def cleanup(self): + if is_prepared('MDT_UUID'): + try: + lctl.cleanup("MDT", "MDT_UUID") + except CommandError, e: + print "cleanup failed: ", self.name + e.dump() + cleanup_error(e.rc) Module.cleanup(self) clean_loop(self.devname) -- 1.8.3.1