From: yury Date: Thu, 29 Jun 2006 10:02:27 +0000 (+0000) Subject: - implemented Mike's idea that seq-controller and seq-server should be different... X-Git-Tag: v1_8_0_110~486^2~1518 X-Git-Url: https://git.whamcloud.com/?a=commitdiff_plain;h=10a3302b62da3955cf944f063896b3ba8d0c8e86;p=fs%2Flustre-release.git - implemented Mike's idea that seq-controller and seq-server should be different instances of lu_server_seq on controller node; - some fixes after DLDINSP,cleanups. --- diff --git a/lustre/fid/fid_handler.c b/lustre/fid/fid_handler.c index bb20f56..e3850bd 100644 --- a/lustre/fid/fid_handler.c +++ b/lustre/fid/fid_handler.c @@ -65,9 +65,9 @@ EXPORT_SYMBOL(LUSTRE_SEQ_ZERO_RANGE); /* assigns client to sequence controller node */ int -seq_server_set_ctlr(struct lu_server_seq *seq, - struct lu_client_seq *cli, - const struct lu_context *ctx) +seq_server_init_ctlr(struct lu_server_seq *seq, + struct lu_client_seq *cli, + const struct lu_context *ctx) { int rc = 0; ENTRY; @@ -92,12 +92,7 @@ seq_server_set_ctlr(struct lu_server_seq *seq, /* get new range from controller only if super-sequence is not yet * initialized from backing store or something else. */ if (range_is_zero(&seq->seq_super)) { - /* release sema to avoid deadlock for case we're asking our - * selves. */ - up(&seq->seq_sem); rc = seq_client_alloc_super(cli); - down(&seq->seq_sem); - if (rc) { CERROR("can't allocate super-sequence, " "rc %d\n", rc); @@ -119,7 +114,20 @@ seq_server_set_ctlr(struct lu_server_seq *seq, up(&seq->seq_sem); RETURN(rc); } -EXPORT_SYMBOL(seq_server_set_ctlr); +EXPORT_SYMBOL(seq_server_init_ctlr); + +void +seq_server_fini_ctlr(struct lu_server_seq *seq) +{ + ENTRY; + + down(&seq->seq_sem); + seq->seq_cli = NULL; + up(&seq->seq_sem); + + EXIT; +} +EXPORT_SYMBOL(seq_server_fini_ctlr); /* on controller node, allocate new super sequence for regular sequnece * server. */ @@ -197,29 +205,16 @@ __seq_server_alloc_meta(struct lu_server_seq *seq, RETURN(-EOPNOTSUPP); } - /* allocate new super-sequence. */ - up(&seq->seq_sem); rc = seq_client_alloc_super(seq->seq_cli); - down(&seq->seq_sem); if (rc) { CERROR("can't allocate new super-sequence, " "rc %d\n", rc); RETURN(rc); } - if (seq->seq_cli->seq_range.lr_start > super->lr_start) { - /* saving new range into allocation space. */ - *super = seq->seq_cli->seq_range; - LASSERT(range_is_sane(super)); - } else { - /* XXX: race is catched, ignore what we have from - * controller node. The only issue is that controller - * node has now this super-sequence lost, what makes - * sequences space smaller. */ - CWARN("SEQ-MGR(srv): race is cached, reject " - "allocated super-sequence\n"); - RETURN(0); - } + /* saving new range into allocation space. */ + *super = seq->seq_cli->seq_range; + LASSERT(range_is_sane(super)); } range_alloc(range, super, seq->seq_meta_width); @@ -269,25 +264,30 @@ seq_server_handle(struct lu_server_seq *seq, rc = seq_server_alloc_meta(seq, range, ctx); break; default: + CERROR("wrong opc 0x%x\n", opc); rc = -EINVAL; break; } - RETURN(rc); } static int seq_req_handle0(const struct lu_context *ctx, - struct lu_server_seq *seq, struct ptlrpc_request *req) { int rep_buf_size[2] = { 0, }; + struct obd_device *obd; struct req_capsule pill; + struct lu_site *site; struct lu_range *out; int rc = -EPROTO; __u32 *opc; ENTRY; + obd = req->rq_export->exp_obd; + site = obd->obd_lu_dev->ld_site; + LASSERT(site != NULL); + req_capsule_init(&pill, req, RCL_SERVER, rep_buf_size); @@ -301,7 +301,22 @@ seq_req_handle0(const struct lu_context *ctx, CERROR("can't get range buffer\n"); GOTO(out_pill, rc= -EPROTO); } - rc = seq_server_handle(seq, ctx, out, *opc); + + if (*opc == SEQ_ALLOC_META) { + if (!site->ls_server_seq) { + CERROR("sequence-server is not initialized\n"); + GOTO(out_pill, rc == -EINVAL); + } + rc = seq_server_handle(site->ls_server_seq, + ctx, out, *opc); + } else { + if (!site->ls_ctlr_seq) { + CERROR("sequence-controller is not initialized\n"); + GOTO(out_pill, rc == -EINVAL); + } + rc = seq_server_handle(site->ls_ctlr_seq, + ctx, out, *opc); + } } else { CERROR("cannot unpack client request\n"); } @@ -317,7 +332,6 @@ seq_req_handle(struct ptlrpc_request *req) { int fail = OBD_FAIL_SEQ_ALL_REPLY_NET; const struct lu_context *ctx; - struct lu_site *site; int rc = -EPROTO; ENTRY; @@ -328,13 +342,7 @@ seq_req_handle(struct ptlrpc_request *req) LASSERT(ctx->lc_thread == req->rq_svc_thread); if (req->rq_reqmsg->opc == SEQ_QUERY) { if (req->rq_export != NULL) { - struct obd_device *obd; - - obd = req->rq_export->exp_obd; - site = obd->obd_lu_dev->ld_site; - LASSERT(site != NULL); - - rc = seq_req_handle0(ctx, site->ls_server_seq, req); + rc = seq_req_handle0(ctx, req); } else { CERROR("Unconnected request\n"); req->rq_status = -ENOTCONN; @@ -418,15 +426,18 @@ int seq_server_init(struct lu_server_seq *seq, struct dt_device *dev, const char *uuid, + lu_server_type_t type, const struct lu_context *ctx) { - int rc; + int rc, portal = (type == LUSTRE_SEQ_SRV) ? + SEQ_SRV_PORTAL : SEQ_CTLR_PORTAL; + struct ptlrpc_service_conf seq_conf = { .psc_nbufs = MDS_NBUFS, .psc_bufsize = MDS_BUFSIZE, .psc_max_req_size = MDS_MAXREQSIZE, .psc_max_reply_size = MDS_MAXREPSIZE, - .psc_req_portal = MDS_SEQ_PORTAL, + .psc_req_portal = portal, .psc_rep_portal = MDC_REPLY_PORTAL, .psc_watchdog_timeout = SEQ_SERVICE_WATCHDOG_TIMEOUT, .psc_num_threads = SEQ_NUM_THREADS @@ -438,13 +449,15 @@ seq_server_init(struct lu_server_seq *seq, seq->seq_dev = dev; seq->seq_cli = NULL; + seq->seq_type = type; sema_init(&seq->seq_sem, 1); seq->seq_super_width = LUSTRE_SEQ_SUPER_WIDTH; seq->seq_meta_width = LUSTRE_SEQ_META_WIDTH; - snprintf(seq->seq_name, sizeof(seq->seq_name), - "%s-%s", LUSTRE_SEQ_NAME, uuid); + snprintf(seq->seq_name, sizeof(seq->seq_name), "%s-%s-%s", + LUSTRE_SEQ_NAME, (type == LUSTRE_SEQ_SRV ? "srv" : "ctlr"), + uuid); seq->seq_space = LUSTRE_SEQ_SPACE_RANGE; seq->seq_super = LUSTRE_SEQ_ZERO_RANGE; @@ -458,8 +471,13 @@ seq_server_init(struct lu_server_seq *seq, /* request backing store for saved sequence info */ rc = seq_store_read(seq, ctx); if (rc == -ENODATA) { - CDEBUG(D_INFO|D_WARNING, "SEQ-MGR(srv): no data on " - "disk found, waiting for controller assign\n"); + if (type == LUSTRE_SEQ_SRV) { + CDEBUG(D_INFO|D_WARNING, "SEQ-MGR(srv): no data on " + "disk found, waiting for controller assign\n"); + } else { + CDEBUG(D_INFO|D_WARNING, "SEQ-MGR(ctlr): no data on " + "disk found, this is first controller run\n"); + } } else if (rc) { CERROR("can't read sequence state, rc = %d\n", rc); @@ -474,23 +492,24 @@ seq_server_init(struct lu_server_seq *seq, seq->seq_service = ptlrpc_init_svc_conf(&seq_conf, seq_req_handle, - LUSTRE_SEQ_NAME, + LUSTRE_SEQ_NAME, seq->seq_proc_entry, NULL); if (seq->seq_service != NULL) rc = ptlrpc_start_threads(NULL, seq->seq_service, - LUSTRE_SEQ_NAME); + LUSTRE_SEQ_NAME); else rc = -ENOMEM; EXIT; out: - if (rc) + if (rc) { seq_server_fini(seq, ctx); - else - CDEBUG(D_INFO|D_WARNING, "Server Sequence " - "Manager\n"); + } else { + CDEBUG(D_INFO|D_WARNING, "%s Sequence Manager\n", + (type == LUSTRE_SEQ_SRV ? "Server" : "Controller")); + } return rc; } EXPORT_SYMBOL(seq_server_init); diff --git a/lustre/fid/fid_request.c b/lustre/fid/fid_request.c index c3f4c57..f41a4c1 100644 --- a/lustre/fid/fid_request.c +++ b/lustre/fid/fid_request.c @@ -72,7 +72,9 @@ seq_client_rpc(struct lu_client_seq *seq, *op = opc; req->rq_replen = lustre_msg_size(1, &repsize); - req->rq_request_portal = MDS_SEQ_PORTAL; + + req->rq_request_portal = (opc == SEQ_ALLOC_SUPER) ? + SEQ_CTLR_PORTAL : SEQ_SRV_PORTAL; rc = ptlrpc_queue_wait(req); if (rc) @@ -96,22 +98,31 @@ out_req: return rc; } -/* request sequence-controller node to allocate new super-sequence. */ static int -__seq_client_alloc_super(struct lu_client_seq *seq) +__seq_client_alloc_opc(struct lu_client_seq *seq, + int opc, const char *opcname) { int rc; ENTRY; - rc = seq_client_rpc(seq, &seq->seq_range, SEQ_ALLOC_SUPER); + rc = seq_client_rpc(seq, &seq->seq_range, opc); if (rc == 0) { - CDEBUG(D_INFO, "SEQ-MGR(cli): allocated super-sequence " - "["LPX64"-"LPX64"]\n", seq->seq_range.lr_start, + CDEBUG(D_INFO, "SEQ-MGR(cli): allocated " + "%s-sequence ["LPX64"-"LPX64"]\n", + opcname, seq->seq_range.lr_start, seq->seq_range.lr_end); } RETURN(rc); } +/* request sequence-controller node to allocate new super-sequence. */ +static int +__seq_client_alloc_super(struct lu_client_seq *seq) +{ + ENTRY; + RETURN(__seq_client_alloc_opc(seq, SEQ_ALLOC_SUPER, "super")); +} + int seq_client_alloc_super(struct lu_client_seq *seq) { @@ -130,16 +141,8 @@ EXPORT_SYMBOL(seq_client_alloc_super); static int __seq_client_alloc_meta(struct lu_client_seq *seq) { - int rc; ENTRY; - - rc = seq_client_rpc(seq, &seq->seq_range, SEQ_ALLOC_META); - if (rc == 0) { - CDEBUG(D_INFO, "SEQ-MGR(cli): allocated meta-sequence " - "["LPX64"-"LPX64"]\n", seq->seq_range.lr_start, - seq->seq_range.lr_end); - } - RETURN(rc); + RETURN(__seq_client_alloc_opc(seq, SEQ_ALLOC_META, "meta")); } int @@ -172,6 +175,7 @@ __seq_client_alloc_seq(struct lu_client_seq *seq, __u64 *seqnr) if (rc) { CERROR("can't allocate new meta-sequence, " "rc %d\n", rc); + RETURN(rc); } } diff --git a/lustre/fld/fld_handler.c b/lustre/fld/fld_handler.c index 2c6aa3d..e428655 100644 --- a/lustre/fld/fld_handler.c +++ b/lustre/fld/fld_handler.c @@ -286,7 +286,7 @@ fld_server_init(struct lu_server_fld *fld, .psc_bufsize = MDS_BUFSIZE, .psc_max_req_size = MDS_MAXREQSIZE, .psc_max_reply_size = MDS_MAXREPSIZE, - .psc_req_portal = MDS_FLD_PORTAL, + .psc_req_portal = FLD_REQUEST_PORTAL, .psc_rep_portal = MDC_REPLY_PORTAL, .psc_watchdog_timeout = FLD_SERVICE_WATCHDOG_TIMEOUT, .psc_num_threads = FLD_NUM_THREADS diff --git a/lustre/fld/fld_request.c b/lustre/fld/fld_request.c index 3f68be7..edb7e35 100644 --- a/lustre/fld/fld_request.c +++ b/lustre/fld/fld_request.c @@ -396,7 +396,7 @@ fld_client_rpc(struct obd_export *exp, memcpy(pmf, mf, sizeof(*mf)); req->rq_replen = lustre_msg_size(1, &mf_size); - req->rq_request_portal = MDS_FLD_PORTAL; + req->rq_request_portal = FLD_REQUEST_PORTAL; rc = ptlrpc_queue_wait(req); if (rc) diff --git a/lustre/include/lu_object.h b/lustre/include/lu_object.h index e56f4bd..e3a8026 100644 --- a/lustre/include/lu_object.h +++ b/lustre/include/lu_object.h @@ -493,12 +493,17 @@ struct lu_site { struct lu_server_seq *ls_server_seq; /* + * Controller Seq Manager + */ + struct lu_server_seq *ls_ctlr_seq; + + /* * Clienbt Seq Manager */ struct lu_client_seq *ls_client_seq; /* sequence controller node */ - struct obd_export *ls_controller; + struct obd_export *ls_ctlr_exp; /* statistical counters. Protected by nothing, races are accepted. */ struct { diff --git a/lustre/include/lustre/lustre_idl.h b/lustre/include/lustre/lustre_idl.h index f407753..6d8496c 100644 --- a/lustre/include/lustre/lustre_idl.h +++ b/lustre/include/lustre/lustre_idl.h @@ -97,8 +97,9 @@ #define MGS_REQUEST_PORTAL 26 #define MGS_REPLY_PORTAL 27 #define OST_REQUEST_PORTAL 28 -#define MDS_FLD_PORTAL 29 -#define MDS_SEQ_PORTAL 30 +#define FLD_REQUEST_PORTAL 29 +#define SEQ_SRV_PORTAL 30 +#define SEQ_CTLR_PORTAL 31 #define SVC_KILLED 1 #define SVC_EVENT 2 diff --git a/lustre/include/lustre_fid.h b/lustre/include/lustre_fid.h index 631ce24..3758af3 100644 --- a/lustre/include/lustre_fid.h +++ b/lustre/include/lustre_fid.h @@ -49,6 +49,13 @@ extern const struct lu_range LUSTRE_SEQ_ZERO_RANGE; * allocated to MDTs. */ #define LUSTRE_SEQ_SUPER_WIDTH (LUSTRE_SEQ_META_WIDTH * LUSTRE_SEQ_META_WIDTH) +enum lu_server_type { + LUSTRE_SEQ_SRV, + LUSTRE_SEQ_CTLR +}; + +typedef enum lu_server_type lu_server_type_t; + /* client sequence manager interface */ struct lu_client_seq { /* sequence-controller export. */ @@ -99,6 +106,8 @@ struct lu_server_seq { cfs_proc_dir_entry_t *seq_proc_entry; cfs_proc_dir_entry_t *seq_proc_dir; + /* LUSTRE_SEQ_SRV or LUSTRE_SEQ_CTLR */ + lu_server_type_t seq_type; /* server side seq service */ struct ptlrpc_service *seq_service; @@ -121,17 +130,21 @@ struct lu_server_seq { #endif #ifdef __KERNEL__ + int seq_server_init(struct lu_server_seq *seq, struct dt_device *dev, const char *uuid, + lu_server_type_t type, const struct lu_context *ctx); void seq_server_fini(struct lu_server_seq *seq, const struct lu_context *ctx); -int seq_server_set_ctlr(struct lu_server_seq *seq, - struct lu_client_seq *cli, - const struct lu_context *ctx); +int seq_server_init_ctlr(struct lu_server_seq *seq, + struct lu_client_seq *cli, + const struct lu_context *ctx); + +void seq_server_fini_ctlr(struct lu_server_seq *seq); #endif int seq_client_init(struct lu_client_seq *seq, diff --git a/lustre/mdt/mdt_handler.c b/lustre/mdt/mdt_handler.c index 0daa1db..53d01a1 100644 --- a/lustre/mdt/mdt_handler.c +++ b/lustre/mdt/mdt_handler.c @@ -1673,10 +1673,10 @@ static int mdt_seq_fini(const struct lu_context *ctx, OBD_FREE_PTR(ls->ls_server_seq); ls->ls_server_seq = NULL; } - if (ls && ls->ls_client_seq) { - seq_client_fini(ls->ls_client_seq); - OBD_FREE_PTR(ls->ls_client_seq); - ls->ls_client_seq = NULL; + if (ls && ls->ls_ctlr_seq) { + seq_server_fini(ls->ls_ctlr_seq, ctx); + OBD_FREE_PTR(ls->ls_ctlr_seq); + ls->ls_ctlr_seq = NULL; } RETURN(0); } @@ -1691,11 +1691,29 @@ static int mdt_seq_init(const struct lu_context *ctx, ls = m->mdt_md_dev.md_lu_dev.ld_site; + /* sequence-controller node */ + if (ls->ls_node_id == 0) { + LASSERT(ls->ls_ctlr_seq == NULL); + OBD_ALLOC_PTR(ls->ls_ctlr_seq); + + if (ls->ls_ctlr_seq != NULL) { + rc = seq_server_init(ls->ls_ctlr_seq, + m->mdt_bottom, uuid, + LUSTRE_SEQ_CTLR, + ctx); + if (rc) + mdt_seq_fini(ctx, m); + } else + rc = -ENOMEM; + } + + LASSERT(ls->ls_server_seq == NULL); OBD_ALLOC_PTR(ls->ls_server_seq); if (ls->ls_server_seq != NULL) { rc = seq_server_init(ls->ls_server_seq, m->mdt_bottom, uuid, + LUSTRE_SEQ_SRV, ctx); if (rc) mdt_seq_fini(ctx, m); @@ -1706,7 +1724,7 @@ static int mdt_seq_init(const struct lu_context *ctx, } /* XXX: this is ugly, should be something else */ -static int mdt_seq_ctlr_init(const struct lu_context *ctx, +static int mdt_seq_init_ctlr(const struct lu_context *ctx, struct mdt_device *m, struct lustre_cfg *cfg) { @@ -1721,7 +1739,7 @@ static int mdt_seq_ctlr_init(const struct lu_context *ctx, /* check if this is first MDC add and controller is not yet * initialized. */ - if (index != 0 || ls->ls_controller) + if (index != 0 || ls->ls_ctlr_exp) RETURN(0); uuid_str = lustre_cfg_string(cfg, 1); @@ -1746,14 +1764,14 @@ static int mdt_seq_ctlr_init(const struct lu_context *ctx, CERROR("target %s connect error %d\n", mdc->obd_name, rc); } else { - ls->ls_controller = class_conn2export(&conn); + ls->ls_ctlr_exp = class_conn2export(&conn); OBD_ALLOC_PTR(ls->ls_client_seq); if (ls->ls_client_seq != NULL) { rc = seq_client_init(ls->ls_client_seq, mdc->obd_name, - ls->ls_controller); + ls->ls_ctlr_exp); } else rc = -ENOMEM; @@ -1762,29 +1780,40 @@ static int mdt_seq_ctlr_init(const struct lu_context *ctx, LASSERT(ls->ls_server_seq != NULL); - rc = seq_server_set_ctlr(ls->ls_server_seq, - ls->ls_client_seq, - ctx); + rc = seq_server_init_ctlr(ls->ls_server_seq, + ls->ls_client_seq, + ctx); } } RETURN(rc); } -static void mdt_seq_ctlr_fini(struct mdt_device *m) +static void mdt_seq_fini_ctlr(struct mdt_device *m) { struct lu_site *ls; ENTRY; ls = m->mdt_md_dev.md_lu_dev.ld_site; - if (ls && ls->ls_controller) { - int rc; - rc = obd_disconnect(ls->ls_controller); - ls->ls_controller = NULL; - if (rc != 0) - CERROR("Failure to disconnect obd: %d\n", rc); + if (ls && ls->ls_server_seq) + seq_server_fini_ctlr(ls->ls_server_seq); + + if (ls && ls->ls_client_seq) { + seq_client_fini(ls->ls_client_seq); + OBD_FREE_PTR(ls->ls_client_seq); + ls->ls_client_seq = NULL; + } + + if (ls && ls->ls_ctlr_exp) { + int rc = obd_disconnect(ls->ls_ctlr_exp); + ls->ls_ctlr_exp = NULL; + + if (rc) { + CERROR("failure to disconnect " + "obd: %d\n", rc); + } } EXIT; } @@ -2014,7 +2043,7 @@ static void mdt_fini(const struct lu_context *ctx, struct mdt_device *m) mdt_fld_fini(ctx, m); mdt_seq_fini(ctx, m); - mdt_seq_ctlr_fini(m); + mdt_seq_fini_ctlr(m); LASSERT(atomic_read(&d->ld_ref) == 0); md_device_fini(&m->mdt_md_dev); @@ -2119,7 +2148,7 @@ static int mdt_process_config(const struct lu_context *ctx, case LCFG_ADD_MDC: /* add mdc hook to get first MDT uuid and connect it to * ls->controller to use for seq manager. */ - err = mdt_seq_ctlr_init(ctx, mdt_dev(d), cfg); + err = mdt_seq_init_ctlr(ctx, mdt_dev(d), cfg); if (err) { CERROR("can't initialize controller export, " "rc %d\n", err);