From: yury Date: Mon, 19 Jun 2006 07:46:24 +0000 (+0000) Subject: - seq-mgr reworked, now it simpler, and cleaner; X-Git-Tag: v1_8_0_110~486^2~1614 X-Git-Url: https://git.whamcloud.com/gitweb?a=commitdiff_plain;h=276baf0cc7d28493556d85d83c871a1cf2f895cf;p=fs%2Flustre-release.git - seq-mgr reworked, now it simpler, and cleaner; - fix in utils/Makefile.am --- diff --git a/lustre/fid/fid_handler.c b/lustre/fid/fid_handler.c index 2f1ceb0..efac264 100644 --- a/lustre/fid/fid_handler.c +++ b/lustre/fid/fid_handler.c @@ -49,12 +49,12 @@ #ifdef __KERNEL__ /* server side seq mgr stuff */ -static const struct lu_range LUSTRE_SEQ_SUPER_INIT = { +static const struct lu_range LUSTRE_SEQ_SPACE_INIT = { LUSTRE_SEQ_SPACE_START, - LUSTRE_SEQ_SPACE_LIMIT + LUSTRE_SEQ_SPACE_END }; -static const struct lu_range LUSTRE_SEQ_META_INIT = { +static const struct lu_range LUSTRE_SEQ_SUPER_INIT = { 0, 0 }; @@ -85,28 +85,32 @@ seq_server_read_state(struct lu_server_seq *seq, RETURN(rc); } +/* on controller node, allocate new super sequence for regular sequnece + * server. */ static int seq_server_alloc_super(struct lu_server_seq *seq, struct lu_range *range) { - struct lu_range *ss_range = &seq->seq_ss_range; + struct lu_range *space = &seq->seq_space; int rc; ENTRY; - if (ss_range->lr_end - ss_range->lr_start < LUSTRE_SEQ_SUPER_CHUNK) { - CWARN("super-sequence is going to exhauste soon. " + LASSERT(range_is_sane(space)); + + if (space->lr_end - space->lr_start < LUSTRE_SEQ_SUPER_CHUNK) { + CWARN("sequences space is going to exhauste soon. " "Only can allocate "LPU64" sequences\n", - ss_range->lr_end - ss_range->lr_start); - *range = *ss_range; - ss_range->lr_start = ss_range->lr_end; + space->lr_end - space->lr_start); + *range = *space; + space->lr_start = space->lr_end; rc = 0; - } else if (ss_range->lr_start >= ss_range->lr_end) { - CERROR("super-sequence is exhausted\n"); + } else if (space->lr_start == space->lr_end) { + CERROR("sequences space is exhausted\n"); rc = -ENOSPC; } else { - range->lr_start = ss_range->lr_start; - ss_range->lr_start += LUSTRE_SEQ_SUPER_CHUNK; - range->lr_end = ss_range->lr_start; + range->lr_start = space->lr_start; + space->lr_start += LUSTRE_SEQ_SUPER_CHUNK; + range->lr_end = space->lr_start; rc = 0; } @@ -122,41 +126,37 @@ static int seq_server_alloc_meta(struct lu_server_seq *seq, struct lu_range *range) { - struct lu_range *ms_range = &seq->seq_ms_range; + struct lu_range *super = &seq->seq_super; int rc; ENTRY; - LASSERT(range_is_sane(ms_range)); - + LASSERT(range_is_sane(super)); + /* XXX: here should avoid cascading RPCs using kind of async * preallocation when meta-sequence is close to exhausting. */ - if (ms_range->lr_start == ms_range->lr_end) { - if (seq->seq_flags & LUSTRE_SRV_SEQ_CONTROLLER) { - /* allocate new range of meta-sequences to allocate new - * meta-sequence from it. */ - rc = seq_server_alloc_super(seq, ms_range); - } else { - /* request controller to allocate new super-sequence for - * us.*/ - rc = seq_client_alloc_super(seq->seq_cli); - if (rc) { - CERROR("can't allocate new super-sequence, " - "rc %d\n", rc); - RETURN(rc); - } - - /* saving new range into allocation space. */ - *ms_range = seq->seq_cli->seq_cl_range; + if (super->lr_start == super->lr_end) { + if (!seq->seq_cli) { + CERROR("no seq-controller client is setup\n"); + RETURN(-EOPNOTSUPP); + } + + /* request controller to allocate new super-sequence for us.*/ + rc = seq_client_alloc_super(seq->seq_cli); + if (rc) { + CERROR("can't allocate new super-sequence, " + "rc %d\n", rc); + RETURN(rc); } - LASSERT(ms_range->lr_start != 0); - LASSERT(ms_range->lr_end > ms_range->lr_start); + /* saving new range into allocation space. */ + *super = seq->seq_cli->seq_range; + LASSERT(range_is_sane(super)); } else { rc = 0; } - range->lr_start = ms_range->lr_start; - ms_range->lr_start += LUSTRE_SEQ_META_CHUNK; - range->lr_end = ms_range->lr_start; + range->lr_start = super->lr_start; + super->lr_start += LUSTRE_SEQ_META_CHUNK; + range->lr_end = super->lr_start; if (rc == 0) { CDEBUG(D_INFO|D_WARNING, "SEQ-MGR(srv): allocated meta-sequence " @@ -282,12 +282,61 @@ out: return 0; } +/* assigns client to sequence controller node */ +int +seq_server_controller(struct lu_server_seq *seq, + struct lu_client_seq *cli, + const struct lu_context *ctx) +{ + int rc; + ENTRY; + + LASSERT(cli != NULL); + + if (seq->seq_cli) { + CERROR("SEQ-MGR(srv): sequence-controller " + "is already assigned\n"); + RETURN(-EINVAL); + } + + CDEBUG(D_INFO|D_WARNING, "SEQ-MGR(srv): assign " + "sequence controller client %s\n", + cli->seq_exp->exp_client_uuid.uuid); + + /* assign controller */ + seq->seq_cli = cli; + + /* allocate new super-sequence */ + rc = seq_client_alloc_super(cli); + if (rc) { + CERROR("can't allocate super-sequence, " + "rc %d\n", rc); + RETURN(rc); + } + + /* take super-seq from client seq mgr */ + LASSERT(range_is_sane(&cli->seq_range)); + + down(&seq->seq_sem); + seq->seq_super = cli->seq_range; + + /* save init seq to backing store. */ + rc = seq_server_write_state(seq, ctx); + if (rc) { + CERROR("can't write sequence state, " + "rc = %d\n", rc); + } + + up(&seq->seq_sem); + + RETURN(rc); +} +EXPORT_SYMBOL(seq_server_controller); + int seq_server_init(struct lu_server_seq *seq, - struct lu_client_seq *cli, - const struct lu_context *ctx, - struct dt_device *dev, - int flags) + struct dt_device *dev, int flags, + const struct lu_context *ctx) { int rc; struct ptlrpc_service_conf seq_conf = { @@ -303,49 +352,28 @@ seq_server_init(struct lu_server_seq *seq, ENTRY; LASSERT(dev != NULL); - LASSERT(cli != NULL); - - LASSERT(flags & (LUSTRE_SRV_SEQ_CONTROLLER | - LUSTRE_SRV_SEQ_REGULAR)); seq->seq_dev = dev; - seq->seq_cli = cli; + seq->seq_cli = NULL; seq->seq_flags = flags; sema_init(&seq->seq_sem, 1); + seq->seq_space = LUSTRE_SEQ_SPACE_INIT; + seq->seq_super = LUSTRE_SEQ_SUPER_INIT; + lu_device_get(&seq->seq_dev->dd_lu_dev); /* request backing store for saved sequence info */ rc = seq_server_read_state(seq, ctx); if (rc == -ENODATA) { - /* first run, no state on disk, init all seqs */ - if (seq->seq_flags & LUSTRE_SRV_SEQ_CONTROLLER) { - /* init super seq by start values on sequence-controller - * node.*/ - seq->seq_ss_range = LUSTRE_SEQ_SUPER_INIT; - } else { - /* take super-seq from client seq mgr */ - LASSERT(range_is_sane(&cli->seq_cl_range)); - seq->seq_ss_range = cli->seq_cl_range; - } - - /* init meta-sequence by start values and get ready for - * allocating it for clients. */ - seq->seq_ms_range = LUSTRE_SEQ_META_INIT; - - /* save init seq to backing store. */ - rc = seq_server_write_state(seq, ctx); - if (rc) { - CERROR("can't write sequence state, " - "rc = %d\n", rc); - GOTO(out, rc); - } + CDEBUG(D_INFO|D_WARNING, "SEQ-MGR(srv): no data on " + "disk found, waiting for controller assign\n"); } else if (rc) { CERROR("can't read sequence state, rc = %d\n", rc); GOTO(out, rc); } - + seq->seq_service = ptlrpc_init_svc_conf(&seq_conf, seq_req_handle, LUSTRE_SEQ0_NAME, @@ -363,7 +391,8 @@ out: if (rc) seq_server_fini(seq, ctx); else - CDEBUG(D_INFO|D_WARNING, "Server Sequence Manager initialized\n"); + CDEBUG(D_INFO|D_WARNING, "Server Sequence " + "Manager initialized\n"); return rc; } EXPORT_SYMBOL(seq_server_init); @@ -372,7 +401,7 @@ void seq_server_fini(struct lu_server_seq *seq, const struct lu_context *ctx) { - int rc; + ENTRY; if (seq->seq_service != NULL) { ptlrpc_unregister_service(seq->seq_service); @@ -380,16 +409,13 @@ seq_server_fini(struct lu_server_seq *seq, } if (seq->seq_dev != NULL) { - rc = seq_server_write_state(seq, ctx); - if (rc) { - CERROR("can't save sequence state, " - "rc = %d\n", rc); - } lu_device_put(&seq->seq_dev->dd_lu_dev); seq->seq_dev = NULL; } - CDEBUG(D_INFO|D_WARNING, "Server Sequence Manager finalized\n"); + CDEBUG(D_INFO|D_WARNING, "Server Sequence " + "Manager finalized\n"); + EXIT; } EXPORT_SYMBOL(seq_server_fini); @@ -427,5 +453,5 @@ MODULE_AUTHOR("Cluster File Systems, Inc. "); MODULE_DESCRIPTION("Lustre FID Module"); MODULE_LICENSE("GPL"); -cfs_module(fid, "0.0.3", fid_mod_init, fid_mod_exit); +cfs_module(fid, "0.0.4", fid_mod_init, fid_mod_exit); #endif diff --git a/lustre/fid/fid_request.c b/lustre/fid/fid_request.c index 964c279..1e75949 100644 --- a/lustre/fid/fid_request.c +++ b/lustre/fid/fid_request.c @@ -96,13 +96,12 @@ seq_client_alloc_super(struct lu_client_seq *seq) int rc; ENTRY; - LASSERT(seq->seq_flags & LUSTRE_CLI_SEQ_SERVER); - rc = seq_client_rpc(seq, &seq->seq_cl_range, + rc = seq_client_rpc(seq, &seq->seq_range, SEQ_ALLOC_SUPER); if (rc == 0) { CDEBUG(D_INFO|D_WARNING, "SEQ-MGR(cli): allocated super-sequence " - "["LPX64"-"LPX64"]\n", seq->seq_cl_range.lr_start, - seq->seq_cl_range.lr_end); + "["LPX64"-"LPX64"]\n", seq->seq_range.lr_start, + seq->seq_range.lr_end); } RETURN(rc); } @@ -115,13 +114,12 @@ seq_client_alloc_meta(struct lu_client_seq *seq) int rc; ENTRY; - LASSERT(seq->seq_flags & LUSTRE_CLI_SEQ_CLIENT); - rc = seq_client_rpc(seq, &seq->seq_cl_range, + rc = seq_client_rpc(seq, &seq->seq_range, SEQ_ALLOC_META); if (rc == 0) { CDEBUG(D_INFO|D_WARNING, "SEQ-MGR(cli): allocated meta-sequence " - "["LPX64"-"LPX64"]\n", seq->seq_cl_range.lr_start, - seq->seq_cl_range.lr_end); + "["LPX64"-"LPX64"]\n", seq->seq_range.lr_start, + seq->seq_range.lr_end); } RETURN(rc); } @@ -135,15 +133,13 @@ seq_client_alloc_seq(struct lu_client_seq *seq, __u64 *seqnr) ENTRY; down(&seq->seq_sem); - - LASSERT(seq->seq_flags & LUSTRE_CLI_SEQ_CLIENT); - LASSERT(range_is_sane(&seq->seq_cl_range)); + LASSERT(range_is_sane(&seq->seq_range)); /* if we still have free sequences in meta-sequence we allocate new seq * from given range. */ - if (seq->seq_cl_range.lr_end > seq->seq_cl_range.lr_start) { - *seqnr = seq->seq_cl_range.lr_start; - seq->seq_cl_range.lr_start += 1; + if (seq->seq_range.lr_end > seq->seq_range.lr_start) { + *seqnr = seq->seq_range.lr_start; + seq->seq_range.lr_start += 1; rc = 0; } else { /* meta-sequence is exhausted, request MDT to allocate new @@ -154,8 +150,8 @@ seq_client_alloc_seq(struct lu_client_seq *seq, __u64 *seqnr) "rc %d\n", rc); } - *seqnr = seq->seq_cl_range.lr_start; - seq->seq_cl_range.lr_start += 1; + *seqnr = seq->seq_range.lr_start; + seq->seq_range.lr_start += 1; } up(&seq->seq_sem); @@ -170,36 +166,42 @@ EXPORT_SYMBOL(seq_client_alloc_seq); int seq_client_alloc_fid(struct lu_client_seq *seq, struct lu_fid *fid) { + __u64 seqnr = 0; int rc; ENTRY; LASSERT(fid != NULL); - LASSERT(fid_is_sane(&seq->seq_fid)); - LASSERT(seq->seq_flags & LUSTRE_CLI_SEQ_CLIENT); down(&seq->seq_sem); - if (fid_oid(&seq->seq_fid) < LUSTRE_SEQ_WIDTH) { - *fid = seq->seq_fid; - seq->seq_fid.f_oid += 1; - rc = 0; - } else { - __u64 seqnr = 0; - + + if (!fid_is_sane(&seq->seq_fid) || + fid_oid(&seq->seq_fid) >= LUSTRE_SEQ_WIDTH) + { + /* allocate new sequence for case client hass no sequence at all + * or sequnece is exhausted and should be switched. */ + up(&seq->seq_sem); rc = seq_client_alloc_seq(seq, &seqnr); + down(&seq->seq_sem); if (rc) { CERROR("can't allocate new sequence, " "rc %d\n", rc); GOTO(out, rc); - } else { - seq->seq_fid.f_oid = LUSTRE_FID_INIT_OID; - seq->seq_fid.f_seq = seqnr; - seq->seq_fid.f_ver = 0; - - *fid = seq->seq_fid; - seq->seq_fid.f_oid += 1; - rc = -ERESTART; } + + /* init new fid */ + seq->seq_fid.f_oid = LUSTRE_FID_INIT_OID; + seq->seq_fid.f_seq = seqnr; + seq->seq_fid.f_ver = 0; + + /* inform caller that sequnece switch is performed to allow it + * to setup FLD for it. */ + rc = -ERESTART; + } else { + seq->seq_fid.f_oid += 1; + rc = 0; } + + *fid = seq->seq_fid; LASSERT(fid_is_sane(fid)); CDEBUG(D_INFO, "SEQ-MGR(cli): allocated FID "DFID3"\n", @@ -217,57 +219,23 @@ seq_client_init(struct lu_client_seq *seq, struct obd_export *exp, int flags) { - int rc; ENTRY; - LASSERT(flags & (LUSTRE_CLI_SEQ_CLIENT | - LUSTRE_CLI_SEQ_SERVER)); - + LASSERT(exp != NULL); + seq->seq_flags = flags; fid_zero(&seq->seq_fid); sema_init(&seq->seq_sem, 1); - seq->seq_cl_range.lr_end = 0; - seq->seq_cl_range.lr_start = 0; + seq->seq_range.lr_end = 0; + seq->seq_range.lr_start = 0; if (exp != NULL) seq->seq_exp = class_export_get(exp); - if (seq->seq_flags & LUSTRE_CLI_SEQ_CLIENT) { - __u64 seqnr = 0; - - /* client (llite or MDC) init case, we need new sequence from - * MDT. This will allocate new meta-sequemce first, because seq - * range in init state and looks the same as exhausted. */ - rc = seq_client_alloc_seq(seq, &seqnr); - if (rc) { - CERROR("can't allocate new sequence, rc %d\n", rc); - GOTO(out, rc); - } else { - seq->seq_fid.f_oid = LUSTRE_FID_INIT_OID; - seq->seq_fid.f_seq = seqnr; - seq->seq_fid.f_ver = 0; - } - - LASSERT(fid_is_sane(&seq->seq_fid)); - } else { - /* check if this is controller node is trying to init client. */ - if (seq->seq_exp) { - /* MDT uses client seq manager to talk to sequence - * controller, and thus, we need super-sequence. */ - rc = seq_client_alloc_super(seq); - } else { - rc = 0; - } - } - - EXIT; -out: - if (rc) - seq_client_fini(seq); - else - CDEBUG(D_INFO|D_WARNING, "Client Sequence Manager initialized\n"); - return rc; + CDEBUG(D_INFO|D_WARNING, "Client Sequence " + "Manager initialized\n"); + RETURN(0); } EXPORT_SYMBOL(seq_client_init); diff --git a/lustre/include/lustre_fid.h b/lustre/include/lustre_fid.h index 69be22c..9946c43 100644 --- a/lustre/include/lustre_fid.h +++ b/lustre/include/lustre_fid.h @@ -38,7 +38,7 @@ struct lu_context; #define LUSTRE_SEQ_SPACE_START 0x400 /* maximal posible seq number */ -#define LUSTRE_SEQ_SPACE_LIMIT ((__u64)~0ULL) +#define LUSTRE_SEQ_SPACE_END ((__u64)~0ULL) /* this is how may FIDs may be allocated in one sequence. */ #define LUSTRE_SEQ_WIDTH 0x00000000000002800 @@ -64,7 +64,7 @@ struct lu_client_seq { /* range of allowed for allocation sequeces. When using lu_client_seq on * clients, this contains meta-sequence range. And for servers this * contains super-sequence range. */ - struct lu_range seq_cl_range; + struct lu_range seq_range; /* seq related proc */ struct proc_dir_entry *seq_proc_entry; @@ -76,14 +76,13 @@ struct lu_client_seq { #ifdef __KERNEL__ /* server sequence manager interface */ struct lu_server_seq { + /* available sequence space */ + struct lu_range seq_space; + /* super-sequence range, all super-sequences for other servers are * allocated from it. */ - struct lu_range seq_ss_range; - - /* meta-sequence range, all meta-sequences for clients are allocated - * from it. */ - struct lu_range seq_ms_range; - + struct lu_range seq_super; + /* device for server side seq manager needs (saving sequences to backing * store). */ struct dt_device *seq_dev; @@ -105,23 +104,17 @@ struct lu_server_seq { }; #endif -/* client seq mgr flags */ -#define LUSTRE_CLI_SEQ_CLIENT (1 << 0) -#define LUSTRE_CLI_SEQ_SERVER (1 << 1) - #ifdef __KERNEL__ -/* server seq mgr flags */ -#define LUSTRE_SRV_SEQ_CONTROLLER (1 << 0) -#define LUSTRE_SRV_SEQ_REGULAR (1 << 1) - int seq_server_init(struct lu_server_seq *seq, - struct lu_client_seq *cli, - const struct lu_context *ctx, - struct dt_device *dev, - int flags); + struct dt_device *dev, int flags, + const struct lu_context *ctx); void seq_server_fini(struct lu_server_seq *seq, - const struct lu_context *ctx) ; + const struct lu_context *ctx); + +int seq_server_controller(struct lu_server_seq *seq, + struct lu_client_seq *cli, + const struct lu_context *ctx); #endif int seq_client_init(struct lu_client_seq *seq, diff --git a/lustre/mdc/mdc_request.c b/lustre/mdc/mdc_request.c index 4904ab4..a35bd4d 100644 --- a/lustre/mdc/mdc_request.c +++ b/lustre/mdc/mdc_request.c @@ -1067,8 +1067,7 @@ static int mdc_fid_init(struct obd_export *exp) RETURN(-ENOMEM); /* init client side of sequence-manager */ - rc = seq_client_init(cli->cl_seq, exp, - LUSTRE_CLI_SEQ_CLIENT); + rc = seq_client_init(cli->cl_seq, exp, 0); if (rc) { OBD_FREE_PTR(cli->cl_seq); cli->cl_seq = NULL; diff --git a/lustre/mdt/mdt_handler.c b/lustre/mdt/mdt_handler.c index 37abf0f..5fcb1d6 100644 --- a/lustre/mdt/mdt_handler.c +++ b/lustre/mdt/mdt_handler.c @@ -1458,31 +1458,12 @@ static int mdt_seq_init(const struct lu_context *ctx, ls = m->mdt_md_dev.md_lu_dev.ld_site; - OBD_ALLOC_PTR(ls->ls_client_seq); - - if (ls->ls_client_seq != NULL) { - rc = seq_client_init(ls->ls_client_seq, - ls->ls_controller, - LUSTRE_CLI_SEQ_SERVER); - } else - rc = -ENOMEM; - - if (rc) - RETURN(rc); - OBD_ALLOC_PTR(ls->ls_server_seq); if (ls->ls_server_seq != NULL) { - int flags; - - flags = (ls->ls_node_id == 0) ? - LUSTRE_SRV_SEQ_CONTROLLER : - LUSTRE_SRV_SEQ_REGULAR; - rc = seq_server_init(ls->ls_server_seq, - ls->ls_client_seq, - ctx, m->mdt_bottom, - flags); + m->mdt_bottom, + 0, ctx); } else rc = -ENOMEM; @@ -1509,18 +1490,22 @@ static int mdt_seq_fini(const struct lu_context *ctx, } /* XXX: this is ugly, should be something else */ -static int mdt_controller_init(struct mdt_device *m, +static int mdt_controller_init(const struct lu_context *ctx, + struct mdt_device *m, struct lustre_cfg *cfg) { + struct lu_site *ls = m->mdt_md_dev.md_lu_dev.ld_site; struct obd_device *mdc; struct obd_uuid uuid; - struct lu_site *ls; char *uuid_str; int rc, index; ENTRY; index = simple_strtol(lustre_cfg_string(cfg, 2), NULL, 10); - if (index != 0) + + /* check if this is first MDC add and controller is not yet + * initialized. */ + if (index != 0 || ls->ls_controller) RETURN(0); uuid_str = lustre_cfg_string(cfg, 1); @@ -1545,8 +1530,25 @@ static int mdt_controller_init(struct mdt_device *m, CERROR("target %s connect error %d\n", mdc->obd_name, rc); } else { - ls = m->mdt_md_dev.md_lu_dev.ld_site; ls->ls_controller = class_conn2export(&conn); + + OBD_ALLOC_PTR(ls->ls_client_seq); + + if (ls->ls_client_seq != NULL) { + rc = seq_client_init(ls->ls_client_seq, + ls->ls_controller, + 0); + } else + rc = -ENOMEM; + + if (rc) + RETURN(rc); + + LASSERT(ls->ls_server_seq != NULL); + + rc = seq_server_controller(ls->ls_server_seq, + ls->ls_client_seq, + ctx); } } @@ -1922,7 +1924,7 @@ static int mdt_process_config(const struct lu_context *ctx, case LCFG_ADD_MDC: /* add mdc hook to get first MDT uuid and connect it to * ls->controller to use for seq manager. */ - err = mdt_controller_init(mdt_dev(d), cfg); + err = mdt_controller_init(ctx, mdt_dev(d), cfg); if (err) { CERROR("can't initialize controller export, " "rc %d\n", err); diff --git a/lustre/utils/Makefile.am b/lustre/utils/Makefile.am index f89a42e..efd09f3 100644 --- a/lustre/utils/Makefile.am +++ b/lustre/utils/Makefile.am @@ -45,7 +45,7 @@ wiretest_SOURCES = wiretest.c obdio_SOURCES = obdio.c obdiolib.c obdiolib.h obdbarrier_SOURCES = obdbarrier.c obdiolib.c obdiolib.h create_iam_SOURCES = create_iam.c -req_layout_SOURCES = create-iam.c +req_layout_SOURCES = req-layout.c llog_reader_SOURCES = llog_reader.c llog_reader_LDADD := $(LIBPTLCTL)