- some fixes after DLDINSP,cleanups.
/* assigns client to sequence controller node */
int
-seq_server_set_ctlr(struct lu_server_seq *seq,
- struct lu_client_seq *cli,
- const struct lu_context *ctx)
+seq_server_init_ctlr(struct lu_server_seq *seq,
+ struct lu_client_seq *cli,
+ const struct lu_context *ctx)
{
int rc = 0;
ENTRY;
/* get new range from controller only if super-sequence is not yet
* initialized from backing store or something else. */
if (range_is_zero(&seq->seq_super)) {
- /* release sema to avoid deadlock for case we're asking our
- * selves. */
- up(&seq->seq_sem);
rc = seq_client_alloc_super(cli);
- down(&seq->seq_sem);
-
if (rc) {
CERROR("can't allocate super-sequence, "
"rc %d\n", rc);
up(&seq->seq_sem);
RETURN(rc);
}
-EXPORT_SYMBOL(seq_server_set_ctlr);
+EXPORT_SYMBOL(seq_server_init_ctlr);
+
+void
+seq_server_fini_ctlr(struct lu_server_seq *seq)
+{
+ ENTRY;
+
+ down(&seq->seq_sem);
+ seq->seq_cli = NULL;
+ up(&seq->seq_sem);
+
+ EXIT;
+}
+EXPORT_SYMBOL(seq_server_fini_ctlr);
/* on controller node, allocate new super sequence for regular sequnece
* server. */
RETURN(-EOPNOTSUPP);
}
- /* allocate new super-sequence. */
- up(&seq->seq_sem);
rc = seq_client_alloc_super(seq->seq_cli);
- down(&seq->seq_sem);
if (rc) {
CERROR("can't allocate new super-sequence, "
"rc %d\n", rc);
RETURN(rc);
}
- if (seq->seq_cli->seq_range.lr_start > super->lr_start) {
- /* saving new range into allocation space. */
- *super = seq->seq_cli->seq_range;
- LASSERT(range_is_sane(super));
- } else {
- /* XXX: race is catched, ignore what we have from
- * controller node. The only issue is that controller
- * node has now this super-sequence lost, what makes
- * sequences space smaller. */
- CWARN("SEQ-MGR(srv): race is cached, reject "
- "allocated super-sequence\n");
- RETURN(0);
- }
+ /* saving new range into allocation space. */
+ *super = seq->seq_cli->seq_range;
+ LASSERT(range_is_sane(super));
}
range_alloc(range, super, seq->seq_meta_width);
rc = seq_server_alloc_meta(seq, range, ctx);
break;
default:
+ CERROR("wrong opc 0x%x\n", opc);
rc = -EINVAL;
break;
}
-
RETURN(rc);
}
static int
seq_req_handle0(const struct lu_context *ctx,
- struct lu_server_seq *seq,
struct ptlrpc_request *req)
{
int rep_buf_size[2] = { 0, };
+ struct obd_device *obd;
struct req_capsule pill;
+ struct lu_site *site;
struct lu_range *out;
int rc = -EPROTO;
__u32 *opc;
ENTRY;
+ obd = req->rq_export->exp_obd;
+ site = obd->obd_lu_dev->ld_site;
+ LASSERT(site != NULL);
+
req_capsule_init(&pill, req, RCL_SERVER,
rep_buf_size);
CERROR("can't get range buffer\n");
GOTO(out_pill, rc= -EPROTO);
}
- rc = seq_server_handle(seq, ctx, out, *opc);
+
+ if (*opc == SEQ_ALLOC_META) {
+ if (!site->ls_server_seq) {
+ CERROR("sequence-server is not initialized\n");
+ GOTO(out_pill, rc == -EINVAL);
+ }
+ rc = seq_server_handle(site->ls_server_seq,
+ ctx, out, *opc);
+ } else {
+ if (!site->ls_ctlr_seq) {
+ CERROR("sequence-controller is not initialized\n");
+ GOTO(out_pill, rc == -EINVAL);
+ }
+ rc = seq_server_handle(site->ls_ctlr_seq,
+ ctx, out, *opc);
+ }
} else {
CERROR("cannot unpack client request\n");
}
{
int fail = OBD_FAIL_SEQ_ALL_REPLY_NET;
const struct lu_context *ctx;
- struct lu_site *site;
int rc = -EPROTO;
ENTRY;
LASSERT(ctx->lc_thread == req->rq_svc_thread);
if (req->rq_reqmsg->opc == SEQ_QUERY) {
if (req->rq_export != NULL) {
- struct obd_device *obd;
-
- obd = req->rq_export->exp_obd;
- site = obd->obd_lu_dev->ld_site;
- LASSERT(site != NULL);
-
- rc = seq_req_handle0(ctx, site->ls_server_seq, req);
+ rc = seq_req_handle0(ctx, req);
} else {
CERROR("Unconnected request\n");
req->rq_status = -ENOTCONN;
seq_server_init(struct lu_server_seq *seq,
struct dt_device *dev,
const char *uuid,
+ lu_server_type_t type,
const struct lu_context *ctx)
{
- int rc;
+ int rc, portal = (type == LUSTRE_SEQ_SRV) ?
+ SEQ_SRV_PORTAL : SEQ_CTLR_PORTAL;
+
struct ptlrpc_service_conf seq_conf = {
.psc_nbufs = MDS_NBUFS,
.psc_bufsize = MDS_BUFSIZE,
.psc_max_req_size = MDS_MAXREQSIZE,
.psc_max_reply_size = MDS_MAXREPSIZE,
- .psc_req_portal = MDS_SEQ_PORTAL,
+ .psc_req_portal = portal,
.psc_rep_portal = MDC_REPLY_PORTAL,
.psc_watchdog_timeout = SEQ_SERVICE_WATCHDOG_TIMEOUT,
.psc_num_threads = SEQ_NUM_THREADS
seq->seq_dev = dev;
seq->seq_cli = NULL;
+ seq->seq_type = type;
sema_init(&seq->seq_sem, 1);
seq->seq_super_width = LUSTRE_SEQ_SUPER_WIDTH;
seq->seq_meta_width = LUSTRE_SEQ_META_WIDTH;
- snprintf(seq->seq_name, sizeof(seq->seq_name),
- "%s-%s", LUSTRE_SEQ_NAME, uuid);
+ snprintf(seq->seq_name, sizeof(seq->seq_name), "%s-%s-%s",
+ LUSTRE_SEQ_NAME, (type == LUSTRE_SEQ_SRV ? "srv" : "ctlr"),
+ uuid);
seq->seq_space = LUSTRE_SEQ_SPACE_RANGE;
seq->seq_super = LUSTRE_SEQ_ZERO_RANGE;
/* request backing store for saved sequence info */
rc = seq_store_read(seq, ctx);
if (rc == -ENODATA) {
- CDEBUG(D_INFO|D_WARNING, "SEQ-MGR(srv): no data on "
- "disk found, waiting for controller assign\n");
+ if (type == LUSTRE_SEQ_SRV) {
+ CDEBUG(D_INFO|D_WARNING, "SEQ-MGR(srv): no data on "
+ "disk found, waiting for controller assign\n");
+ } else {
+ CDEBUG(D_INFO|D_WARNING, "SEQ-MGR(ctlr): no data on "
+ "disk found, this is first controller run\n");
+ }
} else if (rc) {
CERROR("can't read sequence state, rc = %d\n",
rc);
seq->seq_service = ptlrpc_init_svc_conf(&seq_conf,
seq_req_handle,
- LUSTRE_SEQ_NAME,
+ LUSTRE_SEQ_NAME,
seq->seq_proc_entry,
NULL);
if (seq->seq_service != NULL)
rc = ptlrpc_start_threads(NULL, seq->seq_service,
- LUSTRE_SEQ_NAME);
+ LUSTRE_SEQ_NAME);
else
rc = -ENOMEM;
EXIT;
out:
- if (rc)
+ if (rc) {
seq_server_fini(seq, ctx);
- else
- CDEBUG(D_INFO|D_WARNING, "Server Sequence "
- "Manager\n");
+ } else {
+ CDEBUG(D_INFO|D_WARNING, "%s Sequence Manager\n",
+ (type == LUSTRE_SEQ_SRV ? "Server" : "Controller"));
+ }
return rc;
}
EXPORT_SYMBOL(seq_server_init);
*op = opc;
req->rq_replen = lustre_msg_size(1, &repsize);
- req->rq_request_portal = MDS_SEQ_PORTAL;
+
+ req->rq_request_portal = (opc == SEQ_ALLOC_SUPER) ?
+ SEQ_CTLR_PORTAL : SEQ_SRV_PORTAL;
rc = ptlrpc_queue_wait(req);
if (rc)
return rc;
}
-/* request sequence-controller node to allocate new super-sequence. */
static int
-__seq_client_alloc_super(struct lu_client_seq *seq)
+__seq_client_alloc_opc(struct lu_client_seq *seq,
+ int opc, const char *opcname)
{
int rc;
ENTRY;
- rc = seq_client_rpc(seq, &seq->seq_range, SEQ_ALLOC_SUPER);
+ rc = seq_client_rpc(seq, &seq->seq_range, opc);
if (rc == 0) {
- CDEBUG(D_INFO, "SEQ-MGR(cli): allocated super-sequence "
- "["LPX64"-"LPX64"]\n", seq->seq_range.lr_start,
+ CDEBUG(D_INFO, "SEQ-MGR(cli): allocated "
+ "%s-sequence ["LPX64"-"LPX64"]\n",
+ opcname, seq->seq_range.lr_start,
seq->seq_range.lr_end);
}
RETURN(rc);
}
+/* request sequence-controller node to allocate new super-sequence. */
+static int
+__seq_client_alloc_super(struct lu_client_seq *seq)
+{
+ ENTRY;
+ RETURN(__seq_client_alloc_opc(seq, SEQ_ALLOC_SUPER, "super"));
+}
+
int
seq_client_alloc_super(struct lu_client_seq *seq)
{
static int
__seq_client_alloc_meta(struct lu_client_seq *seq)
{
- int rc;
ENTRY;
-
- rc = seq_client_rpc(seq, &seq->seq_range, SEQ_ALLOC_META);
- if (rc == 0) {
- CDEBUG(D_INFO, "SEQ-MGR(cli): allocated meta-sequence "
- "["LPX64"-"LPX64"]\n", seq->seq_range.lr_start,
- seq->seq_range.lr_end);
- }
- RETURN(rc);
+ RETURN(__seq_client_alloc_opc(seq, SEQ_ALLOC_META, "meta"));
}
int
if (rc) {
CERROR("can't allocate new meta-sequence, "
"rc %d\n", rc);
+ RETURN(rc);
}
}
.psc_bufsize = MDS_BUFSIZE,
.psc_max_req_size = MDS_MAXREQSIZE,
.psc_max_reply_size = MDS_MAXREPSIZE,
- .psc_req_portal = MDS_FLD_PORTAL,
+ .psc_req_portal = FLD_REQUEST_PORTAL,
.psc_rep_portal = MDC_REPLY_PORTAL,
.psc_watchdog_timeout = FLD_SERVICE_WATCHDOG_TIMEOUT,
.psc_num_threads = FLD_NUM_THREADS
memcpy(pmf, mf, sizeof(*mf));
req->rq_replen = lustre_msg_size(1, &mf_size);
- req->rq_request_portal = MDS_FLD_PORTAL;
+ req->rq_request_portal = FLD_REQUEST_PORTAL;
rc = ptlrpc_queue_wait(req);
if (rc)
struct lu_server_seq *ls_server_seq;
/*
+ * Controller Seq Manager
+ */
+ struct lu_server_seq *ls_ctlr_seq;
+
+ /*
* Clienbt Seq Manager
*/
struct lu_client_seq *ls_client_seq;
/* sequence controller node */
- struct obd_export *ls_controller;
+ struct obd_export *ls_ctlr_exp;
/* statistical counters. Protected by nothing, races are accepted. */
struct {
#define MGS_REQUEST_PORTAL 26
#define MGS_REPLY_PORTAL 27
#define OST_REQUEST_PORTAL 28
-#define MDS_FLD_PORTAL 29
-#define MDS_SEQ_PORTAL 30
+#define FLD_REQUEST_PORTAL 29
+#define SEQ_SRV_PORTAL 30
+#define SEQ_CTLR_PORTAL 31
#define SVC_KILLED 1
#define SVC_EVENT 2
* allocated to MDTs. */
#define LUSTRE_SEQ_SUPER_WIDTH (LUSTRE_SEQ_META_WIDTH * LUSTRE_SEQ_META_WIDTH)
+enum lu_server_type {
+ LUSTRE_SEQ_SRV,
+ LUSTRE_SEQ_CTLR
+};
+
+typedef enum lu_server_type lu_server_type_t;
+
/* client sequence manager interface */
struct lu_client_seq {
/* sequence-controller export. */
cfs_proc_dir_entry_t *seq_proc_entry;
cfs_proc_dir_entry_t *seq_proc_dir;
+ /* LUSTRE_SEQ_SRV or LUSTRE_SEQ_CTLR */
+ lu_server_type_t seq_type;
/* server side seq service */
struct ptlrpc_service *seq_service;
#endif
#ifdef __KERNEL__
+
int seq_server_init(struct lu_server_seq *seq,
struct dt_device *dev,
const char *uuid,
+ lu_server_type_t type,
const struct lu_context *ctx);
void seq_server_fini(struct lu_server_seq *seq,
const struct lu_context *ctx);
-int seq_server_set_ctlr(struct lu_server_seq *seq,
- struct lu_client_seq *cli,
- const struct lu_context *ctx);
+int seq_server_init_ctlr(struct lu_server_seq *seq,
+ struct lu_client_seq *cli,
+ const struct lu_context *ctx);
+
+void seq_server_fini_ctlr(struct lu_server_seq *seq);
#endif
int seq_client_init(struct lu_client_seq *seq,
OBD_FREE_PTR(ls->ls_server_seq);
ls->ls_server_seq = NULL;
}
- if (ls && ls->ls_client_seq) {
- seq_client_fini(ls->ls_client_seq);
- OBD_FREE_PTR(ls->ls_client_seq);
- ls->ls_client_seq = NULL;
+ if (ls && ls->ls_ctlr_seq) {
+ seq_server_fini(ls->ls_ctlr_seq, ctx);
+ OBD_FREE_PTR(ls->ls_ctlr_seq);
+ ls->ls_ctlr_seq = NULL;
}
RETURN(0);
}
ls = m->mdt_md_dev.md_lu_dev.ld_site;
+ /* sequence-controller node */
+ if (ls->ls_node_id == 0) {
+ LASSERT(ls->ls_ctlr_seq == NULL);
+ OBD_ALLOC_PTR(ls->ls_ctlr_seq);
+
+ if (ls->ls_ctlr_seq != NULL) {
+ rc = seq_server_init(ls->ls_ctlr_seq,
+ m->mdt_bottom, uuid,
+ LUSTRE_SEQ_CTLR,
+ ctx);
+ if (rc)
+ mdt_seq_fini(ctx, m);
+ } else
+ rc = -ENOMEM;
+ }
+
+ LASSERT(ls->ls_server_seq == NULL);
OBD_ALLOC_PTR(ls->ls_server_seq);
if (ls->ls_server_seq != NULL) {
rc = seq_server_init(ls->ls_server_seq,
m->mdt_bottom, uuid,
+ LUSTRE_SEQ_SRV,
ctx);
if (rc)
mdt_seq_fini(ctx, m);
}
/* XXX: this is ugly, should be something else */
-static int mdt_seq_ctlr_init(const struct lu_context *ctx,
+static int mdt_seq_init_ctlr(const struct lu_context *ctx,
struct mdt_device *m,
struct lustre_cfg *cfg)
{
/* check if this is first MDC add and controller is not yet
* initialized. */
- if (index != 0 || ls->ls_controller)
+ if (index != 0 || ls->ls_ctlr_exp)
RETURN(0);
uuid_str = lustre_cfg_string(cfg, 1);
CERROR("target %s connect error %d\n",
mdc->obd_name, rc);
} else {
- ls->ls_controller = class_conn2export(&conn);
+ ls->ls_ctlr_exp = class_conn2export(&conn);
OBD_ALLOC_PTR(ls->ls_client_seq);
if (ls->ls_client_seq != NULL) {
rc = seq_client_init(ls->ls_client_seq,
mdc->obd_name,
- ls->ls_controller);
+ ls->ls_ctlr_exp);
} else
rc = -ENOMEM;
LASSERT(ls->ls_server_seq != NULL);
- rc = seq_server_set_ctlr(ls->ls_server_seq,
- ls->ls_client_seq,
- ctx);
+ rc = seq_server_init_ctlr(ls->ls_server_seq,
+ ls->ls_client_seq,
+ ctx);
}
}
RETURN(rc);
}
-static void mdt_seq_ctlr_fini(struct mdt_device *m)
+static void mdt_seq_fini_ctlr(struct mdt_device *m)
{
struct lu_site *ls;
ENTRY;
ls = m->mdt_md_dev.md_lu_dev.ld_site;
- if (ls && ls->ls_controller) {
- int rc;
- rc = obd_disconnect(ls->ls_controller);
- ls->ls_controller = NULL;
- if (rc != 0)
- CERROR("Failure to disconnect obd: %d\n", rc);
+ if (ls && ls->ls_server_seq)
+ seq_server_fini_ctlr(ls->ls_server_seq);
+
+ if (ls && ls->ls_client_seq) {
+ seq_client_fini(ls->ls_client_seq);
+ OBD_FREE_PTR(ls->ls_client_seq);
+ ls->ls_client_seq = NULL;
+ }
+
+ if (ls && ls->ls_ctlr_exp) {
+ int rc = obd_disconnect(ls->ls_ctlr_exp);
+ ls->ls_ctlr_exp = NULL;
+
+ if (rc) {
+ CERROR("failure to disconnect "
+ "obd: %d\n", rc);
+ }
}
EXIT;
}
mdt_fld_fini(ctx, m);
mdt_seq_fini(ctx, m);
- mdt_seq_ctlr_fini(m);
+ mdt_seq_fini_ctlr(m);
LASSERT(atomic_read(&d->ld_ref) == 0);
md_device_fini(&m->mdt_md_dev);
case LCFG_ADD_MDC:
/* add mdc hook to get first MDT uuid and connect it to
* ls->controller to use for seq manager. */
- err = mdt_seq_ctlr_init(ctx, mdt_dev(d), cfg);
+ err = mdt_seq_init_ctlr(ctx, mdt_dev(d), cfg);
if (err) {
CERROR("can't initialize controller export, "
"rc %d\n", err);
git://git.whamcloud.com / fs / lustre-release.git / commitdiff