From 7fb42104d6467d6ca38a4d4c8de6d5355e4fcf7d Mon Sep 17 00:00:00 2001 From: lincent Date: Sun, 9 Oct 2005 09:26:41 +0000 Subject: [PATCH] adapt to lnet from portals --- lustre/include/linux/lustre_idl.h | 4 ++ lustre/include/linux/lustre_net.h | 31 ++++++------- lustre/include/linux/obd.h | 4 -- lustre/ldlm/ldlm_lib.c | 96 +++++++++++++++------------------------ lustre/obdclass/class_obd.c | 6 +-- lustre/obdclass/obd_config.c | 6 +-- lustre/utils/mkfs_lustre.c | 6 +-- 7 files changed, 63 insertions(+), 90 deletions(-) diff --git a/lustre/include/linux/lustre_idl.h b/lustre/include/linux/lustre_idl.h index 0d1673b..cf94497 100644 --- a/lustre/include/linux/lustre_idl.h +++ b/lustre/include/linux/lustre_idl.h @@ -830,6 +830,9 @@ struct ldlm_flock { __u32 pid; }; +struct ldlm_llog { + /*FIXME*/ +}; /* it's important that the fields of the ldlm_extent structure match * the first fields of the ldlm_flock structure because there is only * one ldlm_swab routine to process the ldlm_policy_data_t union. if @@ -839,6 +842,7 @@ struct ldlm_flock { typedef union { struct ldlm_extent l_extent; struct ldlm_flock l_flock; + struct ldlm_llog l_llog; } ldlm_policy_data_t; extern void lustre_swab_ldlm_policy_data (ldlm_policy_data_t *d); diff --git a/lustre/include/linux/lustre_net.h b/lustre/include/linux/lustre_net.h index 55e7245..95567a8 100644 --- a/lustre/include/linux/lustre_net.h +++ b/lustre/include/linux/lustre_net.h @@ -46,7 +46,7 @@ /* Define maxima for bulk I/O * CAVEAT EMPTOR, with multinet (i.e. routers forwarding between networks) * these limits are system wide and not interface-local. */ -#define PTLRPC_MAX_BRW_SIZE PTL_MTU +#define PTLRPC_MAX_BRW_SIZE LNET_MTU #define PTLRPC_MAX_BRW_PAGES (PTLRPC_MAX_BRW_SIZE/PAGE_SIZE) /* When PAGE_SIZE is a constant, we can check our arithmetic here with cpp! */ @@ -57,10 +57,10 @@ # if (PTLRPC_MAX_BRW_SIZE != (PTLRPC_MAX_BRW_PAGES * PAGE_SIZE)) # error "PTLRPC_MAX_BRW_SIZE isn't PTLRPC_MAX_BRW_PAGES * PAGE_SIZE" # endif -# if (PTLRPC_MAX_BRW_SIZE > PTL_MTU) +# if (PTLRPC_MAX_BRW_SIZE > LNET_MTU) # error "PTLRPC_MAX_BRW_SIZE too big" # endif -# if (PTLRPC_MAX_BRW_PAGES > PTL_MD_MAX_IOV) +# if (PTLRPC_MAX_BRW_PAGES > LNET_MAX_IOV) # error "PTLRPC_MAX_BRW_PAGES too big" # endif #endif /* __KERNEL__ */ @@ -89,13 +89,6 @@ #define LDLM_MAXREQSIZE (5 * 1024) #define LDLM_MAXREPSIZE (1024) -#define MGT_MAX_THREADS 8UL -#define MGT_NUM_THREADS max(min_t(unsigned long, num_physpages / 8192, \ - MGT_MAX_THREADS), 2UL) -#define MGS_NBUFS (64 * smp_num_cpus) -#define MGS_BUFSIZE (8 * 1024) -#define MGS_MAXREQSIZE (5 * 1024) - #define MDT_MAX_THREADS 32UL #define MDT_NUM_THREADS max(min_t(unsigned long, num_physpages / 8192, \ MDT_MAX_THREADS), 2UL) @@ -139,7 +132,8 @@ struct ptlrpc_connection { struct list_head c_link; - lnet_process_id_t c_peer; + lnet_nid_t c_self; + lnet_process_id_t c_peer; struct obd_uuid c_remote_uuid; atomic_t c_refcount; }; @@ -335,7 +329,8 @@ struct ptlrpc_request { wait_queue_head_t rq_reply_waitq; struct ptlrpc_cb_id rq_reply_cbid; - lnet_process_id_t rq_peer; + lnet_nid_t rq_self; + lnet_process_id_t rq_peer; struct obd_export *rq_export; struct obd_import *rq_import; @@ -461,10 +456,9 @@ struct ptlrpc_thread { struct list_head t_link; /* active threads for service, from svc->srv_threads */ + void *t_data; /* thread-private data (preallocated memory) */ __u32 t_flags; - void *t_data; /* thread-private data (preallocated memory) */ - unsigned int t_id; /* service thread index, from ptlrpc_start_threads */ wait_queue_head_t t_ctl_waitq; }; @@ -495,7 +489,8 @@ struct ptlrpc_service { int srv_n_active_reqs; /* # reqs being served */ int srv_rqbd_timeout; /* timeout before re-posting reqs */ int srv_watchdog_timeout; /* soft watchdog timeout, in ms */ - int srv_num_threads; /*# of threads to start/started*/ + int srv_num_threads; /* # threads to start/started */ + unsigned srv_cpu_affinity:1; /* bind threads to CPUs */ __u32 srv_req_portal; __u32 srv_rep_portal; @@ -553,7 +548,8 @@ struct ptlrpc_service { /* ptlrpc/events.c */ extern lnet_handle_eq_t ptlrpc_eq_h; -extern int ptlrpc_uuid_to_peer(struct obd_uuid *uuid, lnet_process_id_t *peer); +extern int ptlrpc_uuid_to_peer(struct obd_uuid *uuid, + lnet_process_id_t *peer, lnet_nid_t *self); extern void request_out_callback (lnet_event_t *ev); extern void reply_in_callback(lnet_event_t *ev); extern void client_bulk_callback (lnet_event_t *ev); @@ -565,7 +561,7 @@ extern void server_bulk_callback (lnet_event_t *ev); void ptlrpc_dump_connections(void); void ptlrpc_readdress_connection(struct ptlrpc_connection *, struct obd_uuid *); struct ptlrpc_connection *ptlrpc_get_connection(lnet_process_id_t peer, - struct obd_uuid *uuid); + lnet_nid_t self, struct obd_uuid *uuid); int ptlrpc_put_connection(struct ptlrpc_connection *c); struct ptlrpc_connection *ptlrpc_connection_addref(struct ptlrpc_connection *); void ptlrpc_init_connection(void); @@ -594,6 +590,7 @@ int ptlrpc_reply(struct ptlrpc_request *req); int ptlrpc_error(struct ptlrpc_request *req); void ptlrpc_resend_req(struct ptlrpc_request *request); int ptl_send_rpc(struct ptlrpc_request *request); +int ptl_send_rpc_nowait(struct ptlrpc_request *request); int ptlrpc_register_rqbd (struct ptlrpc_request_buffer_desc *rqbd); /* ptlrpc/client.c */ diff --git a/lustre/include/linux/obd.h b/lustre/include/linux/obd.h index 1f8316f..327998f 100644 --- a/lustre/include/linux/obd.h +++ b/lustre/include/linux/obd.h @@ -337,10 +337,6 @@ struct mgs_obd { struct vfsmount *mgs_vfsmnt; struct super_block *mgs_sb; struct dentry *mgs_configs_dir; - spinlock_t mgs_llogs_lock; - struct list_head mgs_open_llogs; - spinlock_t mgs_llhs_lock; - struct list_head mgs_update_llhs; }; struct mds_obd { diff --git a/lustre/ldlm/ldlm_lib.c b/lustre/ldlm/ldlm_lib.c index caa70cd..e90bc14 100644 --- a/lustre/ldlm/ldlm_lib.c +++ b/lustre/ldlm/ldlm_lib.c @@ -35,7 +35,6 @@ #include #include /* for LUSTRE_OSC_NAME */ #include /* for LUSTRE_MDC_NAME */ -#include #include #include @@ -96,7 +95,7 @@ static int import_set_conn(struct obd_import *imp, struct obd_uuid *uuid, } else { spin_unlock(&imp->imp_lock); GOTO(out_free, rc = -ENOENT); - + } spin_unlock(&imp->imp_lock); @@ -180,6 +179,13 @@ out: RETURN(rc); } +/* configure an RPC client OBD device + * + * lcfg parameters: + * 1 - client UUID + * 2 - server UUID + * 3 - inactive-on-startup + */ int client_obd_setup(struct obd_device *obddev, obd_count len, void *buf) { struct lustre_cfg* lcfg = buf; @@ -188,10 +194,7 @@ int client_obd_setup(struct obd_device *obddev, obd_count len, void *buf) struct obd_uuid server_uuid; int rq_portal, rp_portal, connect_op; char *name = obddev->obd_type->typ_name; - char *mgmt_name = NULL; int rc; - struct obd_device *mgmt_obd; - mgmtcli_register_for_events_t register_f; ENTRY; /* In a more perfect world, we would hang a ptlrpc_client off of @@ -204,10 +207,6 @@ int client_obd_setup(struct obd_device *obddev, obd_count len, void *buf) rq_portal = MDS_REQUEST_PORTAL; rp_portal = MDC_REPLY_PORTAL; connect_op = MDS_CONNECT; - } else if (!strcmp(name, LUSTRE_MGMTCLI_NAME)) { - rq_portal = MGMT_REQUEST_PORTAL; - rp_portal = MGMT_REPLY_PORTAL; - connect_op = MGMT_CONNECT; } else { CERROR("unknown client OBD type \"%s\", can't setup\n", name); @@ -310,39 +309,9 @@ int client_obd_setup(struct obd_device *obddev, obd_count len, void *buf) name, obddev->obd_name, imp->imp_target_uuid.uuid); imp->imp_invalid = 1; - - if (LUSTRE_CFG_BUFLEN(lcfg, 4) > 0) - mgmt_name = lustre_cfg_string(lcfg, 4); - } else { - mgmt_name = lustre_cfg_string(lcfg, 3); } } - if (mgmt_name != NULL) { - /* Register with management client if we need to. */ - CDEBUG(D_HA, "%s registering with %s for events about %s\n", - obddev->obd_name, mgmt_name, server_uuid.uuid); - - mgmt_obd = class_name2obd(mgmt_name); - if (!mgmt_obd) { - CERROR("can't find mgmtcli %s to register\n", - mgmt_name); - GOTO(err_import, rc = -ENOSYS); - } - - register_f = inter_module_get("mgmtcli_register_for_events"); - if (!register_f) { - CERROR("can't i_m_g mgmtcli_register_for_events\n"); - GOTO(err_import, rc = -ENOSYS); - } - - rc = register_f(mgmt_obd, obddev, &imp->imp_target_uuid); - inter_module_put("mgmtcli_register_for_events"); - - if (!rc) - cli->cl_mgmtcli_obd = mgmt_obd; - } - spin_lock_init(&cli->cl_qchk_lock); cli->cl_qchk_stat = CL_NO_QUOTACHECK; @@ -363,13 +332,6 @@ int client_obd_cleanup(struct obd_device *obddev) if (!cli->cl_import) RETURN(-EINVAL); - if (cli->cl_mgmtcli_obd) { - mgmtcli_deregister_for_events_t dereg_f; - - dereg_f = inter_module_get("mgmtcli_deregister_for_events"); - dereg_f(cli->cl_mgmtcli_obd, obddev); - inter_module_put("mgmtcli_deregister_for_events"); - } class_destroy_import(cli->cl_import); cli->cl_import = NULL; @@ -378,6 +340,7 @@ int client_obd_cleanup(struct obd_device *obddev) RETURN(0); } +/* ->o_connect() method for client side (OSC and MDC) */ int client_connect_import(struct lustre_handle *dlm_handle, struct obd_device *obd, struct obd_uuid *cluuid, struct obd_connect_data *data) @@ -385,6 +348,7 @@ int client_connect_import(struct lustre_handle *dlm_handle, struct client_obd *cli = &obd->u.cli; struct obd_import *imp = cli->cl_import; struct obd_export *exp; + struct obd_connect_data *ocd; int rc; ENTRY; @@ -410,8 +374,10 @@ int client_connect_import(struct lustre_handle *dlm_handle, if (rc != 0) GOTO(out_ldlm, rc); + ocd = &imp->imp_connect_data; if (data) - memcpy(&imp->imp_connect_data, data, sizeof(*data)); + *ocd = *data; + rc = ptlrpc_connect_import(imp, NULL); if (rc != 0) { LASSERT (imp->imp_state == LUSTRE_IMP_DISCON); @@ -419,6 +385,12 @@ int client_connect_import(struct lustre_handle *dlm_handle, } LASSERT(exp->exp_connection); + if (data) { + LASSERT((ocd->ocd_connect_flags & data->ocd_connect_flags) == + ocd->ocd_connect_flags); + data->ocd_connect_flags = ocd->ocd_connect_flags; + } + ptlrpc_pinger_add_import(imp); EXIT; @@ -524,7 +496,7 @@ int target_handle_reconnect(struct lustre_handle *conn, struct obd_export *exp, conn->cookie = exp->exp_handle.h_cookie; CDEBUG(D_INFO, "existing export for UUID '%s' at %p\n", cluuid->uuid, exp); - CDEBUG(D_IOCTL,"connect: cookie "LPX64"\n", conn->cookie); + CDEBUG(D_IOCTL, "connect: cookie "LPX64"\n", conn->cookie); RETURN(0); } @@ -692,7 +664,7 @@ int target_handle_connect(struct ptlrpc_request *req, svc_handler_t handler) spin_lock_irqsave(&export->exp_lock, flags); if (export->exp_conn_cnt >= req->rq_reqmsg->conn_cnt) { CERROR("%s: already connected at a higher conn_cnt: %d > %d\n", - cluuid.uuid, export->exp_conn_cnt, + cluuid.uuid, export->exp_conn_cnt, req->rq_reqmsg->conn_cnt); spin_unlock_irqrestore(&export->exp_lock, flags); GOTO(out, rc = -EALREADY); @@ -703,13 +675,15 @@ int target_handle_connect(struct ptlrpc_request *req, svc_handler_t handler) /* request from liblustre? Don't evict it for not pinging. */ if (lustre_msg_get_op_flags(req->rq_reqmsg) & MSG_CONNECT_LIBCLIENT) { export->exp_libclient = 1; + spin_lock(&target->obd_dev_lock); list_del_init(&export->exp_obd_chain_timed); + spin_unlock(&target->obd_dev_lock); } if (export->exp_connection != NULL) ptlrpc_put_connection(export->exp_connection); export->exp_connection = ptlrpc_get_connection(req->rq_peer, - &remote_uuid); + req->rq_self, &remote_uuid); if (rc == EALREADY) { /* We indicate the reconnection in a flag, not an error code. */ @@ -840,14 +814,14 @@ static void abort_recovery_queue(struct obd_device *obd) } } -/* Called from a cleanup function if the device is being cleaned up - forcefully. The exports should all have been disconnected already, - the only thing left to do is +/* Called from a cleanup function if the device is being cleaned up + forcefully. The exports should all have been disconnected already, + the only thing left to do is - clear the recovery flags - cancel the timer - free queued requests and replies, but don't send replies Because the obd_stopping flag is set, no new requests should be received. - + */ void target_cleanup_recovery(struct obd_device *obd) { @@ -932,10 +906,12 @@ static void reset_recovery_timer(struct obd_device *obd) spin_unlock_bh(&obd->obd_processing_task_lock); return; } - CDEBUG(D_HA, "%s: timer will expire in %u seconds\n", obd->obd_name, - (int)(OBD_RECOVERY_TIMEOUT / HZ)); mod_timer(&obd->obd_recovery_timer, jiffies + OBD_RECOVERY_TIMEOUT); spin_unlock_bh(&obd->obd_processing_task_lock); + CDEBUG(D_HA, "%s: timer will expire in %u seconds\n", obd->obd_name, + (int)(OBD_RECOVERY_TIMEOUT / HZ)); + /* Only used for lprocfs_status */ + obd->obd_recovery_end = CURRENT_SECONDS + OBD_RECOVERY_TIMEOUT/HZ; } @@ -1258,7 +1234,7 @@ target_send_reply_msg (struct ptlrpc_request *req, int rc, int fail_id) return (ptlrpc_send_reply(req, 1)); } -void +void target_send_reply(struct ptlrpc_request *req, int rc, int fail_id) { int netrc; @@ -1299,12 +1275,12 @@ target_send_reply(struct ptlrpc_request *req, int rc, int fail_id) rs->rs_xid = req->rq_xid; rs->rs_transno = req->rq_transno; rs->rs_export = exp; - + spin_lock_irqsave (&obd->obd_uncommitted_replies_lock, flags); if (rs->rs_transno > obd->obd_last_committed) { - /* not committed already */ - list_add_tail (&rs->rs_obd_list, + /* not committed already */ + list_add_tail (&rs->rs_obd_list, &obd->obd_uncommitted_replies); } diff --git a/lustre/obdclass/class_obd.c b/lustre/obdclass/class_obd.c index 00590c0..ccf303e 100644 --- a/lustre/obdclass/class_obd.c +++ b/lustre/obdclass/class_obd.c @@ -175,7 +175,7 @@ int class_handle_ioctl(unsigned int cmd, unsigned long arg) { char *buf = NULL; struct obd_ioctl_data *data; - struct portals_debug_ioctl_data *debug_data; + struct libcfs_debug_ioctl_data *debug_data; struct obd_device *obd = NULL; int err = 0, len = 0; ENTRY; @@ -189,8 +189,8 @@ int class_handle_ioctl(unsigned int cmd, unsigned long arg) RETURN(err = -ENOTTY); /* only for debugging */ - if (cmd == PTL_IOC_DEBUG_MASK) { - debug_data = (struct portals_debug_ioctl_data*)arg; + if (cmd == LIBCFS_IOC_DEBUG_MASK) { + debug_data = (struct libcfs_debug_ioctl_data*)arg; libcfs_subsystem_debug = debug_data->subs; libcfs_debug = debug_data->debug; return 0; diff --git a/lustre/obdclass/obd_config.c b/lustre/obdclass/obd_config.c index d411042..c188756 100644 --- a/lustre/obdclass/obd_config.c +++ b/lustre/obdclass/obd_config.c @@ -723,7 +723,7 @@ static int class_config_llog_handler(struct llog_handle * handle, __u32 addr = (__u32)(lcfg->lcfg_nid & 0xffffffff); lcfg_new->lcfg_nid = - PTL_MKNID(PTL_MKNET(lcfg->lcfg_nal, 0), addr); + LNET_MKNID(LNET_MKNET(lcfg->lcfg_nal, 0), addr); CWARN("Converted pre-newconfig NAL %d NID %x to %s\n", lcfg->lcfg_nal, addr, libcfs_nid2str(lcfg_new->lcfg_nid)); @@ -761,7 +761,7 @@ int class_config_parse_llog(struct llog_ctxt *ctxt, char *name, ENTRY; CDEBUG(D_INFO, "looking up llog %s\n", name); - rc = llog_create(ctxt, &llh, NULL, NULL, name); + rc = llog_create(ctxt, &llh, NULL, name); if (rc) RETURN(rc); @@ -829,7 +829,7 @@ int class_config_dump_llog(struct llog_ctxt *ctxt, char *name, int rc, rc2; ENTRY; - rc = llog_create(ctxt, &llh, NULL, NULL, name); + rc = llog_create(ctxt, &llh, NULL, name); if (rc) RETURN(rc); diff --git a/lustre/utils/mkfs_lustre.c b/lustre/utils/mkfs_lustre.c index d38b155..42a9b53 100644 --- a/lustre/utils/mkfs_lustre.c +++ b/lustre/utils/mkfs_lustre.c @@ -605,14 +605,14 @@ static int jt_setup() /* see jt_ptl_network */ int jt_getnids(lnet_nid_t *nidarray, int maxnids) { - struct portal_ioctl_data data; + struct libcfs_ioctl_data data; int count; int rc; for (count = 0; count < maxnids; count++) { - PORTAL_IOC_INIT (data); + LIBCFS_IOC_INIT (data); data.ioc_count = count; - rc = l_ioctl(LNET_DEV_ID, IOC_PORTAL_GET_NI, &data); + rc = l_ioctl(LNET_DEV_ID, IOC_LIBCFS_GET_NI, &data); if (rc >= 0) { vprint("%s\n", libcfs_nid2str(data.ioc_nid)); -- 1.8.3.1