* FOO_BULK_PORTAL is for incoming bulk on the FOO
*/
-#define OSC_REQUEST_PORTAL 1
-#define OSC_REPLY_PORTAL 2
-#define OSC_BULK_PORTAL 3
-
-#define OST_REQUEST_PORTAL 4
-#define OST_REPLY_PORTAL 5
-#define OST_BULK_PORTAL 6
-
-#define MDC_REQUEST_PORTAL 7
-#define MDC_REPLY_PORTAL 8
-#define MDC_BULK_PORTAL 9
-
-#define MDS_REQUEST_PORTAL 10
-#define MDS_REPLY_PORTAL 11
-#define MDS_BULK_PORTAL 12
+#define CONNMGR_REQUEST_PORTAL 1
+#define CONNMGR_REPLY_PORTAL 2
+#define OSC_REQUEST_PORTAL 3
+#define OSC_REPLY_PORTAL 4
+#define OSC_BULK_PORTAL 5
+#define OST_REQUEST_PORTAL 6
+#define OST_REPLY_PORTAL 7
+#define OST_BULK_PORTAL 8
+#define MDC_REQUEST_PORTAL 9
+#define MDC_REPLY_PORTAL 10
+#define MDC_BULK_PORTAL 11
+#define MDS_REQUEST_PORTAL 12
+#define MDS_REPLY_PORTAL 13
+#define MDS_BULK_PORTAL 14
+#define LDLM_REQUEST_PORTAL 15
+#define LDLM_REPLY_PORTAL 16
+#define LDLM_CLI_REQUEST_PORTAL 17
+#define LDLM_CLI_REPLY_PORTAL 18
/* default rpc ring length */
#define RPC_RING_LENGTH 2
-/* generic wrappable next */
-#define NEXT_INDEX(index, max) (((index+1) >= max) ? 0 : (index+1))
-
-#define SVC_STOPPING 1
-#define SVC_RUNNING 2
-#define SVC_STOPPED 4
-#define SVC_KILLED 8
-#define SVC_EVENT 16
-#define SVC_LIST 32
-#define SVC_SIGNAL 64
-
-typedef int (*rep_unpack_t)(char *, int, struct ptlrep_hdr **, union ptl_rep *);
-typedef int (*req_pack_t)(char *, int, char *, int, struct ptlreq_hdr **,
- union ptl_req*, int *, char **);
-
-typedef int (*req_unpack_t)(char *buf, int len, struct ptlreq_hdr **,
- union ptl_req *);
-typedef int (*rep_pack_t)(char *buf1, int len1, char *buf2, int len2,
- struct ptlrep_hdr **, union ptl_rep*,
- int *replen, char **repbuf);
+#define SVC_KILLED 1
+#define SVC_EVENT 2
+#define SVC_SIGNAL 4
+#define SVC_RUNNING 8
+#define SVC_STOPPING 16
+#define SVC_STOPPED 32
+
+#define RECOVD_STOPPING 1 /* how cleanup tells recovd to quit */
+#define RECOVD_IDLE 2 /* normal state */
+#define RECOVD_STOPPED 4 /* after recovd has stopped */
+#define RECOVD_FAIL 8 /* RPC timeout: wakeup recovd, sets flag */
+#define RECOVD_TIMEOUT 16 /* set when recovd detects a timeout */
+#define RECOVD_UPCALL_WAIT 32 /* an upcall has been placed */
+#define RECOVD_UPCALL_ANSWER 64 /* an upcall has been answered */
+
+#define LUSTRE_CONN_NEW 1
+#define LUSTRE_CONN_CON 2
+#define LUSTRE_CONN_RECOVD 3
+#define LUSTRE_CONN_FULL 4
+
+struct ptlrpc_connection {
+ struct list_head c_link;
+ struct lustre_peer c_peer;
+ __u8 c_local_uuid[37];
+ __u8 c_remote_uuid[37];
+
+ int c_level;
+ __u32 c_generation; /* changes upon new connection */
+ __u32 c_epoch; /* changes when peer changes */
+ __u32 c_bootcount; /* peer's boot count */
+
+ spinlock_t c_lock;
+ __u32 c_xid_in;
+ __u32 c_xid_out;
+
+ atomic_t c_refcount;
+ __u64 c_token;
+ __u64 c_remote_conn;
+ __u64 c_remote_token;
+};
struct ptlrpc_client {
- struct lustre_peer cli_server;
struct obd_device *cli_obd;
__u32 cli_request_portal;
__u32 cli_reply_portal;
- rep_unpack_t cli_rep_unpack;
- req_pack_t cli_req_pack;
+ __u64 cli_last_rcvd;
+ __u64 cli_last_committed;
+
+ struct semaphore cli_rpc_sem; /* limits outstanding requests */
+
+ spinlock_t cli_lock; /* protects lists */
+ struct list_head cli_sending_head;
+ struct list_head cli_sent_head;
+ struct list_head cli_replied_head;
+ struct list_head cli_replay_head;
+ struct list_head cli_ha_item;
+ void (*cli_recover)(struct ptlrpc_client *);
- spinlock_t cli_lock;
- __u32 cli_xid;
+ struct recovd_obd *cli_recovd;
};
-/* These do double-duty in rq_type and rq_flags */
-#define PTL_RPC_INTR 1
-#define PTL_RPC_REQUEST 2
-#define PTL_RPC_REPLY 3
-#define PTL_RPC_BULK 4
-#define PTL_RPC_SENT 5
-#define PTL_BULK_SENT 6
-#define PTL_BULK_RCVD 6
+/* packet types */
+#define PTL_RPC_TYPE_REQUEST 2
+#define PTL_RPC_TYPE_REPLY 3
+
+/* state flags of requests */
+#define PTL_RPC_FL_INTR (1 << 0)
+#define PTL_RPC_FL_REPLY (1 << 1)
+#define PTL_RPC_FL_SENT (1 << 2)
+#define PTL_BULK_FL_SENT (1 << 3)
+#define PTL_BULK_FL_RCVD (1 << 4)
+#define PTL_RPC_FL_ERR (1 << 5)
+#define PTL_RPC_FL_TIMEOUT (1 << 6)
+#define PTL_RPC_FL_RESEND (1 << 7)
+#define PTL_RPC_FL_COMMITTED (1 << 8)
+#define PTL_RPC_FL_FINISHED (1 << 9)
+#define PTL_RPC_FL_RETAIN (1 << 10)
struct ptlrpc_request {
int rq_type; /* one of PTL_RPC_REQUEST, PTL_RPC_REPLY, PTL_RPC_BULK */
- struct list_head rq_list;
- struct obd_device *rq_obd;
- int rq_status;
+ spinlock_t rq_lock;
+ struct list_head rq_list;
+ struct obd_device *rq_obd;
+ int rq_status;
int rq_flags;
__u32 rq_connid;
- __u32 rq_xid;
+ atomic_t rq_refcount;
- char *rq_reqbuf;
- int rq_reqlen;
- struct ptlreq_hdr *rq_reqhdr;
- union ptl_req rq_req;
+ int rq_reqlen;
+ struct lustre_msg *rq_reqmsg;
- char *rq_repbuf;
- int rq_replen;
- struct ptlrep_hdr *rq_rephdr;
- union ptl_rep rq_rep;
+ int rq_replen;
+ struct lustre_msg *rq_repmsg;
+ __u64 rq_transno;
char *rq_bulkbuf;
int rq_bulklen;
- void * rq_reply_handle;
- wait_queue_head_t rq_wait_for_rep;
+ time_t rq_time;
+ time_t rq_timeout;
+ // void * rq_reply_handle;
+ wait_queue_head_t rq_wait_for_rep;
/* incoming reply */
ptl_md_t rq_reply_md;
ptl_md_t rq_req_md;
ptl_handle_md_t rq_req_md_h;
- __u32 rq_reply_portal;
- __u32 rq_req_portal;
-
- struct lustre_peer rq_peer;
+ struct ptlrpc_connection *rq_connection;
+ struct ptlrpc_client *rq_client;
};
struct ptlrpc_bulk_desc {
int b_flags;
- struct lustre_peer b_peer;
+ struct ptlrpc_connection *b_connection;
__u32 b_portal;
char *b_buf;
int b_buflen;
struct obd_conn b_conn;
__u32 b_xid;
- wait_queue_head_t b_waitq;
+ wait_queue_head_t b_waitq;
ptl_md_t b_md;
ptl_handle_md_t b_md_h;
};
struct ptlrpc_service {
+ time_t srv_time;
+ time_t srv_timeout;
+
/* incoming request buffers */
/* FIXME: perhaps a list of EQs, if multiple NIs are used? */
char *srv_buf[RPC_RING_LENGTH];
+ __u32 srv_ref_count[RPC_RING_LENGTH];
+ ptl_handle_me_t srv_me_h[RPC_RING_LENGTH];
__u32 srv_buf_size;
- __u32 srv_me_active;
- __u32 srv_me_tail;
- __u32 srv_md_active;
__u32 srv_ring_length;
__u32 srv_req_portal;
__u32 srv_rep_portal;
- __u32 srv_ref_count[RPC_RING_LENGTH];
- ptl_handle_me_t srv_me_h[RPC_RING_LENGTH];
- ptl_process_id_t srv_id;
- ptl_md_t srv_md[RPC_RING_LENGTH];
- ptl_handle_md_t srv_md_h[RPC_RING_LENGTH];
+
__u32 srv_xid;
/* event queue */
__u32 srv_flags;
struct lustre_peer srv_self;
- struct task_struct *srv_thread;
- wait_queue_head_t srv_waitq;
- wait_queue_head_t srv_ctl_waitq;
- int ost_flags;
+ ptl_process_id_t srv_id;
+
+ struct task_struct *srv_thread;
+ wait_queue_head_t srv_waitq;
+ wait_queue_head_t srv_ctl_waitq;
- spinlock_t srv_lock;
- struct list_head srv_reqs;
+ spinlock_t srv_lock;
+ struct list_head srv_reqs;
ptl_event_t srv_ev;
- req_unpack_t srv_req_unpack;
- rep_pack_t srv_rep_pack;
int (*srv_handler)(struct obd_device *obddev,
struct ptlrpc_service *svc,
struct ptlrpc_request *req);
struct ptlrpc_service *svc,
struct ptlrpc_request *req);
-
+/* rpc/connection.c */
+struct ptlrpc_connection *ptlrpc_get_connection(struct lustre_peer *peer);
+int ptlrpc_put_connection(struct ptlrpc_connection *c);
+struct ptlrpc_connection *ptlrpc_connection_addref(struct ptlrpc_connection *);
+void ptlrpc_init_connection(void);
+void ptlrpc_cleanup_connection(void);
/* rpc/niobuf.c */
int ptlrpc_check_bulk_sent(struct ptlrpc_bulk_desc *);
int ptlrpc_send_bulk(struct ptlrpc_bulk_desc *, int portal);
-int ptl_send_buf(struct ptlrpc_request *, struct lustre_peer *, int portal);
int ptlrpc_register_bulk(struct ptlrpc_bulk_desc *);
-int ptlrpc_reply(struct obd_device *obddev, struct ptlrpc_service *svc,
- struct ptlrpc_request *req);
-int ptlrpc_error(struct obd_device *obddev, struct ptlrpc_service *svc,
- struct ptlrpc_request *req);
-int ptl_send_rpc(struct ptlrpc_request *request, struct lustre_peer *peer);
-int ptl_received_rpc(struct ptlrpc_service *service);
+int ptlrpc_abort_bulk(struct ptlrpc_bulk_desc *bulk);
+int ptlrpc_reply(struct ptlrpc_service *svc, struct ptlrpc_request *req);
+int ptlrpc_error(struct ptlrpc_service *svc, struct ptlrpc_request *req);
+void ptlrpc_resend_req(struct ptlrpc_request *request);
+int ptl_send_rpc(struct ptlrpc_request *request);
+void ptlrpc_link_svc_me(struct ptlrpc_service *service, int i);
/* rpc/client.c */
-int ptlrpc_connect_client(int dev, char *uuid, int req_portal, int rep_portal,
- req_pack_t req_pack, rep_unpack_t rep_unpack,
- struct ptlrpc_client *cl);
-int ptlrpc_queue_wait(struct ptlrpc_client *cl, struct ptlrpc_request *req);
-int ptlrpc_queue_req(struct ptlrpc_client *peer, struct ptlrpc_request *req);
-struct ptlrpc_request *ptlrpc_prep_req(struct ptlrpc_client *cl,
- int opcode, int namelen, char *name,
- int tgtlen, char *tgt);
+void ptlrpc_init_client(struct recovd_obd *,
+ void (*recover)(struct ptlrpc_client *),
+ int req_portal, int rep_portal,
+ struct ptlrpc_client *);
+void ptlrpc_cleanup_client(struct ptlrpc_client *cli);
+__u8 *ptlrpc_req_to_uuid(struct ptlrpc_request *req);
+struct ptlrpc_connection *ptlrpc_uuid_to_connection(char *uuid);
+int ptlrpc_queue_wait(struct ptlrpc_request *req);
+struct ptlrpc_request *ptlrpc_prep_req(struct ptlrpc_client *cl,
+ struct ptlrpc_connection *u, int opcode,
+ int count, int *lengths, char **bufs);
+void ptlrpc_free_bulk(struct ptlrpc_bulk_desc *bulk);
void ptlrpc_free_req(struct ptlrpc_request *request);
-struct ptlrpc_bulk_desc *ptlrpc_prep_bulk(struct lustre_peer *);
+void ptlrpc_req_finished(struct ptlrpc_request *request);
+struct ptlrpc_bulk_desc *ptlrpc_prep_bulk(struct ptlrpc_connection *);
+int ptlrpc_check_status(struct ptlrpc_request *req, int err);
/* rpc/service.c */
struct ptlrpc_service *
ptlrpc_init_svc(__u32 bufsize, int req_portal, int rep_portal, char *uuid,
- req_unpack_t unpack, rep_pack_t pack, svc_handler_t);
+ svc_handler_t);
void ptlrpc_stop_thread(struct ptlrpc_service *svc);
int ptlrpc_start_thread(struct obd_device *dev, struct ptlrpc_service *svc,
char *name);
-int rpc_register_service(struct ptlrpc_service *service, char *uuid);
int rpc_unregister_service(struct ptlrpc_service *service);
struct ptlrpc_svc_data {
struct obd_device *dev;
};
+/* rpc/pack_generic.c */
+int lustre_pack_msg(int count, int *lens, char **bufs, int *len,
+ struct lustre_msg **msg);
+int lustre_msg_size(int count, int *lengths);
+int lustre_unpack_msg(struct lustre_msg *m, int len);
+void *lustre_msg_buf(struct lustre_msg *m, int n);
+
#endif