X-Git-Url: https://git.whamcloud.com/?a=blobdiff_plain;f=lnet%2Fklnds%2Fsocklnd%2Fsocklnd.h;h=6354f62044afda5d31cc1d0efbd9fd37c12bd851;hb=0a9c9e444635dcf35a74bfb2f46efb3040ca17a0;hp=b72df00b36922f25e31119e521be613c179e2d9c;hpb=9190c945141ad12ebec0d589b3d3fdb193e742fa;p=fs%2Flustre-release.git diff --git a/lnet/klnds/socklnd/socklnd.h b/lnet/klnds/socklnd/socklnd.h index b72df00..6354f62 100644 --- a/lnet/klnds/socklnd/socklnd.h +++ b/lnet/klnds/socklnd/socklnd.h @@ -46,13 +46,12 @@ #include #include #include +#include #define SOCKNAL_PEER_HASH_SIZE 101 /* # peer lists */ #define SOCKNAL_RESCHED 100 /* # scheduler loops before reschedule */ #define SOCKNAL_ENOMEM_RETRY CFS_TICK /* jiffies between retries */ -#define SOCKNAL_ROUND_ROBIN 0 /* round robin / load balance */ - #define SOCKNAL_SINGLE_FRAG_TX 0 /* disable multi-fragment sends */ #define SOCKNAL_SINGLE_FRAG_RX 0 /* disable multi-fragment receives */ @@ -68,12 +67,18 @@ typedef struct /* per scheduler state */ { - spinlock_t kss_lock; /* serialise */ + cfs_spinlock_t kss_lock; /* serialise */ struct list_head kss_rx_conns; /* conn waiting to be read */ struct list_head kss_tx_conns; /* conn waiting to be written */ struct list_head kss_zombie_noop_txs; /* zombie noop tx list */ cfs_waitq_t kss_waitq; /* where scheduler sleeps */ int kss_nconns; /* # connections assigned to this scheduler */ +#if !SOCKNAL_SINGLE_FRAG_RX + struct page *kss_rx_scratch_pgs[LNET_MAX_IOV]; +#endif +#if !SOCKNAL_SINGLE_FRAG_TX || !SOCKNAL_SINGLE_FRAG_RX + struct iovec kss_scratch_iov[LNET_MAX_IOV]; +#endif } ksock_sched_t; typedef struct @@ -104,6 +109,8 @@ typedef struct int *ksnd_tx_buffer_size; /* socket tx buffer size */ int *ksnd_rx_buffer_size; /* socket rx buffer size */ int *ksnd_nagle; /* enable NAGLE? */ + int *ksnd_round_robin; /* round robin for multiple interfaces */ + int *ksnd_keepalive; /* # secs for sending keepalive NOOP */ int *ksnd_keepalive_idle; /* # idle secs before 1st probe */ int *ksnd_keepalive_count; /* # probes */ int *ksnd_keepalive_intvl; /* time between probes */ @@ -111,7 +118,9 @@ typedef struct int *ksnd_peercredits; /* # concurrent sends to 1 peer */ int *ksnd_enable_csum; /* enable check sum */ int *ksnd_inject_csum_error; /* set non-zero to inject checksum error */ - unsigned int *ksnd_zc_min_frag; /* minimum zero copy frag size */ + unsigned int *ksnd_zc_min_payload; /* minimum zero copy payload size */ + int *ksnd_zc_recv; /* enable ZC receive (for Chelsio TOE) */ + int *ksnd_zc_recv_min_nfrags; /* minimum # of fragments to enable ZC receive */ #ifdef CPU_AFFINITY int *ksnd_irq_affinity; /* enable IRQ affinity? */ #endif @@ -130,7 +139,7 @@ typedef struct typedef struct { __u64 ksnn_incarnation; /* my epoch */ - spinlock_t ksnn_lock; /* serialise */ + cfs_spinlock_t ksnn_lock; /* serialise */ int ksnn_npeers; /* # peers */ int ksnn_shutdown; /* shutting down? */ int ksnn_ninterfaces; /* IP interfaces */ @@ -142,7 +151,7 @@ typedef struct int ksnd_init; /* initialisation state */ int ksnd_nnets; /* # networks set up */ - rwlock_t ksnd_global_lock; /* stabilize peer/conn ops */ + cfs_rwlock_t ksnd_global_lock; /* stabilize peer/conn ops */ struct list_head *ksnd_peers; /* hash table of all my known peers */ int ksnd_peer_hash_size; /* size of ksnd_peers */ @@ -151,14 +160,14 @@ typedef struct int ksnd_nschedulers; /* # schedulers */ ksock_sched_t *ksnd_schedulers; /* their state */ - atomic_t ksnd_nactive_txs; /* #active txs */ + cfs_atomic_t ksnd_nactive_txs; /* #active txs */ struct list_head ksnd_deathrow_conns; /* conns to close: reaper_lock*/ struct list_head ksnd_zombie_conns; /* conns to free: reaper_lock */ struct list_head ksnd_enomem_conns; /* conns to retry: reaper_lock*/ cfs_waitq_t ksnd_reaper_waitq; /* reaper sleeps here */ cfs_time_t ksnd_reaper_waketime; /* when reaper will wake */ - spinlock_t ksnd_reaper_lock; /* serialise */ + cfs_spinlock_t ksnd_reaper_lock; /* serialise */ int ksnd_enomem_tx; /* test ENOMEM sender */ int ksnd_stall_tx; /* test sluggish sender */ @@ -168,10 +177,10 @@ typedef struct struct list_head ksnd_connd_routes; /* routes waiting to be connected */ cfs_waitq_t ksnd_connd_waitq; /* connds sleep here */ int ksnd_connd_connecting;/* # connds connecting */ - spinlock_t ksnd_connd_lock; /* serialise */ + cfs_spinlock_t ksnd_connd_lock; /* serialise */ struct list_head ksnd_idle_noop_txs; /* list head for freed noop tx */ - spinlock_t ksnd_tx_lock; /* serialise, NOT safe in g_lock */ + cfs_spinlock_t ksnd_tx_lock; /* serialise, NOT safe in g_lock */ ksock_irqinfo_t ksnd_irqinfo[NR_IRQS];/* irq->scheduler lookup */ @@ -199,13 +208,15 @@ typedef struct /* transmit packet */ { struct list_head tx_list; /* queue on conn for transmission etc */ struct list_head tx_zc_list; /* queue on peer for ZC request */ - atomic_t tx_refcount; /* tx reference count */ + cfs_atomic_t tx_refcount; /* tx reference count */ int tx_nob; /* # packet bytes */ int tx_resid; /* residual bytes */ int tx_niov; /* # packet iovec frags */ struct iovec *tx_iov; /* packet iovec frags */ int tx_nkiov; /* # packet page frags */ - unsigned int tx_checked_zc; /* Have I checked if I should ZC? */ + unsigned int tx_zc_capable:1; /* payload is large enough for ZC */ + unsigned int tx_zc_checked:1; /* Have I checked if I should ZC? */ + unsigned int tx_nonblk:1; /* it's a non-blocking ACK */ lnet_kiov_t *tx_kiov; /* packet page frags */ struct ksock_conn *tx_conn; /* owning conn */ lnet_msg_t *tx_lnetmsg; /* lnet message for lnet_finalize() */ @@ -223,7 +234,7 @@ typedef struct /* transmit packet */ } tx_frags; } ksock_tx_t; -#define KSOCK_NOOP_TX_SIZE offsetof(ksock_tx_t, tx_frags.paged.kiov[0]) +#define KSOCK_NOOP_TX_SIZE ((int)offsetof(ksock_tx_t, tx_frags.paged.kiov[0])) /* network zero copy callback descriptor embedded in ksock_tx_t */ @@ -249,8 +260,8 @@ typedef struct ksock_conn cfs_socket_t *ksnc_sock; /* actual socket */ void *ksnc_saved_data_ready; /* socket's original data_ready() callback */ void *ksnc_saved_write_space; /* socket's original write_space() callback */ - atomic_t ksnc_conn_refcount; /* conn refcount */ - atomic_t ksnc_sock_refcount; /* sock refcount */ + cfs_atomic_t ksnc_conn_refcount; /* conn refcount */ + cfs_atomic_t ksnc_sock_refcount; /* sock refcount */ ksock_sched_t *ksnc_scheduler; /* who schedules this connection */ __u32 ksnc_myipaddr; /* my IP */ __u32 ksnc_ipaddr; /* peer's IP */ @@ -285,21 +296,13 @@ typedef struct ksock_conn /* WRITER */ struct list_head ksnc_tx_list; /* where I enq waiting for output space */ struct list_head ksnc_tx_queue; /* packets waiting to be sent */ - ksock_tx_t *ksnc_tx_mono; /* V2.x only, next mono-packet, mono-packet is : - * a. lnet packet without piggyback - * b. noop ZC-ACK packet */ + ksock_tx_t *ksnc_tx_carrier; /* next TX that can carry a LNet message or ZC-ACK */ cfs_time_t ksnc_tx_deadline; /* when (in jiffies) tx times out */ int ksnc_tx_bufnob; /* send buffer marker */ - atomic_t ksnc_tx_nob; /* # bytes queued */ + cfs_atomic_t ksnc_tx_nob; /* # bytes queued */ int ksnc_tx_ready; /* write space */ int ksnc_tx_scheduled; /* being progressed */ - -#if !SOCKNAL_SINGLE_FRAG_RX - struct iovec ksnc_rx_scratch_iov[LNET_MAX_IOV]; -#endif -#if !SOCKNAL_SINGLE_FRAG_TX - struct iovec ksnc_tx_scratch_iov[LNET_MAX_IOV]; -#endif + cfs_time_t ksnc_tx_last_post; /* time stamp of the last posted TX */ } ksock_conn_t; typedef struct ksock_route @@ -307,7 +310,7 @@ typedef struct ksock_route struct list_head ksnr_list; /* chain on peer route list */ struct list_head ksnr_connd_list; /* chain on ksnr_connd_routes */ struct ksock_peer *ksnr_peer; /* owning peer */ - atomic_t ksnr_refcount; /* # users */ + cfs_atomic_t ksnr_refcount; /* # users */ cfs_time_t ksnr_timeout; /* when (in jiffies) reconnection can happen next */ cfs_duration_t ksnr_retry_interval; /* how long between retries */ __u32 ksnr_myipaddr; /* my IP */ @@ -321,11 +324,13 @@ typedef struct ksock_route int ksnr_conn_count; /* # conns established by this route */ } ksock_route_t; +#define SOCKNAL_KEEPALIVE_PING 1 /* cookie for keepalive ping */ + typedef struct ksock_peer { struct list_head ksnp_list; /* stash on global peer list */ lnet_process_id_t ksnp_id; /* who's on the other end(s) */ - atomic_t ksnp_refcount; /* # users */ + cfs_atomic_t ksnp_refcount; /* # users */ int ksnp_sharecount; /* lconf usage counter */ int ksnp_closing; /* being closed */ int ksnp_accepting; /* # passive connections pending */ @@ -336,9 +341,10 @@ typedef struct ksock_peer struct list_head ksnp_conns; /* all active connections */ struct list_head ksnp_routes; /* routes */ struct list_head ksnp_tx_queue; /* waiting packets */ - spinlock_t ksnp_lock; /* serialize, NOT safe in g_lock */ + cfs_spinlock_t ksnp_lock; /* serialize, NOT safe in g_lock */ struct list_head ksnp_zc_req_list; /* zero copy requests wait for ACK */ cfs_time_t ksnp_last_alive; /* when (in jiffies) I was last alive */ + cfs_time_t ksnp_send_keepalive; /* time to send keepalive */ lnet_ni_t *ksnp_ni; /* which network */ int ksnp_n_passive_ips; /* # of... */ __u32 ksnp_passive_ips[LNET_MAX_INTERFACES]; /* preferred local interfaces */ @@ -354,17 +360,31 @@ typedef struct ksock_connreq extern ksock_nal_data_t ksocknal_data; extern ksock_tunables_t ksocknal_tunables; +#define SOCKNAL_MATCH_NO 0 /* TX can't match type of connection */ +#define SOCKNAL_MATCH_YES 1 /* TX matches type of connection */ +#define SOCKNAL_MATCH_MAY 2 /* TX can be sent on the connection, but not preferred */ + typedef struct ksock_proto { - int pro_version; /* version number of protocol */ - int (*pro_send_hello)(ksock_conn_t *, ksock_hello_msg_t *); /* handshake function */ - int (*pro_recv_hello)(ksock_conn_t *, ksock_hello_msg_t *, int);/* handshake function */ - void (*pro_pack)(ksock_tx_t *); /* message pack */ - void (*pro_unpack)(ksock_msg_t *); /* message unpack */ + int pro_version; /* version number of protocol */ + int (*pro_send_hello)(ksock_conn_t *, ksock_hello_msg_t *); /* handshake function */ + int (*pro_recv_hello)(ksock_conn_t *, ksock_hello_msg_t *, int);/* handshake function */ + void (*pro_pack)(ksock_tx_t *); /* message pack */ + void (*pro_unpack)(ksock_msg_t *); /* message unpack */ + ksock_tx_t *(*pro_queue_tx_msg)(ksock_conn_t *, ksock_tx_t *); /* queue tx on the connection */ + int (*pro_queue_tx_zcack)(ksock_conn_t *, ksock_tx_t *, __u64); /* queue ZC ack on the connection */ + int (*pro_handle_zcreq)(ksock_conn_t *, __u64, int); /* handle ZC request */ + int (*pro_handle_zcack)(ksock_conn_t *, __u64, __u64); /* handle ZC ACK */ + int (*pro_match_tx)(ksock_conn_t *, ksock_tx_t *, int); /* msg type matches the connection type: + * return value: + * return MATCH_NO : no + * return MATCH_YES : matching type + * return MATCH_MAY : can be backup */ } ksock_proto_t; extern ksock_proto_t ksocknal_protocol_v1x; extern ksock_proto_t ksocknal_protocol_v2x; +extern ksock_proto_t ksocknal_protocol_v3x; #define KSOCK_PROTO_V1_MAJOR LNET_PROTO_TCP_VERSION_MAJOR #define KSOCK_PROTO_V1_MINOR LNET_PROTO_TCP_VERSION_MINOR @@ -396,8 +416,8 @@ ksocknal_nid2peerlist (lnet_nid_t nid) static inline void ksocknal_conn_addref (ksock_conn_t *conn) { - LASSERT (atomic_read(&conn->ksnc_conn_refcount) > 0); - atomic_inc(&conn->ksnc_conn_refcount); + LASSERT (cfs_atomic_read(&conn->ksnc_conn_refcount) > 0); + cfs_atomic_inc(&conn->ksnc_conn_refcount); } extern void ksocknal_queue_zombie_conn (ksock_conn_t *conn); @@ -406,8 +426,8 @@ extern void ksocknal_finalize_zcreq(ksock_conn_t *conn); static inline void ksocknal_conn_decref (ksock_conn_t *conn) { - LASSERT (atomic_read(&conn->ksnc_conn_refcount) > 0); - if (atomic_dec_and_test(&conn->ksnc_conn_refcount)) + LASSERT (cfs_atomic_read(&conn->ksnc_conn_refcount) > 0); + if (cfs_atomic_dec_and_test(&conn->ksnc_conn_refcount)) ksocknal_queue_zombie_conn(conn); } @@ -416,13 +436,13 @@ ksocknal_connsock_addref (ksock_conn_t *conn) { int rc = -ESHUTDOWN; - read_lock (&ksocknal_data.ksnd_global_lock); + cfs_read_lock (&ksocknal_data.ksnd_global_lock); if (!conn->ksnc_closing) { - LASSERT (atomic_read(&conn->ksnc_sock_refcount) > 0); - atomic_inc(&conn->ksnc_sock_refcount); + LASSERT (cfs_atomic_read(&conn->ksnc_sock_refcount) > 0); + cfs_atomic_inc(&conn->ksnc_sock_refcount); rc = 0; } - read_unlock (&ksocknal_data.ksnd_global_lock); + cfs_read_unlock (&ksocknal_data.ksnd_global_lock); return (rc); } @@ -430,8 +450,8 @@ ksocknal_connsock_addref (ksock_conn_t *conn) static inline void ksocknal_connsock_decref (ksock_conn_t *conn) { - LASSERT (atomic_read(&conn->ksnc_sock_refcount) > 0); - if (atomic_dec_and_test(&conn->ksnc_sock_refcount)) { + LASSERT (cfs_atomic_read(&conn->ksnc_sock_refcount) > 0); + if (cfs_atomic_dec_and_test(&conn->ksnc_sock_refcount)) { LASSERT (conn->ksnc_closing); libcfs_sock_release(conn->ksnc_sock); conn->ksnc_sock = NULL; @@ -442,25 +462,26 @@ ksocknal_connsock_decref (ksock_conn_t *conn) static inline void ksocknal_tx_addref (ksock_tx_t *tx) { - LASSERT (atomic_read(&tx->tx_refcount) > 0); - atomic_inc(&tx->tx_refcount); + LASSERT (cfs_atomic_read(&tx->tx_refcount) > 0); + cfs_atomic_inc(&tx->tx_refcount); } +extern void ksocknal_tx_prep (ksock_conn_t *, ksock_tx_t *tx); extern void ksocknal_tx_done (lnet_ni_t *ni, ksock_tx_t *tx); static inline void ksocknal_tx_decref (ksock_tx_t *tx) { - LASSERT (atomic_read(&tx->tx_refcount) > 0); - if (atomic_dec_and_test(&tx->tx_refcount)) + LASSERT (cfs_atomic_read(&tx->tx_refcount) > 0); + if (cfs_atomic_dec_and_test(&tx->tx_refcount)) ksocknal_tx_done(NULL, tx); } static inline void ksocknal_route_addref (ksock_route_t *route) { - LASSERT (atomic_read(&route->ksnr_refcount) > 0); - atomic_inc(&route->ksnr_refcount); + LASSERT (cfs_atomic_read(&route->ksnr_refcount) > 0); + cfs_atomic_inc(&route->ksnr_refcount); } extern void ksocknal_destroy_route (ksock_route_t *route); @@ -468,16 +489,16 @@ extern void ksocknal_destroy_route (ksock_route_t *route); static inline void ksocknal_route_decref (ksock_route_t *route) { - LASSERT (atomic_read (&route->ksnr_refcount) > 0); - if (atomic_dec_and_test(&route->ksnr_refcount)) + LASSERT (cfs_atomic_read (&route->ksnr_refcount) > 0); + if (cfs_atomic_dec_and_test(&route->ksnr_refcount)) ksocknal_destroy_route (route); } static inline void ksocknal_peer_addref (ksock_peer_t *peer) { - LASSERT (atomic_read (&peer->ksnp_refcount) > 0); - atomic_inc(&peer->ksnp_refcount); + LASSERT (cfs_atomic_read (&peer->ksnp_refcount) > 0); + cfs_atomic_inc(&peer->ksnp_refcount); } extern void ksocknal_destroy_peer (ksock_peer_t *peer); @@ -485,8 +506,8 @@ extern void ksocknal_destroy_peer (ksock_peer_t *peer); static inline void ksocknal_peer_decref (ksock_peer_t *peer) { - LASSERT (atomic_read (&peer->ksnp_refcount) > 0); - if (atomic_dec_and_test(&peer->ksnp_refcount)) + LASSERT (cfs_atomic_read (&peer->ksnp_refcount) > 0); + if (cfs_atomic_dec_and_test(&peer->ksnp_refcount)) ksocknal_destroy_peer (peer); } @@ -513,7 +534,15 @@ extern int ksocknal_close_peer_conns_locked (ksock_peer_t *peer, __u32 ipaddr, int why); extern int ksocknal_close_conn_and_siblings (ksock_conn_t *conn, int why); extern int ksocknal_close_matching_conns (lnet_process_id_t id, __u32 ipaddr); - +extern ksock_conn_t *ksocknal_find_conn_locked(ksock_peer_t *peer, + ksock_tx_t *tx, int nonblk); + +extern int ksocknal_launch_packet(lnet_ni_t *ni, ksock_tx_t *tx, + lnet_process_id_t id); +extern ksock_tx_t *ksocknal_alloc_tx(int type, int size); +extern void ksocknal_free_tx (ksock_tx_t *tx); +extern ksock_tx_t *ksocknal_alloc_tx_noop(__u64 cookie, int nonblk); +extern void ksocknal_next_tx_carrier(ksock_conn_t *conn); extern void ksocknal_queue_tx_locked (ksock_tx_t *tx, ksock_conn_t *conn); extern void ksocknal_txlist_done (lnet_ni_t *ni, struct list_head *txlist, int error); extern void ksocknal_notify (lnet_ni_t *ni, lnet_nid_t gw_nid, int alive); @@ -532,7 +561,7 @@ extern int ksocknal_recv_hello (lnet_ni_t *ni, ksock_conn_t *conn, extern void ksocknal_read_callback(ksock_conn_t *conn); extern void ksocknal_write_callback(ksock_conn_t *conn); -extern int ksocknal_lib_zc_capable(cfs_socket_t *sock); +extern int ksocknal_lib_zc_capable(ksock_conn_t *conn); extern void ksocknal_lib_save_callback(cfs_socket_t *sock, ksock_conn_t *conn); extern void ksocknal_lib_set_callback(cfs_socket_t *sock, ksock_conn_t *conn); extern void ksocknal_lib_reset_callback(cfs_socket_t *sock, ksock_conn_t *conn); @@ -549,7 +578,13 @@ extern int ksocknal_lib_recv_kiov (ksock_conn_t *conn); extern int ksocknal_lib_get_conn_tunables (ksock_conn_t *conn, int *txmem, int *rxmem, int *nagle); +extern int ksocknal_tunables_init(void); +extern void ksocknal_tunables_fini(void); extern int ksocknal_lib_tunables_init(void); extern void ksocknal_lib_tunables_fini(void); extern void ksocknal_lib_csum_tx(ksock_tx_t *tx); + +extern int ksocknal_lib_memory_pressure(ksock_conn_t *conn); +extern __u64 ksocknal_lib_new_incarnation(void); +extern int ksocknal_lib_bind_thread_to_cpu(int id);