1 /* -*- mode: c; c-basic-offset: 8; indent-tabs-mode: nil; -*-
2 * vim:expandtab:shiftwidth=8:tabstop=8:
6 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
8 * This program is free software; you can redistribute it and/or modify
9 * it under the terms of the GNU General Public License version 2 only,
10 * as published by the Free Software Foundation.
12 * This program is distributed in the hope that it will be useful, but
13 * WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
15 * General Public License version 2 for more details (a copy is included
16 * in the LICENSE file that accompanied this code).
18 * You should have received a copy of the GNU General Public License
19 * version 2 along with this program; If not, see [sun.com URL with a
22 * Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa Clara,
23 * CA 95054 USA or visit www.sun.com if you need additional information or
29 * Copyright 2008 Sun Microsystems, Inc. All rights reserved
30 * Use is subject to license terms.
33 * This file is part of Lustre, http://www.lustre.org/
34 * Lustre is a trademark of Sun Microsystems, Inc.
36 * lnet/klnds/socklnd/socklnd_lib-winnt.c
38 * windows socknal library
43 # if CONFIG_SYSCTL && !CFS_SYSFS_MODULE_PARM
44 static ctl_table ksocknal_ctl_table[18];
46 ctl_table ksocknal_top_ctl_table[] = {
47 {200, "socknal", NULL, 0, 0555, ksocknal_ctl_table},
52 ksocknal_lib_tunables_init ()
57 ksocknal_ctl_table[i++] = (ctl_table)
58 {j++, "timeout", ksocknal_tunables.ksnd_timeout,
59 sizeof (int), 0644, NULL, &proc_dointvec};
60 ksocknal_ctl_table[i++] = (ctl_table)
61 {j++, "credits", ksocknal_tunables.ksnd_credits,
62 sizeof (int), 0444, NULL, &proc_dointvec};
63 ksocknal_ctl_table[i++] = (ctl_table)
64 {j++, "peer_credits", ksocknal_tunables.ksnd_peercredits,
65 sizeof (int), 0444, NULL, &proc_dointvec};
66 ksocknal_ctl_table[i++] = (ctl_table)
67 {j++, "nconnds", ksocknal_tunables.ksnd_nconnds,
68 sizeof (int), 0444, NULL, &proc_dointvec};
69 ksocknal_ctl_table[i++] = (ctl_table)
70 {j++, "min_reconnectms", ksocknal_tunables.ksnd_min_reconnectms,
71 sizeof (int), 0444, NULL, &proc_dointvec};
72 ksocknal_ctl_table[i++] = (ctl_table)
73 {j++, "max_reconnectms", ksocknal_tunables.ksnd_max_reconnectms,
74 sizeof (int), 0444, NULL, &proc_dointvec};
75 ksocknal_ctl_table[i++] = (ctl_table)
76 {j++, "eager_ack", ksocknal_tunables.ksnd_eager_ack,
77 sizeof (int), 0644, NULL, &proc_dointvec};
79 ksocknal_ctl_table[i++] = (ctl_table)
80 {j++, "zero_copy", ksocknal_tunables.ksnd_zc_min_frag,
81 sizeof (int), 0644, NULL, &proc_dointvec};
83 ksocknal_ctl_table[i++] = (ctl_table)
84 {j++, "typed", ksocknal_tunables.ksnd_typed_conns,
85 sizeof (int), 0444, NULL, &proc_dointvec};
86 ksocknal_ctl_table[i++] = (ctl_table)
87 {j++, "min_bulk", ksocknal_tunables.ksnd_min_bulk,
88 sizeof (int), 0644, NULL, &proc_dointvec};
89 ksocknal_ctl_table[i++] = (ctl_table)
90 {j++, "buffer_size", ksocknal_tunables.ksnd_buffer_size,
91 sizeof(int), 0644, NULL, &proc_dointvec};
92 ksocknal_ctl_table[i++] = (ctl_table)
93 {j++, "nagle", ksocknal_tunables.ksnd_nagle,
94 sizeof(int), 0644, NULL, &proc_dointvec};
96 ksocknal_ctl_table[i++] = (ctl_table)
97 {j++, "irq_affinity", ksocknal_tunables.ksnd_irq_affinity,
98 sizeof(int), 0644, NULL, &proc_dointvec};
100 ksocknal_ctl_table[i++] = (ctl_table)
101 {j++, "keepalive_idle", ksocknal_tunables.ksnd_keepalive_idle,
102 sizeof(int), 0644, NULL, &proc_dointvec};
103 ksocknal_ctl_table[i++] = (ctl_table)
104 {j++, "keepalive_count", ksocknal_tunables.ksnd_keepalive_count,
105 sizeof(int), 0644, NULL, &proc_dointvec};
106 ksocknal_ctl_table[i++] = (ctl_table)
107 {j++, "keepalive_intvl", ksocknal_tunables.ksnd_keepalive_intvl,
108 sizeof(int), 0644, NULL, &proc_dointvec};
111 LASSERT (i < sizeof(ksocknal_ctl_table)/sizeof(ksocknal_ctl_table[0]));
113 ksocknal_tunables.ksnd_sysctl =
114 register_sysctl_table(ksocknal_top_ctl_table, 0);
116 if (ksocknal_tunables.ksnd_sysctl == NULL)
117 CWARN("Can't setup /proc tunables\n");
123 ksocknal_lib_tunables_fini ()
125 if (ksocknal_tunables.ksnd_sysctl != NULL)
126 unregister_sysctl_table(ksocknal_tunables.ksnd_sysctl);
130 ksocknal_lib_tunables_init ()
136 ksocknal_lib_tunables_fini ()
142 ksocknal_lib_bind_irq (unsigned int irq)
147 ksocknal_lib_get_conn_addrs (ksock_conn_t *conn)
149 int rc = libcfs_sock_getaddr(conn->ksnc_sock, 1,
150 &conn->ksnc_ipaddr, &conn->ksnc_port);
152 /* Didn't need the {get,put}connsock dance to deref ksnc_sock... */
153 LASSERT (!conn->ksnc_closing);
156 CERROR ("Error %d getting sock peer IP\n", rc);
160 rc = libcfs_sock_getaddr(conn->ksnc_sock, 0,
161 &conn->ksnc_myipaddr, NULL);
163 CERROR ("Error %d getting sock local IP\n", rc);
171 ksocknal_lib_sock_irq (struct socket *sock)
176 #if (SOCKNAL_ZC && SOCKNAL_VADDR_ZC)
178 ksocknal_kvaddr_to_page (unsigned long vaddr)
182 if (vaddr >= VMALLOC_START &&
184 page = vmalloc_to_page ((void *)vaddr);
185 #ifdef CONFIG_HIGHMEM
186 else if (vaddr >= PKMAP_BASE &&
187 vaddr < (PKMAP_BASE + LAST_PKMAP * PAGE_SIZE))
188 page = vmalloc_to_page ((void *)vaddr);
189 /* in 2.4 ^ just walks the page tables */
192 page = virt_to_page (vaddr);
204 * Lock the i/o vector buffers into MDL structure
207 * iov: the array of i/o vectors
208 * niov: number of i/o vectors to be locked
209 * len: the real length of the iov vectors
212 * ksock_mdl_t *: the Mdl of the locked buffers or
213 * NULL pointer in failure case
221 IN struct iovec *iov,
230 ksock_mdl_t * mdl = NULL;
231 ksock_mdl_t * tail = NULL;
233 LASSERT(iov != NULL);
235 LASSERT(len != NULL);
237 for (i=0; i < niov; i++) {
239 ksock_mdl_t * Iovec = NULL;
245 recving ? IoWriteAccess : IoReadAccess,
260 total +=iov[i].iov_len;
267 ks_release_mdl(mdl, FALSE);
277 * Lock the kiov pages into MDL structure
280 * kiov: the array of kiov pages
281 * niov: number of kiov to be locked
282 * len: the real length of the kiov arrary
285 * PMDL: the Mdl of the locked buffers or NULL
286 * pointer in failure case
293 IN lnet_kiov_t * kiov,
301 ksock_mdl_t * mdl = NULL;
302 ksock_mdl_t * tail = NULL;
304 LASSERT(kiov != NULL);
306 LASSERT(len != NULL);
308 for (i=0; i < nkiov; i++) {
310 ksock_mdl_t * Iovec = NULL;
314 // Lock the kiov page into Iovec ¡Â
318 (PUCHAR)kiov[i].kiov_page->addr +
322 recving ? IoWriteAccess : IoReadAccess,
331 // Attach the Iovec to the mdl chain
342 total += kiov[i].kiov_len;
350 ks_release_mdl(mdl, FALSE);
360 ksocknal_lib_send_iov (ksock_conn_t *conn, ksock_tx_t *tx)
362 struct socket *sock = conn->ksnc_sock;
363 #if (SOCKNAL_ZC && SOCKNAL_VADDR_ZC)
364 unsigned long vaddr = (unsigned long)iov->iov_base
365 int offset = vaddr & (PAGE_SIZE - 1);
366 int zcsize = MIN (iov->iov_len, PAGE_SIZE - offset);
373 /* NB we can't trust socket ops to either consume our iovs
374 * or leave them alone. */
376 #if (SOCKNAL_ZC && SOCKNAL_VADDR_ZC)
377 if (zcsize >= ksocknal_data.ksnd_zc_min_frag &&
378 (sock->sk->sk_route_caps & NETIF_F_SG) &&
379 (sock->sk->sk_route_caps & (NETIF_F_IP_CSUM | NETIF_F_NO_CSUM | NETIF_F_HW_CSUM)) &&
380 (page = ksocknal_kvaddr_to_page (vaddr)) != NULL) {
381 int msgflg = MSG_DONTWAIT;
383 CDEBUG(D_NET, "vaddr %p, page %p->%p + offset %x for %d\n",
384 (void *)vaddr, page, page_address(page), offset, zcsize);
386 if (!list_empty (&conn->ksnc_tx_queue) ||
387 zcsize < tx->tx_resid)
390 rc = tcp_sendpage_zccd(sock, page, offset, zcsize, msgflg, &tx->tx_zccd);
394 /* lock the whole tx iovs into a single mdl chain */
395 mdl = ks_lock_iovs(tx->tx_iov, tx->tx_niov, FALSE, &nob);
398 /* send the total mdl chain */
399 rc = ks_send_mdl( conn->ksnc_sock, tx, mdl, nob,
400 (!list_empty (&conn->ksnc_tx_queue) || nob < tx->tx_resid) ?
401 (MSG_DONTWAIT | MSG_MORE) : MSG_DONTWAIT);
411 ksocknal_lib_send_kiov (ksock_conn_t *conn, ksock_tx_t *tx)
413 struct socket *sock = conn->ksnc_sock;
414 lnet_kiov_t *kiov = tx->tx_kiov;
419 /* NB we can't trust socket ops to either consume our iovs
420 * or leave them alone. */
423 if (kiov->kiov_len >= *ksocknal_tunables.ksnd_zc_min_frag &&
424 (sock->sk->sk_route_caps & NETIF_F_SG) &&
425 (sock->sk->sk_route_caps & (NETIF_F_IP_CSUM | NETIF_F_NO_CSUM | NETIF_F_HW_CSUM))) {
426 struct page *page = kiov->kiov_page;
427 int offset = kiov->kiov_offset;
428 int fragsize = kiov->kiov_len;
429 int msgflg = MSG_DONTWAIT;
431 CDEBUG(D_NET, "page %p + offset %x for %d\n",
432 page, offset, kiov->kiov_len);
434 if (!list_empty(&conn->ksnc_tx_queue) ||
435 fragsize < tx->tx_resid)
438 rc = tcp_sendpage_zccd(sock, page, offset, fragsize, msgflg,
443 /* lock the whole tx kiovs into a single mdl chain */
444 mdl = ks_lock_kiovs(tx->tx_kiov, tx->tx_nkiov, FALSE, &nob);
447 /* send the total mdl chain */
449 conn->ksnc_sock, tx, mdl, nob,
450 (!list_empty(&conn->ksnc_tx_queue) || nob < tx->tx_resid) ?
451 (MSG_DONTWAIT | MSG_MORE) : MSG_DONTWAIT);
462 ksocknal_lib_recv_iov (ksock_conn_t *conn)
464 struct iovec *iov = conn->ksnc_rx_iov;
469 /* lock the whole tx iovs into a single mdl chain */
470 mdl = ks_lock_iovs(iov, conn->ksnc_rx_niov, TRUE, &size);
476 LASSERT (size <= conn->ksnc_rx_nob_wanted);
478 /* try to request data for the whole mdl chain */
479 rc = ks_recv_mdl (conn->ksnc_sock, mdl, size, MSG_DONTWAIT);
485 ksocknal_lib_recv_kiov (ksock_conn_t *conn)
487 lnet_kiov_t *kiov = conn->ksnc_rx_kiov;
492 /* NB we can't trust socket ops to either consume our iovs
493 * or leave them alone, so we only receive 1 frag at a time. */
494 LASSERT (conn->ksnc_rx_nkiov > 0);
496 /* lock the whole tx kiovs into a single mdl chain */
497 mdl = ks_lock_kiovs(kiov, conn->ksnc_rx_nkiov, TRUE, &size);
504 LASSERT (size <= conn->ksnc_rx_nob_wanted);
506 /* try to request data for the whole mdl chain */
507 rc = ks_recv_mdl (conn->ksnc_sock, mdl, size, MSG_DONTWAIT);
513 ksocknal_lib_eager_ack (ksock_conn_t *conn)
518 rc = ks_set_tcp_option(
519 conn->ksnc_sock, TCP_SOCKET_NODELAY,
520 &option, sizeof(option) );
522 CERROR("Can't disable nagle: %d\n", rc);
527 ksocknal_lib_get_conn_tunables (ksock_conn_t *conn, int *txmem, int *rxmem, int *nagle)
529 ksock_tconn_t * tconn = conn->ksnc_sock;
533 ks_get_tconn (tconn);
537 len = sizeof(*nagle);
539 rc = ks_get_tcp_option(
540 tconn, TCP_SOCKET_NODELAY,
541 (__u32 *)nagle, &len);
543 ks_put_tconn (tconn);
545 printk("ksocknal_get_conn_tunables: nodelay = %d rc = %d\n", *nagle, rc);
550 *txmem = *rxmem = *nagle = 0;
556 ksocknal_lib_buffersize (int current_sz, int tunable_sz)
558 /* ensure >= SOCKNAL_MIN_BUFFER */
559 if (current_sz < SOCKNAL_MIN_BUFFER)
560 return MAX(SOCKNAL_MIN_BUFFER, tunable_sz);
562 if (tunable_sz > SOCKNAL_MIN_BUFFER)
570 ksocknal_lib_setup_sock (struct socket *sock)
581 /* set the window size */
584 tconn->kstc_snd_wnd = ksocknal_tunables.ksnd_buffer_size;
585 tconn->kstc_rcv_wnd = ksocknal_tunables.ksnd_buffer_size;
589 if (!ksocknal_tunables.ksnd_nagle) {
592 rc = ks_set_tcp_option(
593 sock, TCP_SOCKET_NODELAY,
594 &option, sizeof (option));
596 printk ("Can't disable nagle: %d\n", rc);
601 /* snapshot tunables */
602 keep_idle = *ksocknal_tunables.ksnd_keepalive_idle;
603 keep_count = *ksocknal_tunables.ksnd_keepalive_count;
604 keep_intvl = *ksocknal_tunables.ksnd_keepalive_intvl;
606 keep_alive = (keep_idle > 0 && keep_count > 0 && keep_intvl > 0);
608 option = (__u32)(keep_alive ? 1 : 0);
610 rc = ks_set_tcp_option(
611 sock, TCP_SOCKET_KEEPALIVE,
612 &option, sizeof (option));
614 CERROR ("Can't disable nagle: %d\n", rc);
622 ksocknal_lib_push_conn (ksock_conn_t *conn)
624 ksock_tconn_t * tconn;
629 tconn = conn->ksnc_sock;
633 spin_lock(&tconn->kstc_lock);
634 if (tconn->kstc_type == kstt_sender) {
635 nagle = tconn->sender.kstc_info.nagle;
636 tconn->sender.kstc_info.nagle = 0;
638 LASSERT(tconn->kstc_type == kstt_child);
639 nagle = tconn->child.kstc_info.nagle;
640 tconn->child.kstc_info.nagle = 0;
643 spin_unlock(&tconn->kstc_lock);
646 rc = ks_set_tcp_option(
654 spin_lock(&tconn->kstc_lock);
656 if (tconn->kstc_type == kstt_sender) {
657 tconn->sender.kstc_info.nagle = nagle;
659 LASSERT(tconn->kstc_type == kstt_child);
660 tconn->child.kstc_info.nagle = nagle;
662 spin_unlock(&tconn->kstc_lock);
667 /* @mode: 0: receiving mode / 1: sending mode */
669 ksocknal_sched_conn (ksock_conn_t *conn, int mode, ksock_tx_t *tx)
672 ksock_sched_t * sched;
675 /* interleave correctly with closing sockets... */
676 read_lock (&ksocknal_data.ksnd_global_lock);
678 sched = conn->ksnc_scheduler;
680 spin_lock_irqsave (&sched->kss_lock, flags);
682 if (mode) { /* transmission can continue ... */
684 #error "This is out of date - we should be calling ksocknal_write_callback()"
685 conn->ksnc_tx_ready = 1;
688 /* Incomplete send: place tx on HEAD of tx_queue */
689 list_add (&tx->tx_list, &conn->ksnc_tx_queue);
692 if ( !conn->ksnc_tx_scheduled &&
693 !list_empty(&conn->ksnc_tx_queue)) { //packets to send
694 list_add_tail (&conn->ksnc_tx_list,
695 &sched->kss_tx_conns);
696 conn->ksnc_tx_scheduled = 1;
697 /* extra ref for scheduler */
698 atomic_inc (&conn->ksnc_conn_refcount);
700 cfs_waitq_signal (&sched->kss_waitq);
702 } else { /* receiving can continue ... */
704 conn->ksnc_rx_ready = 1;
706 if ( !conn->ksnc_rx_scheduled) { /* not being progressed */
707 list_add_tail(&conn->ksnc_rx_list,
708 &sched->kss_rx_conns);
709 conn->ksnc_rx_scheduled = 1;
710 /* extra ref for scheduler */
711 atomic_inc (&conn->ksnc_conn_refcount);
713 cfs_waitq_signal (&sched->kss_waitq);
717 spin_unlock_irqrestore (&sched->kss_lock, flags);
718 read_unlock (&ksocknal_data.ksnd_global_lock);
723 void ksocknal_schedule_callback(struct socket*sock, int mode, void * tx, ulong_ptr bytes)
725 ksock_conn_t * conn = (ksock_conn_t *) sock->kstc_conn;
728 ksocknal_sched_conn(conn, mode, tx);
730 if ( CAN_BE_SCHED(bytes, (ulong_ptr)conn->ksnc_rx_nob_wanted )) {
731 ksocknal_sched_conn(conn, mode, tx);
737 ksocknal_tx_launched (ksock_tx_t *tx);
740 ksocknal_fini_sending(ksock_tcpx_fini_t *tcpx)
742 ksocknal_tx_launched(tcpx->tx);
748 struct socket* tconn,
753 ksock_tx_t * tx = (ksock_tx_t *)txp;
756 * the transmission was done, we need update the tx
759 LASSERT(tx->tx_resid >= (int)rc);
760 tx->tx_resid -= (int)rc;
763 * just partial of tx is sent out, we need update
764 * the fields of tx and schedule later transmission.
769 if (tx->tx_niov > 0) {
771 /* if there's iov, we need process iov first */
773 if (rc < tx->tx_iov->iov_len) {
774 /* didn't send whole iov entry... */
775 tx->tx_iov->iov_base =
776 (char *)(tx->tx_iov->iov_base) + rc;
777 tx->tx_iov->iov_len -= rc;
780 /* the whole of iov was sent out */
781 rc -= tx->tx_iov->iov_len;
789 /* now we need process the kiov queues ... */
793 if (rc < tx->tx_kiov->kiov_len) {
794 /* didn't send whole kiov entry... */
795 tx->tx_kiov->kiov_offset += rc;
796 tx->tx_kiov->kiov_len -= rc;
799 /* whole kiov was sent out */
800 rc -= tx->tx_kiov->kiov_len;
809 ksock_tcpx_fini_t * tcpx =
810 cfs_alloc(sizeof(ksock_tcpx_fini_t), CFS_ALLOC_ZERO);
812 ASSERT(tx->tx_resid == 0);
816 ksocknal_tx_launched (tx);
821 ExInitializeWorkItem(
823 ksocknal_fini_sending,
839 ksocknal_lib_save_callback(struct socket *sock, ksock_conn_t *conn)
844 ksocknal_lib_set_callback(struct socket *sock, ksock_conn_t *conn)
846 sock->kstc_conn = conn;
847 sock->kstc_sched_cb = ksocknal_schedule_callback;
848 sock->kstc_update_tx = ksocknal_update_tx;
852 ksocknal_lib_reset_callback(struct socket *sock, ksock_conn_t *conn)
854 sock->kstc_conn = NULL;
855 sock->kstc_sched_cb = NULL;
856 sock->kstc_update_tx = NULL;