b=3966

r=nic

- land b1_2_bug3966 branch (HT-aware irq affinity)

diff --git a/lnet/klnds/socklnd/socklnd.c b/lnet/klnds/socklnd/socklnd.c
index 0dd5d11..f17ac4b 100644 (file)
--- a/lnet/klnds/socklnd/socklnd.c
+++ b/lnet/klnds/socklnd/socklnd.c
@@ -159,6 +159,7 @@ ksocknal_bind_irq (unsigned int irq)
 {
 #if (defined(CONFIG_SMP) && CPU_AFFINITY)
         int              bind;
+        int              cpu;
         unsigned long    flags;
         char             cmdline[64];
         ksock_irqinfo_t *info;
@@ -171,7 +172,7 @@ ksocknal_bind_irq (unsigned int irq)
                                    NULL};
 
         LASSERT (irq < NR_IRQS);
-        if (irq == 0)                           /* software NIC */
+        if (irq == 0)              /* software NIC or affinity disabled */
                 return;
 
         info = &ksocknal_data.ksnd_irqinfo[irq];
@@ -187,11 +188,12 @@ ksocknal_bind_irq (unsigned int irq)
         if (!bind)                              /* bound already */
                 return;
 
+        cpu = ksocknal_irqsched2cpu(info->ksni_sched);
         snprintf (cmdline, sizeof (cmdline),
-                  "echo %d > /proc/irq/%u/smp_affinity", 1 << info->ksni_sched, irq);
+                  "echo %d > /proc/irq/%u/smp_affinity", 1 << cpu, irq);
 
         printk (KERN_INFO "Lustre: Binding irq %u to CPU %d with cmd: %s\n",
-                irq, info->ksni_sched, cmdline);
+                irq, cpu, cmdline);
 
         /* FIXME: Find a better method of setting IRQ affinity...
          */

diff --git a/lnet/klnds/socklnd/socklnd.h b/lnet/klnds/socklnd/socklnd.h
index 194ac98..2bef800 100644 (file)
--- a/lnet/klnds/socklnd/socklnd.h
+++ b/lnet/klnds/socklnd/socklnd.h
@@ -67,11 +67,7 @@
 #include <portals/nal.h>
 #include <portals/socknal.h>
 
-#if CONFIG_SMP
-# define SOCKNAL_N_SCHED       num_online_cpus() /* # socknal schedulers */
-#else
-# define SOCKNAL_N_SCHED        1               /* # socknal schedulers */
-#endif
+#define SOCKNAL_N_SCHED         ksocknal_nsched() /* # socknal schedulers */
 #define SOCKNAL_N_AUTOCONNECTD  4               /* # socknal autoconnect daemons */
 
 #define SOCKNAL_MIN_RECONNECT_INTERVAL HZ      /* first failed connection retry... */
@@ -392,6 +388,64 @@ ksocknal_putconnsock (ksock_conn_t *conn)
         fput (conn->ksnc_sock->file);
 }
 
+#ifndef CONFIG_SMP
+static inline 
+int ksocknal_nsched(void)
+{
+        return 1;
+}
+#else
+#include <linux/lustre_version.h>
+# if !(defined(CONFIG_X86) && (LINUX_VERSION_CODE >= KERNEL_VERSION(2,4,21))) || defined(CONFIG_X86_64) || (LUSTRE_KERNEL_VERSION < 39) || ((LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,0)) && !defined(CONFIG_X86_HT))
+static inline int
+ksocknal_nsched(void)
+{
+        return num_online_cpus();
+}
+
+static inline int
+ksocknal_sched2cpu(int i)
+{
+        return i;
+}
+
+static inline int
+ksocknal_irqsched2cpu(int i)
+{
+        return i;
+}
+# else 
+static inline int
+ksocknal_nsched(void)
+{
+        if (smp_num_siblings == 1)
+                return (num_online_cpus());
+
+        /* We need to know if this assumption is crap */
+        LASSERT (smp_num_siblings == 2);
+        return (num_online_cpus()/2);
+}
+
+static inline int
+ksocknal_sched2cpu(int i)
+{
+        if (smp_num_siblings == 1)
+                return i;
+        
+        return (i * 2);
+}
+
+static inline int
+ksocknal_irqsched2cpu(int i)
+{
+        if (smp_num_siblings == 1)
+                return ksocknal_sched2cpu(i);
+
+        return (ksocknal_sched2cpu(i) + 1);
+}
+# endif
+#endif
+
 extern void ksocknal_put_route (ksock_route_t *route);
 extern void ksocknal_put_peer (ksock_peer_t *peer);
 extern ksock_peer_t *ksocknal_find_peer_locked (ptl_nid_t nid);

diff --git a/lustre/kernel_patches/patches/export_num_siblings.patch b/lustre/kernel_patches/patches/export_num_siblings.patch
new file mode 100644 (file)
index 0000000..0fdefb5
--- /dev/null
+++ b/lustre/kernel_patches/patches/export_num_siblings.patch
@@ -0,0 +1,10 @@
+--- linux/arch/i386/kernel/i386_ksyms.c.orig   2003-11-28 11:26:19.000000000 -0700
++++ linux/arch/i386/kernel/i386_ksyms.c        2004-08-13 11:31:33.000000000 -0600
+@@ -131,6 +131,7 @@
+ EXPORT_SYMBOL(cpu_data);
+ EXPORT_SYMBOL(kernel_flag_cacheline);
+ EXPORT_SYMBOL(smp_num_cpus);
++EXPORT_SYMBOL(smp_num_siblings);
+ EXPORT_SYMBOL(cpu_online_map);
+ EXPORT_SYMBOL_NOVERS(__write_lock_failed);
+ EXPORT_SYMBOL_NOVERS(__read_lock_failed);

diff --git a/lustre/kernel_patches/patches/lustre_version.patch b/lustre/kernel_patches/patches/lustre_version.patch
index c938220..5c1887d 100644 (file)
--- a/lustre/kernel_patches/patches/lustre_version.patch
+++ b/lustre/kernel_patches/patches/lustre_version.patch
@@ -1,3 +1,4 @@
+Version 39: add EXPORT_SYMBOL(smp_num_siblings) to vanilla-2.4.24 (b=3966)
 Version 38: drop dentry ref in ext3_add_link from open_connect_dentry (b=3266)
 Version 37: fix htree rename-within-same-dir (b=3417), endianness (b=2447)
 Version 36: don't dput dentry after error (b=2350), zero page->private (3119)
@@ -10,6 +11,6 @@ Version 34: ext3 iopen assertion (b=2517), operations on deleted "." (b=2399)
 --- /dev/null  Fri Aug 30 17:31:37 2002
 +++ linux-2.4.18-18.8.0-l12-braam/include/linux/lustre_version.h       Thu Feb 13 07:58:33 2003
 @@ -0,0 +1 @@
-+#define LUSTRE_KERNEL_VERSION 38
++#define LUSTRE_KERNEL_VERSION 39
 
 _

diff --git a/lustre/kernel_patches/series/vanilla-2.4.24 b/lustre/kernel_patches/series/vanilla-2.4.24
index d050da7..b35ab9f 100644 (file)
--- a/lustre/kernel_patches/series/vanilla-2.4.24
+++ b/lustre/kernel_patches/series/vanilla-2.4.24
@@ -37,3 +37,4 @@ procfs-ndynamic-2.4.patch
 ext3-truncate-buffer-head.patch
 inode-max-readahead-2.4.24.patch
 ext3-nlinks-2.4.24.patch
+export_num_siblings.patch

diff --git a/lustre/portals/knals/socknal/socknal.c b/lustre/portals/knals/socknal/socknal.c
index 0dd5d11..f17ac4b 100644 (file)
--- a/lustre/portals/knals/socknal/socknal.c
+++ b/lustre/portals/knals/socknal/socknal.c
@@ -159,6 +159,7 @@ ksocknal_bind_irq (unsigned int irq)
 {
 #if (defined(CONFIG_SMP) && CPU_AFFINITY)
         int              bind;
+        int              cpu;
         unsigned long    flags;
         char             cmdline[64];
         ksock_irqinfo_t *info;
@@ -171,7 +172,7 @@ ksocknal_bind_irq (unsigned int irq)
                                    NULL};
 
         LASSERT (irq < NR_IRQS);
-        if (irq == 0)                           /* software NIC */
+        if (irq == 0)              /* software NIC or affinity disabled */
                 return;
 
         info = &ksocknal_data.ksnd_irqinfo[irq];
@@ -187,11 +188,12 @@ ksocknal_bind_irq (unsigned int irq)
         if (!bind)                              /* bound already */
                 return;
 
+        cpu = ksocknal_irqsched2cpu(info->ksni_sched);
         snprintf (cmdline, sizeof (cmdline),
-                  "echo %d > /proc/irq/%u/smp_affinity", 1 << info->ksni_sched, irq);
+                  "echo %d > /proc/irq/%u/smp_affinity", 1 << cpu, irq);
 
         printk (KERN_INFO "Lustre: Binding irq %u to CPU %d with cmd: %s\n",
-                irq, info->ksni_sched, cmdline);
+                irq, cpu, cmdline);
 
         /* FIXME: Find a better method of setting IRQ affinity...
          */

diff --git a/lustre/portals/knals/socknal/socknal.h b/lustre/portals/knals/socknal/socknal.h
index 194ac98..2bef800 100644 (file)
--- a/lustre/portals/knals/socknal/socknal.h
+++ b/lustre/portals/knals/socknal/socknal.h
@@ -67,11 +67,7 @@
 #include <portals/nal.h>
 #include <portals/socknal.h>
 
-#if CONFIG_SMP
-# define SOCKNAL_N_SCHED       num_online_cpus() /* # socknal schedulers */
-#else
-# define SOCKNAL_N_SCHED        1               /* # socknal schedulers */
-#endif
+#define SOCKNAL_N_SCHED         ksocknal_nsched() /* # socknal schedulers */
 #define SOCKNAL_N_AUTOCONNECTD  4               /* # socknal autoconnect daemons */
 
 #define SOCKNAL_MIN_RECONNECT_INTERVAL HZ      /* first failed connection retry... */
@@ -392,6 +388,64 @@ ksocknal_putconnsock (ksock_conn_t *conn)
         fput (conn->ksnc_sock->file);
 }
 
+#ifndef CONFIG_SMP
+static inline 
+int ksocknal_nsched(void)
+{
+        return 1;
+}
+#else
+#include <linux/lustre_version.h>
+# if !(defined(CONFIG_X86) && (LINUX_VERSION_CODE >= KERNEL_VERSION(2,4,21))) || defined(CONFIG_X86_64) || (LUSTRE_KERNEL_VERSION < 39) || ((LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,0)) && !defined(CONFIG_X86_HT))
+static inline int
+ksocknal_nsched(void)
+{
+        return num_online_cpus();
+}
+
+static inline int
+ksocknal_sched2cpu(int i)
+{
+        return i;
+}
+
+static inline int
+ksocknal_irqsched2cpu(int i)
+{
+        return i;
+}
+# else 
+static inline int
+ksocknal_nsched(void)
+{
+        if (smp_num_siblings == 1)
+                return (num_online_cpus());
+
+        /* We need to know if this assumption is crap */
+        LASSERT (smp_num_siblings == 2);
+        return (num_online_cpus()/2);
+}
+
+static inline int
+ksocknal_sched2cpu(int i)
+{
+        if (smp_num_siblings == 1)
+                return i;
+        
+        return (i * 2);
+}
+
+static inline int
+ksocknal_irqsched2cpu(int i)
+{
+        if (smp_num_siblings == 1)
+                return ksocknal_sched2cpu(i);
+
+        return (ksocknal_sched2cpu(i) + 1);
+}
+# endif
+#endif
+
 extern void ksocknal_put_route (ksock_route_t *route);
 extern void ksocknal_put_peer (ksock_peer_t *peer);
 extern ksock_peer_t *ksocknal_find_peer_locked (ptl_nid_t nid);