Index: linux-2.6.10/include/asm-ia64/spinlock.h =================================================================== --- linux-2.6.10.orig/include/asm-ia64/spinlock.h 2005-02-17 18:47:00.000000000 -0800 +++ linux-2.6.10/include/asm-ia64/spinlock.h 2005-02-22 10:31:22.000000000 -0800 @@ -11,7 +11,7 @@ #include #include - +#include #include #include #include @@ -24,10 +24,13 @@ typedef struct { #endif } spinlock_t; +/* These definitions do not mean much. Lots of code for IA64 depends on + * zeroed memory containing unlocked spinlocks + */ #define SPIN_LOCK_UNLOCKED (spinlock_t) { 0 } #define spin_lock_init(x) ((x)->lock = 0) -#ifdef ASM_SUPPORTED +#if defined(ASM_SUPPORTED) & !defined(CONFIG_HBO_LOCKS) /* * Try to get the lock. If we fail to get the lock, make a non-standard call to * ia64_spinlock_contention(). We do not use a normal call because that would force all @@ -94,7 +97,31 @@ _raw_spin_lock_flags (spinlock_t *lock, #endif } #define _raw_spin_lock(lock) _raw_spin_lock_flags(lock, 0) +#error CRAP CRAP #else /* !ASM_SUPPORTED */ + +#ifdef CONFIG_HBO_LOCKS + +void hbo_spinlock_contention(__u32 *, unsigned long, unsigned long); +void hbo_spinlock_wait_remote(__u32 *, unsigned long); + +#define _raw_spin_lock_flags(__x, __flags) \ +do { \ + __u32 *ia64_spinlock_ptr = (__u32 *) (__x); \ + __u64 ia64_spinlock_val; \ + \ + if (unlikely(system_state == SYSTEM_RUNNING && local_node_data->remote_lock_addr == ia64_spinlock_ptr)) \ + hbo_spinlock_wait_remote(ia64_spinlock_ptr, __flags); \ + \ + ia64_spinlock_val = ia64_cmpxchg4_acq(ia64_spinlock_ptr, numa_node_id()+ 1, 0); \ + \ + if (unlikely(ia64_spinlock_val)) \ + hbo_spinlock_contention(ia64_spinlock_ptr, ia64_spinlock_val, __flags); \ +} while (0) + +#define _raw_spin_lock(lock) _raw_spin_lock_flags(lock, 0) + +#else #define _raw_spin_lock_flags(lock, flags) _raw_spin_lock(lock) # define _raw_spin_lock(x) \ do { \ @@ -109,11 +136,16 @@ do { \ } while (ia64_spinlock_val); \ } \ } while (0) +#endif #endif /* !ASM_SUPPORTED */ #define spin_is_locked(x) ((x)->lock != 0) #define _raw_spin_unlock(x) do { barrier(); ((spinlock_t *) x)->lock = 0; } while (0) +#ifdef CONFIG_HBO_LOCKS +#define _raw_spin_trylock(x) (cmpxchg_acq(&(x)->lock, 0, numa_node_id() + 1) == 0) +#else #define _raw_spin_trylock(x) (cmpxchg_acq(&(x)->lock, 0, 1) == 0) +#endif #define spin_unlock_wait(x) do { barrier(); } while ((x)->lock) typedef struct { Index: linux-2.6.10/arch/ia64/Kconfig =================================================================== --- linux-2.6.10.orig/arch/ia64/Kconfig 2005-02-17 18:46:27.000000000 -0800 +++ linux-2.6.10/arch/ia64/Kconfig 2005-02-18 18:38:59.000000000 -0800 @@ -272,6 +272,18 @@ config PREEMPT Say Y here if you are building a kernel for a desktop, embedded or real-time system. Say N if you are unsure. +config HBO_LOCKS + bool "HBO Locks" + help + depends on (SMP && NUMA) + default y + help + HBO locks reduces contention for spinlocks by saving the node id when the + lock is taken. The cpu waiting on the spinlock can then select an appropriate + backoff algorithm depending on the distance to the node. This also insures that + it is highly likely that another cpu on the same node gets to a spinlock before + remote nodes to avoid too much cacheline bouncing. + config HAVE_DEC_LOCK bool depends on (SMP || PREEMPT) Index: linux-2.6.10/arch/ia64/mm/numa.c =================================================================== --- linux-2.6.10.orig/arch/ia64/mm/numa.c 2005-02-17 18:46:28.000000000 -0800 +++ linux-2.6.10/arch/ia64/mm/numa.c 2005-02-22 10:33:10.000000000 -0800 @@ -47,3 +47,81 @@ paddr_to_nid(unsigned long paddr) return (i < num_node_memblks) ? node_memblk[i].nid : (num_node_memblks ? -1 : 0); } + +#ifdef CONFIG_HBO_LOCKS + +static inline void maybe_enable_irq(unsigned long flags) +{ + if (flags & IA64_PSR_I_BIT) + local_irq_enable(); +} + +static inline void maybe_disable_irq(unsigned long flags) +{ + if (flags & IA64_PSR_I_BIT) + local_irq_disable(); +} + +void hbo_spinlock_contention(__u32 *lock, unsigned long lockval, unsigned long flags) +{ + int node = numa_node_id()+ 1; + + do { + int backoff = 100; + int cap = 2500; + int remote = 0; + + maybe_enable_irq(flags); + + if (lockval != node) { + /* remote backoff */ + backoff = 200; + cap = 20000; + /* + * Make sure that no other cpu of this node tries + * to acquire the same remote lock. + */ + if (system_state == SYSTEM_RUNNING) { + local_node_data->remote_lock_addr = lock; + remote = 1; + } + } + + do { + int i; + + for(i = 0; i < backoff; i++) + cpu_relax(); + + /* Increase backoff for next cycle */ + backoff = min(backoff << 1, cap); + + ia64_barrier(); + + /* No cmpxchg so that we will not get an exclusive cache line */ + lockval = *lock; + + } while (lockval); + + maybe_disable_irq(flags); + + /* Lock was unlocked. Now use cmpxchg acquiring an exclusive cache line */ + lockval = ia64_cmpxchg4_acq(lock, node, 0); + + /* Release eventually spinning other cpus on this node since either the lock has been + * acquired by this cpu or the node holding the lock may have changed. + */ + if (remote) + local_node_data->remote_lock_addr = NULL; + + } while (lockval); +} + +void hbo_spinlock_wait_remote(__u32 *lock, unsigned long flags) +{ + maybe_enable_irq(flags); + while (local_node_data->remote_lock_addr == lock) + cpu_relax(); + maybe_disable_irq(flags); +} +#endif Index: linux-2.6.10/include/asm-ia64/nodedata.h =================================================================== --- linux-2.6.10.orig/include/asm-ia64/nodedata.h 2005-02-17 18:47:00.000000000 -0800 +++ linux-2.6.10/include/asm-ia64/nodedata.h 2005-02-18 18:49:48.000000000 -0800 @@ -27,6 +27,7 @@ struct pglist_data; struct ia64_node_data { short active_cpu_count; short node; + __u32 *remote_lock_addr; /* active off node spinlock */ struct pglist_data *pg_data_ptrs[MAX_NUMNODES]; }; Index: linux-2.6.10/Makefile =================================================================== --- linux-2.6.10.orig/Makefile 2005-02-17 18:47:21.000000000 -0800 +++ linux-2.6.10/Makefile 2005-02-22 11:24:51.000000000 -0800 @@ -1,7 +1,7 @@ VERSION = 2 PATCHLEVEL = 6 SUBLEVEL = 11 -EXTRAVERSION = -rc4-bk5 +EXTRAVERSION = -rc4-bk5-hbo NAME=Woozy Numbat # *DOCUMENTATION*