diff -urpN numaq-2.5.45/drivers/block/ll_rw_blk.c iowait-2.5.45/drivers/block/ll_rw_blk.c --- numaq-2.5.45/drivers/block/ll_rw_blk.c 2002-11-01 00:53:40.000000000 -0800 +++ iowait-2.5.45/drivers/block/ll_rw_blk.c 2002-11-01 01:29:00.000000000 -0800 @@ -56,7 +56,6 @@ static int queue_nr_requests; static int batch_requests; unsigned long blk_max_low_pfn, blk_max_pfn; -atomic_t nr_iowait_tasks = ATOMIC_INIT(0); int blk_nohighio = 0; static struct congestion_state { @@ -115,27 +114,6 @@ static void set_queue_congested(request_ atomic_inc(&congestion_states[rw].nr_congested_queues); } -/* - * This task is about to go to sleep on IO. Increment nr_iowait_tasks so - * that process accounting knows that this is a task in IO wait state. - * - * But don't do that if it is a deliberate, throttling IO wait (this task - * has set its backing_dev_info: the queue against which it should throttle) - */ -void io_schedule(void) -{ - atomic_inc(&nr_iowait_tasks); - schedule(); - atomic_dec(&nr_iowait_tasks); -} - -void io_schedule_timeout(long timeout) -{ - atomic_inc(&nr_iowait_tasks); - schedule_timeout(timeout); - atomic_dec(&nr_iowait_tasks); -} - /** * blk_get_backing_dev_info - get the address of a queue's backing_dev_info * @dev: device diff -urpN numaq-2.5.45/fs/proc/proc_misc.c iowait-2.5.45/fs/proc/proc_misc.c --- numaq-2.5.45/fs/proc/proc_misc.c 2002-11-01 00:53:40.000000000 -0800 +++ iowait-2.5.45/fs/proc/proc_misc.c 2002-11-01 01:28:45.000000000 -0800 @@ -424,7 +424,7 @@ static int kstat_read_proc(char *page, c xtime.tv_sec - jif / HZ, total_forks, nr_running(), - atomic_read(&nr_iowait_tasks)); + nr_iowait()); return proc_calc_metrics(page, start, off, count, eof, len); } diff -urpN numaq-2.5.45/include/linux/blkdev.h iowait-2.5.45/include/linux/blkdev.h --- numaq-2.5.45/include/linux/blkdev.h 2002-11-01 00:53:40.000000000 -0800 +++ iowait-2.5.45/include/linux/blkdev.h 2002-11-01 01:40:57.000000000 -0800 @@ -467,9 +467,4 @@ static inline void put_dev_sector(Sector #endif - -extern atomic_t nr_iowait_tasks; -void io_schedule(void); -void io_schedule_timeout(long timeout); - #endif diff -urpN numaq-2.5.45/include/linux/sched.h iowait-2.5.45/include/linux/sched.h --- numaq-2.5.45/include/linux/sched.h 2002-11-01 00:53:41.000000000 -0800 +++ iowait-2.5.45/include/linux/sched.h 2002-11-01 01:41:12.000000000 -0800 @@ -89,6 +89,7 @@ extern int nr_threads; extern int last_pid; extern unsigned long nr_running(void); extern unsigned long nr_uninterruptible(void); +extern unsigned long nr_iowait(void); #include #include @@ -161,6 +162,8 @@ extern unsigned long cache_decay_ticks; #define MAX_SCHEDULE_TIMEOUT LONG_MAX extern signed long FASTCALL(schedule_timeout(signed long timeout)); asmlinkage void schedule(void); +void io_schedule(void); +void io_schedule_timeout(long timeout); struct namespace; diff -urpN numaq-2.5.45/kernel/sched.c iowait-2.5.45/kernel/sched.c --- numaq-2.5.45/kernel/sched.c 2002-11-01 00:53:41.000000000 -0800 +++ iowait-2.5.45/kernel/sched.c 2002-11-01 01:36:14.000000000 -0800 @@ -157,6 +157,7 @@ struct runqueue { task_t *migration_thread; struct list_head migration_queue; + atomic_t nr_iowait; } ____cacheline_aligned; static struct runqueue runqueues[NR_CPUS] __cacheline_aligned; @@ -573,6 +574,16 @@ unsigned long nr_context_switches(void) return sum; } +unsigned long nr_iowait(void) +{ + unsigned long i, sum = 0; + + for (i = 0; i < NR_CPUS; ++i) + sum += atomic_read(&cpu_rq(i)->nr_iowait); + + return sum; +} + /* * double_rq_lock - safely lock two runqueues * @@ -875,7 +886,7 @@ void scheduler_tick(int user_ticks, int /* note: this timer irq context must be accounted for as well */ if (irq_count() - HARDIRQ_OFFSET >= SOFTIRQ_OFFSET) kstat_cpu(cpu).cpustat.system += sys_ticks; - else if (atomic_read(&nr_iowait_tasks) > 0) + else if (atomic_read(&rq->nr_iowait) > 0) kstat_cpu(cpu).cpustat.iowait += sys_ticks; else kstat_cpu(cpu).cpustat.idle += sys_ticks; @@ -1712,6 +1723,35 @@ void yield(void) sys_sched_yield(); } +/* + * This task is about to go to sleep on IO. Increment rq->nr_iowait so + * that process accounting knows that this is a task in IO wait state. + * + * But don't do that if it is a deliberate, throttling IO wait (this task + * has set its backing_dev_info: the queue against which it should throttle) + */ +void io_schedule(void) +{ + struct runqueue *rq; + preempt_disable(); + rq = this_rq(); + atomic_inc(&rq->nr_iowait); + schedule(); + atomic_dec(&rq->nr_iowait); + preempt_enable(); +} + +void io_schedule_timeout(long timeout) +{ + struct runqueue *rq; + preempt_disable(); + rq = this_rq(); + atomic_inc(&rq->nr_iowait); + schedule_timeout(timeout); + atomic_dec(&rq->nr_iowait); + preempt_enable(); +} + /** * sys_sched_get_priority_max - return maximum RT priority. * @policy: scheduling class. @@ -2160,6 +2200,7 @@ void __init sched_init(void) rq->expired = rq->arrays + 1; spin_lock_init(&rq->lock); INIT_LIST_HEAD(&rq->migration_queue); + atomic_set(&rq->nr_iowait, 0); for (j = 0; j < 2; j++) { array = rq->arrays + j;